[verified] Publish ReelBox source under GPLv3

server/catalog_api.py

@@ -0,0 +1,431 @@
+"""Read-only ReelBox metadata service. No upstream or media fetches."""
+from collections import OrderedDict
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+import re
+import sqlite3
+import sys
+import threading
+import unicodedata
+
+
+class ApiError(Exception):
+    def __init__(self, status, message):
+        super().__init__(message)
+        self.status = status
+
+
+_FOLD = dict.fromkeys(i for i in range(0x110000)
+                      if unicodedata.category(chr(i)) in {'Mn', 'Cf'})
+_FOLD.update({ord('ي'): 'ی', ord('ى'): 'ی', ord('ك'): 'ک'})
+
+
+def normal(value):
+    return str(value or '').lower().translate(_FOLD)
+
+
+def allowed_text(value):
+    text = normal(unicodedata.normalize('NFKC', str(value or '')))
+    # Upstream notices are often disguised as movie rows, with new IDs after
+    # refreshes. Match their labels rather than IDs or bare English "notice",
+    # which would incorrectly hide films such as Red Notice / Notice to Quit.
+    label = re.sub(r'[\W_ـ]', '', text)
+    if label.startswith((
+            'اطلاعیه', 'اطلاعرسانی', 'بهروزرسانی', 'بروزرسانی',
+            'اعلانجدید', 'اعلانمهم', 'اعلانفوری', 'اعلامیهجدید', 'اعلامیهمهم',
+            'آپدیتجدید', 'اپدیتجدید',
+            'newannouncement', 'importantannouncement', 'urgentannouncement',
+            'newnotice', 'importantnotice', 'urgentnotice',
+            'appupdate', 'applicationupdate', 'softwareupdate')):
+        return False
+    compact = re.sub(r'[\s_\-ـ]', '', text)
+    return not (any(word in compact for word in
+                    ('ملودیباکس', 'ملودیبکس', 'melodybox', 'melodibox', 'نورتون', 'نورتن', 'norton',
+                     'موویباکس', 'موسویباکس', 'نسخهجدیدهورتون', 'اپلیکیشنهورتون', 'نرمافزارهورتون'))
+                or 'تلگرام' in text or re.search(r'\btelegram\b|(^|[\s>(])@[a-z][a-z0-9_]{3,}', text)
+                or re.search(r'(دانلود|نصب|دریافت).{0,30}نسخه.{0,18}(برنامه|اپلیکیشن|سیمبا|مووی)', text)
+                or re.search(r'(install|download|update).{0,24}(latest|new).{0,12}(app|version)', text))
+
+
+def banner_https(value):
+    from urllib.parse import urlsplit
+    if (not isinstance(value, str) or len(value) > 2048
+            or re.search(r'[\x00-\x20\x7f\\]', value)
+            or re.search(r'%(?![0-9a-f]{2})|%(?:0[0-9a-f]|1[0-9a-f]|7f)', value, re.I)):
+        return False
+    try:
+        uri = urlsplit(value)
+        return (uri.scheme == 'https' and bool(uri.hostname) and uri.username is None
+                and uri.password is None and uri.port in (None, 443)
+                and len(uri.hostname) <= 253
+                and all(re.fullmatch(r'[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?', part)
+                        for part in uri.hostname.split('.')))
+    except ValueError:
+        return False
+
+
+def load_banners(path):
+    """Read a bounded operator-owned file, never a remote config or admin input."""
+    try:
+        with Path(path).open('rb') as source:
+            data = source.read(65537)
+        if len(data) > 65536:
+            return {'banners': []}
+        config = json.loads(data)
+    except (OSError, ValueError, RecursionError):
+        return {'banners': []}
+    if not isinstance(config, dict) or not isinstance(config.get('banners'), list):
+        return {'banners': []}
+    result = []
+    for row in config['banners'][:20]:
+        if len(result) == 5:
+            break
+        if not isinstance(row, dict) or not banner_https(row.get('target_url')):
+            continue
+        image = row.get('image_url')
+        if image not in (None, '') and not banner_https(image):
+            continue
+        clean = {}
+        for key, fallback, limit in (
+                ('title_en', '', 80), ('title_fa', row.get('title_en', ''), 80),
+                ('subtitle_en', '', 180), ('subtitle_fa', row.get('subtitle_en', ''), 180)):
+            text = row.get(key, fallback)
+            if (not isinstance(text, str) or len(text) > limit
+                    or re.search(r'[\x00-\x1f\x7f\u202a-\u202e\u2066-\u2069]', text)
+                    or not allowed_text(text)):
+                break
+            clean[key] = text.strip()
+        else:
+            if not clean['title_en'] or not clean['title_fa']:
+                continue
+            clean['target_url'] = row['target_url']
+            if image:
+                clean['image_url'] = image
+            result.append(clean)
+    return {'banners': result}
+
+
+class Catalog:
+    RESPONSE_CACHE_BYTES = 8 * 1024 * 1024
+    RESPONSE_CACHE_ENTRIES = 128
+    RESPONSE_CACHE_ITEM_BYTES = 1024 * 1024
+
+    def __init__(self, path, banner_config=None):
+        self.banner_config = Path(banner_config) if banner_config is not None else Path(__file__).with_name('banners.json')
+        self.db = sqlite3.connect(Path(path).resolve().as_uri() + '?mode=ro&immutable=1',
+                                  uri=True, check_same_thread=False)
+        self.db.execute('PRAGMA query_only=ON')
+        self.db.execute('PRAGMA cache_size=-8192')
+        self.lock = threading.RLock()
+        self._responses = OrderedDict()
+        self._response_bytes = 0
+        self.metadata = {k: json.loads(v) for k, v in self.db.execute('SELECT * FROM metadata')}
+        # The snapshot is immutable until service restart. Normalize searchable
+        # text once instead of redoing Unicode folding for every row/request.
+        # Bound retained strings on the 256 MiB service; overflow stays correct
+        # through the original normalization path rather than dropping results.
+        self.search_text = {}
+        self.search_index_bytes = 0
+        visible = set()
+        for title_id, kind, title, description in self.db.execute(
+                'SELECT id,type,title,description FROM titles'):
+            if kind not in {'movie', 'serie'} or not allowed_text(title) or not allowed_text(description):
+                continue
+            visible.add(title_id)
+            text = (normal(title), normal(description))
+            size = sys.getsizeof(text) + sum(sys.getsizeof(v) for v in text) + sys.getsizeof(title_id) + 64
+            if self.search_index_bytes + size <= 64 * 1024 * 1024:
+                self.search_text[title_id] = text
+                self.search_index_bytes += size
+        self.visible = frozenset(visible)
+        self.db.create_function('visible', 1, lambda value: value in self.visible, deterministic=True)
+        self.db.create_function('search_match', 4, self.search_match, deterministic=True)
+        tables = ('titles', 'genres', 'countries', 'movie_sources', 'seasons', 'episodes', 'episode_sources')
+        self.counts = {name: self.db.execute('SELECT COUNT(*) FROM ' + name).fetchone()[0] for name in tables}
+        self.counts['visible_titles'] = len(self.visible)
+
+    def search_match(self, title_id, title, description, query):
+        text = self.search_text.get(title_id)
+        if text is None:
+            text = (normal(title), normal(description))
+        return query in text[0] or query in text[1]
+
+    def close(self):
+        self.db.close()
+
+    def health(self):
+        built = self.metadata.get('built_at_utc')
+        age = None
+        if built:
+            age = max(0, int((datetime.now(timezone.utc) - datetime.fromisoformat(built)).total_seconds()))
+        return {'status': 'ok', 'counts': self.counts.copy(),
+                'snapshot_built_at_utc': built, 'snapshot_age_seconds': age,
+                'provenance': self.metadata.get('provenance', 'Exported API snapshot'),
+                'offline': True, 'media_proxy': False,
+                'snapshot_limitations': ['Not live-synchronised', 'Media URLs may expire',
+                                         'Cast, public lists and subtitle metadata were not exported']}
+
+    @staticmethod
+    def clean(value):
+        if isinstance(value, dict):
+            return {k: Catalog.clean(v) for k, v in value.items()
+                    if k.lower() not in {'comment', 'comments', 'comments_enabled', 'replies'}}
+        if isinstance(value, list):
+            return [Catalog.clean(v) for v in value]
+        return value
+
+    def rows(self, sql, params=()):
+        return [self.clean(json.loads(row[0])) for row in self.db.execute(sql, params)]
+
+    def tags(self, name):
+        columns = 'id,title,image' if name == 'countries' else 'id,title'
+        cursor = self.db.execute('SELECT ' + columns + ' FROM ' + name + ' ORDER BY id')
+        keys = [column[0] for column in cursor.description]
+        return [dict(zip(keys, row)) for row in cursor
+                if allowed_text(row[1]) and not (name == 'genres' and row[0] == 48)]
+
+    def titles(self, kind='poster', genre=0, country=0, order='created', page=0, query=None):
+        orders = {'created': 't.id DESC', 'updated': 't.id DESC',
+                  'imdb': 't.imdb DESC,t.id DESC', 'rating': 't.rating DESC,t.id DESC',
+                  'year': 't.year DESC,t.id DESC', 'title': 't.title COLLATE NOCASE,t.id',
+                  'random': '(t.id * 2654435761) % 2147483647,t.id'}
+        if order not in orders or kind not in {'movie', 'serie', 'poster'} or not 0 <= page <= 100000:
+            raise ApiError(400, 'Unsupported catalog filter')
+        where, params = ['visible(t.id)'], []
+        if kind != 'poster':
+            where.append('t.type=?')
+            params.append(kind)
+        for table, column, value in [('title_genres', 'genre_id', genre),
+                                      ('title_countries', 'country_id', country)]:
+            if value:
+                where.append('EXISTS (SELECT 1 FROM ' + table + ' f WHERE f.title_id=t.id AND f.' + column + '=?)')
+                params.append(value)
+        if query is not None:
+            where.append('search_match(t.id,t.title,t.description,?)')
+            params.append(normal(query))
+        return self.rows('SELECT t.raw_json FROM titles t WHERE ' + ' AND '.join(where)
+                         + ' ORDER BY ' + orders[order] + ' LIMIT 30 OFFSET ?', params + [page * 30])
+
+    def home(self):
+        return {'genres': [
+            {'id': -2, 'title': 'Latest in catalogue', 'posters': self.titles()},
+            {'id': -3, 'title': 'Highest IMDb ratings', 'posters': self.titles(order='imdb')},
+            {'id': -4, 'title': 'Series in catalogue', 'posters': self.titles(kind='serie')}
+        ], 'actors': [], 'countries': self.tags('countries'), 'slides': []}
+
+    def legacy(self, path, query):
+        from urllib.parse import parse_qs
+        try:
+            raw = parse_qs(query, keep_blank_values=True, max_num_fields=8)
+        except ValueError:
+            raise ApiError(400, 'Invalid query') from None
+        if any(len(values) != 1 for values in raw.values()):
+            raise ApiError(400, 'Repeated query parameter')
+        args = {key: values[0] for key, values in raw.items()}
+        allowed = {'action', 'q', 'pageno'}
+        if path.endswith('/filter.php'):
+            allowed = {'action', 'garid', 'pageno'}
+        elif path.endswith('/star.php'):
+            allowed = {'key', 'action', 'q', 'pageno'}
+        elif path.endswith(('/pishnahad.php', '/bartar.php')):
+            allowed = set()
+        if args.keys() - allowed or len(args.get('q', '')) > 200 or len(args.get('key', '')) > 256:
+            raise ApiError(400, 'Unsupported query parameter')
+        try:
+            page = int(args.get('pageno') or 1) - 1
+            genre = int(args.get('garid') or 0)
+        except ValueError:
+            raise ApiError(400, 'Invalid page or genre') from None
+        if not 0 <= page <= 100000 or not 0 <= genre <= 999999999:
+            raise ApiError(400, 'Invalid page or genre')
+        action = args.get('action', '')
+        if path.endswith(('/pishnahad.php', '/bartar.php')):
+            return [{'title': [tag['title']],
+                     'link': ['/api-user/newapi/filter.php?action=pishnahad&garid=' + str(tag['id']) + '&pageno=']}
+                    for tag in self.tags('genres')
+                    if self.db.execute('SELECT 1 FROM title_genres WHERE genre_id=? AND visible(title_id) LIMIT 1', (tag['id'],)).fetchone()]
+        if path == '/api-user/newapi/filter.php' and action == 'pishnahad':
+            return self.titles(genre=genre, order='imdb', page=page)
+        if path == '/api-user/newapi/like.php':
+            if action in {'search-movie', 'search-movie-en'}:
+                return self.titles(query=args.get('q', '').strip(), page=page) if args.get('q', '').strip() else []
+            if action in {'best-rand-top', 'best-rand-year'}:
+                return self.titles(order='year' if action.endswith('year') else 'imdb', page=page)
+            if action in {'all-actor', 'colection', 'search-trand'}:
+                return []
+        if path == '/api-user/filmahang/star.php' and action in {'starsearch', 'mazmon', 'share', 'new'}:
+            return []
+        raise ApiError(404, 'Metadata route not available in snapshot')
+
+    def response(self, target):
+        # Only immutable REST snapshot metadata. Health/banner freshness and
+        # search privacy are preserved: no search query is retained in this LRU.
+        cacheable = (target.startswith('/api/') and '?' not in target
+                     and target.rstrip('/') not in {'/api/health', '/api/banners'})
+        with self.lock:
+            cached = self._responses.get(target) if cacheable else None
+            if cached is not None:
+                self._responses.move_to_end(target)
+                return cached
+            body = json.dumps(self.dispatch(target), ensure_ascii=False,
+                              separators=(',', ':')).encode('utf-8')
+            if len(body) > 32 * 1024 * 1024:
+                raise ApiError(503, 'Metadata response exceeds safety limit')
+            size = len(body) + len(target.encode('utf-8'))
+            if (cacheable and size <= self.RESPONSE_CACHE_ITEM_BYTES
+                    and size <= self.RESPONSE_CACHE_BYTES and self.RESPONSE_CACHE_ENTRIES > 0):
+                while self._responses and (
+                        self._response_bytes + size > self.RESPONSE_CACHE_BYTES
+                        or len(self._responses) >= self.RESPONSE_CACHE_ENTRIES):
+                    old_target, old_body = self._responses.popitem(last=False)
+                    self._response_bytes -= len(old_body) + len(old_target.encode('utf-8'))
+                self._responses[target] = body
+                self._response_bytes += size
+            return body
+
+    def dispatch(self, target):
+        from urllib.parse import urlsplit
+        if len(target) > 4096 or re.search(r'[\x00-\x20\x7f]', target):
+            raise ApiError(400, 'Invalid metadata route')
+        try:
+            parsed = urlsplit(target)
+        except ValueError:
+            raise ApiError(400, 'Invalid metadata route') from None
+        path = parsed.path
+        if (len(target) > 4096 or parsed.scheme or parsed.netloc or parsed.fragment
+                or '%' in path or '\\' in path or re.search(r'[\x00-\x20\x7f]', path)
+                or re.search(r'(^|/)\.{1,2}(/|$)', path)):
+            raise ApiError(400, 'Invalid metadata route')
+        key = r'[A-Za-z0-9_-]{0,256}'
+        with self.lock:
+            if path in {'/api-user/newapi/like.php', '/api-user/newapi/filter.php',
+                        '/api-user/newapi/pishnahad.php', '/api-user/newapi/bartar.php',
+                        '/api-user/filmahang/star.php'}:
+                return self.legacy(path, parsed.query)
+            if parsed.query:
+                raise ApiError(400, 'Unexpected query parameters')
+            if path in {'/api/banners', '/api/banners/'}:
+                return load_banners(self.banner_config)
+            if path in {'/api/health', '/api/health/'}:
+                return self.health()
+            if re.fullmatch(r'/api/first/' + key + '/', path):
+                return self.home()
+            if re.fullmatch(r'/api/genre/all/' + key + '/', path):
+                return self.tags('genres')
+            if re.fullmatch(r'/api/country/all/' + key + '/', path):
+                return self.tags('countries')
+            match = re.fullmatch(r'/api/(movie|serie|poster)/by/filtres/(\d{1,9})/(?:(\d{1,9})/)?([a-z]+)/([0-9]{1,6})/' + key + '/', path)
+            if match:
+                kind, genre, country, order, page = match.groups()
+                return self.titles(kind, int(genre), int(country or 0), order, int(page))
+            match = re.fullmatch(r'/api/movie/by/([0-9]{1,10})/' + key + '/', path)
+            if match:
+                title_id = int(match[1])
+                if title_id not in self.visible:
+                    raise ApiError(404, 'Title not in snapshot')
+                item = self.rows('SELECT raw_json FROM titles WHERE id=?', (title_id,))[0]
+                item['sources'] = self.rows('SELECT raw_json FROM movie_sources WHERE title_id=? ORDER BY source_index', (title_id,))
+                return item
+            match = re.fullmatch(r'/api/season/by/serie/([0-9]{1,10})/' + key + '/', path)
+            if match:
+                title_id = int(match[1])
+                if title_id not in self.visible:
+                    raise ApiError(404, 'Title not in snapshot')
+                return self.rows('SELECT raw_json FROM seasons WHERE series_id=? ORDER BY season_index', (title_id,))
+            if re.fullmatch(r'/api/(?:role/by/poster|movie/by/actor|subtitles/by/(?:movie|episode))/[0-9]{1,10}/' + key + '/', path):
+                return []
+            raise ApiError(404, 'Unknown metadata route')
+
+
+def make_server(catalog, host='127.0.0.1', port=18765):
+    from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+    class Handler(BaseHTTPRequestHandler):
+        protocol_version = 'HTTP/1.0'
+
+        def setup(self):
+            self.request.settimeout(10)
+            super().setup()
+
+        def log_message(self, *_args):
+            pass  # Never record source links, search terms or shared route keys.
+
+        def reply(self, status, value):
+            body = value if isinstance(value, bytes) else json.dumps(
+                value, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
+            if len(body) > 32 * 1024 * 1024:
+                status, body = 503, b'{"error":"Metadata response exceeds safety limit"}'
+            self.send_response_only(status)
+            self.send_header('Content-Type', 'application/json; charset=utf-8')
+            self.send_header('Content-Length', str(len(body)))
+            self.send_header('X-Content-Type-Options', 'nosniff')
+            self.send_header('Cache-Control', 'no-store')
+            self.send_header('Connection', 'close')
+            self.end_headers()
+            if self.command != 'HEAD':
+                self.wfile.write(body)
+            self.close_connection = True
+
+        def do_GET(self):
+            try:
+                if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
+                    raise ApiError(400, 'Request body not supported')
+                self.reply(200, catalog.response(self.path))
+            except ApiError as error:
+                self.reply(error.status, {'error': str(error)})
+            except (BrokenPipeError, ConnectionResetError, TimeoutError):
+                self.close_connection = True
+            except Exception:
+                self.reply(503, {'error': 'Catalogue temporarily unavailable'})
+
+        do_HEAD = do_GET
+
+        def reject_write(self):
+            self.reply(405, {'error': 'Read-only metadata API'})
+
+        do_POST = do_PUT = do_DELETE = do_PATCH = do_CONNECT = do_OPTIONS = do_TRACE = reject_write
+
+    class Server(ThreadingHTTPServer):
+        daemon_threads = True
+        request_queue_size = 16
+
+        def __init__(self, address):
+            self.slots = threading.BoundedSemaphore(16)
+            super().__init__(address, Handler)
+
+        def process_request(self, request, client_address):
+            if not self.slots.acquire(blocking=False):
+                self.shutdown_request(request)
+                return
+            try:
+                super().process_request(request, client_address)
+            except Exception:
+                self.slots.release()
+                raise
+
+        def process_request_thread(self, request, client_address):
+            try:
+                super().process_request_thread(request, client_address)
+            finally:
+                self.slots.release()
+
+    if host != '127.0.0.1':
+        raise ValueError('Bind behind the HTTPS reverse proxy on IPv4 loopback only')
+    return Server((host, port))
+
+
+def main():
+    import os
+    catalog = Catalog(os.environ.get('REELBOX_CATALOG_DB', '/var/lib/reelbox-api/catalog.sqlite'),
+                      banner_config=os.environ.get('REELBOX_BANNER_CONFIG', '/var/lib/reelbox-api/banners.json'))
+    server = make_server(catalog)
+    try:
+        server.serve_forever()
+    finally:
+        server.server_close()
+        catalog.close()
+
+
+if __name__ == '__main__':
+    main()