432 lines
21 KiB
Python
432 lines
21 KiB
Python
"""Read-only ReelBox metadata service. No upstream or media fetches."""
|
|
from collections import OrderedDict
|
|
from datetime import datetime, timezone
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
import threading
|
|
import unicodedata
|
|
|
|
|
|
class ApiError(Exception):
|
|
def __init__(self, status, message):
|
|
super().__init__(message)
|
|
self.status = status
|
|
|
|
|
|
_FOLD = dict.fromkeys(i for i in range(0x110000)
|
|
if unicodedata.category(chr(i)) in {'Mn', 'Cf'})
|
|
_FOLD.update({ord('ي'): 'ی', ord('ى'): 'ی', ord('ك'): 'ک'})
|
|
|
|
|
|
def normal(value):
|
|
return str(value or '').lower().translate(_FOLD)
|
|
|
|
|
|
def allowed_text(value):
|
|
text = normal(unicodedata.normalize('NFKC', str(value or '')))
|
|
# Upstream notices are often disguised as movie rows, with new IDs after
|
|
# refreshes. Match their labels rather than IDs or bare English "notice",
|
|
# which would incorrectly hide films such as Red Notice / Notice to Quit.
|
|
label = re.sub(r'[\W_ـ]', '', text)
|
|
if label.startswith((
|
|
'اطلاعیه', 'اطلاعرسانی', 'بهروزرسانی', 'بروزرسانی',
|
|
'اعلانجدید', 'اعلانمهم', 'اعلانفوری', 'اعلامیهجدید', 'اعلامیهمهم',
|
|
'آپدیتجدید', 'اپدیتجدید',
|
|
'newannouncement', 'importantannouncement', 'urgentannouncement',
|
|
'newnotice', 'importantnotice', 'urgentnotice',
|
|
'appupdate', 'applicationupdate', 'softwareupdate')):
|
|
return False
|
|
compact = re.sub(r'[\s_\-ـ]', '', text)
|
|
return not (any(word in compact for word in
|
|
('ملودیباکس', 'ملودیبکس', 'melodybox', 'melodibox', 'نورتون', 'نورتن', 'norton',
|
|
'موویباکس', 'موسویباکس', 'نسخهجدیدهورتون', 'اپلیکیشنهورتون', 'نرمافزارهورتون'))
|
|
or 'تلگرام' in text or re.search(r'\btelegram\b|(^|[\s>(])@[a-z][a-z0-9_]{3,}', text)
|
|
or re.search(r'(دانلود|نصب|دریافت).{0,30}نسخه.{0,18}(برنامه|اپلیکیشن|سیمبا|مووی)', text)
|
|
or re.search(r'(install|download|update).{0,24}(latest|new).{0,12}(app|version)', text))
|
|
|
|
|
|
def banner_https(value):
|
|
from urllib.parse import urlsplit
|
|
if (not isinstance(value, str) or len(value) > 2048
|
|
or re.search(r'[\x00-\x20\x7f\\]', value)
|
|
or re.search(r'%(?![0-9a-f]{2})|%(?:0[0-9a-f]|1[0-9a-f]|7f)', value, re.I)):
|
|
return False
|
|
try:
|
|
uri = urlsplit(value)
|
|
return (uri.scheme == 'https' and bool(uri.hostname) and uri.username is None
|
|
and uri.password is None and uri.port in (None, 443)
|
|
and len(uri.hostname) <= 253
|
|
and all(re.fullmatch(r'[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?', part)
|
|
for part in uri.hostname.split('.')))
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def load_banners(path):
|
|
"""Read a bounded operator-owned file, never a remote config or admin input."""
|
|
try:
|
|
with Path(path).open('rb') as source:
|
|
data = source.read(65537)
|
|
if len(data) > 65536:
|
|
return {'banners': []}
|
|
config = json.loads(data)
|
|
except (OSError, ValueError, RecursionError):
|
|
return {'banners': []}
|
|
if not isinstance(config, dict) or not isinstance(config.get('banners'), list):
|
|
return {'banners': []}
|
|
result = []
|
|
for row in config['banners'][:20]:
|
|
if len(result) == 5:
|
|
break
|
|
if not isinstance(row, dict) or not banner_https(row.get('target_url')):
|
|
continue
|
|
image = row.get('image_url')
|
|
if image not in (None, '') and not banner_https(image):
|
|
continue
|
|
clean = {}
|
|
for key, fallback, limit in (
|
|
('title_en', '', 80), ('title_fa', row.get('title_en', ''), 80),
|
|
('subtitle_en', '', 180), ('subtitle_fa', row.get('subtitle_en', ''), 180)):
|
|
text = row.get(key, fallback)
|
|
if (not isinstance(text, str) or len(text) > limit
|
|
or re.search(r'[\x00-\x1f\x7f\u202a-\u202e\u2066-\u2069]', text)
|
|
or not allowed_text(text)):
|
|
break
|
|
clean[key] = text.strip()
|
|
else:
|
|
if not clean['title_en'] or not clean['title_fa']:
|
|
continue
|
|
clean['target_url'] = row['target_url']
|
|
if image:
|
|
clean['image_url'] = image
|
|
result.append(clean)
|
|
return {'banners': result}
|
|
|
|
|
|
class Catalog:
|
|
RESPONSE_CACHE_BYTES = 8 * 1024 * 1024
|
|
RESPONSE_CACHE_ENTRIES = 128
|
|
RESPONSE_CACHE_ITEM_BYTES = 1024 * 1024
|
|
|
|
def __init__(self, path, banner_config=None):
|
|
self.banner_config = Path(banner_config) if banner_config is not None else Path(__file__).with_name('banners.json')
|
|
self.db = sqlite3.connect(Path(path).resolve().as_uri() + '?mode=ro&immutable=1',
|
|
uri=True, check_same_thread=False)
|
|
self.db.execute('PRAGMA query_only=ON')
|
|
self.db.execute('PRAGMA cache_size=-8192')
|
|
self.lock = threading.RLock()
|
|
self._responses = OrderedDict()
|
|
self._response_bytes = 0
|
|
self.metadata = {k: json.loads(v) for k, v in self.db.execute('SELECT * FROM metadata')}
|
|
# The snapshot is immutable until service restart. Normalize searchable
|
|
# text once instead of redoing Unicode folding for every row/request.
|
|
# Bound retained strings on the 256 MiB service; overflow stays correct
|
|
# through the original normalization path rather than dropping results.
|
|
self.search_text = {}
|
|
self.search_index_bytes = 0
|
|
visible = set()
|
|
for title_id, kind, title, description in self.db.execute(
|
|
'SELECT id,type,title,description FROM titles'):
|
|
if kind not in {'movie', 'serie'} or not allowed_text(title) or not allowed_text(description):
|
|
continue
|
|
visible.add(title_id)
|
|
text = (normal(title), normal(description))
|
|
size = sys.getsizeof(text) + sum(sys.getsizeof(v) for v in text) + sys.getsizeof(title_id) + 64
|
|
if self.search_index_bytes + size <= 64 * 1024 * 1024:
|
|
self.search_text[title_id] = text
|
|
self.search_index_bytes += size
|
|
self.visible = frozenset(visible)
|
|
self.db.create_function('visible', 1, lambda value: value in self.visible, deterministic=True)
|
|
self.db.create_function('search_match', 4, self.search_match, deterministic=True)
|
|
tables = ('titles', 'genres', 'countries', 'movie_sources', 'seasons', 'episodes', 'episode_sources')
|
|
self.counts = {name: self.db.execute('SELECT COUNT(*) FROM ' + name).fetchone()[0] for name in tables}
|
|
self.counts['visible_titles'] = len(self.visible)
|
|
|
|
def search_match(self, title_id, title, description, query):
|
|
text = self.search_text.get(title_id)
|
|
if text is None:
|
|
text = (normal(title), normal(description))
|
|
return query in text[0] or query in text[1]
|
|
|
|
def close(self):
|
|
self.db.close()
|
|
|
|
def health(self):
|
|
built = self.metadata.get('built_at_utc')
|
|
age = None
|
|
if built:
|
|
age = max(0, int((datetime.now(timezone.utc) - datetime.fromisoformat(built)).total_seconds()))
|
|
return {'status': 'ok', 'counts': self.counts.copy(),
|
|
'snapshot_built_at_utc': built, 'snapshot_age_seconds': age,
|
|
'provenance': self.metadata.get('provenance', 'Exported API snapshot'),
|
|
'offline': True, 'media_proxy': False,
|
|
'snapshot_limitations': ['Not live-synchronised', 'Media URLs may expire',
|
|
'Cast, public lists and subtitle metadata were not exported']}
|
|
|
|
@staticmethod
|
|
def clean(value):
|
|
if isinstance(value, dict):
|
|
return {k: Catalog.clean(v) for k, v in value.items()
|
|
if k.lower() not in {'comment', 'comments', 'comments_enabled', 'replies'}}
|
|
if isinstance(value, list):
|
|
return [Catalog.clean(v) for v in value]
|
|
return value
|
|
|
|
def rows(self, sql, params=()):
|
|
return [self.clean(json.loads(row[0])) for row in self.db.execute(sql, params)]
|
|
|
|
def tags(self, name):
|
|
columns = 'id,title,image' if name == 'countries' else 'id,title'
|
|
cursor = self.db.execute('SELECT ' + columns + ' FROM ' + name + ' ORDER BY id')
|
|
keys = [column[0] for column in cursor.description]
|
|
return [dict(zip(keys, row)) for row in cursor
|
|
if allowed_text(row[1]) and not (name == 'genres' and row[0] == 48)]
|
|
|
|
def titles(self, kind='poster', genre=0, country=0, order='created', page=0, query=None):
|
|
orders = {'created': 't.id DESC', 'updated': 't.id DESC',
|
|
'imdb': 't.imdb DESC,t.id DESC', 'rating': 't.rating DESC,t.id DESC',
|
|
'year': 't.year DESC,t.id DESC', 'title': 't.title COLLATE NOCASE,t.id',
|
|
'random': '(t.id * 2654435761) % 2147483647,t.id'}
|
|
if order not in orders or kind not in {'movie', 'serie', 'poster'} or not 0 <= page <= 100000:
|
|
raise ApiError(400, 'Unsupported catalog filter')
|
|
where, params = ['visible(t.id)'], []
|
|
if kind != 'poster':
|
|
where.append('t.type=?')
|
|
params.append(kind)
|
|
for table, column, value in [('title_genres', 'genre_id', genre),
|
|
('title_countries', 'country_id', country)]:
|
|
if value:
|
|
where.append('EXISTS (SELECT 1 FROM ' + table + ' f WHERE f.title_id=t.id AND f.' + column + '=?)')
|
|
params.append(value)
|
|
if query is not None:
|
|
where.append('search_match(t.id,t.title,t.description,?)')
|
|
params.append(normal(query))
|
|
return self.rows('SELECT t.raw_json FROM titles t WHERE ' + ' AND '.join(where)
|
|
+ ' ORDER BY ' + orders[order] + ' LIMIT 30 OFFSET ?', params + [page * 30])
|
|
|
|
def home(self):
|
|
return {'genres': [
|
|
{'id': -2, 'title': 'Latest in catalogue', 'posters': self.titles()},
|
|
{'id': -3, 'title': 'Highest IMDb ratings', 'posters': self.titles(order='imdb')},
|
|
{'id': -4, 'title': 'Series in catalogue', 'posters': self.titles(kind='serie')}
|
|
], 'actors': [], 'countries': self.tags('countries'), 'slides': []}
|
|
|
|
def legacy(self, path, query):
|
|
from urllib.parse import parse_qs
|
|
try:
|
|
raw = parse_qs(query, keep_blank_values=True, max_num_fields=8)
|
|
except ValueError:
|
|
raise ApiError(400, 'Invalid query') from None
|
|
if any(len(values) != 1 for values in raw.values()):
|
|
raise ApiError(400, 'Repeated query parameter')
|
|
args = {key: values[0] for key, values in raw.items()}
|
|
allowed = {'action', 'q', 'pageno'}
|
|
if path.endswith('/filter.php'):
|
|
allowed = {'action', 'garid', 'pageno'}
|
|
elif path.endswith('/star.php'):
|
|
allowed = {'key', 'action', 'q', 'pageno'}
|
|
elif path.endswith(('/pishnahad.php', '/bartar.php')):
|
|
allowed = set()
|
|
if args.keys() - allowed or len(args.get('q', '')) > 200 or len(args.get('key', '')) > 256:
|
|
raise ApiError(400, 'Unsupported query parameter')
|
|
try:
|
|
page = int(args.get('pageno') or 1) - 1
|
|
genre = int(args.get('garid') or 0)
|
|
except ValueError:
|
|
raise ApiError(400, 'Invalid page or genre') from None
|
|
if not 0 <= page <= 100000 or not 0 <= genre <= 999999999:
|
|
raise ApiError(400, 'Invalid page or genre')
|
|
action = args.get('action', '')
|
|
if path.endswith(('/pishnahad.php', '/bartar.php')):
|
|
return [{'title': [tag['title']],
|
|
'link': ['/api-user/newapi/filter.php?action=pishnahad&garid=' + str(tag['id']) + '&pageno=']}
|
|
for tag in self.tags('genres')
|
|
if self.db.execute('SELECT 1 FROM title_genres WHERE genre_id=? AND visible(title_id) LIMIT 1', (tag['id'],)).fetchone()]
|
|
if path == '/api-user/newapi/filter.php' and action == 'pishnahad':
|
|
return self.titles(genre=genre, order='imdb', page=page)
|
|
if path == '/api-user/newapi/like.php':
|
|
if action in {'search-movie', 'search-movie-en'}:
|
|
return self.titles(query=args.get('q', '').strip(), page=page) if args.get('q', '').strip() else []
|
|
if action in {'best-rand-top', 'best-rand-year'}:
|
|
return self.titles(order='year' if action.endswith('year') else 'imdb', page=page)
|
|
if action in {'all-actor', 'colection', 'search-trand'}:
|
|
return []
|
|
if path == '/api-user/filmahang/star.php' and action in {'starsearch', 'mazmon', 'share', 'new'}:
|
|
return []
|
|
raise ApiError(404, 'Metadata route not available in snapshot')
|
|
|
|
def response(self, target):
|
|
# Only immutable REST snapshot metadata. Health/banner freshness and
|
|
# search privacy are preserved: no search query is retained in this LRU.
|
|
cacheable = (target.startswith('/api/') and '?' not in target
|
|
and target.rstrip('/') not in {'/api/health', '/api/banners'})
|
|
with self.lock:
|
|
cached = self._responses.get(target) if cacheable else None
|
|
if cached is not None:
|
|
self._responses.move_to_end(target)
|
|
return cached
|
|
body = json.dumps(self.dispatch(target), ensure_ascii=False,
|
|
separators=(',', ':')).encode('utf-8')
|
|
if len(body) > 32 * 1024 * 1024:
|
|
raise ApiError(503, 'Metadata response exceeds safety limit')
|
|
size = len(body) + len(target.encode('utf-8'))
|
|
if (cacheable and size <= self.RESPONSE_CACHE_ITEM_BYTES
|
|
and size <= self.RESPONSE_CACHE_BYTES and self.RESPONSE_CACHE_ENTRIES > 0):
|
|
while self._responses and (
|
|
self._response_bytes + size > self.RESPONSE_CACHE_BYTES
|
|
or len(self._responses) >= self.RESPONSE_CACHE_ENTRIES):
|
|
old_target, old_body = self._responses.popitem(last=False)
|
|
self._response_bytes -= len(old_body) + len(old_target.encode('utf-8'))
|
|
self._responses[target] = body
|
|
self._response_bytes += size
|
|
return body
|
|
|
|
def dispatch(self, target):
|
|
from urllib.parse import urlsplit
|
|
if len(target) > 4096 or re.search(r'[\x00-\x20\x7f]', target):
|
|
raise ApiError(400, 'Invalid metadata route')
|
|
try:
|
|
parsed = urlsplit(target)
|
|
except ValueError:
|
|
raise ApiError(400, 'Invalid metadata route') from None
|
|
path = parsed.path
|
|
if (len(target) > 4096 or parsed.scheme or parsed.netloc or parsed.fragment
|
|
or '%' in path or '\\' in path or re.search(r'[\x00-\x20\x7f]', path)
|
|
or re.search(r'(^|/)\.{1,2}(/|$)', path)):
|
|
raise ApiError(400, 'Invalid metadata route')
|
|
key = r'[A-Za-z0-9_-]{0,256}'
|
|
with self.lock:
|
|
if path in {'/api-user/newapi/like.php', '/api-user/newapi/filter.php',
|
|
'/api-user/newapi/pishnahad.php', '/api-user/newapi/bartar.php',
|
|
'/api-user/filmahang/star.php'}:
|
|
return self.legacy(path, parsed.query)
|
|
if parsed.query:
|
|
raise ApiError(400, 'Unexpected query parameters')
|
|
if path in {'/api/banners', '/api/banners/'}:
|
|
return load_banners(self.banner_config)
|
|
if path in {'/api/health', '/api/health/'}:
|
|
return self.health()
|
|
if re.fullmatch(r'/api/first/' + key + '/', path):
|
|
return self.home()
|
|
if re.fullmatch(r'/api/genre/all/' + key + '/', path):
|
|
return self.tags('genres')
|
|
if re.fullmatch(r'/api/country/all/' + key + '/', path):
|
|
return self.tags('countries')
|
|
match = re.fullmatch(r'/api/(movie|serie|poster)/by/filtres/(\d{1,9})/(?:(\d{1,9})/)?([a-z]+)/([0-9]{1,6})/' + key + '/', path)
|
|
if match:
|
|
kind, genre, country, order, page = match.groups()
|
|
return self.titles(kind, int(genre), int(country or 0), order, int(page))
|
|
match = re.fullmatch(r'/api/movie/by/([0-9]{1,10})/' + key + '/', path)
|
|
if match:
|
|
title_id = int(match[1])
|
|
if title_id not in self.visible:
|
|
raise ApiError(404, 'Title not in snapshot')
|
|
item = self.rows('SELECT raw_json FROM titles WHERE id=?', (title_id,))[0]
|
|
item['sources'] = self.rows('SELECT raw_json FROM movie_sources WHERE title_id=? ORDER BY source_index', (title_id,))
|
|
return item
|
|
match = re.fullmatch(r'/api/season/by/serie/([0-9]{1,10})/' + key + '/', path)
|
|
if match:
|
|
title_id = int(match[1])
|
|
if title_id not in self.visible:
|
|
raise ApiError(404, 'Title not in snapshot')
|
|
return self.rows('SELECT raw_json FROM seasons WHERE series_id=? ORDER BY season_index', (title_id,))
|
|
if re.fullmatch(r'/api/(?:role/by/poster|movie/by/actor|subtitles/by/(?:movie|episode))/[0-9]{1,10}/' + key + '/', path):
|
|
return []
|
|
raise ApiError(404, 'Unknown metadata route')
|
|
|
|
|
|
def make_server(catalog, host='127.0.0.1', port=18765):
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
protocol_version = 'HTTP/1.0'
|
|
|
|
def setup(self):
|
|
self.request.settimeout(10)
|
|
super().setup()
|
|
|
|
def log_message(self, *_args):
|
|
pass # Never record source links, search terms or shared route keys.
|
|
|
|
def reply(self, status, value):
|
|
body = value if isinstance(value, bytes) else json.dumps(
|
|
value, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
|
|
if len(body) > 32 * 1024 * 1024:
|
|
status, body = 503, b'{"error":"Metadata response exceeds safety limit"}'
|
|
self.send_response_only(status)
|
|
self.send_header('Content-Type', 'application/json; charset=utf-8')
|
|
self.send_header('Content-Length', str(len(body)))
|
|
self.send_header('X-Content-Type-Options', 'nosniff')
|
|
self.send_header('Cache-Control', 'no-store')
|
|
self.send_header('Connection', 'close')
|
|
self.end_headers()
|
|
if self.command != 'HEAD':
|
|
self.wfile.write(body)
|
|
self.close_connection = True
|
|
|
|
def do_GET(self):
|
|
try:
|
|
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
|
|
raise ApiError(400, 'Request body not supported')
|
|
self.reply(200, catalog.response(self.path))
|
|
except ApiError as error:
|
|
self.reply(error.status, {'error': str(error)})
|
|
except (BrokenPipeError, ConnectionResetError, TimeoutError):
|
|
self.close_connection = True
|
|
except Exception:
|
|
self.reply(503, {'error': 'Catalogue temporarily unavailable'})
|
|
|
|
do_HEAD = do_GET
|
|
|
|
def reject_write(self):
|
|
self.reply(405, {'error': 'Read-only metadata API'})
|
|
|
|
do_POST = do_PUT = do_DELETE = do_PATCH = do_CONNECT = do_OPTIONS = do_TRACE = reject_write
|
|
|
|
class Server(ThreadingHTTPServer):
|
|
daemon_threads = True
|
|
request_queue_size = 16
|
|
|
|
def __init__(self, address):
|
|
self.slots = threading.BoundedSemaphore(16)
|
|
super().__init__(address, Handler)
|
|
|
|
def process_request(self, request, client_address):
|
|
if not self.slots.acquire(blocking=False):
|
|
self.shutdown_request(request)
|
|
return
|
|
try:
|
|
super().process_request(request, client_address)
|
|
except Exception:
|
|
self.slots.release()
|
|
raise
|
|
|
|
def process_request_thread(self, request, client_address):
|
|
try:
|
|
super().process_request_thread(request, client_address)
|
|
finally:
|
|
self.slots.release()
|
|
|
|
if host != '127.0.0.1':
|
|
raise ValueError('Bind behind the HTTPS reverse proxy on IPv4 loopback only')
|
|
return Server((host, port))
|
|
|
|
|
|
def main():
|
|
import os
|
|
catalog = Catalog(os.environ.get('REELBOX_CATALOG_DB', '/var/lib/reelbox-api/catalog.sqlite'),
|
|
banner_config=os.environ.get('REELBOX_BANNER_CONFIG', '/var/lib/reelbox-api/banners.json'))
|
|
server = make_server(catalog)
|
|
try:
|
|
server.serve_forever()
|
|
finally:
|
|
server.server_close()
|
|
catalog.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|