[verified] Publish ReelBox source under GPLv3
This commit is contained in:
Hermes committed
commit
db2b505ca3
445 files changed
+105274
No files matched your search
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"banners": []
|
||||
}
|
||||
@@ -0,0 +1,431 @@
|
||||
"""Read-only ReelBox metadata service. No upstream or media fetches."""
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import threading
|
||||
import unicodedata
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, message):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
|
||||
|
||||
_FOLD = dict.fromkeys(i for i in range(0x110000)
|
||||
if unicodedata.category(chr(i)) in {'Mn', 'Cf'})
|
||||
_FOLD.update({ord('ي'): 'ی', ord('ى'): 'ی', ord('ك'): 'ک'})
|
||||
|
||||
|
||||
def normal(value):
|
||||
return str(value or '').lower().translate(_FOLD)
|
||||
|
||||
|
||||
def allowed_text(value):
|
||||
text = normal(unicodedata.normalize('NFKC', str(value or '')))
|
||||
# Upstream notices are often disguised as movie rows, with new IDs after
|
||||
# refreshes. Match their labels rather than IDs or bare English "notice",
|
||||
# which would incorrectly hide films such as Red Notice / Notice to Quit.
|
||||
label = re.sub(r'[\W_ـ]', '', text)
|
||||
if label.startswith((
|
||||
'اطلاعیه', 'اطلاعرسانی', 'بهروزرسانی', 'بروزرسانی',
|
||||
'اعلانجدید', 'اعلانمهم', 'اعلانفوری', 'اعلامیهجدید', 'اعلامیهمهم',
|
||||
'آپدیتجدید', 'اپدیتجدید',
|
||||
'newannouncement', 'importantannouncement', 'urgentannouncement',
|
||||
'newnotice', 'importantnotice', 'urgentnotice',
|
||||
'appupdate', 'applicationupdate', 'softwareupdate')):
|
||||
return False
|
||||
compact = re.sub(r'[\s_\-ـ]', '', text)
|
||||
return not (any(word in compact for word in
|
||||
('ملودیباکس', 'ملودیبکس', 'melodybox', 'melodibox', 'نورتون', 'نورتن', 'norton',
|
||||
'موویباکس', 'موسویباکس', 'نسخهجدیدهورتون', 'اپلیکیشنهورتون', 'نرمافزارهورتون'))
|
||||
or 'تلگرام' in text or re.search(r'\btelegram\b|(^|[\s>(])@[a-z][a-z0-9_]{3,}', text)
|
||||
or re.search(r'(دانلود|نصب|دریافت).{0,30}نسخه.{0,18}(برنامه|اپلیکیشن|سیمبا|مووی)', text)
|
||||
or re.search(r'(install|download|update).{0,24}(latest|new).{0,12}(app|version)', text))
|
||||
|
||||
|
||||
def banner_https(value):
|
||||
from urllib.parse import urlsplit
|
||||
if (not isinstance(value, str) or len(value) > 2048
|
||||
or re.search(r'[\x00-\x20\x7f\\]', value)
|
||||
or re.search(r'%(?![0-9a-f]{2})|%(?:0[0-9a-f]|1[0-9a-f]|7f)', value, re.I)):
|
||||
return False
|
||||
try:
|
||||
uri = urlsplit(value)
|
||||
return (uri.scheme == 'https' and bool(uri.hostname) and uri.username is None
|
||||
and uri.password is None and uri.port in (None, 443)
|
||||
and len(uri.hostname) <= 253
|
||||
and all(re.fullmatch(r'[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?', part)
|
||||
for part in uri.hostname.split('.')))
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def load_banners(path):
|
||||
"""Read a bounded operator-owned file, never a remote config or admin input."""
|
||||
try:
|
||||
with Path(path).open('rb') as source:
|
||||
data = source.read(65537)
|
||||
if len(data) > 65536:
|
||||
return {'banners': []}
|
||||
config = json.loads(data)
|
||||
except (OSError, ValueError, RecursionError):
|
||||
return {'banners': []}
|
||||
if not isinstance(config, dict) or not isinstance(config.get('banners'), list):
|
||||
return {'banners': []}
|
||||
result = []
|
||||
for row in config['banners'][:20]:
|
||||
if len(result) == 5:
|
||||
break
|
||||
if not isinstance(row, dict) or not banner_https(row.get('target_url')):
|
||||
continue
|
||||
image = row.get('image_url')
|
||||
if image not in (None, '') and not banner_https(image):
|
||||
continue
|
||||
clean = {}
|
||||
for key, fallback, limit in (
|
||||
('title_en', '', 80), ('title_fa', row.get('title_en', ''), 80),
|
||||
('subtitle_en', '', 180), ('subtitle_fa', row.get('subtitle_en', ''), 180)):
|
||||
text = row.get(key, fallback)
|
||||
if (not isinstance(text, str) or len(text) > limit
|
||||
or re.search(r'[\x00-\x1f\x7f\u202a-\u202e\u2066-\u2069]', text)
|
||||
or not allowed_text(text)):
|
||||
break
|
||||
clean[key] = text.strip()
|
||||
else:
|
||||
if not clean['title_en'] or not clean['title_fa']:
|
||||
continue
|
||||
clean['target_url'] = row['target_url']
|
||||
if image:
|
||||
clean['image_url'] = image
|
||||
result.append(clean)
|
||||
return {'banners': result}
|
||||
|
||||
|
||||
class Catalog:
|
||||
RESPONSE_CACHE_BYTES = 8 * 1024 * 1024
|
||||
RESPONSE_CACHE_ENTRIES = 128
|
||||
RESPONSE_CACHE_ITEM_BYTES = 1024 * 1024
|
||||
|
||||
def __init__(self, path, banner_config=None):
|
||||
self.banner_config = Path(banner_config) if banner_config is not None else Path(__file__).with_name('banners.json')
|
||||
self.db = sqlite3.connect(Path(path).resolve().as_uri() + '?mode=ro&immutable=1',
|
||||
uri=True, check_same_thread=False)
|
||||
self.db.execute('PRAGMA query_only=ON')
|
||||
self.db.execute('PRAGMA cache_size=-8192')
|
||||
self.lock = threading.RLock()
|
||||
self._responses = OrderedDict()
|
||||
self._response_bytes = 0
|
||||
self.metadata = {k: json.loads(v) for k, v in self.db.execute('SELECT * FROM metadata')}
|
||||
# The snapshot is immutable until service restart. Normalize searchable
|
||||
# text once instead of redoing Unicode folding for every row/request.
|
||||
# Bound retained strings on the 256 MiB service; overflow stays correct
|
||||
# through the original normalization path rather than dropping results.
|
||||
self.search_text = {}
|
||||
self.search_index_bytes = 0
|
||||
visible = set()
|
||||
for title_id, kind, title, description in self.db.execute(
|
||||
'SELECT id,type,title,description FROM titles'):
|
||||
if kind not in {'movie', 'serie'} or not allowed_text(title) or not allowed_text(description):
|
||||
continue
|
||||
visible.add(title_id)
|
||||
text = (normal(title), normal(description))
|
||||
size = sys.getsizeof(text) + sum(sys.getsizeof(v) for v in text) + sys.getsizeof(title_id) + 64
|
||||
if self.search_index_bytes + size <= 64 * 1024 * 1024:
|
||||
self.search_text[title_id] = text
|
||||
self.search_index_bytes += size
|
||||
self.visible = frozenset(visible)
|
||||
self.db.create_function('visible', 1, lambda value: value in self.visible, deterministic=True)
|
||||
self.db.create_function('search_match', 4, self.search_match, deterministic=True)
|
||||
tables = ('titles', 'genres', 'countries', 'movie_sources', 'seasons', 'episodes', 'episode_sources')
|
||||
self.counts = {name: self.db.execute('SELECT COUNT(*) FROM ' + name).fetchone()[0] for name in tables}
|
||||
self.counts['visible_titles'] = len(self.visible)
|
||||
|
||||
def search_match(self, title_id, title, description, query):
|
||||
text = self.search_text.get(title_id)
|
||||
if text is None:
|
||||
text = (normal(title), normal(description))
|
||||
return query in text[0] or query in text[1]
|
||||
|
||||
def close(self):
|
||||
self.db.close()
|
||||
|
||||
def health(self):
|
||||
built = self.metadata.get('built_at_utc')
|
||||
age = None
|
||||
if built:
|
||||
age = max(0, int((datetime.now(timezone.utc) - datetime.fromisoformat(built)).total_seconds()))
|
||||
return {'status': 'ok', 'counts': self.counts.copy(),
|
||||
'snapshot_built_at_utc': built, 'snapshot_age_seconds': age,
|
||||
'provenance': self.metadata.get('provenance', 'Exported API snapshot'),
|
||||
'offline': True, 'media_proxy': False,
|
||||
'snapshot_limitations': ['Not live-synchronised', 'Media URLs may expire',
|
||||
'Cast, public lists and subtitle metadata were not exported']}
|
||||
|
||||
@staticmethod
|
||||
def clean(value):
|
||||
if isinstance(value, dict):
|
||||
return {k: Catalog.clean(v) for k, v in value.items()
|
||||
if k.lower() not in {'comment', 'comments', 'comments_enabled', 'replies'}}
|
||||
if isinstance(value, list):
|
||||
return [Catalog.clean(v) for v in value]
|
||||
return value
|
||||
|
||||
def rows(self, sql, params=()):
|
||||
return [self.clean(json.loads(row[0])) for row in self.db.execute(sql, params)]
|
||||
|
||||
def tags(self, name):
|
||||
columns = 'id,title,image' if name == 'countries' else 'id,title'
|
||||
cursor = self.db.execute('SELECT ' + columns + ' FROM ' + name + ' ORDER BY id')
|
||||
keys = [column[0] for column in cursor.description]
|
||||
return [dict(zip(keys, row)) for row in cursor
|
||||
if allowed_text(row[1]) and not (name == 'genres' and row[0] == 48)]
|
||||
|
||||
def titles(self, kind='poster', genre=0, country=0, order='created', page=0, query=None):
|
||||
orders = {'created': 't.id DESC', 'updated': 't.id DESC',
|
||||
'imdb': 't.imdb DESC,t.id DESC', 'rating': 't.rating DESC,t.id DESC',
|
||||
'year': 't.year DESC,t.id DESC', 'title': 't.title COLLATE NOCASE,t.id',
|
||||
'random': '(t.id * 2654435761) % 2147483647,t.id'}
|
||||
if order not in orders or kind not in {'movie', 'serie', 'poster'} or not 0 <= page <= 100000:
|
||||
raise ApiError(400, 'Unsupported catalog filter')
|
||||
where, params = ['visible(t.id)'], []
|
||||
if kind != 'poster':
|
||||
where.append('t.type=?')
|
||||
params.append(kind)
|
||||
for table, column, value in [('title_genres', 'genre_id', genre),
|
||||
('title_countries', 'country_id', country)]:
|
||||
if value:
|
||||
where.append('EXISTS (SELECT 1 FROM ' + table + ' f WHERE f.title_id=t.id AND f.' + column + '=?)')
|
||||
params.append(value)
|
||||
if query is not None:
|
||||
where.append('search_match(t.id,t.title,t.description,?)')
|
||||
params.append(normal(query))
|
||||
return self.rows('SELECT t.raw_json FROM titles t WHERE ' + ' AND '.join(where)
|
||||
+ ' ORDER BY ' + orders[order] + ' LIMIT 30 OFFSET ?', params + [page * 30])
|
||||
|
||||
def home(self):
|
||||
return {'genres': [
|
||||
{'id': -2, 'title': 'Latest in catalogue', 'posters': self.titles()},
|
||||
{'id': -3, 'title': 'Highest IMDb ratings', 'posters': self.titles(order='imdb')},
|
||||
{'id': -4, 'title': 'Series in catalogue', 'posters': self.titles(kind='serie')}
|
||||
], 'actors': [], 'countries': self.tags('countries'), 'slides': []}
|
||||
|
||||
def legacy(self, path, query):
|
||||
from urllib.parse import parse_qs
|
||||
try:
|
||||
raw = parse_qs(query, keep_blank_values=True, max_num_fields=8)
|
||||
except ValueError:
|
||||
raise ApiError(400, 'Invalid query') from None
|
||||
if any(len(values) != 1 for values in raw.values()):
|
||||
raise ApiError(400, 'Repeated query parameter')
|
||||
args = {key: values[0] for key, values in raw.items()}
|
||||
allowed = {'action', 'q', 'pageno'}
|
||||
if path.endswith('/filter.php'):
|
||||
allowed = {'action', 'garid', 'pageno'}
|
||||
elif path.endswith('/star.php'):
|
||||
allowed = {'key', 'action', 'q', 'pageno'}
|
||||
elif path.endswith(('/pishnahad.php', '/bartar.php')):
|
||||
allowed = set()
|
||||
if args.keys() - allowed or len(args.get('q', '')) > 200 or len(args.get('key', '')) > 256:
|
||||
raise ApiError(400, 'Unsupported query parameter')
|
||||
try:
|
||||
page = int(args.get('pageno') or 1) - 1
|
||||
genre = int(args.get('garid') or 0)
|
||||
except ValueError:
|
||||
raise ApiError(400, 'Invalid page or genre') from None
|
||||
if not 0 <= page <= 100000 or not 0 <= genre <= 999999999:
|
||||
raise ApiError(400, 'Invalid page or genre')
|
||||
action = args.get('action', '')
|
||||
if path.endswith(('/pishnahad.php', '/bartar.php')):
|
||||
return [{'title': [tag['title']],
|
||||
'link': ['/api-user/newapi/filter.php?action=pishnahad&garid=' + str(tag['id']) + '&pageno=']}
|
||||
for tag in self.tags('genres')
|
||||
if self.db.execute('SELECT 1 FROM title_genres WHERE genre_id=? AND visible(title_id) LIMIT 1', (tag['id'],)).fetchone()]
|
||||
if path == '/api-user/newapi/filter.php' and action == 'pishnahad':
|
||||
return self.titles(genre=genre, order='imdb', page=page)
|
||||
if path == '/api-user/newapi/like.php':
|
||||
if action in {'search-movie', 'search-movie-en'}:
|
||||
return self.titles(query=args.get('q', '').strip(), page=page) if args.get('q', '').strip() else []
|
||||
if action in {'best-rand-top', 'best-rand-year'}:
|
||||
return self.titles(order='year' if action.endswith('year') else 'imdb', page=page)
|
||||
if action in {'all-actor', 'colection', 'search-trand'}:
|
||||
return []
|
||||
if path == '/api-user/filmahang/star.php' and action in {'starsearch', 'mazmon', 'share', 'new'}:
|
||||
return []
|
||||
raise ApiError(404, 'Metadata route not available in snapshot')
|
||||
|
||||
def response(self, target):
|
||||
# Only immutable REST snapshot metadata. Health/banner freshness and
|
||||
# search privacy are preserved: no search query is retained in this LRU.
|
||||
cacheable = (target.startswith('/api/') and '?' not in target
|
||||
and target.rstrip('/') not in {'/api/health', '/api/banners'})
|
||||
with self.lock:
|
||||
cached = self._responses.get(target) if cacheable else None
|
||||
if cached is not None:
|
||||
self._responses.move_to_end(target)
|
||||
return cached
|
||||
body = json.dumps(self.dispatch(target), ensure_ascii=False,
|
||||
separators=(',', ':')).encode('utf-8')
|
||||
if len(body) > 32 * 1024 * 1024:
|
||||
raise ApiError(503, 'Metadata response exceeds safety limit')
|
||||
size = len(body) + len(target.encode('utf-8'))
|
||||
if (cacheable and size <= self.RESPONSE_CACHE_ITEM_BYTES
|
||||
and size <= self.RESPONSE_CACHE_BYTES and self.RESPONSE_CACHE_ENTRIES > 0):
|
||||
while self._responses and (
|
||||
self._response_bytes + size > self.RESPONSE_CACHE_BYTES
|
||||
or len(self._responses) >= self.RESPONSE_CACHE_ENTRIES):
|
||||
old_target, old_body = self._responses.popitem(last=False)
|
||||
self._response_bytes -= len(old_body) + len(old_target.encode('utf-8'))
|
||||
self._responses[target] = body
|
||||
self._response_bytes += size
|
||||
return body
|
||||
|
||||
def dispatch(self, target):
|
||||
from urllib.parse import urlsplit
|
||||
if len(target) > 4096 or re.search(r'[\x00-\x20\x7f]', target):
|
||||
raise ApiError(400, 'Invalid metadata route')
|
||||
try:
|
||||
parsed = urlsplit(target)
|
||||
except ValueError:
|
||||
raise ApiError(400, 'Invalid metadata route') from None
|
||||
path = parsed.path
|
||||
if (len(target) > 4096 or parsed.scheme or parsed.netloc or parsed.fragment
|
||||
or '%' in path or '\\' in path or re.search(r'[\x00-\x20\x7f]', path)
|
||||
or re.search(r'(^|/)\.{1,2}(/|$)', path)):
|
||||
raise ApiError(400, 'Invalid metadata route')
|
||||
key = r'[A-Za-z0-9_-]{0,256}'
|
||||
with self.lock:
|
||||
if path in {'/api-user/newapi/like.php', '/api-user/newapi/filter.php',
|
||||
'/api-user/newapi/pishnahad.php', '/api-user/newapi/bartar.php',
|
||||
'/api-user/filmahang/star.php'}:
|
||||
return self.legacy(path, parsed.query)
|
||||
if parsed.query:
|
||||
raise ApiError(400, 'Unexpected query parameters')
|
||||
if path in {'/api/banners', '/api/banners/'}:
|
||||
return load_banners(self.banner_config)
|
||||
if path in {'/api/health', '/api/health/'}:
|
||||
return self.health()
|
||||
if re.fullmatch(r'/api/first/' + key + '/', path):
|
||||
return self.home()
|
||||
if re.fullmatch(r'/api/genre/all/' + key + '/', path):
|
||||
return self.tags('genres')
|
||||
if re.fullmatch(r'/api/country/all/' + key + '/', path):
|
||||
return self.tags('countries')
|
||||
match = re.fullmatch(r'/api/(movie|serie|poster)/by/filtres/(\d{1,9})/(?:(\d{1,9})/)?([a-z]+)/([0-9]{1,6})/' + key + '/', path)
|
||||
if match:
|
||||
kind, genre, country, order, page = match.groups()
|
||||
return self.titles(kind, int(genre), int(country or 0), order, int(page))
|
||||
match = re.fullmatch(r'/api/movie/by/([0-9]{1,10})/' + key + '/', path)
|
||||
if match:
|
||||
title_id = int(match[1])
|
||||
if title_id not in self.visible:
|
||||
raise ApiError(404, 'Title not in snapshot')
|
||||
item = self.rows('SELECT raw_json FROM titles WHERE id=?', (title_id,))[0]
|
||||
item['sources'] = self.rows('SELECT raw_json FROM movie_sources WHERE title_id=? ORDER BY source_index', (title_id,))
|
||||
return item
|
||||
match = re.fullmatch(r'/api/season/by/serie/([0-9]{1,10})/' + key + '/', path)
|
||||
if match:
|
||||
title_id = int(match[1])
|
||||
if title_id not in self.visible:
|
||||
raise ApiError(404, 'Title not in snapshot')
|
||||
return self.rows('SELECT raw_json FROM seasons WHERE series_id=? ORDER BY season_index', (title_id,))
|
||||
if re.fullmatch(r'/api/(?:role/by/poster|movie/by/actor|subtitles/by/(?:movie|episode))/[0-9]{1,10}/' + key + '/', path):
|
||||
return []
|
||||
raise ApiError(404, 'Unknown metadata route')
|
||||
|
||||
|
||||
def make_server(catalog, host='127.0.0.1', port=18765):
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = 'HTTP/1.0'
|
||||
|
||||
def setup(self):
|
||||
self.request.settimeout(10)
|
||||
super().setup()
|
||||
|
||||
def log_message(self, *_args):
|
||||
pass # Never record source links, search terms or shared route keys.
|
||||
|
||||
def reply(self, status, value):
|
||||
body = value if isinstance(value, bytes) else json.dumps(
|
||||
value, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
|
||||
if len(body) > 32 * 1024 * 1024:
|
||||
status, body = 503, b'{"error":"Metadata response exceeds safety limit"}'
|
||||
self.send_response_only(status)
|
||||
self.send_header('Content-Type', 'application/json; charset=utf-8')
|
||||
self.send_header('Content-Length', str(len(body)))
|
||||
self.send_header('X-Content-Type-Options', 'nosniff')
|
||||
self.send_header('Cache-Control', 'no-store')
|
||||
self.send_header('Connection', 'close')
|
||||
self.end_headers()
|
||||
if self.command != 'HEAD':
|
||||
self.wfile.write(body)
|
||||
self.close_connection = True
|
||||
|
||||
def do_GET(self):
|
||||
try:
|
||||
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
|
||||
raise ApiError(400, 'Request body not supported')
|
||||
self.reply(200, catalog.response(self.path))
|
||||
except ApiError as error:
|
||||
self.reply(error.status, {'error': str(error)})
|
||||
except (BrokenPipeError, ConnectionResetError, TimeoutError):
|
||||
self.close_connection = True
|
||||
except Exception:
|
||||
self.reply(503, {'error': 'Catalogue temporarily unavailable'})
|
||||
|
||||
do_HEAD = do_GET
|
||||
|
||||
def reject_write(self):
|
||||
self.reply(405, {'error': 'Read-only metadata API'})
|
||||
|
||||
do_POST = do_PUT = do_DELETE = do_PATCH = do_CONNECT = do_OPTIONS = do_TRACE = reject_write
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
request_queue_size = 16
|
||||
|
||||
def __init__(self, address):
|
||||
self.slots = threading.BoundedSemaphore(16)
|
||||
super().__init__(address, Handler)
|
||||
|
||||
def process_request(self, request, client_address):
|
||||
if not self.slots.acquire(blocking=False):
|
||||
self.shutdown_request(request)
|
||||
return
|
||||
try:
|
||||
super().process_request(request, client_address)
|
||||
except Exception:
|
||||
self.slots.release()
|
||||
raise
|
||||
|
||||
def process_request_thread(self, request, client_address):
|
||||
try:
|
||||
super().process_request_thread(request, client_address)
|
||||
finally:
|
||||
self.slots.release()
|
||||
|
||||
if host != '127.0.0.1':
|
||||
raise ValueError('Bind behind the HTTPS reverse proxy on IPv4 loopback only')
|
||||
return Server((host, port))
|
||||
|
||||
|
||||
def main():
|
||||
import os
|
||||
catalog = Catalog(os.environ.get('REELBOX_CATALOG_DB', '/var/lib/reelbox-api/catalog.sqlite'),
|
||||
banner_config=os.environ.get('REELBOX_BANNER_CONFIG', '/var/lib/reelbox-api/banners.json'))
|
||||
server = make_server(catalog)
|
||||
try:
|
||||
server.serve_forever()
|
||||
finally:
|
||||
server.server_close()
|
||||
catalog.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,419 @@
|
||||
"""Metadata-only catalogue synchronization; publication is the caller's job.
|
||||
|
||||
Only fixed-origin, allowlisted metadata GETs are possible. Image/video URLs are
|
||||
opaque stored metadata and are never followed. Supply the key from the calling
|
||||
process (e.g. REELBOX_UPSTREAM_KEY). Reports and errors contain no request URLs.
|
||||
"""
|
||||
from contextlib import closing
|
||||
from datetime import datetime, timezone
|
||||
import http.client
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sqlite3
|
||||
import ssl
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import quote, urlsplit
|
||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||
|
||||
|
||||
UPSTREAM_BASE = 'https://ninwinios.info/api/'
|
||||
MAX_RESPONSE_BYTES = 32 * 1024 * 1024
|
||||
_ROUTE = re.compile(r'(?:genre/all|country/all|(?:movie|serie)/by/filtres/0/created/[0-9]{1,6}'
|
||||
r'|movie/by/[1-9][0-9]{0,9}|season/by/serie/[1-9][0-9]{0,9})')
|
||||
|
||||
|
||||
class SyncError(RuntimeError):
|
||||
"""Safe, key-free error suitable for a scheduler log."""
|
||||
|
||||
|
||||
class _NoRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
raise SyncError('Upstream redirect denied')
|
||||
|
||||
|
||||
def _invalid_constant(_value):
|
||||
raise ValueError('Non-finite JSON number')
|
||||
|
||||
|
||||
class Upstream:
|
||||
"""Single-worker rate-limited client. ``attempts`` is total attempts, <=10.
|
||||
|
||||
``opener`` is an injectable trusted urllib-compatible opener for tests.
|
||||
``cache_dir`` is reserved; responses are intentionally not cached on disk,
|
||||
so a subsequent run always refreshes metadata. No response/key is logged.
|
||||
"""
|
||||
def __init__(self, key, interval=0.25, timeout=30, attempts=3, opener=None, cache_dir=None):
|
||||
if not isinstance(key, str) or not key or len(key) > 4096:
|
||||
raise ValueError('Upstream key required')
|
||||
if (type(attempts) is not int or not 1 <= attempts <= 10
|
||||
or not isinstance(interval, (int, float)) or not math.isfinite(interval) or interval < 0
|
||||
or not isinstance(timeout, (int, float)) or not math.isfinite(timeout) or timeout <= 0):
|
||||
raise ValueError('Invalid upstream retry or timing configuration')
|
||||
self._key = key
|
||||
self.interval, self.timeout, self.attempts = interval, timeout, attempts
|
||||
self._opener = opener if opener is not None else build_opener(_NoRedirect())
|
||||
self.requests_count = 0
|
||||
self._last_request = None
|
||||
|
||||
def get(self, path):
|
||||
if not isinstance(path, str) or not _ROUTE.fullmatch(path):
|
||||
raise ValueError('Unsupported upstream metadata route')
|
||||
url = UPSTREAM_BASE + path + '/' + quote(self._key, safe='') + '/'
|
||||
parsed = urlsplit(url)
|
||||
if (parsed.scheme, parsed.netloc, parsed.query, parsed.fragment) != ('https', 'ninwinios.info', '', ''):
|
||||
raise ValueError('Unsupported upstream origin')
|
||||
request = Request(url, headers={'Accept': 'application/json', 'Accept-Encoding': 'identity'}, method='GET')
|
||||
for attempt in range(self.attempts):
|
||||
if self._last_request is not None:
|
||||
delay = self.interval - (time.monotonic() - self._last_request)
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
self._last_request = time.monotonic()
|
||||
self.requests_count += 1
|
||||
try:
|
||||
with self._opener.open(request, timeout=self.timeout) as response:
|
||||
status = response.status
|
||||
if status == 429 or 500 <= status <= 599:
|
||||
raise HTTPError(url, status, 'Upstream temporarily unavailable', {}, None)
|
||||
if status != 200 or response.geturl() != url:
|
||||
raise SyncError('Upstream status or redirect denied')
|
||||
media_type = response.headers.get('Content-Type', '').split(';')[0].strip().lower()
|
||||
# The original PHP API labels valid JSON as text/html.
|
||||
# Only allow this known legacy label; parsing/schema checks still apply.
|
||||
if media_type and media_type not in {'application/json', 'text/html'} and not media_type.endswith('+json'):
|
||||
raise SyncError('Upstream did not return JSON metadata')
|
||||
declared = response.headers.get('Content-Length')
|
||||
if declared is not None:
|
||||
if not re.fullmatch(r'[0-9]{1,12}', declared) or int(declared) > MAX_RESPONSE_BYTES:
|
||||
raise SyncError('Invalid upstream metadata length')
|
||||
body = response.read(MAX_RESPONSE_BYTES + 1)
|
||||
if len(body) > MAX_RESPONSE_BYTES:
|
||||
raise SyncError('Upstream metadata response exceeds safety limit')
|
||||
if declared is not None and len(body) != int(declared):
|
||||
raise SyncError('Truncated upstream metadata response')
|
||||
value = json.loads(body, parse_constant=_invalid_constant)
|
||||
detail = path.startswith('movie/by/') and '/filtres/' not in path
|
||||
if (detail and not isinstance(value, dict)) or (not detail and not isinstance(value, list)):
|
||||
raise SyncError('Unexpected upstream metadata shape')
|
||||
return value
|
||||
except HTTPError as error:
|
||||
retry = error.code == 429 or 500 <= error.code <= 599
|
||||
error.close()
|
||||
if not retry or attempt + 1 == self.attempts:
|
||||
raise SyncError('Upstream HTTP request failed') from None
|
||||
except (ssl.SSLError, ValueError, RecursionError):
|
||||
raise SyncError('Invalid upstream TLS or JSON metadata') from None
|
||||
except (URLError, OSError, http.client.HTTPException) as error:
|
||||
if isinstance(getattr(error, 'reason', None), ssl.SSLError) or attempt + 1 == self.attempts:
|
||||
raise SyncError('Upstream transport request failed') from None
|
||||
if attempt + 1 < self.attempts:
|
||||
time.sleep(min(2 ** attempt, 8))
|
||||
raise SyncError('Upstream request exhausted')
|
||||
|
||||
|
||||
try:
|
||||
from .catalog_api import allowed_text
|
||||
except ImportError: # Also support `PYTHONPATH=server; import catalog_sync`.
|
||||
from catalog_api import allowed_text
|
||||
|
||||
|
||||
def _encode(value):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False)
|
||||
|
||||
|
||||
def _id(value):
|
||||
if type(value) is not int or not 1 <= value <= 9999999999:
|
||||
raise SyncError('Malformed metadata identifier')
|
||||
return value
|
||||
|
||||
|
||||
def _objects(value):
|
||||
if not isinstance(value, list) or any(not isinstance(item, dict) for item in value):
|
||||
raise SyncError('Malformed metadata list')
|
||||
return value
|
||||
|
||||
|
||||
def _merge(old, incoming):
|
||||
result = dict(old)
|
||||
for key, value in incoming.items():
|
||||
if key in {'sources', 'episodes', 'genres', 'country'}:
|
||||
result[key] = _merge_sequence(old.get(key, []), value)
|
||||
elif key not in result or (value is not None and value != ''):
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _merge_sequence(old, incoming):
|
||||
"""Retain old positions; stable IDs win, missing IDs fall back to index."""
|
||||
old, incoming = _objects(old), _objects(incoming)
|
||||
result = list(old)
|
||||
by_id = {item['id']: index for index, item in enumerate(old) if item.get('id') is not None}
|
||||
seen_ids, matched = set(), set()
|
||||
for index, item in enumerate(incoming):
|
||||
identity = item.get('id')
|
||||
if identity is not None:
|
||||
_id(identity)
|
||||
if identity in seen_ids:
|
||||
raise SyncError('Duplicate nested metadata identifier')
|
||||
seen_ids.add(identity)
|
||||
target = by_id.get(identity)
|
||||
if target is None and index < len(old) and old[index].get('id') is None:
|
||||
target = index
|
||||
else:
|
||||
target = index if index < len(old) else None
|
||||
if target is not None:
|
||||
if target in matched:
|
||||
raise SyncError('Ambiguous nested metadata identity')
|
||||
matched.add(target)
|
||||
result[target] = _merge(old[target], item)
|
||||
else:
|
||||
result.append(_merge({}, item))
|
||||
return result
|
||||
|
||||
|
||||
def _validate_record(item, kind, require_title=False):
|
||||
if not isinstance(item, dict) or item.get('error') or item.get('errors'):
|
||||
raise SyncError('Malformed metadata record')
|
||||
if item.get('id') is not None:
|
||||
_id(item['id'])
|
||||
if kind in {'season', 'episode', 'source'} and item.get('id') is None:
|
||||
identity = item.get('url') if kind == 'source' else item.get('title')
|
||||
if not isinstance(identity, str) or not identity.strip():
|
||||
raise SyncError('Nested metadata has no usable identity')
|
||||
if require_title and (not isinstance(item.get('title'), str) or not item['title'].strip()):
|
||||
raise SyncError('New title has no usable name')
|
||||
for field in ('title', 'description', 'duration', 'classification', 'image', 'cover',
|
||||
'downloadas', 'playas', 'type', 'quality', 'url'):
|
||||
if field in item and item[field] is not None and not isinstance(item[field], str):
|
||||
raise SyncError('Malformed text metadata field')
|
||||
for field in ('year', 'imdb', 'rating'):
|
||||
value = item.get(field)
|
||||
if value is not None and (type(value) not in {int, float} or not math.isfinite(value)):
|
||||
raise SyncError('Malformed numeric metadata field')
|
||||
if 'comment' in item and item['comment'] is not None and type(item['comment']) not in {bool, int}:
|
||||
raise SyncError('Malformed comment metadata flag')
|
||||
for field, nested_kind in [('sources', 'source'), ('episodes', 'episode'),
|
||||
('genres', 'reference'), ('country', 'reference')]:
|
||||
if field in item:
|
||||
for nested in _objects(item[field]):
|
||||
_validate_record(nested, nested_kind)
|
||||
|
||||
|
||||
def _reference(db, table, item):
|
||||
_validate_record(item, 'reference')
|
||||
_id(item.get('id'))
|
||||
columns = ['id', 'title', 'image'] if table == 'countries' else ['id', 'title']
|
||||
db.execute('INSERT INTO ' + table + '(' + ','.join(columns) + ') VALUES (' +
|
||||
','.join('?' for _ in columns) + ') ON CONFLICT(id) DO UPDATE SET ' +
|
||||
','.join(c + '=COALESCE(excluded.' + c + ',' + table + '.' + c + ')' for c in columns[1:]),
|
||||
[item.get(c) for c in columns])
|
||||
|
||||
|
||||
def _save_title(db, incoming):
|
||||
title_id = _id(incoming.get('id'))
|
||||
record = db.execute('SELECT raw_json FROM titles WHERE id=?', (title_id,)).fetchone()
|
||||
_validate_record(incoming, 'title', require_title=not record)
|
||||
old = json.loads(record[0]) if record else {}
|
||||
merged = _merge(old, incoming)
|
||||
raw = _encode(merged)
|
||||
if record and old == merged:
|
||||
return False
|
||||
fields = ['id', 'type', 'title', 'description', 'year', 'imdb', 'rating', 'duration',
|
||||
'classification', 'image', 'cover', 'downloadas', 'playas', 'comments_enabled', 'raw_json']
|
||||
values = [merged.get(c) for c in fields]
|
||||
values[-2], values[-1] = merged.get('comment'), raw
|
||||
db.execute('INSERT INTO titles(' + ','.join(fields) + ') VALUES (' + ','.join('?' for _ in fields) +
|
||||
') ON CONFLICT(id) DO UPDATE SET ' + ','.join(c + '=excluded.' + c for c in fields[1:]), values)
|
||||
for field, table, junction, column in [('genres', 'genres', 'title_genres', 'genre_id'),
|
||||
('country', 'countries', 'title_countries', 'country_id')]:
|
||||
for tag in _objects(merged.get(field, [])):
|
||||
_reference(db, table, tag)
|
||||
db.execute('INSERT OR IGNORE INTO ' + junction + '(title_id,' + column + ') VALUES (?,?)',
|
||||
(title_id, tag['id']))
|
||||
for index, source in enumerate(_objects(merged.get('sources', []))):
|
||||
db.execute('INSERT INTO movie_sources VALUES (?,?,?,?,?,?,?) ON CONFLICT(title_id,source_index) '
|
||||
'DO UPDATE SET source_id=excluded.source_id,quality=excluded.quality,'
|
||||
'media_type=excluded.media_type,url=excluded.url,raw_json=excluded.raw_json',
|
||||
(title_id, index, source.get('id'), source.get('quality'), source.get('type'),
|
||||
source.get('url'), _encode(source)))
|
||||
return True
|
||||
|
||||
|
||||
def _save_seasons(db, series_id, incoming):
|
||||
for season in _objects(incoming):
|
||||
_validate_record(season, 'season')
|
||||
previous = db.execute('SELECT season_index,raw_json FROM seasons WHERE series_id=? ORDER BY season_index',
|
||||
(series_id,)).fetchall()
|
||||
old = [json.loads(raw) for _, raw in previous]
|
||||
merged = _merge_sequence(old, incoming)
|
||||
changed = False
|
||||
# Existing indices are retained even if an exported snapshot has gaps.
|
||||
indices = [index for index, _ in previous]
|
||||
next_index = max(indices, default=-1) + 1
|
||||
indices += list(range(next_index, next_index + len(merged) - len(indices)))
|
||||
for position, season in enumerate(merged):
|
||||
if position < len(old) and season == old[position]:
|
||||
continue
|
||||
changed = True
|
||||
si = indices[position]
|
||||
db.execute('INSERT INTO seasons VALUES (?,?,?,?,?) ON CONFLICT(series_id,season_index) DO UPDATE SET '
|
||||
'id=excluded.id,title=excluded.title,raw_json=excluded.raw_json',
|
||||
(series_id, si, season.get('id'), season.get('title'), _encode(season)))
|
||||
for ei, episode in enumerate(_objects(season.get('episodes', []))):
|
||||
raw = _encode(episode)
|
||||
current = db.execute('SELECT raw_json FROM episodes WHERE series_id=? AND season_index=? AND episode_index=?',
|
||||
(series_id, si, ei)).fetchone()
|
||||
if current and json.loads(current[0]) == episode:
|
||||
continue
|
||||
db.execute('INSERT INTO episodes VALUES (?,?,?,?,?,?,?,?,?,?) '
|
||||
'ON CONFLICT(series_id,season_index,episode_index) DO UPDATE SET '
|
||||
'id=excluded.id,title=excluded.title,description=excluded.description,duration=excluded.duration,'
|
||||
'downloadas=excluded.downloadas,playas=excluded.playas,raw_json=excluded.raw_json',
|
||||
(series_id, si, ei, episode.get('id'), episode.get('title'), episode.get('description'),
|
||||
episode.get('duration'), episode.get('downloadas'), episode.get('playas'), raw))
|
||||
for source_index, source in enumerate(_objects(episode.get('sources', []))):
|
||||
db.execute('INSERT INTO episode_sources VALUES (?,?,?,?,?,?,?,?,?) '
|
||||
'ON CONFLICT(series_id,season_index,episode_index,source_index) DO UPDATE SET '
|
||||
'source_id=excluded.source_id,quality=excluded.quality,media_type=excluded.media_type,'
|
||||
'url=excluded.url,raw_json=excluded.raw_json',
|
||||
(series_id, si, ei, source_index, source.get('id'), source.get('quality'),
|
||||
source.get('type'), source.get('url'), _encode(source)))
|
||||
db.execute('INSERT INTO series_fetch_status VALUES (?,?,?,?) ON CONFLICT(series_id) DO UPDATE SET '
|
||||
'status=excluded.status,season_count=excluded.season_count,source_url=excluded.source_url',
|
||||
(series_id, 'fetched', len(merged), UPSTREAM_BASE + f'season/by/serie/{series_id}/'))
|
||||
return changed
|
||||
|
||||
|
||||
def _counts(db):
|
||||
tables = ('titles', 'genres', 'countries', 'movie_sources', 'seasons', 'episodes', 'episode_sources')
|
||||
counts = {name: db.execute('SELECT COUNT(*) FROM ' + name).fetchone()[0] for name in tables}
|
||||
for name, kind in [('movies', 'movie'), ('series', 'serie')]:
|
||||
counts[name] = db.execute('SELECT COUNT(*) FROM titles WHERE type=?', (kind,)).fetchone()[0]
|
||||
counts['sources'] = counts['movie_sources'] + counts['episode_sources']
|
||||
counts['visible_titles'] = sum(1 for kind, title, description in db.execute('SELECT type,title,description FROM titles')
|
||||
if kind in {'movie', 'serie'} and allowed_text(title) and allowed_text(description))
|
||||
return counts
|
||||
|
||||
|
||||
def synchronize(source_db: Path, staged_db: Path, upstream, max_pages=2000, progress=None) -> dict:
|
||||
"""Copy a snapshot into an exclusively created candidate and refresh metadata.
|
||||
|
||||
Never publishes or mutates the source. A failed candidate is removed. The
|
||||
caller owns locking, free-space checks, atomic publication and API reload.
|
||||
``progress`` accepts dictionaries containing only phase/count/total.
|
||||
Both catalogue types must reach an empty page within ``max_pages`` each;
|
||||
all visible series are refreshed, including those absent from those pages.
|
||||
The returned updated-series IDs are a sorted sample of at most 100, with
|
||||
``updated_series_count`` reporting the full count. No media is fetched.
|
||||
"""
|
||||
if type(max_pages) is not int or not 1 <= max_pages <= 1000000:
|
||||
raise ValueError('Invalid catalogue page limit')
|
||||
source_db, staged_db = Path(source_db), Path(staged_db)
|
||||
if source_db.resolve() == staged_db.resolve():
|
||||
raise ValueError('Candidate must differ from source')
|
||||
created, db = False, None
|
||||
start_requests = getattr(upstream, 'requests_count', None)
|
||||
|
||||
def emit(phase, count, total):
|
||||
if progress is not None:
|
||||
progress({'phase': phase, 'count': count, 'total': total})
|
||||
|
||||
try:
|
||||
emit('copy', 0, 1)
|
||||
with closing(sqlite3.connect(source_db.resolve().as_uri() + '?mode=ro', uri=True)) as source:
|
||||
fd = os.open(staged_db, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
created = True
|
||||
os.close(fd)
|
||||
db = sqlite3.connect(staged_db)
|
||||
source.backup(db)
|
||||
emit('copy', 1, 1)
|
||||
db.execute('PRAGMA foreign_keys=ON')
|
||||
db.execute('PRAGMA journal_mode=DELETE')
|
||||
db.execute('BEGIN IMMEDIATE')
|
||||
before = _counts(db)
|
||||
metadata = {k: json.loads(v) for k, v in db.execute('SELECT * FROM metadata')}
|
||||
for count, (route, table) in enumerate([('genre/all', 'genres'), ('country/all', 'countries')], 1):
|
||||
for item in _objects(upstream.get(route)):
|
||||
_reference(db, table, item)
|
||||
emit('references', count, 2)
|
||||
updated_series = set()
|
||||
for kind in ('movie', 'serie'):
|
||||
seen = set()
|
||||
for page in range(max_pages):
|
||||
items = _objects(upstream.get(f'{kind}/by/filtres/0/created/{page}'))
|
||||
emit(kind + '_pages', page + 1, max_pages)
|
||||
if not items:
|
||||
break
|
||||
for item in items:
|
||||
title_id = _id(item.get('id'))
|
||||
if title_id in seen:
|
||||
continue
|
||||
seen.add(title_id)
|
||||
if item.get('type') == 'channel':
|
||||
continue
|
||||
if item.get('type') != kind:
|
||||
raise SyncError('Unexpected catalogue title type')
|
||||
if not allowed_text(item.get('title')) or not allowed_text(item.get('description')):
|
||||
continue
|
||||
exists = db.execute('SELECT type FROM titles WHERE id=?', (title_id,)).fetchone()
|
||||
if exists and exists[0] != kind:
|
||||
raise SyncError('Catalogue title type changed')
|
||||
if kind == 'movie' and not exists:
|
||||
detail = upstream.get(f'movie/by/{title_id}')
|
||||
if not isinstance(detail, dict) or detail.get('id') != title_id or detail.get('type') != kind:
|
||||
raise SyncError('Malformed movie detail')
|
||||
item = _merge(item, detail)
|
||||
if not allowed_text(item.get('title')) or not allowed_text(item.get('description')):
|
||||
continue
|
||||
if _save_title(db, item) and kind == 'serie':
|
||||
updated_series.add(title_id)
|
||||
else:
|
||||
raise SyncError('Catalogue pagination did not reach an empty terminal page')
|
||||
series = [row[0] for row in db.execute("SELECT id,title,description FROM titles WHERE type='serie'")
|
||||
if allowed_text(row[1]) and allowed_text(row[2])]
|
||||
for index, series_id in enumerate(series, 1):
|
||||
seasons = _objects(upstream.get(f'season/by/serie/{series_id}'))
|
||||
if _save_seasons(db, series_id, seasons):
|
||||
updated_series.add(series_id)
|
||||
emit('series', index, len(series))
|
||||
after = _counts(db)
|
||||
completed = datetime.now(timezone.utc).isoformat()
|
||||
report = {'status': 'ok', 'counts': after,
|
||||
'added': {name: after[name] - before[name] for name in
|
||||
('titles', 'movies', 'series', 'episodes', 'sources')},
|
||||
'updated_series_ids': sorted(updated_series)[:100], 'updated_series_count': len(updated_series),
|
||||
'media_files_downloaded': False, 'synchronized_at_utc': completed,
|
||||
'upstream_base': UPSTREAM_BASE}
|
||||
end_requests = getattr(upstream, 'requests_count', None)
|
||||
if type(start_requests) is int and type(end_requests) is int:
|
||||
report['requests_count'] = end_requests - start_requests
|
||||
metadata.setdefault('original_snapshot_built_at_utc', metadata.get('built_at_utc'))
|
||||
metadata.update({'built_at_utc': completed, 'synchronized_at_utc': completed,
|
||||
'sync_interval_seconds': 259200, 'media_files_downloaded': False,
|
||||
'counts': after, 'last_sync': report, 'catalog_pagination_complete': True,
|
||||
'series_episode_fetch_complete': True})
|
||||
db.executemany('INSERT INTO metadata VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value_json=excluded.value_json',
|
||||
[(k, _encode(v)) for k, v in metadata.items()])
|
||||
if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
|
||||
raise SyncError('Candidate integrity check failed')
|
||||
if db.execute('PRAGMA foreign_key_check').fetchall():
|
||||
raise SyncError('Candidate foreign key check failed')
|
||||
db.commit()
|
||||
db.close()
|
||||
db = None
|
||||
emit('complete', 1, 1)
|
||||
return report
|
||||
except BaseException as error:
|
||||
if db is not None:
|
||||
db.close()
|
||||
if created:
|
||||
for suffix in ('', '-journal', '-wal', '-shm'):
|
||||
Path(str(staged_db) + suffix).unlink(missing_ok=True)
|
||||
if isinstance(error, (KeyboardInterrupt, SystemExit, FileExistsError, SyncError)):
|
||||
raise
|
||||
raise SyncError('Catalogue synchronization failed; candidate discarded') from None
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Stage, validate and atomically publish read-only catalogue updates."""
|
||||
import argparse
|
||||
from contextlib import closing
|
||||
from datetime import datetime, timezone
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import stat
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
class UpdateError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def fsync_directory(path):
|
||||
descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def atomic_json(path, value):
|
||||
temp = path.with_name('.' + path.name + '.' + uuid.uuid4().hex)
|
||||
try:
|
||||
with temp.open('x', encoding='utf-8') as stream:
|
||||
os.chmod(temp, 0o600)
|
||||
json.dump(value, stream, ensure_ascii=False, indent=2)
|
||||
stream.write('\n')
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temp, path)
|
||||
fsync_directory(path.parent)
|
||||
finally:
|
||||
temp.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def check_candidate(source, stage):
|
||||
with closing(sqlite3.connect(source.resolve().as_uri() + '?mode=ro', uri=True)) as old:
|
||||
with closing(sqlite3.connect(stage.resolve().as_uri() + '?mode=ro', uri=True)) as candidate:
|
||||
if candidate.execute('PRAGMA quick_check').fetchone()[0] != 'ok':
|
||||
raise UpdateError('Candidate integrity check failed')
|
||||
if candidate.execute('PRAGMA foreign_key_check').fetchone() is not None:
|
||||
raise UpdateError('Candidate relation check failed')
|
||||
for table in ('titles', 'genres', 'countries', 'title_genres', 'title_countries',
|
||||
'series_fetch_status', 'seasons', 'episodes',
|
||||
'movie_sources', 'episode_sources'):
|
||||
before = old.execute('SELECT COUNT(*) FROM ' + table).fetchone()[0]
|
||||
after = candidate.execute('SELECT COUNT(*) FROM ' + table).fetchone()[0]
|
||||
if after < before:
|
||||
raise UpdateError('Candidate would remove existing catalogue entries')
|
||||
source_counts = (
|
||||
'SELECT title_id, COUNT(*) FROM movie_sources GROUP BY title_id',
|
||||
'''SELECT e.series_id, e.id,
|
||||
CASE WHEN e.id IS NULL THEN e.season_index END,
|
||||
CASE WHEN e.id IS NULL THEN e.episode_index END, COUNT(*)
|
||||
FROM episodes AS e JOIN episode_sources AS s
|
||||
USING (series_id, season_index, episode_index)
|
||||
GROUP BY 1, 2, 3, 4''',
|
||||
)
|
||||
for query in source_counts:
|
||||
after = {row[:-1]: row[-1] for row in candidate.execute(query)}
|
||||
for row in old.execute(query):
|
||||
if after.get(row[:-1], 0) < row[-1]:
|
||||
raise UpdateError('Candidate would remove existing catalogue sources')
|
||||
row = candidate.execute("SELECT value_json FROM metadata WHERE key='built_at_utc'").fetchone()
|
||||
if row is None or not isinstance(json.loads(row[0]), str):
|
||||
raise UpdateError('Candidate timestamp missing')
|
||||
return json.loads(row[0])
|
||||
|
||||
|
||||
def run_update(database, state_dir, upstream, *, synchronize_fn=None,
|
||||
restart_fn=None, health_fn=None, max_pages=2000, progress=None):
|
||||
database, state_dir = Path(database).resolve(), Path(state_dir).resolve()
|
||||
state_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if synchronize_fn is None:
|
||||
from catalog_sync import synchronize
|
||||
synchronize_fn = synchronize
|
||||
restart_fn = restart_fn or restart_api
|
||||
health_fn = health_fn or wait_for_api
|
||||
with (state_dir / 'sync.lock').open('a') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
raise UpdateError('A catalogue update is already running') from None
|
||||
original = database.stat()
|
||||
identity = (original.st_ino, original.st_size, original.st_mtime_ns)
|
||||
stage = database.with_name('.catalog-sync-' + uuid.uuid4().hex + '.sqlite')
|
||||
backup = database.with_name('catalog.previous.sqlite')
|
||||
backup_temp = database.with_name('.catalog-backup-' + uuid.uuid4().hex)
|
||||
published = False
|
||||
started = now()
|
||||
atomic_json(state_dir / 'last-result.json', {'status': 'running', 'started_at_utc': started})
|
||||
try:
|
||||
with closing(sqlite3.connect(database.as_uri() + '?mode=ro', uri=True)) as old:
|
||||
row = old.execute("SELECT value_json FROM metadata WHERE key='built_at_utc'").fetchone()
|
||||
if row is None or not isinstance(json.loads(row[0]), str):
|
||||
raise UpdateError('Previous catalogue timestamp missing')
|
||||
previous_stamp = json.loads(row[0])
|
||||
report = synchronize_fn(database, stage, upstream, max_pages=max_pages, progress=progress)
|
||||
stamp = check_candidate(database, stage)
|
||||
current = database.stat()
|
||||
if identity != (current.st_ino, current.st_size, current.st_mtime_ns):
|
||||
raise UpdateError('Live catalogue changed during synchronisation')
|
||||
os.chown(stage, original.st_uid, original.st_gid)
|
||||
os.chmod(stage, stat.S_IMODE(original.st_mode))
|
||||
with stage.open('rb') as stream:
|
||||
os.fsync(stream.fileno())
|
||||
os.link(database, backup_temp)
|
||||
os.replace(backup_temp, backup)
|
||||
fsync_directory(database.parent)
|
||||
os.replace(stage, database)
|
||||
published = True
|
||||
fsync_directory(database.parent)
|
||||
restart_fn()
|
||||
health_fn(stamp)
|
||||
result = {**report, 'status': 'success', 'started_at_utc': started,
|
||||
'completed_at_utc': now(), 'api_health_verified': True,
|
||||
'media_files_downloaded': False}
|
||||
atomic_json(state_dir / 'last-success.json', result)
|
||||
atomic_json(state_dir / 'last-result.json', result)
|
||||
return result
|
||||
except Exception as error:
|
||||
rollback_failed = False
|
||||
if published:
|
||||
try:
|
||||
os.replace(backup, database)
|
||||
fsync_directory(database.parent)
|
||||
restart_fn()
|
||||
health_fn(previous_stamp)
|
||||
except Exception:
|
||||
rollback_failed = True
|
||||
atomic_json(state_dir / 'last-result.json', {
|
||||
'status': 'failed', 'started_at_utc': started, 'completed_at_utc': now(),
|
||||
'error_type': type(error).__name__, 'rollback_failed': rollback_failed,
|
||||
'media_files_downloaded': False,
|
||||
})
|
||||
if rollback_failed:
|
||||
raise UpdateError('Update failed and API restart needs operator attention') from None
|
||||
raise UpdateError('Update failed; the previous catalogue was retained') from None
|
||||
finally:
|
||||
for suffix in ('', '-journal', '-wal', '-shm'):
|
||||
Path(str(stage) + suffix).unlink(missing_ok=True)
|
||||
backup_temp.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def restart_api():
|
||||
subprocess.run(['/bin/systemctl', 'restart', 'reelbox-api.service'],
|
||||
check=True, capture_output=True, timeout=120)
|
||||
|
||||
|
||||
def wait_for_api(expected_stamp):
|
||||
from urllib.request import ProxyHandler, build_opener
|
||||
opener = build_opener(ProxyHandler({}))
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with opener.open('http://127.0.0.1:18765/api/health', timeout=5) as response:
|
||||
health = json.loads(response.read(65536))
|
||||
if (health.get('status') == 'ok' and health.get('media_proxy') is False
|
||||
and health.get('snapshot_built_at_utc') == expected_stamp):
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(2)
|
||||
raise UpdateError('Updated API did not become healthy in time')
|
||||
|
||||
|
||||
def emit_progress(event):
|
||||
fields = {key: event[key] for key in ('phase', 'total') if key in event}
|
||||
if 'completed' in event or 'count' in event:
|
||||
fields['completed'] = event.get('completed', event.get('count'))
|
||||
print(json.dumps({'progress': fields}), flush=True)
|
||||
|
||||
|
||||
def probe(upstream):
|
||||
genres = upstream.get('genre/all')
|
||||
movies = upstream.get('movie/by/filtres/0/created/0')
|
||||
series = upstream.get('serie/by/filtres/0/created/0')
|
||||
if not all(isinstance(rows, list) and rows for rows in (genres, movies, series)):
|
||||
raise UpdateError('Upstream metadata probe returned an unexpected shape')
|
||||
movie_id = movies[0]['id']
|
||||
series_id = series[0]['id']
|
||||
detail = upstream.get('movie/by/' + str(movie_id))
|
||||
seasons = upstream.get('season/by/serie/' + str(series_id))
|
||||
if not isinstance(detail, dict) or not isinstance(seasons, list):
|
||||
raise UpdateError('Upstream detail or season probe failed')
|
||||
return {'status': 'probe_ok', 'movie_id': movie_id, 'series_id': series_id,
|
||||
'movie_sources': len(detail.get('sources', [])), 'seasons': len(seasons),
|
||||
'media_files_downloaded': False}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--database', type=Path, default=Path('/var/lib/reelbox-api/catalog.sqlite'))
|
||||
parser.add_argument('--state-dir', type=Path, default=Path('/var/lib/reelbox-sync'))
|
||||
parser.add_argument('--interval', type=float, default=0.25)
|
||||
parser.add_argument('--max-pages', type=int, default=2000)
|
||||
parser.add_argument('--probe', action='store_true')
|
||||
args = parser.parse_args()
|
||||
from catalog_sync import Upstream
|
||||
key = os.environ.get('REELBOX_UPSTREAM_KEY', '')
|
||||
if not key:
|
||||
print('REELBOX_UPSTREAM_KEY is required; no requests sent', flush=True)
|
||||
return 2
|
||||
try:
|
||||
upstream = Upstream(key, interval=max(0.25, args.interval))
|
||||
result = probe(upstream) if args.probe else run_update(
|
||||
args.database, args.state_dir, upstream, max_pages=args.max_pages, progress=emit_progress)
|
||||
print(json.dumps(result, ensure_ascii=False), flush=True)
|
||||
return 0
|
||||
except Exception as error:
|
||||
print(json.dumps({'status': 'failed', 'error_type': type(error).__name__,
|
||||
'message': str(error) if isinstance(error, UpdateError) else 'Metadata update failed; see safe status record'}), flush=True)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"format_version": 1,
|
||||
"built_at_utc": "2026-11-02T07:00:18.228980+00:00",
|
||||
"provenance": "API snapshot reconstructed from the read-only endpoints used by the downloaded Simba apps; not an original server-side database dump.",
|
||||
"api_base": "https://ninwinios.info/api/",
|
||||
"discovery_url": "http://ninnies-server.info/api-user/newapi/config-server186.txt",
|
||||
"download_page": "https://zil.ink/movieboxapp",
|
||||
"counts": {
|
||||
"titles": 29931,
|
||||
"genres": 50,
|
||||
"countries": 84,
|
||||
"movie_sources": 53853,
|
||||
"seasons": 19132,
|
||||
"episodes": 348897,
|
||||
"episode_sources": 441945,
|
||||
"movie": 20435,
|
||||
"serie": 9496,
|
||||
"series_seasons_fetched": 9496,
|
||||
"series_seasons_pending": 0,
|
||||
"movie_titles_without_sources": 8
|
||||
},
|
||||
"catalog_coverage": {
|
||||
"movie": {
|
||||
"kind": "movie",
|
||||
"first_empty_page": 682,
|
||||
"pages": 682,
|
||||
"records": 20435,
|
||||
"unique_ids": 20435,
|
||||
"last_nonempty_page_count": 5
|
||||
},
|
||||
"serie": {
|
||||
"kind": "serie",
|
||||
"first_empty_page": 317,
|
||||
"pages": 317,
|
||||
"records": 9496,
|
||||
"unique_ids": 9496,
|
||||
"last_nonempty_page_count": 16
|
||||
}
|
||||
},
|
||||
"catalog_pagination_complete": true,
|
||||
"series_episode_fetch_complete": true,
|
||||
"media_files_downloaded": false,
|
||||
"media_urls_playback_checked": false,
|
||||
"notes": [
|
||||
"API pagination was exhausted, but the service is live and does not offer transactional snapshots or independent total counts.",
|
||||
"Titles and media URLs are preserved as returned; dead URLs and catalog omissions may exist.",
|
||||
"No accounts, private profiles, user data, comments or payment data were exported.",
|
||||
"Extraction does not establish rights to redistribute metadata or stream the referenced videos."
|
||||
],
|
||||
"artifacts": {
|
||||
"catalog.sqlite": {
|
||||
"bytes": 598843392,
|
||||
"sha256": "45213410c51eeee984a02883823e312272bd6c6a44ec75452e9f90f74db1a9d8"
|
||||
},
|
||||
"catalog.jsonl.gz": {
|
||||
"bytes": 13425106,
|
||||
"sha256": "a3ef9cba408e554e4b18cd4f8ededb2d69010972a02d7f0182f15fedc669eeae"
|
||||
},
|
||||
"seasons.jsonl.gz": {
|
||||
"bytes": 6530014,
|
||||
"sha256": "c1dc5e96a40f33587c6bcaf0dd1a3573d92a693c0756bbef11778ccc91ae6668"
|
||||
},
|
||||
"schema.sql": {
|
||||
"bytes": 2375,
|
||||
"sha256": "fd6deb8ed09540aeae8ebb00a621cbfefff19d4fd86e48d33edb28117e298066"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"download_page": "https://zil.ink/movieboxapp",
|
||||
"inspection": "Static archive extraction; neither app was executed",
|
||||
"binaries": [
|
||||
{
|
||||
"file": "Simba-v82.apk",
|
||||
"url": "https://ninamacos.info/apk/Simba-v82.apk",
|
||||
"bytes": 21415761,
|
||||
"sha256": "089b742354ef868cdd1b49e0bb8f593f7c1c0df1e35d8a893da3fff1c05bea3c"
|
||||
},
|
||||
{
|
||||
"file": "Simba-windows.zip",
|
||||
"url": "https://ninamacos.info/apk/windows/Simba-Setup-1.3.5-x64.zip",
|
||||
"bytes": 191886111,
|
||||
"sha256": "e7904367e539f832b69991b067c489d4174b46a1918277e2bfc366e36f748754"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Install API routing on the existing Iran ReelBox HTTPS server."""
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
config = Path('/etc/reelbox/nginx.conf')
|
||||
original = config.read_text()
|
||||
include = ' include /etc/reelbox/api-locations.conf;\n'
|
||||
if include not in original:
|
||||
anchor = ' root /var/www/reelbox/current;\n'
|
||||
if original.count(anchor) != 1:
|
||||
raise RuntimeError('Unexpected site configuration; refusing to alter it')
|
||||
backup = config.with_name('nginx.conf.' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '.backup')
|
||||
shutil.copy2(config, backup)
|
||||
config.write_text(original.replace(anchor, include + anchor))
|
||||
try:
|
||||
subprocess.run(['/usr/sbin/nginx', '-t', '-c', str(config)], check=True)
|
||||
except Exception:
|
||||
config.write_text(original)
|
||||
raise
|
||||
subprocess.run(['systemctl', 'reload', 'reelbox-web'], check=True)
|
||||
print('ReelBox metadata API routes enabled; website root and existing listeners unchanged')
|
||||
@@ -0,0 +1,42 @@
|
||||
# Include inside the ReelBox HTTPS server on 443 (legacy 8443 remains served).
|
||||
# Preserve website/download routes and unrelated services; inspect each host.
|
||||
location /api/ {
|
||||
access_log off;
|
||||
# Public metadata only; low compression level trades little CPU for fewer
|
||||
# transferred bytes. Nginx handles negotiation, HEAD and chunked framing.
|
||||
gzip on;
|
||||
gzip_types application/json;
|
||||
gzip_min_length 1024;
|
||||
gzip_comp_level 2;
|
||||
gzip_vary on;
|
||||
gzip_proxied no-store;
|
||||
limit_except GET { deny all; }
|
||||
proxy_pass http://127.0.0.1:18765;
|
||||
proxy_http_version 1.0;
|
||||
proxy_set_header Host localhost;
|
||||
proxy_set_header Connection close;
|
||||
proxy_connect_timeout 3s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_send_timeout 10s;
|
||||
proxy_max_temp_file_size 0;
|
||||
}
|
||||
location /api-user/ {
|
||||
access_log off;
|
||||
# Public metadata only; low compression level trades little CPU for fewer
|
||||
# transferred bytes. Nginx handles negotiation, HEAD and chunked framing.
|
||||
gzip on;
|
||||
gzip_types application/json;
|
||||
gzip_min_length 1024;
|
||||
gzip_comp_level 2;
|
||||
gzip_vary on;
|
||||
gzip_proxied no-store;
|
||||
limit_except GET { deny all; }
|
||||
proxy_pass http://127.0.0.1:18765;
|
||||
proxy_http_version 1.0;
|
||||
proxy_set_header Host localhost;
|
||||
proxy_set_header Connection close;
|
||||
proxy_connect_timeout 3s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_send_timeout 10s;
|
||||
proxy_max_temp_file_size 0;
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
[Unit]
|
||||
Description=ReelBox read-only offline Catalogue API
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=www-data
|
||||
Group=www-data
|
||||
WorkingDirectory=/opt/reelbox-api
|
||||
ExecStart=/usr/bin/python3 -B /opt/reelbox-api/catalog_api.py
|
||||
Environment=REELBOX_CATALOG_DB=/var/lib/reelbox-api/catalog.sqlite
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
TimeoutStartSec=120
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
PrivateDevices=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
RestrictSUIDSGID=true
|
||||
RestrictNamespaces=true
|
||||
LockPersonality=true
|
||||
RestrictAddressFamilies=AF_INET AF_UNIX
|
||||
IPAddressDeny=any
|
||||
IPAddressAllow=localhost
|
||||
MemoryMax=256M
|
||||
TasksMax=24
|
||||
LimitNOFILE=256
|
||||
UMask=0077
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,40 @@
|
||||
[Unit]
|
||||
Description=Refresh ReelBox catalogue metadata from the original API
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
StartLimitIntervalSec=1d
|
||||
StartLimitBurst=3
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
WorkingDirectory=/opt/reelbox-api
|
||||
EnvironmentFile=/etc/reelbox/sync.env
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
ExecStart=/usr/bin/python3 -B /opt/reelbox-api/catalog_sync_runner.py
|
||||
TimeoutStartSec=8h
|
||||
Restart=on-failure
|
||||
RestartSec=1h
|
||||
StateDirectory=reelbox-sync
|
||||
StateDirectoryMode=0700
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
PrivateDevices=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
RestrictSUIDSGID=true
|
||||
RestrictNamespaces=true
|
||||
LockPersonality=true
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||
ReadWritePaths=/var/lib/reelbox-api /var/lib/reelbox-sync
|
||||
MemoryMax=512M
|
||||
TasksMax=16
|
||||
LimitNOFILE=512
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Refresh ReelBox catalogue every three days
|
||||
|
||||
[Timer]
|
||||
OnBootSec=15min
|
||||
OnUnitActiveSec=3d
|
||||
AccuracySec=1min
|
||||
Unit=reelbox-sync.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,37 @@
|
||||
|
||||
PRAGMA foreign_keys=ON;
|
||||
CREATE TABLE metadata(key TEXT PRIMARY KEY,value_json TEXT NOT NULL);
|
||||
CREATE TABLE titles(
|
||||
id INTEGER PRIMARY KEY,type TEXT NOT NULL,title TEXT,description TEXT,
|
||||
year INTEGER,imdb REAL,rating REAL,duration TEXT,classification TEXT,
|
||||
image TEXT,cover TEXT,downloadas TEXT,playas TEXT,comments_enabled INTEGER,
|
||||
raw_json TEXT NOT NULL);
|
||||
CREATE INDEX titles_type ON titles(type);
|
||||
CREATE INDEX titles_title ON titles(title COLLATE NOCASE);
|
||||
CREATE INDEX titles_year ON titles(year);
|
||||
CREATE TABLE genres(id INTEGER PRIMARY KEY,title TEXT);
|
||||
CREATE TABLE countries(id INTEGER PRIMARY KEY,title TEXT,image TEXT);
|
||||
CREATE TABLE title_genres(title_id INTEGER REFERENCES titles(id),genre_id INTEGER REFERENCES genres(id),PRIMARY KEY(title_id,genre_id));
|
||||
CREATE TABLE title_countries(title_id INTEGER REFERENCES titles(id),country_id INTEGER REFERENCES countries(id),PRIMARY KEY(title_id,country_id));
|
||||
CREATE TABLE movie_sources(
|
||||
title_id INTEGER NOT NULL REFERENCES titles(id),source_index INTEGER NOT NULL,
|
||||
source_id INTEGER,quality TEXT,media_type TEXT,url TEXT,raw_json TEXT NOT NULL,
|
||||
PRIMARY KEY(title_id,source_index));
|
||||
CREATE INDEX movie_sources_title ON movie_sources(title_id);
|
||||
CREATE TABLE series_fetch_status(
|
||||
series_id INTEGER PRIMARY KEY REFERENCES titles(id),status TEXT NOT NULL,
|
||||
season_count INTEGER,source_url TEXT NOT NULL);
|
||||
CREATE TABLE seasons(
|
||||
series_id INTEGER NOT NULL REFERENCES titles(id),season_index INTEGER NOT NULL,
|
||||
id INTEGER,title TEXT,raw_json TEXT NOT NULL,PRIMARY KEY(series_id,season_index));
|
||||
CREATE TABLE episodes(
|
||||
series_id INTEGER NOT NULL,season_index INTEGER NOT NULL,episode_index INTEGER NOT NULL,
|
||||
id INTEGER,title TEXT,description TEXT,duration TEXT,downloadas TEXT,playas TEXT,raw_json TEXT NOT NULL,
|
||||
PRIMARY KEY(series_id,season_index,episode_index),
|
||||
FOREIGN KEY(series_id,season_index) REFERENCES seasons(series_id,season_index));
|
||||
CREATE TABLE episode_sources(
|
||||
series_id INTEGER NOT NULL,season_index INTEGER NOT NULL,episode_index INTEGER NOT NULL,source_index INTEGER NOT NULL,
|
||||
source_id INTEGER,quality TEXT,media_type TEXT,url TEXT,raw_json TEXT NOT NULL,
|
||||
PRIMARY KEY(series_id,season_index,episode_index,source_index),
|
||||
FOREIGN KEY(series_id,season_index,episode_index) REFERENCES episodes(series_id,season_index,episode_index));
|
||||
CREATE INDEX episode_sources_series ON episode_sources(series_id);
|
||||
@@ -0,0 +1,98 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from test_catalog_api import fixture, load_service
|
||||
|
||||
|
||||
class BannerTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(dir=os.environ.get('TMPDIR'))
|
||||
self.path = fixture(self.temp.name)
|
||||
self.api = load_service()
|
||||
self.catalog = self.api.Catalog(self.path)
|
||||
|
||||
def tearDown(self):
|
||||
self.catalog.close()
|
||||
self.temp.cleanup()
|
||||
|
||||
def test_local_config_reloads_safe_bounded_ads_without_admin_endpoint(self):
|
||||
config = Path(self.temp.name) / 'banners.json'
|
||||
self.catalog.close()
|
||||
self.catalog = self.api.Catalog(self.path, banner_config=config)
|
||||
self.assertEqual(self.catalog.dispatch('/api/banners/'), {'banners': []})
|
||||
ad = {'title_en': 'Fixture sponsor', 'title_fa': 'حامی آزمایشی',
|
||||
'subtitle_en': 'Test only', 'target_url': 'https://sponsor.example/offer',
|
||||
'image_url': 'https://sponsor.example/banner.jpg', 'secret': 'not public'}
|
||||
config.write_text(json.dumps({'banners': [ad]}))
|
||||
result = self.catalog.dispatch('/api/banners/')['banners']
|
||||
self.assertEqual(len(result), 1)
|
||||
self.assertEqual(result[0]['title_fa'], ad['title_fa'])
|
||||
self.assertEqual(result[0]['image_url'], ad['image_url'])
|
||||
self.assertNotIn('secret', result[0])
|
||||
config.write_text('{invalid')
|
||||
self.assertEqual(self.catalog.dispatch('/api/banners/'), {'banners': []})
|
||||
for target in ['http://evil.example/', 'javascript:alert(1)', '//evil.example/',
|
||||
'https://a:b@evil.example/', 'https://evil.example:444/',
|
||||
'https://evil.example/ bad', 'https://evil.example/\\\\bad',
|
||||
'https://evil.example/%0abad', 'https://',
|
||||
'https://-bad.example/', 'https://bad..example/', 'https://bad.example./',
|
||||
'https://evil.example/%zz', 'https://evil.example/' + 'x' * 2048]:
|
||||
config.write_text(json.dumps({'banners': [{**ad, 'target_url': target}, ad]}))
|
||||
self.assertEqual(len(self.catalog.dispatch('/api/banners/')['banners']), 1, target)
|
||||
for row in [None, True, 5, {}, {**ad, 'title_en': 'x' * 81},
|
||||
{**ad, 'subtitle_en': 'x' * 181}, {**ad, 'title_fa': 'bad\u202etext'},
|
||||
{**ad, 'image_url': 'http://evil.example/banner.jpg'}]:
|
||||
config.write_text(json.dumps({'banners': [row, ad]}))
|
||||
self.assertEqual(len(self.catalog.dispatch('/api/banners/')['banners']), 1, row)
|
||||
config.write_text(json.dumps({'banners': [ad] * 30}))
|
||||
self.assertEqual(len(self.catalog.dispatch('/api/banners/')['banners']), 5)
|
||||
config.write_text(json.dumps({'banners': [{}] * 20 + [ad]}))
|
||||
self.assertEqual(self.catalog.dispatch('/api/banners/'), {'banners': []})
|
||||
config.write_text(' ' * 65537)
|
||||
self.assertEqual(self.catalog.dispatch('/api/banners/'), {'banners': []})
|
||||
for value in [None, [], {'banners': {}}, {'banners': []}]:
|
||||
config.write_text(json.dumps(value))
|
||||
self.assertEqual(self.catalog.dispatch('/api/banners/'), {'banners': []})
|
||||
for route in ['/api/banners/admin/', '/api/banners/config/']:
|
||||
with self.assertRaises(self.api.ApiError) as error:
|
||||
self.catalog.dispatch(route)
|
||||
self.assertEqual(error.exception.status, 404)
|
||||
|
||||
def test_http_banner_endpoint_is_json_no_store_and_rejects_writes(self):
|
||||
from http.client import HTTPConnection
|
||||
import threading
|
||||
server = self.api.make_server(self.catalog, port=0)
|
||||
worker = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
worker.start()
|
||||
try:
|
||||
for method, expected in [('GET', 200), ('HEAD', 200), ('POST', 405), ('PUT', 405)]:
|
||||
client = HTTPConnection('127.0.0.1', server.server_port, timeout=5)
|
||||
client.request(method, '/api/banners/')
|
||||
response = client.getresponse()
|
||||
self.assertEqual(response.status, expected)
|
||||
self.assertEqual(response.getheader('Cache-Control'), 'no-store')
|
||||
self.assertIn('application/json', response.getheader('Content-Type'))
|
||||
body = response.read()
|
||||
if method == 'GET':
|
||||
self.assertEqual(json.loads(body), {'banners': []})
|
||||
if method == 'HEAD':
|
||||
self.assertEqual(body, b'')
|
||||
client.close()
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
worker.join(timeout=5)
|
||||
|
||||
def test_read_only_banner_route_defaults_to_empty_ad_list(self):
|
||||
self.assertEqual(self.catalog.dispatch('/api/banners/'), {'banners': []})
|
||||
self.assertEqual(self.catalog.dispatch('/api/banners'), {'banners': []})
|
||||
with self.assertRaises(self.api.ApiError) as error:
|
||||
self.catalog.dispatch('/api/banners/?url=https://evil.example/')
|
||||
self.assertEqual(error.exception.status, 400)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Regression coverage for app advertisements disguised as movie records."""
|
||||
import json
|
||||
import sqlite3
|
||||
import unittest
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import test_catalog_api as fixtures
|
||||
|
||||
|
||||
class CatalogueAdTests(unittest.TestCase):
|
||||
setUp = fixtures.CatalogTests.setUp
|
||||
tearDown = fixtures.CatalogTests.tearDown
|
||||
open = fixtures.CatalogTests.open
|
||||
|
||||
def test_app_promotions_and_typographic_variants_are_rejected(self):
|
||||
self.open()
|
||||
for text in [
|
||||
'نورتون', 'نُوْرتـو\u200cن', 'N O R T O N',
|
||||
'آموزش زبان به سبک موسویباکس',
|
||||
'آموزش زبان به سبک مووی باکس',
|
||||
'آموزش زبان به سبک مُووی باكْس',
|
||||
'نسخه جدید هورتون', 'نسخه جدید هُورتـون',
|
||||
'اپلیکیشن هورتون', 'نرم افزار هورتون',
|
||||
]:
|
||||
with self.subTest(text=text):
|
||||
self.assertFalse(self.api.allowed_text(text))
|
||||
for text in [
|
||||
'Horton Hears a Who!', 'نام پارسی انیمیشن: هورتون',
|
||||
'آموزش زبان New Headway', 'The Marriage App',
|
||||
'فیلمی درباره یک اپلیکیشن خیالی', 'بازیگران: Alex Horton',
|
||||
]:
|
||||
with self.subTest(text=text):
|
||||
self.assertTrue(self.api.allowed_text(text))
|
||||
|
||||
def test_ads_are_absent_from_every_title_surface_and_direct_lookup(self):
|
||||
with sqlite3.connect(self.path) as db:
|
||||
original = json.loads(db.execute('SELECT raw_json FROM titles WHERE id=1').fetchone()[0])
|
||||
records = [
|
||||
(66, 'نورتون', ''),
|
||||
(67, 'آموزش زبان به سبک مووی باکس', ''),
|
||||
(68, 'آموزش زبان به سبک موسویباکس', ''),
|
||||
(69, 'نسخه جدید هورتون', ''),
|
||||
(70, 'Horton Hears a Who!', 'نام پارسی انیمیشن: هورتون'),
|
||||
(71, 'آموزش زبان New Headway', 'مجموعه ویدیویی آموزش مهارت مکالمه'),
|
||||
(72, 'The Marriage App', 'فیلمی درباره یک اپلیکیشن خیالی'),
|
||||
(73, 'Language companion', 'اپلیکیشن هورتون'),
|
||||
]
|
||||
for title_id, title, description in records:
|
||||
raw = {**original, 'id': title_id, 'title': title, 'description': description}
|
||||
db.execute('INSERT INTO titles(id,type,title,description,year,imdb,rating,raw_json) VALUES (?,?,?,?,?,?,?,?)',
|
||||
(title_id, 'movie', title, description, 2026, 9.9, 5, json.dumps(raw)))
|
||||
db.execute('INSERT INTO title_genres VALUES (?,2)', (title_id,))
|
||||
db.execute('INSERT INTO title_countries VALUES (?,1)', (title_id,))
|
||||
catalog = self.open()
|
||||
ads = {66, 67, 68, 69, 73}
|
||||
actual = set()
|
||||
for page in range(4):
|
||||
for order in ['created', 'imdb', 'rating', 'year', 'title', 'random']:
|
||||
for path in [f'/api/movie/by/filtres/0/{order}/{page}/public/',
|
||||
f'/api/poster/by/filtres/2/1/{order}/{page}/public/']:
|
||||
rows = catalog.dispatch(path)
|
||||
self.assertFalse(ads & {row['id'] for row in rows}, path)
|
||||
actual.update(row['id'] for row in rows)
|
||||
self.assertTrue({70, 71, 72} <= actual)
|
||||
for group in catalog.dispatch('/api/first/public/')['genres']:
|
||||
self.assertFalse(ads & {row['id'] for row in group['posters']})
|
||||
for action in ['best-rand-top', 'best-rand-year']:
|
||||
rows = catalog.dispatch('/api-user/newapi/like.php?' + urlencode({'action': action}))
|
||||
self.assertFalse(ads & {row['id'] for row in rows})
|
||||
for query in ['نورتون', 'هورتون', 'آموزش زبان', 'موسوی', 'مووی', 'Language companion']:
|
||||
rows = catalog.dispatch('/api-user/newapi/like.php?' + urlencode({'action': 'search-movie', 'q': query}))
|
||||
self.assertFalse(ads & {row['id'] for row in rows}, query)
|
||||
rows = catalog.dispatch('/api-user/newapi/filter.php?action=pishnahad&garid=2&pageno=1')
|
||||
self.assertFalse(ads & {row['id'] for row in rows})
|
||||
for title_id in ads:
|
||||
for path in [f'/api/movie/by/{title_id}/public/', f'/api/season/by/serie/{title_id}/public/']:
|
||||
with self.assertRaises(self.api.ApiError) as error:
|
||||
catalog.dispatch(path)
|
||||
self.assertEqual(error.exception.status, 404)
|
||||
for title_id in [70, 71, 72]:
|
||||
self.assertEqual(catalog.dispatch(f'/api/movie/by/{title_id}/public/')['id'], title_id)
|
||||
detail = catalog.dispatch('/api/movie/by/1/public/')
|
||||
self.assertEqual(detail['sources'], original['sources'])
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Upstream notices are not catalogue films, including after later scrapes."""
|
||||
from contextlib import closing
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from typing import Any
|
||||
import unittest
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import test_catalog_api as fixtures
|
||||
from test_catalog_api import load_service
|
||||
from test_catalog_sync import FakeUpstream, load_sync
|
||||
|
||||
|
||||
class AnnouncementPolicyTests(unittest.TestCase):
|
||||
def test_generic_notice_labels_are_blocked_without_hiding_real_notice_films(self):
|
||||
api = load_service()
|
||||
for text in [
|
||||
'اطلاعیه جدید', 'اطلاعیه مهم ', 'اطلاعیه خیلی مهم',
|
||||
'📢 **اطلاعـیّهٔ جدید**', 'اطلاع يه جديد', 'اطلاعرسانی جدید',
|
||||
'اعلان جدید', 'اعلان مهم شماره ۲',
|
||||
'به روز رسانی جدید', 'بهروزرسانی برنامه', 'بروز رسانی نسخه ویندوز',
|
||||
'آپدیت جدید برنامه', 'New announcement', 'IMPORTANT NOTICE!',
|
||||
'Urgent announcement: service changes', 'App update',
|
||||
]:
|
||||
with self.subTest(blocked=text):
|
||||
self.assertFalse(api.allowed_text(text))
|
||||
for text in [
|
||||
'Red Notice', 'SAS: Red Notice', 'Burn Notice', 'Notice to Quit',
|
||||
'Two Weeks Notice', 'The Announcement', 'اعلان قرمز', 'Upgrade',
|
||||
'News of the World', 'نام پارسی فیلم: اعلان قرمز',
|
||||
'A detective reads an important notice before leaving town.',
|
||||
'در این فیلم یک اطلاعیه جدید، زندگی شخصیت اصلی را تغییر می‌دهد.',
|
||||
]:
|
||||
with self.subTest(allowed=text):
|
||||
self.assertTrue(api.allowed_text(text))
|
||||
|
||||
|
||||
class AnnouncementSurfaceTests(unittest.TestCase):
|
||||
path: Path
|
||||
temp: tempfile.TemporaryDirectory
|
||||
api: Any
|
||||
setUp = fixtures.CatalogTests.setUp
|
||||
tearDown = fixtures.CatalogTests.tearDown
|
||||
open = fixtures.CatalogTests.open
|
||||
|
||||
def test_notices_are_absent_from_all_catalogue_routes_and_direct_lookup(self):
|
||||
bad = {70, 71, 72, 73, 74}
|
||||
with closing(sqlite3.connect(self.path)) as db:
|
||||
original = json.loads(db.execute('SELECT raw_json FROM titles WHERE id=1').fetchone()[0])
|
||||
records = [
|
||||
(70, 'movie', 'اطلاعیه جدید', ''),
|
||||
(71, 'movie', 'اطلاعیه خیلی مهم', ''),
|
||||
(72, 'movie', 'به روز رسانی جدید', ''),
|
||||
(73, 'movie', 'Service changes', 'Important announcement: app changes'),
|
||||
(74, 'serie', 'اعلان جدید', ''),
|
||||
(75, 'movie', 'Red Notice', 'نام پارسی فیلم: اعلان قرمز'),
|
||||
(76, 'serie', 'Burn Notice', 'An ordinary spy series'),
|
||||
]
|
||||
for ident, kind, title, description in records:
|
||||
raw = {**original, 'id': ident, 'type': kind, 'title': title, 'description': description}
|
||||
db.execute('INSERT INTO titles(id,type,title,description,year,imdb,rating,raw_json) VALUES (?,?,?,?,?,?,?,?)',
|
||||
(ident, kind, title, description, 2026, 9.9, 5, json.dumps(raw)))
|
||||
db.execute('INSERT INTO title_genres VALUES (?,2)', (ident,))
|
||||
db.execute('INSERT INTO title_countries VALUES (?,1)', (ident,))
|
||||
db.commit()
|
||||
catalog = self.open()
|
||||
found = set()
|
||||
for kind in ('movie', 'serie', 'poster'):
|
||||
for order in ('created', 'updated', 'imdb', 'rating', 'year', 'title', 'random'):
|
||||
for page in range(4):
|
||||
for path in (f'/api/{kind}/by/filtres/0/{order}/{page}/public/',
|
||||
f'/api/{kind}/by/filtres/2/1/{order}/{page}/public/'):
|
||||
ids = {row['id'] for row in catalog.dispatch(path)}
|
||||
self.assertFalse(bad & ids, path)
|
||||
found.update(ids)
|
||||
self.assertTrue({75, 76} <= found)
|
||||
for group in catalog.dispatch('/api/first/public/')['genres']:
|
||||
self.assertFalse(bad & {row['id'] for row in group['posters']})
|
||||
for action in ('best-rand-top', 'best-rand-year'):
|
||||
self.assertFalse(bad & {row['id'] for row in catalog.dispatch(
|
||||
'/api-user/newapi/like.php?' + urlencode({'action': action}))})
|
||||
for action in ('search-movie', 'search-movie-en'):
|
||||
for query in ('اطلاعیه', 'به روز', 'announcement', 'Notice'):
|
||||
rows = catalog.dispatch('/api-user/newapi/like.php?' + urlencode({'action': action, 'q': query}))
|
||||
self.assertFalse(bad & {row['id'] for row in rows})
|
||||
self.assertFalse(bad & {row['id'] for row in catalog.dispatch(
|
||||
'/api-user/newapi/filter.php?action=pishnahad&garid=2&pageno=1')})
|
||||
for ident in bad:
|
||||
for path in (f'/api/movie/by/{ident}/public/', f'/api/season/by/serie/{ident}/public/'):
|
||||
with self.assertRaises(self.api.ApiError) as error:
|
||||
catalog.dispatch(path)
|
||||
self.assertEqual(error.exception.status, 404)
|
||||
self.assertEqual(catalog.dispatch('/api/movie/by/75/public/')['title'], 'Red Notice')
|
||||
self.assertEqual(catalog.dispatch('/api/movie/by/76/public/')['title'], 'Burn Notice')
|
||||
|
||||
def test_repeated_scrapes_keep_old_and_new_notice_ids_out_of_the_api(self):
|
||||
# Old raw rows stay for snapshot/source preservation, but are never served.
|
||||
with closing(sqlite3.connect(self.path)) as db:
|
||||
item = {'id': 90, 'type': 'movie', 'title': 'اطلاعیه مهم', 'description': ''}
|
||||
db.execute('INSERT INTO titles(id,type,title,description,raw_json) VALUES (?,?,?,?,?)',
|
||||
(90, 'movie', item['title'], '', json.dumps(item)))
|
||||
db.commit()
|
||||
source_bytes = self.path.read_bytes()
|
||||
source = self.path
|
||||
sync = load_sync()
|
||||
for run, offset in enumerate((0, 100), 1):
|
||||
new_notice, series_notice, detail_notice = 91 + offset, 92 + offset, 93 + offset
|
||||
rows = [
|
||||
{'id': 90, 'type': 'movie', 'title': 'اطلاعیه جدید'},
|
||||
{'id': new_notice, 'type': 'movie', 'title': '📢 اطلاعیه جدید'},
|
||||
{'id': detail_notice, 'type': 'movie', 'title': 'Looks like a film'},
|
||||
{'id': 94, 'type': 'movie', 'title': 'Red Notice'},
|
||||
]
|
||||
upstream = FakeUpstream({
|
||||
'movie/by/filtres/0/created/0': rows,
|
||||
f'movie/by/{detail_notice}': {'id': detail_notice, 'type': 'movie', 'title': 'به روز رسانی جدید'},
|
||||
'movie/by/94': {'id': 94, 'type': 'movie', 'title': 'Red Notice', 'sources': []},
|
||||
'serie/by/filtres/0/created/0': [
|
||||
{'id': series_notice, 'type': 'serie', 'title': 'Important announcement'}],
|
||||
})
|
||||
stage = Path(self.temp.name) / f'scrape-{run}.sqlite'
|
||||
sync.synchronize(source, stage, upstream)
|
||||
self.assertNotIn(f'movie/by/{new_notice}', upstream.calls)
|
||||
self.assertNotIn('movie/by/90', upstream.calls)
|
||||
self.assertNotIn(f'season/by/serie/{series_notice}', upstream.calls)
|
||||
with closing(sqlite3.connect(stage)) as db:
|
||||
self.assertEqual(db.execute('SELECT COUNT(*) FROM titles WHERE id IN (?,?,?)',
|
||||
(new_notice, series_notice, detail_notice)).fetchone()[0], 0)
|
||||
self.assertEqual(db.execute('SELECT title FROM titles WHERE id=90').fetchone()[0], 'اطلاعیه مهم')
|
||||
api = load_service()
|
||||
catalog = api.Catalog(stage)
|
||||
try:
|
||||
for ident in (90, new_notice, series_notice, detail_notice):
|
||||
with self.assertRaises(api.ApiError):
|
||||
catalog.dispatch(f'/api/movie/by/{ident}/public/')
|
||||
self.assertEqual(catalog.dispatch('/api/movie/by/94/public/')['title'], 'Red Notice')
|
||||
self.assertEqual(catalog.dispatch('/api/season/by/serie/2/public/')[0]['episodes'][0]['id'], 501)
|
||||
finally:
|
||||
catalog.close()
|
||||
source = stage
|
||||
self.assertEqual(self.path.read_bytes(), source_bytes)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,246 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
SERVER = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_service():
|
||||
spec = importlib.util.spec_from_file_location('catalog_api', SERVER / 'catalog_api.py')
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def fixture(directory):
|
||||
path = Path(directory) / 'catalog.sqlite'
|
||||
con = sqlite3.connect(path)
|
||||
con.executescript((SERVER / 'tests' / 'schema.sql').read_text())
|
||||
metadata = {'built_at_utc': '2026-11-02T07:00:18+00:00',
|
||||
'provenance': 'Realistic test snapshot',
|
||||
'catalog_pagination_complete': True, 'series_episode_fetch_complete': True}
|
||||
con.executemany('INSERT INTO metadata VALUES (?,?)',
|
||||
[(k, json.dumps(v)) for k, v in metadata.items()])
|
||||
con.executemany('INSERT INTO genres VALUES (?,?)', [(1, 'Horror'), (2, 'Action'), (32, 'Top 250'), (48, 'Live')])
|
||||
con.execute('INSERT INTO countries VALUES (1,?,?)', ('America', 'http://art.test/a%2Fb.jpg?token=Ab+Cd'))
|
||||
for i in range(1, 66):
|
||||
kind = 'serie' if i == 2 else 'movie'
|
||||
title = 'Alpha' if i == 1 else 'Warrior' if i == 2 else 'Film ' + str(i)
|
||||
if i == 63:
|
||||
title = 'نسخه جدید ملودی باکس'
|
||||
if i == 64:
|
||||
kind = 'channel'
|
||||
if i == 65:
|
||||
title = 'Telegram @channelname'
|
||||
item = {'id': i, 'type': kind, 'title': title, 'description': 'Actor Example بازیگران',
|
||||
'year': 2020 + i % 7, 'imdb': i / 10, 'rating': 4.1,
|
||||
'image': 'http://art.test/a%2Fb.jpg?token=Ab+Cd',
|
||||
'cover': 'https://art.test/a b.jpg', 'comment': True,
|
||||
'genres': [{'id': 2, 'title': 'Action'}], 'country': [{'id': 1, 'title': 'America'}]}
|
||||
if kind == 'movie':
|
||||
source = {'id': 100 + i, 'type': 'mkv', 'quality': '720',
|
||||
'url': 'http://media.test/Movie%20Name%2F.mkv?token=Ab+Cd&x=%2f#fragment'}
|
||||
item['sources'] = [source]
|
||||
con.execute('INSERT INTO titles(id,type,title,description,year,imdb,rating,raw_json) VALUES (?,?,?,?,?,?,?,?)',
|
||||
(i, kind, title, item['description'], item['year'], item['imdb'], item['rating'], json.dumps(item)))
|
||||
if kind == 'movie':
|
||||
con.execute('INSERT INTO movie_sources VALUES (?,?,?,?,?,?,?)',
|
||||
(i, 0, source['id'], '720', 'mkv', source['url'], json.dumps(source)))
|
||||
con.execute('INSERT INTO title_genres VALUES (?,2)', (i,))
|
||||
con.execute('INSERT INTO title_countries VALUES (?,1)', (i,))
|
||||
con.execute('INSERT INTO title_genres VALUES (1,32)')
|
||||
episode = {'id': 501, 'title': 'Episode 1', 'sources': [{'id': 201, 'type': 'mkv',
|
||||
'quality': None, 'url': 'http://media.test/a%20b.mkv?signature=%2f+Ab'}]}
|
||||
season = {'id': 1001, 'title': 'Season 1', 'episodes': [episode]}
|
||||
con.execute('INSERT INTO seasons VALUES (2,0,1001,?,?)', ('Season 1', json.dumps(season)))
|
||||
con.execute('INSERT INTO episodes(series_id,season_index,episode_index,id,title,raw_json) VALUES (2,0,0,501,?,?)',
|
||||
('Episode 1', json.dumps(episode)))
|
||||
con.execute('INSERT INTO episode_sources VALUES (2,0,0,0,201,NULL,?,?,?)',
|
||||
('mkv', episode['sources'][0]['url'], json.dumps(episode['sources'][0])))
|
||||
con.execute('INSERT INTO series_fetch_status VALUES (2,?,1,?)', ('ok', 'https://old.test/api/season/2'))
|
||||
con.commit()
|
||||
con.close()
|
||||
return path
|
||||
|
||||
|
||||
class CatalogTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(dir=os.environ.get('TMPDIR'))
|
||||
self.path = fixture(self.temp.name)
|
||||
|
||||
def tearDown(self):
|
||||
if hasattr(self, 'catalog'):
|
||||
self.catalog.close()
|
||||
self.temp.cleanup()
|
||||
|
||||
def open(self):
|
||||
self.api = load_service()
|
||||
self.catalog = self.api.Catalog(self.path)
|
||||
return self.catalog
|
||||
|
||||
def test_health_is_real_snapshot_counts_and_freshness(self):
|
||||
catalog = self.open()
|
||||
health = catalog.dispatch('/api/health')
|
||||
self.assertEqual(health['status'], 'ok')
|
||||
self.assertEqual(health['counts']['titles'], 65)
|
||||
self.assertEqual(health['counts']['episodes'], 1)
|
||||
self.assertEqual(health['counts']['visible_titles'], 62)
|
||||
self.assertEqual(health['snapshot_built_at_utc'], '2026-11-02T07:00:18+00:00')
|
||||
self.assertEqual(health['provenance'], 'Realistic test snapshot')
|
||||
self.assertFalse(health['media_proxy'])
|
||||
self.assertTrue(health['offline'])
|
||||
self.assertGreaterEqual(health['snapshot_age_seconds'], 0)
|
||||
with self.assertRaises(sqlite3.OperationalError):
|
||||
catalog.db.execute('DELETE FROM titles')
|
||||
def test_catalog_filters_pages_and_original_media_links(self):
|
||||
catalog = self.open()
|
||||
first = catalog.dispatch('/api/movie/by/filtres/0/created/0/public/')
|
||||
second = catalog.dispatch('/api/movie/by/filtres/0/created/1/public/')
|
||||
self.assertEqual(len(first), 30)
|
||||
self.assertEqual(len(second), 30)
|
||||
self.assertFalse({r['id'] for r in first} & {r['id'] for r in second})
|
||||
self.assertTrue(all(r['id'] <= 62 and r['type'] == 'movie' for r in first + second))
|
||||
self.assertEqual([r['id'] for r in catalog.dispatch('/api/movie/by/filtres/0/created/2/public/')], [1])
|
||||
self.assertEqual(catalog.dispatch('/api/movie/by/filtres/0/created/3/public/'), [])
|
||||
self.assertEqual([r['id'] for r in catalog.dispatch('/api/poster/by/filtres/32/1/imdb/0/public/')], [1])
|
||||
self.assertEqual([r['id'] for r in catalog.dispatch('/api/serie/by/filtres/0/year/0/public/')], [2])
|
||||
film = catalog.dispatch('/api/movie/by/1/public/')
|
||||
original = json.loads(catalog.db.execute('SELECT raw_json FROM titles WHERE id=1').fetchone()[0])
|
||||
self.assertEqual(film['sources'], original['sources'])
|
||||
self.assertEqual(film['image'], original['image'])
|
||||
self.assertNotIn('comment', film)
|
||||
seasons = catalog.dispatch('/api/season/by/serie/2/public/')
|
||||
self.assertEqual(seasons[0]['episodes'][0]['sources'][0]['url'], 'http://media.test/a%20b.mkv?signature=%2f+Ab')
|
||||
self.assertEqual(catalog.dispatch('/api/genre/all/public/')[0]['id'], 1)
|
||||
self.assertNotIn(48, [r['id'] for r in catalog.dispatch('/api/genre/all/public/')])
|
||||
self.assertEqual(catalog.dispatch('/api/country/all/public/')[0]['title'], 'America')
|
||||
for title_id in (63, 64, 65, 999):
|
||||
with self.assertRaises(self.api.ApiError) as error:
|
||||
catalog.dispatch(f'/api/movie/by/{title_id}/public/')
|
||||
self.assertEqual(error.exception.status, 404)
|
||||
def test_home_search_recommendations_curated_and_unexported_metadata(self):
|
||||
catalog = self.open()
|
||||
home = catalog.dispatch('/api/first/public/')
|
||||
self.assertTrue(home['genres'][0]['posters'])
|
||||
self.assertEqual(home['actors'], [])
|
||||
self.assertEqual(home['countries'][0]['id'], 1)
|
||||
search = catalog.dispatch('/api-user/newapi/like.php?action=search-movie-en&q=alpha&pageno=1')
|
||||
self.assertEqual([r['id'] for r in search], [1])
|
||||
self.assertEqual(catalog.dispatch('/api-user/newapi/like.php?action=search-movie&q=no-match&pageno=1'), [])
|
||||
self.assertEqual(catalog.dispatch('/api-user/newapi/like.php?action=search-movie&q=%27%20OR%201%3D1--'), [])
|
||||
self.assertEqual(catalog.dispatch('/api-user/newapi/like.php?action=best-rand-top&pageno=1')[0]['id'], 62)
|
||||
self.assertTrue(catalog.dispatch('/api-user/newapi/like.php?action=best-rand-year&pageno=1'))
|
||||
curated = catalog.dispatch('/api-user/newapi/pishnahad.php')
|
||||
self.assertTrue(curated)
|
||||
self.assertEqual(len(curated[0]['title']), len(curated[0]['link']))
|
||||
self.assertTrue(catalog.dispatch(curated[0]['link'][0] + '1'))
|
||||
self.assertEqual([r['id'] for r in catalog.dispatch('/api-user/newapi/filter.php?action=pishnahad&garid=32&pageno=1')], [1])
|
||||
for target in ['/api-user/newapi/like.php?action=all-actor&pageno=1',
|
||||
'/api-user/newapi/like.php?action=colection&pageno=1',
|
||||
'/api-user/filmahang/star.php?key=public&action=starsearch&q=Example&pageno=1',
|
||||
'/api-user/filmahang/star.php?key=public&action=mazmon&pageno=1',
|
||||
'/api-user/filmahang/star.php?key=public&action=share&pageno=1',
|
||||
'/api/role/by/poster/1/public/', '/api/movie/by/actor/12/public/',
|
||||
'/api/subtitles/by/movie/1/public/', '/api/subtitles/by/episode/501/public/']:
|
||||
self.assertEqual(catalog.dispatch(target), [], target)
|
||||
def test_rejects_proxy_comments_traversal_and_unbounded_inputs(self):
|
||||
catalog = self.open()
|
||||
for target in ['/api/proxy?url=http://169.254.169.254/', '/api/download/movie.mkv',
|
||||
'/api/comments/by/poster/1/public/', '/api/../catalog.sqlite',
|
||||
'/api/%2e%2e/catalog.sqlite', '//evil.test/api/health',
|
||||
'/api/movie/by/1/public/?url=https://media.test/movie.mkv',
|
||||
'/api-user/newapi/like.php?action=comments',
|
||||
'/api-user/newapi/like.php?action=search-movie&q=a&q=b',
|
||||
'/api-user/newapi/like.php?action=search-movie&q=' + 'a' * 201,
|
||||
'/api-user/newapi/like.php?action=search-movie&pageno=-1',
|
||||
'/api-user/newapi/like.php?action=search-movie&pageno=100002',
|
||||
'/api/movie/by/filtres/0/views/0/public/']:
|
||||
with self.assertRaises(self.api.ApiError, msg=target):
|
||||
catalog.dispatch(target)
|
||||
|
||||
def test_raw_controls_and_malformed_urls_are_rejected_before_parsing(self):
|
||||
catalog = self.open()
|
||||
for target in ['/api/\thealth', '\n/api/health', '/api/health\r', 'http://[',
|
||||
'/api-user/newapi/like.php?action=search-movie&q=a\tb']:
|
||||
with self.subTest(target=repr(target)):
|
||||
with self.assertRaises(self.api.ApiError) as error:
|
||||
catalog.dispatch(target)
|
||||
self.assertEqual(error.exception.status, 400)
|
||||
|
||||
def test_real_http_is_read_only_bounded_and_returns_direct_source_urls(self):
|
||||
import threading
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import HTTPError
|
||||
catalog = self.open()
|
||||
server = self.api.make_server(catalog, port=0)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
origin = 'http://127.0.0.1:' + str(server.server_port)
|
||||
try:
|
||||
with urlopen(origin + '/api/movie/by/1/public/', timeout=3) as response:
|
||||
detail = json.load(response)
|
||||
self.assertEqual(response.headers['Content-Type'], 'application/json; charset=utf-8')
|
||||
self.assertNotIn('Server', response.headers)
|
||||
self.assertEqual(detail['sources'][0]['url'], 'http://media.test/Movie%20Name%2F.mkv?token=Ab+Cd&x=%2f#fragment')
|
||||
for method in ['POST', 'PUT', 'DELETE', 'PATCH', 'CONNECT']:
|
||||
with self.assertRaises(HTTPError) as error:
|
||||
urlopen(Request(origin + '/api/health', method=method), timeout=3)
|
||||
self.assertEqual(error.exception.code, 405)
|
||||
error.exception.close()
|
||||
with self.assertRaises(HTTPError) as error:
|
||||
urlopen(origin + '/api/proxy?url=https://media.test/a.mkv', timeout=3)
|
||||
self.assertIn(error.exception.code, (400, 404))
|
||||
error.exception.close()
|
||||
with urlopen(Request(origin + '/api/health', method='HEAD'), timeout=3) as response:
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(response.read(), b'')
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(3)
|
||||
def test_worker_saturation_rejects_extra_connections_and_releases_slots(self):
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
catalog = self.open()
|
||||
server = self.api.make_server(catalog, port=0)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
sockets = []
|
||||
def wait_slots(value):
|
||||
deadline = time.monotonic() + 4
|
||||
while server.slots._value != value and time.monotonic() < deadline:
|
||||
time.sleep(.01)
|
||||
self.assertEqual(server.slots._value, value)
|
||||
try:
|
||||
for _ in range(16):
|
||||
sock = socket.create_connection(server.server_address, timeout=2)
|
||||
sock.sendall(b'GET /api/health HTTP/1.0\r\n')
|
||||
sockets.append(sock)
|
||||
wait_slots(0)
|
||||
with socket.create_connection(server.server_address, timeout=2) as extra:
|
||||
try:
|
||||
self.assertEqual(extra.recv(1), b'')
|
||||
except ConnectionResetError:
|
||||
pass
|
||||
for sock in sockets:
|
||||
sock.close()
|
||||
sockets.clear()
|
||||
wait_slots(16)
|
||||
with socket.create_connection(server.server_address, timeout=2) as recovered:
|
||||
recovered.sendall(b'GET /api/health HTTP/1.0\r\n\r\n')
|
||||
self.assertIn(b'200', recovered.recv(100))
|
||||
finally:
|
||||
for sock in sockets:
|
||||
sock.close()
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(3)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Operation-count regressions; timings are recorded separately, never asserted."""
|
||||
import json
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import HTTPError
|
||||
|
||||
import test_catalog_api as fixtures
|
||||
|
||||
|
||||
class CatalogPerformanceTests(unittest.TestCase):
|
||||
setUp = fixtures.CatalogTests.setUp
|
||||
tearDown = fixtures.CatalogTests.tearDown
|
||||
open = fixtures.CatalogTests.open
|
||||
def http_server(self, catalog):
|
||||
server = self.api.make_server(catalog, port=0)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
def stop():
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(3)
|
||||
self.addCleanup(stop)
|
||||
return 'http://127.0.0.1:' + str(server.server_port)
|
||||
|
||||
def test_repeated_snapshot_http_read_does_not_query_or_serialize_again(self):
|
||||
catalog = self.open()
|
||||
origin = self.http_server(catalog)
|
||||
with urlopen(origin + '/api/first/public/', timeout=3) as response:
|
||||
first = response.read()
|
||||
queries = []
|
||||
catalog.db.set_trace_callback(queries.append)
|
||||
with patch.object(self.api.json, 'dumps', wraps=json.dumps) as encode:
|
||||
with urlopen(origin + '/api/first/public/', timeout=3) as response:
|
||||
self.assertEqual(response.read(), first)
|
||||
self.assertEqual(queries, [], 'Warm snapshot HTTP responses must reuse encoded bytes')
|
||||
self.assertEqual(encode.call_count, 0)
|
||||
|
||||
def test_cache_bounds_lru_and_large_response_bypass(self):
|
||||
catalog = self.open()
|
||||
catalog.RESPONSE_CACHE_ENTRIES = 2
|
||||
one = '/api/movie/by/1/public/'
|
||||
two = '/api/movie/by/2/public/'
|
||||
three = '/api/movie/by/3/public/'
|
||||
first = catalog.response(one)
|
||||
catalog.response(two)
|
||||
self.assertIs(catalog.response(one), first)
|
||||
catalog.response(three)
|
||||
self.assertEqual(list(catalog._responses), [one, three])
|
||||
self.assertLessEqual(catalog._response_bytes, catalog.RESPONSE_CACHE_BYTES)
|
||||
catalog.RESPONSE_CACHE_ITEM_BYTES = 1
|
||||
catalog.response(two)
|
||||
self.assertNotIn(two, catalog._responses)
|
||||
# Byte budget, not just number of entries, forces eviction.
|
||||
catalog.RESPONSE_CACHE_ITEM_BYTES = 1024 * 1024
|
||||
catalog.RESPONSE_CACHE_BYTES = len(catalog.response(three)) + len(three) + 1
|
||||
catalog.response(two)
|
||||
self.assertLessEqual(catalog._response_bytes, catalog.RESPONSE_CACHE_BYTES)
|
||||
self.assertEqual(len(catalog._responses), 1)
|
||||
|
||||
def test_dynamic_responses_errors_and_queries_are_never_retained(self):
|
||||
catalog = self.open()
|
||||
with patch.object(catalog, 'health', side_effect=[{'age': 1}, {'age': 2}]):
|
||||
self.assertNotEqual(catalog.response('/api/health'), catalog.response('/api/health'))
|
||||
with patch.object(self.api, 'load_banners', side_effect=[{'banners': [1]}, {'banners': [2]}]):
|
||||
self.assertNotEqual(catalog.response('/api/banners'), catalog.response('/api/banners'))
|
||||
catalog.response('/api-user/newapi/like.php?action=search-movie&q=Alpha')
|
||||
for _ in range(2):
|
||||
with self.assertRaises(self.api.ApiError):
|
||||
catalog.response('/api/movie/by/999/public/')
|
||||
self.assertEqual(list(catalog._responses), [])
|
||||
|
||||
def test_warm_head_reuses_get_bytes_and_body_requests_still_fail(self):
|
||||
catalog = self.open()
|
||||
origin = self.http_server(catalog)
|
||||
with urlopen(origin + '/api/first/public/', timeout=3) as response:
|
||||
first = response.read()
|
||||
with urlopen(Request(origin + '/api/first/public/', method='HEAD'), timeout=3) as response:
|
||||
self.assertEqual(response.read(), b'')
|
||||
self.assertEqual(int(response.headers['Content-Length']), len(first))
|
||||
with self.assertRaises(HTTPError) as error:
|
||||
urlopen(Request(origin + '/api/first/public/', method='GET', data=b'x'), timeout=3)
|
||||
self.assertEqual(error.exception.code, 400)
|
||||
error.exception.close()
|
||||
|
||||
def test_normalized_search_matches_fallback_with_unicode_and_pagination(self):
|
||||
catalog = self.open()
|
||||
indexed = catalog.search_text
|
||||
self.assertLessEqual(catalog.search_index_bytes, 64 * 1024 * 1024)
|
||||
for query in ['Actor', 'بازیگران', 'بازيگران', 'ALPHA', 'not-present', 'Film 4']:
|
||||
for page in [0, 1, 2]:
|
||||
catalog.search_text = indexed
|
||||
expected = catalog.titles(query=query, page=page)
|
||||
catalog.search_text = {}
|
||||
self.assertEqual(catalog.titles(query=query, page=page), expected)
|
||||
catalog.search_text = indexed
|
||||
|
||||
def test_search_normalizes_query_not_every_snapshot_row(self):
|
||||
catalog = self.open()
|
||||
with patch.object(self.api, 'normal', wraps=self.api.normal) as normalize:
|
||||
catalog.db.create_function('normal', 1, normalize, deterministic=True)
|
||||
result = catalog.dispatch('/api-user/newapi/like.php?action=search-movie&q=ACTOR&pageno=1')
|
||||
self.assertEqual([row['id'] for row in result], list(range(62, 32, -1)))
|
||||
self.assertLessEqual(normalize.call_count, 2,
|
||||
'Immutable title text must not be normalized on every search')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,463 @@
|
||||
"""Synchronizer tests use the exported schema, never the upstream service."""
|
||||
import copy
|
||||
from contextlib import closing
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import ssl
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from urllib.error import HTTPError, URLError
|
||||
|
||||
from test_catalog_api import SERVER, fixture, load_service
|
||||
|
||||
|
||||
def load_sync():
|
||||
spec = importlib.util.spec_from_file_location('catalog_sync', SERVER / 'catalog_sync.py')
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
with patch.object(sys, 'path', [str(SERVER)] + sys.path):
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class Response(io.BytesIO):
|
||||
status = 200
|
||||
headers = {}
|
||||
|
||||
def __init__(self, payload=b'[]', url=None, status=200):
|
||||
super().__init__(payload)
|
||||
self.url, self.status = url, status
|
||||
|
||||
def geturl(self):
|
||||
return self.url
|
||||
|
||||
|
||||
class QueueOpener:
|
||||
def __init__(self, *responses):
|
||||
self.responses = list(responses)
|
||||
self.calls = []
|
||||
|
||||
def open(self, request, timeout):
|
||||
self.calls.append((request.full_url, request.get_method(), timeout))
|
||||
response = self.responses.pop(0)
|
||||
if isinstance(response, Exception):
|
||||
raise response
|
||||
if response.url is None:
|
||||
response.url = request.full_url
|
||||
return response
|
||||
|
||||
|
||||
class UpstreamTests(unittest.TestCase):
|
||||
def test_exact_fixed_origin_metadata_urls_and_quoted_key(self):
|
||||
sync = load_sync()
|
||||
opener = QueueOpener(*(Response(b'{}' if p.startswith('movie/by/3') else b'[]') for p in
|
||||
['genre/all', 'country/all', 'movie/by/35721',
|
||||
'movie/by/filtres/0/created/0', 'serie/by/filtres/0/created/12',
|
||||
'season/by/serie/35716']))
|
||||
client = sync.Upstream('test secret/+#?', interval=0, opener=opener)
|
||||
paths = ['genre/all', 'country/all', 'movie/by/35721', 'movie/by/filtres/0/created/0',
|
||||
'serie/by/filtres/0/created/12', 'season/by/serie/35716']
|
||||
for path in paths:
|
||||
client.get(path)
|
||||
self.assertEqual(opener.calls, [('https://ninwinios.info/api/' + p +
|
||||
'/test%20secret%2F%2B%23%3F/', 'GET', 30) for p in paths])
|
||||
self.assertEqual(client.requests_count, 6)
|
||||
for bad in ['https://evil.test/api/genre/all', '//evil.test/a', '/genre/all',
|
||||
'genre/all/', 'genre/../all', 'genre/%2e%2e/all', 'genre/all?url=x',
|
||||
'genre/all#x', 'movie/by/1/surprise', 'download/movie.mkv',
|
||||
'movie/by/1\n', 'movie/by/0', 'movie/by/-1', 'movie/by/filtres/0/views/0']:
|
||||
with self.subTest(path=repr(bad)), self.assertRaises(ValueError):
|
||||
client.get(bad)
|
||||
self.assertEqual(client.requests_count, 6)
|
||||
|
||||
def test_real_upstream_legacy_html_label_still_requires_valid_json(self):
|
||||
sync = load_sync()
|
||||
response = Response(b'[{"id":1,"title":"Action"}]')
|
||||
response.headers = {'Content-Type': 'text/html; charset=UTF-8'}
|
||||
client = sync.Upstream('k', interval=0, opener=QueueOpener(response))
|
||||
self.assertEqual(client.get('genre/all'), [{'id': 1, 'title': 'Action'}])
|
||||
html = Response(b'<html>Service unavailable</html>')
|
||||
html.headers = {'Content-Type': 'text/html; charset=UTF-8'}
|
||||
opener = QueueOpener(html)
|
||||
with self.assertRaises(sync.SyncError):
|
||||
sync.Upstream('k', interval=0, opener=opener).get('genre/all')
|
||||
self.assertEqual(len(opener.calls), 1)
|
||||
|
||||
def test_wrong_content_type_and_truncated_length_are_not_retried(self):
|
||||
sync = load_sync()
|
||||
for headers in ({'Content-Type': 'video/mp4'}, {'Content-Type': 'image/jpeg'},
|
||||
{'Content-Length': '999'}, {'Content-Length': 'bad'},
|
||||
{'Content-Length': str(32 * 1024 * 1024 + 1)}):
|
||||
response = Response(b'[]')
|
||||
response.headers = headers
|
||||
opener = QueueOpener(response)
|
||||
with self.subTest(headers=headers), self.assertRaises(sync.SyncError):
|
||||
sync.Upstream('k', interval=0, opener=opener).get('genre/all')
|
||||
self.assertEqual(len(opener.calls), 1)
|
||||
|
||||
def test_bounded_retry_and_rate_limit(self):
|
||||
sync = load_sync()
|
||||
for status in (429, 503):
|
||||
error = HTTPError('https://secret.test/key', status, 'private key', {}, None)
|
||||
opener = QueueOpener(error, Response(b'[]'))
|
||||
client = sync.Upstream('private-key', interval=.25, opener=opener)
|
||||
with patch.object(sync.time, 'sleep') as sleep:
|
||||
self.assertEqual(client.get('genre/all'), [])
|
||||
self.assertGreaterEqual(sleep.call_count, 1)
|
||||
self.assertEqual(client.requests_count, 2)
|
||||
opener = QueueOpener(URLError('private URL'), Response(b'[]'))
|
||||
with patch.object(sync.time, 'sleep'):
|
||||
self.assertEqual(sync.Upstream('k', interval=0, opener=opener).get('genre/all'), [])
|
||||
opener = QueueOpener(Response(), Response())
|
||||
client = sync.Upstream('k', interval=.25, opener=opener)
|
||||
with patch.object(sync.time, 'monotonic', return_value=100), patch.object(sync.time, 'sleep') as sleep:
|
||||
client.get('genre/all')
|
||||
client.get('country/all')
|
||||
sleep.assert_called_once_with(.25)
|
||||
|
||||
def test_auth_cert_redirect_json_and_size_fail_closed_without_secrets(self):
|
||||
sync = load_sync()
|
||||
for failure in [HTTPError('https://secret.test/private-key', code, 'private-key', {}, None)
|
||||
for code in (301, 302, 307, 308, 401, 403, 404)] + [
|
||||
URLError(ssl.SSLCertVerificationError('private-key')),
|
||||
ssl.SSLCertVerificationError('private-key')]:
|
||||
opener = QueueOpener(failure)
|
||||
client = sync.Upstream('private-key', interval=0, opener=opener)
|
||||
with self.assertRaises(sync.SyncError) as error:
|
||||
client.get('genre/all')
|
||||
self.assertNotIn('private-key', str(error.exception))
|
||||
self.assertNotIn('https:', str(error.exception))
|
||||
self.assertEqual(len(opener.calls), 1)
|
||||
for response in [Response(b'not JSON'), Response(b'NaN'), Response(b'{}'),
|
||||
Response(b'null'), Response(b'[]', url='https://evil.test/video.mkv'),
|
||||
Response(b'[]', status=302), Response(b'[' * (32 * 1024 * 1024 + 1))]:
|
||||
opener = QueueOpener(response)
|
||||
with self.assertRaises(sync.SyncError):
|
||||
sync.Upstream('private-key', interval=0, opener=opener).get('genre/all')
|
||||
self.assertEqual(len(opener.calls), 1)
|
||||
client = sync.Upstream('k', interval=0)
|
||||
from urllib.request import HTTPRedirectHandler
|
||||
handlers = [h for h in client._opener.handlers if isinstance(h, HTTPRedirectHandler)]
|
||||
self.assertEqual(len(handlers), 1)
|
||||
with self.assertRaises(sync.SyncError):
|
||||
handlers[0].redirect_request(None, None, 302, 'moved', {}, 'https://evil.test/video')
|
||||
|
||||
def test_retry_exhaustion_and_configuration_are_bounded(self):
|
||||
sync = load_sync()
|
||||
opener = QueueOpener(*(HTTPError('https://secret/key', 503, 'key', {}, None) for _ in range(3)))
|
||||
with patch.object(sync.time, 'sleep'), self.assertRaises(sync.SyncError):
|
||||
sync.Upstream('k', interval=0, opener=opener).get('genre/all')
|
||||
self.assertEqual(len(opener.calls), 3)
|
||||
for kwargs in ({'attempts': 0}, {'attempts': 11}, {'interval': -1}, {'timeout': 0},
|
||||
{'interval': float('nan')}, {'timeout': float('inf')}):
|
||||
with self.subTest(kwargs=kwargs), self.assertRaises(ValueError):
|
||||
sync.Upstream('k', **kwargs)
|
||||
|
||||
|
||||
class FakeUpstream:
|
||||
def __init__(self, routes=None):
|
||||
self.routes = {'genre/all': [], 'country/all': [],
|
||||
'movie/by/filtres/0/created/0': [],
|
||||
'movie/by/filtres/0/created/1': [],
|
||||
'serie/by/filtres/0/created/0': [],
|
||||
'serie/by/filtres/0/created/1': [], 'season/by/serie/2': []}
|
||||
self.routes.update(routes or {})
|
||||
self.calls = []
|
||||
self.requests_count = 0
|
||||
|
||||
def get(self, path):
|
||||
self.calls.append(path)
|
||||
self.requests_count += 1
|
||||
if path not in self.routes:
|
||||
raise AssertionError('Unexpected metadata path: ' + path)
|
||||
value = self.routes[path]
|
||||
if isinstance(value, Exception):
|
||||
raise value
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
class SynchronizeTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(dir=os.environ.get('TMPDIR'))
|
||||
self.source = fixture(self.temp.name)
|
||||
self.stage = Path(self.temp.name) / 'stage.sqlite'
|
||||
self.sync = load_sync()
|
||||
|
||||
def tearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
def rows(self, path, sql, args=()):
|
||||
with closing(sqlite3.connect(path)) as db:
|
||||
return db.execute(sql, args).fetchall()
|
||||
|
||||
def metadata(self, path):
|
||||
return {key: json.loads(value) for key, value in self.rows(path, 'SELECT * FROM metadata')}
|
||||
|
||||
def test_all_known_visible_series_refresh_and_series_samples_are_bounded(self):
|
||||
with closing(sqlite3.connect(self.source)) as db:
|
||||
for title_id, title, description in [(90, 'An older series', 'Normal description'),
|
||||
(91, 'Telegram @channelname', ''),
|
||||
(92, 'Disguised advertisement', 'ملودی باکس')]:
|
||||
item = {'id': title_id, 'type': 'serie', 'title': title, 'description': description}
|
||||
db.execute('INSERT INTO titles(id,type,title,description,raw_json) VALUES (?,?,?,?,?)',
|
||||
(title_id, 'serie', title, description, json.dumps(item)))
|
||||
db.commit()
|
||||
upstream = FakeUpstream({'season/by/serie/90': [{'id': 900, 'title': 'Season 1', 'episodes': [
|
||||
{'id': 901, 'title': 'New episode on an old series'}]}]})
|
||||
report = self.sync.synchronize(self.source, self.stage, upstream)
|
||||
self.assertEqual(report['added']['episodes'], 1)
|
||||
self.assertIn('season/by/serie/2', upstream.calls)
|
||||
self.assertIn('season/by/serie/90', upstream.calls)
|
||||
self.assertNotIn('season/by/serie/91', upstream.calls)
|
||||
self.assertNotIn('season/by/serie/92', upstream.calls)
|
||||
self.assertEqual(report['counts']['series'], 4)
|
||||
self.assertEqual(self.rows(self.stage, 'SELECT COUNT(*) FROM titles WHERE id IN (91,92)'), [(2,)])
|
||||
self.stage.unlink()
|
||||
records = [{'id': i, 'type': 'serie', 'title': 'New Series ' + str(i)} for i in range(100, 201)]
|
||||
routes = {'serie/by/filtres/0/created/0': records, 'season/by/serie/90': []}
|
||||
routes.update({f'season/by/serie/{item["id"]}': [] for item in records})
|
||||
report = self.sync.synchronize(self.source, self.stage, FakeUpstream(routes))
|
||||
self.assertEqual(report['updated_series_count'], 101)
|
||||
self.assertEqual(len(report['updated_series_ids']), 100)
|
||||
self.assertEqual(report['updated_series_ids'], list(range(100, 200)))
|
||||
|
||||
def test_progress_failure_discards_candidate_and_missing_source_is_not_created(self):
|
||||
original = self.source.read_bytes()
|
||||
def reject(_event):
|
||||
raise RuntimeError('private-key')
|
||||
with self.assertRaises(self.sync.SyncError) as error:
|
||||
self.sync.synchronize(self.source, self.stage, FakeUpstream(), progress=reject)
|
||||
self.assertNotIn('private-key', str(error.exception))
|
||||
self.assertEqual(self.source.read_bytes(), original)
|
||||
self.assertFalse(self.stage.exists())
|
||||
missing = Path(self.temp.name) / 'missing.sqlite'
|
||||
with self.assertRaises(self.sync.SyncError):
|
||||
self.sync.synchronize(missing, self.stage, FakeUpstream())
|
||||
self.assertFalse(missing.exists())
|
||||
self.assertFalse(self.stage.exists())
|
||||
|
||||
def test_malformed_metadata_discards_candidate_not_source(self):
|
||||
original = self.source.read_bytes()
|
||||
cases = [
|
||||
{'genre/all': {'error': 'not a list'}},
|
||||
{'country/all': [{'error': 'bad metadata'}]},
|
||||
{'movie/by/filtres/0/created/0': {'error': 'failed'}},
|
||||
{'serie/by/filtres/0/created/0': [None]},
|
||||
{'serie/by/filtres/0/created/0': [{'id': 70, 'type': 'serie'}]},
|
||||
{'serie/by/filtres/0/created/0': [{'id': 70, 'type': 'serie', 'title': {'bad': 'title'}}]},
|
||||
{'movie/by/filtres/0/created/0': [{'id': True, 'type': 'movie', 'title': 'Bad'}]},
|
||||
{'movie/by/filtres/0/created/0': [{'id': 1, 'type': 'movie', 'title': 'Alpha', 'sources': None}]},
|
||||
{'season/by/serie/2': {'error': 'not seasons'}},
|
||||
{'season/by/serie/2': [{'error': 'not a season'}]},
|
||||
{'season/by/serie/2': [{'id': 1001, 'episodes': {'error': 'not episodes'}}]},
|
||||
{'season/by/serie/2': [{'id': 1001, 'episodes': [{'error': 'not an episode'}]}]},
|
||||
{'season/by/serie/2': [{'id': 1001, 'episodes': [{'id': 501, 'sources': [{}]}]}]},
|
||||
{'season/by/serie/2': [{'id': 1001, 'episodes': [{'id': 501, 'sources': [{'id': 201, 'url': []}]}]}]},
|
||||
{'season/by/serie/2': [{'id': 1001, 'episodes': [{'id': 501, 'sources': 'not sources'}]}]},
|
||||
{'season/by/serie/2': [{'id': 1001, 'episodes': [{'id': 501}, {'id': 501}]}]},
|
||||
{'season/by/serie/2': ValueError('https://private.test/private-key')},
|
||||
]
|
||||
for routes in cases:
|
||||
with self.subTest(routes=routes):
|
||||
with self.assertRaises(self.sync.SyncError) as error:
|
||||
self.sync.synchronize(self.source, self.stage, FakeUpstream(routes))
|
||||
self.assertNotIn('private-key', str(error.exception))
|
||||
self.assertFalse(self.stage.exists())
|
||||
self.assertEqual(self.source.read_bytes(), original)
|
||||
|
||||
def test_429_503_exhaustion_rolls_back_staged_writes(self):
|
||||
original = self.source.read_bytes()
|
||||
for status in (429, 503):
|
||||
opener = QueueOpener(Response(b'[{"id":99,"title":"A new genre"}]'), Response(),
|
||||
*(HTTPError('https://private.test/private-key', status, 'private-key', {}, None)
|
||||
for _ in range(3)))
|
||||
upstream = self.sync.Upstream('private-key', interval=0, opener=opener)
|
||||
with patch.object(self.sync.time, 'sleep'), self.assertRaises(self.sync.SyncError):
|
||||
self.sync.synchronize(self.source, self.stage, upstream)
|
||||
self.assertEqual(len(opener.calls), 5)
|
||||
self.assertFalse(self.stage.exists())
|
||||
self.assertEqual(self.source.read_bytes(), original)
|
||||
|
||||
def test_page_limit_and_existing_candidates_are_never_published_or_overwritten(self):
|
||||
original = self.source.read_bytes()
|
||||
routes = {'movie/by/filtres/0/created/0': [{'id': 1, 'type': 'movie', 'title': 'Updated'}]}
|
||||
with self.assertRaises(self.sync.SyncError):
|
||||
self.sync.synchronize(self.source, self.stage, FakeUpstream(routes), max_pages=1)
|
||||
self.assertFalse(self.stage.exists())
|
||||
self.assertEqual(self.source.read_bytes(), original)
|
||||
self.stage.write_bytes(b'existing candidate')
|
||||
with self.assertRaises(FileExistsError):
|
||||
self.sync.synchronize(self.source, self.stage, FakeUpstream())
|
||||
self.assertEqual(self.stage.read_bytes(), b'existing candidate')
|
||||
with self.assertRaises(ValueError):
|
||||
self.sync.synchronize(self.source, self.source, FakeUpstream())
|
||||
self.assertEqual(self.source.read_bytes(), original)
|
||||
|
||||
def test_existing_id_does_not_end_pagination_and_missing_ids_use_indices(self):
|
||||
routes = {'movie/by/filtres/0/created/0': [{'id': 1, 'type': 'movie', 'title': 'Alpha'}],
|
||||
'movie/by/filtres/0/created/1': [{'id': 70, 'type': 'movie', 'title': 'Next page'}],
|
||||
'movie/by/filtres/0/created/2': [],
|
||||
'movie/by/70': {'id': 70, 'type': 'movie', 'title': 'Next page', 'sources': []},
|
||||
'season/by/serie/2': [{'title': 'Season 1 refreshed', 'episodes': [
|
||||
{'title': 'Episode 1 refreshed', 'sources': []}, {'title': 'Episode without ID', 'sources': []}]}]}
|
||||
report = self.sync.synchronize(self.source, self.stage, FakeUpstream(routes))
|
||||
self.assertEqual(report['added']['movies'], 1)
|
||||
self.assertEqual(report['added']['episodes'], 1)
|
||||
self.assertEqual(self.rows(self.stage, 'SELECT id,title FROM seasons'), [(1001, 'Season 1 refreshed')])
|
||||
self.assertEqual(self.rows(self.stage, 'SELECT id,title FROM episodes ORDER BY episode_index'),
|
||||
[(501, 'Episode 1 refreshed'), (None, 'Episode without ID')])
|
||||
self.assertEqual(self.rows(self.stage, 'SELECT source_id FROM episode_sources'), [(201,)])
|
||||
|
||||
def test_advertisements_channels_and_detail_ads_are_not_imported(self):
|
||||
items = [
|
||||
{'id': 70, 'type': 'movie', 'title': 'Telegram @channelname'},
|
||||
{'id': 71, 'type': 'movie', 'title': 'Innocent', 'description': 'نسخه جدید ملودی باکس'},
|
||||
{'id': 72, 'type': 'channel', 'title': 'Live channel'},
|
||||
{'id': 73, 'type': 'movie', 'title': 'Looks like a movie'}]
|
||||
upstream = FakeUpstream({'movie/by/filtres/0/created/0': items,
|
||||
'movie/by/filtres/0/created/1': [],
|
||||
'movie/by/73': {'id': 73, 'type': 'movie', 'title': 'Telegram @channelname'}})
|
||||
report = self.sync.synchronize(self.source, self.stage, upstream)
|
||||
self.assertEqual(report['added']['titles'], 0)
|
||||
self.assertNotIn('movie/by/70', upstream.calls)
|
||||
self.assertNotIn('movie/by/71', upstream.calls)
|
||||
self.assertNotIn('movie/by/72', upstream.calls)
|
||||
self.assertEqual(self.rows(self.stage, 'SELECT COUNT(*) FROM titles'), [(65,)])
|
||||
|
||||
def test_reordered_stable_ids_and_truncated_nested_lists_retain_every_old_entry(self):
|
||||
first = {'season/by/serie/2': [
|
||||
{'id': 1001, 'episodes': [{'id': 502, 'title': 'Episode 2', 'sources': [
|
||||
{'id': 202, 'url': 'https://video.test/old2'}]}]},
|
||||
{'id': 1002, 'title': 'Season 2', 'episodes': [{'id': 601, 'title': 'S2 E1'}]}]}
|
||||
self.sync.synchronize(self.source, self.stage, FakeUpstream(first))
|
||||
second = Path(self.temp.name) / 'second.sqlite'
|
||||
routes = {'movie/by/filtres/0/created/0': [{'id': 1, 'type': 'movie', 'title': 'Updated Alpha',
|
||||
'sources': [], 'genres': []}],
|
||||
'movie/by/filtres/0/created/1': [],
|
||||
'season/by/serie/2': [
|
||||
{'id': 1002, 'title': 'Season Two', 'episodes': []},
|
||||
{'id': 1001, 'episodes': [{'id': 502, 'title': 'Updated Episode 2', 'sources': [
|
||||
{'id': 202, 'url': 'https://video.test/refreshed'}]},
|
||||
{'id': 501, 'sources': []}]}]}
|
||||
report = self.sync.synchronize(self.stage, second, FakeUpstream(routes))
|
||||
self.assertEqual(report['added'], dict.fromkeys(report['added'], 0))
|
||||
self.assertEqual(self.rows(second, 'SELECT season_index,id FROM seasons ORDER BY season_index'),
|
||||
[(0, 1001), (1, 1002)])
|
||||
self.assertEqual(self.rows(second, 'SELECT season_index,episode_index,id FROM episodes ORDER BY 1,2'),
|
||||
[(0, 0, 501), (0, 1, 502), (1, 0, 601)])
|
||||
self.assertEqual(self.rows(second, 'SELECT url FROM episode_sources WHERE source_id=202'),
|
||||
[('https://video.test/refreshed',)])
|
||||
for table in ('seasons', 'episodes', 'movie_sources', 'episode_sources'):
|
||||
self.assertGreaterEqual(self.rows(second, 'SELECT COUNT(*) FROM ' + table)[0][0],
|
||||
self.rows(self.stage, 'SELECT COUNT(*) FROM ' + table)[0][0])
|
||||
third = Path(self.temp.name) / 'third.sqlite'
|
||||
self.sync.synchronize(second, third, FakeUpstream())
|
||||
for table in ('titles', 'seasons', 'episodes', 'movie_sources', 'episode_sources'):
|
||||
self.assertEqual(self.rows(second, 'SELECT * FROM ' + table), self.rows(third, 'SELECT * FROM ' + table))
|
||||
season = json.loads(self.rows(second, 'SELECT raw_json FROM seasons WHERE id=1001')[0][0])
|
||||
self.assertEqual([e['id'] for e in season['episodes']], [501, 502])
|
||||
self.assertEqual(season['episodes'][1]['sources'][0]['url'], 'https://video.test/refreshed')
|
||||
movie = json.loads(self.rows(second, 'SELECT raw_json FROM titles WHERE id=1')[0][0])
|
||||
self.assertTrue(movie['sources'])
|
||||
self.assertTrue(movie['genres'])
|
||||
|
||||
def test_old_series_absent_from_catalogue_and_new_series_receive_episodes_idempotently(self):
|
||||
new_series = {'id': 70, 'type': 'serie', 'title': 'New Series'}
|
||||
old_episode = {'id': 501, 'title': 'Episode 1', 'sources': []}
|
||||
new_episode = {'id': 502, 'title': 'Episode 2', 'sources': [
|
||||
{'id': 202, 'quality': '1080', 'type': 'mkv', 'url': 'https://video.test/e2.mkv'}]}
|
||||
routes = {'serie/by/filtres/0/created/0': [new_series],
|
||||
'serie/by/filtres/0/created/1': [],
|
||||
'season/by/serie/2': [{'id': 1001, 'title': 'Season 1', 'episodes': [old_episode, new_episode]}],
|
||||
'season/by/serie/70': [{'id': 1070, 'title': 'New Season', 'episodes': [
|
||||
{'id': 701, 'title': 'Pilot', 'sources': [
|
||||
{'id': 702, 'url': 'https://video.test/pilot.mp4', 'type': 'mp4'}]}]}]}
|
||||
report = self.sync.synchronize(self.source, self.stage, FakeUpstream(routes))
|
||||
self.assertEqual(report['added'], {'titles': 1, 'movies': 0, 'series': 1, 'episodes': 2, 'sources': 2})
|
||||
self.assertEqual(report['updated_series_ids'], [2, 70])
|
||||
self.assertEqual(report['updated_series_count'], 2)
|
||||
self.assertEqual(self.rows(self.stage, 'SELECT id FROM episodes ORDER BY id'), [(501,), (502,), (701,)])
|
||||
raw = json.loads(self.rows(self.stage, 'SELECT raw_json FROM seasons WHERE series_id=2')[0][0])
|
||||
self.assertEqual(len(raw['episodes']), 2)
|
||||
self.assertEqual(raw['episodes'][0]['sources'][0]['id'], 201)
|
||||
self.assertEqual(raw['episodes'][1], new_episode)
|
||||
catalog = load_service().Catalog(self.stage)
|
||||
try:
|
||||
served = catalog.dispatch('/api/season/by/serie/2/public/')
|
||||
self.assertEqual(served[0]['episodes'][1], new_episode)
|
||||
finally:
|
||||
catalog.close()
|
||||
second_stage = Path(self.temp.name) / 'second.sqlite'
|
||||
again = self.sync.synchronize(self.stage, second_stage, FakeUpstream(routes))
|
||||
self.assertEqual(again['added'], dict.fromkeys(report['added'], 0))
|
||||
self.assertEqual(again['updated_series_ids'], [])
|
||||
for table in ('titles', 'seasons', 'episodes', 'movie_sources', 'episode_sources'):
|
||||
self.assertEqual(self.rows(self.stage, 'SELECT * FROM ' + table),
|
||||
self.rows(second_stage, 'SELECT * FROM ' + table))
|
||||
self.assertEqual(self.metadata(second_stage)['original_snapshot_built_at_utc'],
|
||||
self.metadata(self.source)['built_at_utc'])
|
||||
for (url,) in self.rows(second_stage, 'SELECT source_url FROM series_fetch_status'):
|
||||
self.assertTrue(url.startswith('https://ninwinios.info/api/season/by/serie/'))
|
||||
self.assertNotIn('key', url)
|
||||
|
||||
def test_new_movie_details_staged_source_untouched_and_real_counts(self):
|
||||
original = self.source.read_bytes()
|
||||
movie = {'id': 70, 'type': 'movie', 'title': 'New Film', 'year': 2026,
|
||||
'genres': [{'id': 5, 'title': 'Drama'}], 'country': [{'id': 3, 'title': 'Iran'}],
|
||||
'image': 'https://images.test/never-fetch.jpg'}
|
||||
detail = dict(movie, description='New description', sources=[
|
||||
{'id': 700, 'type': 'mkv', 'url': 'https://video.test/never-fetch.mkv?key=opaque'}])
|
||||
upstream = FakeUpstream({'genre/all': [{'id': 5, 'title': 'Drama'}],
|
||||
'country/all': [{'id': 3, 'title': 'Iran'}],
|
||||
'movie/by/filtres/0/created/0': [movie, movie],
|
||||
'movie/by/filtres/0/created/1': [], 'movie/by/70': detail})
|
||||
progress = []
|
||||
report = self.sync.synchronize(self.source, self.stage, upstream, progress=progress.append)
|
||||
self.assertEqual(self.source.read_bytes(), original)
|
||||
self.assertEqual(report['added']['titles'], 1)
|
||||
self.assertEqual(report['added']['movies'], 1)
|
||||
self.assertEqual(report['added']['series'], 0)
|
||||
self.assertEqual(report['added']['sources'], 1)
|
||||
self.assertEqual(report['counts']['titles'], 66)
|
||||
self.assertEqual(report['counts']['episodes'], 1)
|
||||
self.assertEqual(report['requests_count'], upstream.requests_count)
|
||||
self.assertFalse(report['media_files_downloaded'])
|
||||
self.assertEqual(upstream.calls.count('movie/by/70'), 1)
|
||||
self.assertIn('season/by/serie/2', upstream.calls)
|
||||
self.assertTrue(all('http' not in path and '.mkv' not in path and '.jpg' not in path
|
||||
for path in upstream.calls))
|
||||
raw = json.loads(self.rows(self.stage, 'SELECT raw_json FROM titles WHERE id=70')[0][0])
|
||||
self.assertEqual(raw, detail)
|
||||
self.assertEqual(self.rows(self.stage, 'SELECT url FROM movie_sources WHERE title_id=70')[0][0],
|
||||
detail['sources'][0]['url'])
|
||||
catalog = load_service().Catalog(self.stage)
|
||||
try:
|
||||
served = catalog.dispatch('/api/movie/by/70/public/')
|
||||
self.assertEqual(served, detail)
|
||||
finally:
|
||||
catalog.close()
|
||||
self.assertEqual(self.rows(self.stage, 'PRAGMA integrity_check'), [('ok',)])
|
||||
self.assertEqual(self.rows(self.stage, 'PRAGMA foreign_key_check'), [])
|
||||
metadata = self.metadata(self.stage)
|
||||
self.assertEqual(metadata['original_snapshot_built_at_utc'], self.metadata(self.source)['built_at_utc'])
|
||||
self.assertNotEqual(metadata['built_at_utc'], self.metadata(self.source)['built_at_utc'])
|
||||
self.assertEqual(metadata['built_at_utc'], metadata['synchronized_at_utc'])
|
||||
self.assertEqual(metadata['sync_interval_seconds'], 259200)
|
||||
self.assertEqual(metadata['counts']['titles'], 66)
|
||||
self.assertFalse(metadata['media_files_downloaded'])
|
||||
self.assertEqual(metadata['last_sync']['upstream_base'], 'https://ninwinios.info/api/')
|
||||
self.assertEqual(json.loads(json.dumps(report)), report)
|
||||
self.assertTrue(progress)
|
||||
for item in progress:
|
||||
self.assertLessEqual(set(item), {'phase', 'count', 'total'})
|
||||
self.assertIsInstance(item['phase'], str)
|
||||
self.assertIsInstance(item['count'], int)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,271 @@
|
||||
"""Atomic publication and rollback for the catalogue updater."""
|
||||
import grp
|
||||
import hashlib
|
||||
from contextlib import closing, redirect_stdout
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import stat
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import test_catalog_api as fixtures
|
||||
|
||||
SERVER = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def runner_module():
|
||||
spec = importlib.util.spec_from_file_location('catalog_sync_runner', SERVER / 'catalog_sync_runner.py')
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class SyncRunnerTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temp.name)
|
||||
self.database = fixtures.fixture(self.root)
|
||||
self.state = self.root / 'state'
|
||||
self.stamp = '2026-11-03T19:00:00+00:00'
|
||||
self.api = runner_module()
|
||||
|
||||
def tearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
def candidate(self, source, stage, upstream, **kwargs):
|
||||
with closing(sqlite3.connect(source)) as src, closing(sqlite3.connect(stage)) as dest:
|
||||
src.backup(dest)
|
||||
dest.execute('UPDATE metadata SET value_json=? WHERE key=?',
|
||||
(json.dumps(self.stamp), 'built_at_utc'))
|
||||
dest.commit()
|
||||
return {'added_movies': 0, 'added_episodes': 0, 'built_at_utc': self.stamp}
|
||||
|
||||
def digest(self):
|
||||
return hashlib.sha256(self.database.read_bytes()).hexdigest()
|
||||
|
||||
def test_progress_normalizes_core_count_without_logging_other_fields(self):
|
||||
for event, completed in (({'count': 0}, 0), ({'count': 3}, 3),
|
||||
({'completed': 4, 'count': 5}, 4)):
|
||||
with self.subTest(event=event):
|
||||
output = io.StringIO()
|
||||
with redirect_stdout(output):
|
||||
self.api.emit_progress({**event, 'phase': 'movies', 'total': 7,
|
||||
'api_key': 'never-print-this'})
|
||||
self.assertEqual(json.loads(output.getvalue()),
|
||||
{'progress': {'phase': 'movies', 'completed': completed, 'total': 7}})
|
||||
|
||||
def test_publish_is_atomic_and_records_success_after_health(self):
|
||||
before = self.digest()
|
||||
events = []
|
||||
def restart():
|
||||
events.append('restart')
|
||||
with closing(sqlite3.connect(self.database)) as db:
|
||||
stamp = json.loads(db.execute("SELECT value_json FROM metadata WHERE key='built_at_utc'").fetchone()[0])
|
||||
self.assertEqual(stamp, self.stamp)
|
||||
def health(stamp):
|
||||
self.assertEqual(stamp, self.stamp)
|
||||
self.assertFalse((self.state / 'last-success.json').exists())
|
||||
events.append('healthy')
|
||||
result = self.api.run_update(self.database, self.state, object(),
|
||||
synchronize_fn=self.candidate, restart_fn=restart, health_fn=health)
|
||||
self.assertEqual(events, ['restart', 'healthy'])
|
||||
self.assertNotEqual(self.digest(), before)
|
||||
backup = self.database.with_name('catalog.previous.sqlite')
|
||||
self.assertEqual(hashlib.sha256(backup.read_bytes()).hexdigest(), before)
|
||||
self.assertEqual(result['status'], 'success')
|
||||
self.assertEqual(json.loads((self.state / 'last-success.json').read_text())['status'], 'success')
|
||||
|
||||
@unittest.skipUnless(os.geteuid() == 0, 'requires root to set group ownership')
|
||||
def test_publish_preserves_group_readable_database_ownership_and_mode(self):
|
||||
group = grp.getgrnam('www-data').gr_gid
|
||||
os.chown(self.database, 0, group)
|
||||
os.chmod(self.database, 0o640)
|
||||
observed = []
|
||||
def restart():
|
||||
published = self.database.stat()
|
||||
observed.append((published.st_uid, published.st_gid, stat.S_IMODE(published.st_mode)))
|
||||
self.api.run_update(self.database, self.state, object(), synchronize_fn=self.candidate,
|
||||
restart_fn=restart, health_fn=lambda _: None)
|
||||
self.assertEqual(observed, [(0, group, 0o640)])
|
||||
published = self.database.stat()
|
||||
self.assertEqual((published.st_uid, published.st_gid), (0, group))
|
||||
self.assertEqual(stat.S_IMODE(published.st_mode), 0o640)
|
||||
|
||||
def test_fetch_failure_never_changes_live_database_or_restarts(self):
|
||||
before = self.digest()
|
||||
def fail(*_args, **_kwargs):
|
||||
raise RuntimeError('network unavailable')
|
||||
with self.assertRaises(self.api.UpdateError):
|
||||
self.api.run_update(self.database, self.state, object(), synchronize_fn=fail,
|
||||
restart_fn=lambda: self.fail('must not restart'), health_fn=lambda _: None)
|
||||
self.assertEqual(self.digest(), before)
|
||||
self.assertFalse((self.state / 'last-success.json').exists())
|
||||
self.assertEqual(json.loads((self.state / 'last-result.json').read_text())['status'], 'failed')
|
||||
self.assertEqual(list(self.root.glob('.catalog-sync-*.sqlite*')), [])
|
||||
|
||||
def test_failed_health_rolls_back_before_restarting_old_catalogue(self):
|
||||
before = self.digest()
|
||||
with closing(sqlite3.connect(self.database)) as db:
|
||||
old_stamp = json.loads(db.execute(
|
||||
"SELECT value_json FROM metadata WHERE key='built_at_utc'").fetchone()[0])
|
||||
restarts = []
|
||||
health_checks = []
|
||||
def restart():
|
||||
restarts.append(self.digest())
|
||||
def fail_health(stamp):
|
||||
health_checks.append(stamp)
|
||||
if stamp == self.stamp:
|
||||
raise RuntimeError('new API not healthy')
|
||||
self.assertEqual(self.digest(), before)
|
||||
with self.assertRaises(self.api.UpdateError):
|
||||
self.api.run_update(self.database, self.state, object(), synchronize_fn=self.candidate,
|
||||
restart_fn=restart, health_fn=fail_health)
|
||||
self.assertEqual(len(restarts), 2)
|
||||
self.assertNotEqual(restarts[0], before)
|
||||
self.assertEqual(restarts[1], before)
|
||||
self.assertEqual(health_checks, [self.stamp, old_stamp])
|
||||
self.assertEqual(self.digest(), before)
|
||||
self.assertFalse((self.state / 'last-success.json').exists())
|
||||
self.assertFalse(json.loads((self.state / 'last-result.json').read_text())['rollback_failed'])
|
||||
|
||||
def test_failed_rollback_health_requires_operator_attention(self):
|
||||
before = self.digest()
|
||||
with closing(sqlite3.connect(self.database)) as db:
|
||||
old_stamp = json.loads(db.execute(
|
||||
"SELECT value_json FROM metadata WHERE key='built_at_utc'").fetchone()[0])
|
||||
health_checks = []
|
||||
restarts = []
|
||||
def fail_health(stamp):
|
||||
health_checks.append(stamp)
|
||||
raise RuntimeError('API not healthy')
|
||||
with self.assertRaisesRegex(self.api.UpdateError, 'operator attention'):
|
||||
self.api.run_update(self.database, self.state, object(), synchronize_fn=self.candidate,
|
||||
restart_fn=lambda: restarts.append(self.digest()), health_fn=fail_health)
|
||||
self.assertEqual(health_checks, [self.stamp, old_stamp])
|
||||
self.assertEqual(len(restarts), 2)
|
||||
self.assertEqual(restarts[1], before)
|
||||
self.assertEqual(self.digest(), before)
|
||||
self.assertFalse((self.state / 'last-success.json').exists())
|
||||
self.assertTrue(json.loads((self.state / 'last-result.json').read_text())['rollback_failed'])
|
||||
|
||||
def test_concurrent_run_is_rejected_before_fetching(self):
|
||||
import fcntl
|
||||
self.state.mkdir()
|
||||
with (self.state / 'sync.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
with self.assertRaises(self.api.UpdateError):
|
||||
self.api.run_update(self.database, self.state, object(),
|
||||
synchronize_fn=lambda *_a, **_k: self.fail('overlapping fetch'),
|
||||
restart_fn=lambda: None, health_fn=lambda _: None)
|
||||
|
||||
def assert_candidate_rejected(self, mutate):
|
||||
before = self.digest()
|
||||
restarts = []
|
||||
def candidate(source, stage, upstream, **kwargs):
|
||||
report = self.candidate(source, stage, upstream, **kwargs)
|
||||
with closing(sqlite3.connect(stage)) as db:
|
||||
mutate(db)
|
||||
db.commit()
|
||||
return report
|
||||
with self.assertRaises(self.api.UpdateError):
|
||||
self.api.run_update(self.database, self.state, object(), synchronize_fn=candidate,
|
||||
restart_fn=lambda: restarts.append(True), health_fn=lambda _: None)
|
||||
self.assertEqual(restarts, [])
|
||||
self.assertEqual(self.digest(), before)
|
||||
self.assertFalse((self.state / 'last-success.json').exists())
|
||||
|
||||
def test_required_reference_and_source_tables_cannot_be_removed_or_emptied(self):
|
||||
for table in ('genres', 'countries', 'title_genres', 'title_countries',
|
||||
'series_fetch_status', 'movie_sources', 'episode_sources'):
|
||||
for statement in ('DELETE FROM ', 'DROP TABLE '):
|
||||
with self.subTest(table=table, statement=statement):
|
||||
self.assert_candidate_rejected(lambda db: db.execute(statement + table))
|
||||
|
||||
def test_new_movie_sources_cannot_mask_existing_title_source_loss(self):
|
||||
def replace(db):
|
||||
db.execute("INSERT INTO titles(id,type,raw_json) VALUES (66,'movie','{}')")
|
||||
db.execute('UPDATE movie_sources SET title_id=66 WHERE title_id=1')
|
||||
self.assert_candidate_rejected(replace)
|
||||
|
||||
def test_other_title_sources_cannot_mask_existing_title_source_loss(self):
|
||||
def replace(db):
|
||||
db.execute('UPDATE movie_sources SET title_id=3,source_index=1 WHERE title_id=1')
|
||||
self.assert_candidate_rejected(replace)
|
||||
|
||||
def test_new_episode_sources_cannot_mask_existing_episode_source_loss(self):
|
||||
def replace(db):
|
||||
db.execute("INSERT INTO episodes(series_id,season_index,episode_index,id,raw_json) "
|
||||
"VALUES (2,0,1,502,'{}')")
|
||||
db.execute('UPDATE episode_sources SET episode_index=1')
|
||||
self.assert_candidate_rejected(replace)
|
||||
|
||||
def test_null_id_episode_sources_are_preserved_by_index(self):
|
||||
with closing(sqlite3.connect(self.database)) as db:
|
||||
db.execute('UPDATE episodes SET id=NULL')
|
||||
db.commit()
|
||||
def replace(db):
|
||||
db.execute("INSERT INTO episodes(series_id,season_index,episode_index,raw_json) "
|
||||
"VALUES (2,0,1,'{}')")
|
||||
db.execute('UPDATE episode_sources SET episode_index=1')
|
||||
self.assert_candidate_rejected(replace)
|
||||
|
||||
def test_source_url_replacement_with_stable_ids_can_be_published(self):
|
||||
replacement = 'https://media.test/replacement.mkv?token=new'
|
||||
def candidate(source, stage, upstream, **kwargs):
|
||||
report = self.candidate(source, stage, upstream, **kwargs)
|
||||
with closing(sqlite3.connect(stage)) as db:
|
||||
for table in ('movie_sources', 'episode_sources'):
|
||||
raw = json.loads(db.execute(
|
||||
'SELECT raw_json FROM ' + table + ' WHERE source_id=?',
|
||||
(101 if table == 'movie_sources' else 201,)).fetchone()[0])
|
||||
raw['url'] = replacement
|
||||
db.execute('UPDATE ' + table + ' SET url=?,raw_json=? WHERE source_id=?',
|
||||
(replacement, json.dumps(raw), raw['id']))
|
||||
for table, title_id in (('titles', 1), ('episodes', 501)):
|
||||
raw = json.loads(db.execute('SELECT raw_json FROM ' + table + ' WHERE id=?',
|
||||
(title_id,)).fetchone()[0])
|
||||
raw['sources'][0]['url'] = replacement
|
||||
db.execute('UPDATE ' + table + ' SET raw_json=? WHERE id=?',
|
||||
(json.dumps(raw), title_id))
|
||||
season = json.loads(db.execute('SELECT raw_json FROM seasons').fetchone()[0])
|
||||
season['episodes'][0]['sources'][0]['url'] = replacement
|
||||
db.execute('UPDATE seasons SET raw_json=?', (json.dumps(season),))
|
||||
db.commit()
|
||||
return report
|
||||
result = self.api.run_update(self.database, self.state, object(), synchronize_fn=candidate,
|
||||
restart_fn=lambda: None, health_fn=lambda _: None)
|
||||
self.assertEqual(result['status'], 'success')
|
||||
with closing(sqlite3.connect(self.database)) as db:
|
||||
self.assertEqual(db.execute('SELECT source_id,url FROM movie_sources WHERE title_id=1').fetchone(),
|
||||
(101, replacement))
|
||||
self.assertEqual(db.execute('SELECT source_id,url FROM episode_sources').fetchone(),
|
||||
(201, replacement))
|
||||
|
||||
def test_episode_sources_follow_stable_id_when_episode_index_changes(self):
|
||||
def candidate(source, stage, upstream, **kwargs):
|
||||
report = self.candidate(source, stage, upstream, **kwargs)
|
||||
with closing(sqlite3.connect(stage)) as db:
|
||||
db.execute('UPDATE episodes SET episode_index=1 WHERE id=501')
|
||||
db.execute('UPDATE episode_sources SET episode_index=1')
|
||||
db.commit()
|
||||
return report
|
||||
result = self.api.run_update(self.database, self.state, object(), synchronize_fn=candidate,
|
||||
restart_fn=lambda: None, health_fn=lambda _: None)
|
||||
self.assertEqual(result['status'], 'success')
|
||||
|
||||
def test_incomplete_candidate_cannot_replace_the_catalogue(self):
|
||||
before = self.digest()
|
||||
def truncate(source, stage, upstream, **kwargs):
|
||||
result = self.candidate(source, stage, upstream, **kwargs)
|
||||
with closing(sqlite3.connect(stage)) as db:
|
||||
db.execute('DELETE FROM titles WHERE id=65')
|
||||
db.commit()
|
||||
return result
|
||||
with self.assertRaises(self.api.UpdateError):
|
||||
self.api.run_update(self.database, self.state, object(), synchronize_fn=truncate,
|
||||
restart_fn=lambda: self.fail('invalid candidate'), health_fn=lambda _: None)
|
||||
self.assertEqual(self.digest(), before)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Operational boundaries of the installed updater schedule."""
|
||||
import configparser
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
SERVER = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class SyncUnitTests(unittest.TestCase):
|
||||
def unit(self, name):
|
||||
config = configparser.ConfigParser(interpolation=None, strict=False)
|
||||
config.optionxform = str
|
||||
config.read(SERVER / name)
|
||||
return config
|
||||
|
||||
def test_timer_uses_three_day_interval_and_boot_catch_up(self):
|
||||
timer = self.unit('reelbox-sync.timer')
|
||||
self.assertEqual(timer['Timer']['OnUnitActiveSec'], '3d')
|
||||
self.assertEqual(timer['Timer']['OnBootSec'], '15min')
|
||||
self.assertEqual(timer['Timer']['Unit'], 'reelbox-sync.service')
|
||||
self.assertNotIn('OnCalendar', timer['Timer'])
|
||||
self.assertEqual(timer['Install']['WantedBy'], 'timers.target')
|
||||
|
||||
def test_updater_is_bounded_and_separate_from_public_api(self):
|
||||
service = self.unit('reelbox-sync.service')['Service']
|
||||
self.assertEqual(service['Type'], 'oneshot')
|
||||
self.assertEqual(service['TimeoutStartSec'], '8h')
|
||||
self.assertEqual(service['RestartSec'], '1h')
|
||||
self.assertEqual(service['ProtectSystem'], 'strict')
|
||||
self.assertEqual(service['ProtectHome'], 'true')
|
||||
self.assertEqual(service['EnvironmentFile'], '/etc/reelbox/sync.env')
|
||||
self.assertEqual(service['UMask'], '0077')
|
||||
self.assertIn('catalog_sync_runner.py', service['ExecStart'])
|
||||
self.assertEqual(service['ReadWritePaths'], '/var/lib/reelbox-api /var/lib/reelbox-sync')
|
||||
api = self.unit('reelbox-api.service')['Service']
|
||||
self.assertEqual(api['IPAddressDeny'], 'any')
|
||||
self.assertEqual(api['IPAddressAllow'], 'localhost')
|
||||
self.assertNotIn('sync.env', api.get('EnvironmentFile', ''))
|
||||
Reference in new issue
Block a user