"""Upstream notices are not catalogue films, including after later scrapes.""" from contextlib import closing import json from pathlib import Path import sqlite3 import tempfile from typing import Any import unittest from urllib.parse import urlencode import test_catalog_api as fixtures from test_catalog_api import load_service from test_catalog_sync import FakeUpstream, load_sync class AnnouncementPolicyTests(unittest.TestCase): def test_generic_notice_labels_are_blocked_without_hiding_real_notice_films(self): api = load_service() for text in [ 'اطلاعیه جدید', 'اطلاعیه مهم ', 'اطلاعیه خیلی مهم', '📢 **اطلاعـیّهٔ جدید**', 'اطلاع يه جديد', 'اطلاع‌رسانی جدید', 'اعلان جدید', 'اعلان مهم شماره ۲', 'به روز رسانی جدید', 'به‌روزرسانی برنامه', 'بروز رسانی نسخه ویندوز', 'آپدیت جدید برنامه', 'New announcement', 'IMPORTANT NOTICE!', 'Urgent announcement: service changes', 'App update', ]: with self.subTest(blocked=text): self.assertFalse(api.allowed_text(text)) for text in [ 'Red Notice', 'SAS: Red Notice', 'Burn Notice', 'Notice to Quit', 'Two Weeks Notice', 'The Announcement', 'اعلان قرمز', 'Upgrade', 'News of the World', 'نام پارسی فیلم: اعلان قرمز', 'A detective reads an important notice before leaving town.', 'در این فیلم یک اطلاعیه جدید، زندگی شخصیت اصلی را تغییر می‌دهد.', ]: with self.subTest(allowed=text): self.assertTrue(api.allowed_text(text)) class AnnouncementSurfaceTests(unittest.TestCase): path: Path temp: tempfile.TemporaryDirectory api: Any setUp = fixtures.CatalogTests.setUp tearDown = fixtures.CatalogTests.tearDown open = fixtures.CatalogTests.open def test_notices_are_absent_from_all_catalogue_routes_and_direct_lookup(self): bad = {70, 71, 72, 73, 74} with closing(sqlite3.connect(self.path)) as db: original = json.loads(db.execute('SELECT raw_json FROM titles WHERE id=1').fetchone()[0]) records = [ (70, 'movie', 'اطلاعیه جدید', ''), (71, 'movie', 'اطلاعیه خیلی مهم', ''), (72, 'movie', 'به روز رسانی جدید', ''), (73, 'movie', 'Service changes', 'Important announcement: app changes'), (74, 'serie', 'اعلان جدید', ''), (75, 'movie', 'Red Notice', 'نام پارسی فیلم: اعلان قرمز'), (76, 'serie', 'Burn Notice', 'An ordinary spy series'), ] for ident, kind, title, description in records: raw = {**original, 'id': ident, 'type': kind, 'title': title, 'description': description} db.execute('INSERT INTO titles(id,type,title,description,year,imdb,rating,raw_json) VALUES (?,?,?,?,?,?,?,?)', (ident, kind, title, description, 2026, 9.9, 5, json.dumps(raw))) db.execute('INSERT INTO title_genres VALUES (?,2)', (ident,)) db.execute('INSERT INTO title_countries VALUES (?,1)', (ident,)) db.commit() catalog = self.open() found = set() for kind in ('movie', 'serie', 'poster'): for order in ('created', 'updated', 'imdb', 'rating', 'year', 'title', 'random'): for page in range(4): for path in (f'/api/{kind}/by/filtres/0/{order}/{page}/public/', f'/api/{kind}/by/filtres/2/1/{order}/{page}/public/'): ids = {row['id'] for row in catalog.dispatch(path)} self.assertFalse(bad & ids, path) found.update(ids) self.assertTrue({75, 76} <= found) for group in catalog.dispatch('/api/first/public/')['genres']: self.assertFalse(bad & {row['id'] for row in group['posters']}) for action in ('best-rand-top', 'best-rand-year'): self.assertFalse(bad & {row['id'] for row in catalog.dispatch( '/api-user/newapi/like.php?' + urlencode({'action': action}))}) for action in ('search-movie', 'search-movie-en'): for query in ('اطلاعیه', 'به روز', 'announcement', 'Notice'): rows = catalog.dispatch('/api-user/newapi/like.php?' + urlencode({'action': action, 'q': query})) self.assertFalse(bad & {row['id'] for row in rows}) self.assertFalse(bad & {row['id'] for row in catalog.dispatch( '/api-user/newapi/filter.php?action=pishnahad&garid=2&pageno=1')}) for ident in bad: for path in (f'/api/movie/by/{ident}/public/', f'/api/season/by/serie/{ident}/public/'): with self.assertRaises(self.api.ApiError) as error: catalog.dispatch(path) self.assertEqual(error.exception.status, 404) self.assertEqual(catalog.dispatch('/api/movie/by/75/public/')['title'], 'Red Notice') self.assertEqual(catalog.dispatch('/api/movie/by/76/public/')['title'], 'Burn Notice') def test_repeated_scrapes_keep_old_and_new_notice_ids_out_of_the_api(self): # Old raw rows stay for snapshot/source preservation, but are never served. with closing(sqlite3.connect(self.path)) as db: item = {'id': 90, 'type': 'movie', 'title': 'اطلاعیه مهم', 'description': ''} db.execute('INSERT INTO titles(id,type,title,description,raw_json) VALUES (?,?,?,?,?)', (90, 'movie', item['title'], '', json.dumps(item))) db.commit() source_bytes = self.path.read_bytes() source = self.path sync = load_sync() for run, offset in enumerate((0, 100), 1): new_notice, series_notice, detail_notice = 91 + offset, 92 + offset, 93 + offset rows = [ {'id': 90, 'type': 'movie', 'title': 'اطلاعیه جدید'}, {'id': new_notice, 'type': 'movie', 'title': '📢 اطلاعیه جدید'}, {'id': detail_notice, 'type': 'movie', 'title': 'Looks like a film'}, {'id': 94, 'type': 'movie', 'title': 'Red Notice'}, ] upstream = FakeUpstream({ 'movie/by/filtres/0/created/0': rows, f'movie/by/{detail_notice}': {'id': detail_notice, 'type': 'movie', 'title': 'به روز رسانی جدید'}, 'movie/by/94': {'id': 94, 'type': 'movie', 'title': 'Red Notice', 'sources': []}, 'serie/by/filtres/0/created/0': [ {'id': series_notice, 'type': 'serie', 'title': 'Important announcement'}], }) stage = Path(self.temp.name) / f'scrape-{run}.sqlite' sync.synchronize(source, stage, upstream) self.assertNotIn(f'movie/by/{new_notice}', upstream.calls) self.assertNotIn('movie/by/90', upstream.calls) self.assertNotIn(f'season/by/serie/{series_notice}', upstream.calls) with closing(sqlite3.connect(stage)) as db: self.assertEqual(db.execute('SELECT COUNT(*) FROM titles WHERE id IN (?,?,?)', (new_notice, series_notice, detail_notice)).fetchone()[0], 0) self.assertEqual(db.execute('SELECT title FROM titles WHERE id=90').fetchone()[0], 'اطلاعیه مهم') api = load_service() catalog = api.Catalog(stage) try: for ident in (90, new_notice, series_notice, detail_notice): with self.assertRaises(api.ApiError): catalog.dispatch(f'/api/movie/by/{ident}/public/') self.assertEqual(catalog.dispatch('/api/movie/by/94/public/')['title'], 'Red Notice') self.assertEqual(catalog.dispatch('/api/season/by/serie/2/public/')[0]['episodes'][0]['id'], 501) finally: catalog.close() source = stage self.assertEqual(self.path.read_bytes(), source_bytes) if __name__ == '__main__': unittest.main()