148 lines
8.3 KiB
Python
148 lines
8.3 KiB
Python
"""Upstream notices are not catalogue films, including after later scrapes."""
|
|
from contextlib import closing
|
|
import json
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tempfile
|
|
from typing import Any
|
|
import unittest
|
|
from urllib.parse import urlencode
|
|
|
|
import test_catalog_api as fixtures
|
|
from test_catalog_api import load_service
|
|
from test_catalog_sync import FakeUpstream, load_sync
|
|
|
|
|
|
class AnnouncementPolicyTests(unittest.TestCase):
|
|
def test_generic_notice_labels_are_blocked_without_hiding_real_notice_films(self):
|
|
api = load_service()
|
|
for text in [
|
|
'اطلاعیه جدید', 'اطلاعیه مهم ', 'اطلاعیه خیلی مهم',
|
|
'📢 **اطلاعـیّهٔ جدید**', 'اطلاع يه جديد', 'اطلاع‌رسانی جدید',
|
|
'اعلان جدید', 'اعلان مهم شماره ۲',
|
|
'به روز رسانی جدید', 'به‌روزرسانی برنامه', 'بروز رسانی نسخه ویندوز',
|
|
'آپدیت جدید برنامه', 'New announcement', 'IMPORTANT NOTICE!',
|
|
'Urgent announcement: service changes', 'App update',
|
|
]:
|
|
with self.subTest(blocked=text):
|
|
self.assertFalse(api.allowed_text(text))
|
|
for text in [
|
|
'Red Notice', 'SAS: Red Notice', 'Burn Notice', 'Notice to Quit',
|
|
'Two Weeks Notice', 'The Announcement', 'اعلان قرمز', 'Upgrade',
|
|
'News of the World', 'نام پارسی فیلم: اعلان قرمز',
|
|
'A detective reads an important notice before leaving town.',
|
|
'در این فیلم یک اطلاعیه جدید، زندگی شخصیت اصلی را تغییر می‌دهد.',
|
|
]:
|
|
with self.subTest(allowed=text):
|
|
self.assertTrue(api.allowed_text(text))
|
|
|
|
|
|
class AnnouncementSurfaceTests(unittest.TestCase):
|
|
path: Path
|
|
temp: tempfile.TemporaryDirectory
|
|
api: Any
|
|
setUp = fixtures.CatalogTests.setUp
|
|
tearDown = fixtures.CatalogTests.tearDown
|
|
open = fixtures.CatalogTests.open
|
|
|
|
def test_notices_are_absent_from_all_catalogue_routes_and_direct_lookup(self):
|
|
bad = {70, 71, 72, 73, 74}
|
|
with closing(sqlite3.connect(self.path)) as db:
|
|
original = json.loads(db.execute('SELECT raw_json FROM titles WHERE id=1').fetchone()[0])
|
|
records = [
|
|
(70, 'movie', 'اطلاعیه جدید', ''),
|
|
(71, 'movie', 'اطلاعیه خیلی مهم', ''),
|
|
(72, 'movie', 'به روز رسانی جدید', ''),
|
|
(73, 'movie', 'Service changes', 'Important announcement: app changes'),
|
|
(74, 'serie', 'اعلان جدید', ''),
|
|
(75, 'movie', 'Red Notice', 'نام پارسی فیلم: اعلان قرمز'),
|
|
(76, 'serie', 'Burn Notice', 'An ordinary spy series'),
|
|
]
|
|
for ident, kind, title, description in records:
|
|
raw = {**original, 'id': ident, 'type': kind, 'title': title, 'description': description}
|
|
db.execute('INSERT INTO titles(id,type,title,description,year,imdb,rating,raw_json) VALUES (?,?,?,?,?,?,?,?)',
|
|
(ident, kind, title, description, 2026, 9.9, 5, json.dumps(raw)))
|
|
db.execute('INSERT INTO title_genres VALUES (?,2)', (ident,))
|
|
db.execute('INSERT INTO title_countries VALUES (?,1)', (ident,))
|
|
db.commit()
|
|
catalog = self.open()
|
|
found = set()
|
|
for kind in ('movie', 'serie', 'poster'):
|
|
for order in ('created', 'updated', 'imdb', 'rating', 'year', 'title', 'random'):
|
|
for page in range(4):
|
|
for path in (f'/api/{kind}/by/filtres/0/{order}/{page}/public/',
|
|
f'/api/{kind}/by/filtres/2/1/{order}/{page}/public/'):
|
|
ids = {row['id'] for row in catalog.dispatch(path)}
|
|
self.assertFalse(bad & ids, path)
|
|
found.update(ids)
|
|
self.assertTrue({75, 76} <= found)
|
|
for group in catalog.dispatch('/api/first/public/')['genres']:
|
|
self.assertFalse(bad & {row['id'] for row in group['posters']})
|
|
for action in ('best-rand-top', 'best-rand-year'):
|
|
self.assertFalse(bad & {row['id'] for row in catalog.dispatch(
|
|
'/api-user/newapi/like.php?' + urlencode({'action': action}))})
|
|
for action in ('search-movie', 'search-movie-en'):
|
|
for query in ('اطلاعیه', 'به روز', 'announcement', 'Notice'):
|
|
rows = catalog.dispatch('/api-user/newapi/like.php?' + urlencode({'action': action, 'q': query}))
|
|
self.assertFalse(bad & {row['id'] for row in rows})
|
|
self.assertFalse(bad & {row['id'] for row in catalog.dispatch(
|
|
'/api-user/newapi/filter.php?action=pishnahad&garid=2&pageno=1')})
|
|
for ident in bad:
|
|
for path in (f'/api/movie/by/{ident}/public/', f'/api/season/by/serie/{ident}/public/'):
|
|
with self.assertRaises(self.api.ApiError) as error:
|
|
catalog.dispatch(path)
|
|
self.assertEqual(error.exception.status, 404)
|
|
self.assertEqual(catalog.dispatch('/api/movie/by/75/public/')['title'], 'Red Notice')
|
|
self.assertEqual(catalog.dispatch('/api/movie/by/76/public/')['title'], 'Burn Notice')
|
|
|
|
def test_repeated_scrapes_keep_old_and_new_notice_ids_out_of_the_api(self):
|
|
# Old raw rows stay for snapshot/source preservation, but are never served.
|
|
with closing(sqlite3.connect(self.path)) as db:
|
|
item = {'id': 90, 'type': 'movie', 'title': 'اطلاعیه مهم', 'description': ''}
|
|
db.execute('INSERT INTO titles(id,type,title,description,raw_json) VALUES (?,?,?,?,?)',
|
|
(90, 'movie', item['title'], '', json.dumps(item)))
|
|
db.commit()
|
|
source_bytes = self.path.read_bytes()
|
|
source = self.path
|
|
sync = load_sync()
|
|
for run, offset in enumerate((0, 100), 1):
|
|
new_notice, series_notice, detail_notice = 91 + offset, 92 + offset, 93 + offset
|
|
rows = [
|
|
{'id': 90, 'type': 'movie', 'title': 'اطلاعیه جدید'},
|
|
{'id': new_notice, 'type': 'movie', 'title': '📢 اطلاعیه جدید'},
|
|
{'id': detail_notice, 'type': 'movie', 'title': 'Looks like a film'},
|
|
{'id': 94, 'type': 'movie', 'title': 'Red Notice'},
|
|
]
|
|
upstream = FakeUpstream({
|
|
'movie/by/filtres/0/created/0': rows,
|
|
f'movie/by/{detail_notice}': {'id': detail_notice, 'type': 'movie', 'title': 'به روز رسانی جدید'},
|
|
'movie/by/94': {'id': 94, 'type': 'movie', 'title': 'Red Notice', 'sources': []},
|
|
'serie/by/filtres/0/created/0': [
|
|
{'id': series_notice, 'type': 'serie', 'title': 'Important announcement'}],
|
|
})
|
|
stage = Path(self.temp.name) / f'scrape-{run}.sqlite'
|
|
sync.synchronize(source, stage, upstream)
|
|
self.assertNotIn(f'movie/by/{new_notice}', upstream.calls)
|
|
self.assertNotIn('movie/by/90', upstream.calls)
|
|
self.assertNotIn(f'season/by/serie/{series_notice}', upstream.calls)
|
|
with closing(sqlite3.connect(stage)) as db:
|
|
self.assertEqual(db.execute('SELECT COUNT(*) FROM titles WHERE id IN (?,?,?)',
|
|
(new_notice, series_notice, detail_notice)).fetchone()[0], 0)
|
|
self.assertEqual(db.execute('SELECT title FROM titles WHERE id=90').fetchone()[0], 'اطلاعیه مهم')
|
|
api = load_service()
|
|
catalog = api.Catalog(stage)
|
|
try:
|
|
for ident in (90, new_notice, series_notice, detail_notice):
|
|
with self.assertRaises(api.ApiError):
|
|
catalog.dispatch(f'/api/movie/by/{ident}/public/')
|
|
self.assertEqual(catalog.dispatch('/api/movie/by/94/public/')['title'], 'Red Notice')
|
|
self.assertEqual(catalog.dispatch('/api/season/by/serie/2/public/')[0]['episodes'][0]['id'], 501)
|
|
finally:
|
|
catalog.close()
|
|
source = stage
|
|
self.assertEqual(self.path.read_bytes(), source_bytes)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|