diff --git a/paper_atlas/collectors/rss2026.py b/paper_atlas/collectors/rss2026.py index 93cfe0b..8a40beb 100644 --- a/paper_atlas/collectors/rss2026.py +++ b/paper_atlas/collectors/rss2026.py @@ -6,7 +6,6 @@ import hashlib import json from pathlib import Path import re -import tempfile import unicodedata from paper_atlas.io.html import Document, Node, clean @@ -147,11 +146,15 @@ def collect(out, *, snapshots=None, cache=None): evidence = {}; receipts = {} if snapshots is not None: root = Path(snapshots) - for r in json.loads((root/'receipts.json').read_bytes()): - name = r['file'] - if not isinstance(name, str) or Path(name).name != name or name in receipts: - raise ValueError('Unsafe or repeated snapshot receipt') - receipts[name] = r + try: + for r in json.loads((root/'receipts.json').read_bytes()): + name = r['file'] + if not isinstance(name, str) or Path(name).name != name or name in receipts: + raise ValueError('Unsafe or repeated snapshot receipt') + receipts[name] = r + except Exception as error: + (out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n') + raise def page(name, url): if snapshots is not None: path = root/name diff --git a/tests/test_rss2026.py b/tests/test_rss2026.py index 52e04df..661ce63 100644 --- a/tests/test_rss2026.py +++ b/tests/test_rss2026.py @@ -96,3 +96,5 @@ class Rss2026Tests(unittest.TestCase): receipts=[*self.fixture['receipts'],{**self.fixture['receipts'][0],'file':'../outside'}] (self.snap/'receipts.json').write_text(json.dumps(receipts)) with self.assertRaisesRegex(ValueError,'Unsafe'):rss.collect(self.root/'unsafe',snapshots=self.snap) + self.assertTrue((self.root/'unsafe/failed.json').exists()) + self.assertFalse((self.root/'unsafe/collection-report.json').exists())