152 lines
9.0 KiB
Python
152 lines
9.0 KiB
Python
"""Publisher evidence, section boundaries and native-ingestion regression checks."""
|
||
import hashlib
|
||
import json
|
||
import re
|
||
from pathlib import Path
|
||
import sqlite3
|
||
import tempfile
|
||
import unittest
|
||
from unittest.mock import patch
|
||
|
||
from paper_atlas.collectors import aamas
|
||
from paper_atlas.ingestion.records import ingest_file
|
||
|
||
|
||
def paper(pid, abstract=True):
|
||
summary = '<span class="abstract">Research with <i>evidence</i>.<br>More detail.</span>' if abstract else ''
|
||
return f'''<div class="paper" id="{pid}"><span class="id">{pid}</span>
|
||
<p class="paper"><span class="title"><a href="../papers/{pid}.pdf">{summary}
|
||
A structured & auditable paper {pid}</a></span><span class="authors">
|
||
<a href="authors.html#Jane">Jane Doe<span class="amount">has 7 papers</span></a>, Bob Smith
|
||
</span></p></div>'''
|
||
|
||
|
||
def page(section, pid, abstract=True):
|
||
heading = aamas.SECTIONS[section][0]
|
||
return (f'<html><head><title>{heading} - AAMAS 2012</title></head><body>'
|
||
f'<h1>{heading}</h1><h2 class="section" id="s-1A">Session 1A – Robotics I</h2>'
|
||
+paper(pid, abstract)+'</body></html>')
|
||
|
||
|
||
class AamasProceedingsTests(unittest.TestCase):
|
||
def snapshots(self, root):
|
||
root.mkdir()
|
||
receipts = []
|
||
items = {'introduction': ('<title>AAMAS 2012</title><p>Aggregate withdrawals do not identify papers.</p>', aamas.ROOT)}
|
||
for index, section in enumerate(aamas.SECTIONS):
|
||
items[section] = (page(section, f'P{index}_1', section != 'extabstracts'), aamas.ROOT+'resources/'+section+'.html')
|
||
for key, (html, url) in items.items():
|
||
data = html.encode()
|
||
(root/(key+'.html')).write_bytes(data)
|
||
receipts.append({'url': url, 'status': 200, 'file': key+'.html', 'bytes': len(data),
|
||
'sha256': hashlib.sha256(data).hexdigest(), 'finished_at': '2026-11-09T00:00:00+00:00'})
|
||
(root/'receipts.json').write_text(json.dumps({'receipts': receipts}))
|
||
return root
|
||
|
||
def test_hidden_abstract_and_author_badges_are_not_title_or_authors(self):
|
||
rows = aamas.parse_section(page('fullpapers', '1A_1'), 'fullpapers', '2026-11-09T00:00:00Z', {'sha256': 'proof'})
|
||
row, = rows
|
||
self.assertEqual(row['title'], 'A structured & auditable paper 1A_1')
|
||
self.assertEqual(row['abstract'], 'Research with evidence. More detail.')
|
||
self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith'])
|
||
self.assertEqual(row['metadata']['authors_display'], 'Jane Doe, Bob Smith')
|
||
self.assertEqual(row['url'], aamas.ROOT+'resources/fullpapers.html#1A_1')
|
||
self.assertEqual(row['metadata']['paper_url'], aamas.ROOT+'papers/1A_1.pdf')
|
||
self.assertEqual(row['metadata']['source_evidence'], {'sha256': 'proof'})
|
||
|
||
def test_publication_formats_and_session_topics_do_not_invent_submission_tracks(self):
|
||
for section, (_, fmt, track, _) in aamas.SECTIONS.items():
|
||
row, = aamas.parse_section(page(section, 'P_1', False), section, '2026-11-09T00:00:00Z', {})
|
||
self.assertEqual((row['track'], row['metadata']['publication_format']), (track, fmt))
|
||
self.assertIsNone(row['abstract'])
|
||
self.assertEqual(row['status'], 'accepted')
|
||
self.assertEqual(row['metadata']['decision_date_precision'], 'not_supplied')
|
||
self.assertEqual(row['metadata']['session']['title'], 'Session 1A – Robotics I')
|
||
self.assertIn({'namespace': 'aamas_proceedings:publication_format', 'value': fmt}, row['tags'])
|
||
self.assertEqual(aamas.SECTIONS['fullpapers'][2], 'proceedings_unspecified')
|
||
self.assertEqual(aamas.SECTIONS['extabstracts'][2], 'proceedings_unspecified')
|
||
|
||
def test_changed_edition_section_duplicate_id_or_pdf_link_is_rejected(self):
|
||
html = page('fullpapers', '1A_1')
|
||
for changed in [html.replace('AAMAS 2012', 'AAMAS 2025'),
|
||
html.replace('<h1>Full Papers</h1>', '<h1>Rejected Papers</h1>'),
|
||
html.replace('</body>', paper('1A_1')+'</body>'),
|
||
html.replace('../papers/1A_1.pdf', 'https://example.org/1A_1.pdf'),
|
||
html.replace('../papers/1A_1.pdf', '../papers/1A_2.pdf')]:
|
||
with self.subTest(changed=changed):
|
||
with self.assertRaises(ValueError):
|
||
aamas.parse_section(changed, 'fullpapers', '2026-11-09T00:00:00Z', {})
|
||
|
||
def test_frozen_responses_require_checksums_success_and_safe_paths(self):
|
||
for failure in ['checksum', 'status', 'path']:
|
||
with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary:
|
||
root = self.snapshots(Path(temporary)/'source')
|
||
path = root/'receipts.json'
|
||
receipts = json.loads(path.read_text())
|
||
item = receipts['receipts'][0]
|
||
if failure == 'checksum': item['sha256'] = '0'*64
|
||
elif failure == 'status': item['status'] = 404
|
||
else: item['file'] = '../outside.html'
|
||
path.write_text(json.dumps(receipts))
|
||
with self.assertRaises(ValueError):
|
||
aamas.read_pages(snapshots=root)
|
||
|
||
def test_reviewed_section_counts_and_year_are_required(self):
|
||
with tempfile.TemporaryDirectory() as temporary:
|
||
root = Path(temporary)
|
||
source = self.snapshots(root/'source')
|
||
with self.assertRaises(ValueError):
|
||
aamas.collect(2012, root/'out', snapshots=source)
|
||
self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
|
||
with self.assertRaises(ValueError):
|
||
aamas.collect(2027, root/'future', snapshots=source)
|
||
self.assertFalse((root/'future').exists())
|
||
|
||
def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self):
|
||
with tempfile.TemporaryDirectory() as temporary:
|
||
root = Path(temporary)
|
||
source = self.snapshots(root/'source')
|
||
receipts_path = source/'receipts.json'
|
||
receipts = json.loads(receipts_path.read_text())
|
||
for item in receipts['receipts'][1:]:
|
||
path = source/item['file']
|
||
data = re.sub(r'P[0-9]_1', 'SAME_1', path.read_text()).encode()
|
||
path.write_bytes(data)
|
||
item.update(bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
|
||
receipts_path.write_text(json.dumps(receipts))
|
||
small = {key: (*value[:3], 1) for key, value in aamas.SECTIONS.items()}
|
||
with patch.dict(aamas.SECTIONS, small, clear=True), self.assertRaises(ValueError):
|
||
aamas.collect(2012, root/'out', snapshots=source)
|
||
self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
|
||
|
||
def test_collection_and_native_ingestion_preserve_formats_evidence_and_missing_abstracts(self):
|
||
with tempfile.TemporaryDirectory() as temporary:
|
||
root = Path(temporary)
|
||
source = self.snapshots(root/'source')
|
||
small = {key: (*value[:3], 1) for key, value in aamas.SECTIONS.items()}
|
||
with patch.dict(aamas.SECTIONS, small, clear=True):
|
||
result = aamas.collect(2012, root/'out', snapshots=source)
|
||
self.assertEqual(result['records'], 4)
|
||
self.assertEqual(len(result['files']), 3)
|
||
self.assertFalse(result['database_written'])
|
||
self.assertFalse(result['pdfs_downloaded'])
|
||
db = root/'sample.sqlite'
|
||
for manifest in result['files']:
|
||
path = Path(manifest['file'])
|
||
self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), manifest['file_sha256'])
|
||
for raw in path.read_text().splitlines():
|
||
record = json.loads(raw)
|
||
evidence = record['metadata']['source_evidence']
|
||
self.assertEqual(hashlib.sha256((root/'out'/evidence['file']).read_bytes()).hexdigest(), evidence['sha256'])
|
||
ingest_file(db, path)
|
||
with sqlite3.connect(db) as connection:
|
||
self.assertEqual(connection.execute('SELECT count(*) FROM papers').fetchone()[0], 4)
|
||
self.assertEqual(connection.execute('SELECT count(*) FROM submissions').fetchone()[0], 4)
|
||
self.assertEqual(connection.execute("SELECT count(*) FROM decisions WHERE status='accepted'").fetchone()[0], 4)
|
||
self.assertEqual(connection.execute('SELECT count(*) FROM papers WHERE abstract IS NULL').fetchone()[0], 1)
|
||
metadata = [json.loads(r[0]) for r in connection.execute('SELECT metadata_json FROM source_records')]
|
||
self.assertEqual({r['publication_format'] for r in metadata}, {'full_paper','extended_abstract','demo','doctoral_abstract'})
|
||
namespaces = {r[0] for r in connection.execute('SELECT DISTINCT namespace FROM tags')}
|
||
self.assertEqual(namespaces, {'aamas_proceedings:publication_format', 'aamas_proceedings:session'})
|
||
self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(), [])
|