156 lines
14 KiB
Python
156 lines
14 KiB
Python
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
|
|
from paper_atlas.collectors.www_archive import (ROOT, SPRING, FALL, Sources, collect, decode,
|
|
early_review, spring_rows)
|
|
|
|
|
|
class ArchiveTests(unittest.TestCase):
|
|
def setUp(self):
|
|
t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);self.cache=self.root/'cache';self.cache.mkdir()
|
|
self.pages={ROOT+'www1/PrelimProcs.html':'''<h1>Formal presentations</h1><ol><li><em>Keywords:</em> BUSINESS TRACK?<br><em>Title:</em><b> First contribution</b><br><em>Author:</em>Jane Doe <jane@example.org><br><li><em>Title:</em><b>Unlinked contribution</b><br><em>Author:</em>John Roe<br></ol>''',
|
|
FALL:'<h1>Conference Tracks</h1><ol><li><a href="Agents.html">Agents</a><li><a href="Tutor.html">Tutorials</a></ol>',
|
|
ROOT+'www2/Proceedings/Agents.html':'<h3>Panel</h3><dl><dt><a href="a.html">Linked talk</a><dd>Jane Doe<dt>Unlinked talk</a><dd>John Roe</dl>',
|
|
ROOT+'www2/Proceedings/Tutor.html':'<h1>Tutorials</h1><dl><dt>Training session<dd>Teacher</dl>'}
|
|
for track in ['papers','posters']:
|
|
self.pages[SPRING+track+'/author.html']='<b>Doe, Jane</b><ul><li><a href="01/paper.html">Same contribution</a></li></ul><b>Roe, John</b><a href="01/paper.html">Same contribution</a>'
|
|
self.pages[SPRING+track+'/abstract.html']='''<h1>Abstracts</h1><a href="#one">Same contribution</a><h3><a href="01/paper.html">Same contribution</a></h3><i>Author: Jane Doe, John Roe<br>Institute</i><p>Actual abstract with <i>inline emphasis</i> and a separator\u2028 preserved.</p><hr><h2>Another session</h2>'''
|
|
self.pages[SPRING+'papers/session.html']='<a href="01/paper.html">Same contribution</a>'
|
|
|
|
def sources(self):
|
|
rows=[]
|
|
for i,(url,html) in enumerate(sorted(self.pages.items())):
|
|
raw=html.encode()if isinstance(html,str)else html;name='source-'+str(i)+'.html';(self.cache/name).write_bytes(raw)
|
|
rows.append(dict(url=url,status=200,file=name,bytes=len(raw),sha256=hashlib.sha256(raw).hexdigest()))
|
|
p=self.cache/'manifest.json';p.write_text(json.dumps({'sources':rows}));sha=hashlib.sha256(p.read_bytes()).hexdigest()
|
|
return Sources(self.cache,sha),sha
|
|
|
|
def test_offline_batch_separates_tracks_review_and_importable_rows(self):
|
|
_,sha=self.sources();out=self.root/'batch';report=collect(self.cache,sha,out)
|
|
self.assertEqual(report['ready'],2);self.assertEqual(report['held'],5)
|
|
queue=json.loads((out/'import-queue.json').read_text())['files'];self.assertEqual(len(queue),2)
|
|
for name in queue:
|
|
p=Path(name);rows=[json.loads(l)for l in p.read_text().split('\n') if l];self.assertEqual(len(rows),1)
|
|
row=rows[0];self.assertEqual(row['event_key'],'www-1995-spring');self.assertEqual(row['authors'],['Jane Doe','John Roe'])
|
|
self.assertIn(row['track'],['main','poster']);self.assertIn('inline emphasis',row['abstract']);self.assertNotIn('Another session',row['abstract'])
|
|
self.assertIn('\u2028',row['abstract']);manifest=json.loads(p.with_suffix('.manifest.json').read_text());self.assertEqual(manifest['file_sha256'],hashlib.sha256(p.read_bytes()).hexdigest());self.assertEqual(manifest['status'],'partial')
|
|
held=[json.loads(l)for l in(out/'review-candidates.jsonl').read_text().split('\n') if l]
|
|
self.assertTrue(all('status'not in r and 'source_evidence'in r for r in held))
|
|
self.assertFalse(any('review-candidates'in p for p in queue))
|
|
|
|
def test_partial_manifest_preserves_full_listing_size_separately_from_ready_count(self):
|
|
self.pages[SPRING+'posters/abstract.html']+='<h3><a href="02/other.html">Unlinked poster</a></h3><i>Author: Other Person</i><p>Actual abstract</p>'
|
|
_,sha=self.sources();out=self.root/'batch';collect(self.cache,sha,out)
|
|
manifest=json.loads((out/'www-1995-spring-poster.manifest.json').read_text())
|
|
self.assertEqual((manifest['count'],manifest['source_listing_count'],manifest['held_entry_count']),(1,2,1))
|
|
self.assertNotIn('coverage_previous_count',manifest) # Live parent must be independently preflighted.
|
|
|
|
def test_malformed_href_requires_exact_id_and_two_clean_index_anchors(self):
|
|
self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('01/paper.html','01/ paper.html')
|
|
sources,_=self.sources();rows,held,_=spring_rows(sources,'papers');self.assertEqual(held,[])
|
|
self.assertEqual(rows[0]['url'],SPRING+'papers/01/paper.html');self.assertTrue(rows[0]['metadata']['url_corroborated_by_author_and_session'])
|
|
self.assertIn(' ',rows[0]['metadata']['abstract_href_raw'])
|
|
|
|
def test_main_index_disagreement_fails_without_publishing_batch(self):
|
|
self.pages[SPRING+'papers/session.html']='<a href="02/other.html">Other work</a>';_,sha=self.sources();out=self.root/'batch'
|
|
with self.assertRaisesRegex(ValueError,'ID sets disagree'):collect(self.cache,sha,out)
|
|
self.assertFalse(out.exists())
|
|
|
|
def test_same_entry_id_with_conflicting_clean_url_is_rejected(self):
|
|
self.pages[SPRING+'papers/session.html']='<a href="01/other.html">Same contribution</a>';sources,_=self.sources()
|
|
with self.assertRaisesRegex(ValueError,'URLs disagree'):spring_rows(sources,'papers')
|
|
|
|
def test_poster_without_linked_author_entry_is_held_even_with_abstract(self):
|
|
self.pages[SPRING+'posters/author.html']='<b>Doe, Jane</b><a href="#">Same contribution</a>';sources,_=self.sources();rows,held,counts=spring_rows(sources,'posters')
|
|
self.assertEqual(rows,[]);self.assertEqual(counts['abstract_entries'],1);self.assertEqual(held[0]['reason'],'abstract_entry_missing_linked_author_index')
|
|
|
|
def test_unlinked_and_panel_entries_never_become_acceptance_or_rejection(self):
|
|
sources,_=self.sources();held,counts=early_review(sources);self.assertEqual(counts,{'first_conference_fields':2,'fall_categories':2,'fall_program_entries':3})
|
|
self.assertTrue(any(r['candidate']['title']=='Unlinked talk'for r in held));self.assertTrue(all('status'not in r for r in held))
|
|
self.assertEqual(held[0]['candidate']['authors_raw'],['Jane Doe'])
|
|
|
|
def test_title_disagreement_goes_to_review_with_source_claims(self):
|
|
self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('Same contribution','Different contribution');sources,_=self.sources();rows,held,_=spring_rows(sources,'papers')
|
|
self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'abstract_and_author_index_titles_disagree')
|
|
|
|
def test_cache_tampering_and_manifest_hash_mismatch_fail_closed(self):
|
|
_,sha=self.sources()
|
|
with self.assertRaisesRegex(ValueError,'manifest checksum'):Sources(self.cache,'0'*64)
|
|
p=next(self.cache.glob('source-*.html'));p.write_bytes(p.read_bytes()+b'x')
|
|
with self.assertRaisesRegex(ValueError,'checksum or size'):Sources(self.cache,sha)
|
|
|
|
def test_cache_cannot_read_paths_outside_snapshot(self):
|
|
_,sha=self.sources();p=self.cache/'manifest.json';d=json.loads(p.read_text());d['sources'][0]['file']='../outside.html';p.write_text(json.dumps(d))
|
|
with self.assertRaisesRegex(ValueError,'plain files'):Sources(self.cache,hashlib.sha256(p.read_bytes()).hexdigest())
|
|
|
|
def test_keywords_only_is_missing_abstract_not_invented_summary(self):
|
|
self.pages[SPRING+'papers/abstract.html']='<h3><a href="01/paper.html">Same contribution</a></h3><i>Author: Jane Doe</i><p>Keywords: networking, hypertext</p>';sources,_=self.sources();rows,held,_=spring_rows(sources,'papers')
|
|
self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'missing_abstract')
|
|
|
|
def test_old_encoding_and_duplicate_abstract_identity(self):
|
|
self.assertEqual(decode(b'M\xfcller')[0],'Müller')
|
|
self.pages[SPRING+'papers/abstract.html']*=2;sources,_=self.sources()
|
|
with self.assertRaisesRegex(ValueError,'Duplicate abstract'):spring_rows(sources,'papers')
|
|
|
|
def test_existing_output_is_never_overwritten(self):
|
|
_,sha=self.sources();out=self.root/'batch';out.mkdir();(out/'keep').write_text('original')
|
|
with self.assertRaisesRegex(ValueError,'overwrite'):collect(self.cache,sha,out)
|
|
self.assertEqual((out/'keep').read_text(),'original')
|
|
|
|
def review(self,*,authors=None,author_source='canonical_byline',missing='',page=None):
|
|
url=SPRING+'posters/01/paper.html'
|
|
self.pages[url]=page or '<h1>Canonical full title</h1><p>Jane Doe, John Roe</p><p>Keywords: web</p><h2>Abstract</h2><p>Body</p>'
|
|
_,sha=self.sources();raw=self.pages[url].encode()if isinstance(self.pages[url],str)else self.pages[url]
|
|
row=dict(entry_id='01',individual_url=url,individual_sha256=hashlib.sha256(raw).hexdigest(),
|
|
title='Canonical full title',authors=authors or ['Jane Doe','John Roe'],author_source=author_source,
|
|
reason='Reviewed exact canonical entry and its explicit authorship; retain original title variants',missing_abstract_reason=missing)
|
|
p=self.root/'review.json';p.write_text(json.dumps({'format':1,'source_cache_sha256':sha,'records':[row]}));return sha,p,hashlib.sha256(p.read_bytes()).hexdigest()
|
|
|
|
def test_reviewed_variant_keeps_primary_title_and_all_old_title_claims(self):
|
|
sha,p,h=self.review();out=self.root/'reviewed';r=collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
|
|
row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['title'],'Canonical full title')
|
|
self.assertEqual(row['metadata']['abstract_heading_title_raw'],'Same contribution');self.assertEqual(row['metadata']['author_index_titles_raw'],['Same contribution'])
|
|
self.assertEqual(len(row['metadata']['source_evidence']),3);self.assertEqual(r['reviewed_posters_sha256'],h)
|
|
self.assertEqual(row['external_id'],'www3:posters:01');self.assertEqual(row['status'],'accepted')
|
|
|
|
def test_canonical_byline_can_resolve_unlinked_author_index_without_guessing(self):
|
|
self.pages[SPRING+'posters/author.html']='<b>Doe, Jane</b><a href="#">Broken index link</a>'
|
|
sha,p,h=self.review();out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
|
|
row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['authors'],['Jane Doe','John Roe'])
|
|
self.assertEqual(row['metadata']['author_index_raw'],[]);self.assertEqual(row['metadata']['author_order'],'canonical_byline')
|
|
|
|
def test_reviewed_missing_abstract_is_explicit_partial_not_body_summary(self):
|
|
self.pages[SPRING+'posters/abstract.html']='<h3><a href="01/paper.html">Same contribution</a></h3><i>Author: Jane Doe</i><p>Keywords: web</p>'
|
|
sha,p,h=self.review(missing='Neither listing nor canonical page supplies an explicit abstract')
|
|
out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
|
|
row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());m=json.loads((out/'www-1995-spring-poster.manifest.json').read_text())
|
|
self.assertIsNone(row['abstract']);self.assertEqual((m['missing_abstract_count'],m['status']),(1,'partial'))
|
|
sha,p,h=self.review()
|
|
with self.assertRaisesRegex(ValueError,'Missing abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
|
self.assertFalse((self.root/'bad').exists())
|
|
|
|
def test_reviewed_names_must_be_in_byline_region_not_citations_or_contact_guess(self):
|
|
sha,p,h=self.review(authors=['Unrelated Person'],page='<h1>Canonical full title</h1><p>Jane Doe</p><h2>Abstract</h2><p>Unrelated Person cited here</p>')
|
|
with self.assertRaisesRegex(ValueError,'byline region'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
|
sha,p,h=self.review(author_source='author_index',authors=['Contact Person'])
|
|
with self.assertRaisesRegex(ValueError,'author-index'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
|
|
|
def test_review_hash_cache_binding_and_canonical_h1_are_all_required(self):
|
|
sha,p,h=self.review()
|
|
with self.assertRaisesRegex(ValueError,'checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256='0'*64)
|
|
d=json.loads(p.read_text());d['source_cache_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
|
|
with self.assertRaisesRegex(ValueError,'binding'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
|
d['source_cache_sha256']=sha;d['records'][0]['title']='Wrong title';p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
|
|
with self.assertRaisesRegex(ValueError,'canonical H1'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
|
|
|
def test_unlisted_review_entry_and_changed_primary_digest_fail_closed(self):
|
|
sha,p,h=self.review();d=json.loads(p.read_text());d['records'][0]['individual_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
|
|
with self.assertRaisesRegex(ValueError,'page checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
|
self.pages[SPRING+'posters/abstract.html']=self.pages[SPRING+'posters/abstract.html'].replace('01/paper.html','02/other.html')
|
|
self.pages[SPRING+'posters/author.html']=self.pages[SPRING+'posters/author.html'].replace('01/paper.html','02/other.html')
|
|
sha,p,h=self.review()
|
|
with self.assertRaisesRegex(ValueError,'outside abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|