175 lines
13 KiB
Python
175 lines
13 KiB
Python
from importlib.resources import files
|
|
import json
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
from paper_atlas.ingestion.records import ingest_file
|
|
from paper_atlas.db.store import Store
|
|
from paper_atlas.db.submissions import identities_for_row
|
|
from paper_atlas.ingestion.backfill import backfill,queue_files
|
|
|
|
|
|
class SubmissionTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
|
|
self.db=Path(self.tmp.name)/'papers.sqlite';self.store=Store(self.db);self.addCleanup(self.store.conn.close)
|
|
|
|
def record(self,source,external,authors):
|
|
return self.store.record(source,external,'https://example.org/'+external,'A long paper about reliable metadata and historical events','Abstract',authors)
|
|
|
|
def decision(self,paper,record,external,identities,status='accepted',raw='Accept',url='https://example.org/decision',observed='2026-05-08T00:00:00Z',date=None):
|
|
return self.store.decision(paper,record,'ICLR',2026,'main',external,url,status,raw,identities=identities,observed_at=observed,decision_date=date)
|
|
|
|
def query(self,*args):
|
|
self.store.conn.commit()
|
|
command=[sys.executable,'-B','-m','paper_atlas','query','--db',str(self.db),*args]
|
|
result=subprocess.run(command,capture_output=True,text=True)
|
|
self.assertEqual(result.returncode,0,result.stderr)
|
|
return json.loads(result.stdout)
|
|
|
|
def test_same_submission_across_sources_has_one_event_and_both_provenances(self):
|
|
first,one=self.store.record('iclr_program','2026/number-42','https://example.org/program','A long paper about reliable metadata and historical events',None,['Jane Doe','John Smith'])
|
|
second,two=self.record('openreview_snapshot','realForum',['Anonymous'])
|
|
self.assertNotEqual(first,second)
|
|
self.decision(first,one,'2026/number-42',[('iclr_number','42')])
|
|
self.decision(second,two,'realForum',[('iclr_number','42'),('openreview','realForum')],url='https://openreview.net/forum?id=realForum¬eId=note1')
|
|
c=self.store.conn
|
|
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
|
|
self.assertEqual(c.execute('SELECT abstract FROM papers').fetchone()[0],'Abstract')
|
|
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
|
|
self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0],2)
|
|
details=self.query('--paper',str(first))[0]['submissions'][0]
|
|
self.assertEqual({r['name'] for r in details['sources']},{'iclr_program','openreview_snapshot'})
|
|
self.assertEqual(len(details['decisions']),2)
|
|
self.assertTrue(all(d['sources'] for d in details['decisions']))
|
|
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
|
|
|
|
def test_same_paper_can_have_two_distinct_submission_attempts(self):
|
|
paper,one=self.store.record('openreview_snapshot','forum1','https://example.org/1','First version','Abstract',[],{'arxiv':'2601.00001','openreview':'forum1'})
|
|
other,two=self.store.record('openreview_snapshot','forum2','https://example.org/2','Another version','Abstract',[],{'arxiv':'2601.00001','openreview':'forum2'})
|
|
self.assertEqual(paper,other)
|
|
self.decision(paper,one,'forum1',[('openreview','forum1'),('iclr_number','1')],status='rejected',raw='Reject')
|
|
self.decision(paper,two,'forum2',[('openreview','forum2'),('iclr_number','2')])
|
|
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],2)
|
|
|
|
def test_conflicting_forums_cannot_be_bridged_by_one_submission_number(self):
|
|
paper,one=self.record('snapshot','realForum',['Jane Doe'])
|
|
other,two=self.record('snapshot','differentForum',['Other Author'])
|
|
self.decision(paper,one,'realForum',[('openreview','realForum'),('iclr_number','42')])
|
|
with self.assertRaisesRegex(ValueError,'Conflicting authoritative submission identities'):
|
|
self.decision(other,two,'differentForum',[('openreview','differentForum'),('iclr_number','42')])
|
|
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],2)
|
|
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
|
|
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submission_merges').fetchone()[0],0)
|
|
|
|
def test_late_identity_bridge_preserves_decisions_and_does_not_reuse_merged_ids(self):
|
|
first,one=self.record('iclr_program','number-42',['Jane Doe','John Smith'])
|
|
second,two=self.record('openreview_snapshot','realForum',['Anonymous'])
|
|
self.decision(first,one,'number-42',[('iclr_number','42')])
|
|
self.decision(second,two,'realForum',[('openreview','realForum')])
|
|
self.decision(second,two,'realForum',[('openreview','realForum'),('iclr_number','42')])
|
|
c=self.store.conn
|
|
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
|
|
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],1)
|
|
self.assertEqual(c.execute('SELECT count(*) FROM decision_sources').fetchone()[0],2)
|
|
self.assertEqual(c.execute('SELECT count(*) FROM submission_merges').fetchone()[0],1)
|
|
self.assertEqual(c.execute('SELECT count(*) FROM decision_merges').fetchone()[0],1)
|
|
third,record=self.record('other','different',['Other Author'])
|
|
self.decision(third,record,'different',[('openreview','differentForum')],status='rejected',raw='Reject')
|
|
self.assertGreater(c.execute('SELECT max(id) FROM submissions').fetchone()[0],2)
|
|
self.assertGreater(c.execute('SELECT max(id) FROM decisions').fetchone()[0],2)
|
|
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
|
|
|
|
def test_old_snapshot_does_not_override_a_newer_official_observation(self):
|
|
paper,one=self.record('iclr_program','number-42',['Jane Doe','John Smith'])
|
|
_,two=self.record('openreview_snapshot','realForum',['Jane Doe','John Smith'])
|
|
self.decision(paper,one,'number-42',[('iclr_number','42')],observed='2026-11-06T00:00:00Z')
|
|
self.decision(paper,two,'realForum',[('iclr_number','42')],status='rejected',raw='Reject',observed='2026-05-08T00:00:00Z')
|
|
self.assertEqual(len(self.query('--venue','ICLR','--status','accepted')),1)
|
|
self.assertEqual(self.query('--venue','ICLR','--status','rejected'),[])
|
|
|
|
def test_decision_date_orders_events_with_the_same_snapshot_time(self):
|
|
paper,record=self.record('openreview_snapshot','realForum',['Jane Doe'])
|
|
self.decision(paper,record,'realForum',[('openreview','realForum')],date='2026-02-01T00:00:00Z')
|
|
self.decision(paper,record,'realForum',[('openreview','realForum')],status='rejected',raw='Reject',date='2026-01-01T00:00:00Z')
|
|
self.assertEqual(len(self.query('--venue','ICLR','--status','accepted')),1)
|
|
self.assertEqual(self.query('--venue','ICLR','--status','rejected'),[])
|
|
|
|
def test_unknown_without_a_decision_is_queryable_without_invented_verdict(self):
|
|
path=Path(self.tmp.name)/'unknown.jsonl'
|
|
row={'source':'openreview_snapshot','external_id':'realForum','url':'https://example.org/paper','title':'Unknown result','abstract':'Abstract','authors':[],'identifiers':{'openreview':'realForum'},'venue':'ICLR','year':2026,'track':'main','status':'unknown','decision_events':[]}
|
|
path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':row['source'],'list_url':'https://example.org/list','count':1}))
|
|
ingest_file(self.db,path)
|
|
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM decisions').fetchone()[0],0)
|
|
self.assertEqual(len(self.query('--venue','ICLR','--status','unknown')),1)
|
|
|
|
def test_known_summary_without_corresponding_decision_rolls_back(self):
|
|
path=Path(self.tmp.name)/'missing-evidence.jsonl'
|
|
row={'source':'openreview_snapshot','external_id':'realForum','url':'https://example.org/paper','title':'Unsupported result','abstract':'Abstract','authors':[],'venue':'ICLR','year':2026,'track':'main','status':'accepted','decision_events':[{'status':None}]}
|
|
path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':row['source'],'list_url':'https://example.org/list','count':1}))
|
|
with self.assertRaisesRegex(ValueError,'no corresponding decision evidence'):
|
|
ingest_file(self.db,path)
|
|
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],0)
|
|
|
|
def test_event_number_is_not_assumed_to_be_a_submission_number(self):
|
|
row={'source':'iclr_program','venue':'ICLR','metadata':{'program_source_id':42,'program_identity_type':'event'},'identifiers':{}}
|
|
self.assertEqual(identities_for_row(row),[])
|
|
row['metadata']['program_identity_type']='submission'
|
|
self.assertEqual(identities_for_row(row),[])
|
|
row['metadata']['program_submission_number_verified']=True
|
|
self.assertEqual(identities_for_row(row),[('iclr_number','42')])
|
|
with self.assertRaisesRegex(ValueError,'Invalid authoritative'):
|
|
identities_for_row({'source':'snapshot','venue':'ICLR','submission_identifiers':[{'scheme':'openreview','value':'2026-Oral--42-placeholder'}]})
|
|
|
|
def test_legacy_database_migration_preserves_source_and_decision_links(self):
|
|
legacy=Path(self.tmp.name)/'legacy.sqlite';sql=files('paper_atlas.db').joinpath('schema.sql').read_text()
|
|
sql=sql[:sql.index('CREATE TABLE IF NOT EXISTS submission_sources')]+sql[sql.index('CREATE TABLE IF NOT EXISTS tags'):]
|
|
c=sqlite3.connect(legacy);c.executescript(sql)
|
|
c.execute("INSERT INTO sources VALUES(1,'legacy','https://example.org')")
|
|
c.execute("INSERT INTO papers(id,title,title_key) VALUES(1,'Paper','paper')")
|
|
c.execute("INSERT INTO source_records VALUES(1,1,'paper',1,'https://example.org/p','Paper','Abstract','[]','{}',CURRENT_TIMESTAMP)")
|
|
c.execute("INSERT INTO venues VALUES(1,'ICLR','ICLR','machine_learning','conference')")
|
|
c.execute("INSERT INTO editions(id,venue_id,year,track) VALUES(1,1,2026,'main')")
|
|
c.execute("INSERT INTO submissions VALUES(1,1,1,1,'paper',NULL)")
|
|
c.execute("INSERT INTO decisions VALUES(1,1,'accepted','Accept','https://example.org/decision','2026-01-01T00:00:00Z',NULL)")
|
|
c.commit();c.close()
|
|
store=Store(legacy);self.addCleanup(store.conn.close)
|
|
self.assertEqual(store.conn.execute('SELECT * FROM submission_sources').fetchall(),[(1,1,'paper')])
|
|
self.assertEqual(store.conn.execute('SELECT * FROM decision_sources').fetchall(),[(1,1,'')])
|
|
self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(),[])
|
|
|
|
def test_existing_program_identity_backfill_is_idempotent_and_preserves_text(self):
|
|
paper,record=self.record('iclr_program','2026/program-42',['Jane Doe','John Smith'])
|
|
self.decision(paper,record,'2026/program-42',[])
|
|
self.store.conn.commit()
|
|
original=self.store.conn.execute('SELECT title,abstract,retrieved_at FROM source_records WHERE id=?',(record,)).fetchone()
|
|
path=Path(self.tmp.name)/'program.jsonl'
|
|
row={'source':'iclr_program','external_id':'2026/program-42','venue':'ICLR','year':2026,'track':'main','status':'accepted','identifiers':{},'metadata':{'program_identity_type':'submission','program_source_id':42,'program_submission_number_verified':True}}
|
|
path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','parser_version':2,'venue':'ICLR','year':2026,'track':'main','count':1}))
|
|
first=backfill(self.db,[path]);second=backfill(self.db,[path])
|
|
self.assertEqual(first,second)
|
|
self.assertEqual(self.store.conn.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),[('iclr_number','42')])
|
|
self.assertEqual(self.store.conn.execute('SELECT title,abstract,retrieved_at FROM source_records WHERE id=?',(record,)).fetchone(),original)
|
|
other,snapshot=self.record('openreview_snapshot','realForum',['Anonymous'])
|
|
self.decision(other,snapshot,'realForum',[('iclr_number','42'),('openreview','realForum')])
|
|
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
|
|
|
|
def test_evidence_queue_rejects_changed_file_before_mutation(self):
|
|
path=Path(self.tmp.name)/'program.jsonl';path.write_text('changed bytes')
|
|
queue=Path(self.tmp.name)/'queue.json';queue.write_text(json.dumps({'files':[str(path)],'file_sha256':{str(path):'0'*64}}))
|
|
with self.assertRaisesRegex(ValueError,'differs from audited'):
|
|
queue_files(queue)
|
|
|
|
def test_old_untyped_program_files_require_recollection(self):
|
|
path=Path(self.tmp.name)/'old.jsonl';path.write_text('')
|
|
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','parser_version':1}))
|
|
with self.assertRaisesRegex(ValueError,'Recollect ICLR'):
|
|
ingest_file(self.db,path)
|
|
|
|
|
|
if __name__=='__main__':unittest.main()
|