92 lines
5.9 KiB
Python
92 lines
5.9 KiB
Python
import json
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from index_core import Store
|
|
from import_records import ingest_file
|
|
|
|
|
|
class StoreTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.db = Path(self.tmp.name)/'papers.sqlite'
|
|
self.store = Store(self.db)
|
|
self.addCleanup(self.store.conn.close)
|
|
|
|
def record(self,source,identifier,ids=None,authors=None):
|
|
return self.store.record(source,identifier,'https://example.org/'+identifier,'A sufficiently long example research paper title','Example abstract',authors or ['Jane Doe','John Smith'],ids)
|
|
|
|
def test_bridge_identifiers_preserves_submission_and_tags(self):
|
|
first,record = self.record('arxiv','2601.00001',{'arxiv':'2601.00001'},['First Author'])
|
|
second,other_record = self.record('pmlr','v1/other',{'doi':'10.0000/example'},['Other Author'])
|
|
self.store.accepted(second,other_record,'ICML',2025,'main','v1/other','https://example.org/other')
|
|
tag = self.store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','test')").lastrowid
|
|
self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)',(second,tag,other_record,'manual'))
|
|
resolved,_ = self.record('openreview','forum1',{'arxiv':'2601.00001','doi':'10.0000/example'},['Anonymous'])
|
|
self.assertEqual(resolved,first)
|
|
c=self.store.conn
|
|
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
|
|
self.assertEqual(c.execute('SELECT paper_id FROM submissions').fetchone()[0],first)
|
|
self.assertEqual(c.execute('SELECT paper_id FROM paper_tags').fetchone()[0],first)
|
|
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
|
|
self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1)
|
|
|
|
def test_cross_platform_exact_title_needs_author_corroboration(self):
|
|
paper,_=self.record('arxiv','2601.00001')
|
|
same,_=self.record('pmlr','v1/paper')
|
|
other,_=self.record('acl','2025.acl.1',authors=['Unrelated Person'])
|
|
self.assertEqual(same,paper)
|
|
self.assertNotEqual(other,paper)
|
|
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
|
|
|
|
def test_manifest_count_mismatch_rolls_back_records(self):
|
|
path=Path(self.tmp.name)/'source.jsonl'
|
|
path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'})+'\n')
|
|
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':2}))
|
|
with self.assertRaisesRegex(ValueError,'differs'):
|
|
ingest_file(self.db,path)
|
|
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],0)
|
|
self.assertEqual(self.store.conn.execute('SELECT status FROM ingest_runs').fetchone()[0],'failed')
|
|
|
|
def test_source_update_refreshes_canonical_text_and_fts(self):
|
|
paper,_=self.record('arxiv','2601.00001')
|
|
updated,_=self.store.record('arxiv','2601.00001','https://arxiv.org/abs/2601.00001','Updated research title','New searchable content',['Jane Doe'])
|
|
self.assertEqual(updated,paper)
|
|
self.assertEqual(self.store.conn.execute('SELECT title FROM papers').fetchone()[0],'Updated research title')
|
|
self.assertEqual(self.store.conn.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'searchable'").fetchone()[0],1)
|
|
|
|
def test_tag_refresh_keeps_other_record_evidence_and_custom_tags(self):
|
|
paper,first=self.record('iclr_program','2025/a',{'openreview':'sharedForum'})
|
|
same,second=self.record('iclr_program','2025/b',{'openreview':'sharedForum'})
|
|
self.store.record_tags(paper,first,'iclr_program',[{'namespace':'iclr_program:topic','value':'Shared'}])
|
|
self.store.record_tags(same,second,'iclr_program',[{'namespace':'iclr_program:topic','value':'Shared'}])
|
|
tag=self.store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','Manual')").lastrowid
|
|
self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)',(paper,tag,None,'manual'))
|
|
self.store.record_tags(paper,first,'iclr_program',[])
|
|
self.assertEqual(self.store.conn.execute('SELECT source_record_id FROM paper_tags ORDER BY source_record_id').fetchall(),[(None,),(second,)])
|
|
with self.assertRaisesRegex(ValueError,'namespace'):
|
|
self.store.record_tags(paper,second,'iclr_program',[{'namespace':'custom:topic','value':'No overwrite'}])
|
|
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM paper_tags').fetchone()[0],2)
|
|
|
|
def test_missing_abstract_does_not_become_complete_coverage(self):
|
|
path=Path(self.tmp.name)/'partial.jsonl'
|
|
path.write_text(json.dumps({'source':'colm','external_id':'paper1','url':'https://example.org/paper1','title':'Paper','abstract':None,'authors':[],'venue':'COLM','year':2024,'track':'main','status':'accepted'})+'\n')
|
|
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'colm','parser_version':2,'list_url':'https://example.org/list','count':1,'status':'partial','missing_abstract_count':1}))
|
|
ingest_file(self.db,path)
|
|
self.assertEqual(self.store.conn.execute('SELECT status FROM edition_sources').fetchone()[0],'partial')
|
|
|
|
def test_legacy_proceedings_manifest_is_rejected_before_import(self):
|
|
path=Path(self.tmp.name)/'old.jsonl'
|
|
path.write_text('')
|
|
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'colm','count':1}))
|
|
with self.assertRaisesRegex(ValueError,'parser_version'):
|
|
ingest_file(self.db,path)
|
|
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],0)
|
|
|
|
|
|
if __name__=='__main__':
|
|
unittest.main()
|