diff --git a/docs/usage.md b/docs/usage.md index 9f0e198..133a458 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -102,6 +102,8 @@ paper-atlas query --pg-service foundry-research \ `--stats` 中 papers 是当前规则下的论文实体数；source_records 是来源记录数；decisions 是决定观察数。一个投稿可能有多个来源和多条观察，不能相加当作论文数。 +已有稳定标识或原来源记录匹配时，该身份优先；同标题、同作者的另一 arXiv 存档或扩展稿不会仅因相似标题被并入。没有身份锚点时，作者佐证的精确标题只有一个候选才可匹配；多个候选会报 `Ambiguous paper identity`，整个采集文件回滚并记录失败运行。应补有证据的标识或先核查版本关系，不删除重复候选或任取第一个。一个新来源明确同时携带两个既有稳定标识时，仍可按既有规则归并并保留投稿、标签和合并日志；未归并的同标题关系继续保留为待核查候选。这些规则不证明全库去重完成。 + ## 5. 投稿历史关系审计 ```sh diff --git a/paper_atlas/db/store.py b/paper_atlas/db/store.py index 7d57533..b40698f 100644 --- a/paper_atlas/db/store.py +++ b/paper_atlas/db/store.py @@ -95,13 +95,19 @@ class Store: matches.add(old[0]) key = normalized(title) title_matches = c.execute('SELECT id,authors_json,abstract FROM papers WHERE title_key=?',(key,)).fetchall() - authors_set = author_keys(authors) - for paper,old_authors,old_abstract in title_matches: - overlap = authors_set & author_keys(json.loads(old_authors)) - # Cross-platform title matching needs independent author corroboration. - same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone() - if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())): - matches.add(paper) + # An existing identifier/source record anchors the entity. Title + # similarity must not pull in another deposit or an extended version. + if not matches: + title_only_matches = set() + authors_set = author_keys(authors) + for paper,old_authors,old_abstract in title_matches: + overlap = authors_set & author_keys(json.loads(old_authors)) + same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone() + if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())): + title_only_matches.add(paper) + if len(title_only_matches)>1: + raise ValueError('Ambiguous paper identity: multiple author-corroborated title matches; provide verified identifiers') + matches = title_only_matches if matches: paper = min(matches) for other in sorted(matches-{paper}): diff --git a/tests/test_index_core.py b/tests/test_index_core.py index 0e18a06..dc9e2cf 100644 --- a/tests/test_index_core.py +++ b/tests/test_index_core.py @@ -53,6 +53,50 @@ class StoreTests(unittest.TestCase): self.assertNotEqual(other,paper) self.assertEqual(self.store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1) + def test_identifier_anchor_does_not_absorb_a_same_title_arxiv_version(self): + conference,first=self.record('arxiv','1408.1485') + extended,second=self.record('arxiv','cs/0307069') + self.store.decision(extended,second,'ICLR',2025,'main','separate','https://example.org/reject','rejected','Reject') + resolved,_=self.record('dblp','conf/uai/example',{'arxiv':'1408.1485'}) + self.assertEqual(resolved,conference);self.assertNotEqual(conference,extended) + c=self.store.conn + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2) + self.assertEqual(c.execute('SELECT paper_id FROM submissions').fetchone()[0],extended) + self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],0) + self.assertEqual(c.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1) + # A refresh of the anchored original source is equally unambiguous. + self.assertEqual(self.record('arxiv','1408.1485')[0],conference) + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2) + + def test_ambiguous_title_aborts_whole_file_without_merging_existing_entities(self): + self.record('arxiv','1408.1485');self.record('arxiv','cs/0307069') + self.store.conn.commit() + rows=[{'source':'dblp','external_id':'conf/uai/new','url':'https://example.org/new', + 'title':'An unrelated earlier record in this file','abstract':None,'authors':[], + 'venue':'UAI','year':2001,'track':'main','status':'accepted'}, + {'source':'dblp','external_id':'conf/uai/ambiguous','url':'https://example.org/ambiguous', + 'title':'A sufficiently long example research paper title','abstract':None, + 'authors':['Jane Doe','John Smith'],'identifiers':{'doi':'10.0000/not-yet-known'}, + 'venue':'UAI','year':2001,'track':'main','status':'accepted'}] + path=Path(self.tmp.name)/'ambiguous.jsonl' + path.write_text(''.join(json.dumps(row)+'\n' for row in rows)) + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','list_url':'https://example.org/list','count':2})) + with self.assertRaisesRegex(ValueError,'Ambiguous paper identity'):ingest_file(self.db,path) + c=self.store.conn + for table,count in [('papers',2),('source_records',2),('submissions',0),('decisions',0),('paper_merges',0)]: + self.assertEqual(c.execute('SELECT count(*) FROM '+table).fetchone()[0],count,table) + self.assertEqual(c.execute('SELECT status FROM ingest_runs').fetchone()[0],'failed') + + def test_explicit_bridge_merges_only_its_identifier_matches(self): + first,_=self.record('arxiv','2601.00001',authors=['First Author']) + second,_=self.record('arxiv','2601.00002',{'doi':'10.0000/bridge'},authors=['Second Author']) + unrelated,_=self.record('arxiv','2601.00003') + resolved,_=self.record('official','bridge',{'arxiv':'2601.00001','doi':'10.0000/bridge'}) + self.assertEqual(resolved,min(first,second)) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],2) + self.assertIsNotNone(self.store.conn.execute('SELECT 1 FROM papers WHERE id=?',(unrelated,)).fetchone()) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1) + def test_manifest_count_mismatch_rolls_back_records(self): path=Path(self.tmp.name)/'source.jsonl' path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2023,'track':'main','status':'accepted'})+'\n') diff --git a/tests/test_postgres_ingestion.py b/tests/test_postgres_ingestion.py index 7769b6d..a37c5fa 100644 --- a/tests/test_postgres_ingestion.py +++ b/tests/test_postgres_ingestion.py @@ -110,6 +110,34 @@ class PostgreSQLIngestionTests(unittest.TestCase): finally: reader.close() + def test_identity_anchor_keeps_other_arxiv_version_and_ambiguous_file_rolls_back(self): + title='An identical long research title shared by conference and extended versions' + authors=['Jane Doe','John Smith'] + store=self.store() + try: + first,_=store.record('arxiv','1408.1485','https://arxiv.org/abs/1408.1485',title,'Conference abstract',authors) + second,_=store.record('arxiv','cs/0307069','https://arxiv.org/abs/cs/0307069',title,'Extended abstract',authors) + resolved,_=store.record('dblp','known','https://example.org/known',title,None,authors,{'arxiv':'1408.1485'}) + self.assertEqual(resolved,first);self.assertNotEqual(first,second) + store.conn.commit() + self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],3) + self.assertEqual(store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1) # migration fixture only + self.assertEqual(store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1) + finally:store.conn.close() + rows=[self.row('official','earlier',status='accepted',venue='UAI',year=2001), + dict(self.row('official','ambiguous',status='accepted',venue='UAI',year=2001), + title=title,abstract=None,authors=authors,identifiers={})] + with self.assertRaisesRegex(ValueError,'Ambiguous paper identity'):self.ingest(self.file('ambiguous-identity',rows)) + store=self.store() + try: + self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],3) + self.assertEqual(store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],4) + self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + self.assertEqual(store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1) + self.assertEqual(store.conn.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed') + self.assertEqual(store.conn.execute("SELECT count(*) FROM source_records WHERE external_id IN (?,?)",('earlier','ambiguous')).fetchone()[0],0) + finally:store.conn.close() + def test_file_import_is_idempotent_and_refreshes_only_own_tags(self): row = self.row(); row['tags'] = [{'namespace': 'openreview_snapshot:area', 'value': 'theory'}] path = self.file('reject', [row]); self.ingest(path)