diff --git a/docs/mm2026-publications.md b/docs/mm2026-publications.md index a8172d3..374d114 100644 --- a/docs/mm2026-publications.md +++ b/docs/mm2026-publications.md @@ -50,7 +50,7 @@ ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 ` 完整2143个节目ID和4326个展示位置仍在匹配证据或417个未匹配节目中,28个原始track均保留。417个节目与192个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。 -注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`crossref:type` 与 `mm_program:track` 分别保留来源命名空间,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。 +注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`mm_registry_program:crossref:type` 与 `mm_registry_program:mm_program:track` 分别保存注册书目类型和节目 track;共同的派生来源前缀满足写入器的来源校验,两个标签命名空间仍相互独立,原始平台字段与证据继续保留在各自元数据中,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。 ## 只读数据库预检与导入边界 @@ -63,3 +63,5 @@ ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 ` 完整注册目录及独立节目比对恢复包:`/data/paper-index/validation/mm2026-container-registry-recovery-v2.tar.gz`,SHA256 `ddb05a8134f6437b3fb40250e45ed6350d55431d953812abda3300871b6dd0ac`。其中含注册源目录、父记录 `validation/mm2026-parent.json`、逐项比对和请求方法证据。安全解包后可把该父记录路径传给 `--parent-snapshot`。 节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。 + +实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1726个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。 diff --git a/paper_atlas/collectors/mm_publications2026.py b/paper_atlas/collectors/mm_publications2026.py index dd8039e..22d8393 100644 --- a/paper_atlas/collectors/mm_publications2026.py +++ b/paper_atlas/collectors/mm_publications2026.py @@ -162,8 +162,8 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot): abstract=p['abstract'] if p and p['abstract'] else None missing+=abstract is None;copied+=abstract is not None if p:known_tracks[p['track']]+=1 - tags=[{'namespace':'crossref:type','value':'proceedings-article'}] - if p:tags.append({'namespace':'mm_program:track','value':p['track']}) + tags=[{'namespace':SOURCE+':crossref:type','value':'proceedings-article'}] + if p:tags.append({'namespace':SOURCE+':mm_program:track','value':p['track']}) buckets[track].append({'source':SOURCE,'external_id':doi,'url':'https://doi.org/'+doi, 'title':item['title'],'authors':item['authors'],'abstract':abstract, 'identifiers':{'doi':doi},'submission_identifiers':[{'scheme':'publication_doi','value':doi}], diff --git a/tests/test_mm_publications2026.py b/tests/test_mm_publications2026.py index d0b96eb..3614a7e 100644 --- a/tests/test_mm_publications2026.py +++ b/tests/test_mm_publications2026.py @@ -1,3 +1,7 @@ +from contextlib import redirect_stdout +import io +import os +import uuid import gzip import hashlib import json @@ -82,8 +86,8 @@ class MMPublications2026Tests(unittest.TestCase): self.assertFalse(m['original_decision_signature_verified']);self.assertFalse(m['research_type_classified']) self.assertFalse(m['scientific_work_identity_approved']) self.assertTrue({'decided_at','decision_date','source_decision_id','paper_type'}.isdisjoint(r)) - self.assertIn({'namespace':'crossref:type','value':'proceedings-article'},r['tags']) - self.assertTrue({x['namespace'] for x in r['tags']} <= {'crossref:type','mm_program:track'}) + self.assertIn({'namespace':c.SOURCE+':crossref:type','value':'proceedings-article'},r['tags']) + self.assertTrue({x['namespace'] for x in r['tags']} <= {c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'}) for track in ('interactive_art','industry_talk','demo_and_video'): self.assertTrue(any(r['track']==track for r in self.rows)) variants=[r for r in self.rows if r['track']=='proceedings_unclassified'] @@ -153,3 +157,62 @@ class MMPublications2026Tests(unittest.TestCase): self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written']) self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0) self.assertIn('unknown',self.report['public_negative_availability']) + + + def assert_native_ingestion(self, pg_service=None): + # An isolated synthetic database exercises the writer contract, not a + # production identity approval. All source rows/track partitions run. + from paper_atlas.db.store import Store + from paper_atlas.db.backend import open_store + from paper_atlas.ingestion.records import ingest_file + from paper_atlas.postgres.client import connect + from paper_atlas.postgres.migration import migrate + from paper_atlas.postgres.schema import quote + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);db=root/'seed.sqlite';seed=self.rows[0] + store=Store(db) + paper,record=store.record('arxiv','synthetic-mm-anchor','https://example.org/arxiv', + seed['title'],'Existing canonical abstract',seed['authors'],seed['identifiers']) + store.decision(paper,record,'ICLR',2025,'main','synthetic-old-forum', + 'https://example.org/decision','rejected','Reject') + before=tuple(store.conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone()) + store.conn.commit();store.conn.close() + schema='pa_mm_pub_'+uuid.uuid4().hex[:12] if pg_service else None + try: + with redirect_stdout(io.StringIO()): + if pg_service:migrate(db,pg_service,root/'migration.json',schema=schema,min_free_gib=0) + for f in self.report['files']: + ingest_file(None if pg_service else db,f['file'],pg_service=pg_service,pg_schema=schema) + store=open_store(None if pg_service else db,pg_service=pg_service,pg_schema=schema) + try: + conn=store.conn + self.assertEqual(tuple(conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone()),before) + self.assertEqual(conn.execute('SELECT status FROM decisions WHERE submission_id=1').fetchone()[0],'rejected') + actual={r[0]:r for r in conn.execute("SELECT sr.external_id,sr.paper_id,sr.title,sr.abstract,sr.authors_json,sr.metadata_json,sr.id FROM source_records sr JOIN sources s ON s.id=sr.source_id WHERE s.name=?",(c.SOURCE,))} + self.assertEqual(set(actual),set(self.by_doi)) + self.assertEqual(actual[seed['external_id']][1],paper) + for doi,row in self.by_doi.items(): + got=actual[doi];self.assertEqual((got[2],got[3],json.loads(got[4])),(row['title'],row['abstract'],row['authors'])) + meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata']) + tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall() + self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918) + self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1726) + self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'}) + decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall() + self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)]) + self.assertEqual(conn.execute("SELECT count(*) FROM submission_identifiers WHERE scheme='publication_doi'").fetchone()[0],1918) + coverage=conn.execute("SELECT es.status,sum(es.imported_count) FROM edition_sources es JOIN sources s ON s.id=es.source_id WHERE s.name=? GROUP BY es.status",(c.SOURCE,)).fetchall() + self.assertEqual([tuple(r) for r in coverage],[('partial',1918)]) + finally:store.conn.close() + finally: + if pg_service: + with connect(pg_service) as pg: + for name, in pg.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(schema,)).fetchall(): + pg.execute('DROP SCHEMA '+quote(name)+' CASCADE') + + def test_all1918_native_sqlite_source_tags_and_prior_rejection_survive(self): + self.assert_native_ingestion() + + @unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL test service') + def test_all1918_native_postgres_source_tags_and_prior_rejection_survive(self): + self.assert_native_ingestion(os.environ['PAPER_ATLAS_TEST_PG_SERVICE'])