Fix MM publication tag provenance and exercise full native ingestion
Metadata integrity checks / metadata (push) Successful in 6m46s

This commit is contained in:
2026-11-18 01:31:59 +08:00
parent 13536d6268
commit 83f3ebd26b
3 changed files with 70 additions and 5 deletions
+3 -1
View File
@@ -50,7 +50,7 @@ ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `
完整2143个节目ID和4326个展示位置仍在匹配证据或417个未匹配节目中,28个原始track均保留。417个节目与192个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`crossref:type` 与 `mm_program:track` 分别保留来源命名空间,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`mm_registry_program:crossref:type` 与 `mm_registry_program:mm_program:track` 分别保存注册书目类型和节目 track;共同的派生来源前缀满足写入器的来源校验,两个标签命名空间仍相互独立,原始平台字段与证据继续保留在各自元数据中,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
## 只读数据库预检与导入边界
@@ -63,3 +63,5 @@ ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `
完整注册目录及独立节目比对恢复包:`/data/paper-index/validation/mm2026-container-registry-recovery-v2.tar.gz`,SHA256 `ddb05a8134f6437b3fb40250e45ed6350d55431d953812abda3300871b6dd0ac`。其中含注册源目录、父记录 `validation/mm2026-parent.json`、逐项比对和请求方法证据。安全解包后可把该父记录路径传给 `--parent-snapshot`。
节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。
实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1726个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。
@@ -162,8 +162,8 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
abstract=p['abstract'] if p and p['abstract'] else None
missing+=abstract is None;copied+=abstract is not None
if p:known_tracks[p['track']]+=1
tags=[{'namespace':'crossref:type','value':'proceedings-article'}]
if p:tags.append({'namespace':'mm_program:track','value':p['track']})
tags=[{'namespace':SOURCE+':crossref:type','value':'proceedings-article'}]
if p:tags.append({'namespace':SOURCE+':mm_program:track','value':p['track']})
buckets[track].append({'source':SOURCE,'external_id':doi,'url':'https://doi.org/'+doi,
'title':item['title'],'authors':item['authors'],'abstract':abstract,
'identifiers':{'doi':doi},'submission_identifiers':[{'scheme':'publication_doi','value':doi}],
+65 -2
View File
@@ -1,3 +1,7 @@
from contextlib import redirect_stdout
import io
import os
import uuid
import gzip
import hashlib
import json
@@ -82,8 +86,8 @@ class MMPublications2026Tests(unittest.TestCase):
self.assertFalse(m['original_decision_signature_verified']);self.assertFalse(m['research_type_classified'])
self.assertFalse(m['scientific_work_identity_approved'])
self.assertTrue({'decided_at','decision_date','source_decision_id','paper_type'}.isdisjoint(r))
self.assertIn({'namespace':'crossref:type','value':'proceedings-article'},r['tags'])
self.assertTrue({x['namespace'] for x in r['tags']} <= {'crossref:type','mm_program:track'})
self.assertIn({'namespace':c.SOURCE+':crossref:type','value':'proceedings-article'},r['tags'])
self.assertTrue({x['namespace'] for x in r['tags']} <= {c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
for track in ('interactive_art','industry_talk','demo_and_video'):
self.assertTrue(any(r['track']==track for r in self.rows))
variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
@@ -153,3 +157,62 @@ class MMPublications2026Tests(unittest.TestCase):
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
self.assertIn('unknown',self.report['public_negative_availability'])
def assert_native_ingestion(self, pg_service=None):
# An isolated synthetic database exercises the writer contract, not a
# production identity approval. All source rows/track partitions run.
from paper_atlas.db.store import Store
from paper_atlas.db.backend import open_store
from paper_atlas.ingestion.records import ingest_file
from paper_atlas.postgres.client import connect
from paper_atlas.postgres.migration import migrate
from paper_atlas.postgres.schema import quote
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);db=root/'seed.sqlite';seed=self.rows[0]
store=Store(db)
paper,record=store.record('arxiv','synthetic-mm-anchor','https://example.org/arxiv',
seed['title'],'Existing canonical abstract',seed['authors'],seed['identifiers'])
store.decision(paper,record,'ICLR',2025,'main','synthetic-old-forum',
'https://example.org/decision','rejected','Reject')
before=tuple(store.conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone())
store.conn.commit();store.conn.close()
schema='pa_mm_pub_'+uuid.uuid4().hex[:12] if pg_service else None
try:
with redirect_stdout(io.StringIO()):
if pg_service:migrate(db,pg_service,root/'migration.json',schema=schema,min_free_gib=0)
for f in self.report['files']:
ingest_file(None if pg_service else db,f['file'],pg_service=pg_service,pg_schema=schema)
store=open_store(None if pg_service else db,pg_service=pg_service,pg_schema=schema)
try:
conn=store.conn
self.assertEqual(tuple(conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone()),before)
self.assertEqual(conn.execute('SELECT status FROM decisions WHERE submission_id=1').fetchone()[0],'rejected')
actual={r[0]:r for r in conn.execute("SELECT sr.external_id,sr.paper_id,sr.title,sr.abstract,sr.authors_json,sr.metadata_json,sr.id FROM source_records sr JOIN sources s ON s.id=sr.source_id WHERE s.name=?",(c.SOURCE,))}
self.assertEqual(set(actual),set(self.by_doi))
self.assertEqual(actual[seed['external_id']][1],paper)
for doi,row in self.by_doi.items():
got=actual[doi];self.assertEqual((got[2],got[3],json.loads(got[4])),(row['title'],row['abstract'],row['authors']))
meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata'])
tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall()
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918)
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1726)
self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall()
self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)])
self.assertEqual(conn.execute("SELECT count(*) FROM submission_identifiers WHERE scheme='publication_doi'").fetchone()[0],1918)
coverage=conn.execute("SELECT es.status,sum(es.imported_count) FROM edition_sources es JOIN sources s ON s.id=es.source_id WHERE s.name=? GROUP BY es.status",(c.SOURCE,)).fetchall()
self.assertEqual([tuple(r) for r in coverage],[('partial',1918)])
finally:store.conn.close()
finally:
if pg_service:
with connect(pg_service) as pg:
for name, in pg.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(schema,)).fetchall():
pg.execute('DROP SCHEMA '+quote(name)+' CASCADE')
def test_all1918_native_sqlite_source_tags_and_prior_rejection_survive(self):
self.assert_native_ingestion()
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL test service')
def test_all1918_native_postgres_source_tags_and_prior_rejection_survive(self):
self.assert_native_ingestion(os.environ['PAPER_ATLAS_TEST_PG_SERVICE'])