From 09a2d4ed207781cafef517b8c8f05d26eae7378d Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Tue, 17 Nov 2026 22:49:53 +0800 Subject: [PATCH] Audit MM program identities without inventing decisions or import records --- .gitea/workflows/check.yml | 7 ++ docs/mm2026-source-status.md | 18 +++++ paper_atlas/audits/source_identity.py | 83 ++++++++++++++++++++- tests/test_mm_source_identity.py | 101 ++++++++++++++++++++++++++ 4 files changed, 205 insertions(+), 4 deletions(-) create mode 100644 tests/test_mm_source_identity.py diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 1d8ae31..9726482 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -66,6 +66,13 @@ jobs: assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256'] assert [c['source_pointer'] for c in row['contexts']] == gold['locations'] assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row + if service_file.is_file(): + subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True) + mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text()) + assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28 + assert mm_preflight['transaction_read_only'] and not mm_preflight['database_written'] and mm_preflight['approved_for_import'] == 0 + assert sum(len(r['program_contexts']) for r in mm_preflight['entries']) == 4326 + assert all(r['status'] == 'unknown' and not r['identifiers'] and not r['contribution_type_verified'] and not r['publication_identity_verified'] and not r['final_decision_verified'] for r in mm_preflight['entries']) subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True) diff --git a/docs/mm2026-source-status.md b/docs/mm2026-source-status.md index 0d44474..78c52db 100644 --- a/docs/mm2026-source-status.md +++ b/docs/mm2026-source-status.md @@ -90,3 +90,21 @@ paper-atlas collect mm2026-program \ 这些文件的格式为 `mm_program_observation_v1_not_ingestion_record`,所有项 `approved_for_import=false`。节目ID仅为 `mm_program:program_id` 来源定位符,track标签使用 `mm_program:track`;不建立科学作品、原始投稿ID或最终决定,不推断 DOI/arXiv/OpenReview 标识、论文类型、拒稿或撤稿。行业报告、艺术、workshop及其他来源贡献均保留,尚需逐轨核对贡献类型与正式出版依据。 本批尚未形成生产导入队列。后续做贡献类型、作品/出版身份及既往投稿审查,再生成经批准的增量队列。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。 + +## 直接审计节目身份与既往历史 + +全量节目观察可以直接进入只读候选审计,无需伪造成录用论文队列: + +```sh +PGSERVICEFILE=/etc/paper-atlas/pg_service.conf \ +paper-atlas audit source-identities \ + --mm-program-report /data/paper-index/collected/mm2026-native-v1/collection-report.json \ + --pg-service foundry-research --pg-schema research \ + --out /data/paper-index/collected/mm2026-preflight-v1 +``` + +`--mm-program-report` 与常规 `--files` 互斥。审计先核对冻结原文、全部28轨的逐文件哈希/数量,以及每个节目ID的完整字段和展示上下文;缺轨、重复ID、篡改后的标识/决定或上下文均在数据库连接前拒绝。报告中记录的绝对文件路径仅作来源证据,实际读取报告同目录中的对应文件。 + +候选审计中的 `status=unknown` 仅为内部查询占位符,不代表原始提交的最终状态。节目ID以 `mm_program` 来源键查询,不转成全局 DOI、arXiv 或 OpenReview 标识;保留原始 `listed_in_official_program` 观察与全部展示位置。所有贡献类型、出版身份、最终决定核验标志均为false;缺作者的两项也保留并标记。 + +查询使用 PostgreSQL 只读一致性快照,候选作品的原始来源、标识、历次投稿和逐次决定压缩保存并记录哈希;截断会显式标记。标题/作者候选匹配不是作品身份批准,没有匹配也不证明是新论文。既往拒稿和后续录用不能互相覆盖,评分或评审推荐不能充当最终决定。输出目录必须不存在;遇到已有输出或静默运行,先检查具体单元和报告,不重启重放。 diff --git a/paper_atlas/audits/source_identity.py b/paper_atlas/audits/source_identity.py index 24be057..11c97d8 100644 --- a/paper_atlas/audits/source_identity.py +++ b/paper_atlas/audits/source_identity.py @@ -74,6 +74,74 @@ def load_inputs(files): return records, inputs +def load_mm_program_inputs(report_path): + """Read every MM observation without turning a program into import records. + + Reconcile the complete output against its frozen raw source before any DB + connection. The synthetic unknown status is an audit placeholder only. + """ + from paper_atlas.collectors import mm2026 + + report_path = Path(report_path).absolute() + raw_report = mm2026.safe_read(report_path, 256*1024) + report = json.loads(raw_report) + if (report.get('source') != 'mm_program' or report.get('records') != 2143 + or report.get('program_observations') != 4326 or len(report.get('files', [])) != 28 + or report.get('source_body_sha256') != mm2026.BODY_SHA + or report.get('source_manifest_sha256') != mm2026.MANIFEST_SHA + or report.get('database_written') is not False or report.get('approved_for_import') != 0): + raise ValueError('Unreviewed MM observation report') + root = report_path.parent + raw_source = mm2026.safe_read(root/'evidence/program-data.json', 12*1024**2) + raw_receipts = mm2026.safe_read(root/'evidence/manifest.json', 128*1024) + if digest(raw_source) != mm2026.BODY_SHA or digest(raw_receipts) != mm2026.MANIFEST_SHA: + raise ValueError('MM raw evidence changed') + expected_rows, _ = mm2026.parse_program(raw_source) + expected = {r['paper_id']:r for r in expected_rows} + records, inputs, seen, tracks = [], [], set(), set() + for item in report['files']: + track = item.get('track_literal') + if (track not in mm2026.TRACK_COUNTS or track in tracks + or item.get('format') != 'mm_program_observation_v1_not_ingestion_record' + or item.get('approved_for_import') != 0): + raise ValueError('Duplicate or unreviewed MM track format') + tracks.add(track) + # Stored absolute paths are provenance, not instructions to read elsewhere. + path = root/Path(item['file']).name + if path.suffix != '.jsonl': + raise ValueError('Invalid MM observation filename') + raw = mm2026.safe_read(path, 16*1024**2) + lines = raw.splitlines(keepends=True) + if digest(raw) != item.get('file_sha256') or len(lines) != item.get('count') or len(lines) != mm2026.TRACK_COUNTS[track]: + raise ValueError('MM observation file count/hash mismatch') + for number,line in enumerate(lines, 1): + row = json.loads(line) + pid = row.get('paper_id') + if not isinstance(pid,str) or pid in seen or row != expected.get(pid) or row['track'] != track: + raise ValueError('MM observation differs from frozen source or repeats an ID') + seen.add(pid) + records.append({'source':'mm_program','external_id':'2026/'+pid, + 'title':row['title'],'authors':row['authors'],'venue':'MM','year':2026,'track':track, + 'status':'unknown','identifiers':{},'file':str(path),'line':number, + 'raw_line_sha256':digest(line),'source_record_hits':[], + 'identifier_hits':[],'title_candidate_ids':[], + 'input_format':'mm_program_observation_v1_not_ingestion_record', + 'program_id':pid,'program_contexts':row['contexts'], + 'source_observed_status':row['observed_status'], + 'missing_source_fields':row['missing_fields'], + 'publication_identity_verified':False,'final_decision_verified':False, + 'contribution_type_verified':False}) + inputs.append({'file':str(path),'file_sha256':digest(raw),'count':len(lines), + 'manifest_sha256':digest(raw_report),'manifest':{ + 'source':'mm_program','venue':'MM','year':2026,'track':track,'status':'unknown', + 'format':'mm_program_observation_v1_not_ingestion_record', + 'scope':'Read-only lookup input only; unknown is not a source decision.', + 'source_report':str(report_path),'source_body_sha256':digest(raw_source)}}) + if seen != expected.keys() or tracks != mm2026.TRACK_COUNTS.keys(): + raise ValueError('Incomplete MM observation input; all28tracks required') + return records, inputs + + def _lookup(connection, values, columns, lateral): """Bound scalar VALUES allow the existing text codec to encode each key.""" width = len(columns) @@ -187,9 +255,11 @@ def _atomic_json(path, value): temporary.replace(path) -def audit(files, out, *, pg_service, pg_schema): +def audit(files, out, *, pg_service, pg_schema, mm_program_report=None): # Reject input corruption before opening a database connection or reserving output. - records, inputs = load_inputs(files) + if bool(files) == bool(mm_program_report): + raise ValueError('Choose source files or one MM program report') + records, inputs = load_mm_program_inputs(mm_program_report) if mm_program_report else load_inputs(files) out = Path(out) out.mkdir(parents=True, exist_ok=False) (out / 'audit.lock').write_text(timestamp() + '\n') @@ -229,6 +299,7 @@ def audit(files, out, *, pg_service, pg_schema): 'Name agreement does not prove person or scientific-work identity.', 'Bound identifiers can be incorrect; review titles, authors and original evidence.', 'Source/history limits are explicit in compressed entity packets.', + 'Program observations and unknown audit status do not establish contribution type, publication or a final decision.', 'Not a final whole-database deduplication or coverage audit.']} _atomic_json(out / 'postgresql-preflight.json', report) return report @@ -243,10 +314,14 @@ def audit(files, out, *, pg_service, pg_schema): def main(argv=None, *, prog=None): parser = argparse.ArgumentParser(prog=prog, description=__doc__) - parser.add_argument('--files', nargs='+', required=True, type=Path) + inputs = parser.add_mutually_exclusive_group(required=True) + inputs.add_argument('--files', nargs='+', type=Path) + inputs.add_argument('--mm-program-report', type=Path, + help='Read all frozen MM program observations directly; never generate an import queue') parser.add_argument('--pg-service', required=True) parser.add_argument('--pg-schema', required=True) parser.add_argument('--out', required=True, type=Path) args = parser.parse_args(argv) - report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema) + report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema, + mm_program_report=args.mm_program_report) print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False)) diff --git a/tests/test_mm_source_identity.py b/tests/test_mm_source_identity.py new file mode 100644 index 0000000..1708547 --- /dev/null +++ b/tests/test_mm_source_identity.py @@ -0,0 +1,101 @@ +import gzip +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from paper_atlas.collectors import mm2026 +from paper_atlas.audits.source_identity import audit, load_inputs, load_mm_program_inputs + + +class MMProgramIdentityInputTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes())) + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + src = self.root/'source' + for name,body in self.fixture['files'].items(): + path=src/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode()) + mm2026.collect(self.root/'program', snapshots=src) + self.report_path = self.root/'program/collection-report.json' + + def change_report(self, change): + report=json.loads(self.report_path.read_text());change(report) + self.report_path.write_text(json.dumps(report)) + + def assert_rejected_before_database(self): + with patch('paper_atlas.audits.source_identity.ReadConnection') as connection: + with self.assertRaises(ValueError): + audit(None,self.root/'audit',pg_service='unused',pg_schema='research',mm_program_report=self.report_path) + connection.assert_not_called() + self.assertFalse((self.root/'audit').exists()) + + def test_all2143_programs_are_unknown_readonly_inputs_not_import_records(self): + rows,inputs=load_mm_program_inputs(self.report_path) + self.assertEqual(len(rows),2143);self.assertEqual(len(inputs),28) + self.assertEqual(sum(len(r['program_contexts']) for r in rows),4326) + actual={r['program_id']:r for r in rows} + for gold in self.fixture['independent_inventory']['inventory']: + r=actual[gold['paper_id']] + self.assertEqual((r['title'],r['authors'],r['track']), + (gold['title'],gold['authors'],gold['track_literal'])) + self.assertEqual(r['external_id'],'2026/'+gold['paper_id']) + self.assertEqual(r['status'],'unknown');self.assertEqual(r['identifiers'],{}) + self.assertEqual(r['source_observed_status'],'listed_in_official_program') + self.assertFalse(r['publication_identity_verified']);self.assertFalse(r['final_decision_verified']) + self.assertFalse(r['contribution_type_verified']) + self.assertEqual([c['source_pointer'] for c in r['program_contexts']],gold['locations']) + self.assertEqual({r['program_id'] for r in rows if not r['authors']},{'mfp4971','mfp7801'}) + self.assertTrue(all('unknown is not a source decision' in i['manifest']['scope'] for i in inputs)) + self.assertFalse(any(self.root.rglob('*.manifest.json'))) + + def test_untrusted_absolute_file_paths_are_not_followed(self): + self.change_report(lambda r:[f.update(file='/untrusted/'+Path(f['file']).name) for f in r['files']]) + rows,inputs=load_mm_program_inputs(self.report_path) + self.assertEqual(len(rows),2143) + self.assertTrue(all(Path(i['file']).parent==self.report_path.parent for i in inputs)) + + def test_program_files_are_not_silently_accepted_by_regular_ingestion_input_loader(self): + report=json.loads(self.report_path.read_text()) + with self.assertRaises(FileNotFoundError):load_inputs([report['files'][0]['file']]) + + def test_corrupt_jsonl_fails_before_connection_or_output(self): + report=json.loads(self.report_path.read_text());path=Path(report['files'][0]['file']) + path.write_bytes(path.read_bytes()+b' ') + self.assert_rejected_before_database() + + def test_rehashed_forged_identifiers_decisions_and_contexts_rejected(self): + original=self.report_path.read_bytes();report=json.loads(original);path=Path(report['files'][0]['file']);raw=path.read_bytes() + for field,value in [('identifiers',{'openreview':'invented'}),('status','accepted'),('contexts',[])]: + path.write_bytes(raw);self.report_path.write_bytes(original) + lines=raw.splitlines();row=json.loads(lines[0]);row[field]=value;lines[0]=json.dumps(row).encode() + changed=b'\n'.join(lines)+b'\n';path.write_bytes(changed) + self.change_report(lambda r:r['files'][0].update(file_sha256=hashlib.sha256(changed).hexdigest())) + with self.subTest(field=field):self.assert_rejected_before_database() + + def test_missing_or_repeated_track_rejected(self): + original=self.report_path.read_bytes() + for kind in ('missing','repeated'): + self.report_path.write_bytes(original) + if kind=='missing':self.change_report(lambda r:r['files'].pop()) + else:self.change_report(lambda r:r['files'].__setitem__(1,r['files'][0])) + with self.subTest(kind=kind):self.assert_rejected_before_database() + + def test_raw_source_corruption_or_symlink_rejected(self): + path=self.report_path.parent/'evidence/program-data.json';raw=path.read_bytes() + path.write_bytes(raw+b' ');self.assert_rejected_before_database() + path.write_bytes(raw);saved=path.with_name('original.json');path.rename(saved);path.symlink_to(saved) + self.assert_rejected_before_database() + + def test_mixed_input_modes_rejected_before_connection(self): + with patch('paper_atlas.audits.source_identity.ReadConnection') as connection: + for files,program in [([],None),([Path('unused')],self.report_path)]: + with self.assertRaisesRegex(ValueError,'Choose source files'): + audit(files,self.root/'out',pg_service='unused',pg_schema='research',mm_program_report=program) + connection.assert_not_called() -- 2.52.0