直接审计 MM 节目身份与既往投稿历史 #66

Merged
Zayden-Jung merged 1 commits from codex/mm-identity into main 2026-11-17 14:57:43 +00:00
4 changed files with 205 additions and 4 deletions
+7
View File
@@ -66,6 +66,13 @@ jobs:
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
if service_file.is_file():
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True)
mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text())
assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28
assert mm_preflight['transaction_read_only'] and not mm_preflight['database_written'] and mm_preflight['approved_for_import'] == 0
assert sum(len(r['program_contexts']) for r in mm_preflight['entries']) == 4326
assert all(r['status'] == 'unknown' and not r['identifiers'] and not r['contribution_type_verified'] and not r['publication_identity_verified'] and not r['final_decision_verified'] for r in mm_preflight['entries'])
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
+18
View File
@@ -90,3 +90,21 @@ paper-atlas collect mm2026-program \
这些文件的格式为 `mm_program_observation_v1_not_ingestion_record`,所有项 `approved_for_import=false`。节目ID仅为 `mm_program:program_id` 来源定位符,track标签使用 `mm_program:track`;不建立科学作品、原始投稿ID或最终决定,不推断 DOI/arXiv/OpenReview 标识、论文类型、拒稿或撤稿。行业报告、艺术、workshop及其他来源贡献均保留,尚需逐轨核对贡献类型与正式出版依据。
本批尚未形成生产导入队列。后续做贡献类型、作品/出版身份及既往投稿审查,再生成经批准的增量队列。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。
## 直接审计节目身份与既往历史
全量节目观察可以直接进入只读候选审计,无需伪造成录用论文队列:
```sh
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf \
paper-atlas audit source-identities \
--mm-program-report /data/paper-index/collected/mm2026-native-v1/collection-report.json \
--pg-service foundry-research --pg-schema research \
--out /data/paper-index/collected/mm2026-preflight-v1
```
`--mm-program-report` 与常规 `--files` 互斥。审计先核对冻结原文、全部28轨的逐文件哈希/数量,以及每个节目ID的完整字段和展示上下文;缺轨、重复ID、篡改后的标识/决定或上下文均在数据库连接前拒绝。报告中记录的绝对文件路径仅作来源证据,实际读取报告同目录中的对应文件。
候选审计中的 `status=unknown` 仅为内部查询占位符,不代表原始提交的最终状态。节目ID以 `mm_program` 来源键查询,不转成全局 DOI、arXiv 或 OpenReview 标识;保留原始 `listed_in_official_program` 观察与全部展示位置。所有贡献类型、出版身份、最终决定核验标志均为false;缺作者的两项也保留并标记。
查询使用 PostgreSQL 只读一致性快照,候选作品的原始来源、标识、历次投稿和逐次决定压缩保存并记录哈希;截断会显式标记。标题/作者候选匹配不是作品身份批准,没有匹配也不证明是新论文。既往拒稿和后续录用不能互相覆盖,评分或评审推荐不能充当最终决定。输出目录必须不存在;遇到已有输出或静默运行,先检查具体单元和报告,不重启重放。
+79 -4
View File
@@ -74,6 +74,74 @@ def load_inputs(files):
return records, inputs
def load_mm_program_inputs(report_path):
"""Read every MM observation without turning a program into import records.
Reconcile the complete output against its frozen raw source before any DB
connection. The synthetic unknown status is an audit placeholder only.
"""
from paper_atlas.collectors import mm2026
report_path = Path(report_path).absolute()
raw_report = mm2026.safe_read(report_path, 256*1024)
report = json.loads(raw_report)
if (report.get('source') != 'mm_program' or report.get('records') != 2143
or report.get('program_observations') != 4326 or len(report.get('files', [])) != 28
or report.get('source_body_sha256') != mm2026.BODY_SHA
or report.get('source_manifest_sha256') != mm2026.MANIFEST_SHA
or report.get('database_written') is not False or report.get('approved_for_import') != 0):
raise ValueError('Unreviewed MM observation report')
root = report_path.parent
raw_source = mm2026.safe_read(root/'evidence/program-data.json', 12*1024**2)
raw_receipts = mm2026.safe_read(root/'evidence/manifest.json', 128*1024)
if digest(raw_source) != mm2026.BODY_SHA or digest(raw_receipts) != mm2026.MANIFEST_SHA:
raise ValueError('MM raw evidence changed')
expected_rows, _ = mm2026.parse_program(raw_source)
expected = {r['paper_id']:r for r in expected_rows}
records, inputs, seen, tracks = [], [], set(), set()
for item in report['files']:
track = item.get('track_literal')
if (track not in mm2026.TRACK_COUNTS or track in tracks
or item.get('format') != 'mm_program_observation_v1_not_ingestion_record'
or item.get('approved_for_import') != 0):
raise ValueError('Duplicate or unreviewed MM track format')
tracks.add(track)
# Stored absolute paths are provenance, not instructions to read elsewhere.
path = root/Path(item['file']).name
if path.suffix != '.jsonl':
raise ValueError('Invalid MM observation filename')
raw = mm2026.safe_read(path, 16*1024**2)
lines = raw.splitlines(keepends=True)
if digest(raw) != item.get('file_sha256') or len(lines) != item.get('count') or len(lines) != mm2026.TRACK_COUNTS[track]:
raise ValueError('MM observation file count/hash mismatch')
for number,line in enumerate(lines, 1):
row = json.loads(line)
pid = row.get('paper_id')
if not isinstance(pid,str) or pid in seen or row != expected.get(pid) or row['track'] != track:
raise ValueError('MM observation differs from frozen source or repeats an ID')
seen.add(pid)
records.append({'source':'mm_program','external_id':'2026/'+pid,
'title':row['title'],'authors':row['authors'],'venue':'MM','year':2026,'track':track,
'status':'unknown','identifiers':{},'file':str(path),'line':number,
'raw_line_sha256':digest(line),'source_record_hits':[],
'identifier_hits':[],'title_candidate_ids':[],
'input_format':'mm_program_observation_v1_not_ingestion_record',
'program_id':pid,'program_contexts':row['contexts'],
'source_observed_status':row['observed_status'],
'missing_source_fields':row['missing_fields'],
'publication_identity_verified':False,'final_decision_verified':False,
'contribution_type_verified':False})
inputs.append({'file':str(path),'file_sha256':digest(raw),'count':len(lines),
'manifest_sha256':digest(raw_report),'manifest':{
'source':'mm_program','venue':'MM','year':2026,'track':track,'status':'unknown',
'format':'mm_program_observation_v1_not_ingestion_record',
'scope':'Read-only lookup input only; unknown is not a source decision.',
'source_report':str(report_path),'source_body_sha256':digest(raw_source)}})
if seen != expected.keys() or tracks != mm2026.TRACK_COUNTS.keys():
raise ValueError('Incomplete MM observation input; all28tracks required')
return records, inputs
def _lookup(connection, values, columns, lateral):
"""Bound scalar VALUES allow the existing text codec to encode each key."""
width = len(columns)
@@ -187,9 +255,11 @@ def _atomic_json(path, value):
temporary.replace(path)
def audit(files, out, *, pg_service, pg_schema):
def audit(files, out, *, pg_service, pg_schema, mm_program_report=None):
# Reject input corruption before opening a database connection or reserving output.
records, inputs = load_inputs(files)
if bool(files) == bool(mm_program_report):
raise ValueError('Choose source files or one MM program report')
records, inputs = load_mm_program_inputs(mm_program_report) if mm_program_report else load_inputs(files)
out = Path(out)
out.mkdir(parents=True, exist_ok=False)
(out / 'audit.lock').write_text(timestamp() + '\n')
@@ -229,6 +299,7 @@ def audit(files, out, *, pg_service, pg_schema):
'Name agreement does not prove person or scientific-work identity.',
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
'Source/history limits are explicit in compressed entity packets.',
'Program observations and unknown audit status do not establish contribution type, publication or a final decision.',
'Not a final whole-database deduplication or coverage audit.']}
_atomic_json(out / 'postgresql-preflight.json', report)
return report
@@ -243,10 +314,14 @@ def audit(files, out, *, pg_service, pg_schema):
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--files', nargs='+', required=True, type=Path)
inputs = parser.add_mutually_exclusive_group(required=True)
inputs.add_argument('--files', nargs='+', type=Path)
inputs.add_argument('--mm-program-report', type=Path,
help='Read all frozen MM program observations directly; never generate an import queue')
parser.add_argument('--pg-service', required=True)
parser.add_argument('--pg-schema', required=True)
parser.add_argument('--out', required=True, type=Path)
args = parser.parse_args(argv)
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
mm_program_report=args.mm_program_report)
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
+101
View File
@@ -0,0 +1,101 @@
import gzip
import hashlib
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from paper_atlas.collectors import mm2026
from paper_atlas.audits.source_identity import audit, load_inputs, load_mm_program_inputs
class MMProgramIdentityInputTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
src = self.root/'source'
for name,body in self.fixture['files'].items():
path=src/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode())
mm2026.collect(self.root/'program', snapshots=src)
self.report_path = self.root/'program/collection-report.json'
def change_report(self, change):
report=json.loads(self.report_path.read_text());change(report)
self.report_path.write_text(json.dumps(report))
def assert_rejected_before_database(self):
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
with self.assertRaises(ValueError):
audit(None,self.root/'audit',pg_service='unused',pg_schema='research',mm_program_report=self.report_path)
connection.assert_not_called()
self.assertFalse((self.root/'audit').exists())
def test_all2143_programs_are_unknown_readonly_inputs_not_import_records(self):
rows,inputs=load_mm_program_inputs(self.report_path)
self.assertEqual(len(rows),2143);self.assertEqual(len(inputs),28)
self.assertEqual(sum(len(r['program_contexts']) for r in rows),4326)
actual={r['program_id']:r for r in rows}
for gold in self.fixture['independent_inventory']['inventory']:
r=actual[gold['paper_id']]
self.assertEqual((r['title'],r['authors'],r['track']),
(gold['title'],gold['authors'],gold['track_literal']))
self.assertEqual(r['external_id'],'2026/'+gold['paper_id'])
self.assertEqual(r['status'],'unknown');self.assertEqual(r['identifiers'],{})
self.assertEqual(r['source_observed_status'],'listed_in_official_program')
self.assertFalse(r['publication_identity_verified']);self.assertFalse(r['final_decision_verified'])
self.assertFalse(r['contribution_type_verified'])
self.assertEqual([c['source_pointer'] for c in r['program_contexts']],gold['locations'])
self.assertEqual({r['program_id'] for r in rows if not r['authors']},{'mfp4971','mfp7801'})
self.assertTrue(all('unknown is not a source decision' in i['manifest']['scope'] for i in inputs))
self.assertFalse(any(self.root.rglob('*.manifest.json')))
def test_untrusted_absolute_file_paths_are_not_followed(self):
self.change_report(lambda r:[f.update(file='/untrusted/'+Path(f['file']).name) for f in r['files']])
rows,inputs=load_mm_program_inputs(self.report_path)
self.assertEqual(len(rows),2143)
self.assertTrue(all(Path(i['file']).parent==self.report_path.parent for i in inputs))
def test_program_files_are_not_silently_accepted_by_regular_ingestion_input_loader(self):
report=json.loads(self.report_path.read_text())
with self.assertRaises(FileNotFoundError):load_inputs([report['files'][0]['file']])
def test_corrupt_jsonl_fails_before_connection_or_output(self):
report=json.loads(self.report_path.read_text());path=Path(report['files'][0]['file'])
path.write_bytes(path.read_bytes()+b' ')
self.assert_rejected_before_database()
def test_rehashed_forged_identifiers_decisions_and_contexts_rejected(self):
original=self.report_path.read_bytes();report=json.loads(original);path=Path(report['files'][0]['file']);raw=path.read_bytes()
for field,value in [('identifiers',{'openreview':'invented'}),('status','accepted'),('contexts',[])]:
path.write_bytes(raw);self.report_path.write_bytes(original)
lines=raw.splitlines();row=json.loads(lines[0]);row[field]=value;lines[0]=json.dumps(row).encode()
changed=b'\n'.join(lines)+b'\n';path.write_bytes(changed)
self.change_report(lambda r:r['files'][0].update(file_sha256=hashlib.sha256(changed).hexdigest()))
with self.subTest(field=field):self.assert_rejected_before_database()
def test_missing_or_repeated_track_rejected(self):
original=self.report_path.read_bytes()
for kind in ('missing','repeated'):
self.report_path.write_bytes(original)
if kind=='missing':self.change_report(lambda r:r['files'].pop())
else:self.change_report(lambda r:r['files'].__setitem__(1,r['files'][0]))
with self.subTest(kind=kind):self.assert_rejected_before_database()
def test_raw_source_corruption_or_symlink_rejected(self):
path=self.report_path.parent/'evidence/program-data.json';raw=path.read_bytes()
path.write_bytes(raw+b' ');self.assert_rejected_before_database()
path.write_bytes(raw);saved=path.with_name('original.json');path.rename(saved);path.symlink_to(saved)
self.assert_rejected_before_database()
def test_mixed_input_modes_rejected_before_connection(self):
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
for files,program in [([],None),([Path('unused')],self.report_path)]:
with self.assertRaisesRegex(ValueError,'Choose source files'):
audit(files,self.root/'out',pg_service='unused',pg_schema='research',mm_program_report=program)
connection.assert_not_called()