Files
paper-atlas/paper_atlas/audits/iclr_legacy.py
2026-11-06 21:19:10 +08:00

173 lines
11 KiB
Python

"""Reconcile the 2020 program with independently identified official papers."""
import argparse
from collections import Counter, defaultdict
import copy
import fcntl
import hashlib
import json
from pathlib import Path
import re
import tempfile
from paper_atlas.audits.iclr import load
from paper_atlas.db.store import author_keys, normalized
from paper_atlas.db.submissions import timestamp
LEGACY_URL = 'https://iclr.cc/virtual_2020/papers.json'
LEGACY_SHA256 = '2405f2da6e2641fa600762f7d7cdf7c9d27f4d0ae65799168f3aee46036d30d2'
LEGACY_BYTES = 391687
EXPECTED = 687
# This one paper has three documented titles across the pinned official legacy
# list, current official program, and public submission snapshot. The exception
# also requires independent public-author agreement; it is not fuzzy matching.
REVIEWED_TITLE_VARIANT = {
'forum': 'SJgzLkBKPB',
'program': 'Explain Your Move: Understanding Agent Actions Using Salient and Relevant Feature Attribution',
'legacy': 'Explain Your Move: Understanding Agent Actions Using Focused Feature Saliency',
'snapshot': 'Explain Your Move: Understanding Agent Actions Using Specific and Relevant Feature Attribution',
}
def reconcile(program, snapshot, legacy):
by_number = defaultdict(list)
for row in snapshot.values():
by_number[row['metadata'].get('source_submission_number')].append(row)
legacy_by_forum = {}
for note in legacy:
forum = note.get('forum')
content = note.get('content') or {}
if not isinstance(forum, str) or not re.fullmatch(r'[A-Za-z0-9_-]+', forum):
raise ValueError('Invalid legacy forum identity')
if note.get('id') != forum or content.get('iclr_id') != forum or forum in legacy_by_forum:
raise ValueError('Conflicting or repeated official legacy forum')
if not isinstance(content.get('title'), str) or not content['title'].strip() or not isinstance(content.get('authors'), list):
raise ValueError('Legacy paper lacks title or public authors')
legacy_by_forum[forum] = content
if len(program) != len(legacy_by_forum):
raise ValueError('Official program and legacy inventories differ')
title_counts = Counter(normalized(row['title']) for row in legacy_by_forum.values())
enriched = []
assigned = set()
numbers = set()
evidence_counts = Counter()
for original in program:
if any(original.get(k) != v for k, v in {'source': 'iclr_program', 'venue': 'ICLR', 'year': 2020, 'track': 'main', 'status': 'accepted'}.items()):
raise ValueError('Program record is outside the accepted 2020 main track')
metadata = original.get('metadata') or {}
number = metadata.get('program_source_id')
if metadata.get('program_identity_type') != 'submission' or type(number) is not int or number <= 0 or number in numbers:
raise ValueError('Missing or repeated program submission number')
numbers.add(number)
candidates = by_number[number]
if len(candidates) != 1:
raise ValueError('Submission number is absent or ambiguous in the public snapshot')
submission = candidates[0]
forum = submission['identifiers']['openreview']
if submission['status'] != 'accepted' or forum not in legacy_by_forum or forum in assigned:
raise ValueError('Snapshot acceptance and official forum inventory do not agree')
content = legacy_by_forum[forum]
shared_legacy = len(author_keys(original['authors']) & author_keys(content['authors']))
shared_snapshot = len(author_keys(original['authors']) & author_keys(submission['authors']))
checks = {'legacy_title': normalized(original['title']) == normalized(content['title']),
'snapshot_title': normalized(original['title']) == normalized(submission['title']),
'snapshot_abstract': bool(original.get('abstract')) and original['abstract'].strip() == (submission.get('abstract') or '').strip(),
'two_snapshot_authors': shared_snapshot >= 2}
independent_snapshot = sum(checks[k] for k in ('snapshot_title', 'snapshot_abstract', 'two_snapshot_authors')) >= 2
checks['legacy_title_unique'] = title_counts[normalized(content['title'])] == 1
independent_legacy = checks['legacy_title'] and checks['legacy_title_unique'] and shared_legacy >= 1
reviewed_variant = (forum == REVIEWED_TITLE_VARIANT['forum']
and normalized(original['title']) == normalized(REVIEWED_TITLE_VARIANT['program'])
and normalized(content['title']) == normalized(REVIEWED_TITLE_VARIANT['legacy'])
and normalized(submission['title']) == normalized(REVIEWED_TITLE_VARIANT['snapshot'])
and shared_legacy >= 2 and shared_snapshot >= 2)
if not (independent_snapshot or independent_legacy or reviewed_variant):
raise ValueError('Submission number alone cannot establish forum identity: ' + original['external_id'])
own = original.get('identifiers') or {}
if own.get('openreview') not in (None, forum):
raise ValueError('Existing program forum conflicts with independent evidence')
row = copy.deepcopy(original)
row['identifiers'] = {**own, 'openreview': forum}
row['url'] = 'https://iclr.cc/virtual_2020/poster_' + forum + '.html'
row['metadata'].update(identity_state='forum', program_submission_number_verified=True,
program_identity_verification={'method': 'official_legacy_and_public_submission_crosscheck',
'legacy_url': LEGACY_URL, 'legacy_sha256': LEGACY_SHA256,
'original_program_url': original['url'], 'legacy_forum': forum,
'checks': checks, 'shared_legacy_authors': shared_legacy,
'shared_snapshot_authors': shared_snapshot, 'reviewed_title_variant': reviewed_variant})
enriched.append(row)
assigned.add(forum)
evidence_counts.update({'independent_snapshot': int(independent_snapshot),
'independent_legacy': int(independent_legacy),
'reviewed_title_variant': int(reviewed_variant)})
if assigned != legacy_by_forum.keys():
raise ValueError('Unmatched official legacy forum identities remain')
if assigned != {forum for forum, row in snapshot.items() if row['status'] == 'accepted'}:
raise ValueError('Accepted snapshot forum inventory differs from the official program')
return enriched, dict(evidence_counts)
def audit(program_path, snapshot_path, legacy_path, out):
program_path, snapshot_path, legacy_path = map(lambda p: Path(p).resolve(), (program_path, snapshot_path, legacy_path))
raw = legacy_path.read_bytes()
if len(raw) != LEGACY_BYTES or hashlib.sha256(raw).hexdigest() != LEGACY_SHA256:
raise ValueError('Official legacy snapshot checksum mismatch')
legacy = json.loads(raw)
if not isinstance(legacy, list) or len(legacy) != EXPECTED:
raise ValueError('Official legacy count mismatch')
manifest = json.loads(program_path.with_suffix('.manifest.json').read_text())
if manifest.get('parser_version', 0) < 2 or any(manifest.get(k) != v for k, v in {'source': 'iclr_program', 'venue': 'ICLR', 'year': 2020, 'track': 'main', 'count': EXPECTED}.items()):
raise ValueError('Invalid collected program inventory')
program_bytes = program_path.read_bytes()
program = [json.loads(line) for line in program_bytes.splitlines()]
if len(program) != EXPECTED or len({r['external_id'] for r in program}) != EXPECTED:
raise ValueError('Program count or source identity mismatch')
snapshot, snapshot_manifest, snapshot_sha = load(snapshot_path, 'openreview_snapshot', 2020)
rows, checks = reconcile(program, snapshot, legacy)
out = Path(out).resolve()
if out / 'iclr-program-2020-main.jsonl' in (program_path, snapshot_path, legacy_path):
raise ValueError('Output must preserve the original input files')
out.mkdir(parents=True, exist_ok=True)
with (out / 'iclr-legacy.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (out / 'iclr-2020-identity-audit.json').exists():
raise ValueError('Completed identity audit exists; choose a new output directory')
with tempfile.TemporaryDirectory(prefix='.iclr-legacy-', dir=out) as temporary:
stage = Path(temporary)
path = stage / 'iclr-program-2020-main.jsonl'
path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in rows))
digest = hashlib.sha256(path.read_bytes()).hexdigest()
manifest.update(parser_version=3, file=str(out / path.name), file_sha256=digest,
missing_forum_count=0, legacy_identity_sha256=LEGACY_SHA256)
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
backfill = {'files': [str(out / path.name)], 'file_sha256': {str(out / path.name): digest}}
snapshots = {'files': [str(snapshot_path)], 'file_sha256': {str(snapshot_path): snapshot_sha}}
result = {'year': 2020, 'checked_at': timestamp(), 'program_records': len(rows),
'snapshot_records': len(snapshot), 'snapshot_status_counts': snapshot_manifest['submission_status_counts'],
'legacy_url': LEGACY_URL, 'legacy_sha256': LEGACY_SHA256,
'program_input_sha256': hashlib.sha256(program_bytes).hexdigest(),
'corroboration_counts': checks, 'backfill_queue': backfill, 'snapshot_files': snapshots,
'note': 'No database writes. Backfill existing program identities before importing the snapshot; source IDs, titles and abstracts are preserved.'}
(stage / 'iclr-2020-backfill-queue.json').write_text(json.dumps(backfill, indent=2) + '\n')
(stage / 'iclr-2020-snapshot-queue.json').write_text(json.dumps(snapshots, indent=2) + '\n')
report = stage / 'iclr-2020-identity-audit.json'
report.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
for child in sorted(stage.iterdir(), key=lambda child: child == report):
child.replace(out / child.name)
print(json.dumps(result), flush=True)
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--program', required=True)
parser.add_argument('--snapshot', required=True)
parser.add_argument('--legacy', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
audit(args.program, args.snapshot, args.legacy, args.out)
if __name__ == '__main__':
main()