173 lines
11 KiB
Python
173 lines
11 KiB
Python
"""Reconcile the 2020 program with independently identified official papers."""
|
|
import argparse
|
|
from collections import Counter, defaultdict
|
|
import copy
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import tempfile
|
|
|
|
from paper_atlas.audits.iclr import load
|
|
from paper_atlas.db.store import author_keys, normalized
|
|
from paper_atlas.db.submissions import timestamp
|
|
|
|
|
|
LEGACY_URL = 'https://iclr.cc/virtual_2020/papers.json'
|
|
LEGACY_SHA256 = '2405f2da6e2641fa600762f7d7cdf7c9d27f4d0ae65799168f3aee46036d30d2'
|
|
LEGACY_BYTES = 391687
|
|
EXPECTED = 687
|
|
# This one paper has three documented titles across the pinned official legacy
|
|
# list, current official program, and public submission snapshot. The exception
|
|
# also requires independent public-author agreement; it is not fuzzy matching.
|
|
REVIEWED_TITLE_VARIANT = {
|
|
'forum': 'SJgzLkBKPB',
|
|
'program': 'Explain Your Move: Understanding Agent Actions Using Salient and Relevant Feature Attribution',
|
|
'legacy': 'Explain Your Move: Understanding Agent Actions Using Focused Feature Saliency',
|
|
'snapshot': 'Explain Your Move: Understanding Agent Actions Using Specific and Relevant Feature Attribution',
|
|
}
|
|
|
|
|
|
def reconcile(program, snapshot, legacy):
|
|
by_number = defaultdict(list)
|
|
for row in snapshot.values():
|
|
by_number[row['metadata'].get('source_submission_number')].append(row)
|
|
legacy_by_forum = {}
|
|
for note in legacy:
|
|
forum = note.get('forum')
|
|
content = note.get('content') or {}
|
|
if not isinstance(forum, str) or not re.fullmatch(r'[A-Za-z0-9_-]+', forum):
|
|
raise ValueError('Invalid legacy forum identity')
|
|
if note.get('id') != forum or content.get('iclr_id') != forum or forum in legacy_by_forum:
|
|
raise ValueError('Conflicting or repeated official legacy forum')
|
|
if not isinstance(content.get('title'), str) or not content['title'].strip() or not isinstance(content.get('authors'), list):
|
|
raise ValueError('Legacy paper lacks title or public authors')
|
|
legacy_by_forum[forum] = content
|
|
if len(program) != len(legacy_by_forum):
|
|
raise ValueError('Official program and legacy inventories differ')
|
|
title_counts = Counter(normalized(row['title']) for row in legacy_by_forum.values())
|
|
enriched = []
|
|
assigned = set()
|
|
numbers = set()
|
|
evidence_counts = Counter()
|
|
for original in program:
|
|
if any(original.get(k) != v for k, v in {'source': 'iclr_program', 'venue': 'ICLR', 'year': 2020, 'track': 'main', 'status': 'accepted'}.items()):
|
|
raise ValueError('Program record is outside the accepted 2020 main track')
|
|
metadata = original.get('metadata') or {}
|
|
number = metadata.get('program_source_id')
|
|
if metadata.get('program_identity_type') != 'submission' or type(number) is not int or number <= 0 or number in numbers:
|
|
raise ValueError('Missing or repeated program submission number')
|
|
numbers.add(number)
|
|
candidates = by_number[number]
|
|
if len(candidates) != 1:
|
|
raise ValueError('Submission number is absent or ambiguous in the public snapshot')
|
|
submission = candidates[0]
|
|
forum = submission['identifiers']['openreview']
|
|
if submission['status'] != 'accepted' or forum not in legacy_by_forum or forum in assigned:
|
|
raise ValueError('Snapshot acceptance and official forum inventory do not agree')
|
|
content = legacy_by_forum[forum]
|
|
shared_legacy = len(author_keys(original['authors']) & author_keys(content['authors']))
|
|
shared_snapshot = len(author_keys(original['authors']) & author_keys(submission['authors']))
|
|
checks = {'legacy_title': normalized(original['title']) == normalized(content['title']),
|
|
'snapshot_title': normalized(original['title']) == normalized(submission['title']),
|
|
'snapshot_abstract': bool(original.get('abstract')) and original['abstract'].strip() == (submission.get('abstract') or '').strip(),
|
|
'two_snapshot_authors': shared_snapshot >= 2}
|
|
independent_snapshot = sum(checks[k] for k in ('snapshot_title', 'snapshot_abstract', 'two_snapshot_authors')) >= 2
|
|
checks['legacy_title_unique'] = title_counts[normalized(content['title'])] == 1
|
|
independent_legacy = checks['legacy_title'] and checks['legacy_title_unique'] and shared_legacy >= 1
|
|
reviewed_variant = (forum == REVIEWED_TITLE_VARIANT['forum']
|
|
and normalized(original['title']) == normalized(REVIEWED_TITLE_VARIANT['program'])
|
|
and normalized(content['title']) == normalized(REVIEWED_TITLE_VARIANT['legacy'])
|
|
and normalized(submission['title']) == normalized(REVIEWED_TITLE_VARIANT['snapshot'])
|
|
and shared_legacy >= 2 and shared_snapshot >= 2)
|
|
if not (independent_snapshot or independent_legacy or reviewed_variant):
|
|
raise ValueError('Submission number alone cannot establish forum identity: ' + original['external_id'])
|
|
own = original.get('identifiers') or {}
|
|
if own.get('openreview') not in (None, forum):
|
|
raise ValueError('Existing program forum conflicts with independent evidence')
|
|
row = copy.deepcopy(original)
|
|
row['identifiers'] = {**own, 'openreview': forum}
|
|
row['url'] = 'https://iclr.cc/virtual_2020/poster_' + forum + '.html'
|
|
row['metadata'].update(identity_state='forum', program_submission_number_verified=True,
|
|
program_identity_verification={'method': 'official_legacy_and_public_submission_crosscheck',
|
|
'legacy_url': LEGACY_URL, 'legacy_sha256': LEGACY_SHA256,
|
|
'original_program_url': original['url'], 'legacy_forum': forum,
|
|
'checks': checks, 'shared_legacy_authors': shared_legacy,
|
|
'shared_snapshot_authors': shared_snapshot, 'reviewed_title_variant': reviewed_variant})
|
|
enriched.append(row)
|
|
assigned.add(forum)
|
|
evidence_counts.update({'independent_snapshot': int(independent_snapshot),
|
|
'independent_legacy': int(independent_legacy),
|
|
'reviewed_title_variant': int(reviewed_variant)})
|
|
if assigned != legacy_by_forum.keys():
|
|
raise ValueError('Unmatched official legacy forum identities remain')
|
|
if assigned != {forum for forum, row in snapshot.items() if row['status'] == 'accepted'}:
|
|
raise ValueError('Accepted snapshot forum inventory differs from the official program')
|
|
return enriched, dict(evidence_counts)
|
|
|
|
|
|
def audit(program_path, snapshot_path, legacy_path, out):
|
|
program_path, snapshot_path, legacy_path = map(lambda p: Path(p).resolve(), (program_path, snapshot_path, legacy_path))
|
|
raw = legacy_path.read_bytes()
|
|
if len(raw) != LEGACY_BYTES or hashlib.sha256(raw).hexdigest() != LEGACY_SHA256:
|
|
raise ValueError('Official legacy snapshot checksum mismatch')
|
|
legacy = json.loads(raw)
|
|
if not isinstance(legacy, list) or len(legacy) != EXPECTED:
|
|
raise ValueError('Official legacy count mismatch')
|
|
manifest = json.loads(program_path.with_suffix('.manifest.json').read_text())
|
|
if manifest.get('parser_version', 0) < 2 or any(manifest.get(k) != v for k, v in {'source': 'iclr_program', 'venue': 'ICLR', 'year': 2020, 'track': 'main', 'count': EXPECTED}.items()):
|
|
raise ValueError('Invalid collected program inventory')
|
|
program_bytes = program_path.read_bytes()
|
|
program = [json.loads(line) for line in program_bytes.splitlines()]
|
|
if len(program) != EXPECTED or len({r['external_id'] for r in program}) != EXPECTED:
|
|
raise ValueError('Program count or source identity mismatch')
|
|
snapshot, snapshot_manifest, snapshot_sha = load(snapshot_path, 'openreview_snapshot', 2020)
|
|
rows, checks = reconcile(program, snapshot, legacy)
|
|
out = Path(out).resolve()
|
|
if out / 'iclr-program-2020-main.jsonl' in (program_path, snapshot_path, legacy_path):
|
|
raise ValueError('Output must preserve the original input files')
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
with (out / 'iclr-legacy.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (out / 'iclr-2020-identity-audit.json').exists():
|
|
raise ValueError('Completed identity audit exists; choose a new output directory')
|
|
with tempfile.TemporaryDirectory(prefix='.iclr-legacy-', dir=out) as temporary:
|
|
stage = Path(temporary)
|
|
path = stage / 'iclr-program-2020-main.jsonl'
|
|
path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in rows))
|
|
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
manifest.update(parser_version=3, file=str(out / path.name), file_sha256=digest,
|
|
missing_forum_count=0, legacy_identity_sha256=LEGACY_SHA256)
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
|
|
backfill = {'files': [str(out / path.name)], 'file_sha256': {str(out / path.name): digest}}
|
|
snapshots = {'files': [str(snapshot_path)], 'file_sha256': {str(snapshot_path): snapshot_sha}}
|
|
result = {'year': 2020, 'checked_at': timestamp(), 'program_records': len(rows),
|
|
'snapshot_records': len(snapshot), 'snapshot_status_counts': snapshot_manifest['submission_status_counts'],
|
|
'legacy_url': LEGACY_URL, 'legacy_sha256': LEGACY_SHA256,
|
|
'program_input_sha256': hashlib.sha256(program_bytes).hexdigest(),
|
|
'corroboration_counts': checks, 'backfill_queue': backfill, 'snapshot_files': snapshots,
|
|
'note': 'No database writes. Backfill existing program identities before importing the snapshot; source IDs, titles and abstracts are preserved.'}
|
|
(stage / 'iclr-2020-backfill-queue.json').write_text(json.dumps(backfill, indent=2) + '\n')
|
|
(stage / 'iclr-2020-snapshot-queue.json').write_text(json.dumps(snapshots, indent=2) + '\n')
|
|
report = stage / 'iclr-2020-identity-audit.json'
|
|
report.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
|
|
for child in sorted(stage.iterdir(), key=lambda child: child == report):
|
|
child.replace(out / child.name)
|
|
print(json.dumps(result), flush=True)
|
|
return result
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog)
|
|
parser.add_argument('--program', required=True)
|
|
parser.add_argument('--snapshot', required=True)
|
|
parser.add_argument('--legacy', required=True)
|
|
parser.add_argument('--out', required=True)
|
|
args = parser.parse_args(argv)
|
|
audit(args.program, args.snapshot, args.legacy, args.out)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|