Files
paper-atlas/paper_atlas/collectors/icml_public_archive.py
codex-bot ac08edf572
Metadata integrity checks / metadata (push) Successful in 35s
Add ICML 2026 public rejected-submission archive with platform evidence
2026-11-06 20:55:49 +08:00

126 lines
6.9 KiB
Python

"""ICML 2026 public submission roots with explicit Rejected_Submission IDs."""
import argparse
import fcntl
import hashlib
import json
from pathlib import Path
import re
import shutil
import tempfile
from paper_atlas.collectors.openreview import date_ms, invitations, public, value
from paper_atlas.collectors.public_openreview import download
from paper_atlas.db.submissions import timestamp
SOURCE = 'openreview_public_archive'
REVISION = '1a1b9d973a8dfbf254f740fb8100a2fcf36757ea'
AS_OF = '2026-08-08T12:03:26.000Z'
URL = 'https://raw.githubusercontent.com/HemantKumar01/ICML-2026-data/' + REVISION + '/data/rejected.json'
BLOB_SHA1 = 'cebdcddffe3eb2145d32d328db89a1626e3f85eb'
BYTES = 1600856
EXPECTED = 214
DOMAIN = 'ICML.cc/2026/Conference'
VENUEID = DOMAIN + '/Rejected_Submission'
def normalize(note):
forum = note.get('id')
if not isinstance(forum, str) or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', forum) or note.get('forum') != forum:
raise ValueError('Not a public submission root with a real forum identity')
if not public(note) or note.get('ddate'):
raise ValueError('Submission is not explicitly public and live in the archive')
if note.get('domain') != DOMAIN or DOMAIN + '/-/Submission' not in invitations(note):
raise ValueError('Submission is outside the ICML 2026 conference track')
content = note.get('content') or {}
if value(content, 'venueid') != VENUEID:
raise ValueError('Missing explicit platform Rejected_Submission classification')
title, abstract = value(content, 'title'), value(content, 'abstract')
if not isinstance(title, str) or not title.strip():
raise ValueError('Submission lacks a title')
if abstract is not None and not isinstance(abstract, str):
raise ValueError('Submission abstract must be text')
authors = value(content, 'authors', []) or []
if not isinstance(authors, list):
raise ValueError('Unexpected public author format')
authors = [a.strip() for a in authors if isinstance(a, str) and a.strip() and '@' not in a
and not re.fullmatch(r'anonymous(?: authors?)?', a.strip(), re.I)]
tags = []
for field in ('keywords', 'primary_area'):
raw = value(content, field)
terms = raw if isinstance(raw, list) else re.split(r'[,;]', raw) if field == 'keywords' and isinstance(raw, str) else [raw]
tags.extend({'namespace': SOURCE + ':' + field, 'value': term.strip()}
for term in terms if isinstance(term, str) and term.strip())
url = 'https://openreview.net/forum?id=' + forum
return {'source': SOURCE, 'external_id': 'icml/2026/' + forum, 'url': url,
'title': ' '.join(title.split()), 'abstract': abstract.strip() if abstract else None,
'authors': authors, 'identifiers': {'openreview': forum},
'submission_identifiers': [{'scheme': 'openreview', 'value': forum}],
'venue': 'ICML', 'year': 2026, 'track': 'main', 'status': 'rejected',
'raw_decision': VENUEID, 'evidence_url': url, 'observed_at': AS_OF,
'submitted_at': date_ms(note.get('cdate') or note.get('tcdate')), 'tags': tags,
'metadata': {'source_revision': REVISION, 'source_blob_sha1': BLOB_SHA1,
'source_file': URL, 'original_forum': forum, 'license': note.get('license'),
'venue': value(content, 'venue'), 'venueid': VENUEID,
'public_readers_verified': True, 'original_invitations': invitations(note),
'source_note_modified_at': date_ms(note.get('mdate')),
'evidence_kind': 'public_submission_venueid',
'original_decision_signature_verified': False,
'source_snapshot_upper_bound': AS_OF}}
def collect(out, fetcher=None):
fetcher = fetcher or download
out = Path(out)
out.mkdir(parents=True, exist_ok=True)
with (out / 'collect.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (out / 'public-openreview-inventory.json').exists():
raise ValueError('Completed inventory exists; select a new output directory')
if shutil.disk_usage(out).free < 8 * 1024**3:
raise RuntimeError('Less than 8 GiB free before collection')
data = fetcher(URL, BYTES)
blob = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest()
if len(data) != BYTES or blob != BLOB_SHA1:
raise ValueError('Public archive checksum mismatch')
archive = json.loads(data)
notes = archive['notes']
if archive.get('count') != EXPECTED or not isinstance(notes, list) or len(notes) != EXPECTED:
raise ValueError('Public archive inventory count mismatch')
records = [normalize(note) for note in notes]
if len({row['identifiers']['openreview'] for row in records}) != EXPECTED:
raise ValueError('Repeated submission forum in public archive')
records.sort(key=lambda row: row['external_id'])
with tempfile.TemporaryDirectory(prefix='.icml-public-', dir=out) as temporary:
stage = Path(temporary)
path = stage / 'public-openreview-icml-2026-main.jsonl'
path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in records))
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICML', 'year': 2026,
'track': 'main', 'count': EXPECTED, 'status': 'partial', 'list_url': URL,
'file': str(out / path.name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
'source_revision': REVISION, 'missing_abstract_count': sum(not row['abstract'] for row in records),
'evidence_kind': 'public_submission_venueid',
'note': 'Public opt-in subset; explicit platform venueid, not original signed PC notes.'}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
result = {'source': SOURCE, 'revision': REVISION, 'finished_at': timestamp(),
'input_rows': EXPECTED, 'normalized_records': EXPECTED, 'files': [manifest],
'source_files': [{'url': URL, 'git_blob_sha1': blob, 'input_bytes': len(data)}],
'note': 'Snapshot preserves per-note public visibility and licenses; no reviews, authorids or PDFs retained.'}
inventory = stage / 'public-openreview-inventory.json'
inventory.write_text(json.dumps(result, indent=2) + '\n')
for child in sorted(stage.iterdir(), key=lambda child: child == inventory):
child.replace(out / child.name)
print(json.dumps(result), flush=True)
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
collect(args.out)
if __name__ == '__main__':
main()