126 lines
6.9 KiB
Python
126 lines
6.9 KiB
Python
"""ICML 2026 public submission roots with explicit Rejected_Submission IDs."""
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import tempfile
|
|
|
|
from paper_atlas.collectors.openreview import date_ms, invitations, public, value
|
|
from paper_atlas.collectors.public_openreview import download
|
|
from paper_atlas.db.submissions import timestamp
|
|
|
|
|
|
SOURCE = 'openreview_public_archive'
|
|
REVISION = '1a1b9d973a8dfbf254f740fb8100a2fcf36757ea'
|
|
AS_OF = '2026-08-08T12:03:26.000Z'
|
|
URL = 'https://raw.githubusercontent.com/HemantKumar01/ICML-2026-data/' + REVISION + '/data/rejected.json'
|
|
BLOB_SHA1 = 'cebdcddffe3eb2145d32d328db89a1626e3f85eb'
|
|
BYTES = 1600856
|
|
EXPECTED = 214
|
|
DOMAIN = 'ICML.cc/2026/Conference'
|
|
VENUEID = DOMAIN + '/Rejected_Submission'
|
|
|
|
|
|
def normalize(note):
|
|
forum = note.get('id')
|
|
if not isinstance(forum, str) or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', forum) or note.get('forum') != forum:
|
|
raise ValueError('Not a public submission root with a real forum identity')
|
|
if not public(note) or note.get('ddate'):
|
|
raise ValueError('Submission is not explicitly public and live in the archive')
|
|
if note.get('domain') != DOMAIN or DOMAIN + '/-/Submission' not in invitations(note):
|
|
raise ValueError('Submission is outside the ICML 2026 conference track')
|
|
content = note.get('content') or {}
|
|
if value(content, 'venueid') != VENUEID:
|
|
raise ValueError('Missing explicit platform Rejected_Submission classification')
|
|
title, abstract = value(content, 'title'), value(content, 'abstract')
|
|
if not isinstance(title, str) or not title.strip():
|
|
raise ValueError('Submission lacks a title')
|
|
if abstract is not None and not isinstance(abstract, str):
|
|
raise ValueError('Submission abstract must be text')
|
|
authors = value(content, 'authors', []) or []
|
|
if not isinstance(authors, list):
|
|
raise ValueError('Unexpected public author format')
|
|
authors = [a.strip() for a in authors if isinstance(a, str) and a.strip() and '@' not in a
|
|
and not re.fullmatch(r'anonymous(?: authors?)?', a.strip(), re.I)]
|
|
tags = []
|
|
for field in ('keywords', 'primary_area'):
|
|
raw = value(content, field)
|
|
terms = raw if isinstance(raw, list) else re.split(r'[,;]', raw) if field == 'keywords' and isinstance(raw, str) else [raw]
|
|
tags.extend({'namespace': SOURCE + ':' + field, 'value': term.strip()}
|
|
for term in terms if isinstance(term, str) and term.strip())
|
|
url = 'https://openreview.net/forum?id=' + forum
|
|
return {'source': SOURCE, 'external_id': 'icml/2026/' + forum, 'url': url,
|
|
'title': ' '.join(title.split()), 'abstract': abstract.strip() if abstract else None,
|
|
'authors': authors, 'identifiers': {'openreview': forum},
|
|
'submission_identifiers': [{'scheme': 'openreview', 'value': forum}],
|
|
'venue': 'ICML', 'year': 2026, 'track': 'main', 'status': 'rejected',
|
|
'raw_decision': VENUEID, 'evidence_url': url, 'observed_at': AS_OF,
|
|
'submitted_at': date_ms(note.get('cdate') or note.get('tcdate')), 'tags': tags,
|
|
'metadata': {'source_revision': REVISION, 'source_blob_sha1': BLOB_SHA1,
|
|
'source_file': URL, 'original_forum': forum, 'license': note.get('license'),
|
|
'venue': value(content, 'venue'), 'venueid': VENUEID,
|
|
'public_readers_verified': True, 'original_invitations': invitations(note),
|
|
'source_note_modified_at': date_ms(note.get('mdate')),
|
|
'evidence_kind': 'public_submission_venueid',
|
|
'original_decision_signature_verified': False,
|
|
'source_snapshot_upper_bound': AS_OF}}
|
|
|
|
|
|
def collect(out, fetcher=None):
|
|
fetcher = fetcher or download
|
|
out = Path(out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
with (out / 'collect.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (out / 'public-openreview-inventory.json').exists():
|
|
raise ValueError('Completed inventory exists; select a new output directory')
|
|
if shutil.disk_usage(out).free < 8 * 1024**3:
|
|
raise RuntimeError('Less than 8 GiB free before collection')
|
|
data = fetcher(URL, BYTES)
|
|
blob = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest()
|
|
if len(data) != BYTES or blob != BLOB_SHA1:
|
|
raise ValueError('Public archive checksum mismatch')
|
|
archive = json.loads(data)
|
|
notes = archive['notes']
|
|
if archive.get('count') != EXPECTED or not isinstance(notes, list) or len(notes) != EXPECTED:
|
|
raise ValueError('Public archive inventory count mismatch')
|
|
records = [normalize(note) for note in notes]
|
|
if len({row['identifiers']['openreview'] for row in records}) != EXPECTED:
|
|
raise ValueError('Repeated submission forum in public archive')
|
|
records.sort(key=lambda row: row['external_id'])
|
|
with tempfile.TemporaryDirectory(prefix='.icml-public-', dir=out) as temporary:
|
|
stage = Path(temporary)
|
|
path = stage / 'public-openreview-icml-2026-main.jsonl'
|
|
path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in records))
|
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICML', 'year': 2026,
|
|
'track': 'main', 'count': EXPECTED, 'status': 'partial', 'list_url': URL,
|
|
'file': str(out / path.name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
|
|
'source_revision': REVISION, 'missing_abstract_count': sum(not row['abstract'] for row in records),
|
|
'evidence_kind': 'public_submission_venueid',
|
|
'note': 'Public opt-in subset; explicit platform venueid, not original signed PC notes.'}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
|
|
result = {'source': SOURCE, 'revision': REVISION, 'finished_at': timestamp(),
|
|
'input_rows': EXPECTED, 'normalized_records': EXPECTED, 'files': [manifest],
|
|
'source_files': [{'url': URL, 'git_blob_sha1': blob, 'input_bytes': len(data)}],
|
|
'note': 'Snapshot preserves per-note public visibility and licenses; no reviews, authorids or PDFs retained.'}
|
|
inventory = stage / 'public-openreview-inventory.json'
|
|
inventory.write_text(json.dumps(result, indent=2) + '\n')
|
|
for child in sorted(stage.iterdir(), key=lambda child: child == inventory):
|
|
child.replace(out / child.name)
|
|
print(json.dumps(result), flush=True)
|
|
return result
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog)
|
|
parser.add_argument('--out', required=True)
|
|
args = parser.parse_args(argv)
|
|
collect(args.out)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|