Files
paper-atlas/collect_iclr.py
T
codex-bot 15bb4920c6
Metadata integrity checks / metadata (push) Successful in 7s
Preserve canonical submission identities and decision source evidence
2026-11-06 14:05:37 +08:00

248 lines
13 KiB
Python

"""Collect public ICLR program metadata, excluding blog/journal presentations."""
import argparse
from collections import Counter, defaultdict
from datetime import datetime, timezone
from difflib import SequenceMatcher
import json
from pathlib import Path
import re
from urllib.error import HTTPError
from urllib.parse import parse_qs, urljoin, urlsplit
from collect_pmlr import fetch
from collect_proceedings import Document, clean
from index_core import author_keys, normalized
ROOT = 'https://iclr.cc/'
SOURCE = 'iclr_program'
class NotYetPublished(ValueError):
pass
def discover(html):
years = {}
for node in Document(html).root.walk():
text = clean(node.text())
if node.tag == 'a' and re.fullmatch(r'20\d{2}', text):
url = urljoin(ROOT, node.attrs.get('href', ''))
if urlsplit(url).netloc == 'iclr.cc':
years[int(text)] = url
if not years:
raise ValueError('No ICLR years in the official navigation')
return sorted(years.items())
def asset_urls(html, page_url, year):
refs = re.findall(r'''["']([^"'\s]+\.json)["']''', html)
result = {}
for kind in ('orals-posters', 'abstracts'):
matches = {urljoin(page_url, ref) for ref in refs
if urlsplit(urljoin(page_url, ref)).netloc == 'iclr.cc'
and urlsplit(urljoin(page_url, ref)).path == f'/static/virtual/data/iclr-{year}-{kind}.json'}
if len(matches) > 1:
raise ValueError('Ambiguous official program asset')
if matches:
result[kind] = matches.pop()
if 'orals-posters' not in result:
raise ValueError('Official page has no supported program JSON reference')
return result
def main_event(row, year):
source = row.get('sourceurl')
if source:
parsed = urlsplit(source)
return (parsed.netloc == 'openreview.net' and parsed.path == '/group'
and parse_qs(parsed.query).get('id') == [f'ICLR.cc/{year}/Conference'])
# These historical official paper lists predate sourceurl/decision fields.
return year in (2020, 2021) and row.get('eventtype') in ('Poster', 'Oral', 'Spotlight')
def source_number(row):
value = row.get('sourceid')
return int(value) if re.fullmatch(r'-?[1-9]\d*', str(value)) else None
def identifiers(row):
# The program creates negative-ID oral events with fabricated forum URLs.
if (source_number(row) or 0) < 0:
return {}
links = [row.get('paper_url'), row.get('paper_pdf_url')]
links += [media.get('uri') for media in row.get('eventmedia', [])
if str(media.get('name') or '').casefold() in ('openreview', 'paper')]
forums = set()
for link in links:
if not isinstance(link, str):
continue
parsed = urlsplit(link)
if parsed.netloc == 'openreview.net' and parsed.path in ('/forum', '/pdf'):
values = parse_qs(parsed.query).get('id', [])
if len(values) == 1 and re.fullmatch(r'[\w-]+', values[0]) and not re.match(r'20\d{2}-Oral-', values[0]):
forums.add(values[0])
if len(forums) > 1:
raise ValueError('Conflicting paper self-links in one program event')
return {'openreview': forums.pop()} if forums else {}
def parse_program(data, abstracts, year, index_url):
if not isinstance(data, dict) or not isinstance(data.get('results'), list):
raise ValueError('Expected a program results array with a declared count')
events = data['results']
if data.get('next') or data.get('previous') or data.get('count') != len(events):
raise ValueError('Program JSON is paginated or its count differs from events')
if not isinstance(abstracts, dict):
raise ValueError('Expected an abstract mapping keyed by event ID')
event_ids = [str(row['id']) for row in events]
if len(event_ids) != len(set(event_ids)):
raise ValueError('Duplicate event IDs in program JSON')
grouped = defaultdict(list)
excluded = Counter()
for row in events:
if not main_event(row, year):
excluded[str(row.get('sourceurl') or 'unclassified')] += 1
continue
decision = row.get('decision')
if decision and not (str(decision).casefold().startswith('accept') or decision in ('Poster', 'Oral', 'Spotlight')):
raise ValueError('Unexpected decision in an accepted conference program')
number = source_number(row)
identity = ('submission', abs(number)) if number else ('forum', identifiers(row).get('openreview'))
if identity == ('forum', None):
identity = ('event', row['id'])
grouped[identity].append(row)
records = []
missing_identity = 0
for identity, group in grouped.items():
# Verify the negative oral event really describes its positive counterpart.
group.sort(key=lambda r: ((source_number(r) or 0) < 0, str(r['id'])))
primary = group[0]
title = normalized(primary.get('name', ''))
authors = [author_keys([a['fullname'] for a in row.get('authors', [])]) for row in group]
title_match = all(normalized(row.get('name', '')) == title or (
identity[0] == 'submission' and (source_number(row) or 0) < 0
and (source_number(primary) or 0) > 0 and len(authors[0]) >= 2
and SequenceMatcher(None, title, normalized(row.get('name', ''))).ratio() >= 0.85)
for row in group)
if not title or not title_match or any(a != authors[0] for a in authors):
raise ValueError('Program submission ID collides across different papers')
own_ids = {}
for row in group:
for scheme, value in identifiers(row).items():
if scheme in own_ids and own_ids[scheme] != value:
raise ValueError('Conflicting identifiers across presentation events')
own_ids[scheme] = value
abstract = next((text for row in group
if (text := row.get('abstract') or abstracts.get(str(row['id'])))), None)
if abstract is not None and not isinstance(abstract, str):
raise ValueError('Nontext abstract')
presentation_ids = [row['id'] for row in group]
forum = own_ids.get('openreview')
external = f'{year}/{forum}' if forum else f'{year}/program-{identity[1]}'
missing_identity += not bool(forum)
page = urljoin(ROOT, primary.get('virtualsite_url') or f'/virtual/{year}/{str(primary.get("eventtype", "Poster")).lower()}/{primary["id"]}')
topics = sorted({str(row['topic']).strip() for row in group if row.get('topic')})
keywords = sorted({str(keyword).strip() for row in group for keyword in row.get('keywords', []) if keyword})
records.append({'source': SOURCE, 'external_id': external, 'url': page,
'title': clean(primary['name']), 'abstract': abstract.strip() if abstract else None,
'authors': [a['fullname'] for a in primary.get('authors', [])],
'identifiers': own_ids, 'venue': 'ICLR', 'year': year, 'track': 'main',
'status': 'accepted', 'raw_decision': str(primary.get('decision') or 'listed in official conference paper program'),
'evidence_url': index_url, 'list_url': index_url,
'tags': [{'namespace': SOURCE+':'+kind, 'value': value}
for kind, values in [('topic', topics), ('keyword', keywords)] for value in values],
'metadata': {'program_event_ids': presentation_ids,
'program_identity_type': identity[0],
'presentation_titles': sorted({clean(r['name']) for r in group}),
'program_source_id': identity[1], 'presentation_types': sorted({str(r.get('event_type') or r.get('eventtype')) for r in group}),
'raw_decisions': sorted({str(r.get('decision')) for r in group if r.get('decision')}),
'identity_state': 'forum' if forum else 'program_only'}})
# Different source numbers can still identify the same forum. Keep one record.
by_id = {}
for row in records:
previous = by_id.get(row['external_id'])
if previous:
if normalized(previous['title']) != normalized(row['title']) or author_keys(previous['authors']) != author_keys(row['authors']):
raise ValueError('One forum identifies different program papers')
previous['metadata']['program_event_ids'] += row['metadata']['program_event_ids']
previous['metadata']['presentation_types'] = sorted(set(previous['metadata']['presentation_types'] + row['metadata']['presentation_types']))
previous['abstract'] = previous['abstract'] or row['abstract']
else:
by_id[row['external_id']] = row
result = list(by_id.values())
if not result:
raise ValueError('No verified main-conference papers in program')
return result, {'program_event_count': len(events), 'main_event_count': sum(map(len, grouped.values())),
'excluded_events_by_source': dict(excluded), 'missing_forum_count': missing_identity,
'missing_abstract_count': sum(not r['abstract'] for r in result)}
def collect_year(year, cache, out):
page = f'https://iclr.cc/virtual/{year}/papers.html'
assets = asset_urls(fetch(page, cache, 16*1024**2, delay=1).decode(), page, year)
data = json.loads(fetch(assets['orals-posters'], cache, 32*1024**2, delay=1))
if data.get('count') == 0 and data.get('results') == [] and year > datetime.now(timezone.utc).year:
raise NotYetPublished('Official program is empty for a future conference')
abstracts = {}
abstract_asset_status = 'not_referenced'
if 'abstracts' in assets:
try:
abstracts = json.loads(fetch(assets['abstracts'], cache, 32*1024**2, delay=1))
abstract_asset_status = 'available'
except HTTPError as exc:
if exc.code != 404:
raise
# Historical programs embed abstracts but retain a stale split-file URL.
exc.close()
abstract_asset_status = 'unavailable_404'
records, audit = parse_program(data, abstracts, year, page)
path = Path(out)/f'iclr-program-{year}-main.jsonl'
part = path.with_suffix('.part')
part.write_text(''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in records))
part.replace(path)
manifest = {'source': SOURCE, 'parser_version': 2, 'venue': 'ICLR', 'year': year, 'track': 'main',
'count': len(records), 'list_url': page, 'asset_urls': assets,
'abstract_asset_status': abstract_asset_status, 'file': str(path),
'status': 'partial' if audit['missing_abstract_count'] else 'collected', **audit}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2))
return manifest
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--cache', required=True)
parser.add_argument('--out', required=True)
parser.add_argument('--year', type=int, action='append')
args = parser.parse_args()
out = Path(args.out); out.mkdir(parents=True, exist_ok=True)
years = discover(fetch(ROOT, args.cache, 4*1024**2, delay=1).decode())
if args.year and set(args.year)-{year for year, _ in years}:
parser.error('Requested year is absent from the official navigation')
status_path = out/'iclr-program-year-status.json'
catalogue = json.loads(status_path.read_text()) if status_path.exists() else {}
requested = []
for year, conference_url in years:
if args.year and year not in args.year:
continue
requested.append(year)
try:
result = collect_year(year, args.cache, out)
except NotYetPublished as exc:
result = {'year': year, 'status': 'not_yet_published', 'error': str(exc)}
except HTTPError as exc:
result = {'year': year, 'status': 'not_yet_published' if year > datetime.now(timezone.utc).year and exc.code == 404 else 'unavailable' if exc.code == 404 else 'blocked', 'error': str(exc)}
exc.close()
except Exception as exc:
result = {'year': year, 'status': 'blocked', 'error': str(exc)}
result['conference_url'] = conference_url
catalogue[str(year)] = result
part = status_path.with_suffix('.part'); part.write_text(json.dumps(catalogue, indent=2)); part.replace(status_path)
print(json.dumps(result), flush=True)
(out/'iclr-program-last-run.json').write_text(json.dumps({'requested_years': requested, 'year_filter': args.year}, indent=2))
if any(catalogue[str(y)]['status'] == 'blocked' for y in requested):
raise SystemExit(1)
if __name__ == '__main__':
main()