"""Collect public ICLR program metadata, excluding blog/journal presentations.""" import argparse from collections import Counter, defaultdict from datetime import datetime, timezone from difflib import SequenceMatcher import json from pathlib import Path import re from urllib.error import HTTPError from urllib.parse import parse_qs, urljoin, urlsplit from collect_pmlr import fetch from collect_proceedings import Document, clean from index_core import author_keys, normalized ROOT = 'https://iclr.cc/' SOURCE = 'iclr_program' class NotYetPublished(ValueError): pass def discover(html): years = {} for node in Document(html).root.walk(): text = clean(node.text()) if node.tag == 'a' and re.fullmatch(r'20\d{2}', text): url = urljoin(ROOT, node.attrs.get('href', '')) if urlsplit(url).netloc == 'iclr.cc': years[int(text)] = url if not years: raise ValueError('No ICLR years in the official navigation') return sorted(years.items()) def asset_urls(html, page_url, year): refs = re.findall(r'''["']([^"'\s]+\.json)["']''', html) result = {} for kind in ('orals-posters', 'abstracts'): matches = {urljoin(page_url, ref) for ref in refs if urlsplit(urljoin(page_url, ref)).netloc == 'iclr.cc' and urlsplit(urljoin(page_url, ref)).path == f'/static/virtual/data/iclr-{year}-{kind}.json'} if len(matches) > 1: raise ValueError('Ambiguous official program asset') if matches: result[kind] = matches.pop() if 'orals-posters' not in result: raise ValueError('Official page has no supported program JSON reference') return result def main_event(row, year): source = row.get('sourceurl') if source: parsed = urlsplit(source) return (parsed.netloc == 'openreview.net' and parsed.path == '/group' and parse_qs(parsed.query).get('id') == [f'ICLR.cc/{year}/Conference']) # These historical official paper lists predate sourceurl/decision fields. return year in (2020, 2021) and row.get('eventtype') in ('Poster', 'Oral', 'Spotlight') def source_number(row): value = row.get('sourceid') return int(value) if re.fullmatch(r'-?[1-9]\d*', str(value)) else None def identifiers(row): # The program creates negative-ID oral events with fabricated forum URLs. if (source_number(row) or 0) < 0: return {} links = [row.get('paper_url'), row.get('paper_pdf_url')] links += [media.get('uri') for media in row.get('eventmedia', []) if str(media.get('name') or '').casefold() in ('openreview', 'paper')] forums = set() for link in links: if not isinstance(link, str): continue parsed = urlsplit(link) if parsed.netloc == 'openreview.net' and parsed.path in ('/forum', '/pdf'): values = parse_qs(parsed.query).get('id', []) if len(values) == 1 and re.fullmatch(r'[\w-]+', values[0]) and not re.match(r'20\d{2}-Oral-', values[0]): forums.add(values[0]) if len(forums) > 1: raise ValueError('Conflicting paper self-links in one program event') return {'openreview': forums.pop()} if forums else {} def parse_program(data, abstracts, year, index_url): if not isinstance(data, dict) or not isinstance(data.get('results'), list): raise ValueError('Expected a program results array with a declared count') events = data['results'] if data.get('next') or data.get('previous') or data.get('count') != len(events): raise ValueError('Program JSON is paginated or its count differs from events') if not isinstance(abstracts, dict): raise ValueError('Expected an abstract mapping keyed by event ID') event_ids = [str(row['id']) for row in events] if len(event_ids) != len(set(event_ids)): raise ValueError('Duplicate event IDs in program JSON') grouped = defaultdict(list) excluded = Counter() for row in events: if not main_event(row, year): excluded[str(row.get('sourceurl') or 'unclassified')] += 1 continue decision = row.get('decision') if decision and not (str(decision).casefold().startswith('accept') or decision in ('Poster', 'Oral', 'Spotlight')): raise ValueError('Unexpected decision in an accepted conference program') number = source_number(row) identity = ('submission', abs(number)) if number else ('forum', identifiers(row).get('openreview')) if identity == ('forum', None): identity = ('event', row['id']) grouped[identity].append(row) records = [] missing_identity = 0 for identity, group in grouped.items(): # Verify the negative oral event really describes its positive counterpart. group.sort(key=lambda r: ((source_number(r) or 0) < 0, str(r['id']))) primary = group[0] title = normalized(primary.get('name', '')) authors = [author_keys([a['fullname'] for a in row.get('authors', [])]) for row in group] title_match = all(normalized(row.get('name', '')) == title or ( identity[0] == 'submission' and (source_number(row) or 0) < 0 and (source_number(primary) or 0) > 0 and len(authors[0]) >= 2 and SequenceMatcher(None, title, normalized(row.get('name', ''))).ratio() >= 0.85) for row in group) if not title or not title_match or any(a != authors[0] for a in authors): raise ValueError('Program submission ID collides across different papers') own_ids = {} for row in group: for scheme, value in identifiers(row).items(): if scheme in own_ids and own_ids[scheme] != value: raise ValueError('Conflicting identifiers across presentation events') own_ids[scheme] = value abstract = next((text for row in group if (text := row.get('abstract') or abstracts.get(str(row['id'])))), None) if abstract is not None and not isinstance(abstract, str): raise ValueError('Nontext abstract') presentation_ids = [row['id'] for row in group] forum = own_ids.get('openreview') external = f'{year}/{forum}' if forum else f'{year}/program-{identity[1]}' missing_identity += not bool(forum) page = urljoin(ROOT, primary.get('virtualsite_url') or f'/virtual/{year}/{str(primary.get("eventtype", "Poster")).lower()}/{primary["id"]}') topics = sorted({str(row['topic']).strip() for row in group if row.get('topic')}) keywords = sorted({str(keyword).strip() for row in group for keyword in row.get('keywords', []) if keyword}) records.append({'source': SOURCE, 'external_id': external, 'url': page, 'title': clean(primary['name']), 'abstract': abstract.strip() if abstract else None, 'authors': [a['fullname'] for a in primary.get('authors', [])], 'identifiers': own_ids, 'venue': 'ICLR', 'year': year, 'track': 'main', 'status': 'accepted', 'raw_decision': str(primary.get('decision') or 'listed in official conference paper program'), 'evidence_url': index_url, 'list_url': index_url, 'tags': [{'namespace': SOURCE+':'+kind, 'value': value} for kind, values in [('topic', topics), ('keyword', keywords)] for value in values], 'metadata': {'program_event_ids': presentation_ids, 'presentation_titles': sorted({clean(r['name']) for r in group}), 'program_source_id': identity[1], 'presentation_types': sorted({str(r.get('event_type') or r.get('eventtype')) for r in group}), 'raw_decisions': sorted({str(r.get('decision')) for r in group if r.get('decision')}), 'identity_state': 'forum' if forum else 'program_only'}}) # Different source numbers can still identify the same forum. Keep one record. by_id = {} for row in records: previous = by_id.get(row['external_id']) if previous: if normalized(previous['title']) != normalized(row['title']) or author_keys(previous['authors']) != author_keys(row['authors']): raise ValueError('One forum identifies different program papers') previous['metadata']['program_event_ids'] += row['metadata']['program_event_ids'] previous['metadata']['presentation_types'] = sorted(set(previous['metadata']['presentation_types'] + row['metadata']['presentation_types'])) previous['abstract'] = previous['abstract'] or row['abstract'] else: by_id[row['external_id']] = row result = list(by_id.values()) if not result: raise ValueError('No verified main-conference papers in program') return result, {'program_event_count': len(events), 'main_event_count': sum(map(len, grouped.values())), 'excluded_events_by_source': dict(excluded), 'missing_forum_count': missing_identity, 'missing_abstract_count': sum(not r['abstract'] for r in result)} def collect_year(year, cache, out): page = f'https://iclr.cc/virtual/{year}/papers.html' assets = asset_urls(fetch(page, cache, 16*1024**2, delay=1).decode(), page, year) data = json.loads(fetch(assets['orals-posters'], cache, 32*1024**2, delay=1)) if data.get('count') == 0 and data.get('results') == [] and year > datetime.now(timezone.utc).year: raise NotYetPublished('Official program is empty for a future conference') abstracts = {} abstract_asset_status = 'not_referenced' if 'abstracts' in assets: try: abstracts = json.loads(fetch(assets['abstracts'], cache, 32*1024**2, delay=1)) abstract_asset_status = 'available' except HTTPError as exc: if exc.code != 404: raise # Historical programs embed abstracts but retain a stale split-file URL. exc.close() abstract_asset_status = 'unavailable_404' records, audit = parse_program(data, abstracts, year, page) path = Path(out)/f'iclr-program-{year}-main.jsonl' part = path.with_suffix('.part') part.write_text(''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in records)) part.replace(path) manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICLR', 'year': year, 'track': 'main', 'count': len(records), 'list_url': page, 'asset_urls': assets, 'abstract_asset_status': abstract_asset_status, 'file': str(path), 'status': 'partial' if audit['missing_abstract_count'] else 'collected', **audit} path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)) return manifest def main(): parser = argparse.ArgumentParser() parser.add_argument('--cache', required=True) parser.add_argument('--out', required=True) parser.add_argument('--year', type=int, action='append') args = parser.parse_args() out = Path(args.out); out.mkdir(parents=True, exist_ok=True) years = discover(fetch(ROOT, args.cache, 4*1024**2, delay=1).decode()) if args.year and set(args.year)-{year for year, _ in years}: parser.error('Requested year is absent from the official navigation') status_path = out/'iclr-program-year-status.json' catalogue = json.loads(status_path.read_text()) if status_path.exists() else {} requested = [] for year, conference_url in years: if args.year and year not in args.year: continue requested.append(year) try: result = collect_year(year, args.cache, out) except NotYetPublished as exc: result = {'year': year, 'status': 'not_yet_published', 'error': str(exc)} except HTTPError as exc: result = {'year': year, 'status': 'not_yet_published' if year > datetime.now(timezone.utc).year and exc.code == 404 else 'unavailable' if exc.code == 404 else 'blocked', 'error': str(exc)} exc.close() except Exception as exc: result = {'year': year, 'status': 'blocked', 'error': str(exc)} result['conference_url'] = conference_url catalogue[str(year)] = result part = status_path.with_suffix('.part'); part.write_text(json.dumps(catalogue, indent=2)); part.replace(status_path) print(json.dumps(result), flush=True) (out/'iclr-program-last-run.json').write_text(json.dumps({'requested_years': requested, 'year_filter': args.year}, indent=2)) if any(catalogue[str(y)]['status'] == 'blocked' for y in requested): raise SystemExit(1) if __name__ == '__main__': main()