252 lines
13 KiB
Python
252 lines
13 KiB
Python
"""Collect public ICLR program metadata, excluding blog/journal presentations."""
|
|
import argparse
|
|
from collections import Counter, defaultdict
|
|
from datetime import datetime, timezone
|
|
from difflib import SequenceMatcher
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
from urllib.error import HTTPError
|
|
from urllib.parse import parse_qs, urljoin, urlsplit
|
|
|
|
from paper_atlas.io.http import fetch
|
|
from paper_atlas.io.html import Document, clean
|
|
from paper_atlas.db.store import author_keys, normalized
|
|
|
|
ROOT = 'https://iclr.cc/'
|
|
SOURCE = 'iclr_program'
|
|
|
|
|
|
class NotYetPublished(ValueError):
|
|
pass
|
|
|
|
|
|
def discover(html):
|
|
years = {}
|
|
for node in Document(html).root.walk():
|
|
text = clean(node.text())
|
|
if node.tag == 'a' and re.fullmatch(r'20\d{2}', text):
|
|
url = urljoin(ROOT, node.attrs.get('href', ''))
|
|
if urlsplit(url).netloc == 'iclr.cc':
|
|
years[int(text)] = url
|
|
if not years:
|
|
raise ValueError('No ICLR years in the official navigation')
|
|
return sorted(years.items())
|
|
|
|
|
|
def asset_urls(html, page_url, year):
|
|
refs = re.findall(r'''["']([^"'\s]+\.json)["']''', html)
|
|
result = {}
|
|
for kind in ('orals-posters', 'abstracts'):
|
|
matches = {urljoin(page_url, ref) for ref in refs
|
|
if urlsplit(urljoin(page_url, ref)).netloc == 'iclr.cc'
|
|
and urlsplit(urljoin(page_url, ref)).path == f'/static/virtual/data/iclr-{year}-{kind}.json'}
|
|
if len(matches) > 1:
|
|
raise ValueError('Ambiguous official program asset')
|
|
if matches:
|
|
result[kind] = matches.pop()
|
|
if 'orals-posters' not in result:
|
|
raise ValueError('Official page has no supported program JSON reference')
|
|
return result
|
|
|
|
|
|
def main_event(row, year):
|
|
source = row.get('sourceurl')
|
|
if source:
|
|
parsed = urlsplit(source)
|
|
return (parsed.netloc == 'openreview.net' and parsed.path == '/group'
|
|
and parse_qs(parsed.query).get('id') == [f'ICLR.cc/{year}/Conference'])
|
|
# These historical official paper lists predate sourceurl/decision fields.
|
|
return year in (2020, 2021) and row.get('eventtype') in ('Poster', 'Oral', 'Spotlight')
|
|
|
|
|
|
def source_number(row):
|
|
value = row.get('sourceid')
|
|
return int(value) if re.fullmatch(r'-?[1-9]\d*', str(value)) else None
|
|
|
|
|
|
def identifiers(row):
|
|
# The program creates negative-ID oral events with fabricated forum URLs.
|
|
if (source_number(row) or 0) < 0:
|
|
return {}
|
|
links = [row.get('paper_url'), row.get('paper_pdf_url')]
|
|
links += [media.get('uri') for media in row.get('eventmedia', [])
|
|
if str(media.get('name') or '').casefold() in ('openreview', 'paper')]
|
|
forums = set()
|
|
for link in links:
|
|
if not isinstance(link, str):
|
|
continue
|
|
parsed = urlsplit(link)
|
|
if parsed.netloc == 'openreview.net' and parsed.path in ('/forum', '/pdf'):
|
|
values = parse_qs(parsed.query).get('id', [])
|
|
if len(values) == 1 and re.fullmatch(r'[\w-]+', values[0]) and not re.match(r'20\d{2}-Oral-', values[0]):
|
|
forums.add(values[0])
|
|
if len(forums) > 1:
|
|
raise ValueError('Conflicting paper self-links in one program event')
|
|
return {'openreview': forums.pop()} if forums else {}
|
|
|
|
|
|
def parse_program(data, abstracts, year, index_url):
|
|
if not isinstance(data, dict) or not isinstance(data.get('results'), list):
|
|
raise ValueError('Expected a program results array with a declared count')
|
|
events = data['results']
|
|
if data.get('next') or data.get('previous') or data.get('count') != len(events):
|
|
raise ValueError('Program JSON is paginated or its count differs from events')
|
|
if not isinstance(abstracts, dict):
|
|
raise ValueError('Expected an abstract mapping keyed by event ID')
|
|
event_ids = [str(row['id']) for row in events]
|
|
if len(event_ids) != len(set(event_ids)):
|
|
raise ValueError('Duplicate event IDs in program JSON')
|
|
grouped = defaultdict(list)
|
|
excluded = Counter()
|
|
for row in events:
|
|
if not main_event(row, year):
|
|
excluded[str(row.get('sourceurl') or 'unclassified')] += 1
|
|
continue
|
|
decision = row.get('decision')
|
|
if decision and not (str(decision).casefold().startswith('accept') or decision in ('Poster', 'Oral', 'Spotlight')):
|
|
raise ValueError('Unexpected decision in an accepted conference program')
|
|
number = source_number(row)
|
|
identity = ('submission', abs(number)) if number else ('forum', identifiers(row).get('openreview'))
|
|
if identity == ('forum', None):
|
|
identity = ('event', row['id'])
|
|
grouped[identity].append(row)
|
|
records = []
|
|
missing_identity = 0
|
|
for identity, group in grouped.items():
|
|
# Verify the negative oral event really describes its positive counterpart.
|
|
group.sort(key=lambda r: ((source_number(r) or 0) < 0, str(r['id'])))
|
|
primary = group[0]
|
|
title = normalized(primary.get('name', ''))
|
|
authors = [author_keys([a['fullname'] for a in row.get('authors', [])]) for row in group]
|
|
title_match = all(normalized(row.get('name', '')) == title or (
|
|
identity[0] == 'submission' and (source_number(row) or 0) < 0
|
|
and (source_number(primary) or 0) > 0 and len(authors[0]) >= 2
|
|
and SequenceMatcher(None, title, normalized(row.get('name', ''))).ratio() >= 0.85)
|
|
for row in group)
|
|
if not title or not title_match or any(a != authors[0] for a in authors):
|
|
raise ValueError('Program submission ID collides across different papers')
|
|
own_ids = {}
|
|
for row in group:
|
|
for scheme, value in identifiers(row).items():
|
|
if scheme in own_ids and own_ids[scheme] != value:
|
|
raise ValueError('Conflicting identifiers across presentation events')
|
|
own_ids[scheme] = value
|
|
abstract = next((text for row in group
|
|
if (text := row.get('abstract') or abstracts.get(str(row['id'])))), None)
|
|
if abstract is not None and not isinstance(abstract, str):
|
|
raise ValueError('Nontext abstract')
|
|
presentation_ids = [row['id'] for row in group]
|
|
forum = own_ids.get('openreview')
|
|
external = f'{year}/{forum}' if forum else f'{year}/program-{identity[1]}'
|
|
missing_identity += not bool(forum)
|
|
page = urljoin(ROOT, primary.get('virtualsite_url') or f'/virtual/{year}/{str(primary.get("eventtype", "Poster")).lower()}/{primary["id"]}')
|
|
if year == 2020 and not primary.get('virtualsite_url'):
|
|
# Numeric modern event IDs do not identify legacy poster pages.
|
|
# The independently verified forum bridge supplies the paper URL.
|
|
page = index_url
|
|
topics = sorted({str(row['topic']).strip() for row in group if row.get('topic')})
|
|
keywords = sorted({str(keyword).strip() for row in group for keyword in row.get('keywords', []) if keyword})
|
|
records.append({'source': SOURCE, 'external_id': external, 'url': page,
|
|
'title': clean(primary['name']), 'abstract': abstract.strip() if abstract else None,
|
|
'authors': [a['fullname'] for a in primary.get('authors', [])],
|
|
'identifiers': own_ids, 'venue': 'ICLR', 'year': year, 'track': 'main',
|
|
'status': 'accepted', 'raw_decision': str(primary.get('decision') or 'listed in official conference paper program'),
|
|
'evidence_url': index_url, 'list_url': index_url,
|
|
'tags': [{'namespace': SOURCE+':'+kind, 'value': value}
|
|
for kind, values in [('topic', topics), ('keyword', keywords)] for value in values],
|
|
'metadata': {'program_event_ids': presentation_ids,
|
|
'program_identity_type': identity[0],
|
|
'presentation_titles': sorted({clean(r['name']) for r in group}),
|
|
'program_source_id': identity[1], 'presentation_types': sorted({str(r.get('event_type') or r.get('eventtype')) for r in group}),
|
|
'raw_decisions': sorted({str(r.get('decision')) for r in group if r.get('decision')}),
|
|
'identity_state': 'forum' if forum else 'program_only'}})
|
|
# Different source numbers can still identify the same forum. Keep one record.
|
|
by_id = {}
|
|
for row in records:
|
|
previous = by_id.get(row['external_id'])
|
|
if previous:
|
|
if normalized(previous['title']) != normalized(row['title']) or author_keys(previous['authors']) != author_keys(row['authors']):
|
|
raise ValueError('One forum identifies different program papers')
|
|
previous['metadata']['program_event_ids'] += row['metadata']['program_event_ids']
|
|
previous['metadata']['presentation_types'] = sorted(set(previous['metadata']['presentation_types'] + row['metadata']['presentation_types']))
|
|
previous['abstract'] = previous['abstract'] or row['abstract']
|
|
else:
|
|
by_id[row['external_id']] = row
|
|
result = list(by_id.values())
|
|
if not result:
|
|
raise ValueError('No verified main-conference papers in program')
|
|
return result, {'program_event_count': len(events), 'main_event_count': sum(map(len, grouped.values())),
|
|
'excluded_events_by_source': dict(excluded), 'missing_forum_count': missing_identity,
|
|
'missing_abstract_count': sum(not r['abstract'] for r in result)}
|
|
|
|
|
|
def collect_year(year, cache, out):
|
|
page = f'https://iclr.cc/virtual/{year}/papers.html'
|
|
assets = asset_urls(fetch(page, cache, 16*1024**2, delay=1).decode(), page, year)
|
|
data = json.loads(fetch(assets['orals-posters'], cache, 32*1024**2, delay=1))
|
|
if data.get('count') == 0 and data.get('results') == [] and year > datetime.now(timezone.utc).year:
|
|
raise NotYetPublished('Official program is empty for a future conference')
|
|
abstracts = {}
|
|
abstract_asset_status = 'not_referenced'
|
|
if 'abstracts' in assets:
|
|
try:
|
|
abstracts = json.loads(fetch(assets['abstracts'], cache, 32*1024**2, delay=1))
|
|
abstract_asset_status = 'available'
|
|
except HTTPError as exc:
|
|
if exc.code != 404:
|
|
raise
|
|
# Historical programs embed abstracts but retain a stale split-file URL.
|
|
exc.close()
|
|
abstract_asset_status = 'unavailable_404'
|
|
records, audit = parse_program(data, abstracts, year, page)
|
|
path = Path(out)/f'iclr-program-{year}-main.jsonl'
|
|
part = path.with_suffix('.part')
|
|
part.write_text(''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in records))
|
|
part.replace(path)
|
|
manifest = {'source': SOURCE, 'parser_version': 2, 'venue': 'ICLR', 'year': year, 'track': 'main',
|
|
'count': len(records), 'list_url': page, 'asset_urls': assets,
|
|
'abstract_asset_status': abstract_asset_status, 'file': str(path),
|
|
'status': 'partial' if audit['missing_abstract_count'] else 'collected', **audit}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2))
|
|
return manifest
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog)
|
|
parser.add_argument('--cache', required=True)
|
|
parser.add_argument('--out', required=True)
|
|
parser.add_argument('--year', type=int, action='append')
|
|
args = parser.parse_args(argv)
|
|
out = Path(args.out); out.mkdir(parents=True, exist_ok=True)
|
|
years = discover(fetch(ROOT, args.cache, 4*1024**2, delay=1).decode())
|
|
if args.year and set(args.year)-{year for year, _ in years}:
|
|
parser.error('Requested year is absent from the official navigation')
|
|
status_path = out/'iclr-program-year-status.json'
|
|
catalogue = json.loads(status_path.read_text()) if status_path.exists() else {}
|
|
requested = []
|
|
for year, conference_url in years:
|
|
if args.year and year not in args.year:
|
|
continue
|
|
requested.append(year)
|
|
try:
|
|
result = collect_year(year, args.cache, out)
|
|
except NotYetPublished as exc:
|
|
result = {'year': year, 'status': 'not_yet_published', 'error': str(exc)}
|
|
except HTTPError as exc:
|
|
result = {'year': year, 'status': 'not_yet_published' if year > datetime.now(timezone.utc).year and exc.code == 404 else 'unavailable' if exc.code == 404 else 'blocked', 'error': str(exc)}
|
|
exc.close()
|
|
except Exception as exc:
|
|
result = {'year': year, 'status': 'blocked', 'error': str(exc)}
|
|
result['conference_url'] = conference_url
|
|
catalogue[str(year)] = result
|
|
part = status_path.with_suffix('.part'); part.write_text(json.dumps(catalogue, indent=2)); part.replace(status_path)
|
|
print(json.dumps(result), flush=True)
|
|
(out/'iclr-program-last-run.json').write_text(json.dumps({'requested_years': requested, 'year_filter': args.year}, indent=2))
|
|
if any(catalogue[str(y)]['status'] == 'blocked' for y in requested):
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|