141 lines
7.6 KiB
Python
141 lines
7.6 KiB
Python
"""Resolve mixed ICML proceedings tracks from the official paper program."""
|
|
from collections import Counter
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
from paper_atlas.collectors.iclr import identifiers, source_number
|
|
from paper_atlas.db.store import author_keys, normalized
|
|
from paper_atlas.io.html import Document, clean
|
|
from paper_atlas.io.http import fetch
|
|
|
|
|
|
POSITION_LIST_POLICY = 'https://icml.cc/Conferences/2025/CallForPositionPapers'
|
|
PUBLISHER_LINK_ALIASES = {'v235/theo24a': 'v235/guyard24a'}
|
|
|
|
|
|
def position_list_policy(data):
|
|
"""The organizers explicitly link this title filter as the 2024 position list."""
|
|
for node in Document(data.decode()).root.walk():
|
|
if node.tag != 'a' or clean(node.text()).casefold() != 'position papers':
|
|
continue
|
|
url = node.attrs.get('href', '')
|
|
parsed = urlsplit(url)
|
|
query = parse_qs(parsed.query)
|
|
if (parsed.netloc in ('', 'icml.cc') and parsed.path == '/virtual/2024/papers.html'
|
|
and query.get('filter') == ['titles'] and query.get('search') == ['position:']):
|
|
return {'url': POSITION_LIST_POLICY, 'sha256': hashlib.sha256(data).hexdigest(),
|
|
'position_list_url': 'https://icml.cc' + parsed.path + '?' + parsed.query}
|
|
raise ValueError('Official policy no longer identifies the 2024 position-paper title list')
|
|
|
|
|
|
def program_tracks(data, year, policy=None):
|
|
program = json.loads(data)
|
|
events = program.get('results') if isinstance(program, dict) else None
|
|
if (not isinstance(events, list) or program.get('count') != len(events)
|
|
or program.get('next') or program.get('previous')):
|
|
raise ValueError('Official ICML program must be a complete counted event list')
|
|
if len({str(row['id']) for row in events}) != len(events):
|
|
raise ValueError('Repeated event identity in official ICML program')
|
|
if year == 2024 and policy is None:
|
|
raise ValueError('The 2024 position-list policy is required')
|
|
mapping = {}
|
|
excluded = Counter()
|
|
for row in events:
|
|
number = source_number(row)
|
|
if number is not None and number < 0:
|
|
excluded['synthetic_oral_event'] += 1
|
|
continue
|
|
parsed = urlsplit(row.get('sourceurl') or '')
|
|
group = parse_qs(parsed.query).get('id', [])
|
|
groups = {f'ICML.cc/{year}/Conference': 'main',
|
|
f'ICML.cc/{year}/Position_Paper_Track': 'position'}
|
|
if (parsed.netloc != 'openreview.net' or parsed.path != '/group'
|
|
or len(group) != 1 or group[0] not in groups):
|
|
excluded['other_source'] += 1
|
|
continue
|
|
if not str(row.get('decision') or '').casefold().startswith('accept'):
|
|
raise ValueError('Non-accepted event in official ICML paper program')
|
|
if year == 2024:
|
|
# The 2024 program flattens both tracks into Conference and links
|
|
# PMLR publication IDs instead of forums. Use only this year-specific,
|
|
# organizer-endorsed list rule, never a global title heuristic.
|
|
link = urlsplit(row.get('paper_pdf_url') or '')
|
|
if link.netloc != 'proceedings.mlr.press' or not re.fullmatch(r'/v\d+/[^/]+\.html', link.path):
|
|
raise ValueError('2024 ICML paper has no publisher self-link')
|
|
key = ('pmlr', link.path.lstrip('/').removesuffix('.html'))
|
|
track = 'position' if re.match(r'^Position(?: Paper)?:', row.get('name') or '', re.I) else 'main'
|
|
kind = 'official_2024_position_list'
|
|
else:
|
|
forum = identifiers(row).get('openreview')
|
|
if not forum:
|
|
raise ValueError('ICML paper has no real OpenReview self-identity')
|
|
key = ('openreview', forum)
|
|
track, kind = groups[group[0]], 'official_openreview_track_group'
|
|
evidence = {'track': track, 'kind': kind, 'event_ids': [row['id']],
|
|
'source_group': group[0], 'program_title': row.get('name')}
|
|
if key[0] == 'pmlr' and key[1] in PUBLISHER_LINK_ALIASES:
|
|
evidence['publisher_link_alias'] = key[1]
|
|
evidence['_authors'] = [author['fullname'] for author in row.get('authors', [])]
|
|
key = ('pmlr', PUBLISHER_LINK_ALIASES[key[1]])
|
|
if key in mapping:
|
|
if mapping[key]['track'] != track:
|
|
raise ValueError('One paper identity has conflicting official tracks')
|
|
mapping[key]['event_ids'].append(row['id'])
|
|
else:
|
|
mapping[key] = evidence
|
|
if not mapping:
|
|
raise ValueError('Official ICML program contains no supported paper identities')
|
|
return mapping, {'event_count': len(events), 'paper_identity_count': len(mapping),
|
|
'excluded_events': dict(excluded)}
|
|
|
|
|
|
def collect_track_evidence(year, cache):
|
|
url = f'https://icml.cc/static/virtual/data/icml-{year}-orals-posters.json'
|
|
data = fetch(url, cache)
|
|
policy = position_list_policy(fetch(POSITION_LIST_POLICY, cache)) if year == 2024 else None
|
|
mapping, inventory = program_tracks(data, year, policy)
|
|
provenance = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), **inventory}
|
|
if policy:
|
|
provenance['position_list_policy'] = policy
|
|
return mapping, provenance
|
|
|
|
|
|
def record_track(record, mapping, provenance):
|
|
key = ('pmlr', record['external_id']) if record['year'] == 2024 else ('openreview', record['identifiers'].get('openreview'))
|
|
if key not in mapping:
|
|
raise ValueError('Published ICML paper is absent from the verified official track inventory: ' + record['external_id'])
|
|
evidence = {**mapping[key], 'identity_scheme': key[0], 'identity_value': key[1],
|
|
'program_url': provenance['url'], 'program_sha256': provenance['sha256']}
|
|
if 'publisher_link_alias' in evidence:
|
|
overlap = author_keys(record.get('authors', [])) & author_keys(evidence.pop('_authors', []))
|
|
if normalized(record['title']) != normalized(evidence['program_title']) or len(overlap) < 2:
|
|
raise ValueError('Reviewed publisher-link alias lacks exact title and independent author corroboration')
|
|
evidence['corroborating_author_count'] = len(overlap)
|
|
if 'position_list_policy' in provenance:
|
|
evidence['position_list_policy'] = provenance['position_list_policy']
|
|
record['track'] = evidence['track']
|
|
record['metadata']['track_evidence'] = evidence
|
|
|
|
|
|
def require_track_manifest(manifest):
|
|
if (manifest.get('source') == 'pmlr' and str(manifest.get('venue', '')).upper() == 'ICML'
|
|
and int(manifest.get('year', 0)) >= 2024
|
|
and (manifest.get('parser_version', 0) < 3 or not manifest.get('track_program'))):
|
|
raise ValueError('Recollect mixed ICML PMLR proceedings with parser_version>=3 and official track evidence')
|
|
|
|
|
|
def require_track_record(row, manifest):
|
|
require_track_manifest({**manifest, **{key: row.get(key) for key in ('source', 'venue', 'year')}})
|
|
if manifest.get('track_program'):
|
|
evidence = row.get('metadata', {}).get('track_evidence', {})
|
|
program = manifest['track_program']
|
|
if (evidence.get('track') != row.get('track') or evidence.get('program_url') != program.get('url')
|
|
or evidence.get('program_sha256') != program.get('sha256')
|
|
or not evidence.get('event_ids') or evidence.get('identity_scheme') not in ('pmlr', 'openreview')):
|
|
raise ValueError('PMLR record differs from its official track evidence')
|
|
own = row['external_id'] if evidence['identity_scheme'] == 'pmlr' else row.get('identifiers', {}).get('openreview')
|
|
if not own or evidence.get('identity_value') != own:
|
|
raise ValueError('PMLR track evidence belongs to a different paper identity')
|