Files
codex-bot 17bdaa77fb
Metadata integrity checks / metadata (push) Successful in 40s
Resolve mixed ICML PMLR tracks from verified official programs
2026-11-06 22:12:08 +08:00

230 lines
12 KiB
Python

"""Collect official PMLR metadata, never the PDFs linked by the metadata."""
import argparse
from collections import defaultdict
import hashlib
from paper_atlas.config import DEFAULT_REGISTRY
import io
import json
import re
import os
import subprocess
import tarfile
import time
import tempfile
from pathlib import Path
import yaml
class MetadataLoader(yaml.SafeLoader):
pass
def binary_text(loader,node):
return loader.construct_yaml_binary(node).decode('utf-8')
MetadataLoader.add_constructor('!binary',binary_text)
MetadataLoader.add_constructor('tag:yaml.org,2002:binary',binary_text)
def load_metadata(value):
documents=[doc for doc in yaml.load_all(value,Loader=MetadataLoader) if doc is not None]
if len(documents)!=1 or not isinstance(documents[0],dict):
raise ValueError('Expected exactly one nonempty metadata mapping')
return documents[0]
def venue_key(shortname):
return re.sub(r'[\s_-]*(?:19|20)\d{2}$','',str(shortname).strip().upper())
from paper_atlas.io.http import fetch
from paper_atlas.collectors.pmlr_tracks import collect_track_evidence, record_track
def metadata_texts(volume,cache):
archive = None
if volume>50:
try:
archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
except RuntimeError as exc:
if 'exceeds byte limit' not in str(exc):
raise
if archive is not None:
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
for member in tar:
if member.isfile() and '/_posts/' in member.name and member.name.endswith(('.md','.markdown','.html')):
if member.size>8*1024**2:
raise RuntimeError('Unexpectedly large metadata member')
yield member.name,tar.extractfile(member).read().decode('utf-8-sig')
return
# Old volumes may contain PDFs. Partial clone plus sparse checkout retrieves
# trees and metadata blobs only, without checking out or executing publisher code.
checkout = Path(cache)/'metadata-checkouts'/f'pmlr-v{volume}'
checkout.parent.mkdir(parents=True,exist_ok=True)
env = dict(os.environ,GIT_TERMINAL_PROMPT='0')
def git(*args):
result = subprocess.run(['git','-c','credential.helper=',*args],env=env,capture_output=True,text=True,timeout=180)
if result.returncode:
raise RuntimeError('Public metadata Git operation failed: '+result.stderr[-500:])
return result.stdout
if not checkout.exists():
git('clone','--filter=blob:none','--depth','1','--single-branch','--branch','gh-pages','--no-checkout',f'https://github.com/mlresearch/v{volume}.git',str(checkout))
git('-C',str(checkout),'sparse-checkout','set','--no-cone','/_posts/','/_config.yml')
git('-C',str(checkout),'checkout','gh-pages')
else:
if git('-C',str(checkout),'status','--porcelain').strip():
raise RuntimeError('Metadata checkout has local changes; refusing to overwrite')
git('-C',str(checkout),'fetch','--filter=blob:none','--depth','1','origin','gh-pages')
git('-C',str(checkout),'checkout','--detach','FETCH_HEAD')
posts = checkout/'_posts'
for path in sorted(posts.rglob('*')):
if path.is_file() and path.suffix in ('.md','.markdown','.html'):
if path.stat().st_size>8*1024**2:
raise RuntimeError('Unexpectedly large metadata member')
yield str(path.relative_to(checkout)),path.read_text(encoding='utf-8-sig')
def collect_volume(volume,cache,out,allowed=None):
config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml'
config = load_metadata(fetch(config_url,cache))
venue = venue_key(config.get('shortname',''))
if allowed and venue not in allowed:
return {'source':'pmlr','volume':volume,'status':'excluded_by_registry','raw_shortname':venue,'booktitle':str(config.get('booktitle',''))}
year_value = config.get('year')
if not year_value:
year_value = str(config.get('start',''))[:4]
if not re.fullmatch(r'\d{4}',str(year_value)):
raise ValueError('No explicit year or conference start date')
year = int(year_value)
title = str(config.get('booktitle',''))
track = 'workshop' if 'workshop' in title.lower() else 'main'
path = Path(out)/f'pmlr-v{volume}.jsonl'
path.parent.mkdir(parents=True,exist_ok=True)
mixed_icml = venue == 'ICML' and year >= 2024
mapping, track_program = collect_track_evidence(year, cache) if mixed_icml else ({}, None)
records = {}
duplicate_posts = 0
for member_name,content in metadata_texts(volume,cache):
parts = re.split(r'(?m)^---\s*$',content,maxsplit=2)
if len(parts)<3 or parts[0].strip():
raise RuntimeError('Missing YAML frontmatter: '+member_name)
row = load_metadata(parts[1])
if row.get('layout') not in ('inproceedings','article'):
continue
if not row.get('title'):
raise ValueError('Paper metadata has no title: '+member_name)
if not row.get('id'):
raise ValueError('Paper metadata has no publication ID: '+member_name)
external = f"v{volume}/{row['id']}"
ids = {}
for key in ('doi','openreview','arxiv'):
value = row.get(key)
if value:
value = str(value)
if key=='arxiv':
value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
if key=='openreview':
value = value.split('id=')[-1].split('&')[0]
ids[key] = value
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
version={'path':member_name,'date':row.get('date')}
if external in records:
previous=records[external]
if ({k:v for k,v in previous.items() if k!='metadata'} != {k:v for k,v in record.items() if k!='metadata'}
or {k:v for k,v in previous['metadata'].items() if k not in ('date','source_versions')} != {k:v for k,v in record['metadata'].items() if k!='date'}):
raise ValueError('Conflicting metadata posts share a publication ID: '+external)
previous['metadata']['source_versions'].append(version)
if str(row.get('date') or '')>str(previous['metadata'].get('date') or ''):
previous['metadata']['date']=row['date']
duplicate_posts+=1
continue
record['metadata']['source_versions']=[version]
records[external]=record
count=len(records)
missing_abstracts=sum(not (record.get('abstract') or '').strip() for record in records.values())
if count==0:
raise RuntimeError('No papers parsed; refusing an empty successful collection')
grouped = defaultdict(list)
for record in records.values():
if mixed_icml:
record_track(record, mapping, track_program)
grouped[record['track']].append(record)
# Parse and verify the entire volume before replacing any published output.
manifests = []
with tempfile.TemporaryDirectory(prefix=f'.pmlr-v{volume}-', dir=path.parent) as temporary:
stage = Path(temporary)
for current_track, rows in sorted(grouped.items()):
filename = f'pmlr-v{volume}-{current_track}.jsonl' if mixed_icml else path.name
output = stage / filename
output.write_text(''.join(json.dumps(row, ensure_ascii=False, default=str)+'\n' for row in rows))
absent = sum(not (row.get('abstract') or '').strip() for row in rows)
manifest = {'source': 'pmlr', 'parser_version': 3, 'venue': venue, 'year': year,
'track': current_track, 'volume': volume, 'count': len(rows),
'missing_abstract_count': absent, 'list_url': f'https://proceedings.mlr.press/v{volume}/',
'file': str(path.parent / filename), 'file_sha256': hashlib.sha256(output.read_bytes()).hexdigest(),
'status': 'partial' if absent else 'collected'}
if track_program:
manifest['track_program'] = track_program
output.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
manifests.append(manifest)
result = {'source': 'pmlr', 'parser_version': 3, 'venue': venue, 'year': year,
'track': next(iter(grouped)) if len(grouped) == 1 else 'mixed', 'volume': volume,
'count': count, 'metadata_post_count': count + duplicate_posts,
'duplicate_post_count': duplicate_posts, 'missing_abstract_count': missing_abstracts,
'list_url': f'https://proceedings.mlr.press/v{volume}/', 'files': manifests,
'status': 'partial' if missing_abstracts else 'collected'}
if len(manifests) == 1:
result['file'] = manifests[0]['file']
if track_program:
result['track_program'] = track_program
inventory = stage / f'pmlr-v{volume}.inventory.json'
inventory.write_text(json.dumps(result, indent=2)+'\n')
for child in sorted(stage.iterdir(), key=lambda child: child == inventory):
child.replace(path.parent / child.name)
print(json.dumps(result), flush=True)
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--cache',required=True)
parser.add_argument('--out',required=True)
parser.add_argument('--volume',type=int,action='append')
parser.add_argument('--registry',default=str(DEFAULT_REGISTRY))
args = parser.parse_args(argv)
allowed = {venue_key(v['acronym']) for v in json.loads(Path(args.registry).read_text())['venues']}
volumes = args.volume
if not volumes:
index = fetch('https://proceedings.mlr.press/',args.cache).decode()
volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index)))
if not volumes:
raise RuntimeError('PMLR index yielded no volumes')
Path(args.out).mkdir(parents=True,exist_ok=True)
catalogue_path=Path(args.out)/'pmlr-volume-status.json'
catalogue=json.loads(catalogue_path.read_text()) if catalogue_path.exists() else {}
errors = []
for volume in volumes:
try:
result=collect_volume(volume,args.cache,args.out,None if args.volume else allowed)
result['selection_mode']='explicit' if args.volume else 'registry'
result['selected_by_registry']=result.get('venue',result.get('raw_shortname')) in allowed
catalogue[str(volume)]=result
except Exception as exc:
error = {'volume':volume,'status':'blocked','error':str(exc)}
errors.append(error)
catalogue[str(volume)]=error
print(json.dumps(error),flush=True)
part=catalogue_path.with_suffix('.part')
part.write_text(json.dumps(catalogue,indent=2))
part.replace(catalogue_path)
(Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2))
(Path(args.out)/'pmlr-last-run.json').write_text(json.dumps({'requested_volumes':volumes,'selection_mode':'explicit' if args.volume else 'registry','errors':errors},indent=2))
if errors:
raise SystemExit(1)
if __name__=='__main__':
main()