230 lines
12 KiB
Python
230 lines
12 KiB
Python
"""Collect official PMLR metadata, never the PDFs linked by the metadata."""
|
|
import argparse
|
|
from collections import defaultdict
|
|
import hashlib
|
|
from paper_atlas.config import DEFAULT_REGISTRY
|
|
import io
|
|
import json
|
|
import re
|
|
import os
|
|
import subprocess
|
|
import tarfile
|
|
import time
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
|
|
class MetadataLoader(yaml.SafeLoader):
|
|
pass
|
|
|
|
|
|
def binary_text(loader,node):
|
|
return loader.construct_yaml_binary(node).decode('utf-8')
|
|
|
|
|
|
MetadataLoader.add_constructor('!binary',binary_text)
|
|
MetadataLoader.add_constructor('tag:yaml.org,2002:binary',binary_text)
|
|
|
|
|
|
def load_metadata(value):
|
|
documents=[doc for doc in yaml.load_all(value,Loader=MetadataLoader) if doc is not None]
|
|
if len(documents)!=1 or not isinstance(documents[0],dict):
|
|
raise ValueError('Expected exactly one nonempty metadata mapping')
|
|
return documents[0]
|
|
|
|
|
|
def venue_key(shortname):
|
|
return re.sub(r'[\s_-]*(?:19|20)\d{2}$','',str(shortname).strip().upper())
|
|
|
|
|
|
from paper_atlas.io.http import fetch
|
|
from paper_atlas.collectors.pmlr_tracks import collect_track_evidence, record_track
|
|
|
|
|
|
def metadata_texts(volume,cache):
|
|
archive = None
|
|
if volume>50:
|
|
try:
|
|
archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
|
|
except RuntimeError as exc:
|
|
if 'exceeds byte limit' not in str(exc):
|
|
raise
|
|
if archive is not None:
|
|
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
|
|
for member in tar:
|
|
if member.isfile() and '/_posts/' in member.name and member.name.endswith(('.md','.markdown','.html')):
|
|
if member.size>8*1024**2:
|
|
raise RuntimeError('Unexpectedly large metadata member')
|
|
yield member.name,tar.extractfile(member).read().decode('utf-8-sig')
|
|
return
|
|
# Old volumes may contain PDFs. Partial clone plus sparse checkout retrieves
|
|
# trees and metadata blobs only, without checking out or executing publisher code.
|
|
checkout = Path(cache)/'metadata-checkouts'/f'pmlr-v{volume}'
|
|
checkout.parent.mkdir(parents=True,exist_ok=True)
|
|
env = dict(os.environ,GIT_TERMINAL_PROMPT='0')
|
|
def git(*args):
|
|
result = subprocess.run(['git','-c','credential.helper=',*args],env=env,capture_output=True,text=True,timeout=180)
|
|
if result.returncode:
|
|
raise RuntimeError('Public metadata Git operation failed: '+result.stderr[-500:])
|
|
return result.stdout
|
|
if not checkout.exists():
|
|
git('clone','--filter=blob:none','--depth','1','--single-branch','--branch','gh-pages','--no-checkout',f'https://github.com/mlresearch/v{volume}.git',str(checkout))
|
|
git('-C',str(checkout),'sparse-checkout','set','--no-cone','/_posts/','/_config.yml')
|
|
git('-C',str(checkout),'checkout','gh-pages')
|
|
else:
|
|
if git('-C',str(checkout),'status','--porcelain').strip():
|
|
raise RuntimeError('Metadata checkout has local changes; refusing to overwrite')
|
|
git('-C',str(checkout),'fetch','--filter=blob:none','--depth','1','origin','gh-pages')
|
|
git('-C',str(checkout),'checkout','--detach','FETCH_HEAD')
|
|
posts = checkout/'_posts'
|
|
for path in sorted(posts.rglob('*')):
|
|
if path.is_file() and path.suffix in ('.md','.markdown','.html'):
|
|
if path.stat().st_size>8*1024**2:
|
|
raise RuntimeError('Unexpectedly large metadata member')
|
|
yield str(path.relative_to(checkout)),path.read_text(encoding='utf-8-sig')
|
|
|
|
|
|
def collect_volume(volume,cache,out,allowed=None):
|
|
config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml'
|
|
config = load_metadata(fetch(config_url,cache))
|
|
venue = venue_key(config.get('shortname',''))
|
|
if allowed and venue not in allowed:
|
|
return {'source':'pmlr','volume':volume,'status':'excluded_by_registry','raw_shortname':venue,'booktitle':str(config.get('booktitle',''))}
|
|
year_value = config.get('year')
|
|
if not year_value:
|
|
year_value = str(config.get('start',''))[:4]
|
|
if not re.fullmatch(r'\d{4}',str(year_value)):
|
|
raise ValueError('No explicit year or conference start date')
|
|
year = int(year_value)
|
|
title = str(config.get('booktitle',''))
|
|
track = 'workshop' if 'workshop' in title.lower() else 'main'
|
|
path = Path(out)/f'pmlr-v{volume}.jsonl'
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
mixed_icml = venue == 'ICML' and year >= 2024
|
|
mapping, track_program = collect_track_evidence(year, cache) if mixed_icml else ({}, None)
|
|
records = {}
|
|
duplicate_posts = 0
|
|
for member_name,content in metadata_texts(volume,cache):
|
|
parts = re.split(r'(?m)^---\s*$',content,maxsplit=2)
|
|
if len(parts)<3 or parts[0].strip():
|
|
raise RuntimeError('Missing YAML frontmatter: '+member_name)
|
|
row = load_metadata(parts[1])
|
|
if row.get('layout') not in ('inproceedings','article'):
|
|
continue
|
|
if not row.get('title'):
|
|
raise ValueError('Paper metadata has no title: '+member_name)
|
|
if not row.get('id'):
|
|
raise ValueError('Paper metadata has no publication ID: '+member_name)
|
|
external = f"v{volume}/{row['id']}"
|
|
ids = {}
|
|
for key in ('doi','openreview','arxiv'):
|
|
value = row.get(key)
|
|
if value:
|
|
value = str(value)
|
|
if key=='arxiv':
|
|
value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
|
|
if key=='openreview':
|
|
value = value.split('id=')[-1].split('&')[0]
|
|
ids[key] = value
|
|
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
|
|
version={'path':member_name,'date':row.get('date')}
|
|
if external in records:
|
|
previous=records[external]
|
|
if ({k:v for k,v in previous.items() if k!='metadata'} != {k:v for k,v in record.items() if k!='metadata'}
|
|
or {k:v for k,v in previous['metadata'].items() if k not in ('date','source_versions')} != {k:v for k,v in record['metadata'].items() if k!='date'}):
|
|
raise ValueError('Conflicting metadata posts share a publication ID: '+external)
|
|
previous['metadata']['source_versions'].append(version)
|
|
if str(row.get('date') or '')>str(previous['metadata'].get('date') or ''):
|
|
previous['metadata']['date']=row['date']
|
|
duplicate_posts+=1
|
|
continue
|
|
record['metadata']['source_versions']=[version]
|
|
records[external]=record
|
|
count=len(records)
|
|
missing_abstracts=sum(not (record.get('abstract') or '').strip() for record in records.values())
|
|
if count==0:
|
|
raise RuntimeError('No papers parsed; refusing an empty successful collection')
|
|
grouped = defaultdict(list)
|
|
for record in records.values():
|
|
if mixed_icml:
|
|
record_track(record, mapping, track_program)
|
|
grouped[record['track']].append(record)
|
|
# Parse and verify the entire volume before replacing any published output.
|
|
manifests = []
|
|
with tempfile.TemporaryDirectory(prefix=f'.pmlr-v{volume}-', dir=path.parent) as temporary:
|
|
stage = Path(temporary)
|
|
for current_track, rows in sorted(grouped.items()):
|
|
filename = f'pmlr-v{volume}-{current_track}.jsonl' if mixed_icml else path.name
|
|
output = stage / filename
|
|
output.write_text(''.join(json.dumps(row, ensure_ascii=False, default=str)+'\n' for row in rows))
|
|
absent = sum(not (row.get('abstract') or '').strip() for row in rows)
|
|
manifest = {'source': 'pmlr', 'parser_version': 3, 'venue': venue, 'year': year,
|
|
'track': current_track, 'volume': volume, 'count': len(rows),
|
|
'missing_abstract_count': absent, 'list_url': f'https://proceedings.mlr.press/v{volume}/',
|
|
'file': str(path.parent / filename), 'file_sha256': hashlib.sha256(output.read_bytes()).hexdigest(),
|
|
'status': 'partial' if absent else 'collected'}
|
|
if track_program:
|
|
manifest['track_program'] = track_program
|
|
output.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
|
manifests.append(manifest)
|
|
result = {'source': 'pmlr', 'parser_version': 3, 'venue': venue, 'year': year,
|
|
'track': next(iter(grouped)) if len(grouped) == 1 else 'mixed', 'volume': volume,
|
|
'count': count, 'metadata_post_count': count + duplicate_posts,
|
|
'duplicate_post_count': duplicate_posts, 'missing_abstract_count': missing_abstracts,
|
|
'list_url': f'https://proceedings.mlr.press/v{volume}/', 'files': manifests,
|
|
'status': 'partial' if missing_abstracts else 'collected'}
|
|
if len(manifests) == 1:
|
|
result['file'] = manifests[0]['file']
|
|
if track_program:
|
|
result['track_program'] = track_program
|
|
inventory = stage / f'pmlr-v{volume}.inventory.json'
|
|
inventory.write_text(json.dumps(result, indent=2)+'\n')
|
|
for child in sorted(stage.iterdir(), key=lambda child: child == inventory):
|
|
child.replace(path.parent / child.name)
|
|
print(json.dumps(result), flush=True)
|
|
return result
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog)
|
|
parser.add_argument('--cache',required=True)
|
|
parser.add_argument('--out',required=True)
|
|
parser.add_argument('--volume',type=int,action='append')
|
|
parser.add_argument('--registry',default=str(DEFAULT_REGISTRY))
|
|
args = parser.parse_args(argv)
|
|
allowed = {venue_key(v['acronym']) for v in json.loads(Path(args.registry).read_text())['venues']}
|
|
volumes = args.volume
|
|
if not volumes:
|
|
index = fetch('https://proceedings.mlr.press/',args.cache).decode()
|
|
volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index)))
|
|
if not volumes:
|
|
raise RuntimeError('PMLR index yielded no volumes')
|
|
Path(args.out).mkdir(parents=True,exist_ok=True)
|
|
catalogue_path=Path(args.out)/'pmlr-volume-status.json'
|
|
catalogue=json.loads(catalogue_path.read_text()) if catalogue_path.exists() else {}
|
|
errors = []
|
|
for volume in volumes:
|
|
try:
|
|
result=collect_volume(volume,args.cache,args.out,None if args.volume else allowed)
|
|
result['selection_mode']='explicit' if args.volume else 'registry'
|
|
result['selected_by_registry']=result.get('venue',result.get('raw_shortname')) in allowed
|
|
catalogue[str(volume)]=result
|
|
except Exception as exc:
|
|
error = {'volume':volume,'status':'blocked','error':str(exc)}
|
|
errors.append(error)
|
|
catalogue[str(volume)]=error
|
|
print(json.dumps(error),flush=True)
|
|
part=catalogue_path.with_suffix('.part')
|
|
part.write_text(json.dumps(catalogue,indent=2))
|
|
part.replace(catalogue_path)
|
|
(Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2))
|
|
(Path(args.out)/'pmlr-last-run.json').write_text(json.dumps({'requested_volumes':volumes,'selection_mode':'explicit' if args.volume else 'registry','errors':errors},indent=2))
|
|
if errors:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|