Files
paper-atlas/collect_pmlr.py
T
2026-11-06 11:34:58 +08:00

203 lines
9.8 KiB
Python

"""Collect official PMLR metadata, never the PDFs linked by the metadata."""
import argparse
import hashlib
import io
import json
import re
import os
import subprocess
import tarfile
import time
import urllib.error
import urllib.request
from pathlib import Path
import yaml
class MetadataLoader(yaml.SafeLoader):
pass
def binary_text(loader,node):
return loader.construct_yaml_binary(node).decode('utf-8')
MetadataLoader.add_constructor('!binary',binary_text)
MetadataLoader.add_constructor('tag:yaml.org,2002:binary',binary_text)
def load_metadata(value):
documents=[doc for doc in yaml.load_all(value,Loader=MetadataLoader) if doc is not None]
if len(documents)!=1 or not isinstance(documents[0],dict):
raise ValueError('Expected exactly one nonempty metadata mapping')
return documents[0]
def venue_key(shortname):
return re.sub(r'[\s_-]*(?:19|20)\d{2}$','',str(shortname).strip().upper())
def fetch(url,cache,limit=64*1024**2):
cache = Path(cache)
cache.mkdir(parents=True,exist_ok=True)
path = cache/hashlib.sha256(url.encode()).hexdigest()
if path.exists() and time.time()-path.stat().st_mtime < 24*3600:
return path.read_bytes()
for attempt in range(3):
try:
request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'})
with urllib.request.urlopen(request,timeout=90) as response:
if response.status != 200:
raise RuntimeError('Expected complete HTTP 200 response')
if int(response.headers.get('Content-Length','0'))>limit:
raise RuntimeError('Metadata response exceeds byte limit')
data = response.read(limit+1)
if len(data)>limit:
raise RuntimeError('Metadata response exceeds byte limit')
part = path.with_suffix('.part')
part.write_bytes(data)
part.replace(path)
path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
time.sleep(0.5)
return data
except urllib.error.HTTPError as exc:
if exc.code not in (429,500,502,503,504) or attempt==2:
raise
time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1')))))
def metadata_texts(volume,cache):
archive = None
if volume>50:
try:
archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
except RuntimeError as exc:
if 'exceeds byte limit' not in str(exc):
raise
if archive is not None:
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
for member in tar:
if member.isfile() and '/_posts/' in member.name and member.name.endswith(('.md','.markdown','.html')):
if member.size>8*1024**2:
raise RuntimeError('Unexpectedly large metadata member')
yield member.name,tar.extractfile(member).read().decode('utf-8-sig')
return
# Old volumes may contain PDFs. Partial clone plus sparse checkout retrieves
# trees and metadata blobs only, without checking out or executing publisher code.
checkout = Path(cache)/'metadata-checkouts'/f'pmlr-v{volume}'
checkout.parent.mkdir(parents=True,exist_ok=True)
env = dict(os.environ,GIT_TERMINAL_PROMPT='0')
def git(*args):
result = subprocess.run(['git','-c','credential.helper=',*args],env=env,capture_output=True,text=True,timeout=180)
if result.returncode:
raise RuntimeError('Public metadata Git operation failed: '+result.stderr[-500:])
return result.stdout
if not checkout.exists():
git('clone','--filter=blob:none','--depth','1','--single-branch','--branch','gh-pages','--no-checkout',f'https://github.com/mlresearch/v{volume}.git',str(checkout))
git('-C',str(checkout),'sparse-checkout','set','--no-cone','/_posts/','/_config.yml')
git('-C',str(checkout),'checkout','gh-pages')
else:
if git('-C',str(checkout),'status','--porcelain').strip():
raise RuntimeError('Metadata checkout has local changes; refusing to overwrite')
git('-C',str(checkout),'fetch','--filter=blob:none','--depth','1','origin','gh-pages')
git('-C',str(checkout),'checkout','--detach','FETCH_HEAD')
posts = checkout/'_posts'
for path in sorted(posts.rglob('*')):
if path.is_file() and path.suffix in ('.md','.markdown','.html'):
if path.stat().st_size>8*1024**2:
raise RuntimeError('Unexpectedly large metadata member')
yield str(path.relative_to(checkout)),path.read_text(encoding='utf-8-sig')
def collect_volume(volume,cache,out,allowed=None):
config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml'
config = load_metadata(fetch(config_url,cache))
venue = venue_key(config.get('shortname',''))
if allowed and venue not in allowed:
return {'source':'pmlr','volume':volume,'status':'excluded_by_registry','raw_shortname':venue,'booktitle':str(config.get('booktitle',''))}
year_value = config.get('year')
if not year_value:
year_value = str(config.get('start',''))[:4]
if not re.fullmatch(r'\d{4}',str(year_value)):
raise ValueError('No explicit year or conference start date')
year = int(year_value)
title = str(config.get('booktitle',''))
track = 'workshop' if 'workshop' in title.lower() else 'main'
path = Path(out)/f'pmlr-v{volume}.jsonl'
path.parent.mkdir(parents=True,exist_ok=True)
part = path.with_suffix('.part')
count = 0
with part.open('w') as output:
for member_name,content in metadata_texts(volume,cache):
parts = re.split(r'(?m)^---\s*$',content,maxsplit=2)
if len(parts)<3 or parts[0].strip():
raise RuntimeError('Missing YAML frontmatter: '+member_name)
row = load_metadata(parts[1])
if row.get('layout') not in ('inproceedings','article') or not row.get('title'):
continue
external = f"v{volume}/{row.get('id') or Path(member_name).stem}"
ids = {}
for key in ('doi','openreview','arxiv'):
value = row.get(key)
if value:
value = str(value)
if key=='arxiv':
value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
if key=='openreview':
value = value.split('id=')[-1].split('&')[0]
ids[key] = value
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n')
count += 1
if count==0:
part.unlink(missing_ok=True)
raise RuntimeError('No papers parsed; refusing an empty successful collection')
part.replace(path)
result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'}
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
print(json.dumps(result),flush=True)
return result
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--cache',required=True)
parser.add_argument('--out',required=True)
parser.add_argument('--volume',type=int,action='append')
parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json')))
args = parser.parse_args()
allowed = {venue_key(v['acronym']) for v in json.loads(Path(args.registry).read_text())['venues']}
volumes = args.volume
if not volumes:
index = fetch('https://proceedings.mlr.press/',args.cache).decode()
volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index)))
if not volumes:
raise RuntimeError('PMLR index yielded no volumes')
Path(args.out).mkdir(parents=True,exist_ok=True)
catalogue_path=Path(args.out)/'pmlr-volume-status.json'
catalogue=json.loads(catalogue_path.read_text()) if catalogue_path.exists() else {}
errors = []
for volume in volumes:
try:
result=collect_volume(volume,args.cache,args.out,None if args.volume else allowed)
result['selection_mode']='explicit' if args.volume else 'registry'
result['selected_by_registry']=result.get('venue',result.get('raw_shortname')) in allowed
catalogue[str(volume)]=result
except Exception as exc:
error = {'volume':volume,'status':'blocked','error':str(exc)}
errors.append(error)
catalogue[str(volume)]=error
print(json.dumps(error),flush=True)
part=catalogue_path.with_suffix('.part')
part.write_text(json.dumps(catalogue,indent=2))
part.replace(catalogue_path)
(Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2))
(Path(args.out)/'pmlr-last-run.json').write_text(json.dumps({'requested_volumes':volumes,'selection_mode':'explicit' if args.volume else 'registry','errors':errors},indent=2))
if errors:
raise SystemExit(1)
if __name__=='__main__':
main()