Files
paper-atlas/collect_pmlr.py
T
codex-bot ad9967c1b2
Metadata integrity checks / metadata (push) Successful in 4s
Collect official COLM and NeurIPS abstracts with track-aware coverage
2026-11-06 12:04:18 +08:00

208 lines
10 KiB
Python

"""Collect official PMLR metadata, never the PDFs linked by the metadata."""
import argparse
import hashlib
import io
import json
import re
import os
import subprocess
import tarfile
import time
import urllib.error
import urllib.request
from pathlib import Path
import yaml
class MetadataLoader(yaml.SafeLoader):
pass
def binary_text(loader,node):
return loader.construct_yaml_binary(node).decode('utf-8')
MetadataLoader.add_constructor('!binary',binary_text)
MetadataLoader.add_constructor('tag:yaml.org,2002:binary',binary_text)
def load_metadata(value):
documents=[doc for doc in yaml.load_all(value,Loader=MetadataLoader) if doc is not None]
if len(documents)!=1 or not isinstance(documents[0],dict):
raise ValueError('Expected exactly one nonempty metadata mapping')
return documents[0]
def venue_key(shortname):
return re.sub(r'[\s_-]*(?:19|20)\d{2}$','',str(shortname).strip().upper())
def fetch(url,cache,limit=64*1024**2,delay=0.5):
cache = Path(cache)
cache.mkdir(parents=True,exist_ok=True)
path = cache/hashlib.sha256(url.encode()).hexdigest()
if path.exists() and time.time()-path.stat().st_mtime < 24*3600:
return path.read_bytes()
for attempt in range(3):
try:
request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'})
with urllib.request.urlopen(request,timeout=90) as response:
if response.status != 200:
raise RuntimeError('Expected complete HTTP 200 response')
if int(response.headers.get('Content-Length','0'))>limit:
raise RuntimeError('Metadata response exceeds byte limit')
data = response.read(limit+1)
if len(data)>limit:
raise RuntimeError('Metadata response exceeds byte limit')
part = path.with_suffix('.part')
part.write_bytes(data)
part.replace(path)
path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
time.sleep(delay)
return data
except urllib.error.HTTPError as exc:
if exc.code not in (429,500,502,503,504) or attempt==2:
raise
time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1')))))
def metadata_texts(volume,cache):
archive = None
if volume>50:
try:
archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
except RuntimeError as exc:
if 'exceeds byte limit' not in str(exc):
raise
if archive is not None:
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
for member in tar:
if member.isfile() and '/_posts/' in member.name and member.name.endswith(('.md','.markdown','.html')):
if member.size>8*1024**2:
raise RuntimeError('Unexpectedly large metadata member')
yield member.name,tar.extractfile(member).read().decode('utf-8-sig')
return
# Old volumes may contain PDFs. Partial clone plus sparse checkout retrieves
# trees and metadata blobs only, without checking out or executing publisher code.
checkout = Path(cache)/'metadata-checkouts'/f'pmlr-v{volume}'
checkout.parent.mkdir(parents=True,exist_ok=True)
env = dict(os.environ,GIT_TERMINAL_PROMPT='0')
def git(*args):
result = subprocess.run(['git','-c','credential.helper=',*args],env=env,capture_output=True,text=True,timeout=180)
if result.returncode:
raise RuntimeError('Public metadata Git operation failed: '+result.stderr[-500:])
return result.stdout
if not checkout.exists():
git('clone','--filter=blob:none','--depth','1','--single-branch','--branch','gh-pages','--no-checkout',f'https://github.com/mlresearch/v{volume}.git',str(checkout))
git('-C',str(checkout),'sparse-checkout','set','--no-cone','/_posts/','/_config.yml')
git('-C',str(checkout),'checkout','gh-pages')
else:
if git('-C',str(checkout),'status','--porcelain').strip():
raise RuntimeError('Metadata checkout has local changes; refusing to overwrite')
git('-C',str(checkout),'fetch','--filter=blob:none','--depth','1','origin','gh-pages')
git('-C',str(checkout),'checkout','--detach','FETCH_HEAD')
posts = checkout/'_posts'
for path in sorted(posts.rglob('*')):
if path.is_file() and path.suffix in ('.md','.markdown','.html'):
if path.stat().st_size>8*1024**2:
raise RuntimeError('Unexpectedly large metadata member')
yield str(path.relative_to(checkout)),path.read_text(encoding='utf-8-sig')
def collect_volume(volume,cache,out,allowed=None):
config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml'
config = load_metadata(fetch(config_url,cache))
venue = venue_key(config.get('shortname',''))
if allowed and venue not in allowed:
return {'source':'pmlr','volume':volume,'status':'excluded_by_registry','raw_shortname':venue,'booktitle':str(config.get('booktitle',''))}
year_value = config.get('year')
if not year_value:
year_value = str(config.get('start',''))[:4]
if not re.fullmatch(r'\d{4}',str(year_value)):
raise ValueError('No explicit year or conference start date')
year = int(year_value)
title = str(config.get('booktitle',''))
track = 'workshop' if 'workshop' in title.lower() else 'main'
path = Path(out)/f'pmlr-v{volume}.jsonl'
path.parent.mkdir(parents=True,exist_ok=True)
part = path.with_suffix('.part')
count = 0
missing_abstracts = 0
with part.open('w') as output:
for member_name,content in metadata_texts(volume,cache):
parts = re.split(r'(?m)^---\s*$',content,maxsplit=2)
if len(parts)<3 or parts[0].strip():
raise RuntimeError('Missing YAML frontmatter: '+member_name)
row = load_metadata(parts[1])
if row.get('layout') not in ('inproceedings','article'):
continue
if not row.get('title'):
raise ValueError('Paper metadata has no title: '+member_name)
if not (row.get('abstract') or '').strip():
missing_abstracts+=1
external = f"v{volume}/{row.get('id') or Path(member_name).stem}"
ids = {}
for key in ('doi','openreview','arxiv'):
value = row.get(key)
if value:
value = str(value)
if key=='arxiv':
value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
if key=='openreview':
value = value.split('id=')[-1].split('&')[0]
ids[key] = value
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n')
count += 1
if count==0:
part.unlink(missing_ok=True)
raise RuntimeError('No papers parsed; refusing an empty successful collection')
part.replace(path)
result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'missing_abstract_count':missing_abstracts,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'partial' if missing_abstracts else 'collected'}
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
print(json.dumps(result),flush=True)
return result
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--cache',required=True)
parser.add_argument('--out',required=True)
parser.add_argument('--volume',type=int,action='append')
parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json')))
args = parser.parse_args()
allowed = {venue_key(v['acronym']) for v in json.loads(Path(args.registry).read_text())['venues']}
volumes = args.volume
if not volumes:
index = fetch('https://proceedings.mlr.press/',args.cache).decode()
volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index)))
if not volumes:
raise RuntimeError('PMLR index yielded no volumes')
Path(args.out).mkdir(parents=True,exist_ok=True)
catalogue_path=Path(args.out)/'pmlr-volume-status.json'
catalogue=json.loads(catalogue_path.read_text()) if catalogue_path.exists() else {}
errors = []
for volume in volumes:
try:
result=collect_volume(volume,args.cache,args.out,None if args.volume else allowed)
result['selection_mode']='explicit' if args.volume else 'registry'
result['selected_by_registry']=result.get('venue',result.get('raw_shortname')) in allowed
catalogue[str(volume)]=result
except Exception as exc:
error = {'volume':volume,'status':'blocked','error':str(exc)}
errors.append(error)
catalogue[str(volume)]=error
print(json.dumps(error),flush=True)
part=catalogue_path.with_suffix('.part')
part.write_text(json.dumps(catalogue,indent=2))
part.replace(catalogue_path)
(Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2))
(Path(args.out)/'pmlr-last-run.json').write_text(json.dumps({'requested_volumes':volumes,'selection_mode':'explicit' if args.volume else 'registry','errors':errors},indent=2))
if errors:
raise SystemExit(1)
if __name__=='__main__':
main()