Files
paper-atlas/collect_pmlr.py
T
codex-bot 66b98fa375
Metadata integrity checks / metadata (push) Successful in 3s
Fetch old PMLR metadata without entire PDF archives
2026-11-06 03:19:30 +08:00

165 lines
8.0 KiB
Python

"""Collect official PMLR metadata, never the PDFs linked by the metadata."""
import argparse
import hashlib
import io
import json
import re
import os
import subprocess
import tarfile
import time
import urllib.error
import urllib.request
from pathlib import Path
import yaml
def fetch(url,cache,limit=64*1024**2):
cache = Path(cache)
cache.mkdir(parents=True,exist_ok=True)
path = cache/hashlib.sha256(url.encode()).hexdigest()
if path.exists() and time.time()-path.stat().st_mtime < 24*3600:
return path.read_bytes()
for attempt in range(3):
try:
request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'})
with urllib.request.urlopen(request,timeout=90) as response:
if response.status != 200:
raise RuntimeError('Expected complete HTTP 200 response')
if int(response.headers.get('Content-Length','0'))>limit:
raise RuntimeError('Metadata response exceeds byte limit')
data = response.read(limit+1)
if len(data)>limit:
raise RuntimeError('Metadata response exceeds byte limit')
part = path.with_suffix('.part')
part.write_bytes(data)
part.replace(path)
path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
time.sleep(0.5)
return data
except urllib.error.HTTPError as exc:
if exc.code not in (429,500,502,503,504) or attempt==2:
raise
time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1')))))
def metadata_texts(volume,cache):
archive = None
if volume>50:
try:
archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
except RuntimeError as exc:
if 'exceeds byte limit' not in str(exc):
raise
if archive is not None:
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
for member in tar:
if member.isfile() and '/_posts/' in member.name and member.name.endswith(('.md','.markdown','.html')):
if member.size>8*1024**2:
raise RuntimeError('Unexpectedly large metadata member')
yield member.name,tar.extractfile(member).read().decode('utf-8-sig')
return
# Old volumes may contain PDFs. Partial clone plus sparse checkout retrieves
# trees and metadata blobs only, without checking out or executing publisher code.
checkout = Path(cache)/'metadata-checkouts'/f'pmlr-v{volume}'
checkout.parent.mkdir(parents=True,exist_ok=True)
env = dict(os.environ,GIT_TERMINAL_PROMPT='0')
def git(*args):
result = subprocess.run(['git','-c','credential.helper=',*args],env=env,capture_output=True,text=True,timeout=180)
if result.returncode:
raise RuntimeError('Public metadata Git operation failed: '+result.stderr[-500:])
return result.stdout
if not checkout.exists():
git('clone','--filter=blob:none','--depth','1','--single-branch','--branch','gh-pages','--no-checkout',f'https://github.com/mlresearch/v{volume}.git',str(checkout))
git('-C',str(checkout),'sparse-checkout','set','--no-cone','/_posts/','/_config.yml')
git('-C',str(checkout),'checkout','gh-pages')
else:
if git('-C',str(checkout),'status','--porcelain').strip():
raise RuntimeError('Metadata checkout has local changes; refusing to overwrite')
git('-C',str(checkout),'fetch','--filter=blob:none','--depth','1','origin','gh-pages')
git('-C',str(checkout),'checkout','--detach','FETCH_HEAD')
posts = checkout/'_posts'
for path in sorted(posts.rglob('*')):
if path.is_file() and path.suffix in ('.md','.markdown','.html'):
if path.stat().st_size>8*1024**2:
raise RuntimeError('Unexpectedly large metadata member')
yield str(path.relative_to(checkout)),path.read_text(encoding='utf-8-sig')
def collect_volume(volume,cache,out,allowed=None):
config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml'
config = yaml.safe_load(fetch(config_url,cache))
venue = str(config.get('shortname','')).upper()
if allowed and venue not in allowed:
return None
year = int(config['year'])
title = str(config.get('booktitle',''))
track = 'workshop' if 'workshop' in title.lower() else 'main'
path = Path(out)/f'pmlr-v{volume}.jsonl'
path.parent.mkdir(parents=True,exist_ok=True)
part = path.with_suffix('.part')
count = 0
with part.open('w') as output:
for member_name,content in metadata_texts(volume,cache):
parts = content.split('---',2)
if len(parts)<3 or parts[0].strip():
raise RuntimeError('Missing YAML frontmatter: '+member_name)
row = yaml.safe_load(parts[1])
if row.get('layout') not in ('inproceedings','article') or not row.get('title'):
continue
external = f"v{volume}/{row.get('id') or Path(member_name).stem}"
ids = {}
for key in ('doi','openreview','arxiv'):
value = row.get(key)
if value:
value = str(value)
if key=='arxiv':
value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
if key=='openreview':
value = value.split('id=')[-1].split('&')[0]
ids[key] = value
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n')
count += 1
if count==0:
part.unlink(missing_ok=True)
raise RuntimeError('No papers parsed; refusing an empty successful collection')
part.replace(path)
result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'}
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
print(json.dumps(result),flush=True)
return result
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--cache',required=True)
parser.add_argument('--out',required=True)
parser.add_argument('--volume',type=int,action='append')
parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json')))
args = parser.parse_args()
allowed = {v['acronym'].upper() for v in json.loads(Path(args.registry).read_text())['venues']}
volumes = args.volume
if not volumes:
index = fetch('https://proceedings.mlr.press/',args.cache).decode()
volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index)))
if not volumes:
raise RuntimeError('PMLR index yielded no volumes')
errors = []
for volume in volumes:
try:
collect_volume(volume,args.cache,args.out,None if args.volume else allowed)
except Exception as exc:
error = {'volume':volume,'status':'blocked','error':str(exc)}
errors.append(error)
print(json.dumps(error),flush=True)
Path(args.out).mkdir(parents=True,exist_ok=True)
(Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2))
if errors:
raise SystemExit(1)
if __name__=='__main__':
main()