Files
paper-atlas/collect_pmlr.py
T
codex-bot 4a4272219e
Metadata integrity checks / metadata (push) Successful in 2s
Collect explicit public ICLR decisions and refresh metadata indexes
2026-11-06 03:12:44 +08:00

124 lines
5.9 KiB
Python

"""Collect official PMLR metadata, never the PDFs linked by the metadata."""
import argparse
import hashlib
import io
import json
import re
import tarfile
import time
import urllib.error
import urllib.request
from pathlib import Path
import yaml
def fetch(url,cache,limit=64*1024**2):
cache = Path(cache)
cache.mkdir(parents=True,exist_ok=True)
path = cache/hashlib.sha256(url.encode()).hexdigest()
if path.exists() and time.time()-path.stat().st_mtime < 24*3600:
return path.read_bytes()
for attempt in range(3):
try:
request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'})
with urllib.request.urlopen(request,timeout=90) as response:
if response.status != 200:
raise RuntimeError('Expected complete HTTP 200 response')
data = response.read(limit+1)
if len(data)>limit:
raise RuntimeError('Metadata response exceeds byte limit')
part = path.with_suffix('.part')
part.write_bytes(data)
part.replace(path)
path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
time.sleep(0.5)
return data
except urllib.error.HTTPError as exc:
if exc.code not in (429,500,502,503,504) or attempt==2:
raise
time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1')))))
def collect_volume(volume,cache,out,allowed=None):
config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml'
config = yaml.safe_load(fetch(config_url,cache))
venue = str(config.get('shortname','')).upper()
if allowed and venue not in allowed:
return None
year = int(config['year'])
title = str(config.get('booktitle',''))
track = 'workshop' if 'workshop' in title.lower() else 'main'
archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
path = Path(out)/f'pmlr-v{volume}.jsonl'
path.parent.mkdir(parents=True,exist_ok=True)
part = path.with_suffix('.part')
count = 0
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar,part.open('w') as output:
for member in tar:
if not member.isfile() or '/_posts/' not in member.name or not member.name.endswith(('.md','.markdown','.html')):
continue
if member.size>8*1024**2:
raise RuntimeError('Unexpectedly large metadata member')
content = tar.extractfile(member).read().decode('utf-8-sig')
parts = content.split('---',2)
if len(parts)<3 or parts[0].strip():
raise RuntimeError('Missing YAML frontmatter: '+member.name)
row = yaml.safe_load(parts[1])
if row.get('layout') not in ('inproceedings','article') or not row.get('title'):
continue
external = f"v{volume}/{row.get('id') or Path(member.name).stem}"
ids = {}
for key in ('doi','openreview','arxiv'):
value = row.get(key)
if value:
value = str(value)
if key=='arxiv':
value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
if key=='openreview':
value = value.split('id=')[-1].split('&')[0]
ids[key] = value
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n')
count += 1
if count==0:
part.unlink(missing_ok=True)
raise RuntimeError('No papers parsed; refusing an empty successful collection')
part.replace(path)
result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'}
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
print(json.dumps(result),flush=True)
return result
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--cache',required=True)
parser.add_argument('--out',required=True)
parser.add_argument('--volume',type=int,action='append')
parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json')))
args = parser.parse_args()
allowed = {v['acronym'].upper() for v in json.loads(Path(args.registry).read_text())['venues']}
volumes = args.volume
if not volumes:
index = fetch('https://proceedings.mlr.press/',args.cache).decode()
volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index)))
if not volumes:
raise RuntimeError('PMLR index yielded no volumes')
errors = []
for volume in volumes:
try:
collect_volume(volume,args.cache,args.out,None if args.volume else allowed)
except Exception as exc:
error = {'volume':volume,'status':'blocked','error':str(exc)}
errors.append(error)
print(json.dumps(error),flush=True)
Path(args.out).mkdir(parents=True,exist_ok=True)
(Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2))
if errors:
raise SystemExit(1)
if __name__=='__main__':
main()