"""Collect official PMLR metadata, never the PDFs linked by the metadata.""" import argparse import hashlib import io import json import re import os import subprocess import tarfile import time import urllib.error import urllib.request from pathlib import Path import yaml def fetch(url,cache,limit=64*1024**2): cache = Path(cache) cache.mkdir(parents=True,exist_ok=True) path = cache/hashlib.sha256(url.encode()).hexdigest() if path.exists() and time.time()-path.stat().st_mtime < 24*3600: return path.read_bytes() for attempt in range(3): try: request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'}) with urllib.request.urlopen(request,timeout=90) as response: if response.status != 200: raise RuntimeError('Expected complete HTTP 200 response') if int(response.headers.get('Content-Length','0'))>limit: raise RuntimeError('Metadata response exceeds byte limit') data = response.read(limit+1) if len(data)>limit: raise RuntimeError('Metadata response exceeds byte limit') part = path.with_suffix('.part') part.write_bytes(data) part.replace(path) path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())})) time.sleep(0.5) return data except urllib.error.HTTPError as exc: if exc.code not in (429,500,502,503,504) or attempt==2: raise time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1'))))) def metadata_texts(volume,cache): archive = None if volume>50: try: archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache) except RuntimeError as exc: if 'exceeds byte limit' not in str(exc): raise if archive is not None: with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar: for member in tar: if member.isfile() and '/_posts/' in member.name and member.name.endswith(('.md','.markdown','.html')): if member.size>8*1024**2: raise RuntimeError('Unexpectedly large metadata member') yield member.name,tar.extractfile(member).read().decode('utf-8-sig') return # Old volumes may contain PDFs. Partial clone plus sparse checkout retrieves # trees and metadata blobs only, without checking out or executing publisher code. checkout = Path(cache)/'metadata-checkouts'/f'pmlr-v{volume}' checkout.parent.mkdir(parents=True,exist_ok=True) env = dict(os.environ,GIT_TERMINAL_PROMPT='0') def git(*args): result = subprocess.run(['git','-c','credential.helper=',*args],env=env,capture_output=True,text=True,timeout=180) if result.returncode: raise RuntimeError('Public metadata Git operation failed: '+result.stderr[-500:]) return result.stdout if not checkout.exists(): git('clone','--filter=blob:none','--depth','1','--single-branch','--branch','gh-pages','--no-checkout',f'https://github.com/mlresearch/v{volume}.git',str(checkout)) git('-C',str(checkout),'sparse-checkout','set','--no-cone','/_posts/','/_config.yml') git('-C',str(checkout),'checkout','gh-pages') else: if git('-C',str(checkout),'status','--porcelain').strip(): raise RuntimeError('Metadata checkout has local changes; refusing to overwrite') git('-C',str(checkout),'fetch','--filter=blob:none','--depth','1','origin','gh-pages') git('-C',str(checkout),'checkout','--detach','FETCH_HEAD') posts = checkout/'_posts' for path in sorted(posts.rglob('*')): if path.is_file() and path.suffix in ('.md','.markdown','.html'): if path.stat().st_size>8*1024**2: raise RuntimeError('Unexpectedly large metadata member') yield str(path.relative_to(checkout)),path.read_text(encoding='utf-8-sig') def collect_volume(volume,cache,out,allowed=None): config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml' config = yaml.safe_load(fetch(config_url,cache)) venue = str(config.get('shortname','')).upper() if allowed and venue not in allowed: return None year = int(config['year']) title = str(config.get('booktitle','')) track = 'workshop' if 'workshop' in title.lower() else 'main' path = Path(out)/f'pmlr-v{volume}.jsonl' path.parent.mkdir(parents=True,exist_ok=True) part = path.with_suffix('.part') count = 0 with part.open('w') as output: for member_name,content in metadata_texts(volume,cache): parts = content.split('---',2) if len(parts)<3 or parts[0].strip(): raise RuntimeError('Missing YAML frontmatter: '+member_name) row = yaml.safe_load(parts[1]) if row.get('layout') not in ('inproceedings','article') or not row.get('title'): continue external = f"v{volume}/{row.get('id') or Path(member_name).stem}" ids = {} for key in ('doi','openreview','arxiv'): value = row.get(key) if value: value = str(value) if key=='arxiv': value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf') if key=='openreview': value = value.split('id=')[-1].split('&')[0] ids[key] = value record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}} output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n') count += 1 if count==0: part.unlink(missing_ok=True) raise RuntimeError('No papers parsed; refusing an empty successful collection') part.replace(path) result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'} path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2)) print(json.dumps(result),flush=True) return result def main(): parser = argparse.ArgumentParser() parser.add_argument('--cache',required=True) parser.add_argument('--out',required=True) parser.add_argument('--volume',type=int,action='append') parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json'))) args = parser.parse_args() allowed = {v['acronym'].upper() for v in json.loads(Path(args.registry).read_text())['venues']} volumes = args.volume if not volumes: index = fetch('https://proceedings.mlr.press/',args.cache).decode() volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index))) if not volumes: raise RuntimeError('PMLR index yielded no volumes') errors = [] for volume in volumes: try: collect_volume(volume,args.cache,args.out,None if args.volume else allowed) except Exception as exc: error = {'volume':volume,'status':'blocked','error':str(exc)} errors.append(error) print(json.dumps(error),flush=True) Path(args.out).mkdir(parents=True,exist_ok=True) (Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2)) if errors: raise SystemExit(1) if __name__=='__main__': main()