124 lines
5.9 KiB
Python
124 lines
5.9 KiB
Python
"""Collect official PMLR metadata, never the PDFs linked by the metadata."""
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import re
|
|
import tarfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
|
|
def fetch(url,cache,limit=64*1024**2):
|
|
cache = Path(cache)
|
|
cache.mkdir(parents=True,exist_ok=True)
|
|
path = cache/hashlib.sha256(url.encode()).hexdigest()
|
|
if path.exists() and time.time()-path.stat().st_mtime < 24*3600:
|
|
return path.read_bytes()
|
|
for attempt in range(3):
|
|
try:
|
|
request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'})
|
|
with urllib.request.urlopen(request,timeout=90) as response:
|
|
if response.status != 200:
|
|
raise RuntimeError('Expected complete HTTP 200 response')
|
|
data = response.read(limit+1)
|
|
if len(data)>limit:
|
|
raise RuntimeError('Metadata response exceeds byte limit')
|
|
part = path.with_suffix('.part')
|
|
part.write_bytes(data)
|
|
part.replace(path)
|
|
path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
|
|
time.sleep(0.5)
|
|
return data
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code not in (429,500,502,503,504) or attempt==2:
|
|
raise
|
|
time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1')))))
|
|
|
|
|
|
def collect_volume(volume,cache,out,allowed=None):
|
|
config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml'
|
|
config = yaml.safe_load(fetch(config_url,cache))
|
|
venue = str(config.get('shortname','')).upper()
|
|
if allowed and venue not in allowed:
|
|
return None
|
|
year = int(config['year'])
|
|
title = str(config.get('booktitle',''))
|
|
track = 'workshop' if 'workshop' in title.lower() else 'main'
|
|
archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
|
|
path = Path(out)/f'pmlr-v{volume}.jsonl'
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
part = path.with_suffix('.part')
|
|
count = 0
|
|
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar,part.open('w') as output:
|
|
for member in tar:
|
|
if not member.isfile() or '/_posts/' not in member.name or not member.name.endswith(('.md','.markdown','.html')):
|
|
continue
|
|
if member.size>8*1024**2:
|
|
raise RuntimeError('Unexpectedly large metadata member')
|
|
content = tar.extractfile(member).read().decode('utf-8-sig')
|
|
parts = content.split('---',2)
|
|
if len(parts)<3 or parts[0].strip():
|
|
raise RuntimeError('Missing YAML frontmatter: '+member.name)
|
|
row = yaml.safe_load(parts[1])
|
|
if row.get('layout') not in ('inproceedings','article') or not row.get('title'):
|
|
continue
|
|
external = f"v{volume}/{row.get('id') or Path(member.name).stem}"
|
|
ids = {}
|
|
for key in ('doi','openreview','arxiv'):
|
|
value = row.get(key)
|
|
if value:
|
|
value = str(value)
|
|
if key=='arxiv':
|
|
value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
|
|
if key=='openreview':
|
|
value = value.split('id=')[-1].split('&')[0]
|
|
ids[key] = value
|
|
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
|
|
output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n')
|
|
count += 1
|
|
if count==0:
|
|
part.unlink(missing_ok=True)
|
|
raise RuntimeError('No papers parsed; refusing an empty successful collection')
|
|
part.replace(path)
|
|
result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
|
|
print(json.dumps(result),flush=True)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--cache',required=True)
|
|
parser.add_argument('--out',required=True)
|
|
parser.add_argument('--volume',type=int,action='append')
|
|
parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json')))
|
|
args = parser.parse_args()
|
|
allowed = {v['acronym'].upper() for v in json.loads(Path(args.registry).read_text())['venues']}
|
|
volumes = args.volume
|
|
if not volumes:
|
|
index = fetch('https://proceedings.mlr.press/',args.cache).decode()
|
|
volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index)))
|
|
if not volumes:
|
|
raise RuntimeError('PMLR index yielded no volumes')
|
|
errors = []
|
|
for volume in volumes:
|
|
try:
|
|
collect_volume(volume,args.cache,args.out,None if args.volume else allowed)
|
|
except Exception as exc:
|
|
error = {'volume':volume,'status':'blocked','error':str(exc)}
|
|
errors.append(error)
|
|
print(json.dumps(error),flush=True)
|
|
Path(args.out).mkdir(parents=True,exist_ok=True)
|
|
(Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2))
|
|
if errors:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|