165 lines
8.0 KiB
Python
165 lines
8.0 KiB
Python
"""Collect official PMLR metadata, never the PDFs linked by the metadata."""
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import re
|
|
import os
|
|
import subprocess
|
|
import tarfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
|
|
def fetch(url,cache,limit=64*1024**2):
|
|
cache = Path(cache)
|
|
cache.mkdir(parents=True,exist_ok=True)
|
|
path = cache/hashlib.sha256(url.encode()).hexdigest()
|
|
if path.exists() and time.time()-path.stat().st_mtime < 24*3600:
|
|
return path.read_bytes()
|
|
for attempt in range(3):
|
|
try:
|
|
request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'})
|
|
with urllib.request.urlopen(request,timeout=90) as response:
|
|
if response.status != 200:
|
|
raise RuntimeError('Expected complete HTTP 200 response')
|
|
if int(response.headers.get('Content-Length','0'))>limit:
|
|
raise RuntimeError('Metadata response exceeds byte limit')
|
|
data = response.read(limit+1)
|
|
if len(data)>limit:
|
|
raise RuntimeError('Metadata response exceeds byte limit')
|
|
part = path.with_suffix('.part')
|
|
part.write_bytes(data)
|
|
part.replace(path)
|
|
path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
|
|
time.sleep(0.5)
|
|
return data
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code not in (429,500,502,503,504) or attempt==2:
|
|
raise
|
|
time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1')))))
|
|
|
|
|
|
def metadata_texts(volume,cache):
|
|
archive = None
|
|
if volume>50:
|
|
try:
|
|
archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
|
|
except RuntimeError as exc:
|
|
if 'exceeds byte limit' not in str(exc):
|
|
raise
|
|
if archive is not None:
|
|
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
|
|
for member in tar:
|
|
if member.isfile() and '/_posts/' in member.name and member.name.endswith(('.md','.markdown','.html')):
|
|
if member.size>8*1024**2:
|
|
raise RuntimeError('Unexpectedly large metadata member')
|
|
yield member.name,tar.extractfile(member).read().decode('utf-8-sig')
|
|
return
|
|
# Old volumes may contain PDFs. Partial clone plus sparse checkout retrieves
|
|
# trees and metadata blobs only, without checking out or executing publisher code.
|
|
checkout = Path(cache)/'metadata-checkouts'/f'pmlr-v{volume}'
|
|
checkout.parent.mkdir(parents=True,exist_ok=True)
|
|
env = dict(os.environ,GIT_TERMINAL_PROMPT='0')
|
|
def git(*args):
|
|
result = subprocess.run(['git','-c','credential.helper=',*args],env=env,capture_output=True,text=True,timeout=180)
|
|
if result.returncode:
|
|
raise RuntimeError('Public metadata Git operation failed: '+result.stderr[-500:])
|
|
return result.stdout
|
|
if not checkout.exists():
|
|
git('clone','--filter=blob:none','--depth','1','--single-branch','--branch','gh-pages','--no-checkout',f'https://github.com/mlresearch/v{volume}.git',str(checkout))
|
|
git('-C',str(checkout),'sparse-checkout','set','--no-cone','/_posts/','/_config.yml')
|
|
git('-C',str(checkout),'checkout','gh-pages')
|
|
else:
|
|
if git('-C',str(checkout),'status','--porcelain').strip():
|
|
raise RuntimeError('Metadata checkout has local changes; refusing to overwrite')
|
|
git('-C',str(checkout),'fetch','--filter=blob:none','--depth','1','origin','gh-pages')
|
|
git('-C',str(checkout),'checkout','--detach','FETCH_HEAD')
|
|
posts = checkout/'_posts'
|
|
for path in sorted(posts.rglob('*')):
|
|
if path.is_file() and path.suffix in ('.md','.markdown','.html'):
|
|
if path.stat().st_size>8*1024**2:
|
|
raise RuntimeError('Unexpectedly large metadata member')
|
|
yield str(path.relative_to(checkout)),path.read_text(encoding='utf-8-sig')
|
|
|
|
|
|
def collect_volume(volume,cache,out,allowed=None):
|
|
config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml'
|
|
config = yaml.safe_load(fetch(config_url,cache))
|
|
venue = str(config.get('shortname','')).upper()
|
|
if allowed and venue not in allowed:
|
|
return None
|
|
year = int(config['year'])
|
|
title = str(config.get('booktitle',''))
|
|
track = 'workshop' if 'workshop' in title.lower() else 'main'
|
|
path = Path(out)/f'pmlr-v{volume}.jsonl'
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
part = path.with_suffix('.part')
|
|
count = 0
|
|
with part.open('w') as output:
|
|
for member_name,content in metadata_texts(volume,cache):
|
|
parts = content.split('---',2)
|
|
if len(parts)<3 or parts[0].strip():
|
|
raise RuntimeError('Missing YAML frontmatter: '+member_name)
|
|
row = yaml.safe_load(parts[1])
|
|
if row.get('layout') not in ('inproceedings','article') or not row.get('title'):
|
|
continue
|
|
external = f"v{volume}/{row.get('id') or Path(member_name).stem}"
|
|
ids = {}
|
|
for key in ('doi','openreview','arxiv'):
|
|
value = row.get(key)
|
|
if value:
|
|
value = str(value)
|
|
if key=='arxiv':
|
|
value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
|
|
if key=='openreview':
|
|
value = value.split('id=')[-1].split('&')[0]
|
|
ids[key] = value
|
|
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
|
|
output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n')
|
|
count += 1
|
|
if count==0:
|
|
part.unlink(missing_ok=True)
|
|
raise RuntimeError('No papers parsed; refusing an empty successful collection')
|
|
part.replace(path)
|
|
result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
|
|
print(json.dumps(result),flush=True)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--cache',required=True)
|
|
parser.add_argument('--out',required=True)
|
|
parser.add_argument('--volume',type=int,action='append')
|
|
parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json')))
|
|
args = parser.parse_args()
|
|
allowed = {v['acronym'].upper() for v in json.loads(Path(args.registry).read_text())['venues']}
|
|
volumes = args.volume
|
|
if not volumes:
|
|
index = fetch('https://proceedings.mlr.press/',args.cache).decode()
|
|
volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index)))
|
|
if not volumes:
|
|
raise RuntimeError('PMLR index yielded no volumes')
|
|
errors = []
|
|
for volume in volumes:
|
|
try:
|
|
collect_volume(volume,args.cache,args.out,None if args.volume else allowed)
|
|
except Exception as exc:
|
|
error = {'volume':volume,'status':'blocked','error':str(exc)}
|
|
errors.append(error)
|
|
print(json.dumps(error),flush=True)
|
|
Path(args.out).mkdir(parents=True,exist_ok=True)
|
|
(Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2))
|
|
if errors:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|