198 lines
9.8 KiB
Python
198 lines
9.8 KiB
Python
"""Collect COLM, NeurIPS and MLSys official metadata without downloading PDFs."""
|
|
import argparse
|
|
import json
|
|
import re
|
|
import time
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
from urllib.parse import urljoin,urlsplit
|
|
|
|
from paper_atlas.io.http import fetch
|
|
from paper_atlas.io.publication_ids import mlsys_id
|
|
|
|
ROOTS={'colm':'https://proceedings.colm.cc/','neurips':'https://proceedings.neurips.cc/',
|
|
'mlsys':'https://proceedings.mlsys.org/'}
|
|
VENUES={'colm':'COLM','neurips':'NeurIPS','mlsys':'MLSys'}
|
|
|
|
|
|
from paper_atlas.io.html import Document,Node,clean
|
|
|
|
|
|
def track_key(text):
|
|
text=text.lower()
|
|
if 'dataset' in text or 'benchmark' in text:
|
|
return 'datasets_benchmarks'
|
|
if 'creative' in text:
|
|
return 'creative_ai'
|
|
if 'position' in text:
|
|
return 'position'
|
|
return 'main'
|
|
|
|
|
|
def paper_links(html,index_url):
|
|
doc=Document(html)
|
|
links=sorted({urljoin(index_url,n.attrs['href']) for n in doc.root.walk() if n.tag=='a' and '-Abstract' in n.attrs.get('href','') and urlsplit(urljoin(index_url,n.attrs['href'])).path.endswith('.html') and urlsplit(urljoin(index_url,n.attrs['href'])).netloc==urlsplit(index_url).netloc})
|
|
declared=next((match for n in doc.root.walk() if n.tag in ('p','span','div') and not any(child.tag=='a' for child in n.walk()) and (match:=re.fullmatch(r'([\d,]+)\s+papers',clean(n.text()),re.I))),None)
|
|
if not links:
|
|
raise ValueError('Official index contains no abstract-page links')
|
|
if declared and int(declared[1].replace(',',''))!=len(links):
|
|
raise ValueError('Declared paper count differs from unique abstract-page links')
|
|
return links
|
|
|
|
|
|
def parse_paper(html,url,source,year,index_url):
|
|
nodes=list(Document(html).root.walk())
|
|
meta=defaultdict(list)
|
|
for node in nodes:
|
|
if node.tag=='meta':
|
|
meta[node.attrs.get('name','').lower()].append(node.attrs.get('content',''))
|
|
title=next(iter(meta['citation_title']),None)
|
|
if not title:
|
|
title=next((clean(n.text()) for n in nodes if n.tag=='h1'),None)
|
|
abstract=next((clean(n.text()) for n in nodes if 'paper-abstract' in n.attrs.get('class','').split()),None)
|
|
if not abstract:
|
|
for index,node in enumerate(nodes):
|
|
if node.tag in ('h2','h3','h4','h5') and clean(node.text()).lower()=='abstract':
|
|
following=[]
|
|
covered=set()
|
|
for candidate in nodes[index+1:]:
|
|
if candidate.tag in ('h1','h2','h3','h4','h5'):
|
|
break
|
|
if candidate.tag=='p' and candidate not in covered:
|
|
following.append(clean(candidate.text()))
|
|
covered.update(candidate.walk())
|
|
abstract=' '.join(following).strip() or None
|
|
break
|
|
if not title:
|
|
raise ValueError('Missing title on '+url)
|
|
# Never fall back to footer/policy text when the abstract is absent.
|
|
authors=[]
|
|
for author in meta['citation_author']:
|
|
if ',' in author:
|
|
family,given=author.split(',',1)
|
|
author=given.strip()+' '+family.strip()
|
|
authors.append(clean(author))
|
|
track_text=' '.join(clean(n.text()) for n in nodes if 'paper-track' in n.attrs.get('class','').split())
|
|
track=track_key(track_text or url)
|
|
identifiers={}
|
|
if source == 'mlsys':
|
|
if year < 2019:
|
|
raise ValueError('SysML 2018 was non-archival; official proceedings begin in 2019')
|
|
own = mlsys_id(url)
|
|
if not own or int(own.split('/')[0]) != year:
|
|
raise ValueError('MLSys paper URL does not identify this publication year')
|
|
identifiers['publication_mlsys'] = own
|
|
if track_text:
|
|
explicit = {'conference': 'main', 'main track': 'main', 'research track': 'main',
|
|
'industry track': 'industry', 'industrial track': 'industry',
|
|
'mlsys' + str(year): 'main'}
|
|
if track_text.casefold() not in explicit:
|
|
raise ValueError('Unclassified official MLSys track: ' + track_text)
|
|
track = explicit[track_text.casefold()]
|
|
elif year >= 2025 and not url.endswith('-Abstract-Conference.html'):
|
|
raise ValueError('Modern MLSys paper lacks an explicit conference track')
|
|
if meta['citation_doi']:
|
|
identifiers['doi']=meta['citation_doi'][0]
|
|
action_nodes=[]
|
|
for container in nodes:
|
|
if 'paper-actions' in container.attrs.get('class','').split():
|
|
action_nodes.extend(container.walk())
|
|
# Abstract links can cite other papers; only labelled self links identify this one.
|
|
for node in action_nodes:
|
|
if node.tag!='a':
|
|
continue
|
|
link=urljoin(url,node.attrs.get('href',''))
|
|
parsed=urlsplit(link)
|
|
if parsed.netloc in ('arxiv.org','www.arxiv.org') and re.match(r'^/(abs|pdf)/',parsed.path):
|
|
identifiers['arxiv']=re.sub(r'^/(abs|pdf)/','',parsed.path).removesuffix('.pdf')
|
|
if parsed.netloc=='openreview.net' and parsed.path=='/forum':
|
|
from urllib.parse import parse_qs
|
|
forum=parse_qs(parsed.query).get('id')
|
|
if forum: identifiers['openreview']=forum[0]
|
|
venue=VENUES[source]
|
|
result = {'source':source,'external_id':urlsplit(url).path,'url':url,'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,'venue':venue,'year':year,'track':track,'status':'accepted','raw_decision':'listed in official proceedings','list_url':index_url,'metadata':{'citation':dict(meta),'publication_state':'pre-proceedings' if 'pre-proceedings' in html.lower() else 'proceedings'}}
|
|
if source == 'mlsys' and year == 2019:
|
|
result['venue_aliases'] = ['SysML']
|
|
return result
|
|
|
|
|
|
def discover(source,cache):
|
|
root=ROOTS[source]
|
|
html=fetch(root,cache,8*1024**2,delay=1.0).decode()
|
|
result={}
|
|
for node in Document(html).root.walk():
|
|
if node.tag!='a':continue
|
|
url=urljoin(root,node.attrs.get('href',''))
|
|
parsed=urlsplit(url)
|
|
match=re.search(r'/paper(?:_files/paper)?/((?:19|20)\d{2})(?:/|$)',parsed.path)
|
|
if parsed.netloc==urlsplit(root).netloc and match and not any(term in parsed.path for term in ('/hash/','/file/','.pdf','-Abstract')):
|
|
result[url]=int(match[1])
|
|
if source=='neurips' and parsed.netloc=='datasets-benchmarks-proceedings.neurips.cc' and match:
|
|
result[url]=int(match[1])
|
|
if not result:
|
|
raise ValueError('No official year indexes found')
|
|
return sorted(result.items(),key=lambda entry:(entry[1],entry[0]))
|
|
|
|
|
|
def collect_index(source,index_url,year,cache,out):
|
|
html=fetch(index_url,cache,16*1024**2,delay=1.0).decode()
|
|
links=paper_links(html,index_url)
|
|
grouped=defaultdict(list)
|
|
missing=[]
|
|
for number,url in enumerate(links,1):
|
|
record=parse_paper(fetch(url,cache,2*1024**2,delay=1.0).decode(),url,source,year,index_url)
|
|
if not record['abstract']:missing.append(url)
|
|
grouped[record['track']].append(record)
|
|
if number%25==0:
|
|
print(json.dumps({'source':source,'year':year,'collected':number,'expected':len(links)}),flush=True)
|
|
results=[]
|
|
for track,records in grouped.items():
|
|
# Distinct official index URLs may publish multiple volumes in one year.
|
|
import hashlib
|
|
suffix=hashlib.sha256(index_url.encode()).hexdigest()[:10]
|
|
path=Path(out)/f'{source}-{year}-{track}-{suffix}.jsonl'
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
part=path.with_suffix('.part')
|
|
with part.open('w') as output:
|
|
for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n')
|
|
part.replace(path)
|
|
missing_count=sum(not r['abstract'] for r in records)
|
|
result={'source':source,'parser_version':2,'venue':records[0]['venue'],'year':year,'track':track,'count':len(records),'index_link_count':len(links),'index_url':index_url,'list_url':index_url,'missing_abstract_count':missing_count,'file':str(path),'status':'collected' if not missing_count else 'partial'}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
|
|
results.append(result)
|
|
print(json.dumps(result),flush=True)
|
|
return results
|
|
|
|
|
|
def main(argv=None, *, prog=None, source=None):
|
|
parser=argparse.ArgumentParser(prog=prog)
|
|
if source is None:parser.add_argument('--source',choices=ROOTS,required=True)
|
|
else:parser.set_defaults(source=source)
|
|
parser.add_argument('--cache',required=True)
|
|
parser.add_argument('--out',required=True)
|
|
parser.add_argument('--year',type=int,action='append')
|
|
args=parser.parse_args(argv)
|
|
out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
|
|
status_path=out/f'{args.source}-index-status.json'
|
|
catalogue=json.loads(status_path.read_text()) if status_path.exists() else {}
|
|
indexes=discover(args.source,args.cache)
|
|
if args.year and set(args.year)-{year for _,year in indexes}:
|
|
parser.error('Requested year is absent from the current official proceedings index')
|
|
requested=[]
|
|
for url,year in indexes:
|
|
if args.year and year not in args.year:continue
|
|
requested.append(url)
|
|
try:
|
|
files=collect_index(args.source,url,year,args.cache,args.out)
|
|
catalogue[url]={'status':'partial' if any(f['status']=='partial' for f in files) else 'collected','files':files}
|
|
except Exception as exc:
|
|
catalogue[url]={'status':'blocked','error':str(exc),'year':year}
|
|
print(json.dumps(catalogue[url]),flush=True)
|
|
part=status_path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_path)
|
|
(out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year},indent=2))
|
|
if any(catalogue[url]['status']=='blocked' for url in requested):raise SystemExit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|