Files
paper-atlas/paper_atlas/collectors/proceedings.py
2026-11-06 22:58:50 +08:00

198 lines
9.8 KiB
Python

"""Collect COLM, NeurIPS and MLSys official metadata without downloading PDFs."""
import argparse
import json
import re
import time
from collections import defaultdict
from pathlib import Path
from urllib.parse import urljoin,urlsplit
from paper_atlas.io.http import fetch
from paper_atlas.io.publication_ids import mlsys_id
ROOTS={'colm':'https://proceedings.colm.cc/','neurips':'https://proceedings.neurips.cc/',
'mlsys':'https://proceedings.mlsys.org/'}
VENUES={'colm':'COLM','neurips':'NeurIPS','mlsys':'MLSys'}
from paper_atlas.io.html import Document,Node,clean
def track_key(text):
text=text.lower()
if 'dataset' in text or 'benchmark' in text:
return 'datasets_benchmarks'
if 'creative' in text:
return 'creative_ai'
if 'position' in text:
return 'position'
return 'main'
def paper_links(html,index_url):
doc=Document(html)
links=sorted({urljoin(index_url,n.attrs['href']) for n in doc.root.walk() if n.tag=='a' and '-Abstract' in n.attrs.get('href','') and urlsplit(urljoin(index_url,n.attrs['href'])).path.endswith('.html') and urlsplit(urljoin(index_url,n.attrs['href'])).netloc==urlsplit(index_url).netloc})
declared=next((match for n in doc.root.walk() if n.tag in ('p','span','div') and not any(child.tag=='a' for child in n.walk()) and (match:=re.fullmatch(r'([\d,]+)\s+papers',clean(n.text()),re.I))),None)
if not links:
raise ValueError('Official index contains no abstract-page links')
if declared and int(declared[1].replace(',',''))!=len(links):
raise ValueError('Declared paper count differs from unique abstract-page links')
return links
def parse_paper(html,url,source,year,index_url):
nodes=list(Document(html).root.walk())
meta=defaultdict(list)
for node in nodes:
if node.tag=='meta':
meta[node.attrs.get('name','').lower()].append(node.attrs.get('content',''))
title=next(iter(meta['citation_title']),None)
if not title:
title=next((clean(n.text()) for n in nodes if n.tag=='h1'),None)
abstract=next((clean(n.text()) for n in nodes if 'paper-abstract' in n.attrs.get('class','').split()),None)
if not abstract:
for index,node in enumerate(nodes):
if node.tag in ('h2','h3','h4','h5') and clean(node.text()).lower()=='abstract':
following=[]
covered=set()
for candidate in nodes[index+1:]:
if candidate.tag in ('h1','h2','h3','h4','h5'):
break
if candidate.tag=='p' and candidate not in covered:
following.append(clean(candidate.text()))
covered.update(candidate.walk())
abstract=' '.join(following).strip() or None
break
if not title:
raise ValueError('Missing title on '+url)
# Never fall back to footer/policy text when the abstract is absent.
authors=[]
for author in meta['citation_author']:
if ',' in author:
family,given=author.split(',',1)
author=given.strip()+' '+family.strip()
authors.append(clean(author))
track_text=' '.join(clean(n.text()) for n in nodes if 'paper-track' in n.attrs.get('class','').split())
track=track_key(track_text or url)
identifiers={}
if source == 'mlsys':
if year < 2019:
raise ValueError('SysML 2018 was non-archival; official proceedings begin in 2019')
own = mlsys_id(url)
if not own or int(own.split('/')[0]) != year:
raise ValueError('MLSys paper URL does not identify this publication year')
identifiers['publication_mlsys'] = own
if track_text:
explicit = {'conference': 'main', 'main track': 'main', 'research track': 'main',
'industry track': 'industry', 'industrial track': 'industry',
'mlsys' + str(year): 'main'}
if track_text.casefold() not in explicit:
raise ValueError('Unclassified official MLSys track: ' + track_text)
track = explicit[track_text.casefold()]
elif year >= 2025 and not url.endswith('-Abstract-Conference.html'):
raise ValueError('Modern MLSys paper lacks an explicit conference track')
if meta['citation_doi']:
identifiers['doi']=meta['citation_doi'][0]
action_nodes=[]
for container in nodes:
if 'paper-actions' in container.attrs.get('class','').split():
action_nodes.extend(container.walk())
# Abstract links can cite other papers; only labelled self links identify this one.
for node in action_nodes:
if node.tag!='a':
continue
link=urljoin(url,node.attrs.get('href',''))
parsed=urlsplit(link)
if parsed.netloc in ('arxiv.org','www.arxiv.org') and re.match(r'^/(abs|pdf)/',parsed.path):
identifiers['arxiv']=re.sub(r'^/(abs|pdf)/','',parsed.path).removesuffix('.pdf')
if parsed.netloc=='openreview.net' and parsed.path=='/forum':
from urllib.parse import parse_qs
forum=parse_qs(parsed.query).get('id')
if forum: identifiers['openreview']=forum[0]
venue=VENUES[source]
result = {'source':source,'external_id':urlsplit(url).path,'url':url,'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,'venue':venue,'year':year,'track':track,'status':'accepted','raw_decision':'listed in official proceedings','list_url':index_url,'metadata':{'citation':dict(meta),'publication_state':'pre-proceedings' if 'pre-proceedings' in html.lower() else 'proceedings'}}
if source == 'mlsys' and year == 2019:
result['venue_aliases'] = ['SysML']
return result
def discover(source,cache):
root=ROOTS[source]
html=fetch(root,cache,8*1024**2,delay=1.0).decode()
result={}
for node in Document(html).root.walk():
if node.tag!='a':continue
url=urljoin(root,node.attrs.get('href',''))
parsed=urlsplit(url)
match=re.search(r'/paper(?:_files/paper)?/((?:19|20)\d{2})(?:/|$)',parsed.path)
if parsed.netloc==urlsplit(root).netloc and match and not any(term in parsed.path for term in ('/hash/','/file/','.pdf','-Abstract')):
result[url]=int(match[1])
if source=='neurips' and parsed.netloc=='datasets-benchmarks-proceedings.neurips.cc' and match:
result[url]=int(match[1])
if not result:
raise ValueError('No official year indexes found')
return sorted(result.items(),key=lambda entry:(entry[1],entry[0]))
def collect_index(source,index_url,year,cache,out):
html=fetch(index_url,cache,16*1024**2,delay=1.0).decode()
links=paper_links(html,index_url)
grouped=defaultdict(list)
missing=[]
for number,url in enumerate(links,1):
record=parse_paper(fetch(url,cache,2*1024**2,delay=1.0).decode(),url,source,year,index_url)
if not record['abstract']:missing.append(url)
grouped[record['track']].append(record)
if number%25==0:
print(json.dumps({'source':source,'year':year,'collected':number,'expected':len(links)}),flush=True)
results=[]
for track,records in grouped.items():
# Distinct official index URLs may publish multiple volumes in one year.
import hashlib
suffix=hashlib.sha256(index_url.encode()).hexdigest()[:10]
path=Path(out)/f'{source}-{year}-{track}-{suffix}.jsonl'
path.parent.mkdir(parents=True,exist_ok=True)
part=path.with_suffix('.part')
with part.open('w') as output:
for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n')
part.replace(path)
missing_count=sum(not r['abstract'] for r in records)
result={'source':source,'parser_version':2,'venue':records[0]['venue'],'year':year,'track':track,'count':len(records),'index_link_count':len(links),'index_url':index_url,'list_url':index_url,'missing_abstract_count':missing_count,'file':str(path),'status':'collected' if not missing_count else 'partial'}
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
results.append(result)
print(json.dumps(result),flush=True)
return results
def main(argv=None, *, prog=None, source=None):
parser=argparse.ArgumentParser(prog=prog)
if source is None:parser.add_argument('--source',choices=ROOTS,required=True)
else:parser.set_defaults(source=source)
parser.add_argument('--cache',required=True)
parser.add_argument('--out',required=True)
parser.add_argument('--year',type=int,action='append')
args=parser.parse_args(argv)
out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
status_path=out/f'{args.source}-index-status.json'
catalogue=json.loads(status_path.read_text()) if status_path.exists() else {}
indexes=discover(args.source,args.cache)
if args.year and set(args.year)-{year for _,year in indexes}:
parser.error('Requested year is absent from the current official proceedings index')
requested=[]
for url,year in indexes:
if args.year and year not in args.year:continue
requested.append(url)
try:
files=collect_index(args.source,url,year,args.cache,args.out)
catalogue[url]={'status':'partial' if any(f['status']=='partial' for f in files) else 'collected','files':files}
except Exception as exc:
catalogue[url]={'status':'blocked','error':str(exc),'year':year}
print(json.dumps(catalogue[url]),flush=True)
part=status_path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_path)
(out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year},indent=2))
if any(catalogue[url]['status']=='blocked' for url in requested):raise SystemExit(1)
if __name__ == '__main__':
main()