Files
paper-atlas/collect_proceedings.py
T
2026-11-06 15:01:15 +08:00

217 lines
9.8 KiB
Python

"""Collect COLM/NeurIPS official abstract pages without downloading PDFs."""
import argparse
import json
import re
import time
from collections import defaultdict
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import urljoin,urlsplit
from collect_pmlr import fetch
ROOTS={'colm':'https://proceedings.colm.cc/','neurips':'https://proceedings.neurips.cc/'}
class Node:
def __init__(self,tag='',attrs=None):
self.tag=tag
self.attrs={key:value or '' for key,value in attrs or []}
self.children=[]
def text(self):
if self.tag in ('script','style'):
return ''
content=''.join(child.text() if isinstance(child,Node) else child for child in self.children)
return ' '+content+' ' if self.tag in ('p','div','section','li','br') else content
def walk(self):
yield self
for child in self.children:
if isinstance(child,Node):
yield from child.walk()
class Document(HTMLParser):
VOID={'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'}
def __init__(self,html):
super().__init__(convert_charrefs=True)
self.root=Node()
self.stack=[self.root]
self.feed(html)
def handle_starttag(self,tag,attrs):
node=Node(tag,attrs)
self.stack[-1].children.append(node)
if tag not in self.VOID:
self.stack.append(node)
def handle_endtag(self,tag):
for index in range(len(self.stack)-1,0,-1):
if self.stack[index].tag==tag:
del self.stack[index:]
break
def handle_data(self,data):
self.stack[-1].children.append(data)
def clean(text):
return ' '.join(text.split())
def track_key(text):
text=text.lower()
if 'dataset' in text or 'benchmark' in text:
return 'datasets_benchmarks'
if 'creative' in text:
return 'creative_ai'
if 'position' in text:
return 'position'
return 'main'
def paper_links(html,index_url):
doc=Document(html)
links=sorted({urljoin(index_url,n.attrs['href']) for n in doc.root.walk() if n.tag=='a' and '-Abstract' in n.attrs.get('href','') and urlsplit(urljoin(index_url,n.attrs['href'])).path.endswith('.html') and urlsplit(urljoin(index_url,n.attrs['href'])).netloc==urlsplit(index_url).netloc})
declared=next((match for n in doc.root.walk() if n.tag in ('p','span','div') and not any(child.tag=='a' for child in n.walk()) and (match:=re.fullmatch(r'([\d,]+)\s+papers',clean(n.text()),re.I))),None)
if not links:
raise ValueError('Official index contains no abstract-page links')
if declared and int(declared[1].replace(',',''))!=len(links):
raise ValueError('Declared paper count differs from unique abstract-page links')
return links
def parse_paper(html,url,source,year,index_url):
nodes=list(Document(html).root.walk())
meta=defaultdict(list)
for node in nodes:
if node.tag=='meta':
meta[node.attrs.get('name','').lower()].append(node.attrs.get('content',''))
title=next(iter(meta['citation_title']),None)
if not title:
title=next((clean(n.text()) for n in nodes if n.tag=='h1'),None)
abstract=next((clean(n.text()) for n in nodes if 'paper-abstract' in n.attrs.get('class','').split()),None)
if not abstract:
for index,node in enumerate(nodes):
if node.tag in ('h2','h3','h4','h5') and clean(node.text()).lower()=='abstract':
following=[]
covered=set()
for candidate in nodes[index+1:]:
if candidate.tag in ('h1','h2','h3','h4','h5'):
break
if candidate.tag=='p' and candidate not in covered:
following.append(clean(candidate.text()))
covered.update(candidate.walk())
abstract=' '.join(following).strip() or None
break
if not title:
raise ValueError('Missing title on '+url)
# Never fall back to footer/policy text when the abstract is absent.
authors=[]
for author in meta['citation_author']:
if ',' in author:
family,given=author.split(',',1)
author=given.strip()+' '+family.strip()
authors.append(clean(author))
track_text=' '.join(clean(n.text()) for n in nodes if 'paper-track' in n.attrs.get('class','').split())
track=track_key(track_text or url)
identifiers={}
if meta['citation_doi']:
identifiers['doi']=meta['citation_doi'][0]
action_nodes=[]
for container in nodes:
if 'paper-actions' in container.attrs.get('class','').split():
action_nodes.extend(container.walk())
# Abstract links can cite other papers; only labelled self links identify this one.
for node in action_nodes:
if node.tag!='a':
continue
link=urljoin(url,node.attrs.get('href',''))
parsed=urlsplit(link)
if parsed.netloc in ('arxiv.org','www.arxiv.org') and re.match(r'^/(abs|pdf)/',parsed.path):
identifiers['arxiv']=re.sub(r'^/(abs|pdf)/','',parsed.path).removesuffix('.pdf')
if parsed.netloc=='openreview.net' and parsed.path=='/forum':
from urllib.parse import parse_qs
forum=parse_qs(parsed.query).get('id')
if forum: identifiers['openreview']=forum[0]
venue='COLM' if source=='colm' else 'NeurIPS'
return {'source':source,'external_id':urlsplit(url).path,'url':url,'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,'venue':venue,'year':year,'track':track,'status':'accepted','raw_decision':'listed in official proceedings','list_url':index_url,'metadata':{'citation':dict(meta),'publication_state':'pre-proceedings' if 'pre-proceedings' in html.lower() else 'proceedings'}}
def discover(source,cache):
root=ROOTS[source]
html=fetch(root,cache,8*1024**2,delay=1.0).decode()
result={}
for node in Document(html).root.walk():
if node.tag!='a':continue
url=urljoin(root,node.attrs.get('href',''))
parsed=urlsplit(url)
match=re.search(r'/paper(?:_files/paper)?/((?:19|20)\d{2})(?:/|$)',parsed.path)
if parsed.netloc==urlsplit(root).netloc and match and not any(term in parsed.path for term in ('/hash/','/file/','.pdf','-Abstract')):
result[url]=int(match[1])
if source=='neurips' and parsed.netloc=='datasets-benchmarks-proceedings.neurips.cc' and match:
result[url]=int(match[1])
if not result:
raise ValueError('No official year indexes found')
return sorted(result.items(),key=lambda entry:(entry[1],entry[0]))
def collect_index(source,index_url,year,cache,out):
html=fetch(index_url,cache,16*1024**2,delay=1.0).decode()
links=paper_links(html,index_url)
grouped=defaultdict(list)
missing=[]
for number,url in enumerate(links,1):
record=parse_paper(fetch(url,cache,2*1024**2,delay=1.0).decode(),url,source,year,index_url)
if not record['abstract']:missing.append(url)
grouped[record['track']].append(record)
if number%25==0:
print(json.dumps({'source':source,'year':year,'collected':number,'expected':len(links)}),flush=True)
results=[]
for track,records in grouped.items():
# Distinct official index URLs may publish multiple volumes in one year.
import hashlib
suffix=hashlib.sha256(index_url.encode()).hexdigest()[:10]
path=Path(out)/f'{source}-{year}-{track}-{suffix}.jsonl'
path.parent.mkdir(parents=True,exist_ok=True)
part=path.with_suffix('.part')
with part.open('w') as output:
for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n')
part.replace(path)
missing_count=sum(not r['abstract'] for r in records)
result={'source':source,'parser_version':2,'venue':records[0]['venue'],'year':year,'track':track,'count':len(records),'index_link_count':len(links),'index_url':index_url,'list_url':index_url,'missing_abstract_count':missing_count,'file':str(path),'status':'collected' if not missing_count else 'partial'}
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
results.append(result)
print(json.dumps(result),flush=True)
return results
if __name__=='__main__':
parser=argparse.ArgumentParser()
parser.add_argument('--source',choices=ROOTS,required=True)
parser.add_argument('--cache',required=True)
parser.add_argument('--out',required=True)
parser.add_argument('--year',type=int,action='append')
args=parser.parse_args()
out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
status_path=out/f'{args.source}-index-status.json'
catalogue=json.loads(status_path.read_text()) if status_path.exists() else {}
indexes=discover(args.source,args.cache)
if args.year and set(args.year)-{year for _,year in indexes}:
parser.error('Requested year is absent from the current official proceedings index')
requested=[]
for url,year in indexes:
if args.year and year not in args.year:continue
requested.append(url)
try:
files=collect_index(args.source,url,year,args.cache,args.out)
catalogue[url]={'status':'partial' if any(f['status']=='partial' for f in files) else 'collected','files':files}
except Exception as exc:
catalogue[url]={'status':'blocked','error':str(exc),'year':year}
print(json.dumps(catalogue[url]),flush=True)
part=status_path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_path)
(out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year},indent=2))
if any(catalogue[url]['status']=='blocked' for url in requested):raise SystemExit(1)