217 lines
9.8 KiB
Python
217 lines
9.8 KiB
Python
"""Collect COLM/NeurIPS official abstract pages without downloading PDFs."""
|
|
import argparse
|
|
import json
|
|
import re
|
|
import time
|
|
from collections import defaultdict
|
|
from html.parser import HTMLParser
|
|
from pathlib import Path
|
|
from urllib.parse import urljoin,urlsplit
|
|
|
|
from collect_pmlr import fetch
|
|
|
|
ROOTS={'colm':'https://proceedings.colm.cc/','neurips':'https://proceedings.neurips.cc/'}
|
|
|
|
|
|
class Node:
|
|
def __init__(self,tag='',attrs=None):
|
|
self.tag=tag
|
|
self.attrs={key:value or '' for key,value in attrs or []}
|
|
self.children=[]
|
|
|
|
def text(self):
|
|
if self.tag in ('script','style'):
|
|
return ''
|
|
content=''.join(child.text() if isinstance(child,Node) else child for child in self.children)
|
|
return ' '+content+' ' if self.tag in ('p','div','section','li','br') else content
|
|
|
|
def walk(self):
|
|
yield self
|
|
for child in self.children:
|
|
if isinstance(child,Node):
|
|
yield from child.walk()
|
|
|
|
|
|
class Document(HTMLParser):
|
|
VOID={'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'}
|
|
|
|
def __init__(self,html):
|
|
super().__init__(convert_charrefs=True)
|
|
self.root=Node()
|
|
self.stack=[self.root]
|
|
self.feed(html)
|
|
|
|
def handle_starttag(self,tag,attrs):
|
|
node=Node(tag,attrs)
|
|
self.stack[-1].children.append(node)
|
|
if tag not in self.VOID:
|
|
self.stack.append(node)
|
|
|
|
def handle_endtag(self,tag):
|
|
for index in range(len(self.stack)-1,0,-1):
|
|
if self.stack[index].tag==tag:
|
|
del self.stack[index:]
|
|
break
|
|
|
|
def handle_data(self,data):
|
|
self.stack[-1].children.append(data)
|
|
|
|
|
|
def clean(text):
|
|
return ' '.join(text.split())
|
|
|
|
|
|
def track_key(text):
|
|
text=text.lower()
|
|
if 'dataset' in text or 'benchmark' in text:
|
|
return 'datasets_benchmarks'
|
|
if 'creative' in text:
|
|
return 'creative_ai'
|
|
if 'position' in text:
|
|
return 'position'
|
|
return 'main'
|
|
|
|
|
|
def paper_links(html,index_url):
|
|
doc=Document(html)
|
|
links=sorted({urljoin(index_url,n.attrs['href']) for n in doc.root.walk() if n.tag=='a' and '-Abstract' in n.attrs.get('href','') and urlsplit(urljoin(index_url,n.attrs['href'])).path.endswith('.html') and urlsplit(urljoin(index_url,n.attrs['href'])).netloc==urlsplit(index_url).netloc})
|
|
declared=next((match for n in doc.root.walk() if n.tag in ('p','span','div') and not any(child.tag=='a' for child in n.walk()) and (match:=re.fullmatch(r'([\d,]+)\s+papers',clean(n.text()),re.I))),None)
|
|
if not links:
|
|
raise ValueError('Official index contains no abstract-page links')
|
|
if declared and int(declared[1].replace(',',''))!=len(links):
|
|
raise ValueError('Declared paper count differs from unique abstract-page links')
|
|
return links
|
|
|
|
|
|
def parse_paper(html,url,source,year,index_url):
|
|
nodes=list(Document(html).root.walk())
|
|
meta=defaultdict(list)
|
|
for node in nodes:
|
|
if node.tag=='meta':
|
|
meta[node.attrs.get('name','').lower()].append(node.attrs.get('content',''))
|
|
title=next(iter(meta['citation_title']),None)
|
|
if not title:
|
|
title=next((clean(n.text()) for n in nodes if n.tag=='h1'),None)
|
|
abstract=next((clean(n.text()) for n in nodes if 'paper-abstract' in n.attrs.get('class','').split()),None)
|
|
if not abstract:
|
|
for index,node in enumerate(nodes):
|
|
if node.tag in ('h2','h3','h4','h5') and clean(node.text()).lower()=='abstract':
|
|
following=[]
|
|
covered=set()
|
|
for candidate in nodes[index+1:]:
|
|
if candidate.tag in ('h1','h2','h3','h4','h5'):
|
|
break
|
|
if candidate.tag=='p' and candidate not in covered:
|
|
following.append(clean(candidate.text()))
|
|
covered.update(candidate.walk())
|
|
abstract=' '.join(following).strip() or None
|
|
break
|
|
if not title:
|
|
raise ValueError('Missing title on '+url)
|
|
# Never fall back to footer/policy text when the abstract is absent.
|
|
authors=[]
|
|
for author in meta['citation_author']:
|
|
if ',' in author:
|
|
family,given=author.split(',',1)
|
|
author=given.strip()+' '+family.strip()
|
|
authors.append(clean(author))
|
|
track_text=' '.join(clean(n.text()) for n in nodes if 'paper-track' in n.attrs.get('class','').split())
|
|
track=track_key(track_text or url)
|
|
identifiers={}
|
|
if meta['citation_doi']:
|
|
identifiers['doi']=meta['citation_doi'][0]
|
|
action_nodes=[]
|
|
for container in nodes:
|
|
if 'paper-actions' in container.attrs.get('class','').split():
|
|
action_nodes.extend(container.walk())
|
|
# Abstract links can cite other papers; only labelled self links identify this one.
|
|
for node in action_nodes:
|
|
if node.tag!='a':
|
|
continue
|
|
link=urljoin(url,node.attrs.get('href',''))
|
|
parsed=urlsplit(link)
|
|
if parsed.netloc in ('arxiv.org','www.arxiv.org') and re.match(r'^/(abs|pdf)/',parsed.path):
|
|
identifiers['arxiv']=re.sub(r'^/(abs|pdf)/','',parsed.path).removesuffix('.pdf')
|
|
if parsed.netloc=='openreview.net' and parsed.path=='/forum':
|
|
from urllib.parse import parse_qs
|
|
forum=parse_qs(parsed.query).get('id')
|
|
if forum: identifiers['openreview']=forum[0]
|
|
venue='COLM' if source=='colm' else 'NeurIPS'
|
|
return {'source':source,'external_id':urlsplit(url).path,'url':url,'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,'venue':venue,'year':year,'track':track,'status':'accepted','raw_decision':'listed in official proceedings','list_url':index_url,'metadata':{'citation':dict(meta),'publication_state':'pre-proceedings' if 'pre-proceedings' in html.lower() else 'proceedings'}}
|
|
|
|
|
|
def discover(source,cache):
|
|
root=ROOTS[source]
|
|
html=fetch(root,cache,8*1024**2,delay=1.0).decode()
|
|
result={}
|
|
for node in Document(html).root.walk():
|
|
if node.tag!='a':continue
|
|
url=urljoin(root,node.attrs.get('href',''))
|
|
parsed=urlsplit(url)
|
|
match=re.search(r'/paper(?:_files/paper)?/((?:19|20)\d{2})(?:/|$)',parsed.path)
|
|
if parsed.netloc==urlsplit(root).netloc and match and not any(term in parsed.path for term in ('/hash/','/file/','.pdf','-Abstract')):
|
|
result[url]=int(match[1])
|
|
if source=='neurips' and parsed.netloc=='datasets-benchmarks-proceedings.neurips.cc' and match:
|
|
result[url]=int(match[1])
|
|
if not result:
|
|
raise ValueError('No official year indexes found')
|
|
return sorted(result.items(),key=lambda entry:(entry[1],entry[0]))
|
|
|
|
|
|
def collect_index(source,index_url,year,cache,out):
|
|
html=fetch(index_url,cache,16*1024**2,delay=1.0).decode()
|
|
links=paper_links(html,index_url)
|
|
grouped=defaultdict(list)
|
|
missing=[]
|
|
for number,url in enumerate(links,1):
|
|
record=parse_paper(fetch(url,cache,2*1024**2,delay=1.0).decode(),url,source,year,index_url)
|
|
if not record['abstract']:missing.append(url)
|
|
grouped[record['track']].append(record)
|
|
if number%25==0:
|
|
print(json.dumps({'source':source,'year':year,'collected':number,'expected':len(links)}),flush=True)
|
|
results=[]
|
|
for track,records in grouped.items():
|
|
# Distinct official index URLs may publish multiple volumes in one year.
|
|
import hashlib
|
|
suffix=hashlib.sha256(index_url.encode()).hexdigest()[:10]
|
|
path=Path(out)/f'{source}-{year}-{track}-{suffix}.jsonl'
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
part=path.with_suffix('.part')
|
|
with part.open('w') as output:
|
|
for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n')
|
|
part.replace(path)
|
|
missing_count=sum(not r['abstract'] for r in records)
|
|
result={'source':source,'parser_version':2,'venue':records[0]['venue'],'year':year,'track':track,'count':len(records),'index_link_count':len(links),'index_url':index_url,'list_url':index_url,'missing_abstract_count':missing_count,'file':str(path),'status':'collected' if not missing_count else 'partial'}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
|
|
results.append(result)
|
|
print(json.dumps(result),flush=True)
|
|
return results
|
|
|
|
|
|
if __name__=='__main__':
|
|
parser=argparse.ArgumentParser()
|
|
parser.add_argument('--source',choices=ROOTS,required=True)
|
|
parser.add_argument('--cache',required=True)
|
|
parser.add_argument('--out',required=True)
|
|
parser.add_argument('--year',type=int,action='append')
|
|
args=parser.parse_args()
|
|
out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
|
|
status_path=out/f'{args.source}-index-status.json'
|
|
catalogue=json.loads(status_path.read_text()) if status_path.exists() else {}
|
|
indexes=discover(args.source,args.cache)
|
|
if args.year and set(args.year)-{year for _,year in indexes}:
|
|
parser.error('Requested year is absent from the current official proceedings index')
|
|
requested=[]
|
|
for url,year in indexes:
|
|
if args.year and year not in args.year:continue
|
|
requested.append(url)
|
|
try:
|
|
files=collect_index(args.source,url,year,args.cache,args.out)
|
|
catalogue[url]={'status':'partial' if any(f['status']=='partial' for f in files) else 'collected','files':files}
|
|
except Exception as exc:
|
|
catalogue[url]={'status':'blocked','error':str(exc),'year':year}
|
|
print(json.dumps(catalogue[url]),flush=True)
|
|
part=status_path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_path)
|
|
(out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year},indent=2))
|
|
if any(catalogue[url]['status']=='blocked' for url in requested):raise SystemExit(1)
|