"""Collect COLM/NeurIPS official abstract pages without downloading PDFs.""" import argparse import json import re import time from collections import defaultdict from html.parser import HTMLParser from pathlib import Path from urllib.parse import urljoin,urlsplit from collect_pmlr import fetch ROOTS={'colm':'https://proceedings.colm.cc/','neurips':'https://proceedings.neurips.cc/'} class Node: def __init__(self,tag='',attrs=None): self.tag=tag self.attrs=dict(attrs or []) self.children=[] def text(self): if self.tag in ('script','style'): return '' content=''.join(child.text() if isinstance(child,Node) else child for child in self.children) return ' '+content+' ' if self.tag in ('p','div','section','li','br') else content def walk(self): yield self for child in self.children: if isinstance(child,Node): yield from child.walk() class Document(HTMLParser): VOID={'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'} def __init__(self,html): super().__init__(convert_charrefs=True) self.root=Node() self.stack=[self.root] self.feed(html) def handle_starttag(self,tag,attrs): node=Node(tag,attrs) self.stack[-1].children.append(node) if tag not in self.VOID: self.stack.append(node) def handle_endtag(self,tag): for index in range(len(self.stack)-1,0,-1): if self.stack[index].tag==tag: del self.stack[index:] break def handle_data(self,data): self.stack[-1].children.append(data) def clean(text): return ' '.join(text.split()) def track_key(text): text=text.lower() if 'dataset' in text or 'benchmark' in text: return 'datasets_benchmarks' if 'creative' in text: return 'creative_ai' if 'position' in text: return 'position' return 'main' def paper_links(html,index_url): doc=Document(html) links=sorted({urljoin(index_url,n.attrs['href']) for n in doc.root.walk() if n.tag=='a' and '-Abstract' in n.attrs.get('href','') and urlsplit(urljoin(index_url,n.attrs['href'])).path.endswith('.html') and urlsplit(urljoin(index_url,n.attrs['href'])).netloc==urlsplit(index_url).netloc}) declared=next((match for n in doc.root.walk() if n.tag in ('p','span','div') and not any(child.tag=='a' for child in n.walk()) and (match:=re.fullmatch(r'([\d,]+)\s+papers',clean(n.text()),re.I))),None) if not links: raise ValueError('Official index contains no abstract-page links') if declared and int(declared[1].replace(',',''))!=len(links): raise ValueError('Declared paper count differs from unique abstract-page links') return links def parse_paper(html,url,source,year,index_url): nodes=list(Document(html).root.walk()) meta=defaultdict(list) for node in nodes: if node.tag=='meta': meta[node.attrs.get('name','').lower()].append(node.attrs.get('content','')) title=next(iter(meta['citation_title']),None) if not title: title=next((clean(n.text()) for n in nodes if n.tag=='h1'),None) abstract=next((clean(n.text()) for n in nodes if 'paper-abstract' in n.attrs.get('class','').split()),None) if not abstract: for index,node in enumerate(nodes): if node.tag in ('h2','h3','h4','h5') and clean(node.text()).lower()=='abstract': following=[] covered=set() for candidate in nodes[index+1:]: if candidate.tag in ('h1','h2','h3','h4','h5'): break if candidate.tag=='p' and candidate not in covered: following.append(clean(candidate.text())) covered.update(candidate.walk()) abstract=' '.join(following).strip() or None break if not title: raise ValueError('Missing title on '+url) # Never fall back to footer/policy text when the abstract is absent. authors=[] for author in meta['citation_author']: if ',' in author: family,given=author.split(',',1) author=given.strip()+' '+family.strip() authors.append(clean(author)) track_text=' '.join(clean(n.text()) for n in nodes if 'paper-track' in n.attrs.get('class','').split()) track=track_key(track_text or url) identifiers={} if meta['citation_doi']: identifiers['doi']=meta['citation_doi'][0] action_nodes=[] for container in nodes: if 'paper-actions' in container.attrs.get('class','').split(): action_nodes.extend(container.walk()) # Abstract links can cite other papers; only labelled self links identify this one. for node in action_nodes: if node.tag!='a': continue link=urljoin(url,node.attrs.get('href','')) parsed=urlsplit(link) if parsed.netloc in ('arxiv.org','www.arxiv.org') and re.match(r'^/(abs|pdf)/',parsed.path): identifiers['arxiv']=re.sub(r'^/(abs|pdf)/','',parsed.path).removesuffix('.pdf') if parsed.netloc=='openreview.net' and parsed.path=='/forum': from urllib.parse import parse_qs forum=parse_qs(parsed.query).get('id') if forum: identifiers['openreview']=forum[0] venue='COLM' if source=='colm' else 'NeurIPS' return {'source':source,'external_id':urlsplit(url).path,'url':url,'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,'venue':venue,'year':year,'track':track,'status':'accepted','raw_decision':'listed in official proceedings','list_url':index_url,'metadata':{'citation':dict(meta),'publication_state':'pre-proceedings' if 'pre-proceedings' in html.lower() else 'proceedings'}} def discover(source,cache): root=ROOTS[source] html=fetch(root,cache,8*1024**2,delay=1.0).decode() result={} for node in Document(html).root.walk(): if node.tag!='a':continue url=urljoin(root,node.attrs.get('href','')) parsed=urlsplit(url) match=re.search(r'/paper(?:_files/paper)?/((?:19|20)\d{2})(?:/|$)',parsed.path) if parsed.netloc==urlsplit(root).netloc and match and not any(term in parsed.path for term in ('/hash/','/file/','.pdf','-Abstract')): result[url]=int(match[1]) if source=='neurips' and parsed.netloc=='datasets-benchmarks-proceedings.neurips.cc' and match: result[url]=int(match[1]) if not result: raise ValueError('No official year indexes found') return sorted(result.items(),key=lambda entry:(entry[1],entry[0])) def collect_index(source,index_url,year,cache,out): html=fetch(index_url,cache,16*1024**2,delay=1.0).decode() links=paper_links(html,index_url) grouped=defaultdict(list) missing=[] for number,url in enumerate(links,1): record=parse_paper(fetch(url,cache,2*1024**2,delay=1.0).decode(),url,source,year,index_url) if not record['abstract']:missing.append(url) grouped[record['track']].append(record) if number%25==0: print(json.dumps({'source':source,'year':year,'collected':number,'expected':len(links)}),flush=True) results=[] for track,records in grouped.items(): # Distinct official index URLs may publish multiple volumes in one year. import hashlib suffix=hashlib.sha256(index_url.encode()).hexdigest()[:10] path=Path(out)/f'{source}-{year}-{track}-{suffix}.jsonl' path.parent.mkdir(parents=True,exist_ok=True) part=path.with_suffix('.part') with part.open('w') as output: for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n') part.replace(path) missing_count=sum(not r['abstract'] for r in records) result={'source':source,'parser_version':2,'venue':records[0]['venue'],'year':year,'track':track,'count':len(records),'index_link_count':len(links),'index_url':index_url,'list_url':index_url,'missing_abstract_count':missing_count,'file':str(path),'status':'collected' if not missing_count else 'partial'} path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2)) results.append(result) print(json.dumps(result),flush=True) return results if __name__=='__main__': parser=argparse.ArgumentParser() parser.add_argument('--source',choices=ROOTS,required=True) parser.add_argument('--cache',required=True) parser.add_argument('--out',required=True) parser.add_argument('--year',type=int,action='append') args=parser.parse_args() out=Path(args.out);out.mkdir(parents=True,exist_ok=True) status_path=out/f'{args.source}-index-status.json' catalogue=json.loads(status_path.read_text()) if status_path.exists() else {} indexes=discover(args.source,args.cache) if args.year and set(args.year)-{year for _,year in indexes}: parser.error('Requested year is absent from the current official proceedings index') requested=[] for url,year in indexes: if args.year and year not in args.year:continue requested.append(url) try: files=collect_index(args.source,url,year,args.cache,args.out) catalogue[url]={'status':'partial' if any(f['status']=='partial' for f in files) else 'collected','files':files} except Exception as exc: catalogue[url]={'status':'blocked','error':str(exc),'year':year} print(json.dumps(catalogue[url]),flush=True) part=status_path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_path) (out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year},indent=2)) if any(catalogue[url]['status']=='blocked' for url in requested):raise SystemExit(1)