Collect official COLM and NeurIPS abstracts with track-aware coverage
Metadata integrity checks / metadata (push) Successful in 4s

This commit is contained in:
2026-11-06 12:04:18 +08:00
parent a3539be4b1
commit ad9967c1b2
9 changed files with 323 additions and 13 deletions
+18 -1
View File
@@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工
目标包括:完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。
**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集,其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集,其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
## 数据范围与来源
@@ -123,6 +123,23 @@ python3 -B collect_peerread.py --cache /data/paper-index/raw/http-cache \
数据任务以一次性 systemd 服务运行:`paper-atlas-arxiv-import`、`paper-atlas-pmlr-collect`、`paper-atlas-peerread-collect`;`paper-atlas-pmlr-verify` 用于全索引复核,`paper-atlas-conference-merge` 用于单写入合并。它们不是定时任务。主库只允许一个写入进程;会议采集先写独立 JSONL,待 arXiv 主批次结束后再合并;也可以使用 `merge_conferences.py` 在保存断点处暂停本项目的 arXiv unit、顺序合并会议清单并恢复。该操作应作为独立的一次性 systemd 任务运行,避免 SSH 断开影响恢复;不能同时启动多个合并任务。运行日志位于 `/data/paper-index/*-import.log`、`/data/paper-index/*-collect.log`。任何服务启动成功都不等于批次已完成,需检查批次状态、清单与完整性结果。标签按来源刷新使用 `paper_tags_source_evidence` 索引,避免每条论文都扫描整张标签表。运行中的 arXiv 可以通过同一协调脚本的 `--schema-only` 模式在断点更新索引后恢复;此模式要求已有 arXiv unit 正在运行。
### COLM 与 NeurIPS 官方摘要
```sh
python3 -B collect_proceedings.py --source colm \
--cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
python3 -B collect_proceedings.py --source neurips \
--cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
```
采集器从官方根目录发现实际发表届次,仅访问索引和 Abstract HTML 页面,不请求 PDF、补充材料或源码。可用重复 `--year` 限定实际存在的届次;不存在的年份会报错,局部重试保留其他届次的已有状态。每个来源串行请求,实际网络响应后等待 1 秒;缓存命中不额外等待。
COLM 当前目录提供 2024–2026 三届 pre-proceedings,按会议年份保存,保留平台实际出版日期和出版状态,不把晚补刊日期当作会议年份。NeurIPS 当前目录提供 1987–2025,并包含 2025 独立卷及 2021 独立 Datasets and Benchmarks 索引。主会、datasets_benchmarks、position、creative_ai 分开出文件,不混为同一个 track。首次真实样本核对:COLM 2024 官方目录 299 个摘要入口;NeurIPS 1987 为 90 个,作者与摘要均已验证解析。样本成功不代表所有页面采集完成。
`colm-index-status.json`、`neurips-index-status.json` 保存逐索引状态;对应 `*-last-run.json` 保存本次实际选择范围。每个 JSONL 的 manifest 记录索引入口数、track 记录数和缺失摘要数。论文数量与官方页面声明不一致会失败,缺失摘要会标记 partial;不会把页脚、姓名政策或其他正文当作摘要。
这两个采集器只写采集文件,合并仍通过单写入协调流程。全量历史覆盖和所有未实现会议仍待最终核验。
## 长期维护与更新
1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发,遵守来源限流、重试与人机验证要求。
+9 -4
View File
@@ -38,7 +38,7 @@ def venue_key(shortname):
return re.sub(r'[\s_-]*(?:19|20)\d{2}$','',str(shortname).strip().upper())
def fetch(url,cache,limit=64*1024**2):
def fetch(url,cache,limit=64*1024**2,delay=0.5):
cache = Path(cache)
cache.mkdir(parents=True,exist_ok=True)
path = cache/hashlib.sha256(url.encode()).hexdigest()
@@ -59,7 +59,7 @@ def fetch(url,cache,limit=64*1024**2):
part.write_bytes(data)
part.replace(path)
path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
time.sleep(0.5)
time.sleep(delay)
return data
except urllib.error.HTTPError as exc:
if exc.code not in (429,500,502,503,504) or attempt==2:
@@ -128,14 +128,19 @@ def collect_volume(volume,cache,out,allowed=None):
path.parent.mkdir(parents=True,exist_ok=True)
part = path.with_suffix('.part')
count = 0
missing_abstracts = 0
with part.open('w') as output:
for member_name,content in metadata_texts(volume,cache):
parts = re.split(r'(?m)^---\s*$',content,maxsplit=2)
if len(parts)<3 or parts[0].strip():
raise RuntimeError('Missing YAML frontmatter: '+member_name)
row = load_metadata(parts[1])
if row.get('layout') not in ('inproceedings','article') or not row.get('title'):
if row.get('layout') not in ('inproceedings','article'):
continue
if not row.get('title'):
raise ValueError('Paper metadata has no title: '+member_name)
if not (row.get('abstract') or '').strip():
missing_abstracts+=1
external = f"v{volume}/{row.get('id') or Path(member_name).stem}"
ids = {}
for key in ('doi','openreview','arxiv'):
@@ -154,7 +159,7 @@ def collect_volume(volume,cache,out,allowed=None):
part.unlink(missing_ok=True)
raise RuntimeError('No papers parsed; refusing an empty successful collection')
part.replace(path)
result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'}
result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'missing_abstract_count':missing_abstracts,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'partial' if missing_abstracts else 'collected'}
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
print(json.dumps(result),flush=True)
return result
+211
View File
@@ -0,0 +1,211 @@
"""Collect COLM/NeurIPS official abstract pages without downloading PDFs."""
import argparse
import json
import re
import time
from collections import defaultdict
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import urljoin,urlsplit
from collect_pmlr import fetch
ROOTS={'colm':'https://proceedings.colm.cc/','neurips':'https://proceedings.neurips.cc/'}
class Node:
def __init__(self,tag='',attrs=None):
self.tag=tag
self.attrs=dict(attrs or [])
self.children=[]
def text(self):
if self.tag in ('script','style'):
return ''
content=''.join(child.text() if isinstance(child,Node) else child for child in self.children)
return ' '+content+' ' if self.tag in ('p','div','section','li','br') else content
def walk(self):
yield self
for child in self.children:
if isinstance(child,Node):
yield from child.walk()
class Document(HTMLParser):
VOID={'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'}
def __init__(self,html):
super().__init__(convert_charrefs=True)
self.root=Node()
self.stack=[self.root]
self.feed(html)
def handle_starttag(self,tag,attrs):
node=Node(tag,attrs)
self.stack[-1].children.append(node)
if tag not in self.VOID:
self.stack.append(node)
def handle_endtag(self,tag):
for index in range(len(self.stack)-1,0,-1):
if self.stack[index].tag==tag:
del self.stack[index:]
break
def handle_data(self,data):
self.stack[-1].children.append(data)
def clean(text):
return ' '.join(text.split())
def track_key(text):
text=text.lower()
if 'dataset' in text or 'benchmark' in text:
return 'datasets_benchmarks'
if 'creative' in text:
return 'creative_ai'
if 'position' in text:
return 'position'
return 'main'
def paper_links(html,index_url):
doc=Document(html)
links=sorted({urljoin(index_url,n.attrs['href']) for n in doc.root.walk() if n.tag=='a' and '-Abstract' in n.attrs.get('href','') and urlsplit(urljoin(index_url,n.attrs['href'])).path.endswith('.html') and urlsplit(urljoin(index_url,n.attrs['href'])).netloc==urlsplit(index_url).netloc})
declared=next((match for n in doc.root.walk() if n.tag in ('p','span','div') and not any(child.tag=='a' for child in n.walk()) and (match:=re.fullmatch(r'([\d,]+)\s+papers',clean(n.text()),re.I))),None)
if not links:
raise ValueError('Official index contains no abstract-page links')
if declared and int(declared[1].replace(',',''))!=len(links):
raise ValueError('Declared paper count differs from unique abstract-page links')
return links
def parse_paper(html,url,source,year,index_url):
nodes=list(Document(html).root.walk())
meta=defaultdict(list)
for node in nodes:
if node.tag=='meta':
meta[node.attrs.get('name','').lower()].append(node.attrs.get('content',''))
title=next(iter(meta['citation_title']),None)
if not title:
title=next((clean(n.text()) for n in nodes if n.tag=='h1'),None)
abstract=next((clean(n.text()) for n in nodes if 'paper-abstract' in n.attrs.get('class','').split()),None)
if not abstract:
for index,node in enumerate(nodes):
if node.tag in ('h2','h3','h4','h5') and clean(node.text()).lower()=='abstract':
following=[]
covered=set()
for candidate in nodes[index+1:]:
if candidate.tag in ('h1','h2','h3','h4','h5'):
break
if candidate.tag=='p' and candidate not in covered:
following.append(clean(candidate.text()))
covered.update(candidate.walk())
abstract=' '.join(following).strip() or None
break
if not title:
raise ValueError('Missing title on '+url)
# Never fall back to footer/policy text when the abstract is absent.
authors=[]
for author in meta['citation_author']:
if ',' in author:
family,given=author.split(',',1)
author=given.strip()+' '+family.strip()
authors.append(clean(author))
track_text=' '.join(clean(n.text()) for n in nodes if 'paper-track' in n.attrs.get('class','').split())
track=track_key(track_text or url)
identifiers={}
if meta['citation_doi']:
identifiers['doi']=meta['citation_doi'][0]
for node in nodes:
if node.tag!='a':
continue
link=urljoin(url,node.attrs.get('href',''))
parsed=urlsplit(link)
if parsed.netloc in ('arxiv.org','www.arxiv.org') and re.match(r'^/(abs|pdf)/',parsed.path):
identifiers['arxiv']=re.sub(r'^/(abs|pdf)/','',parsed.path).removesuffix('.pdf')
if parsed.netloc=='openreview.net' and parsed.path=='/forum':
from urllib.parse import parse_qs
forum=parse_qs(parsed.query).get('id')
if forum: identifiers['openreview']=forum[0]
venue='COLM' if source=='colm' else 'NeurIPS'
return {'source':source,'external_id':urlsplit(url).path,'url':url,'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,'venue':venue,'year':year,'track':track,'status':'accepted','raw_decision':'listed in official proceedings','list_url':index_url,'metadata':{'citation':dict(meta),'publication_state':'pre-proceedings' if 'pre-proceedings' in html.lower() else 'proceedings'}}
def discover(source,cache):
root=ROOTS[source]
html=fetch(root,cache,8*1024**2,delay=1.0).decode()
result={}
for node in Document(html).root.walk():
if node.tag!='a':continue
url=urljoin(root,node.attrs.get('href',''))
parsed=urlsplit(url)
match=re.search(r'/paper(?:_files/paper)?/((?:19|20)\d{2})(?:/|$)',parsed.path)
if parsed.netloc==urlsplit(root).netloc and match and not any(term in parsed.path for term in ('/hash/','/file/','.pdf','-Abstract')):
result[url]=int(match[1])
if source=='neurips' and parsed.netloc=='datasets-benchmarks-proceedings.neurips.cc' and match:
result[url]=int(match[1])
if not result:
raise ValueError('No official year indexes found')
return sorted(result.items(),key=lambda entry:(entry[1],entry[0]))
def collect_index(source,index_url,year,cache,out):
html=fetch(index_url,cache,16*1024**2,delay=1.0).decode()
links=paper_links(html,index_url)
grouped=defaultdict(list)
missing=[]
for number,url in enumerate(links,1):
record=parse_paper(fetch(url,cache,2*1024**2,delay=1.0).decode(),url,source,year,index_url)
if not record['abstract']:missing.append(url)
grouped[record['track']].append(record)
if number%25==0:
print(json.dumps({'source':source,'year':year,'collected':number,'expected':len(links)}),flush=True)
results=[]
for track,records in grouped.items():
# Distinct official index URLs may publish multiple volumes in one year.
import hashlib
suffix=hashlib.sha256(index_url.encode()).hexdigest()[:10]
path=Path(out)/f'{source}-{year}-{track}-{suffix}.jsonl'
path.parent.mkdir(parents=True,exist_ok=True)
part=path.with_suffix('.part')
with part.open('w') as output:
for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n')
part.replace(path)
missing_count=sum(not r['abstract'] for r in records)
result={'source':source,'venue':records[0]['venue'],'year':year,'track':track,'count':len(records),'index_link_count':len(links),'index_url':index_url,'list_url':index_url,'missing_abstract_count':missing_count,'file':str(path),'status':'collected' if not missing_count else 'partial'}
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
results.append(result)
print(json.dumps(result),flush=True)
return results
if __name__=='__main__':
parser=argparse.ArgumentParser()
parser.add_argument('--source',choices=ROOTS,required=True)
parser.add_argument('--cache',required=True)
parser.add_argument('--out',required=True)
parser.add_argument('--year',type=int,action='append')
args=parser.parse_args()
out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
status_path=out/f'{args.source}-index-status.json'
catalogue=json.loads(status_path.read_text()) if status_path.exists() else {}
indexes=discover(args.source,args.cache)
if args.year and set(args.year)-{year for _,year in indexes}:
parser.error('Requested year is absent from the current official proceedings index')
requested=[]
for url,year in indexes:
if args.year and year not in args.year:continue
requested.append(url)
try:
files=collect_index(args.source,url,year,args.cache,args.out)
catalogue[url]={'status':'partial' if any(f['status']=='partial' for f in files) else 'collected','files':files}
except Exception as exc:
catalogue[url]={'status':'blocked','error':str(exc),'year':year}
print(json.dumps(catalogue[url]),flush=True)
part=status_path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_path)
(out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year},indent=2))
if any(catalogue[url]['status']=='blocked' for url in requested):raise SystemExit(1)
+2 -1
View File
@@ -47,7 +47,8 @@ def ingest_file(db,path):
expected = manifest.get('count')
if expected is not None and expected != count:
raise ValueError(f'Manifest count {expected} differs from records {count}')
c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if expected==count else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
complete=expected==count and manifest.get('status') not in ('partial','blocked') and not manifest.get('missing_abstract_count',0)
c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if complete else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
c.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=? WHERE id=?",(count,count,run))
c.commit()
print(json.dumps({'file':str(path),'status':'imported','records':count}),flush=True)
+3 -2
View File
@@ -4,6 +4,7 @@ Run as a transient systemd job so an SSH disconnect cannot strand the importer.
Only the Paper Atlas arXiv unit is controlled; business services are untouched.
"""
import argparse
from contextlib import closing
import json
import sqlite3
import subprocess
@@ -34,14 +35,14 @@ def merge(db,archive,files):
raise RuntimeError('Expected the existing arXiv unit to be active; inspect it before proceeding')
subprocess.run(['systemctl','stop',UNIT],check=True)
try:
with sqlite3.connect(db) as c:
with closing(sqlite3.connect(db)) as c, c:
checkpoint=c.execute("SELECT records_written FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone()
print('Saved arxiv checkpoint:',checkpoint,flush=True)
schema_store=Store(db)
schema_store.conn.close()
for filename in files:
ingest_file(db,filename)
with sqlite3.connect(db) as c:
with closing(sqlite3.connect(db)) as c, c:
if files:
errors=c.execute('PRAGMA foreign_key_check').fetchall()
if errors:
+5 -4
View File
@@ -1,4 +1,5 @@
import json
from contextlib import closing
import sqlite3
import tempfile
import unittest
@@ -27,7 +28,7 @@ class ImportTests(unittest.TestCase):
self.write([self.row('2601.00001','10.0000/example'),self.row('2601.00002','10.0000/example')])
ingest(self.db,self.archive)
ingest(self.db,self.archive)
with sqlite3.connect(self.db) as c:
with closing(sqlite3.connect(self.db)) as c, c:
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2)
self.assertEqual(c.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'Searchable'").fetchone()[0],1)
@@ -38,18 +39,18 @@ class ImportTests(unittest.TestCase):
def test_title_alone_does_not_merge(self):
self.write([self.row('2601.00001'),self.row('2601.00002')])
ingest(self.db,self.archive)
with sqlite3.connect(self.db) as c:
with closing(sqlite3.connect(self.db)) as c, c:
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2)
def test_platform_and_custom_tags_remain_distinct(self):
self.write([self.row('2601.00001')])
ingest(self.db,self.archive)
with sqlite3.connect(self.db) as c:
with closing(sqlite3.connect(self.db)) as c, c:
paper = c.execute('SELECT id FROM papers').fetchone()[0]
tag = c.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','cs.AI')").lastrowid
c.execute('INSERT INTO paper_tags VALUES(?,?,NULL,?)',(paper,tag,'manual:1'))
ingest(self.db,self.archive)
with sqlite3.connect(self.db) as c:
with closing(sqlite3.connect(self.db)) as c, c:
self.assertEqual(c.execute("SELECT count(*) FROM paper_tags pt JOIN tags t ON pt.tag_id=t.id WHERE t.value='cs.AI'").fetchone()[0],2)
+7
View File
@@ -58,6 +58,13 @@ class StoreTests(unittest.TestCase):
self.assertEqual(self.store.conn.execute('SELECT title FROM papers').fetchone()[0],'Updated research title')
self.assertEqual(self.store.conn.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'searchable'").fetchone()[0],1)
def test_missing_abstract_does_not_become_complete_coverage(self):
path=Path(self.tmp.name)/'partial.jsonl'
path.write_text(json.dumps({'source':'colm','external_id':'paper1','url':'https://example.org/paper1','title':'Paper','abstract':None,'authors':[],'venue':'COLM','year':2024,'track':'main','status':'accepted'})+'\n')
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'colm','list_url':'https://example.org/list','count':1,'status':'partial','missing_abstract_count':1}))
ingest_file(self.db,path)
self.assertEqual(self.store.conn.execute('SELECT status FROM edition_sources').fetchone()[0],'partial')
if __name__=='__main__':
unittest.main()
+12 -1
View File
@@ -1,6 +1,8 @@
import unittest
import tempfile
from unittest.mock import patch
from collect_pmlr import load_metadata,venue_key
from collect_pmlr import load_metadata,venue_key,collect_volume
class MetadataTests(unittest.TestCase):
@@ -20,6 +22,15 @@ class MetadataTests(unittest.TestCase):
self.assertEqual(venue_key('COLT 2012'),'COLT')
self.assertEqual(venue_key('ECML-PKDD'),'ECML-PKDD')
def test_missing_abstract_is_reported_as_partial_collection(self):
config=b'shortname: ICML\nyear: 2025\nbooktitle: ICML\n'
paper='---\nid: paper1\nlayout: inproceedings\ntitle: Paper\nabstract:\n---\n'
with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.fetch',return_value=config),patch('collect_pmlr.metadata_texts',return_value=iter([('paper.md',paper)])):
result=collect_volume(999,tmp,tmp)
self.assertEqual(result['count'],1)
self.assertEqual(result['missing_abstract_count'],1)
self.assertEqual(result['status'],'partial')
if __name__=='__main__':
unittest.main()
+56
View File
@@ -0,0 +1,56 @@
import unittest
import json
import runpy
import tempfile
from pathlib import Path
from unittest.mock import patch
from collect_proceedings import paper_links,parse_paper
class ProceedingsTests(unittest.TestCase):
def test_colm_nested_abstract_and_citation_authors(self):
html='''<meta name="citation_title" content="A paper"><meta name="citation_author" content="Doe, Jane"><section><h2>Abstract</h2><p class="paper-abstract"><p>Real <em>abstract</em> text.</p></p></section><h4>Name Change Policy</h4><p>Do not ingest footer.</p>'''
row=parse_paper(html,'https://proceedings.colm.cc/paper_files/paper/2024/hash/a-Abstract-Conference.html','colm',2024,'https://proceedings.colm.cc/paper_files/paper/2024')
self.assertEqual(row['abstract'],'Real abstract text.')
self.assertEqual(row['authors'],['Jane Doe'])
self.assertEqual(row['track'],'main')
self.assertEqual(row['year'],2024)
def test_declared_index_count_detects_truncation(self):
html='<p>2 papers</p><a href="/paper/2024/hash/a-Abstract.html">Paper</a>'
with self.assertRaisesRegex(ValueError,'count differs'):
paper_links(html,'https://proceedings.colm.cc/paper/2024')
def test_missing_abstract_does_not_capture_footer(self):
row=parse_paper('<h1>Paper</h1><h4>Name Change Policy</h4><p>Footer</p>','https://example.org/a','colm',2024,'https://example.org/')
self.assertIsNone(row['abstract'])
def test_paper_title_is_not_mistaken_for_index_paper_count(self):
html='<li><a href="/paper/2021/hash/a-Abstract.html">Lessons from 1000 papers</a></li>'
self.assertEqual(len(paper_links(html,'https://datasets-benchmarks-proceedings.neurips.cc/paper/2021')),1)
def test_neurips_track_and_legacy_abstract(self):
html='<h1>Paper</h1><span class="paper-track">Datasets and Benchmarks Track</span><h4>Abstract</h4><p>Actual abstract.</p><h4>Name Change Policy</h4><p>Footer</p>'
row=parse_paper(html,'https://proceedings.neurips.cc/a','neurips',2025,'https://proceedings.neurips.cc/')
self.assertEqual(row['abstract'],'Actual abstract.')
self.assertEqual(row['track'],'datasets_benchmarks')
def test_focused_retry_preserves_other_index_status(self):
root='https://proceedings.colm.cc/'
url=root+'paper_files/paper/2024'
other=root+'paper_files/paper/2025'
paper=root+'paper_files/paper/2024/hash/a-Abstract-Conference.html'
documents={root:f'<a href="{url}">2024</a><a href="{other}">2025</a>',url:f'<p>1 papers</p><a href="{paper}">Paper</a>',paper:'<h1>Paper</h1><p class="paper-abstract">Abstract</p>'}
with tempfile.TemporaryDirectory() as tmp:
status=Path(tmp)/'colm-index-status.json'
status.write_text(json.dumps({other:{'status':'collected','files':[]}}))
argv=['collect_proceedings.py','--source','colm','--cache',tmp,'--out',tmp,'--year','2024']
with patch('sys.argv',argv),patch('collect_pmlr.fetch',side_effect=lambda u,*a,**kw:documents[u].encode()):
runpy.run_path(str(Path(__file__).with_name('collect_proceedings.py')),run_name='__main__')
result=json.loads(status.read_text())
self.assertEqual(result[other]['status'],'collected')
self.assertEqual(result[url]['files'][0]['count'],1)
if __name__=='__main__':unittest.main()