From ad9967c1b21e26fee3565617cd26e3aae2c61b82 Mon Sep 17 00:00:00 2001
From: Codex Bot
Date: Fri, 6 Nov 2026 12:04:18 +0800
Subject: [PATCH] Collect official COLM and NeurIPS abstracts with track-aware
coverage
---
README.md | 19 +++-
collect_pmlr.py | 13 ++-
collect_proceedings.py | 211 +++++++++++++++++++++++++++++++++++++++++
import_records.py | 3 +-
merge_conferences.py | 5 +-
test_import_arxiv.py | 9 +-
test_index_core.py | 7 ++
test_pmlr.py | 13 ++-
test_proceedings.py | 56 +++++++++++
9 files changed, 323 insertions(+), 13 deletions(-)
create mode 100644 collect_proceedings.py
create mode 100644 test_proceedings.py
diff --git a/README.md b/README.md
index 3c44c31..5b35682 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工
目标包括:完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。
-**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集,其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
+**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集,其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
## 数据范围与来源
@@ -123,6 +123,23 @@ python3 -B collect_peerread.py --cache /data/paper-index/raw/http-cache \
数据任务以一次性 systemd 服务运行:`paper-atlas-arxiv-import`、`paper-atlas-pmlr-collect`、`paper-atlas-peerread-collect`;`paper-atlas-pmlr-verify` 用于全索引复核,`paper-atlas-conference-merge` 用于单写入合并。它们不是定时任务。主库只允许一个写入进程;会议采集先写独立 JSONL,待 arXiv 主批次结束后再合并;也可以使用 `merge_conferences.py` 在保存断点处暂停本项目的 arXiv unit、顺序合并会议清单并恢复。该操作应作为独立的一次性 systemd 任务运行,避免 SSH 断开影响恢复;不能同时启动多个合并任务。运行日志位于 `/data/paper-index/*-import.log`、`/data/paper-index/*-collect.log`。任何服务启动成功都不等于批次已完成,需检查批次状态、清单与完整性结果。标签按来源刷新使用 `paper_tags_source_evidence` 索引,避免每条论文都扫描整张标签表。运行中的 arXiv 可以通过同一协调脚本的 `--schema-only` 模式在断点更新索引后恢复;此模式要求已有 arXiv unit 正在运行。
+### COLM 与 NeurIPS 官方摘要
+
+```sh
+python3 -B collect_proceedings.py --source colm \
+ --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
+python3 -B collect_proceedings.py --source neurips \
+ --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected
+```
+
+采集器从官方根目录发现实际发表届次,仅访问索引和 Abstract HTML 页面,不请求 PDF、补充材料或源码。可用重复 `--year` 限定实际存在的届次;不存在的年份会报错,局部重试保留其他届次的已有状态。每个来源串行请求,实际网络响应后等待 1 秒;缓存命中不额外等待。
+
+COLM 当前目录提供 2024–2026 三届 pre-proceedings,按会议年份保存,保留平台实际出版日期和出版状态,不把晚补刊日期当作会议年份。NeurIPS 当前目录提供 1987–2025,并包含 2025 独立卷及 2021 独立 Datasets and Benchmarks 索引。主会、datasets_benchmarks、position、creative_ai 分开出文件,不混为同一个 track。首次真实样本核对:COLM 2024 官方目录 299 个摘要入口;NeurIPS 1987 为 90 个,作者与摘要均已验证解析。样本成功不代表所有页面采集完成。
+
+`colm-index-status.json`、`neurips-index-status.json` 保存逐索引状态;对应 `*-last-run.json` 保存本次实际选择范围。每个 JSONL 的 manifest 记录索引入口数、track 记录数和缺失摘要数。论文数量与官方页面声明不一致会失败,缺失摘要会标记 partial;不会把页脚、姓名政策或其他正文当作摘要。
+
+这两个采集器只写采集文件,合并仍通过单写入协调流程。全量历史覆盖和所有未实现会议仍待最终核验。
+
## 长期维护与更新
1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发,遵守来源限流、重试与人机验证要求。
diff --git a/collect_pmlr.py b/collect_pmlr.py
index f43cfe3..2ba97c7 100644
--- a/collect_pmlr.py
+++ b/collect_pmlr.py
@@ -38,7 +38,7 @@ def venue_key(shortname):
return re.sub(r'[\s_-]*(?:19|20)\d{2}$','',str(shortname).strip().upper())
-def fetch(url,cache,limit=64*1024**2):
+def fetch(url,cache,limit=64*1024**2,delay=0.5):
cache = Path(cache)
cache.mkdir(parents=True,exist_ok=True)
path = cache/hashlib.sha256(url.encode()).hexdigest()
@@ -59,7 +59,7 @@ def fetch(url,cache,limit=64*1024**2):
part.write_bytes(data)
part.replace(path)
path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
- time.sleep(0.5)
+ time.sleep(delay)
return data
except urllib.error.HTTPError as exc:
if exc.code not in (429,500,502,503,504) or attempt==2:
@@ -128,14 +128,19 @@ def collect_volume(volume,cache,out,allowed=None):
path.parent.mkdir(parents=True,exist_ok=True)
part = path.with_suffix('.part')
count = 0
+ missing_abstracts = 0
with part.open('w') as output:
for member_name,content in metadata_texts(volume,cache):
parts = re.split(r'(?m)^---\s*$',content,maxsplit=2)
if len(parts)<3 or parts[0].strip():
raise RuntimeError('Missing YAML frontmatter: '+member_name)
row = load_metadata(parts[1])
- if row.get('layout') not in ('inproceedings','article') or not row.get('title'):
+ if row.get('layout') not in ('inproceedings','article'):
continue
+ if not row.get('title'):
+ raise ValueError('Paper metadata has no title: '+member_name)
+ if not (row.get('abstract') or '').strip():
+ missing_abstracts+=1
external = f"v{volume}/{row.get('id') or Path(member_name).stem}"
ids = {}
for key in ('doi','openreview','arxiv'):
@@ -154,7 +159,7 @@ def collect_volume(volume,cache,out,allowed=None):
part.unlink(missing_ok=True)
raise RuntimeError('No papers parsed; refusing an empty successful collection')
part.replace(path)
- result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'}
+ result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'missing_abstract_count':missing_abstracts,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'partial' if missing_abstracts else 'collected'}
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
print(json.dumps(result),flush=True)
return result
diff --git a/collect_proceedings.py b/collect_proceedings.py
new file mode 100644
index 0000000..19917c4
--- /dev/null
+++ b/collect_proceedings.py
@@ -0,0 +1,211 @@
+"""Collect COLM/NeurIPS official abstract pages without downloading PDFs."""
+import argparse
+import json
+import re
+import time
+from collections import defaultdict
+from html.parser import HTMLParser
+from pathlib import Path
+from urllib.parse import urljoin,urlsplit
+
+from collect_pmlr import fetch
+
+ROOTS={'colm':'https://proceedings.colm.cc/','neurips':'https://proceedings.neurips.cc/'}
+
+
+class Node:
+ def __init__(self,tag='',attrs=None):
+ self.tag=tag
+ self.attrs=dict(attrs or [])
+ self.children=[]
+
+ def text(self):
+ if self.tag in ('script','style'):
+ return ''
+ content=''.join(child.text() if isinstance(child,Node) else child for child in self.children)
+ return ' '+content+' ' if self.tag in ('p','div','section','li','br') else content
+
+ def walk(self):
+ yield self
+ for child in self.children:
+ if isinstance(child,Node):
+ yield from child.walk()
+
+
+class Document(HTMLParser):
+ VOID={'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'}
+
+ def __init__(self,html):
+ super().__init__(convert_charrefs=True)
+ self.root=Node()
+ self.stack=[self.root]
+ self.feed(html)
+
+ def handle_starttag(self,tag,attrs):
+ node=Node(tag,attrs)
+ self.stack[-1].children.append(node)
+ if tag not in self.VOID:
+ self.stack.append(node)
+
+ def handle_endtag(self,tag):
+ for index in range(len(self.stack)-1,0,-1):
+ if self.stack[index].tag==tag:
+ del self.stack[index:]
+ break
+
+ def handle_data(self,data):
+ self.stack[-1].children.append(data)
+
+
+def clean(text):
+ return ' '.join(text.split())
+
+
+def track_key(text):
+ text=text.lower()
+ if 'dataset' in text or 'benchmark' in text:
+ return 'datasets_benchmarks'
+ if 'creative' in text:
+ return 'creative_ai'
+ if 'position' in text:
+ return 'position'
+ return 'main'
+
+
+def paper_links(html,index_url):
+ doc=Document(html)
+ links=sorted({urljoin(index_url,n.attrs['href']) for n in doc.root.walk() if n.tag=='a' and '-Abstract' in n.attrs.get('href','') and urlsplit(urljoin(index_url,n.attrs['href'])).path.endswith('.html') and urlsplit(urljoin(index_url,n.attrs['href'])).netloc==urlsplit(index_url).netloc})
+ declared=next((match for n in doc.root.walk() if n.tag in ('p','span','div') and not any(child.tag=='a' for child in n.walk()) and (match:=re.fullmatch(r'([\d,]+)\s+papers',clean(n.text()),re.I))),None)
+ if not links:
+ raise ValueError('Official index contains no abstract-page links')
+ if declared and int(declared[1].replace(',',''))!=len(links):
+ raise ValueError('Declared paper count differs from unique abstract-page links')
+ return links
+
+
+def parse_paper(html,url,source,year,index_url):
+ nodes=list(Document(html).root.walk())
+ meta=defaultdict(list)
+ for node in nodes:
+ if node.tag=='meta':
+ meta[node.attrs.get('name','').lower()].append(node.attrs.get('content',''))
+ title=next(iter(meta['citation_title']),None)
+ if not title:
+ title=next((clean(n.text()) for n in nodes if n.tag=='h1'),None)
+ abstract=next((clean(n.text()) for n in nodes if 'paper-abstract' in n.attrs.get('class','').split()),None)
+ if not abstract:
+ for index,node in enumerate(nodes):
+ if node.tag in ('h2','h3','h4','h5') and clean(node.text()).lower()=='abstract':
+ following=[]
+ covered=set()
+ for candidate in nodes[index+1:]:
+ if candidate.tag in ('h1','h2','h3','h4','h5'):
+ break
+ if candidate.tag=='p' and candidate not in covered:
+ following.append(clean(candidate.text()))
+ covered.update(candidate.walk())
+ abstract=' '.join(following).strip() or None
+ break
+ if not title:
+ raise ValueError('Missing title on '+url)
+ # Never fall back to footer/policy text when the abstract is absent.
+ authors=[]
+ for author in meta['citation_author']:
+ if ',' in author:
+ family,given=author.split(',',1)
+ author=given.strip()+' '+family.strip()
+ authors.append(clean(author))
+ track_text=' '.join(clean(n.text()) for n in nodes if 'paper-track' in n.attrs.get('class','').split())
+ track=track_key(track_text or url)
+ identifiers={}
+ if meta['citation_doi']:
+ identifiers['doi']=meta['citation_doi'][0]
+ for node in nodes:
+ if node.tag!='a':
+ continue
+ link=urljoin(url,node.attrs.get('href',''))
+ parsed=urlsplit(link)
+ if parsed.netloc in ('arxiv.org','www.arxiv.org') and re.match(r'^/(abs|pdf)/',parsed.path):
+ identifiers['arxiv']=re.sub(r'^/(abs|pdf)/','',parsed.path).removesuffix('.pdf')
+ if parsed.netloc=='openreview.net' and parsed.path=='/forum':
+ from urllib.parse import parse_qs
+ forum=parse_qs(parsed.query).get('id')
+ if forum: identifiers['openreview']=forum[0]
+ venue='COLM' if source=='colm' else 'NeurIPS'
+ return {'source':source,'external_id':urlsplit(url).path,'url':url,'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,'venue':venue,'year':year,'track':track,'status':'accepted','raw_decision':'listed in official proceedings','list_url':index_url,'metadata':{'citation':dict(meta),'publication_state':'pre-proceedings' if 'pre-proceedings' in html.lower() else 'proceedings'}}
+
+
+def discover(source,cache):
+ root=ROOTS[source]
+ html=fetch(root,cache,8*1024**2,delay=1.0).decode()
+ result={}
+ for node in Document(html).root.walk():
+ if node.tag!='a':continue
+ url=urljoin(root,node.attrs.get('href',''))
+ parsed=urlsplit(url)
+ match=re.search(r'/paper(?:_files/paper)?/((?:19|20)\d{2})(?:/|$)',parsed.path)
+ if parsed.netloc==urlsplit(root).netloc and match and not any(term in parsed.path for term in ('/hash/','/file/','.pdf','-Abstract')):
+ result[url]=int(match[1])
+ if source=='neurips' and parsed.netloc=='datasets-benchmarks-proceedings.neurips.cc' and match:
+ result[url]=int(match[1])
+ if not result:
+ raise ValueError('No official year indexes found')
+ return sorted(result.items(),key=lambda entry:(entry[1],entry[0]))
+
+
+def collect_index(source,index_url,year,cache,out):
+ html=fetch(index_url,cache,16*1024**2,delay=1.0).decode()
+ links=paper_links(html,index_url)
+ grouped=defaultdict(list)
+ missing=[]
+ for number,url in enumerate(links,1):
+ record=parse_paper(fetch(url,cache,2*1024**2,delay=1.0).decode(),url,source,year,index_url)
+ if not record['abstract']:missing.append(url)
+ grouped[record['track']].append(record)
+ if number%25==0:
+ print(json.dumps({'source':source,'year':year,'collected':number,'expected':len(links)}),flush=True)
+ results=[]
+ for track,records in grouped.items():
+ # Distinct official index URLs may publish multiple volumes in one year.
+ import hashlib
+ suffix=hashlib.sha256(index_url.encode()).hexdigest()[:10]
+ path=Path(out)/f'{source}-{year}-{track}-{suffix}.jsonl'
+ path.parent.mkdir(parents=True,exist_ok=True)
+ part=path.with_suffix('.part')
+ with part.open('w') as output:
+ for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n')
+ part.replace(path)
+ missing_count=sum(not r['abstract'] for r in records)
+ result={'source':source,'venue':records[0]['venue'],'year':year,'track':track,'count':len(records),'index_link_count':len(links),'index_url':index_url,'list_url':index_url,'missing_abstract_count':missing_count,'file':str(path),'status':'collected' if not missing_count else 'partial'}
+ path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
+ results.append(result)
+ print(json.dumps(result),flush=True)
+ return results
+
+
+if __name__=='__main__':
+ parser=argparse.ArgumentParser()
+ parser.add_argument('--source',choices=ROOTS,required=True)
+ parser.add_argument('--cache',required=True)
+ parser.add_argument('--out',required=True)
+ parser.add_argument('--year',type=int,action='append')
+ args=parser.parse_args()
+ out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
+ status_path=out/f'{args.source}-index-status.json'
+ catalogue=json.loads(status_path.read_text()) if status_path.exists() else {}
+ indexes=discover(args.source,args.cache)
+ if args.year and set(args.year)-{year for _,year in indexes}:
+ parser.error('Requested year is absent from the current official proceedings index')
+ requested=[]
+ for url,year in indexes:
+ if args.year and year not in args.year:continue
+ requested.append(url)
+ try:
+ files=collect_index(args.source,url,year,args.cache,args.out)
+ catalogue[url]={'status':'partial' if any(f['status']=='partial' for f in files) else 'collected','files':files}
+ except Exception as exc:
+ catalogue[url]={'status':'blocked','error':str(exc),'year':year}
+ print(json.dumps(catalogue[url]),flush=True)
+ part=status_path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_path)
+ (out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year},indent=2))
+ if any(catalogue[url]['status']=='blocked' for url in requested):raise SystemExit(1)
diff --git a/import_records.py b/import_records.py
index 02ed87b..2b45792 100644
--- a/import_records.py
+++ b/import_records.py
@@ -47,7 +47,8 @@ def ingest_file(db,path):
expected = manifest.get('count')
if expected is not None and expected != count:
raise ValueError(f'Manifest count {expected} differs from records {count}')
- c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if expected==count else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
+ complete=expected==count and manifest.get('status') not in ('partial','blocked') and not manifest.get('missing_abstract_count',0)
+ c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if complete else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
c.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=? WHERE id=?",(count,count,run))
c.commit()
print(json.dumps({'file':str(path),'status':'imported','records':count}),flush=True)
diff --git a/merge_conferences.py b/merge_conferences.py
index c869b14..d971af9 100644
--- a/merge_conferences.py
+++ b/merge_conferences.py
@@ -4,6 +4,7 @@ Run as a transient systemd job so an SSH disconnect cannot strand the importer.
Only the Paper Atlas arXiv unit is controlled; business services are untouched.
"""
import argparse
+from contextlib import closing
import json
import sqlite3
import subprocess
@@ -34,14 +35,14 @@ def merge(db,archive,files):
raise RuntimeError('Expected the existing arXiv unit to be active; inspect it before proceeding')
subprocess.run(['systemctl','stop',UNIT],check=True)
try:
- with sqlite3.connect(db) as c:
+ with closing(sqlite3.connect(db)) as c, c:
checkpoint=c.execute("SELECT records_written FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone()
print('Saved arxiv checkpoint:',checkpoint,flush=True)
schema_store=Store(db)
schema_store.conn.close()
for filename in files:
ingest_file(db,filename)
- with sqlite3.connect(db) as c:
+ with closing(sqlite3.connect(db)) as c, c:
if files:
errors=c.execute('PRAGMA foreign_key_check').fetchall()
if errors:
diff --git a/test_import_arxiv.py b/test_import_arxiv.py
index 4c2c6e8..d973cd4 100644
--- a/test_import_arxiv.py
+++ b/test_import_arxiv.py
@@ -1,4 +1,5 @@
import json
+from contextlib import closing
import sqlite3
import tempfile
import unittest
@@ -27,7 +28,7 @@ class ImportTests(unittest.TestCase):
self.write([self.row('2601.00001','10.0000/example'),self.row('2601.00002','10.0000/example')])
ingest(self.db,self.archive)
ingest(self.db,self.archive)
- with sqlite3.connect(self.db) as c:
+ with closing(sqlite3.connect(self.db)) as c, c:
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2)
self.assertEqual(c.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'Searchable'").fetchone()[0],1)
@@ -38,18 +39,18 @@ class ImportTests(unittest.TestCase):
def test_title_alone_does_not_merge(self):
self.write([self.row('2601.00001'),self.row('2601.00002')])
ingest(self.db,self.archive)
- with sqlite3.connect(self.db) as c:
+ with closing(sqlite3.connect(self.db)) as c, c:
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2)
def test_platform_and_custom_tags_remain_distinct(self):
self.write([self.row('2601.00001')])
ingest(self.db,self.archive)
- with sqlite3.connect(self.db) as c:
+ with closing(sqlite3.connect(self.db)) as c, c:
paper = c.execute('SELECT id FROM papers').fetchone()[0]
tag = c.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','cs.AI')").lastrowid
c.execute('INSERT INTO paper_tags VALUES(?,?,NULL,?)',(paper,tag,'manual:1'))
ingest(self.db,self.archive)
- with sqlite3.connect(self.db) as c:
+ with closing(sqlite3.connect(self.db)) as c, c:
self.assertEqual(c.execute("SELECT count(*) FROM paper_tags pt JOIN tags t ON pt.tag_id=t.id WHERE t.value='cs.AI'").fetchone()[0],2)
diff --git a/test_index_core.py b/test_index_core.py
index 91a5979..29cb122 100644
--- a/test_index_core.py
+++ b/test_index_core.py
@@ -58,6 +58,13 @@ class StoreTests(unittest.TestCase):
self.assertEqual(self.store.conn.execute('SELECT title FROM papers').fetchone()[0],'Updated research title')
self.assertEqual(self.store.conn.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'searchable'").fetchone()[0],1)
+ def test_missing_abstract_does_not_become_complete_coverage(self):
+ path=Path(self.tmp.name)/'partial.jsonl'
+ path.write_text(json.dumps({'source':'colm','external_id':'paper1','url':'https://example.org/paper1','title':'Paper','abstract':None,'authors':[],'venue':'COLM','year':2024,'track':'main','status':'accepted'})+'\n')
+ path.with_suffix('.manifest.json').write_text(json.dumps({'source':'colm','list_url':'https://example.org/list','count':1,'status':'partial','missing_abstract_count':1}))
+ ingest_file(self.db,path)
+ self.assertEqual(self.store.conn.execute('SELECT status FROM edition_sources').fetchone()[0],'partial')
+
if __name__=='__main__':
unittest.main()
diff --git a/test_pmlr.py b/test_pmlr.py
index 35122f5..5713a93 100644
--- a/test_pmlr.py
+++ b/test_pmlr.py
@@ -1,6 +1,8 @@
import unittest
+import tempfile
+from unittest.mock import patch
-from collect_pmlr import load_metadata,venue_key
+from collect_pmlr import load_metadata,venue_key,collect_volume
class MetadataTests(unittest.TestCase):
@@ -20,6 +22,15 @@ class MetadataTests(unittest.TestCase):
self.assertEqual(venue_key('COLT 2012'),'COLT')
self.assertEqual(venue_key('ECML-PKDD'),'ECML-PKDD')
+ def test_missing_abstract_is_reported_as_partial_collection(self):
+ config=b'shortname: ICML\nyear: 2025\nbooktitle: ICML\n'
+ paper='---\nid: paper1\nlayout: inproceedings\ntitle: Paper\nabstract:\n---\n'
+ with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.fetch',return_value=config),patch('collect_pmlr.metadata_texts',return_value=iter([('paper.md',paper)])):
+ result=collect_volume(999,tmp,tmp)
+ self.assertEqual(result['count'],1)
+ self.assertEqual(result['missing_abstract_count'],1)
+ self.assertEqual(result['status'],'partial')
+
if __name__=='__main__':
unittest.main()
diff --git a/test_proceedings.py b/test_proceedings.py
new file mode 100644
index 0000000..44c1153
--- /dev/null
+++ b/test_proceedings.py
@@ -0,0 +1,56 @@
+import unittest
+import json
+import runpy
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+from collect_proceedings import paper_links,parse_paper
+
+
+class ProceedingsTests(unittest.TestCase):
+ def test_colm_nested_abstract_and_citation_authors(self):
+ html='''Abstract
Real abstract text.
Name Change Policy
Do not ingest footer.
'''
+ row=parse_paper(html,'https://proceedings.colm.cc/paper_files/paper/2024/hash/a-Abstract-Conference.html','colm',2024,'https://proceedings.colm.cc/paper_files/paper/2024')
+ self.assertEqual(row['abstract'],'Real abstract text.')
+ self.assertEqual(row['authors'],['Jane Doe'])
+ self.assertEqual(row['track'],'main')
+ self.assertEqual(row['year'],2024)
+
+ def test_declared_index_count_detects_truncation(self):
+ html='2 papers
Paper'
+ with self.assertRaisesRegex(ValueError,'count differs'):
+ paper_links(html,'https://proceedings.colm.cc/paper/2024')
+
+ def test_missing_abstract_does_not_capture_footer(self):
+ row=parse_paper('Paper
Name Change Policy
Footer
','https://example.org/a','colm',2024,'https://example.org/')
+ self.assertIsNone(row['abstract'])
+
+ def test_paper_title_is_not_mistaken_for_index_paper_count(self):
+ html='Lessons from 1000 papers'
+ self.assertEqual(len(paper_links(html,'https://datasets-benchmarks-proceedings.neurips.cc/paper/2021')),1)
+
+ def test_neurips_track_and_legacy_abstract(self):
+ html='Paper
Datasets and Benchmarks TrackAbstract
Actual abstract.
Name Change Policy
Footer
'
+ row=parse_paper(html,'https://proceedings.neurips.cc/a','neurips',2025,'https://proceedings.neurips.cc/')
+ self.assertEqual(row['abstract'],'Actual abstract.')
+ self.assertEqual(row['track'],'datasets_benchmarks')
+
+ def test_focused_retry_preserves_other_index_status(self):
+ root='https://proceedings.colm.cc/'
+ url=root+'paper_files/paper/2024'
+ other=root+'paper_files/paper/2025'
+ paper=root+'paper_files/paper/2024/hash/a-Abstract-Conference.html'
+ documents={root:f'20242025',url:f'1 papers
Paper',paper:'Paper
Abstract
'}
+ with tempfile.TemporaryDirectory() as tmp:
+ status=Path(tmp)/'colm-index-status.json'
+ status.write_text(json.dumps({other:{'status':'collected','files':[]}}))
+ argv=['collect_proceedings.py','--source','colm','--cache',tmp,'--out',tmp,'--year','2024']
+ with patch('sys.argv',argv),patch('collect_pmlr.fetch',side_effect=lambda u,*a,**kw:documents[u].encode()):
+ runpy.run_path(str(Path(__file__).with_name('collect_proceedings.py')),run_name='__main__')
+ result=json.loads(status.read_text())
+ self.assertEqual(result[other]['status'],'collected')
+ self.assertEqual(result[url]['files'][0]['count'],1)
+
+
+if __name__=='__main__':unittest.main()