diff --git a/README.md b/README.md index 3c44c31..5b35682 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 目标包括:完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。 -**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集,其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。 +**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集,其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。 ## 数据范围与来源 @@ -123,6 +123,23 @@ python3 -B collect_peerread.py --cache /data/paper-index/raw/http-cache \ 数据任务以一次性 systemd 服务运行:`paper-atlas-arxiv-import`、`paper-atlas-pmlr-collect`、`paper-atlas-peerread-collect`;`paper-atlas-pmlr-verify` 用于全索引复核,`paper-atlas-conference-merge` 用于单写入合并。它们不是定时任务。主库只允许一个写入进程;会议采集先写独立 JSONL,待 arXiv 主批次结束后再合并;也可以使用 `merge_conferences.py` 在保存断点处暂停本项目的 arXiv unit、顺序合并会议清单并恢复。该操作应作为独立的一次性 systemd 任务运行,避免 SSH 断开影响恢复;不能同时启动多个合并任务。运行日志位于 `/data/paper-index/*-import.log`、`/data/paper-index/*-collect.log`。任何服务启动成功都不等于批次已完成,需检查批次状态、清单与完整性结果。标签按来源刷新使用 `paper_tags_source_evidence` 索引,避免每条论文都扫描整张标签表。运行中的 arXiv 可以通过同一协调脚本的 `--schema-only` 模式在断点更新索引后恢复;此模式要求已有 arXiv unit 正在运行。 +### COLM 与 NeurIPS 官方摘要 + +```sh +python3 -B collect_proceedings.py --source colm \ + --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected +python3 -B collect_proceedings.py --source neurips \ + --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected +``` + +采集器从官方根目录发现实际发表届次,仅访问索引和 Abstract HTML 页面,不请求 PDF、补充材料或源码。可用重复 `--year` 限定实际存在的届次;不存在的年份会报错,局部重试保留其他届次的已有状态。每个来源串行请求,实际网络响应后等待 1 秒;缓存命中不额外等待。 + +COLM 当前目录提供 2024–2026 三届 pre-proceedings,按会议年份保存,保留平台实际出版日期和出版状态,不把晚补刊日期当作会议年份。NeurIPS 当前目录提供 1987–2025,并包含 2025 独立卷及 2021 独立 Datasets and Benchmarks 索引。主会、datasets_benchmarks、position、creative_ai 分开出文件,不混为同一个 track。首次真实样本核对:COLM 2024 官方目录 299 个摘要入口;NeurIPS 1987 为 90 个,作者与摘要均已验证解析。样本成功不代表所有页面采集完成。 + +`colm-index-status.json`、`neurips-index-status.json` 保存逐索引状态;对应 `*-last-run.json` 保存本次实际选择范围。每个 JSONL 的 manifest 记录索引入口数、track 记录数和缺失摘要数。论文数量与官方页面声明不一致会失败,缺失摘要会标记 partial;不会把页脚、姓名政策或其他正文当作摘要。 + +这两个采集器只写采集文件,合并仍通过单写入协调流程。全量历史覆盖和所有未实现会议仍待最终核验。 + ## 长期维护与更新 1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发,遵守来源限流、重试与人机验证要求。 diff --git a/collect_pmlr.py b/collect_pmlr.py index f43cfe3..2ba97c7 100644 --- a/collect_pmlr.py +++ b/collect_pmlr.py @@ -38,7 +38,7 @@ def venue_key(shortname): return re.sub(r'[\s_-]*(?:19|20)\d{2}$','',str(shortname).strip().upper()) -def fetch(url,cache,limit=64*1024**2): +def fetch(url,cache,limit=64*1024**2,delay=0.5): cache = Path(cache) cache.mkdir(parents=True,exist_ok=True) path = cache/hashlib.sha256(url.encode()).hexdigest() @@ -59,7 +59,7 @@ def fetch(url,cache,limit=64*1024**2): part.write_bytes(data) part.replace(path) path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())})) - time.sleep(0.5) + time.sleep(delay) return data except urllib.error.HTTPError as exc: if exc.code not in (429,500,502,503,504) or attempt==2: @@ -128,14 +128,19 @@ def collect_volume(volume,cache,out,allowed=None): path.parent.mkdir(parents=True,exist_ok=True) part = path.with_suffix('.part') count = 0 + missing_abstracts = 0 with part.open('w') as output: for member_name,content in metadata_texts(volume,cache): parts = re.split(r'(?m)^---\s*$',content,maxsplit=2) if len(parts)<3 or parts[0].strip(): raise RuntimeError('Missing YAML frontmatter: '+member_name) row = load_metadata(parts[1]) - if row.get('layout') not in ('inproceedings','article') or not row.get('title'): + if row.get('layout') not in ('inproceedings','article'): continue + if not row.get('title'): + raise ValueError('Paper metadata has no title: '+member_name) + if not (row.get('abstract') or '').strip(): + missing_abstracts+=1 external = f"v{volume}/{row.get('id') or Path(member_name).stem}" ids = {} for key in ('doi','openreview','arxiv'): @@ -154,7 +159,7 @@ def collect_volume(volume,cache,out,allowed=None): part.unlink(missing_ok=True) raise RuntimeError('No papers parsed; refusing an empty successful collection') part.replace(path) - result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'} + result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'missing_abstract_count':missing_abstracts,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'partial' if missing_abstracts else 'collected'} path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2)) print(json.dumps(result),flush=True) return result diff --git a/collect_proceedings.py b/collect_proceedings.py new file mode 100644 index 0000000..19917c4 --- /dev/null +++ b/collect_proceedings.py @@ -0,0 +1,211 @@ +"""Collect COLM/NeurIPS official abstract pages without downloading PDFs.""" +import argparse +import json +import re +import time +from collections import defaultdict +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import urljoin,urlsplit + +from collect_pmlr import fetch + +ROOTS={'colm':'https://proceedings.colm.cc/','neurips':'https://proceedings.neurips.cc/'} + + +class Node: + def __init__(self,tag='',attrs=None): + self.tag=tag + self.attrs=dict(attrs or []) + self.children=[] + + def text(self): + if self.tag in ('script','style'): + return '' + content=''.join(child.text() if isinstance(child,Node) else child for child in self.children) + return ' '+content+' ' if self.tag in ('p','div','section','li','br') else content + + def walk(self): + yield self + for child in self.children: + if isinstance(child,Node): + yield from child.walk() + + +class Document(HTMLParser): + VOID={'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'} + + def __init__(self,html): + super().__init__(convert_charrefs=True) + self.root=Node() + self.stack=[self.root] + self.feed(html) + + def handle_starttag(self,tag,attrs): + node=Node(tag,attrs) + self.stack[-1].children.append(node) + if tag not in self.VOID: + self.stack.append(node) + + def handle_endtag(self,tag): + for index in range(len(self.stack)-1,0,-1): + if self.stack[index].tag==tag: + del self.stack[index:] + break + + def handle_data(self,data): + self.stack[-1].children.append(data) + + +def clean(text): + return ' '.join(text.split()) + + +def track_key(text): + text=text.lower() + if 'dataset' in text or 'benchmark' in text: + return 'datasets_benchmarks' + if 'creative' in text: + return 'creative_ai' + if 'position' in text: + return 'position' + return 'main' + + +def paper_links(html,index_url): + doc=Document(html) + links=sorted({urljoin(index_url,n.attrs['href']) for n in doc.root.walk() if n.tag=='a' and '-Abstract' in n.attrs.get('href','') and urlsplit(urljoin(index_url,n.attrs['href'])).path.endswith('.html') and urlsplit(urljoin(index_url,n.attrs['href'])).netloc==urlsplit(index_url).netloc}) + declared=next((match for n in doc.root.walk() if n.tag in ('p','span','div') and not any(child.tag=='a' for child in n.walk()) and (match:=re.fullmatch(r'([\d,]+)\s+papers',clean(n.text()),re.I))),None) + if not links: + raise ValueError('Official index contains no abstract-page links') + if declared and int(declared[1].replace(',',''))!=len(links): + raise ValueError('Declared paper count differs from unique abstract-page links') + return links + + +def parse_paper(html,url,source,year,index_url): + nodes=list(Document(html).root.walk()) + meta=defaultdict(list) + for node in nodes: + if node.tag=='meta': + meta[node.attrs.get('name','').lower()].append(node.attrs.get('content','')) + title=next(iter(meta['citation_title']),None) + if not title: + title=next((clean(n.text()) for n in nodes if n.tag=='h1'),None) + abstract=next((clean(n.text()) for n in nodes if 'paper-abstract' in n.attrs.get('class','').split()),None) + if not abstract: + for index,node in enumerate(nodes): + if node.tag in ('h2','h3','h4','h5') and clean(node.text()).lower()=='abstract': + following=[] + covered=set() + for candidate in nodes[index+1:]: + if candidate.tag in ('h1','h2','h3','h4','h5'): + break + if candidate.tag=='p' and candidate not in covered: + following.append(clean(candidate.text())) + covered.update(candidate.walk()) + abstract=' '.join(following).strip() or None + break + if not title: + raise ValueError('Missing title on '+url) + # Never fall back to footer/policy text when the abstract is absent. + authors=[] + for author in meta['citation_author']: + if ',' in author: + family,given=author.split(',',1) + author=given.strip()+' '+family.strip() + authors.append(clean(author)) + track_text=' '.join(clean(n.text()) for n in nodes if 'paper-track' in n.attrs.get('class','').split()) + track=track_key(track_text or url) + identifiers={} + if meta['citation_doi']: + identifiers['doi']=meta['citation_doi'][0] + for node in nodes: + if node.tag!='a': + continue + link=urljoin(url,node.attrs.get('href','')) + parsed=urlsplit(link) + if parsed.netloc in ('arxiv.org','www.arxiv.org') and re.match(r'^/(abs|pdf)/',parsed.path): + identifiers['arxiv']=re.sub(r'^/(abs|pdf)/','',parsed.path).removesuffix('.pdf') + if parsed.netloc=='openreview.net' and parsed.path=='/forum': + from urllib.parse import parse_qs + forum=parse_qs(parsed.query).get('id') + if forum: identifiers['openreview']=forum[0] + venue='COLM' if source=='colm' else 'NeurIPS' + return {'source':source,'external_id':urlsplit(url).path,'url':url,'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,'venue':venue,'year':year,'track':track,'status':'accepted','raw_decision':'listed in official proceedings','list_url':index_url,'metadata':{'citation':dict(meta),'publication_state':'pre-proceedings' if 'pre-proceedings' in html.lower() else 'proceedings'}} + + +def discover(source,cache): + root=ROOTS[source] + html=fetch(root,cache,8*1024**2,delay=1.0).decode() + result={} + for node in Document(html).root.walk(): + if node.tag!='a':continue + url=urljoin(root,node.attrs.get('href','')) + parsed=urlsplit(url) + match=re.search(r'/paper(?:_files/paper)?/((?:19|20)\d{2})(?:/|$)',parsed.path) + if parsed.netloc==urlsplit(root).netloc and match and not any(term in parsed.path for term in ('/hash/','/file/','.pdf','-Abstract')): + result[url]=int(match[1]) + if source=='neurips' and parsed.netloc=='datasets-benchmarks-proceedings.neurips.cc' and match: + result[url]=int(match[1]) + if not result: + raise ValueError('No official year indexes found') + return sorted(result.items(),key=lambda entry:(entry[1],entry[0])) + + +def collect_index(source,index_url,year,cache,out): + html=fetch(index_url,cache,16*1024**2,delay=1.0).decode() + links=paper_links(html,index_url) + grouped=defaultdict(list) + missing=[] + for number,url in enumerate(links,1): + record=parse_paper(fetch(url,cache,2*1024**2,delay=1.0).decode(),url,source,year,index_url) + if not record['abstract']:missing.append(url) + grouped[record['track']].append(record) + if number%25==0: + print(json.dumps({'source':source,'year':year,'collected':number,'expected':len(links)}),flush=True) + results=[] + for track,records in grouped.items(): + # Distinct official index URLs may publish multiple volumes in one year. + import hashlib + suffix=hashlib.sha256(index_url.encode()).hexdigest()[:10] + path=Path(out)/f'{source}-{year}-{track}-{suffix}.jsonl' + path.parent.mkdir(parents=True,exist_ok=True) + part=path.with_suffix('.part') + with part.open('w') as output: + for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n') + part.replace(path) + missing_count=sum(not r['abstract'] for r in records) + result={'source':source,'venue':records[0]['venue'],'year':year,'track':track,'count':len(records),'index_link_count':len(links),'index_url':index_url,'list_url':index_url,'missing_abstract_count':missing_count,'file':str(path),'status':'collected' if not missing_count else 'partial'} + path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2)) + results.append(result) + print(json.dumps(result),flush=True) + return results + + +if __name__=='__main__': + parser=argparse.ArgumentParser() + parser.add_argument('--source',choices=ROOTS,required=True) + parser.add_argument('--cache',required=True) + parser.add_argument('--out',required=True) + parser.add_argument('--year',type=int,action='append') + args=parser.parse_args() + out=Path(args.out);out.mkdir(parents=True,exist_ok=True) + status_path=out/f'{args.source}-index-status.json' + catalogue=json.loads(status_path.read_text()) if status_path.exists() else {} + indexes=discover(args.source,args.cache) + if args.year and set(args.year)-{year for _,year in indexes}: + parser.error('Requested year is absent from the current official proceedings index') + requested=[] + for url,year in indexes: + if args.year and year not in args.year:continue + requested.append(url) + try: + files=collect_index(args.source,url,year,args.cache,args.out) + catalogue[url]={'status':'partial' if any(f['status']=='partial' for f in files) else 'collected','files':files} + except Exception as exc: + catalogue[url]={'status':'blocked','error':str(exc),'year':year} + print(json.dumps(catalogue[url]),flush=True) + part=status_path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_path) + (out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year},indent=2)) + if any(catalogue[url]['status']=='blocked' for url in requested):raise SystemExit(1) diff --git a/import_records.py b/import_records.py index 02ed87b..2b45792 100644 --- a/import_records.py +++ b/import_records.py @@ -47,7 +47,8 @@ def ingest_file(db,path): expected = manifest.get('count') if expected is not None and expected != count: raise ValueError(f'Manifest count {expected} differs from records {count}') - c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if expected==count else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()))) + complete=expected==count and manifest.get('status') not in ('partial','blocked') and not manifest.get('missing_abstract_count',0) + c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if complete else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()))) c.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=? WHERE id=?",(count,count,run)) c.commit() print(json.dumps({'file':str(path),'status':'imported','records':count}),flush=True) diff --git a/merge_conferences.py b/merge_conferences.py index c869b14..d971af9 100644 --- a/merge_conferences.py +++ b/merge_conferences.py @@ -4,6 +4,7 @@ Run as a transient systemd job so an SSH disconnect cannot strand the importer. Only the Paper Atlas arXiv unit is controlled; business services are untouched. """ import argparse +from contextlib import closing import json import sqlite3 import subprocess @@ -34,14 +35,14 @@ def merge(db,archive,files): raise RuntimeError('Expected the existing arXiv unit to be active; inspect it before proceeding') subprocess.run(['systemctl','stop',UNIT],check=True) try: - with sqlite3.connect(db) as c: + with closing(sqlite3.connect(db)) as c, c: checkpoint=c.execute("SELECT records_written FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone() print('Saved arxiv checkpoint:',checkpoint,flush=True) schema_store=Store(db) schema_store.conn.close() for filename in files: ingest_file(db,filename) - with sqlite3.connect(db) as c: + with closing(sqlite3.connect(db)) as c, c: if files: errors=c.execute('PRAGMA foreign_key_check').fetchall() if errors: diff --git a/test_import_arxiv.py b/test_import_arxiv.py index 4c2c6e8..d973cd4 100644 --- a/test_import_arxiv.py +++ b/test_import_arxiv.py @@ -1,4 +1,5 @@ import json +from contextlib import closing import sqlite3 import tempfile import unittest @@ -27,7 +28,7 @@ class ImportTests(unittest.TestCase): self.write([self.row('2601.00001','10.0000/example'),self.row('2601.00002','10.0000/example')]) ingest(self.db,self.archive) ingest(self.db,self.archive) - with sqlite3.connect(self.db) as c: + with closing(sqlite3.connect(self.db)) as c, c: self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1) self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2) self.assertEqual(c.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'Searchable'").fetchone()[0],1) @@ -38,18 +39,18 @@ class ImportTests(unittest.TestCase): def test_title_alone_does_not_merge(self): self.write([self.row('2601.00001'),self.row('2601.00002')]) ingest(self.db,self.archive) - with sqlite3.connect(self.db) as c: + with closing(sqlite3.connect(self.db)) as c, c: self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2) def test_platform_and_custom_tags_remain_distinct(self): self.write([self.row('2601.00001')]) ingest(self.db,self.archive) - with sqlite3.connect(self.db) as c: + with closing(sqlite3.connect(self.db)) as c, c: paper = c.execute('SELECT id FROM papers').fetchone()[0] tag = c.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','cs.AI')").lastrowid c.execute('INSERT INTO paper_tags VALUES(?,?,NULL,?)',(paper,tag,'manual:1')) ingest(self.db,self.archive) - with sqlite3.connect(self.db) as c: + with closing(sqlite3.connect(self.db)) as c, c: self.assertEqual(c.execute("SELECT count(*) FROM paper_tags pt JOIN tags t ON pt.tag_id=t.id WHERE t.value='cs.AI'").fetchone()[0],2) diff --git a/test_index_core.py b/test_index_core.py index 91a5979..29cb122 100644 --- a/test_index_core.py +++ b/test_index_core.py @@ -58,6 +58,13 @@ class StoreTests(unittest.TestCase): self.assertEqual(self.store.conn.execute('SELECT title FROM papers').fetchone()[0],'Updated research title') self.assertEqual(self.store.conn.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'searchable'").fetchone()[0],1) + def test_missing_abstract_does_not_become_complete_coverage(self): + path=Path(self.tmp.name)/'partial.jsonl' + path.write_text(json.dumps({'source':'colm','external_id':'paper1','url':'https://example.org/paper1','title':'Paper','abstract':None,'authors':[],'venue':'COLM','year':2024,'track':'main','status':'accepted'})+'\n') + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'colm','list_url':'https://example.org/list','count':1,'status':'partial','missing_abstract_count':1})) + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT status FROM edition_sources').fetchone()[0],'partial') + if __name__=='__main__': unittest.main() diff --git a/test_pmlr.py b/test_pmlr.py index 35122f5..5713a93 100644 --- a/test_pmlr.py +++ b/test_pmlr.py @@ -1,6 +1,8 @@ import unittest +import tempfile +from unittest.mock import patch -from collect_pmlr import load_metadata,venue_key +from collect_pmlr import load_metadata,venue_key,collect_volume class MetadataTests(unittest.TestCase): @@ -20,6 +22,15 @@ class MetadataTests(unittest.TestCase): self.assertEqual(venue_key('COLT 2012'),'COLT') self.assertEqual(venue_key('ECML-PKDD'),'ECML-PKDD') + def test_missing_abstract_is_reported_as_partial_collection(self): + config=b'shortname: ICML\nyear: 2025\nbooktitle: ICML\n' + paper='---\nid: paper1\nlayout: inproceedings\ntitle: Paper\nabstract:\n---\n' + with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.fetch',return_value=config),patch('collect_pmlr.metadata_texts',return_value=iter([('paper.md',paper)])): + result=collect_volume(999,tmp,tmp) + self.assertEqual(result['count'],1) + self.assertEqual(result['missing_abstract_count'],1) + self.assertEqual(result['status'],'partial') + if __name__=='__main__': unittest.main() diff --git a/test_proceedings.py b/test_proceedings.py new file mode 100644 index 0000000..44c1153 --- /dev/null +++ b/test_proceedings.py @@ -0,0 +1,56 @@ +import unittest +import json +import runpy +import tempfile +from pathlib import Path +from unittest.mock import patch + +from collect_proceedings import paper_links,parse_paper + + +class ProceedingsTests(unittest.TestCase): + def test_colm_nested_abstract_and_citation_authors(self): + html='''

Abstract

Real abstract text.

Name Change Policy

Do not ingest footer.

''' + row=parse_paper(html,'https://proceedings.colm.cc/paper_files/paper/2024/hash/a-Abstract-Conference.html','colm',2024,'https://proceedings.colm.cc/paper_files/paper/2024') + self.assertEqual(row['abstract'],'Real abstract text.') + self.assertEqual(row['authors'],['Jane Doe']) + self.assertEqual(row['track'],'main') + self.assertEqual(row['year'],2024) + + def test_declared_index_count_detects_truncation(self): + html='

2 papers

Paper' + with self.assertRaisesRegex(ValueError,'count differs'): + paper_links(html,'https://proceedings.colm.cc/paper/2024') + + def test_missing_abstract_does_not_capture_footer(self): + row=parse_paper('

Paper

Name Change Policy

Footer

','https://example.org/a','colm',2024,'https://example.org/') + self.assertIsNone(row['abstract']) + + def test_paper_title_is_not_mistaken_for_index_paper_count(self): + html='
  • Lessons from 1000 papers
  • ' + self.assertEqual(len(paper_links(html,'https://datasets-benchmarks-proceedings.neurips.cc/paper/2021')),1) + + def test_neurips_track_and_legacy_abstract(self): + html='

    Paper

    Datasets and Benchmarks Track

    Abstract

    Actual abstract.

    Name Change Policy

    Footer

    ' + row=parse_paper(html,'https://proceedings.neurips.cc/a','neurips',2025,'https://proceedings.neurips.cc/') + self.assertEqual(row['abstract'],'Actual abstract.') + self.assertEqual(row['track'],'datasets_benchmarks') + + def test_focused_retry_preserves_other_index_status(self): + root='https://proceedings.colm.cc/' + url=root+'paper_files/paper/2024' + other=root+'paper_files/paper/2025' + paper=root+'paper_files/paper/2024/hash/a-Abstract-Conference.html' + documents={root:f'20242025',url:f'

    1 papers

    Paper',paper:'

    Paper

    Abstract

    '} + with tempfile.TemporaryDirectory() as tmp: + status=Path(tmp)/'colm-index-status.json' + status.write_text(json.dumps({other:{'status':'collected','files':[]}})) + argv=['collect_proceedings.py','--source','colm','--cache',tmp,'--out',tmp,'--year','2024'] + with patch('sys.argv',argv),patch('collect_pmlr.fetch',side_effect=lambda u,*a,**kw:documents[u].encode()): + runpy.run_path(str(Path(__file__).with_name('collect_proceedings.py')),run_name='__main__') + result=json.loads(status.read_text()) + self.assertEqual(result[other]['status'],'collected') + self.assertEqual(result[url]['files'][0]['count'],1) + + +if __name__=='__main__':unittest.main()