From 4a4272219e77df7d2a735386e2348641cf7d1a29 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Fri, 6 Nov 2026 03:12:44 +0800 Subject: [PATCH] Collect explicit public ICLR decisions and refresh metadata indexes --- README.md | 31 ++++++++++++++++++-- collect_peerread.py | 71 +++++++++++++++++++++++++++++++++++++++++++++ collect_pmlr.py | 4 +-- import_records.py | 4 +-- index_core.py | 13 ++++++--- test_index_core.py | 7 +++++ test_peerread.py | 33 +++++++++++++++++++++ 7 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 collect_peerread.py create mode 100644 test_peerread.py diff --git a/README.md b/README.md index 162d6b2..e61530e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 目标包括:完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。 -**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上六项合成回归检查通过;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 全量快照下载与历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。 +**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集,其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。 ## 数据范围与来源 @@ -30,6 +30,22 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 - [AAAI](https://ojs.aaai.org/index.php/AAAI/issue/archive) - [DBLP](https://dblp.org/db/conf/) +## 候选会议覆盖目录 + +| 领域 | 会议系列 | 主要发现入口 | +| --- | --- | --- | +| 机器学习、理论 | NeurIPS、ICML、ICLR、COLM、COLT、AISTATS、UAI、CLeaR | OpenReview、PMLR、NeurIPS、COLM 官方论文集 | +| 自然语言 | ACL、EMNLP、NAACL、EACL、COLING、CoNLL | ACL Anthology | +| 视觉 | CVPR、ICCV、ECCV | CVF、ECVA | +| 综合 AI | AAAI、IJCAI、ECAI | 官方论文集、DBLP | +| 检索与数据 | KDD、SIGIR、WWW、WSDM、RecSys、CIKM、ICDM、SDM、ECML-PKDD | 官方出版方、DBLP,具体年份入口待逐项登记 | +| 机器人 | ICRA、IROS、RSS、CoRL | 官方论文集、PMLR、DBLP | +| 规划、知识与智能体 | ICAPS、KR、AAMAS | 官方论文集、DBLP | +| 语音与音频 | ICASSP、INTERSPEECH、ISMIR | 官方论文集、ISCA、DBLP | +| 相关专门领域 | MICCAI、FAccT | 官方出版方、DBLP | + +这些是领域覆盖候选,不构成统一排名,也不能据此声称已全部入库。逐年目录以真实举办届次和正式出版清单为准:双年会议不补造不存在的年份;主会、Findings、workshop 分开。`edition_sources` 保存实际采集状态,不能用候选名单代替覆盖率报告。 + ## 数据模型 | 表 | 用途 | @@ -52,7 +68,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 基础导入和查询依赖 Python 3 标准库和带 FTS5 的 SQLite;PMLR 采集另需 `requirements.txt` 中锁定的 PyYAML 6.0.3。无项目构建步骤。服务器已检查为 Python 3.14.4。所有运行数据放在仓库外,不提交数据库、下载包或访问凭据。 -服务器暂存目录:`/data/paper-index/`,快照目录:`/data/paper-index/raw/`,运行代码目录:`/data/paper-index/code/`。这些路径不是对外访问的 API。 +服务器暂存目录:`/data/paper-index/`,快照目录:`/data/paper-index/raw/`,运行代码目录:`/data/paper-index/code/`。这些路径不是对外访问的 API。本地工作区只用于编辑代码、文档和 Git;采集、全量导入和运行检查均在用户确认的服务器执行。 下载完成并验证 ZIP 后,在服务器运行: @@ -94,7 +110,16 @@ python3 -B import_records.py --db /data/paper-index/papers.sqlite \ /data/paper-index/collected/pmlr-v267.jsonl ``` -采集输出 JSONL 和对应 manifest,只有显式官方录用记录才产生 accepted 决定。清单数目不一致时整个文件导入回滚,记录失败批次,不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。 +采集输出 JSONL 和对应 manifest,只有显式官方录用记录才产生 accepted 决定。清单数目不一致时整个文件导入回滚,记录失败批次,不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。缓存默认有效 24 小时;同一次批次可复用缓存,后续更新会重新读取动态入口。 + +PeerRead 采集仅选择作者公开发布的 ICLR 2017 投稿元数据,固定到源仓库 commit。只依据明确的布尔 `accepted` 标签形成 accepted/rejected;空标签为 unknown,不使用其他数据分区中的 probably-rejected 推断。不会保存 reviews 或 parsed_pdfs 正文。 + +```sh +python3 -B collect_peerread.py --cache /data/paper-index/raw/http-cache \ + --out /data/paper-index/collected +``` + +数据任务以一次性 systemd 服务运行:`paper-atlas-arxiv-import`、`paper-atlas-pmlr-collect`、`paper-atlas-peerread-collect`。它们不是定时任务。主库只允许一个写入进程;会议采集先写独立 JSONL,待 arXiv 主批次结束后再合并。运行日志位于 `/data/paper-index/*-import.log`、`/data/paper-index/*-collect.log`。任何服务启动成功都不等于批次已完成,需检查批次状态、清单与完整性结果。 ## 长期维护与更新 diff --git a/collect_peerread.py b/collect_peerread.py new file mode 100644 index 0000000..ca18641 --- /dev/null +++ b/collect_peerread.py @@ -0,0 +1,71 @@ +"""Collect explicit public ICLR decisions from the authors' PeerRead release. + +Review bodies and parsed full paper text are deliberately not persisted. +""" +import argparse +import json +import time +import urllib.request +from collections import defaultdict +from pathlib import Path + +from collect_pmlr import fetch + + +def collect(cache,out): + root = Path(out) + root.mkdir(parents=True,exist_ok=True) + commit = json.loads(fetch('https://api.github.com/repos/allenai/PeerRead/commits/master',cache))['sha'] + tree = json.loads(fetch(f'https://api.github.com/repos/allenai/PeerRead/git/trees/{commit}?recursive=1',cache)) + if tree.get('truncated'): + raise RuntimeError('Truncated repository tree') + paths = [v['path'] for v in tree['tree'] if v['path'].startswith('data/iclr_2017/') and '/reviews/' in v['path'] and v['path'].endswith('.json')] + if not paths: + raise RuntimeError('No explicit ICLR2017 metadata files found') + cache_dir = root/'peerread-normalized'/commit + cache_dir.mkdir(parents=True,exist_ok=True) + grouped = defaultdict(list) + for number,path in enumerate(paths,1): + external = '/'.join(path.split('/')[1:]) + normalized_cache = cache_dir/(path.replace('/','_')) + if normalized_cache.exists(): + record = json.loads(normalized_cache.read_text()) + else: + raw_url = f'https://raw.githubusercontent.com/allenai/PeerRead/{commit}/{path}' + with urllib.request.urlopen(urllib.request.Request(raw_url,headers={'User-Agent':'PaperAtlas/0.1 metadata research'}),timeout=60) as response: + payload = response.read(3*1024**2+1) + if len(payload)>3*1024**2: + raise RuntimeError('Unexpected metadata size') + data = json.loads(payload) + conference = data.get('conference','') + if 'iclr' not in conference.lower() or '2017' not in conference: + raise RuntimeError('Unexpected conference metadata: '+conference) + accepted = data.get('accepted') + status = ('accepted' if accepted else 'rejected') if isinstance(accepted,bool) else 'unknown' + authors = data.get('authors') or [] + if isinstance(authors,str): + authors = [author.strip() for author in authors.split(',') if author.strip()] + record = {'source':'peerread','external_id':external,'url':f'https://github.com/allenai/PeerRead/blob/{commit}/{path}','title':data['title'],'abstract':data.get('abstract'),'authors':authors,'identifiers':{},'venue':'ICLR','year':2017,'track':'workshop' if 'workshop' in conference.lower() else 'main','status':status,'raw_decision':json.dumps({'accepted':accepted}),'metadata':{'conference':conference,'dataset_commit':commit,'dataset_path':path,'original_id':data.get('id'),'accepted':accepted,'decision_provenance':'PeerRead authors public release; explicit label, not inferred from publication absence'}} + normalized_cache.write_text(json.dumps(record,ensure_ascii=False)) + time.sleep(0.25) + grouped[record['track']].append(record) + if number%25==0: + print(json.dumps({'source':'peerread','collected':number,'expected':len(paths)}),flush=True) + for track,records in grouped.items(): + target = root/f'peerread-iclr2017-{track}.jsonl' + part = target.with_suffix('.part') + with part.open('w') as output: + for record in records: + output.write(json.dumps(record,ensure_ascii=False)+'\n') + part.replace(target) + manifest = {'source':'peerread','venue':'ICLR','year':2017,'track':track,'count':len(records),'list_url':f'https://github.com/allenai/PeerRead/tree/{commit}/data/iclr_2017','dataset_commit':commit,'file':str(target),'decision_counts':{status:sum(r['status']==status for r in records) for status in ('accepted','rejected','unknown')}} + target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)) + print(json.dumps(manifest),flush=True) + + +if __name__=='__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--cache',required=True) + parser.add_argument('--out',required=True) + args = parser.parse_args() + collect(args.cache,args.out) diff --git a/collect_pmlr.py b/collect_pmlr.py index 1e9da16..09910b0 100644 --- a/collect_pmlr.py +++ b/collect_pmlr.py @@ -17,7 +17,7 @@ def fetch(url,cache,limit=64*1024**2): cache = Path(cache) cache.mkdir(parents=True,exist_ok=True) path = cache/hashlib.sha256(url.encode()).hexdigest() - if path.exists(): + if path.exists() and time.time()-path.stat().st_mtime < 24*3600: return path.read_bytes() for attempt in range(3): try: @@ -102,7 +102,7 @@ def main(): volumes = args.volume if not volumes: index = fetch('https://proceedings.mlr.press/',args.cache).decode() - volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press)?/v(\d+)/?["\']',index))) + volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index))) if not volumes: raise RuntimeError('PMLR index yielded no volumes') errors = [] diff --git a/import_records.py b/import_records.py index ea66aee..02ed87b 100644 --- a/import_records.py +++ b/import_records.py @@ -33,8 +33,8 @@ def ingest_file(db,path): raise ValueError('Duplicate external ID in one collected file: '+external) seen.add(external) paper,record = store.record(source_name,external,row['url'],row['title'],row.get('abstract'),row.get('authors',[]),row.get('identifiers'),row.get('metadata')) - if row.get('status') == 'accepted': - edition = store.accepted(paper,record,row['venue'],row['year'],row['track'],external,row['url']) + if row.get('status') in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'): + edition = store.decision(paper,record,row['venue'],row['year'],row['track'],external,row.get('evidence_url',row['url']),row['status'],row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status'])) editions.add(edition) else: raise ValueError('This importer requires an explicit supported decision status') diff --git a/index_core.py b/index_core.py index 2bb2898..2c012e1 100644 --- a/index_core.py +++ b/index_core.py @@ -79,7 +79,7 @@ class Store: found = c.execute('SELECT paper_id FROM identifiers WHERE scheme=? AND value=?',(scheme,value)).fetchone() if found: matches.add(found[0]) - old = c.execute('SELECT paper_id FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone() + old = c.execute('SELECT paper_id,title,abstract FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone() if old: matches.add(old[0]) key = normalized(title) @@ -95,6 +95,8 @@ class Store: paper = min(matches) for other in sorted(matches-{paper}): self.merge(paper,other,'explicit identifier or exact title corroborated by authors') + if old: + c.execute("UPDATE papers SET title=?,abstract=?,title_key=?,authors_json=? WHERE id=? AND title=? AND COALESCE(abstract,'')=COALESCE(?,'')",(title,abstract,key,json.dumps(authors,ensure_ascii=False),paper,old[1],old[2])) c.execute("UPDATE papers SET abstract=CASE WHEN abstract IS NULL OR abstract='' THEN ? ELSE abstract END WHERE id=?",(abstract,paper)) else: paper = c.execute('INSERT INTO papers(title,abstract,title_key,authors_json,first_published) VALUES(?,?,?,?,?)',(title,abstract,key,json.dumps(authors,ensure_ascii=False),published)).lastrowid @@ -107,7 +109,7 @@ class Store: c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(paper,other),max(paper,other),'exact_title_unconfirmed',None,json.dumps({'title':title}), 'pending')) return paper,record - def accepted(self,paper,record,venue,year,track,external_id,url): + def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value): c = self.conn c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified')) venue_id = c.execute('SELECT id FROM venues WHERE acronym=?',(venue,)).fetchone()[0] @@ -115,6 +117,9 @@ class Store: edition = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?',(venue_id,int(year),track)).fetchone()[0] c.execute('INSERT OR IGNORE INTO submissions(paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?)',(paper,edition,record,external_id)) submission = c.execute('SELECT id FROM submissions WHERE source_record_id=? AND edition_id=? AND source_submission_id=?',(record,edition,external_id)).fetchone()[0] - if not c.execute("SELECT 1 FROM decisions WHERE submission_id=? AND status='accepted' AND evidence_url=?",(submission,url)).fetchone(): - c.execute("INSERT INTO decisions(submission_id,status,raw_value,evidence_url,observed_at) VALUES(?,'accepted','listed in official proceedings',?,strftime('%Y-%m-%dT%H:%M:%fZ','now'))",(submission,url)) + if not c.execute('SELECT 1 FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=?',(submission,status,raw_value,url)).fetchone(): + c.execute("INSERT INTO decisions(submission_id,status,raw_value,evidence_url,observed_at) VALUES(?,?,?,?,strftime('%Y-%m-%dT%H:%M:%fZ','now'))",(submission,status,raw_value,url)) return edition + + def accepted(self,paper,record,venue,year,track,external_id,url): + return self.decision(paper,record,venue,year,track,external_id,url,'accepted','listed in official proceedings') diff --git a/test_index_core.py b/test_index_core.py index 1d7c0f6..91a5979 100644 --- a/test_index_core.py +++ b/test_index_core.py @@ -51,6 +51,13 @@ class StoreTests(unittest.TestCase): self.assertEqual(self.store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],0) self.assertEqual(self.store.conn.execute('SELECT status FROM ingest_runs').fetchone()[0],'failed') + def test_source_update_refreshes_canonical_text_and_fts(self): + paper,_=self.record('arxiv','2601.00001') + updated,_=self.store.record('arxiv','2601.00001','https://arxiv.org/abs/2601.00001','Updated research title','New searchable content',['Jane Doe']) + self.assertEqual(updated,paper) + self.assertEqual(self.store.conn.execute('SELECT title FROM papers').fetchone()[0],'Updated research title') + self.assertEqual(self.store.conn.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'searchable'").fetchone()[0],1) + if __name__=='__main__': unittest.main() diff --git a/test_peerread.py b/test_peerread.py new file mode 100644 index 0000000..40ac185 --- /dev/null +++ b/test_peerread.py @@ -0,0 +1,33 @@ +import io +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from collect_peerread import collect +from import_records import ingest_file +from index_core import Store + + +class PeerReadTests(unittest.TestCase): + def test_explicit_rejection_and_no_review_body_storage(self): + tree={'sha':'a'*40,'tree':[{'path':'data/iclr_2017/test/reviews/123.json'}]} + paper={'conference':'ICLR 2017 conference submission','id':'123','title':'Public paper','abstract':'Abstract','authors':'Jane Doe, John Smith','accepted':False,'reviews':[{'text':'Do not persist this review body'}]} + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp) + with patch('collect_peerread.fetch',side_effect=[json.dumps({'sha':'a'*40}).encode(),json.dumps(tree).encode()]),patch('collect_peerread.urllib.request.urlopen',return_value=io.BytesIO(json.dumps(paper).encode())),patch('collect_peerread.time.sleep'): + collect(root/'cache',root/'out') + path=root/'out/peerread-iclr2017-main.jsonl' + record=json.loads(path.read_text()) + self.assertEqual(record['status'],'rejected') + self.assertNotIn('Do not persist',path.read_text()) + ingest_file(root/'db.sqlite',path) + store=Store(root/'db.sqlite') + self.addCleanup(store.conn.close) + self.assertEqual(store.conn.execute('SELECT status FROM decisions').fetchone()[0],'rejected') + self.assertEqual(store.conn.execute('SELECT status FROM edition_sources').fetchone()[0],'complete') + + +if __name__=='__main__': + unittest.main()