diff --git a/README.md b/README.md index 3da3ff5..3362078 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,9 @@ python3 -B query.py --db /data/paper-index/papers.sqlite --tag arxiv:category=cs python3 -B query.py --db /data/paper-index/papers.sqlite --venue COLM --year 2025 --status rejected ``` -查询连接使用 SQLite 只读模式,结果输出 JSON。当前没有公共 Web 页面或网络查询服务。 +查询连接使用 SQLite 只读模式,结果输出 JSON。`--stats` 中 `papers` 是按当前规则去重后的论文实体数,`records_by_source` 是各平台已入库记录数,`decision_events_by_status` 是决定事件数。采集文件 manifest 的 `count` 是已采集数量,不能等同已入库数量,也不能把各平台记录数相加当成独立论文数。 + +用 `--paper 531782` 可查看单篇论文的所有已知来源、平台标签、投稿届次及带证据的决定历史。会议筛选不区分大小写,状态筛选使用该次投稿的最新已知决定。当前没有公共 Web 页面或网络查询服务。 ## 官方会议元数据采集 @@ -110,7 +112,7 @@ python3 -B import_records.py --db /data/paper-index/papers.sqlite \ /data/paper-index/collected/pmlr-v267.jsonl ``` -采集输出 JSONL 和对应 manifest,只有显式官方录用记录才产生 accepted 决定。清单数目不一致时整个文件导入回滚,记录失败批次,不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。旧卷可能把 PDF 放入归档;采集器对前 50 卷使用 Git partial clone 和 sparse checkout,仅取 `_posts` 和卷配置,不执行出版方项目代码。较新归档若超出 64 MiB 限额,也转为此方式。已对原本超限的 AISTATS 2009(第 5 卷)验证采集 84 条且工作区无 PDF。缓存默认有效 24 小时;同一次批次可复用缓存,后续更新会重新读取动态入口。 +采集输出 JSONL 和对应 manifest,只有明确来源标签才产生相应决定:官方论文集记录为 accepted;PeerRead 使用其明确的 accepted 布尔标签,缺失标签保留 unknown。清单数目不一致时整个文件导入回滚,记录失败批次,不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。旧卷可能把 PDF 放入归档;采集器对前 50 卷使用 Git partial clone 和 sparse checkout,仅取 `_posts` 和卷配置,不执行出版方项目代码。较新归档若超出 64 MiB 限额,也转为此方式。已对原本超限的 AISTATS 2009(第 5 卷)验证采集 84 条且工作区无 PDF。旧 YAML 的 `!binary` 文本、空文档尾标记会安全解析;带年份的 AISTATS/COLT 等历史 shortname 会规范化,避免名称别名漏届;缺失 year 时仅从官方会议 start 日期提取年份。`pmlr-volume-status.json` 逐卷保存结果和候选范围归属,`pmlr-last-run.json` 记录本批实际请求范围,局部重试不能证明全历史完成。缓存默认有效 24 小时;同一次批次可复用缓存,后续更新会重新读取动态入口。 PeerRead 采集仅选择作者公开发布的 ICLR 2017 投稿元数据,固定到源仓库 commit。只依据明确的布尔 `accepted` 标签形成 accepted/rejected;空标签为 unknown,不使用其他数据分区中的 probably-rejected 推断。不会保存 reviews 或 parsed_pdfs 正文。 @@ -119,7 +121,7 @@ python3 -B collect_peerread.py --cache /data/paper-index/raw/http-cache \ --out /data/paper-index/collected ``` -数据任务以一次性 systemd 服务运行:`paper-atlas-arxiv-import`、`paper-atlas-pmlr-collect`、`paper-atlas-peerread-collect`。它们不是定时任务。主库只允许一个写入进程;会议采集先写独立 JSONL,待 arXiv 主批次结束后再合并。运行日志位于 `/data/paper-index/*-import.log`、`/data/paper-index/*-collect.log`。任何服务启动成功都不等于批次已完成,需检查批次状态、清单与完整性结果。 +数据任务以一次性 systemd 服务运行:`paper-atlas-arxiv-import`、`paper-atlas-pmlr-collect`、`paper-atlas-peerread-collect`;`paper-atlas-pmlr-verify` 用于全索引复核,`paper-atlas-conference-merge` 用于单写入合并。它们不是定时任务。主库只允许一个写入进程;会议采集先写独立 JSONL,待 arXiv 主批次结束后再合并;也可以使用 `merge_conferences.py` 在保存断点处暂停本项目的 arXiv unit、顺序合并会议清单并恢复。该操作应作为独立的一次性 systemd 任务运行,避免 SSH 断开影响恢复;不能同时启动多个合并任务。运行日志位于 `/data/paper-index/*-import.log`、`/data/paper-index/*-collect.log`。任何服务启动成功都不等于批次已完成,需检查批次状态、清单与完整性结果。 ## 长期维护与更新 @@ -135,4 +137,6 @@ python3 -B collect_peerread.py --cache /data/paper-index/raw/http-cache \ 2026-11-06 从目标服务器测试:Kaggle 元数据 GET 可下载;OpenReview 会议 group 接口可读,但 notes 接口返回 `ChallengeRequiredError`。拒稿记录导入尚受此限制,不能用录用名单替代全部投稿,也不能声称已覆盖全部拒稿。需通过平台支持的访问方式或有合法来源的公开数据导出继续补全。 -全量导入、会议适配器、正式 runner 配置与 PR 交付状态应随实现及时更新本 README。 +已验证的阶段性结果:ICML 2025 的 3,330 条记录和 PeerRead ICLR 2017 的 427 条记录已合并主库,合计 3,502 条 accepted 决定、255 条 rejected 决定;arXiv 曾在 529,000 条断点暂停合并,已验证恢复。这些是阶段性值,不是全量交付数据量。 + +代码在 `codex/metadata-index` 工作分支交付,[Draft PR #1](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/1) 尚未合并,等待全量就绪后用户手动合并。已推送的前三个开发提交均有成功的远程 CI;后续提交仍需检查其各自 CI。全量导入、其他会议适配器、去重复核和覆盖率核验继续进行,应随实际结果更新此说明。 diff --git a/collect_pmlr.py b/collect_pmlr.py index 51ea9e9..f43cfe3 100644 --- a/collect_pmlr.py +++ b/collect_pmlr.py @@ -15,6 +15,29 @@ from pathlib import Path import yaml +class MetadataLoader(yaml.SafeLoader): + pass + + +def binary_text(loader,node): + return loader.construct_yaml_binary(node).decode('utf-8') + + +MetadataLoader.add_constructor('!binary',binary_text) +MetadataLoader.add_constructor('tag:yaml.org,2002:binary',binary_text) + + +def load_metadata(value): + documents=[doc for doc in yaml.load_all(value,Loader=MetadataLoader) if doc is not None] + if len(documents)!=1 or not isinstance(documents[0],dict): + raise ValueError('Expected exactly one nonempty metadata mapping') + return documents[0] + + +def venue_key(shortname): + return re.sub(r'[\s_-]*(?:19|20)\d{2}$','',str(shortname).strip().upper()) + + def fetch(url,cache,limit=64*1024**2): cache = Path(cache) cache.mkdir(parents=True,exist_ok=True) @@ -89,11 +112,16 @@ def metadata_texts(volume,cache): def collect_volume(volume,cache,out,allowed=None): config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml' - config = yaml.safe_load(fetch(config_url,cache)) - venue = str(config.get('shortname','')).upper() + config = load_metadata(fetch(config_url,cache)) + venue = venue_key(config.get('shortname','')) if allowed and venue not in allowed: - return None - year = int(config['year']) + return {'source':'pmlr','volume':volume,'status':'excluded_by_registry','raw_shortname':venue,'booktitle':str(config.get('booktitle',''))} + year_value = config.get('year') + if not year_value: + year_value = str(config.get('start',''))[:4] + if not re.fullmatch(r'\d{4}',str(year_value)): + raise ValueError('No explicit year or conference start date') + year = int(year_value) title = str(config.get('booktitle','')) track = 'workshop' if 'workshop' in title.lower() else 'main' path = Path(out)/f'pmlr-v{volume}.jsonl' @@ -102,10 +130,10 @@ def collect_volume(volume,cache,out,allowed=None): count = 0 with part.open('w') as output: for member_name,content in metadata_texts(volume,cache): - parts = content.split('---',2) + parts = re.split(r'(?m)^---\s*$',content,maxsplit=2) if len(parts)<3 or parts[0].strip(): raise RuntimeError('Missing YAML frontmatter: '+member_name) - row = yaml.safe_load(parts[1]) + row = load_metadata(parts[1]) if row.get('layout') not in ('inproceedings','article') or not row.get('title'): continue external = f"v{volume}/{row.get('id') or Path(member_name).stem}" @@ -139,23 +167,33 @@ def main(): parser.add_argument('--volume',type=int,action='append') parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json'))) args = parser.parse_args() - allowed = {v['acronym'].upper() for v in json.loads(Path(args.registry).read_text())['venues']} + allowed = {venue_key(v['acronym']) for v in json.loads(Path(args.registry).read_text())['venues']} volumes = args.volume if not volumes: index = fetch('https://proceedings.mlr.press/',args.cache).decode() volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index))) if not volumes: raise RuntimeError('PMLR index yielded no volumes') + Path(args.out).mkdir(parents=True,exist_ok=True) + catalogue_path=Path(args.out)/'pmlr-volume-status.json' + catalogue=json.loads(catalogue_path.read_text()) if catalogue_path.exists() else {} errors = [] for volume in volumes: try: - collect_volume(volume,args.cache,args.out,None if args.volume else allowed) + result=collect_volume(volume,args.cache,args.out,None if args.volume else allowed) + result['selection_mode']='explicit' if args.volume else 'registry' + result['selected_by_registry']=result.get('venue',result.get('raw_shortname')) in allowed + catalogue[str(volume)]=result except Exception as exc: error = {'volume':volume,'status':'blocked','error':str(exc)} errors.append(error) + catalogue[str(volume)]=error print(json.dumps(error),flush=True) - Path(args.out).mkdir(parents=True,exist_ok=True) + part=catalogue_path.with_suffix('.part') + part.write_text(json.dumps(catalogue,indent=2)) + part.replace(catalogue_path) (Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2)) + (Path(args.out)/'pmlr-last-run.json').write_text(json.dumps({'requested_volumes':volumes,'selection_mode':'explicit' if args.volume else 'registry','errors':errors},indent=2)) if errors: raise SystemExit(1) diff --git a/merge_conferences.py b/merge_conferences.py new file mode 100644 index 0000000..a70098a --- /dev/null +++ b/merge_conferences.py @@ -0,0 +1,60 @@ +"""One-shot server operation: merge collected files at an arXiv checkpoint. + +Run as a transient systemd job so an SSH disconnect cannot strand the importer. +Only the Paper Atlas arXiv unit is controlled; business services are untouched. +""" +import argparse +import json +import sqlite3 +import subprocess +from pathlib import Path + +from import_records import ingest_file + + +UNIT='paper-atlas-arxiv-import.service' + + +def property_value(name): + return subprocess.check_output(['systemctl','show',UNIT,'-p',name,'--value'],text=True).strip() + + +def merge(db,archive,files): + # Validate inputs before stopping the existing writer. + for filename in files: + path=Path(filename) + if not path.is_file() or not path.with_suffix('.manifest.json').is_file(): + raise ValueError('Missing collected file or manifest: '+filename) + if not Path(archive).is_file(): + raise ValueError('Missing arXiv archive') + if property_value('ActiveState')!='active': + raise RuntimeError('Expected the existing arXiv unit to be active; inspect it before proceeding') + subprocess.run(['systemctl','stop',UNIT],check=True) + try: + with sqlite3.connect(db) as c: + checkpoint=c.execute("SELECT records_written FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone() + print('Saved arxiv checkpoint:',checkpoint,flush=True) + for filename in files: + ingest_file(db,filename) + with sqlite3.connect(db) as c: + errors=c.execute('PRAGMA foreign_key_check').fetchall() + if errors: + raise RuntimeError('Foreign key violations after merge') + print('Decision totals:',c.execute('SELECT status,count(*) FROM decisions GROUP BY status').fetchall(),flush=True) + finally: + if property_value('LoadState')=='loaded': + subprocess.run(['systemctl','start',UNIT],check=True) + else: + subprocess.run(['systemd-run','--unit='+UNIT,'--property=WorkingDirectory='+str(Path(__file__).parent),'--property=CPUQuota=50%','--property=MemoryMax=512M','--property=Nice=15','--property=IOWeight=10','--property=StandardOutput=append:/data/paper-index/arxiv-import.log','--property=StandardError=append:/data/paper-index/arxiv-import.log','/usr/bin/env','PYTHONDONTWRITEBYTECODE=1','python3','-B',str(Path(__file__).with_name('import_arxiv.py')),'--db',db,'--archive',archive],check=True) + if property_value('ActiveState')!='active': + raise RuntimeError('arXiv unit did not resume; inspect the logs') + print('ARXIV_RESUMED',flush=True) + + +if __name__=='__main__': + parser=argparse.ArgumentParser() + parser.add_argument('--db',required=True) + parser.add_argument('--archive',required=True) + parser.add_argument('files',nargs='+') + args=parser.parse_args() + merge(args.db,args.archive,args.files) diff --git a/query.py b/query.py index b293291..c322f43 100644 --- a/query.py +++ b/query.py @@ -9,6 +9,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--db', required=True) parser.add_argument('--query', help='SQLite FTS5 query over title and abstract') + parser.add_argument('--paper',type=int,help='Paper ID; include sources, tags and submission history') parser.add_argument('--tag', help='Filter as namespace=value, e.g. arxiv:category=cs.AI') parser.add_argument('--venue', help='Conference acronym') parser.add_argument('--year', type=int) @@ -18,34 +19,50 @@ def main(): args = parser.parse_args() conn = sqlite3.connect(Path(args.db).resolve().as_uri()+'?mode=ro', uri=True) conn.row_factory = sqlite3.Row + conn.execute('BEGIN') # One consistent read snapshot while ingestion continues. if args.stats: result = {name:conn.execute('SELECT count(*) FROM '+name).fetchone()[0] for name in ('papers','source_records','submissions','decisions','tags','match_candidates')} + result['records_by_source'] = [dict(r) for r in conn.execute('SELECT s.name,count(r.id) AS imported_records FROM sources s LEFT JOIN source_records r ON r.source_id=s.id GROUP BY s.id ORDER BY s.name')] + result['decision_events_by_status'] = [dict(r) for r in conn.execute('SELECT status,count(*) AS events FROM decisions GROUP BY status')] result['coverage'] = [dict(r) for r in conn.execute('SELECT status,count(*) AS sources FROM edition_sources GROUP BY status')] result['runs'] = [dict(r) for r in conn.execute('SELECT * FROM ingest_runs ORDER BY id DESC LIMIT 10')] else: where = []; params = [] + if args.paper is not None: + where.append('p.id=?'); params.append(args.paper) if args.query: where.append('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)'); params.append(args.query) if args.tag: namespace,value = args.tag.split('=',1) where.append('EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'); params.extend([namespace,value]) if args.venue or args.year or args.status: - clause = ['s.paper_id=p.id'] + clause = ['1=1'] if args.venue: - clause.append('v.acronym=?'); params.append(args.venue) + clause.append('v.acronym=? COLLATE NOCASE'); params.append(args.venue) if args.year: clause.append('e.year=?'); params.append(args.year) if args.status: clause.append('EXISTS(SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=? AND NOT EXISTS(SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id AND (newer.observed_at>d.observed_at OR (newer.observed_at=d.observed_at AND newer.id>d.id))))'); params.append(args.status) - where.append('EXISTS(SELECT 1 FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+' AND '.join(clause)+')') + where.append('p.id IN (SELECT s.paper_id FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+' AND '.join(clause)+')') sql = 'SELECT p.* FROM papers p'+(' WHERE '+' AND '.join(where) if where else '')+' ORDER BY p.id LIMIT ?' params.append(max(1,min(args.limit,200))) result = [] for row in conn.execute(sql,params): paper = dict(row) + paper['authors'] = json.loads(paper.pop('authors_json')) paper['identifiers'] = [dict(r) for r in conn.execute('SELECT scheme,value FROM identifiers WHERE paper_id=?',(row['id'],))] + if args.paper is not None: + paper['tags'] = [dict(r) for r in conn.execute('SELECT t.namespace,t.value,pt.evidence_key FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=?',(row['id'],))] + paper['sources'] = [dict(r) for r in conn.execute('SELECT s.name,r.external_id,r.url,r.title,r.abstract,r.retrieved_at FROM source_records r JOIN sources s ON s.id=r.source_id WHERE r.paper_id=?',(row['id'],))] + submissions = [] + for sub in conn.execute('SELECT s.id,s.source_submission_id,s.submitted_at,v.acronym,e.year,e.track FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE s.paper_id=? ORDER BY e.year,s.id',(row['id'],)): + submission = dict(sub) + submission['decisions'] = [dict(d) for d in conn.execute('SELECT status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=? ORDER BY observed_at,id',(sub['id'],))] + submissions.append(submission) + paper['submissions'] = submissions result.append(paper) print(json.dumps(result,ensure_ascii=False,indent=2)) + conn.close() if __name__ == '__main__': diff --git a/schema.sql b/schema.sql index 8a93f24..58c947d 100644 --- a/schema.sql +++ b/schema.sql @@ -53,6 +53,7 @@ CREATE TABLE IF NOT EXISTS decisions ( raw_value TEXT NOT NULL, evidence_url TEXT NOT NULL, observed_at TEXT NOT NULL, decision_date TEXT, UNIQUE(submission_id,status,raw_value,evidence_url,observed_at) ); +CREATE INDEX IF NOT EXISTS decisions_submission_order ON decisions(submission_id,observed_at DESC,id DESC); CREATE TABLE IF NOT EXISTS tags ( id INTEGER PRIMARY KEY, namespace TEXT NOT NULL, value TEXT NOT NULL, description TEXT, UNIQUE(namespace,value) diff --git a/test_pmlr.py b/test_pmlr.py new file mode 100644 index 0000000..35122f5 --- /dev/null +++ b/test_pmlr.py @@ -0,0 +1,25 @@ +import unittest + +from collect_pmlr import load_metadata,venue_key + + +class MetadataTests(unittest.TestCase): + def test_legacy_binary_scalar_is_decoded_as_text(self): + self.assertEqual(load_metadata('title: !binary |\n SGVsbG8=\n')['title'],'Hello') + + def test_trailing_empty_document_is_allowed(self): + self.assertEqual(load_metadata('---\nyear: 2024\n---\n'),{'year':2024}) + + def test_conflicting_documents_are_not_silently_combined(self): + with self.assertRaisesRegex(ValueError,'exactly one'): + load_metadata('year: 2024\n---\nyear: 2025\n') + + def test_historical_conference_name_aliases(self): + self.assertEqual(venue_key('AISTATS 2010'),'AISTATS') + self.assertEqual(venue_key('AISTATS 2012'),'AISTATS') + self.assertEqual(venue_key('COLT 2012'),'COLT') + self.assertEqual(venue_key('ECML-PKDD'),'ECML-PKDD') + + +if __name__=='__main__': + unittest.main() diff --git a/test_query.py b/test_query.py new file mode 100644 index 0000000..0b83b50 --- /dev/null +++ b/test_query.py @@ -0,0 +1,39 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from index_core import Store + + +class QueryTests(unittest.TestCase): + def test_venue_filter_is_case_insensitive_and_uses_latest_decision(self): + with tempfile.TemporaryDirectory() as tmp: + db=Path(tmp)/'papers.sqlite' + store=Store(db) + self.addCleanup(store.conn.close) + paper,record=store.record('peerread','paper1','https://example.org/paper1','A real submission','Abstract',['Jane Doe']) + store.decision(paper,record,'ICLR',2017,'main','paper1','https://example.org/paper1','rejected','explicit rejection') + store.record('arxiv','2601.00001','https://example.org/other','Another paper','Abstract',['John Smith']) + store.conn.commit() + command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--venue','iclr','--year','2017','--status','rejected'] + result=json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout) + self.assertEqual([r['id'] for r in result],[paper]) + details_command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--paper',str(paper)] + details=json.loads(subprocess.run(details_command,check=True,capture_output=True,text=True).stdout)[0] + self.assertEqual(details['authors'],['Jane Doe']) + self.assertEqual(details['submissions'][0]['decisions'][0]['status'],'rejected') + self.assertEqual(details['sources'][0]['external_id'],'paper1') + stats_command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--stats'] + stats=json.loads(subprocess.run(stats_command,check=True,capture_output=True,text=True).stdout) + self.assertEqual(stats['papers'],2) + self.assertEqual({r['name']:r['imported_records'] for r in stats['records_by_source']},{'arxiv':1,'peerread':1}) + store.decision(paper,record,'ICLR',2017,'main','paper1','https://example.org/paper1','accepted','changed decision') + store.conn.commit() + self.assertEqual(json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout),[]) + + +if __name__=='__main__': + unittest.main()