Add provenance-aware paper metadata storage and PMLR collection
Metadata integrity checks / metadata (push) Successful in 1s
Metadata integrity checks / metadata (push) Successful in 1s
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
name: Metadata integrity checks
|
||||
on:
|
||||
push:
|
||||
branches: [main, 'codex/**']
|
||||
jobs:
|
||||
metadata:
|
||||
runs-on: ces-control
|
||||
env:
|
||||
ARCHIVE_SHA: ${{ github.sha }}
|
||||
PYTHONDONTWRITEBYTECODE: '1'
|
||||
steps:
|
||||
- name: Verify metadata storage and query commands
|
||||
run: |
|
||||
python3 -B - <<'PY'
|
||||
import io, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||
sha = os.environ['ARCHIVE_SHA']
|
||||
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
||||
raise SystemExit('Invalid source commit')
|
||||
# Public repository archive: no owner credentials or runner SSH keys.
|
||||
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
|
||||
with urllib.request.urlopen(url, timeout=60) as response:
|
||||
archive = response.read(16*1024*1024+1)
|
||||
if len(archive)>16*1024*1024:
|
||||
raise SystemExit('Unexpectedly large source archive')
|
||||
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
|
||||
root = pathlib.Path(tmp)
|
||||
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
|
||||
tar.extractall(root, filter='data')
|
||||
candidates = list(root.glob('*/test_import_arxiv.py'))
|
||||
if len(candidates)!=1:
|
||||
raise SystemExit('Unexpected repository archive layout')
|
||||
src = candidates[0].parent
|
||||
subprocess.run(['python3','-B','-m','unittest','discover','-v'],cwd=src,check=True)
|
||||
subprocess.run(['python3','-B','query.py','--help'],cwd=src,check=True)
|
||||
PY
|
||||
@@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工
|
||||
|
||||
目标包括:完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。
|
||||
|
||||
**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、DOI/标识合并和只读查询脚本。服务器上进行过合成样本验证;全量快照下载与历史导入尚未完成。会议适配器、跨平台模糊匹配复核流程、完整覆盖率核验与 CI 仍待实现。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
|
||||
**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上六项合成回归检查通过;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 全量快照下载与历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
|
||||
|
||||
## 数据范围与来源
|
||||
|
||||
@@ -41,16 +41,16 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工
|
||||
| `edition_sources` | 每届每个来源的覆盖状态、预计/已导入数量、错误与断点 |
|
||||
| `submissions` / `decisions` | 多次投稿及带证据的决定历史,支持改投、撤稿和决定变更 |
|
||||
| `tags` / `paper_tags` | 平台标签与人工标签,使用命名空间隔离 |
|
||||
| `match_candidates` | 证据不充分的潜在重复记录及人工复核状态 |
|
||||
| `match_candidates` / `paper_merges` | 潜在重复复核状态、确定合并审计 |
|
||||
| `ingest_runs` | 导入批次、状态、错误和恢复位置 |
|
||||
|
||||
决定状态为 `accepted`、`rejected`、`desk_rejected`、`withdrawn`、`under_review`、`unknown`。一篇论文可以对应多次投稿和多个决定,不能用单一“最终会议”字段覆盖历史。查询中按观察时间选择最近状态;后续分析需要同时检查来源覆盖率。
|
||||
|
||||
当前确定性去重使用 arXiv ID 和单个 DOI。保留各平台源记录,避免合并后丢失出处。规范化标题只建检索索引,尚未实现模糊匹配流程;**现阶段不保证跨平台去重已经完成**。不同平台标签不自动等同,自定义标签应使用 `custom:*` 命名空间。
|
||||
当前确定性去重使用 arXiv ID、单个 DOI 和 OpenReview forum ID;跨平台同标题还需至少两位作者吻合,或作者吻合且摘要相同,才自动合并。保留各平台源记录,避免合并后丢失出处。规范化标题只建检索索引,尚未实现模糊匹配流程;**现阶段不保证跨平台去重已经完成**。不同平台标签不自动等同,自定义标签应使用 `custom:*` 命名空间。
|
||||
|
||||
## 运行与查询
|
||||
|
||||
依赖 Python 3 标准库和带 FTS5 的 SQLite,目前无第三方 Python 依赖,无项目构建步骤。服务器已检查为 Python 3.14.4。所有运行数据放在仓库外,不提交数据库、下载包或访问凭据。
|
||||
基础导入和查询依赖 Python 3 标准库和带 FTS5 的 SQLite;PMLR 采集另需 `requirements.txt` 中锁定的 PyYAML 6.0.3。无项目构建步骤。服务器已检查为 Python 3.14.4。所有运行数据放在仓库外,不提交数据库、下载包或访问凭据。
|
||||
|
||||
服务器暂存目录:`/data/paper-index/`,快照目录:`/data/paper-index/raw/`,运行代码目录:`/data/paper-index/code/`。这些路径不是对外访问的 API。
|
||||
|
||||
@@ -82,6 +82,20 @@ python3 -B query.py --db /data/paper-index/papers.sqlite --venue COLM --year 202
|
||||
|
||||
查询连接使用 SQLite 只读模式,结果输出 JSON。当前没有公共 Web 页面或网络查询服务。
|
||||
|
||||
## 官方会议元数据采集
|
||||
|
||||
```sh
|
||||
# 单卷示例:ICML 2025,读取官方 gh-pages 元数据归档,不下载 PDF
|
||||
python3 -B collect_pmlr.py --cache /data/paper-index/raw/http-cache \
|
||||
--out /data/paper-index/collected --volume 267
|
||||
|
||||
# 待主库完成 arXiv 导入、且没有其他写入进程后,合并会议记录
|
||||
python3 -B import_records.py --db /data/paper-index/papers.sqlite \
|
||||
/data/paper-index/collected/pmlr-v267.jsonl
|
||||
```
|
||||
|
||||
采集输出 JSONL 和对应 manifest,只有显式官方录用记录才产生 accepted 决定。清单数目不一致时整个文件导入回滚,记录失败批次,不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。
|
||||
|
||||
## 长期维护与更新
|
||||
|
||||
1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发,遵守来源限流、重试与人机验证要求。
|
||||
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
"""Collect official PMLR metadata, never the PDFs linked by the metadata."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import tarfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def fetch(url,cache,limit=64*1024**2):
|
||||
cache = Path(cache)
|
||||
cache.mkdir(parents=True,exist_ok=True)
|
||||
path = cache/hashlib.sha256(url.encode()).hexdigest()
|
||||
if path.exists():
|
||||
return path.read_bytes()
|
||||
for attempt in range(3):
|
||||
try:
|
||||
request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'})
|
||||
with urllib.request.urlopen(request,timeout=90) as response:
|
||||
if response.status != 200:
|
||||
raise RuntimeError('Expected complete HTTP 200 response')
|
||||
data = response.read(limit+1)
|
||||
if len(data)>limit:
|
||||
raise RuntimeError('Metadata response exceeds byte limit')
|
||||
part = path.with_suffix('.part')
|
||||
part.write_bytes(data)
|
||||
part.replace(path)
|
||||
path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
|
||||
time.sleep(0.5)
|
||||
return data
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code not in (429,500,502,503,504) or attempt==2:
|
||||
raise
|
||||
time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1')))))
|
||||
|
||||
|
||||
def collect_volume(volume,cache,out,allowed=None):
|
||||
config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml'
|
||||
config = yaml.safe_load(fetch(config_url,cache))
|
||||
venue = str(config.get('shortname','')).upper()
|
||||
if allowed and venue not in allowed:
|
||||
return None
|
||||
year = int(config['year'])
|
||||
title = str(config.get('booktitle',''))
|
||||
track = 'workshop' if 'workshop' in title.lower() else 'main'
|
||||
archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
|
||||
path = Path(out)/f'pmlr-v{volume}.jsonl'
|
||||
path.parent.mkdir(parents=True,exist_ok=True)
|
||||
part = path.with_suffix('.part')
|
||||
count = 0
|
||||
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar,part.open('w') as output:
|
||||
for member in tar:
|
||||
if not member.isfile() or '/_posts/' not in member.name or not member.name.endswith(('.md','.markdown','.html')):
|
||||
continue
|
||||
if member.size>8*1024**2:
|
||||
raise RuntimeError('Unexpectedly large metadata member')
|
||||
content = tar.extractfile(member).read().decode('utf-8-sig')
|
||||
parts = content.split('---',2)
|
||||
if len(parts)<3 or parts[0].strip():
|
||||
raise RuntimeError('Missing YAML frontmatter: '+member.name)
|
||||
row = yaml.safe_load(parts[1])
|
||||
if row.get('layout') not in ('inproceedings','article') or not row.get('title'):
|
||||
continue
|
||||
external = f"v{volume}/{row.get('id') or Path(member.name).stem}"
|
||||
ids = {}
|
||||
for key in ('doi','openreview','arxiv'):
|
||||
value = row.get(key)
|
||||
if value:
|
||||
value = str(value)
|
||||
if key=='arxiv':
|
||||
value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
|
||||
if key=='openreview':
|
||||
value = value.split('id=')[-1].split('&')[0]
|
||||
ids[key] = value
|
||||
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
|
||||
output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n')
|
||||
count += 1
|
||||
if count==0:
|
||||
part.unlink(missing_ok=True)
|
||||
raise RuntimeError('No papers parsed; refusing an empty successful collection')
|
||||
part.replace(path)
|
||||
result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
|
||||
print(json.dumps(result),flush=True)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--cache',required=True)
|
||||
parser.add_argument('--out',required=True)
|
||||
parser.add_argument('--volume',type=int,action='append')
|
||||
parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json')))
|
||||
args = parser.parse_args()
|
||||
allowed = {v['acronym'].upper() for v in json.loads(Path(args.registry).read_text())['venues']}
|
||||
volumes = args.volume
|
||||
if not volumes:
|
||||
index = fetch('https://proceedings.mlr.press/',args.cache).decode()
|
||||
volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press)?/v(\d+)/?["\']',index)))
|
||||
if not volumes:
|
||||
raise RuntimeError('PMLR index yielded no volumes')
|
||||
errors = []
|
||||
for volume in volumes:
|
||||
try:
|
||||
collect_volume(volume,args.cache,args.out,None if args.volume else allowed)
|
||||
except Exception as exc:
|
||||
error = {'volume':volume,'status':'blocked','error':str(exc)}
|
||||
errors.append(error)
|
||||
print(json.dumps(error),flush=True)
|
||||
Path(args.out).mkdir(parents=True,exist_ok=True)
|
||||
(Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2))
|
||||
if errors:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Stream the official metadata ZIP into SQLite. No PDFs, source files or builds."""
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
import shutil
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
from index_core import Store
|
||||
|
||||
|
||||
def ingest(db, archive):
|
||||
store = Store(db)
|
||||
conn = store.conn
|
||||
conn.execute("INSERT OR IGNORE INTO sources(name,homepage) VALUES('arxiv','https://arxiv.org')")
|
||||
source = conn.execute("SELECT id FROM sources WHERE name='arxiv'").fetchone()[0]
|
||||
previous = conn.execute('SELECT id,checkpoint_json FROM ingest_runs WHERE source_id=? AND status IN (\'running\',\'failed\') ORDER BY id DESC LIMIT 1', (source,)).fetchone()
|
||||
fingerprint = {'path':str(Path(archive).resolve()), 'bytes':Path(archive).stat().st_size, 'mtime_ns':Path(archive).stat().st_mtime_ns}
|
||||
start = 0
|
||||
if previous:
|
||||
checkpoint = json.loads(previous[1] or '{}')
|
||||
if checkpoint.get('archive') == fingerprint:
|
||||
run = previous[0]
|
||||
start = checkpoint.get('line', 0)
|
||||
conn.execute("UPDATE ingest_runs SET status='running',error=NULL WHERE id=?", (run,))
|
||||
else:
|
||||
raise RuntimeError('Pending run belongs to a different archive; resolve it explicitly')
|
||||
else:
|
||||
run = conn.execute('INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)', (source,json.dumps({'archive':fingerprint,'line':0}))).lastrowid
|
||||
conn.commit()
|
||||
line_number = start
|
||||
last = time.monotonic()
|
||||
try:
|
||||
with zipfile.ZipFile(archive) as zipped:
|
||||
# Reading the member to EOF also checks its CRC.
|
||||
with zipped.open('arxiv-metadata-oai-snapshot.json') as stream:
|
||||
for line_number, line in enumerate(stream, 1):
|
||||
if line_number <= start:
|
||||
continue
|
||||
row = json.loads(line)
|
||||
external_id = row['id']
|
||||
title = ' '.join(row['title'].split())
|
||||
abstract = row.get('abstract', '').strip()
|
||||
authors = row.get('authors_parsed') or row.get('authors') or []
|
||||
metadata = {k:v for k,v in row.items() if k not in ('title','abstract','authors','authors_parsed')}
|
||||
paper,record = store.record('arxiv',external_id,'https://arxiv.org/abs/'+external_id,title,abstract,authors,{'arxiv':external_id,'doi':row.get('doi')},metadata,(row.get('versions') or [{}])[0].get('created'))
|
||||
conn.execute("DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key='arxiv:categories'", (record,))
|
||||
for category in (row.get('categories') or '').split():
|
||||
conn.execute("INSERT OR IGNORE INTO tags(namespace,value) VALUES('arxiv:category',?)",(category,))
|
||||
tag = conn.execute("SELECT id FROM tags WHERE namespace='arxiv:category' AND value=?",(category,)).fetchone()[0]
|
||||
conn.execute('INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,'arxiv:categories'))
|
||||
if line_number % 500 == 0:
|
||||
if shutil.disk_usage(Path(db).parent).free < 4*1024**3:
|
||||
raise RuntimeError('Stopped before exhausting disk: less than 4 GiB free')
|
||||
conn.execute('UPDATE ingest_runs SET records_seen=?,records_written=?,checkpoint_json=? WHERE id=?',(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
|
||||
conn.commit()
|
||||
if time.monotonic()-last >= 20:
|
||||
print(json.dumps({'run':run,'rows':line_number}),flush=True)
|
||||
last = time.monotonic()
|
||||
conn.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
|
||||
conn.commit()
|
||||
print(json.dumps({'status':'complete','rows':line_number}),flush=True)
|
||||
except BaseException as exc:
|
||||
conn.rollback()
|
||||
conn.execute("UPDATE ingest_runs SET status='failed',error=? WHERE id=?",(str(exc),run))
|
||||
conn.commit()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--db', required=True)
|
||||
parser.add_argument('--archive', required=True)
|
||||
args = parser.parse_args()
|
||||
ingest(args.db,args.archive)
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Import normalized conference JSONL files with coverage evidence."""
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from index_core import Store
|
||||
|
||||
|
||||
def ingest_file(db,path):
|
||||
path = Path(path)
|
||||
manifest_path = path.with_suffix('.manifest.json')
|
||||
manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
|
||||
store = Store(db)
|
||||
c = store.conn
|
||||
source_name = manifest.get('source')
|
||||
if not source_name:
|
||||
raise ValueError('A source manifest is required')
|
||||
source_id = store.source(source_name,manifest['list_url'])
|
||||
run = c.execute('INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)',(source_id,json.dumps({'file':str(path)}))).lastrowid
|
||||
c.commit()
|
||||
count = 0
|
||||
seen = set()
|
||||
editions = set()
|
||||
try:
|
||||
with path.open() as stream:
|
||||
for line in stream:
|
||||
row = json.loads(line)
|
||||
if row['source'] != source_name:
|
||||
raise ValueError('Source differs from manifest')
|
||||
external = row['external_id']
|
||||
if external in seen:
|
||||
raise ValueError('Duplicate external ID in one collected file: '+external)
|
||||
seen.add(external)
|
||||
paper,record = store.record(source_name,external,row['url'],row['title'],row.get('abstract'),row.get('authors',[]),row.get('identifiers'),row.get('metadata'))
|
||||
if row.get('status') == 'accepted':
|
||||
edition = store.accepted(paper,record,row['venue'],row['year'],row['track'],external,row['url'])
|
||||
editions.add(edition)
|
||||
else:
|
||||
raise ValueError('This importer requires an explicit supported decision status')
|
||||
count += 1
|
||||
if not count:
|
||||
raise ValueError('Empty metadata file')
|
||||
if len(editions) != 1:
|
||||
raise ValueError('A manifest must describe exactly one conference edition and track')
|
||||
edition = next(iter(editions))
|
||||
expected = manifest.get('count')
|
||||
if expected is not None and expected != count:
|
||||
raise ValueError(f'Manifest count {expected} differs from records {count}')
|
||||
c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if expected==count else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
|
||||
c.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=? WHERE id=?",(count,count,run))
|
||||
c.commit()
|
||||
print(json.dumps({'file':str(path),'status':'imported','records':count}),flush=True)
|
||||
except BaseException as exc:
|
||||
c.rollback()
|
||||
c.execute("UPDATE ingest_runs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?",(str(exc),run))
|
||||
c.commit()
|
||||
raise
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--db',required=True)
|
||||
parser.add_argument('files',nargs='+')
|
||||
args = parser.parse_args()
|
||||
for filename in args.files:
|
||||
ingest_file(args.db,filename)
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
"""Shared identity resolution and provenance-preserving metadata storage."""
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def normalized(value):
|
||||
return ''.join(c for c in unicodedata.normalize('NFKD',value).casefold() if c.isalnum())
|
||||
|
||||
|
||||
def author_keys(authors):
|
||||
if not isinstance(authors,list):
|
||||
return set()
|
||||
result = set()
|
||||
for author in authors:
|
||||
if isinstance(author,list):
|
||||
author = ' '.join(str(x) for x in author[1:2]+author[:1])
|
||||
elif isinstance(author,dict):
|
||||
author = ' '.join(str(author.get(k,'')) for k in ('given','family'))
|
||||
key = normalized(str(author))
|
||||
if key:
|
||||
result.add(key)
|
||||
return result
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self,path):
|
||||
self.conn = sqlite3.connect(path,timeout=60)
|
||||
self.conn.executescript(Path(__file__).with_name('schema.sql').read_text())
|
||||
self.conn.execute('PRAGMA synchronous=NORMAL')
|
||||
self.conn.execute('PRAGMA cache_size=-32768')
|
||||
|
||||
def source(self,name,url):
|
||||
self.conn.execute('INSERT OR IGNORE INTO sources(name,homepage) VALUES(?,?)',(name,url))
|
||||
return self.conn.execute('SELECT id FROM sources WHERE name=?',(name,)).fetchone()[0]
|
||||
|
||||
def merge(self,target,other,reason):
|
||||
c = self.conn
|
||||
if target == other:
|
||||
return
|
||||
c.execute('INSERT INTO paper_merges(former_paper_id,surviving_paper_id,reason) VALUES(?,?,?)',(other,target,reason))
|
||||
c.execute('UPDATE identifiers SET paper_id=? WHERE paper_id=?',(target,other))
|
||||
c.execute('UPDATE source_records SET paper_id=? WHERE paper_id=?',(target,other))
|
||||
c.execute('UPDATE submissions SET paper_id=? WHERE paper_id=?',(target,other))
|
||||
c.execute('INSERT OR IGNORE INTO paper_tags SELECT ?,tag_id,source_record_id,evidence_key FROM paper_tags WHERE paper_id=?',(target,other))
|
||||
c.execute('DELETE FROM paper_tags WHERE paper_id=?',(other,))
|
||||
c.execute('UPDATE paper_merges SET surviving_paper_id=? WHERE surviving_paper_id=?',(target,other))
|
||||
# Reattach pending candidate evidence without losing colliding rows.
|
||||
candidates = c.execute('SELECT left_paper_id,right_paper_id,method,score,evidence_json,status FROM match_candidates WHERE left_paper_id=? OR right_paper_id=?',(other,other)).fetchall()
|
||||
c.execute('DELETE FROM match_candidates WHERE left_paper_id=? OR right_paper_id=?',(other,other))
|
||||
for left,right,method,score,evidence,status in candidates:
|
||||
left = target if left == other else left
|
||||
right = target if right == other else right
|
||||
if left != right:
|
||||
c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(left,right),max(left,right),method,score,evidence,status))
|
||||
c.execute('DELETE FROM papers WHERE id=?',(other,))
|
||||
|
||||
def record(self,source,external_id,url,title,abstract,authors,identifiers=None,metadata=None,published=None):
|
||||
c = self.conn
|
||||
identifiers = dict(identifiers or {})
|
||||
# External IDs are scoped to platforms; an OpenReview ID is a forum ID.
|
||||
identifiers[source] = external_id
|
||||
ids = []
|
||||
for scheme,value in identifiers.items():
|
||||
if not value:
|
||||
continue
|
||||
value = str(value).strip()
|
||||
if scheme == 'doi':
|
||||
value = re.sub(r'^https?://(?:dx\.)?doi\.org/','',value,flags=re.I).lower()
|
||||
if not re.fullmatch(r'10\.\d{4,9}/\S+',value):
|
||||
continue
|
||||
if scheme == 'arxiv':
|
||||
value = re.sub(r'v\d+$','',value)
|
||||
ids.append((scheme,value))
|
||||
matches = set()
|
||||
for scheme,value in ids:
|
||||
found = c.execute('SELECT paper_id FROM identifiers WHERE scheme=? AND value=?',(scheme,value)).fetchone()
|
||||
if found:
|
||||
matches.add(found[0])
|
||||
old = c.execute('SELECT paper_id FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
|
||||
if old:
|
||||
matches.add(old[0])
|
||||
key = normalized(title)
|
||||
title_matches = c.execute('SELECT id,authors_json,abstract FROM papers WHERE title_key=?',(key,)).fetchall()
|
||||
authors_set = author_keys(authors)
|
||||
for paper,old_authors,old_abstract in title_matches:
|
||||
overlap = authors_set & author_keys(json.loads(old_authors))
|
||||
# Cross-platform title matching needs independent author corroboration.
|
||||
same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone()
|
||||
if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())):
|
||||
matches.add(paper)
|
||||
if matches:
|
||||
paper = min(matches)
|
||||
for other in sorted(matches-{paper}):
|
||||
self.merge(paper,other,'explicit identifier or exact title corroborated by authors')
|
||||
c.execute("UPDATE papers SET abstract=CASE WHEN abstract IS NULL OR abstract='' THEN ? ELSE abstract END WHERE id=?",(abstract,paper))
|
||||
else:
|
||||
paper = c.execute('INSERT INTO papers(title,abstract,title_key,authors_json,first_published) VALUES(?,?,?,?,?)',(title,abstract,key,json.dumps(authors,ensure_ascii=False),published)).lastrowid
|
||||
for scheme,value in ids:
|
||||
c.execute('INSERT OR IGNORE INTO identifiers VALUES(?,?,?)',(scheme,value,paper))
|
||||
c.execute('INSERT INTO source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_id) DO UPDATE SET paper_id=excluded.paper_id,title=excluded.title,abstract=excluded.abstract,authors_json=excluded.authors_json,metadata_json=excluded.metadata_json,retrieved_at=CURRENT_TIMESTAMP',(source_id,external_id,paper,url,title,abstract,json.dumps(authors,ensure_ascii=False),json.dumps(metadata or {},ensure_ascii=False,default=str)))
|
||||
record = c.execute('SELECT id FROM source_records WHERE source_id=? AND external_id=?',(source_id,external_id)).fetchone()[0]
|
||||
for other,_,_ in title_matches:
|
||||
if other != paper and c.execute('SELECT 1 FROM papers WHERE id=?',(other,)).fetchone():
|
||||
c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(paper,other),max(paper,other),'exact_title_unconfirmed',None,json.dumps({'title':title}), 'pending'))
|
||||
return paper,record
|
||||
|
||||
def accepted(self,paper,record,venue,year,track,external_id,url):
|
||||
c = self.conn
|
||||
c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified'))
|
||||
venue_id = c.execute('SELECT id FROM venues WHERE acronym=?',(venue,)).fetchone()[0]
|
||||
c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)',(venue_id,int(year),track))
|
||||
edition = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?',(venue_id,int(year),track)).fetchone()[0]
|
||||
c.execute('INSERT OR IGNORE INTO submissions(paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?)',(paper,edition,record,external_id))
|
||||
submission = c.execute('SELECT id FROM submissions WHERE source_record_id=? AND edition_id=? AND source_submission_id=?',(record,edition,external_id)).fetchone()[0]
|
||||
if not c.execute("SELECT 1 FROM decisions WHERE submission_id=? AND status='accepted' AND evidence_url=?",(submission,url)).fetchone():
|
||||
c.execute("INSERT INTO decisions(submission_id,status,raw_value,evidence_url,observed_at) VALUES(?,'accepted','listed in official proceedings',?,strftime('%Y-%m-%dT%H:%M:%fZ','now'))",(submission,url))
|
||||
return edition
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Read-only command-line queries for the paper index."""
|
||||
import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--db', required=True)
|
||||
parser.add_argument('--query', help='SQLite FTS5 query over title and abstract')
|
||||
parser.add_argument('--tag', help='Filter as namespace=value, e.g. arxiv:category=cs.AI')
|
||||
parser.add_argument('--venue', help='Conference acronym')
|
||||
parser.add_argument('--year', type=int)
|
||||
parser.add_argument('--status', choices=['accepted','rejected','desk_rejected','withdrawn','under_review','unknown'])
|
||||
parser.add_argument('--limit', type=int, default=20)
|
||||
parser.add_argument('--stats', action='store_true')
|
||||
args = parser.parse_args()
|
||||
conn = sqlite3.connect(Path(args.db).resolve().as_uri()+'?mode=ro', uri=True)
|
||||
conn.row_factory = sqlite3.Row
|
||||
if args.stats:
|
||||
result = {name:conn.execute('SELECT count(*) FROM '+name).fetchone()[0] for name in ('papers','source_records','submissions','decisions','tags','match_candidates')}
|
||||
result['coverage'] = [dict(r) for r in conn.execute('SELECT status,count(*) AS sources FROM edition_sources GROUP BY status')]
|
||||
result['runs'] = [dict(r) for r in conn.execute('SELECT * FROM ingest_runs ORDER BY id DESC LIMIT 10')]
|
||||
else:
|
||||
where = []; params = []
|
||||
if args.query:
|
||||
where.append('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)'); params.append(args.query)
|
||||
if args.tag:
|
||||
namespace,value = args.tag.split('=',1)
|
||||
where.append('EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'); params.extend([namespace,value])
|
||||
if args.venue or args.year or args.status:
|
||||
clause = ['s.paper_id=p.id']
|
||||
if args.venue:
|
||||
clause.append('v.acronym=?'); params.append(args.venue)
|
||||
if args.year:
|
||||
clause.append('e.year=?'); params.append(args.year)
|
||||
if args.status:
|
||||
clause.append('EXISTS(SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=? AND NOT EXISTS(SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id AND (newer.observed_at>d.observed_at OR (newer.observed_at=d.observed_at AND newer.id>d.id))))'); params.append(args.status)
|
||||
where.append('EXISTS(SELECT 1 FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+' AND '.join(clause)+')')
|
||||
sql = 'SELECT p.* FROM papers p'+(' WHERE '+' AND '.join(where) if where else '')+' ORDER BY p.id LIMIT ?'
|
||||
params.append(max(1,min(args.limit,200)))
|
||||
result = []
|
||||
for row in conn.execute(sql,params):
|
||||
paper = dict(row)
|
||||
paper['identifiers'] = [dict(r) for r in conn.execute('SELECT scheme,value FROM identifiers WHERE paper_id=?',(row['id'],))]
|
||||
result.append(paper)
|
||||
print(json.dumps(result,ensure_ascii=False,indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
PyYAML==6.0.3
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
PRAGMA foreign_keys=ON;
|
||||
PRAGMA journal_mode=WAL;
|
||||
CREATE TABLE IF NOT EXISTS sources (
|
||||
id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, homepage TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS papers (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, abstract TEXT,
|
||||
title_key TEXT NOT NULL, authors_json TEXT NOT NULL DEFAULT '[]',
|
||||
first_published TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS papers_title_key ON papers(title_key);
|
||||
CREATE TABLE IF NOT EXISTS identifiers (
|
||||
scheme TEXT NOT NULL, value TEXT NOT NULL, paper_id INTEGER NOT NULL REFERENCES papers(id),
|
||||
PRIMARY KEY(scheme,value)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS identifiers_paper ON identifiers(paper_id);
|
||||
CREATE TABLE IF NOT EXISTS source_records (
|
||||
id INTEGER PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id),
|
||||
external_id TEXT NOT NULL, paper_id INTEGER NOT NULL REFERENCES papers(id),
|
||||
url TEXT NOT NULL, title TEXT NOT NULL, abstract TEXT, authors_json TEXT NOT NULL,
|
||||
metadata_json TEXT NOT NULL, retrieved_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(source_id,external_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS source_records_paper ON source_records(paper_id);
|
||||
CREATE TABLE IF NOT EXISTS venues (
|
||||
id INTEGER PRIMARY KEY, acronym TEXT NOT NULL UNIQUE, name TEXT NOT NULL,
|
||||
field TEXT NOT NULL, venue_type TEXT NOT NULL CHECK(venue_type IN ('conference','journal','review_platform'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS editions (
|
||||
id INTEGER PRIMARY KEY, venue_id INTEGER NOT NULL REFERENCES venues(id),
|
||||
year INTEGER NOT NULL, track TEXT NOT NULL DEFAULT 'main',
|
||||
UNIQUE(venue_id,year,track)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS edition_sources (
|
||||
edition_id INTEGER NOT NULL REFERENCES editions(id), source_id INTEGER NOT NULL REFERENCES sources(id),
|
||||
list_url TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK(status IN ('pending','running','complete','partial','blocked','unavailable','not_yet_published')),
|
||||
expected_count INTEGER, imported_count INTEGER NOT NULL DEFAULT 0,
|
||||
cursor TEXT, last_success_at TEXT, last_error TEXT,
|
||||
PRIMARY KEY(edition_id,source_id,list_url)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS submissions (
|
||||
id INTEGER PRIMARY KEY, paper_id INTEGER NOT NULL REFERENCES papers(id),
|
||||
edition_id INTEGER NOT NULL REFERENCES editions(id),
|
||||
source_record_id INTEGER NOT NULL REFERENCES source_records(id),
|
||||
source_submission_id TEXT NOT NULL, submitted_at TEXT,
|
||||
UNIQUE(source_record_id,edition_id,source_submission_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS submissions_paper ON submissions(paper_id);
|
||||
CREATE TABLE IF NOT EXISTS decisions (
|
||||
id INTEGER PRIMARY KEY, submission_id INTEGER NOT NULL REFERENCES submissions(id),
|
||||
status TEXT NOT NULL CHECK(status IN ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown')),
|
||||
raw_value TEXT NOT NULL, evidence_url TEXT NOT NULL, observed_at TEXT NOT NULL,
|
||||
decision_date TEXT, UNIQUE(submission_id,status,raw_value,evidence_url,observed_at)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
id INTEGER PRIMARY KEY, namespace TEXT NOT NULL, value TEXT NOT NULL,
|
||||
description TEXT, UNIQUE(namespace,value)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS paper_tags (
|
||||
paper_id INTEGER NOT NULL REFERENCES papers(id), tag_id INTEGER NOT NULL REFERENCES tags(id),
|
||||
source_record_id INTEGER REFERENCES source_records(id),
|
||||
evidence_key TEXT NOT NULL, PRIMARY KEY(paper_id,tag_id,evidence_key)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS match_candidates (
|
||||
left_paper_id INTEGER NOT NULL REFERENCES papers(id), right_paper_id INTEGER NOT NULL REFERENCES papers(id),
|
||||
method TEXT NOT NULL, score REAL, evidence_json TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','merged','distinct')),
|
||||
PRIMARY KEY(left_paper_id,right_paper_id,method)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS paper_merges (
|
||||
id INTEGER PRIMARY KEY, former_paper_id INTEGER NOT NULL,
|
||||
surviving_paper_id INTEGER NOT NULL REFERENCES papers(id),
|
||||
reason TEXT NOT NULL, merged_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ingest_runs (
|
||||
id INTEGER PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id),
|
||||
started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'running', records_seen INTEGER NOT NULL DEFAULT 0,
|
||||
records_written INTEGER NOT NULL DEFAULT 0, error TEXT, checkpoint_json TEXT
|
||||
);
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS papers_fts USING fts5(title,abstract,content='papers',content_rowid='id');
|
||||
CREATE TRIGGER IF NOT EXISTS papers_insert AFTER INSERT ON papers BEGIN
|
||||
INSERT INTO papers_fts(rowid,title,abstract) VALUES(new.id,new.title,new.abstract);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS papers_delete AFTER DELETE ON papers BEGIN
|
||||
INSERT INTO papers_fts(papers_fts,rowid,title,abstract) VALUES('delete',old.id,old.title,old.abstract);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS papers_update AFTER UPDATE OF title,abstract ON papers BEGIN
|
||||
INSERT INTO papers_fts(papers_fts,rowid,title,abstract) VALUES('delete',old.id,old.title,old.abstract);
|
||||
INSERT INTO papers_fts(rowid,title,abstract) VALUES(new.id,new.title,new.abstract);
|
||||
END;
|
||||
@@ -0,0 +1,57 @@
|
||||
import json
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from import_arxiv import ingest
|
||||
|
||||
|
||||
class ImportTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.root = Path(self.tmp.name)
|
||||
self.db = self.root/'papers.sqlite'
|
||||
self.archive = self.root/'metadata.zip'
|
||||
|
||||
def write(self, rows):
|
||||
with zipfile.ZipFile(self.archive,'w') as z:
|
||||
z.writestr('arxiv-metadata-oai-snapshot.json','\n'.join(json.dumps(r) for r in rows))
|
||||
|
||||
def row(self, identifier, doi=None):
|
||||
return dict(id=identifier,title='A shared title',abstract='Searchable abstract',authors_parsed=[['Doe','Jane','']],categories='cs.AI cs.LG',doi=doi,versions=[])
|
||||
|
||||
def test_identity_provenance_and_idempotency(self):
|
||||
self.write([self.row('2601.00001','10.0000/example'),self.row('2601.00002','10.0000/example')])
|
||||
ingest(self.db,self.archive)
|
||||
ingest(self.db,self.archive)
|
||||
with sqlite3.connect(self.db) as c:
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2)
|
||||
self.assertEqual(c.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'Searchable'").fetchone()[0],1)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],0)
|
||||
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
|
||||
self.assertEqual(c.execute('PRAGMA integrity_check').fetchone()[0],'ok')
|
||||
|
||||
def test_title_alone_does_not_merge(self):
|
||||
self.write([self.row('2601.00001'),self.row('2601.00002')])
|
||||
ingest(self.db,self.archive)
|
||||
with sqlite3.connect(self.db) as c:
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2)
|
||||
|
||||
def test_platform_and_custom_tags_remain_distinct(self):
|
||||
self.write([self.row('2601.00001')])
|
||||
ingest(self.db,self.archive)
|
||||
with sqlite3.connect(self.db) as c:
|
||||
paper = c.execute('SELECT id FROM papers').fetchone()[0]
|
||||
tag = c.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','cs.AI')").lastrowid
|
||||
c.execute('INSERT INTO paper_tags VALUES(?,?,NULL,?)',(paper,tag,'manual:1'))
|
||||
ingest(self.db,self.archive)
|
||||
with sqlite3.connect(self.db) as c:
|
||||
self.assertEqual(c.execute("SELECT count(*) FROM paper_tags pt JOIN tags t ON pt.tag_id=t.id WHERE t.value='cs.AI'").fetchone()[0],2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,56 @@
|
||||
import json
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from index_core import Store
|
||||
from import_records import ingest_file
|
||||
|
||||
|
||||
class StoreTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.db = Path(self.tmp.name)/'papers.sqlite'
|
||||
self.store = Store(self.db)
|
||||
self.addCleanup(self.store.conn.close)
|
||||
|
||||
def record(self,source,identifier,ids=None,authors=None):
|
||||
return self.store.record(source,identifier,'https://example.org/'+identifier,'A sufficiently long example research paper title','Example abstract',authors or ['Jane Doe','John Smith'],ids)
|
||||
|
||||
def test_bridge_identifiers_preserves_submission_and_tags(self):
|
||||
first,record = self.record('arxiv','2601.00001',{'arxiv':'2601.00001'},['First Author'])
|
||||
second,other_record = self.record('pmlr','v1/other',{'doi':'10.0000/example'},['Other Author'])
|
||||
self.store.accepted(second,other_record,'ICML',2025,'main','v1/other','https://example.org/other')
|
||||
tag = self.store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','test')").lastrowid
|
||||
self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)',(second,tag,other_record,'manual'))
|
||||
resolved,_ = self.record('openreview','forum1',{'arxiv':'2601.00001','doi':'10.0000/example'},['Anonymous'])
|
||||
self.assertEqual(resolved,first)
|
||||
c=self.store.conn
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
|
||||
self.assertEqual(c.execute('SELECT paper_id FROM submissions').fetchone()[0],first)
|
||||
self.assertEqual(c.execute('SELECT paper_id FROM paper_tags').fetchone()[0],first)
|
||||
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1)
|
||||
|
||||
def test_cross_platform_exact_title_needs_author_corroboration(self):
|
||||
paper,_=self.record('arxiv','2601.00001')
|
||||
same,_=self.record('pmlr','v1/paper')
|
||||
other,_=self.record('acl','2025.acl.1',authors=['Unrelated Person'])
|
||||
self.assertEqual(same,paper)
|
||||
self.assertNotEqual(other,paper)
|
||||
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
|
||||
|
||||
def test_manifest_count_mismatch_rolls_back_records(self):
|
||||
path=Path(self.tmp.name)/'source.jsonl'
|
||||
path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'})+'\n')
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':2}))
|
||||
with self.assertRaisesRegex(ValueError,'differs'):
|
||||
ingest_file(self.db,path)
|
||||
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],0)
|
||||
self.assertEqual(self.store.conn.execute('SELECT status FROM ingest_runs').fetchone()[0],'failed')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
unittest.main()
|
||||
+664
@@ -0,0 +1,664 @@
|
||||
{
|
||||
"scope_note": "Broad field coverage, not a claim of objective ranking relative to COLM. Main tracks, Findings, workshops, journals remain distinct. Missing editions must be reported, not silently skipped.",
|
||||
"sources": {
|
||||
"arxiv": "https://info.arxiv.org/help/bulk_data/index.html",
|
||||
"openreview": "https://docs.openreview.net/getting-started/using-the-api",
|
||||
"acl_anthology": "https://github.com/acl-org/acl-anthology/tree/master/data/xml",
|
||||
"pmlr": "https://proceedings.mlr.press/",
|
||||
"neurips": "https://proceedings.neurips.cc/",
|
||||
"cvf": "https://openaccess.thecvf.com/",
|
||||
"ecva": "https://www.ecva.net/papers.php",
|
||||
"aaai": "https://ojs.aaai.org/index.php/AAAI/issue/archive",
|
||||
"ijcai": "https://www.ijcai.org/all_proceedings/",
|
||||
"dblp": "https://dblp.org/db/conf/",
|
||||
"colm": "https://proceedings.colm.cc/"
|
||||
},
|
||||
"venues": [
|
||||
{
|
||||
"acronym": "NeurIPS",
|
||||
"field": "machine_learning",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"openreview",
|
||||
"pmlr",
|
||||
"neurips"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ICML",
|
||||
"field": "machine_learning",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"openreview",
|
||||
"pmlr",
|
||||
"neurips"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ICLR",
|
||||
"field": "machine_learning",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"openreview",
|
||||
"pmlr",
|
||||
"neurips"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "COLM",
|
||||
"field": "machine_learning",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"openreview",
|
||||
"pmlr",
|
||||
"neurips"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "COLT",
|
||||
"field": "machine_learning",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"openreview",
|
||||
"pmlr",
|
||||
"neurips"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "AISTATS",
|
||||
"field": "machine_learning",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"openreview",
|
||||
"pmlr",
|
||||
"neurips"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "UAI",
|
||||
"field": "machine_learning",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"openreview",
|
||||
"pmlr",
|
||||
"neurips"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "CLeaR",
|
||||
"field": "machine_learning",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"openreview",
|
||||
"pmlr",
|
||||
"neurips"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ACL",
|
||||
"field": "language",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"acl_anthology",
|
||||
"openreview"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "EMNLP",
|
||||
"field": "language",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"acl_anthology",
|
||||
"openreview"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "NAACL",
|
||||
"field": "language",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"acl_anthology",
|
||||
"openreview"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "EACL",
|
||||
"field": "language",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"acl_anthology",
|
||||
"openreview"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "COLING",
|
||||
"field": "language",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"acl_anthology",
|
||||
"openreview"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "CoNLL",
|
||||
"field": "language",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"acl_anthology",
|
||||
"openreview"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "CVPR",
|
||||
"field": "vision",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"cvf",
|
||||
"ecva",
|
||||
"openreview"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ICCV",
|
||||
"field": "vision",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"cvf",
|
||||
"ecva",
|
||||
"openreview"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ECCV",
|
||||
"field": "vision",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"cvf",
|
||||
"ecva",
|
||||
"openreview"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "AAAI",
|
||||
"field": "general_ai",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"aaai",
|
||||
"ijcai",
|
||||
"dblp"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "IJCAI",
|
||||
"field": "general_ai",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"aaai",
|
||||
"ijcai",
|
||||
"dblp"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ECAI",
|
||||
"field": "general_ai",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"aaai",
|
||||
"ijcai",
|
||||
"dblp"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "KDD",
|
||||
"field": "retrieval_data",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"publisher"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "SIGIR",
|
||||
"field": "retrieval_data",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"publisher"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "WWW",
|
||||
"field": "retrieval_data",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"publisher"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "WSDM",
|
||||
"field": "retrieval_data",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"publisher"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "RecSys",
|
||||
"field": "retrieval_data",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"publisher"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "CIKM",
|
||||
"field": "retrieval_data",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"publisher"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ICDM",
|
||||
"field": "retrieval_data",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"publisher"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "SDM",
|
||||
"field": "retrieval_data",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"publisher"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ECML-PKDD",
|
||||
"field": "retrieval_data",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"publisher"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ICRA",
|
||||
"field": "robotics",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"pmlr",
|
||||
"rss"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "IROS",
|
||||
"field": "robotics",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"pmlr",
|
||||
"rss"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "RSS",
|
||||
"field": "robotics",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"pmlr",
|
||||
"rss"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "CoRL",
|
||||
"field": "robotics",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"pmlr",
|
||||
"rss"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ICAPS",
|
||||
"field": "reasoning_agents",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"proceedings"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "KR",
|
||||
"field": "reasoning_agents",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"proceedings"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "AAMAS",
|
||||
"field": "reasoning_agents",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"proceedings"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ICASSP",
|
||||
"field": "speech_audio",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"isca",
|
||||
"ismir"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "INTERSPEECH",
|
||||
"field": "speech_audio",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"isca",
|
||||
"ismir"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "ISMIR",
|
||||
"field": "speech_audio",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"isca",
|
||||
"ismir"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "MICCAI",
|
||||
"field": "specialist",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"publisher"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
},
|
||||
{
|
||||
"acronym": "FAccT",
|
||||
"field": "specialist",
|
||||
"venue_type": "conference",
|
||||
"discovery_sources": [
|
||||
"dblp",
|
||||
"publisher"
|
||||
],
|
||||
"scope": "candidate_pending_scope_review",
|
||||
"years": "all_publicly_available",
|
||||
"tracks": [
|
||||
"main"
|
||||
],
|
||||
"coverage": "pending"
|
||||
}
|
||||
],
|
||||
"additional_tracks": [
|
||||
"Findings",
|
||||
"workshop"
|
||||
],
|
||||
"additional_nonconference_venues": [
|
||||
"TMLR",
|
||||
"JMLR",
|
||||
"TACL",
|
||||
"ACL ARR"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user