Add provenance-aware paper metadata storage and PMLR collection
Metadata integrity checks / metadata (push) Successful in 1s

This commit is contained in:
2026-11-06 03:01:43 +08:00
parent 1922d45896
commit 4d018a14c3
12 changed files with 1365 additions and 4 deletions
+35
View File
@@ -0,0 +1,35 @@
name: Metadata integrity checks
on:
push:
branches: [main, 'codex/**']
jobs:
metadata:
runs-on: ces-control
env:
ARCHIVE_SHA: ${{ github.sha }}
PYTHONDONTWRITEBYTECODE: '1'
steps:
- name: Verify metadata storage and query commands
run: |
python3 -B - <<'PY'
import io, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
sha = os.environ['ARCHIVE_SHA']
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
raise SystemExit('Invalid source commit')
# Public repository archive: no owner credentials or runner SSH keys.
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
with urllib.request.urlopen(url, timeout=60) as response:
archive = response.read(16*1024*1024+1)
if len(archive)>16*1024*1024:
raise SystemExit('Unexpectedly large source archive')
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
root = pathlib.Path(tmp)
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
tar.extractall(root, filter='data')
candidates = list(root.glob('*/test_import_arxiv.py'))
if len(candidates)!=1:
raise SystemExit('Unexpected repository archive layout')
src = candidates[0].parent
subprocess.run(['python3','-B','-m','unittest','discover','-v'],cwd=src,check=True)
subprocess.run(['python3','-B','query.py','--help'],cwd=src,check=True)
PY
+18 -4
View File
@@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工
目标包括:完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。
**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、DOI/标识合并和只读查询脚本。服务器上进行过合成样本验证;全量快照下载与历史导入尚未完成。会议适配器、跨平台模糊匹配复核流程、完整覆盖率核验与 CI 仍待实现。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上六项合成回归检查通过;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 全量快照下载与历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
## 数据范围与来源
@@ -41,16 +41,16 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工
| `edition_sources` | 每届每个来源的覆盖状态、预计/已导入数量、错误与断点 |
| `submissions` / `decisions` | 多次投稿及带证据的决定历史,支持改投、撤稿和决定变更 |
| `tags` / `paper_tags` | 平台标签与人工标签,使用命名空间隔离 |
| `match_candidates` | 证据不充分的潜在重复记录及人工复核状态 |
| `match_candidates` / `paper_merges` | 潜在重复复核状态、确定合并审计 |
| `ingest_runs` | 导入批次、状态、错误和恢复位置 |
决定状态为 `accepted`、`rejected`、`desk_rejected`、`withdrawn`、`under_review`、`unknown`。一篇论文可以对应多次投稿和多个决定,不能用单一“最终会议”字段覆盖历史。查询中按观察时间选择最近状态;后续分析需要同时检查来源覆盖率。
当前确定性去重使用 arXiv ID 和单个 DOI。保留各平台源记录,避免合并后丢失出处。规范化标题只建检索索引,尚未实现模糊匹配流程;**现阶段不保证跨平台去重已经完成**。不同平台标签不自动等同,自定义标签应使用 `custom:*` 命名空间。
当前确定性去重使用 arXiv ID、单个 DOI 和 OpenReview forum ID;跨平台同标题还需至少两位作者吻合,或作者吻合且摘要相同,才自动合并。保留各平台源记录,避免合并后丢失出处。规范化标题只建检索索引,尚未实现模糊匹配流程;**现阶段不保证跨平台去重已经完成**。不同平台标签不自动等同,自定义标签应使用 `custom:*` 命名空间。
## 运行与查询
依赖 Python 3 标准库和带 FTS5 的 SQLite,目前无第三方 Python 依赖,无项目构建步骤。服务器已检查为 Python 3.14.4。所有运行数据放在仓库外,不提交数据库、下载包或访问凭据。
基础导入和查询依赖 Python 3 标准库和带 FTS5 的 SQLite;PMLR 采集另需 `requirements.txt` 中锁定的 PyYAML 6.0.3。无项目构建步骤。服务器已检查为 Python 3.14.4。所有运行数据放在仓库外,不提交数据库、下载包或访问凭据。
服务器暂存目录:`/data/paper-index/`,快照目录:`/data/paper-index/raw/`,运行代码目录:`/data/paper-index/code/`。这些路径不是对外访问的 API。
@@ -82,6 +82,20 @@ python3 -B query.py --db /data/paper-index/papers.sqlite --venue COLM --year 202
查询连接使用 SQLite 只读模式,结果输出 JSON。当前没有公共 Web 页面或网络查询服务。
## 官方会议元数据采集
```sh
# 单卷示例:ICML 2025,读取官方 gh-pages 元数据归档,不下载 PDF
python3 -B collect_pmlr.py --cache /data/paper-index/raw/http-cache \
--out /data/paper-index/collected --volume 267
# 待主库完成 arXiv 导入、且没有其他写入进程后,合并会议记录
python3 -B import_records.py --db /data/paper-index/papers.sqlite \
/data/paper-index/collected/pmlr-v267.jsonl
```
采集输出 JSONL 和对应 manifest,只有显式官方录用记录才产生 accepted 决定。清单数目不一致时整个文件导入回滚,记录失败批次,不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。
## 长期维护与更新
1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发,遵守来源限流、重试与人机验证要求。
+123
View File
@@ -0,0 +1,123 @@
"""Collect official PMLR metadata, never the PDFs linked by the metadata."""
import argparse
import hashlib
import io
import json
import re
import tarfile
import time
import urllib.error
import urllib.request
from pathlib import Path
import yaml
def fetch(url,cache,limit=64*1024**2):
cache = Path(cache)
cache.mkdir(parents=True,exist_ok=True)
path = cache/hashlib.sha256(url.encode()).hexdigest()
if path.exists():
return path.read_bytes()
for attempt in range(3):
try:
request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'})
with urllib.request.urlopen(request,timeout=90) as response:
if response.status != 200:
raise RuntimeError('Expected complete HTTP 200 response')
data = response.read(limit+1)
if len(data)>limit:
raise RuntimeError('Metadata response exceeds byte limit')
part = path.with_suffix('.part')
part.write_bytes(data)
part.replace(path)
path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
time.sleep(0.5)
return data
except urllib.error.HTTPError as exc:
if exc.code not in (429,500,502,503,504) or attempt==2:
raise
time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1')))))
def collect_volume(volume,cache,out,allowed=None):
config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml'
config = yaml.safe_load(fetch(config_url,cache))
venue = str(config.get('shortname','')).upper()
if allowed and venue not in allowed:
return None
year = int(config['year'])
title = str(config.get('booktitle',''))
track = 'workshop' if 'workshop' in title.lower() else 'main'
archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
path = Path(out)/f'pmlr-v{volume}.jsonl'
path.parent.mkdir(parents=True,exist_ok=True)
part = path.with_suffix('.part')
count = 0
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar,part.open('w') as output:
for member in tar:
if not member.isfile() or '/_posts/' not in member.name or not member.name.endswith(('.md','.markdown','.html')):
continue
if member.size>8*1024**2:
raise RuntimeError('Unexpectedly large metadata member')
content = tar.extractfile(member).read().decode('utf-8-sig')
parts = content.split('---',2)
if len(parts)<3 or parts[0].strip():
raise RuntimeError('Missing YAML frontmatter: '+member.name)
row = yaml.safe_load(parts[1])
if row.get('layout') not in ('inproceedings','article') or not row.get('title'):
continue
external = f"v{volume}/{row.get('id') or Path(member.name).stem}"
ids = {}
for key in ('doi','openreview','arxiv'):
value = row.get(key)
if value:
value = str(value)
if key=='arxiv':
value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
if key=='openreview':
value = value.split('id=')[-1].split('&')[0]
ids[key] = value
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n')
count += 1
if count==0:
part.unlink(missing_ok=True)
raise RuntimeError('No papers parsed; refusing an empty successful collection')
part.replace(path)
result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'}
path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
print(json.dumps(result),flush=True)
return result
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--cache',required=True)
parser.add_argument('--out',required=True)
parser.add_argument('--volume',type=int,action='append')
parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json')))
args = parser.parse_args()
allowed = {v['acronym'].upper() for v in json.loads(Path(args.registry).read_text())['venues']}
volumes = args.volume
if not volumes:
index = fetch('https://proceedings.mlr.press/',args.cache).decode()
volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press)?/v(\d+)/?["\']',index)))
if not volumes:
raise RuntimeError('PMLR index yielded no volumes')
errors = []
for volume in volumes:
try:
collect_volume(volume,args.cache,args.out,None if args.volume else allowed)
except Exception as exc:
error = {'volume':volume,'status':'blocked','error':str(exc)}
errors.append(error)
print(json.dumps(error),flush=True)
Path(args.out).mkdir(parents=True,exist_ok=True)
(Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2))
if errors:
raise SystemExit(1)
if __name__=='__main__':
main()
+78
View File
@@ -0,0 +1,78 @@
"""Stream the official metadata ZIP into SQLite. No PDFs, source files or builds."""
import argparse
import json
import time
import shutil
import zipfile
from pathlib import Path
from index_core import Store
def ingest(db, archive):
store = Store(db)
conn = store.conn
conn.execute("INSERT OR IGNORE INTO sources(name,homepage) VALUES('arxiv','https://arxiv.org')")
source = conn.execute("SELECT id FROM sources WHERE name='arxiv'").fetchone()[0]
previous = conn.execute('SELECT id,checkpoint_json FROM ingest_runs WHERE source_id=? AND status IN (\'running\',\'failed\') ORDER BY id DESC LIMIT 1', (source,)).fetchone()
fingerprint = {'path':str(Path(archive).resolve()), 'bytes':Path(archive).stat().st_size, 'mtime_ns':Path(archive).stat().st_mtime_ns}
start = 0
if previous:
checkpoint = json.loads(previous[1] or '{}')
if checkpoint.get('archive') == fingerprint:
run = previous[0]
start = checkpoint.get('line', 0)
conn.execute("UPDATE ingest_runs SET status='running',error=NULL WHERE id=?", (run,))
else:
raise RuntimeError('Pending run belongs to a different archive; resolve it explicitly')
else:
run = conn.execute('INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)', (source,json.dumps({'archive':fingerprint,'line':0}))).lastrowid
conn.commit()
line_number = start
last = time.monotonic()
try:
with zipfile.ZipFile(archive) as zipped:
# Reading the member to EOF also checks its CRC.
with zipped.open('arxiv-metadata-oai-snapshot.json') as stream:
for line_number, line in enumerate(stream, 1):
if line_number <= start:
continue
row = json.loads(line)
external_id = row['id']
title = ' '.join(row['title'].split())
abstract = row.get('abstract', '').strip()
authors = row.get('authors_parsed') or row.get('authors') or []
metadata = {k:v for k,v in row.items() if k not in ('title','abstract','authors','authors_parsed')}
paper,record = store.record('arxiv',external_id,'https://arxiv.org/abs/'+external_id,title,abstract,authors,{'arxiv':external_id,'doi':row.get('doi')},metadata,(row.get('versions') or [{}])[0].get('created'))
conn.execute("DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key='arxiv:categories'", (record,))
for category in (row.get('categories') or '').split():
conn.execute("INSERT OR IGNORE INTO tags(namespace,value) VALUES('arxiv:category',?)",(category,))
tag = conn.execute("SELECT id FROM tags WHERE namespace='arxiv:category' AND value=?",(category,)).fetchone()[0]
conn.execute('INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,'arxiv:categories'))
if line_number % 500 == 0:
if shutil.disk_usage(Path(db).parent).free < 4*1024**3:
raise RuntimeError('Stopped before exhausting disk: less than 4 GiB free')
conn.execute('UPDATE ingest_runs SET records_seen=?,records_written=?,checkpoint_json=? WHERE id=?',(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
conn.commit()
if time.monotonic()-last >= 20:
print(json.dumps({'run':run,'rows':line_number}),flush=True)
last = time.monotonic()
conn.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
conn.commit()
print(json.dumps({'status':'complete','rows':line_number}),flush=True)
except BaseException as exc:
conn.rollback()
conn.execute("UPDATE ingest_runs SET status='failed',error=? WHERE id=?",(str(exc),run))
conn.commit()
raise
finally:
conn.close()
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--db', required=True)
parser.add_argument('--archive', required=True)
args = parser.parse_args()
ingest(args.db,args.archive)
+69
View File
@@ -0,0 +1,69 @@
"""Import normalized conference JSONL files with coverage evidence."""
import argparse
import json
import time
from pathlib import Path
from index_core import Store
def ingest_file(db,path):
path = Path(path)
manifest_path = path.with_suffix('.manifest.json')
manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
store = Store(db)
c = store.conn
source_name = manifest.get('source')
if not source_name:
raise ValueError('A source manifest is required')
source_id = store.source(source_name,manifest['list_url'])
run = c.execute('INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)',(source_id,json.dumps({'file':str(path)}))).lastrowid
c.commit()
count = 0
seen = set()
editions = set()
try:
with path.open() as stream:
for line in stream:
row = json.loads(line)
if row['source'] != source_name:
raise ValueError('Source differs from manifest')
external = row['external_id']
if external in seen:
raise ValueError('Duplicate external ID in one collected file: '+external)
seen.add(external)
paper,record = store.record(source_name,external,row['url'],row['title'],row.get('abstract'),row.get('authors',[]),row.get('identifiers'),row.get('metadata'))
if row.get('status') == 'accepted':
edition = store.accepted(paper,record,row['venue'],row['year'],row['track'],external,row['url'])
editions.add(edition)
else:
raise ValueError('This importer requires an explicit supported decision status')
count += 1
if not count:
raise ValueError('Empty metadata file')
if len(editions) != 1:
raise ValueError('A manifest must describe exactly one conference edition and track')
edition = next(iter(editions))
expected = manifest.get('count')
if expected is not None and expected != count:
raise ValueError(f'Manifest count {expected} differs from records {count}')
c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if expected==count else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
c.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=? WHERE id=?",(count,count,run))
c.commit()
print(json.dumps({'file':str(path),'status':'imported','records':count}),flush=True)
except BaseException as exc:
c.rollback()
c.execute("UPDATE ingest_runs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?",(str(exc),run))
c.commit()
raise
finally:
c.close()
if __name__=='__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--db',required=True)
parser.add_argument('files',nargs='+')
args = parser.parse_args()
for filename in args.files:
ingest_file(args.db,filename)
+120
View File
@@ -0,0 +1,120 @@
"""Shared identity resolution and provenance-preserving metadata storage."""
import json
import re
import sqlite3
import unicodedata
from pathlib import Path
def normalized(value):
return ''.join(c for c in unicodedata.normalize('NFKD',value).casefold() if c.isalnum())
def author_keys(authors):
if not isinstance(authors,list):
return set()
result = set()
for author in authors:
if isinstance(author,list):
author = ' '.join(str(x) for x in author[1:2]+author[:1])
elif isinstance(author,dict):
author = ' '.join(str(author.get(k,'')) for k in ('given','family'))
key = normalized(str(author))
if key:
result.add(key)
return result
class Store:
def __init__(self,path):
self.conn = sqlite3.connect(path,timeout=60)
self.conn.executescript(Path(__file__).with_name('schema.sql').read_text())
self.conn.execute('PRAGMA synchronous=NORMAL')
self.conn.execute('PRAGMA cache_size=-32768')
def source(self,name,url):
self.conn.execute('INSERT OR IGNORE INTO sources(name,homepage) VALUES(?,?)',(name,url))
return self.conn.execute('SELECT id FROM sources WHERE name=?',(name,)).fetchone()[0]
def merge(self,target,other,reason):
c = self.conn
if target == other:
return
c.execute('INSERT INTO paper_merges(former_paper_id,surviving_paper_id,reason) VALUES(?,?,?)',(other,target,reason))
c.execute('UPDATE identifiers SET paper_id=? WHERE paper_id=?',(target,other))
c.execute('UPDATE source_records SET paper_id=? WHERE paper_id=?',(target,other))
c.execute('UPDATE submissions SET paper_id=? WHERE paper_id=?',(target,other))
c.execute('INSERT OR IGNORE INTO paper_tags SELECT ?,tag_id,source_record_id,evidence_key FROM paper_tags WHERE paper_id=?',(target,other))
c.execute('DELETE FROM paper_tags WHERE paper_id=?',(other,))
c.execute('UPDATE paper_merges SET surviving_paper_id=? WHERE surviving_paper_id=?',(target,other))
# Reattach pending candidate evidence without losing colliding rows.
candidates = c.execute('SELECT left_paper_id,right_paper_id,method,score,evidence_json,status FROM match_candidates WHERE left_paper_id=? OR right_paper_id=?',(other,other)).fetchall()
c.execute('DELETE FROM match_candidates WHERE left_paper_id=? OR right_paper_id=?',(other,other))
for left,right,method,score,evidence,status in candidates:
left = target if left == other else left
right = target if right == other else right
if left != right:
c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(left,right),max(left,right),method,score,evidence,status))
c.execute('DELETE FROM papers WHERE id=?',(other,))
def record(self,source,external_id,url,title,abstract,authors,identifiers=None,metadata=None,published=None):
c = self.conn
identifiers = dict(identifiers or {})
# External IDs are scoped to platforms; an OpenReview ID is a forum ID.
identifiers[source] = external_id
ids = []
for scheme,value in identifiers.items():
if not value:
continue
value = str(value).strip()
if scheme == 'doi':
value = re.sub(r'^https?://(?:dx\.)?doi\.org/','',value,flags=re.I).lower()
if not re.fullmatch(r'10\.\d{4,9}/\S+',value):
continue
if scheme == 'arxiv':
value = re.sub(r'v\d+$','',value)
ids.append((scheme,value))
matches = set()
for scheme,value in ids:
found = c.execute('SELECT paper_id FROM identifiers WHERE scheme=? AND value=?',(scheme,value)).fetchone()
if found:
matches.add(found[0])
old = c.execute('SELECT paper_id FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
if old:
matches.add(old[0])
key = normalized(title)
title_matches = c.execute('SELECT id,authors_json,abstract FROM papers WHERE title_key=?',(key,)).fetchall()
authors_set = author_keys(authors)
for paper,old_authors,old_abstract in title_matches:
overlap = authors_set & author_keys(json.loads(old_authors))
# Cross-platform title matching needs independent author corroboration.
same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone()
if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())):
matches.add(paper)
if matches:
paper = min(matches)
for other in sorted(matches-{paper}):
self.merge(paper,other,'explicit identifier or exact title corroborated by authors')
c.execute("UPDATE papers SET abstract=CASE WHEN abstract IS NULL OR abstract='' THEN ? ELSE abstract END WHERE id=?",(abstract,paper))
else:
paper = c.execute('INSERT INTO papers(title,abstract,title_key,authors_json,first_published) VALUES(?,?,?,?,?)',(title,abstract,key,json.dumps(authors,ensure_ascii=False),published)).lastrowid
for scheme,value in ids:
c.execute('INSERT OR IGNORE INTO identifiers VALUES(?,?,?)',(scheme,value,paper))
c.execute('INSERT INTO source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_id) DO UPDATE SET paper_id=excluded.paper_id,title=excluded.title,abstract=excluded.abstract,authors_json=excluded.authors_json,metadata_json=excluded.metadata_json,retrieved_at=CURRENT_TIMESTAMP',(source_id,external_id,paper,url,title,abstract,json.dumps(authors,ensure_ascii=False),json.dumps(metadata or {},ensure_ascii=False,default=str)))
record = c.execute('SELECT id FROM source_records WHERE source_id=? AND external_id=?',(source_id,external_id)).fetchone()[0]
for other,_,_ in title_matches:
if other != paper and c.execute('SELECT 1 FROM papers WHERE id=?',(other,)).fetchone():
c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(paper,other),max(paper,other),'exact_title_unconfirmed',None,json.dumps({'title':title}), 'pending'))
return paper,record
def accepted(self,paper,record,venue,year,track,external_id,url):
c = self.conn
c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified'))
venue_id = c.execute('SELECT id FROM venues WHERE acronym=?',(venue,)).fetchone()[0]
c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)',(venue_id,int(year),track))
edition = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?',(venue_id,int(year),track)).fetchone()[0]
c.execute('INSERT OR IGNORE INTO submissions(paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?)',(paper,edition,record,external_id))
submission = c.execute('SELECT id FROM submissions WHERE source_record_id=? AND edition_id=? AND source_submission_id=?',(record,edition,external_id)).fetchone()[0]
if not c.execute("SELECT 1 FROM decisions WHERE submission_id=? AND status='accepted' AND evidence_url=?",(submission,url)).fetchone():
c.execute("INSERT INTO decisions(submission_id,status,raw_value,evidence_url,observed_at) VALUES(?,'accepted','listed in official proceedings',?,strftime('%Y-%m-%dT%H:%M:%fZ','now'))",(submission,url))
return edition
+52
View File
@@ -0,0 +1,52 @@
"""Read-only command-line queries for the paper index."""
import argparse
import json
import sqlite3
from pathlib import Path
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--db', required=True)
parser.add_argument('--query', help='SQLite FTS5 query over title and abstract')
parser.add_argument('--tag', help='Filter as namespace=value, e.g. arxiv:category=cs.AI')
parser.add_argument('--venue', help='Conference acronym')
parser.add_argument('--year', type=int)
parser.add_argument('--status', choices=['accepted','rejected','desk_rejected','withdrawn','under_review','unknown'])
parser.add_argument('--limit', type=int, default=20)
parser.add_argument('--stats', action='store_true')
args = parser.parse_args()
conn = sqlite3.connect(Path(args.db).resolve().as_uri()+'?mode=ro', uri=True)
conn.row_factory = sqlite3.Row
if args.stats:
result = {name:conn.execute('SELECT count(*) FROM '+name).fetchone()[0] for name in ('papers','source_records','submissions','decisions','tags','match_candidates')}
result['coverage'] = [dict(r) for r in conn.execute('SELECT status,count(*) AS sources FROM edition_sources GROUP BY status')]
result['runs'] = [dict(r) for r in conn.execute('SELECT * FROM ingest_runs ORDER BY id DESC LIMIT 10')]
else:
where = []; params = []
if args.query:
where.append('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)'); params.append(args.query)
if args.tag:
namespace,value = args.tag.split('=',1)
where.append('EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'); params.extend([namespace,value])
if args.venue or args.year or args.status:
clause = ['s.paper_id=p.id']
if args.venue:
clause.append('v.acronym=?'); params.append(args.venue)
if args.year:
clause.append('e.year=?'); params.append(args.year)
if args.status:
clause.append('EXISTS(SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=? AND NOT EXISTS(SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id AND (newer.observed_at>d.observed_at OR (newer.observed_at=d.observed_at AND newer.id>d.id))))'); params.append(args.status)
where.append('EXISTS(SELECT 1 FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+' AND '.join(clause)+')')
sql = 'SELECT p.* FROM papers p'+(' WHERE '+' AND '.join(where) if where else '')+' ORDER BY p.id LIMIT ?'
params.append(max(1,min(args.limit,200)))
result = []
for row in conn.execute(sql,params):
paper = dict(row)
paper['identifiers'] = [dict(r) for r in conn.execute('SELECT scheme,value FROM identifiers WHERE paper_id=?',(row['id'],))]
result.append(paper)
print(json.dumps(result,ensure_ascii=False,indent=2))
if __name__ == '__main__':
main()
+1
View File
@@ -0,0 +1 @@
PyYAML==6.0.3
+92
View File
@@ -0,0 +1,92 @@
PRAGMA foreign_keys=ON;
PRAGMA journal_mode=WAL;
CREATE TABLE IF NOT EXISTS sources (
id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, homepage TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS papers (
id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, abstract TEXT,
title_key TEXT NOT NULL, authors_json TEXT NOT NULL DEFAULT '[]',
first_published TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS papers_title_key ON papers(title_key);
CREATE TABLE IF NOT EXISTS identifiers (
scheme TEXT NOT NULL, value TEXT NOT NULL, paper_id INTEGER NOT NULL REFERENCES papers(id),
PRIMARY KEY(scheme,value)
);
CREATE INDEX IF NOT EXISTS identifiers_paper ON identifiers(paper_id);
CREATE TABLE IF NOT EXISTS source_records (
id INTEGER PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id),
external_id TEXT NOT NULL, paper_id INTEGER NOT NULL REFERENCES papers(id),
url TEXT NOT NULL, title TEXT NOT NULL, abstract TEXT, authors_json TEXT NOT NULL,
metadata_json TEXT NOT NULL, retrieved_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE(source_id,external_id)
);
CREATE INDEX IF NOT EXISTS source_records_paper ON source_records(paper_id);
CREATE TABLE IF NOT EXISTS venues (
id INTEGER PRIMARY KEY, acronym TEXT NOT NULL UNIQUE, name TEXT NOT NULL,
field TEXT NOT NULL, venue_type TEXT NOT NULL CHECK(venue_type IN ('conference','journal','review_platform'))
);
CREATE TABLE IF NOT EXISTS editions (
id INTEGER PRIMARY KEY, venue_id INTEGER NOT NULL REFERENCES venues(id),
year INTEGER NOT NULL, track TEXT NOT NULL DEFAULT 'main',
UNIQUE(venue_id,year,track)
);
CREATE TABLE IF NOT EXISTS edition_sources (
edition_id INTEGER NOT NULL REFERENCES editions(id), source_id INTEGER NOT NULL REFERENCES sources(id),
list_url TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending'
CHECK(status IN ('pending','running','complete','partial','blocked','unavailable','not_yet_published')),
expected_count INTEGER, imported_count INTEGER NOT NULL DEFAULT 0,
cursor TEXT, last_success_at TEXT, last_error TEXT,
PRIMARY KEY(edition_id,source_id,list_url)
);
CREATE TABLE IF NOT EXISTS submissions (
id INTEGER PRIMARY KEY, paper_id INTEGER NOT NULL REFERENCES papers(id),
edition_id INTEGER NOT NULL REFERENCES editions(id),
source_record_id INTEGER NOT NULL REFERENCES source_records(id),
source_submission_id TEXT NOT NULL, submitted_at TEXT,
UNIQUE(source_record_id,edition_id,source_submission_id)
);
CREATE INDEX IF NOT EXISTS submissions_paper ON submissions(paper_id);
CREATE TABLE IF NOT EXISTS decisions (
id INTEGER PRIMARY KEY, submission_id INTEGER NOT NULL REFERENCES submissions(id),
status TEXT NOT NULL CHECK(status IN ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown')),
raw_value TEXT NOT NULL, evidence_url TEXT NOT NULL, observed_at TEXT NOT NULL,
decision_date TEXT, UNIQUE(submission_id,status,raw_value,evidence_url,observed_at)
);
CREATE TABLE IF NOT EXISTS tags (
id INTEGER PRIMARY KEY, namespace TEXT NOT NULL, value TEXT NOT NULL,
description TEXT, UNIQUE(namespace,value)
);
CREATE TABLE IF NOT EXISTS paper_tags (
paper_id INTEGER NOT NULL REFERENCES papers(id), tag_id INTEGER NOT NULL REFERENCES tags(id),
source_record_id INTEGER REFERENCES source_records(id),
evidence_key TEXT NOT NULL, PRIMARY KEY(paper_id,tag_id,evidence_key)
);
CREATE TABLE IF NOT EXISTS match_candidates (
left_paper_id INTEGER NOT NULL REFERENCES papers(id), right_paper_id INTEGER NOT NULL REFERENCES papers(id),
method TEXT NOT NULL, score REAL, evidence_json TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','merged','distinct')),
PRIMARY KEY(left_paper_id,right_paper_id,method)
);
CREATE TABLE IF NOT EXISTS paper_merges (
id INTEGER PRIMARY KEY, former_paper_id INTEGER NOT NULL,
surviving_paper_id INTEGER NOT NULL REFERENCES papers(id),
reason TEXT NOT NULL, merged_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS ingest_runs (
id INTEGER PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id),
started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT,
status TEXT NOT NULL DEFAULT 'running', records_seen INTEGER NOT NULL DEFAULT 0,
records_written INTEGER NOT NULL DEFAULT 0, error TEXT, checkpoint_json TEXT
);
CREATE VIRTUAL TABLE IF NOT EXISTS papers_fts USING fts5(title,abstract,content='papers',content_rowid='id');
CREATE TRIGGER IF NOT EXISTS papers_insert AFTER INSERT ON papers BEGIN
INSERT INTO papers_fts(rowid,title,abstract) VALUES(new.id,new.title,new.abstract);
END;
CREATE TRIGGER IF NOT EXISTS papers_delete AFTER DELETE ON papers BEGIN
INSERT INTO papers_fts(papers_fts,rowid,title,abstract) VALUES('delete',old.id,old.title,old.abstract);
END;
CREATE TRIGGER IF NOT EXISTS papers_update AFTER UPDATE OF title,abstract ON papers BEGIN
INSERT INTO papers_fts(papers_fts,rowid,title,abstract) VALUES('delete',old.id,old.title,old.abstract);
INSERT INTO papers_fts(rowid,title,abstract) VALUES(new.id,new.title,new.abstract);
END;
+57
View File
@@ -0,0 +1,57 @@
import json
import sqlite3
import tempfile
import unittest
import zipfile
from pathlib import Path
from import_arxiv import ingest
class ImportTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.db = self.root/'papers.sqlite'
self.archive = self.root/'metadata.zip'
def write(self, rows):
with zipfile.ZipFile(self.archive,'w') as z:
z.writestr('arxiv-metadata-oai-snapshot.json','\n'.join(json.dumps(r) for r in rows))
def row(self, identifier, doi=None):
return dict(id=identifier,title='A shared title',abstract='Searchable abstract',authors_parsed=[['Doe','Jane','']],categories='cs.AI cs.LG',doi=doi,versions=[])
def test_identity_provenance_and_idempotency(self):
self.write([self.row('2601.00001','10.0000/example'),self.row('2601.00002','10.0000/example')])
ingest(self.db,self.archive)
ingest(self.db,self.archive)
with sqlite3.connect(self.db) as c:
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2)
self.assertEqual(c.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'Searchable'").fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],0)
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
self.assertEqual(c.execute('PRAGMA integrity_check').fetchone()[0],'ok')
def test_title_alone_does_not_merge(self):
self.write([self.row('2601.00001'),self.row('2601.00002')])
ingest(self.db,self.archive)
with sqlite3.connect(self.db) as c:
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2)
def test_platform_and_custom_tags_remain_distinct(self):
self.write([self.row('2601.00001')])
ingest(self.db,self.archive)
with sqlite3.connect(self.db) as c:
paper = c.execute('SELECT id FROM papers').fetchone()[0]
tag = c.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','cs.AI')").lastrowid
c.execute('INSERT INTO paper_tags VALUES(?,?,NULL,?)',(paper,tag,'manual:1'))
ingest(self.db,self.archive)
with sqlite3.connect(self.db) as c:
self.assertEqual(c.execute("SELECT count(*) FROM paper_tags pt JOIN tags t ON pt.tag_id=t.id WHERE t.value='cs.AI'").fetchone()[0],2)
if __name__ == '__main__':
unittest.main()
+56
View File
@@ -0,0 +1,56 @@
import json
import sqlite3
import tempfile
import unittest
from pathlib import Path
from index_core import Store
from import_records import ingest_file
class StoreTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.db = Path(self.tmp.name)/'papers.sqlite'
self.store = Store(self.db)
self.addCleanup(self.store.conn.close)
def record(self,source,identifier,ids=None,authors=None):
return self.store.record(source,identifier,'https://example.org/'+identifier,'A sufficiently long example research paper title','Example abstract',authors or ['Jane Doe','John Smith'],ids)
def test_bridge_identifiers_preserves_submission_and_tags(self):
first,record = self.record('arxiv','2601.00001',{'arxiv':'2601.00001'},['First Author'])
second,other_record = self.record('pmlr','v1/other',{'doi':'10.0000/example'},['Other Author'])
self.store.accepted(second,other_record,'ICML',2025,'main','v1/other','https://example.org/other')
tag = self.store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','test')").lastrowid
self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)',(second,tag,other_record,'manual'))
resolved,_ = self.record('openreview','forum1',{'arxiv':'2601.00001','doi':'10.0000/example'},['Anonymous'])
self.assertEqual(resolved,first)
c=self.store.conn
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
self.assertEqual(c.execute('SELECT paper_id FROM submissions').fetchone()[0],first)
self.assertEqual(c.execute('SELECT paper_id FROM paper_tags').fetchone()[0],first)
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1)
def test_cross_platform_exact_title_needs_author_corroboration(self):
paper,_=self.record('arxiv','2601.00001')
same,_=self.record('pmlr','v1/paper')
other,_=self.record('acl','2025.acl.1',authors=['Unrelated Person'])
self.assertEqual(same,paper)
self.assertNotEqual(other,paper)
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
def test_manifest_count_mismatch_rolls_back_records(self):
path=Path(self.tmp.name)/'source.jsonl'
path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'})+'\n')
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':2}))
with self.assertRaisesRegex(ValueError,'differs'):
ingest_file(self.db,path)
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],0)
self.assertEqual(self.store.conn.execute('SELECT status FROM ingest_runs').fetchone()[0],'failed')
if __name__=='__main__':
unittest.main()
+664
View File
@@ -0,0 +1,664 @@
{
"scope_note": "Broad field coverage, not a claim of objective ranking relative to COLM. Main tracks, Findings, workshops, journals remain distinct. Missing editions must be reported, not silently skipped.",
"sources": {
"arxiv": "https://info.arxiv.org/help/bulk_data/index.html",
"openreview": "https://docs.openreview.net/getting-started/using-the-api",
"acl_anthology": "https://github.com/acl-org/acl-anthology/tree/master/data/xml",
"pmlr": "https://proceedings.mlr.press/",
"neurips": "https://proceedings.neurips.cc/",
"cvf": "https://openaccess.thecvf.com/",
"ecva": "https://www.ecva.net/papers.php",
"aaai": "https://ojs.aaai.org/index.php/AAAI/issue/archive",
"ijcai": "https://www.ijcai.org/all_proceedings/",
"dblp": "https://dblp.org/db/conf/",
"colm": "https://proceedings.colm.cc/"
},
"venues": [
{
"acronym": "NeurIPS",
"field": "machine_learning",
"venue_type": "conference",
"discovery_sources": [
"openreview",
"pmlr",
"neurips"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ICML",
"field": "machine_learning",
"venue_type": "conference",
"discovery_sources": [
"openreview",
"pmlr",
"neurips"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ICLR",
"field": "machine_learning",
"venue_type": "conference",
"discovery_sources": [
"openreview",
"pmlr",
"neurips"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "COLM",
"field": "machine_learning",
"venue_type": "conference",
"discovery_sources": [
"openreview",
"pmlr",
"neurips"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "COLT",
"field": "machine_learning",
"venue_type": "conference",
"discovery_sources": [
"openreview",
"pmlr",
"neurips"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "AISTATS",
"field": "machine_learning",
"venue_type": "conference",
"discovery_sources": [
"openreview",
"pmlr",
"neurips"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "UAI",
"field": "machine_learning",
"venue_type": "conference",
"discovery_sources": [
"openreview",
"pmlr",
"neurips"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "CLeaR",
"field": "machine_learning",
"venue_type": "conference",
"discovery_sources": [
"openreview",
"pmlr",
"neurips"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ACL",
"field": "language",
"venue_type": "conference",
"discovery_sources": [
"acl_anthology",
"openreview"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "EMNLP",
"field": "language",
"venue_type": "conference",
"discovery_sources": [
"acl_anthology",
"openreview"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "NAACL",
"field": "language",
"venue_type": "conference",
"discovery_sources": [
"acl_anthology",
"openreview"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "EACL",
"field": "language",
"venue_type": "conference",
"discovery_sources": [
"acl_anthology",
"openreview"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "COLING",
"field": "language",
"venue_type": "conference",
"discovery_sources": [
"acl_anthology",
"openreview"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "CoNLL",
"field": "language",
"venue_type": "conference",
"discovery_sources": [
"acl_anthology",
"openreview"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "CVPR",
"field": "vision",
"venue_type": "conference",
"discovery_sources": [
"cvf",
"ecva",
"openreview"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ICCV",
"field": "vision",
"venue_type": "conference",
"discovery_sources": [
"cvf",
"ecva",
"openreview"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ECCV",
"field": "vision",
"venue_type": "conference",
"discovery_sources": [
"cvf",
"ecva",
"openreview"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "AAAI",
"field": "general_ai",
"venue_type": "conference",
"discovery_sources": [
"aaai",
"ijcai",
"dblp"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "IJCAI",
"field": "general_ai",
"venue_type": "conference",
"discovery_sources": [
"aaai",
"ijcai",
"dblp"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ECAI",
"field": "general_ai",
"venue_type": "conference",
"discovery_sources": [
"aaai",
"ijcai",
"dblp"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "KDD",
"field": "retrieval_data",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"publisher"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "SIGIR",
"field": "retrieval_data",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"publisher"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "WWW",
"field": "retrieval_data",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"publisher"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "WSDM",
"field": "retrieval_data",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"publisher"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "RecSys",
"field": "retrieval_data",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"publisher"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "CIKM",
"field": "retrieval_data",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"publisher"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ICDM",
"field": "retrieval_data",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"publisher"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "SDM",
"field": "retrieval_data",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"publisher"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ECML-PKDD",
"field": "retrieval_data",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"publisher"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ICRA",
"field": "robotics",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"pmlr",
"rss"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "IROS",
"field": "robotics",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"pmlr",
"rss"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "RSS",
"field": "robotics",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"pmlr",
"rss"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "CoRL",
"field": "robotics",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"pmlr",
"rss"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ICAPS",
"field": "reasoning_agents",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"proceedings"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "KR",
"field": "reasoning_agents",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"proceedings"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "AAMAS",
"field": "reasoning_agents",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"proceedings"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ICASSP",
"field": "speech_audio",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"isca",
"ismir"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "INTERSPEECH",
"field": "speech_audio",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"isca",
"ismir"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "ISMIR",
"field": "speech_audio",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"isca",
"ismir"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "MICCAI",
"field": "specialist",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"publisher"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
},
{
"acronym": "FAccT",
"field": "specialist",
"venue_type": "conference",
"discovery_sources": [
"dblp",
"publisher"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
"tracks": [
"main"
],
"coverage": "pending"
}
],
"additional_tracks": [
"Findings",
"workshop"
],
"additional_nonconference_venues": [
"TMLR",
"JMLR",
"TACL",
"ACL ARR"
]
}