diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml new file mode 100644 index 0000000..5fbeb95 --- /dev/null +++ b/.gitea/workflows/check.yml @@ -0,0 +1,35 @@ +name: Metadata integrity checks +on: + push: + branches: [main, 'codex/**'] +jobs: + metadata: + runs-on: ces-control + env: + ARCHIVE_SHA: ${{ github.sha }} + PYTHONDONTWRITEBYTECODE: '1' + steps: + - name: Verify metadata storage and query commands + run: | + python3 -B - <<'PY' + import io, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request + sha = os.environ['ARCHIVE_SHA'] + if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha): + raise SystemExit('Invalid source commit') + # Public repository archive: no owner credentials or runner SSH keys. + url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz' + with urllib.request.urlopen(url, timeout=60) as response: + archive = response.read(16*1024*1024+1) + if len(archive)>16*1024*1024: + raise SystemExit('Unexpectedly large source archive') + with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp: + root = pathlib.Path(tmp) + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + tar.extractall(root, filter='data') + candidates = list(root.glob('*/test_import_arxiv.py')) + if len(candidates)!=1: + raise SystemExit('Unexpected repository archive layout') + src = candidates[0].parent + subprocess.run(['python3','-B','-m','unittest','discover','-v'],cwd=src,check=True) + subprocess.run(['python3','-B','query.py','--help'],cwd=src,check=True) + PY diff --git a/README.md b/README.md index 17edc73..162d6b2 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 目标包括：完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。 -**目前仍在开发，不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、DOI/标识合并和只读查询脚本。服务器上进行过合成样本验证；全量快照下载与历史导入尚未完成。会议适配器、跨平台模糊匹配复核流程、完整覆盖率核验与 CI 仍待实现。`venues.json` 的 41 个会议是候选覆盖范围，不代表已完成采集，也不是与 COLM 比较的客观排名。 +**目前仍在开发，不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上六项合成回归检查通过；完整 ICML 2025 元数据已采集 3,330 条，尚未作为全量交付。arXiv 全量快照下载与历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流，实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围，不代表已完成采集，也不是与 COLM 比较的客观排名。 ## 数据范围与来源 @@ -41,16 +41,16 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 | `edition_sources` | 每届每个来源的覆盖状态、预计/已导入数量、错误与断点 | | `submissions` / `decisions` | 多次投稿及带证据的决定历史，支持改投、撤稿和决定变更 | | `tags` / `paper_tags` | 平台标签与人工标签，使用命名空间隔离 | -| `match_candidates` | 证据不充分的潜在重复记录及人工复核状态 | +| `match_candidates` / `paper_merges` | 潜在重复复核状态、确定合并审计 | | `ingest_runs` | 导入批次、状态、错误和恢复位置 | 决定状态为 `accepted`、`rejected`、`desk_rejected`、`withdrawn`、`under_review`、`unknown`。一篇论文可以对应多次投稿和多个决定，不能用单一“最终会议”字段覆盖历史。查询中按观察时间选择最近状态；后续分析需要同时检查来源覆盖率。 -当前确定性去重使用 arXiv ID 和单个 DOI。保留各平台源记录，避免合并后丢失出处。规范化标题只建检索索引，尚未实现模糊匹配流程；**现阶段不保证跨平台去重已经完成**。不同平台标签不自动等同，自定义标签应使用 `custom:*` 命名空间。 +当前确定性去重使用 arXiv ID、单个 DOI 和 OpenReview forum ID；跨平台同标题还需至少两位作者吻合，或作者吻合且摘要相同，才自动合并。保留各平台源记录，避免合并后丢失出处。规范化标题只建检索索引，尚未实现模糊匹配流程；**现阶段不保证跨平台去重已经完成**。不同平台标签不自动等同，自定义标签应使用 `custom:*` 命名空间。 ## 运行与查询 -依赖 Python 3 标准库和带 FTS5 的 SQLite，目前无第三方 Python 依赖，无项目构建步骤。服务器已检查为 Python 3.14.4。所有运行数据放在仓库外，不提交数据库、下载包或访问凭据。 +基础导入和查询依赖 Python 3 标准库和带 FTS5 的 SQLite；PMLR 采集另需 `requirements.txt` 中锁定的 PyYAML 6.0.3。无项目构建步骤。服务器已检查为 Python 3.14.4。所有运行数据放在仓库外，不提交数据库、下载包或访问凭据。 服务器暂存目录：`/data/paper-index/`，快照目录：`/data/paper-index/raw/`，运行代码目录：`/data/paper-index/code/`。这些路径不是对外访问的 API。 @@ -82,6 +82,20 @@ python3 -B query.py --db /data/paper-index/papers.sqlite --venue COLM --year 202 查询连接使用 SQLite 只读模式，结果输出 JSON。当前没有公共 Web 页面或网络查询服务。 +## 官方会议元数据采集 + +```sh +# 单卷示例：ICML 2025，读取官方 gh-pages 元数据归档，不下载 PDF +python3 -B collect_pmlr.py --cache /data/paper-index/raw/http-cache \ + --out /data/paper-index/collected --volume 267 + +# 待主库完成 arXiv 导入、且没有其他写入进程后，合并会议记录 +python3 -B import_records.py --db /data/paper-index/papers.sqlite \ + /data/paper-index/collected/pmlr-v267.jsonl +``` + +采集输出 JSONL 和对应 manifest，只有显式官方录用记录才产生 accepted 决定。清单数目不一致时整个文件导入回滚，记录失败批次，不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。 + ## 长期维护与更新 1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发，遵守来源限流、重试与人机验证要求。 diff --git a/collect_pmlr.py b/collect_pmlr.py new file mode 100644 index 0000000..1e9da16 --- /dev/null +++ b/collect_pmlr.py @@ -0,0 +1,123 @@ +"""Collect official PMLR metadata, never the PDFs linked by the metadata.""" +import argparse +import hashlib +import io +import json +import re +import tarfile +import time +import urllib.error +import urllib.request +from pathlib import Path + +import yaml + + +def fetch(url,cache,limit=64*1024**2): + cache = Path(cache) + cache.mkdir(parents=True,exist_ok=True) + path = cache/hashlib.sha256(url.encode()).hexdigest() + if path.exists(): + return path.read_bytes() + for attempt in range(3): + try: + request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'}) + with urllib.request.urlopen(request,timeout=90) as response: + if response.status != 200: + raise RuntimeError('Expected complete HTTP 200 response') + data = response.read(limit+1) + if len(data)>limit: + raise RuntimeError('Metadata response exceeds byte limit') + part = path.with_suffix('.part') + part.write_bytes(data) + part.replace(path) + path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())})) + time.sleep(0.5) + return data + except urllib.error.HTTPError as exc: + if exc.code not in (429,500,502,503,504) or attempt==2: + raise + time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1'))))) + + +def collect_volume(volume,cache,out,allowed=None): + config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml' + config = yaml.safe_load(fetch(config_url,cache)) + venue = str(config.get('shortname','')).upper() + if allowed and venue not in allowed: + return None + year = int(config['year']) + title = str(config.get('booktitle','')) + track = 'workshop' if 'workshop' in title.lower() else 'main' + archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache) + path = Path(out)/f'pmlr-v{volume}.jsonl' + path.parent.mkdir(parents=True,exist_ok=True) + part = path.with_suffix('.part') + count = 0 + with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar,part.open('w') as output: + for member in tar: + if not member.isfile() or '/_posts/' not in member.name or not member.name.endswith(('.md','.markdown','.html')): + continue + if member.size>8*1024**2: + raise RuntimeError('Unexpectedly large metadata member') + content = tar.extractfile(member).read().decode('utf-8-sig') + parts = content.split('---',2) + if len(parts)<3 or parts[0].strip(): + raise RuntimeError('Missing YAML frontmatter: '+member.name) + row = yaml.safe_load(parts[1]) + if row.get('layout') not in ('inproceedings','article') or not row.get('title'): + continue + external = f"v{volume}/{row.get('id') or Path(member.name).stem}" + ids = {} + for key in ('doi','openreview','arxiv'): + value = row.get(key) + if value: + value = str(value) + if key=='arxiv': + value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf') + if key=='openreview': + value = value.split('id=')[-1].split('&')[0] + ids[key] = value + record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}} + output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n') + count += 1 + if count==0: + part.unlink(missing_ok=True) + raise RuntimeError('No papers parsed; refusing an empty successful collection') + part.replace(path) + result = {'source':'pmlr','venue':venue,'year':year,'track':track,'volume':volume,'count':count,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'collected'} + path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2)) + print(json.dumps(result),flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--cache',required=True) + parser.add_argument('--out',required=True) + parser.add_argument('--volume',type=int,action='append') + parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json'))) + args = parser.parse_args() + allowed = {v['acronym'].upper() for v in json.loads(Path(args.registry).read_text())['venues']} + volumes = args.volume + if not volumes: + index = fetch('https://proceedings.mlr.press/',args.cache).decode() + volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press)?/v(\d+)/?["\']',index))) + if not volumes: + raise RuntimeError('PMLR index yielded no volumes') + errors = [] + for volume in volumes: + try: + collect_volume(volume,args.cache,args.out,None if args.volume else allowed) + except Exception as exc: + error = {'volume':volume,'status':'blocked','error':str(exc)} + errors.append(error) + print(json.dumps(error),flush=True) + Path(args.out).mkdir(parents=True,exist_ok=True) + (Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2)) + if errors: + raise SystemExit(1) + + +if __name__=='__main__': + main() diff --git a/import_arxiv.py b/import_arxiv.py new file mode 100644 index 0000000..8e71dfd --- /dev/null +++ b/import_arxiv.py @@ -0,0 +1,78 @@ +"""Stream the official metadata ZIP into SQLite. No PDFs, source files or builds.""" +import argparse +import json +import time +import shutil +import zipfile +from pathlib import Path + + +from index_core import Store + + +def ingest(db, archive): + store = Store(db) + conn = store.conn + conn.execute("INSERT OR IGNORE INTO sources(name,homepage) VALUES('arxiv','https://arxiv.org')") + source = conn.execute("SELECT id FROM sources WHERE name='arxiv'").fetchone()[0] + previous = conn.execute('SELECT id,checkpoint_json FROM ingest_runs WHERE source_id=? AND status IN (\'running\',\'failed\') ORDER BY id DESC LIMIT 1', (source,)).fetchone() + fingerprint = {'path':str(Path(archive).resolve()), 'bytes':Path(archive).stat().st_size, 'mtime_ns':Path(archive).stat().st_mtime_ns} + start = 0 + if previous: + checkpoint = json.loads(previous[1] or '{}') + if checkpoint.get('archive') == fingerprint: + run = previous[0] + start = checkpoint.get('line', 0) + conn.execute("UPDATE ingest_runs SET status='running',error=NULL WHERE id=?", (run,)) + else: + raise RuntimeError('Pending run belongs to a different archive; resolve it explicitly') + else: + run = conn.execute('INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)', (source,json.dumps({'archive':fingerprint,'line':0}))).lastrowid + conn.commit() + line_number = start + last = time.monotonic() + try: + with zipfile.ZipFile(archive) as zipped: + # Reading the member to EOF also checks its CRC. + with zipped.open('arxiv-metadata-oai-snapshot.json') as stream: + for line_number, line in enumerate(stream, 1): + if line_number <= start: + continue + row = json.loads(line) + external_id = row['id'] + title = ' '.join(row['title'].split()) + abstract = row.get('abstract', '').strip() + authors = row.get('authors_parsed') or row.get('authors') or [] + metadata = {k:v for k,v in row.items() if k not in ('title','abstract','authors','authors_parsed')} + paper,record = store.record('arxiv',external_id,'https://arxiv.org/abs/'+external_id,title,abstract,authors,{'arxiv':external_id,'doi':row.get('doi')},metadata,(row.get('versions') or [{}])[0].get('created')) + conn.execute("DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key='arxiv:categories'", (record,)) + for category in (row.get('categories') or '').split(): + conn.execute("INSERT OR IGNORE INTO tags(namespace,value) VALUES('arxiv:category',?)",(category,)) + tag = conn.execute("SELECT id FROM tags WHERE namespace='arxiv:category' AND value=?",(category,)).fetchone()[0] + conn.execute('INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,'arxiv:categories')) + if line_number % 500 == 0: + if shutil.disk_usage(Path(db).parent).free < 4*1024**3: + raise RuntimeError('Stopped before exhausting disk: less than 4 GiB free') + conn.execute('UPDATE ingest_runs SET records_seen=?,records_written=?,checkpoint_json=? WHERE id=?',(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run)) + conn.commit() + if time.monotonic()-last >= 20: + print(json.dumps({'run':run,'rows':line_number}),flush=True) + last = time.monotonic() + conn.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run)) + conn.commit() + print(json.dumps({'status':'complete','rows':line_number}),flush=True) + except BaseException as exc: + conn.rollback() + conn.execute("UPDATE ingest_runs SET status='failed',error=? WHERE id=?",(str(exc),run)) + conn.commit() + raise + finally: + conn.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--db', required=True) + parser.add_argument('--archive', required=True) + args = parser.parse_args() + ingest(args.db,args.archive) diff --git a/import_records.py b/import_records.py new file mode 100644 index 0000000..ea66aee --- /dev/null +++ b/import_records.py @@ -0,0 +1,69 @@ +"""Import normalized conference JSONL files with coverage evidence.""" +import argparse +import json +import time +from pathlib import Path + +from index_core import Store + + +def ingest_file(db,path): + path = Path(path) + manifest_path = path.with_suffix('.manifest.json') + manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {} + store = Store(db) + c = store.conn + source_name = manifest.get('source') + if not source_name: + raise ValueError('A source manifest is required') + source_id = store.source(source_name,manifest['list_url']) + run = c.execute('INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)',(source_id,json.dumps({'file':str(path)}))).lastrowid + c.commit() + count = 0 + seen = set() + editions = set() + try: + with path.open() as stream: + for line in stream: + row = json.loads(line) + if row['source'] != source_name: + raise ValueError('Source differs from manifest') + external = row['external_id'] + if external in seen: + raise ValueError('Duplicate external ID in one collected file: '+external) + seen.add(external) + paper,record = store.record(source_name,external,row['url'],row['title'],row.get('abstract'),row.get('authors',[]),row.get('identifiers'),row.get('metadata')) + if row.get('status') == 'accepted': + edition = store.accepted(paper,record,row['venue'],row['year'],row['track'],external,row['url']) + editions.add(edition) + else: + raise ValueError('This importer requires an explicit supported decision status') + count += 1 + if not count: + raise ValueError('Empty metadata file') + if len(editions) != 1: + raise ValueError('A manifest must describe exactly one conference edition and track') + edition = next(iter(editions)) + expected = manifest.get('count') + if expected is not None and expected != count: + raise ValueError(f'Manifest count {expected} differs from records {count}') + c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if expected==count else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()))) + c.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=? WHERE id=?",(count,count,run)) + c.commit() + print(json.dumps({'file':str(path),'status':'imported','records':count}),flush=True) + except BaseException as exc: + c.rollback() + c.execute("UPDATE ingest_runs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?",(str(exc),run)) + c.commit() + raise + finally: + c.close() + + +if __name__=='__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--db',required=True) + parser.add_argument('files',nargs='+') + args = parser.parse_args() + for filename in args.files: + ingest_file(args.db,filename) diff --git a/index_core.py b/index_core.py new file mode 100644 index 0000000..2bb2898 --- /dev/null +++ b/index_core.py @@ -0,0 +1,120 @@ +"""Shared identity resolution and provenance-preserving metadata storage.""" +import json +import re +import sqlite3 +import unicodedata +from pathlib import Path + + +def normalized(value): + return ''.join(c for c in unicodedata.normalize('NFKD',value).casefold() if c.isalnum()) + + +def author_keys(authors): + if not isinstance(authors,list): + return set() + result = set() + for author in authors: + if isinstance(author,list): + author = ' '.join(str(x) for x in author[1:2]+author[:1]) + elif isinstance(author,dict): + author = ' '.join(str(author.get(k,'')) for k in ('given','family')) + key = normalized(str(author)) + if key: + result.add(key) + return result + + +class Store: + def __init__(self,path): + self.conn = sqlite3.connect(path,timeout=60) + self.conn.executescript(Path(__file__).with_name('schema.sql').read_text()) + self.conn.execute('PRAGMA synchronous=NORMAL') + self.conn.execute('PRAGMA cache_size=-32768') + + def source(self,name,url): + self.conn.execute('INSERT OR IGNORE INTO sources(name,homepage) VALUES(?,?)',(name,url)) + return self.conn.execute('SELECT id FROM sources WHERE name=?',(name,)).fetchone()[0] + + def merge(self,target,other,reason): + c = self.conn + if target == other: + return + c.execute('INSERT INTO paper_merges(former_paper_id,surviving_paper_id,reason) VALUES(?,?,?)',(other,target,reason)) + c.execute('UPDATE identifiers SET paper_id=? WHERE paper_id=?',(target,other)) + c.execute('UPDATE source_records SET paper_id=? WHERE paper_id=?',(target,other)) + c.execute('UPDATE submissions SET paper_id=? WHERE paper_id=?',(target,other)) + c.execute('INSERT OR IGNORE INTO paper_tags SELECT ?,tag_id,source_record_id,evidence_key FROM paper_tags WHERE paper_id=?',(target,other)) + c.execute('DELETE FROM paper_tags WHERE paper_id=?',(other,)) + c.execute('UPDATE paper_merges SET surviving_paper_id=? WHERE surviving_paper_id=?',(target,other)) + # Reattach pending candidate evidence without losing colliding rows. + candidates = c.execute('SELECT left_paper_id,right_paper_id,method,score,evidence_json,status FROM match_candidates WHERE left_paper_id=? OR right_paper_id=?',(other,other)).fetchall() + c.execute('DELETE FROM match_candidates WHERE left_paper_id=? OR right_paper_id=?',(other,other)) + for left,right,method,score,evidence,status in candidates: + left = target if left == other else left + right = target if right == other else right + if left != right: + c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(left,right),max(left,right),method,score,evidence,status)) + c.execute('DELETE FROM papers WHERE id=?',(other,)) + + def record(self,source,external_id,url,title,abstract,authors,identifiers=None,metadata=None,published=None): + c = self.conn + identifiers = dict(identifiers or {}) + # External IDs are scoped to platforms; an OpenReview ID is a forum ID. + identifiers[source] = external_id + ids = [] + for scheme,value in identifiers.items(): + if not value: + continue + value = str(value).strip() + if scheme == 'doi': + value = re.sub(r'^https?://(?:dx\.)?doi\.org/','',value,flags=re.I).lower() + if not re.fullmatch(r'10\.\d{4,9}/\S+',value): + continue + if scheme == 'arxiv': + value = re.sub(r'v\d+$','',value) + ids.append((scheme,value)) + matches = set() + for scheme,value in ids: + found = c.execute('SELECT paper_id FROM identifiers WHERE scheme=? AND value=?',(scheme,value)).fetchone() + if found: + matches.add(found[0]) + old = c.execute('SELECT paper_id FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone() + if old: + matches.add(old[0]) + key = normalized(title) + title_matches = c.execute('SELECT id,authors_json,abstract FROM papers WHERE title_key=?',(key,)).fetchall() + authors_set = author_keys(authors) + for paper,old_authors,old_abstract in title_matches: + overlap = authors_set & author_keys(json.loads(old_authors)) + # Cross-platform title matching needs independent author corroboration. + same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone() + if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())): + matches.add(paper) + if matches: + paper = min(matches) + for other in sorted(matches-{paper}): + self.merge(paper,other,'explicit identifier or exact title corroborated by authors') + c.execute("UPDATE papers SET abstract=CASE WHEN abstract IS NULL OR abstract='' THEN ? ELSE abstract END WHERE id=?",(abstract,paper)) + else: + paper = c.execute('INSERT INTO papers(title,abstract,title_key,authors_json,first_published) VALUES(?,?,?,?,?)',(title,abstract,key,json.dumps(authors,ensure_ascii=False),published)).lastrowid + for scheme,value in ids: + c.execute('INSERT OR IGNORE INTO identifiers VALUES(?,?,?)',(scheme,value,paper)) + c.execute('INSERT INTO source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_id) DO UPDATE SET paper_id=excluded.paper_id,title=excluded.title,abstract=excluded.abstract,authors_json=excluded.authors_json,metadata_json=excluded.metadata_json,retrieved_at=CURRENT_TIMESTAMP',(source_id,external_id,paper,url,title,abstract,json.dumps(authors,ensure_ascii=False),json.dumps(metadata or {},ensure_ascii=False,default=str))) + record = c.execute('SELECT id FROM source_records WHERE source_id=? AND external_id=?',(source_id,external_id)).fetchone()[0] + for other,_,_ in title_matches: + if other != paper and c.execute('SELECT 1 FROM papers WHERE id=?',(other,)).fetchone(): + c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(paper,other),max(paper,other),'exact_title_unconfirmed',None,json.dumps({'title':title}), 'pending')) + return paper,record + + def accepted(self,paper,record,venue,year,track,external_id,url): + c = self.conn + c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified')) + venue_id = c.execute('SELECT id FROM venues WHERE acronym=?',(venue,)).fetchone()[0] + c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)',(venue_id,int(year),track)) + edition = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?',(venue_id,int(year),track)).fetchone()[0] + c.execute('INSERT OR IGNORE INTO submissions(paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?)',(paper,edition,record,external_id)) + submission = c.execute('SELECT id FROM submissions WHERE source_record_id=? AND edition_id=? AND source_submission_id=?',(record,edition,external_id)).fetchone()[0] + if not c.execute("SELECT 1 FROM decisions WHERE submission_id=? AND status='accepted' AND evidence_url=?",(submission,url)).fetchone(): + c.execute("INSERT INTO decisions(submission_id,status,raw_value,evidence_url,observed_at) VALUES(?,'accepted','listed in official proceedings',?,strftime('%Y-%m-%dT%H:%M:%fZ','now'))",(submission,url)) + return edition diff --git a/query.py b/query.py new file mode 100644 index 0000000..b293291 --- /dev/null +++ b/query.py @@ -0,0 +1,52 @@ +"""Read-only command-line queries for the paper index.""" +import argparse +import json +import sqlite3 +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--db', required=True) + parser.add_argument('--query', help='SQLite FTS5 query over title and abstract') + parser.add_argument('--tag', help='Filter as namespace=value, e.g. arxiv:category=cs.AI') + parser.add_argument('--venue', help='Conference acronym') + parser.add_argument('--year', type=int) + parser.add_argument('--status', choices=['accepted','rejected','desk_rejected','withdrawn','under_review','unknown']) + parser.add_argument('--limit', type=int, default=20) + parser.add_argument('--stats', action='store_true') + args = parser.parse_args() + conn = sqlite3.connect(Path(args.db).resolve().as_uri()+'?mode=ro', uri=True) + conn.row_factory = sqlite3.Row + if args.stats: + result = {name:conn.execute('SELECT count(*) FROM '+name).fetchone()[0] for name in ('papers','source_records','submissions','decisions','tags','match_candidates')} + result['coverage'] = [dict(r) for r in conn.execute('SELECT status,count(*) AS sources FROM edition_sources GROUP BY status')] + result['runs'] = [dict(r) for r in conn.execute('SELECT * FROM ingest_runs ORDER BY id DESC LIMIT 10')] + else: + where = []; params = [] + if args.query: + where.append('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)'); params.append(args.query) + if args.tag: + namespace,value = args.tag.split('=',1) + where.append('EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'); params.extend([namespace,value]) + if args.venue or args.year or args.status: + clause = ['s.paper_id=p.id'] + if args.venue: + clause.append('v.acronym=?'); params.append(args.venue) + if args.year: + clause.append('e.year=?'); params.append(args.year) + if args.status: + clause.append('EXISTS(SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=? AND NOT EXISTS(SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id AND (newer.observed_at>d.observed_at OR (newer.observed_at=d.observed_at AND newer.id>d.id))))'); params.append(args.status) + where.append('EXISTS(SELECT 1 FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+' AND '.join(clause)+')') + sql = 'SELECT p.* FROM papers p'+(' WHERE '+' AND '.join(where) if where else '')+' ORDER BY p.id LIMIT ?' + params.append(max(1,min(args.limit,200))) + result = [] + for row in conn.execute(sql,params): + paper = dict(row) + paper['identifiers'] = [dict(r) for r in conn.execute('SELECT scheme,value FROM identifiers WHERE paper_id=?',(row['id'],))] + result.append(paper) + print(json.dumps(result,ensure_ascii=False,indent=2)) + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f62ce0c --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +PyYAML==6.0.3 diff --git a/schema.sql b/schema.sql new file mode 100644 index 0000000..8a93f24 --- /dev/null +++ b/schema.sql @@ -0,0 +1,92 @@ +PRAGMA foreign_keys=ON; +PRAGMA journal_mode=WAL; +CREATE TABLE IF NOT EXISTS sources ( + id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, homepage TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS papers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, abstract TEXT, + title_key TEXT NOT NULL, authors_json TEXT NOT NULL DEFAULT '[]', + first_published TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS papers_title_key ON papers(title_key); +CREATE TABLE IF NOT EXISTS identifiers ( + scheme TEXT NOT NULL, value TEXT NOT NULL, paper_id INTEGER NOT NULL REFERENCES papers(id), + PRIMARY KEY(scheme,value) +); +CREATE INDEX IF NOT EXISTS identifiers_paper ON identifiers(paper_id); +CREATE TABLE IF NOT EXISTS source_records ( + id INTEGER PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id), + external_id TEXT NOT NULL, paper_id INTEGER NOT NULL REFERENCES papers(id), + url TEXT NOT NULL, title TEXT NOT NULL, abstract TEXT, authors_json TEXT NOT NULL, + metadata_json TEXT NOT NULL, retrieved_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(source_id,external_id) +); +CREATE INDEX IF NOT EXISTS source_records_paper ON source_records(paper_id); +CREATE TABLE IF NOT EXISTS venues ( + id INTEGER PRIMARY KEY, acronym TEXT NOT NULL UNIQUE, name TEXT NOT NULL, + field TEXT NOT NULL, venue_type TEXT NOT NULL CHECK(venue_type IN ('conference','journal','review_platform')) +); +CREATE TABLE IF NOT EXISTS editions ( + id INTEGER PRIMARY KEY, venue_id INTEGER NOT NULL REFERENCES venues(id), + year INTEGER NOT NULL, track TEXT NOT NULL DEFAULT 'main', + UNIQUE(venue_id,year,track) +); +CREATE TABLE IF NOT EXISTS edition_sources ( + edition_id INTEGER NOT NULL REFERENCES editions(id), source_id INTEGER NOT NULL REFERENCES sources(id), + list_url TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending' + CHECK(status IN ('pending','running','complete','partial','blocked','unavailable','not_yet_published')), + expected_count INTEGER, imported_count INTEGER NOT NULL DEFAULT 0, + cursor TEXT, last_success_at TEXT, last_error TEXT, + PRIMARY KEY(edition_id,source_id,list_url) +); +CREATE TABLE IF NOT EXISTS submissions ( + id INTEGER PRIMARY KEY, paper_id INTEGER NOT NULL REFERENCES papers(id), + edition_id INTEGER NOT NULL REFERENCES editions(id), + source_record_id INTEGER NOT NULL REFERENCES source_records(id), + source_submission_id TEXT NOT NULL, submitted_at TEXT, + UNIQUE(source_record_id,edition_id,source_submission_id) +); +CREATE INDEX IF NOT EXISTS submissions_paper ON submissions(paper_id); +CREATE TABLE IF NOT EXISTS decisions ( + id INTEGER PRIMARY KEY, submission_id INTEGER NOT NULL REFERENCES submissions(id), + status TEXT NOT NULL CHECK(status IN ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown')), + raw_value TEXT NOT NULL, evidence_url TEXT NOT NULL, observed_at TEXT NOT NULL, + decision_date TEXT, UNIQUE(submission_id,status,raw_value,evidence_url,observed_at) +); +CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY, namespace TEXT NOT NULL, value TEXT NOT NULL, + description TEXT, UNIQUE(namespace,value) +); +CREATE TABLE IF NOT EXISTS paper_tags ( + paper_id INTEGER NOT NULL REFERENCES papers(id), tag_id INTEGER NOT NULL REFERENCES tags(id), + source_record_id INTEGER REFERENCES source_records(id), + evidence_key TEXT NOT NULL, PRIMARY KEY(paper_id,tag_id,evidence_key) +); +CREATE TABLE IF NOT EXISTS match_candidates ( + left_paper_id INTEGER NOT NULL REFERENCES papers(id), right_paper_id INTEGER NOT NULL REFERENCES papers(id), + method TEXT NOT NULL, score REAL, evidence_json TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','merged','distinct')), + PRIMARY KEY(left_paper_id,right_paper_id,method) +); +CREATE TABLE IF NOT EXISTS paper_merges ( + id INTEGER PRIMARY KEY, former_paper_id INTEGER NOT NULL, + surviving_paper_id INTEGER NOT NULL REFERENCES papers(id), + reason TEXT NOT NULL, merged_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE TABLE IF NOT EXISTS ingest_runs ( + id INTEGER PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id), + started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT, + status TEXT NOT NULL DEFAULT 'running', records_seen INTEGER NOT NULL DEFAULT 0, + records_written INTEGER NOT NULL DEFAULT 0, error TEXT, checkpoint_json TEXT +); +CREATE VIRTUAL TABLE IF NOT EXISTS papers_fts USING fts5(title,abstract,content='papers',content_rowid='id'); +CREATE TRIGGER IF NOT EXISTS papers_insert AFTER INSERT ON papers BEGIN + INSERT INTO papers_fts(rowid,title,abstract) VALUES(new.id,new.title,new.abstract); +END; +CREATE TRIGGER IF NOT EXISTS papers_delete AFTER DELETE ON papers BEGIN + INSERT INTO papers_fts(papers_fts,rowid,title,abstract) VALUES('delete',old.id,old.title,old.abstract); +END; +CREATE TRIGGER IF NOT EXISTS papers_update AFTER UPDATE OF title,abstract ON papers BEGIN + INSERT INTO papers_fts(papers_fts,rowid,title,abstract) VALUES('delete',old.id,old.title,old.abstract); + INSERT INTO papers_fts(rowid,title,abstract) VALUES(new.id,new.title,new.abstract); +END; diff --git a/test_import_arxiv.py b/test_import_arxiv.py new file mode 100644 index 0000000..4c2c6e8 --- /dev/null +++ b/test_import_arxiv.py @@ -0,0 +1,57 @@ +import json +import sqlite3 +import tempfile +import unittest +import zipfile +from pathlib import Path + +from import_arxiv import ingest + + +class ImportTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.db = self.root/'papers.sqlite' + self.archive = self.root/'metadata.zip' + + def write(self, rows): + with zipfile.ZipFile(self.archive,'w') as z: + z.writestr('arxiv-metadata-oai-snapshot.json','\n'.join(json.dumps(r) for r in rows)) + + def row(self, identifier, doi=None): + return dict(id=identifier,title='A shared title',abstract='Searchable abstract',authors_parsed=[['Doe','Jane','']],categories='cs.AI cs.LG',doi=doi,versions=[]) + + def test_identity_provenance_and_idempotency(self): + self.write([self.row('2601.00001','10.0000/example'),self.row('2601.00002','10.0000/example')]) + ingest(self.db,self.archive) + ingest(self.db,self.archive) + with sqlite3.connect(self.db) as c: + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2) + self.assertEqual(c.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'Searchable'").fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],0) + self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[]) + self.assertEqual(c.execute('PRAGMA integrity_check').fetchone()[0],'ok') + + def test_title_alone_does_not_merge(self): + self.write([self.row('2601.00001'),self.row('2601.00002')]) + ingest(self.db,self.archive) + with sqlite3.connect(self.db) as c: + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2) + + def test_platform_and_custom_tags_remain_distinct(self): + self.write([self.row('2601.00001')]) + ingest(self.db,self.archive) + with sqlite3.connect(self.db) as c: + paper = c.execute('SELECT id FROM papers').fetchone()[0] + tag = c.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','cs.AI')").lastrowid + c.execute('INSERT INTO paper_tags VALUES(?,?,NULL,?)',(paper,tag,'manual:1')) + ingest(self.db,self.archive) + with sqlite3.connect(self.db) as c: + self.assertEqual(c.execute("SELECT count(*) FROM paper_tags pt JOIN tags t ON pt.tag_id=t.id WHERE t.value='cs.AI'").fetchone()[0],2) + + +if __name__ == '__main__': + unittest.main() diff --git a/test_index_core.py b/test_index_core.py new file mode 100644 index 0000000..1d7c0f6 --- /dev/null +++ b/test_index_core.py @@ -0,0 +1,56 @@ +import json +import sqlite3 +import tempfile +import unittest +from pathlib import Path + +from index_core import Store +from import_records import ingest_file + + +class StoreTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.db = Path(self.tmp.name)/'papers.sqlite' + self.store = Store(self.db) + self.addCleanup(self.store.conn.close) + + def record(self,source,identifier,ids=None,authors=None): + return self.store.record(source,identifier,'https://example.org/'+identifier,'A sufficiently long example research paper title','Example abstract',authors or ['Jane Doe','John Smith'],ids) + + def test_bridge_identifiers_preserves_submission_and_tags(self): + first,record = self.record('arxiv','2601.00001',{'arxiv':'2601.00001'},['First Author']) + second,other_record = self.record('pmlr','v1/other',{'doi':'10.0000/example'},['Other Author']) + self.store.accepted(second,other_record,'ICML',2025,'main','v1/other','https://example.org/other') + tag = self.store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','test')").lastrowid + self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)',(second,tag,other_record,'manual')) + resolved,_ = self.record('openreview','forum1',{'arxiv':'2601.00001','doi':'10.0000/example'},['Anonymous']) + self.assertEqual(resolved,first) + c=self.store.conn + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1) + self.assertEqual(c.execute('SELECT paper_id FROM submissions').fetchone()[0],first) + self.assertEqual(c.execute('SELECT paper_id FROM paper_tags').fetchone()[0],first) + self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[]) + self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1) + + def test_cross_platform_exact_title_needs_author_corroboration(self): + paper,_=self.record('arxiv','2601.00001') + same,_=self.record('pmlr','v1/paper') + other,_=self.record('acl','2025.acl.1',authors=['Unrelated Person']) + self.assertEqual(same,paper) + self.assertNotEqual(other,paper) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1) + + def test_manifest_count_mismatch_rolls_back_records(self): + path=Path(self.tmp.name)/'source.jsonl' + path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'})+'\n') + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':2})) + with self.assertRaisesRegex(ValueError,'differs'): + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],0) + self.assertEqual(self.store.conn.execute('SELECT status FROM ingest_runs').fetchone()[0],'failed') + + +if __name__=='__main__': + unittest.main() diff --git a/venues.json b/venues.json new file mode 100644 index 0000000..5f7adc6 --- /dev/null +++ b/venues.json @@ -0,0 +1,664 @@ +{ + "scope_note": "Broad field coverage, not a claim of objective ranking relative to COLM. Main tracks, Findings, workshops, journals remain distinct. Missing editions must be reported, not silently skipped.", + "sources": { + "arxiv": "https://info.arxiv.org/help/bulk_data/index.html", + "openreview": "https://docs.openreview.net/getting-started/using-the-api", + "acl_anthology": "https://github.com/acl-org/acl-anthology/tree/master/data/xml", + "pmlr": "https://proceedings.mlr.press/", + "neurips": "https://proceedings.neurips.cc/", + "cvf": "https://openaccess.thecvf.com/", + "ecva": "https://www.ecva.net/papers.php", + "aaai": "https://ojs.aaai.org/index.php/AAAI/issue/archive", + "ijcai": "https://www.ijcai.org/all_proceedings/", + "dblp": "https://dblp.org/db/conf/", + "colm": "https://proceedings.colm.cc/" + }, + "venues": [ + { + "acronym": "NeurIPS", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICML", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICLR", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "COLM", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "COLT", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "AISTATS", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "UAI", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "CLeaR", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ACL", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "EMNLP", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "NAACL", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "EACL", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "COLING", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "CoNLL", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "CVPR", + "field": "vision", + "venue_type": "conference", + "discovery_sources": [ + "cvf", + "ecva", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICCV", + "field": "vision", + "venue_type": "conference", + "discovery_sources": [ + "cvf", + "ecva", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ECCV", + "field": "vision", + "venue_type": "conference", + "discovery_sources": [ + "cvf", + "ecva", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "AAAI", + "field": "general_ai", + "venue_type": "conference", + "discovery_sources": [ + "aaai", + "ijcai", + "dblp" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "IJCAI", + "field": "general_ai", + "venue_type": "conference", + "discovery_sources": [ + "aaai", + "ijcai", + "dblp" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ECAI", + "field": "general_ai", + "venue_type": "conference", + "discovery_sources": [ + "aaai", + "ijcai", + "dblp" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "KDD", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "SIGIR", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "WWW", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "WSDM", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "RecSys", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "CIKM", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICDM", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "SDM", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ECML-PKDD", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICRA", + "field": "robotics", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "pmlr", + "rss" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "IROS", + "field": "robotics", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "pmlr", + "rss" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "RSS", + "field": "robotics", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "pmlr", + "rss" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "CoRL", + "field": "robotics", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "pmlr", + "rss" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICAPS", + "field": "reasoning_agents", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "proceedings" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "KR", + "field": "reasoning_agents", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "proceedings" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "AAMAS", + "field": "reasoning_agents", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "proceedings" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICASSP", + "field": "speech_audio", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "isca", + "ismir" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "INTERSPEECH", + "field": "speech_audio", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "isca", + "ismir" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ISMIR", + "field": "speech_audio", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "isca", + "ismir" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "MICCAI", + "field": "specialist", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "FAccT", + "field": "specialist", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + } + ], + "additional_tracks": [ + "Findings", + "workshop" + ], + "additional_nonconference_venues": [ + "TMLR", + "JMLR", + "TACL", + "ACL ARR" + ] +}