Preserve canonical submission identities and decision source evidence
Metadata integrity checks / metadata (push) Successful in 7s

This commit is contained in:
2026-11-06 14:05:37 +08:00
parent a7a0729df2
commit 15bb4920c6
14 changed files with 515 additions and 27 deletions
+1
View File
@@ -9,6 +9,7 @@ __pycache__/
*.part
*.part.json
*.download.lock
*.maintenance.lock
.env
.DS_Store
raw/
+21 -1
View File
@@ -55,7 +55,9 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工
| `source_records` / `sources` | 平台记录、原平台标题摘要与索引元数据、来源 URL 和抓取时间 |
| `venues` / `venue_aliases` / `editions` | 会议或期刊系列、年份与 track,区分主会、Findings 和 workshop,联合会议用别名查询,不重复生成事件 |
| `edition_sources` | 每届每个来源的覆盖状态、预计/已导入数量、错误与断点 |
| `submissions` / `decisions` | 多次投稿及带证据的决定历史,支持改投、撤稿和决定变更 |
| `submissions` / `submission_identifiers` / `submission_sources` | 投稿事件、按届次限定的原平台强标识、多来源证据;同稿的不同投稿尝试分别保留 |
| `decisions` / `decision_sources` | 明确决定、原始决定日期、来源观察日期及证据,不把缺失决定编造成拒稿 |
| `submission_merges` / `decision_merges` | 投稿事件与重复决定观察的合并审计,保留来源关联并避免重用已合并的编号 |
| `tags` / `paper_tags` | 平台标签与人工标签,使用命名空间隔离 |
| `match_candidates` / `paper_merges` | 潜在重复复核状态、确定合并审计 |
| `ingest_runs` | 导入批次、状态、错误和恢复位置 |
@@ -127,6 +129,24 @@ python3 -B collect_peerread.py --cache /data/paper-index/raw/http-cache \
数据任务以一次性 systemd 服务运行:`paper-atlas-arxiv-import`、`paper-atlas-pmlr-collect`、`paper-atlas-peerread-collect`;`paper-atlas-pmlr-verify` 用于全索引复核,`paper-atlas-conference-merge` 用于单写入合并。它们不是定时任务。主库只允许一个写入进程;会议采集先写独立 JSONL,待 arXiv 主批次结束后再合并;也可以使用 `merge_conferences.py` 在保存断点处暂停本项目的 arXiv unit、顺序合并会议清单并恢复。该操作应作为独立的一次性 systemd 任务运行,避免 SSH 断开影响恢复;不能同时启动多个合并任务。运行日志位于 `/data/paper-index/*-import.log`、`/data/paper-index/*-collect.log`。任何服务启动成功都不等于批次已完成,需检查批次状态、清单与完整性结果。标签按来源刷新使用 `paper_tags_source_evidence` 索引,避免每条论文都扫描整张标签表。运行中的 arXiv 可以通过同一协调脚本的 `--schema-only` 模式在断点更新索引后恢复;此模式要求已有 arXiv unit 正在运行。
### 投稿事件身份与旧数据回填
论文实体与投稿事件分开去重。同一篇论文可以在同一年向同一会议投稿多次,不能仅凭 paper_id、会议、标题相同就把投稿历史压成一次。跨来源只有原平台 forum ID、已验证的投稿编号,或正式发表稿的同届同 track 出版 DOI 等强证据才能关联同一事件。程序展示的内部编号仅在经过独立核验并带有 program_submission_number_verified 标记时才能用作投稿编号。
新版 ICLR manifest 的 parser_version 为 2,明确区别投稿编号、forum 和展示事件编号。旧版文件应先用缓存重新采集;事件证据回填只接受已入库、与 manifest 的会议/年份/track 和数量一致的文件。队列 JSON 包含 files 及对应的 file_sha256,文件变化后必须重新核验,不能继续使用旧哈希。
```sh
python3 -B merge_conferences.py --db /data/paper-index/papers.sqlite \
--archive /data/paper-index/raw/arxiv-metadata.zip \
--backfill-queue /data/paper-index/collected/submission-evidence-queue.json
```
协调器只维护本项目的 arXiv unit,并对维护过程加互斥锁。若 arXiv 正在运行,会保存断点、应用 schema、回填来源身份、导入本次附加文件、检查外键后恢复;若批次已经读至 EOF 并标记 complete,则不再启动它。若导入停下但未完成,会报错要求先调查,不以“unit 不存在”推断成功或重新全量运行。应以一次性 systemd 任务运行该命令,保持单写入。
首次升级会给已有投稿与决定补上来源关联;身份回填不重新导入标题摘要,不更改原抓取时间。确认同一次投稿后,可以合并镜像事件并保留所有决定及来源。当前稿件的原始强标识也会写入 source_records 元数据,后续源 URL 更新会同步保存。
通用 JSONL 导入支持 decision_events 数组及 observed_at、decision_date、submitted_at。没有取得明确决定时,可用 unknown 状态和空数组仅保存投稿,不生成虚假的决定;`--status unknown` 也能查询这些投稿。历史快照使用其实际观察时间,不能因今天导入就覆盖较新来源的状态;同一观察时间内按原始决定日期排序。反复导入相同证据只补来源关联,不重复制造决定事件。全量结束后仍须单独核验重复候选、冲突决定及来源覆盖,迁移检查不能代替最终全库审计。
### 固定版本的补充元数据快照
`download_snapshot.py` 流式下载到 `.part`,核对完整 HTTP 响应、预期大小和可信来源发布的校验和后,才发布正式文件及 manifest。已存在的正式文件会先重新核验;错误校验和不会覆盖它。默认保留 8 GiB 空闲空间,同一输出只允许一个下载进程。
+63
View File
@@ -0,0 +1,63 @@
"""Attach authoritative identities to existing events without reimporting text."""
import argparse
import hashlib
import json
from pathlib import Path
from index_core import Store
from submission_evidence import attach_submission,identities_for_row
def queue_files(path):
queue=json.loads(Path(path).read_text())
files=queue['files']
if len(files)!=len(set(files)):raise ValueError('Duplicate paths in evidence queue')
for filename in files:
if hashlib.sha256(Path(filename).read_bytes()).hexdigest()!=queue['file_sha256'][filename]:
raise ValueError('Evidence file differs from audited queue: '+filename)
return files
def backfill(db,files):
for filename in files:
manifest=json.loads(Path(filename).with_suffix('.manifest.json').read_text())
if manifest['source'] in ('colm','neurips','iclr_program') and manifest.get('parser_version',0)<2:
raise ValueError('Recollect old parser output before submission evidence backfill')
store=Store(db);c=store.conn;count=0
before=c.execute('SELECT count(*) FROM submissions').fetchone()[0]
try:
for filename in files:
path=Path(filename);manifest=json.loads(path.with_suffix('.manifest.json').read_text());seen=set()
with path.open() as stream:
for line in stream:
row=json.loads(line)
if row['external_id'] in seen:raise ValueError('Repeated source ID in evidence file')
seen.add(row['external_id'])
if any(row.get(key)!=manifest.get(key) for key in ('source','venue','year','track')):
raise ValueError('Evidence row differs from edition manifest')
record=c.execute('SELECT r.id,r.paper_id,r.metadata_json FROM source_records r JOIN sources s ON s.id=r.source_id WHERE s.name=? AND r.external_id=?',(row['source'],row['external_id'])).fetchone()
edition=c.execute('SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? AND e.year=? AND e.track=?',(row['venue'],row['year'],row['track'])).fetchone()
if not record or not edition:raise ValueError('Evidence backfill requires an already imported record and edition')
if not c.execute('SELECT 1 FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id WHERE ss.source_record_id=? AND ss.source_submission_id=? AND s.edition_id=?',(record[0],row['external_id'],edition[0])).fetchone():
raise ValueError('No existing submission to backfill')
attach_submission(store,edition[0],record[1],record[0],row['external_id'],identities_for_row(row))
metadata=json.loads(record[2]);metadata['record_identifiers_raw']=row.get('identifiers') or {}
if row['source']=='iclr_program':
for key in ('program_identity_type','program_submission_number_verified','program_identity_verification'):
if key in row['metadata']:metadata[key]=row['metadata'][key]
c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(metadata,ensure_ascii=False),record[0]))
count+=1
if len(seen)!=manifest['count']:raise ValueError('Evidence count differs from manifest')
conflict=c.execute('SELECT 1 FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id JOIN source_records r ON r.id=ss.source_record_id WHERE s.paper_id!=r.paper_id LIMIT 1').fetchone()
if conflict:raise ValueError('Submission evidence points to a different paper entity')
c.commit()
result={'status':'backfilled','records':count,'submissions_before':before,'submissions_after':c.execute('SELECT count(*) FROM submissions').fetchone()[0]}
print(json.dumps(result),flush=True);return result
except BaseException:
c.rollback();raise
finally:c.close()
if __name__=='__main__':
parser=argparse.ArgumentParser();parser.add_argument('--db',required=True);parser.add_argument('--queue',required=True)
args=parser.parse_args();backfill(args.db,queue_files(args.queue))
+2 -1
View File
@@ -152,6 +152,7 @@ def parse_program(data, abstracts, year, index_url):
'tags': [{'namespace': SOURCE+':'+kind, 'value': value}
for kind, values in [('topic', topics), ('keyword', keywords)] for value in values],
'metadata': {'program_event_ids': presentation_ids,
'program_identity_type': identity[0],
'presentation_titles': sorted({clean(r['name']) for r in group}),
'program_source_id': identity[1], 'presentation_types': sorted({str(r.get('event_type') or r.get('eventtype')) for r in group}),
'raw_decisions': sorted({str(r.get('decision')) for r in group if r.get('decision')}),
@@ -199,7 +200,7 @@ def collect_year(year, cache, out):
part = path.with_suffix('.part')
part.write_text(''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in records))
part.replace(path)
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICLR', 'year': year, 'track': 'main',
manifest = {'source': SOURCE, 'parser_version': 2, 'venue': 'ICLR', 'year': year, 'track': 'main',
'count': len(records), 'list_url': page, 'asset_urls': assets,
'abstract_asset_status': abstract_asset_status, 'file': str(path),
'status': 'partial' if audit['missing_abstract_count'] else 'collected', **audit}
+16 -1
View File
@@ -5,6 +5,7 @@ import time
from pathlib import Path
from index_core import Store
from submission_evidence import identities_for_row,timestamp
def ingest_file(db,path):
@@ -13,6 +14,8 @@ def ingest_file(db,path):
manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
if manifest.get('source') in ('colm','neurips') and manifest.get('parser_version',0)<2:
raise ValueError('Recollect proceedings with parser_version>=2 before identity matching')
if manifest.get('source')=='iclr_program' and manifest.get('parser_version',0)<2:
raise ValueError('Recollect ICLR with parser_version>=2 before submission identity matching')
store = Store(db)
c = store.conn
source_name = manifest.get('source')
@@ -41,7 +44,19 @@ def ingest_file(db,path):
if 'tags' in row:
store.record_tags(paper,record,source_name,row['tags'])
if row.get('status') in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):
edition = store.decision(paper,record,row['venue'],row['year'],row['track'],external,row.get('evidence_url',row['url']),row['status'],row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status']),row.get('venue_aliases'))
identities=identities_for_row(row)
events=row.get('decision_events')
if events is None:
events=[{'status':row['status'],'raw_value':row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status']),'evidence_url':row.get('evidence_url',row['url'])}]
if not isinstance(events,list):raise ValueError('Decision events must be a list')
if not events and row['status']!='unknown':raise ValueError('Known status requires decision evidence')
if row['status']!='unknown' and not any(event.get('status')==row['status'] for event in events):raise ValueError('Summary status has no corresponding decision evidence')
row_observed=row.get('observed_at') or timestamp()
for event in events or [{'status':None}]:
status=event['status']
if events and status is None:raise ValueError('Explicit decision event requires a status')
if status is not None and status not in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):raise ValueError('Unsupported decision event')
edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at'))
editions.add(edition)
else:
raise ValueError('This importer requires an explicit supported decision status')
+18 -6
View File
@@ -4,6 +4,7 @@ import re
import sqlite3
import unicodedata
from pathlib import Path
from submission_evidence import attach_submission,migrate_sources,write_decision
def normalized(value):
@@ -31,6 +32,7 @@ class Store:
self.conn.executescript(Path(__file__).with_name('schema.sql').read_text())
self.conn.execute('PRAGMA synchronous=NORMAL')
self.conn.execute('PRAGMA cache_size=-32768')
migrate_sources(self.conn)
def source(self,name,url):
self.conn.execute('INSERT OR IGNORE INTO sources(name,homepage) VALUES(?,?)',(name,url))
@@ -40,6 +42,12 @@ class Store:
c = self.conn
if target == other:
return
columns=('abstract','authors_json','first_published')
target_values=c.execute('SELECT abstract,authors_json,first_published FROM papers WHERE id=?',(target,)).fetchone()
other_values=c.execute('SELECT abstract,authors_json,first_published FROM papers WHERE id=?',(other,)).fetchone()
if not target_values or not other_values:raise ValueError('Missing paper entity during merge')
fill=[(column,value) for column,current,value in zip(columns,target_values,other_values) if current in (None,'','[]') and value not in (None,'','[]')]
if fill:c.execute('UPDATE papers SET '+','.join(column+'=?' for column,_ in fill)+' WHERE id=?',(*[value for _,value in fill],target))
c.execute('INSERT INTO paper_merges(former_paper_id,surviving_paper_id,reason) VALUES(?,?,?)',(other,target,reason))
c.execute('UPDATE identifiers SET paper_id=? WHERE paper_id=?',(target,other))
c.execute('UPDATE source_records SET paper_id=? WHERE paper_id=?',(target,other))
@@ -102,7 +110,8 @@ class Store:
paper = c.execute('INSERT INTO papers(title,abstract,title_key,authors_json,first_published) VALUES(?,?,?,?,?)',(title,abstract,key,json.dumps(authors,ensure_ascii=False),published)).lastrowid
for scheme,value in ids:
c.execute('INSERT OR IGNORE INTO identifiers VALUES(?,?,?)',(scheme,value,paper))
c.execute('INSERT INTO source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_id) DO UPDATE SET paper_id=excluded.paper_id,title=excluded.title,abstract=excluded.abstract,authors_json=excluded.authors_json,metadata_json=excluded.metadata_json,retrieved_at=CURRENT_TIMESTAMP',(source_id,external_id,paper,url,title,abstract,json.dumps(authors,ensure_ascii=False),json.dumps(metadata or {},ensure_ascii=False,default=str)))
metadata=dict(metadata or {});metadata['record_identifiers']=dict(ids)
c.execute('INSERT INTO source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_id) DO UPDATE SET paper_id=excluded.paper_id,url=excluded.url,title=excluded.title,abstract=excluded.abstract,authors_json=excluded.authors_json,metadata_json=excluded.metadata_json,retrieved_at=CURRENT_TIMESTAMP',(source_id,external_id,paper,url,title,abstract,json.dumps(authors,ensure_ascii=False),json.dumps(metadata,ensure_ascii=False,default=str)))
record = c.execute('SELECT id FROM source_records WHERE source_id=? AND external_id=?',(source_id,external_id)).fetchone()[0]
for other,_,_ in title_matches:
if other != paper and c.execute('SELECT 1 FROM papers WHERE id=?',(other,)).fetchone():
@@ -125,7 +134,7 @@ class Store:
tag=c.execute('SELECT id FROM tags WHERE namespace=? AND value=?',(namespace,value)).fetchone()[0]
c.execute('INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,evidence))
def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None):
def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None,identities=None,observed_at=None,decision_date=None,source_decision_id='',submitted_at=None):
c = self.conn
c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified'))
venue_id = c.execute('SELECT id FROM venues WHERE acronym=?',(venue,)).fetchone()[0]
@@ -133,10 +142,13 @@ class Store:
c.execute('INSERT OR IGNORE INTO venue_aliases VALUES(?,?)',(venue_id,alias))
c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)',(venue_id,int(year),track))
edition = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?',(venue_id,int(year),track)).fetchone()[0]
c.execute('INSERT OR IGNORE INTO submissions(paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?)',(paper,edition,record,external_id))
submission = c.execute('SELECT id FROM submissions WHERE source_record_id=? AND edition_id=? AND source_submission_id=?',(record,edition,external_id)).fetchone()[0]
if not c.execute('SELECT 1 FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=?',(submission,status,raw_value,url)).fetchone():
c.execute("INSERT INTO decisions(submission_id,status,raw_value,evidence_url,observed_at) VALUES(?,?,?,?,strftime('%Y-%m-%dT%H:%M:%fZ','now'))",(submission,status,raw_value,url))
# A previous event in this row may already have merged the input paper.
paper=c.execute('SELECT paper_id FROM source_records WHERE id=?',(record,)).fetchone()[0]
submission=attach_submission(self,edition,paper,record,external_id,identities or [])
if submitted_at:
from submission_evidence import timestamp
c.execute('UPDATE submissions SET submitted_at=COALESCE(submitted_at,?) WHERE id=?',(timestamp(submitted_at),submission))
if status is not None:write_decision(self,submission,record,status,raw_value,url,observed_at,decision_date,source_decision_id)
return edition
def accepted(self,paper,record,venue,year,track,external_id,url):
+34 -15
View File
@@ -6,12 +6,14 @@ Only the Paper Atlas arXiv unit is controlled; business services are untouched.
import argparse
from contextlib import closing
import json
import fcntl
import sqlite3
import subprocess
from pathlib import Path
from import_records import ingest_file
from index_core import Store
from backfill_submission_evidence import backfill,queue_files
UNIT='paper-atlas-arxiv-import.service'
@@ -21,29 +23,37 @@ def property_value(name):
return subprocess.check_output(['systemctl','show',UNIT,'-p',name,'--value'],text=True).strip()
def merge(db,archive,files):
def merge_locked(db,archive,files,backfill_queue=None):
# Validate inputs before stopping the existing writer.
for filename in files:
path=Path(filename)
if not path.is_file() or not path.with_suffix('.manifest.json').is_file():
raise ValueError('Missing collected file or manifest: '+filename)
evidence_files=queue_files(backfill_queue) if backfill_queue else []
if not Path(archive).is_file():
raise ValueError('Missing arXiv archive')
if not Path(db).is_file():
raise ValueError('Missing existing database')
if property_value('ActiveState')!='active':
raise RuntimeError('Expected the existing arXiv unit to be active; inspect it before proceeding')
subprocess.run(['systemctl','stop',UNIT],check=True)
resume_arxiv=property_value('ActiveState')=='active'
if not resume_arxiv:
with closing(sqlite3.connect(Path(db).resolve().as_uri()+'?mode=ro',uri=True)) as c:
last=c.execute("SELECT status FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone()
if not last or last[0]!='complete':
raise RuntimeError('arXiv is neither active nor verified complete; inspect its state before maintenance')
try:
if resume_arxiv:subprocess.run(['systemctl','stop',UNIT],check=True)
with closing(sqlite3.connect(db)) as c, c:
checkpoint=c.execute("SELECT records_written FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone()
checkpoint=c.execute("SELECT records_written,status FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone()
print('Saved arxiv checkpoint:',checkpoint,flush=True)
if checkpoint and checkpoint[1]=='complete':resume_arxiv=False
schema_store=Store(db)
schema_store.conn.close()
if evidence_files:backfill(db,evidence_files)
for filename in files:
ingest_file(db,filename)
with closing(sqlite3.connect(db)) as c, c:
if files:
c.execute('PRAGMA cache_size=-32768')
if files or evidence_files:
errors=c.execute('PRAGMA foreign_key_check').fetchall()
if errors:
raise RuntimeError('Foreign key violations after merge')
@@ -51,13 +61,21 @@ def merge(db,archive,files):
print('Tag refresh plan:',c.execute('EXPLAIN QUERY PLAN DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key=?',(1,'arxiv:categories')).fetchall(),flush=True)
print('Decision totals:',c.execute('SELECT status,count(*) FROM decisions GROUP BY status').fetchall(),flush=True)
finally:
if property_value('LoadState')=='loaded':
subprocess.run(['systemctl','start',UNIT],check=True)
else:
subprocess.run(['systemd-run','--unit='+UNIT,'--property=WorkingDirectory='+str(Path(__file__).parent),'--property=CPUQuota=50%','--property=MemoryMax=512M','--property=Nice=15','--property=IOWeight=10','--property=StandardOutput=append:/data/paper-index/arxiv-import.log','--property=StandardError=append:/data/paper-index/arxiv-import.log','/usr/bin/env','PYTHONDONTWRITEBYTECODE=1','python3','-B',str(Path(__file__).with_name('import_arxiv.py')),'--db',db,'--archive',archive],check=True)
if property_value('ActiveState')!='active':
raise RuntimeError('arXiv unit did not resume; inspect the logs')
print('ARXIV_RESUMED',flush=True)
if resume_arxiv:
if property_value('LoadState')=='loaded':
subprocess.run(['systemctl','start',UNIT],check=True)
else:
subprocess.run(['systemd-run','--unit='+UNIT,'--property=WorkingDirectory='+str(Path(__file__).parent),'--property=CPUQuota=50%','--property=MemoryMax=512M','--property=Nice=15','--property=IOWeight=10','--property=StandardOutput=append:/data/paper-index/arxiv-import.log','--property=StandardError=append:/data/paper-index/arxiv-import.log','/usr/bin/env','PYTHONDONTWRITEBYTECODE=1','python3','-B',str(Path(__file__).with_name('import_arxiv.py')),'--db',db,'--archive',archive],check=True)
if property_value('ActiveState')!='active':
raise RuntimeError('arXiv unit did not resume; inspect the logs')
print('ARXIV_RESUMED',flush=True)
else:print('ARXIV_ALREADY_COMPLETE',flush=True)
def merge(db,archive,files,backfill_queue=None):
with Path(str(db)+'.maintenance.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return merge_locked(db,archive,files,backfill_queue)
if __name__=='__main__':
@@ -65,8 +83,9 @@ if __name__=='__main__':
parser.add_argument('--db',required=True)
parser.add_argument('--archive',required=True)
parser.add_argument('--schema-only',action='store_true',help='Apply idempotent schema/index updates at a saved checkpoint')
parser.add_argument('--backfill-queue',help='Audited existing-source queue for canonical submission evidence')
parser.add_argument('files',nargs='*')
args=parser.parse_args()
if not args.files and not args.schema_only:
if not args.files and not args.schema_only and not args.backfill_queue:
parser.error('Provide collected files or --schema-only')
merge(args.db,args.archive,args.files)
merge(args.db,args.archive,args.files,args.backfill_queue)
+17 -2
View File
@@ -48,7 +48,15 @@ def main():
if args.year:
clause.append('e.year=?'); params.append(args.year)
if args.status:
clause.append('EXISTS(SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=? AND NOT EXISTS(SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id AND (newer.observed_at>d.observed_at OR (newer.observed_at=d.observed_at AND newer.id>d.id))))'); params.append(args.status)
latest='''EXISTS (
SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=?
AND NOT EXISTS (
SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id
AND (newer.observed_at,COALESCE(newer.decision_date,newer.observed_at),newer.id)
> (d.observed_at,COALESCE(d.decision_date,d.observed_at),d.id)
)
)'''
clause.append('(NOT EXISTS(SELECT 1 FROM decisions d WHERE d.submission_id=s.id) OR '+latest+')' if args.status=='unknown' else latest); params.append(args.status)
where.append('p.id IN (SELECT s.paper_id FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+' AND '.join(clause)+')')
sql = 'SELECT p.* FROM papers p'+(' WHERE '+' AND '.join(where) if where else '')+' ORDER BY p.id LIMIT ?'
params.append(max(1,min(args.limit,200)))
@@ -63,7 +71,14 @@ def main():
submissions = []
for sub in conn.execute('SELECT s.id,s.source_submission_id,s.submitted_at,v.acronym,e.year,e.track FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE s.paper_id=? ORDER BY e.year,s.id',(row['id'],)):
submission = dict(sub)
submission['decisions'] = [dict(d) for d in conn.execute('SELECT status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=? ORDER BY observed_at,id',(sub['id'],))]
submission['decisions'] = []
for d in conn.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=? ORDER BY observed_at,COALESCE(decision_date,observed_at),id',(sub['id'],)):
decision=dict(d)
if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='decision_sources'").fetchone():
decision['sources']=[dict(r) for r in conn.execute('SELECT s.name,r.external_id,ds.source_decision_id FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?',(d['id'],))]
submission['decisions'].append(decision)
if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='submission_sources'").fetchone():
submission['sources']=[dict(r) for r in conn.execute('SELECT so.name,r.external_id,ss.source_submission_id FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id JOIN sources so ON so.id=r.source_id WHERE ss.submission_id=?',(sub['id'],))]
submissions.append(submission)
paper['submissions'] = submissions
result.append(paper)
+27
View File
@@ -58,6 +58,33 @@ CREATE TABLE IF NOT EXISTS decisions (
decision_date TEXT, UNIQUE(submission_id,status,raw_value,evidence_url,observed_at)
);
CREATE INDEX IF NOT EXISTS decisions_submission_order ON decisions(submission_id,observed_at DESC,id DESC);
CREATE TABLE IF NOT EXISTS submission_sources (
submission_id INTEGER NOT NULL REFERENCES submissions(id),
source_record_id INTEGER NOT NULL REFERENCES source_records(id), source_submission_id TEXT NOT NULL,
PRIMARY KEY(submission_id,source_record_id,source_submission_id)
);
CREATE INDEX IF NOT EXISTS submission_sources_record ON submission_sources(source_record_id,source_submission_id);
CREATE TABLE IF NOT EXISTS submission_identifiers (
edition_id INTEGER NOT NULL REFERENCES editions(id), scheme TEXT NOT NULL, value TEXT NOT NULL,
submission_id INTEGER NOT NULL REFERENCES submissions(id), PRIMARY KEY(edition_id,scheme,value)
);
CREATE INDEX IF NOT EXISTS submission_identifiers_submission ON submission_identifiers(submission_id);
CREATE TABLE IF NOT EXISTS decision_sources (
decision_id INTEGER NOT NULL REFERENCES decisions(id),
source_record_id INTEGER NOT NULL REFERENCES source_records(id), source_decision_id TEXT NOT NULL DEFAULT '',
PRIMARY KEY(decision_id,source_record_id,source_decision_id)
);
CREATE TABLE IF NOT EXISTS submission_merges (
former_submission_id INTEGER PRIMARY KEY, surviving_submission_id INTEGER NOT NULL REFERENCES submissions(id),
reason TEXT NOT NULL, merged_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS decision_merges (
former_decision_id INTEGER PRIMARY KEY, surviving_decision_id INTEGER NOT NULL REFERENCES decisions(id),
reason TEXT NOT NULL, merged_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS schema_migrations (
name TEXT PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS tags (
id INTEGER PRIMARY KEY, namespace TEXT NOT NULL, value TEXT NOT NULL,
description TEXT, UNIQUE(namespace,value)
+110
View File
@@ -0,0 +1,110 @@
"""Canonical submission events with independent source and decision evidence."""
from datetime import datetime, timezone
import re
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','cvf','ecva','dblp'}
def timestamp(value=None):
date=datetime.now(timezone.utc) if value is None else datetime.fromisoformat(value.replace('Z','+00:00'))
if date.tzinfo is None:raise ValueError('Decision timestamp requires a timezone')
return date.astimezone(timezone.utc).isoformat(timespec='milliseconds').replace('+00:00','Z')
def identities_for_row(row):
identities=list(row.get('submission_identifiers') or [])
own=row.get('identifiers') or {}
if own.get('openreview'):
identities.append({'scheme':'openreview','value':own['openreview']})
if row['source'] in PUBLICATIONS and row.get('status')=='accepted' and own.get('doi'):
doi=re.sub(r'^https?://(?:dx\.)?doi\.org/','',str(own['doi']),flags=re.I).strip().lower()
if re.fullmatch(r'10\.\d{4,9}/\S+',doi):identities.append({'scheme':'publication_doi','value':doi})
meta=row.get('metadata') or {}
if row['source']=='iclr_program' and row['venue']=='ICLR' and meta.get('program_identity_type')=='submission' and meta.get('program_submission_number_verified') is True:
identities.append({'scheme':'iclr_number','value':str(meta['program_source_id'])})
result=set()
for identity in identities:
scheme,value=identity['scheme'],str(identity['value']).strip()
if scheme=='openreview':
valid=re.fullmatch(r'[A-Za-z0-9_-]+',value) and not re.match(r'20\d{2}-Oral-',value)
elif scheme=='iclr_number':valid=re.fullmatch(r'[1-9]\d*',value) and row['venue']=='ICLR'
elif scheme=='publication_doi':valid=re.fullmatch(r'10\.\d{4,9}/\S+',value)
else:valid=False
if not valid:raise ValueError('Invalid authoritative submission identity')
result.add((scheme,value))
return sorted(result)
def migrate_sources(c):
if c.execute("SELECT 1 FROM schema_migrations WHERE name='submission_source_baseline_v1'").fetchone():return
with c:
c.execute('INSERT OR IGNORE INTO submission_sources SELECT id,source_record_id,source_submission_id FROM submissions')
c.execute("INSERT OR IGNORE INTO decision_sources SELECT d.id,s.source_record_id,'' FROM decisions d JOIN submissions s ON s.id=d.submission_id")
c.execute("INSERT INTO schema_migrations(name) VALUES('submission_source_baseline_v1')")
def next_id(c,table,audit,former):
# Merged IDs are kept in audit; never allocate them to a later unrelated event.
return max(c.execute('SELECT COALESCE(max(id),0) FROM '+table).fetchone()[0],c.execute('SELECT COALESCE(max('+former+'),0) FROM '+audit).fetchone()[0])+1
def merge_submissions(c,target,other):
if target==other:return
editions=c.execute('SELECT id,edition_id FROM submissions WHERE id IN (?,?)',(target,other)).fetchall()
if len(editions)!=2 or editions[0][1]!=editions[1][1]:raise ValueError('Cannot combine submissions from different editions')
c.execute('INSERT OR IGNORE INTO submission_sources SELECT ?,source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(target,other))
c.execute('DELETE FROM submission_sources WHERE submission_id=?',(other,))
for decision,status,raw,url,observed,date in c.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=?',(other,)).fetchall():
duplicate=c.execute('SELECT id,decision_date FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=? AND observed_at=?',(target,status,raw,url,observed)).fetchone()
if duplicate:
surviving=duplicate[0]
if date and duplicate[1] and date!=duplicate[1]:raise ValueError('Conflicting dates for the same decision observation')
if date and not duplicate[1]:c.execute('UPDATE decisions SET decision_date=? WHERE id=?',(date,surviving))
c.execute('INSERT OR IGNORE INTO decision_sources SELECT ?,source_record_id,source_decision_id FROM decision_sources WHERE decision_id=?',(surviving,decision))
c.execute('DELETE FROM decision_sources WHERE decision_id=?',(decision,))
c.execute('UPDATE decision_merges SET surviving_decision_id=? WHERE surviving_decision_id=?',(surviving,decision))
c.execute("INSERT INTO decision_merges(former_decision_id,surviving_decision_id,reason) VALUES(?,?,'same decision observation on merged submission')",(decision,surviving))
c.execute('DELETE FROM decisions WHERE id=?',(decision,))
else:c.execute('UPDATE decisions SET submission_id=? WHERE id=?',(target,decision))
c.execute('UPDATE submission_identifiers SET submission_id=? WHERE submission_id=?',(target,other))
c.execute('UPDATE submission_merges SET surviving_submission_id=? WHERE surviving_submission_id=?',(target,other))
c.execute("INSERT INTO submission_merges(former_submission_id,surviving_submission_id,reason) VALUES(?,?,'shared authoritative submission identity')",(other,target))
c.execute('DELETE FROM submissions WHERE id=?',(other,))
def attach_submission(store,edition,paper,record,external_id,identities):
c=store.conn
matches={r[0] for r in c.execute('SELECT s.id FROM submission_sources p JOIN submissions s ON s.id=p.submission_id WHERE p.source_record_id=? AND p.source_submission_id=? AND s.edition_id=?',(record,external_id,edition))}
matches.update(r[0] for r in c.execute('SELECT id FROM submissions WHERE source_record_id=? AND source_submission_id=? AND edition_id=?',(record,external_id,edition)))
for scheme,value in identities:
found=c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?',(edition,scheme,value)).fetchone()
if found:matches.add(found[0])
if matches:
papers={paper}|{c.execute('SELECT paper_id FROM submissions WHERE id=?',(sid,)).fetchone()[0] for sid in matches}
surviving=min(papers)
for other in sorted(papers-{surviving}):store.merge(surviving,other,'shared authoritative conference submission identity')
submission=min(matches)
for other in sorted(matches-{submission}):merge_submissions(c,submission,other)
c.execute('UPDATE submissions SET paper_id=? WHERE id=?',(surviving,submission))
else:
submission=next_id(c,'submissions','submission_merges','former_submission_id')
c.execute('INSERT INTO submissions(id,paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?,?)',(submission,paper,edition,record,external_id))
c.execute('INSERT OR IGNORE INTO submission_sources VALUES(?,?,?)',(submission,record,external_id))
for scheme,value in identities:
c.execute('INSERT OR IGNORE INTO submission_identifiers VALUES(?,?,?,?)',(edition,scheme,value,submission))
return submission
def write_decision(store,submission,record,status,raw,url,observed_at=None,decision_date=None,source_decision_id=''):
c=store.conn
observed=timestamp(observed_at)
date=timestamp(decision_date) if decision_date else None
# Re-observing unchanged evidence adds provenance, not a new decision event.
old=c.execute('SELECT id FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=? AND COALESCE(decision_date,\'\')=COALESCE(?,\'\')',(submission,status,raw,url,date)).fetchone()
if old:decision=old[0]
else:
decision=next_id(c,'decisions','decision_merges','former_decision_id')
c.execute('INSERT INTO decisions(id,submission_id,status,raw_value,evidence_url,observed_at,decision_date) VALUES(?,?,?,?,?,?,?)',(decision,submission,status,raw,url,observed,date))
c.execute('INSERT OR IGNORE INTO decision_sources VALUES(?,?,?)',(decision,record,source_decision_id or ''))
return decision
+1 -1
View File
@@ -86,7 +86,7 @@ class ICLRTests(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmp:
path=Path(tmp)/'iclr.jsonl';db=Path(tmp)/'papers.sqlite'
path.write_text(''.join(json.dumps(r)+'\n' for r in rows))
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','list_url':'https://iclr.cc/virtual/2026/papers.html','count':1,'status':'collected'}))
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','parser_version':2,'list_url':'https://iclr.cc/virtual/2026/papers.html','count':1,'status':'collected'}))
ingest_file(db,path);ingest_file(db,path)
store=Store(db);self.addCleanup(store.conn.close)
self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
+1
View File
@@ -56,6 +56,7 @@ class StoreTests(unittest.TestCase):
updated,_=self.store.record('arxiv','2601.00001','https://arxiv.org/abs/2601.00001','Updated research title','New searchable content',['Jane Doe'])
self.assertEqual(updated,paper)
self.assertEqual(self.store.conn.execute('SELECT title FROM papers').fetchone()[0],'Updated research title')
self.assertEqual(self.store.conn.execute('SELECT url FROM source_records').fetchone()[0],'https://arxiv.org/abs/2601.00001')
self.assertEqual(self.store.conn.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'searchable'").fetchone()[0],1)
def test_tag_refresh_keeps_other_record_evidence_and_custom_tags(self):
+41
View File
@@ -0,0 +1,41 @@
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from index_core import Store
from merge_conferences import merge
class MaintenanceTests(unittest.TestCase):
def fixture(self,tmp,status):
db=Path(tmp)/'papers.sqlite';archive=Path(tmp)/'metadata.zip';archive.write_bytes(b'not read by schema-only maintenance')
store=Store(db);source=store.source('arxiv','https://arxiv.org')
store.conn.execute('INSERT INTO ingest_runs(source_id,status) VALUES(?,?)',(source,status));store.conn.commit();store.conn.close()
return db,archive
def test_completed_archive_is_not_started_again_after_maintenance(self):
with tempfile.TemporaryDirectory() as tmp:
db,archive=self.fixture(tmp,'complete')
with patch('merge_conferences.property_value',return_value='inactive'),patch('merge_conferences.subprocess.run') as run:
merge(db,archive,[])
run.assert_not_called()
def test_inactive_incomplete_arxiv_requires_investigation(self):
with tempfile.TemporaryDirectory() as tmp:
db,archive=self.fixture(tmp,'failed')
with patch('merge_conferences.property_value',return_value='inactive'),patch('merge_conferences.subprocess.run') as run:
with self.assertRaisesRegex(RuntimeError,'neither active nor verified complete'):
merge(db,archive,[])
run.assert_not_called()
def test_arxiv_finishing_at_the_stop_boundary_is_not_restarted(self):
with tempfile.TemporaryDirectory() as tmp:
db,archive=self.fixture(tmp,'complete')
with patch('merge_conferences.property_value',return_value='active'),patch('merge_conferences.subprocess.run') as run:
merge(db,archive,[])
self.assertEqual(run.call_args_list[0].args[0],['systemctl','stop','paper-atlas-arxiv-import.service'])
self.assertEqual(run.call_count,1)
if __name__=='__main__':unittest.main()
+163
View File
@@ -0,0 +1,163 @@
import json
from pathlib import Path
import sqlite3
import subprocess
import sys
import tempfile
import unittest
from import_records import ingest_file
from index_core import Store
from submission_evidence import identities_for_row
from backfill_submission_evidence import backfill,queue_files
class SubmissionTests(unittest.TestCase):
def setUp(self):
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
self.db=Path(self.tmp.name)/'papers.sqlite';self.store=Store(self.db);self.addCleanup(self.store.conn.close)
def record(self,source,external,authors):
return self.store.record(source,external,'https://example.org/'+external,'A long paper about reliable metadata and historical events','Abstract',authors)
def decision(self,paper,record,external,identities,status='accepted',raw='Accept',url='https://example.org/decision',observed='2026-05-08T00:00:00Z',date=None):
return self.store.decision(paper,record,'ICLR',2026,'main',external,url,status,raw,identities=identities,observed_at=observed,decision_date=date)
def query(self,*args):
self.store.conn.commit()
command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(self.db),*args]
result=subprocess.run(command,capture_output=True,text=True)
self.assertEqual(result.returncode,0,result.stderr)
return json.loads(result.stdout)
def test_same_submission_across_sources_has_one_event_and_both_provenances(self):
first,one=self.store.record('iclr_program','2026/number-42','https://example.org/program','A long paper about reliable metadata and historical events',None,['Jane Doe','John Smith'])
second,two=self.record('openreview_snapshot','realForum',['Anonymous'])
self.assertNotEqual(first,second)
self.decision(first,one,'2026/number-42',[('iclr_number','42')])
self.decision(second,two,'realForum',[('iclr_number','42'),('openreview','realForum')],url='https://openreview.net/forum?id=realForum&noteId=note1')
c=self.store.conn
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
self.assertEqual(c.execute('SELECT abstract FROM papers').fetchone()[0],'Abstract')
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0],2)
details=self.query('--paper',str(first))[0]['submissions'][0]
self.assertEqual({r['name'] for r in details['sources']},{'iclr_program','openreview_snapshot'})
self.assertEqual(len(details['decisions']),2)
self.assertTrue(all(d['sources'] for d in details['decisions']))
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
def test_same_paper_can_have_two_distinct_submission_attempts(self):
paper,one=self.store.record('openreview_snapshot','forum1','https://example.org/1','First version','Abstract',[],{'arxiv':'2601.00001','openreview':'forum1'})
other,two=self.store.record('openreview_snapshot','forum2','https://example.org/2','Another version','Abstract',[],{'arxiv':'2601.00001','openreview':'forum2'})
self.assertEqual(paper,other)
self.decision(paper,one,'forum1',[('openreview','forum1'),('iclr_number','1')],status='rejected',raw='Reject')
self.decision(paper,two,'forum2',[('openreview','forum2'),('iclr_number','2')])
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],2)
def test_late_identity_bridge_preserves_decisions_and_does_not_reuse_merged_ids(self):
first,one=self.record('iclr_program','number-42',['Jane Doe','John Smith'])
second,two=self.record('openreview_snapshot','realForum',['Anonymous'])
self.decision(first,one,'number-42',[('iclr_number','42')])
self.decision(second,two,'realForum',[('openreview','realForum')])
self.decision(second,two,'realForum',[('openreview','realForum'),('iclr_number','42')])
c=self.store.conn
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM decision_sources').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM submission_merges').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM decision_merges').fetchone()[0],1)
third,record=self.record('other','different',['Other Author'])
self.decision(third,record,'different',[('openreview','differentForum')],status='rejected',raw='Reject')
self.assertGreater(c.execute('SELECT max(id) FROM submissions').fetchone()[0],2)
self.assertGreater(c.execute('SELECT max(id) FROM decisions').fetchone()[0],2)
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
def test_old_snapshot_does_not_override_a_newer_official_observation(self):
paper,one=self.record('iclr_program','number-42',['Jane Doe','John Smith'])
_,two=self.record('openreview_snapshot','realForum',['Jane Doe','John Smith'])
self.decision(paper,one,'number-42',[('iclr_number','42')],observed='2026-11-06T00:00:00Z')
self.decision(paper,two,'realForum',[('iclr_number','42')],status='rejected',raw='Reject',observed='2026-05-08T00:00:00Z')
self.assertEqual(len(self.query('--venue','ICLR','--status','accepted')),1)
self.assertEqual(self.query('--venue','ICLR','--status','rejected'),[])
def test_decision_date_orders_events_with_the_same_snapshot_time(self):
paper,record=self.record('openreview_snapshot','realForum',['Jane Doe'])
self.decision(paper,record,'realForum',[('openreview','realForum')],date='2026-02-01T00:00:00Z')
self.decision(paper,record,'realForum',[('openreview','realForum')],status='rejected',raw='Reject',date='2026-01-01T00:00:00Z')
self.assertEqual(len(self.query('--venue','ICLR','--status','accepted')),1)
self.assertEqual(self.query('--venue','ICLR','--status','rejected'),[])
def test_unknown_without_a_decision_is_queryable_without_invented_verdict(self):
path=Path(self.tmp.name)/'unknown.jsonl'
row={'source':'openreview_snapshot','external_id':'realForum','url':'https://example.org/paper','title':'Unknown result','abstract':'Abstract','authors':[],'identifiers':{'openreview':'realForum'},'venue':'ICLR','year':2026,'track':'main','status':'unknown','decision_events':[]}
path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':row['source'],'list_url':'https://example.org/list','count':1}))
ingest_file(self.db,path)
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM decisions').fetchone()[0],0)
self.assertEqual(len(self.query('--venue','ICLR','--status','unknown')),1)
def test_known_summary_without_corresponding_decision_rolls_back(self):
path=Path(self.tmp.name)/'missing-evidence.jsonl'
row={'source':'openreview_snapshot','external_id':'realForum','url':'https://example.org/paper','title':'Unsupported result','abstract':'Abstract','authors':[],'venue':'ICLR','year':2026,'track':'main','status':'accepted','decision_events':[{'status':None}]}
path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':row['source'],'list_url':'https://example.org/list','count':1}))
with self.assertRaisesRegex(ValueError,'no corresponding decision evidence'):
ingest_file(self.db,path)
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],0)
def test_event_number_is_not_assumed_to_be_a_submission_number(self):
row={'source':'iclr_program','venue':'ICLR','metadata':{'program_source_id':42,'program_identity_type':'event'},'identifiers':{}}
self.assertEqual(identities_for_row(row),[])
row['metadata']['program_identity_type']='submission'
self.assertEqual(identities_for_row(row),[])
row['metadata']['program_submission_number_verified']=True
self.assertEqual(identities_for_row(row),[('iclr_number','42')])
with self.assertRaisesRegex(ValueError,'Invalid authoritative'):
identities_for_row({'source':'snapshot','venue':'ICLR','submission_identifiers':[{'scheme':'openreview','value':'2026-Oral--42-placeholder'}]})
def test_legacy_database_migration_preserves_source_and_decision_links(self):
legacy=Path(self.tmp.name)/'legacy.sqlite';sql=Path(__file__).with_name('schema.sql').read_text()
sql=sql[:sql.index('CREATE TABLE IF NOT EXISTS submission_sources')]+sql[sql.index('CREATE TABLE IF NOT EXISTS tags'):]
c=sqlite3.connect(legacy);c.executescript(sql)
c.execute("INSERT INTO sources VALUES(1,'legacy','https://example.org')")
c.execute("INSERT INTO papers(id,title,title_key) VALUES(1,'Paper','paper')")
c.execute("INSERT INTO source_records VALUES(1,1,'paper',1,'https://example.org/p','Paper','Abstract','[]','{}',CURRENT_TIMESTAMP)")
c.execute("INSERT INTO venues VALUES(1,'ICLR','ICLR','machine_learning','conference')")
c.execute("INSERT INTO editions VALUES(1,1,2026,'main')")
c.execute("INSERT INTO submissions VALUES(1,1,1,1,'paper',NULL)")
c.execute("INSERT INTO decisions VALUES(1,1,'accepted','Accept','https://example.org/decision','2026-01-01T00:00:00Z',NULL)")
c.commit();c.close()
store=Store(legacy);self.addCleanup(store.conn.close)
self.assertEqual(store.conn.execute('SELECT * FROM submission_sources').fetchall(),[(1,1,'paper')])
self.assertEqual(store.conn.execute('SELECT * FROM decision_sources').fetchall(),[(1,1,'')])
self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(),[])
def test_existing_program_identity_backfill_is_idempotent_and_preserves_text(self):
paper,record=self.record('iclr_program','2026/program-42',['Jane Doe','John Smith'])
self.decision(paper,record,'2026/program-42',[])
self.store.conn.commit()
original=self.store.conn.execute('SELECT title,abstract,retrieved_at FROM source_records WHERE id=?',(record,)).fetchone()
path=Path(self.tmp.name)/'program.jsonl'
row={'source':'iclr_program','external_id':'2026/program-42','venue':'ICLR','year':2026,'track':'main','status':'accepted','identifiers':{},'metadata':{'program_identity_type':'submission','program_source_id':42,'program_submission_number_verified':True}}
path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','parser_version':2,'venue':'ICLR','year':2026,'track':'main','count':1}))
first=backfill(self.db,[path]);second=backfill(self.db,[path])
self.assertEqual(first,second)
self.assertEqual(self.store.conn.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),[('iclr_number','42')])
self.assertEqual(self.store.conn.execute('SELECT title,abstract,retrieved_at FROM source_records WHERE id=?',(record,)).fetchone(),original)
other,snapshot=self.record('openreview_snapshot','realForum',['Anonymous'])
self.decision(other,snapshot,'realForum',[('iclr_number','42'),('openreview','realForum')])
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
def test_evidence_queue_rejects_changed_file_before_mutation(self):
path=Path(self.tmp.name)/'program.jsonl';path.write_text('changed bytes')
queue=Path(self.tmp.name)/'queue.json';queue.write_text(json.dumps({'files':[str(path)],'file_sha256':{str(path):'0'*64}}))
with self.assertRaisesRegex(ValueError,'differs from audited'):
queue_files(queue)
def test_old_untyped_program_files_require_recollection(self):
path=Path(self.tmp.name)/'old.jsonl';path.write_text('')
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','parser_version':1}))
with self.assertRaisesRegex(ValueError,'Recollect ICLR'):
ingest_file(self.db,path)
if __name__=='__main__':unittest.main()