diff --git a/.gitignore b/.gitignore index f0d97c8..b0a58ff 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ *.part *.part.json *.download.lock +*.maintenance.lock .env .DS_Store raw/ diff --git a/README.md b/README.md index 9d5f8f2..73b9b4e 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,9 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 | `source_records` / `sources` | 平台记录、原平台标题摘要与索引元数据、来源 URL 和抓取时间 | | `venues` / `venue_aliases` / `editions` | 会议或期刊系列、年份与 track,区分主会、Findings 和 workshop,联合会议用别名查询,不重复生成事件 | | `edition_sources` | 每届每个来源的覆盖状态、预计/已导入数量、错误与断点 | -| `submissions` / `decisions` | 多次投稿及带证据的决定历史,支持改投、撤稿和决定变更 | +| `submissions` / `submission_identifiers` / `submission_sources` | 投稿事件、按届次限定的原平台强标识、多来源证据;同稿的不同投稿尝试分别保留 | +| `decisions` / `decision_sources` | 明确决定、原始决定日期、来源观察日期及证据,不把缺失决定编造成拒稿 | +| `submission_merges` / `decision_merges` | 投稿事件与重复决定观察的合并审计,保留来源关联并避免重用已合并的编号 | | `tags` / `paper_tags` | 平台标签与人工标签,使用命名空间隔离 | | `match_candidates` / `paper_merges` | 潜在重复复核状态、确定合并审计 | | `ingest_runs` | 导入批次、状态、错误和恢复位置 | @@ -127,6 +129,24 @@ python3 -B collect_peerread.py --cache /data/paper-index/raw/http-cache \ 数据任务以一次性 systemd 服务运行:`paper-atlas-arxiv-import`、`paper-atlas-pmlr-collect`、`paper-atlas-peerread-collect`;`paper-atlas-pmlr-verify` 用于全索引复核,`paper-atlas-conference-merge` 用于单写入合并。它们不是定时任务。主库只允许一个写入进程;会议采集先写独立 JSONL,待 arXiv 主批次结束后再合并;也可以使用 `merge_conferences.py` 在保存断点处暂停本项目的 arXiv unit、顺序合并会议清单并恢复。该操作应作为独立的一次性 systemd 任务运行,避免 SSH 断开影响恢复;不能同时启动多个合并任务。运行日志位于 `/data/paper-index/*-import.log`、`/data/paper-index/*-collect.log`。任何服务启动成功都不等于批次已完成,需检查批次状态、清单与完整性结果。标签按来源刷新使用 `paper_tags_source_evidence` 索引,避免每条论文都扫描整张标签表。运行中的 arXiv 可以通过同一协调脚本的 `--schema-only` 模式在断点更新索引后恢复;此模式要求已有 arXiv unit 正在运行。 +### 投稿事件身份与旧数据回填 + +论文实体与投稿事件分开去重。同一篇论文可以在同一年向同一会议投稿多次,不能仅凭 paper_id、会议、标题相同就把投稿历史压成一次。跨来源只有原平台 forum ID、已验证的投稿编号,或正式发表稿的同届同 track 出版 DOI 等强证据才能关联同一事件。程序展示的内部编号仅在经过独立核验并带有 program_submission_number_verified 标记时才能用作投稿编号。 + +新版 ICLR manifest 的 parser_version 为 2,明确区别投稿编号、forum 和展示事件编号。旧版文件应先用缓存重新采集;事件证据回填只接受已入库、与 manifest 的会议/年份/track 和数量一致的文件。队列 JSON 包含 files 及对应的 file_sha256,文件变化后必须重新核验,不能继续使用旧哈希。 + +```sh +python3 -B merge_conferences.py --db /data/paper-index/papers.sqlite \ + --archive /data/paper-index/raw/arxiv-metadata.zip \ + --backfill-queue /data/paper-index/collected/submission-evidence-queue.json +``` + +协调器只维护本项目的 arXiv unit,并对维护过程加互斥锁。若 arXiv 正在运行,会保存断点、应用 schema、回填来源身份、导入本次附加文件、检查外键后恢复;若批次已经读至 EOF 并标记 complete,则不再启动它。若导入停下但未完成,会报错要求先调查,不以“unit 不存在”推断成功或重新全量运行。应以一次性 systemd 任务运行该命令,保持单写入。 + +首次升级会给已有投稿与决定补上来源关联;身份回填不重新导入标题摘要,不更改原抓取时间。确认同一次投稿后,可以合并镜像事件并保留所有决定及来源。当前稿件的原始强标识也会写入 source_records 元数据,后续源 URL 更新会同步保存。 + +通用 JSONL 导入支持 decision_events 数组及 observed_at、decision_date、submitted_at。没有取得明确决定时,可用 unknown 状态和空数组仅保存投稿,不生成虚假的决定;`--status unknown` 也能查询这些投稿。历史快照使用其实际观察时间,不能因今天导入就覆盖较新来源的状态;同一观察时间内按原始决定日期排序。反复导入相同证据只补来源关联,不重复制造决定事件。全量结束后仍须单独核验重复候选、冲突决定及来源覆盖,迁移检查不能代替最终全库审计。 + ### 固定版本的补充元数据快照 `download_snapshot.py` 流式下载到 `.part`,核对完整 HTTP 响应、预期大小和可信来源发布的校验和后,才发布正式文件及 manifest。已存在的正式文件会先重新核验;错误校验和不会覆盖它。默认保留 8 GiB 空闲空间,同一输出只允许一个下载进程。 diff --git a/backfill_submission_evidence.py b/backfill_submission_evidence.py new file mode 100644 index 0000000..6f84430 --- /dev/null +++ b/backfill_submission_evidence.py @@ -0,0 +1,63 @@ +"""Attach authoritative identities to existing events without reimporting text.""" +import argparse +import hashlib +import json +from pathlib import Path + +from index_core import Store +from submission_evidence import attach_submission,identities_for_row + + +def queue_files(path): + queue=json.loads(Path(path).read_text()) + files=queue['files'] + if len(files)!=len(set(files)):raise ValueError('Duplicate paths in evidence queue') + for filename in files: + if hashlib.sha256(Path(filename).read_bytes()).hexdigest()!=queue['file_sha256'][filename]: + raise ValueError('Evidence file differs from audited queue: '+filename) + return files + + +def backfill(db,files): + for filename in files: + manifest=json.loads(Path(filename).with_suffix('.manifest.json').read_text()) + if manifest['source'] in ('colm','neurips','iclr_program') and manifest.get('parser_version',0)<2: + raise ValueError('Recollect old parser output before submission evidence backfill') + store=Store(db);c=store.conn;count=0 + before=c.execute('SELECT count(*) FROM submissions').fetchone()[0] + try: + for filename in files: + path=Path(filename);manifest=json.loads(path.with_suffix('.manifest.json').read_text());seen=set() + with path.open() as stream: + for line in stream: + row=json.loads(line) + if row['external_id'] in seen:raise ValueError('Repeated source ID in evidence file') + seen.add(row['external_id']) + if any(row.get(key)!=manifest.get(key) for key in ('source','venue','year','track')): + raise ValueError('Evidence row differs from edition manifest') + record=c.execute('SELECT r.id,r.paper_id,r.metadata_json FROM source_records r JOIN sources s ON s.id=r.source_id WHERE s.name=? AND r.external_id=?',(row['source'],row['external_id'])).fetchone() + edition=c.execute('SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? AND e.year=? AND e.track=?',(row['venue'],row['year'],row['track'])).fetchone() + if not record or not edition:raise ValueError('Evidence backfill requires an already imported record and edition') + if not c.execute('SELECT 1 FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id WHERE ss.source_record_id=? AND ss.source_submission_id=? AND s.edition_id=?',(record[0],row['external_id'],edition[0])).fetchone(): + raise ValueError('No existing submission to backfill') + attach_submission(store,edition[0],record[1],record[0],row['external_id'],identities_for_row(row)) + metadata=json.loads(record[2]);metadata['record_identifiers_raw']=row.get('identifiers') or {} + if row['source']=='iclr_program': + for key in ('program_identity_type','program_submission_number_verified','program_identity_verification'): + if key in row['metadata']:metadata[key]=row['metadata'][key] + c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(metadata,ensure_ascii=False),record[0])) + count+=1 + if len(seen)!=manifest['count']:raise ValueError('Evidence count differs from manifest') + conflict=c.execute('SELECT 1 FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id JOIN source_records r ON r.id=ss.source_record_id WHERE s.paper_id!=r.paper_id LIMIT 1').fetchone() + if conflict:raise ValueError('Submission evidence points to a different paper entity') + c.commit() + result={'status':'backfilled','records':count,'submissions_before':before,'submissions_after':c.execute('SELECT count(*) FROM submissions').fetchone()[0]} + print(json.dumps(result),flush=True);return result + except BaseException: + c.rollback();raise + finally:c.close() + + +if __name__=='__main__': + parser=argparse.ArgumentParser();parser.add_argument('--db',required=True);parser.add_argument('--queue',required=True) + args=parser.parse_args();backfill(args.db,queue_files(args.queue)) diff --git a/collect_iclr.py b/collect_iclr.py index ef0978f..6eb4eb0 100644 --- a/collect_iclr.py +++ b/collect_iclr.py @@ -152,6 +152,7 @@ def parse_program(data, abstracts, year, index_url): 'tags': [{'namespace': SOURCE+':'+kind, 'value': value} for kind, values in [('topic', topics), ('keyword', keywords)] for value in values], 'metadata': {'program_event_ids': presentation_ids, + 'program_identity_type': identity[0], 'presentation_titles': sorted({clean(r['name']) for r in group}), 'program_source_id': identity[1], 'presentation_types': sorted({str(r.get('event_type') or r.get('eventtype')) for r in group}), 'raw_decisions': sorted({str(r.get('decision')) for r in group if r.get('decision')}), @@ -199,7 +200,7 @@ def collect_year(year, cache, out): part = path.with_suffix('.part') part.write_text(''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in records)) part.replace(path) - manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICLR', 'year': year, 'track': 'main', + manifest = {'source': SOURCE, 'parser_version': 2, 'venue': 'ICLR', 'year': year, 'track': 'main', 'count': len(records), 'list_url': page, 'asset_urls': assets, 'abstract_asset_status': abstract_asset_status, 'file': str(path), 'status': 'partial' if audit['missing_abstract_count'] else 'collected', **audit} diff --git a/import_records.py b/import_records.py index eaed872..fd5c61d 100644 --- a/import_records.py +++ b/import_records.py @@ -5,6 +5,7 @@ import time from pathlib import Path from index_core import Store +from submission_evidence import identities_for_row,timestamp def ingest_file(db,path): @@ -13,6 +14,8 @@ def ingest_file(db,path): manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {} if manifest.get('source') in ('colm','neurips') and manifest.get('parser_version',0)<2: raise ValueError('Recollect proceedings with parser_version>=2 before identity matching') + if manifest.get('source')=='iclr_program' and manifest.get('parser_version',0)<2: + raise ValueError('Recollect ICLR with parser_version>=2 before submission identity matching') store = Store(db) c = store.conn source_name = manifest.get('source') @@ -41,7 +44,19 @@ def ingest_file(db,path): if 'tags' in row: store.record_tags(paper,record,source_name,row['tags']) if row.get('status') in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'): - edition = store.decision(paper,record,row['venue'],row['year'],row['track'],external,row.get('evidence_url',row['url']),row['status'],row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status']),row.get('venue_aliases')) + identities=identities_for_row(row) + events=row.get('decision_events') + if events is None: + events=[{'status':row['status'],'raw_value':row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status']),'evidence_url':row.get('evidence_url',row['url'])}] + if not isinstance(events,list):raise ValueError('Decision events must be a list') + if not events and row['status']!='unknown':raise ValueError('Known status requires decision evidence') + if row['status']!='unknown' and not any(event.get('status')==row['status'] for event in events):raise ValueError('Summary status has no corresponding decision evidence') + row_observed=row.get('observed_at') or timestamp() + for event in events or [{'status':None}]: + status=event['status'] + if events and status is None:raise ValueError('Explicit decision event requires a status') + if status is not None and status not in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):raise ValueError('Unsupported decision event') + edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at')) editions.add(edition) else: raise ValueError('This importer requires an explicit supported decision status') diff --git a/index_core.py b/index_core.py index 7227a30..df7ce7f 100644 --- a/index_core.py +++ b/index_core.py @@ -4,6 +4,7 @@ import re import sqlite3 import unicodedata from pathlib import Path +from submission_evidence import attach_submission,migrate_sources,write_decision def normalized(value): @@ -31,6 +32,7 @@ class Store: self.conn.executescript(Path(__file__).with_name('schema.sql').read_text()) self.conn.execute('PRAGMA synchronous=NORMAL') self.conn.execute('PRAGMA cache_size=-32768') + migrate_sources(self.conn) def source(self,name,url): self.conn.execute('INSERT OR IGNORE INTO sources(name,homepage) VALUES(?,?)',(name,url)) @@ -40,6 +42,12 @@ class Store: c = self.conn if target == other: return + columns=('abstract','authors_json','first_published') + target_values=c.execute('SELECT abstract,authors_json,first_published FROM papers WHERE id=?',(target,)).fetchone() + other_values=c.execute('SELECT abstract,authors_json,first_published FROM papers WHERE id=?',(other,)).fetchone() + if not target_values or not other_values:raise ValueError('Missing paper entity during merge') + fill=[(column,value) for column,current,value in zip(columns,target_values,other_values) if current in (None,'','[]') and value not in (None,'','[]')] + if fill:c.execute('UPDATE papers SET '+','.join(column+'=?' for column,_ in fill)+' WHERE id=?',(*[value for _,value in fill],target)) c.execute('INSERT INTO paper_merges(former_paper_id,surviving_paper_id,reason) VALUES(?,?,?)',(other,target,reason)) c.execute('UPDATE identifiers SET paper_id=? WHERE paper_id=?',(target,other)) c.execute('UPDATE source_records SET paper_id=? WHERE paper_id=?',(target,other)) @@ -102,7 +110,8 @@ class Store: paper = c.execute('INSERT INTO papers(title,abstract,title_key,authors_json,first_published) VALUES(?,?,?,?,?)',(title,abstract,key,json.dumps(authors,ensure_ascii=False),published)).lastrowid for scheme,value in ids: c.execute('INSERT OR IGNORE INTO identifiers VALUES(?,?,?)',(scheme,value,paper)) - c.execute('INSERT INTO source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_id) DO UPDATE SET paper_id=excluded.paper_id,title=excluded.title,abstract=excluded.abstract,authors_json=excluded.authors_json,metadata_json=excluded.metadata_json,retrieved_at=CURRENT_TIMESTAMP',(source_id,external_id,paper,url,title,abstract,json.dumps(authors,ensure_ascii=False),json.dumps(metadata or {},ensure_ascii=False,default=str))) + metadata=dict(metadata or {});metadata['record_identifiers']=dict(ids) + c.execute('INSERT INTO source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_id) DO UPDATE SET paper_id=excluded.paper_id,url=excluded.url,title=excluded.title,abstract=excluded.abstract,authors_json=excluded.authors_json,metadata_json=excluded.metadata_json,retrieved_at=CURRENT_TIMESTAMP',(source_id,external_id,paper,url,title,abstract,json.dumps(authors,ensure_ascii=False),json.dumps(metadata,ensure_ascii=False,default=str))) record = c.execute('SELECT id FROM source_records WHERE source_id=? AND external_id=?',(source_id,external_id)).fetchone()[0] for other,_,_ in title_matches: if other != paper and c.execute('SELECT 1 FROM papers WHERE id=?',(other,)).fetchone(): @@ -125,7 +134,7 @@ class Store: tag=c.execute('SELECT id FROM tags WHERE namespace=? AND value=?',(namespace,value)).fetchone()[0] c.execute('INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,evidence)) - def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None): + def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None,identities=None,observed_at=None,decision_date=None,source_decision_id='',submitted_at=None): c = self.conn c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified')) venue_id = c.execute('SELECT id FROM venues WHERE acronym=?',(venue,)).fetchone()[0] @@ -133,10 +142,13 @@ class Store: c.execute('INSERT OR IGNORE INTO venue_aliases VALUES(?,?)',(venue_id,alias)) c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)',(venue_id,int(year),track)) edition = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?',(venue_id,int(year),track)).fetchone()[0] - c.execute('INSERT OR IGNORE INTO submissions(paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?)',(paper,edition,record,external_id)) - submission = c.execute('SELECT id FROM submissions WHERE source_record_id=? AND edition_id=? AND source_submission_id=?',(record,edition,external_id)).fetchone()[0] - if not c.execute('SELECT 1 FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=?',(submission,status,raw_value,url)).fetchone(): - c.execute("INSERT INTO decisions(submission_id,status,raw_value,evidence_url,observed_at) VALUES(?,?,?,?,strftime('%Y-%m-%dT%H:%M:%fZ','now'))",(submission,status,raw_value,url)) + # A previous event in this row may already have merged the input paper. + paper=c.execute('SELECT paper_id FROM source_records WHERE id=?',(record,)).fetchone()[0] + submission=attach_submission(self,edition,paper,record,external_id,identities or []) + if submitted_at: + from submission_evidence import timestamp + c.execute('UPDATE submissions SET submitted_at=COALESCE(submitted_at,?) WHERE id=?',(timestamp(submitted_at),submission)) + if status is not None:write_decision(self,submission,record,status,raw_value,url,observed_at,decision_date,source_decision_id) return edition def accepted(self,paper,record,venue,year,track,external_id,url): diff --git a/merge_conferences.py b/merge_conferences.py index d971af9..0d07a41 100644 --- a/merge_conferences.py +++ b/merge_conferences.py @@ -6,12 +6,14 @@ Only the Paper Atlas arXiv unit is controlled; business services are untouched. import argparse from contextlib import closing import json +import fcntl import sqlite3 import subprocess from pathlib import Path from import_records import ingest_file from index_core import Store +from backfill_submission_evidence import backfill,queue_files UNIT='paper-atlas-arxiv-import.service' @@ -21,29 +23,37 @@ def property_value(name): return subprocess.check_output(['systemctl','show',UNIT,'-p',name,'--value'],text=True).strip() -def merge(db,archive,files): +def merge_locked(db,archive,files,backfill_queue=None): # Validate inputs before stopping the existing writer. for filename in files: path=Path(filename) if not path.is_file() or not path.with_suffix('.manifest.json').is_file(): raise ValueError('Missing collected file or manifest: '+filename) + evidence_files=queue_files(backfill_queue) if backfill_queue else [] if not Path(archive).is_file(): raise ValueError('Missing arXiv archive') if not Path(db).is_file(): raise ValueError('Missing existing database') - if property_value('ActiveState')!='active': - raise RuntimeError('Expected the existing arXiv unit to be active; inspect it before proceeding') - subprocess.run(['systemctl','stop',UNIT],check=True) + resume_arxiv=property_value('ActiveState')=='active' + if not resume_arxiv: + with closing(sqlite3.connect(Path(db).resolve().as_uri()+'?mode=ro',uri=True)) as c: + last=c.execute("SELECT status FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone() + if not last or last[0]!='complete': + raise RuntimeError('arXiv is neither active nor verified complete; inspect its state before maintenance') try: + if resume_arxiv:subprocess.run(['systemctl','stop',UNIT],check=True) with closing(sqlite3.connect(db)) as c, c: - checkpoint=c.execute("SELECT records_written FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone() + checkpoint=c.execute("SELECT records_written,status FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone() print('Saved arxiv checkpoint:',checkpoint,flush=True) + if checkpoint and checkpoint[1]=='complete':resume_arxiv=False schema_store=Store(db) schema_store.conn.close() + if evidence_files:backfill(db,evidence_files) for filename in files: ingest_file(db,filename) with closing(sqlite3.connect(db)) as c, c: - if files: + c.execute('PRAGMA cache_size=-32768') + if files or evidence_files: errors=c.execute('PRAGMA foreign_key_check').fetchall() if errors: raise RuntimeError('Foreign key violations after merge') @@ -51,13 +61,21 @@ def merge(db,archive,files): print('Tag refresh plan:',c.execute('EXPLAIN QUERY PLAN DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key=?',(1,'arxiv:categories')).fetchall(),flush=True) print('Decision totals:',c.execute('SELECT status,count(*) FROM decisions GROUP BY status').fetchall(),flush=True) finally: - if property_value('LoadState')=='loaded': - subprocess.run(['systemctl','start',UNIT],check=True) - else: - subprocess.run(['systemd-run','--unit='+UNIT,'--property=WorkingDirectory='+str(Path(__file__).parent),'--property=CPUQuota=50%','--property=MemoryMax=512M','--property=Nice=15','--property=IOWeight=10','--property=StandardOutput=append:/data/paper-index/arxiv-import.log','--property=StandardError=append:/data/paper-index/arxiv-import.log','/usr/bin/env','PYTHONDONTWRITEBYTECODE=1','python3','-B',str(Path(__file__).with_name('import_arxiv.py')),'--db',db,'--archive',archive],check=True) - if property_value('ActiveState')!='active': - raise RuntimeError('arXiv unit did not resume; inspect the logs') - print('ARXIV_RESUMED',flush=True) + if resume_arxiv: + if property_value('LoadState')=='loaded': + subprocess.run(['systemctl','start',UNIT],check=True) + else: + subprocess.run(['systemd-run','--unit='+UNIT,'--property=WorkingDirectory='+str(Path(__file__).parent),'--property=CPUQuota=50%','--property=MemoryMax=512M','--property=Nice=15','--property=IOWeight=10','--property=StandardOutput=append:/data/paper-index/arxiv-import.log','--property=StandardError=append:/data/paper-index/arxiv-import.log','/usr/bin/env','PYTHONDONTWRITEBYTECODE=1','python3','-B',str(Path(__file__).with_name('import_arxiv.py')),'--db',db,'--archive',archive],check=True) + if property_value('ActiveState')!='active': + raise RuntimeError('arXiv unit did not resume; inspect the logs') + print('ARXIV_RESUMED',flush=True) + else:print('ARXIV_ALREADY_COMPLETE',flush=True) + + +def merge(db,archive,files,backfill_queue=None): + with Path(str(db)+'.maintenance.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + return merge_locked(db,archive,files,backfill_queue) if __name__=='__main__': @@ -65,8 +83,9 @@ if __name__=='__main__': parser.add_argument('--db',required=True) parser.add_argument('--archive',required=True) parser.add_argument('--schema-only',action='store_true',help='Apply idempotent schema/index updates at a saved checkpoint') + parser.add_argument('--backfill-queue',help='Audited existing-source queue for canonical submission evidence') parser.add_argument('files',nargs='*') args=parser.parse_args() - if not args.files and not args.schema_only: + if not args.files and not args.schema_only and not args.backfill_queue: parser.error('Provide collected files or --schema-only') - merge(args.db,args.archive,args.files) + merge(args.db,args.archive,args.files,args.backfill_queue) diff --git a/query.py b/query.py index f1ba580..04354fb 100644 --- a/query.py +++ b/query.py @@ -48,7 +48,15 @@ def main(): if args.year: clause.append('e.year=?'); params.append(args.year) if args.status: - clause.append('EXISTS(SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=? AND NOT EXISTS(SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id AND (newer.observed_at>d.observed_at OR (newer.observed_at=d.observed_at AND newer.id>d.id))))'); params.append(args.status) + latest='''EXISTS ( + SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=? + AND NOT EXISTS ( + SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id + AND (newer.observed_at,COALESCE(newer.decision_date,newer.observed_at),newer.id) + > (d.observed_at,COALESCE(d.decision_date,d.observed_at),d.id) + ) + )''' + clause.append('(NOT EXISTS(SELECT 1 FROM decisions d WHERE d.submission_id=s.id) OR '+latest+')' if args.status=='unknown' else latest); params.append(args.status) where.append('p.id IN (SELECT s.paper_id FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+' AND '.join(clause)+')') sql = 'SELECT p.* FROM papers p'+(' WHERE '+' AND '.join(where) if where else '')+' ORDER BY p.id LIMIT ?' params.append(max(1,min(args.limit,200))) @@ -63,7 +71,14 @@ def main(): submissions = [] for sub in conn.execute('SELECT s.id,s.source_submission_id,s.submitted_at,v.acronym,e.year,e.track FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE s.paper_id=? ORDER BY e.year,s.id',(row['id'],)): submission = dict(sub) - submission['decisions'] = [dict(d) for d in conn.execute('SELECT status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=? ORDER BY observed_at,id',(sub['id'],))] + submission['decisions'] = [] + for d in conn.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=? ORDER BY observed_at,COALESCE(decision_date,observed_at),id',(sub['id'],)): + decision=dict(d) + if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='decision_sources'").fetchone(): + decision['sources']=[dict(r) for r in conn.execute('SELECT s.name,r.external_id,ds.source_decision_id FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?',(d['id'],))] + submission['decisions'].append(decision) + if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='submission_sources'").fetchone(): + submission['sources']=[dict(r) for r in conn.execute('SELECT so.name,r.external_id,ss.source_submission_id FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id JOIN sources so ON so.id=r.source_id WHERE ss.submission_id=?',(sub['id'],))] submissions.append(submission) paper['submissions'] = submissions result.append(paper) diff --git a/schema.sql b/schema.sql index 2e5cfb1..5a0b268 100644 --- a/schema.sql +++ b/schema.sql @@ -58,6 +58,33 @@ CREATE TABLE IF NOT EXISTS decisions ( decision_date TEXT, UNIQUE(submission_id,status,raw_value,evidence_url,observed_at) ); CREATE INDEX IF NOT EXISTS decisions_submission_order ON decisions(submission_id,observed_at DESC,id DESC); +CREATE TABLE IF NOT EXISTS submission_sources ( + submission_id INTEGER NOT NULL REFERENCES submissions(id), + source_record_id INTEGER NOT NULL REFERENCES source_records(id), source_submission_id TEXT NOT NULL, + PRIMARY KEY(submission_id,source_record_id,source_submission_id) +); +CREATE INDEX IF NOT EXISTS submission_sources_record ON submission_sources(source_record_id,source_submission_id); +CREATE TABLE IF NOT EXISTS submission_identifiers ( + edition_id INTEGER NOT NULL REFERENCES editions(id), scheme TEXT NOT NULL, value TEXT NOT NULL, + submission_id INTEGER NOT NULL REFERENCES submissions(id), PRIMARY KEY(edition_id,scheme,value) +); +CREATE INDEX IF NOT EXISTS submission_identifiers_submission ON submission_identifiers(submission_id); +CREATE TABLE IF NOT EXISTS decision_sources ( + decision_id INTEGER NOT NULL REFERENCES decisions(id), + source_record_id INTEGER NOT NULL REFERENCES source_records(id), source_decision_id TEXT NOT NULL DEFAULT '', + PRIMARY KEY(decision_id,source_record_id,source_decision_id) +); +CREATE TABLE IF NOT EXISTS submission_merges ( + former_submission_id INTEGER PRIMARY KEY, surviving_submission_id INTEGER NOT NULL REFERENCES submissions(id), + reason TEXT NOT NULL, merged_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE TABLE IF NOT EXISTS decision_merges ( + former_decision_id INTEGER PRIMARY KEY, surviving_decision_id INTEGER NOT NULL REFERENCES decisions(id), + reason TEXT NOT NULL, merged_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE TABLE IF NOT EXISTS schema_migrations ( + name TEXT PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); CREATE TABLE IF NOT EXISTS tags ( id INTEGER PRIMARY KEY, namespace TEXT NOT NULL, value TEXT NOT NULL, description TEXT, UNIQUE(namespace,value) diff --git a/submission_evidence.py b/submission_evidence.py new file mode 100644 index 0000000..43d3502 --- /dev/null +++ b/submission_evidence.py @@ -0,0 +1,110 @@ +"""Canonical submission events with independent source and decision evidence.""" +from datetime import datetime, timezone +import re + + +PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','cvf','ecva','dblp'} + + +def timestamp(value=None): + date=datetime.now(timezone.utc) if value is None else datetime.fromisoformat(value.replace('Z','+00:00')) + if date.tzinfo is None:raise ValueError('Decision timestamp requires a timezone') + return date.astimezone(timezone.utc).isoformat(timespec='milliseconds').replace('+00:00','Z') + + +def identities_for_row(row): + identities=list(row.get('submission_identifiers') or []) + own=row.get('identifiers') or {} + if own.get('openreview'): + identities.append({'scheme':'openreview','value':own['openreview']}) + if row['source'] in PUBLICATIONS and row.get('status')=='accepted' and own.get('doi'): + doi=re.sub(r'^https?://(?:dx\.)?doi\.org/','',str(own['doi']),flags=re.I).strip().lower() + if re.fullmatch(r'10\.\d{4,9}/\S+',doi):identities.append({'scheme':'publication_doi','value':doi}) + meta=row.get('metadata') or {} + if row['source']=='iclr_program' and row['venue']=='ICLR' and meta.get('program_identity_type')=='submission' and meta.get('program_submission_number_verified') is True: + identities.append({'scheme':'iclr_number','value':str(meta['program_source_id'])}) + result=set() + for identity in identities: + scheme,value=identity['scheme'],str(identity['value']).strip() + if scheme=='openreview': + valid=re.fullmatch(r'[A-Za-z0-9_-]+',value) and not re.match(r'20\d{2}-Oral-',value) + elif scheme=='iclr_number':valid=re.fullmatch(r'[1-9]\d*',value) and row['venue']=='ICLR' + elif scheme=='publication_doi':valid=re.fullmatch(r'10\.\d{4,9}/\S+',value) + else:valid=False + if not valid:raise ValueError('Invalid authoritative submission identity') + result.add((scheme,value)) + return sorted(result) + + +def migrate_sources(c): + if c.execute("SELECT 1 FROM schema_migrations WHERE name='submission_source_baseline_v1'").fetchone():return + with c: + c.execute('INSERT OR IGNORE INTO submission_sources SELECT id,source_record_id,source_submission_id FROM submissions') + c.execute("INSERT OR IGNORE INTO decision_sources SELECT d.id,s.source_record_id,'' FROM decisions d JOIN submissions s ON s.id=d.submission_id") + c.execute("INSERT INTO schema_migrations(name) VALUES('submission_source_baseline_v1')") + + +def next_id(c,table,audit,former): + # Merged IDs are kept in audit; never allocate them to a later unrelated event. + return max(c.execute('SELECT COALESCE(max(id),0) FROM '+table).fetchone()[0],c.execute('SELECT COALESCE(max('+former+'),0) FROM '+audit).fetchone()[0])+1 + + +def merge_submissions(c,target,other): + if target==other:return + editions=c.execute('SELECT id,edition_id FROM submissions WHERE id IN (?,?)',(target,other)).fetchall() + if len(editions)!=2 or editions[0][1]!=editions[1][1]:raise ValueError('Cannot combine submissions from different editions') + c.execute('INSERT OR IGNORE INTO submission_sources SELECT ?,source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(target,other)) + c.execute('DELETE FROM submission_sources WHERE submission_id=?',(other,)) + for decision,status,raw,url,observed,date in c.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=?',(other,)).fetchall(): + duplicate=c.execute('SELECT id,decision_date FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=? AND observed_at=?',(target,status,raw,url,observed)).fetchone() + if duplicate: + surviving=duplicate[0] + if date and duplicate[1] and date!=duplicate[1]:raise ValueError('Conflicting dates for the same decision observation') + if date and not duplicate[1]:c.execute('UPDATE decisions SET decision_date=? WHERE id=?',(date,surviving)) + c.execute('INSERT OR IGNORE INTO decision_sources SELECT ?,source_record_id,source_decision_id FROM decision_sources WHERE decision_id=?',(surviving,decision)) + c.execute('DELETE FROM decision_sources WHERE decision_id=?',(decision,)) + c.execute('UPDATE decision_merges SET surviving_decision_id=? WHERE surviving_decision_id=?',(surviving,decision)) + c.execute("INSERT INTO decision_merges(former_decision_id,surviving_decision_id,reason) VALUES(?,?,'same decision observation on merged submission')",(decision,surviving)) + c.execute('DELETE FROM decisions WHERE id=?',(decision,)) + else:c.execute('UPDATE decisions SET submission_id=? WHERE id=?',(target,decision)) + c.execute('UPDATE submission_identifiers SET submission_id=? WHERE submission_id=?',(target,other)) + c.execute('UPDATE submission_merges SET surviving_submission_id=? WHERE surviving_submission_id=?',(target,other)) + c.execute("INSERT INTO submission_merges(former_submission_id,surviving_submission_id,reason) VALUES(?,?,'shared authoritative submission identity')",(other,target)) + c.execute('DELETE FROM submissions WHERE id=?',(other,)) + + +def attach_submission(store,edition,paper,record,external_id,identities): + c=store.conn + matches={r[0] for r in c.execute('SELECT s.id FROM submission_sources p JOIN submissions s ON s.id=p.submission_id WHERE p.source_record_id=? AND p.source_submission_id=? AND s.edition_id=?',(record,external_id,edition))} + matches.update(r[0] for r in c.execute('SELECT id FROM submissions WHERE source_record_id=? AND source_submission_id=? AND edition_id=?',(record,external_id,edition))) + for scheme,value in identities: + found=c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?',(edition,scheme,value)).fetchone() + if found:matches.add(found[0]) + if matches: + papers={paper}|{c.execute('SELECT paper_id FROM submissions WHERE id=?',(sid,)).fetchone()[0] for sid in matches} + surviving=min(papers) + for other in sorted(papers-{surviving}):store.merge(surviving,other,'shared authoritative conference submission identity') + submission=min(matches) + for other in sorted(matches-{submission}):merge_submissions(c,submission,other) + c.execute('UPDATE submissions SET paper_id=? WHERE id=?',(surviving,submission)) + else: + submission=next_id(c,'submissions','submission_merges','former_submission_id') + c.execute('INSERT INTO submissions(id,paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?,?)',(submission,paper,edition,record,external_id)) + c.execute('INSERT OR IGNORE INTO submission_sources VALUES(?,?,?)',(submission,record,external_id)) + for scheme,value in identities: + c.execute('INSERT OR IGNORE INTO submission_identifiers VALUES(?,?,?,?)',(edition,scheme,value,submission)) + return submission + + +def write_decision(store,submission,record,status,raw,url,observed_at=None,decision_date=None,source_decision_id=''): + c=store.conn + observed=timestamp(observed_at) + date=timestamp(decision_date) if decision_date else None + # Re-observing unchanged evidence adds provenance, not a new decision event. + old=c.execute('SELECT id FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=? AND COALESCE(decision_date,\'\')=COALESCE(?,\'\')',(submission,status,raw,url,date)).fetchone() + if old:decision=old[0] + else: + decision=next_id(c,'decisions','decision_merges','former_decision_id') + c.execute('INSERT INTO decisions(id,submission_id,status,raw_value,evidence_url,observed_at,decision_date) VALUES(?,?,?,?,?,?,?)',(decision,submission,status,raw,url,observed,date)) + c.execute('INSERT OR IGNORE INTO decision_sources VALUES(?,?,?)',(decision,record,source_decision_id or '')) + return decision diff --git a/test_iclr.py b/test_iclr.py index d9f577e..fb9b983 100644 --- a/test_iclr.py +++ b/test_iclr.py @@ -86,7 +86,7 @@ class ICLRTests(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp: path=Path(tmp)/'iclr.jsonl';db=Path(tmp)/'papers.sqlite' path.write_text(''.join(json.dumps(r)+'\n' for r in rows)) - path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','list_url':'https://iclr.cc/virtual/2026/papers.html','count':1,'status':'collected'})) + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','parser_version':2,'list_url':'https://iclr.cc/virtual/2026/papers.html','count':1,'status':'collected'})) ingest_file(db,path);ingest_file(db,path) store=Store(db);self.addCleanup(store.conn.close) self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1) diff --git a/test_index_core.py b/test_index_core.py index 36f5c18..c55c331 100644 --- a/test_index_core.py +++ b/test_index_core.py @@ -56,6 +56,7 @@ class StoreTests(unittest.TestCase): updated,_=self.store.record('arxiv','2601.00001','https://arxiv.org/abs/2601.00001','Updated research title','New searchable content',['Jane Doe']) self.assertEqual(updated,paper) self.assertEqual(self.store.conn.execute('SELECT title FROM papers').fetchone()[0],'Updated research title') + self.assertEqual(self.store.conn.execute('SELECT url FROM source_records').fetchone()[0],'https://arxiv.org/abs/2601.00001') self.assertEqual(self.store.conn.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'searchable'").fetchone()[0],1) def test_tag_refresh_keeps_other_record_evidence_and_custom_tags(self): diff --git a/test_maintenance.py b/test_maintenance.py new file mode 100644 index 0000000..057f9d2 --- /dev/null +++ b/test_maintenance.py @@ -0,0 +1,41 @@ +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from index_core import Store +from merge_conferences import merge + + +class MaintenanceTests(unittest.TestCase): + def fixture(self,tmp,status): + db=Path(tmp)/'papers.sqlite';archive=Path(tmp)/'metadata.zip';archive.write_bytes(b'not read by schema-only maintenance') + store=Store(db);source=store.source('arxiv','https://arxiv.org') + store.conn.execute('INSERT INTO ingest_runs(source_id,status) VALUES(?,?)',(source,status));store.conn.commit();store.conn.close() + return db,archive + + def test_completed_archive_is_not_started_again_after_maintenance(self): + with tempfile.TemporaryDirectory() as tmp: + db,archive=self.fixture(tmp,'complete') + with patch('merge_conferences.property_value',return_value='inactive'),patch('merge_conferences.subprocess.run') as run: + merge(db,archive,[]) + run.assert_not_called() + + def test_inactive_incomplete_arxiv_requires_investigation(self): + with tempfile.TemporaryDirectory() as tmp: + db,archive=self.fixture(tmp,'failed') + with patch('merge_conferences.property_value',return_value='inactive'),patch('merge_conferences.subprocess.run') as run: + with self.assertRaisesRegex(RuntimeError,'neither active nor verified complete'): + merge(db,archive,[]) + run.assert_not_called() + + def test_arxiv_finishing_at_the_stop_boundary_is_not_restarted(self): + with tempfile.TemporaryDirectory() as tmp: + db,archive=self.fixture(tmp,'complete') + with patch('merge_conferences.property_value',return_value='active'),patch('merge_conferences.subprocess.run') as run: + merge(db,archive,[]) + self.assertEqual(run.call_args_list[0].args[0],['systemctl','stop','paper-atlas-arxiv-import.service']) + self.assertEqual(run.call_count,1) + + +if __name__=='__main__':unittest.main() diff --git a/test_submission_evidence.py b/test_submission_evidence.py new file mode 100644 index 0000000..b0ed694 --- /dev/null +++ b/test_submission_evidence.py @@ -0,0 +1,163 @@ +import json +from pathlib import Path +import sqlite3 +import subprocess +import sys +import tempfile +import unittest + +from import_records import ingest_file +from index_core import Store +from submission_evidence import identities_for_row +from backfill_submission_evidence import backfill,queue_files + + +class SubmissionTests(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup) + self.db=Path(self.tmp.name)/'papers.sqlite';self.store=Store(self.db);self.addCleanup(self.store.conn.close) + + def record(self,source,external,authors): + return self.store.record(source,external,'https://example.org/'+external,'A long paper about reliable metadata and historical events','Abstract',authors) + + def decision(self,paper,record,external,identities,status='accepted',raw='Accept',url='https://example.org/decision',observed='2026-05-08T00:00:00Z',date=None): + return self.store.decision(paper,record,'ICLR',2026,'main',external,url,status,raw,identities=identities,observed_at=observed,decision_date=date) + + def query(self,*args): + self.store.conn.commit() + command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(self.db),*args] + result=subprocess.run(command,capture_output=True,text=True) + self.assertEqual(result.returncode,0,result.stderr) + return json.loads(result.stdout) + + def test_same_submission_across_sources_has_one_event_and_both_provenances(self): + first,one=self.store.record('iclr_program','2026/number-42','https://example.org/program','A long paper about reliable metadata and historical events',None,['Jane Doe','John Smith']) + second,two=self.record('openreview_snapshot','realForum',['Anonymous']) + self.assertNotEqual(first,second) + self.decision(first,one,'2026/number-42',[('iclr_number','42')]) + self.decision(second,two,'realForum',[('iclr_number','42'),('openreview','realForum')],url='https://openreview.net/forum?id=realForum¬eId=note1') + c=self.store.conn + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1) + self.assertEqual(c.execute('SELECT abstract FROM papers').fetchone()[0],'Abstract') + self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0],2) + details=self.query('--paper',str(first))[0]['submissions'][0] + self.assertEqual({r['name'] for r in details['sources']},{'iclr_program','openreview_snapshot'}) + self.assertEqual(len(details['decisions']),2) + self.assertTrue(all(d['sources'] for d in details['decisions'])) + self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[]) + + def test_same_paper_can_have_two_distinct_submission_attempts(self): + paper,one=self.store.record('openreview_snapshot','forum1','https://example.org/1','First version','Abstract',[],{'arxiv':'2601.00001','openreview':'forum1'}) + other,two=self.store.record('openreview_snapshot','forum2','https://example.org/2','Another version','Abstract',[],{'arxiv':'2601.00001','openreview':'forum2'}) + self.assertEqual(paper,other) + self.decision(paper,one,'forum1',[('openreview','forum1'),('iclr_number','1')],status='rejected',raw='Reject') + self.decision(paper,two,'forum2',[('openreview','forum2'),('iclr_number','2')]) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],2) + + def test_late_identity_bridge_preserves_decisions_and_does_not_reuse_merged_ids(self): + first,one=self.record('iclr_program','number-42',['Jane Doe','John Smith']) + second,two=self.record('openreview_snapshot','realForum',['Anonymous']) + self.decision(first,one,'number-42',[('iclr_number','42')]) + self.decision(second,two,'realForum',[('openreview','realForum')]) + self.decision(second,two,'realForum',[('openreview','realForum'),('iclr_number','42')]) + c=self.store.conn + self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM decision_sources').fetchone()[0],2) + self.assertEqual(c.execute('SELECT count(*) FROM submission_merges').fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM decision_merges').fetchone()[0],1) + third,record=self.record('other','different',['Other Author']) + self.decision(third,record,'different',[('openreview','differentForum')],status='rejected',raw='Reject') + self.assertGreater(c.execute('SELECT max(id) FROM submissions').fetchone()[0],2) + self.assertGreater(c.execute('SELECT max(id) FROM decisions').fetchone()[0],2) + self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[]) + + def test_old_snapshot_does_not_override_a_newer_official_observation(self): + paper,one=self.record('iclr_program','number-42',['Jane Doe','John Smith']) + _,two=self.record('openreview_snapshot','realForum',['Jane Doe','John Smith']) + self.decision(paper,one,'number-42',[('iclr_number','42')],observed='2026-11-06T00:00:00Z') + self.decision(paper,two,'realForum',[('iclr_number','42')],status='rejected',raw='Reject',observed='2026-05-08T00:00:00Z') + self.assertEqual(len(self.query('--venue','ICLR','--status','accepted')),1) + self.assertEqual(self.query('--venue','ICLR','--status','rejected'),[]) + + def test_decision_date_orders_events_with_the_same_snapshot_time(self): + paper,record=self.record('openreview_snapshot','realForum',['Jane Doe']) + self.decision(paper,record,'realForum',[('openreview','realForum')],date='2026-02-01T00:00:00Z') + self.decision(paper,record,'realForum',[('openreview','realForum')],status='rejected',raw='Reject',date='2026-01-01T00:00:00Z') + self.assertEqual(len(self.query('--venue','ICLR','--status','accepted')),1) + self.assertEqual(self.query('--venue','ICLR','--status','rejected'),[]) + + def test_unknown_without_a_decision_is_queryable_without_invented_verdict(self): + path=Path(self.tmp.name)/'unknown.jsonl' + row={'source':'openreview_snapshot','external_id':'realForum','url':'https://example.org/paper','title':'Unknown result','abstract':'Abstract','authors':[],'identifiers':{'openreview':'realForum'},'venue':'ICLR','year':2026,'track':'main','status':'unknown','decision_events':[]} + path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':row['source'],'list_url':'https://example.org/list','count':1})) + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM decisions').fetchone()[0],0) + self.assertEqual(len(self.query('--venue','ICLR','--status','unknown')),1) + + def test_known_summary_without_corresponding_decision_rolls_back(self): + path=Path(self.tmp.name)/'missing-evidence.jsonl' + row={'source':'openreview_snapshot','external_id':'realForum','url':'https://example.org/paper','title':'Unsupported result','abstract':'Abstract','authors':[],'venue':'ICLR','year':2026,'track':'main','status':'accepted','decision_events':[{'status':None}]} + path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':row['source'],'list_url':'https://example.org/list','count':1})) + with self.assertRaisesRegex(ValueError,'no corresponding decision evidence'): + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],0) + + def test_event_number_is_not_assumed_to_be_a_submission_number(self): + row={'source':'iclr_program','venue':'ICLR','metadata':{'program_source_id':42,'program_identity_type':'event'},'identifiers':{}} + self.assertEqual(identities_for_row(row),[]) + row['metadata']['program_identity_type']='submission' + self.assertEqual(identities_for_row(row),[]) + row['metadata']['program_submission_number_verified']=True + self.assertEqual(identities_for_row(row),[('iclr_number','42')]) + with self.assertRaisesRegex(ValueError,'Invalid authoritative'): + identities_for_row({'source':'snapshot','venue':'ICLR','submission_identifiers':[{'scheme':'openreview','value':'2026-Oral--42-placeholder'}]}) + + def test_legacy_database_migration_preserves_source_and_decision_links(self): + legacy=Path(self.tmp.name)/'legacy.sqlite';sql=Path(__file__).with_name('schema.sql').read_text() + sql=sql[:sql.index('CREATE TABLE IF NOT EXISTS submission_sources')]+sql[sql.index('CREATE TABLE IF NOT EXISTS tags'):] + c=sqlite3.connect(legacy);c.executescript(sql) + c.execute("INSERT INTO sources VALUES(1,'legacy','https://example.org')") + c.execute("INSERT INTO papers(id,title,title_key) VALUES(1,'Paper','paper')") + c.execute("INSERT INTO source_records VALUES(1,1,'paper',1,'https://example.org/p','Paper','Abstract','[]','{}',CURRENT_TIMESTAMP)") + c.execute("INSERT INTO venues VALUES(1,'ICLR','ICLR','machine_learning','conference')") + c.execute("INSERT INTO editions VALUES(1,1,2026,'main')") + c.execute("INSERT INTO submissions VALUES(1,1,1,1,'paper',NULL)") + c.execute("INSERT INTO decisions VALUES(1,1,'accepted','Accept','https://example.org/decision','2026-01-01T00:00:00Z',NULL)") + c.commit();c.close() + store=Store(legacy);self.addCleanup(store.conn.close) + self.assertEqual(store.conn.execute('SELECT * FROM submission_sources').fetchall(),[(1,1,'paper')]) + self.assertEqual(store.conn.execute('SELECT * FROM decision_sources').fetchall(),[(1,1,'')]) + self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(),[]) + + def test_existing_program_identity_backfill_is_idempotent_and_preserves_text(self): + paper,record=self.record('iclr_program','2026/program-42',['Jane Doe','John Smith']) + self.decision(paper,record,'2026/program-42',[]) + self.store.conn.commit() + original=self.store.conn.execute('SELECT title,abstract,retrieved_at FROM source_records WHERE id=?',(record,)).fetchone() + path=Path(self.tmp.name)/'program.jsonl' + row={'source':'iclr_program','external_id':'2026/program-42','venue':'ICLR','year':2026,'track':'main','status':'accepted','identifiers':{},'metadata':{'program_identity_type':'submission','program_source_id':42,'program_submission_number_verified':True}} + path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','parser_version':2,'venue':'ICLR','year':2026,'track':'main','count':1})) + first=backfill(self.db,[path]);second=backfill(self.db,[path]) + self.assertEqual(first,second) + self.assertEqual(self.store.conn.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),[('iclr_number','42')]) + self.assertEqual(self.store.conn.execute('SELECT title,abstract,retrieved_at FROM source_records WHERE id=?',(record,)).fetchone(),original) + other,snapshot=self.record('openreview_snapshot','realForum',['Anonymous']) + self.decision(other,snapshot,'realForum',[('iclr_number','42'),('openreview','realForum')]) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + + def test_evidence_queue_rejects_changed_file_before_mutation(self): + path=Path(self.tmp.name)/'program.jsonl';path.write_text('changed bytes') + queue=Path(self.tmp.name)/'queue.json';queue.write_text(json.dumps({'files':[str(path)],'file_sha256':{str(path):'0'*64}})) + with self.assertRaisesRegex(ValueError,'differs from audited'): + queue_files(queue) + + def test_old_untyped_program_files_require_recollection(self): + path=Path(self.tmp.name)/'old.jsonl';path.write_text('') + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','parser_version':1})) + with self.assertRaisesRegex(ValueError,'Recollect ICLR'): + ingest_file(self.db,path) + + +if __name__=='__main__':unittest.main()