Compare commits

..

1 Commits

Author SHA1 Message Date
codex-bot 2000d340fa Add snapshot-bound publication source scope revisions
Metadata integrity checks / metadata (push) Successful in 7m16s
2026-11-18 06:59:21 +08:00
17 changed files with 738 additions and 762 deletions
+1 -1
View File
@@ -35,7 +35,7 @@ jobs:
venv = root / 'venv' venv = root / 'venv'
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True) subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
python = str(venv / 'bin/python') python = str(venv / 'bin/python')
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8','pyarrow==26.0.1'],check=True) subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8'],check=True)
testenv = os.environ.copy() testenv = os.environ.copy()
service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf') service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf')
if service_file.is_file(): if service_file.is_file():
-88
View File
@@ -1,88 +0,0 @@
name: ReviewArena metadata projection
on:
workflow_dispatch:
inputs:
split:
description: 'Pinned public mirror family; metadata columns only'
required: true
default: emnlp
projection_name:
description: 'Optional existing metadata output directory name for offline replay'
required: false
default: ''
projection_sha256:
description: 'Exact cached collection-report.json SHA256, required for offline replay'
required: false
default: ''
jobs:
project:
runs-on: ces-control
env:
ARCHIVE_SHA: ${{ github.sha }}
MIRROR_SPLIT: ${{ inputs.split }}
MIRROR_PROJECTION: ${{ inputs.projection_name }}
MIRROR_PROJECTION_SHA: ${{ inputs.projection_sha256 }}
PYTHONDONTWRITEBYTECODE: '1'
steps:
- name: Project permitted metadata on the approved remote runner
run: |
python3 -B - <<'PY'
import hashlib, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
sha = os.environ['ARCHIVE_SHA']
split = os.environ['MIRROR_SPLIT']
cached = os.environ.get('MIRROR_PROJECTION','')
cached_sha = os.environ.get('MIRROR_PROJECTION_SHA','')
if not re.fullmatch(r'[0-9a-f]{40}', sha) or split not in ('colm','corl','emnlp','iclr','icml','neurips','tmlr'):
raise SystemExit('Explicit approved source commit/split required')
base = pathlib.Path('/data/paper-index')
if not (base/'collected').is_dir() or not (base/'validation').is_dir():
raise SystemExit('Confirmed ces-control host dataset paths unavailable; do not use another host')
out = base/'collected'/('reviewarena-c2978add-'+split+'-'+sha[:8])
if out.exists():
raise SystemExit('Inspect existing output; no overwrite or automatic retry')
extra=[]
if cached:
if not re.fullmatch('reviewarena-c2978add-'+split+r'-[0-9a-f]{8}',cached) or not re.fullmatch('[0-9a-f]{64}',cached_sha):
raise SystemExit('Explicit permitted cached projection name and report hash required')
extra=['--from-projection',str(base/'collected'/cached/'collection-report.json'),'--projection-sha256',cached_sha]
elif cached_sha:
raise SystemExit('Cached projection name required')
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/'+sha+'.tar.gz'
with urllib.request.urlopen(url,timeout=60) as response:
archive = response.read(24*1024*1024+1)
if len(archive)>24*1024*1024:
raise SystemExit('Oversized source archive')
with tempfile.TemporaryDirectory(prefix='paper-atlas-mirror-') as tmp:
root = pathlib.Path(tmp)
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
tar.extractall(root,filter='data')
candidates=list(root.glob('*/pyproject.toml'))
if len(candidates)!=1:
raise SystemExit('Unexpected repository archive')
src=candidates[0].parent
venv=root/'venv'
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
python=str(venv/'bin/python')
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','pyarrow==26.0.1'],check=True)
wheels=root/'wheels'
subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True)
wheel,=wheels.glob('paper_atlas-*.whl')
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
outside=root/'outside';outside.mkdir()
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out),*extra],cwd=outside,check=True)
report=json.loads((out/'collection-report.json').read_bytes())
assert report['database_written'] is False and report['approved_for_import']==0
receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'cached_projection_report_sha256':report.get('cached_projection_report_sha256'),'raw_parquet_saved':False,'database_written':False}
(out/'runner-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n')
backup=base/'validation'/(out.name+'-recovery.tar.gz')
if backup.exists():
raise SystemExit('Recovery archive already exists; inspect')
with tarfile.open(backup,'w:gz') as tar:
tar.add(out,arcname=str(out.relative_to(base)))
backup.chmod(0o444)
for path in out.iterdir():
if path.is_file():path.chmod(0o444)
result={'status':'metadata_projection_backed_up_not_imported','archive':str(backup),'bytes':backup.stat().st_size,'sha256':hashlib.sha256(backup.read_bytes()).hexdigest(),**receipt}
(base/'validation'/(out.name+'-recovery.json')).write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result),flush=True)
PY
+2 -2
View File
@@ -2,8 +2,6 @@
**长期维护、按需运行的论文元数据与公开投稿历史工具。** **长期维护、按需运行的论文元数据与公开投稿历史工具。**
其他会议的公开评审镜像可按 [ReviewArena 元数据投影流程](docs/reviewarena-metadata.md) 核查:仅取固定版本的标题、摘要、作者及决定等索引字段,保留镜像声明和稳定行定位,不下载正文/评审数据页,也不将镜像 ID 自动提升为官方论坛身份。
Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。 Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。 项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
@@ -119,6 +117,8 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。 已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
完整出版来源的新旧版本导致分轨或摘要变化时,使用 [source-revisions 流程](docs/source-revisions.md):核对同一 DOI 全集,保存完整修正前数据与版本链,支持已有记录的目标 track,按新版本重算分区覆盖数,并阻止旧版元数据刷新。普通增量导入仍禁止缩小预期数量。
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。 入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。 RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
-52
View File
@@ -1,52 +0,0 @@
# ReviewArena 镜像的索引字段投影
ReviewArena 是公开评审镜像,固定版本为 `c2978add17c2099219eaddbc2599974d69d4d09b`。上游数据卡描述七类会议、2020–2026 年的部分来源;这不是逐会议、逐年度的公开决定全集。文件清单有 15 个 Parquet 分片,覆盖 COLM、CoRL、EMNLP、ICLR、ICML、NeurIPS、TMLR。这里不包括 UAI,不能用它填补 UAI 缺口。
- [固定版本上游数据卡](https://huggingface.co/datasets/anonymousNeurIPS2026submission4281/reviewarena/blob/c2978add17c2099219eaddbc2599974d69d4d09b/README.md)
- [固定版本文件元数据 API](https://huggingface.co/api/datasets/anonymousNeurIPS2026submission4281/reviewarena/tree/c2978add17c2099219eaddbc2599974d69d4d09b/data?recursive=false&expand=false)
- [Arrow ParquetFile 投影接口](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html)
## 运行
仅在已授权远程 runner 安装 `paper-atlas[mirrors]`,使用固定 PyArrow 26.0.1 二进制 wheel。本机不执行安装、构建或项目测试。常规 CI 包含投影、安全边界和镜像决定语义测试;真实网络采集由独立的 `ReviewArena metadata projection` workflow 手动触发,指定已通过 CI 的代码 ref 和 split,例如 `emnlp`。该 workflow 仍使用 `ces-control` / `aliyun-ces-control`,在服务器固定数据目录生成输出和恢复归档,不更改默认 runtime 或数据库。
包内命令如下,`--out` 必须是既有父目录内的新目录:
```sh
paper-atlas collect reviewarena-metadata \
--split emnlp \
--out /data/paper-index/collected/reviewarena-emnlp-reviewed-version
```
输出目录已存在时拒绝执行。遇到禁止访问、来源变化或读取失败,保留失败报告,不自动重试或切换到整文件下载。不能把原始访问拒绝解释为没有拒稿。
已保存的完整元数据投影可以离线重新解释决定字面值。提供原 `collection-report.json` 及其已独立验证的精确文件 SHA-256,程序核对全部元数据文件哈希、逐行来源身份和原始字段,生成新目录;不会再次访问上游 Parquet:
```sh
paper-atlas collect reviewarena-metadata --split emnlp \
--from-projection /data/paper-index/collected/<既有投影>/collection-report.json \
--projection-sha256 <已验证的原报告SHA256> \
--out /data/paper-index/collected/<新解释版本>
```
手动 workflow 对应输入为 `projection_name` 与 `projection_sha256`。离线报告的 `metadata_bytes_read` 为本次请求字节数 `0`,`source_projection_bytes` 保留原始列块投影字节数;原报告哈希建立恢复链,原始记录与来源定位不变。
## 实际读取和保存范围
仅投影以下字段:`forum_id`、`conference`、`year`、`track`、`venue_id`、`paper_number`、`title`、`abstract`、`authors`、`keywords`、`venue`、`decision`。
程序先读取文件魔数、尾部长度和有界 Parquet 页脚,在内存中解析 schema 和列块偏移,然后只请求这些列的字典/数据页。Arrow 可能请求较大的页脚窗口,未获准区域由本地稀疏文件合成零字节,不会为该窗口补下载相邻正文页。逐个选中列块还要检查不与未选列块重叠。单文件上限为 64 MiB 所需字节、512 个范围请求和 100,000 行;响应必须是准确的 HTTP 206 / Content-Range,ETag 变化时停止。该边界适用于每个分片,不能据此启动其他分片的未授权全文件下载。
不读取 `markdown`、`reviews_json`、`decision_comment`、`author_rebuttal` 等正文或评审内容的数据页;不保存原 Parquet、页脚原文或未选字段统计值。保存的 JSONL 只有上述索引字段、来源定位和审计标记。报告记录每个请求范围及其 SHA-256、选中列名、页脚哈希、投影数量和保存文件哈希,便于独立复核。
上游 LFS SHA-256 与大小是固定版本 API 提供的对象声明。由于不下载整个 Parquet,本流程**不声称验证了完整 Parquet 内容哈希**。范围对象大小/ETag 和各段哈希用于检查本次读取的一致性。
## 决定和身份边界
`mirror_decision_category` 只解释明确的镜像字面值;未知、缺失或未审核的决定保持 `unknown`,不会根据 venue、是否录用、摘要缺失或目录缺席推断拒稿。类别数量是镜像行数,不能解释为独立作品数量或原始官方决定数量。
真实 EMNLP 2023 固定分片包含 2,009 行:原始决定为 `Accept-Main` 994 行、`Accept-Findings` 1,015 行,原始 `track` 却全部为 `main`。两种已核对字面值属于镜像录用声明;`mirror_decision_track_claim` 分开保留其明确的 main/findings 声明,原 `track` 和所有 `raw_fields` 不被重写。`Accept (Poster)` 等展示形式不能自行推出主会归属。此分片没有明确负面决定,不能用于补齐拒稿缺口,更不能由此断言 EMNLP 原始平台没有公开拒稿。首次实际投影仅请求 1,720,498 字节;原文件 90,794,691 字节,未下载完整文件。
上游 `forum_id`、`venue_id`、投稿号、作者和决定均作为原始声明保留。`forum_id` 不直接成为全局 OpenReview 标识;稳定行定位由数据集、固定 commit、文件和行号组成。输出 `identifiers` 为空,原始论坛身份和决定签名标记为未验证,保留匿名和缺失作者。源数据中的字符串命名不能被当成真实人员别名。
输出故意不是可直接 `import records` 的规范化队列,没有授予导入或合并权限。下一步必须检查原始来源、跨来源重合和作品身份,再决定哪些来源观察能进入生产。即使明确镜像拒稿,也要保留该投稿历史并继续查找后续其他会议的录用;不能把“目前没有关联录用”表述为从未录用。本采集不代替最终全库去重、决定冲突、来源覆盖和查询验收。
+57
View File
@@ -0,0 +1,57 @@
# 出版来源版本与覆盖修正
`paper-atlas maintain source-revisions` 用于同一出版清单的新版本:稳定 DOI 全集、标题、作者和决定不变,但官方分轨、摘要或对应来源证据已经过重新核对。它不导入新论文、不合并实体、不推断拒稿,不把出版分类当成研究类型。
例如,同一清单的“未分类”由 192 条减少到 166 条,而主会等分区增加。直接放松增量导入的计数保护会隐藏清单丢失。本命令先核验完整的新旧清单及数据库快照,再单独修正版本;普通 `import records` 的单调计数规则保持原样。
## 准备完整证据
在工作区外保存 inventory JSON,包含以下字段:
| 字段 | 内容 |
| --- | --- |
| `format` | `1` |
| `source`, `venue`, `year`, `event_key`, `list_url` | 明确的已有来源、会议缩写、年份、事件键和 HTTPS 出版清单地址 |
| `previous_rows`, `rows` | 完整旧版、新版规范化来源记录数组,每版最多 10,000 条;每条包含稳定 DOI、来源字段、作者、摘要、track、决定、标签、原始来源 metadata |
| `evidence` | 一个或多个 `{ "url": "https://…", "sha256": "…" }`,绑定已独立核验的原始证据文件 |
两版必须有完全相同的 DOI 集合;仅允许 `abstract`、`track`、`tags`、`metadata` 的有据变化。必须保留 collector 原始字段,不能将工作身份核查产生的额外锚点混入原始来源版本。只有明确 `accepted` 的出版观察可进入本流程。证据 URL/hash 是审核输入,本命令不联网确认它们;执行者必须另行验证文件哈希、完整性及对应关系。
数据库可以部分导入,甚至各分区的预期数暂时来自不同版本。现有覆盖必须为 `partial`、无错误,导入数与实际来源记录一致;预期数必须恰好属于旧版或新版的同一分区。已有目标 edition 和对应覆盖记录必须存在,目标中可以有其他来源及既有投稿。本版本不会新建 edition,也不会处理共享来源投稿、多次既有 scope curation 的再迁移,或成员集合变化;遇到这些情况会拒绝并要求单独核查。
## 先生成只读计划
```sh
paper-atlas maintain source-revisions \
--pg-service foundry-research --pg-schema research \
--inventory /data/paper-index/validation/reviewed-source-version.json \
--out /data/paper-index/validation/source-revision-plan.json
```
计划包含明确数据库身份、两个完整来源版本、当前版本父节点、覆盖表原值和修正值、逐条变更,以及受影响会议 scope 内所有论文的规范记录、稳定标识、全部跨会议投稿和决定、来源关系、标签与已存在的 annotation。拒稿后录用的历史也在保护范围内,不能只校验当前会议的录用观察。
计划和原始证据应保存到服务器与独立恢复位置,核对完整差异及计划文件 SHA-256。执行前重新核对生产保护基线与单写入安排。输出文件必须是不存在的新路径,禁止覆盖旧报告。
## 按文件哈希应用并验收
```sh
paper-atlas maintain source-revisions \
--pg-service foundry-research --pg-schema research \
--plan /data/paper-index/validation/source-revision-plan.json \
--plan-sha256 <已审核的计划文件SHA256> \
--out /data/paper-index/validation/source-revision-result.json
```
应用时重新生成计划并要求完全一致;PostgreSQL 使用已有 writer 锁及相关表写锁,SQLite 使用立即事务。任何保护数据、当前版本或目标标识冲突都使操作失败。操作保留投稿和决定 ID、原始决定证据、规范论文文本、全部稳定标识、来源关系和现有标签;只修改计划列出的投稿/投稿标识 edition,以及来源记录的摘要与 metadata 当前投影。旧投影完整保存在版本历史中,来源原始获取时间不会伪造为修正时间。
覆盖只调整明确分区的预期数和来源观察数,保留 `partial`、cursor、成功时间和错误字段原值。尚未导入的分区不产生虚构投稿或来源观察。覆盖数不能被解读为科学作品数量,或所有 track、摘要、负面决定已完整。
同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。
提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。
## 后续刷新及限制
`import records` 检查当前版本的逐条完整来源字段;`Store.record` 也检查原始文本、作者、来源 metadata 和基础标识,防止直接调用绕回旧版摘要。来源字段必须与当前版本相符,可以额外携带已经核验的工作身份/curation 证据,不能替换来源自身字段。相同来源、会议事件和清单地址下未登记的成员必须另行核验新版本。
版本表通过包内接口追加,未声明数据库管理员的任意 SQL 为安全接口。新来源或新版本在导入前仍需要独立工作身份核查;姓名、标题或无候选都不证明作品身份。实现测试和版本修正验收也不替代所有导入结束后的全库去重、标识冲突、近似标题/作者、摘要、来源覆盖与查询审计。
+1 -1
View File
@@ -20,7 +20,6 @@ COMMANDS = {
('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'), ('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'),
('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'), ('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'),
('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'), ('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'),
('collect','reviewarena-metadata'): ('paper_atlas.collectors.reviewarena', {}, '仅投影固定版本评审镜像的索引字段,保留决定声明而不下载正文或评审页'),
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'), ('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'), ('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
@@ -47,6 +46,7 @@ COMMANDS = {
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'), ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'), ('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'), ('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'),
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'), ('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'), ('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'), ('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
-216
View File
@@ -1,216 +0,0 @@
"""Source-attributed mirror metadata; no paper/review text or identity promotion."""
import argparse
from collections import Counter
from datetime import datetime, timezone
import hashlib
from importlib.resources import files
import json
from pathlib import Path
import re
from paper_atlas.io.parquet_metadata import RangeSource, projection
DATASET = 'anonymousNeurIPS2026submission4281/reviewarena'
COMMIT = 'c2978add17c2099219eaddbc2599974d69d4d09b'
TREE_SHA256 = 'bbc75f5774b019c4a9ee3efe5c0f37b14761bc3c5a054570db1df8f5603eac20'
SOURCE = 'reviewarena_mirror'
SPLITS = ('colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr')
COLUMNS = {'forum_id': 'string', 'conference': 'string', 'year': 'integer',
'track': 'string', 'venue_id': 'string', 'paper_number': 'integer',
'title': 'string', 'abstract': 'string', 'authors': 'strings',
'keywords': 'strings', 'venue': 'string', 'decision': 'string'}
def inventory(split):
if split not in SPLITS:
raise ValueError('Explicit supported mirror split required')
raw = files('paper_atlas.resources').joinpath('reviewarena-files.json').read_bytes()
if hashlib.sha256(raw).hexdigest() != TREE_SHA256:
raise ValueError('Pinned mirror file inventory changed')
items = json.loads(raw)
if len(items) != 15:
raise ValueError('Pinned mirror shard inventory changed')
selected = []
for item in items:
path = item['path']
if not re.fullmatch(r'data/(colm|corl|emnlp|iclr|icml|neurips|tmlr)-\d{5}-of-\d{5}\.parquet', path):
raise ValueError('Unexpected mirror shard path')
if (item['type'] != 'file' or type(item['size']) is not int or item['size'] < 12
or item['lfs']['size'] != item['size'] or not re.fullmatch('[0-9a-f]{64}', item['lfs']['oid'])):
raise ValueError('Missing source object size/checksum claim')
if path.startswith('data/'+split+'-'):
selected.append(item)
parts = sorted(selected, key=lambda x: x['path'])
if not parts or any(int(x['path'][-13:-8]) != len(parts) for x in parts):
raise ValueError('Incomplete selected mirror split inventory')
if [int(x['path'].split('-')[-3]) for x in parts] != list(range(len(parts))):
raise ValueError('Mirror shard sequence changed')
return parts
def decision_claim(raw):
value = raw.strip().casefold() if isinstance(raw, str) else ''
if value in {'reject', 'rejected'}:
return 'rejected'
if value in {'desk reject', 'desk rejected', 'desk_rejected'}:
return 'desk_rejected'
if value in {'withdraw', 'withdrawn'}:
return 'withdrawn'
if value in {'accept', 'accepted', 'accept-main', 'accept-findings', 'accept (main)', 'accept (findings)',
'accept (poster)', 'accept (oral)', 'accept (spotlight)', 'accept (talk)'}:
return 'accepted'
return 'unknown'
def decision_track_claim(raw):
value = raw.strip().casefold() if isinstance(raw, str) else ''
if value in {'accept-findings', 'accept (findings)'}:
return 'findings'
if value in {'accept-main', 'accept (main)'}:
return 'main'
return None
def record(row, split, item, row_index):
if set(row) != set(COLUMNS):
raise ValueError('Unexpected projected metadata fields')
if (not isinstance(row['conference'], str) or row['conference'].casefold() != split
or type(row['year']) is not int or not 2020 <= row['year'] <= 2026):
raise ValueError('Mirror conference/year outside pinned source scope')
locator = {'dataset': DATASET, 'commit': COMMIT, 'file': item['path'], 'row_index': row_index,
'lfs_sha256_claim_not_full_file_verification': item['lfs']['oid']}
return {'source': SOURCE, 'external_id': COMMIT+':'+item['path']+':'+str(row_index),
'url': 'https://huggingface.co/datasets/'+DATASET+'/blob/'+COMMIT+'/'+item['path'],
'title': row['title'], 'abstract': row['abstract'], 'authors': row['authors'],
'conference': row['conference'], 'year': row['year'], 'track': row['track'],
'mirror_decision_category': decision_claim(row['decision']),
'mirror_decision_track_claim': decision_track_claim(row['decision']), 'raw_fields': row,
'source_locator': locator, 'identifiers': {},
'metadata': {'forum_id_claim': row['forum_id'], 'original_forum_id_verified': False,
'original_decision_signature_verified': False, 'source_visibility': 'public_mirror',
'scientific_work_identity_approved': False, 'research_type_classified': False,
'approved_for_import': False,
'scope': 'Mirror row only. No original forum/person/decision verification; absence is not rejection or proof of no later acceptance.'}}
def cached_projections(path, expected_sha256, split, items):
path = Path(path)
raw = path.read_bytes()
if (not isinstance(expected_sha256, str) or not re.fullmatch('[0-9a-f]{64}', expected_sha256)
or hashlib.sha256(raw).hexdigest() != expected_sha256):
raise ValueError('Cached projection report hash differs')
report = json.loads(raw)
if (report.get('dataset') != DATASET or report.get('commit') != COMMIT
or report.get('split') != split or report.get('inventory_sha256') != TREE_SHA256
or report.get('raw_parquet_saved') is not False or report.get('database_written') is not False
or report.get('paper_body_saved') is not False or report.get('review_text_saved') is not False
or report.get('approved_for_import') != 0):
raise ValueError('Cached report scope differs from permitted mirror projection')
declared = report.get('files', [])
if len(declared) != len(items) or len({f['source_file'] for f in declared}) != len(items):
raise ValueError('Incomplete cached mirror shard set')
result = {}
for item in items:
fm = next((f for f in declared if f['source_file'] == item['path']), None)
filename = Path(item['path']).stem+'.metadata.jsonl'
if not fm or Path(fm['file']).name != filename or fm['source_lfs_sha256_claim'] != item['lfs']['oid']:
raise ValueError('Cached mirror source differs')
file = path.parent/filename
if file.is_symlink():
raise ValueError('Symlink metadata cache forbidden')
content = file.read_bytes()
if hashlib.sha256(content).hexdigest() != fm['file_sha256']:
raise ValueError('Cached metadata file hash differs')
rows = []
for index, line in enumerate(content.splitlines()):
previous = json.loads(line); row = previous['raw_fields']; expected = record(row, split, item, index)
for key in ('source', 'external_id', 'url', 'title', 'abstract', 'authors', 'conference', 'year', 'track', 'raw_fields', 'source_locator', 'identifiers', 'metadata'):
if previous.get(key) != expected[key]:
raise ValueError('Cached raw metadata/identity differs: '+key)
rows.append(row)
proof = fm['projection']
if (len(rows) != fm['records'] or len(rows) != proof['rows'] or proof['columns'] != COLUMNS
or proof['source_bytes'] != item['size'] or proof['raw_parquet_saved'] is not False
or proof['footer_saved'] is not False or proof['excluded_payload_pages_downloaded'] is not False
or proof['full_file_checksum_verified'] is not False):
raise ValueError('Cached projection proof differs')
result[item['path']] = (rows, proof)
if sum(len(rows) for rows, _ in result.values()) != report['records']:
raise ValueError('Cached projection row count differs')
return result
def collect(split, out, *, source_factory=RangeSource, progress=None, from_projection=None, projection_sha256=None):
items = inventory(split); out = Path(out)
if bool(from_projection) != bool(projection_sha256):
raise ValueError('Cached projection and exact report hash required together')
if out.exists() or not out.parent.is_dir() or out.is_symlink():
raise ValueError('New output directory in existing parent required')
out.mkdir()
counts = Counter(); track_claims = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = []
observed = datetime.now(timezone.utc).isoformat()
try:
cached = cached_projections(from_projection, projection_sha256, split, items) if from_projection else None
for item in items:
if cached is None:
source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path']
source = source_factory(source_url, item['size'])
rows, proof = projection(source, COLUMNS, progress=progress)
else:
rows, proof = cached[item['path']]
if progress:
progress({'phase': 'cached_metadata_projection', 'file': item['path'], 'rows': len(rows)})
path = out/(Path(item['path']).stem+'.metadata.jsonl')
part = path.with_suffix(path.suffix+'.part'); h = hashlib.sha256()
with part.open('wb') as stream:
for index, row in enumerate(rows):
item_record = record(row, split, item, index)
raw = (json.dumps(item_record, ensure_ascii=False)+'\n').encode()
stream.write(raw); h.update(raw); records += 1
counts[item_record['mirror_decision_category']] += 1
track_claims[item_record['mirror_decision_track_claim'] or 'unspecified'] += 1
raw_counts[json.dumps(row['decision'], ensure_ascii=False)] += 1
years[(row['year'], row['track'])] += 1
part.rename(path)
reports.append({'source_file': item['path'], 'file': str(path), 'records': len(rows),
'file_sha256': h.hexdigest(), 'source_lfs_sha256_claim': item['lfs']['oid'],
'projection': proof})
report = {'status': 'collected_mirror_metadata_pending_original_provenance_and_work_identity',
'dataset': DATASET, 'commit': COMMIT, 'inventory_sha256': TREE_SHA256,
'split': split, 'observed_at': observed, 'records': records, 'files': reports,
'mirror_decision_counts': dict(counts), 'raw_mirror_decision_counts': dict(raw_counts),
'mirror_decision_track_claim_counts': dict(track_claims),
'year_track_counts': [{'year': y, 'track': t, 'rows': n} for (y, t), n in sorted(years.items(), key=lambda x: (x[0][0], str(x[0][1])))],
'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports) if cached is None else 0,
'source_projection_bytes': sum(x['projection']['metadata_bytes_read'] for x in reports),
'cached_projection_report_sha256': projection_sha256,
'excluded_payload_pages_downloaded': False, 'raw_parquet_saved': False,
'paper_body_saved': False, 'review_text_saved': False, 'database_written': False,
'approved_for_import': 0,
'limitations': ['Public mirror copies do not verify original forum IDs, visibility or signed decisions.',
'Original source LFS SHA256 is a pinned claim, not a computed full-Parquet checksum.',
'Every selected row and row locator is retained, including missing or anonymous author data.',
'Decision categories describe only explicit mirror literals; venue labels and missing decisions do not imply rejection.',
'Rows are not independent scientific works or a complete venue/year public negative census.',
'Cross-source identity, chronological rejection/acceptance history and final dedup remain.']}
(out/'collection-report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
return report
except BaseException as error:
# No source-specific body or signed redirect URLs in failure reports.
failure = {'status': 'incomplete_do_not_import', 'split': split, 'commit': COMMIT,
'completed_files': reports, 'error_type': type(error).__name__,
'database_written': False, 'approved_for_import': 0, 'no_automatic_retry': True}
(out/'failure.json').write_text(json.dumps(failure, ensure_ascii=False, indent=2)+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--split', required=True, choices=SPLITS)
parser.add_argument('--out', required=True)
parser.add_argument('--from-projection', help='Offline replay of a complete permitted metadata projection')
parser.add_argument('--projection-sha256', help='Exact cached collection-report.json SHA256')
args = parser.parse_args(argv)
report = collect(args.split, args.out, from_projection=args.from_projection, projection_sha256=args.projection_sha256,
progress=lambda row: print(json.dumps(row), flush=True))
print(json.dumps({'status': report['status'], 'records': report['records'], 'out': args.out}), flush=True)
+128
View File
@@ -0,0 +1,128 @@
"""Versioned publication-source scopes and guards against stale source refreshes."""
import json
import re
TABLE = 'source_scope_revisions'
MARKER = 'source_scope_revisions_v1'
COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url',
'parent_sha256', 'records_json', 'plan_json', 'applied_at')
def available(c):
if getattr(c, 'postgres', False):
return bool(c.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (TABLE,)).fetchone())
return bool(c.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)).fetchone())
def initialize(c):
exists = available(c)
marker = bool(c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone())
if exists != marker:
raise ValueError('Source revision table/marker mismatch')
if exists:
result = c.execute('SELECT * FROM source_scope_revisions LIMIT 0')
description = result.cursor.description if getattr(c, 'postgres', False) else result.description
if tuple(x[0] for x in description) != COLUMNS:
raise ValueError('Unexpected source revision schema')
return
ddl = '''CREATE TABLE source_scope_revisions (
version_sha256 TEXT PRIMARY KEY,
source_id INTEGER NOT NULL REFERENCES sources(id),
venue_id INTEGER NOT NULL REFERENCES venues(id),
year INTEGER NOT NULL, event_key TEXT NOT NULL, list_url TEXT NOT NULL,
parent_sha256 TEXT UNIQUE REFERENCES source_scope_revisions(version_sha256),
records_json TEXT NOT NULL, plan_json TEXT NOT NULL,
applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)'''
if getattr(c, 'postgres', False):
ddl = re.sub(r'\bINTEGER\b', 'BIGINT', ddl)
ddl = re.sub(r'\bTEXT\b', 'TEXT COLLATE "C"', ddl)
c.execute(ddl)
c.execute('CREATE INDEX source_scope_revisions_scope ON source_scope_revisions(source_id,venue_id,year,event_key,list_url)')
c.execute('INSERT INTO schema_migrations(name) VALUES(?)', (MARKER,))
def heads(c, source_id):
if not available(c):
return []
if not c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone():
raise ValueError('Source revision table/marker mismatch')
return c.execute('''SELECT v.version_sha256,v.venue_id,v.year,v.event_key,v.list_url
FROM source_scope_revisions v WHERE v.source_id=? AND NOT EXISTS
(SELECT 1 FROM source_scope_revisions child WHERE child.parent_sha256=v.version_sha256)
ORDER BY v.version_sha256''', (source_id,)).fetchall()
def current_rows(c, source_id, cache):
active = heads(c, source_id)
signatures = tuple(r[0] for r in active)
# A new head invalidates the cache, even when maintenance uses the same Store.
if cache.get('signatures') != (source_id, signatures):
records = {}; scopes = {}
for signature in signatures:
raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0]
scopes[signature] = json.loads(raw)
for row in scopes[signature]:
ext = row['external_id']
if ext in records and records[ext] != row:
raise ValueError('Conflicting active source snapshots')
records[ext] = row
cache.clear()
cache.update(signatures=(source_id, signatures), records=records, scopes=scopes)
return cache['records']
def subset(expected, actual):
"""Reviewed source fields must survive additional work-identity/curation evidence."""
return isinstance(actual, dict) and all(k in actual and actual[k] == v for k, v in expected.items())
def require_record(c, source_id, external, url, title, abstract, authors, identifiers, metadata, cache):
expected = current_rows(c, source_id, cache).get(external)
if expected is None:
return
actual = {'url': url, 'title': title, 'abstract': abstract, 'authors': authors}
if any(actual[k] != expected.get(k) for k in actual):
raise ValueError('Stale source snapshot: reconcile the source version before refresh')
if not subset(expected.get('metadata', {}), metadata or {}) or not subset(expected.get('identifiers', {}), identifiers or {}):
raise ValueError('Stale source snapshot metadata or identifiers')
def scope_head(c, source_id, manifest):
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (manifest.get('venue'),)).fetchone()
if not venue:
return None
matches = [r for r in heads(c, source_id)
if [r[i] for i in range(1, 5)] == [venue[0], manifest.get('year'), manifest.get('event_key', ''), manifest.get('list_url')]]
if len(matches) > 1:
raise ValueError('Forked source revision history')
return matches[0][0] if matches else None
def require_manifest(c, source_id, manifest, cache):
current_rows(c, source_id, cache)
signature = scope_head(c, source_id, manifest)
if signature is None:
return
expected = sum(r['track'] == manifest.get('track') for r in cache['scopes'][signature])
bound = manifest.get('coverage_expected_count') if manifest.get('coverage_mode') == 'incremental' else manifest.get('count')
if expected < 1 or bound != expected or manifest.get('status') != 'partial':
raise ValueError('Source coverage manifest differs from current source snapshot')
def require_row(c, source_id, row, manifest, cache):
expected = current_rows(c, source_id, cache).get(row['external_id'])
scoped_manifest = {**manifest, 'venue': row['venue'], 'year': row['year'], 'event_key': row.get('event_key', '')}
signature = scope_head(c, source_id, scoped_manifest)
if expected is None:
if signature is not None:
raise ValueError('Publication absent from current source snapshot; reconcile source membership first')
return
if signature is None or not any(r['external_id'] == row['external_id'] for r in cache['scopes'][signature]):
raise ValueError('Source listing differs from current source snapshot')
for key, value in expected.items():
if key in ('metadata', 'identifiers'):
okay = subset(value, row.get(key))
else:
okay = row.get(key) == value
if not okay:
raise ValueError('Stale source snapshot field: ' + key)
+4
View File
@@ -96,6 +96,10 @@ class Store:
if found: if found:
matches.add(found[0]) matches.add(found[0])
old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone() old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
from paper_atlas.db.source_revisions import require_record
if not hasattr(self, '_source_revision_cache'):
self._source_revision_cache = {}
require_record(c, source_id, external_id, url, title, abstract, authors, identifiers, metadata, self._source_revision_cache)
from paper_atlas.db.edition_scopes import preserve from paper_atlas.db.edition_scopes import preserve
metadata = preserve(dict(metadata or {}), old[4] if old else None) metadata = preserve(dict(metadata or {}), old[4] if old else None)
if old: if old:
+7
View File
@@ -101,6 +101,9 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
c.commit() c.commit()
if incremental is not None: if incremental is not None:
require_incremental_parent(c, source_id, manifest, incremental) require_incremental_parent(c, source_id, manifest, incremental)
from paper_atlas.db.source_revisions import require_manifest
store._source_revision_cache = {}
require_manifest(c, source_id, manifest, store._source_revision_cache)
digest = hashlib.sha256() digest = hashlib.sha256()
with path.open('rb') as stream: with path.open('rb') as stream:
for line in stream: for line in stream:
@@ -113,6 +116,10 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
require_track_record(row, manifest) require_track_record(row, manifest)
from paper_atlas.db.events import validate_event_key from paper_atlas.db.events import validate_event_key
validate_event_key(row.get('event_key')) validate_event_key(row.get('event_key'))
from paper_atlas.db.source_revisions import require_row
if not hasattr(store, '_source_revision_cache'):
store._source_revision_cache = {}
require_row(c, source_id, row, manifest, store._source_revision_cache)
external = row['external_id'] external = row['external_id']
if external in seen: if external in seen:
raise ValueError('Duplicate external ID in one collected file: '+external) raise ValueError('Duplicate external ID in one collected file: '+external)
+327
View File
@@ -0,0 +1,327 @@
"""Plan-bound publication scope revisions with complete before-images, never reimport."""
import argparse
from collections import Counter
from contextlib import closing
import copy
import hashlib
import json
from pathlib import Path
import re
from paper_atlas.db import source_revisions as versions
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.ingestion.edition_tracks import curation
from paper_atlas.ingestion.event_identity import database_identity, digest, values
MAX_ROWS = 10000
def validate(inventory):
required = {'format', 'source', 'venue', 'year', 'event_key', 'list_url',
'previous_rows', 'rows', 'evidence'}
if not isinstance(inventory, dict) or set(inventory) != required or inventory['format'] != 1:
raise ValueError('Unsupported source revision inventory')
if (type(inventory['year']) is not int or inventory['year'] < 2000
or not all(isinstance(inventory[k], str) and inventory[k] for k in ('source', 'venue', 'list_url'))
or not isinstance(inventory['event_key'], str)):
raise ValueError('Explicit publication source scope required')
if not inventory['list_url'].startswith('https://'):
raise ValueError('HTTPS source listing required')
evidence = inventory['evidence']
if not isinstance(evidence, list) or not 1 <= len(evidence) <= 100:
raise ValueError('Bounded source evidence required')
for item in evidence:
if (not isinstance(item, dict) or set(item) != {'url', 'sha256'}
or not isinstance(item['url'], str) or not item['url'].startswith('https://')
or not isinstance(item['sha256'], str) or not re.fullmatch('[0-9a-f]{64}', item['sha256'])):
raise ValueError('Source evidence URL/hash required')
snapshots = []
for key in ('previous_rows', 'rows'):
items = inventory[key]
if not isinstance(items, list) or not 1 <= len(items) <= MAX_ROWS:
raise ValueError('Complete bounded source snapshots required')
records = {}
for row in items:
if not isinstance(row, dict) or any(row.get(k) != inventory[k] for k in ('source', 'venue', 'year')):
raise ValueError('Source snapshot scope mismatch')
ext = row.get('external_id')
if (not isinstance(ext, str) or ext in records or row.get('status') != 'accepted'
or not re.fullmatch(r'10\.\d{4,9}/\S+', ext)
or row.get('identifiers', {}).get('doi') != ext
or not isinstance(row.get('track'), str) or not row['track']
or row.get('event_key', '') != inventory['event_key']
or not isinstance(row.get('title'), str) or not row['title']
or not isinstance(row.get('authors'), list)
or not isinstance(row.get('metadata'), dict)):
raise ValueError('Unique DOI-bearing accepted publication records required')
records[ext] = row
snapshots.append(records)
previous, current = snapshots
if set(previous) != set(current):
raise ValueError('Source membership changed; this operation cannot add or remove publications')
for ext, row in current.items():
before = previous[ext]
stable = (set(before) | set(row)) - {'abstract', 'track', 'tags', 'metadata'}
if any(before.get(k) != row.get(k) for k in stable):
raise ValueError('Stable publication identity or decision changed')
return previous, current
def bounded(c, sql, params):
result = values(c.execute(sql + ' LIMIT 100001', params))
if len(result) > 100000:
raise ValueError('Source revision protection exceeds bounded scope')
return result
def capture(c, inventory):
previous, current = validate(inventory)
source = c.execute('SELECT id FROM sources WHERE name=?', (inventory['source'],)).fetchone()
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (inventory['venue'],)).fetchone()
if not source or not venue:
raise ValueError('Existing source and venue required')
sid, vid = source[0], venue[0]
editions = bounded(c, 'SELECT * FROM editions WHERE venue_id=? AND year=? AND event_key=? ORDER BY id',
(vid, inventory['year'], inventory['event_key']))
by_track = {r[3]: r[0] for r in editions}
# The reviewed scope includes all its existing target attempts, even when
# they originate from another source. No empty-target shortcut is used.
scope_subs = 'SELECT s.id FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE e.venue_id=? AND e.year=? AND e.event_key=?'
scope_params = (vid, inventory['year'], inventory['event_key'])
scoped = bounded(c, 'SELECT * FROM submissions WHERE id IN (' + scope_subs + ') ORDER BY id', scope_params)
own_records = bounded(c, '''SELECT DISTINCT r.* FROM source_records r JOIN submission_sources ss ON ss.source_record_id=r.id
WHERE r.source_id=? AND ss.submission_id IN (''' + scope_subs + ') ORDER BY r.id', (sid, *scope_params))
if not own_records:
raise ValueError('Existing source observations required')
if any(r[2] not in current for r in own_records):
raise ValueError('Snapshot omits an existing source observation')
pids = sorted({r[1] for r in scoped})
if not pids or len(pids) > MAX_ROWS:
raise ValueError('Bounded nonempty protected paper scope required')
marks = ','.join('?' for _ in pids)
papers = 'SELECT id FROM papers WHERE id IN (' + marks + ')'
subs = 'SELECT id FROM submissions WHERE paper_id IN (' + papers + ')'
decisions = 'SELECT id FROM decisions WHERE submission_id IN (' + subs + ')'
queries = {
'papers': 'SELECT * FROM papers WHERE id IN (' + papers + ') ORDER BY id',
'identifiers': 'SELECT * FROM identifiers WHERE paper_id IN (' + papers + ') ORDER BY scheme,value',
'source_records': 'SELECT * FROM source_records WHERE paper_id IN (' + papers + ') ORDER BY id',
'submissions': 'SELECT * FROM submissions WHERE paper_id IN (' + papers + ') ORDER BY id',
'decisions': 'SELECT * FROM decisions WHERE submission_id IN (' + subs + ') ORDER BY id',
'submission_sources': 'SELECT * FROM submission_sources WHERE submission_id IN (' + subs + ') ORDER BY submission_id,source_record_id,source_submission_id',
'submission_identifiers': 'SELECT * FROM submission_identifiers WHERE submission_id IN (' + subs + ') ORDER BY edition_id,scheme,value',
'decision_sources': 'SELECT * FROM decision_sources WHERE decision_id IN (' + decisions + ') ORDER BY decision_id,source_record_id,source_decision_id',
'paper_tags': 'SELECT * FROM paper_tags WHERE paper_id IN (' + papers + ') ORDER BY paper_id,tag_id,evidence_key',
}
from paper_atlas.curation.annotations import available
if available(c):
queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN (' + papers + ') ORDER BY id'
packet = {k: bounded(c, q, pids) for k, q in queries.items()}
coverage = bounded(c, '''SELECT es.* FROM edition_sources es JOIN editions e ON e.id=es.edition_id
WHERE es.source_id=? AND e.venue_id=? AND e.year=? AND e.event_key=? ORDER BY es.edition_id,es.list_url''', (sid, *scope_params))
if not coverage or any(r[2] != inventory['list_url'] for r in coverage):
raise ValueError('Unambiguous source listing required')
if len({r[0] for r in coverage}) != len(coverage):
raise ValueError('One source coverage entry per edition required')
heads = [[r[i] for i in range(len(r))] for r in versions.heads(c, sid)]
own_heads = [r for r in heads if r[1:] == [vid, inventory['year'], inventory['event_key'], inventory['list_url']]]
if len(own_heads) > 1:
raise ValueError('Forked source revision history')
parent = own_heads[0][0] if own_heads else None
return {'source_id': sid, 'venue_id': vid, 'editions': editions, 'coverage': coverage,
'packet': packet, 'own_records': own_records, 'revision_heads': heads,
'parent_sha256': parent, 'by_track': by_track,
'venue_names': sorted({inventory['venue']} | {r[0] for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (vid,))})}
def changes(c, inventory, before):
previous, current = validate(inventory)
packet = before['packet']; result = []; sid = before['source_id']
editions = {r[0]: r for r in before['editions']}
for raw in before['own_records']:
ext = raw[2]; old = previous[ext]; new = current[ext]
metadata = json.loads(raw[8])
candidates = [old, new]
if not any(raw[4] == r['url'] and raw[5] == r['title'] and raw[6] == r.get('abstract')
and json.loads(raw[7]) == r['authors'] and versions.subset(r['metadata'], metadata) for r in candidates):
raise ValueError('Current raw source differs from both reviewed snapshots')
if metadata.get('record_identifiers', {}).get('doi') != ext:
raise ValueError('Current stable DOI differs from source evidence')
attempts = [s for s in packet['submissions'] if s[3] == raw[0] and s[2] in editions]
if len(attempts) != 1:
raise ValueError('Exactly one publication attempt per source record required')
sub = attempts[0]; origin = editions[sub[2]]
if origin[3] not in {old['track'], new['track']} or sub[4] != ext or sub[1] != raw[3]:
raise ValueError('Current source attempt identity/scope differs')
if [x for x in packet['submission_sources'] if x[0] == sub[0]] != [[sub[0], raw[0], ext]]:
raise ValueError('Shared-source attempt requires separate reconciliation')
decisions = [d for d in packet['decisions'] if d[1] == sub[0]]
if not decisions or any(d[2] != 'accepted' for d in decisions):
raise ValueError('Only accepted publication observations can be revised')
target = before['by_track'].get(new['track'])
if target is None:
raise ValueError('Reviewed target edition must already exist')
identities = [x for x in packet['submission_identifiers'] if x[3] == sub[0]]
if not identities or any(x[0] != sub[2] for x in identities):
raise ValueError('Inconsistent scoped submission identifiers')
for _, scheme, value, _ in identities:
conflict = c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?', (target, scheme, value)).fetchone()
if conflict and conflict[0] != sub[0]:
raise ValueError('Target submission identifier collision')
replacement = copy.deepcopy(metadata); replacement.update(new['metadata'])
if sub[2] != target and KEY in metadata:
raise ValueError('An already curated attempt needs explicit scope-chain reconciliation')
if sub[2] != target or raw[6] != new.get('abstract') or replacement != metadata:
result.append({'source_record_id': raw[0], 'paper_id': raw[3], 'submission_id': sub[0],
'external_id': ext, 'from_edition_id': sub[2], 'target_edition_id': target,
'from_track': origin[3], 'track': new['track'],
'new_abstract': new.get('abstract'), 'metadata': replacement})
old_expected = Counter(r['track'] for r in previous.values()); new_expected = Counter(r['track'] for r in current.values())
observed = Counter()
own_ids = {r[0] for r in before['own_records']}
for s in packet['submissions']:
if s[3] in own_ids and s[2] in editions:
observed[s[2]] += 1
coverage = []
for raw in before['coverage']:
track = editions[raw[0]][3]
if raw[3] != 'partial' or raw[8] is not None or raw[5] != observed[raw[0]]:
raise ValueError('Coverage parent is not the verified partial source observation count')
if raw[4] not in {old_expected[track], new_expected[track]}:
raise ValueError('Coverage expected count differs from both source versions')
count = raw[5] + sum((r['target_edition_id'] == raw[0]) - (r['from_edition_id'] == raw[0]) for r in result)
if count < 0 or count > new_expected[track]:
raise ValueError('Corrected coverage exceeds current source membership')
coverage.append({'before': raw, 'expected_count': new_expected[track], 'imported_count': count})
covered = {r['before'][0] for r in coverage}
if any(r['target_edition_id'] not in covered for r in result):
raise ValueError('Target source coverage must already exist')
return result, coverage
def proposal(c, inventory, identity):
before = capture(c, inventory)
if before['parent_sha256']:
saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']):
raise ValueError('Previous snapshot differs from active source version')
records, coverage = changes(c, inventory, before)
return {'format': 1, 'database': identity, 'inventory': inventory, 'before': before,
'changes': records, 'coverage': coverage}
def verify(c, plan, signature):
after = capture(c, plan['inventory']); expected = copy.deepcopy(plan['before']['packet'])
for change in plan['changes']:
for row in expected['submissions']:
if row[0] == change['submission_id']:
row[2] = change['target_edition_id']
for row in expected['submission_identifiers']:
if row[3] == change['submission_id']:
row[0] = change['target_edition_id']
for row in expected['source_records']:
if row[0] == change['source_record_id']:
row[6] = change['new_abstract']
row[8] = encoded_metadata(plan, change, signature)
expected['submission_identifiers'].sort(key=lambda r: (r[0], r[1], r[2]))
if after['packet'] != expected:
raise ValueError('Protected profiles/history/tags/identifiers differ; roll back')
if after['editions'] != plan['before']['editions']:
raise ValueError('Edition identities changed')
expected_heads = [r for r in plan['before']['revision_heads'] if r[0] != plan['before']['parent_sha256']]
inventory = plan['inventory']
expected_heads.append([signature, plan['before']['venue_id'], inventory['year'], inventory['event_key'], inventory['list_url']])
if after['revision_heads'] != sorted(expected_heads):
raise ValueError('Active source version differs')
expected_coverage = []
for item in plan['coverage']:
row = item['before'][:]; row[4] = item['expected_count']; row[5] = item['imported_count']
expected_coverage.append(row)
if after['coverage'] != expected_coverage:
raise ValueError('Reconciled coverage differs; roll back')
stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()
if (not stored or json.loads(stored[0]) != plan['inventory']['rows']
or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
raise ValueError('Versioned source or complete before-images changed')
return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']),
'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])}
def encoded_metadata(plan, change, signature):
metadata = copy.deepcopy(change['metadata'])
if change['from_edition_id'] != change['target_edition_id']:
inventory = plan['inventory']
evidence = inventory['evidence'][0]
row = {**change, 'source': inventory['source'], 'doi': change['external_id'],
'venue_id': plan['before']['venue_id'], 'year': inventory['year'],
'event_key': inventory['event_key'], 'venue_names': plan['before']['venue_names'],
'section': change['track'], 'evidence_url': evidence['url'], 'evidence_sha256': evidence['sha256']}
metadata[KEY] = [curation(row, change['target_edition_id'], signature)]
return json.dumps(metadata, ensure_ascii=False)
def apply_plan(c, plan):
if plan.get('format') != 1:
raise ValueError('Unsupported source revision plan')
if not getattr(c, 'postgres', False) and not c.in_transaction:
c.execute('BEGIN IMMEDIATE')
signature = digest(plan)
if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone():
return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)}
if getattr(c, 'postgres', False):
c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE')
if proposal(c, plan['inventory'], plan['database']) != plan:
raise ValueError('Stale source revision plan; create and review a fresh plan')
versions.initialize(c)
for row in plan['changes']:
if row['from_edition_id'] != row['target_edition_id']:
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?', (row['target_edition_id'], row['submission_id']))
c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?', (row['target_edition_id'], row['submission_id'], row['from_edition_id']))
c.execute('UPDATE source_records SET abstract=?,metadata_json=? WHERE id=?',
(row['new_abstract'], encoded_metadata(plan, row, signature), row['source_record_id']))
for row in plan['coverage']:
before = row['before']
c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
(row['expected_count'], row['imported_count'], before[0], before[1], before[2]))
inventory = plan['inventory']
c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json)
VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'],
inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False)))
return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)}
def run(db, out, *, pg_service=None, pg_schema=None, inventory=None, plan=None, plan_sha256=None):
out = Path(out)
if out.exists() or not out.parent.is_dir():
raise ValueError('New output in existing directory required')
if bool(inventory) == bool(plan) or bool(plan) != bool(plan_sha256):
raise ValueError('Choose inventory preflight or plan plus exact file hash')
if inventory:
with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
result = proposal(c, json.loads(Path(inventory).read_bytes()), database_identity(c, db))
else:
raw = Path(plan).read_bytes()
if hashlib.sha256(raw).hexdigest() != plan_sha256:
raise ValueError('Plan file hash differs')
proposed = json.loads(raw)
with closing(open_store(db, pg_service=pg_service, pg_schema=pg_schema).conn) as c:
try:
if proposed['database'] != database_identity(c, db):
raise ValueError('Plan belongs to another database')
if not getattr(c, 'postgres', False):
c.commit(); c.execute('BEGIN IMMEDIATE')
result = apply_plan(c, proposed); c.commit()
except BaseException:
c.rollback(); raise
out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog); database_arguments(parser)
parser.add_argument('--inventory'); parser.add_argument('--plan'); parser.add_argument('--plan-sha256'); parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
report = run(args.db, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
inventory=args.inventory, plan=args.plan, plan_sha256=args.plan_sha256)
print(json.dumps({'status': report.get('status', 'planned'), 'out': args.out}), flush=True)
-195
View File
@@ -1,195 +0,0 @@
"""Read only selected Parquet column chunks; never fetch other payload pages."""
import hashlib
import io
import re
import struct
import urllib.error
import urllib.request
MAX_FOOTER = 4 * 1024**2
MAX_SELECTED_BYTES = 64 * 1024**2
MAX_REQUESTS = 512
class RangeSource:
"""Strict bounded HTTP ranges, no full-file fallback, redirects kept private."""
def __init__(self, url, size, *, opener=None):
self.url, self.size = url, size
self.opener = opener or urllib.request.build_opener()
self.etag = None
self.receipts = []
self.bytes_read = 0
def read(self, start, length):
if (type(start) is not int or type(length) is not int or length <= 0
or start < 0 or start + length > self.size
or self.bytes_read + length > MAX_SELECTED_BYTES
or len(self.receipts) >= MAX_REQUESTS):
raise ValueError('Bounded metadata range budget exceeded')
headers = {'Range': f'bytes={start}-{start + length - 1}', 'Accept-Encoding': 'identity'}
if self.etag and not self.etag.startswith('W/'):
headers['If-Match'] = self.etag
request = urllib.request.Request(self.url, headers=headers)
try:
with self.opener.open(request, timeout=45) as response:
if response.status != 206:
raise ValueError('Server did not honor metadata range; full download forbidden')
if response.headers.get('Content-Range') != f'bytes {start}-{start + length - 1}/{self.size}':
raise ValueError('Object size or range differs from pinned inventory')
encoding = response.headers.get('Content-Encoding', 'identity')
if encoding != 'identity':
raise ValueError('Encoded HTTP ranges are unsupported')
etag = response.headers.get('ETag')
if self.receipts and etag != self.etag:
raise ValueError('Object ETag changed between metadata ranges')
raw = response.read(length + 1)
if len(raw) != length:
raise ValueError('Truncated or oversized metadata range')
if not self.receipts:
self.etag = etag
except urllib.error.HTTPError as error:
# Do not expose signed CDN redirect URLs or retry denied resources.
raise RuntimeError('Metadata source HTTP ' + str(error.code) + '; no retry or full-file fallback') from None
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
self.bytes_read += length
return raw
class SparseFile(io.RawIOBase):
"""A logical file backed only by approved bytes; all other positions are zeros.
Arrow may read a 64 KiB footer window. Synthesizing its unused prefix avoids
fetching the preceding OCR/review page. Data pages are explicitly selected
from footer offsets before they can be downloaded.
"""
def __init__(self, size, segments):
self.size, self.segments, self.position = size, segments, 0
def readable(self):
return True
def seekable(self):
return True
def tell(self):
return self.position
def seek(self, offset, whence=0):
base = {0: 0, 1: self.position, 2: self.size}.get(whence)
if base is None or base + offset < 0:
raise ValueError('Invalid sparse metadata seek')
self.position = base + offset
return self.position
def read(self, length=-1):
if length < 0:
length = self.size - self.position
length = min(length, max(0, self.size - self.position))
if length > MAX_SELECTED_BYTES:
raise ValueError('Unbounded sparse read forbidden')
start, end = self.position, self.position + length
raw = bytearray(length)
for offset, data in self.segments:
left, right = max(offset, start), min(offset + len(data), end)
if left < right:
raw[left-start:right-start] = data[left-offset:right-offset]
self.position = end
return bytes(raw)
def readinto(self, buffer):
raw = self.read(len(buffer)); buffer[:len(raw)] = raw
return len(raw)
def arrow():
try:
import pyarrow as pa
import pyarrow.parquet as pq
except ImportError:
raise RuntimeError('Install paper-atlas[mirrors] only on the approved remote runner') from None
return pa, pq
def projection(source, columns, *, progress=None):
"""Return complete metadata rows and range receipts, no full Parquet snapshot."""
pa, pq = arrow()
if type(source.size) is not int or not 12 <= source.size <= 4 * 1024**3:
raise ValueError('Bounded pinned Parquet size required')
if (not isinstance(columns, dict) or not columns
or any(not re.fullmatch('[a-z][a-z0-9_]*', key) for key in columns)):
raise ValueError('Explicit top-level metadata columns required')
header = source.read(0, 4); trailer = source.read(source.size - 8, 8)
if header != b'PAR1' or trailer[4:] != b'PAR1':
raise ValueError('Expected unencrypted Parquet framing')
length = struct.unpack('<I', trailer[:4])[0]
if not 0 < length <= min(MAX_FOOTER, source.size - 12):
raise ValueError('Invalid or oversized Parquet footer')
footer_start = source.size - 8 - length
footer = source.read(footer_start, length)
segments = [(0, header), (footer_start, footer), (source.size - 8, trailer)]
sparse = SparseFile(source.size, segments)
parquet = pq.ParquetFile(sparse, pre_buffer=False, buffer_size=0)
try:
schema = parquet.schema_arrow
if len(set(schema.names)) != len(schema.names) or not set(columns) <= set(schema.names):
raise ValueError('Missing or repeated metadata columns')
for key, kind in columns.items():
value = schema.field(key).type
if kind == 'string':
okay = pa.types.is_string(value) or pa.types.is_large_string(value)
elif kind == 'integer':
okay = pa.types.is_integer(value)
elif kind == 'strings':
okay = (pa.types.is_list(value) or pa.types.is_large_list(value)) and (
pa.types.is_string(value.value_type) or pa.types.is_large_string(value.value_type))
else:
raise ValueError('Unsupported safe metadata type')
if not okay:
raise ValueError('Metadata column type differs: ' + key)
metadata = parquet.metadata
if not 0 < metadata.num_rows <= 100000 or metadata.num_row_groups > 1000:
raise ValueError('Bounded metadata row count required')
wanted, excluded = [], []
for group in range(metadata.num_row_groups):
rg = metadata.row_group(group)
for i in range(rg.num_columns):
col = rg.column(i)
if col.file_path:
raise ValueError('External Parquet column files forbidden')
offsets = [x for x in (col.dictionary_page_offset, col.data_page_offset) if x is not None and x > 0]
if not offsets:
raise ValueError('Missing column chunk offset')
start = min(offsets); end = start + col.total_compressed_size
if not 4 <= start < end <= footer_start:
raise ValueError('Column chunk lies outside the pinned payload')
item = (start, end, group, col.path_in_schema)
(wanted if col.path_in_schema.split('.')[0] in columns else excluded).append(item)
if any(max(a, c) < min(b, d) for a, b, _, _ in wanted for c, d, _, _ in excluded):
raise ValueError('Selected range overlaps an excluded payload column')
if sum(end-start for start, end, _, _ in wanted) + length + 12 > MAX_SELECTED_BYTES:
raise ValueError('Metadata projection exceeds byte budget')
if len(wanted) + 3 > MAX_REQUESTS:
raise ValueError('Metadata projection exceeds request budget')
for start, end, group, path in sorted(wanted):
segments.append((start, source.read(start, end-start)))
if progress:
progress({'phase': 'metadata_column_range', 'row_group': group,
'column': path, 'metadata_bytes_read': source.bytes_read})
rows = []
for group in range(metadata.num_row_groups):
table = parquet.read_row_group(group, columns=list(columns), use_threads=False)
if set(table.column_names) != set(columns) or len(table.column_names) != len(columns):
raise ValueError('Unexpected projected metadata fields')
rows.extend(table.select(list(columns)).to_pylist())
if len(rows) != metadata.num_rows:
raise ValueError('Incomplete metadata projection')
report = {'rows': len(rows), 'row_groups': metadata.num_row_groups,
'columns': dict(columns), 'footer_sha256': hashlib.sha256(footer).hexdigest(),
'column_ranges': [{'start': a, 'length': b-a, 'row_group': g, 'column': p} for a, b, g, p in wanted],
'range_receipts': source.receipts, 'metadata_bytes_read': source.bytes_read,
'source_bytes': source.size, 'full_file_checksum_verified': False,
'excluded_payload_pages_downloaded': False,
'footer_saved': False, 'raw_parquet_saved': False}
return rows, report
finally:
parquet.close()
+1 -1
View File
@@ -8,7 +8,7 @@ TABLES = ('sources', 'papers', 'identifiers', 'source_records', 'venues', 'editi
'submission_sources', 'submission_identifiers', 'decision_sources', 'submission_sources', 'submission_identifiers', 'decision_sources',
'submission_merges', 'decision_merges', 'schema_migrations', 'tags', 'submission_merges', 'decision_merges', 'schema_migrations', 'tags',
'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs') 'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs')
OPTIONAL_TABLES = ('annotation_events',) OPTIONAL_TABLES = ('annotation_events', 'source_scope_revisions')
CHECKS = { CHECKS = {
'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'], 'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'],
'venues': ["venue_type IN ('conference','journal','review_platform')"], 'venues': ["venue_type IN ('conference','journal','review_platform')"],
@@ -1 +0,0 @@
[{"type":"file","oid":"f87639c8a74ab77de91ea6bb4c88718a52e4250f","size":37291354,"lfs":{"oid":"bcb446a0398dfe62593e1db617f82f43bc792fc413d6d4658b0e0d32c358b888","size":37291354,"pointerSize":133},"xetHash":"c9a6d5d70b6f3a9ee09b4e085327e4f61cad91b9b61244e6bf99031fef9dd85b","path":"data/colm-00000-of-00001.parquet"},{"type":"file","oid":"27ef0f337b76d92ac1aaa4e1dc41a1253768545b","size":28627916,"lfs":{"oid":"930dd4ef90fa352fe6bccca7b73eb36eeabfb99f1e77e783572e2db83e15d779","size":28627916,"pointerSize":133},"xetHash":"468b49fe8b9a921e852c1995c4902e175ad7b54fcefb048a485e3fbad8b62bfd","path":"data/corl-00000-of-00001.parquet"},{"type":"file","oid":"c8de3418de080cbd24667e28c55abf684a8a3d9f","size":90794691,"lfs":{"oid":"e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da","size":90794691,"pointerSize":133},"xetHash":"c21b83e5820cea8b7a6bcac34bbd334a0a6ac5b7acb1c4c681715a52ec2cceda","path":"data/emnlp-00000-of-00001.parquet"},{"type":"file","oid":"0a226079bb3fe5256a51750aa874c0e7325f60fa","size":167662887,"lfs":{"oid":"3a65801f95fe490b30282df2cb668fa33fe4182e9d80544148ea66da35215ad8","size":167662887,"pointerSize":134},"xetHash":"2225125f6d18015bb3c44037c82a7f53363eeb636d309de3445e1f1b49f51a03","path":"data/iclr-00000-of-00005.parquet"},{"type":"file","oid":"7b53c49c39f2c287a778e64b18969690926710a8","size":196573710,"lfs":{"oid":"736c05be45eed9db00848471d88dc9cd5a1b1a7d5a0db6432639a21dfb8b4562","size":196573710,"pointerSize":134},"xetHash":"48da5a74c33ca8b04411f4d6f41e34a4ba57d8c4aecb1a72be5dcad89a72bd65","path":"data/iclr-00001-of-00005.parquet"},{"type":"file","oid":"cbf09e34f7f00e33f5bb501461a915ab4bda2ec2","size":210807163,"lfs":{"oid":"1c0b77c370b8735e3d4ebd578706ec8dfbb5e53eb6a71e8cbfeaec36c78ee13f","size":210807163,"pointerSize":134},"xetHash":"fa8bf4848b55d274464b6d06413e1183321e6ffd467e1efa369ee32e0749e8af","path":"data/iclr-00002-of-00005.parquet"},{"type":"file","oid":"4082a290b2e2785413340920e3475c4b5a58b05f","size":250525528,"lfs":{"oid":"078d441ef3854be30f8fd73d34e8442a2a119cd26dbaddf9ab8aba2008df507f","size":250525528,"pointerSize":134},"xetHash":"63ba7c9757b53b02b1793f2186ddc55b79149dbda9a37bacedaf2b0466604d61","path":"data/iclr-00003-of-00005.parquet"},{"type":"file","oid":"d4528dcfcd6d7f56619d2bed75d4705c11811bfb","size":263725824,"lfs":{"oid":"de5efe8578b744f59c8ff9d7c1171cdaa760008920dde190c04b73e5c45901a5","size":263725824,"pointerSize":134},"xetHash":"5934eb773ec49ebf9f48c89d9d7e88ae0d2eb5f5e7f83bfd13f51080aa6d4090","path":"data/iclr-00004-of-00005.parquet"},{"type":"file","oid":"68cd41fa1983b2ba666deb74d1e6da2f9fa9003d","size":204752048,"lfs":{"oid":"4f50e2a0fd7e40a0fdb239595b636c753ca89af1ad9692b7d63f366e7a25eda0","size":204752048,"pointerSize":134},"xetHash":"de8ac963c1738ffe68728255bf7f32fbfff3863b3b07ca1f7026f4c76f123ec0","path":"data/icml-00000-of-00001.parquet"},{"type":"file","oid":"580a98d667a2fc7252fd2bcc2eeeaff116f7030d","size":126503813,"lfs":{"oid":"d2c4359745d30900bd19a44762c9fb6b8415554e2aead6f22f902362aca3e61a","size":126503813,"pointerSize":134},"xetHash":"27cd8f2318a0d096cf3a6aa7178fc502ce52c5d026887f64661b46af74b4b9d5","path":"data/neurips-00000-of-00005.parquet"},{"type":"file","oid":"e90e69bdd1985a36f9a72b4b3b3a9b54d90b7b9d","size":166685127,"lfs":{"oid":"32be2572bc42d33ff227ceadbdcc64e27c996b31e82195d25eced11ec4617963","size":166685127,"pointerSize":134},"xetHash":"2e3528c026d585513df96b23efe39d174b368abd3088e494d85fcf806504a241","path":"data/neurips-00001-of-00005.parquet"},{"type":"file","oid":"b7be2e96d3c3726b87c4a997fcb2c144288b70b4","size":229115194,"lfs":{"oid":"7f9dc8025aebe1d78f74bfccc68e1792f6a076142a920305065d805ff04ab248","size":229115194,"pointerSize":134},"xetHash":"77a4b411199fee5bd1b3fb8752ad4bf1ac090d560a4bfde54b75cf19d48c545b","path":"data/neurips-00002-of-00005.parquet"},{"type":"file","oid":"127763711c2fe76631137432291fa839746536a3","size":264212756,"lfs":{"oid":"32d86e74f00d3fce8eef1cf1db1dd78eed341ddda0e09a924ba17d08c3d566b2","size":264212756,"pointerSize":134},"xetHash":"abe6a12cf6f6f2db52ba28b84a0b91fcfab1411dfa0eb0ab7cdae540b86c5c9d","path":"data/neurips-00003-of-00005.parquet"},{"type":"file","oid":"89eb22ff72a8f44d461e601176239d4f997bbec7","size":279556753,"lfs":{"oid":"b4a536cdb6d306049a32121272b8224ef0e37360423ca04c274f0da74d4dc64c","size":279556753,"pointerSize":134},"xetHash":"d2b68be869a02a2cbe81c74e50fcac3c35585422dbaffcb19985e2376a90e0a7","path":"data/neurips-00004-of-00005.parquet"},{"type":"file","oid":"2b6048e08f894bb47b874a485d3bc5655a43a449","size":211604858,"lfs":{"oid":"df21b892ec531bc223b114d03cd51be6f4f2c48563edde0bdd1b153c36a74a98","size":211604858,"pointerSize":134},"xetHash":"1e81fc6612959ef821e97b7b5ed06a9ebbb3748f0ecc2fe2027b41f7626ef863","path":"data/tmlr-00000-of-00001.parquet"}]
+1 -2
View File
@@ -12,7 +12,6 @@ dependencies = ["PyYAML==6.0.3"]
[project.optional-dependencies] [project.optional-dependencies]
postgres = ["psycopg[binary]==3.3.8"] postgres = ["psycopg[binary]==3.3.8"]
mirrors = ["pyarrow==26.0.1"]
[project.scripts] [project.scripts]
paper-atlas = "paper_atlas.cli:main" paper-atlas = "paper_atlas.cli:main"
@@ -23,4 +22,4 @@ exclude = ["tests*"]
[tool.setuptools.package-data] [tool.setuptools.package-data]
"paper_atlas.db" = ["schema.sql", "annotations.sql"] "paper_atlas.db" = ["schema.sql", "annotations.sql"]
"paper_atlas.resources" = ["venues.json", "reviewarena-files.json"] "paper_atlas.resources" = ["venues.json"]
-203
View File
@@ -1,203 +0,0 @@
"""Metadata projection never requests OCR/review column pages or promotes mirror IDs."""
from collections import Counter
import hashlib
import importlib.util
import io
import json
from pathlib import Path
import re
import struct
import tempfile
import unittest
from unittest.mock import patch
import urllib.error
from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, decision_track_claim, inventory, record
from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection
class MemorySource:
def __init__(self, raw):
self.raw, self.size, self.bytes_read, self.receipts = raw, len(raw), 0, []
def read(self, start, length):
raw = self.raw[start:start+length]
self.bytes_read += len(raw)
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
return raw
def fixture():
import pyarrow as pa
import pyarrow.parquet as pq
rows = []
for i, decision in enumerate(['Accept-Findings', 'Accept-Main', 'Reject', 'Withdrawn', None]):
row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023,
'track': 'main', 'venue_id': 'source-venue-claim',
'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i),
'authors': ['Anonymous'] if i == 2 else ['Ada Author', 'Yi Researcher'],
'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision,
'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000),
'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000),
'decision_comment': 'META_REVIEW_SENTINEL '+('fixture meta-review '*10000)}
rows.append(row)
sink = io.BytesIO(); pq.write_table(pa.Table.from_pylist(rows), sink, row_group_size=2, compression='snappy')
return sink.getvalue(), [{k: v for k, v in r.items() if k in COLUMNS} for r in rows]
class RangeHTTPTests(unittest.TestCase):
def response(self, data, status=206, start=2, size=10, etag='"stable"'):
response = io.BytesIO(data); response.status = status
response.headers = {'Content-Range': f'bytes {start}-{start+len(data)-1}/{size}', 'ETag': etag}
return response
def test_strict_range_and_changed_object_detection(self):
opener = unittest.mock.Mock(); opener.open.side_effect = [self.response(b'ab'), self.response(b'cd', start=4, etag='"changed"')]
source = RangeSource('https://example.org/public', 10, opener=opener)
self.assertEqual(source.read(2, 2), b'ab')
self.assertEqual(opener.open.call_args.args[0].get_header('Range'), 'bytes=2-3')
with self.assertRaisesRegex(ValueError, 'ETag'):
source.read(4, 2)
self.assertEqual(source.bytes_read, 2)
def test_no_full_file_fallback_or_denied_source_retry(self):
opener = unittest.mock.Mock(); response = self.response(b'ab', status=200)
response.read = unittest.mock.Mock(side_effect=AssertionError('Must not consume full body'))
opener.open.return_value = response
with self.assertRaisesRegex(ValueError, 'full download forbidden'):
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
response.read.assert_not_called()
opener = unittest.mock.Mock(); opener.open.side_effect = urllib.error.HTTPError('https://cdn.example/signed?SECRET=hidden', 403, 'Forbidden', {}, None)
with self.assertRaisesRegex(RuntimeError, '^Metadata source HTTP 403; no retry or full-file fallback$'):
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
self.assertEqual(opener.open.call_count, 1)
def test_sparse_footer_windows_are_synthetic_not_network_reads(self):
source = SparseFile(100, [(0, b'PAR1'), (95, b'footer')])
source.seek(90); self.assertEqual(source.read(10), b'\x00'*5+b'foote')
source.seek(-4, 2); self.assertEqual(source.tell(), 96)
with self.assertRaises(ValueError):
source.seek(-101, 2)
@unittest.skipUnless(importlib.util.find_spec('pyarrow'), 'Install mirrors extra on the approved remote runner')
class ParquetProjectionTests(unittest.TestCase):
def test_complete_selected_rows_and_no_excluded_page_requests(self):
import pyarrow.parquet as pq
raw, expected = fixture(); source = MemorySource(raw); rows, proof = projection(source, COLUMNS)
self.assertEqual(rows, expected)
metadata = pq.ParquetFile(io.BytesIO(raw)).metadata
excluded = []
for i in range(metadata.num_row_groups):
group = metadata.row_group(i)
for j in range(group.num_columns):
col = group.column(j)
if col.path_in_schema.split('.')[0] not in COLUMNS:
start = min(x for x in [col.data_page_offset, col.dictionary_page_offset] if x is not None and x > 0)
excluded.append((start, start+col.total_compressed_size))
for request in source.receipts:
a, b = request['start'], request['start']+request['length']
self.assertFalse(any(max(a, c) < min(b, d) for c, d in excluded))
self.assertFalse(proof['raw_parquet_saved']); self.assertFalse(proof['full_file_checksum_verified'])
self.assertLess(source.bytes_read, len(raw))
encoded = json.dumps(rows)+json.dumps(proof)
self.assertNotIn('OCR_FULLTEXT_SENTINEL', encoded); self.assertNotIn('REVIEW_FULLTEXT_SENTINEL', encoded)
self.assertNotIn('META_REVIEW_SENTINEL', encoded)
def test_wrong_schema_and_footer_fail_before_any_payload_download(self):
raw, _ = fixture(); source = MemorySource(raw)
with self.assertRaisesRegex(ValueError, 'Missing'):
projection(source, {'missing_column': 'string'})
self.assertEqual(len(source.receipts), 3)
source = MemorySource(raw)
with self.assertRaisesRegex(ValueError, 'type differs'):
projection(source, {**COLUMNS, 'year': 'string'})
self.assertEqual(len(source.receipts), 3)
bad = bytearray(raw); bad[-8:-4] = struct.pack('<I', len(raw))
source = MemorySource(bytes(bad))
with self.assertRaisesRegex(ValueError, 'footer'):
projection(source, COLUMNS)
self.assertEqual(len(source.receipts), 2)
def test_collector_output_preserves_claims_without_original_identity(self):
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp:
out = Path(temp)/'projected'
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
report = collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
self.assertEqual(report['mirror_decision_counts'], {'accepted': 2, 'rejected': 1, 'withdrawn': 1, 'unknown': 1})
self.assertEqual(report['records'], 5); self.assertFalse(report['database_written'])
projected = [json.loads(line) for line in next(out.glob('*.jsonl')).read_text().splitlines()]
for row in projected:
self.assertEqual(row['identifiers'], {})
self.assertFalse(row['metadata']['original_forum_id_verified'])
self.assertFalse(row['metadata']['original_decision_signature_verified'])
self.assertNotIn('status', row)
self.assertIn(COMMIT, row['external_id'])
self.assertEqual(projected[2]['authors'], ['Anonymous'])
self.assertEqual(projected[2]['metadata']['forum_id_claim'], 'unverified-source-2.xml')
self.assertEqual(projected[0]['track'], 'main')
self.assertEqual(projected[0]['mirror_decision_track_claim'], 'findings')
all_saved=''.join(p.read_text() for p in out.iterdir())
self.assertNotIn('FULLTEXT_SENTINEL', all_saved); self.assertNotIn('META_REVIEW_SENTINEL', all_saved)
with self.assertRaisesRegex(ValueError, 'New output'):
collect('emnlp', out)
def test_offline_replay_reclassifies_literals_preserving_all_raw_fields_without_network(self):
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp, patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
out = Path(temp)/'v1'; collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
file, = out.glob('*.jsonl'); before = [json.loads(x) for x in file.read_text().splitlines()]
# Mimic the first reviewed source version: new literal spellings were unknown.
for row in before[:2]:
row['mirror_decision_category'] = 'unknown'; row.pop('mirror_decision_track_claim')
file.write_text(''.join(json.dumps(x)+'\n' for x in before))
report_path = out/'collection-report.json'; report = json.loads(report_path.read_text())
report['files'][0]['file_sha256'] = hashlib.sha256(file.read_bytes()).hexdigest()
report_path.write_text(json.dumps(report)); report_hash = hashlib.sha256(report_path.read_bytes()).hexdigest()
no_network = unittest.mock.Mock(side_effect=AssertionError('Offline replay must not contact source'))
result = collect('emnlp', Path(temp)/'v2', from_projection=report_path, projection_sha256=report_hash, source_factory=no_network)
no_network.assert_not_called(); self.assertEqual(result['metadata_bytes_read'], 0)
self.assertEqual(result['source_projection_bytes'], report['metadata_bytes_read'])
self.assertEqual(result['mirror_decision_counts']['accepted'], 2)
self.assertEqual(result['mirror_decision_track_claim_counts'], {'findings': 1, 'main': 1, 'unspecified': 3})
new = [json.loads(x) for x in next((Path(temp)/'v2').glob('*.jsonl')).read_text().splitlines()]
self.assertEqual([r['raw_fields'] for r in new], [r['raw_fields'] for r in before])
self.assertEqual([r['source_locator'] for r in new], [r['source_locator'] for r in before])
with self.assertRaisesRegex(ValueError, 'hash differs'):
collect('emnlp', Path(temp)/'bad', from_projection=report_path, projection_sha256='0'*64, source_factory=no_network)
no_network.assert_not_called()
def test_failure_is_retained_without_source_payload_or_retries(self):
item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': 50, 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp:
out = Path(temp)/'failed'
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]), patch('paper_atlas.collectors.reviewarena.projection', side_effect=RuntimeError('private body example')):
with self.assertRaises(RuntimeError):
collect('emnlp', out, source_factory=lambda url, size: None)
report = json.loads((out/'failure.json').read_text())
self.assertEqual(report['status'], 'incomplete_do_not_import')
self.assertTrue(report['no_automatic_retry']); self.assertNotIn('private body', (out/'failure.json').read_text())
class MirrorSemanticsTests(unittest.TestCase):
def test_pinned_inventory_covers_all_seven_families_and_fifteen_shards(self):
splits = ['colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr']
result = {s: inventory(s) for s in splits}
self.assertEqual(sum(len(v) for v in result.values()), 15)
self.assertEqual([len(result[s]) for s in splits], [1, 1, 1, 5, 1, 5, 1])
self.assertEqual(result['emnlp'][0]['size'], 90794691)
self.assertEqual(result['emnlp'][0]['lfs']['oid'], 'e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da')
with self.assertRaises(ValueError):
inventory('uai')
def test_missing_or_ambiguous_decision_is_never_inferred_from_venue(self):
for literal in ['', None, 'Under review', 'Final venue pending', 'Accept maybe', 'Reject?']:
self.assertEqual(decision_claim(literal), 'unknown')
self.assertEqual(decision_claim('Desk Rejected'), 'desk_rejected')
self.assertEqual(decision_claim('Withdrawn'), 'withdrawn')
self.assertEqual(decision_claim('Accept (Findings)'), 'accepted')
self.assertEqual(decision_claim('Accept-Main'), 'accepted')
self.assertEqual(decision_claim('Accept-Findings'), 'accepted')
self.assertEqual(decision_track_claim('Accept-Findings'), 'findings')
self.assertIsNone(decision_track_claim('Accept (Poster)'))
+209
View File
@@ -0,0 +1,209 @@
"""Whole-source scope correction, stale refresh, full history and atomicity on both engines."""
from contextlib import closing, redirect_stdout
import copy
import hashlib
import io
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import uuid
from paper_atlas.db.backend import open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.db.source_revisions import available, require_row, require_manifest
from paper_atlas.db.store import Store
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
from paper_atlas.ingestion.event_identity import database_identity, digest, values
from paper_atlas.ingestion.records import coverage_options, ingest_file
from paper_atlas.postgres.client import connect
from paper_atlas.postgres.migration import migrate
from paper_atlas.postgres.schema import quote
URL = 'https://example.org/volume'
def seed(path):
old = []
for i in range(4):
doi = '10.1234/publication'+str(i)
old.append({'source': 'fixture', 'venue': 'MM', 'year': 2026, 'track': 'main' if i == 0 else 'unclassified',
'external_id': doi, 'url': 'https://doi.org/'+doi, 'title': 'Distinct publication '+str(i),
'authors': ['Alice Author', 'Bob Writer'], 'abstract': None if i == 1 else 'Source abstract '+str(i),
'identifiers': {'doi': doi}, 'submission_identifiers': {'publication_doi': doi},
'status': 'accepted', 'raw_decision': 'Publication observation', 'evidence_url': URL,
'observed_at': '2026-06-01T00:00:00Z', 'tags': [{'namespace': 'fixture:track', 'value': 'main' if i == 0 else 'unclassified'}],
'metadata': {'original': i, 'program': None, 'scientific_work_identity_approved': False}})
current = copy.deepcopy(old)
current[1].update(track='main', abstract='Reviewed program abstract with NUL\x00 and SOH\x01')
current[1]['metadata']['program'] = {'id': 'p1', 'evidence_url': URL}
current[1]['tags'][0]['value'] = 'main'
store = Store(path)
for row in old[:3]:
p, r = store.record(row['source'], row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
store.decision(p, r, 'MM', 2026, row['track'], row['external_id'], URL, 'accepted', row['raw_decision'],
identities=[('publication_doi', row['external_id'])], observed_at=row['observed_at'])
store.record_tags(p, r, 'fixture', row['tags'])
if row == old[1]:
store.conn.execute('UPDATE papers SET abstract=? WHERE id=?', ('Canonical preprint abstract', p))
p, other = store.record('prior', 'old-forum', 'https://example.org/prior', row['title'], 'Canonical preprint abstract', row['authors'], row['identifiers'])
store.decision(p, other, 'PRIOR', 2025, 'main', 'old-forum', 'https://example.org/prior', 'rejected', 'Reject', observed_at='2025-01-01T00:00:00Z')
# An unrelated source already has an attempt in the populated target edition.
p, r = store.record('other', 'other', 'https://example.org/other', 'Unrelated target paper', 'Other abstract', ['Someone Else'])
store.decision(p, r, 'MM', 2026, 'main', 'other', 'https://example.org/other', 'accepted', 'Published')
sid = store.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
for track, expected, imported in [('main', 1, 1), ('unclassified', 3, 2)]:
eid = store.conn.execute('SELECT id FROM editions WHERE year=2026 AND track=?', (track,)).fetchone()[0]
store.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'partial',?,?,?,'2026-06-01',NULL)", (eid, sid, URL, expected, imported, 'checkpoint'))
store.conn.commit(); store.conn.close()
return {'format': 1, 'source': 'fixture', 'venue': 'MM', 'year': 2026, 'event_key': '', 'list_url': URL,
'previous_rows': old, 'rows': current, 'evidence': [{'url': URL, 'sha256': 'a'*64}]}
class RevisionCases:
def store(self):
return open_store(**self.target)
def plan(self, c):
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
with closing(self.store().conn) as c:
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
self.assertEqual(len(plan['before']['packet']['submissions']), 5)
before_papers = plan['before']['packet']['papers']
before_tags = plan['before']['packet']['paper_tags']
result = apply_plan(c, plan); c.commit()
self.assertEqual((result['events_moved'], result['records_changed']), (1, 1))
self.assertEqual(result['protected_decisions'], 5)
self.assertEqual(values(c.execute('SELECT * FROM papers ORDER BY id')), before_papers)
self.assertEqual(values(c.execute('SELECT * FROM paper_tags ORDER BY paper_id,tag_id,evidence_key')), before_tags)
self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status='rejected'").fetchone()[0], 1)
self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources ORDER BY edition_id')),
[['partial', 2, 2, 'checkpoint', '2026-06-01', None], ['partial', 2, 1, 'checkpoint', '2026-06-01', None]])
saved = json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0])
self.assertEqual(saved, plan)
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
def test_stale_direct_store_and_jsonl_refresh_fail_but_current_refresh_succeeds(self):
with closing(self.store().conn) as c:
plan = self.plan(c); apply_plan(c, plan); c.commit()
with closing(self.store().conn) as c:
store = object.__new__(Store); store.conn = c; store.event_identity = True
old = self.inventory['previous_rows'][1]
with self.assertRaisesRegex(ValueError, 'Stale source'):
store.record('fixture', old['external_id'], old['url'], old['title'], old['abstract'], old['authors'], old['identifiers'], old['metadata'])
c.rollback()
row = self.inventory['rows'][1]
p, r = store.record('fixture', row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
self.assertEqual(c.execute('SELECT abstract FROM papers WHERE id=?', (p,)).fetchone()[0], 'Canonical preprint abstract')
self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?', (r,)).fetchone()[0])[KEY][0]['target']['track'], 'main')
c.rollback()
path = self.root/'old.jsonl'; raw = (json.dumps(old)+'\n').encode(); path.write_bytes(raw)
path.with_suffix('.manifest.json').write_text(json.dumps({'source': 'fixture', 'venue': 'MM', 'year': 2026,
'track': 'unclassified', 'list_url': URL, 'status': 'partial', 'count': 1, 'file_sha256': hashlib.sha256(raw).hexdigest()}))
with self.assertRaisesRegex(ValueError, 'source snapshot'):
ingest_file(self.target.get('db'), path, **{k: v for k, v in self.target.items() if k != 'db'})
def test_unregistered_member_and_changed_program_metadata_are_rejected(self):
with closing(self.store().conn) as c:
apply_plan(c, self.plan(c)); c.commit()
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
cache = {}; unknown = copy.deepcopy(self.inventory['rows'][0]); unknown['external_id'] = '10.1234/unlisted'
with self.assertRaisesRegex(ValueError, 'absent from current'):
require_row(c, sid, unknown, {'list_url': URL}, cache)
bad = copy.deepcopy(self.inventory['rows'][1]); bad['metadata']['program'] = None
with self.assertRaisesRegex(ValueError, 'Stale source'):
require_row(c, sid, bad, {'list_url': URL}, cache)
good = copy.deepcopy(self.inventory['rows'][1]); good['metadata']['verified_work_anchor'] = {'reason': 'extra evidence'}
require_row(c, sid, good, {'list_url': URL}, cache)
with self.assertRaisesRegex(ValueError, 'coverage manifest'):
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 1, 'status': 'partial'}, cache)
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 2, 'status': 'partial'}, cache)
def test_stale_plan_and_target_identifier_collision_prevent_writes(self):
with closing(self.store().conn) as c:
plan = self.plan(c)
c.execute("UPDATE decisions SET raw_value='Additional evidence' WHERE status='rejected'")
with self.assertRaisesRegex(ValueError, 'Stale'):
apply_plan(c, plan)
c.rollback()
move = plan['changes'][0]
target_sub = c.execute('SELECT id FROM submissions WHERE edition_id=? LIMIT 1', (move['target_edition_id'],)).fetchone()[0]
c.execute('INSERT INTO submission_identifiers VALUES(?,?,?,?)', (move['target_edition_id'], 'publication_doi', move['external_id'], target_sub))
with self.assertRaisesRegex(ValueError, 'collision'):
self.plan(c)
c.rollback()
self.assertFalse(available(c))
def test_failed_postcheck_rolls_back_rows_extension_and_coverage(self):
with closing(self.store().conn) as c:
plan = self.plan(c)
with patch('paper_atlas.ingestion.source_revisions.verify', side_effect=ValueError('Injected verification failure')):
with self.assertRaisesRegex(ValueError, 'Injected'):
apply_plan(c, plan)
c.rollback()
self.assertFalse(available(c))
self.assertEqual(self.plan(c), plan)
def test_cli_binds_database_and_file_hash_without_writing_during_preflight(self):
inventory = self.root/'inventory.json'; inventory.write_text(json.dumps(self.inventory))
path = self.root/'plan.json'
args = {k: v for k, v in self.target.items() if k != 'db'}
run(self.target.get('db'), path, inventory=inventory, **args)
with closing(self.store().conn) as c:
self.assertFalse(available(c))
with self.assertRaisesRegex(ValueError, 'hash'):
run(self.target.get('db'), self.root/'bad.json', plan=path, plan_sha256='0'*64, **args)
report = run(self.target.get('db'), self.root/'applied.json', plan=path, plan_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), **args)
self.assertEqual(report['events_moved'], 1)
def test_membership_identity_changes_and_incremental_shrink_remain_forbidden(self):
for mutate in [lambda x: x['rows'].pop(), lambda x: x['rows'][0].update(title='Different work'),
lambda x: x['rows'][0].update(status='rejected')]:
bad = copy.deepcopy(self.inventory); mutate(bad)
with self.assertRaises(ValueError):
validate(bad)
with self.assertRaisesRegex(ValueError, 'incremental'):
coverage_options({'coverage_mode': 'incremental', 'venue': 'MM', 'year': 2026, 'track': 'unclassified',
'status': 'partial', 'count': 1, 'coverage_previous_count': 2,
'coverage_previous_expected_count': 3, 'coverage_expected_count': 2})
class SQLiteSourceRevisionTests(RevisionCases, unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
db = self.root/'source.sqlite'; self.inventory = seed(db); self.target = {'db': db}
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
def test_optional_revision_table_migrates_with_full_history(self):
with closing(self.store().conn) as c:
plan = self.plan(c); apply_plan(c, plan); c.commit()
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_copy_'+uuid.uuid4().hex[:10]
self.addCleanup(drop_schema, service, schema)
with redirect_stdout(io.StringIO()):
migrate(self.target['db'], service, self.root/'copied.json', schema=schema, min_free_gib=0)
with closing(open_store(pg_service=service, pg_schema=schema).conn) as c:
self.assertEqual(json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0]), plan)
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
def drop_schema(service, schema):
with connect(service) as c:
for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)):
c.execute('DROP SCHEMA '+quote(name)+' CASCADE')
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
class PostgreSQLSourceRevisionTests(RevisionCases, unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
db = self.root/'source.sqlite'; self.inventory = seed(db)
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_'+uuid.uuid4().hex[:12]
self.addCleanup(drop_schema, service, schema)
with redirect_stdout(io.StringIO()):
migrate(db, service, self.root/'migration.json', schema=schema, min_free_gib=0)
self.target = {'pg_service': service, 'pg_schema': schema}