Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 codex-bot
|
3f0ac694a0 | ||
 Zayden-Jung
|
32c7edd0a5 | ||
|
Zayden-Jung
|
658d799334 | ||
|
codex-bot
|
53fbe9c75c | ||
|
codex-bot
|
dac6edbe1e | ||
|
Zayden-Jung
|
7a674ac797 | ||
|
codex-bot
|
eacc1e3292 | ||
|
codex-bot
|
2275bc7530 | ||
|
codex-bot
|
d2c016e309 | ||
|
codex-bot
|
ea3393144d | ||
|
codex-bot
|
7035cf33b9 | ||
|
Zayden-Jung
|
60726a8c6b | ||
|
codex-bot
|
a1bbd26a5c | ||
|
Zayden-Jung
|
90f3fb213e | ||
|
codex-bot
|
a68e36907f | ||
|
codex-bot
|
9040649320 | ||
|
codex-bot
|
64ced7af42 |
@@ -48,7 +48,9 @@ jobs:
|
||||
outside.mkdir()
|
||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','event-identity','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','import','records','--help'],cwd=outside,check=True)
|
||||
|
||||
@@ -88,8 +88,11 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel，并从源码目
|
||||
|
||||
开发通过工作分支和 PR 交付，由用户手动合并。
|
||||
|
||||
早期WWW档案的可重放元数据采集、论文/海报分轨与待核查条目见 [WWW档案采集](docs/www-archive.md)。
|
||||
历史联合会议的既有届次归属可通过 [`maintain joint-editions`](docs/joint-editions.md) 先只读规划，再按计划哈希执行；保留论文与投稿决定来源，不重导入。
|
||||
同年多届的 `event_key`、显式迁移、旧文件刷新与查询边界见 [会议事件身份](docs/event-identity.md)。
|
||||
研究类型与自定义标签可通过 [`annotate`](docs/annotations.md) 显式初始化、添加和撤回；两者独立查询，保留证据及完整更正历史，不从会议轨道推断研究类型。
|
||||
|
||||
整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。
|
||||
|
||||
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md)：按完整证据生成只读计划，保留原投稿和决定 ID，事务性修正 edition、标识与覆盖，并防止旧整卷刷新覆盖分类。
|
||||
|
||||
@@ -188,3 +188,9 @@ MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明
|
||||
WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`：1995春季论文三目录均为38个相同条目ID，海报摘要53个条目与作者链接47个条目不齐；1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察，不是已导入或全部录用数量；主会、海报、教程等继续独立核验。
|
||||
|
||||
WWW事件标识生产迁移已独立验收（报告SHA `f5dbb8a8`）：保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源（约223KB），只作为后续维护式采集器的输入；尚未补导早期WWW论文，也未启用可选注解表。
|
||||
|
||||
IJCAI 联办届次的逐条后续核查见 [2018/2022 分区与目录差集](ijcai-sections-2026-11-08.md) 和 [稳定 ID 映射](ijcai-sections-2026-11-08.json)。已定位 345 条非 main 分类及 2022 官方 HTML 缺失但官方文件仍存在的一条记录；生产分类尚未修正，不能把整卷数量作为 main 覆盖率。
|
||||
|
||||
2026年11月9日的[当前历史年度缺口核对](historical-gaps-2026-11-09.md)单列135个历史会议—年份、14个2026年度及85个经范围修正的初步候选来源年度；它使用当前生产edition/别名快照，保留原固定快照不变。年度有记录不等于逐篇、逐track或公开拒稿历史完整。
|
||||
|
||||
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希，排除4个COLING卷对ACL缺口的误匹配，并区分会议举办记录、目录可获取性与论文级覆盖；尚未导入或关闭这些年度缺口。
|
||||
|
||||
@@ -0,0 +1,867 @@
|
||||
{
|
||||
"snapshot_at": "2026-11-08 16:34:40.741334+00:00",
|
||||
"scope": "Official first-pass conference-year census vs current populated editions and real aliases. Missing year means no imported attempt in that year, not missing paper count or proof all other years complete. Includes announced/nonarchival exceptions below.",
|
||||
"venues": [
|
||||
{
|
||||
"venue": "NeurIPS",
|
||||
"expected_years": 40,
|
||||
"observed_expected_years": 39,
|
||||
"missing_years": [
|
||||
2026
|
||||
],
|
||||
"missing_count": 1,
|
||||
"canonical_matches": [
|
||||
"NeurIPS"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ICML",
|
||||
"expected_years": 43,
|
||||
"observed_expected_years": 14,
|
||||
"missing_years": [
|
||||
1980,
|
||||
1983,
|
||||
1985,
|
||||
1987,
|
||||
1988,
|
||||
1989,
|
||||
1990,
|
||||
1991,
|
||||
1992,
|
||||
1993,
|
||||
1994,
|
||||
1995,
|
||||
1996,
|
||||
1997,
|
||||
1998,
|
||||
1999,
|
||||
2000,
|
||||
2001,
|
||||
2002,
|
||||
2003,
|
||||
2004,
|
||||
2005,
|
||||
2006,
|
||||
2007,
|
||||
2008,
|
||||
2009,
|
||||
2010,
|
||||
2011,
|
||||
2012
|
||||
],
|
||||
"missing_count": 29,
|
||||
"canonical_matches": [
|
||||
"ICML"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ICLR",
|
||||
"expected_years": 14,
|
||||
"observed_expected_years": 11,
|
||||
"missing_years": [
|
||||
2014,
|
||||
2015,
|
||||
2016
|
||||
],
|
||||
"missing_count": 3,
|
||||
"canonical_matches": [
|
||||
"ICLR"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "COLM",
|
||||
"expected_years": 3,
|
||||
"observed_expected_years": 3,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"COLM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "COLT",
|
||||
"expected_years": 39,
|
||||
"observed_expected_years": 39,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"COLT"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "AISTATS",
|
||||
"expected_years": 29,
|
||||
"observed_expected_years": 25,
|
||||
"missing_years": [
|
||||
1985,
|
||||
1989,
|
||||
1991,
|
||||
1993
|
||||
],
|
||||
"missing_count": 4,
|
||||
"canonical_matches": [
|
||||
"AISTATS"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "UAI",
|
||||
"expected_years": 42,
|
||||
"observed_expected_years": 42,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"UAI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "CLeaR",
|
||||
"expected_years": 5,
|
||||
"observed_expected_years": 5,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"CLEAR"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ACL",
|
||||
"expected_years": 64,
|
||||
"observed_expected_years": 48,
|
||||
"missing_years": [
|
||||
1963,
|
||||
1964,
|
||||
1965,
|
||||
1966,
|
||||
1967,
|
||||
1968,
|
||||
1969,
|
||||
1970,
|
||||
1971,
|
||||
1972,
|
||||
1973,
|
||||
1974,
|
||||
1975,
|
||||
1976,
|
||||
1977,
|
||||
1978
|
||||
],
|
||||
"missing_count": 16,
|
||||
"canonical_matches": [
|
||||
"ACL",
|
||||
"ACL+COLING",
|
||||
"ACL+EACL"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "EMNLP",
|
||||
"expected_years": 31,
|
||||
"observed_expected_years": 31,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"CoNLL+EMNLP",
|
||||
"EMNLP"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "NAACL",
|
||||
"expected_years": 18,
|
||||
"observed_expected_years": 18,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"NAACL"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "EACL",
|
||||
"expected_years": 19,
|
||||
"observed_expected_years": 19,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"ACL+EACL",
|
||||
"EACL"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "COLING",
|
||||
"expected_years": 31,
|
||||
"observed_expected_years": 28,
|
||||
"missing_years": [
|
||||
1971,
|
||||
1976,
|
||||
1978
|
||||
],
|
||||
"missing_count": 3,
|
||||
"canonical_matches": [
|
||||
"ACL+COLING",
|
||||
"COLING",
|
||||
"COLING+LREC"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "CoNLL",
|
||||
"expected_years": 30,
|
||||
"observed_expected_years": 30,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"CoNLL",
|
||||
"CoNLL+EMNLP"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "CVPR",
|
||||
"expected_years": 39,
|
||||
"observed_expected_years": 14,
|
||||
"missing_years": [
|
||||
1983,
|
||||
1985,
|
||||
1986,
|
||||
1988,
|
||||
1989,
|
||||
1991,
|
||||
1992,
|
||||
1993,
|
||||
1994,
|
||||
1996,
|
||||
1997,
|
||||
1998,
|
||||
1999,
|
||||
2000,
|
||||
2001,
|
||||
2003,
|
||||
2004,
|
||||
2005,
|
||||
2006,
|
||||
2007,
|
||||
2008,
|
||||
2009,
|
||||
2010,
|
||||
2011,
|
||||
2012
|
||||
],
|
||||
"missing_count": 25,
|
||||
"canonical_matches": [
|
||||
"CVPR"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ICCV",
|
||||
"expected_years": 20,
|
||||
"observed_expected_years": 7,
|
||||
"missing_years": [
|
||||
1987,
|
||||
1988,
|
||||
1990,
|
||||
1993,
|
||||
1995,
|
||||
1998,
|
||||
1999,
|
||||
2001,
|
||||
2003,
|
||||
2005,
|
||||
2007,
|
||||
2009,
|
||||
2011
|
||||
],
|
||||
"missing_count": 13,
|
||||
"canonical_matches": [
|
||||
"ICCV"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ECCV",
|
||||
"expected_years": 19,
|
||||
"observed_expected_years": 4,
|
||||
"missing_years": [
|
||||
1990,
|
||||
1992,
|
||||
1994,
|
||||
1996,
|
||||
1998,
|
||||
2000,
|
||||
2002,
|
||||
2004,
|
||||
2006,
|
||||
2008,
|
||||
2010,
|
||||
2012,
|
||||
2014,
|
||||
2016,
|
||||
2026
|
||||
],
|
||||
"missing_count": 15,
|
||||
"canonical_matches": [
|
||||
"ECCV"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "AAAI",
|
||||
"expected_years": 40,
|
||||
"observed_expected_years": 40,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"AAAI",
|
||||
"AAAI+IAAI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "IJCAI",
|
||||
"expected_years": 35,
|
||||
"observed_expected_years": 35,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"ECAI+IJCAI",
|
||||
"IJCAI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ECAI",
|
||||
"expected_years": 29,
|
||||
"observed_expected_years": 27,
|
||||
"missing_years": [
|
||||
1974,
|
||||
1980
|
||||
],
|
||||
"missing_count": 2,
|
||||
"canonical_matches": [
|
||||
"ECAI",
|
||||
"ECAI+IJCAI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "KDD",
|
||||
"expected_years": 32,
|
||||
"observed_expected_years": 32,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"KDD"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "SIGIR",
|
||||
"expected_years": 50,
|
||||
"observed_expected_years": 50,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"SIGIR"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "WWW",
|
||||
"expected_years": 33,
|
||||
"observed_expected_years": 27,
|
||||
"missing_years": [
|
||||
1994,
|
||||
1996,
|
||||
1997,
|
||||
1998,
|
||||
1999,
|
||||
2000
|
||||
],
|
||||
"missing_count": 6,
|
||||
"canonical_matches": [
|
||||
"WWW"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "WSDM",
|
||||
"expected_years": 19,
|
||||
"observed_expected_years": 19,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"WSDM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "RecSys",
|
||||
"expected_years": 20,
|
||||
"observed_expected_years": 19,
|
||||
"missing_years": [
|
||||
2026
|
||||
],
|
||||
"missing_count": 1,
|
||||
"canonical_matches": [
|
||||
"RecSys"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "CIKM",
|
||||
"expected_years": 35,
|
||||
"observed_expected_years": 34,
|
||||
"missing_years": [
|
||||
2026
|
||||
],
|
||||
"missing_count": 1,
|
||||
"canonical_matches": [
|
||||
"CIKM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ICDM",
|
||||
"expected_years": 26,
|
||||
"observed_expected_years": 25,
|
||||
"missing_years": [
|
||||
2026
|
||||
],
|
||||
"missing_count": 1,
|
||||
"canonical_matches": [
|
||||
"ICDM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "SDM",
|
||||
"expected_years": 26,
|
||||
"observed_expected_years": 25,
|
||||
"missing_years": [
|
||||
2026
|
||||
],
|
||||
"missing_count": 1,
|
||||
"canonical_matches": [
|
||||
"SDM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ECML-PKDD",
|
||||
"expected_years": 38,
|
||||
"observed_expected_years": 33,
|
||||
"missing_years": [
|
||||
1986,
|
||||
1987,
|
||||
1988,
|
||||
1989,
|
||||
1991
|
||||
],
|
||||
"missing_count": 5,
|
||||
"canonical_matches": [
|
||||
"ECML",
|
||||
"ECML-PKDD",
|
||||
"PKDD"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ICRA",
|
||||
"expected_years": 43,
|
||||
"observed_expected_years": 43,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"ICRA"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "IROS",
|
||||
"expected_years": 39,
|
||||
"observed_expected_years": 35,
|
||||
"missing_years": [
|
||||
1988,
|
||||
1989,
|
||||
1990,
|
||||
2026
|
||||
],
|
||||
"missing_count": 4,
|
||||
"canonical_matches": [
|
||||
"IROS"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "RSS",
|
||||
"expected_years": 22,
|
||||
"observed_expected_years": 20,
|
||||
"missing_years": [
|
||||
2025,
|
||||
2026
|
||||
],
|
||||
"missing_count": 2,
|
||||
"canonical_matches": [
|
||||
"RSS"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "CoRL",
|
||||
"expected_years": 10,
|
||||
"observed_expected_years": 9,
|
||||
"missing_years": [
|
||||
2026
|
||||
],
|
||||
"missing_count": 1,
|
||||
"canonical_matches": [
|
||||
"CORL"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ICAPS",
|
||||
"expected_years": 37,
|
||||
"observed_expected_years": 28,
|
||||
"missing_years": [
|
||||
1990,
|
||||
1991,
|
||||
1992,
|
||||
1993,
|
||||
1995,
|
||||
1997,
|
||||
1999,
|
||||
2001,
|
||||
2026
|
||||
],
|
||||
"missing_count": 9,
|
||||
"canonical_matches": [
|
||||
"AIPS",
|
||||
"ICAPS"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "KR",
|
||||
"expected_years": 23,
|
||||
"observed_expected_years": 23,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"KR"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "AAMAS",
|
||||
"expected_years": 25,
|
||||
"observed_expected_years": 22,
|
||||
"missing_years": [
|
||||
2012,
|
||||
2025,
|
||||
2026
|
||||
],
|
||||
"missing_count": 3,
|
||||
"canonical_matches": [
|
||||
"AAMAS"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ICASSP",
|
||||
"expected_years": 51,
|
||||
"observed_expected_years": 49,
|
||||
"missing_years": [
|
||||
1977,
|
||||
2026
|
||||
],
|
||||
"missing_count": 2,
|
||||
"canonical_matches": [
|
||||
"ICASSP"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "INTERSPEECH",
|
||||
"expected_years": 39,
|
||||
"observed_expected_years": 39,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"ECST",
|
||||
"EUROSPEECH",
|
||||
"ICSLP",
|
||||
"INTERSPEECH"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ISMIR",
|
||||
"expected_years": 27,
|
||||
"observed_expected_years": 26,
|
||||
"missing_years": [
|
||||
2026
|
||||
],
|
||||
"missing_count": 1,
|
||||
"canonical_matches": [
|
||||
"ISMIR"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "MICCAI",
|
||||
"expected_years": 29,
|
||||
"observed_expected_years": 29,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"MICCAI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "FAccT",
|
||||
"expected_years": 9,
|
||||
"observed_expected_years": 9,
|
||||
"missing_years": [],
|
||||
"missing_count": 0,
|
||||
"canonical_matches": [
|
||||
"FAccT"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "MLSys",
|
||||
"expected_years": 9,
|
||||
"observed_expected_years": 8,
|
||||
"missing_years": [
|
||||
2018
|
||||
],
|
||||
"missing_count": 1,
|
||||
"canonical_matches": [
|
||||
"MLSys"
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "MM",
|
||||
"expected_years": 34,
|
||||
"observed_expected_years": 33,
|
||||
"missing_years": [
|
||||
2026
|
||||
],
|
||||
"missing_count": 1,
|
||||
"canonical_matches": [
|
||||
"MM"
|
||||
]
|
||||
}
|
||||
],
|
||||
"expected_venue_years": 1266,
|
||||
"missing_venue_years": 150,
|
||||
"venues_with_missing_years": 25,
|
||||
"known_nonpublication_exceptions": [
|
||||
{
|
||||
"venue": "NeurIPS",
|
||||
"year": 2026,
|
||||
"reason": "announced;not in proceedings index at census"
|
||||
},
|
||||
{
|
||||
"venue": "CoRL",
|
||||
"year": 2026,
|
||||
"reason": "not in publisher proceedings index at census"
|
||||
},
|
||||
{
|
||||
"venue": "MLSys",
|
||||
"year": 2018,
|
||||
"reason": "first non-archival SysML meeting; not a missing formal publication volume"
|
||||
}
|
||||
],
|
||||
"historical_missing_years_excluding_nonarchival_2018": 135,
|
||||
"historical_venues_with_missing_years": 16,
|
||||
"historical_gaps": [
|
||||
{
|
||||
"venue": "ICML",
|
||||
"years": [
|
||||
1980,
|
||||
1983,
|
||||
1985,
|
||||
1987,
|
||||
1988,
|
||||
1989,
|
||||
1990,
|
||||
1991,
|
||||
1992,
|
||||
1993,
|
||||
1994,
|
||||
1995,
|
||||
1996,
|
||||
1997,
|
||||
1998,
|
||||
1999,
|
||||
2000,
|
||||
2001,
|
||||
2002,
|
||||
2003,
|
||||
2004,
|
||||
2005,
|
||||
2006,
|
||||
2007,
|
||||
2008,
|
||||
2009,
|
||||
2010,
|
||||
2011,
|
||||
2012
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ICLR",
|
||||
"years": [
|
||||
2014,
|
||||
2015,
|
||||
2016
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "AISTATS",
|
||||
"years": [
|
||||
1985,
|
||||
1989,
|
||||
1991,
|
||||
1993
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ACL",
|
||||
"years": [
|
||||
1963,
|
||||
1964,
|
||||
1965,
|
||||
1966,
|
||||
1967,
|
||||
1968,
|
||||
1969,
|
||||
1970,
|
||||
1971,
|
||||
1972,
|
||||
1973,
|
||||
1974,
|
||||
1975,
|
||||
1976,
|
||||
1977,
|
||||
1978
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "COLING",
|
||||
"years": [
|
||||
1971,
|
||||
1976,
|
||||
1978
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "CVPR",
|
||||
"years": [
|
||||
1983,
|
||||
1985,
|
||||
1986,
|
||||
1988,
|
||||
1989,
|
||||
1991,
|
||||
1992,
|
||||
1993,
|
||||
1994,
|
||||
1996,
|
||||
1997,
|
||||
1998,
|
||||
1999,
|
||||
2000,
|
||||
2001,
|
||||
2003,
|
||||
2004,
|
||||
2005,
|
||||
2006,
|
||||
2007,
|
||||
2008,
|
||||
2009,
|
||||
2010,
|
||||
2011,
|
||||
2012
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ICCV",
|
||||
"years": [
|
||||
1987,
|
||||
1988,
|
||||
1990,
|
||||
1993,
|
||||
1995,
|
||||
1998,
|
||||
1999,
|
||||
2001,
|
||||
2003,
|
||||
2005,
|
||||
2007,
|
||||
2009,
|
||||
2011
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ECCV",
|
||||
"years": [
|
||||
1990,
|
||||
1992,
|
||||
1994,
|
||||
1996,
|
||||
1998,
|
||||
2000,
|
||||
2002,
|
||||
2004,
|
||||
2006,
|
||||
2008,
|
||||
2010,
|
||||
2012,
|
||||
2014,
|
||||
2016
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ECAI",
|
||||
"years": [
|
||||
1974,
|
||||
1980
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "WWW",
|
||||
"years": [
|
||||
1994,
|
||||
1996,
|
||||
1997,
|
||||
1998,
|
||||
1999,
|
||||
2000
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ECML-PKDD",
|
||||
"years": [
|
||||
1986,
|
||||
1987,
|
||||
1988,
|
||||
1989,
|
||||
1991
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "IROS",
|
||||
"years": [
|
||||
1988,
|
||||
1989,
|
||||
1990
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "RSS",
|
||||
"years": [
|
||||
2025
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ICAPS",
|
||||
"years": [
|
||||
1990,
|
||||
1991,
|
||||
1992,
|
||||
1993,
|
||||
1995,
|
||||
1997,
|
||||
1999,
|
||||
2001
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "AAMAS",
|
||||
"years": [
|
||||
2012,
|
||||
2025
|
||||
]
|
||||
},
|
||||
{
|
||||
"venue": "ICASSP",
|
||||
"years": [
|
||||
1977
|
||||
]
|
||||
}
|
||||
],
|
||||
"current_2026_gaps": 14
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
# 历史年度缺口的当前核对
|
||||
|
||||
生产只读快照：2026-11-08 16:34:40.741334 UTC（北京时间11月9日00:34），foundry_data/research。对比已核实的43份首轮官方会议年度清单，使用当前有投稿记录的edition及真实venue_aliases，不沿用旧固定快照的missing_years。
|
||||
|
||||
- 清单共1266个会议—年份位置，当前有150个位置尚无投稿记录，涉及25个目标会议。
|
||||
- 其中2025年及以前136个；排除已明确非存档的SysML2018首届，剩**135个历史年度待补采或核实，涉及16个会议**。
|
||||
- 另有14个2026年度待核实。已知NeurIPS2026和CoRL2026在首轮核查时尚未进入论文集索引，不能一律当已公开论文遗漏；其他年度也需逐个核实发布时间。
|
||||
|
||||
这些是“整年没有任何投稿记录”的数量，不是缺论文篇数，也不是完整性百分比。一个年份只有少数记录、只有公开拒稿或缺了某个track，仍会被计作“已有年度”；这些更细缺口及公开拒稿的逐年可获得性尚须继续核验。会议别名共享的联合届次不能当成两批独立论文，未见录用也不能推断拒稿。
|
||||
|
||||
| 会议 | 历史缺口年份数 |
|
||||
| --- | ---: |
|
||||
| ICML | 29 |
|
||||
| CVPR | 25 |
|
||||
| ACL | 16 |
|
||||
| ECCV | 14 |
|
||||
| ICCV | 13 |
|
||||
| ICAPS | 8 |
|
||||
| WWW | 6 |
|
||||
| ECML-PKDD | 5 |
|
||||
| AISTATS | 4 |
|
||||
| ICLR、COLING、IROS | 各3 |
|
||||
| ECAI、AAMAS | 各2 |
|
||||
| RSS、ICASSP | 各1 |
|
||||
|
||||
[完整逐年结果](historical-gaps-2026-11-09.json) 保留43目标的官方预期、现有年份数、缺失年份与规范会议匹配。原始只读生产导出保存在 `/data/paper-index/collected/historical-gap-current-20261109-v1.json`。 导出SHA256 `580cdea6dd9869adafe9bc28fb75cf3854bb290ad65aa5a600d9bc368a528b93`。年度对照清单为 `docs/coverage/inventory-2026-11-08.json`；候选来源文件绑定的输入哈希与本目录缺口JSON逐字节一致。来源清单的年份和哈希沿用相应 `*-editions-2026-11-08.json`，不是本轮重新访问所有会议网站；2026发布状态需要后续更新。
|
||||
|
||||
## 候选来源与下一步
|
||||
|
||||
另一次只读目录核查在现有、已校验的DBLP快照中，为135个历史年度中的**85个**匹配到初步候选论文集，另**50个**尚未按本轮规则匹配到，需要继续识别目录键或寻找官方来源。未匹配不等于会议没举办、未出版或没有公开来源。
|
||||
|
||||
[候选卷清单](historical-source-candidates-2026-11-09.json) 保留卷key、标题、出版年份、原始bibliographic counts和出版者链接。初筛使用目标family映射内的四位卷key年份或卷标题唯一显式年份，再按具体会议身份复核；两位键、出版年份与会议年份差异、前身系列及其他目录命名可能漏匹配。原始计数可能含workshop、前言、索引或多个卷，**不能当可直接导入的录用论文数量**。
|
||||
|
||||
后续[来源范围复核](historical-source-scope-2026-11-09.md)排除了4个误归ACL的COLING卷，因此候选年度由89修正为85、未匹配年度由46增至50。原卷及排除理由保留在JSON中；135个历史年度缺口不因此改变。ACL1963–1978与COLING1971/1976/1978在当前Anthology目录未列出，继续记为来源待核实，不能一律称作已公开但漏采的论文集。
|
||||
|
||||
优先核查已有候选卷较集中的CVPR22年、ICCV12年、ECCV14年和ICML25年。其中ICML早期workshop名称可能属于官方主系列前身，也可能是附属workshop，须对照官方届次证据逐卷区分，不能靠标题包含workshop一刀切。所有候选须经过范围、稳定标识、重复/冲突、原始字段和覆盖预检；这一步没有导入候选卷。
|
||||
|
||||
## 复核方法
|
||||
|
||||
只读获取当前有记录的edition，不连接papers大文本或标签表：
|
||||
|
||||
```sql
|
||||
WITH counts AS (
|
||||
SELECT edition_id,count(*) AS submissions
|
||||
FROM submissions GROUP BY edition_id
|
||||
)
|
||||
SELECT e.id,e.venue_id,v.acronym,e.year,e.track,e.event_key,counts.submissions
|
||||
FROM editions e JOIN venues v ON v.id=e.venue_id
|
||||
JOIN counts ON counts.edition_id=e.id ORDER BY e.id;
|
||||
|
||||
SELECT venue_id,alias FROM venue_aliases ORDER BY venue_id,alias;
|
||||
```
|
||||
|
||||
逐目标取规范会议名或别名相等的edition年份集合，与该目标edition_census.expected_years做集合差；再把2026与明确非存档的SysML2018单列。保留原始导出与校验哈希，不把每个matched year误当逐篇/逐track完整。
|
||||
|
||||
与本清单同一阶段的WWW补采只补1995年现有poster轨道，不新增缺失年度，所以即使其15条增量已验收，135这个年度计数也不会随之下降。早期1994年的50条正式展示候选、307条混合节目候选共357条仍需核实；其中并非每条都是论文。
|
||||
|
||||
本清单用于明确第14步剩余范围，不代表第14步完成，也不替代所有导入结束后的第16步全库去重、摘要、约束和实际查询验收。
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,32 @@
|
||||
# 早期 NLP 来源范围复核
|
||||
|
||||
核查时间：2026-11-08 18:27 UTC（北京时间11月9日）。本轮只修正来源候选和记录可获得性，没有操作生产数据库。
|
||||
|
||||
## 跨会议候选修正
|
||||
|
||||
先前候选清单将 `conf/coling/1965`、`1967`、`1969`、`1973` 匹配到 ACL 年会缺口。这四卷标题明确属于 COLING；[COLING 官方 Anthology 目录](https://aclanthology.org/venues/coling/)将它们列在 COLING 下。出版者写作 ACL 或目标映射包含 COLING，不足以证明某一年是 ACL 联办届次。
|
||||
|
||||
从 ACL 候选中排除上述四卷，原始卷字段和排除理由保留在[候选JSON](historical-source-candidates-2026-11-09.json)的 `excluded_cross_venue_candidates` 中。135个历史会议—年份位置仍未关闭；有初步候选来源的年度由89修正为85，尚未匹配的由46增至50。这些数字是目录初筛结果，不是已验证可导入的论文数。
|
||||
|
||||
联合届次仍按具体年度的明确会议身份处理，例如有官方联办证据的卷可以属于多个会议。不能一律排除跨family卷，也不能把联合卷重复导入为两批论文。
|
||||
|
||||
## 来源待核实的早期年份
|
||||
|
||||
| 范围 | 本轮官方来源观察 | 处理 |
|
||||
| --- | --- | --- |
|
||||
| ACL 1963–1978，共16年 | [ACL 官方 Anthology 目录](https://aclanthology.org/venues/acl/)目前从1979年列起 | 保留历史举办清单；逐年核实是否有可索引发表材料，不能据会议举办就断言漏采正式论文集，也不据目录缺失断言未出版 |
|
||||
| COLING 1971、1976、1978 | [主办委员会历史](https://ufal.mff.cuni.cz/iccl-old)列出这三届；当前Anthology COLING目录未列出 | 来源缺口继续开放，需要主办方或出版者逐篇索引；不生成占位论文或录用/拒稿记录 |
|
||||
|
||||
这19个位置不能用其他会议的卷补齐，也不能因本轮未找到目录而删除。公开拒稿可获得性是独立核查，不从论文集或录用清单缺失推断。
|
||||
|
||||
## 证据与复核
|
||||
|
||||
三份原始HTML及 `receipts.json` 保存在工作证据目录 `paper-index/nlp-history-source-review-v1/`；结构化响应URL、时间、状态、字节数和完整SHA256收录于候选JSON的 `scope_review.evidence`。三次响应均为HTTP200；未获取论文全文。
|
||||
|
||||
| 官方来源 | 响应SHA256 |
|
||||
| --- | --- |
|
||||
| [acl-venue](https://aclanthology.org/venues/acl/) | `d04c188c0769e065f18ca9c0246f70086e2f7027ce3efe5ea0f2cdc641796b72` |
|
||||
| [coling-venue](https://aclanthology.org/venues/coling/) | `8e7ec0a2915bfbccaa26b7a7558a9cf977c4aadc119794516ecebb826f61623f` |
|
||||
| [iccl-history](https://ufal.mff.cuni.cz/iccl-old) | `c5286dc51236c90685d22543ff1f7b209adcb3eedb82a7b1ac2115c8dd0169fc` |
|
||||
|
||||
复核时先校验保存的原始响应哈希，再读取目录中的会议身份和年度。实时网页若已变化，应另存新响应和时间，不能把新内容套用旧哈希。本次不改最初会议年度清单或生产年度快照，不代表第14步覆盖验收或第16步最终全库审计完成。
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,40 @@
|
||||
# IJCAI 2018、2022 逐条目录与 track 核查
|
||||
|
||||
2026-11-08 的只读生产快照中，edition 867（2018）和 871（2022）已正确属于 IJCAI/ECAI 联办会议，但整本论文集的所有投稿仍归在 `main`。本核查用官方目录的逐条 DOI 和分区定位问题，不修改生产数据。
|
||||
|
||||
| 年份 | 库中投稿 | 官方目录条目 | DOI 一对一对应 | 官方 Main track | 明确非 main | 当前目录缺项 |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| 2018 | 870 | 870 | 870 | 709 | 161 | 0 |
|
||||
| 2022 | 863 | 862 | 862 | 678 | 184 | 1 |
|
||||
|
||||
2018 官方分区为 Main track 709、Invited Speakers 3、Evolution of the contours of AI 6、Sister Conferences Best Papers 46、Survey track 16、Journal track 24、Early Career 19、Doctoral Consortium 17、Demos 30。
|
||||
|
||||
2022 官方分区为 Main Track 678、Special Track on AI, the Arts and Creativity 17、Special Track on AI for Good 33、Sister Conferences Best Papers 26、Survey Track 38、Journal Track 19、Early Career 8、Doctoral Consortium 25、Demo Track 18。Special Track 保留独立范围，不因同在正式论文集就合进 main。
|
||||
|
||||
## 862/863 差异已定位，不能删一条凑数
|
||||
|
||||
数据库独有条目：`conf/ijcai/Fang22`，题名 **Fixed-Budget Pure Exploration in Multinomial Logit Bandits**，DOI `10.24963/ijcai.2022/409`，页码 2951–2957。submission 333599、source_record 3578445、paper 3442268。
|
||||
|
||||
[当前官方总目录](https://ijcai.org/proceedings/2022/) 没有这一条，详情页 `/proceedings/2022/409` 本次 HTTP 404；但[官方论文文件](https://ijcai.org/proceedings/2022/0409.pdf) HEAD 返回 HTTP 200、application/pdf、153657 字节。浏览器读取的官方文件标题、作者 Boli Fang、IJCAI-22 页脚及首页 2951 与数据库对应，共七页。服务器没有下载保存 PDF 正文。
|
||||
|
||||
因此这是已定位的“当前 HTML 目录缺项但官方文件仍存在”的来源差异。没有证据证明撤稿、拒稿或数据库多收了一篇；保留原条目与发表证据。其当前官方 track 仍待明确，不根据相邻编号擅自归 main。
|
||||
|
||||
## 可复核证据
|
||||
|
||||
同目录 JSON 保存两份官方 HTML 的 URL、HTTP 状态、时间、字节数和 SHA256，以及生产只读导出 SHA256。服务器快照目录 `/data/paper-index/collected/ijcai-section-audit-v1/` 保留 HTML、来源回执、原始数据库来源导出和 409 HTTP 回执。
|
||||
|
||||
JSON 的 `years[].sections[].records` 每条为 `[submission_id, source_record_id, doi_suffix]`；DOI 为 `10.24963/ijcai.<year>/<doi_suffix>`。这些是现有稳定 ID，不是待插入的新投稿。全部 1732 个当前目录条目均对应唯一数据库来源 DOI，目录与数据库均无重复 DOI，官方目录独有条目为零。
|
||||
|
||||
标题仅作辅助核验：NFKC、casefold、去除 Unicode 非 word 字符后，2018 全部一致；2022 有一个拼写差异，DOI `/552` 的 DBLP 标题将 Revisiting、Re-Construction 拼为 Revisting、Re-Constrcution。原始标题均保留，未因此创建或合并论文。
|
||||
|
||||
## 未完成事项及修正边界
|
||||
|
||||
本次交付只完成来源核查和逐条分区映射，**345 条非 main 的生产分类尚未修正**，第 14 步尚未完成。下一步需要可复用的、带快照前提和验收报告的分区修正流程：
|
||||
|
||||
- 保持原 submission、source_record、paper、decision 的 ID 和证据内容；纠正 edition 归属，不制造“新一次投稿”。
|
||||
- 同步处理依赖 edition 的 submission_identifiers、覆盖记录与联合会议别名；不能只移动 submissions 一个字段。
|
||||
- 保存官方分区文字、URL、源文件 SHA 和原始分类，防止以后 DBLP 整卷刷新把精确 track 覆盖回 main。
|
||||
- 对 409 保持单独待核实状态，不推断撤稿、拒稿或当前 main 归属。
|
||||
- 修正后独立核验完整投稿/决定历史、稳定标识、各 track 行数及实际查询。研究方法类型不能由 track 自动生成。
|
||||
|
||||
这是 2018、2022 两届的完整目录差集和分区证据，不代表其他年份已逐条验收，也不替代所有导入结束后的全库去重。
|
||||
@@ -0,0 +1,201 @@
|
||||
{
|
||||
"format": 1,
|
||||
"source_cache_sha256": "8a65562f3490497cada2919ec77827134beabd5ec9c3690dec3625e2318aa54e",
|
||||
"records": [
|
||||
{
|
||||
"entry_id": "45",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/45/index.html",
|
||||
"individual_sha256": "83bede6afc9a7039fae2dd1f28274a394418b34f5d5632590b913617580bda1a",
|
||||
"title": "WISE - World Wide Information System for R&D; Efforts",
|
||||
"authors": [
|
||||
"Fritz Loseries",
|
||||
"Veronika Samara",
|
||||
"Bodo Urban"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "16",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/16/index.html",
|
||||
"individual_sha256": "cf76da903eaeaa27d19857fdaa13220f6821559bf4ac22844fae10abbaa6f3f3",
|
||||
"title": "The World Wide Web as a Tool for Collaborative Inter-Agency Projects: Creation of an Environmental Metadata Database for the Analysis of the Venice Lagoon Ecosystem.",
|
||||
"authors": [
|
||||
"Mirko Labbri"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "17",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/17/index.html",
|
||||
"individual_sha256": "0578fec9c82e8f2f98c4de0f1bcc44011ebb84c53353fdcc423a69cf70673bc2",
|
||||
"title": "NetRepreneur",
|
||||
"authors": [
|
||||
"Kari-Pekka Turtiainen",
|
||||
"Antti Auer"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "29",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/29/index.html",
|
||||
"individual_sha256": "624ad3f685de1163411e33e90488975d848a63243fd7ffbed0862c6fb58c4140",
|
||||
"title": "Mosaic TueV 2.4.3: Multilingual GUI and Improvements towards User Friendliness",
|
||||
"authors": [
|
||||
"Mike Bretz",
|
||||
"Uwe Koch",
|
||||
"Paul de Bra"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "13",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/13/index.html",
|
||||
"individual_sha256": "02409321a3403a94d8362a9862f4960386ea9c344fb47f4bcdf9eae128e8b46e",
|
||||
"title": "Using a WWW Browser as an Alternative User Interface for Interactive Applications",
|
||||
"authors": [
|
||||
"Frank Lonczewski"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "36",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/36/index.html",
|
||||
"individual_sha256": "ed9dada4ea5903cb8f0a4e75f2958416a6e32647fbf3530e58edd9abf26070ce",
|
||||
"title": "HyperSource: A Hypermedia Program Development And Documentation System",
|
||||
"authors": [
|
||||
"Jan Oliver Borchers"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "57",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/57/index.html",
|
||||
"individual_sha256": "3ad4206ad28a19823172cd2845f636fe7a461c251940e9a93deb8f6484b8a984",
|
||||
"title": "Progressive Image Transmission Capability (PROTRAC) on the World Wide Web",
|
||||
"authors": [
|
||||
"Vojislav Lalich-Petrich",
|
||||
"Gaurav Bhatia",
|
||||
"Larry Davis"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "38",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/38/index.html",
|
||||
"individual_sha256": "940370acc5e6a954b197053fe451e466a904d1f0ac82c8fe4e6ca455faf2c927",
|
||||
"title": "Measuring the Web with Lycos",
|
||||
"authors": [
|
||||
"Michael L. Mauldin"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": "Official abstract listing has no abstract. Canonical HTML has body sections but no explicit abstract; body is not substituted as a summary."
|
||||
},
|
||||
{
|
||||
"entry_id": "05",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/05/index.html",
|
||||
"individual_sha256": "1858be87d6e7f47476e1cb8fb4793c0fce3b8a6ff653bcfd4a35089e0520f177",
|
||||
"title": "Virtual Places - Adding people to the web",
|
||||
"authors": [
|
||||
"Yosi Mass"
|
||||
],
|
||||
"author_source": "author_index",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": "Official abstract listing and canonical HTML have no abstract. Canonical page identifies a contact and links PostScript; no full-text download. Author identity comes from author index, not inferred from contact role."
|
||||
},
|
||||
{
|
||||
"entry_id": "37",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/37/index.html",
|
||||
"individual_sha256": "b700a0a51b7a82a5f88373ae0283bb369d1cf6f455acff7d91cd7313fa8b3adf",
|
||||
"title": "The Internet Public Library: Serving a Global Community",
|
||||
"authors": [
|
||||
"Susan Schweitzer"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "41",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/41/index.html",
|
||||
"individual_sha256": "397b474109f9c1240a77d0f490f2b02dd35fae93febbb7397d1b2a31759a8ec1",
|
||||
"title": "Graphical Browsing in the WorldWideWeb and the MultiMedia Forum",
|
||||
"authors": [
|
||||
"Rainer Pagé",
|
||||
"Klaus Süllow"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "47",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/47/index.html",
|
||||
"individual_sha256": "52eb4bfdd658730a222c98170b72dabf9cbcc5ffd2ada1e900b826baf2b02833",
|
||||
"title": "Interoperating Advanced Hypermedia Systems with WWW",
|
||||
"authors": [
|
||||
"Jian Zhang",
|
||||
"Martin Fruehauf",
|
||||
"Thomas Kirste"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "50",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/50/index.html",
|
||||
"individual_sha256": "8dc8b0561bd8b1b106c5b209a973c695c3cc9df5288450d4a01fe32541476b7e",
|
||||
"title": "Web Enabled Agile Manufacturing",
|
||||
"authors": [
|
||||
"S. V. Iyer",
|
||||
"T. Singh",
|
||||
"U. Ravaioli",
|
||||
"R. E. DeVor",
|
||||
"S. G. Kapoor",
|
||||
"G. Veknatasubramian"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "55",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/55/index.html",
|
||||
"individual_sha256": "21780cd4047f513e8c9aeced5247002e27cf71719f08de85c6531b57a9e5058d",
|
||||
"title": "Weblets: Fundamental Building Blocks for WWW Tools",
|
||||
"authors": [
|
||||
"Thomas J. Watt Jr."
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "59",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/59/index.html",
|
||||
"individual_sha256": "fed4cc9dcaee350a12c5e4b6fa9c99be9bead25a81632512d31566400d3d0fad",
|
||||
"title": "Usenet and the World-Wide Web",
|
||||
"authors": [
|
||||
"M. L. Grant"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
# 按证据修正既有投稿的轨道
|
||||
|
||||
`paper-atlas maintain edition-tracks` 用于已导入整卷论文集、后来取得逐条官方分区证据的情形。它更正原投稿的 edition 归属，保留投稿 ID、所有决定和来源关系；不会重新导入论文、合并实体、生成研究方法标签或推断拒稿。支持独立 SQLite 样本和共享 PostgreSQL；不得写入已冻结的生产 SQLite 备份。
|
||||
|
||||
## 证据与完整范围
|
||||
|
||||
先核实官方目录、稳定 DOI、原始来源记录和分区，保存原始页面及 SHA256。输入 JSON 为 `{"format":1,"records":[...]}`，每条必须包含：
|
||||
|
||||
| 字段 | 含义 |
|
||||
| --- | --- |
|
||||
| `submission_id`、`source_record_id`、`external_id` | 原有投稿、主来源记录及主来源投稿标识 |
|
||||
| `source`、`doi` | 原来源名称及其 metadata.record_identifiers 中的 DOI |
|
||||
| `from_edition_id`、`year`、`venue` | 原 edition 及其真实年份和规范会议名 |
|
||||
| `venue_names` | 规范会议名加全部已有别名；必须与库中一致 |
|
||||
| `track` | 经核实的目标轨道，使用小写字母、数字、下划线；范围未明确可用 proceedings_unspecified |
|
||||
| `section` | 官方分区原文或明确的未确定说明，不推断研究方法 |
|
||||
| `evidence_url`、`evidence_sha256` | 已人工/独立核实的 HTTPS 原始证据和内容哈希 |
|
||||
|
||||
输入必须列出每个原 edition 的**全部**投稿，包含留在原轨道的记录。工具核对身份、DOI、单来源发表观察、现有别名、完整范围以及目标冲突；不自动解释 HTML，也不把填写了 URL/哈希当成已证明内容。核查者应先独立证明每条分区对应关系。
|
||||
|
||||
多来源共同拥有的投稿、有非 accepted 决定历史的投稿、已有分类修订、非空目标 edition、目标已有覆盖或标识等情形会停止，需要专门核对。每个原 edition 须有一个明确来源清单；其他来源或多清单覆盖需先另行协调。本功能不是任意批量移动或跨会合并工具。
|
||||
|
||||
## 只读计划、事务执行和验收
|
||||
|
||||
```sh
|
||||
paper-atlas maintain edition-tracks \
|
||||
--pg-service foundry-research --pg-schema research \
|
||||
--inventory reviewed-inventory.json --out plan.json
|
||||
|
||||
sha256sum plan.json
|
||||
|
||||
paper-atlas maintain edition-tracks \
|
||||
--pg-service foundry-research --pg-schema research \
|
||||
--plan plan.json --plan-sha256 <上一步的完整SHA256> --out applied.json
|
||||
```
|
||||
|
||||
报告路径必须是已有目录中的新文件。预检只读。执行使用 PostgreSQL 单写入 advisory lock，或 SQLite BEGIN IMMEDIATE；整个分类修正、逐来源证据、标识、覆盖及验收在一个事务中完成。计划绑定数据库身份、全部来源文本/非修订元数据、标签、可选 annotation_events、投稿/决定/来源/标识内容及原覆盖快照；过期即停止。失败回滚数据，PostgreSQL 序列可能留下正常间隙，不能用 max(id) 当数量。
|
||||
|
||||
目标 edition 沿用原会议、年份、event_key，不删除或重命名原 edition。只更正既有 submissions.edition_id 和对应 submission_identifiers.edition_id。原文、来源检索时间、决定 ID/状态/证据及各表来源关系不变。
|
||||
|
||||
分类证据写入来源元数据的保留字段 `edition_scope_curation_v1`，包括原范围、目标 edition、官方分区、URL、哈希和计划摘要。普通来源导入不能伪造或替换此字段；后续 record 刷新保留修订。相同来源投稿通过旧范围调用 decision 时仍路由到修正后的 edition。旧整卷 main 文件通过 import records 刷新会报错并回滚，要求先按核实轨道重新整理，避免覆盖正确的统计。
|
||||
|
||||
覆盖保留原清单 URL、cursor、成功时间，按实际目标分配 expected_count/imported_count 并标 partial；本工具核实分类，未核实摘要或所有来源完整性。原覆盖整行保留在计划中，不能继续将整卷数当 main 论文数。应用后校验原 edition 身份、全部受保护内容、分类、标识、修订字段和覆盖；相同计划再次执行仅在这些验收条件仍成立时返回 already_applied。
|
||||
|
||||
生产使用前必须在已确认 runner 完成软件验证。正式数据修正后仍需独立验收，并在最终全库审计中复核；单元测试和应用回执不能代替生产验收。
|
||||
@@ -0,0 +1,111 @@
|
||||
# Early WWW archive collector
|
||||
|
||||
`paper-atlas collect www-archive` is a maintained metadata adapter. It reads official
|
||||
IW3C2 archive HTML, keeps exact source hashes, and separates importable rows from
|
||||
program entries that still need review. It fetches no PDF, follows no paper-body or
|
||||
reference link, and makes no database connection.
|
||||
|
||||
For a fresh source snapshot on the server:
|
||||
|
||||
```sh
|
||||
paper-atlas collect www-archive \
|
||||
--http-cache /data/paper-index/cache/www-archive \
|
||||
--source-cache /data/paper-index/collected/www-archive-source-v2 \
|
||||
--out /data/paper-index/collected/www-archive-normalized-v2
|
||||
```
|
||||
|
||||
For reproducible offline replay of the already verified snapshot:
|
||||
|
||||
```sh
|
||||
paper-atlas collect www-archive \
|
||||
--source-cache /data/paper-index/collected/www-early-archive-source-v1 \
|
||||
--source-cache-sha256 c9388ccef44486e0684c9ee9e8620a465cdc4574e6bc9ea9391d743e01a3312c \
|
||||
--out /data/paper-index/collected/www-archive-normalized-v1
|
||||
```
|
||||
|
||||
Use a new output directory for another batch. Existing snapshots and normalized
|
||||
batches are never overwritten. The manifest hash binds every source URL, byte count,
|
||||
response hash and cache filename. Plain files must remain directly within the cache;
|
||||
symlinks, escaping paths, duplicate URLs, non-200 responses and altered bytes fail.
|
||||
The command uses UTF-8 where valid and explicitly records an ISO-8859-1 fallback for
|
||||
older pages; original bytes remain authoritative for encoding corrections.
|
||||
|
||||
## Evidence and outputs
|
||||
|
||||
- WWW3 Spring 1995 papers use `event_key=www-1995-spring`, `track=main`. The author,
|
||||
session and abstract indexes must have the same nonempty entry-ID set. The ID is
|
||||
local to this exact event and proceedings section, never a DOI or OpenReview ID.
|
||||
- Posters remain `track=poster`. An abstract without a linked author-index entry
|
||||
stays in review even when another index has a placeholder `#` link for it.
|
||||
- Paper URLs must agree between author and session indexes. A malformed whitespace
|
||||
URL in the abstract list may use the exact event/entry-ID match only when both
|
||||
independent indexes corroborate the same clean URL. The original href is retained.
|
||||
- Canonical names come from the explicitly labeled author index. The source lists
|
||||
them alphabetically; this is **not** original byline order. Both the raw index names
|
||||
and the full abstract-page byline/affiliation text remain in metadata. No person ID
|
||||
or cross-source identity is inferred from a name. Conflicting titles or missing
|
||||
abstracts stay in review. Topic keywords are not treated as an abstract.
|
||||
- WWW1 1994 formal presentations and WWW2 1994 mixed categories remain review
|
||||
candidates. The first list contains uncertain business-track annotations and a
|
||||
count discrepancy; the second contains panels, tutorials, products and other
|
||||
activities. Neither presentation listing nor a missing full-text link establishes
|
||||
a regular-paper acceptance or a rejection. Raw category/title/name claims are kept.
|
||||
|
||||
The main and poster JSONL files have independent manifests with explicit event key,
|
||||
track, count and SHA256. `source_listing_count` records the full recognized abstract
|
||||
listing, while `held_entry_count` records excluded entries; neither is a ready-row
|
||||
count. Their status stays `partial`: a verified linked subset does
|
||||
not prove complete event coverage. `review-candidates.jsonl` contains nested
|
||||
candidates and reasons, with no importable decision status. It is excluded from
|
||||
`import-queue.json`. `report.json` accounts for ready/held records and every source
|
||||
used. Newlines delimit JSONL; Unicode U+2028/U+2029 inside metadata are not line breaks.
|
||||
The output is published only after all required source and scope checks pass.
|
||||
|
||||
## Before importing
|
||||
|
||||
This independent collector branch starts from main; it does not include or silently
|
||||
install the event-aware writer. Its output requires the reviewed integrated runtime
|
||||
from PR31 (or a main revision containing that implementation). The production event
|
||||
schema migration has been independently accepted, preserving existing edition1212
|
||||
and its 59 Fall submissions; that is not an import of the Spring proceedings.
|
||||
|
||||
Inspect held entries and original author names, preflight every ready row against
|
||||
current PostgreSQL stable identifiers and candidate identities, and verify the
|
||||
source/year/track/event coverage before running `import records`. When a ready subset
|
||||
excludes held entries, preserve the independently known listing total in the existing
|
||||
incremental-coverage protocol: bind a freshly checked parent count and expected count,
|
||||
and set the full expected count from this manifest. A new source uses previous count
|
||||
0 and previous expected count null; an existing source requires its actual current
|
||||
values. These live parent values are not guessed by the offline collector. Do not automatically
|
||||
run this queue against an older writer. Preserve extended versions and independent
|
||||
submission/decision evidence, and keep unresolved identities held. A normalization
|
||||
report or passing tests alone do not establish production import, complete historical
|
||||
coverage or final global deduplication.
|
||||
|
||||
Initial fixed-cache replay (64ced7a, CI1057): 38 main papers and38 posters ready;
|
||||
372 entries held (50 first-conference fields,307 mixed program entries,15 posters).
|
||||
Independent report44dfd070 verified source hashes,counts and author evidence, finding
|
||||
one existing exact-title candidate. That publication is explicitly linked to
|
||||
arXiv1811.12161 by its WWW1995/journal-reference declaration; arXiv1810.07232 is a
|
||||
different1997 invited chapter and remains separate despite the overlap note. This
|
||||
is a preparation observation, not evidence of completed production import or final
|
||||
global deduplication.
|
||||
|
||||
## 以单篇官方页面核实待处理 poster
|
||||
|
||||
作者索引漏链接、摘要目录使用短标题或标题存在拼写差异时，不放宽全局标题匹配。可以先取得单篇官方 HTML，人工核对 H1、署名区与原目录同一 entry URL，再提供带 SHA256 的逐条核查文件：
|
||||
|
||||
```sh
|
||||
paper-atlas collect www-archive \
|
||||
--source-cache /data/paper-index/collected/www-poster-reviewed-source-v2 \
|
||||
--source-cache-sha256 8a65562f3490497cada2919ec77827134beabd5ec9c3690dec3625e2318aa54e \
|
||||
--reviewed-posters docs/coverage/www-1995-posters-reviewed-2026-11-09.json \
|
||||
--reviewed-posters-sha256 <核查文件的SHA256> \
|
||||
--out <新的输出目录>
|
||||
```
|
||||
|
||||
[1995 年15条poster的核查输入](coverage/www-1995-posters-reviewed-2026-11-09.json) 绑定61页来源快照：原46页和补取的15页单篇HTML。程序验证输入文件及快照哈希、单篇页面哈希、完全一致的 canonical H1、原摘要目录中的同一URL/entry ID及作者证据，不通过相似标题猜身份。规范标题采用单篇H1，摘要目录标题及作者索引标题均在 metadata 中保留。
|
||||
|
||||
`author_source=canonical_byline` 表示名字出现在H1之后、摘要/关键词或下一节标题之前的署名区域，由逐条核查明确选出，不从全文引用或机构猜作者；保留原始缩写、顺序、重音和拼写。只有 `author_source=author_index` 时使用原作者索引顺序并逐项核对；05的单篇页面写的是联系人，作者身份使用原作者索引证据，不由联系人角色推断。核查输入是审阅后的数据声明，程序的字符串/哈希检查不代替人工识别署名语义。
|
||||
|
||||
38和05的来源没有可明确引用的摘要。保留 `abstract=null` 和逐条缺失原因，manifest准确报告 missing_abstract_count，继续标partial；正文、联系方式及PostScript链接不当作摘要，也不下载正文。缺摘要不等于拒稿或未发表。本批输出仍包含原先已验收记录，正式导入前必须只取新增external ID，检查实时覆盖父记录并按incremental模式准备，不能直接重导入全部输出。
|
||||
@@ -23,10 +23,12 @@ COMMANDS = {
|
||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
|
||||
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
|
||||
('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'),
|
||||
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
|
||||
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
|
||||
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
|
||||
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
|
||||
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
|
||||
('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
"""Reproducible early WWW metadata; unresolved program entries stay outside imports."""
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
import hashlib
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
from paper_atlas.io.html import Document, clean
|
||||
from paper_atlas.io.http import fetch
|
||||
|
||||
ROOT = 'https://archives.iw3c2.org/'
|
||||
SPRING = ROOT+'www3/proceedings/'
|
||||
FALL = ROOT+'www2/Proceedings/WWW2_Proceedings.html'
|
||||
SEEDS = [ROOT+'www1/PrelimProcs.html', ROOT+'www1/Proceedings.html', FALL,
|
||||
SPRING+'proceedings.html', *[SPRING+'papers/'+x+'.html' for x in ('author','session','abstract')],
|
||||
*[SPRING+'posters/'+x+'.html' for x in ('author','abstract')]]
|
||||
|
||||
|
||||
def decode(raw):
|
||||
try:
|
||||
return raw.decode('utf-8-sig'), 'utf-8'
|
||||
except UnicodeDecodeError:
|
||||
return raw.decode('iso-8859-1'), 'iso-8859-1 fallback; original bytes retained'
|
||||
|
||||
|
||||
def text(html):
|
||||
return re.sub(r'[ \t\r\n\f\v]+',' ',Document(html).root.text()).strip(' \t\r\n\f\v')
|
||||
|
||||
|
||||
def anchors(html):
|
||||
return [(n.attrs.get('href',''),clean(n.text())) for n in Document(html).root.walk() if n.tag=='a']
|
||||
|
||||
|
||||
def source_url(url):
|
||||
p=urlsplit(url)
|
||||
if p.scheme!='https' or p.netloc!='archives.iw3c2.org' or p.query or p.fragment or not p.path.endswith('.html'):
|
||||
raise ValueError('Only exact official archive HTML source URLs are allowed')
|
||||
return url
|
||||
|
||||
|
||||
class Sources:
|
||||
def __init__(self, directory, expected_sha256):
|
||||
self.root=Path(directory).resolve();manifest=self.root/'manifest.json';raw=manifest.read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest()!=expected_sha256:
|
||||
raise ValueError('Source cache manifest checksum mismatch')
|
||||
self.rows={};self.used=set()
|
||||
for row in json.loads(raw)['sources']:
|
||||
url=source_url(row['url'])
|
||||
if url in self.rows or row.get('status')!=200:
|
||||
raise ValueError('Duplicate or incomplete source response')
|
||||
p=self.root/row['file']
|
||||
if p.is_symlink() or p.parent!=self.root or not p.is_file():
|
||||
raise ValueError('Source cache paths must be plain files directly within the cache')
|
||||
data=p.read_bytes()
|
||||
if len(data)>2*1024**2 or len(data)!=row['bytes'] or hashlib.sha256(data).hexdigest()!=row['sha256']:
|
||||
raise ValueError('Source response checksum or size mismatch')
|
||||
self.rows[url]={'raw':data,'receipt':row}
|
||||
|
||||
def read(self,url):
|
||||
source_url(url)
|
||||
if url not in self.rows:
|
||||
raise ValueError('Required source missing from cache: '+url)
|
||||
self.used.add(url)
|
||||
return decode(self.rows[url]['raw'])[0]
|
||||
|
||||
def proof(self,urls):
|
||||
return [{'url':url,'sha256':self.rows[url]['receipt']['sha256'],
|
||||
'encoding':decode(self.rows[url]['raw'])[1]} for url in sorted(set(urls))]
|
||||
|
||||
|
||||
def download_sources(http_cache, directory):
|
||||
directory=Path(directory)
|
||||
if directory.exists():
|
||||
raise ValueError('Refusing to overwrite a source snapshot')
|
||||
directory.mkdir(parents=True)
|
||||
responses={url:fetch(url,http_cache,2*1024**2) for url in SEEDS}
|
||||
html=decode(responses[FALL])[0]
|
||||
for href,label in anchors(html):
|
||||
if href!='Author_Inst.html' and href.endswith('.html'):
|
||||
url=source_url(urljoin(FALL,href))
|
||||
if not url.startswith(ROOT+'www2/Proceedings/'):
|
||||
raise ValueError('Unexpected fall category source')
|
||||
responses[url]=fetch(url,http_cache,2*1024**2)
|
||||
receipts=[]
|
||||
for i,(url,raw) in enumerate(sorted(responses.items())):
|
||||
name='source-'+str(i).zfill(2)+'.html';(directory/name).write_bytes(raw)
|
||||
receipts.append({'url':url,'status':200,'file':name,'bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()})
|
||||
manifest=directory/'manifest.json';manifest.write_text(json.dumps({'sources':receipts},indent=2)+'\n')
|
||||
return hashlib.sha256(manifest.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def identity(href,track):
|
||||
url=urljoin(SPRING+track+'/',href);p=urlsplit(url)
|
||||
match=re.fullmatch(r'/www3/proceedings/'+track+r'/(\d+)/[^?#]+',p.path)
|
||||
if p.scheme=='https' and p.netloc=='archives.iw3c2.org' and not p.query and not p.fragment and match:
|
||||
return match[1],url
|
||||
return None
|
||||
|
||||
|
||||
class AuthorIndex(HTMLParser):
|
||||
def __init__(self,html,track):
|
||||
super().__init__(convert_charrefs=True);self.track=track;self.name='';self.bold=None;self.link=None;self.link_text=[]
|
||||
self.entries=defaultdict(lambda:{'authors':[],'urls':set(),'titles':set()});self.feed(html)
|
||||
|
||||
def handle_starttag(self,tag,attrs):
|
||||
if tag=='b':self.bold=[]
|
||||
if tag=='a':self.link=dict(attrs).get('href');self.link_text=[]
|
||||
|
||||
def handle_data(self,data):
|
||||
if self.bold is not None:self.bold.append(data)
|
||||
if self.link is not None:self.link_text.append(data)
|
||||
|
||||
def handle_endtag(self,tag):
|
||||
if tag=='b' and self.bold is not None:
|
||||
self.name=clean(''.join(self.bold));self.bold=None
|
||||
if tag=='a' and self.link is not None:
|
||||
own=identity(self.link,self.track)
|
||||
if own:
|
||||
key,url=own;entry=self.entries[key];entry['urls'].add(url);entry['titles'].add(clean(''.join(self.link_text)))
|
||||
if not self.name:raise ValueError('Paper link has no archive author context')
|
||||
if self.name not in entry['authors']:entry['authors'].append(self.name)
|
||||
self.link=None
|
||||
|
||||
|
||||
def sections(html):
|
||||
"""Use explicit h3 boundaries; old HTML omits many p/li closing tags."""
|
||||
result=[]
|
||||
for chunk in re.split(r'<h3\b[^>]*>',html,flags=re.I)[1:]:
|
||||
parts=re.split(r'</h3\s*>',chunk,maxsplit=1,flags=re.I)
|
||||
if len(parts)!=2:raise ValueError('Unclosed archive entry heading')
|
||||
heading,body=parts
|
||||
body=re.split(r'<(?:hr|h2|address)\b',body,maxsplit=1,flags=re.I)[0]
|
||||
author=re.search(r'<i\b[^>]*>(.*?)</i\s*>',body,re.I|re.S)
|
||||
paragraphs=[]
|
||||
for paragraph in re.findall(r'<p\b[^>]*>(.*?)(?=</p\s*>|<p\b|$)',body,re.I|re.S):
|
||||
value=text(paragraph)
|
||||
if value and not value.casefold().startswith('keywords:'):paragraphs.append(value)
|
||||
result.append({'heading':heading,'title':text(heading),'abstract':'\n\n'.join(paragraphs)or None,
|
||||
'author_byline_raw':text(author[1])if author else None})
|
||||
if not result:raise ValueError('No explicit archive entry headings')
|
||||
return result
|
||||
|
||||
|
||||
def reviewed_posters(sources, path, expected_sha256, cache_sha256):
|
||||
if path is None:
|
||||
if expected_sha256:raise ValueError('Reviewed poster path required')
|
||||
return {},None
|
||||
raw=Path(path).read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest()!=expected_sha256:raise ValueError('Reviewed poster checksum mismatch')
|
||||
review=json.loads(raw)
|
||||
if set(review)!={'format','source_cache_sha256','records'} or review['format']!=1 or review['source_cache_sha256']!=cache_sha256:
|
||||
raise ValueError('Reviewed poster snapshot binding differs')
|
||||
if not isinstance(review['records'],list) or not 0<len(review['records'])<=100:
|
||||
raise ValueError('Bounded nonempty poster review required')
|
||||
result={}
|
||||
for row in review['records']:
|
||||
if set(row)!={'entry_id','individual_url','individual_sha256','title','authors','author_source','reason','missing_abstract_reason'}:
|
||||
raise ValueError('Unexpected reviewed poster fields')
|
||||
key=row['entry_id'];own=identity(row['individual_url'],'posters')
|
||||
if not isinstance(key,str) or not own or own[0]!=key or key in result:
|
||||
raise ValueError('Reviewed poster identity duplicated or outside canonical entry')
|
||||
if row['author_source'] not in ('canonical_byline','author_index') or not isinstance(row['authors'],list) or not row['authors'] or any(not isinstance(a,str) or not a.strip()for a in row['authors']):
|
||||
raise ValueError('Explicit author evidence required')
|
||||
if len(set(row['authors']))!=len(row['authors']) or not isinstance(row['reason'],str) or not row['reason'].strip():
|
||||
raise ValueError('Reviewed author order/reason invalid')
|
||||
if sources.rows.get(own[1],{}).get('receipt',{}).get('sha256')!=row['individual_sha256']:
|
||||
raise ValueError('Individual poster page checksum differs')
|
||||
page=sources.read(own[1]);headings=re.findall(r'<h1\b[^>]*>(.*?)</h1\s*>',page,re.I|re.S)
|
||||
if len(headings)!=1 or text(headings[0])!=row['title']:
|
||||
raise ValueError('Reviewed title differs from exact canonical H1')
|
||||
header=re.split(r'</h1\s*>',page,maxsplit=1,flags=re.I)[1]
|
||||
header=re.split(r'<h[23]\b',header,maxsplit=1,flags=re.I)[0]
|
||||
header=re.split(r'\b(?:keywords|abstract)\s*:',text(header),maxsplit=1,flags=re.I)[0]
|
||||
if row['author_source']=='canonical_byline' and any(a.casefold() not in header.casefold()for a in row['authors']):
|
||||
raise ValueError('Reviewed author absent from canonical byline region')
|
||||
result[key]={**row,'canonical_byline_region_raw':header}
|
||||
return result,hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def reviewed_row(sources,key,url,item,authors,review):
|
||||
if url!=review['individual_url']:
|
||||
raise ValueError('Reviewed canonical page differs from abstract-entry URL')
|
||||
if authors and authors['urls']!={url}:
|
||||
raise ValueError('Reviewed poster author-index URL disagrees')
|
||||
if review['author_source']=='author_index':
|
||||
names=[]
|
||||
for name in authors['authors']if authors else []:
|
||||
parts=name.split(',');names.append(clean(parts[1]+' '+parts[0])if len(parts)==2 else name)
|
||||
if names!=review['authors']:raise ValueError('Reviewed author-index evidence differs')
|
||||
missing=review['missing_abstract_reason']
|
||||
if not isinstance(missing,str) or bool(missing.strip())==bool(item['abstract']):
|
||||
raise ValueError('Missing abstract requires explicit reason; never invent a summary')
|
||||
urls=[SPRING+'posters/author.html',SPRING+'posters/abstract.html',url]
|
||||
return {'source':'www_archive','external_id':'www3:posters:'+key,'url':url,'title':review['title'],
|
||||
'abstract':item['abstract'],'authors':review['authors'],'identifiers':{},'venue':'WWW','year':1995,
|
||||
'track':'poster','event_key':'www-1995-spring','status':'accepted','raw_decision':'listed in official posters proceedings',
|
||||
'metadata':{'entry_id':key,'source_evidence':sources.proof(urls),'abstract_heading_title_raw':item['title'],
|
||||
'author_byline_raw':item['author_byline_raw'],'author_index_raw':authors['authors']if authors else [],
|
||||
'author_index_titles_raw':sorted(authors['titles'])if authors else [],'abstract_href_raw':url,
|
||||
'author_order':review['author_source'],'reviewed_poster_identity':review,
|
||||
'missing_abstract_reason':missing or None},
|
||||
'tags':[{'namespace':'www_archive:proceedings_section','value':'posters'}]}
|
||||
|
||||
|
||||
def spring_rows(sources,track,reviews=None):
|
||||
base=SPRING+track+'/';author_url=base+'author.html';abstract_url=base+'abstract.html'
|
||||
authors=AuthorIndex(sources.read(author_url),track).entries
|
||||
listing=sections(sources.read(abstract_url));by_id={};held=[];session={}
|
||||
if track=='papers':
|
||||
session_url=base+'session.html'
|
||||
for href,title in anchors(sources.read(session_url)):
|
||||
own=identity(href,track)
|
||||
if own:
|
||||
key,url=own
|
||||
if key in session:raise ValueError('Duplicate main session entry ID')
|
||||
session[key]=url
|
||||
for item in listing:
|
||||
links=[identity(h,track)for h,t in anchors(item['heading'])if identity(h,track)]
|
||||
if len(links)!=1:
|
||||
held.append({'reason':'unverified_heading_identity','evidence':abstract_url,'candidate':item});continue
|
||||
key,url=links[0]
|
||||
if key in by_id:raise ValueError('Duplicate abstract entry ID')
|
||||
by_id[key]=(url,item)
|
||||
if not by_id:raise ValueError('No stable abstract entry IDs')
|
||||
if track=='papers' and (set(authors)!=set(by_id) or set(session)!=set(by_id)):
|
||||
raise ValueError('Main author/session/abstract entry ID sets disagree')
|
||||
if set(authors)-set(by_id):raise ValueError('Author index has entries outside the abstract listing')
|
||||
reviews=reviews or {}
|
||||
if set(reviews)-set(by_id):raise ValueError('Reviewed poster outside abstract listing')
|
||||
rows=[]
|
||||
for key,(url,item) in by_id.items():
|
||||
if key in reviews:
|
||||
if track!='posters':raise ValueError('Reviews only apply to poster entries')
|
||||
rows.append(reviewed_row(sources,key,url,item,authors.get(key),reviews[key]));continue
|
||||
if key not in authors:
|
||||
held.append({'reason':'abstract_entry_missing_linked_author_index','entry_id':key,'evidence':abstract_url,'candidate':item});continue
|
||||
alternatives=authors[key]['urls']
|
||||
if len(alternatives)!=1:raise ValueError('Ambiguous author-index paper URL')
|
||||
canonical=next(iter(alternatives))
|
||||
if track=='papers' and session[key]!=canonical:raise ValueError('Main author/session paper URLs disagree')
|
||||
if url!=canonical and (track!='papers' or not re.search(r'\s',url)):
|
||||
raise ValueError('Uncorroborated archive URL discrepancy')
|
||||
if not item['abstract']:
|
||||
held.append({'reason':'missing_abstract','entry_id':key,'evidence':abstract_url,'candidate':item});continue
|
||||
if item['title'].casefold() not in {t.casefold() for t in authors[key]['titles']}:
|
||||
held.append({'reason':'abstract_and_author_index_titles_disagree','entry_id':key,'evidence':abstract_url,'candidate':item,'author_index_titles':sorted(authors[key]['titles'])});continue
|
||||
names=[]
|
||||
for name in authors[key]['authors']:
|
||||
parts=name.split(',')
|
||||
names.append(clean(parts[1]+' '+parts[0])if len(parts)==2 else name)
|
||||
proof_urls=[author_url,abstract_url]+([session_url]if track=='papers'else[])
|
||||
rows.append({'source':'www_archive','external_id':'www3:'+track+':'+key,'url':canonical,
|
||||
'title':item['title'],'abstract':item['abstract'],'authors':names,'identifiers':{},
|
||||
'venue':'WWW','year':1995,'track':'main'if track=='papers'else'poster','event_key':'www-1995-spring',
|
||||
'status':'accepted','raw_decision':'listed in official '+track+' proceedings',
|
||||
'metadata':{'entry_id':key,'source_evidence':sources.proof(proof_urls),
|
||||
'author_index_raw':authors[key]['authors'],'author_index_titles_raw':sorted(authors[key]['titles']),'author_order':'archive author-index order, not original byline rank',
|
||||
'author_byline_raw':item['author_byline_raw'],'abstract_href_raw':url,
|
||||
'url_corroborated_by_author_and_session':url!=canonical},
|
||||
'tags':[{'namespace':'www_archive:proceedings_section','value':track}]})
|
||||
return rows,held,{'abstract_entries':len(by_id),'linked_author_entries':len(authors),'ready':len(rows),'held':len(held)}
|
||||
|
||||
|
||||
def early_review(sources):
|
||||
held=[];url=ROOT+'www1/PrelimProcs.html';html=sources.read(url)
|
||||
for ordinal,chunk in enumerate(re.split(r'<li\b[^>]*>',html,flags=re.I)[1:],1):
|
||||
title=re.search(r'<em>Title:</em>\s*<b>(.*?)</b>',chunk,re.I|re.S)
|
||||
if not title:raise ValueError('Unclassified first-conference entry')
|
||||
names=re.findall(r'<em>Author:</em>(.*?)(?=<br\b)',chunk,re.I|re.S)
|
||||
held.append({'reason':'formal_presentation_scope_and_count_need_review','event_key':'www-1994-spring',
|
||||
'ordinal':ordinal,'evidence':url,'candidate':{'title':text(title[1]),
|
||||
'authors_raw':[re.sub(r'\s*<[^>]+>','',text(n)).strip()for n in names],
|
||||
'keywords_raw':text(re.search(r'<em>Keywords:</em>(.*?)(?=<br\b)',chunk,re.I|re.S)[1]) if re.search(r'<em>Keywords:</em>(.*?)(?=<br\b)',chunk,re.I|re.S)else None},
|
||||
'source_evidence':sources.proof([url])})
|
||||
if not held:raise ValueError('First-conference presentation listing empty')
|
||||
category_urls={}
|
||||
for href,label in anchors(sources.read(FALL)):
|
||||
if href!='Author_Inst.html' and href.endswith('.html'):
|
||||
category_url=source_url(urljoin(FALL,href))
|
||||
if not category_url.startswith(ROOT+'www2/Proceedings/'):raise ValueError('Unexpected fall category')
|
||||
category_urls[category_url]=label
|
||||
if not category_urls:raise ValueError('Fall archive category listing empty')
|
||||
for url,label in sorted(category_urls.items()):
|
||||
html=sources.read(url)
|
||||
for ordinal,chunk in enumerate(re.split(r'<dt\b[^>]*>',html,flags=re.I)[1:],1):
|
||||
fields=re.split(r'<dd\b[^>]*>',chunk,maxsplit=1,flags=re.I)
|
||||
title=text(fields[0]);raw_byline=text(re.split(r'</dl\b',fields[1],maxsplit=1,flags=re.I)[0]) if len(fields)==2 else None
|
||||
if title:
|
||||
held.append({'reason':'mixed_program_category_not_verified_paper_acceptance','event_key':'www-1994-fall',
|
||||
'category':label,'ordinal':ordinal,'evidence':url,'candidate':{'title':title,'authors_raw':raw_byline},
|
||||
'source_evidence':sources.proof([FALL,url])})
|
||||
return held,{'first_conference_fields':sum(r['event_key']=='www-1994-spring'for r in held),'fall_categories':len(category_urls),
|
||||
'fall_program_entries':sum(r['event_key']=='www-1994-fall'for r in held)}
|
||||
|
||||
|
||||
def collect(source_cache,cache_sha256,out,*,reviewed=None,reviewed_sha256=None):
|
||||
out=Path(out)
|
||||
if out.exists():raise ValueError('Refusing to overwrite an existing normalized batch')
|
||||
sources=Sources(source_cache,cache_sha256);groups={};held=[];counts={}
|
||||
reviews,review_sha=reviewed_posters(sources,reviewed,reviewed_sha256,cache_sha256)
|
||||
for track in ('papers','posters'):
|
||||
rows,review,counts[track]=spring_rows(sources,track,reviews if track=='posters'else None);groups[track]=rows;held.extend(review)
|
||||
review,counts['earlier']=early_review(sources);held.extend(review)
|
||||
for item in held:
|
||||
if 'source_evidence' not in item:item['source_evidence']=sources.proof([item['evidence']])
|
||||
out.parent.mkdir(parents=True,exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix='.www-archive-',dir=out.parent)as tmp:
|
||||
stage=Path(tmp);queue=[]
|
||||
for track,rows in groups.items():
|
||||
if not rows:continue
|
||||
filename='www-1995-spring-'+('main'if track=='papers'else'poster')+'.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n'for r in rows).encode();(stage/filename).write_bytes(raw)
|
||||
manifest={'source':'www_archive','parser_version':2 if review_sha else 1,'venue':'WWW','year':1995,'track':rows[0]['track'],'event_key':'www-1995-spring',
|
||||
'list_url':SPRING+track+'/abstract.html','count':len(rows),'status':'partial','file_sha256':hashlib.sha256(raw).hexdigest(),
|
||||
'missing_abstract_count':sum(not bool((r['abstract']or '').strip())for r in rows),'source_listing_count':counts[track]['abstract_entries'],'held_entry_count':counts[track]['held'],'source_cache_sha256':cache_sha256,'scope':'Verified linked archive entries; held entries and other event tracks excluded, not complete event coverage.'}
|
||||
(stage/filename).with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');queue.append(str(out/filename))
|
||||
(stage/'review-candidates.jsonl').write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n'for r in held))
|
||||
report={'status':'normalized_not_imported','source_cache_sha256':cache_sha256,'counts':counts,'ready':sum(len(r)for r in groups.values()),'held':len(held),
|
||||
'source_evidence':sources.proof(sources.used),'files':queue,'scope':'Early WWW archive only; no PDF fetching, no rejected/withdrawn inference, no database access. Event-aware importer required.'}
|
||||
if review_sha:report['reviewed_posters_sha256']=review_sha
|
||||
(stage/'report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||
(stage/'import-queue.json').write_text(json.dumps({'files':queue},indent=2)+'\n')
|
||||
stage.rename(out)
|
||||
return report
|
||||
|
||||
|
||||
def main(argv=None,*,prog=None):
|
||||
p=argparse.ArgumentParser(prog=prog);p.add_argument('--source-cache',required=True);p.add_argument('--source-cache-sha256');p.add_argument('--http-cache');p.add_argument('--out',required=True)
|
||||
p.add_argument('--reviewed-posters');p.add_argument('--reviewed-posters-sha256')
|
||||
args=p.parse_args(argv)
|
||||
if bool(args.http_cache)==bool(args.source_cache_sha256):p.error('Choose a fresh --http-cache download or an existing --source-cache-sha256 snapshot')
|
||||
sha=download_sources(args.http_cache,args.source_cache)if args.http_cache else args.source_cache_sha256
|
||||
if bool(args.reviewed_posters)!=bool(args.reviewed_posters_sha256):p.error('Reviewed poster file and SHA256 must be supplied together')
|
||||
print(json.dumps(collect(args.source_cache,sha,args.out,reviewed=args.reviewed_posters,reviewed_sha256=args.reviewed_posters_sha256),ensure_ascii=False),flush=True)
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Retain reviewed event scopes when a source refresh repeats a whole-volume label."""
|
||||
import json
|
||||
|
||||
KEY = 'edition_scope_curation_v1'
|
||||
|
||||
|
||||
def preserve(metadata, previous):
|
||||
old = json.loads(previous).get(KEY) if previous else None
|
||||
if KEY in metadata and metadata[KEY] != old:
|
||||
raise ValueError('Source import cannot supply or replace reviewed edition scope')
|
||||
if old is not None:
|
||||
metadata[KEY] = old
|
||||
return metadata
|
||||
|
||||
|
||||
def resolve(c, record, external, venue, year, track, event_key):
|
||||
row = c.execute('SELECT metadata_json FROM source_records WHERE id=?', (record,)).fetchone()
|
||||
entries = json.loads(row[0]).get(KEY, []) if row else []
|
||||
if not isinstance(entries, list):
|
||||
raise ValueError('Invalid reviewed edition scope')
|
||||
applicable = []
|
||||
for entry in entries:
|
||||
if entry['external_id'] != external:
|
||||
continue
|
||||
old, target = entry['from'], entry['target']
|
||||
if int(year) != old['year'] or venue not in entry['venue_names']:
|
||||
continue
|
||||
if track not in (old['track'], target['track']):
|
||||
continue
|
||||
if event_key is not None and event_key != old['event_key']:
|
||||
continue
|
||||
current = c.execute('SELECT venue_id,year,track,event_key FROM editions WHERE id=?', (target['edition_id'],)).fetchone()
|
||||
if not current or [current[i] for i in range(4)] != [target[k] for k in ('venue_id','year','track','event_key')]:
|
||||
raise ValueError('Reviewed target edition has changed')
|
||||
applicable.append(target['edition_id'])
|
||||
if len(applicable) > 1:
|
||||
raise ValueError('Ambiguous reviewed edition scope')
|
||||
return applicable[0] if applicable else None
|
||||
@@ -96,6 +96,8 @@ class Store:
|
||||
if found:
|
||||
matches.add(found[0])
|
||||
old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
|
||||
from paper_atlas.db.edition_scopes import preserve
|
||||
metadata = preserve(dict(metadata or {}), old[4] if old else None)
|
||||
if old:
|
||||
matches.add(old[0])
|
||||
key = normalized(title)
|
||||
@@ -175,7 +177,10 @@ class Store:
|
||||
for alias in aliases or []:
|
||||
c.execute('INSERT INTO venue_aliases VALUES(?,?) ON CONFLICT DO NOTHING',(venue_id,alias))
|
||||
from paper_atlas.db.events import find_edition
|
||||
edition = find_edition(self,venue_id,venue,int(year),track,record,external_id,event_key)
|
||||
from paper_atlas.db.edition_scopes import resolve
|
||||
edition = resolve(c,record,external_id,venue,year,track,event_key)
|
||||
if edition is None:
|
||||
edition = find_edition(self,venue_id,venue,int(year),track,record,external_id,event_key)
|
||||
# A previous event in this row may already have merged the input paper.
|
||||
paper=c.execute('SELECT paper_id FROM source_records WHERE id=?',(record,)).fetchone()[0]
|
||||
submission=attach_submission(self,edition,paper,record,external_id,identities or [])
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Snapshot-bound reclassification of existing attempts, without reimport or merge."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from contextlib import closing
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store, inserted_id
|
||||
from paper_atlas.db.edition_scopes import KEY
|
||||
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
||||
|
||||
MARKER = 'edition_tracks_v1:'
|
||||
|
||||
|
||||
def validate(inventory):
|
||||
if (not isinstance(inventory, dict) or inventory.get('format') != 1
|
||||
or not isinstance(inventory.get('records'), list) or not 0 < len(inventory['records']) <= 10000):
|
||||
raise ValueError('Nonempty bounded reviewed inventory required')
|
||||
ids = set(); records = set()
|
||||
for row in inventory['records']:
|
||||
required = {'submission_id','source_record_id','external_id','source','doi','from_edition_id',
|
||||
'year','venue','venue_names','track','section','evidence_url','evidence_sha256'}
|
||||
if not isinstance(row, dict) or set(row) != required:
|
||||
raise ValueError('Unexpected inventory record fields')
|
||||
for field in ('submission_id','source_record_id','from_edition_id','year'):
|
||||
if type(row[field]) is not int or row[field] <= 0:
|
||||
raise ValueError('Positive integer inventory identity required')
|
||||
if row['submission_id'] in ids or row['source_record_id'] in records:
|
||||
raise ValueError('Repeated submission/source identity in inventory')
|
||||
ids.add(row['submission_id']); records.add(row['source_record_id'])
|
||||
for field in ('external_id','source','doi','venue','section','evidence_url'):
|
||||
if not isinstance(row[field], str) or not row[field].strip():
|
||||
raise ValueError('Missing inventory evidence field')
|
||||
if (not isinstance(row['venue_names'], list) or not row['venue_names']
|
||||
or not all(isinstance(v,str) and v for v in row['venue_names'])
|
||||
or row['venue'] not in row['venue_names']
|
||||
or not re.fullmatch(r'[a-z][a-z0-9_]{0,63}', row['track'])
|
||||
or not re.fullmatch(r'[0-9a-f]{64}', row['evidence_sha256'])
|
||||
or not row['evidence_url'].startswith('https://')):
|
||||
raise ValueError('Invalid scope or evidence binding')
|
||||
return inventory
|
||||
|
||||
|
||||
def bounded(c, sql, params):
|
||||
rows = values(c.execute(sql+' LIMIT 100001', params))
|
||||
if len(rows) > 100000:
|
||||
raise ValueError('Protected scope exceeds bounded reclassification limit')
|
||||
return rows
|
||||
|
||||
|
||||
def protect(c, ids):
|
||||
marks = ','.join('?' for _ in ids)
|
||||
subs = 'SELECT id FROM submissions WHERE id IN ('+marks+')'
|
||||
papers = 'SELECT paper_id FROM submissions WHERE id IN ('+marks+')'
|
||||
decisions = 'SELECT id FROM decisions WHERE submission_id IN ('+subs+')'
|
||||
queries = {
|
||||
'submissions': ('SELECT id,paper_id,source_record_id,source_submission_id,submitted_at FROM submissions WHERE id IN ('+marks+') ORDER BY id'),
|
||||
'submission_identifiers': ('SELECT scheme,value,submission_id FROM submission_identifiers WHERE submission_id IN ('+subs+') ORDER BY submission_id,scheme,value'),
|
||||
'submission_sources': ('SELECT * FROM submission_sources WHERE submission_id IN ('+subs+') ORDER BY submission_id,source_record_id,source_submission_id'),
|
||||
'decisions': ('SELECT * FROM decisions WHERE submission_id IN ('+subs+') ORDER BY id'),
|
||||
'decision_sources': ('SELECT * FROM decision_sources WHERE decision_id IN ('+decisions+') ORDER BY decision_id,source_record_id,source_decision_id'),
|
||||
'papers': ('SELECT * FROM papers WHERE id IN ('+papers+') ORDER BY id'),
|
||||
'identifiers': ('SELECT * FROM identifiers WHERE paper_id IN ('+papers+') ORDER BY scheme,value'),
|
||||
'paper_tags': ('SELECT * FROM paper_tags WHERE paper_id IN ('+papers+') ORDER BY paper_id,tag_id,evidence_key'),
|
||||
}
|
||||
from paper_atlas.curation.annotations import available
|
||||
if available(c):
|
||||
queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN ('+papers+') ORDER BY id'
|
||||
result = {table: {'rows':len(rows), 'sha256':digest(rows)} for table, sql in queries.items()
|
||||
for rows in [bounded(c,sql,ids)]}
|
||||
rows = bounded(c, 'SELECT r.* FROM source_records r WHERE r.id IN (SELECT source_record_id FROM submission_sources WHERE submission_id IN ('+subs+')) ORDER BY r.id', ids)
|
||||
# Only the reviewed curation field may change; every other metadata value
|
||||
# and all raw text/IDs/retrieval times remain protected.
|
||||
for row in rows:
|
||||
metadata = json.loads(row[8]); metadata.pop(KEY,None); row[8] = metadata
|
||||
result['source_records'] = {'rows':len(rows), 'sha256':digest(rows)}
|
||||
return result
|
||||
|
||||
|
||||
def inspect(c, inventory):
|
||||
validate(inventory); changes=[]; ids=tuple(sorted(r['submission_id']for r in inventory['records']))
|
||||
original = sorted({r['from_edition_id'] for r in inventory['records']})
|
||||
actual_ids = {r[0] for r in c.execute('SELECT id FROM submissions WHERE edition_id IN ('+','.join('?'for _ in original)+')', original)}
|
||||
if actual_ids != set(ids):
|
||||
raise ValueError('Inventory must cover every attempt in each original edition')
|
||||
for row in inventory['records']:
|
||||
current = c.execute('''SELECT s.edition_id,s.source_record_id,s.source_submission_id,s.paper_id,
|
||||
e.venue_id,e.year,e.track,e.event_key,v.acronym,r.external_id,r.metadata_json,src.name
|
||||
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
|
||||
JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
|
||||
if not current or [current[i]for i in (0,1,2,5,8,9,11)] != [row[k]for k in ('from_edition_id','source_record_id','external_id','year','venue','external_id','source')]:
|
||||
raise ValueError('Submission identity/scope differs from inventory')
|
||||
aliases={r[0]for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?',(current[4],))}|{current[8]}
|
||||
if set(row['venue_names'])!=aliases:
|
||||
raise ValueError('Venue names differ from current aliases')
|
||||
metadata=json.loads(current[10])
|
||||
if KEY in metadata:
|
||||
raise ValueError('Existing curated scope requires explicit reconciliation')
|
||||
if (metadata.get('record_identifiers') or {}).get('doi') != row['doi']:
|
||||
raise ValueError('Stable DOI differs from reviewed source evidence')
|
||||
links=values(c.execute('SELECT source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(row['submission_id'],)))
|
||||
if links != [[row['source_record_id'],row['external_id']]]:
|
||||
raise ValueError('Shared-source attempt requires separate reconciliation')
|
||||
decisions=values(c.execute('SELECT status FROM decisions WHERE submission_id=?',(row['submission_id'],)))
|
||||
if not decisions or any(d[0]!='accepted'for d in decisions):
|
||||
raise ValueError('Only publication observations can use this track correction')
|
||||
target=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',(current[4],current[5],row['track'],current[7])).fetchone()
|
||||
if target and target[0] != current[0]:
|
||||
if c.execute('SELECT 1 FROM submissions WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
|
||||
raise ValueError('Nonempty target edition requires separate reconciliation')
|
||||
if c.execute('SELECT 1 FROM submission_identifiers WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
|
||||
raise ValueError('Target identifier scope already exists')
|
||||
if c.execute('SELECT 1 FROM edition_sources WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
|
||||
raise ValueError('Target coverage already exists; reconcile first')
|
||||
if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],current[0])).fetchone():
|
||||
raise ValueError('Existing identifier edition is inconsistent')
|
||||
changes.append({**row,'from_track':current[6],'event_key':current[7], 'venue_id':current[4],
|
||||
'target_edition_id':target[0]if target else None})
|
||||
coverage=bounded(c,'SELECT * FROM edition_sources WHERE edition_id IN ('+','.join('?'for _ in original)+') ORDER BY edition_id,source_id,list_url',original)
|
||||
if Counter(row[0]for row in coverage)!=Counter({edition:1 for edition in original}):
|
||||
raise ValueError('Exactly one original coverage listing per edition required')
|
||||
return {'records':changes, 'protected':protect(c,ids), 'coverage':coverage,
|
||||
'editions':bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)}
|
||||
|
||||
|
||||
def proposal(c, inventory, identity):
|
||||
return {'format':1,'database':identity,'inventory':inventory,'before':inspect(c,inventory)}
|
||||
|
||||
|
||||
def verify(c, plan):
|
||||
records=plan['before']['records']; ids=tuple(sorted(r['submission_id']for r in records))
|
||||
if protect(c,ids)!=plan['before']['protected']:
|
||||
raise ValueError('Protected history/text/tags/identities changed; roll back')
|
||||
for row in records:
|
||||
event=c.execute('''SELECT e.id,e.venue_id,e.year,e.track,e.event_key FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
|
||||
if not event or [event[i]for i in range(1,5)] != [row['venue_id'],row['year'],row['track'],row['event_key']]:
|
||||
raise ValueError('Target track assignment failed')
|
||||
if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],event[0])).fetchone():
|
||||
raise ValueError('Target identifier assignment failed')
|
||||
metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0])
|
||||
if row['track'] != row['from_track']:
|
||||
if metadata.get(KEY) != [curation(row,event[0],digest(plan))]:
|
||||
raise ValueError('Reviewed refresh protection missing or changed')
|
||||
elif KEY in metadata:
|
||||
raise ValueError('Unexpected scope revision on unchanged track')
|
||||
original=sorted({r['from_edition_id']for r in records})
|
||||
if bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)!=plan['before']['editions']:
|
||||
raise ValueError('Original edition identities changed')
|
||||
for old in plan['before']['coverage']:
|
||||
edition,source,url,_,_,_,cursor,success,_=old
|
||||
expected=Counter()
|
||||
for row in records:
|
||||
if row['from_edition_id']==edition:
|
||||
target=c.execute('SELECT edition_id FROM submissions WHERE id=?',(row['submission_id'],)).fetchone()[0]
|
||||
expected[target]+=1
|
||||
for target,number in expected.items():
|
||||
actual=c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources WHERE edition_id=? AND source_id=? AND list_url=?',(target,source,url)).fetchone()
|
||||
note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+digest(plan)
|
||||
if actual is None or [actual[i]for i in range(6)]!=['partial',number,number,cursor,success,note]:
|
||||
raise ValueError('Corrected coverage differs from reviewed scope')
|
||||
return {'records_verified':len(records),'protected':plan['before']['protected']}
|
||||
|
||||
|
||||
def curation(row, target, signature):
|
||||
return {'external_id':row['external_id'], 'venue_names':sorted(set(row['venue_names'])),
|
||||
'from':{'edition_id':row['from_edition_id'],'year':row['year'],'track':row['from_track'],'event_key':row['event_key']},
|
||||
'target':{'edition_id':target,'venue_id':row['venue_id'],'year':row['year'],'track':row['track'],'event_key':row['event_key']},
|
||||
'section':row['section'],'evidence_url':row['evidence_url'],'evidence_sha256':row['evidence_sha256'],'plan_digest':signature}
|
||||
|
||||
|
||||
def apply_plan(c, plan):
|
||||
if plan.get('format') != 1:
|
||||
raise ValueError('Unsupported plan format')
|
||||
signature=digest(plan);marker=MARKER+signature
|
||||
if c.execute('SELECT 1 FROM schema_migrations WHERE name=?',(marker,)).fetchone():
|
||||
return {'status':'already_applied','events_moved':0,**verify(c,plan)}
|
||||
if inspect(c,plan['inventory']) != plan['before']:
|
||||
raise ValueError('Stale track plan; create and review a fresh plan')
|
||||
targets={}; counts=Counter(); moved=0
|
||||
for row in plan['before']['records']:
|
||||
key=(row['venue_id'],row['year'],row['track'],row['event_key'])
|
||||
if key not in targets:
|
||||
found=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',key).fetchone()
|
||||
targets[key]=found[0]if found else inserted_id(c,'INSERT INTO editions(venue_id,year,track,event_key) VALUES(?,?,?,?)',key)
|
||||
target=targets[key];counts[(row['from_edition_id'],target)]+=1
|
||||
if target != row['from_edition_id']:
|
||||
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?',(target,row['submission_id']))
|
||||
if c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?',(target,row['submission_id'],row['from_edition_id'])).rowcount!=1:
|
||||
raise ValueError('Concurrent submission change')
|
||||
metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0]);metadata[KEY]=[curation(row,target,signature)]
|
||||
c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(metadata,ensure_ascii=False),row['source_record_id']));moved+=1
|
||||
# Preserve each original source URL and timestamps, but do not retain its
|
||||
# old whole-volume count as main-track coverage. Partial is deliberate:
|
||||
# this operation verifies scope, not abstracts or all possible sources.
|
||||
for old in plan['before']['coverage']:
|
||||
edition,source,url,status,expected,imported,cursor,success,error=old
|
||||
own={r['submission_id'] for r in plan['before']['records']if r['from_edition_id']==edition}
|
||||
source_name=c.execute('SELECT name FROM sources WHERE id=?',(source,)).fetchone()[0]
|
||||
if {r['source']for r in plan['before']['records']if r['submission_id']in own}!={source_name}:
|
||||
raise ValueError('Coverage includes a source outside the reviewed inventory')
|
||||
for (origin,target),number in counts.items():
|
||||
if origin!=edition:continue
|
||||
note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+signature
|
||||
c.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,cursor,last_success_at,last_error)
|
||||
VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,
|
||||
expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_error=excluded.last_error''',
|
||||
(target,source,url,'partial',number,number,cursor,success,note))
|
||||
result=verify(c,plan)
|
||||
c.execute('INSERT INTO schema_migrations(name) VALUES(?)',(marker,))
|
||||
return {'status':'applied','events_moved':moved,**result}
|
||||
|
||||
|
||||
def run(db,out,*,pg_service=None,pg_schema=None,inventory=None,plan=None,plan_sha256=None):
|
||||
out=Path(out)
|
||||
if out.exists() or not out.parent.is_dir():raise ValueError('New report path in existing directory required')
|
||||
expected=None
|
||||
if plan:
|
||||
raw=Path(plan).read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest()!=plan_sha256:raise ValueError('Plan hash differs')
|
||||
expected=json.loads(raw);validate(expected['inventory'])
|
||||
elif plan_sha256:raise ValueError('Plan path required')
|
||||
if bool(expected)==bool(inventory):raise ValueError('Choose inventory preflight or apply plan')
|
||||
if expected is None:
|
||||
with closing(open_read_connection(db,pg_service=pg_service,pg_schema=pg_schema))as c:
|
||||
result=proposal(c,validate(json.loads(Path(inventory).read_bytes())),database_identity(c,db))
|
||||
else:
|
||||
store=open_store(db,pg_service=pg_service,pg_schema=pg_schema)
|
||||
with closing(store.conn)as c:
|
||||
try:
|
||||
if expected['database']!=database_identity(c,db):raise ValueError('Plan belongs to another database')
|
||||
if not getattr(c,'postgres',False):c.commit();c.execute('BEGIN IMMEDIATE')
|
||||
result=apply_plan(c,expected);c.commit()
|
||||
except BaseException:c.rollback();raise
|
||||
out.write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n');return result
|
||||
|
||||
|
||||
def main(argv=None,*,prog=None):
|
||||
p=argparse.ArgumentParser(prog=prog);database_arguments(p)
|
||||
p.add_argument('--inventory');p.add_argument('--plan');p.add_argument('--plan-sha256');p.add_argument('--out',required=True)
|
||||
a=p.parse_args(argv)
|
||||
if bool(a.plan)!=bool(a.plan_sha256):p.error('--plan and --plan-sha256 must be supplied together')
|
||||
r=run(a.db,a.out,pg_service=a.pg_service,pg_schema=a.pg_schema,inventory=a.inventory,plan=a.plan,plan_sha256=a.plan_sha256)
|
||||
print(json.dumps({'status':r.get('status','planned'),'events_moved':r.get('events_moved',0),'out':a.out}),flush=True)
|
||||
@@ -134,6 +134,12 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
|
||||
if events and status is None:raise ValueError('Explicit decision event requires a status')
|
||||
if status is not None and status not in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):raise ValueError('Unsupported decision event')
|
||||
edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at'),guard_status=source_name in (PUBLIC_SOURCE,ARCHIVE_SOURCE),event_key=row.get('event_key'))
|
||||
from paper_atlas.db.edition_scopes import KEY
|
||||
scope_metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0])
|
||||
if scope_metadata.get(KEY):
|
||||
actual_track=c.execute('SELECT track FROM editions WHERE id=?',(edition,)).fetchone()[0]
|
||||
if row['track']!=actual_track:
|
||||
raise ValueError('Recollect this whole-volume source with reviewed per-record tracks before refresh')
|
||||
editions.add(edition)
|
||||
else:
|
||||
raise ValueError('This importer requires an explicit supported decision status')
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Stable-attempt correction, rollback and stale whole-volume refresh on both engines."""
|
||||
from contextlib import closing, redirect_stdout
|
||||
import copy
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from paper_atlas.db.backend import open_store
|
||||
from paper_atlas.db.edition_scopes import KEY
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.edition_tracks import proposal,apply_plan,verify,validate,run
|
||||
from paper_atlas.ingestion.event_identity import database_identity,values
|
||||
from paper_atlas.ingestion.records import ingest_file
|
||||
from paper_atlas.postgres.client import connect
|
||||
from paper_atlas.postgres.migration import migrate
|
||||
from paper_atlas.postgres.schema import quote
|
||||
|
||||
|
||||
def seed(db):
|
||||
s=Store(db); rows=[]
|
||||
for i,track in enumerate(['main','survey','doctoral_consortium','proceedings_unspecified'],1):
|
||||
external='entry'+str(i);doi='10.1234/entry'+str(i)
|
||||
p,r=s.record('fixture',external,'https://example.org/'+external,'Publication '+external,
|
||||
'Abstract with NUL\x00 and SOH\x01',[],{'doi':doi})
|
||||
eid=s.decision(p,r,'JOINT',2022,'main',external,'https://example.org/'+external,'accepted','published',identities=[('doi',doi)],observed_at='2022-01-01T00:00:00Z')
|
||||
sid=s.conn.execute('SELECT id FROM submissions WHERE source_record_id=?',(r,)).fetchone()[0]
|
||||
s.record_tags(p,r,'fixture',[{'namespace':'fixture:subject','value':'Original'}])
|
||||
rows.append(dict(submission_id=sid,source_record_id=r,external_id=external,source='fixture',doi=doi,
|
||||
from_edition_id=eid,year=2022,venue='JOINT',venue_names=['JOINT'],track=track,
|
||||
section=track,evidence_url='https://example.org/proceedings/2022/',evidence_sha256='a'*64))
|
||||
source=s.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||
s.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'complete',4,4,'checkpoint','2022-01-01T00:00:00Z',NULL)",(eid,source,'https://example.org/volume'))
|
||||
s.conn.commit();s.conn.close();return {'format':1,'records':rows}
|
||||
|
||||
|
||||
class TrackCases:
|
||||
def store(self):return open_store(**self.target)
|
||||
|
||||
def plan(self,c):return proposal(c,self.inventory,database_identity(c,self.target.get('db')))
|
||||
|
||||
def test_move_preserves_all_history_tags_text_and_stable_identities(self):
|
||||
with closing(self.store().conn)as c:
|
||||
plan=self.plan(c);before=plan['before']['protected'];r=apply_plan(c,plan);c.commit()
|
||||
self.assertEqual((r['events_moved'],r['records_verified']),(3,4))
|
||||
self.assertEqual(r['protected'],before)
|
||||
self.assertEqual(apply_plan(c,plan)['status'],'already_applied')
|
||||
self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count FROM edition_sources ORDER BY edition_id')),[['partial',1,1]]*4)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],4)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],0)
|
||||
|
||||
def test_source_record_refresh_keeps_scope_and_original_attempt(self):
|
||||
with closing(self.store().conn)as c:
|
||||
plan=self.plan(c);apply_plan(c,plan);c.commit()
|
||||
with closing(self.store().conn)as c:
|
||||
# Use Store methods with the same writer connection as production.
|
||||
s=object.__new__(Store);s.conn=c;s.event_identity=True
|
||||
r=self.inventory['records'][1];metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(r['source_record_id'],)).fetchone()[0]);saved=metadata[KEY]
|
||||
p,record=s.record('fixture',r['external_id'],'https://example.org/new','Refreshed title',None,[],{'doi':r['doi']},{'new_source_field':True})
|
||||
eid=s.decision(p,record,'JOINT',2022,'main',r['external_id'],'https://example.org/entry2','accepted','published',identities=[('doi',r['doi'])],observed_at='2022-01-01T00:00:00Z')
|
||||
self.assertEqual(c.execute('SELECT track FROM editions WHERE id=?',(eid,)).fetchone()[0],'survey')
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4)
|
||||
self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0])[KEY],saved)
|
||||
with self.assertRaisesRegex(ValueError,'cannot supply'):
|
||||
s.record('fixture',r['external_id'],'https://example.org/new','Forged',None,[],metadata={KEY:[]})
|
||||
c.rollback()
|
||||
|
||||
def test_stale_snapshot_and_partial_inventory_are_rejected(self):
|
||||
with closing(self.store().conn)as c:
|
||||
plan=self.plan(c)
|
||||
c.execute("UPDATE papers SET title='Changed' WHERE id=(SELECT paper_id FROM submissions WHERE id=?)",(self.inventory['records'][0]['submission_id'],))
|
||||
with self.assertRaisesRegex(ValueError,'Stale'):apply_plan(c,plan)
|
||||
c.rollback()
|
||||
bad=copy.deepcopy(self.inventory);bad['records'].pop()
|
||||
with self.assertRaisesRegex(ValueError,'every attempt'):proposal(c,bad,database_identity(c,self.target.get('db')))
|
||||
bad=copy.deepcopy(self.inventory);bad['records'][0]['doi']='10.1234/wrong'
|
||||
with self.assertRaisesRegex(ValueError,'DOI'):proposal(c,bad,database_identity(c,self.target.get('db')))
|
||||
|
||||
def test_failure_after_updates_rolls_back_all_scope_and_metadata_changes(self):
|
||||
with closing(self.store().conn)as c:
|
||||
plan=self.plan(c)
|
||||
# A late coverage failure must roll back earlier moved attempts.
|
||||
source=c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||
c.execute("INSERT INTO sources(name,homepage) VALUES('other','https://example.org')")
|
||||
other=c.execute("SELECT id FROM sources WHERE name='other'").fetchone()[0]
|
||||
c.execute('UPDATE edition_sources SET source_id=? WHERE source_id=?',(other,source));c.commit()
|
||||
plan=self.plan(c)
|
||||
try:
|
||||
with self.assertRaisesRegex(ValueError,'outside'):apply_plan(c,plan)
|
||||
finally:c.rollback()
|
||||
self.assertEqual(self.plan(c)['before'],plan['before'])
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM editions').fetchone()[0],1)
|
||||
|
||||
def test_import_old_whole_volume_fails_without_corrupting_coverage(self):
|
||||
with closing(self.store().conn)as c:
|
||||
plan=self.plan(c);apply_plan(c,plan);c.commit()
|
||||
r=self.inventory['records'][1];path=self.root/'old.jsonl'
|
||||
row=dict(source='fixture',external_id=r['external_id'],url='https://example.org/entry2',title='Old title',authors=[],identifiers={'doi':r['doi']},venue='JOINT',year=2022,track='main',status='accepted')
|
||||
path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps(dict(source='fixture',venue='JOINT',year=2022,track='main',count=1,list_url='https://example.org/volume',status='complete')))
|
||||
with self.assertRaisesRegex(ValueError,'Recollect'):ingest_file(self.target.get('db'),path,**{k:v for k,v in self.target.items()if k!='db'})
|
||||
with closing(self.store().conn)as c:verify(c,plan)
|
||||
|
||||
def test_nonempty_target_and_alias_drift_fail_before_moving(self):
|
||||
with closing(self.store().conn)as c:
|
||||
c.execute("INSERT INTO venue_aliases VALUES((SELECT id FROM venues WHERE acronym='JOINT'),'ADDED')")
|
||||
with self.assertRaisesRegex(ValueError,'aliases'):self.plan(c)
|
||||
c.rollback()
|
||||
s=object.__new__(Store);s.conn=c;s.event_identity=True
|
||||
p,r=s.record('fixture','collision','https://example.org/collision','Distinct contribution',None,[],{'doi':'10.1234/collision'})
|
||||
s.decision(p,r,'JOINT',2022,'survey','collision','https://example.org/collision','accepted','published')
|
||||
with self.assertRaisesRegex(ValueError,'Nonempty target'):self.plan(c)
|
||||
c.rollback()
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4)
|
||||
|
||||
def test_repeated_plan_detects_coverage_or_scope_evidence_drift(self):
|
||||
with closing(self.store().conn)as c:
|
||||
plan=self.plan(c);apply_plan(c,plan);c.commit()
|
||||
c.execute("UPDATE edition_sources SET imported_count=99")
|
||||
with self.assertRaisesRegex(ValueError,'coverage'):apply_plan(c,plan)
|
||||
c.rollback()
|
||||
record=self.inventory['records'][0]['source_record_id']
|
||||
m=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0]);m[KEY]=[]
|
||||
c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(m),record))
|
||||
with self.assertRaisesRegex(ValueError,'unchanged track'):apply_plan(c,plan)
|
||||
c.rollback()
|
||||
|
||||
def test_cli_hash_binding_preflight_and_duplicate_inventory(self):
|
||||
inventory=self.root/'inventory.json';inventory.write_text(json.dumps(self.inventory))
|
||||
planfile=self.root/'plan.json';p=run(self.target.get('db'),planfile,inventory=inventory,**{k:v for k,v in self.target.items()if k!='db'})
|
||||
with self.assertRaisesRegex(ValueError,'hash'):run(self.target.get('db'),self.root/'bad.json',plan=planfile,plan_sha256='0'*64,**{k:v for k,v in self.target.items()if k!='db'})
|
||||
report=run(self.target.get('db'),self.root/'applied.json',plan=planfile,plan_sha256=hashlib.sha256(planfile.read_bytes()).hexdigest(),**{k:v for k,v in self.target.items()if k!='db'})
|
||||
self.assertEqual(report['events_moved'],3)
|
||||
bad=copy.deepcopy(self.inventory);bad['records'].append(bad['records'][0])
|
||||
with self.assertRaisesRegex(ValueError,'Repeated'):validate(bad)
|
||||
|
||||
|
||||
class SQLiteTrackTests(TrackCases,unittest.TestCase):
|
||||
def setUp(self):
|
||||
t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);db=self.root/'source.sqlite'
|
||||
self.inventory=seed(db);self.target={'db':db}
|
||||
|
||||
|
||||
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL service')
|
||||
class PostgreSQLTrackTests(TrackCases,unittest.TestCase):
|
||||
def setUp(self):
|
||||
t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);db=self.root/'source.sqlite';self.inventory=seed(db)
|
||||
self.service=os.environ['PAPER_ATLAS_TEST_PG_SERVICE'];self.schema='pa_tracks_'+uuid.uuid4().hex[:12];self.addCleanup(self.cleanup)
|
||||
with redirect_stdout(io.StringIO()):migrate(db,self.service,self.root/'migration.json',schema=self.schema,min_free_gib=0)
|
||||
self.target={'pg_service':self.service,'pg_schema':self.schema}
|
||||
|
||||
def cleanup(self):
|
||||
with connect(self.service)as c:
|
||||
for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(self.schema,)):
|
||||
c.execute('DROP SCHEMA '+quote(name)+' CASCADE')
|
||||
@@ -0,0 +1,155 @@
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors.www_archive import (ROOT, SPRING, FALL, Sources, collect, decode,
|
||||
early_review, spring_rows)
|
||||
|
||||
|
||||
class ArchiveTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);self.cache=self.root/'cache';self.cache.mkdir()
|
||||
self.pages={ROOT+'www1/PrelimProcs.html':'''<h1>Formal presentations</h1><ol><li><em>Keywords:</em> BUSINESS TRACK?<br><em>Title:</em><b> First contribution</b><br><em>Author:</em>Jane Doe <jane@example.org><br><li><em>Title:</em><b>Unlinked contribution</b><br><em>Author:</em>John Roe<br></ol>''',
|
||||
FALL:'<h1>Conference Tracks</h1><ol><li><a href="Agents.html">Agents</a><li><a href="Tutor.html">Tutorials</a></ol>',
|
||||
ROOT+'www2/Proceedings/Agents.html':'<h3>Panel</h3><dl><dt><a href="a.html">Linked talk</a><dd>Jane Doe<dt>Unlinked talk</a><dd>John Roe</dl>',
|
||||
ROOT+'www2/Proceedings/Tutor.html':'<h1>Tutorials</h1><dl><dt>Training session<dd>Teacher</dl>'}
|
||||
for track in ['papers','posters']:
|
||||
self.pages[SPRING+track+'/author.html']='<b>Doe, Jane</b><ul><li><a href="01/paper.html">Same contribution</a></li></ul><b>Roe, John</b><a href="01/paper.html">Same contribution</a>'
|
||||
self.pages[SPRING+track+'/abstract.html']='''<h1>Abstracts</h1><a href="#one">Same contribution</a><h3><a href="01/paper.html">Same contribution</a></h3><i>Author: Jane Doe, John Roe<br>Institute</i><p>Actual abstract with <i>inline emphasis</i> and a separator\u2028 preserved.</p><hr><h2>Another session</h2>'''
|
||||
self.pages[SPRING+'papers/session.html']='<a href="01/paper.html">Same contribution</a>'
|
||||
|
||||
def sources(self):
|
||||
rows=[]
|
||||
for i,(url,html) in enumerate(sorted(self.pages.items())):
|
||||
raw=html.encode()if isinstance(html,str)else html;name='source-'+str(i)+'.html';(self.cache/name).write_bytes(raw)
|
||||
rows.append(dict(url=url,status=200,file=name,bytes=len(raw),sha256=hashlib.sha256(raw).hexdigest()))
|
||||
p=self.cache/'manifest.json';p.write_text(json.dumps({'sources':rows}));sha=hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
return Sources(self.cache,sha),sha
|
||||
|
||||
def test_offline_batch_separates_tracks_review_and_importable_rows(self):
|
||||
_,sha=self.sources();out=self.root/'batch';report=collect(self.cache,sha,out)
|
||||
self.assertEqual(report['ready'],2);self.assertEqual(report['held'],5)
|
||||
queue=json.loads((out/'import-queue.json').read_text())['files'];self.assertEqual(len(queue),2)
|
||||
for name in queue:
|
||||
p=Path(name);rows=[json.loads(l)for l in p.read_text().split('\n') if l];self.assertEqual(len(rows),1)
|
||||
row=rows[0];self.assertEqual(row['event_key'],'www-1995-spring');self.assertEqual(row['authors'],['Jane Doe','John Roe'])
|
||||
self.assertIn(row['track'],['main','poster']);self.assertIn('inline emphasis',row['abstract']);self.assertNotIn('Another session',row['abstract'])
|
||||
self.assertIn('\u2028',row['abstract']);manifest=json.loads(p.with_suffix('.manifest.json').read_text());self.assertEqual(manifest['file_sha256'],hashlib.sha256(p.read_bytes()).hexdigest());self.assertEqual(manifest['status'],'partial')
|
||||
held=[json.loads(l)for l in(out/'review-candidates.jsonl').read_text().split('\n') if l]
|
||||
self.assertTrue(all('status'not in r and 'source_evidence'in r for r in held))
|
||||
self.assertFalse(any('review-candidates'in p for p in queue))
|
||||
|
||||
def test_partial_manifest_preserves_full_listing_size_separately_from_ready_count(self):
|
||||
self.pages[SPRING+'posters/abstract.html']+='<h3><a href="02/other.html">Unlinked poster</a></h3><i>Author: Other Person</i><p>Actual abstract</p>'
|
||||
_,sha=self.sources();out=self.root/'batch';collect(self.cache,sha,out)
|
||||
manifest=json.loads((out/'www-1995-spring-poster.manifest.json').read_text())
|
||||
self.assertEqual((manifest['count'],manifest['source_listing_count'],manifest['held_entry_count']),(1,2,1))
|
||||
self.assertNotIn('coverage_previous_count',manifest) # Live parent must be independently preflighted.
|
||||
|
||||
def test_malformed_href_requires_exact_id_and_two_clean_index_anchors(self):
|
||||
self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('01/paper.html','01/ paper.html')
|
||||
sources,_=self.sources();rows,held,_=spring_rows(sources,'papers');self.assertEqual(held,[])
|
||||
self.assertEqual(rows[0]['url'],SPRING+'papers/01/paper.html');self.assertTrue(rows[0]['metadata']['url_corroborated_by_author_and_session'])
|
||||
self.assertIn(' ',rows[0]['metadata']['abstract_href_raw'])
|
||||
|
||||
def test_main_index_disagreement_fails_without_publishing_batch(self):
|
||||
self.pages[SPRING+'papers/session.html']='<a href="02/other.html">Other work</a>';_,sha=self.sources();out=self.root/'batch'
|
||||
with self.assertRaisesRegex(ValueError,'ID sets disagree'):collect(self.cache,sha,out)
|
||||
self.assertFalse(out.exists())
|
||||
|
||||
def test_same_entry_id_with_conflicting_clean_url_is_rejected(self):
|
||||
self.pages[SPRING+'papers/session.html']='<a href="01/other.html">Same contribution</a>';sources,_=self.sources()
|
||||
with self.assertRaisesRegex(ValueError,'URLs disagree'):spring_rows(sources,'papers')
|
||||
|
||||
def test_poster_without_linked_author_entry_is_held_even_with_abstract(self):
|
||||
self.pages[SPRING+'posters/author.html']='<b>Doe, Jane</b><a href="#">Same contribution</a>';sources,_=self.sources();rows,held,counts=spring_rows(sources,'posters')
|
||||
self.assertEqual(rows,[]);self.assertEqual(counts['abstract_entries'],1);self.assertEqual(held[0]['reason'],'abstract_entry_missing_linked_author_index')
|
||||
|
||||
def test_unlinked_and_panel_entries_never_become_acceptance_or_rejection(self):
|
||||
sources,_=self.sources();held,counts=early_review(sources);self.assertEqual(counts,{'first_conference_fields':2,'fall_categories':2,'fall_program_entries':3})
|
||||
self.assertTrue(any(r['candidate']['title']=='Unlinked talk'for r in held));self.assertTrue(all('status'not in r for r in held))
|
||||
self.assertEqual(held[0]['candidate']['authors_raw'],['Jane Doe'])
|
||||
|
||||
def test_title_disagreement_goes_to_review_with_source_claims(self):
|
||||
self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('Same contribution','Different contribution');sources,_=self.sources();rows,held,_=spring_rows(sources,'papers')
|
||||
self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'abstract_and_author_index_titles_disagree')
|
||||
|
||||
def test_cache_tampering_and_manifest_hash_mismatch_fail_closed(self):
|
||||
_,sha=self.sources()
|
||||
with self.assertRaisesRegex(ValueError,'manifest checksum'):Sources(self.cache,'0'*64)
|
||||
p=next(self.cache.glob('source-*.html'));p.write_bytes(p.read_bytes()+b'x')
|
||||
with self.assertRaisesRegex(ValueError,'checksum or size'):Sources(self.cache,sha)
|
||||
|
||||
def test_cache_cannot_read_paths_outside_snapshot(self):
|
||||
_,sha=self.sources();p=self.cache/'manifest.json';d=json.loads(p.read_text());d['sources'][0]['file']='../outside.html';p.write_text(json.dumps(d))
|
||||
with self.assertRaisesRegex(ValueError,'plain files'):Sources(self.cache,hashlib.sha256(p.read_bytes()).hexdigest())
|
||||
|
||||
def test_keywords_only_is_missing_abstract_not_invented_summary(self):
|
||||
self.pages[SPRING+'papers/abstract.html']='<h3><a href="01/paper.html">Same contribution</a></h3><i>Author: Jane Doe</i><p>Keywords: networking, hypertext</p>';sources,_=self.sources();rows,held,_=spring_rows(sources,'papers')
|
||||
self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'missing_abstract')
|
||||
|
||||
def test_old_encoding_and_duplicate_abstract_identity(self):
|
||||
self.assertEqual(decode(b'M\xfcller')[0],'Müller')
|
||||
self.pages[SPRING+'papers/abstract.html']*=2;sources,_=self.sources()
|
||||
with self.assertRaisesRegex(ValueError,'Duplicate abstract'):spring_rows(sources,'papers')
|
||||
|
||||
def test_existing_output_is_never_overwritten(self):
|
||||
_,sha=self.sources();out=self.root/'batch';out.mkdir();(out/'keep').write_text('original')
|
||||
with self.assertRaisesRegex(ValueError,'overwrite'):collect(self.cache,sha,out)
|
||||
self.assertEqual((out/'keep').read_text(),'original')
|
||||
|
||||
def review(self,*,authors=None,author_source='canonical_byline',missing='',page=None):
|
||||
url=SPRING+'posters/01/paper.html'
|
||||
self.pages[url]=page or '<h1>Canonical full title</h1><p>Jane Doe, John Roe</p><p>Keywords: web</p><h2>Abstract</h2><p>Body</p>'
|
||||
_,sha=self.sources();raw=self.pages[url].encode()if isinstance(self.pages[url],str)else self.pages[url]
|
||||
row=dict(entry_id='01',individual_url=url,individual_sha256=hashlib.sha256(raw).hexdigest(),
|
||||
title='Canonical full title',authors=authors or ['Jane Doe','John Roe'],author_source=author_source,
|
||||
reason='Reviewed exact canonical entry and its explicit authorship; retain original title variants',missing_abstract_reason=missing)
|
||||
p=self.root/'review.json';p.write_text(json.dumps({'format':1,'source_cache_sha256':sha,'records':[row]}));return sha,p,hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
|
||||
def test_reviewed_variant_keeps_primary_title_and_all_old_title_claims(self):
|
||||
sha,p,h=self.review();out=self.root/'reviewed';r=collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
|
||||
row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['title'],'Canonical full title')
|
||||
self.assertEqual(row['metadata']['abstract_heading_title_raw'],'Same contribution');self.assertEqual(row['metadata']['author_index_titles_raw'],['Same contribution'])
|
||||
self.assertEqual(len(row['metadata']['source_evidence']),3);self.assertEqual(r['reviewed_posters_sha256'],h)
|
||||
self.assertEqual(row['external_id'],'www3:posters:01');self.assertEqual(row['status'],'accepted')
|
||||
|
||||
def test_canonical_byline_can_resolve_unlinked_author_index_without_guessing(self):
|
||||
self.pages[SPRING+'posters/author.html']='<b>Doe, Jane</b><a href="#">Broken index link</a>'
|
||||
sha,p,h=self.review();out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
|
||||
row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['authors'],['Jane Doe','John Roe'])
|
||||
self.assertEqual(row['metadata']['author_index_raw'],[]);self.assertEqual(row['metadata']['author_order'],'canonical_byline')
|
||||
|
||||
def test_reviewed_missing_abstract_is_explicit_partial_not_body_summary(self):
|
||||
self.pages[SPRING+'posters/abstract.html']='<h3><a href="01/paper.html">Same contribution</a></h3><i>Author: Jane Doe</i><p>Keywords: web</p>'
|
||||
sha,p,h=self.review(missing='Neither listing nor canonical page supplies an explicit abstract')
|
||||
out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
|
||||
row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());m=json.loads((out/'www-1995-spring-poster.manifest.json').read_text())
|
||||
self.assertIsNone(row['abstract']);self.assertEqual((m['missing_abstract_count'],m['status']),(1,'partial'))
|
||||
sha,p,h=self.review()
|
||||
with self.assertRaisesRegex(ValueError,'Missing abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
self.assertFalse((self.root/'bad').exists())
|
||||
|
||||
def test_reviewed_names_must_be_in_byline_region_not_citations_or_contact_guess(self):
|
||||
sha,p,h=self.review(authors=['Unrelated Person'],page='<h1>Canonical full title</h1><p>Jane Doe</p><h2>Abstract</h2><p>Unrelated Person cited here</p>')
|
||||
with self.assertRaisesRegex(ValueError,'byline region'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
sha,p,h=self.review(author_source='author_index',authors=['Contact Person'])
|
||||
with self.assertRaisesRegex(ValueError,'author-index'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
|
||||
def test_review_hash_cache_binding_and_canonical_h1_are_all_required(self):
|
||||
sha,p,h=self.review()
|
||||
with self.assertRaisesRegex(ValueError,'checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256='0'*64)
|
||||
d=json.loads(p.read_text());d['source_cache_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
with self.assertRaisesRegex(ValueError,'binding'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
d['source_cache_sha256']=sha;d['records'][0]['title']='Wrong title';p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
with self.assertRaisesRegex(ValueError,'canonical H1'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
|
||||
def test_unlisted_review_entry_and_changed_primary_digest_fail_closed(self):
|
||||
sha,p,h=self.review();d=json.loads(p.read_text());d['records'][0]['individual_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
with self.assertRaisesRegex(ValueError,'page checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
self.pages[SPRING+'posters/abstract.html']=self.pages[SPRING+'posters/abstract.html'].replace('01/paper.html','02/other.html')
|
||||
self.pages[SPRING+'posters/author.html']=self.pages[SPRING+'posters/author.html'].replace('01/paper.html','02/other.html')
|
||||
sha,p,h=self.review()
|
||||
with self.assertRaisesRegex(ValueError,'outside abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
Reference in New Issue
Block a user