Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b14193063a |
+15
-16
@@ -77,22 +77,21 @@ jobs:
|
||||
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
||||
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
||||
assert '&' in eccv_literal_row['title']
|
||||
icassp_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||
icassp_snapshots = root/'icassp2026-snapshots'
|
||||
icassp_snapshots.mkdir()
|
||||
for name, encoded in icassp_fixture['files'].items():
|
||||
target = icassp_snapshots/name
|
||||
target.parent.mkdir(parents=True,exist_ok=True)
|
||||
target.write_bytes(base64.b64decode(encoded))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icassp2026-registry','--snapshots',str(icassp_snapshots),'--out',str(root/'installed-icassp2026')],cwd=outside,check=True)
|
||||
icassp_report = json.loads((root/'installed-icassp2026/collection-report.json').read_text())
|
||||
assert icassp_report['records'] == 4582 and icassp_report['missing_abstract_count'] == 4582
|
||||
assert icassp_report['single_name_author_fields'] == 23
|
||||
assert len(icassp_report['excluded_nonpaper_items']) == 7 and len(icassp_report['excluded_other_conference_dois']) == 56
|
||||
assert not icassp_report['database_written'] and icassp_report['network_requests'] == 0 and icassp_report['approved_for_import'] == 0
|
||||
icassp_rows = [json.loads(line) for line in pathlib.Path(icassp_report['files'][0]['file']).read_text().splitlines()]
|
||||
assert [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in icassp_rows] == sorted(icassp_fixture['independent_gold'],key=lambda r:r['doi'])
|
||||
assert all(r['abstract'] is None and r['track'] == 'proceedings_unclassified' for r in icassp_rows)
|
||||
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||
recsys_snapshots = root/'recsys2026-snapshots'
|
||||
recsys_snapshots.mkdir()
|
||||
for name,encoded in recsys_fixture['files'].items():
|
||||
path = recsys_snapshots/name
|
||||
path.parent.mkdir(parents=True,exist_ok=True)
|
||||
path.write_bytes(base64.b64decode(encoded))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
|
||||
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
|
||||
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
|
||||
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
|
||||
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
|
||||
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||
sdm_snapshots = root/'sdm2026-snapshots'
|
||||
for name, text in sdm_fixture['files'].items():
|
||||
|
||||
@@ -131,6 +131,6 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷
|
||||
|
||||
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||
|
||||
`collect icassp2026-registry` 从冻结的 10 页 Crossref 元数据核验 4,582 条论文,明确排除 7 条非论文及 56 条其他会议记录;保留摘要缺失和单字段作者姓名,不自动融合节目表。见 [ICASSP 2026 采集与恢复说明](docs/icassp2026-registry.md)。
|
||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||
|
||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
# ICASSP 2026 注册元数据采集
|
||||
|
||||
这是长期维护项目中的可重复运行采集器,按需离线处理冻结的公开元数据;不会启动数据库导入、联网或下载论文 PDF。命令仅输出待身份审查的来源版本。
|
||||
|
||||
## 输入与使用
|
||||
|
||||
2026-11-17 在数据服务器采集 Crossref 公开 REST API。先从正式节目表抽查的两篇标题确定注册会议名称和 DOI 前缀,再依据 IEEE DOI 前缀、proceedings-article 类型及注册的活动开始日期 `2026-05-03` 分页获取;日期过滤并不意味着所有结果都属于 ICASSP。
|
||||
|
||||
需要服务器 `/data/paper-index/collected/icassp2026-registry-capture-v1` 下的 `manifest.json` 与 `page-001.json` 至 `page-010.json`。将其父目录传给:
|
||||
|
||||
```sh
|
||||
paper-atlas collect icassp2026-registry \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/icassp2026-normalized-<new-run>
|
||||
```
|
||||
|
||||
输出目录必须不存在。复跑前先检查原任务和产物,不能覆盖成功或失败输出。采集器验证固定清单 SHA256、全部页面原始字节、HTTP 回执、时区、页面数量、总数及唯一 DOI,再生成 JSONL、导入格式的部分覆盖清单、全部原始来源证据和采集报告。导入格式不代表身份审查通过;本命令批准导入数始终为零。
|
||||
|
||||
## 数据范围
|
||||
|
||||
- 10 页共有 4,645 个唯一 DOI;56 个属于 ICPEA,明确排除。
|
||||
- ICASSP 的 4,589 个注册条目中,7 个是封面、目录、作者索引、委员会和审稿人名单;依据已审查的准确 DOI、标题和页码排除,不以“没有作者”泛化排除论文。
|
||||
- 剩余 **4,582 条论文**全部缺少注册摘要,输出 `null`。保留完整注册字段、作者顺序、机构、原始标题、日期及原始来源哈希。
|
||||
- 23 个作者条目只提供单个姓名字段,原样保留并标记;不猜测名字拆分、拼音边界或人物别名。
|
||||
- `accepted` 表示已登记在该会议论文集中的发表观察,并非原始签名录用决定。DOI 用 `publication_doi`,不复制先前投稿系统的 ID。
|
||||
- track 为 `proceedings_unclassified`,不从技术主题推断 main、special session、workshop 或研究类型;没有自造标签。
|
||||
|
||||
正式节目表有 4,589 个编号和标题,与注册条目不是已证明的一一对应关系。独立标题规范化比较得到 4,415 个来源候选和 174 个未匹配节目标题;标题匹配不是作品身份批准,未匹配不等于拒稿、撤稿或未发表。节目表作者与注册作者的版本也不应自动拼接。
|
||||
|
||||
注册活动日期为 5 月 3–8 日,会议网站为 5 月 4–8 日,两者原始值均保留。Crossref 记录中的链接只是原始字段,不会访问 PDF。IEEE Open Preview 入口读取遇到 HTTP 418,未绕过访问限制。公开拒稿元数据是否可得仍为 unknown,本批不是拒稿覆盖证明,也不是第 14 项整体完成证明。
|
||||
|
||||
## 证据与恢复
|
||||
|
||||
注册来源清单 SHA256:`1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93`。
|
||||
|
||||
原始证据恢复包在服务器 `/data/paper-index/validation/icassp2026-registry-capture-v1-recovery.tar.gz`,本地备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件,SHA256 为 `cdc506e95800fb8d37ed87f85740e96f56150f2d96daa971de21a5a7a9ab8388`。安全解包并核验后使用上述命令;测试夹具也冻结了全部 10 页及独立论文清单,供远程 CI 验证。
|
||||
|
||||
本批还需生产规范化验收、数据库当前 DOI/作品关联及历次投稿审查。所有导入结束后仍须执行全库去重、覆盖、摘要和历史完整性审计。编译、打包及测试均交给用户指定的 `ces-control:host` runner。
|
||||
|
||||
参考:[Crossref 筛选器文档](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/)、[正式录用标题表](https://cmsworkshops.com/ICASSP2026/papers/accepted_papers.php)、[会议日程](https://2026.ieeeicassp.org/technical-program/)、[发表和展示类型说明](https://2026.ieeeicassp.org/publishing-and-paper-presentation-options/)。
|
||||
@@ -0,0 +1,53 @@
|
||||
# RecSys 2026 公开录用贡献采集
|
||||
|
||||
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不连接数据库、不访问论文链接或下载 PDF。输出须经过作品、DOI 和投稿历史审查才能导入。
|
||||
|
||||
## 使用
|
||||
|
||||
```sh
|
||||
paper-atlas collect recsys2026-program \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/recsys2026-normalized-<new-run>
|
||||
```
|
||||
|
||||
输入父目录应包含 `remaining2026-listings-research-v2/manifest.json` 与 `recsys-contributions.html`。采集器检查固定清单 SHA256、官方 URL、HTTP 200、采集时区、原始字节哈希、九个 track 的菜单及完整条目数。现有输出目录不可覆盖;失败保留 `failed.json`,成功才发布 `collection-report.json`。再次运行前先检查原任务是否结束。
|
||||
|
||||
## 数据范围与重复项
|
||||
|
||||
来源是 [RecSys 官方录用贡献页面](https://recsys.acm.org/recsys26/contributions/),冻结时间为 2026-11-17。九个来源 track:
|
||||
|
||||
| track | 观察数 |
|
||||
| --- | ---: |
|
||||
| long | 65 |
|
||||
| short | 29 |
|
||||
| past_present_future | 13 |
|
||||
| reproducibility | 4 |
|
||||
| resource | 7 |
|
||||
| industry | 83 |
|
||||
| research_practice_notes | 34 |
|
||||
| demo | 18 |
|
||||
| doctoral_symposium | 10 |
|
||||
|
||||
**263 项来源观察对应 260 个不同 DOI,并非 263 篇不同论文。** 每条记录用官方页面中的 accordion locator 标识,分别保留来源标题、作者原文、摘要和位置,不假设 locator 是原始投稿 ID。
|
||||
|
||||
以下 DOI 各出现两次,六个观察均显式标记待审查:
|
||||
|
||||
- `10.1145/3773078.3831839`:Dart 两条来源记录的标题、作者和摘要都有差异;不得静默选一个覆盖另一个。
|
||||
- `10.1145/3773078.3831819`:标题与作者相同、摘要版本不同。
|
||||
- `10.1145/3773078.3831861`:重复展示的文字相同,保留两个来源位置。
|
||||
|
||||
所有观察都有公开摘要。正文只转换为 HTML 展示文字并规范空白,原始页面完整保留;遇到未审查的数学或图片结构会停止,不把上标扁平化成数字。作者按来源的逗号和最后一个 and 分隔,原始名单同时保留;这不是已验证的作者人物身份或别名。
|
||||
|
||||
`accepted` 仅指官方页面明示录用,不是原始签名评审决定。DOI 来自官方超链接,尚未独立核查实时出版商版本;归入 `publication_doi`,不复制其他会议的投稿 ID。track 明确来自页面区块;`recsys_program:contribution_type` 仅保存页面原始类型标签(Research、PastPresentFuture、Doctoral Symposium),各 track 的 RES/IND/RPN 等缩写同时保留在来源字段,不统一推断研究方法类型。
|
||||
|
||||
本页只含录用贡献,不能据此推断未出现的稿件被拒。公开拒稿导出仍为 unknown,征稿页的 desk rejection 政策不等于具体拒稿记录。不得将本批采集成功当成会议全部来源、负面决定或全库去重完成。
|
||||
|
||||
## 输出与恢复
|
||||
|
||||
输出九份 JSONL、对应部分覆盖清单、原始 HTML 和清单证据,以及记录观察数和不同 DOI 数的报告;批准导入数为零。
|
||||
|
||||
清单 SHA256:`fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9`。
|
||||
|
||||
服务器恢复包 `/data/paper-index/validation/remaining2026-listings-research-v2-recovery.tar.gz`,本地备份 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复原始目录、核对哈希后重跑上述命令。
|
||||
|
||||
完整来源与独立 263 项基准已保存在测试夹具,所有测试、构建和安装验证交给指定的 `ces-control:host` runner。生产导入前还需独立核验规范化结果,并检查当前数据库已有 DOI 绑定及历次投稿。最终全库去重审计仍是独立必做阶段。
|
||||
+1
-1
@@ -25,7 +25,7 @@ COMMANDS = {
|
||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
"""Normalize the frozen ICASSP 2026 Crossref census without network or database writes."""
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import clean
|
||||
|
||||
SOURCE = 'crossref_icassp_registry'
|
||||
TRACK = 'proceedings_unclassified'
|
||||
SNAPSHOT = 'icassp2026-registry-capture-v1'
|
||||
MANIFEST_SHA = '1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93'
|
||||
CONTAINER = 'ICASSP 2026 - 2026 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)'
|
||||
OTHER_CONTAINER = '2026 IEEE 6th International Conference in Power Engineering Applications (ICPEA)'
|
||||
PREFIX = '10.1109/icassp55912.2026.'
|
||||
NONPAPERS = {
|
||||
PREFIX+'11463724': ('ICASSP 2026 Reviewers', 'xii-lxxxvi'),
|
||||
PREFIX+'11464828': ('Author Index', 'A-1-A-152'),
|
||||
PREFIX+'11464776': ('Title Page', 'i-ii'),
|
||||
PREFIX+'11462944': ('ICASSP 2026 Committees', 'vi-xii'),
|
||||
PREFIX+'11464852': ('ICASSP 2026 Organizing Committee', 'iii-iii'),
|
||||
PREFIX+'11460797': ('ICASSP 2026 Cover Page', 'c1-c1'),
|
||||
PREFIX+'11463523': ('Table of Contents', 'xlix-cdlxii'),
|
||||
}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def load_snapshot(root):
|
||||
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||
raise ValueError('Unsafe snapshot manifest')
|
||||
raw = path.read_bytes()
|
||||
if digest(raw) != MANIFEST_SHA:
|
||||
raise ValueError('Reviewed manifest hash changed')
|
||||
manifest = json.loads(raw)
|
||||
if (manifest['status'] != 'registry_census_completed_not_import_approval'
|
||||
or manifest['records_captured'] != 4645 or manifest['unique_dois'] != 4645
|
||||
or manifest['database_written'] is not False or manifest['approved_for_import'] != 0
|
||||
or len(manifest['pages']) != 10):
|
||||
raise ValueError('Incomplete frozen census receipt')
|
||||
files = {'manifest.json': raw}; rows = []
|
||||
for index, receipt in enumerate(manifest['pages'], 1):
|
||||
name = receipt['file']; path = directory/name
|
||||
if (name != f'page-{index:03}.json' or path.is_symlink()
|
||||
or path.stat().st_size > 4*1024**2):
|
||||
raise ValueError('Unsafe snapshot page')
|
||||
body = path.read_bytes()
|
||||
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||
or not receipt['url'].startswith('https://api.crossref.org/works?')
|
||||
or datetime.fromisoformat(receipt['at']).utcoffset() is None
|
||||
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||
raise ValueError('Snapshot receipt/body disagree')
|
||||
data = json.loads(body); message = data['message']
|
||||
expected = 500 if index < 10 else 145
|
||||
if data['status'] != 'ok' or message['total-results'] != 4645 or len(message['items']) != expected:
|
||||
raise ValueError('Incomplete registry page')
|
||||
files[name] = body
|
||||
for original in message['items']:
|
||||
rows.append((original, receipt))
|
||||
return manifest, files, rows
|
||||
|
||||
|
||||
def registry_papers(rows):
|
||||
seen = set(); papers = []; nonpapers = []; other = []
|
||||
for original, receipt in rows:
|
||||
doi = original.get('DOI')
|
||||
if not isinstance(doi, str) or doi in seen:
|
||||
raise ValueError('Missing or repeated registry DOI')
|
||||
seen.add(doi)
|
||||
if original.get('container-title') == [OTHER_CONTAINER]:
|
||||
if doi.startswith(PREFIX):
|
||||
raise ValueError('Conference DOI/container conflict')
|
||||
other.append(doi)
|
||||
continue
|
||||
if (original.get('container-title') != [CONTAINER]
|
||||
or original.get('event', {}).get('name') != CONTAINER
|
||||
or not re.fullmatch(re.escape(PREFIX)+r'[0-9]+', doi)
|
||||
or original.get('publisher') != 'IEEE' or original.get('type') != 'proceedings-article'
|
||||
or original.get('URL') != 'https://doi.org/'+doi
|
||||
or original.get('published') != {'date-parts': [[2026, 5, 3]]}
|
||||
or original.get('event', {}).get('start') != {'date-parts': [[2026, 5, 3]]}
|
||||
or original.get('event', {}).get('end') != {'date-parts': [[2026, 5, 8]]}):
|
||||
raise ValueError('Wrong registered conference, DOI, publisher or dates')
|
||||
if doi in NONPAPERS:
|
||||
title, page = NONPAPERS[doi]
|
||||
if original.get('title') != [title] or original.get('page') != page or original.get('author'):
|
||||
raise ValueError('Changed reviewed nonpaper evidence')
|
||||
nonpapers.append({'doi': doi, 'title': title, 'page': page})
|
||||
continue
|
||||
titles = original.get('title')
|
||||
if not isinstance(titles, list) or len(titles) != 1 or not isinstance(titles[0], str) or not clean(titles[0]):
|
||||
raise ValueError('Missing registry title')
|
||||
authors = original.get('author')
|
||||
if not isinstance(authors, list) or not authors or original.get('abstract'):
|
||||
raise ValueError('Missing authors or changed abstract availability')
|
||||
names = []; flags = []
|
||||
for position, author in enumerate(authors):
|
||||
if (not isinstance(author, dict) or not isinstance(author.get('family'), str)
|
||||
or not clean(author['family']) or ('given' in author and not isinstance(author['given'], str))
|
||||
or author.get('sequence') != ('first' if position == 0 else 'additional')):
|
||||
raise ValueError('Incomplete registry author or sequence')
|
||||
name = clean(' '.join(x for x in (author.get('given'), author['family']) if x))
|
||||
if not name or '@' in name or name.casefold() == 'anonymous':
|
||||
raise ValueError('Unreviewed author identity')
|
||||
if not author.get('given'):
|
||||
flags.append({'kind': 'registry_single_name_field_not_split', 'author_position': position+1})
|
||||
names.append(name)
|
||||
papers.append({'original': original, 'receipt': receipt, 'title': clean(titles[0]), 'authors': names, 'review_flags': flags})
|
||||
if len(seen) != 4645 or len(papers) != 4582 or len(other) != 56 or {r['doi'] for r in nonpapers} != set(NONPAPERS):
|
||||
raise ValueError('Incomplete reviewed registry partition')
|
||||
return sorted(papers, key=lambda p: p['original']['DOI']), nonpapers, other
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
manifest, files, items = load_snapshot(snapshots)
|
||||
papers, nonpapers, other = registry_papers(items)
|
||||
evidence_dir = out/'evidence'; evidence_dir.mkdir()
|
||||
for name, body in files.items():
|
||||
(evidence_dir/name).write_bytes(body)
|
||||
rows = []
|
||||
for paper in papers:
|
||||
original = paper['original']; receipt = paper['receipt']; doi = original['DOI']
|
||||
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
|
||||
'title': paper['title'], 'authors': paper['authors'], 'abstract': None,
|
||||
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||
'venue': 'ICASSP', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||
'raw_decision': 'Publication registered in Crossref ICASSP 2026 proceedings; not an original signed decision.',
|
||||
'evidence_url': receipt['url'], 'observed_at': receipt['at'], 'tags': [],
|
||||
'metadata': {'registry_fields': original, 'review_flags': paper['review_flags'],
|
||||
'source_evidence': {'file': 'evidence/'+receipt['file'], 'sha256': receipt['sha256'], 'url': receipt['url']},
|
||||
'source_manifest': {'file': 'evidence/manifest.json', 'sha256': MANIFEST_SHA},
|
||||
'original_decision_signature_verified': False,
|
||||
'abstract_availability': 'Absent in frozen registry deposit; no abstract inferred or fused from program versions.',
|
||||
'author_transform': 'Whitespace normalization of given + family in registered order; single fields retained, not guessed or split; all original author fields retained.',
|
||||
'publication_dates': {k: original[k] for k in ('published', 'published-print', 'published-online') if k in original},
|
||||
'date_scope': 'Registered event starts May 3; conference website says May 4–8. Source values preserved without substitution.',
|
||||
'scope': 'Registered publication version only. Requires work/DOI/history review before import. No automatic program match, aliases, track/type or public-negative inference.'}})
|
||||
path = out/'icassp-crossref-registry-2026-unclassified.jsonl'
|
||||
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(raw)
|
||||
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICASSP', 'year': 2026, 'track': TRACK,
|
||||
'count': len(rows), 'file': str(path), 'file_sha256': digest(raw), 'list_url': manifest['pages'][0]['url'],
|
||||
'status': 'partial', 'missing_abstract_count': len(rows)}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
|
||||
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'files': [file_manifest],
|
||||
'filtered_registry_records': 4645, 'excluded_nonpaper_items': nonpapers, 'excluded_other_conference_dois': other,
|
||||
'missing_abstract_count': len(rows), 'single_name_author_fields': sum(len(p['review_flags']) for p in papers),
|
||||
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False, 'approved_for_import': 0,
|
||||
'public_rejected_availability': 'unknown', 'source_manifest_sha256': MANIFEST_SHA,
|
||||
'scope': 'Registry census, not completeness of program, abstracts, tracks, negative decisions or work identities. Official program has4589 titles;174 unmatched normalized title candidates remain separate source discrepancies, not rejection or withdrawal evidence.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--snapshots', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Frozen RecSys 2026 accepted-contribution observations, preserving source variants."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
|
||||
SOURCE = 'recsys_program'
|
||||
SNAPSHOT = 'remaining2026-listings-research-v2'
|
||||
MANIFEST_SHA = 'fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9'
|
||||
URL = 'https://recsys.acm.org/recsys26/contributions/'
|
||||
TRACKS = [('long', 'Long Papers', 65), ('short', 'Short Papers', 29),
|
||||
('past_present_future', 'PPF Papers', 13), ('reproducibility', 'Reproducibility', 4),
|
||||
('resource', 'Resource', 7), ('industry', 'Industry', 83),
|
||||
('research_practice_notes', 'R&P Notes', 34), ('demo', 'Demos', 18),
|
||||
('doctoral_symposium', 'Doctoral', 10)]
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected exactly one reviewed source field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def load_snapshot(root):
|
||||
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||
raise ValueError('Unsafe snapshot manifest')
|
||||
raw = path.read_bytes()
|
||||
if digest(raw) != MANIFEST_SHA:
|
||||
raise ValueError('Reviewed manifest hash changed')
|
||||
manifest = json.loads(raw)
|
||||
if manifest['status'] != 'research_completed_not_coverage_or_import_approval' or manifest['database_written'] is not False:
|
||||
raise ValueError('Unreviewed snapshot scope')
|
||||
receipt = one([e for e in manifest['entries'] if e['venue'] == 'RecSys' and e['label'] == 'contributions'])
|
||||
name = receipt['file']; path = directory/name
|
||||
if Path(name).name != name or path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||
raise ValueError('Unsafe snapshot body')
|
||||
body = path.read_bytes()
|
||||
if (receipt['url'] != URL or receipt['final_url'] != URL or receipt['http_status'] != 200
|
||||
or not receipt['content_type'].startswith('text/html') or receipt['bytes'] != len(body)
|
||||
or receipt['sha256'] != digest(body) or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||
raise ValueError('Snapshot receipt/body disagree')
|
||||
return receipt, {'manifest.json': raw, name: body}, body
|
||||
|
||||
|
||||
def body_text(node, doi_url):
|
||||
if not isinstance(node, Node):
|
||||
return node
|
||||
if node.tag in ('script', 'style') or (node.tag == 'a' and node.attrs.get('href') == doi_url):
|
||||
return ''
|
||||
text = ''.join(body_text(c, doi_url) for c in node.children)
|
||||
return ' '+text+' ' if node.tag in ('p','div','li','br','section') else text
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
doc = Document(raw.decode('utf-8')); nodes = list(doc.root.walk())
|
||||
if not any(n.tag == 'title' and 'RecSys 26 - Accepted Contributions' in n.text() for n in nodes):
|
||||
raise ValueError('Wrong conference edition')
|
||||
menu = one([n for n in nodes if n.tag == 'ul' and n.attrs.get('id') == 'tab-1'])
|
||||
labels = [(n.attrs.get('href'), clean(n.text())) for n in menu.walk() if n.tag == 'a']
|
||||
if labels != [(f'#content-tab-1-{i}', label) for i, (_,label,_) in enumerate(TRACKS)]:
|
||||
raise ValueError('Unreviewed track menu')
|
||||
observations = []; seen = set()
|
||||
for tab, (track, label, count) in enumerate(TRACKS):
|
||||
section = one([n for n in nodes if n.tag == 'div' and n.attrs.get('id') == f'content-tab-1-{tab}'])
|
||||
listing = one([n for n in section.walk() if n.tag == 'ul' and n.attrs.get('id') == f'accordion-{tab+1}'])
|
||||
rows = [n for n in listing.children if isinstance(n, Node) and n.tag == 'li']
|
||||
if len(rows) != count:
|
||||
raise ValueError('Incomplete reviewed track census')
|
||||
for position, row in enumerate(rows, 1):
|
||||
anchor = one([n for n in row.children if isinstance(n, Node) and n.tag == 'a'])
|
||||
locator = anchor.attrs.get('rel', '').removeprefix('#')
|
||||
if not re.fullmatch(f'accordion-{tab+1}-slide-[0-9]+', locator) or locator in seen:
|
||||
raise ValueError('Missing or repeated program locator')
|
||||
seen.add(locator)
|
||||
types = [n for n in anchor.walk() if n.tag == 'span' and n.attrs.get('class') == 'paper-type']
|
||||
type_node = one(types); type_literal = type_node.attrs.get('title'); abbreviation = clean(type_node.text())
|
||||
expected_type = [('Research','RES'),('Research','RES'),('PastPresentFuture','PPF'),
|
||||
('Research','REPR'),('Research','RSC'),('Research','IND'),('Research','RPN'),('Research','DEMO'),('Doctoral Symposium','DS')][tab]
|
||||
if (type_literal,abbreviation) != expected_type:
|
||||
raise ValueError('Unreviewed source contribution type')
|
||||
title = clean(''.join(c.text() if isinstance(c, Node) else c for c in anchor.children
|
||||
if not isinstance(c, Node) or c.tag not in ('span','em','br')))
|
||||
authors_literal = clean(one([n for n in anchor.walk() if n.tag == 'em']).text())
|
||||
if not authors_literal.startswith('by '):
|
||||
raise ValueError('Missing program author prefix')
|
||||
authors_literal = authors_literal[3:]
|
||||
authors = [n.strip() for n in re.split(r',\s+|\s+and\s+', authors_literal)]
|
||||
if not title or any(not n or '@' in n for n in authors):
|
||||
raise ValueError('Missing title or program author')
|
||||
body = one([n for n in row.children if isinstance(n, Node) and n.tag == 'div'])
|
||||
doi_url = one([n.attrs.get('href') for n in body.walk() if n.tag == 'a'
|
||||
and n.attrs.get('href', '').startswith('https://doi.org/')])
|
||||
if not re.fullmatch(r'https://doi.org/10\.1145/3773078\.[0-9]+', doi_url):
|
||||
raise ValueError('Wrong publication DOI family')
|
||||
abstract = clean(body_text(body, doi_url))
|
||||
if len(abstract) < 100 or any(n.tag in ('math','sup','sub','img','svg') for n in body.walk()):
|
||||
raise ValueError('Missing abstract or unreviewed structured markup')
|
||||
observations.append({'doi': doi_url.removeprefix('https://doi.org/'), 'tab': tab, 'position': position,
|
||||
'locator': locator, 'title': title, 'authors': authors, 'authors_literal': authors_literal,
|
||||
'abstract': abstract, 'type_labels': [type_literal], 'source_type_abbreviation': abbreviation,
|
||||
'track': track, 'track_label': label})
|
||||
counts = Counter(r['doi'] for r in observations)
|
||||
if len(observations) != 263 or len(counts) != 260 or {d:n for d,n in counts.items() if n>1} != {
|
||||
'10.1145/3773078.3831839':2,'10.1145/3773078.3831819':2,'10.1145/3773078.3831861':2}:
|
||||
raise ValueError('Unreviewed DOI observation census')
|
||||
for item in observations:
|
||||
item['review_flags'] = ['repeated_program_doi_preserve_versions_requires_review'] if counts[item['doi']] > 1 else []
|
||||
return observations
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
receipt, files, body = load_snapshot(snapshots); observations = parse_program(body)
|
||||
evidence = out/'evidence'; evidence.mkdir()
|
||||
for name, raw in files.items():
|
||||
(evidence/name).write_bytes(raw)
|
||||
outputs = []
|
||||
for track,label,count in TRACKS:
|
||||
rows = []
|
||||
for item in observations:
|
||||
if item['track'] != track:
|
||||
continue
|
||||
doi = item['doi']; url = URL+'#'+item['locator']
|
||||
rows.append({'source': SOURCE, 'external_id': '2026/'+item['locator'], 'url': url,
|
||||
'title': item['title'], 'authors': item['authors'], 'abstract': item['abstract'],
|
||||
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme':'publication_doi','value':doi}],
|
||||
'venue':'RecSys', 'year':2026, 'track':track, 'status':'accepted', 'observed_at':receipt['at'],
|
||||
'evidence_url':URL, 'raw_decision':'Listed as an accepted contribution in the official RecSys2026 '+label+' section; not a signed review decision.',
|
||||
'tags':[{'namespace':SOURCE+':contribution_type','value':item['type_labels'][0]}],
|
||||
'metadata':{'program_fields':item, 'program_receipt':receipt,
|
||||
'source_evidence':{'file':'evidence/'+receipt['file'],'sha256':receipt['sha256'],'url':URL},
|
||||
'source_manifest':{'file':'evidence/manifest.json','sha256':MANIFEST_SHA},
|
||||
'original_decision_signature_verified':False,
|
||||
'author_scope':'Source-reported comma/and-delimited names; original literal retained. Not verified publisher person identities or aliases.',
|
||||
'doi_scope':'DOI asserted by official program hyperlink, not independently verified live publisher metadata.',
|
||||
'abstract_transform':'HTML display text and whitespace normalization; source retained. No PDF or cross-source fusion.',
|
||||
'scope':'Program observation keyed by source locator. Repeated DOI variants retained separately and flagged, not counted as distinct scientific works. Requires DOI/work/history review before import.'}})
|
||||
path = out/f'recsys-program-2026-{track}.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||
manifest={'source':SOURCE,'parser_version':1,'venue':'RecSys','year':2026,'track':track,'count':len(rows),
|
||||
'file':str(path),'file_sha256':digest(raw),'list_url':URL,'status':'partial','missing_abstract_count':0,
|
||||
'scope':'Official accepted-contribution observations; unique DOI counts differ from observation counts.'}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');outputs.append(manifest)
|
||||
report={'status':'collected_pending_identity_review','records':263,'unique_dois':260,'files':outputs,
|
||||
'track_counts':{track:count for track,_,count in TRACKS},'missing_abstract_count':0,
|
||||
'duplicate_doi_observations':6,'repeated_dois':{d:n for d,n in Counter(r['doi'] for r in observations).items() if n>1},
|
||||
'source_manifest_sha256':MANIFEST_SHA,'database_written':False,'network_requests':0,'pdfs_downloaded':False,
|
||||
'approved_for_import':0,'public_rejected_availability':'unknown_no_public_export_verified',
|
||||
'scope':'263 program observations for260 DOIs, not263 unique papers or final work identities. Track/type labels are source scoped. Not complete public-negative coverage or a final whole-database audit.'}
|
||||
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||
parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -1,95 +0,0 @@
|
||||
import base64
|
||||
import copy
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import icassp2026 as c
|
||||
|
||||
|
||||
class ICASSP2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||
cls.files = {name: base64.b64decode(body) for name, body in cls.fixture['files'].items()}
|
||||
cls.items = []
|
||||
cls.manifest = json.loads(cls.files[c.SNAPSHOT+'/manifest.json'])
|
||||
for receipt in cls.manifest['pages']:
|
||||
cls.items.extend((r, receipt) for r in json.loads(cls.files[c.SNAPSHOT+'/'+receipt['file']])['message']['items'])
|
||||
|
||||
def materialize(self, root):
|
||||
for name, body in self.files.items():
|
||||
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(body)
|
||||
|
||||
def test_full_census_matches_independent_gold_and_preserves_source(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = root/'src'; self.materialize(src)
|
||||
report = c.collect(root/'out', snapshots=src)
|
||||
self.assertEqual((report['records'],report['missing_abstract_count'],report['single_name_author_fields']), (4582,4582,23))
|
||||
self.assertEqual(len(report['excluded_nonpaper_items']),7); self.assertEqual(len(report['excluded_other_conference_dois']),56)
|
||||
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded']); self.assertEqual(report['network_requests'],0)
|
||||
self.assertEqual(report['approved_for_import'],0); self.assertEqual(report['public_rejected_availability'],'unknown')
|
||||
manifest, = report['files']; path = Path(manifest['file']); raw = path.read_bytes()
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']); self.assertEqual(manifest['status'],'partial')
|
||||
rows = [json.loads(l) for l in raw.splitlines()]
|
||||
actual = [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in rows]
|
||||
gold = sorted(self.fixture['independent_gold'],key=lambda r:r['doi'])
|
||||
self.assertEqual(actual,gold)
|
||||
original = {r['DOI']:r for r,_ in self.items}
|
||||
evidence_hashes = {str(p.relative_to(root/'out')):hashlib.sha256(p.read_bytes()).hexdigest() for p in (root/'out/evidence').iterdir()}
|
||||
for r in rows:
|
||||
doi = r['external_id']; self.assertEqual(r['metadata']['registry_fields'],original[doi])
|
||||
self.assertEqual(r['identifiers'],{'doi':doi}); self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
||||
self.assertEqual(r['track'],'proceedings_unclassified'); self.assertFalse(r['tags']); self.assertFalse(r['abstract'])
|
||||
self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||
evidence = r['metadata']['source_evidence']
|
||||
self.assertEqual(evidence_hashes[evidence['file']],evidence['sha256'])
|
||||
self.assertIn('Fareedha',next(r for r in rows if r['external_id'].endswith('.11461474'))['authors'])
|
||||
with self.assertRaisesRegex(ValueError,'Output exists'):
|
||||
c.collect(root/'out',snapshots=src)
|
||||
|
||||
def test_wrong_conference_doi_date_and_unexpected_abstract_rejected(self):
|
||||
for kind in ('container','event','doi','date','author','sequence','abstract'):
|
||||
with self.subTest(kind=kind):
|
||||
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r.get('author') and r['DOI'].startswith(c.PREFIX))
|
||||
if kind=='container': r['container-title']=['Foreign']
|
||||
if kind=='event': r['event']['name']='Other'
|
||||
if kind=='doi': r['DOI']='10.1109/foreign.2026.1'
|
||||
if kind=='date': r['published']['date-parts']=[[2025,5,3]]
|
||||
if kind=='author': r['author']=[]
|
||||
if kind=='sequence': r['author'][0]['sequence']='additional'
|
||||
if kind=='abstract': r['abstract']='new source version'
|
||||
with self.assertRaises(ValueError): c.registry_papers(items)
|
||||
|
||||
def test_nonpaper_exclusion_requires_exact_reviewed_evidence(self):
|
||||
for kind in ('title','page','author'):
|
||||
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r['DOI'] in c.NONPAPERS)
|
||||
if kind=='title': r['title']=['A research paper']
|
||||
if kind=='page': r['page']='1-5'
|
||||
if kind=='author': r['author']=[{'family':'Smith'}]
|
||||
with self.assertRaisesRegex(ValueError,'nonpaper'): c.registry_papers(items)
|
||||
|
||||
def test_duplicate_and_incomplete_census_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError,'repeated'): c.registry_papers(self.items+[self.items[0]])
|
||||
with self.assertRaisesRegex(ValueError,'partition'): c.registry_papers(self.items[:-1])
|
||||
|
||||
def test_corruption_and_partial_input_preserve_failure_without_success(self):
|
||||
for kind in ('manifest','body','missing'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||
path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'page-002.json')
|
||||
if kind=='missing': path.unlink()
|
||||
else: path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.assertRaises((ValueError,FileNotFoundError)): c.collect(root/'out',snapshots=src)
|
||||
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_symlink_page_and_snapshot_rejected(self):
|
||||
for kind in ('page','directory'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||
path=src/c.SNAPSHOT/('page-001.json' if kind=='page' else '')
|
||||
renamed=path.with_name(path.name+'-original'); path.rename(renamed); path.symlink_to(renamed,target_is_directory=kind=='directory')
|
||||
with self.assertRaisesRegex(ValueError,'Unsafe'): c.collect(root/'out',snapshots=src)
|
||||
@@ -0,0 +1,79 @@
|
||||
import base64
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import recsys2026 as c
|
||||
|
||||
|
||||
class RecSys2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||
cls.files={n:base64.b64decode(raw) for n,raw in cls.fixture['files'].items()}
|
||||
cls.body=cls.files[c.SNAPSHOT+'/recsys-contributions.html']
|
||||
|
||||
def materialize(self,root):
|
||||
for name,raw in self.files.items():
|
||||
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(raw)
|
||||
|
||||
def test_all263_observations_against_independent_gold(self):
|
||||
items=c.parse_program(self.body)
|
||||
keys=self.fixture['independent_gold'][0].keys()
|
||||
self.assertEqual([{k:r[k] for k in keys} for r in items],self.fixture['independent_gold'])
|
||||
self.assertEqual(len(items),263);self.assertEqual(len({r['doi'] for r in items}),260)
|
||||
self.assertEqual(sum(bool(r['review_flags']) for r in items),6)
|
||||
variants=[r for r in items if r['doi']=='10.1145/3773078.3831839']
|
||||
self.assertNotEqual(variants[0]['title'],variants[1]['title']);self.assertNotEqual(variants[0]['authors'],variants[1]['authors'])
|
||||
self.assertNotEqual(variants[0]['abstract'],variants[1]['abstract'])
|
||||
|
||||
def test_installed_format_tracks_evidence_and_original_fields(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out'
|
||||
report=c.collect(out,snapshots=src)
|
||||
self.assertEqual((report['records'],report['unique_dois'],report['missing_abstract_count']),(263,260,0))
|
||||
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||
self.assertEqual(report['track_counts'],{k:n for k,_,n in c.TRACKS});self.assertEqual(report['duplicate_doi_observations'],6)
|
||||
gold={r['locator']:r for r in self.fixture['independent_gold']};seen=set()
|
||||
for manifest in report['files']:
|
||||
path=Path(manifest['file']);raw=path.read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256'])
|
||||
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),manifest['count']);self.assertEqual(manifest['status'],'partial')
|
||||
for r in rows:
|
||||
item=r['metadata']['program_fields'];locator=item['locator'];seen.add(locator);expected=gold[locator]
|
||||
for key in expected:self.assertEqual(item[key],expected[key])
|
||||
self.assertEqual(r['external_id'],'2026/'+locator);self.assertEqual(r['title'],expected['title']);self.assertEqual(r['authors'],expected['authors']);self.assertEqual(r['abstract'],expected['abstract'])
|
||||
self.assertEqual(r['track'],manifest['track']);self.assertEqual(r['identifiers'],{'doi':expected['doi']});self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':expected['doi']}])
|
||||
self.assertEqual(r['tags'],[{'namespace':'recsys_program:contribution_type','value':expected['type_labels'][0]}]);self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||
self.assertEqual(seen,set(gold));self.assertEqual((out/'evidence/recsys-contributions.html').read_bytes(),self.body)
|
||||
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||
|
||||
def test_edition_and_track_label_drift_rejected(self):
|
||||
for old,new in [(b'RecSys 26 - Accepted Contributions',b'RecSys 25 - Accepted Contributions'),(b'>Long Papers</a>',b'>Workshops</a>'),(b"title='Research'",b"title='Unreviewed'")]:
|
||||
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||
|
||||
def test_duplicate_locator_missing_author_and_foreign_doi_rejected(self):
|
||||
for old,new in [(b'accordion-1-slide-2"',b'accordion-1-slide-1"'),(b'by Md Aminul Islam, Kathryn Vasilaky and Elena Zheleva',b'by '),(b'https://doi.org/10.1145/3773078.3831824',b'https://doi.org/10.1145/9999999.3831824')]:
|
||||
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||
|
||||
def test_math_markup_requires_review_and_is_not_flattened(self):
|
||||
changed=self.body.replace(b'Learning-to-rank (LTR)',b'Learning-to-rank <sup>2</sup> (LTR)',1)
|
||||
with self.assertRaisesRegex(ValueError,'markup'):c.parse_program(changed)
|
||||
|
||||
def test_corruption_and_missing_receipt_body_preserve_failure(self):
|
||||
for kind in ('manifest','body','missing'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'recsys-contributions.html')
|
||||
if kind=='missing':path.unlink()
|
||||
else:path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_symlink_source_and_root_rejected(self):
|
||||
for kind in ('source','root'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/'recsys-contributions.html' if kind=='source' else src
|
||||
original=path.with_name(path.name+'-original');path.rename(original);path.symlink_to(original,target_is_directory=kind=='root')
|
||||
with self.assertRaisesRegex(ValueError,'Unsafe'):c.collect(root/'out',snapshots=src)
|
||||
Reference in New Issue
Block a user