diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index e8d1b42..99b66b1 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -77,6 +77,23 @@ jobs: eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844') assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title'] assert '&' in eccv_literal_row['title'] + sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes())) + sdm_snapshots = root/'sdm2026-snapshots' + for name, text in sdm_fixture['files'].items(): + target = sdm_snapshots/name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(text.encode('utf-8')) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','sdm2026-proceedings','--snapshots',str(sdm_snapshots),'--out',str(root/'installed-sdm2026')],cwd=outside,check=True) + sdm_report = json.loads((root/'installed-sdm2026/collection-report.json').read_text()) + assert sdm_report['records'] == 53 and sdm_report['missing_abstract_count'] == 53 + assert not sdm_report['database_written'] and sdm_report['network_requests'] == 0 + assert sdm_report['program_observations']['count'] == 53 and sdm_report['program_observations']['abstract_pages'] == 52 + assert sdm_report['program_registry_author_sequence_differences'] == 20 + sdm_rows = [json.loads(line) for line in pathlib.Path(sdm_report['files'][0]['file']).read_text().splitlines()] + assert len(sdm_rows) == 53 and all(row['abstract'] is None for row in sdm_rows) + sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()] + assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1 + assert all('status' not in row and 'identifiers' not in row for row in sdm_observations) icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes())) icaps_snapshots = root/'icaps2026-snapshots' icaps_snapshots.mkdir() diff --git a/README.md b/README.md index c49d5f2..e8b9d96 100644 --- a/README.md +++ b/README.md @@ -130,3 +130,5 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 `collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。 ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。 + +`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。 diff --git a/docs/sdm2026-proceedings.md b/docs/sdm2026-proceedings.md new file mode 100644 index 0000000..4a4502a --- /dev/null +++ b/docs/sdm2026-proceedings.md @@ -0,0 +1,44 @@ +# SDM 2026:论文集与日程版本 + +这是长期维护、按需运行的离线来源适配器。它复用固定的公开元数据快照,不请求网络、不下载论文 PDF、不连接数据库,也不会把候选版本关联当成已通过的作品去重。 + +```sh +paper-atlas collect sdm2026-proceedings \ + --snapshots /data/paper-index/collected \ + --out /data/paper-index/collected/sdm2026-normalized-pass01 +``` + +`--snapshots` 下需保留四个目录:`sdm2026-official-research-v1`、`sdm2026-crossref-research-v1`、`sdm2026-program-research-v1`、`sdm2026-public-abstract-research-v1`。适配器固定核验各自 manifest 和全部成功响应的字节数、哈希、URL 与状态。测试 fixture 保留原始响应字节及独立盘点结果。使用全新输出目录恢复,保留失败目录中的 `failed.json`;成功报告最后以原子替换发布。来源更新时另存版本并重新审阅,不能修改旧 manifest 后冒充旧快照。 + +## 两种输出 + +[论文集](https://epubs.siam.org/doi/book/10.1137/1.9781611979329)的 ISBN 为 `9781611979329`。[Crossref 精确 ISBN 查询](https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000)的冻结响应有 55 项:53 个论文章节、1 本论文集、1 份前言。适配器只将 53 个章节放入 `report.files` 的待身份审阅 JSONL,排除论文集和前言;核对完整章节 DOI 集合、父 DOI、容器标题、ISBN、作者顺序证据及出版年份。 + +正式记录保留 Crossref 作者版本与原始字段,使用 `proceedings_unclassified`,不按主题或分会场推断 track。注册发表记录表达已收入论文集的事实,不冒充原始签名录用决定。印刷与在线出版日期分别保留,不能用它们替代会议举办日期或决定日期。两个标题含 `*`:显示标题保留脚注星号并标记待审阅,原始标题完整保留;去掉这一特定脚注的文字仅用于列出待审阅候选,不用于自动改名或绑定作品。 + +**53 个 Crossref 章节都没有摘要字段。**主记录的 `abstract` 因此为空,报告如实记为 53 个摘要缺失,不能因另一来源有同标题摘要而直接填入。 + +[官方在线日程](https://meetings.siam.org/program.cfm?CONFCODE=sdm26)有 9 个研究论文分会场、53 项日程条目和 52 个公开摘要页。`program-observations.jsonl` 单独保留日程版本、公开页标题、来源报告的作者姓名、摘要、会场引用、回执与哈希;它不是入库 manifest,也不在 `report.files` 中。页面中作者行的单位、国家及联系邮箱不规范化成人物字段,原始页面留在证据文件中。 + +52 个公开页中,32 个作者序列与 Crossref 完全一致,20 个不同,包括姓名拼写、顺序和作者缺失。例如 *Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening* 的日程页列出 Xin Wang、Qinwen Ge,Crossref 列出 Xin Wang、Yu Wang、Yunchao Liu、Jens Meiler、Tyler Derr。它们必须保留为独立来源版本,不能仅凭标题相同拼接摘要与作者,或生成姓名别名。候选 DOI 字段明确表示待审阅关系,不是逐篇身份批准;作者列表相同也不能代替全库作品身份核查。 + +*A Trajectory View of Statistical Policy Comparison* 的日程展示标为取消,且没有公开摘要链接;它仍出现在冻结的 Crossref 论文章节集合中。适配器保留这两项事实,**不把取消展示转换成论文撤回、拒稿或删除发表记录**。邀请报告、博士论坛和其他非研究论文活动也不计入这里的 53 个章节。 + +## 公开拒稿与访问限制 + +[官方投稿说明](https://www.siam.org/conferences-events/siam-conferences/sdm26/submissions/)链接 OpenReview group `SIAM.org/2026/Conference_on_Data_Mining`。匿名读取 group 配置成功;按实际 `rejected_venue_id` 查询 notes 返回 HTTP 403,随后停止,没有绕过或继续查询其他负面状态。因此公开拒稿、desk-rejected 和 withdrawn 决定覆盖仍未核实,不能报告为零条或从录用名单缺席推断拒稿。 + +来源冻结时,服务器访问 SIAM 主站的两份说明页及出版社目录也返回 403;回执如实保留。可读取的是公开在线日程及独立公开的 Crossref 注册元数据。Crossref 记录不是受限出版社页面的重建,也不被称作已经核验的出版社摘要版本。 + +## 来源恢复与后续核查 + +| 冻结目录 | manifest SHA-256 | +| --- | --- | +| sdm2026-official-research-v1 | a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61 | +| sdm2026-crossref-research-v1 | 51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5 | +| sdm2026-program-research-v1 | 689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77 | +| sdm2026-public-abstract-research-v1 | 8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5 | + +服务器恢复包:`/data/paper-index/validation/sdm2026-sources-recovery-v1.tar.gz`,SHA-256 为 `d6d14c0e4bb2c26e16a4888e3b150e44f982d689351638b1b9a28644736c4387`;本机工作记录目录有同名备份。包内保留原始来源、独立盘点及逐页版本比较记录,含义是可复现来源证据,不是已完成导入或身份批准。 + +下次继续时,先核对来源版本和访问状态,再做当前数据库的 DOI 绑定、标题/作者近似候选与完整投稿历史核查,逐条解决作者及摘要版本问题。批准后另建入库队列、保护基线和导入后验收;所有来源结束后仍需全库去重、覆盖、摘要、约束和实际查询验收。测试、构建和安装验证只在已授权 `ces-control:host` 远程 runner 执行。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 2a82c9e..0c867bf 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -25,6 +25,7 @@ COMMANDS = { ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), ('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'), ('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'), + ('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'), ('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'), ('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), diff --git a/paper_atlas/collectors/sdm2026.py b/paper_atlas/collectors/sdm2026.py new file mode 100644 index 0000000..0976515 --- /dev/null +++ b/paper_atlas/collectors/sdm2026.py @@ -0,0 +1,313 @@ +"""Frozen SDM 2026 publication records and separate public program observations.""" +import argparse +from collections import Counter +from datetime import date, datetime +import hashlib +from html.parser import HTMLParser +import json +from pathlib import Path +import re +from urllib.parse import parse_qs, urljoin, urlsplit + +from paper_atlas.io.html import Document, clean + +SOURCE = 'crossref_sdm_registry' +TRACK = 'proceedings_unclassified' +BOOK = '10.1137/1.9781611979329' +ISBN = '9781611979329' +TITLE = 'Proceedings of the 2026 SIAM International Conference on Data Mining (SDM)' +QUERY = 'https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000' +PROGRAM = 'https://meetings.siam.org/program.cfm?CONFCODE=sdm26' +SNAPSHOTS = { + 'sdm2026-official-research-v1': ('a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61', 'sources'), + 'sdm2026-crossref-research-v1': ('51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5', 'sources'), + 'sdm2026-program-research-v1': ('689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77', 'sessions'), + 'sdm2026-public-abstract-research-v1': ('8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5', 'records'), +} + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def one(items): + if len(items) != 1: + raise ValueError('Expected exactly one reviewed source field') + return items[0] + + +def load_snapshot(root, name): + expected, key = SNAPSHOTS[name] + directory = root/name; path = directory/'manifest.json' + if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 256*1024: + raise ValueError('Unsafe snapshot manifest') + raw = path.read_bytes() + if digest(raw) != expected: + raise ValueError('Reviewed snapshot manifest hash changed') + manifest = json.loads(raw) + if datetime.fromisoformat(manifest['at']).utcoffset() is None: + raise ValueError('Snapshot timestamp requires timezone') + files = {'manifest.json': raw} + for receipt in manifest[key]: + if 'file' not in receipt: + # A recorded 403 is an access observation, not an empty data set. + continue + filename = receipt['file']; path = directory/filename + if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 24*1024**2: + raise ValueError('Unsafe snapshot source file') + body = path.read_bytes() + if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url'] + or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)): + raise ValueError('Snapshot receipt/body disagree') + if filename in files: + raise ValueError('Repeated snapshot source file') + files[filename] = body + return manifest, files + + +def registry_papers(message): + items = message.get('items', []) + if message.get('total-results') != 55 or len(items) != 55: + raise ValueError('Incomplete exact-ISBN registry census') + if Counter(x.get('type') for x in items) != {'book-chapter': 53, 'book': 1, 'other': 1}: + raise ValueError('Unreviewed publication item types') + papers = []; excluded = []; seen = set() + for original in items: + doi = original.get('DOI') + if not isinstance(doi, str) or doi in seen or original.get('ISBN') != [ISBN]: + raise ValueError('Missing/repeated DOI or wrong ISBN') + seen.add(doi) + if doi in (BOOK, BOOK+'.fm'): + expected = ('book', [TITLE]) if doi == BOOK else ('other', ['Front Matter']) + if (original['type'], original.get('title')) != expected: + raise ValueError('Unreviewed excluded volume/front matter') + excluded.append({'doi': doi, 'type': original['type'], 'title': original['title']}) + continue + if (original.get('type') != 'book-chapter' or original.get('container-title') != [TITLE] + or not re.fullmatch(re.escape(BOOK)+r'\.[1-9][0-9]*', doi) + or original.get('URL') != 'https://doi.org/'+doi + or original.get('publisher') != 'Society for Industrial and Applied Mathematics' + or original.get('relation', {}).get('is-part-of') != [ + {'id-type': 'doi', 'id': 'https://doi.org/'+BOOK, 'asserted-by': 'object'}]): + raise ValueError('Wrong publication edition, publisher or parent DOI') + title = one(original.get('title', [])) + if not isinstance(title, str) or not clean(title): + raise ValueError('Missing registry title') + flags = [] + if '<' in title or '>' in title: + match = re.fullmatch(r'([^<>]+)\*\s*', title, re.S) + if not match: + raise ValueError('Unreviewed title markup') + comparison_title = clean(match[1]) + flags.append('title_sup_asterisk_footnote_requires_review') + else: + comparison_title = clean(Document(title).root.text()) + displayed_title = clean(Document(title).root.text()) + authors = original.get('author') + if (not isinstance(authors, list) or not authors + or any(not isinstance(a.get('given'), str) or not isinstance(a.get('family'), str) + or not clean(a['given']) or not clean(a['family']) for a in authors) + or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)): + raise ValueError('Incomplete registry authors or order evidence') + names = [clean(a['given']+' '+a['family']) for a in authors] + if any('@' in n or 'anonymous' in n.casefold() for n in names): + raise ValueError('Unreviewed registry author identity') + dates = {} + for field in ('published', 'published-print', 'published-online', 'issued'): + if field in original: + parts = one(original[field]['date-parts']) + if len(parts) != 3 or any(type(x) is not int for x in parts) or parts[0] != 2026: + raise ValueError('Unreviewed registry publication date') + dates[field] = date(*parts).isoformat() + if not dates or original.get('abstract'): + raise ValueError('Reviewed registry date/abstract availability changed') + papers.append({'original': original, 'title': displayed_title, 'comparison_title': comparison_title, + 'authors': names, 'review_flags': flags, 'publication_dates': dates}) + if {p['original']['DOI'] for p in papers} != {BOOK+'.'+str(n) for n in range(1, 54)} or len(excluded) != 2: + raise ValueError('Incomplete reviewed 53-chapter DOI set') + return sorted(papers, key=lambda p: int(p['original']['DOI'].rsplit('.', 1)[1])), excluded + + +class SessionParser(HTMLParser): + """The source omits closing dt/dd tags; each next dt starts a new event.""" + def __init__(self): + super().__init__(); self.rows = []; self.current = None; self.strong = False; self.author = False + + def finish(self): + if self.current is not None: + match = re.fullmatch(r'(\d{1,2}:\d{2}-\d{1,2}:\d{2})\s+(.+)', clean(self.current.pop('timed_title'))) + if not match or not clean(self.current['authors_literal']): + raise ValueError('Incomplete contributed program event') + self.current['time_literal'], self.current['title'] = match.groups() + self.current['authors_literal'] = clean(self.current['authors_literal']) + self.rows.append(self.current) + self.current = None; self.strong = False; self.author = False + + def handle_starttag(self, tag, attrs): + attrs = dict(attrs) + if tag == 'dt': + self.finish() + self.current = {'timed_title': '', 'authors_literal': '', 'cancelled_presentation': False, 'abstract_links': []} + if self.current is not None: + if tag == 'strong': self.strong = True + elif tag == 'dd': self.author = True + elif tag == 's': self.current['cancelled_presentation'] = True + elif tag == 'a' and attrs.get('href', '').startswith('dsp_talk.cfm?p='): + self.current['abstract_links'].append(attrs['href']) + + def handle_endtag(self, tag): + if tag == 'strong': self.strong = False + elif tag == 'dl': self.finish() + + def handle_data(self, text): + if self.current is not None: + if self.strong: self.current['timed_title'] += text + elif self.author: self.current['authors_literal'] += text + elif clean(text) == 'Cancelled': self.current['cancelled_presentation'] = True + + +def talk_fields(raw, session_url): + nodes = list(Document(raw.decode('utf-8')).root.walk()) + if one([clean(n.text()) for n in nodes if n.tag == 'h1']) != '2026 SIAM International Conference on Data Mining': + raise ValueError('Wrong program page edition') + title = clean(one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'ptitle']).text()) + abstract_node = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'abstract']) + abstract = clean(abstract_node.text()) + if not title or not abstract.startswith('Abstract.') or not abstract[len('Abstract.'):].strip(): + raise ValueError('Missing public program title/abstract') + reference = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'sesref']) + link = one([n.attrs.get('href', '') for n in reference.walk() if n.tag == 'a']) + def session_key(url): + u = urlsplit(url) + query = {k.lower(): v for k, v in parse_qs(u.query).items()} + if u.scheme != 'https' or u.netloc != 'meetings.siam.org' or u.path != '/sess/dsp_programsess.cfm' or set(query) != {'sessioncode'}: + raise ValueError('Foreign program session link') + return query['sessioncode'] + if session_key(link) != session_key(session_url): + raise ValueError('Program page belongs to a different session') + authors = [] + for n in nodes: + if n.tag == 'li': + # Source rows contain name, affiliations, country and contact email. + # Only the independently checked leading name is a metadata field. + literal = clean(n.text()); name, separator, _ = literal.partition(',') + if not separator or not name or '@' in name: + raise ValueError('Unreviewed program author row') + authors.append(name) + if not authors: + raise ValueError('Missing source-reported program authors') + markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math', 'sup', 'sub', 'svg', 'img')}) + return {'title': title, 'source_reported_authors': authors, 'abstract': abstract[len('Abstract.'):].strip(), + 'abstract_markup_tags': markup, 'review_flags': ['structured_abstract_markup_requires_review'] if markup else [], + 'author_completeness': 'Program-reported names only; not asserted to be the complete or final publisher author sequence.'} + + +def program_observations(snapshots, papers): + official, official_files = snapshots['sdm2026-official-research-v1'] + sessions, session_files = snapshots['sdm2026-program-research-v1'] + pages, page_files = snapshots['sdm2026-public-abstract-research-v1'] + main_nodes = list(Document(official_files['program.html'].decode()).root.walk()) + links = {urljoin(PROGRAM, n.attrs.get('href', '')): clean(n.text()) for n in main_nodes + if n.tag == 'a' and re.match(r'CP[1-9]\s', clean(n.text()))} + if len(links) != 9 or {s['url']: s['title'] for s in sessions['sessions']} != links: + raise ValueError('Contributed session membership disagrees with main program') + receipts = {r['url']: r for r in pages['records']} + if len(receipts) != len(pages['records']) or len(receipts) != 52: + raise ValueError('Incomplete or repeated public talk receipts') + rows = []; used = set() + for session in sessions['sessions']: + parser = SessionParser(); parser.feed(session_files[session['file']].decode()); parser.finish() + for position, event in enumerate(parser.rows, 1): + urls = [urljoin(session['url'], link) for link in event.pop('abstract_links')] + event.update(record_kind='source_observation_not_import_record', session=session['title'], + session_url=session['url'], session_sha256=session['sha256'], session_position=position) + if not urls and event['cancelled_presentation']: + event.update(program_page=None, public_page_fields=None) + else: + url = one(urls) + if url not in receipts or url in used: + raise ValueError('Missing or repeated public program page') + used.add(url); receipt = receipts[url] + if receipt['session_url'] != session['url']: + raise ValueError('Public page receipt session disagrees') + fields = talk_fields(page_files[receipt['file']], session['url']) + if fields['title'] != event['title']: + fields['review_flags'].append('session_page_title_versions_differ_requires_review') + event.update(program_page=url, public_page_receipt=receipt, public_page_fields=fields) + candidates = [] + for paper in papers: + if paper['comparison_title'].casefold() == event['title'].casefold(): + fields = event['public_page_fields'] + candidates.append({'registry_doi_candidate': paper['original']['DOI'], + 'comparison': 'title_candidate_only_not_identity_approval', + 'registry_title_review_flags': paper['review_flags'], + 'author_sequence_exact': fields['source_reported_authors'] == paper['authors'] if fields else None}) + event['registry_candidates'] = candidates + event['scope'] = 'Separate program version; cancellation is presentation status only. No original submission ID, decision, work identity or author alias inferred.' + rows.append(event) + if len(rows) != 53 or used != set(receipts) or sum(r['cancelled_presentation'] for r in rows) != 1: + raise ValueError('Incomplete reviewed program event census') + return rows + + +def collect(out, *, snapshots): + out = Path(out) + if out.exists(): + raise ValueError('Output exists; inspect previous operation before retrying') + out.mkdir(parents=True) + try: + loaded = {name: load_snapshot(Path(snapshots), name) for name in SNAPSHOTS} + registry_manifest, registry_files = loaded['sdm2026-crossref-research-v1'] + message = json.loads(registry_files['isbn-works.json'])['message'] + papers, excluded = registry_papers(message) + observations = program_observations(loaded, papers) + evidence = []; directory = out/'evidence'; directory.mkdir() + for name, (manifest, files) in loaded.items(): + for filename, raw in files.items(): + rel = 'evidence/'+digest(raw)+Path(filename).suffix + (out/rel).write_bytes(raw) + evidence.append({'snapshot': name, 'file': filename, 'output_file': rel, 'sha256': digest(raw)}) + receipt = one([r for r in registry_manifest['sources'] if r.get('file') == 'isbn-works.json']) + rows = [] + for p in papers: + original = p['original']; doi = original['DOI'] + rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'], 'title': p['title'], + 'authors': p['authors'], 'abstract': None, 'identifiers': {'doi': doi}, + 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}], + 'venue': 'SDM', 'year': 2026, 'track': TRACK, 'status': 'accepted', 'tags': [], + 'observed_at': receipt['at'], 'evidence_url': QUERY, + 'raw_decision': 'Registered research chapter in the exact SDM2026 proceedings ISBN; not an original signed review decision.', + 'metadata': {'registry_fields': original, 'publication_dates': p['publication_dates'], + 'registry_response_sha256': digest(registry_files['isbn-works.json']), 'review_flags': p['review_flags'], + 'original_decision_signature_verified': False, + 'abstract_availability': 'No abstract in frozen Crossref deposit; public program abstracts remain separate source versions.', + 'title_transform': 'Display text with footnote asterisk retained; raw title preserved. Footnote-free text is used only to list review candidates.', + 'scope': 'Requires fresh work/DOI/history review before import. No automatic program fusion, track, person alias or negative decision inference.'}}) + path = out/'sdm-crossref-registry-2026-unclassified.jsonl' + payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload) + file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'SDM', 'year': 2026, 'track': TRACK, + 'count': 53, 'file': str(path), 'file_sha256': digest(payload), 'list_url': QUERY, 'status': 'partial', 'missing_abstract_count': 53} + path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n') + sidecar = out/'program-observations.jsonl' + raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in observations).encode(); sidecar.write_bytes(raw) + report = {'status': 'collected_pending_identity_review', 'records': 53, 'files': [file_manifest], + 'missing_abstract_count': 53, 'excluded_registry_items': excluded, + 'program_observations': {'file': str(sidecar), 'sha256': digest(raw), 'count': 53, 'abstract_pages': 52, + 'cancelled_presentations': 1, 'not_an_import_manifest': True}, + 'program_registry_author_sequence_differences': sum(c['author_sequence_exact'] is False for o in observations for c in o['registry_candidates']), + 'evidence': evidence, 'public_rejected_metadata': 'unknown_anonymous_query_http403', + 'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False, + 'scope': 'Registry publication records and program source observations remain separate; no work merge, metadata fusion or final coverage/dedup claim.'} + part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json') + return report + except Exception as error: + (out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n') + raise + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog, description=__doc__) + parser.add_argument('--snapshots', required=True) + parser.add_argument('--out', required=True) + args = parser.parse_args(argv) + print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False)) diff --git a/tests/fixtures/sdm2026-source.json.gz b/tests/fixtures/sdm2026-source.json.gz new file mode 100644 index 0000000..965dbfb Binary files /dev/null and b/tests/fixtures/sdm2026-source.json.gz differ diff --git a/tests/test_sdm2026.py b/tests/test_sdm2026.py new file mode 100644 index 0000000..f14bdd0 --- /dev/null +++ b/tests/test_sdm2026.py @@ -0,0 +1,142 @@ +import copy +import gzip +import hashlib +import json +from pathlib import Path +import tempfile +import unittest + +from paper_atlas.collectors import sdm2026 as c + + +class SDM2026Tests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/sdm2026-source.json.gz').read_bytes())) + cls.message = json.loads(cls.fixture['files']['sdm2026-crossref-research-v1/isbn-works.json'])['message'] + + def snapshots(self, root): + for name, text in self.fixture['files'].items(): + path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode('utf-8')) + + def test_publication_census_excludes_book_and_front_matter_not_cancelled_paper(self): + papers, excluded = c.registry_papers(self.message) + self.assertEqual(len(papers), 53) + self.assertEqual({x['doi'] for x in excluded}, {c.BOOK, c.BOOK+'.fm'}) + self.assertIn(c.BOOK+'.7', {p['original']['DOI'] for p in papers}) + marked = [p for p in papers if p['review_flags']] + self.assertEqual({p['original']['DOI'] for p in marked}, {c.BOOK+'.12', c.BOOK+'.35'}) + for p in marked: + self.assertTrue(p['title'].endswith('*')) + self.assertIn('*', p['original']['title'][0]) + + def test_full_collection_preserves_separate_source_versions_and_unknown_negatives(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.snapshots(root/'src') + report = c.collect(root/'out', snapshots=root/'src') + self.assertEqual((report['records'], report['missing_abstract_count']), (53, 53)) + self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0) + self.assertEqual(report['public_rejected_metadata'], 'unknown_anonymous_query_http403') + self.assertEqual(report['program_registry_author_sequence_differences'], 20) + manifest, = report['files']; self.assertEqual(manifest['status'], 'partial') + rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()] + originals = {x['DOI']: x for x in self.message['items'] if x['type'] == 'book-chapter'} + for row in rows: + self.assertEqual(row['metadata']['registry_fields'], originals[row['external_id']]) + self.assertIsNone(row['abstract']); self.assertFalse(row['tags']) + self.assertEqual(row['identifiers'], {'doi': row['external_id']}) + self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': row['external_id']}]) + self.assertEqual((row['status'], row['track']), ('accepted', 'proceedings_unclassified')) + self.assertEqual(row['metadata']['publication_dates']['published-online'], '2026-11-12') + sidecar = report['program_observations'] + self.assertTrue(sidecar['not_an_import_manifest']) + observations = [json.loads(line) for line in Path(sidecar['file']).read_text().splitlines()] + self.assertEqual(len(observations), 53) + self.assertEqual(sum(x['public_page_fields'] is not None for x in observations), 52) + for observation in observations: + self.assertNotIn('status', observation); self.assertNotIn('identifiers', observation) + self.assertEqual(observation['record_kind'], 'source_observation_not_import_record') + cancelled, = [x for x in observations if x['cancelled_presentation']] + self.assertEqual(cancelled['title'], 'A Trajectory View of Statistical Policy Comparison') + self.assertIsNone(cancelled['public_page_fields']) + # The real source has conflicting author sets under the same title. + conflict = next(x for x in observations if x['title'].startswith('Scaffold-Aware Generative')) + self.assertEqual(conflict['public_page_fields']['source_reported_authors'], ['Xin Wang', 'Qinwen Ge']) + candidate, = conflict['registry_candidates']; self.assertFalse(candidate['author_sequence_exact']) + registry_row = next(x for x in rows if x['external_id'] == candidate['registry_doi_candidate']) + self.assertEqual(registry_row['authors'], ['Xin Wang', 'Yu Wang', 'Yunchao Liu', 'Jens Meiler', 'Tyler Derr']) + # Every copied source and manifest remains byte-identical, including CRLF. + for evidence in report['evidence']: + body = (root/'out'/evidence['output_file']).read_bytes() + self.assertEqual(body, self.fixture['files'][evidence['snapshot']+'/'+evidence['file']].encode('utf-8')) + self.assertEqual(hashlib.sha256(body).hexdigest(), evidence['sha256']) + with self.assertRaisesRegex(ValueError, 'Output exists'): + c.collect(root/'out', snapshots=root/'src') + + def test_all_program_events_match_independent_inventory(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.snapshots(root) + loaded = {name: c.load_snapshot(root, name) for name in c.SNAPSHOTS} + papers, _ = c.registry_papers(self.message) + observations = c.program_observations(loaded, papers) + gold = {(x['session_url'], x['title']): x for x in self.fixture['program_gold']} + self.assertEqual({(x['session_url'], x['title']) for x in observations}, set(gold)) + for item in observations: + original = gold[(item['session_url'], item['title'])] + for key in ('authors_literal', 'time_literal', 'cancelled_presentation'): + self.assertEqual(item[key], original[key]) + + def test_all_public_abstracts_and_source_reported_names_match_independent_gold(self): + files = self.fixture['files'] + manifest = json.loads(files['sdm2026-public-abstract-research-v1/manifest.json']) + receipts = {r['url']: r for r in manifest['records']} + self.assertEqual(len(self.fixture['page_gold']), 52) + for gold in self.fixture['page_gold']: + receipt = receipts[gold['program_url']] + raw = files['sdm2026-public-abstract-research-v1/'+receipt['file']].encode('utf-8') + self.assertEqual(hashlib.sha256(raw).hexdigest(), gold['page_sha256']) + fields = c.talk_fields(raw, receipt['session_url']) + self.assertEqual(fields['title'], gold['title']) + self.assertEqual(fields['source_reported_authors'], gold['authors']) + self.assertEqual(fields['abstract'], gold['abstract']) + self.assertTrue(all('@' not in name for name in fields['source_reported_authors'])) + + def test_wrong_isbn_parent_doi_and_incomplete_census_rejected(self): + for kind in ('isbn', 'parent', 'repeat', 'type', 'year', 'author', 'markup', 'abstract', 'missing'): + with self.subTest(kind=kind): + message = copy.deepcopy(self.message) + row = next(x for x in message['items'] if x['type'] == 'book-chapter') + if kind == 'isbn': row['ISBN'] = ['9780000000000'] + if kind == 'parent': row['relation']['is-part-of'][0]['id'] = 'https://doi.org/10.9999/other' + if kind == 'repeat': row['DOI'] = c.BOOK + if kind == 'type': row['type'] = 'other' + if kind == 'year': row['published']['date-parts'] = [[2025, 10, 31]] + if kind == 'author': row['author'][0]['sequence'] = 'additional' + if kind == 'markup': row['title'] = ['Changed 2 title'] + if kind == 'abstract': row['abstract'] = 'Unreviewed new version' + if kind == 'missing': message['items'].pop() + with self.assertRaises(ValueError): c.registry_papers(message) + + def test_foreign_session_and_changed_edition_rejected(self): + raw = self.fixture['files']['sdm2026-public-abstract-research-v1/162766.html'].encode() + url = 'https://meetings.siam.org/sess/dsp_programsess.cfm?SESSIONCODE=91511' + for changed in [raw.replace(b'sessioncode=91511', b'sessioncode=91507'), + raw.replace(b'2026 SIAM International Conference', b'2025 SIAM International Conference')]: + with self.assertRaises(ValueError): c.talk_fields(changed, url) + + def test_corruption_preserves_failure_without_success_report(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.snapshots(root/'src') + (root/'src/sdm2026-public-abstract-research-v1/162766.html').write_bytes(b'corrupt') + with self.assertRaisesRegex(ValueError, 'receipt/body'): + c.collect(root/'out', snapshots=root/'src') + self.assertTrue((root/'out/failed.json').exists()) + self.assertFalse((root/'out/collection-report.json').exists()) + + def test_source_symlink_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.snapshots(root/'src') + p = root/'src/sdm2026-crossref-research-v1/isbn-works.json' + original = p.read_bytes(); p.unlink(); (root/'data.json').write_bytes(original); p.symlink_to(root/'data.json') + with self.assertRaisesRegex(ValueError, 'Unsafe'): + c.collect(root/'out', snapshots=root/'src')