diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index f27ecf3..994ca16 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -273,6 +273,23 @@ jobs: subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True) + publisher_fixture = "\n".join(( + "import pathlib, runpy, sys", + "fixture = runpy.run_path(sys.argv[1])", + "root = pathlib.Path(sys.argv[2])", + "root.mkdir()", + "fixture['snapshots'](root)", + )) + publisher_input = root/'eccv-publisher-snapshots' + subprocess.run([python,'-I','-B','-c',publisher_fixture, + str(src/'tests/test_eccv2026_publications.py'),str(publisher_input)],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-publications', + '--toc-snapshots',str(publisher_input/'toc'), + '--chapter-snapshots',str(publisher_input/'chapters'), + '--out',str(root/'installed-eccv-publisher')],cwd=outside,check=True) + publisher_report = json.loads((root/'installed-eccv-publisher/collection-report.json').read_bytes()) + assert publisher_report['records'] == 1 and publisher_report['toc_chapters'] == 2835 + assert not publisher_report['database_written'] springer = json.loads(gzip.decompress((src/'tests/fixtures/springer-aistats1993-fixture.json.gz').read_bytes())) for group in ('toc','chapters'): folder = root/'springer-snapshots'/group diff --git a/README.md b/README.md index 9c4d492..eaa197c 100644 --- a/README.md +++ b/README.md @@ -148,3 +148,5 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 `collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。 `collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。 + +ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。 diff --git a/docs/eccv2026-publications.md b/docs/eccv2026-publications.md new file mode 100644 index 0000000..e22741d --- /dev/null +++ b/docs/eccv2026-publications.md @@ -0,0 +1,16 @@ +# ECCV 2026 正式出版章节采集 + +`collect eccv2026-publications` 读取已归档的 Springer LNCS 17001–17078 目录及所选章节的公开 HTML,输出可复核的 Paper Atlas 记录。目录覆盖 2,835 个章节 DOI。会议节目单与出版方卷册简介均为 2,834 条,逐条差异仍待核。Workshop 论文集另行处理。 + +```sh +paper-atlas collect eccv2026-publications \ + --toc-snapshots /data/paper-index/collected/eccv2026-springer-toc-v1 \ + --chapter-snapshots /data/paper-index/validation/eccv-verified-chapter-pages \ + --out /data/paper-index/collected/eccv-publisher-metadata +``` + +目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。已有输出目录需另查回执或换新空目录后运行。 + +输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。 + +生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index baa3928..084bccf 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -27,6 +27,7 @@ COMMANDS = { ('collect','ismir2026-program'): ('paper_atlas.collectors.ismir2026', {}, '核验 ISMIR 2026 公开节目与录用名单并保留原始摘要、来源差异和自愿公开评审'), ('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'), ('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'), + ('collect','eccv2026-publications'): ('paper_atlas.collectors.eccv2026_publications', {}, '核验ECCV2026 Springer 正式出版章节'), ('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'), ('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'), ('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'), diff --git a/paper_atlas/collectors/eccv2026_publications.py b/paper_atlas/collectors/eccv2026_publications.py new file mode 100644 index 0000000..dddbbf9 --- /dev/null +++ b/paper_atlas/collectors/eccv2026_publications.py @@ -0,0 +1,184 @@ +"""Normalize frozen ECCV 2026 Springer proceedings and preserve publisher evidence.""" +import argparse +import fcntl +import hashlib +import json +from pathlib import Path +import re +import tempfile + +from paper_atlas.io.html import Document, clean +from paper_atlas.io.springer import parse_chapter, read_page, snapshot_index + +SOURCE = 'eccv2026_springer' +ORIGIN = 'https://link.springer.com' +BOOK_TITLE = 'Computer Vision – ECCV 2026' +LIST_URL = ORIGIN + '/conference/eccv' +BOOK = re.compile(r'10\.1007/978-3-032-\d{5}-[\dXx]') +CHAPTER = re.compile(BOOK.pattern + r'_[1-9]\d*') +VOLUMES = set(range(17001, 17079)) +PUBLICATIONS = 2835 + + +def fingerprint(raw): + return hashlib.sha256(raw).hexdigest() + + +def toc_page(html, book_doi, volume): + tree = Document(html).root + text = clean(tree.text()) + if BOOK_TITLE not in text or 'Malmö' not in text or f'(LNCS, volume {volume})' not in text: + raise ValueError('Unexpected Springer ECCV volume scope') + entries = {} + for node in tree.walk(): + if node.tag != 'li' or node.attrs.get('data-test') != 'chapter': + continue + headings = [n for n in node.walk() if n.tag == 'h3'] + if len(headings) != 1: + raise ValueError('Ambiguous Springer contents heading') + heading, = headings + title = clean(heading.text()) + links = [n.attrs.get('href', '') for n in heading.walk() if n.tag == 'a'] + if title in ('Front Matter', 'Back Matter') and not links: + continue + if len(links) != 1 or not title: + raise ValueError('Authored contents entry lacks one chapter link') + link, = links + doi = link.removeprefix('/chapter/') + if (not link.startswith('/chapter/') or not CHAPTER.fullmatch(doi) + or not doi.startswith(book_doi + '_') or doi in entries): + raise ValueError('Unexpected or duplicate Springer chapter DOI') + entries[doi] = title + if not entries: + raise ValueError('Empty Springer contents page') + return entries + + +def publisher_inventory(root): + root = Path(root).resolve() + receipt_bytes = (root / 'receipts.jsonl').read_bytes() + inventory_bytes = (root / 'volume-checkpoint.json').read_bytes() + summary = json.loads((root / 'summary.json').read_bytes()) + volumes = json.loads(inventory_bytes) + receipt_rows = [json.loads(line) for line in receipt_bytes.splitlines()] + receipts = {row['file']: row for row in receipt_rows} + if (summary['raw_receipts_sha256'] != fingerprint(receipt_bytes) + or summary['volume_checkpoint_sha256'] != fingerprint(inventory_bytes) + or set(volumes) != {str(n) for n in VOLUMES} + or len(receipts) != len(receipt_rows)): + raise ValueError('Incomplete or inconsistent Springer contents snapshot') + entries, used = {}, set() + for number, volume in sorted(volumes.items()): + book_url = volume['book'] + book_doi = book_url.removeprefix(ORIGIN + '/book/') + if not BOOK.fullmatch(book_doi) or book_url != ORIGIN + '/book/' + book_doi: + raise ValueError('Unexpected Springer volume URL') + if set(volume['files']) != {'1', '2'}: + raise ValueError('Missing reviewed Springer contents page') + volume_dois = set() + for index, file in volume['files'].items(): + receipt = receipts[file] + path = root / file + page_url = book_url + ('?page=2' if index == '2' else '') + if (file in used or Path(file).name != file or not path.resolve().is_relative_to(root) + or receipt['url'] != page_url or receipt['status'] != 200 + or not 0 < receipt['bytes'] <= 3_000_000): + raise ValueError('Unsafe, reused or mismatched Springer contents receipt') + raw = path.read_bytes() + if len(raw) != receipt['bytes'] or fingerprint(raw) != receipt['sha256']: + raise ValueError('Springer contents bytes disagree with receipt') + used.add(file) + page_entries = toc_page(raw.decode(), book_doi, int(number)) + if entries.keys() & page_entries.keys(): + raise ValueError('Springer chapter repeats across contents pages') + volume_dois.update(page_entries) + entries.update({doi: {'title': title, 'lncs_volume': int(number), 'book_doi': book_doi, + 'evidence_url': page_url, 'evidence_sha256': receipt['sha256']} + for doi, title in page_entries.items()}) + if volume_dois != set(volume['chapter_dois']) or len(volume_dois) != len(volume['chapter_dois']): + raise ValueError('Springer volume chapter set differs from checkpoint') + if used != receipts.keys() or len(entries) != PUBLICATIONS or summary['chapter_link_count'] != PUBLICATIONS: + raise ValueError('Springer contents chapter total differs from the reviewed archive') + return entries + + +def publisher_row(chapter, toc, evidence): + doi = chapter['doi'] + if clean(chapter['title']) != toc['title']: + raise ValueError('Chapter title differs from Springer contents, review the version') + url = ORIGIN + '/chapter/' + doi + return { + 'source': SOURCE, 'external_id': doi, 'url': url, + 'venue': 'ECCV', 'year': 2026, 'track': 'main', + 'title': chapter['title'], 'authors': chapter['authors'], 'abstract': chapter['abstract'], + 'status': 'accepted', 'identifiers': {'doi': doi}, + 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}], + 'evidence_url': url, 'observed_at': evidence['observed_at'], + 'raw_decision': 'Chapter published in the refereed ECCV 2026 conference proceedings.', + 'tags': [{'namespace': SOURCE + ':publication_format', 'value': 'proceedings_chapter'}], + 'metadata': { + 'book_title': BOOK_TITLE, 'book_doi': toc['book_doi'], 'lncs_volume': toc['lncs_volume'], + 'authors_citation_raw': chapter['authors_citation_raw'], 'first_page': chapter['first_page'], + 'last_page': chapter['last_page'], 'publisher_chapter_evidence': evidence, + 'publisher_contents_evidence': {'url': toc['evidence_url'], 'sha256': toc['evidence_sha256']}, + 'scope': 'Refereed conference proceedings, LNCS 17001–17078.', + 'original_decision_signature_verified': False, + }, + } + + +def collect(out, *, toc_snapshots, chapter_snapshots): + contents = publisher_inventory(toc_snapshots) + chapters = snapshot_index(chapter_snapshots) + urls = {ORIGIN + '/chapter/' + doi: doi for doi in contents} + if not chapters or not chapters.keys() <= urls.keys(): + raise ValueError('No chapter snapshots or chapter outside reviewed ECCV proceedings') + rows, raw = [], {} + for url in sorted(chapters): + doi = urls[url] + body, evidence = read_page(chapters, url) + chapter = parse_chapter(body.decode(), doi, book_title=BOOK_TITLE, publication_year=2026) + rows.append(publisher_row(chapter, contents[doi], evidence)) + raw[evidence['file']] = body + out = Path(out) + out.mkdir(parents=True, exist_ok=True) + lock_path = out / 'eccv2026-publications.collect.lock' + filename = 'eccv2026-springer-main.jsonl' + with lock_path.open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if any(p != lock_path for p in out.iterdir()): + raise FileExistsError('Existing output must be inspected, not overwritten') + with tempfile.TemporaryDirectory(prefix='.eccv2026-publications-', dir=out) as temporary: + stage = Path(temporary) + (stage / 'evidence').mkdir() + for path, body in raw.items(): + (stage / path).write_bytes(body) + data = ''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in rows).encode() + (stage / filename).write_bytes(data) + manifest = { + 'source': SOURCE, 'venue': 'ECCV', 'year': 2026, 'track': 'main', 'parser_version': 1, + 'file': str(out / filename), 'file_sha256': fingerprint(data), 'count': len(rows), + 'list_url': LIST_URL, 'source_expected_count': PUBLICATIONS, 'status': 'partial', + 'missing_abstract_count': sum(not r['abstract'] for r in rows), + } + report = { + 'status': 'collected_pending_identity_review', 'records': len(rows), 'files': [manifest], + 'toc_chapters': len(contents), 'chapter_details_missing': len(contents) - len(rows), + 'database_written': False, + } + metadata = Path(filename).with_suffix('.manifest.json') + (stage / metadata).write_text(json.dumps(manifest, indent=2) + '\n') + (stage / 'collection-report.json').write_text(json.dumps(report, indent=2) + '\n') + for item in ('evidence', filename, metadata, 'collection-report.json'): + (stage / item).replace(out / item) + return report + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog) + parser.add_argument('--toc-snapshots', required=True) + parser.add_argument('--chapter-snapshots', required=True) + parser.add_argument('--out', required=True) + args = parser.parse_args(argv) + print(json.dumps(collect(args.out, toc_snapshots=args.toc_snapshots, + chapter_snapshots=args.chapter_snapshots), ensure_ascii=False)) diff --git a/paper_atlas/collectors/springer_selected.py b/paper_atlas/collectors/springer_selected.py index a801219..be3e1e3 100644 --- a/paper_atlas/collectors/springer_selected.py +++ b/paper_atlas/collectors/springer_selected.py @@ -1,16 +1,14 @@ """Reviewed postconference selected volumes from frozen public HTML metadata.""" import argparse -from collections import defaultdict import fcntl import hashlib import json from pathlib import Path import re import tempfile -from urllib.parse import parse_qs, urlsplit -from paper_atlas.db.submissions import timestamp from paper_atlas.io.html import Document, clean +from paper_atlas.io.springer import parse_chapter as parse_citation_chapter, read_page, snapshot_index SOURCE = 'springer_selected_volume' BOOK = '10.1007/978-1-4612-2660-4' @@ -26,50 +24,6 @@ SCOPE = { } -def snapshot_index(root): - root = Path(root).resolve() - receipts = json.loads((root/'receipts.json').read_bytes())['responses'] - index = {} - for receipt in receipts: - url = receipt['url'] - if url in index: - raise ValueError('Duplicate snapshot URL') - index[url] = (root, receipt) - return index - - -def read_page(index, url): - if url not in index: - raise ValueError('Missing required public HTML snapshot: '+url) - root, receipt = index[url] - filename = Path(receipt['file']) - path = root/filename - content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '') - requested, final = urlsplit(url), urlsplit(receipt.get('final_url', '')) - query = parse_qs(final.query, keep_blank_values=True) - original_query = parse_qs(requested.query, keep_blank_values=True) - # Springer returns the public metadata with an explicit cookie diagnostic. - # Retain that final URL verbatim; accept no other redirect or changed page. - if query != original_query and requested.netloc == 'link.springer.com': - if (query.pop('error', None) != ['cookies_not_supported'] - or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))): - raise ValueError('Unreviewed Springer redirect query') - if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root) - or receipt.get('http_status') != 200 - or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path) - or final.fragment or query != original_query - or 'text/html' not in content_type.lower()): - raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot') - raw = path.read_bytes() - digest = hashlib.sha256(raw).hexdigest() - if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']: - raise ValueError('Snapshot checksum or size differs') - evidence = {'url':url, 'sha256':digest, 'bytes':len(raw), - 'file':'evidence/'+digest+'.html', 'http_status':200, - 'observed_at':timestamp(receipt['at']), 'response_receipt':receipt} - return raw, evidence - - def parse_toc(html, *, excluded_materials=None): entries = {} for node in Document(html).root.walk(): @@ -97,40 +51,8 @@ def parse_toc(html, *, excluded_materials=None): def parse_chapter(html, doi): - nodes = list(Document(html).root.walk()) - meta = defaultdict(list) - for node in nodes: - if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'): - meta[node.attrs['name']].append(node.attrs.get('content', '')) - def one(key): - values = meta[key] - if len(values) != 1 or not values[0]: - raise ValueError('Missing or ambiguous citation field: '+key) - return values[0] - if (one('citation_doi') != doi or one('citation_inbook_title') != SCOPE['book_title'] - or one('citation_publication_date') != str(SCOPE['publication_year'])): - raise ValueError('Chapter DOI, book or publication year differs from reviewed scope') - raw_authors = meta['citation_author'] - if not raw_authors: - raise ValueError('Missing named chapter authors') - authors = [] - for raw in raw_authors: - if raw.count(',') != 1: - raise ValueError('Unreviewed citation author format') - family, given = [clean(s) for s in raw.split(',', 1)] - if not family or not given: - raise ValueError('Incomplete citation author') - authors.append({'given':given, 'family':family}) - abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content'] - if len(abstract_nodes) > 1: - raise ValueError('Ambiguous abstract section') - abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None - first, last = one('citation_firstpage'), one('citation_lastpage') - if not first.isdigit() or not last.isdigit() or int(first) > int(last): - raise ValueError('Invalid chapter pages') - return {'doi':doi, 'title':one('citation_title'), 'authors':authors, - 'authors_citation_raw':list(raw_authors), 'abstract':abstract or None, - 'first_page':first, 'last_page':last} + return parse_citation_chapter(html, doi, book_title=SCOPE['book_title'], + publication_year=SCOPE['publication_year']) def record_for(chapter, evidence, scope_evidence): diff --git a/paper_atlas/io/springer.py b/paper_atlas/io/springer.py new file mode 100644 index 0000000..4c52d05 --- /dev/null +++ b/paper_atlas/io/springer.py @@ -0,0 +1,91 @@ +"""Frozen Springer public HTML receipts and chapter citation metadata.""" +from collections import defaultdict +import hashlib +from pathlib import Path +import json +import re +from urllib.parse import parse_qs, urlsplit + +from paper_atlas.db.submissions import timestamp +from paper_atlas.io.html import Document, clean + + +def snapshot_index(root): + root = Path(root).resolve() + receipts = json.loads((root/'receipts.json').read_bytes())['responses'] + index = {} + for receipt in receipts: + url = receipt['url'] + if url in index: + raise ValueError('Duplicate snapshot URL') + index[url] = (root, receipt) + return index + + +def read_page(index, url): + if url not in index: + raise ValueError('Missing required public HTML snapshot: '+url) + root, receipt = index[url] + filename = Path(receipt['file']) + path = root/filename + content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '') + requested, final = urlsplit(url), urlsplit(receipt.get('final_url', '')) + query = parse_qs(final.query, keep_blank_values=True) + original_query = parse_qs(requested.query, keep_blank_values=True) + # Springer returns the public metadata with an explicit cookie diagnostic. + # Retain that final URL verbatim; accept no other redirect or changed page. + if query != original_query and requested.netloc == 'link.springer.com': + if (query.pop('error', None) != ['cookies_not_supported'] + or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))): + raise ValueError('Unreviewed Springer redirect query') + if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root) + or receipt.get('http_status') != 200 + or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path) + or final.fragment or query != original_query + or 'text/html' not in content_type.lower()): + raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot') + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']: + raise ValueError('Snapshot checksum or size differs') + evidence = {'url':url, 'sha256':digest, 'bytes':len(raw), + 'file':'evidence/'+digest+'.html', 'http_status':200, + 'observed_at':timestamp(receipt['at']), 'response_receipt':receipt} + return raw, evidence + + +def parse_chapter(html, doi, *, book_title, publication_year): + nodes = list(Document(html).root.walk()) + meta = defaultdict(list) + for node in nodes: + if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'): + meta[node.attrs['name']].append(node.attrs.get('content', '')) + def one(key): + values = meta[key] + if len(values) != 1 or not values[0]: + raise ValueError('Missing or ambiguous citation field: '+key) + return values[0] + if (one('citation_doi') != doi or one('citation_inbook_title') != book_title + or one('citation_publication_date') != str(publication_year)): + raise ValueError('Chapter DOI, book or publication year differs from reviewed scope') + raw_authors = meta['citation_author'] + if not raw_authors: + raise ValueError('Missing named chapter authors') + authors = [] + for raw in raw_authors: + if raw.count(',') != 1: + raise ValueError('Unreviewed citation author format') + family, given = [clean(s) for s in raw.split(',', 1)] + if not family or not given: + raise ValueError('Incomplete citation author') + authors.append({'given':given, 'family':family}) + abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content'] + if len(abstract_nodes) > 1: + raise ValueError('Ambiguous abstract section') + abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None + first, last = one('citation_firstpage'), one('citation_lastpage') + if not first.isdigit() or not last.isdigit() or int(first) > int(last): + raise ValueError('Invalid chapter pages') + return {'doi':doi, 'title':one('citation_title'), 'authors':authors, + 'authors_citation_raw':list(raw_authors), 'abstract':abstract or None, + 'first_page':first, 'last_page':last} diff --git a/tests/test_eccv2026_publications.py b/tests/test_eccv2026_publications.py new file mode 100644 index 0000000..8be6714 --- /dev/null +++ b/tests/test_eccv2026_publications.py @@ -0,0 +1,124 @@ +"""Checks for published ECCV metadata and preserved submission histories.""" +from hashlib import sha256 +import json +from pathlib import Path +import sqlite3 +import tempfile +import unittest + +from paper_atlas.collectors.eccv2026_publications import BOOK_TITLE, SOURCE, collect +from paper_atlas.db.store import Store +from paper_atlas.ingestion.records import ingest_file + + +FIRST_BOOK = '10.1007/978-3-032-10000-0' +FIRST_DOI = FIRST_BOOK + '_1' + + +def snapshots(root): + toc, chapters = root / 'toc', root / 'chapters' + toc.mkdir() + chapters.mkdir() + receipts, volumes = [], {} + for offset in range(78): + volume = 17001 + offset + book = f'10.1007/978-3-032-{10000 + offset}-0' + book_url = 'https://link.springer.com/book/' + book + count = 37 if offset < 27 else 36 + files, chapter_dois = {}, [] + for page, indices in ((1, range(1, 20)), (2, range(20, count + 1))): + filename = f'{volume}-{page}.html' + html = f'

{BOOK_TITLE}

Malmö (LNCS, volume {volume})

' + html += '
  • Front Matter

  • ' + for i in indices: + doi = f'{book}_{i}' + html += f'
  • Title {volume}-{i}

  • ' + chapter_dois.append(doi) + raw = html.encode() + (toc / filename).write_bytes(raw) + files[str(page)] = filename + receipts.append({'file': filename, 'status': 200, 'bytes': len(raw), 'sha256': sha256(raw).hexdigest(), + 'url': book_url + ('?page=2' if page == 2 else '')}) + volumes[str(volume)] = {'book': book_url, 'files': files, 'chapter_dois': chapter_dois} + data = ''.join(json.dumps(row) + '\n' for row in receipts).encode() + (toc / 'receipts.jsonl').write_bytes(data) + checkpoint = json.dumps(volumes).encode() + (toc / 'volume-checkpoint.json').write_bytes(checkpoint) + (toc / 'summary.json').write_text(json.dumps({ + 'raw_receipts_sha256': sha256(data).hexdigest(), + 'volume_checkpoint_sha256': sha256(checkpoint).hexdigest(), + 'chapter_link_count': 2835, + })) + url = 'https://link.springer.com/chapter/' + FIRST_DOI + citation = { + 'citation_doi': FIRST_DOI, 'citation_title': 'Title 17001-1', + 'citation_inbook_title': BOOK_TITLE, 'citation_publication_date': '2026', + 'citation_firstpage': '1', 'citation_lastpage': '10', + 'citation_author': 'Chen, Lei', + } + body = ''.join(f'' for k, v in citation.items()) + body += '

    Published abstract.

    ' + raw = body.encode() + (chapters / 'first.html').write_bytes(raw) + (chapters / 'receipts.json').write_text(json.dumps({'responses': [{ + 'url': url, 'final_url': url, 'file': 'first.html', 'at': '2026-11-18T00:00:00Z', + 'http_status': 200, 'bytes': len(raw), 'sha256': sha256(raw).hexdigest(), + 'headers': {'Content-Type': 'text/html'}, + }]})) + return {'toc_snapshots': toc, 'chapter_snapshots': chapters} + + +class ECCV2026PublicationsTests(unittest.TestCase): + def test_frozen_snapshot_and_existing_history(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = snapshots(root) + report = collect(root / 'out', **source) + self.assertEqual((report['toc_chapters'], report['chapter_details_missing']), (2835, 2834)) + manifest, = report['files'] + self.assertEqual((manifest['track'], manifest['status']), ('main', 'partial')) + row = json.loads(Path(manifest['file']).read_text()) + self.assertEqual(row['identifiers'], {'doi': FIRST_DOI}) + self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': FIRST_DOI}]) + self.assertEqual(row['authors'], [{'family': 'Chen', 'given': 'Lei'}]) + self.assertEqual(row['abstract'], 'Published abstract.') + self.assertEqual(row['metadata']['lncs_volume'], 17001) + self.assertFalse(row['metadata']['original_decision_signature_verified']) + detail = row['metadata']['publisher_chapter_evidence'] + self.assertEqual(sha256((root / 'out' / detail['file']).read_bytes()).hexdigest(), detail['sha256']) + db = root / 'example.sqlite' + store = Store(db) + pid, rid = store.record('prior', 'work', 'https://example.org/work', + row['title'], 'Original preprint abstract.', row['authors'], + {'arxiv': '2601.00001'}) + store.decision(pid, rid, 'ICLR', 2026, 'main', 'work', 'https://example.org/decision', + 'rejected', 'Original ICLR result.') + store.conn.commit() + store.conn.close() + row['identifiers']['arxiv'] = '2601.00001' + path = Path(manifest['file']) + raw = (json.dumps(row) + '\n').encode() + path.write_bytes(raw) + manifest['file_sha256'] = sha256(raw).hexdigest() + path.with_suffix('.manifest.json').write_text(json.dumps(manifest)) + ingest_file(db, path) + ingest_file(db, path) + with sqlite3.connect(db) as conn: + self.assertEqual(conn.execute('SELECT id,abstract FROM papers').fetchall(), + [(pid, 'Original preprint abstract.')]) + self.assertEqual(conn.execute('SELECT status FROM decisions ORDER BY id').fetchall(), + [('rejected',), ('accepted',)]) + self.assertEqual(conn.execute('SELECT scheme,value FROM submission_identifiers').fetchall(), + [('publication_doi', FIRST_DOI)]) + self.assertEqual(conn.execute('SELECT name FROM sources ORDER BY id').fetchall(), [('prior',), (SOURCE,)]) + with self.assertRaises(FileExistsError): + collect(root / 'out', **source) + + def test_changed_contents_receipt_stops_before_output(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = snapshots(root) + (source['toc_snapshots'] / '17001-1.html').write_text('unverified bytes') + with self.assertRaisesRegex(ValueError, 'contents bytes disagree'): + collect(root / 'out', **source) + self.assertFalse((root / 'out').exists())