Collect verified ECCV 2026 Springer proceedings metadata
Metadata integrity checks / metadata (push) Successful in 7m51s
Metadata integrity checks / metadata (push) Successful in 7m51s
This commit is contained in:
@@ -273,6 +273,23 @@ jobs:
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True)
|
||||
publisher_fixture = "\n".join((
|
||||
"import pathlib, runpy, sys",
|
||||
"fixture = runpy.run_path(sys.argv[1])",
|
||||
"root = pathlib.Path(sys.argv[2])",
|
||||
"root.mkdir()",
|
||||
"fixture['snapshots'](root)",
|
||||
))
|
||||
publisher_input = root/'eccv-publisher-snapshots'
|
||||
subprocess.run([python,'-I','-B','-c',publisher_fixture,
|
||||
str(src/'tests/test_eccv2026_publications.py'),str(publisher_input)],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-publications',
|
||||
'--toc-snapshots',str(publisher_input/'toc'),
|
||||
'--chapter-snapshots',str(publisher_input/'chapters'),
|
||||
'--out',str(root/'installed-eccv-publisher')],cwd=outside,check=True)
|
||||
publisher_report = json.loads((root/'installed-eccv-publisher/collection-report.json').read_bytes())
|
||||
assert publisher_report['records'] == 1 and publisher_report['toc_chapters'] == 2835
|
||||
assert not publisher_report['database_written']
|
||||
springer = json.loads(gzip.decompress((src/'tests/fixtures/springer-aistats1993-fixture.json.gz').read_bytes()))
|
||||
for group in ('toc','chapters'):
|
||||
folder = root/'springer-snapshots'/group
|
||||
|
||||
@@ -148,3 +148,5 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
||||
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||
|
||||
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||
|
||||
ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# ECCV 2026 正式出版章节采集
|
||||
|
||||
`collect eccv2026-publications` 读取已归档的 Springer LNCS 17001–17078 目录及所选章节的公开 HTML,输出可复核的 Paper Atlas 记录。目录覆盖 2,835 个章节 DOI。会议节目单与出版方卷册简介均为 2,834 条,逐条差异仍待核。Workshop 论文集另行处理。
|
||||
|
||||
```sh
|
||||
paper-atlas collect eccv2026-publications \
|
||||
--toc-snapshots /data/paper-index/collected/eccv2026-springer-toc-v1 \
|
||||
--chapter-snapshots /data/paper-index/validation/eccv-verified-chapter-pages \
|
||||
--out /data/paper-index/collected/eccv-publisher-metadata
|
||||
```
|
||||
|
||||
目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。已有输出目录需另查回执或换新空目录后运行。
|
||||
|
||||
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
|
||||
|
||||
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
|
||||
@@ -27,6 +27,7 @@ COMMANDS = {
|
||||
('collect','ismir2026-program'): ('paper_atlas.collectors.ismir2026', {}, '核验 ISMIR 2026 公开节目与录用名单并保留原始摘要、来源差异和自愿公开评审'),
|
||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||
('collect','eccv2026-publications'): ('paper_atlas.collectors.eccv2026_publications', {}, '核验ECCV2026 Springer 正式出版章节'),
|
||||
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
||||
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
|
||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Normalize frozen ECCV 2026 Springer proceedings and preserve publisher evidence."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
|
||||
from paper_atlas.io.html import Document, clean
|
||||
from paper_atlas.io.springer import parse_chapter, read_page, snapshot_index
|
||||
|
||||
SOURCE = 'eccv2026_springer'
|
||||
ORIGIN = 'https://link.springer.com'
|
||||
BOOK_TITLE = 'Computer Vision – ECCV 2026'
|
||||
LIST_URL = ORIGIN + '/conference/eccv'
|
||||
BOOK = re.compile(r'10\.1007/978-3-032-\d{5}-[\dXx]')
|
||||
CHAPTER = re.compile(BOOK.pattern + r'_[1-9]\d*')
|
||||
VOLUMES = set(range(17001, 17079))
|
||||
PUBLICATIONS = 2835
|
||||
|
||||
|
||||
def fingerprint(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def toc_page(html, book_doi, volume):
|
||||
tree = Document(html).root
|
||||
text = clean(tree.text())
|
||||
if BOOK_TITLE not in text or 'Malmö' not in text or f'(LNCS, volume {volume})' not in text:
|
||||
raise ValueError('Unexpected Springer ECCV volume scope')
|
||||
entries = {}
|
||||
for node in tree.walk():
|
||||
if node.tag != 'li' or node.attrs.get('data-test') != 'chapter':
|
||||
continue
|
||||
headings = [n for n in node.walk() if n.tag == 'h3']
|
||||
if len(headings) != 1:
|
||||
raise ValueError('Ambiguous Springer contents heading')
|
||||
heading, = headings
|
||||
title = clean(heading.text())
|
||||
links = [n.attrs.get('href', '') for n in heading.walk() if n.tag == 'a']
|
||||
if title in ('Front Matter', 'Back Matter') and not links:
|
||||
continue
|
||||
if len(links) != 1 or not title:
|
||||
raise ValueError('Authored contents entry lacks one chapter link')
|
||||
link, = links
|
||||
doi = link.removeprefix('/chapter/')
|
||||
if (not link.startswith('/chapter/') or not CHAPTER.fullmatch(doi)
|
||||
or not doi.startswith(book_doi + '_') or doi in entries):
|
||||
raise ValueError('Unexpected or duplicate Springer chapter DOI')
|
||||
entries[doi] = title
|
||||
if not entries:
|
||||
raise ValueError('Empty Springer contents page')
|
||||
return entries
|
||||
|
||||
|
||||
def publisher_inventory(root):
|
||||
root = Path(root).resolve()
|
||||
receipt_bytes = (root / 'receipts.jsonl').read_bytes()
|
||||
inventory_bytes = (root / 'volume-checkpoint.json').read_bytes()
|
||||
summary = json.loads((root / 'summary.json').read_bytes())
|
||||
volumes = json.loads(inventory_bytes)
|
||||
receipt_rows = [json.loads(line) for line in receipt_bytes.splitlines()]
|
||||
receipts = {row['file']: row for row in receipt_rows}
|
||||
if (summary['raw_receipts_sha256'] != fingerprint(receipt_bytes)
|
||||
or summary['volume_checkpoint_sha256'] != fingerprint(inventory_bytes)
|
||||
or set(volumes) != {str(n) for n in VOLUMES}
|
||||
or len(receipts) != len(receipt_rows)):
|
||||
raise ValueError('Incomplete or inconsistent Springer contents snapshot')
|
||||
entries, used = {}, set()
|
||||
for number, volume in sorted(volumes.items()):
|
||||
book_url = volume['book']
|
||||
book_doi = book_url.removeprefix(ORIGIN + '/book/')
|
||||
if not BOOK.fullmatch(book_doi) or book_url != ORIGIN + '/book/' + book_doi:
|
||||
raise ValueError('Unexpected Springer volume URL')
|
||||
if set(volume['files']) != {'1', '2'}:
|
||||
raise ValueError('Missing reviewed Springer contents page')
|
||||
volume_dois = set()
|
||||
for index, file in volume['files'].items():
|
||||
receipt = receipts[file]
|
||||
path = root / file
|
||||
page_url = book_url + ('?page=2' if index == '2' else '')
|
||||
if (file in used or Path(file).name != file or not path.resolve().is_relative_to(root)
|
||||
or receipt['url'] != page_url or receipt['status'] != 200
|
||||
or not 0 < receipt['bytes'] <= 3_000_000):
|
||||
raise ValueError('Unsafe, reused or mismatched Springer contents receipt')
|
||||
raw = path.read_bytes()
|
||||
if len(raw) != receipt['bytes'] or fingerprint(raw) != receipt['sha256']:
|
||||
raise ValueError('Springer contents bytes disagree with receipt')
|
||||
used.add(file)
|
||||
page_entries = toc_page(raw.decode(), book_doi, int(number))
|
||||
if entries.keys() & page_entries.keys():
|
||||
raise ValueError('Springer chapter repeats across contents pages')
|
||||
volume_dois.update(page_entries)
|
||||
entries.update({doi: {'title': title, 'lncs_volume': int(number), 'book_doi': book_doi,
|
||||
'evidence_url': page_url, 'evidence_sha256': receipt['sha256']}
|
||||
for doi, title in page_entries.items()})
|
||||
if volume_dois != set(volume['chapter_dois']) or len(volume_dois) != len(volume['chapter_dois']):
|
||||
raise ValueError('Springer volume chapter set differs from checkpoint')
|
||||
if used != receipts.keys() or len(entries) != PUBLICATIONS or summary['chapter_link_count'] != PUBLICATIONS:
|
||||
raise ValueError('Springer contents chapter total differs from the reviewed archive')
|
||||
return entries
|
||||
|
||||
|
||||
def publisher_row(chapter, toc, evidence):
|
||||
doi = chapter['doi']
|
||||
if clean(chapter['title']) != toc['title']:
|
||||
raise ValueError('Chapter title differs from Springer contents, review the version')
|
||||
url = ORIGIN + '/chapter/' + doi
|
||||
return {
|
||||
'source': SOURCE, 'external_id': doi, 'url': url,
|
||||
'venue': 'ECCV', 'year': 2026, 'track': 'main',
|
||||
'title': chapter['title'], 'authors': chapter['authors'], 'abstract': chapter['abstract'],
|
||||
'status': 'accepted', 'identifiers': {'doi': doi},
|
||||
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||
'evidence_url': url, 'observed_at': evidence['observed_at'],
|
||||
'raw_decision': 'Chapter published in the refereed ECCV 2026 conference proceedings.',
|
||||
'tags': [{'namespace': SOURCE + ':publication_format', 'value': 'proceedings_chapter'}],
|
||||
'metadata': {
|
||||
'book_title': BOOK_TITLE, 'book_doi': toc['book_doi'], 'lncs_volume': toc['lncs_volume'],
|
||||
'authors_citation_raw': chapter['authors_citation_raw'], 'first_page': chapter['first_page'],
|
||||
'last_page': chapter['last_page'], 'publisher_chapter_evidence': evidence,
|
||||
'publisher_contents_evidence': {'url': toc['evidence_url'], 'sha256': toc['evidence_sha256']},
|
||||
'scope': 'Refereed conference proceedings, LNCS 17001–17078.',
|
||||
'original_decision_signature_verified': False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def collect(out, *, toc_snapshots, chapter_snapshots):
|
||||
contents = publisher_inventory(toc_snapshots)
|
||||
chapters = snapshot_index(chapter_snapshots)
|
||||
urls = {ORIGIN + '/chapter/' + doi: doi for doi in contents}
|
||||
if not chapters or not chapters.keys() <= urls.keys():
|
||||
raise ValueError('No chapter snapshots or chapter outside reviewed ECCV proceedings')
|
||||
rows, raw = [], {}
|
||||
for url in sorted(chapters):
|
||||
doi = urls[url]
|
||||
body, evidence = read_page(chapters, url)
|
||||
chapter = parse_chapter(body.decode(), doi, book_title=BOOK_TITLE, publication_year=2026)
|
||||
rows.append(publisher_row(chapter, contents[doi], evidence))
|
||||
raw[evidence['file']] = body
|
||||
out = Path(out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
lock_path = out / 'eccv2026-publications.collect.lock'
|
||||
filename = 'eccv2026-springer-main.jsonl'
|
||||
with lock_path.open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if any(p != lock_path for p in out.iterdir()):
|
||||
raise FileExistsError('Existing output must be inspected, not overwritten')
|
||||
with tempfile.TemporaryDirectory(prefix='.eccv2026-publications-', dir=out) as temporary:
|
||||
stage = Path(temporary)
|
||||
(stage / 'evidence').mkdir()
|
||||
for path, body in raw.items():
|
||||
(stage / path).write_bytes(body)
|
||||
data = ''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in rows).encode()
|
||||
(stage / filename).write_bytes(data)
|
||||
manifest = {
|
||||
'source': SOURCE, 'venue': 'ECCV', 'year': 2026, 'track': 'main', 'parser_version': 1,
|
||||
'file': str(out / filename), 'file_sha256': fingerprint(data), 'count': len(rows),
|
||||
'list_url': LIST_URL, 'source_expected_count': PUBLICATIONS, 'status': 'partial',
|
||||
'missing_abstract_count': sum(not r['abstract'] for r in rows),
|
||||
}
|
||||
report = {
|
||||
'status': 'collected_pending_identity_review', 'records': len(rows), 'files': [manifest],
|
||||
'toc_chapters': len(contents), 'chapter_details_missing': len(contents) - len(rows),
|
||||
'database_written': False,
|
||||
}
|
||||
metadata = Path(filename).with_suffix('.manifest.json')
|
||||
(stage / metadata).write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
(stage / 'collection-report.json').write_text(json.dumps(report, indent=2) + '\n')
|
||||
for item in ('evidence', filename, metadata, 'collection-report.json'):
|
||||
(stage / item).replace(out / item)
|
||||
return report
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog)
|
||||
parser.add_argument('--toc-snapshots', required=True)
|
||||
parser.add_argument('--chapter-snapshots', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, toc_snapshots=args.toc_snapshots,
|
||||
chapter_snapshots=args.chapter_snapshots), ensure_ascii=False))
|
||||
@@ -1,16 +1,14 @@
|
||||
"""Reviewed postconference selected volumes from frozen public HTML metadata."""
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from paper_atlas.db.submissions import timestamp
|
||||
from paper_atlas.io.html import Document, clean
|
||||
from paper_atlas.io.springer import parse_chapter as parse_citation_chapter, read_page, snapshot_index
|
||||
|
||||
SOURCE = 'springer_selected_volume'
|
||||
BOOK = '10.1007/978-1-4612-2660-4'
|
||||
@@ -26,50 +24,6 @@ SCOPE = {
|
||||
}
|
||||
|
||||
|
||||
def snapshot_index(root):
|
||||
root = Path(root).resolve()
|
||||
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
|
||||
index = {}
|
||||
for receipt in receipts:
|
||||
url = receipt['url']
|
||||
if url in index:
|
||||
raise ValueError('Duplicate snapshot URL')
|
||||
index[url] = (root, receipt)
|
||||
return index
|
||||
|
||||
|
||||
def read_page(index, url):
|
||||
if url not in index:
|
||||
raise ValueError('Missing required public HTML snapshot: '+url)
|
||||
root, receipt = index[url]
|
||||
filename = Path(receipt['file'])
|
||||
path = root/filename
|
||||
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
|
||||
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
|
||||
query = parse_qs(final.query, keep_blank_values=True)
|
||||
original_query = parse_qs(requested.query, keep_blank_values=True)
|
||||
# Springer returns the public metadata with an explicit cookie diagnostic.
|
||||
# Retain that final URL verbatim; accept no other redirect or changed page.
|
||||
if query != original_query and requested.netloc == 'link.springer.com':
|
||||
if (query.pop('error', None) != ['cookies_not_supported']
|
||||
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
|
||||
raise ValueError('Unreviewed Springer redirect query')
|
||||
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
|
||||
or receipt.get('http_status') != 200
|
||||
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
|
||||
or final.fragment or query != original_query
|
||||
or 'text/html' not in content_type.lower()):
|
||||
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
|
||||
raw = path.read_bytes()
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
|
||||
raise ValueError('Snapshot checksum or size differs')
|
||||
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
|
||||
'file':'evidence/'+digest+'.html', 'http_status':200,
|
||||
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
|
||||
return raw, evidence
|
||||
|
||||
|
||||
def parse_toc(html, *, excluded_materials=None):
|
||||
entries = {}
|
||||
for node in Document(html).root.walk():
|
||||
@@ -97,40 +51,8 @@ def parse_toc(html, *, excluded_materials=None):
|
||||
|
||||
|
||||
def parse_chapter(html, doi):
|
||||
nodes = list(Document(html).root.walk())
|
||||
meta = defaultdict(list)
|
||||
for node in nodes:
|
||||
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
|
||||
meta[node.attrs['name']].append(node.attrs.get('content', ''))
|
||||
def one(key):
|
||||
values = meta[key]
|
||||
if len(values) != 1 or not values[0]:
|
||||
raise ValueError('Missing or ambiguous citation field: '+key)
|
||||
return values[0]
|
||||
if (one('citation_doi') != doi or one('citation_inbook_title') != SCOPE['book_title']
|
||||
or one('citation_publication_date') != str(SCOPE['publication_year'])):
|
||||
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
|
||||
raw_authors = meta['citation_author']
|
||||
if not raw_authors:
|
||||
raise ValueError('Missing named chapter authors')
|
||||
authors = []
|
||||
for raw in raw_authors:
|
||||
if raw.count(',') != 1:
|
||||
raise ValueError('Unreviewed citation author format')
|
||||
family, given = [clean(s) for s in raw.split(',', 1)]
|
||||
if not family or not given:
|
||||
raise ValueError('Incomplete citation author')
|
||||
authors.append({'given':given, 'family':family})
|
||||
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
||||
if len(abstract_nodes) > 1:
|
||||
raise ValueError('Ambiguous abstract section')
|
||||
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
|
||||
first, last = one('citation_firstpage'), one('citation_lastpage')
|
||||
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
|
||||
raise ValueError('Invalid chapter pages')
|
||||
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
|
||||
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
|
||||
'first_page':first, 'last_page':last}
|
||||
return parse_citation_chapter(html, doi, book_title=SCOPE['book_title'],
|
||||
publication_year=SCOPE['publication_year'])
|
||||
|
||||
|
||||
def record_for(chapter, evidence, scope_evidence):
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Frozen Springer public HTML receipts and chapter citation metadata."""
|
||||
from collections import defaultdict
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from paper_atlas.db.submissions import timestamp
|
||||
from paper_atlas.io.html import Document, clean
|
||||
|
||||
|
||||
def snapshot_index(root):
|
||||
root = Path(root).resolve()
|
||||
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
|
||||
index = {}
|
||||
for receipt in receipts:
|
||||
url = receipt['url']
|
||||
if url in index:
|
||||
raise ValueError('Duplicate snapshot URL')
|
||||
index[url] = (root, receipt)
|
||||
return index
|
||||
|
||||
|
||||
def read_page(index, url):
|
||||
if url not in index:
|
||||
raise ValueError('Missing required public HTML snapshot: '+url)
|
||||
root, receipt = index[url]
|
||||
filename = Path(receipt['file'])
|
||||
path = root/filename
|
||||
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
|
||||
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
|
||||
query = parse_qs(final.query, keep_blank_values=True)
|
||||
original_query = parse_qs(requested.query, keep_blank_values=True)
|
||||
# Springer returns the public metadata with an explicit cookie diagnostic.
|
||||
# Retain that final URL verbatim; accept no other redirect or changed page.
|
||||
if query != original_query and requested.netloc == 'link.springer.com':
|
||||
if (query.pop('error', None) != ['cookies_not_supported']
|
||||
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
|
||||
raise ValueError('Unreviewed Springer redirect query')
|
||||
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
|
||||
or receipt.get('http_status') != 200
|
||||
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
|
||||
or final.fragment or query != original_query
|
||||
or 'text/html' not in content_type.lower()):
|
||||
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
|
||||
raw = path.read_bytes()
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
|
||||
raise ValueError('Snapshot checksum or size differs')
|
||||
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
|
||||
'file':'evidence/'+digest+'.html', 'http_status':200,
|
||||
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
|
||||
return raw, evidence
|
||||
|
||||
|
||||
def parse_chapter(html, doi, *, book_title, publication_year):
|
||||
nodes = list(Document(html).root.walk())
|
||||
meta = defaultdict(list)
|
||||
for node in nodes:
|
||||
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
|
||||
meta[node.attrs['name']].append(node.attrs.get('content', ''))
|
||||
def one(key):
|
||||
values = meta[key]
|
||||
if len(values) != 1 or not values[0]:
|
||||
raise ValueError('Missing or ambiguous citation field: '+key)
|
||||
return values[0]
|
||||
if (one('citation_doi') != doi or one('citation_inbook_title') != book_title
|
||||
or one('citation_publication_date') != str(publication_year)):
|
||||
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
|
||||
raw_authors = meta['citation_author']
|
||||
if not raw_authors:
|
||||
raise ValueError('Missing named chapter authors')
|
||||
authors = []
|
||||
for raw in raw_authors:
|
||||
if raw.count(',') != 1:
|
||||
raise ValueError('Unreviewed citation author format')
|
||||
family, given = [clean(s) for s in raw.split(',', 1)]
|
||||
if not family or not given:
|
||||
raise ValueError('Incomplete citation author')
|
||||
authors.append({'given':given, 'family':family})
|
||||
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
||||
if len(abstract_nodes) > 1:
|
||||
raise ValueError('Ambiguous abstract section')
|
||||
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
|
||||
first, last = one('citation_firstpage'), one('citation_lastpage')
|
||||
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
|
||||
raise ValueError('Invalid chapter pages')
|
||||
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
|
||||
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
|
||||
'first_page':first, 'last_page':last}
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Checks for published ECCV metadata and preserved submission histories."""
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors.eccv2026_publications import BOOK_TITLE, SOURCE, collect
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.records import ingest_file
|
||||
|
||||
|
||||
FIRST_BOOK = '10.1007/978-3-032-10000-0'
|
||||
FIRST_DOI = FIRST_BOOK + '_1'
|
||||
|
||||
|
||||
def snapshots(root):
|
||||
toc, chapters = root / 'toc', root / 'chapters'
|
||||
toc.mkdir()
|
||||
chapters.mkdir()
|
||||
receipts, volumes = [], {}
|
||||
for offset in range(78):
|
||||
volume = 17001 + offset
|
||||
book = f'10.1007/978-3-032-{10000 + offset}-0'
|
||||
book_url = 'https://link.springer.com/book/' + book
|
||||
count = 37 if offset < 27 else 36
|
||||
files, chapter_dois = {}, []
|
||||
for page, indices in ((1, range(1, 20)), (2, range(20, count + 1))):
|
||||
filename = f'{volume}-{page}.html'
|
||||
html = f'<h1>{BOOK_TITLE}</h1><p>Malmö (LNCS, volume {volume})</p>'
|
||||
html += '<li data-test="chapter"><h3>Front Matter</h3></li>'
|
||||
for i in indices:
|
||||
doi = f'{book}_{i}'
|
||||
html += f'<li data-test="chapter"><h3><a href="/chapter/{doi}">Title {volume}-{i}</a></h3></li>'
|
||||
chapter_dois.append(doi)
|
||||
raw = html.encode()
|
||||
(toc / filename).write_bytes(raw)
|
||||
files[str(page)] = filename
|
||||
receipts.append({'file': filename, 'status': 200, 'bytes': len(raw), 'sha256': sha256(raw).hexdigest(),
|
||||
'url': book_url + ('?page=2' if page == 2 else '')})
|
||||
volumes[str(volume)] = {'book': book_url, 'files': files, 'chapter_dois': chapter_dois}
|
||||
data = ''.join(json.dumps(row) + '\n' for row in receipts).encode()
|
||||
(toc / 'receipts.jsonl').write_bytes(data)
|
||||
checkpoint = json.dumps(volumes).encode()
|
||||
(toc / 'volume-checkpoint.json').write_bytes(checkpoint)
|
||||
(toc / 'summary.json').write_text(json.dumps({
|
||||
'raw_receipts_sha256': sha256(data).hexdigest(),
|
||||
'volume_checkpoint_sha256': sha256(checkpoint).hexdigest(),
|
||||
'chapter_link_count': 2835,
|
||||
}))
|
||||
url = 'https://link.springer.com/chapter/' + FIRST_DOI
|
||||
citation = {
|
||||
'citation_doi': FIRST_DOI, 'citation_title': 'Title 17001-1',
|
||||
'citation_inbook_title': BOOK_TITLE, 'citation_publication_date': '2026',
|
||||
'citation_firstpage': '1', 'citation_lastpage': '10',
|
||||
'citation_author': 'Chen, Lei',
|
||||
}
|
||||
body = ''.join(f'<meta name="{k}" content="{v}">' for k, v in citation.items())
|
||||
body += '<div id="Abs1-content"><p>Published abstract.</p></div>'
|
||||
raw = body.encode()
|
||||
(chapters / 'first.html').write_bytes(raw)
|
||||
(chapters / 'receipts.json').write_text(json.dumps({'responses': [{
|
||||
'url': url, 'final_url': url, 'file': 'first.html', 'at': '2026-11-18T00:00:00Z',
|
||||
'http_status': 200, 'bytes': len(raw), 'sha256': sha256(raw).hexdigest(),
|
||||
'headers': {'Content-Type': 'text/html'},
|
||||
}]}))
|
||||
return {'toc_snapshots': toc, 'chapter_snapshots': chapters}
|
||||
|
||||
|
||||
class ECCV2026PublicationsTests(unittest.TestCase):
|
||||
def test_frozen_snapshot_and_existing_history(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
source = snapshots(root)
|
||||
report = collect(root / 'out', **source)
|
||||
self.assertEqual((report['toc_chapters'], report['chapter_details_missing']), (2835, 2834))
|
||||
manifest, = report['files']
|
||||
self.assertEqual((manifest['track'], manifest['status']), ('main', 'partial'))
|
||||
row = json.loads(Path(manifest['file']).read_text())
|
||||
self.assertEqual(row['identifiers'], {'doi': FIRST_DOI})
|
||||
self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': FIRST_DOI}])
|
||||
self.assertEqual(row['authors'], [{'family': 'Chen', 'given': 'Lei'}])
|
||||
self.assertEqual(row['abstract'], 'Published abstract.')
|
||||
self.assertEqual(row['metadata']['lncs_volume'], 17001)
|
||||
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||
detail = row['metadata']['publisher_chapter_evidence']
|
||||
self.assertEqual(sha256((root / 'out' / detail['file']).read_bytes()).hexdigest(), detail['sha256'])
|
||||
db = root / 'example.sqlite'
|
||||
store = Store(db)
|
||||
pid, rid = store.record('prior', 'work', 'https://example.org/work',
|
||||
row['title'], 'Original preprint abstract.', row['authors'],
|
||||
{'arxiv': '2601.00001'})
|
||||
store.decision(pid, rid, 'ICLR', 2026, 'main', 'work', 'https://example.org/decision',
|
||||
'rejected', 'Original ICLR result.')
|
||||
store.conn.commit()
|
||||
store.conn.close()
|
||||
row['identifiers']['arxiv'] = '2601.00001'
|
||||
path = Path(manifest['file'])
|
||||
raw = (json.dumps(row) + '\n').encode()
|
||||
path.write_bytes(raw)
|
||||
manifest['file_sha256'] = sha256(raw).hexdigest()
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
|
||||
ingest_file(db, path)
|
||||
ingest_file(db, path)
|
||||
with sqlite3.connect(db) as conn:
|
||||
self.assertEqual(conn.execute('SELECT id,abstract FROM papers').fetchall(),
|
||||
[(pid, 'Original preprint abstract.')])
|
||||
self.assertEqual(conn.execute('SELECT status FROM decisions ORDER BY id').fetchall(),
|
||||
[('rejected',), ('accepted',)])
|
||||
self.assertEqual(conn.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),
|
||||
[('publication_doi', FIRST_DOI)])
|
||||
self.assertEqual(conn.execute('SELECT name FROM sources ORDER BY id').fetchall(), [('prior',), (SOURCE,)])
|
||||
with self.assertRaises(FileExistsError):
|
||||
collect(root / 'out', **source)
|
||||
|
||||
def test_changed_contents_receipt_stops_before_output(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
source = snapshots(root)
|
||||
(source['toc_snapshots'] / '17001-1.html').write_text('unverified bytes')
|
||||
with self.assertRaisesRegex(ValueError, 'contents bytes disagree'):
|
||||
collect(root / 'out', **source)
|
||||
self.assertFalse((root / 'out').exists())
|
||||
Reference in New Issue
Block a user