diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 204e40a..2d4b3f9 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -48,6 +48,7 @@ jobs: outside.mkdir() subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True) diff --git a/README.md b/README.md index f6d8487..234e8d0 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,7 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目 开发通过工作分支和 PR 交付,由用户手动合并。 +早期WWW档案的可重放元数据采集、论文/海报分轨与待核查条目见 [WWW档案采集](docs/www-archive.md)。 历史联合会议的既有届次归属可通过 [`maintain joint-editions`](docs/joint-editions.md) 先只读规划,再按计划哈希执行;保留论文与投稿决定来源,不重导入。 同年多届的 `event_key`、显式迁移、旧文件刷新与查询边界见 [会议事件身份](docs/event-identity.md)。 研究类型与自定义标签可通过 [`annotate`](docs/annotations.md) 显式初始化、添加和撤回;两者独立查询,保留证据及完整更正历史,不从会议轨道推断研究类型。 diff --git a/docs/coverage/www-1995-posters-reviewed-2026-11-09.json b/docs/coverage/www-1995-posters-reviewed-2026-11-09.json new file mode 100644 index 0000000..3afe54b --- /dev/null +++ b/docs/coverage/www-1995-posters-reviewed-2026-11-09.json @@ -0,0 +1,201 @@ +{ + "format": 1, + "source_cache_sha256": "8a65562f3490497cada2919ec77827134beabd5ec9c3690dec3625e2318aa54e", + "records": [ + { + "entry_id": "45", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/45/index.html", + "individual_sha256": "83bede6afc9a7039fae2dd1f28274a394418b34f5d5632590b913617580bda1a", + "title": "WISE - World Wide Information System for R&D; Efforts", + "authors": [ + "Fritz Loseries", + "Veronika Samara", + "Bodo Urban" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "16", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/16/index.html", + "individual_sha256": "cf76da903eaeaa27d19857fdaa13220f6821559bf4ac22844fae10abbaa6f3f3", + "title": "The World Wide Web as a Tool for Collaborative Inter-Agency Projects: Creation of an Environmental Metadata Database for the Analysis of the Venice Lagoon Ecosystem.", + "authors": [ + "Mirko Labbri" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "17", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/17/index.html", + "individual_sha256": "0578fec9c82e8f2f98c4de0f1bcc44011ebb84c53353fdcc423a69cf70673bc2", + "title": "NetRepreneur", + "authors": [ + "Kari-Pekka Turtiainen", + "Antti Auer" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "29", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/29/index.html", + "individual_sha256": "624ad3f685de1163411e33e90488975d848a63243fd7ffbed0862c6fb58c4140", + "title": "Mosaic TueV 2.4.3: Multilingual GUI and Improvements towards User Friendliness", + "authors": [ + "Mike Bretz", + "Uwe Koch", + "Paul de Bra" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "13", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/13/index.html", + "individual_sha256": "02409321a3403a94d8362a9862f4960386ea9c344fb47f4bcdf9eae128e8b46e", + "title": "Using a WWW Browser as an Alternative User Interface for Interactive Applications", + "authors": [ + "Frank Lonczewski" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "36", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/36/index.html", + "individual_sha256": "ed9dada4ea5903cb8f0a4e75f2958416a6e32647fbf3530e58edd9abf26070ce", + "title": "HyperSource: A Hypermedia Program Development And Documentation System", + "authors": [ + "Jan Oliver Borchers" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "57", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/57/index.html", + "individual_sha256": "3ad4206ad28a19823172cd2845f636fe7a461c251940e9a93deb8f6484b8a984", + "title": "Progressive Image Transmission Capability (PROTRAC) on the World Wide Web", + "authors": [ + "Vojislav Lalich-Petrich", + "Gaurav Bhatia", + "Larry Davis" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "38", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/38/index.html", + "individual_sha256": "940370acc5e6a954b197053fe451e466a904d1f0ac82c8fe4e6ca455faf2c927", + "title": "Measuring the Web with Lycos", + "authors": [ + "Michael L. Mauldin" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "Official abstract listing has no abstract. Canonical HTML has body sections but no explicit abstract; body is not substituted as a summary." + }, + { + "entry_id": "05", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/05/index.html", + "individual_sha256": "1858be87d6e7f47476e1cb8fb4793c0fce3b8a6ff653bcfd4a35089e0520f177", + "title": "Virtual Places - Adding people to the web", + "authors": [ + "Yosi Mass" + ], + "author_source": "author_index", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "Official abstract listing and canonical HTML have no abstract. Canonical page identifies a contact and links PostScript; no full-text download. Author identity comes from author index, not inferred from contact role." + }, + { + "entry_id": "37", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/37/index.html", + "individual_sha256": "b700a0a51b7a82a5f88373ae0283bb369d1cf6f455acff7d91cd7313fa8b3adf", + "title": "The Internet Public Library: Serving a Global Community", + "authors": [ + "Susan Schweitzer" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "41", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/41/index.html", + "individual_sha256": "397b474109f9c1240a77d0f490f2b02dd35fae93febbb7397d1b2a31759a8ec1", + "title": "Graphical Browsing in the WorldWideWeb and the MultiMedia Forum", + "authors": [ + "Rainer Pagé", + "Klaus Süllow" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "47", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/47/index.html", + "individual_sha256": "52eb4bfdd658730a222c98170b72dabf9cbcc5ffd2ada1e900b826baf2b02833", + "title": "Interoperating Advanced Hypermedia Systems with WWW", + "authors": [ + "Jian Zhang", + "Martin Fruehauf", + "Thomas Kirste" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "50", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/50/index.html", + "individual_sha256": "8dc8b0561bd8b1b106c5b209a973c695c3cc9df5288450d4a01fe32541476b7e", + "title": "Web Enabled Agile Manufacturing", + "authors": [ + "S. V. Iyer", + "T. Singh", + "U. Ravaioli", + "R. E. DeVor", + "S. G. Kapoor", + "G. Veknatasubramian" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "55", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/55/index.html", + "individual_sha256": "21780cd4047f513e8c9aeced5247002e27cf71719f08de85c6531b57a9e5058d", + "title": "Weblets: Fundamental Building Blocks for WWW Tools", + "authors": [ + "Thomas J. Watt Jr." + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "59", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/59/index.html", + "individual_sha256": "fed4cc9dcaee350a12c5e4b6fa9c99be9bead25a81632512d31566400d3d0fad", + "title": "Usenet and the World-Wide Web", + "authors": [ + "M. L. Grant" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + } + ] +} diff --git a/docs/www-archive.md b/docs/www-archive.md new file mode 100644 index 0000000..2a91c9e --- /dev/null +++ b/docs/www-archive.md @@ -0,0 +1,111 @@ +# Early WWW archive collector + +`paper-atlas collect www-archive` is a maintained metadata adapter. It reads official +IW3C2 archive HTML, keeps exact source hashes, and separates importable rows from +program entries that still need review. It fetches no PDF, follows no paper-body or +reference link, and makes no database connection. + +For a fresh source snapshot on the server: + +```sh +paper-atlas collect www-archive \ + --http-cache /data/paper-index/cache/www-archive \ + --source-cache /data/paper-index/collected/www-archive-source-v2 \ + --out /data/paper-index/collected/www-archive-normalized-v2 +``` + +For reproducible offline replay of the already verified snapshot: + +```sh +paper-atlas collect www-archive \ + --source-cache /data/paper-index/collected/www-early-archive-source-v1 \ + --source-cache-sha256 c9388ccef44486e0684c9ee9e8620a465cdc4574e6bc9ea9391d743e01a3312c \ + --out /data/paper-index/collected/www-archive-normalized-v1 +``` + +Use a new output directory for another batch. Existing snapshots and normalized +batches are never overwritten. The manifest hash binds every source URL, byte count, +response hash and cache filename. Plain files must remain directly within the cache; +symlinks, escaping paths, duplicate URLs, non-200 responses and altered bytes fail. +The command uses UTF-8 where valid and explicitly records an ISO-8859-1 fallback for +older pages; original bytes remain authoritative for encoding corrections. + +## Evidence and outputs + +- WWW3 Spring 1995 papers use `event_key=www-1995-spring`, `track=main`. The author, + session and abstract indexes must have the same nonempty entry-ID set. The ID is + local to this exact event and proceedings section, never a DOI or OpenReview ID. +- Posters remain `track=poster`. An abstract without a linked author-index entry + stays in review even when another index has a placeholder `#` link for it. +- Paper URLs must agree between author and session indexes. A malformed whitespace + URL in the abstract list may use the exact event/entry-ID match only when both + independent indexes corroborate the same clean URL. The original href is retained. +- Canonical names come from the explicitly labeled author index. The source lists + them alphabetically; this is **not** original byline order. Both the raw index names + and the full abstract-page byline/affiliation text remain in metadata. No person ID + or cross-source identity is inferred from a name. Conflicting titles or missing + abstracts stay in review. Topic keywords are not treated as an abstract. +- WWW1 1994 formal presentations and WWW2 1994 mixed categories remain review + candidates. The first list contains uncertain business-track annotations and a + count discrepancy; the second contains panels, tutorials, products and other + activities. Neither presentation listing nor a missing full-text link establishes + a regular-paper acceptance or a rejection. Raw category/title/name claims are kept. + +The main and poster JSONL files have independent manifests with explicit event key, +track, count and SHA256. `source_listing_count` records the full recognized abstract +listing, while `held_entry_count` records excluded entries; neither is a ready-row +count. Their status stays `partial`: a verified linked subset does +not prove complete event coverage. `review-candidates.jsonl` contains nested +candidates and reasons, with no importable decision status. It is excluded from +`import-queue.json`. `report.json` accounts for ready/held records and every source +used. Newlines delimit JSONL; Unicode U+2028/U+2029 inside metadata are not line breaks. +The output is published only after all required source and scope checks pass. + +## Before importing + +This independent collector branch starts from main; it does not include or silently +install the event-aware writer. Its output requires the reviewed integrated runtime +from PR31 (or a main revision containing that implementation). The production event +schema migration has been independently accepted, preserving existing edition1212 +and its 59 Fall submissions; that is not an import of the Spring proceedings. + +Inspect held entries and original author names, preflight every ready row against +current PostgreSQL stable identifiers and candidate identities, and verify the +source/year/track/event coverage before running `import records`. When a ready subset +excludes held entries, preserve the independently known listing total in the existing +incremental-coverage protocol: bind a freshly checked parent count and expected count, +and set the full expected count from this manifest. A new source uses previous count +0 and previous expected count null; an existing source requires its actual current +values. These live parent values are not guessed by the offline collector. Do not automatically +run this queue against an older writer. Preserve extended versions and independent +submission/decision evidence, and keep unresolved identities held. A normalization +report or passing tests alone do not establish production import, complete historical +coverage or final global deduplication. + +Initial fixed-cache replay (64ced7a, CI1057): 38 main papers and38 posters ready; +372 entries held (50 first-conference fields,307 mixed program entries,15 posters). +Independent report44dfd070 verified source hashes,counts and author evidence, finding +one existing exact-title candidate. That publication is explicitly linked to +arXiv1811.12161 by its WWW1995/journal-reference declaration; arXiv1810.07232 is a +different1997 invited chapter and remains separate despite the overlap note. This +is a preparation observation, not evidence of completed production import or final +global deduplication. + +## 以单篇官方页面核实待处理 poster + +作者索引漏链接、摘要目录使用短标题或标题存在拼写差异时,不放宽全局标题匹配。可以先取得单篇官方 HTML,人工核对 H1、署名区与原目录同一 entry URL,再提供带 SHA256 的逐条核查文件: + +```sh +paper-atlas collect www-archive \ + --source-cache /data/paper-index/collected/www-poster-reviewed-source-v2 \ + --source-cache-sha256 8a65562f3490497cada2919ec77827134beabd5ec9c3690dec3625e2318aa54e \ + --reviewed-posters docs/coverage/www-1995-posters-reviewed-2026-11-09.json \ + --reviewed-posters-sha256 <核查文件的SHA256> \ + --out <新的输出目录> +``` + +[1995 年15条poster的核查输入](coverage/www-1995-posters-reviewed-2026-11-09.json) 绑定61页来源快照:原46页和补取的15页单篇HTML。程序验证输入文件及快照哈希、单篇页面哈希、完全一致的 canonical H1、原摘要目录中的同一URL/entry ID及作者证据,不通过相似标题猜身份。规范标题采用单篇H1,摘要目录标题及作者索引标题均在 metadata 中保留。 + +`author_source=canonical_byline` 表示名字出现在H1之后、摘要/关键词或下一节标题之前的署名区域,由逐条核查明确选出,不从全文引用或机构猜作者;保留原始缩写、顺序、重音和拼写。只有 `author_source=author_index` 时使用原作者索引顺序并逐项核对;05的单篇页面写的是联系人,作者身份使用原作者索引证据,不由联系人角色推断。核查输入是审阅后的数据声明,程序的字符串/哈希检查不代替人工识别署名语义。 + +38和05的来源没有可明确引用的摘要。保留 `abstract=null` 和逐条缺失原因,manifest准确报告 missing_abstract_count,继续标partial;正文、联系方式及PostScript链接不当作摘要,也不下载正文。缺摘要不等于拒稿或未发表。本批输出仍包含原先已验收记录,正式导入前必须只取新增external ID,检查实时覆盖父记录并按incremental模式准备,不能直接重导入全部输出。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index fefb48c..384c3d8 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -23,6 +23,7 @@ COMMANDS = { ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'), + ('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'), ('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'), ('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'), ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'), diff --git a/paper_atlas/collectors/www_archive.py b/paper_atlas/collectors/www_archive.py new file mode 100644 index 0000000..6eec958 --- /dev/null +++ b/paper_atlas/collectors/www_archive.py @@ -0,0 +1,340 @@ +"""Reproducible early WWW metadata; unresolved program entries stay outside imports.""" +import argparse +from collections import defaultdict +import hashlib +from html.parser import HTMLParser +import json +from pathlib import Path +import re +import tempfile +from urllib.parse import urljoin, urlsplit + +from paper_atlas.io.html import Document, clean +from paper_atlas.io.http import fetch + +ROOT = 'https://archives.iw3c2.org/' +SPRING = ROOT+'www3/proceedings/' +FALL = ROOT+'www2/Proceedings/WWW2_Proceedings.html' +SEEDS = [ROOT+'www1/PrelimProcs.html', ROOT+'www1/Proceedings.html', FALL, + SPRING+'proceedings.html', *[SPRING+'papers/'+x+'.html' for x in ('author','session','abstract')], + *[SPRING+'posters/'+x+'.html' for x in ('author','abstract')]] + + +def decode(raw): + try: + return raw.decode('utf-8-sig'), 'utf-8' + except UnicodeDecodeError: + return raw.decode('iso-8859-1'), 'iso-8859-1 fallback; original bytes retained' + + +def text(html): + return re.sub(r'[ \t\r\n\f\v]+',' ',Document(html).root.text()).strip(' \t\r\n\f\v') + + +def anchors(html): + return [(n.attrs.get('href',''),clean(n.text())) for n in Document(html).root.walk() if n.tag=='a'] + + +def source_url(url): + p=urlsplit(url) + if p.scheme!='https' or p.netloc!='archives.iw3c2.org' or p.query or p.fragment or not p.path.endswith('.html'): + raise ValueError('Only exact official archive HTML source URLs are allowed') + return url + + +class Sources: + def __init__(self, directory, expected_sha256): + self.root=Path(directory).resolve();manifest=self.root/'manifest.json';raw=manifest.read_bytes() + if hashlib.sha256(raw).hexdigest()!=expected_sha256: + raise ValueError('Source cache manifest checksum mismatch') + self.rows={};self.used=set() + for row in json.loads(raw)['sources']: + url=source_url(row['url']) + if url in self.rows or row.get('status')!=200: + raise ValueError('Duplicate or incomplete source response') + p=self.root/row['file'] + if p.is_symlink() or p.parent!=self.root or not p.is_file(): + raise ValueError('Source cache paths must be plain files directly within the cache') + data=p.read_bytes() + if len(data)>2*1024**2 or len(data)!=row['bytes'] or hashlib.sha256(data).hexdigest()!=row['sha256']: + raise ValueError('Source response checksum or size mismatch') + self.rows[url]={'raw':data,'receipt':row} + + def read(self,url): + source_url(url) + if url not in self.rows: + raise ValueError('Required source missing from cache: '+url) + self.used.add(url) + return decode(self.rows[url]['raw'])[0] + + def proof(self,urls): + return [{'url':url,'sha256':self.rows[url]['receipt']['sha256'], + 'encoding':decode(self.rows[url]['raw'])[1]} for url in sorted(set(urls))] + + +def download_sources(http_cache, directory): + directory=Path(directory) + if directory.exists(): + raise ValueError('Refusing to overwrite a source snapshot') + directory.mkdir(parents=True) + responses={url:fetch(url,http_cache,2*1024**2) for url in SEEDS} + html=decode(responses[FALL])[0] + for href,label in anchors(html): + if href!='Author_Inst.html' and href.endswith('.html'): + url=source_url(urljoin(FALL,href)) + if not url.startswith(ROOT+'www2/Proceedings/'): + raise ValueError('Unexpected fall category source') + responses[url]=fetch(url,http_cache,2*1024**2) + receipts=[] + for i,(url,raw) in enumerate(sorted(responses.items())): + name='source-'+str(i).zfill(2)+'.html';(directory/name).write_bytes(raw) + receipts.append({'url':url,'status':200,'file':name,'bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()}) + manifest=directory/'manifest.json';manifest.write_text(json.dumps({'sources':receipts},indent=2)+'\n') + return hashlib.sha256(manifest.read_bytes()).hexdigest() + + +def identity(href,track): + url=urljoin(SPRING+track+'/',href);p=urlsplit(url) + match=re.fullmatch(r'/www3/proceedings/'+track+r'/(\d+)/[^?#]+',p.path) + if p.scheme=='https' and p.netloc=='archives.iw3c2.org' and not p.query and not p.fragment and match: + return match[1],url + return None + + +class AuthorIndex(HTMLParser): + def __init__(self,html,track): + super().__init__(convert_charrefs=True);self.track=track;self.name='';self.bold=None;self.link=None;self.link_text=[] + self.entries=defaultdict(lambda:{'authors':[],'urls':set(),'titles':set()});self.feed(html) + + def handle_starttag(self,tag,attrs): + if tag=='b':self.bold=[] + if tag=='a':self.link=dict(attrs).get('href');self.link_text=[] + + def handle_data(self,data): + if self.bold is not None:self.bold.append(data) + if self.link is not None:self.link_text.append(data) + + def handle_endtag(self,tag): + if tag=='b' and self.bold is not None: + self.name=clean(''.join(self.bold));self.bold=None + if tag=='a' and self.link is not None: + own=identity(self.link,self.track) + if own: + key,url=own;entry=self.entries[key];entry['urls'].add(url);entry['titles'].add(clean(''.join(self.link_text))) + if not self.name:raise ValueError('Paper link has no archive author context') + if self.name not in entry['authors']:entry['authors'].append(self.name) + self.link=None + + +def sections(html): + """Use explicit h3 boundaries; old HTML omits many p/li closing tags.""" + result=[] + for chunk in re.split(r']*>',html,flags=re.I)[1:]: + parts=re.split(r'',chunk,maxsplit=1,flags=re.I) + if len(parts)!=2:raise ValueError('Unclosed archive entry heading') + heading,body=parts + body=re.split(r'<(?:hr|h2|address)\b',body,maxsplit=1,flags=re.I)[0] + author=re.search(r']*>(.*?)',body,re.I|re.S) + paragraphs=[] + for paragraph in re.findall(r']*>(.*?)(?=|]*>(.*?)',page,re.I|re.S) + if len(headings)!=1 or text(headings[0])!=row['title']: + raise ValueError('Reviewed title differs from exact canonical H1') + header=re.split(r'',page,maxsplit=1,flags=re.I)[1] + header=re.split(r']*>',html,flags=re.I)[1:],1): + title=re.search(r'Title:\s*(.*?)',chunk,re.I|re.S) + if not title:raise ValueError('Unclassified first-conference entry') + names=re.findall(r'Author:(.*?)(?=]+>','',text(n)).strip()for n in names], + 'keywords_raw':text(re.search(r'Keywords:(.*?)(?=Keywords:(.*?)(?=]*>',html,flags=re.I)[1:],1): + fields=re.split(r']*>',chunk,maxsplit=1,flags=re.I) + title=text(fields[0]);raw_byline=text(re.split(r'Formal presentations
  1. Keywords: BUSINESS TRACK?
    Title: First contribution
    Author:Jane Doe <jane@example.org>
  2. Title:Unlinked contribution
    Author:John Roe
''', + FALL:'

Conference Tracks

  1. Agents
  2. Tutorials
', + ROOT+'www2/Proceedings/Agents.html':'

Panel

Linked talk
Jane Doe
Unlinked talk
John Roe
', + ROOT+'www2/Proceedings/Tutor.html':'

Tutorials

Training session
Teacher
'} + for track in ['papers','posters']: + self.pages[SPRING+track+'/author.html']='Doe, JaneRoe, JohnSame contribution' + self.pages[SPRING+track+'/abstract.html']='''

Abstracts

Same contribution

Same contribution

Author: Jane Doe, John Roe
Institute

Actual abstract with inline emphasis and a separator\u2028 preserved.


Another session

''' + self.pages[SPRING+'papers/session.html']='Same contribution' + + def sources(self): + rows=[] + for i,(url,html) in enumerate(sorted(self.pages.items())): + raw=html.encode()if isinstance(html,str)else html;name='source-'+str(i)+'.html';(self.cache/name).write_bytes(raw) + rows.append(dict(url=url,status=200,file=name,bytes=len(raw),sha256=hashlib.sha256(raw).hexdigest())) + p=self.cache/'manifest.json';p.write_text(json.dumps({'sources':rows}));sha=hashlib.sha256(p.read_bytes()).hexdigest() + return Sources(self.cache,sha),sha + + def test_offline_batch_separates_tracks_review_and_importable_rows(self): + _,sha=self.sources();out=self.root/'batch';report=collect(self.cache,sha,out) + self.assertEqual(report['ready'],2);self.assertEqual(report['held'],5) + queue=json.loads((out/'import-queue.json').read_text())['files'];self.assertEqual(len(queue),2) + for name in queue: + p=Path(name);rows=[json.loads(l)for l in p.read_text().split('\n') if l];self.assertEqual(len(rows),1) + row=rows[0];self.assertEqual(row['event_key'],'www-1995-spring');self.assertEqual(row['authors'],['Jane Doe','John Roe']) + self.assertIn(row['track'],['main','poster']);self.assertIn('inline emphasis',row['abstract']);self.assertNotIn('Another session',row['abstract']) + self.assertIn('\u2028',row['abstract']);manifest=json.loads(p.with_suffix('.manifest.json').read_text());self.assertEqual(manifest['file_sha256'],hashlib.sha256(p.read_bytes()).hexdigest());self.assertEqual(manifest['status'],'partial') + held=[json.loads(l)for l in(out/'review-candidates.jsonl').read_text().split('\n') if l] + self.assertTrue(all('status'not in r and 'source_evidence'in r for r in held)) + self.assertFalse(any('review-candidates'in p for p in queue)) + + def test_partial_manifest_preserves_full_listing_size_separately_from_ready_count(self): + self.pages[SPRING+'posters/abstract.html']+='

Unlinked poster

Author: Other Person

Actual abstract

' + _,sha=self.sources();out=self.root/'batch';collect(self.cache,sha,out) + manifest=json.loads((out/'www-1995-spring-poster.manifest.json').read_text()) + self.assertEqual((manifest['count'],manifest['source_listing_count'],manifest['held_entry_count']),(1,2,1)) + self.assertNotIn('coverage_previous_count',manifest) # Live parent must be independently preflighted. + + def test_malformed_href_requires_exact_id_and_two_clean_index_anchors(self): + self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('01/paper.html','01/ paper.html') + sources,_=self.sources();rows,held,_=spring_rows(sources,'papers');self.assertEqual(held,[]) + self.assertEqual(rows[0]['url'],SPRING+'papers/01/paper.html');self.assertTrue(rows[0]['metadata']['url_corroborated_by_author_and_session']) + self.assertIn(' ',rows[0]['metadata']['abstract_href_raw']) + + def test_main_index_disagreement_fails_without_publishing_batch(self): + self.pages[SPRING+'papers/session.html']='Other work';_,sha=self.sources();out=self.root/'batch' + with self.assertRaisesRegex(ValueError,'ID sets disagree'):collect(self.cache,sha,out) + self.assertFalse(out.exists()) + + def test_same_entry_id_with_conflicting_clean_url_is_rejected(self): + self.pages[SPRING+'papers/session.html']='Same contribution';sources,_=self.sources() + with self.assertRaisesRegex(ValueError,'URLs disagree'):spring_rows(sources,'papers') + + def test_poster_without_linked_author_entry_is_held_even_with_abstract(self): + self.pages[SPRING+'posters/author.html']='Doe, JaneSame contribution';sources,_=self.sources();rows,held,counts=spring_rows(sources,'posters') + self.assertEqual(rows,[]);self.assertEqual(counts['abstract_entries'],1);self.assertEqual(held[0]['reason'],'abstract_entry_missing_linked_author_index') + + def test_unlinked_and_panel_entries_never_become_acceptance_or_rejection(self): + sources,_=self.sources();held,counts=early_review(sources);self.assertEqual(counts,{'first_conference_fields':2,'fall_categories':2,'fall_program_entries':3}) + self.assertTrue(any(r['candidate']['title']=='Unlinked talk'for r in held));self.assertTrue(all('status'not in r for r in held)) + self.assertEqual(held[0]['candidate']['authors_raw'],['Jane Doe']) + + def test_title_disagreement_goes_to_review_with_source_claims(self): + self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('Same contribution','Different contribution');sources,_=self.sources();rows,held,_=spring_rows(sources,'papers') + self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'abstract_and_author_index_titles_disagree') + + def test_cache_tampering_and_manifest_hash_mismatch_fail_closed(self): + _,sha=self.sources() + with self.assertRaisesRegex(ValueError,'manifest checksum'):Sources(self.cache,'0'*64) + p=next(self.cache.glob('source-*.html'));p.write_bytes(p.read_bytes()+b'x') + with self.assertRaisesRegex(ValueError,'checksum or size'):Sources(self.cache,sha) + + def test_cache_cannot_read_paths_outside_snapshot(self): + _,sha=self.sources();p=self.cache/'manifest.json';d=json.loads(p.read_text());d['sources'][0]['file']='../outside.html';p.write_text(json.dumps(d)) + with self.assertRaisesRegex(ValueError,'plain files'):Sources(self.cache,hashlib.sha256(p.read_bytes()).hexdigest()) + + def test_keywords_only_is_missing_abstract_not_invented_summary(self): + self.pages[SPRING+'papers/abstract.html']='

Same contribution

Author: Jane Doe

Keywords: networking, hypertext

';sources,_=self.sources();rows,held,_=spring_rows(sources,'papers') + self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'missing_abstract') + + def test_old_encoding_and_duplicate_abstract_identity(self): + self.assertEqual(decode(b'M\xfcller')[0],'Müller') + self.pages[SPRING+'papers/abstract.html']*=2;sources,_=self.sources() + with self.assertRaisesRegex(ValueError,'Duplicate abstract'):spring_rows(sources,'papers') + + def test_existing_output_is_never_overwritten(self): + _,sha=self.sources();out=self.root/'batch';out.mkdir();(out/'keep').write_text('original') + with self.assertRaisesRegex(ValueError,'overwrite'):collect(self.cache,sha,out) + self.assertEqual((out/'keep').read_text(),'original') + + def review(self,*,authors=None,author_source='canonical_byline',missing='',page=None): + url=SPRING+'posters/01/paper.html' + self.pages[url]=page or '

Canonical full title

Jane Doe, John Roe

Keywords: web

Abstract

Body

' + _,sha=self.sources();raw=self.pages[url].encode()if isinstance(self.pages[url],str)else self.pages[url] + row=dict(entry_id='01',individual_url=url,individual_sha256=hashlib.sha256(raw).hexdigest(), + title='Canonical full title',authors=authors or ['Jane Doe','John Roe'],author_source=author_source, + reason='Reviewed exact canonical entry and its explicit authorship; retain original title variants',missing_abstract_reason=missing) + p=self.root/'review.json';p.write_text(json.dumps({'format':1,'source_cache_sha256':sha,'records':[row]}));return sha,p,hashlib.sha256(p.read_bytes()).hexdigest() + + def test_reviewed_variant_keeps_primary_title_and_all_old_title_claims(self): + sha,p,h=self.review();out=self.root/'reviewed';r=collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h) + row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['title'],'Canonical full title') + self.assertEqual(row['metadata']['abstract_heading_title_raw'],'Same contribution');self.assertEqual(row['metadata']['author_index_titles_raw'],['Same contribution']) + self.assertEqual(len(row['metadata']['source_evidence']),3);self.assertEqual(r['reviewed_posters_sha256'],h) + self.assertEqual(row['external_id'],'www3:posters:01');self.assertEqual(row['status'],'accepted') + + def test_canonical_byline_can_resolve_unlinked_author_index_without_guessing(self): + self.pages[SPRING+'posters/author.html']='Doe, JaneBroken index link' + sha,p,h=self.review();out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h) + row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['authors'],['Jane Doe','John Roe']) + self.assertEqual(row['metadata']['author_index_raw'],[]);self.assertEqual(row['metadata']['author_order'],'canonical_byline') + + def test_reviewed_missing_abstract_is_explicit_partial_not_body_summary(self): + self.pages[SPRING+'posters/abstract.html']='

Same contribution

Author: Jane Doe

Keywords: web

' + sha,p,h=self.review(missing='Neither listing nor canonical page supplies an explicit abstract') + out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h) + row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());m=json.loads((out/'www-1995-spring-poster.manifest.json').read_text()) + self.assertIsNone(row['abstract']);self.assertEqual((m['missing_abstract_count'],m['status']),(1,'partial')) + sha,p,h=self.review() + with self.assertRaisesRegex(ValueError,'Missing abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + self.assertFalse((self.root/'bad').exists()) + + def test_reviewed_names_must_be_in_byline_region_not_citations_or_contact_guess(self): + sha,p,h=self.review(authors=['Unrelated Person'],page='

Canonical full title

Jane Doe

Abstract

Unrelated Person cited here

') + with self.assertRaisesRegex(ValueError,'byline region'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + sha,p,h=self.review(author_source='author_index',authors=['Contact Person']) + with self.assertRaisesRegex(ValueError,'author-index'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + + def test_review_hash_cache_binding_and_canonical_h1_are_all_required(self): + sha,p,h=self.review() + with self.assertRaisesRegex(ValueError,'checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256='0'*64) + d=json.loads(p.read_text());d['source_cache_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest() + with self.assertRaisesRegex(ValueError,'binding'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + d['source_cache_sha256']=sha;d['records'][0]['title']='Wrong title';p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest() + with self.assertRaisesRegex(ValueError,'canonical H1'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + + def test_unlisted_review_entry_and_changed_primary_digest_fail_closed(self): + sha,p,h=self.review();d=json.loads(p.read_text());d['records'][0]['individual_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest() + with self.assertRaisesRegex(ValueError,'page checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + self.pages[SPRING+'posters/abstract.html']=self.pages[SPRING+'posters/abstract.html'].replace('01/paper.html','02/other.html') + self.pages[SPRING+'posters/author.html']=self.pages[SPRING+'posters/author.html'].replace('01/paper.html','02/other.html') + sha,p,h=self.review() + with self.assertRaisesRegex(ValueError,'outside abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)