From 64ced7af429b50368a8f8552b7b534c1facd0b49 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Sun, 8 Nov 2026 21:41:02 +0800 Subject: [PATCH 1/3] Package reproducible early WWW metadata collection with explicit review holds --- .gitea/workflows/check.yml | 1 + README.md | 2 + docs/www-archive.md | 76 ++++++++ paper_atlas/cli.py | 1 + paper_atlas/collectors/www_archive.py | 270 ++++++++++++++++++++++++++ tests/test_www_archive.py | 94 +++++++++ 6 files changed, 444 insertions(+) create mode 100644 docs/www-archive.md create mode 100644 paper_atlas/collectors/www_archive.py create mode 100644 tests/test_www_archive.py diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 950c95a..ddc29f1 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -48,6 +48,7 @@ jobs: outside.mkdir() subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','import','records','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','import','arxiv','--help'],cwd=outside,check=True) diff --git a/README.md b/README.md index 09dcf66..024d864 100644 --- a/README.md +++ b/README.md @@ -86,3 +86,5 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \ CI 在已授权的远程 runner 上运行测试、构建 wheel，并从源码目录外验证安装后的命令及随包资源。数据库全量去重和覆盖率是独立的数据验收步骤，不因代码测试通过而自动完成。 开发通过工作分支和 PR 交付，由用户手动合并。 + +早期WWW档案的可重放元数据采集、论文/海报分轨与待核查条目见 [WWW档案采集](docs/www-archive.md)。 diff --git a/docs/www-archive.md b/docs/www-archive.md new file mode 100644 index 0000000..59d1fcf --- /dev/null +++ b/docs/www-archive.md @@ -0,0 +1,76 @@ +# Early WWW archive collector + +`paper-atlas collect www-archive` is a maintained metadata adapter. It reads official +IW3C2 archive HTML, keeps exact source hashes, and separates importable rows from +program entries that still need review. It fetches no PDF, follows no paper-body or +reference link, and makes no database connection. + +For a fresh source snapshot on the server: + +```sh +paper-atlas collect www-archive \ + --http-cache /data/paper-index/cache/www-archive \ + --source-cache /data/paper-index/collected/www-archive-source-v2 \ + --out /data/paper-index/collected/www-archive-normalized-v2 +``` + +For reproducible offline replay of the already verified snapshot: + +```sh +paper-atlas collect www-archive \ + --source-cache /data/paper-index/collected/www-early-archive-source-v1 \ + --source-cache-sha256 c9388ccef44486e0684c9ee9e8620a465cdc4574e6bc9ea9391d743e01a3312c \ + --out /data/paper-index/collected/www-archive-normalized-v1 +``` + +Use a new output directory for another batch. Existing snapshots and normalized +batches are never overwritten. The manifest hash binds every source URL, byte count, +response hash and cache filename. Plain files must remain directly within the cache; +symlinks, escaping paths, duplicate URLs, non-200 responses and altered bytes fail. +The command uses UTF-8 where valid and explicitly records an ISO-8859-1 fallback for +older pages; original bytes remain authoritative for encoding corrections. + +## Evidence and outputs + +- WWW3 Spring 1995 papers use `event_key=www-1995-spring`, `track=main`. The author, + session and abstract indexes must have the same nonempty entry-ID set. The ID is + local to this exact event and proceedings section, never a DOI or OpenReview ID. +- Posters remain `track=poster`. An abstract without a linked author-index entry + stays in review even when another index has a placeholder `#` link for it. +- Paper URLs must agree between author and session indexes. A malformed whitespace + URL in the abstract list may use the exact event/entry-ID match only when both + independent indexes corroborate the same clean URL. The original href is retained. +- Canonical names come from the explicitly labeled author index. The source lists + them alphabetically; this is **not** original byline order. Both the raw index names + and the full abstract-page byline/affiliation text remain in metadata. No person ID + or cross-source identity is inferred from a name. Conflicting titles or missing + abstracts stay in review. Topic keywords are not treated as an abstract. +- WWW1 1994 formal presentations and WWW2 1994 mixed categories remain review + candidates. The first list contains uncertain business-track annotations and a + count discrepancy; the second contains panels, tutorials, products and other + activities. Neither presentation listing nor a missing full-text link establishes + a regular-paper acceptance or a rejection. Raw category/title/name claims are kept. + +The main and poster JSONL files have independent manifests with explicit event key, +track, count and SHA256. Their status stays `partial`: a verified linked subset does +not prove complete event coverage. `review-candidates.jsonl` contains nested +candidates and reasons, with no importable decision status. It is excluded from +`import-queue.json`. `report.json` accounts for ready/held records and every source +used. Newlines delimit JSONL; Unicode U+2028/U+2029 inside metadata are not line breaks. +The output is published only after all required source and scope checks pass. + +## Before importing + +This independent collector branch starts from main; it does not include or silently +install the event-aware writer. Its output requires the reviewed integrated runtime +from PR31 (or a main revision containing that implementation). The production event +schema migration has been independently accepted, preserving existing edition1212 +and its 59 Fall submissions; that is not an import of the Spring proceedings. + +Inspect held entries and original author names, preflight every ready row against +current PostgreSQL stable identifiers and candidate identities, and verify the +source/year/track/event coverage before running `import records`. Do not automatically +run this queue against an older writer. Preserve extended versions and independent +submission/decision evidence, and keep unresolved identities held. A normalization +report or passing tests alone do not establish production import, complete historical +coverage or final global deduplication. diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 549bfc0..72fab61 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -21,6 +21,7 @@ COMMANDS = { ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'), + ('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'), ('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'), ('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'), ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'), diff --git a/paper_atlas/collectors/www_archive.py b/paper_atlas/collectors/www_archive.py new file mode 100644 index 0000000..015d14b --- /dev/null +++ b/paper_atlas/collectors/www_archive.py @@ -0,0 +1,270 @@ +"""Reproducible early WWW metadata; unresolved program entries stay outside imports.""" +import argparse +from collections import defaultdict +import hashlib +from html.parser import HTMLParser +import json +from pathlib import Path +import re +import tempfile +from urllib.parse import urljoin, urlsplit + +from paper_atlas.io.html import Document, clean +from paper_atlas.io.http import fetch + +ROOT = 'https://archives.iw3c2.org/' +SPRING = ROOT+'www3/proceedings/' +FALL = ROOT+'www2/Proceedings/WWW2_Proceedings.html' +SEEDS = [ROOT+'www1/PrelimProcs.html', ROOT+'www1/Proceedings.html', FALL, + SPRING+'proceedings.html', *[SPRING+'papers/'+x+'.html' for x in ('author','session','abstract')], + *[SPRING+'posters/'+x+'.html' for x in ('author','abstract')]] + + +def decode(raw): + try: + return raw.decode('utf-8-sig'), 'utf-8' + except UnicodeDecodeError: + return raw.decode('iso-8859-1'), 'iso-8859-1 fallback; original bytes retained' + + +def text(html): + return re.sub(r'[ \t\r\n\f\v]+',' ',Document(html).root.text()).strip(' \t\r\n\f\v') + + +def anchors(html): + return [(n.attrs.get('href',''),clean(n.text())) for n in Document(html).root.walk() if n.tag=='a'] + + +def source_url(url): + p=urlsplit(url) + if p.scheme!='https' or p.netloc!='archives.iw3c2.org' or p.query or p.fragment or not p.path.endswith('.html'): + raise ValueError('Only exact official archive HTML source URLs are allowed') + return url + + +class Sources: + def __init__(self, directory, expected_sha256): + self.root=Path(directory).resolve();manifest=self.root/'manifest.json';raw=manifest.read_bytes() + if hashlib.sha256(raw).hexdigest()!=expected_sha256: + raise ValueError('Source cache manifest checksum mismatch') + self.rows={};self.used=set() + for row in json.loads(raw)['sources']: + url=source_url(row['url']) + if url in self.rows or row.get('status')!=200: + raise ValueError('Duplicate or incomplete source response') + p=self.root/row['file'] + if p.is_symlink() or p.parent!=self.root or not p.is_file(): + raise ValueError('Source cache paths must be plain files directly within the cache') + data=p.read_bytes() + if len(data)>2*1024**2 or len(data)!=row['bytes'] or hashlib.sha256(data).hexdigest()!=row['sha256']: + raise ValueError('Source response checksum or size mismatch') + self.rows[url]={'raw':data,'receipt':row} + + def read(self,url): + source_url(url) + if url not in self.rows: + raise ValueError('Required source missing from cache: '+url) + self.used.add(url) + return decode(self.rows[url]['raw'])[0] + + def proof(self,urls): + return [{'url':url,'sha256':self.rows[url]['receipt']['sha256'], + 'encoding':decode(self.rows[url]['raw'])[1]} for url in sorted(set(urls))] + + +def download_sources(http_cache, directory): + directory=Path(directory) + if directory.exists(): + raise ValueError('Refusing to overwrite a source snapshot') + directory.mkdir(parents=True) + responses={url:fetch(url,http_cache,2*1024**2) for url in SEEDS} + html=decode(responses[FALL])[0] + for href,label in anchors(html): + if href!='Author_Inst.html' and href.endswith('.html'): + url=source_url(urljoin(FALL,href)) + if not url.startswith(ROOT+'www2/Proceedings/'): + raise ValueError('Unexpected fall category source') + responses[url]=fetch(url,http_cache,2*1024**2) + receipts=[] + for i,(url,raw) in enumerate(sorted(responses.items())): + name='source-'+str(i).zfill(2)+'.html';(directory/name).write_bytes(raw) + receipts.append({'url':url,'status':200,'file':name,'bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()}) + manifest=directory/'manifest.json';manifest.write_text(json.dumps({'sources':receipts},indent=2)+'\n') + return hashlib.sha256(manifest.read_bytes()).hexdigest() + + +def identity(href,track): + url=urljoin(SPRING+track+'/',href);p=urlsplit(url) + match=re.fullmatch(r'/www3/proceedings/'+track+r'/(\d+)/[^?#]+',p.path) + if p.scheme=='https' and p.netloc=='archives.iw3c2.org' and not p.query and not p.fragment and match: + return match[1],url + return None + + +class AuthorIndex(HTMLParser): + def __init__(self,html,track): + super().__init__(convert_charrefs=True);self.track=track;self.name='';self.bold=None;self.link=None;self.link_text=[] + self.entries=defaultdict(lambda:{'authors':[],'urls':set(),'titles':set()});self.feed(html) + + def handle_starttag(self,tag,attrs): + if tag=='b':self.bold=[] + if tag=='a':self.link=dict(attrs).get('href');self.link_text=[] + + def handle_data(self,data): + if self.bold is not None:self.bold.append(data) + if self.link is not None:self.link_text.append(data) + + def handle_endtag(self,tag): + if tag=='b' and self.bold is not None: + self.name=clean(''.join(self.bold));self.bold=None + if tag=='a' and self.link is not None: + own=identity(self.link,self.track) + if own: + key,url=own;entry=self.entries[key];entry['urls'].add(url);entry['titles'].add(clean(''.join(self.link_text))) + if not self.name:raise ValueError('Paper link has no archive author context') + if self.name not in entry['authors']:entry['authors'].append(self.name) + self.link=None + + +def sections(html): + """Use explicit h3 boundaries; old HTML omits many p/li closing tags.""" + result=[] + for chunk in re.split(r']*>',html,flags=re.I)[1:]: + parts=re.split(r'',chunk,maxsplit=1,flags=re.I) + if len(parts)!=2:raise ValueError('Unclosed archive entry heading') + heading,body=parts + body=re.split(r'<(?:hr|h2|address)\b',body,maxsplit=1,flags=re.I)[0] + author=re.search(r']*>(.*?)',body,re.I|re.S) + paragraphs=[] + for paragraph in re.findall(r']*>(.*?)(?=|]*>',html,flags=re.I)[1:],1): + title=re.search(r'Title:\s*(.*?)',chunk,re.I|re.S) + if not title:raise ValueError('Unclassified first-conference entry') + names=re.findall(r'Author:(.*?)(?=]+>','',text(n)).strip()for n in names], + 'keywords_raw':text(re.search(r'Keywords:(.*?)(?=Keywords:(.*?)(?=]*>',html,flags=re.I)[1:],1): + fields=re.split(r']*>',chunk,maxsplit=1,flags=re.I) + title=text(fields[0]);raw_byline=text(re.split(r'Formal presentations
  1. Keywords: BUSINESS TRACK?
    Title: First contribution
    Author:Jane Doe <jane@example.org>
  2. Title:Unlinked contribution
    Author:John Roe
''', + FALL:'

Conference Tracks

  1. Agents
  2. Tutorials
', + ROOT+'www2/Proceedings/Agents.html':'

Panel

Linked talk
Jane Doe
Unlinked talk
John Roe
', + ROOT+'www2/Proceedings/Tutor.html':'

Tutorials

Training session
Teacher
'} + for track in ['papers','posters']: + self.pages[SPRING+track+'/author.html']='Doe, JaneRoe, JohnSame contribution' + self.pages[SPRING+track+'/abstract.html']='''

Abstracts

Same contribution

Same contribution

Author: Jane Doe, John Roe
Institute

Actual abstract with inline emphasis and a separator\u2028 preserved.

Another session

''' + self.pages[SPRING+'papers/session.html']='Same contribution' + + def sources(self): + rows=[] + for i,(url,html) in enumerate(sorted(self.pages.items())): + raw=html.encode()if isinstance(html,str)else html;name='source-'+str(i)+'.html';(self.cache/name).write_bytes(raw) + rows.append(dict(url=url,status=200,file=name,bytes=len(raw),sha256=hashlib.sha256(raw).hexdigest())) + p=self.cache/'manifest.json';p.write_text(json.dumps({'sources':rows}));sha=hashlib.sha256(p.read_bytes()).hexdigest() + return Sources(self.cache,sha),sha + + def test_offline_batch_separates_tracks_review_and_importable_rows(self): + _,sha=self.sources();out=self.root/'batch';report=collect(self.cache,sha,out) + self.assertEqual(report['ready'],2);self.assertEqual(report['held'],5) + queue=json.loads((out/'import-queue.json').read_text())['files'];self.assertEqual(len(queue),2) + for name in queue: + p=Path(name);rows=[json.loads(l)for l in p.read_text().split('\n') if l];self.assertEqual(len(rows),1) + row=rows[0];self.assertEqual(row['event_key'],'www-1995-spring');self.assertEqual(row['authors'],['Jane Doe','John Roe']) + self.assertIn(row['track'],['main','poster']);self.assertIn('inline emphasis',row['abstract']);self.assertNotIn('Another session',row['abstract']) + self.assertIn('\u2028',row['abstract']);manifest=json.loads(p.with_suffix('.manifest.json').read_text());self.assertEqual(manifest['file_sha256'],hashlib.sha256(p.read_bytes()).hexdigest());self.assertEqual(manifest['status'],'partial') + held=[json.loads(l)for l in(out/'review-candidates.jsonl').read_text().split('\n') if l] + self.assertTrue(all('status'not in r and 'source_evidence'in r for r in held)) + self.assertFalse(any('review-candidates'in p for p in queue)) + + def test_malformed_href_requires_exact_id_and_two_clean_index_anchors(self): + self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('01/paper.html','01/ paper.html') + sources,_=self.sources();rows,held,_=spring_rows(sources,'papers');self.assertEqual(held,[]) + self.assertEqual(rows[0]['url'],SPRING+'papers/01/paper.html');self.assertTrue(rows[0]['metadata']['url_corroborated_by_author_and_session']) + self.assertIn(' ',rows[0]['metadata']['abstract_href_raw']) + + def test_main_index_disagreement_fails_without_publishing_batch(self): + self.pages[SPRING+'papers/session.html']='Other work';_,sha=self.sources();out=self.root/'batch' + with self.assertRaisesRegex(ValueError,'ID sets disagree'):collect(self.cache,sha,out) + self.assertFalse(out.exists()) + + def test_same_entry_id_with_conflicting_clean_url_is_rejected(self): + self.pages[SPRING+'papers/session.html']='Same contribution';sources,_=self.sources() + with self.assertRaisesRegex(ValueError,'URLs disagree'):spring_rows(sources,'papers') + + def test_poster_without_linked_author_entry_is_held_even_with_abstract(self): + self.pages[SPRING+'posters/author.html']='Doe, JaneSame contribution';sources,_=self.sources();rows,held,counts=spring_rows(sources,'posters') + self.assertEqual(rows,[]);self.assertEqual(counts['abstract_entries'],1);self.assertEqual(held[0]['reason'],'abstract_entry_missing_linked_author_index') + + def test_unlinked_and_panel_entries_never_become_acceptance_or_rejection(self): + sources,_=self.sources();held,counts=early_review(sources);self.assertEqual(counts,{'first_conference_fields':2,'fall_categories':2,'fall_program_entries':3}) + self.assertTrue(any(r['candidate']['title']=='Unlinked talk'for r in held));self.assertTrue(all('status'not in r for r in held)) + self.assertEqual(held[0]['candidate']['authors_raw'],['Jane Doe']) + + def test_title_disagreement_goes_to_review_with_source_claims(self): + self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('Same contribution','Different contribution');sources,_=self.sources();rows,held,_=spring_rows(sources,'papers') + self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'abstract_and_author_index_titles_disagree') + + def test_cache_tampering_and_manifest_hash_mismatch_fail_closed(self): + _,sha=self.sources() + with self.assertRaisesRegex(ValueError,'manifest checksum'):Sources(self.cache,'0'*64) + p=next(self.cache.glob('source-*.html'));p.write_bytes(p.read_bytes()+b'x') + with self.assertRaisesRegex(ValueError,'checksum or size'):Sources(self.cache,sha) + + def test_cache_cannot_read_paths_outside_snapshot(self): + _,sha=self.sources();p=self.cache/'manifest.json';d=json.loads(p.read_text());d['sources'][0]['file']='../outside.html';p.write_text(json.dumps(d)) + with self.assertRaisesRegex(ValueError,'plain files'):Sources(self.cache,hashlib.sha256(p.read_bytes()).hexdigest()) + + def test_keywords_only_is_missing_abstract_not_invented_summary(self): + self.pages[SPRING+'papers/abstract.html']='

Same contribution

Author: Jane Doe

Keywords: networking, hypertext

';sources,_=self.sources();rows,held,_=spring_rows(sources,'papers') + self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'missing_abstract') + + def test_old_encoding_and_duplicate_abstract_identity(self): + self.assertEqual(decode(b'M\xfcller')[0],'Müller') + self.pages[SPRING+'papers/abstract.html']*=2;sources,_=self.sources() + with self.assertRaisesRegex(ValueError,'Duplicate abstract'):spring_rows(sources,'papers') + + def test_existing_output_is_never_overwritten(self): + _,sha=self.sources();out=self.root/'batch';out.mkdir();(out/'keep').write_text('original') + with self.assertRaisesRegex(ValueError,'overwrite'):collect(self.cache,sha,out) + self.assertEqual((out/'keep').read_text(),'original') -- 2.52.0 From 9040649320227b46d69b6834901e8f055b548be4 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Sun, 8 Nov 2026 21:54:43 +0800 Subject: [PATCH 2/3] Preserve full source listing totals beside ready archive subsets --- docs/www-archive.md | 20 ++++++++++++++++++-- paper_atlas/collectors/www_archive.py | 2 +- tests/test_www_archive.py | 7 +++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/www-archive.md b/docs/www-archive.md index 59d1fcf..24f1053 100644 --- a/docs/www-archive.md +++ b/docs/www-archive.md @@ -52,7 +52,9 @@ older pages; original bytes remain authoritative for encoding corrections. a regular-paper acceptance or a rejection. Raw category/title/name claims are kept. The main and poster JSONL files have independent manifests with explicit event key, -track, count and SHA256. Their status stays `partial`: a verified linked subset does +track, count and SHA256. `source_listing_count` records the full recognized abstract +listing, while `held_entry_count` records excluded entries; neither is a ready-row +count. Their status stays `partial`: a verified linked subset does not prove complete event coverage. `review-candidates.jsonl` contains nested candidates and reasons, with no importable decision status. It is excluded from `import-queue.json`. `report.json` accounts for ready/held records and every source @@ -69,8 +71,22 @@ and its 59 Fall submissions; that is not an import of the Spring proceedings. Inspect held entries and original author names, preflight every ready row against current PostgreSQL stable identifiers and candidate identities, and verify the -source/year/track/event coverage before running `import records`. Do not automatically +source/year/track/event coverage before running `import records`. When a ready subset +excludes held entries, preserve the independently known listing total in the existing +incremental-coverage protocol: bind a freshly checked parent count and expected count, +and set the full expected count from this manifest. A new source uses previous count +0 and previous expected count null; an existing source requires its actual current +values. These live parent values are not guessed by the offline collector. Do not automatically run this queue against an older writer. Preserve extended versions and independent submission/decision evidence, and keep unresolved identities held. A normalization report or passing tests alone do not establish production import, complete historical coverage or final global deduplication. + +Initial fixed-cache replay (64ced7a, CI1057): 38 main papers and38 posters ready; +372 entries held (50 first-conference fields,307 mixed program entries,15 posters). +Independent report44dfd070 verified source hashes,counts and author evidence, finding +one existing exact-title candidate. That publication is explicitly linked to +arXiv1811.12161 by its WWW1995/journal-reference declaration; arXiv1810.07232 is a +different1997 invited chapter and remains separate despite the overlap note. This +is a preparation observation, not evidence of completed production import or final +global deduplication. diff --git a/paper_atlas/collectors/www_archive.py b/paper_atlas/collectors/www_archive.py index 015d14b..8644978 100644 --- a/paper_atlas/collectors/www_archive.py +++ b/paper_atlas/collectors/www_archive.py @@ -248,7 +248,7 @@ def collect(source_cache,cache_sha256,out): filename='www-1995-spring-'+('main'if track=='papers'else'poster')+'.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n'for r in rows).encode();(stage/filename).write_bytes(raw) manifest={'source':'www_archive','parser_version':1,'venue':'WWW','year':1995,'track':rows[0]['track'],'event_key':'www-1995-spring', 'list_url':SPRING+track+'/abstract.html','count':len(rows),'status':'partial','file_sha256':hashlib.sha256(raw).hexdigest(), - 'missing_abstract_count':0,'source_cache_sha256':cache_sha256,'scope':'Verified linked archive entries; held entries and other event tracks excluded, not complete event coverage.'} + 'missing_abstract_count':0,'source_listing_count':counts[track]['abstract_entries'],'held_entry_count':counts[track]['held'],'source_cache_sha256':cache_sha256,'scope':'Verified linked archive entries; held entries and other event tracks excluded, not complete event coverage.'} (stage/filename).with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');queue.append(str(out/filename)) (stage/'review-candidates.jsonl').write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n'for r in held)) report={'status':'normalized_not_imported','source_cache_sha256':cache_sha256,'counts':counts,'ready':sum(len(r)for r in groups.values()),'held':len(held), diff --git a/tests/test_www_archive.py b/tests/test_www_archive.py index edd7a9e..83e251a 100644 --- a/tests/test_www_archive.py +++ b/tests/test_www_archive.py @@ -41,6 +41,13 @@ class ArchiveTests(unittest.TestCase): self.assertTrue(all('status'not in r and 'source_evidence'in r for r in held)) self.assertFalse(any('review-candidates'in p for p in queue)) + def test_partial_manifest_preserves_full_listing_size_separately_from_ready_count(self): + self.pages[SPRING+'posters/abstract.html']+='

Unlinked poster

Author: Other Person

Actual abstract

' + _,sha=self.sources();out=self.root/'batch';collect(self.cache,sha,out) + manifest=json.loads((out/'www-1995-spring-poster.manifest.json').read_text()) + self.assertEqual((manifest['count'],manifest['source_listing_count'],manifest['held_entry_count']),(1,2,1)) + self.assertNotIn('coverage_previous_count',manifest) # Live parent must be independently preflighted. + def test_malformed_href_requires_exact_id_and_two_clean_index_anchors(self): self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('01/paper.html','01/ paper.html') sources,_=self.sources();rows,held,_=spring_rows(sources,'papers');self.assertEqual(held,[]) -- 2.52.0 From eacc1e3292bf207113e7b3b7fdb2c66c4b3b6e44 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Mon, 9 Nov 2026 00:18:32 +0800 Subject: [PATCH 3/3] Resolve reviewed WWW poster entries with canonical page evidence --- .../www-1995-posters-reviewed-2026-11-09.json | 201 ++++++++++++++++++ docs/www-archive.md | 19 ++ paper_atlas/collectors/www_archive.py | 82 ++++++- tests/test_www_archive.py | 54 +++++ 4 files changed, 350 insertions(+), 6 deletions(-) create mode 100644 docs/coverage/www-1995-posters-reviewed-2026-11-09.json diff --git a/docs/coverage/www-1995-posters-reviewed-2026-11-09.json b/docs/coverage/www-1995-posters-reviewed-2026-11-09.json new file mode 100644 index 0000000..3afe54b --- /dev/null +++ b/docs/coverage/www-1995-posters-reviewed-2026-11-09.json @@ -0,0 +1,201 @@ +{ + "format": 1, + "source_cache_sha256": "8a65562f3490497cada2919ec77827134beabd5ec9c3690dec3625e2318aa54e", + "records": [ + { + "entry_id": "45", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/45/index.html", + "individual_sha256": "83bede6afc9a7039fae2dd1f28274a394418b34f5d5632590b913617580bda1a", + "title": "WISE - World Wide Information System for R&D; Efforts", + "authors": [ + "Fritz Loseries", + "Veronika Samara", + "Bodo Urban" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "16", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/16/index.html", + "individual_sha256": "cf76da903eaeaa27d19857fdaa13220f6821559bf4ac22844fae10abbaa6f3f3", + "title": "The World Wide Web as a Tool for Collaborative Inter-Agency Projects: Creation of an Environmental Metadata Database for the Analysis of the Venice Lagoon Ecosystem.", + "authors": [ + "Mirko Labbri" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "17", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/17/index.html", + "individual_sha256": "0578fec9c82e8f2f98c4de0f1bcc44011ebb84c53353fdcc423a69cf70673bc2", + "title": "NetRepreneur", + "authors": [ + "Kari-Pekka Turtiainen", + "Antti Auer" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "29", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/29/index.html", + "individual_sha256": "624ad3f685de1163411e33e90488975d848a63243fd7ffbed0862c6fb58c4140", + "title": "Mosaic TueV 2.4.3: Multilingual GUI and Improvements towards User Friendliness", + "authors": [ + "Mike Bretz", + "Uwe Koch", + "Paul de Bra" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "13", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/13/index.html", + "individual_sha256": "02409321a3403a94d8362a9862f4960386ea9c344fb47f4bcdf9eae128e8b46e", + "title": "Using a WWW Browser as an Alternative User Interface for Interactive Applications", + "authors": [ + "Frank Lonczewski" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "36", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/36/index.html", + "individual_sha256": "ed9dada4ea5903cb8f0a4e75f2958416a6e32647fbf3530e58edd9abf26070ce", + "title": "HyperSource: A Hypermedia Program Development And Documentation System", + "authors": [ + "Jan Oliver Borchers" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "57", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/57/index.html", + "individual_sha256": "3ad4206ad28a19823172cd2845f636fe7a461c251940e9a93deb8f6484b8a984", + "title": "Progressive Image Transmission Capability (PROTRAC) on the World Wide Web", + "authors": [ + "Vojislav Lalich-Petrich", + "Gaurav Bhatia", + "Larry Davis" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "38", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/38/index.html", + "individual_sha256": "940370acc5e6a954b197053fe451e466a904d1f0ac82c8fe4e6ca455faf2c927", + "title": "Measuring the Web with Lycos", + "authors": [ + "Michael L. Mauldin" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "Official abstract listing has no abstract. Canonical HTML has body sections but no explicit abstract; body is not substituted as a summary." + }, + { + "entry_id": "05", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/05/index.html", + "individual_sha256": "1858be87d6e7f47476e1cb8fb4793c0fce3b8a6ff653bcfd4a35089e0520f177", + "title": "Virtual Places - Adding people to the web", + "authors": [ + "Yosi Mass" + ], + "author_source": "author_index", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "Official abstract listing and canonical HTML have no abstract. Canonical page identifies a contact and links PostScript; no full-text download. Author identity comes from author index, not inferred from contact role." + }, + { + "entry_id": "37", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/37/index.html", + "individual_sha256": "b700a0a51b7a82a5f88373ae0283bb369d1cf6f455acff7d91cd7313fa8b3adf", + "title": "The Internet Public Library: Serving a Global Community", + "authors": [ + "Susan Schweitzer" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "41", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/41/index.html", + "individual_sha256": "397b474109f9c1240a77d0f490f2b02dd35fae93febbb7397d1b2a31759a8ec1", + "title": "Graphical Browsing in the WorldWideWeb and the MultiMedia Forum", + "authors": [ + "Rainer Pagé", + "Klaus Süllow" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "47", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/47/index.html", + "individual_sha256": "52eb4bfdd658730a222c98170b72dabf9cbcc5ffd2ada1e900b826baf2b02833", + "title": "Interoperating Advanced Hypermedia Systems with WWW", + "authors": [ + "Jian Zhang", + "Martin Fruehauf", + "Thomas Kirste" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "50", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/50/index.html", + "individual_sha256": "8dc8b0561bd8b1b106c5b209a973c695c3cc9df5288450d4a01fe32541476b7e", + "title": "Web Enabled Agile Manufacturing", + "authors": [ + "S. V. Iyer", + "T. Singh", + "U. Ravaioli", + "R. E. DeVor", + "S. G. Kapoor", + "G. Veknatasubramian" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "55", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/55/index.html", + "individual_sha256": "21780cd4047f513e8c9aeced5247002e27cf71719f08de85c6531b57a9e5058d", + "title": "Weblets: Fundamental Building Blocks for WWW Tools", + "authors": [ + "Thomas J. Watt Jr." + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + }, + { + "entry_id": "59", + "individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/59/index.html", + "individual_sha256": "fed4cc9dcaee350a12c5e4b6fa9c99be9bead25a81632512d31566400d3d0fad", + "title": "Usenet and the World-Wide Web", + "authors": [ + "M. L. Grant" + ], + "author_source": "canonical_byline", + "reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.", + "missing_abstract_reason": "" + } + ] +} diff --git a/docs/www-archive.md b/docs/www-archive.md index 24f1053..2a91c9e 100644 --- a/docs/www-archive.md +++ b/docs/www-archive.md @@ -90,3 +90,22 @@ arXiv1811.12161 by its WWW1995/journal-reference declaration; arXiv1810.07232 is different1997 invited chapter and remains separate despite the overlap note. This is a preparation observation, not evidence of completed production import or final global deduplication. + +## 以单篇官方页面核实待处理 poster + +作者索引漏链接、摘要目录使用短标题或标题存在拼写差异时，不放宽全局标题匹配。可以先取得单篇官方 HTML，人工核对 H1、署名区与原目录同一 entry URL，再提供带 SHA256 的逐条核查文件： + +```sh +paper-atlas collect www-archive \ + --source-cache /data/paper-index/collected/www-poster-reviewed-source-v2 \ + --source-cache-sha256 8a65562f3490497cada2919ec77827134beabd5ec9c3690dec3625e2318aa54e \ + --reviewed-posters docs/coverage/www-1995-posters-reviewed-2026-11-09.json \ + --reviewed-posters-sha256 <核查文件的SHA256> \ + --out <新的输出目录> +``` + +[1995 年15条poster的核查输入](coverage/www-1995-posters-reviewed-2026-11-09.json) 绑定61页来源快照：原46页和补取的15页单篇HTML。程序验证输入文件及快照哈希、单篇页面哈希、完全一致的 canonical H1、原摘要目录中的同一URL/entry ID及作者证据，不通过相似标题猜身份。规范标题采用单篇H1，摘要目录标题及作者索引标题均在 metadata 中保留。 + +`author_source=canonical_byline` 表示名字出现在H1之后、摘要/关键词或下一节标题之前的署名区域，由逐条核查明确选出，不从全文引用或机构猜作者；保留原始缩写、顺序、重音和拼写。只有 `author_source=author_index` 时使用原作者索引顺序并逐项核对；05的单篇页面写的是联系人，作者身份使用原作者索引证据，不由联系人角色推断。核查输入是审阅后的数据声明，程序的字符串/哈希检查不代替人工识别署名语义。 + +38和05的来源没有可明确引用的摘要。保留 `abstract=null` 和逐条缺失原因，manifest准确报告 missing_abstract_count，继续标partial；正文、联系方式及PostScript链接不当作摘要，也不下载正文。缺摘要不等于拒稿或未发表。本批输出仍包含原先已验收记录，正式导入前必须只取新增external ID，检查实时覆盖父记录并按incremental模式准备，不能直接重导入全部输出。 diff --git a/paper_atlas/collectors/www_archive.py b/paper_atlas/collectors/www_archive.py index 8644978..6eec958 100644 --- a/paper_atlas/collectors/www_archive.py +++ b/paper_atlas/collectors/www_archive.py @@ -145,7 +145,68 @@ def sections(html): return result -def spring_rows(sources,track): +def reviewed_posters(sources, path, expected_sha256, cache_sha256): + if path is None: + if expected_sha256:raise ValueError('Reviewed poster path required') + return {},None + raw=Path(path).read_bytes() + if hashlib.sha256(raw).hexdigest()!=expected_sha256:raise ValueError('Reviewed poster checksum mismatch') + review=json.loads(raw) + if set(review)!={'format','source_cache_sha256','records'} or review['format']!=1 or review['source_cache_sha256']!=cache_sha256: + raise ValueError('Reviewed poster snapshot binding differs') + if not isinstance(review['records'],list) or not 0]*>(.*?)',page,re.I|re.S) + if len(headings)!=1 or text(headings[0])!=row['title']: + raise ValueError('Reviewed title differs from exact canonical H1') + header=re.split(r'',page,maxsplit=1,flags=re.I)[1] + header=re.split(r'Canonical full title

Jane Doe, John Roe

Keywords: web

Abstract

Body

' + _,sha=self.sources();raw=self.pages[url].encode()if isinstance(self.pages[url],str)else self.pages[url] + row=dict(entry_id='01',individual_url=url,individual_sha256=hashlib.sha256(raw).hexdigest(), + title='Canonical full title',authors=authors or ['Jane Doe','John Roe'],author_source=author_source, + reason='Reviewed exact canonical entry and its explicit authorship; retain original title variants',missing_abstract_reason=missing) + p=self.root/'review.json';p.write_text(json.dumps({'format':1,'source_cache_sha256':sha,'records':[row]}));return sha,p,hashlib.sha256(p.read_bytes()).hexdigest() + + def test_reviewed_variant_keeps_primary_title_and_all_old_title_claims(self): + sha,p,h=self.review();out=self.root/'reviewed';r=collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h) + row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['title'],'Canonical full title') + self.assertEqual(row['metadata']['abstract_heading_title_raw'],'Same contribution');self.assertEqual(row['metadata']['author_index_titles_raw'],['Same contribution']) + self.assertEqual(len(row['metadata']['source_evidence']),3);self.assertEqual(r['reviewed_posters_sha256'],h) + self.assertEqual(row['external_id'],'www3:posters:01');self.assertEqual(row['status'],'accepted') + + def test_canonical_byline_can_resolve_unlinked_author_index_without_guessing(self): + self.pages[SPRING+'posters/author.html']='Doe, JaneBroken index link' + sha,p,h=self.review();out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h) + row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['authors'],['Jane Doe','John Roe']) + self.assertEqual(row['metadata']['author_index_raw'],[]);self.assertEqual(row['metadata']['author_order'],'canonical_byline') + + def test_reviewed_missing_abstract_is_explicit_partial_not_body_summary(self): + self.pages[SPRING+'posters/abstract.html']='

Same contribution

Author: Jane Doe

Keywords: web

' + sha,p,h=self.review(missing='Neither listing nor canonical page supplies an explicit abstract') + out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h) + row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());m=json.loads((out/'www-1995-spring-poster.manifest.json').read_text()) + self.assertIsNone(row['abstract']);self.assertEqual((m['missing_abstract_count'],m['status']),(1,'partial')) + sha,p,h=self.review() + with self.assertRaisesRegex(ValueError,'Missing abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + self.assertFalse((self.root/'bad').exists()) + + def test_reviewed_names_must_be_in_byline_region_not_citations_or_contact_guess(self): + sha,p,h=self.review(authors=['Unrelated Person'],page='

Canonical full title

Jane Doe

Abstract

Unrelated Person cited here

') + with self.assertRaisesRegex(ValueError,'byline region'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + sha,p,h=self.review(author_source='author_index',authors=['Contact Person']) + with self.assertRaisesRegex(ValueError,'author-index'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + + def test_review_hash_cache_binding_and_canonical_h1_are_all_required(self): + sha,p,h=self.review() + with self.assertRaisesRegex(ValueError,'checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256='0'*64) + d=json.loads(p.read_text());d['source_cache_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest() + with self.assertRaisesRegex(ValueError,'binding'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + d['source_cache_sha256']=sha;d['records'][0]['title']='Wrong title';p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest() + with self.assertRaisesRegex(ValueError,'canonical H1'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + + def test_unlisted_review_entry_and_changed_primary_digest_fail_closed(self): + sha,p,h=self.review();d=json.loads(p.read_text());d['records'][0]['individual_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest() + with self.assertRaisesRegex(ValueError,'page checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) + self.pages[SPRING+'posters/abstract.html']=self.pages[SPRING+'posters/abstract.html'].replace('01/paper.html','02/other.html') + self.pages[SPRING+'posters/author.html']=self.pages[SPRING+'posters/author.html'].replace('01/paper.html','02/other.html') + sha,p,h=self.review() + with self.assertRaisesRegex(ValueError,'outside abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h) -- 2.52.0