diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 950c95a..ddc29f1 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -48,6 +48,7 @@ jobs: outside.mkdir() subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','import','records','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','import','arxiv','--help'],cwd=outside,check=True) diff --git a/README.md b/README.md index 09dcf66..024d864 100644 --- a/README.md +++ b/README.md @@ -86,3 +86,5 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目录外验证安装后的命令及随包资源。数据库全量去重和覆盖率是独立的数据验收步骤,不因代码测试通过而自动完成。 开发通过工作分支和 PR 交付,由用户手动合并。 + +早期WWW档案的可重放元数据采集、论文/海报分轨与待核查条目见 [WWW档案采集](docs/www-archive.md)。 diff --git a/docs/www-archive.md b/docs/www-archive.md new file mode 100644 index 0000000..59d1fcf --- /dev/null +++ b/docs/www-archive.md @@ -0,0 +1,76 @@ +# Early WWW archive collector + +`paper-atlas collect www-archive` is a maintained metadata adapter. It reads official +IW3C2 archive HTML, keeps exact source hashes, and separates importable rows from +program entries that still need review. It fetches no PDF, follows no paper-body or +reference link, and makes no database connection. + +For a fresh source snapshot on the server: + +```sh +paper-atlas collect www-archive \ + --http-cache /data/paper-index/cache/www-archive \ + --source-cache /data/paper-index/collected/www-archive-source-v2 \ + --out /data/paper-index/collected/www-archive-normalized-v2 +``` + +For reproducible offline replay of the already verified snapshot: + +```sh +paper-atlas collect www-archive \ + --source-cache /data/paper-index/collected/www-early-archive-source-v1 \ + --source-cache-sha256 c9388ccef44486e0684c9ee9e8620a465cdc4574e6bc9ea9391d743e01a3312c \ + --out /data/paper-index/collected/www-archive-normalized-v1 +``` + +Use a new output directory for another batch. Existing snapshots and normalized +batches are never overwritten. The manifest hash binds every source URL, byte count, +response hash and cache filename. Plain files must remain directly within the cache; +symlinks, escaping paths, duplicate URLs, non-200 responses and altered bytes fail. +The command uses UTF-8 where valid and explicitly records an ISO-8859-1 fallback for +older pages; original bytes remain authoritative for encoding corrections. + +## Evidence and outputs + +- WWW3 Spring 1995 papers use `event_key=www-1995-spring`, `track=main`. The author, + session and abstract indexes must have the same nonempty entry-ID set. The ID is + local to this exact event and proceedings section, never a DOI or OpenReview ID. +- Posters remain `track=poster`. An abstract without a linked author-index entry + stays in review even when another index has a placeholder `#` link for it. +- Paper URLs must agree between author and session indexes. A malformed whitespace + URL in the abstract list may use the exact event/entry-ID match only when both + independent indexes corroborate the same clean URL. The original href is retained. +- Canonical names come from the explicitly labeled author index. The source lists + them alphabetically; this is **not** original byline order. Both the raw index names + and the full abstract-page byline/affiliation text remain in metadata. No person ID + or cross-source identity is inferred from a name. Conflicting titles or missing + abstracts stay in review. Topic keywords are not treated as an abstract. +- WWW1 1994 formal presentations and WWW2 1994 mixed categories remain review + candidates. The first list contains uncertain business-track annotations and a + count discrepancy; the second contains panels, tutorials, products and other + activities. Neither presentation listing nor a missing full-text link establishes + a regular-paper acceptance or a rejection. Raw category/title/name claims are kept. + +The main and poster JSONL files have independent manifests with explicit event key, +track, count and SHA256. Their status stays `partial`: a verified linked subset does +not prove complete event coverage. `review-candidates.jsonl` contains nested +candidates and reasons, with no importable decision status. It is excluded from +`import-queue.json`. `report.json` accounts for ready/held records and every source +used. Newlines delimit JSONL; Unicode U+2028/U+2029 inside metadata are not line breaks. +The output is published only after all required source and scope checks pass. + +## Before importing + +This independent collector branch starts from main; it does not include or silently +install the event-aware writer. Its output requires the reviewed integrated runtime +from PR31 (or a main revision containing that implementation). The production event +schema migration has been independently accepted, preserving existing edition1212 +and its 59 Fall submissions; that is not an import of the Spring proceedings. + +Inspect held entries and original author names, preflight every ready row against +current PostgreSQL stable identifiers and candidate identities, and verify the +source/year/track/event coverage before running `import records`. Do not automatically +run this queue against an older writer. Preserve extended versions and independent +submission/decision evidence, and keep unresolved identities held. A normalization +report or passing tests alone do not establish production import, complete historical +coverage or final global deduplication. diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 549bfc0..72fab61 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -21,6 +21,7 @@ COMMANDS = { ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'), + ('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'), ('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'), ('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'), ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'), diff --git a/paper_atlas/collectors/www_archive.py b/paper_atlas/collectors/www_archive.py new file mode 100644 index 0000000..015d14b --- /dev/null +++ b/paper_atlas/collectors/www_archive.py @@ -0,0 +1,270 @@ +"""Reproducible early WWW metadata; unresolved program entries stay outside imports.""" +import argparse +from collections import defaultdict +import hashlib +from html.parser import HTMLParser +import json +from pathlib import Path +import re +import tempfile +from urllib.parse import urljoin, urlsplit + +from paper_atlas.io.html import Document, clean +from paper_atlas.io.http import fetch + +ROOT = 'https://archives.iw3c2.org/' +SPRING = ROOT+'www3/proceedings/' +FALL = ROOT+'www2/Proceedings/WWW2_Proceedings.html' +SEEDS = [ROOT+'www1/PrelimProcs.html', ROOT+'www1/Proceedings.html', FALL, + SPRING+'proceedings.html', *[SPRING+'papers/'+x+'.html' for x in ('author','session','abstract')], + *[SPRING+'posters/'+x+'.html' for x in ('author','abstract')]] + + +def decode(raw): + try: + return raw.decode('utf-8-sig'), 'utf-8' + except UnicodeDecodeError: + return raw.decode('iso-8859-1'), 'iso-8859-1 fallback; original bytes retained' + + +def text(html): + return re.sub(r'[ \t\r\n\f\v]+',' ',Document(html).root.text()).strip(' \t\r\n\f\v') + + +def anchors(html): + return [(n.attrs.get('href',''),clean(n.text())) for n in Document(html).root.walk() if n.tag=='a'] + + +def source_url(url): + p=urlsplit(url) + if p.scheme!='https' or p.netloc!='archives.iw3c2.org' or p.query or p.fragment or not p.path.endswith('.html'): + raise ValueError('Only exact official archive HTML source URLs are allowed') + return url + + +class Sources: + def __init__(self, directory, expected_sha256): + self.root=Path(directory).resolve();manifest=self.root/'manifest.json';raw=manifest.read_bytes() + if hashlib.sha256(raw).hexdigest()!=expected_sha256: + raise ValueError('Source cache manifest checksum mismatch') + self.rows={};self.used=set() + for row in json.loads(raw)['sources']: + url=source_url(row['url']) + if url in self.rows or row.get('status')!=200: + raise ValueError('Duplicate or incomplete source response') + p=self.root/row['file'] + if p.is_symlink() or p.parent!=self.root or not p.is_file(): + raise ValueError('Source cache paths must be plain files directly within the cache') + data=p.read_bytes() + if len(data)>2*1024**2 or len(data)!=row['bytes'] or hashlib.sha256(data).hexdigest()!=row['sha256']: + raise ValueError('Source response checksum or size mismatch') + self.rows[url]={'raw':data,'receipt':row} + + def read(self,url): + source_url(url) + if url not in self.rows: + raise ValueError('Required source missing from cache: '+url) + self.used.add(url) + return decode(self.rows[url]['raw'])[0] + + def proof(self,urls): + return [{'url':url,'sha256':self.rows[url]['receipt']['sha256'], + 'encoding':decode(self.rows[url]['raw'])[1]} for url in sorted(set(urls))] + + +def download_sources(http_cache, directory): + directory=Path(directory) + if directory.exists(): + raise ValueError('Refusing to overwrite a source snapshot') + directory.mkdir(parents=True) + responses={url:fetch(url,http_cache,2*1024**2) for url in SEEDS} + html=decode(responses[FALL])[0] + for href,label in anchors(html): + if href!='Author_Inst.html' and href.endswith('.html'): + url=source_url(urljoin(FALL,href)) + if not url.startswith(ROOT+'www2/Proceedings/'): + raise ValueError('Unexpected fall category source') + responses[url]=fetch(url,http_cache,2*1024**2) + receipts=[] + for i,(url,raw) in enumerate(sorted(responses.items())): + name='source-'+str(i).zfill(2)+'.html';(directory/name).write_bytes(raw) + receipts.append({'url':url,'status':200,'file':name,'bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()}) + manifest=directory/'manifest.json';manifest.write_text(json.dumps({'sources':receipts},indent=2)+'\n') + return hashlib.sha256(manifest.read_bytes()).hexdigest() + + +def identity(href,track): + url=urljoin(SPRING+track+'/',href);p=urlsplit(url) + match=re.fullmatch(r'/www3/proceedings/'+track+r'/(\d+)/[^?#]+',p.path) + if p.scheme=='https' and p.netloc=='archives.iw3c2.org' and not p.query and not p.fragment and match: + return match[1],url + return None + + +class AuthorIndex(HTMLParser): + def __init__(self,html,track): + super().__init__(convert_charrefs=True);self.track=track;self.name='';self.bold=None;self.link=None;self.link_text=[] + self.entries=defaultdict(lambda:{'authors':[],'urls':set(),'titles':set()});self.feed(html) + + def handle_starttag(self,tag,attrs): + if tag=='b':self.bold=[] + if tag=='a':self.link=dict(attrs).get('href');self.link_text=[] + + def handle_data(self,data): + if self.bold is not None:self.bold.append(data) + if self.link is not None:self.link_text.append(data) + + def handle_endtag(self,tag): + if tag=='b' and self.bold is not None: + self.name=clean(''.join(self.bold));self.bold=None + if tag=='a' and self.link is not None: + own=identity(self.link,self.track) + if own: + key,url=own;entry=self.entries[key];entry['urls'].add(url);entry['titles'].add(clean(''.join(self.link_text))) + if not self.name:raise ValueError('Paper link has no archive author context') + if self.name not in entry['authors']:entry['authors'].append(self.name) + self.link=None + + +def sections(html): + """Use explicit h3 boundaries; old HTML omits many p/li closing tags.""" + result=[] + for chunk in re.split(r']*>',html,flags=re.I)[1:]: + parts=re.split(r'',chunk,maxsplit=1,flags=re.I) + if len(parts)!=2:raise ValueError('Unclosed archive entry heading') + heading,body=parts + body=re.split(r'<(?:hr|h2|address)\b',body,maxsplit=1,flags=re.I)[0] + author=re.search(r']*>(.*?)',body,re.I|re.S) + paragraphs=[] + for paragraph in re.findall(r']*>(.*?)(?=|]*>',html,flags=re.I)[1:],1): + title=re.search(r'Title:\s*(.*?)',chunk,re.I|re.S) + if not title:raise ValueError('Unclassified first-conference entry') + names=re.findall(r'Author:(.*?)(?=]+>','',text(n)).strip()for n in names], + 'keywords_raw':text(re.search(r'Keywords:(.*?)(?=Keywords:(.*?)(?=]*>',html,flags=re.I)[1:],1): + fields=re.split(r']*>',chunk,maxsplit=1,flags=re.I) + title=text(fields[0]);raw_byline=text(re.split(r'Formal presentations
  1. Keywords: BUSINESS TRACK?
    Title: First contribution
    Author:Jane Doe <jane@example.org>
  2. Title:Unlinked contribution
    Author:John Roe
''', + FALL:'

Conference Tracks

  1. Agents
  2. Tutorials
', + ROOT+'www2/Proceedings/Agents.html':'

Panel

Linked talk
Jane Doe
Unlinked talk
John Roe
', + ROOT+'www2/Proceedings/Tutor.html':'

Tutorials

Training session
Teacher
'} + for track in ['papers','posters']: + self.pages[SPRING+track+'/author.html']='Doe, JaneRoe, JohnSame contribution' + self.pages[SPRING+track+'/abstract.html']='''

Abstracts

Same contribution

Same contribution

Author: Jane Doe, John Roe
Institute

Actual abstract with inline emphasis and a separator\u2028 preserved.


Another session

''' + self.pages[SPRING+'papers/session.html']='Same contribution' + + def sources(self): + rows=[] + for i,(url,html) in enumerate(sorted(self.pages.items())): + raw=html.encode()if isinstance(html,str)else html;name='source-'+str(i)+'.html';(self.cache/name).write_bytes(raw) + rows.append(dict(url=url,status=200,file=name,bytes=len(raw),sha256=hashlib.sha256(raw).hexdigest())) + p=self.cache/'manifest.json';p.write_text(json.dumps({'sources':rows}));sha=hashlib.sha256(p.read_bytes()).hexdigest() + return Sources(self.cache,sha),sha + + def test_offline_batch_separates_tracks_review_and_importable_rows(self): + _,sha=self.sources();out=self.root/'batch';report=collect(self.cache,sha,out) + self.assertEqual(report['ready'],2);self.assertEqual(report['held'],5) + queue=json.loads((out/'import-queue.json').read_text())['files'];self.assertEqual(len(queue),2) + for name in queue: + p=Path(name);rows=[json.loads(l)for l in p.read_text().split('\n') if l];self.assertEqual(len(rows),1) + row=rows[0];self.assertEqual(row['event_key'],'www-1995-spring');self.assertEqual(row['authors'],['Jane Doe','John Roe']) + self.assertIn(row['track'],['main','poster']);self.assertIn('inline emphasis',row['abstract']);self.assertNotIn('Another session',row['abstract']) + self.assertIn('\u2028',row['abstract']);manifest=json.loads(p.with_suffix('.manifest.json').read_text());self.assertEqual(manifest['file_sha256'],hashlib.sha256(p.read_bytes()).hexdigest());self.assertEqual(manifest['status'],'partial') + held=[json.loads(l)for l in(out/'review-candidates.jsonl').read_text().split('\n') if l] + self.assertTrue(all('status'not in r and 'source_evidence'in r for r in held)) + self.assertFalse(any('review-candidates'in p for p in queue)) + + def test_malformed_href_requires_exact_id_and_two_clean_index_anchors(self): + self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('01/paper.html','01/ paper.html') + sources,_=self.sources();rows,held,_=spring_rows(sources,'papers');self.assertEqual(held,[]) + self.assertEqual(rows[0]['url'],SPRING+'papers/01/paper.html');self.assertTrue(rows[0]['metadata']['url_corroborated_by_author_and_session']) + self.assertIn(' ',rows[0]['metadata']['abstract_href_raw']) + + def test_main_index_disagreement_fails_without_publishing_batch(self): + self.pages[SPRING+'papers/session.html']='Other work';_,sha=self.sources();out=self.root/'batch' + with self.assertRaisesRegex(ValueError,'ID sets disagree'):collect(self.cache,sha,out) + self.assertFalse(out.exists()) + + def test_same_entry_id_with_conflicting_clean_url_is_rejected(self): + self.pages[SPRING+'papers/session.html']='Same contribution';sources,_=self.sources() + with self.assertRaisesRegex(ValueError,'URLs disagree'):spring_rows(sources,'papers') + + def test_poster_without_linked_author_entry_is_held_even_with_abstract(self): + self.pages[SPRING+'posters/author.html']='Doe, JaneSame contribution';sources,_=self.sources();rows,held,counts=spring_rows(sources,'posters') + self.assertEqual(rows,[]);self.assertEqual(counts['abstract_entries'],1);self.assertEqual(held[0]['reason'],'abstract_entry_missing_linked_author_index') + + def test_unlinked_and_panel_entries_never_become_acceptance_or_rejection(self): + sources,_=self.sources();held,counts=early_review(sources);self.assertEqual(counts,{'first_conference_fields':2,'fall_categories':2,'fall_program_entries':3}) + self.assertTrue(any(r['candidate']['title']=='Unlinked talk'for r in held));self.assertTrue(all('status'not in r for r in held)) + self.assertEqual(held[0]['candidate']['authors_raw'],['Jane Doe']) + + def test_title_disagreement_goes_to_review_with_source_claims(self): + self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('Same contribution','Different contribution');sources,_=self.sources();rows,held,_=spring_rows(sources,'papers') + self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'abstract_and_author_index_titles_disagree') + + def test_cache_tampering_and_manifest_hash_mismatch_fail_closed(self): + _,sha=self.sources() + with self.assertRaisesRegex(ValueError,'manifest checksum'):Sources(self.cache,'0'*64) + p=next(self.cache.glob('source-*.html'));p.write_bytes(p.read_bytes()+b'x') + with self.assertRaisesRegex(ValueError,'checksum or size'):Sources(self.cache,sha) + + def test_cache_cannot_read_paths_outside_snapshot(self): + _,sha=self.sources();p=self.cache/'manifest.json';d=json.loads(p.read_text());d['sources'][0]['file']='../outside.html';p.write_text(json.dumps(d)) + with self.assertRaisesRegex(ValueError,'plain files'):Sources(self.cache,hashlib.sha256(p.read_bytes()).hexdigest()) + + def test_keywords_only_is_missing_abstract_not_invented_summary(self): + self.pages[SPRING+'papers/abstract.html']='

Same contribution

Author: Jane Doe

Keywords: networking, hypertext

';sources,_=self.sources();rows,held,_=spring_rows(sources,'papers') + self.assertEqual(rows,[]);self.assertEqual(held[0]['reason'],'missing_abstract') + + def test_old_encoding_and_duplicate_abstract_identity(self): + self.assertEqual(decode(b'M\xfcller')[0],'Müller') + self.pages[SPRING+'papers/abstract.html']*=2;sources,_=self.sources() + with self.assertRaisesRegex(ValueError,'Duplicate abstract'):spring_rows(sources,'papers') + + def test_existing_output_is_never_overwritten(self): + _,sha=self.sources();out=self.root/'batch';out.mkdir();(out/'keep').write_text('original') + with self.assertRaisesRegex(ValueError,'overwrite'):collect(self.cache,sha,out) + self.assertEqual((out/'keep').read_text(),'original')