Resolve reviewed WWW poster entries with canonical page evidence
Metadata integrity checks / metadata (push) Successful in 1m32s
Metadata integrity checks / metadata (push) Successful in 1m32s
This commit is contained in:
@@ -0,0 +1,201 @@
|
||||
{
|
||||
"format": 1,
|
||||
"source_cache_sha256": "8a65562f3490497cada2919ec77827134beabd5ec9c3690dec3625e2318aa54e",
|
||||
"records": [
|
||||
{
|
||||
"entry_id": "45",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/45/index.html",
|
||||
"individual_sha256": "83bede6afc9a7039fae2dd1f28274a394418b34f5d5632590b913617580bda1a",
|
||||
"title": "WISE - World Wide Information System for R&D; Efforts",
|
||||
"authors": [
|
||||
"Fritz Loseries",
|
||||
"Veronika Samara",
|
||||
"Bodo Urban"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "16",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/16/index.html",
|
||||
"individual_sha256": "cf76da903eaeaa27d19857fdaa13220f6821559bf4ac22844fae10abbaa6f3f3",
|
||||
"title": "The World Wide Web as a Tool for Collaborative Inter-Agency Projects: Creation of an Environmental Metadata Database for the Analysis of the Venice Lagoon Ecosystem.",
|
||||
"authors": [
|
||||
"Mirko Labbri"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "17",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/17/index.html",
|
||||
"individual_sha256": "0578fec9c82e8f2f98c4de0f1bcc44011ebb84c53353fdcc423a69cf70673bc2",
|
||||
"title": "NetRepreneur",
|
||||
"authors": [
|
||||
"Kari-Pekka Turtiainen",
|
||||
"Antti Auer"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "29",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/29/index.html",
|
||||
"individual_sha256": "624ad3f685de1163411e33e90488975d848a63243fd7ffbed0862c6fb58c4140",
|
||||
"title": "Mosaic TueV 2.4.3: Multilingual GUI and Improvements towards User Friendliness",
|
||||
"authors": [
|
||||
"Mike Bretz",
|
||||
"Uwe Koch",
|
||||
"Paul de Bra"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "13",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/13/index.html",
|
||||
"individual_sha256": "02409321a3403a94d8362a9862f4960386ea9c344fb47f4bcdf9eae128e8b46e",
|
||||
"title": "Using a WWW Browser as an Alternative User Interface for Interactive Applications",
|
||||
"authors": [
|
||||
"Frank Lonczewski"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "36",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/36/index.html",
|
||||
"individual_sha256": "ed9dada4ea5903cb8f0a4e75f2958416a6e32647fbf3530e58edd9abf26070ce",
|
||||
"title": "HyperSource: A Hypermedia Program Development And Documentation System",
|
||||
"authors": [
|
||||
"Jan Oliver Borchers"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "57",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/57/index.html",
|
||||
"individual_sha256": "3ad4206ad28a19823172cd2845f636fe7a461c251940e9a93deb8f6484b8a984",
|
||||
"title": "Progressive Image Transmission Capability (PROTRAC) on the World Wide Web",
|
||||
"authors": [
|
||||
"Vojislav Lalich-Petrich",
|
||||
"Gaurav Bhatia",
|
||||
"Larry Davis"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "38",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/38/index.html",
|
||||
"individual_sha256": "940370acc5e6a954b197053fe451e466a904d1f0ac82c8fe4e6ca455faf2c927",
|
||||
"title": "Measuring the Web with Lycos",
|
||||
"authors": [
|
||||
"Michael L. Mauldin"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": "Official abstract listing has no abstract. Canonical HTML has body sections but no explicit abstract; body is not substituted as a summary."
|
||||
},
|
||||
{
|
||||
"entry_id": "05",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/05/index.html",
|
||||
"individual_sha256": "1858be87d6e7f47476e1cb8fb4793c0fce3b8a6ff653bcfd4a35089e0520f177",
|
||||
"title": "Virtual Places - Adding people to the web",
|
||||
"authors": [
|
||||
"Yosi Mass"
|
||||
],
|
||||
"author_source": "author_index",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": "Official abstract listing and canonical HTML have no abstract. Canonical page identifies a contact and links PostScript; no full-text download. Author identity comes from author index, not inferred from contact role."
|
||||
},
|
||||
{
|
||||
"entry_id": "37",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/37/index.html",
|
||||
"individual_sha256": "b700a0a51b7a82a5f88373ae0283bb369d1cf6f455acff7d91cd7313fa8b3adf",
|
||||
"title": "The Internet Public Library: Serving a Global Community",
|
||||
"authors": [
|
||||
"Susan Schweitzer"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "41",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/41/index.html",
|
||||
"individual_sha256": "397b474109f9c1240a77d0f490f2b02dd35fae93febbb7397d1b2a31759a8ec1",
|
||||
"title": "Graphical Browsing in the WorldWideWeb and the MultiMedia Forum",
|
||||
"authors": [
|
||||
"Rainer Pagé",
|
||||
"Klaus Süllow"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "47",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/47/index.html",
|
||||
"individual_sha256": "52eb4bfdd658730a222c98170b72dabf9cbcc5ffd2ada1e900b826baf2b02833",
|
||||
"title": "Interoperating Advanced Hypermedia Systems with WWW",
|
||||
"authors": [
|
||||
"Jian Zhang",
|
||||
"Martin Fruehauf",
|
||||
"Thomas Kirste"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "50",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/50/index.html",
|
||||
"individual_sha256": "8dc8b0561bd8b1b106c5b209a973c695c3cc9df5288450d4a01fe32541476b7e",
|
||||
"title": "Web Enabled Agile Manufacturing",
|
||||
"authors": [
|
||||
"S. V. Iyer",
|
||||
"T. Singh",
|
||||
"U. Ravaioli",
|
||||
"R. E. DeVor",
|
||||
"S. G. Kapoor",
|
||||
"G. Veknatasubramian"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "55",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/55/index.html",
|
||||
"individual_sha256": "21780cd4047f513e8c9aeced5247002e27cf71719f08de85c6531b57a9e5058d",
|
||||
"title": "Weblets: Fundamental Building Blocks for WWW Tools",
|
||||
"authors": [
|
||||
"Thomas J. Watt Jr."
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
},
|
||||
{
|
||||
"entry_id": "59",
|
||||
"individual_url": "https://archives.iw3c2.org/www3/proceedings/posters/59/index.html",
|
||||
"individual_sha256": "fed4cc9dcaee350a12c5e4b6fa9c99be9bead25a81632512d31566400d3d0fad",
|
||||
"title": "Usenet and the World-Wide Web",
|
||||
"authors": [
|
||||
"M. L. Grant"
|
||||
],
|
||||
"author_source": "canonical_byline",
|
||||
"reason": "Reviewed canonical H1 and explicit authorship for this exact linked poster entry; retain abstract-heading and author-index title variants without fuzzy matching.",
|
||||
"missing_abstract_reason": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -90,3 +90,22 @@ arXiv1811.12161 by its WWW1995/journal-reference declaration; arXiv1810.07232 is
|
||||
different1997 invited chapter and remains separate despite the overlap note. This
|
||||
is a preparation observation, not evidence of completed production import or final
|
||||
global deduplication.
|
||||
|
||||
## 以单篇官方页面核实待处理 poster
|
||||
|
||||
作者索引漏链接、摘要目录使用短标题或标题存在拼写差异时,不放宽全局标题匹配。可以先取得单篇官方 HTML,人工核对 H1、署名区与原目录同一 entry URL,再提供带 SHA256 的逐条核查文件:
|
||||
|
||||
```sh
|
||||
paper-atlas collect www-archive \
|
||||
--source-cache /data/paper-index/collected/www-poster-reviewed-source-v2 \
|
||||
--source-cache-sha256 8a65562f3490497cada2919ec77827134beabd5ec9c3690dec3625e2318aa54e \
|
||||
--reviewed-posters docs/coverage/www-1995-posters-reviewed-2026-11-09.json \
|
||||
--reviewed-posters-sha256 <核查文件的SHA256> \
|
||||
--out <新的输出目录>
|
||||
```
|
||||
|
||||
[1995 年15条poster的核查输入](coverage/www-1995-posters-reviewed-2026-11-09.json) 绑定61页来源快照:原46页和补取的15页单篇HTML。程序验证输入文件及快照哈希、单篇页面哈希、完全一致的 canonical H1、原摘要目录中的同一URL/entry ID及作者证据,不通过相似标题猜身份。规范标题采用单篇H1,摘要目录标题及作者索引标题均在 metadata 中保留。
|
||||
|
||||
`author_source=canonical_byline` 表示名字出现在H1之后、摘要/关键词或下一节标题之前的署名区域,由逐条核查明确选出,不从全文引用或机构猜作者;保留原始缩写、顺序、重音和拼写。只有 `author_source=author_index` 时使用原作者索引顺序并逐项核对;05的单篇页面写的是联系人,作者身份使用原作者索引证据,不由联系人角色推断。核查输入是审阅后的数据声明,程序的字符串/哈希检查不代替人工识别署名语义。
|
||||
|
||||
38和05的来源没有可明确引用的摘要。保留 `abstract=null` 和逐条缺失原因,manifest准确报告 missing_abstract_count,继续标partial;正文、联系方式及PostScript链接不当作摘要,也不下载正文。缺摘要不等于拒稿或未发表。本批输出仍包含原先已验收记录,正式导入前必须只取新增external ID,检查实时覆盖父记录并按incremental模式准备,不能直接重导入全部输出。
|
||||
|
||||
@@ -145,7 +145,68 @@ def sections(html):
|
||||
return result
|
||||
|
||||
|
||||
def spring_rows(sources,track):
|
||||
def reviewed_posters(sources, path, expected_sha256, cache_sha256):
|
||||
if path is None:
|
||||
if expected_sha256:raise ValueError('Reviewed poster path required')
|
||||
return {},None
|
||||
raw=Path(path).read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest()!=expected_sha256:raise ValueError('Reviewed poster checksum mismatch')
|
||||
review=json.loads(raw)
|
||||
if set(review)!={'format','source_cache_sha256','records'} or review['format']!=1 or review['source_cache_sha256']!=cache_sha256:
|
||||
raise ValueError('Reviewed poster snapshot binding differs')
|
||||
if not isinstance(review['records'],list) or not 0<len(review['records'])<=100:
|
||||
raise ValueError('Bounded nonempty poster review required')
|
||||
result={}
|
||||
for row in review['records']:
|
||||
if set(row)!={'entry_id','individual_url','individual_sha256','title','authors','author_source','reason','missing_abstract_reason'}:
|
||||
raise ValueError('Unexpected reviewed poster fields')
|
||||
key=row['entry_id'];own=identity(row['individual_url'],'posters')
|
||||
if not isinstance(key,str) or not own or own[0]!=key or key in result:
|
||||
raise ValueError('Reviewed poster identity duplicated or outside canonical entry')
|
||||
if row['author_source'] not in ('canonical_byline','author_index') or not isinstance(row['authors'],list) or not row['authors'] or any(not isinstance(a,str) or not a.strip()for a in row['authors']):
|
||||
raise ValueError('Explicit author evidence required')
|
||||
if len(set(row['authors']))!=len(row['authors']) or not isinstance(row['reason'],str) or not row['reason'].strip():
|
||||
raise ValueError('Reviewed author order/reason invalid')
|
||||
if sources.rows.get(own[1],{}).get('receipt',{}).get('sha256')!=row['individual_sha256']:
|
||||
raise ValueError('Individual poster page checksum differs')
|
||||
page=sources.read(own[1]);headings=re.findall(r'<h1\b[^>]*>(.*?)</h1\s*>',page,re.I|re.S)
|
||||
if len(headings)!=1 or text(headings[0])!=row['title']:
|
||||
raise ValueError('Reviewed title differs from exact canonical H1')
|
||||
header=re.split(r'</h1\s*>',page,maxsplit=1,flags=re.I)[1]
|
||||
header=re.split(r'<h[23]\b',header,maxsplit=1,flags=re.I)[0]
|
||||
header=re.split(r'\b(?:keywords|abstract)\s*:',text(header),maxsplit=1,flags=re.I)[0]
|
||||
if row['author_source']=='canonical_byline' and any(a.casefold() not in header.casefold()for a in row['authors']):
|
||||
raise ValueError('Reviewed author absent from canonical byline region')
|
||||
result[key]={**row,'canonical_byline_region_raw':header}
|
||||
return result,hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def reviewed_row(sources,key,url,item,authors,review):
|
||||
if url!=review['individual_url']:
|
||||
raise ValueError('Reviewed canonical page differs from abstract-entry URL')
|
||||
if authors and authors['urls']!={url}:
|
||||
raise ValueError('Reviewed poster author-index URL disagrees')
|
||||
if review['author_source']=='author_index':
|
||||
names=[]
|
||||
for name in authors['authors']if authors else []:
|
||||
parts=name.split(',');names.append(clean(parts[1]+' '+parts[0])if len(parts)==2 else name)
|
||||
if names!=review['authors']:raise ValueError('Reviewed author-index evidence differs')
|
||||
missing=review['missing_abstract_reason']
|
||||
if not isinstance(missing,str) or bool(missing.strip())==bool(item['abstract']):
|
||||
raise ValueError('Missing abstract requires explicit reason; never invent a summary')
|
||||
urls=[SPRING+'posters/author.html',SPRING+'posters/abstract.html',url]
|
||||
return {'source':'www_archive','external_id':'www3:posters:'+key,'url':url,'title':review['title'],
|
||||
'abstract':item['abstract'],'authors':review['authors'],'identifiers':{},'venue':'WWW','year':1995,
|
||||
'track':'poster','event_key':'www-1995-spring','status':'accepted','raw_decision':'listed in official posters proceedings',
|
||||
'metadata':{'entry_id':key,'source_evidence':sources.proof(urls),'abstract_heading_title_raw':item['title'],
|
||||
'author_byline_raw':item['author_byline_raw'],'author_index_raw':authors['authors']if authors else [],
|
||||
'author_index_titles_raw':sorted(authors['titles'])if authors else [],'abstract_href_raw':url,
|
||||
'author_order':review['author_source'],'reviewed_poster_identity':review,
|
||||
'missing_abstract_reason':missing or None},
|
||||
'tags':[{'namespace':'www_archive:proceedings_section','value':'posters'}]}
|
||||
|
||||
|
||||
def spring_rows(sources,track,reviews=None):
|
||||
base=SPRING+track+'/';author_url=base+'author.html';abstract_url=base+'abstract.html'
|
||||
authors=AuthorIndex(sources.read(author_url),track).entries
|
||||
listing=sections(sources.read(abstract_url));by_id={};held=[];session={}
|
||||
@@ -168,8 +229,13 @@ def spring_rows(sources,track):
|
||||
if track=='papers' and (set(authors)!=set(by_id) or set(session)!=set(by_id)):
|
||||
raise ValueError('Main author/session/abstract entry ID sets disagree')
|
||||
if set(authors)-set(by_id):raise ValueError('Author index has entries outside the abstract listing')
|
||||
reviews=reviews or {}
|
||||
if set(reviews)-set(by_id):raise ValueError('Reviewed poster outside abstract listing')
|
||||
rows=[]
|
||||
for key,(url,item) in by_id.items():
|
||||
if key in reviews:
|
||||
if track!='posters':raise ValueError('Reviews only apply to poster entries')
|
||||
rows.append(reviewed_row(sources,key,url,item,authors.get(key),reviews[key]));continue
|
||||
if key not in authors:
|
||||
held.append({'reason':'abstract_entry_missing_linked_author_index','entry_id':key,'evidence':abstract_url,'candidate':item});continue
|
||||
alternatives=authors[key]['urls']
|
||||
@@ -231,12 +297,13 @@ def early_review(sources):
|
||||
'fall_program_entries':sum(r['event_key']=='www-1994-fall'for r in held)}
|
||||
|
||||
|
||||
def collect(source_cache,cache_sha256,out):
|
||||
def collect(source_cache,cache_sha256,out,*,reviewed=None,reviewed_sha256=None):
|
||||
out=Path(out)
|
||||
if out.exists():raise ValueError('Refusing to overwrite an existing normalized batch')
|
||||
sources=Sources(source_cache,cache_sha256);groups={};held=[];counts={}
|
||||
reviews,review_sha=reviewed_posters(sources,reviewed,reviewed_sha256,cache_sha256)
|
||||
for track in ('papers','posters'):
|
||||
rows,review,counts[track]=spring_rows(sources,track);groups[track]=rows;held.extend(review)
|
||||
rows,review,counts[track]=spring_rows(sources,track,reviews if track=='posters'else None);groups[track]=rows;held.extend(review)
|
||||
review,counts['earlier']=early_review(sources);held.extend(review)
|
||||
for item in held:
|
||||
if 'source_evidence' not in item:item['source_evidence']=sources.proof([item['evidence']])
|
||||
@@ -246,13 +313,14 @@ def collect(source_cache,cache_sha256,out):
|
||||
for track,rows in groups.items():
|
||||
if not rows:continue
|
||||
filename='www-1995-spring-'+('main'if track=='papers'else'poster')+'.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n'for r in rows).encode();(stage/filename).write_bytes(raw)
|
||||
manifest={'source':'www_archive','parser_version':1,'venue':'WWW','year':1995,'track':rows[0]['track'],'event_key':'www-1995-spring',
|
||||
manifest={'source':'www_archive','parser_version':2 if review_sha else 1,'venue':'WWW','year':1995,'track':rows[0]['track'],'event_key':'www-1995-spring',
|
||||
'list_url':SPRING+track+'/abstract.html','count':len(rows),'status':'partial','file_sha256':hashlib.sha256(raw).hexdigest(),
|
||||
'missing_abstract_count':0,'source_listing_count':counts[track]['abstract_entries'],'held_entry_count':counts[track]['held'],'source_cache_sha256':cache_sha256,'scope':'Verified linked archive entries; held entries and other event tracks excluded, not complete event coverage.'}
|
||||
'missing_abstract_count':sum(not bool((r['abstract']or '').strip())for r in rows),'source_listing_count':counts[track]['abstract_entries'],'held_entry_count':counts[track]['held'],'source_cache_sha256':cache_sha256,'scope':'Verified linked archive entries; held entries and other event tracks excluded, not complete event coverage.'}
|
||||
(stage/filename).with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');queue.append(str(out/filename))
|
||||
(stage/'review-candidates.jsonl').write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n'for r in held))
|
||||
report={'status':'normalized_not_imported','source_cache_sha256':cache_sha256,'counts':counts,'ready':sum(len(r)for r in groups.values()),'held':len(held),
|
||||
'source_evidence':sources.proof(sources.used),'files':queue,'scope':'Early WWW archive only; no PDF fetching, no rejected/withdrawn inference, no database access. Event-aware importer required.'}
|
||||
if review_sha:report['reviewed_posters_sha256']=review_sha
|
||||
(stage/'report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||
(stage/'import-queue.json').write_text(json.dumps({'files':queue},indent=2)+'\n')
|
||||
stage.rename(out)
|
||||
@@ -261,10 +329,12 @@ def collect(source_cache,cache_sha256,out):
|
||||
|
||||
def main(argv=None,*,prog=None):
|
||||
p=argparse.ArgumentParser(prog=prog);p.add_argument('--source-cache',required=True);p.add_argument('--source-cache-sha256');p.add_argument('--http-cache');p.add_argument('--out',required=True)
|
||||
p.add_argument('--reviewed-posters');p.add_argument('--reviewed-posters-sha256')
|
||||
args=p.parse_args(argv)
|
||||
if bool(args.http_cache)==bool(args.source_cache_sha256):p.error('Choose a fresh --http-cache download or an existing --source-cache-sha256 snapshot')
|
||||
sha=download_sources(args.http_cache,args.source_cache)if args.http_cache else args.source_cache_sha256
|
||||
print(json.dumps(collect(args.source_cache,sha,args.out),ensure_ascii=False),flush=True)
|
||||
if bool(args.reviewed_posters)!=bool(args.reviewed_posters_sha256):p.error('Reviewed poster file and SHA256 must be supplied together')
|
||||
print(json.dumps(collect(args.source_cache,sha,args.out,reviewed=args.reviewed_posters,reviewed_sha256=args.reviewed_posters_sha256),ensure_ascii=False),flush=True)
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
|
||||
@@ -99,3 +99,57 @@ class ArchiveTests(unittest.TestCase):
|
||||
_,sha=self.sources();out=self.root/'batch';out.mkdir();(out/'keep').write_text('original')
|
||||
with self.assertRaisesRegex(ValueError,'overwrite'):collect(self.cache,sha,out)
|
||||
self.assertEqual((out/'keep').read_text(),'original')
|
||||
|
||||
def review(self,*,authors=None,author_source='canonical_byline',missing='',page=None):
|
||||
url=SPRING+'posters/01/paper.html'
|
||||
self.pages[url]=page or '<h1>Canonical full title</h1><p>Jane Doe, John Roe</p><p>Keywords: web</p><h2>Abstract</h2><p>Body</p>'
|
||||
_,sha=self.sources();raw=self.pages[url].encode()if isinstance(self.pages[url],str)else self.pages[url]
|
||||
row=dict(entry_id='01',individual_url=url,individual_sha256=hashlib.sha256(raw).hexdigest(),
|
||||
title='Canonical full title',authors=authors or ['Jane Doe','John Roe'],author_source=author_source,
|
||||
reason='Reviewed exact canonical entry and its explicit authorship; retain original title variants',missing_abstract_reason=missing)
|
||||
p=self.root/'review.json';p.write_text(json.dumps({'format':1,'source_cache_sha256':sha,'records':[row]}));return sha,p,hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
|
||||
def test_reviewed_variant_keeps_primary_title_and_all_old_title_claims(self):
|
||||
sha,p,h=self.review();out=self.root/'reviewed';r=collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
|
||||
row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['title'],'Canonical full title')
|
||||
self.assertEqual(row['metadata']['abstract_heading_title_raw'],'Same contribution');self.assertEqual(row['metadata']['author_index_titles_raw'],['Same contribution'])
|
||||
self.assertEqual(len(row['metadata']['source_evidence']),3);self.assertEqual(r['reviewed_posters_sha256'],h)
|
||||
self.assertEqual(row['external_id'],'www3:posters:01');self.assertEqual(row['status'],'accepted')
|
||||
|
||||
def test_canonical_byline_can_resolve_unlinked_author_index_without_guessing(self):
|
||||
self.pages[SPRING+'posters/author.html']='<b>Doe, Jane</b><a href="#">Broken index link</a>'
|
||||
sha,p,h=self.review();out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
|
||||
row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());self.assertEqual(row['authors'],['Jane Doe','John Roe'])
|
||||
self.assertEqual(row['metadata']['author_index_raw'],[]);self.assertEqual(row['metadata']['author_order'],'canonical_byline')
|
||||
|
||||
def test_reviewed_missing_abstract_is_explicit_partial_not_body_summary(self):
|
||||
self.pages[SPRING+'posters/abstract.html']='<h3><a href="01/paper.html">Same contribution</a></h3><i>Author: Jane Doe</i><p>Keywords: web</p>'
|
||||
sha,p,h=self.review(missing='Neither listing nor canonical page supplies an explicit abstract')
|
||||
out=self.root/'reviewed';collect(self.cache,sha,out,reviewed=p,reviewed_sha256=h)
|
||||
row=json.loads((out/'www-1995-spring-poster.jsonl').read_text());m=json.loads((out/'www-1995-spring-poster.manifest.json').read_text())
|
||||
self.assertIsNone(row['abstract']);self.assertEqual((m['missing_abstract_count'],m['status']),(1,'partial'))
|
||||
sha,p,h=self.review()
|
||||
with self.assertRaisesRegex(ValueError,'Missing abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
self.assertFalse((self.root/'bad').exists())
|
||||
|
||||
def test_reviewed_names_must_be_in_byline_region_not_citations_or_contact_guess(self):
|
||||
sha,p,h=self.review(authors=['Unrelated Person'],page='<h1>Canonical full title</h1><p>Jane Doe</p><h2>Abstract</h2><p>Unrelated Person cited here</p>')
|
||||
with self.assertRaisesRegex(ValueError,'byline region'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
sha,p,h=self.review(author_source='author_index',authors=['Contact Person'])
|
||||
with self.assertRaisesRegex(ValueError,'author-index'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
|
||||
def test_review_hash_cache_binding_and_canonical_h1_are_all_required(self):
|
||||
sha,p,h=self.review()
|
||||
with self.assertRaisesRegex(ValueError,'checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256='0'*64)
|
||||
d=json.loads(p.read_text());d['source_cache_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
with self.assertRaisesRegex(ValueError,'binding'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
d['source_cache_sha256']=sha;d['records'][0]['title']='Wrong title';p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
with self.assertRaisesRegex(ValueError,'canonical H1'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
|
||||
def test_unlisted_review_entry_and_changed_primary_digest_fail_closed(self):
|
||||
sha,p,h=self.review();d=json.loads(p.read_text());d['records'][0]['individual_sha256']='0'*64;p.write_text(json.dumps(d));h=hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
with self.assertRaisesRegex(ValueError,'page checksum'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
self.pages[SPRING+'posters/abstract.html']=self.pages[SPRING+'posters/abstract.html'].replace('01/paper.html','02/other.html')
|
||||
self.pages[SPRING+'posters/author.html']=self.pages[SPRING+'posters/author.html'].replace('01/paper.html','02/other.html')
|
||||
sha,p,h=self.review()
|
||||
with self.assertRaisesRegex(ValueError,'outside abstract'):collect(self.cache,sha,self.root/'bad',reviewed=p,reviewed_sha256=h)
|
||||
|
||||
Reference in New Issue
Block a user