diff --git a/docs/www-archive.md b/docs/www-archive.md index 59d1fcf..24f1053 100644 --- a/docs/www-archive.md +++ b/docs/www-archive.md @@ -52,7 +52,9 @@ older pages; original bytes remain authoritative for encoding corrections. a regular-paper acceptance or a rejection. Raw category/title/name claims are kept. The main and poster JSONL files have independent manifests with explicit event key, -track, count and SHA256. Their status stays `partial`: a verified linked subset does +track, count and SHA256. `source_listing_count` records the full recognized abstract +listing, while `held_entry_count` records excluded entries; neither is a ready-row +count. Their status stays `partial`: a verified linked subset does not prove complete event coverage. `review-candidates.jsonl` contains nested candidates and reasons, with no importable decision status. It is excluded from `import-queue.json`. `report.json` accounts for ready/held records and every source @@ -69,8 +71,22 @@ and its 59 Fall submissions; that is not an import of the Spring proceedings. Inspect held entries and original author names, preflight every ready row against current PostgreSQL stable identifiers and candidate identities, and verify the -source/year/track/event coverage before running `import records`. Do not automatically +source/year/track/event coverage before running `import records`. When a ready subset +excludes held entries, preserve the independently known listing total in the existing +incremental-coverage protocol: bind a freshly checked parent count and expected count, +and set the full expected count from this manifest. A new source uses previous count +0 and previous expected count null; an existing source requires its actual current +values. These live parent values are not guessed by the offline collector. Do not automatically run this queue against an older writer. Preserve extended versions and independent submission/decision evidence, and keep unresolved identities held. A normalization report or passing tests alone do not establish production import, complete historical coverage or final global deduplication. + +Initial fixed-cache replay (64ced7a, CI1057): 38 main papers and38 posters ready; +372 entries held (50 first-conference fields,307 mixed program entries,15 posters). +Independent report44dfd070 verified source hashes,counts and author evidence, finding +one existing exact-title candidate. That publication is explicitly linked to +arXiv1811.12161 by its WWW1995/journal-reference declaration; arXiv1810.07232 is a +different1997 invited chapter and remains separate despite the overlap note. This +is a preparation observation, not evidence of completed production import or final +global deduplication. diff --git a/paper_atlas/collectors/www_archive.py b/paper_atlas/collectors/www_archive.py index 015d14b..8644978 100644 --- a/paper_atlas/collectors/www_archive.py +++ b/paper_atlas/collectors/www_archive.py @@ -248,7 +248,7 @@ def collect(source_cache,cache_sha256,out): filename='www-1995-spring-'+('main'if track=='papers'else'poster')+'.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n'for r in rows).encode();(stage/filename).write_bytes(raw) manifest={'source':'www_archive','parser_version':1,'venue':'WWW','year':1995,'track':rows[0]['track'],'event_key':'www-1995-spring', 'list_url':SPRING+track+'/abstract.html','count':len(rows),'status':'partial','file_sha256':hashlib.sha256(raw).hexdigest(), - 'missing_abstract_count':0,'source_cache_sha256':cache_sha256,'scope':'Verified linked archive entries; held entries and other event tracks excluded, not complete event coverage.'} + 'missing_abstract_count':0,'source_listing_count':counts[track]['abstract_entries'],'held_entry_count':counts[track]['held'],'source_cache_sha256':cache_sha256,'scope':'Verified linked archive entries; held entries and other event tracks excluded, not complete event coverage.'} (stage/filename).with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');queue.append(str(out/filename)) (stage/'review-candidates.jsonl').write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n'for r in held)) report={'status':'normalized_not_imported','source_cache_sha256':cache_sha256,'counts':counts,'ready':sum(len(r)for r in groups.values()),'held':len(held), diff --git a/tests/test_www_archive.py b/tests/test_www_archive.py index edd7a9e..83e251a 100644 --- a/tests/test_www_archive.py +++ b/tests/test_www_archive.py @@ -41,6 +41,13 @@ class ArchiveTests(unittest.TestCase): self.assertTrue(all('status'not in r and 'source_evidence'in r for r in held)) self.assertFalse(any('review-candidates'in p for p in queue)) + def test_partial_manifest_preserves_full_listing_size_separately_from_ready_count(self): + self.pages[SPRING+'posters/abstract.html']+='

Unlinked poster

Author: Other Person

Actual abstract

' + _,sha=self.sources();out=self.root/'batch';collect(self.cache,sha,out) + manifest=json.loads((out/'www-1995-spring-poster.manifest.json').read_text()) + self.assertEqual((manifest['count'],manifest['source_listing_count'],manifest['held_entry_count']),(1,2,1)) + self.assertNotIn('coverage_previous_count',manifest) # Live parent must be independently preflighted. + def test_malformed_href_requires_exact_id_and_two_clean_index_anchors(self): self.pages[SPRING+'papers/abstract.html']=self.pages[SPRING+'papers/abstract.html'].replace('01/paper.html','01/ paper.html') sources,_=self.sources();rows,held,_=spring_rows(sources,'papers');self.assertEqual(held,[])