Deduplicate CVF index links by HTTP resource

paper_atlas/audits/public_openreview.py

@@ -39,9 +39,7 @@ def audit(db, inventory, out, *, pg_service=None, pg_schema=None):
             if json.loads(path.with_suffix('.manifest.json').read_text()) != manifest:
                 raise ValueError('Collected manifest differs from inventory')
             count = 0
-            # Unicode paragraph/line separators are valid inside JSON strings;
-            # split physical byte lines, not Unicode text boundaries.
-            for line in path.read_bytes().splitlines():
+            for line in path.read_text().splitlines():
                 row = json.loads(line)
                 if any(row[key] != manifest[key] for key in ('source', 'venue', 'year', 'track')):
                     raise ValueError('Record edition differs from inventory')

paper_atlas/collectors/vision.py

@@ -7,7 +7,7 @@ import re
 import time
 from datetime import date
 from urllib.error import HTTPError
-from urllib.parse import parse_qs, unquote, urljoin, urlsplit
+from urllib.parse import parse_qs, unquote, urldefrag, urljoin, urlsplit
 
 from paper_atlas.io.http import fetch
 from paper_atlas.io.html import Document, Node, clean
@@ -147,7 +147,8 @@ def read_index(index,cache):
     links=set()
     for node in Document(html).root.walk():
         if node.tag!='a':continue
-        link=urljoin(url,node.attrs.get('href',''));day=parse_qs(urlsplit(link).query).get('day',[])
+        # Fragments select positions within a page, not separate HTTP resources.
+        link=urldefrag(urljoin(url,node.attrs.get('href','')))[0];day=parse_qs(urlsplit(link).query).get('day',[])
         if index_identity(link)!=index_identity(url) or index_identity(url) is None or len(day)!=1:continue
         if day[0]!='all':
             try:valid=date.fromisoformat(day[0]).year==index['year']

tests/test_public_openreview.py

@@ -152,35 +152,6 @@ class PublicOpenReviewTests(unittest.TestCase):
             finally:
                 store.conn.close()
 
-    def test_unicode_separators_in_abstract_survive_audit_and_import(self):
-        for separator in ('\u0085', '\u2028', '\u2029'):
-            with self.subTest(separator=repr(separator)), tempfile.TemporaryDirectory() as tmp:
-                root = Path(tmp)
-                abstract = 'First paragraph' + separator + 'Second paragraph'
-                result = self.collect(root / 'source', {
-                    'venues/icml/icml2025.jsonl': [self.row(abstract=abstract)]})
-                path = Path(result['files'][0]['file'])
-                data = path.read_bytes()
-                self.assertIn(separator.encode(), data)
-                self.assertEqual(data.count(b'\n'), 1)
-                db = root / 'db.sqlite'
-                store = Store(db)
-                store.conn.close()
-                inventory = root / 'source/public-openreview-inventory.json'
-                before = audit(db, inventory, root / 'before.json')
-                self.assertEqual(before['status'], 'ready_for_serial_import')
-                self.assertEqual(before['records_checked'], 1)
-                ingest_file(db, path)
-                after = audit(db, inventory, root / 'after.json')
-                self.assertEqual(after['status'], 'ready_for_serial_import')
-                self.assertEqual(after['existing_same_edition_attempts'], 1)
-                store = Store(db)
-                try:
-                    self.assertEqual(store.conn.execute(
-                        'SELECT abstract FROM source_records').fetchone()[0], abstract)
-                finally:
-                    store.conn.close()
-
 
 if __name__ == '__main__':
     unittest.main()

tests/test_vision.py

@@ -37,6 +37,27 @@ class VisionTests(unittest.TestCase):
                 collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
             self.assertFalse(list(Path(tmp).glob('*.jsonl')))
 
+    def test_fragment_variants_share_one_all_papers_resource(self):
+        root,papers,entry=self.cvf_fixture();all_url=root+'?day=all'
+        pages={root:''.join(f'<a href="{all_url}{fragment}">All</a>' for fragment in ('','#','#papers')),
+               all_url:'<dl>'+entry(papers[0])+entry(papers[1])+'</dl>'}
+        pages.update({paper:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>' for paper in papers[:2]})
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
+            manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
+        self.assertEqual(manifest['count'],2)
+        self.assertEqual(manifest['index_urls'],[all_url])
+        self.assertEqual([call.args[0] for call in fetch.call_args_list].count(all_url),1)
+        self.assertFalse(any('#' in call.args[0] for call in fetch.call_args_list))
+
+    def test_distinct_all_papers_queries_still_fail_before_publishing(self):
+        root,_,_=self.cvf_fixture();urls=[root+'?day=all',root+'?day=all&view=alternate']
+        html=''.join(f'<a href="{url}">All</a>' for url in urls).encode()
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',return_value=html) as fetch:
+            with self.assertRaisesRegex(ValueError,'Ambiguous all-papers index'):
+                collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
+            self.assertEqual(fetch.call_count,1)
+            self.assertFalse(list(Path(tmp).glob('*.jsonl')))
+
     def test_missing_abstract_page_keeps_official_index_metadata_and_marks_partial(self):
         root,papers,entry=self.cvf_fixture();error=HTTPError(papers[0],404,'Not Found',{},io.BytesIO())
         def fetch(url,*args,**kwargs):