Deduplicate CVF index links by HTTP resource

Reviewed-on: #16

paper_atlas/collectors/vision.py

@@ -7,7 +7,7 @@ import re
 import time
 from datetime import date
 from urllib.error import HTTPError
-from urllib.parse import parse_qs, unquote, urljoin, urlsplit
+from urllib.parse import parse_qs, unquote, urldefrag, urljoin, urlsplit
 
 from paper_atlas.io.http import fetch
 from paper_atlas.io.html import Document, Node, clean
@@ -147,7 +147,8 @@ def read_index(index,cache):
     links=set()
     for node in Document(html).root.walk():
         if node.tag!='a':continue
-        link=urljoin(url,node.attrs.get('href',''));day=parse_qs(urlsplit(link).query).get('day',[])
+        # Fragments select positions within a page, not separate HTTP resources.
+        link=urldefrag(urljoin(url,node.attrs.get('href','')))[0];day=parse_qs(urlsplit(link).query).get('day',[])
         if index_identity(link)!=index_identity(url) or index_identity(url) is None or len(day)!=1:continue
         if day[0]!='all':
             try:valid=date.fromisoformat(day[0]).year==index['year']

tests/test_vision.py

@@ -37,6 +37,27 @@ class VisionTests(unittest.TestCase):
                 collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
             self.assertFalse(list(Path(tmp).glob('*.jsonl')))
 
+    def test_fragment_variants_share_one_all_papers_resource(self):
+        root,papers,entry=self.cvf_fixture();all_url=root+'?day=all'
+        pages={root:''.join(f'<a href="{all_url}{fragment}">All</a>' for fragment in ('','#','#papers')),
+               all_url:'<dl>'+entry(papers[0])+entry(papers[1])+'</dl>'}
+        pages.update({paper:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>' for paper in papers[:2]})
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
+            manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
+        self.assertEqual(manifest['count'],2)
+        self.assertEqual(manifest['index_urls'],[all_url])
+        self.assertEqual([call.args[0] for call in fetch.call_args_list].count(all_url),1)
+        self.assertFalse(any('#' in call.args[0] for call in fetch.call_args_list))
+
+    def test_distinct_all_papers_queries_still_fail_before_publishing(self):
+        root,_,_=self.cvf_fixture();urls=[root+'?day=all',root+'?day=all&view=alternate']
+        html=''.join(f'<a href="{url}">All</a>' for url in urls).encode()
+        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',return_value=html) as fetch:
+            with self.assertRaisesRegex(ValueError,'Ambiguous all-papers index'):
+                collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
+            self.assertEqual(fetch.call_count,1)
+            self.assertFalse(list(Path(tmp).glob('*.jsonl')))
+
     def test_missing_abstract_page_keeps_official_index_metadata_and_marks_partial(self):
         root,papers,entry=self.cvf_fixture();error=HTTPError(papers[0],404,'Not Found',{},io.BytesIO())
         def fetch(url,*args,**kwargs):