diff --git a/paper_atlas/collectors/vision.py b/paper_atlas/collectors/vision.py index 68233d1..dc39732 100644 --- a/paper_atlas/collectors/vision.py +++ b/paper_atlas/collectors/vision.py @@ -7,7 +7,7 @@ import re import time from datetime import date from urllib.error import HTTPError -from urllib.parse import parse_qs, unquote, urljoin, urlsplit +from urllib.parse import parse_qs, unquote, urldefrag, urljoin, urlsplit from paper_atlas.io.http import fetch from paper_atlas.io.html import Document, Node, clean @@ -147,7 +147,8 @@ def read_index(index,cache): links=set() for node in Document(html).root.walk(): if node.tag!='a':continue - link=urljoin(url,node.attrs.get('href',''));day=parse_qs(urlsplit(link).query).get('day',[]) + # Fragments select positions within a page, not separate HTTP resources. + link=urldefrag(urljoin(url,node.attrs.get('href','')))[0];day=parse_qs(urlsplit(link).query).get('day',[]) if index_identity(link)!=index_identity(url) or index_identity(url) is None or len(day)!=1:continue if day[0]!='all': try:valid=date.fromisoformat(day[0]).year==index['year'] diff --git a/tests/test_vision.py b/tests/test_vision.py index 6572b30..7a7c743 100644 --- a/tests/test_vision.py +++ b/tests/test_vision.py @@ -37,6 +37,27 @@ class VisionTests(unittest.TestCase): collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp) self.assertFalse(list(Path(tmp).glob('*.jsonl'))) + def test_fragment_variants_share_one_all_papers_resource(self): + root,papers,entry=self.cvf_fixture();all_url=root+'?day=all' + pages={root:''.join(f'All' for fragment in ('','#','#papers')), + all_url:'