168 lines
13 KiB
Python
168 lines
13 KiB
Python
import tempfile
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from urllib.error import HTTPError
|
|
|
|
from paper_atlas.collectors.vision import collect_index, decode_html, discover, index_entries, parse_paper,main
|
|
|
|
|
|
class VisionTests(unittest.TestCase):
|
|
def cvf_fixture(self,year=2018):
|
|
root=f'https://openaccess.thecvf.com/CVPR{year}.py'
|
|
papers=[f'https://openaccess.thecvf.com/content_cvpr_{year}/html/Paper{i}_CVPR_{year}_paper.html' for i in (1,2,3)]
|
|
def entry(url,title='Paper'):
|
|
return f'<dt><a href="{url}">{title}</a></dt><dd>Jane Doe, John Smith</dd>'
|
|
return root,papers,entry
|
|
|
|
def test_all_explicit_days_are_combined_and_overlapping_papers_are_not_duplicated(self):
|
|
root,papers,entry=self.cvf_fixture();day1=root+'?day=2018-06-19';day2=root+'?day=2018-06-20'
|
|
pages={root:f'<a href="{day1}">Day1</a><a href="{day2}">Day2</a><a href="https://evil.example/CVPR2018.py?day=all">Other</a><a href="/CVPR2019.py?day=all">Wrong year</a>',day1:'<dl>'+entry(papers[0])+entry(papers[1])+'</dl>',day2:'<dl>'+entry(papers[1])+entry(papers[2])+'</dl>'}
|
|
pages.update({paper:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>' for paper in papers})
|
|
with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
|
|
manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
|
|
rows=[json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
|
|
self.assertEqual(manifest['count'],3);self.assertEqual(manifest['raw_index_entry_count'],4)
|
|
self.assertEqual(manifest['index_urls'],[day1,day2]);self.assertEqual(manifest['status'],'collected')
|
|
self.assertEqual(rows[1]['metadata']['index_urls'],[day1,day2])
|
|
self.assertEqual({call.args[0] for call in fetch.call_args_list},set(pages))
|
|
|
|
def test_conflicting_copies_across_days_fail_before_publishing(self):
|
|
root,papers,entry=self.cvf_fixture();days=[root+'?day=2018-06-19',root+'?day=2018-06-20']
|
|
pages={root:''.join(f'<a href="{day}">Day</a>' for day in days),days[0]:'<dl>'+entry(papers[0],'First title')+'</dl>',days[1]:'<dl>'+entry(papers[0],'Different title')+'</dl>'}
|
|
with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()):
|
|
with self.assertRaisesRegex(ValueError,'Conflicting metadata'):
|
|
collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
|
|
self.assertFalse(list(Path(tmp).glob('*.jsonl')))
|
|
|
|
def test_fragment_variants_share_one_all_papers_resource(self):
|
|
root,papers,entry=self.cvf_fixture();all_url=root+'?day=all'
|
|
pages={root:''.join(f'<a href="{all_url}{fragment}">All</a>' for fragment in ('','#','#papers')),
|
|
all_url:'<dl>'+entry(papers[0])+entry(papers[1])+'</dl>'}
|
|
pages.update({paper:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>' for paper in papers[:2]})
|
|
with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
|
|
manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
|
|
self.assertEqual(manifest['count'],2)
|
|
self.assertEqual(manifest['index_urls'],[all_url])
|
|
self.assertEqual([call.args[0] for call in fetch.call_args_list].count(all_url),1)
|
|
self.assertFalse(any('#' in call.args[0] for call in fetch.call_args_list))
|
|
|
|
def test_distinct_all_papers_queries_still_fail_before_publishing(self):
|
|
root,_,_=self.cvf_fixture();urls=[root+'?day=all',root+'?day=all&view=alternate']
|
|
html=''.join(f'<a href="{url}">All</a>' for url in urls).encode()
|
|
with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',return_value=html) as fetch:
|
|
with self.assertRaisesRegex(ValueError,'Ambiguous all-papers index'):
|
|
collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
|
|
self.assertEqual(fetch.call_count,1)
|
|
self.assertFalse(list(Path(tmp).glob('*.jsonl')))
|
|
|
|
def test_missing_abstract_page_keeps_official_index_metadata_and_marks_partial(self):
|
|
root,papers,entry=self.cvf_fixture();error=HTTPError(papers[0],404,'Not Found',{},io.BytesIO())
|
|
def fetch(url,*args,**kwargs):
|
|
if url==root:return ('<dl>'+entry(papers[0],'Known publication')+'</dl>').encode()
|
|
raise error
|
|
with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch):
|
|
manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
|
|
row=json.loads(Path(manifest['file']).read_text())
|
|
self.assertEqual(row['title'],'Known publication');self.assertIsNone(row['abstract'])
|
|
self.assertEqual(row['authors'],['Jane Doe','John Smith']);self.assertEqual(row['evidence_url'],root)
|
|
self.assertEqual(manifest['status'],'partial');self.assertEqual(manifest['missing_abstract_pages'],[papers[0]])
|
|
self.assertEqual(row['metadata']['abstract_fetch_error']['status'],404)
|
|
|
|
def test_forbidden_paper_is_not_replaced_with_a_fallback(self):
|
|
root,papers,entry=self.cvf_fixture()
|
|
def fetch(url,*args,**kwargs):
|
|
if url==root:return ('<dl>'+entry(papers[0])+'</dl>').encode()
|
|
raise HTTPError(url,403,'Forbidden',{},io.BytesIO())
|
|
with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch):
|
|
with self.assertRaises(HTTPError):collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
|
|
self.assertFalse(list(Path(tmp).glob('*.jsonl')))
|
|
|
|
def test_broken_all_papers_link_can_use_all_advertised_days(self):
|
|
root,papers,entry=self.cvf_fixture();day=root+'?day=2018-06-19';all_url=root+'?day=all'
|
|
pages={root:f'<a href="{day}">Day</a><a href="{all_url}">All</a>',day:'<dl>'+entry(papers[0])+'</dl>',papers[0]:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>'}
|
|
def fetch(url,*a,**kw):
|
|
if url==all_url:raise HTTPError(url,404,'Not Found',{},io.BytesIO())
|
|
return pages[url].encode()
|
|
with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch):
|
|
manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
|
|
self.assertEqual(manifest['index_urls'],[day]);self.assertEqual(manifest['index_errors'],[{'url':all_url,'status':404}])
|
|
|
|
def test_selective_retry_uses_separate_output_and_leaves_original_catalogue_intact(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
old=Path(tmp)/'old.json';payload={'CVPR:2018:main':{'status':'blocked'},'CVPR:2019:main':{'status':'collected'}};old.write_text(json.dumps(payload))
|
|
indexes=[{'venue':'CVPR','year':year,'track':'main','index_url':f'https://openaccess.thecvf.com/CVPR{year}'} for year in (2018,2019)]
|
|
output=Path(tmp)/'repaired'
|
|
with patch('paper_atlas.collectors.vision.discover',return_value=indexes),patch('paper_atlas.collectors.vision.collect_index',return_value={'status':'collected','count':1}) as collect:
|
|
main(['--cache',tmp,'--out',str(output),'--retry-catalogue',str(old)],source='cvf')
|
|
self.assertEqual(collect.call_count,1);self.assertEqual(collect.call_args.args[1]['year'],2018)
|
|
self.assertEqual(json.loads(old.read_text()),payload)
|
|
self.assertEqual(set(json.loads((output/'cvf-index-status.json').read_text())),{'CVPR:2018:main'})
|
|
|
|
def entry(self):
|
|
return {'url':'https://www.ecva.net/papers/eccv_2024/papers_ECCV/html/1_ECCV_2024_paper.php','index_title':'Paper','index_authors':'Jane Doe*, John Smith','doi':'10.1007/example_1'}
|
|
|
|
def test_ecva_doi_belongs_to_its_own_title_entry(self):
|
|
html='<dl><dt class="ptitle"><a href="papers/eccv_2024/papers_ECCV/html/1_ECCV_2024_paper.php">First paper</a></dt><dd>Jane Doe*, John Smith</dd><dd><a href="https://link.springer.com/chapter/10.1007/example_1">DOI</a></dd><dt class="ptitle"><a href="papers/eccv_2024/papers_ECCV/html/2_ECCV_2024_paper.php">Second paper</a></dt><dd>Other Author</dd><dd><a href="https://doi.org/10.1007/example_2">DOI</a></dd></dl>'
|
|
entries,count=index_entries(html,'https://www.ecva.net/papers.php')
|
|
self.assertEqual(count,2)
|
|
self.assertEqual([e['doi'] for e in entries],['10.1007/example_1','10.1007/example_2'])
|
|
html='<div id="papertitle">Paper</div><div id="abstract">Abstract citing <a href="https://arxiv.org/abs/1706.03762">another paper</a> and <a href="https://doi.org/10.1007/wrong">a reference</a>.</div>'
|
|
row=parse_paper(html,entries[0],'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php')
|
|
self.assertEqual(row['identifiers'],{'doi':'10.1007/example_1'})
|
|
self.assertEqual(row['authors'],['Jane Doe','John Smith'])
|
|
|
|
def test_cvf_meta_authors_and_missing_abstract_never_capture_footer(self):
|
|
html='<meta name="citation_title" content="A vision paper"><meta name="citation_author" content="Doe, Jane"><div id="authors">Do not include the proceedings citation</div><p>Copyright footer</p>'
|
|
entry=dict(self.entry(),doi=None)
|
|
row=parse_paper(html,entry,'cvf','CVPR',2013,'main','https://openaccess.thecvf.com/CVPR2013')
|
|
self.assertEqual(row['authors'],['Jane Doe'])
|
|
self.assertIsNone(row['abstract'])
|
|
self.assertEqual(row['title'],'A vision paper')
|
|
|
|
def test_conflicting_self_doi_is_rejected(self):
|
|
html='<meta name="citation_title" content="Paper"><meta name="citation_doi" content="10.1007/different">'
|
|
with self.assertRaisesRegex(ValueError,'DOI differs'):
|
|
parse_paper(html,self.entry(),'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php')
|
|
|
|
def test_legacy_ecva_author_formats_do_not_become_extra_people(self):
|
|
entry=dict(self.entry(),index_authors='Doe, Jane and Smith, John')
|
|
page='<div id="papertitle">Paper</div><div id="authors">Jane Doe, John Smith; The European Conference on Computer Vision (ECCV), 2018, pp. 1-12</div>'
|
|
row=parse_paper(page,entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php')
|
|
self.assertEqual(row['authors'],['Jane Doe','John Smith'])
|
|
row=parse_paper('<div id="papertitle">Paper</div>',entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php')
|
|
self.assertEqual(row['authors'],['Jane Doe','John Smith'])
|
|
|
|
def test_cvpr_day_one_is_replaced_by_explicit_all_papers_index(self):
|
|
root='https://openaccess.thecvf.com/CVPR2026';all_url=root+'?day=all'
|
|
first='https://openaccess.thecvf.com/content/CVPR2026/html/First_CVPR_2026_paper.html'
|
|
second='https://openaccess.thecvf.com/content/CVPR2026/html/Second_CVPR_2026_paper.html'
|
|
def item(url):return f'<dt class="ptitle"><a href="{url}">Paper</a></dt><dd>Jane Doe, John Smith</dd><dd>[<a href="never-download.pdf">pdf</a>]</dd>'
|
|
pages={root:f'<a href="?day=all">All papers</a><dl>{item(first)}</dl>',all_url:f'<dl>{item(first)}{item(second)}</dl>',first:'<div id="papertitle">First paper</div><div id="abstract">First abstract</div>',second:'<div id="papertitle">Second paper</div><p>Footer without an abstract</p>'}
|
|
with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
|
|
manifest=collect_index('cvf',{'venue':'CVPR','year':2026,'track':'main','index_url':root},tmp,tmp)
|
|
self.assertEqual(manifest['count'],2)
|
|
self.assertEqual(manifest['list_url'],all_url)
|
|
self.assertEqual(manifest['status'],'partial')
|
|
self.assertEqual(manifest['missing_abstract_count'],1)
|
|
self.assertEqual({call.args[0] for call in fetch.call_args_list},{root,all_url,first,second})
|
|
|
|
def test_discovery_keeps_findings_separate_and_excludes_workshops(self):
|
|
html='<a href="/CVPR2026">Main</a><a href="/CVPR2026_findings">Findings</a><a href="/ICCV2025">Main</a><a href="/CVPR2013.py">Legacy main</a><a href="/CVPR2026_workshops/menu">Workshops</a><a href="/WACV2026">WACV</a>'
|
|
with patch('paper_atlas.collectors.vision.fetch',return_value=html.encode()):indexes=discover('cvf','unused')
|
|
self.assertEqual({(x['venue'],x['year'],x['track']) for x in indexes},{('CVPR',2013,'main'),('CVPR',2026,'main'),('CVPR',2026,'findings'),('ICCV',2025,'main')})
|
|
|
|
def test_unparsed_publication_link_does_not_silently_reduce_coverage(self):
|
|
html='<a href="content/CVPR2026/html/Unparsed_CVPR_2026_paper.html">Other paper</a><dl><dt><a href="content/CVPR2026/html/First_CVPR_2026_paper.html">Paper</a></dt></dl>'
|
|
with self.assertRaisesRegex(ValueError,'outside parsed'):
|
|
index_entries(html,'https://openaccess.thecvf.com/')
|
|
|
|
def test_legacy_declared_encoding_is_respected(self):
|
|
html='<meta content="text/html; charset=ISO-8859-1"><div>André</div>'
|
|
self.assertIn('André',decode_html(html.encode('iso-8859-1')))
|
|
|
|
|
|
if __name__=='__main__':unittest.main()
|