import tempfile
import io
import json
from pathlib import Path
import unittest
from unittest.mock import patch
from urllib.error import HTTPError

from paper_atlas.collectors.vision import collect_index, decode_html, discover, index_entries, parse_paper,main


class VisionTests(unittest.TestCase):
    def cvf_fixture(self,year=2018):
        root=f'https://openaccess.thecvf.com/CVPR{year}.py'
        papers=[f'https://openaccess.thecvf.com/content_cvpr_{year}/html/Paper{i}_CVPR_{year}_paper.html' for i in (1,2,3)]
        def entry(url,title='Paper'):
            return f'<dt><a href="{url}">{title}</a></dt><dd>Jane Doe, John Smith</dd>'
        return root,papers,entry

    def test_all_explicit_days_are_combined_and_overlapping_papers_are_not_duplicated(self):
        root,papers,entry=self.cvf_fixture();day1=root+'?day=2018-06-19';day2=root+'?day=2018-06-20'
        pages={root:f'<a href="{day1}">Day1</a><a href="{day2}">Day2</a><a href="https://evil.example/CVPR2018.py?day=all">Other</a><a href="/CVPR2019.py?day=all">Wrong year</a>',day1:'<dl>'+entry(papers[0])+entry(papers[1])+'</dl>',day2:'<dl>'+entry(papers[1])+entry(papers[2])+'</dl>'}
        pages.update({paper:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>' for paper in papers})
        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
            manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
            rows=[json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
        self.assertEqual(manifest['count'],3);self.assertEqual(manifest['raw_index_entry_count'],4)
        self.assertEqual(manifest['index_urls'],[day1,day2]);self.assertEqual(manifest['status'],'collected')
        self.assertEqual(rows[1]['metadata']['index_urls'],[day1,day2])
        self.assertEqual({call.args[0] for call in fetch.call_args_list},set(pages))

    def test_conflicting_copies_across_days_fail_before_publishing(self):
        root,papers,entry=self.cvf_fixture();days=[root+'?day=2018-06-19',root+'?day=2018-06-20']
        pages={root:''.join(f'<a href="{day}">Day</a>' for day in days),days[0]:'<dl>'+entry(papers[0],'First title')+'</dl>',days[1]:'<dl>'+entry(papers[0],'Different title')+'</dl>'}
        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()):
            with self.assertRaisesRegex(ValueError,'Conflicting metadata'):
                collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
            self.assertFalse(list(Path(tmp).glob('*.jsonl')))

    def test_fragment_variants_share_one_all_papers_resource(self):
        root,papers,entry=self.cvf_fixture();all_url=root+'?day=all'
        pages={root:''.join(f'<a href="{all_url}{fragment}">All</a>' for fragment in ('','#','#papers')),
               all_url:'<dl>'+entry(papers[0])+entry(papers[1])+'</dl>'}
        pages.update({paper:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>' for paper in papers[:2]})
        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
            manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
        self.assertEqual(manifest['count'],2)
        self.assertEqual(manifest['index_urls'],[all_url])
        self.assertEqual([call.args[0] for call in fetch.call_args_list].count(all_url),1)
        self.assertFalse(any('#' in call.args[0] for call in fetch.call_args_list))

    def test_distinct_all_papers_queries_still_fail_before_publishing(self):
        root,_,_=self.cvf_fixture();urls=[root+'?day=all',root+'?day=all&view=alternate']
        html=''.join(f'<a href="{url}">All</a>' for url in urls).encode()
        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',return_value=html) as fetch:
            with self.assertRaisesRegex(ValueError,'Ambiguous all-papers index'):
                collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
            self.assertEqual(fetch.call_count,1)
            self.assertFalse(list(Path(tmp).glob('*.jsonl')))

    def test_missing_abstract_page_keeps_official_index_metadata_and_marks_partial(self):
        root,papers,entry=self.cvf_fixture();error=HTTPError(papers[0],404,'Not Found',{},io.BytesIO())
        def fetch(url,*args,**kwargs):
            if url==root:return ('<dl>'+entry(papers[0],'Known publication')+'</dl>').encode()
            raise error
        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch):
            manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
            row=json.loads(Path(manifest['file']).read_text())
        self.assertEqual(row['title'],'Known publication');self.assertIsNone(row['abstract'])
        self.assertEqual(row['authors'],['Jane Doe','John Smith']);self.assertEqual(row['evidence_url'],root)
        self.assertEqual(manifest['status'],'partial');self.assertEqual(manifest['missing_abstract_pages'],[papers[0]])
        self.assertEqual(row['metadata']['abstract_fetch_error']['status'],404)

    def test_forbidden_paper_is_not_replaced_with_a_fallback(self):
        root,papers,entry=self.cvf_fixture()
        def fetch(url,*args,**kwargs):
            if url==root:return ('<dl>'+entry(papers[0])+'</dl>').encode()
            raise HTTPError(url,403,'Forbidden',{},io.BytesIO())
        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch):
            with self.assertRaises(HTTPError):collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
            self.assertFalse(list(Path(tmp).glob('*.jsonl')))

    def test_broken_all_papers_link_can_use_all_advertised_days(self):
        root,papers,entry=self.cvf_fixture();day=root+'?day=2018-06-19';all_url=root+'?day=all'
        pages={root:f'<a href="{day}">Day</a><a href="{all_url}">All</a>',day:'<dl>'+entry(papers[0])+'</dl>',papers[0]:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>'}
        def fetch(url,*a,**kw):
            if url==all_url:raise HTTPError(url,404,'Not Found',{},io.BytesIO())
            return pages[url].encode()
        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch):
            manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
        self.assertEqual(manifest['index_urls'],[day]);self.assertEqual(manifest['index_errors'],[{'url':all_url,'status':404}])

    def test_selective_retry_uses_separate_output_and_leaves_original_catalogue_intact(self):
        with tempfile.TemporaryDirectory() as tmp:
            old=Path(tmp)/'old.json';payload={'CVPR:2018:main':{'status':'blocked'},'CVPR:2019:main':{'status':'collected'}};old.write_text(json.dumps(payload))
            indexes=[{'venue':'CVPR','year':year,'track':'main','index_url':f'https://openaccess.thecvf.com/CVPR{year}'} for year in (2018,2019)]
            output=Path(tmp)/'repaired'
            with patch('paper_atlas.collectors.vision.discover',return_value=indexes),patch('paper_atlas.collectors.vision.collect_index',return_value={'status':'collected','count':1}) as collect:
                main(['--cache',tmp,'--out',str(output),'--retry-catalogue',str(old)],source='cvf')
            self.assertEqual(collect.call_count,1);self.assertEqual(collect.call_args.args[1]['year'],2018)
            self.assertEqual(json.loads(old.read_text()),payload)
            self.assertEqual(set(json.loads((output/'cvf-index-status.json').read_text())),{'CVPR:2018:main'})

    def entry(self):
        return {'url':'https://www.ecva.net/papers/eccv_2024/papers_ECCV/html/1_ECCV_2024_paper.php','index_title':'Paper','index_authors':'Jane Doe*, John Smith','doi':'10.1007/example_1'}

    def test_ecva_doi_belongs_to_its_own_title_entry(self):
        html='<dl><dt class="ptitle"><a href="papers/eccv_2024/papers_ECCV/html/1_ECCV_2024_paper.php">First paper</a></dt><dd>Jane Doe*, John Smith</dd><dd><a href="https://link.springer.com/chapter/10.1007/example_1">DOI</a></dd><dt class="ptitle"><a href="papers/eccv_2024/papers_ECCV/html/2_ECCV_2024_paper.php">Second paper</a></dt><dd>Other Author</dd><dd><a href="https://doi.org/10.1007/example_2">DOI</a></dd></dl>'
        entries,count=index_entries(html,'https://www.ecva.net/papers.php')
        self.assertEqual(count,2)
        self.assertEqual([e['doi'] for e in entries],['10.1007/example_1','10.1007/example_2'])
        html='<div id="papertitle">Paper</div><div id="abstract">Abstract citing <a href="https://arxiv.org/abs/1706.03762">another paper</a> and <a href="https://doi.org/10.1007/wrong">a reference</a>.</div>'
        row=parse_paper(html,entries[0],'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php')
        self.assertEqual(row['identifiers'],{'doi':'10.1007/example_1'})
        self.assertEqual(row['authors'],['Jane Doe','John Smith'])

    def test_cvf_meta_authors_and_missing_abstract_never_capture_footer(self):
        html='<meta name="citation_title" content="A vision paper"><meta name="citation_author" content="Doe, Jane"><div id="authors">Do not include the proceedings citation</div><p>Copyright footer</p>'
        entry=dict(self.entry(),doi=None)
        row=parse_paper(html,entry,'cvf','CVPR',2013,'main','https://openaccess.thecvf.com/CVPR2013')
        self.assertEqual(row['authors'],['Jane Doe'])
        self.assertIsNone(row['abstract'])
        self.assertEqual(row['title'],'A vision paper')

    def test_conflicting_self_doi_is_rejected(self):
        html='<meta name="citation_title" content="Paper"><meta name="citation_doi" content="10.1007/different">'
        with self.assertRaisesRegex(ValueError,'DOI differs'):
            parse_paper(html,self.entry(),'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php')

    def test_legacy_ecva_author_formats_do_not_become_extra_people(self):
        entry=dict(self.entry(),index_authors='Doe, Jane and Smith, John')
        page='<div id="papertitle">Paper</div><div id="authors">Jane Doe, John Smith; The European Conference on Computer Vision (ECCV), 2018, pp. 1-12</div>'
        row=parse_paper(page,entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php')
        self.assertEqual(row['authors'],['Jane Doe','John Smith'])
        row=parse_paper('<div id="papertitle">Paper</div>',entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php')
        self.assertEqual(row['authors'],['Jane Doe','John Smith'])

    def test_cvpr_day_one_is_replaced_by_explicit_all_papers_index(self):
        root='https://openaccess.thecvf.com/CVPR2026';all_url=root+'?day=all'
        first='https://openaccess.thecvf.com/content/CVPR2026/html/First_CVPR_2026_paper.html'
        second='https://openaccess.thecvf.com/content/CVPR2026/html/Second_CVPR_2026_paper.html'
        def item(url):return f'<dt class="ptitle"><a href="{url}">Paper</a></dt><dd>Jane Doe, John Smith</dd><dd>[<a href="never-download.pdf">pdf</a>]</dd>'
        pages={root:f'<a href="?day=all">All papers</a><dl>{item(first)}</dl>',all_url:f'<dl>{item(first)}{item(second)}</dl>',first:'<div id="papertitle">First paper</div><div id="abstract">First abstract</div>',second:'<div id="papertitle">Second paper</div><p>Footer without an abstract</p>'}
        with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
            manifest=collect_index('cvf',{'venue':'CVPR','year':2026,'track':'main','index_url':root},tmp,tmp)
        self.assertEqual(manifest['count'],2)
        self.assertEqual(manifest['list_url'],all_url)
        self.assertEqual(manifest['status'],'partial')
        self.assertEqual(manifest['missing_abstract_count'],1)
        self.assertEqual({call.args[0] for call in fetch.call_args_list},{root,all_url,first,second})

    def test_discovery_keeps_findings_separate_and_excludes_workshops(self):
        html='<a href="/CVPR2026">Main</a><a href="/CVPR2026_findings">Findings</a><a href="/ICCV2025">Main</a><a href="/CVPR2013.py">Legacy main</a><a href="/CVPR2026_workshops/menu">Workshops</a><a href="/WACV2026">WACV</a>'
        with patch('paper_atlas.collectors.vision.fetch',return_value=html.encode()):indexes=discover('cvf','unused')
        self.assertEqual({(x['venue'],x['year'],x['track']) for x in indexes},{('CVPR',2013,'main'),('CVPR',2026,'main'),('CVPR',2026,'findings'),('ICCV',2025,'main')})

    def test_unparsed_publication_link_does_not_silently_reduce_coverage(self):
        html='<a href="content/CVPR2026/html/Unparsed_CVPR_2026_paper.html">Other paper</a><dl><dt><a href="content/CVPR2026/html/First_CVPR_2026_paper.html">Paper</a></dt></dl>'
        with self.assertRaisesRegex(ValueError,'outside parsed'):
            index_entries(html,'https://openaccess.thecvf.com/')

    def test_legacy_declared_encoding_is_respected(self):
        html='<meta content="text/html; charset=ISO-8859-1"><div>André</div>'
        self.assertIn('André',decode_html(html.encode('iso-8859-1')))


if __name__=='__main__':unittest.main()