diff --git a/docs/data-sources.md b/docs/data-sources.md index e949f2e..8759b90 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -215,8 +215,20 @@ python3 -B -m paper_atlas collect ecva \ --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected ``` -CVF 从官方根目录发现 CVPR、ICCV 主会与 Findings,分别保存 track;不把 workshop 或 WACV 算入这些会议。优先跟随页面明确提供的 All Papers 链接,避免只收录默认显示的第一天。ECVA 从官方总目录按论文 URL 的会议与年份分组,当前总目录列有 ECCV 2018、2020、2022、2024。不存在于来源的 `--venue`、`--year` 或二者组合会报错,不补造双年会议届次,也不能因此推断旧届次未举办。 +CVF 从官方根目录发现 CVPR、ICCV 主会与 Findings,分别保存 track;不把 workshop 或 WACV 算入这些会议。优先跟随页面明确提供的 All Papers 链接;只有逐日目录时遍历全部明确链接的日期,再按论文 URL 合并重复条目并检查标题、作者、DOI 一致性。`.py` 与无后缀的同届入口可关联,不跟随其他会议、年份或站点的日期链接。ECVA 从官方总目录按论文 URL 的会议与年份分组,当前总目录列有 ECCV 2018、2020、2022、2024。不存在于来源的 `--venue`、`--year` 或二者组合会报错,不补造双年会议届次,也不能因此推断旧届次未举办。 只访问论文索引与摘要 HTML/PHP 页面。作者优先取 citation 元数据,旧页面尊重声明编码;无 citation 作者时取索引中的作者行,排除论文集出版信息。DOI 只取该论文条目所属的 DOI 链接或 citation 字段;冲突会失败,不从摘要引用中抽取当前论文 ID。不请求 PDF、补充材料、视频或代码。 每份清单记录索引入口数、论文数、缺失摘要与作者数,缺失内容保留为空并标记 partial。存在未被标题条目解析的论文链接时会失败,防止静默漏项。逐来源的 discovery、index-status、last-run 文件分别保存发现目录、逐届结果和实际请求范围。可重复使用 `--year` 或 `--venue` 限定重试;其他届次状态会保留。官方开放目录之外的早期届次、尚未刊出的新届次和独立出版方历史数据仍需后续补齐,采集器完成不代表全历史覆盖已完成。 + +单个摘要页面 404 时,仍保存官方目录提供的题目、作者和发表证据,摘要留空并记录失败 URL;整届文件标为 partial。403 等访问拒绝会终止该届采集。刷新缺失摘要时,存储层保留原来已知的论文摘要及其来源观察信息,后续页面恢复时可以更新。 + +针对旧 catalogue 的 blocked 届次可单独补采到新目录,避免与现有采集任务同时改写状态文件: + +```sh +paper-atlas collect cvf --cache /data/paper-index/raw/http-cache \ + --retry-catalogue /data/paper-index/collected/cvf-index-status.json \ + --out /data/paper-index/collected/cvf-repair +``` + +parser_version 2 的清单还保存实际遍历的 index_urls、入口错误及缺失摘要页面。补采结果经过清单与身份核验后才导入,不能因重试任务启动就标记缺口已解决。 diff --git a/paper_atlas/collectors/vision.py b/paper_atlas/collectors/vision.py index 1247148..68233d1 100644 --- a/paper_atlas/collectors/vision.py +++ b/paper_atlas/collectors/vision.py @@ -5,6 +5,8 @@ import json from pathlib import Path import re import time +from datetime import date +from urllib.error import HTTPError from urllib.parse import parse_qs, unquote, urljoin, urlsplit from paper_atlas.io.http import fetch @@ -98,6 +100,14 @@ def discover(source,cache): return sorted(indexes.values(),key=lambda row:(row['year'],row['venue'],row['track']!='main',row['track'])) +def author_names(raw): + raw=(raw or '').split(';',1)[0] + if ' and ' in raw and all(',' in name for name in raw.split(' and ')): + names=[' '.join(reversed([part.strip() for part in name.split(',',1)])) for name in raw.split(' and ')] + else:names=raw.split(',') + return [clean(name).removesuffix('*').strip() for name in names if clean(name).removesuffix('*').strip()] + + def parse_paper(html,entry,source,venue,year,track,index_url): nodes=list(Document(html).root.walk());meta=defaultdict(list) for node in nodes: @@ -112,11 +122,7 @@ def parse_paper(html,entry,source,venue,year,track,index_url): if clean(author):authors.append(clean(author)) if not authors: raw=next((clean(n.text()) for n in nodes if n.attrs.get('id')=='authors'),'') or entry.get('index_authors') or '' - raw=raw.split(';',1)[0] - if ' and ' in raw and all(',' in name for name in raw.split(' and ')): - names=[' '.join(reversed([part.strip() for part in name.split(',',1)])) for name in raw.split(' and ')] - else:names=raw.split(',') - authors=[clean(name).removesuffix('*').strip() for name in names if clean(name).removesuffix('*').strip()] + authors=author_names(raw) dois={value for doi in meta['citation_doi'] if doi and (value:=doi_link(doi) or doi.strip().lower()) and re.fullmatch(r'10\.\d{4,9}/\S+',value)} if entry.get('doi'):dois.add(entry['doi']) if len(dois)>1:raise ValueError('Citation DOI differs from the official index') @@ -126,35 +132,95 @@ def parse_paper(html,entry,source,venue,year,track,index_url): 'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers, 'venue':venue,'year':year,'track':track,'status':'accepted', 'raw_decision':'listed in official proceedings','list_url':index_url, - 'metadata':{'citation':dict(meta),'index_title':entry['index_title'],'index_authors':entry.get('index_authors')}} + 'metadata':{'citation':dict(meta),'index_title':entry['index_title'],'index_authors':entry.get('index_authors'), + 'index_urls':entry.get('index_urls',[index_url])}} + + +def index_identity(url): + parsed=urlsplit(url) + match=re.fullmatch(r'/(CVPR|ICCV)(20\d{2})(_findings)?(?:\.py)?/?',parsed.path) + return match.groups() if parsed.netloc=='openaccess.thecvf.com' and match else None + + +def read_index(index,cache): + url=index['index_url'];html=decode_html(fetch(url,cache,32*1024**2,delay=1)) + links=set() + for node in Document(html).root.walk(): + if node.tag!='a':continue + link=urljoin(url,node.attrs.get('href',''));day=parse_qs(urlsplit(link).query).get('day',[]) + if index_identity(link)!=index_identity(url) or index_identity(url) is None or len(day)!=1:continue + if day[0]!='all': + try:valid=date.fromisoformat(day[0]).year==index['year'] + except ValueError:valid=False + if not valid:continue + links.add(link) + all_pages=sorted(link for link in links if parse_qs(urlsplit(link).query)['day']==['all']) + day_pages=sorted(links-set(all_pages));index_errors=[] + if len(all_pages)>1:raise ValueError('Ambiguous all-papers index') + pages=[] + if all_pages: + try:pages=[(all_pages[0],decode_html(fetch(all_pages[0],cache,32*1024**2,delay=1)))] + except HTTPError as error: + if error.code!=404 or not day_pages: + error.close();raise + index_errors.append({'url':all_pages[0],'status':404});error.close() + if not pages and day_pages: + pages=[(page,decode_html(fetch(page,cache,32*1024**2,delay=1))) for page in day_pages] + if not pages:pages=[(url,html)] + entries={};raw_count=0 + for page,document in pages: + parsed,count=index_entries(document,page);raw_count+=count + for entry in parsed: + entry['index_urls']=[page];old=entries.get(entry['url']) + if old: + for key in ('index_title','index_authors','doi'): + if old.get(key) and entry.get(key) and old[key]!=entry[key]:raise ValueError('Conflicting metadata across official day indexes') + if not old.get(key):old[key]=entry.get(key) + old['index_urls'].append(page) + else:entries[entry['url']]=entry + return list(entries.values()),raw_count,[page for page,_ in pages],index_errors + + +def index_only_record(entry,source,venue,year,track,index_url,error): + if not entry['index_title'].strip():raise ValueError('Missing title in official index') + return {'source':source,'external_id':urlsplit(entry['url']).path,'url':entry['url'], + 'title':entry['index_title'],'abstract':None,'authors':author_names(entry.get('index_authors')), + 'identifiers':{'doi':entry['doi']} if entry.get('doi') else {},'venue':venue,'year':year,'track':track, + 'status':'accepted','raw_decision':'listed in official proceedings index', + 'evidence_url':entry['index_urls'][0],'list_url':index_url, + 'metadata':{'index_title':entry['index_title'],'index_authors':entry.get('index_authors'), + 'index_urls':entry['index_urls'],'abstract_fetch_error':{'status':error.code,'url':error.url,'message':str(error)}}} def collect_index(source,index,cache,out): url=index['index_url'];venue=index['venue'];year=index['year'];track=index['track'] entries=index.get('entries') raw_count=len(entries) if entries is not None else None + index_urls=[url];index_errors=[];missing_pages=[] if entries is None: - html=decode_html(fetch(url,cache,32*1024**2,delay=1)) - # Some defaults contain only day one; prefer the explicitly linked all-papers page. - all_pages={urljoin(url,n.attrs.get('href','')) for n in Document(html).root.walk() if n.tag=='a' and parse_qs(urlsplit(urljoin(url,n.attrs.get('href',''))).query).get('day')==['all'] and urlsplit(urljoin(url,n.attrs.get('href',''))).netloc==urlsplit(url).netloc and urlsplit(urljoin(url,n.attrs.get('href',''))).path==urlsplit(url).path} - if len(all_pages)>1:raise ValueError('Ambiguous all-papers index') - if all_pages: - url=all_pages.pop();html=decode_html(fetch(url,cache,32*1024**2,delay=1)) - entries,raw_count=index_entries(html,url) + entries,raw_count,index_urls,index_errors=read_index(index,cache) + if len(index_urls)==1:url=index_urls[0] records=[] for number,entry in enumerate(entries,1): if publication(entry['url'])!=(venue,year):raise ValueError('Publication path differs from requested edition') - record=parse_paper(decode_html(fetch(entry['url'],cache,2*1024**2,delay=1)),entry,source,venue,year,track,url) + try:record=parse_paper(decode_html(fetch(entry['url'],cache,2*1024**2,delay=1)),entry,source,venue,year,track,url) + except HTTPError as error: + if error.code!=404: + error.close();raise + entry.setdefault('index_urls',[url]) + record=index_only_record(entry,source,venue,year,track,url,error) + missing_pages.append(entry['url']);error.close() records.append(record) if number%25==0:print(json.dumps({'source':source,'venue':venue,'year':year,'track':track,'collected':number,'expected':len(entries)}),flush=True) if not records:raise ValueError('Empty publication list') path=Path(out)/f'{source}-{venue}{year}-{track}.jsonl';path.parent.mkdir(parents=True,exist_ok=True) part=path.with_suffix('.part');part.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records));part.replace(path) missing=sum(not r['abstract'] for r in records);missing_authors=sum(not r['authors'] for r in records) - manifest={'source':source,'parser_version':1,'venue':venue,'year':year,'track':track, + manifest={'source':source,'parser_version':2,'venue':venue,'year':year,'track':track, 'count':len(records),'index_link_count':len(entries),'raw_index_entry_count':raw_count, 'missing_abstract_count':missing,'missing_authors_count':missing_authors,'file':str(path), - 'list_url':url,'status':'partial' if missing or missing_authors else 'collected'} + 'index_urls':index_urls,'index_errors':index_errors,'missing_abstract_pages':missing_pages, + 'list_url':url,'status':'partial' if missing or missing_authors or index_errors else 'collected'} path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(manifest),flush=True) return manifest @@ -165,8 +231,13 @@ def main(argv=None, *, prog=None, source=None): else:parser.set_defaults(source=source) parser.add_argument('--cache',required=True);parser.add_argument('--out',required=True) parser.add_argument('--year',type=int,action='append');parser.add_argument('--venue',choices=['CVPR','ICCV','ECCV'],action='append') + parser.add_argument('--retry-catalogue',help='Select only blocked editions from an existing catalogue; write results to --out') args=parser.parse_args(argv);out=Path(args.out);out.mkdir(parents=True,exist_ok=True) indexes=discover(args.source,args.cache) + if args.retry_catalogue: + retry=json.loads(Path(args.retry_catalogue).read_text()) + indexes=[index for index in indexes if retry.get(f'{index["venue"]}:{index["year"]}:{index["track"]}',{}).get('status')=='blocked'] + if not indexes:parser.error('Retry catalogue has no blocked editions for this source') if args.year and set(args.year)-{r['year'] for r in indexes}:parser.error('Requested year is absent from the official source') if args.venue and set(args.venue)-{r['venue'] for r in indexes}:parser.error('Requested venue is absent from this source') if not any((not args.year or r['year'] in args.year) and (not args.venue or r['venue'] in args.venue) for r in indexes):parser.error('Requested venue/year combination is absent from the official source') @@ -181,7 +252,7 @@ def main(argv=None, *, prog=None, source=None): try: manifest=collect_index(args.source,index,args.cache,args.out);catalogue[key]={'status':manifest['status'],'files':[manifest]} except Exception as exc: - catalogue[key]={'status':'blocked','year':index['year'],'venue':index['venue'],'track':index['track'],'index_url':index['index_url'],'error':str(exc)} + catalogue[key]={'status':'blocked','year':index['year'],'venue':index['venue'],'track':index['track'],'index_url':index['index_url'],'error':str(exc),'error_url':getattr(exc,'url',None)} print(json.dumps(catalogue[key]),flush=True) part=path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(path) (out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year,'venue_filter':args.venue},indent=2)) diff --git a/paper_atlas/db/store.py b/paper_atlas/db/store.py index 11a8ca0..57fa8db 100644 --- a/paper_atlas/db/store.py +++ b/paper_atlas/db/store.py @@ -88,7 +88,7 @@ class Store: found = c.execute('SELECT paper_id FROM identifiers WHERE scheme=? AND value=?',(scheme,value)).fetchone() if found: matches.add(found[0]) - old = c.execute('SELECT paper_id,title,abstract FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone() + old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone() if old: matches.add(old[0]) key = normalized(title) @@ -105,13 +105,17 @@ class Store: for other in sorted(matches-{paper}): self.merge(paper,other,'explicit identifier or exact title corroborated by authors') if old: - c.execute("UPDATE papers SET title=?,abstract=?,title_key=?,authors_json=? WHERE id=? AND title=? AND COALESCE(abstract,'')=COALESCE(?,'')",(title,abstract,key,json.dumps(authors,ensure_ascii=False),paper,old[1],old[2])) + old_abstract=old[2] or json.loads(old[4]).get('last_known_abstract',{}).get('abstract') + c.execute("UPDATE papers SET title=?,abstract=COALESCE(NULLIF(?,''),abstract),title_key=?,authors_json=CASE WHEN ?='[]' THEN authors_json ELSE ? END WHERE id=? AND title=? AND COALESCE(abstract,'')=COALESCE(?,'')",(title,abstract,key,json.dumps(authors,ensure_ascii=False),json.dumps(authors,ensure_ascii=False),paper,old[1],old_abstract)) c.execute("UPDATE papers SET abstract=CASE WHEN abstract IS NULL OR abstract='' THEN ? ELSE abstract END WHERE id=?",(abstract,paper)) else: paper = c.execute('INSERT INTO papers(title,abstract,title_key,authors_json,first_published) VALUES(?,?,?,?,?)',(title,abstract,key,json.dumps(authors,ensure_ascii=False),published)).lastrowid for scheme,value in ids: c.execute('INSERT OR IGNORE INTO identifiers VALUES(?,?,?)',(scheme,value,paper)) metadata=dict(metadata or {});metadata['record_identifiers']=dict(ids) + if old and not abstract: + previous={'abstract':old[2],'observed_at':old[3]} if old[2] else json.loads(old[4]).get('last_known_abstract') + if previous:metadata['last_known_abstract']=previous c.execute('INSERT INTO source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_id) DO UPDATE SET paper_id=excluded.paper_id,url=excluded.url,title=excluded.title,abstract=excluded.abstract,authors_json=excluded.authors_json,metadata_json=excluded.metadata_json,retrieved_at=CURRENT_TIMESTAMP',(source_id,external_id,paper,url,title,abstract,json.dumps(authors,ensure_ascii=False),json.dumps(metadata,ensure_ascii=False,default=str))) record = c.execute('SELECT id FROM source_records WHERE source_id=? AND external_id=?',(source_id,external_id)).fetchone()[0] for other,_,_ in title_matches: diff --git a/tests/test_index_core.py b/tests/test_index_core.py index edb2441..2f4373d 100644 --- a/tests/test_index_core.py +++ b/tests/test_index_core.py @@ -9,6 +9,17 @@ from paper_atlas.ingestion.records import ingest_file class StoreTests(unittest.TestCase): + def test_partial_source_refresh_preserves_known_abstract_and_can_later_restore_it(self): + paper,record=self.store.record('cvf','paper','https://example.org/paper','Known title','Known abstract',['Jane Doe']) + self.store.record('cvf','paper','https://example.org/paper','Known title',None,[],metadata={'abstract_fetch_error':{'status':404}}) + self.store.record('cvf','paper','https://example.org/paper','Known title',None,[],metadata={'abstract_fetch_error':{'status':404}}) + c=self.store.conn + self.assertEqual(c.execute('SELECT abstract,authors_json FROM papers').fetchone(),('Known abstract','["Jane Doe"]')) + source=c.execute('SELECT abstract,metadata_json FROM source_records WHERE id=?',(record,)).fetchone() + self.assertIsNone(source[0]);self.assertEqual(json.loads(source[1])['last_known_abstract']['abstract'],'Known abstract') + self.store.record('cvf','paper','https://example.org/paper','Updated title','Restored revised abstract',['Jane Doe']) + self.assertEqual(c.execute('SELECT title,abstract FROM papers').fetchone(),('Updated title','Restored revised abstract')) + def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) diff --git a/tests/test_vision.py b/tests/test_vision.py index a93e31f..6572b30 100644 --- a/tests/test_vision.py +++ b/tests/test_vision.py @@ -1,11 +1,85 @@ import tempfile +import io +import json +from pathlib import Path import unittest from unittest.mock import patch +from urllib.error import HTTPError -from paper_atlas.collectors.vision import collect_index, decode_html, discover, index_entries, parse_paper +from paper_atlas.collectors.vision import collect_index, decode_html, discover, index_entries, parse_paper,main class VisionTests(unittest.TestCase): + def cvf_fixture(self,year=2018): + root=f'https://openaccess.thecvf.com/CVPR{year}.py' + papers=[f'https://openaccess.thecvf.com/content_cvpr_{year}/html/Paper{i}_CVPR_{year}_paper.html' for i in (1,2,3)] + def entry(url,title='Paper'): + return f'
{title}
Jane Doe, John Smith
' + return root,papers,entry + + def test_all_explicit_days_are_combined_and_overlapping_papers_are_not_duplicated(self): + root,papers,entry=self.cvf_fixture();day1=root+'?day=2018-06-19';day2=root+'?day=2018-06-20' + pages={root:f'Day1Day2OtherWrong year',day1:'
'+entry(papers[0])+entry(papers[1])+'
',day2:'
'+entry(papers[1])+entry(papers[2])+'
'} + pages.update({paper:'
Paper
Abstract
' for paper in papers}) + with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch: + manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp) + rows=[json.loads(line) for line in Path(manifest['file']).read_text().splitlines()] + self.assertEqual(manifest['count'],3);self.assertEqual(manifest['raw_index_entry_count'],4) + self.assertEqual(manifest['index_urls'],[day1,day2]);self.assertEqual(manifest['status'],'collected') + self.assertEqual(rows[1]['metadata']['index_urls'],[day1,day2]) + self.assertEqual({call.args[0] for call in fetch.call_args_list},set(pages)) + + def test_conflicting_copies_across_days_fail_before_publishing(self): + root,papers,entry=self.cvf_fixture();days=[root+'?day=2018-06-19',root+'?day=2018-06-20'] + pages={root:''.join(f'Day' for day in days),days[0]:'
'+entry(papers[0],'First title')+'
',days[1]:'
'+entry(papers[0],'Different title')+'
'} + with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()): + with self.assertRaisesRegex(ValueError,'Conflicting metadata'): + collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp) + self.assertFalse(list(Path(tmp).glob('*.jsonl'))) + + def test_missing_abstract_page_keeps_official_index_metadata_and_marks_partial(self): + root,papers,entry=self.cvf_fixture();error=HTTPError(papers[0],404,'Not Found',{},io.BytesIO()) + def fetch(url,*args,**kwargs): + if url==root:return ('
'+entry(papers[0],'Known publication')+'
').encode() + raise error + with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch): + manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp) + row=json.loads(Path(manifest['file']).read_text()) + self.assertEqual(row['title'],'Known publication');self.assertIsNone(row['abstract']) + self.assertEqual(row['authors'],['Jane Doe','John Smith']);self.assertEqual(row['evidence_url'],root) + self.assertEqual(manifest['status'],'partial');self.assertEqual(manifest['missing_abstract_pages'],[papers[0]]) + self.assertEqual(row['metadata']['abstract_fetch_error']['status'],404) + + def test_forbidden_paper_is_not_replaced_with_a_fallback(self): + root,papers,entry=self.cvf_fixture() + def fetch(url,*args,**kwargs): + if url==root:return ('
'+entry(papers[0])+'
').encode() + raise HTTPError(url,403,'Forbidden',{},io.BytesIO()) + with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch): + with self.assertRaises(HTTPError):collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp) + self.assertFalse(list(Path(tmp).glob('*.jsonl'))) + + def test_broken_all_papers_link_can_use_all_advertised_days(self): + root,papers,entry=self.cvf_fixture();day=root+'?day=2018-06-19';all_url=root+'?day=all' + pages={root:f'DayAll',day:'
'+entry(papers[0])+'
',papers[0]:'
Paper
Abstract
'} + def fetch(url,*a,**kw): + if url==all_url:raise HTTPError(url,404,'Not Found',{},io.BytesIO()) + return pages[url].encode() + with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=fetch): + manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp) + self.assertEqual(manifest['index_urls'],[day]);self.assertEqual(manifest['index_errors'],[{'url':all_url,'status':404}]) + + def test_selective_retry_uses_separate_output_and_leaves_original_catalogue_intact(self): + with tempfile.TemporaryDirectory() as tmp: + old=Path(tmp)/'old.json';payload={'CVPR:2018:main':{'status':'blocked'},'CVPR:2019:main':{'status':'collected'}};old.write_text(json.dumps(payload)) + indexes=[{'venue':'CVPR','year':year,'track':'main','index_url':f'https://openaccess.thecvf.com/CVPR{year}'} for year in (2018,2019)] + output=Path(tmp)/'repaired' + with patch('paper_atlas.collectors.vision.discover',return_value=indexes),patch('paper_atlas.collectors.vision.collect_index',return_value={'status':'collected','count':1}) as collect: + main(['--cache',tmp,'--out',str(output),'--retry-catalogue',str(old)],source='cvf') + self.assertEqual(collect.call_count,1);self.assertEqual(collect.call_args.args[1]['year'],2018) + self.assertEqual(json.loads(old.read_text()),payload) + self.assertEqual(set(json.loads((output/'cvf-index-status.json').read_text())),{'CVPR:2018:main'}) + def entry(self): return {'url':'https://www.ecva.net/papers/eccv_2024/papers_ECCV/html/1_ECCV_2024_paper.php','index_title':'Paper','index_authors':'Jane Doe*, John Smith','doi':'10.1007/example_1'}