"""Collect CVF/ECVA public titles, abstracts and bibliographic identifiers.""" import argparse from collections import defaultdict import json from pathlib import Path import re import time from urllib.parse import parse_qs, unquote, urljoin, urlsplit from collect_pmlr import fetch from collect_proceedings import Document, Node, clean ROOTS={'cvf':'https://openaccess.thecvf.com/','ecva':'https://www.ecva.net/papers.php'} HOSTS={'openaccess.thecvf.com','www.ecva.net','ecva.net'} def decode_html(data): try: return data.decode('utf-8-sig') except UnicodeDecodeError: charset=re.search(br'charset\s*=\s*["\x27]?([\w-]+)',data[:4096],re.I) if not charset: raise return data.decode(charset[1].decode('ascii')) def publication(url): parsed=urlsplit(url) match=re.search(r'(CVPR|ICCV|ECCV)[_/]?(20\d{2})',parsed.path,re.I) if parsed.netloc in HOSTS and '/html/' in parsed.path and parsed.path.endswith(('.html','.php')) and match: return match[1].upper(),int(match[2]) def doi_link(url): parsed=urlsplit(url) value=None if parsed.netloc in ('doi.org','dx.doi.org'): value=unquote(parsed.path.lstrip('/')) elif parsed.netloc=='link.springer.com' and parsed.path.startswith('/chapter/'): value=unquote(parsed.path.removeprefix('/chapter/')) return value.lower() if value and re.fullmatch(r'10\.\d{4,9}/\S+',value) else None def index_entries(html,index_url): records={} raw_count=0 for dl in (n for n in Document(html).root.walk() if n.tag=='dl'): current=None for node in (n for n in dl.children if isinstance(n,Node)): if node.tag=='dt': current=None links=[urljoin(index_url,n.attrs['href']) for n in node.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))] if not links: continue if len(set(links))!=1: raise ValueError('Ambiguous publication links in one index title') url=links[0];raw_count+=1 if url in records and records[url]['index_title']!=clean(node.text()): raise ValueError('Different index titles share a publication URL') current=records.setdefault(url,{'url':url,'index_title':clean(node.text()),'index_authors':None,'doi':None}) elif node.tag=='dd' and current is not None: text=clean(node.text()) # Author rows precede download/bibliography rows. if current['index_authors'] is None and text and not text.startswith('['): current['index_authors']=text for link in (n for n in node.walk() if n.tag=='a' and clean(n.text()).casefold()=='doi'): value=doi_link(urljoin(index_url,link.attrs.get('href',''))) if value: if current['doi'] and current['doi']!=value: raise ValueError('Conflicting DOI links in one index entry') current['doi']=value if not records: raise ValueError('No official paper entries found') all_links={urljoin(index_url,n.attrs['href']) for n in Document(html).root.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))} if all_links!=set(records):raise ValueError('Paper links exist outside parsed title entries') return list(records.values()),raw_count def discover(source,cache): root=ROOTS[source] html=decode_html(fetch(root,cache,32*1024**2,delay=1)) if source=='ecva': entries,_=index_entries(html,root) groups=defaultdict(list) for entry in entries: venue,year=publication(entry['url']) if venue=='ECCV':groups[(venue,year)].append(entry) return [{'venue':venue,'year':year,'track':'main','index_url':root,'entries':entries} for (venue,year),entries in sorted(groups.items(),key=lambda item:item[0][1])] indexes={} for node in Document(html).root.walk(): if node.tag!='a':continue url=urljoin(root,node.attrs.get('href',''));parsed=urlsplit(url) match=re.fullmatch(r'/(CVPR|ICCV)(20\d{2})(_findings)?(?:\.py)?/?',parsed.path) if parsed.netloc=='openaccess.thecvf.com' and match: indexes[url]={'venue':match[1],'year':int(match[2]),'track':'findings' if match[3] else 'main','index_url':url} if not indexes:raise ValueError('No official CVPR/ICCV indexes found') return sorted(indexes.values(),key=lambda row:(row['year'],row['venue'],row['track']!='main',row['track'])) def parse_paper(html,entry,source,venue,year,track,index_url): nodes=list(Document(html).root.walk());meta=defaultdict(list) for node in nodes: if node.tag=='meta':meta[node.attrs.get('name','').lower()].append(node.attrs.get('content','')) title=next(iter(meta['citation_title']),None) or next((clean(n.text()) for n in nodes if n.attrs.get('id')=='papertitle'),None) if not title:raise ValueError('Missing official paper title: '+entry['url']) abstract=next((clean(n.text()) or None for n in nodes if n.attrs.get('id')=='abstract'),None) authors=[] for author in meta['citation_author']: if ',' in author: family,given=author.split(',',1);author=given.strip()+' '+family.strip() if clean(author):authors.append(clean(author)) if not authors: raw=next((clean(n.text()) for n in nodes if n.attrs.get('id')=='authors'),'') or entry.get('index_authors') or '' raw=raw.split(';',1)[0] if ' and ' in raw and all(',' in name for name in raw.split(' and ')): names=[' '.join(reversed([part.strip() for part in name.split(',',1)])) for name in raw.split(' and ')] else:names=raw.split(',') authors=[clean(name).removesuffix('*').strip() for name in names if clean(name).removesuffix('*').strip()] dois={value for doi in meta['citation_doi'] if doi and (value:=doi_link(doi) or doi.strip().lower()) and re.fullmatch(r'10\.\d{4,9}/\S+',value)} if entry.get('doi'):dois.add(entry['doi']) if len(dois)>1:raise ValueError('Citation DOI differs from the official index') identifiers={'doi':dois.pop()} if dois else {} # Abstract/reference links are never interpreted as this paper's identifiers. return {'source':source,'external_id':urlsplit(entry['url']).path,'url':entry['url'], 'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers, 'venue':venue,'year':year,'track':track,'status':'accepted', 'raw_decision':'listed in official proceedings','list_url':index_url, 'metadata':{'citation':dict(meta),'index_title':entry['index_title'],'index_authors':entry.get('index_authors')}} def collect_index(source,index,cache,out): url=index['index_url'];venue=index['venue'];year=index['year'];track=index['track'] entries=index.get('entries') raw_count=len(entries) if entries is not None else None if entries is None: html=decode_html(fetch(url,cache,32*1024**2,delay=1)) # Some defaults contain only day one; prefer the explicitly linked all-papers page. all_pages={urljoin(url,n.attrs.get('href','')) for n in Document(html).root.walk() if n.tag=='a' and parse_qs(urlsplit(urljoin(url,n.attrs.get('href',''))).query).get('day')==['all'] and urlsplit(urljoin(url,n.attrs.get('href',''))).netloc==urlsplit(url).netloc and urlsplit(urljoin(url,n.attrs.get('href',''))).path==urlsplit(url).path} if len(all_pages)>1:raise ValueError('Ambiguous all-papers index') if all_pages: url=all_pages.pop();html=decode_html(fetch(url,cache,32*1024**2,delay=1)) entries,raw_count=index_entries(html,url) records=[] for number,entry in enumerate(entries,1): if publication(entry['url'])!=(venue,year):raise ValueError('Publication path differs from requested edition') record=parse_paper(decode_html(fetch(entry['url'],cache,2*1024**2,delay=1)),entry,source,venue,year,track,url) records.append(record) if number%25==0:print(json.dumps({'source':source,'venue':venue,'year':year,'track':track,'collected':number,'expected':len(entries)}),flush=True) if not records:raise ValueError('Empty publication list') path=Path(out)/f'{source}-{venue}{year}-{track}.jsonl';path.parent.mkdir(parents=True,exist_ok=True) part=path.with_suffix('.part');part.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records));part.replace(path) missing=sum(not r['abstract'] for r in records);missing_authors=sum(not r['authors'] for r in records) manifest={'source':source,'parser_version':1,'venue':venue,'year':year,'track':track, 'count':len(records),'index_link_count':len(entries),'raw_index_entry_count':raw_count, 'missing_abstract_count':missing,'missing_authors_count':missing_authors,'file':str(path), 'list_url':url,'status':'partial' if missing or missing_authors else 'collected'} path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(manifest),flush=True) return manifest def main(): parser=argparse.ArgumentParser();parser.add_argument('--source',choices=ROOTS,required=True) parser.add_argument('--cache',required=True);parser.add_argument('--out',required=True) parser.add_argument('--year',type=int,action='append');parser.add_argument('--venue',choices=['CVPR','ICCV','ECCV'],action='append') args=parser.parse_args();out=Path(args.out);out.mkdir(parents=True,exist_ok=True) indexes=discover(args.source,args.cache) if args.year and set(args.year)-{r['year'] for r in indexes}:parser.error('Requested year is absent from the official source') if args.venue and set(args.venue)-{r['venue'] for r in indexes}:parser.error('Requested venue is absent from this source') if not any((not args.year or r['year'] in args.year) and (not args.venue or r['venue'] in args.venue) for r in indexes):parser.error('Requested venue/year combination is absent from the official source') path=out/f'{args.source}-index-status.json';catalogue=json.loads(path.read_text()) if path.exists() else {};requested=[] (out/f'{args.source}-discovery.json').write_text(json.dumps({'source':args.source,'root_url':ROOTS[args.source], 'discovered_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'indexes':[ {**{k:index[k] for k in ('venue','year','track','index_url')}, 'index_link_count':len(index['entries']) if 'entries' in index else None} for index in indexes]},indent=2)) for index in indexes: if args.year and index['year'] not in args.year or args.venue and index['venue'] not in args.venue:continue key=f'{index["venue"]}:{index["year"]}:{index["track"]}';requested.append(key) try: manifest=collect_index(args.source,index,args.cache,args.out);catalogue[key]={'status':manifest['status'],'files':[manifest]} except Exception as exc: catalogue[key]={'status':'blocked','year':index['year'],'venue':index['venue'],'track':index['track'],'index_url':index['index_url'],'error':str(exc)} print(json.dumps(catalogue[key]),flush=True) part=path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(path) (out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year,'venue_filter':args.venue},indent=2)) if any(catalogue[key]['status']=='blocked' for key in requested):raise SystemExit(1) if __name__=='__main__':main()