"""Collect CVF/ECVA public titles, abstracts and bibliographic identifiers.""" import argparse from collections import defaultdict import json from pathlib import Path import re import time from datetime import date from urllib.error import HTTPError from urllib.parse import parse_qs, unquote, urljoin, urlsplit from paper_atlas.io.http import fetch from paper_atlas.io.html import Document, Node, clean ROOTS={'cvf':'https://openaccess.thecvf.com/','ecva':'https://www.ecva.net/papers.php'} HOSTS={'openaccess.thecvf.com','www.ecva.net','ecva.net'} def decode_html(data): try: return data.decode('utf-8-sig') except UnicodeDecodeError: charset=re.search(br'charset\s*=\s*["\x27]?([\w-]+)',data[:4096],re.I) if not charset: raise return data.decode(charset[1].decode('ascii')) def publication(url): parsed=urlsplit(url) match=re.search(r'(CVPR|ICCV|ECCV)[_/]?(20\d{2})',parsed.path,re.I) if parsed.netloc in HOSTS and '/html/' in parsed.path and parsed.path.endswith(('.html','.php')) and match: return match[1].upper(),int(match[2]) def doi_link(url): parsed=urlsplit(url) value=None if parsed.netloc in ('doi.org','dx.doi.org'): value=unquote(parsed.path.lstrip('/')) elif parsed.netloc=='link.springer.com' and parsed.path.startswith('/chapter/'): value=unquote(parsed.path.removeprefix('/chapter/')) return value.lower() if value and re.fullmatch(r'10\.\d{4,9}/\S+',value) else None def index_entries(html,index_url): records={} raw_count=0 for dl in (n for n in Document(html).root.walk() if n.tag=='dl'): current=None for node in (n for n in dl.children if isinstance(n,Node)): if node.tag=='dt': current=None links=[urljoin(index_url,n.attrs['href']) for n in node.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))] if not links: continue if len(set(links))!=1: raise ValueError('Ambiguous publication links in one index title') url=links[0];raw_count+=1 if url in records and records[url]['index_title']!=clean(node.text()): raise ValueError('Different index titles share a publication URL') current=records.setdefault(url,{'url':url,'index_title':clean(node.text()),'index_authors':None,'doi':None}) elif node.tag=='dd' and current is not None: text=clean(node.text()) # Author rows precede download/bibliography rows. if current['index_authors'] is None and text and not text.startswith('['): current['index_authors']=text for link in (n for n in node.walk() if n.tag=='a' and clean(n.text()).casefold()=='doi'): value=doi_link(urljoin(index_url,link.attrs.get('href',''))) if value: if current['doi'] and current['doi']!=value: raise ValueError('Conflicting DOI links in one index entry') current['doi']=value if not records: raise ValueError('No official paper entries found') all_links={urljoin(index_url,n.attrs['href']) for n in Document(html).root.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))} if all_links!=set(records):raise ValueError('Paper links exist outside parsed title entries') return list(records.values()),raw_count def discover(source,cache): root=ROOTS[source] html=decode_html(fetch(root,cache,32*1024**2,delay=1)) if source=='ecva': entries,_=index_entries(html,root) groups=defaultdict(list) for entry in entries: venue,year=publication(entry['url']) if venue=='ECCV':groups[(venue,year)].append(entry) return [{'venue':venue,'year':year,'track':'main','index_url':root,'entries':entries} for (venue,year),entries in sorted(groups.items(),key=lambda item:item[0][1])] indexes={} for node in Document(html).root.walk(): if node.tag!='a':continue url=urljoin(root,node.attrs.get('href',''));parsed=urlsplit(url) match=re.fullmatch(r'/(CVPR|ICCV)(20\d{2})(_findings)?(?:\.py)?/?',parsed.path) if parsed.netloc=='openaccess.thecvf.com' and match: indexes[url]={'venue':match[1],'year':int(match[2]),'track':'findings' if match[3] else 'main','index_url':url} if not indexes:raise ValueError('No official CVPR/ICCV indexes found') return sorted(indexes.values(),key=lambda row:(row['year'],row['venue'],row['track']!='main',row['track'])) def author_names(raw): raw=(raw or '').split(';',1)[0] if ' and ' in raw and all(',' in name for name in raw.split(' and ')): names=[' '.join(reversed([part.strip() for part in name.split(',',1)])) for name in raw.split(' and ')] else:names=raw.split(',') return [clean(name).removesuffix('*').strip() for name in names if clean(name).removesuffix('*').strip()] def parse_paper(html,entry,source,venue,year,track,index_url): nodes=list(Document(html).root.walk());meta=defaultdict(list) for node in nodes: if node.tag=='meta':meta[node.attrs.get('name','').lower()].append(node.attrs.get('content','')) title=next(iter(meta['citation_title']),None) or next((clean(n.text()) for n in nodes if n.attrs.get('id')=='papertitle'),None) if not title:raise ValueError('Missing official paper title: '+entry['url']) abstract=next((clean(n.text()) or None for n in nodes if n.attrs.get('id')=='abstract'),None) authors=[] for author in meta['citation_author']: if ',' in author: family,given=author.split(',',1);author=given.strip()+' '+family.strip() if clean(author):authors.append(clean(author)) if not authors: raw=next((clean(n.text()) for n in nodes if n.attrs.get('id')=='authors'),'') or entry.get('index_authors') or '' authors=author_names(raw) dois={value for doi in meta['citation_doi'] if doi and (value:=doi_link(doi) or doi.strip().lower()) and re.fullmatch(r'10\.\d{4,9}/\S+',value)} if entry.get('doi'):dois.add(entry['doi']) if len(dois)>1:raise ValueError('Citation DOI differs from the official index') identifiers={'doi':dois.pop()} if dois else {} # Abstract/reference links are never interpreted as this paper's identifiers. return {'source':source,'external_id':urlsplit(entry['url']).path,'url':entry['url'], 'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers, 'venue':venue,'year':year,'track':track,'status':'accepted', 'raw_decision':'listed in official proceedings','list_url':index_url, 'metadata':{'citation':dict(meta),'index_title':entry['index_title'],'index_authors':entry.get('index_authors'), 'index_urls':entry.get('index_urls',[index_url])}} def index_identity(url): parsed=urlsplit(url) match=re.fullmatch(r'/(CVPR|ICCV)(20\d{2})(_findings)?(?:\.py)?/?',parsed.path) return match.groups() if parsed.netloc=='openaccess.thecvf.com' and match else None def read_index(index,cache): url=index['index_url'];html=decode_html(fetch(url,cache,32*1024**2,delay=1)) links=set() for node in Document(html).root.walk(): if node.tag!='a':continue link=urljoin(url,node.attrs.get('href',''));day=parse_qs(urlsplit(link).query).get('day',[]) if index_identity(link)!=index_identity(url) or index_identity(url) is None or len(day)!=1:continue if day[0]!='all': try:valid=date.fromisoformat(day[0]).year==index['year'] except ValueError:valid=False if not valid:continue links.add(link) all_pages=sorted(link for link in links if parse_qs(urlsplit(link).query)['day']==['all']) day_pages=sorted(links-set(all_pages));index_errors=[] if len(all_pages)>1:raise ValueError('Ambiguous all-papers index') pages=[] if all_pages: try:pages=[(all_pages[0],decode_html(fetch(all_pages[0],cache,32*1024**2,delay=1)))] except HTTPError as error: if error.code!=404 or not day_pages: error.close();raise index_errors.append({'url':all_pages[0],'status':404});error.close() if not pages and day_pages: pages=[(page,decode_html(fetch(page,cache,32*1024**2,delay=1))) for page in day_pages] if not pages:pages=[(url,html)] entries={};raw_count=0 for page,document in pages: parsed,count=index_entries(document,page);raw_count+=count for entry in parsed: entry['index_urls']=[page];old=entries.get(entry['url']) if old: for key in ('index_title','index_authors','doi'): if old.get(key) and entry.get(key) and old[key]!=entry[key]:raise ValueError('Conflicting metadata across official day indexes') if not old.get(key):old[key]=entry.get(key) old['index_urls'].append(page) else:entries[entry['url']]=entry return list(entries.values()),raw_count,[page for page,_ in pages],index_errors def index_only_record(entry,source,venue,year,track,index_url,error): if not entry['index_title'].strip():raise ValueError('Missing title in official index') return {'source':source,'external_id':urlsplit(entry['url']).path,'url':entry['url'], 'title':entry['index_title'],'abstract':None,'authors':author_names(entry.get('index_authors')), 'identifiers':{'doi':entry['doi']} if entry.get('doi') else {},'venue':venue,'year':year,'track':track, 'status':'accepted','raw_decision':'listed in official proceedings index', 'evidence_url':entry['index_urls'][0],'list_url':index_url, 'metadata':{'index_title':entry['index_title'],'index_authors':entry.get('index_authors'), 'index_urls':entry['index_urls'],'abstract_fetch_error':{'status':error.code,'url':error.url,'message':str(error)}}} def collect_index(source,index,cache,out): url=index['index_url'];venue=index['venue'];year=index['year'];track=index['track'] entries=index.get('entries') raw_count=len(entries) if entries is not None else None index_urls=[url];index_errors=[];missing_pages=[] if entries is None: entries,raw_count,index_urls,index_errors=read_index(index,cache) if len(index_urls)==1:url=index_urls[0] records=[] for number,entry in enumerate(entries,1): if publication(entry['url'])!=(venue,year):raise ValueError('Publication path differs from requested edition') try:record=parse_paper(decode_html(fetch(entry['url'],cache,2*1024**2,delay=1)),entry,source,venue,year,track,url) except HTTPError as error: if error.code!=404: error.close();raise entry.setdefault('index_urls',[url]) record=index_only_record(entry,source,venue,year,track,url,error) missing_pages.append(entry['url']);error.close() records.append(record) if number%25==0:print(json.dumps({'source':source,'venue':venue,'year':year,'track':track,'collected':number,'expected':len(entries)}),flush=True) if not records:raise ValueError('Empty publication list') path=Path(out)/f'{source}-{venue}{year}-{track}.jsonl';path.parent.mkdir(parents=True,exist_ok=True) part=path.with_suffix('.part');part.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records));part.replace(path) missing=sum(not r['abstract'] for r in records);missing_authors=sum(not r['authors'] for r in records) manifest={'source':source,'parser_version':2,'venue':venue,'year':year,'track':track, 'count':len(records),'index_link_count':len(entries),'raw_index_entry_count':raw_count, 'missing_abstract_count':missing,'missing_authors_count':missing_authors,'file':str(path), 'index_urls':index_urls,'index_errors':index_errors,'missing_abstract_pages':missing_pages, 'list_url':url,'status':'partial' if missing or missing_authors or index_errors else 'collected'} path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(manifest),flush=True) return manifest def main(argv=None, *, prog=None, source=None): parser=argparse.ArgumentParser(prog=prog) if source is None:parser.add_argument('--source',choices=ROOTS,required=True) else:parser.set_defaults(source=source) parser.add_argument('--cache',required=True);parser.add_argument('--out',required=True) parser.add_argument('--year',type=int,action='append');parser.add_argument('--venue',choices=['CVPR','ICCV','ECCV'],action='append') parser.add_argument('--retry-catalogue',help='Select only blocked editions from an existing catalogue; write results to --out') args=parser.parse_args(argv);out=Path(args.out);out.mkdir(parents=True,exist_ok=True) indexes=discover(args.source,args.cache) if args.retry_catalogue: retry=json.loads(Path(args.retry_catalogue).read_text()) indexes=[index for index in indexes if retry.get(f'{index["venue"]}:{index["year"]}:{index["track"]}',{}).get('status')=='blocked'] if not indexes:parser.error('Retry catalogue has no blocked editions for this source') if args.year and set(args.year)-{r['year'] for r in indexes}:parser.error('Requested year is absent from the official source') if args.venue and set(args.venue)-{r['venue'] for r in indexes}:parser.error('Requested venue is absent from this source') if not any((not args.year or r['year'] in args.year) and (not args.venue or r['venue'] in args.venue) for r in indexes):parser.error('Requested venue/year combination is absent from the official source') path=out/f'{args.source}-index-status.json';catalogue=json.loads(path.read_text()) if path.exists() else {};requested=[] (out/f'{args.source}-discovery.json').write_text(json.dumps({'source':args.source,'root_url':ROOTS[args.source], 'discovered_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'indexes':[ {**{k:index[k] for k in ('venue','year','track','index_url')}, 'index_link_count':len(index['entries']) if 'entries' in index else None} for index in indexes]},indent=2)) for index in indexes: if args.year and index['year'] not in args.year or args.venue and index['venue'] not in args.venue:continue key=f'{index["venue"]}:{index["year"]}:{index["track"]}';requested.append(key) try: manifest=collect_index(args.source,index,args.cache,args.out);catalogue[key]={'status':manifest['status'],'files':[manifest]} except Exception as exc: catalogue[key]={'status':'blocked','year':index['year'],'venue':index['venue'],'track':index['track'],'index_url':index['index_url'],'error':str(exc),'error_url':getattr(exc,'url',None)} print(json.dumps(catalogue[key]),flush=True) part=path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(path) (out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year,'venue_filter':args.venue},indent=2)) if any(catalogue[key]['status']=='blocked' for key in requested):raise SystemExit(1) if __name__=='__main__':main()