190 lines
12 KiB
Python
190 lines
12 KiB
Python
"""Collect CVF/ECVA public titles, abstracts and bibliographic identifiers."""
|
|
import argparse
|
|
from collections import defaultdict
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import time
|
|
from urllib.parse import parse_qs, unquote, urljoin, urlsplit
|
|
|
|
from collect_pmlr import fetch
|
|
from collect_proceedings import Document, Node, clean
|
|
|
|
ROOTS={'cvf':'https://openaccess.thecvf.com/','ecva':'https://www.ecva.net/papers.php'}
|
|
HOSTS={'openaccess.thecvf.com','www.ecva.net','ecva.net'}
|
|
|
|
|
|
def decode_html(data):
|
|
try:
|
|
return data.decode('utf-8-sig')
|
|
except UnicodeDecodeError:
|
|
charset=re.search(br'charset\s*=\s*["\x27]?([\w-]+)',data[:4096],re.I)
|
|
if not charset:
|
|
raise
|
|
return data.decode(charset[1].decode('ascii'))
|
|
|
|
|
|
def publication(url):
|
|
parsed=urlsplit(url)
|
|
match=re.search(r'(CVPR|ICCV|ECCV)[_/]?(20\d{2})',parsed.path,re.I)
|
|
if parsed.netloc in HOSTS and '/html/' in parsed.path and parsed.path.endswith(('.html','.php')) and match:
|
|
return match[1].upper(),int(match[2])
|
|
|
|
|
|
def doi_link(url):
|
|
parsed=urlsplit(url)
|
|
value=None
|
|
if parsed.netloc in ('doi.org','dx.doi.org'):
|
|
value=unquote(parsed.path.lstrip('/'))
|
|
elif parsed.netloc=='link.springer.com' and parsed.path.startswith('/chapter/'):
|
|
value=unquote(parsed.path.removeprefix('/chapter/'))
|
|
return value.lower() if value and re.fullmatch(r'10\.\d{4,9}/\S+',value) else None
|
|
|
|
|
|
def index_entries(html,index_url):
|
|
records={}
|
|
raw_count=0
|
|
for dl in (n for n in Document(html).root.walk() if n.tag=='dl'):
|
|
current=None
|
|
for node in (n for n in dl.children if isinstance(n,Node)):
|
|
if node.tag=='dt':
|
|
current=None
|
|
links=[urljoin(index_url,n.attrs['href']) for n in node.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))]
|
|
if not links:
|
|
continue
|
|
if len(set(links))!=1:
|
|
raise ValueError('Ambiguous publication links in one index title')
|
|
url=links[0];raw_count+=1
|
|
if url in records and records[url]['index_title']!=clean(node.text()):
|
|
raise ValueError('Different index titles share a publication URL')
|
|
current=records.setdefault(url,{'url':url,'index_title':clean(node.text()),'index_authors':None,'doi':None})
|
|
elif node.tag=='dd' and current is not None:
|
|
text=clean(node.text())
|
|
# Author rows precede download/bibliography rows.
|
|
if current['index_authors'] is None and text and not text.startswith('['):
|
|
current['index_authors']=text
|
|
for link in (n for n in node.walk() if n.tag=='a' and clean(n.text()).casefold()=='doi'):
|
|
value=doi_link(urljoin(index_url,link.attrs.get('href','')))
|
|
if value:
|
|
if current['doi'] and current['doi']!=value:
|
|
raise ValueError('Conflicting DOI links in one index entry')
|
|
current['doi']=value
|
|
if not records:
|
|
raise ValueError('No official paper entries found')
|
|
all_links={urljoin(index_url,n.attrs['href']) for n in Document(html).root.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))}
|
|
if all_links!=set(records):raise ValueError('Paper links exist outside parsed title entries')
|
|
return list(records.values()),raw_count
|
|
|
|
|
|
def discover(source,cache):
|
|
root=ROOTS[source]
|
|
html=decode_html(fetch(root,cache,32*1024**2,delay=1))
|
|
if source=='ecva':
|
|
entries,_=index_entries(html,root)
|
|
groups=defaultdict(list)
|
|
for entry in entries:
|
|
venue,year=publication(entry['url'])
|
|
if venue=='ECCV':groups[(venue,year)].append(entry)
|
|
return [{'venue':venue,'year':year,'track':'main','index_url':root,'entries':entries}
|
|
for (venue,year),entries in sorted(groups.items(),key=lambda item:item[0][1])]
|
|
indexes={}
|
|
for node in Document(html).root.walk():
|
|
if node.tag!='a':continue
|
|
url=urljoin(root,node.attrs.get('href',''));parsed=urlsplit(url)
|
|
match=re.fullmatch(r'/(CVPR|ICCV)(20\d{2})(_findings)?(?:\.py)?/?',parsed.path)
|
|
if parsed.netloc=='openaccess.thecvf.com' and match:
|
|
indexes[url]={'venue':match[1],'year':int(match[2]),'track':'findings' if match[3] else 'main','index_url':url}
|
|
if not indexes:raise ValueError('No official CVPR/ICCV indexes found')
|
|
return sorted(indexes.values(),key=lambda row:(row['year'],row['venue'],row['track']!='main',row['track']))
|
|
|
|
|
|
def parse_paper(html,entry,source,venue,year,track,index_url):
|
|
nodes=list(Document(html).root.walk());meta=defaultdict(list)
|
|
for node in nodes:
|
|
if node.tag=='meta':meta[node.attrs.get('name','').lower()].append(node.attrs.get('content',''))
|
|
title=next(iter(meta['citation_title']),None) or next((clean(n.text()) for n in nodes if n.attrs.get('id')=='papertitle'),None)
|
|
if not title:raise ValueError('Missing official paper title: '+entry['url'])
|
|
abstract=next((clean(n.text()) or None for n in nodes if n.attrs.get('id')=='abstract'),None)
|
|
authors=[]
|
|
for author in meta['citation_author']:
|
|
if ',' in author:
|
|
family,given=author.split(',',1);author=given.strip()+' '+family.strip()
|
|
if clean(author):authors.append(clean(author))
|
|
if not authors:
|
|
raw=next((clean(n.text()) for n in nodes if n.attrs.get('id')=='authors'),'') or entry.get('index_authors') or ''
|
|
raw=raw.split(';',1)[0]
|
|
if ' and ' in raw and all(',' in name for name in raw.split(' and ')):
|
|
names=[' '.join(reversed([part.strip() for part in name.split(',',1)])) for name in raw.split(' and ')]
|
|
else:names=raw.split(',')
|
|
authors=[clean(name).removesuffix('*').strip() for name in names if clean(name).removesuffix('*').strip()]
|
|
dois={value for doi in meta['citation_doi'] if doi and (value:=doi_link(doi) or doi.strip().lower()) and re.fullmatch(r'10\.\d{4,9}/\S+',value)}
|
|
if entry.get('doi'):dois.add(entry['doi'])
|
|
if len(dois)>1:raise ValueError('Citation DOI differs from the official index')
|
|
identifiers={'doi':dois.pop()} if dois else {}
|
|
# Abstract/reference links are never interpreted as this paper's identifiers.
|
|
return {'source':source,'external_id':urlsplit(entry['url']).path,'url':entry['url'],
|
|
'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,
|
|
'venue':venue,'year':year,'track':track,'status':'accepted',
|
|
'raw_decision':'listed in official proceedings','list_url':index_url,
|
|
'metadata':{'citation':dict(meta),'index_title':entry['index_title'],'index_authors':entry.get('index_authors')}}
|
|
|
|
|
|
def collect_index(source,index,cache,out):
|
|
url=index['index_url'];venue=index['venue'];year=index['year'];track=index['track']
|
|
entries=index.get('entries')
|
|
raw_count=len(entries) if entries is not None else None
|
|
if entries is None:
|
|
html=decode_html(fetch(url,cache,32*1024**2,delay=1))
|
|
# Some defaults contain only day one; prefer the explicitly linked all-papers page.
|
|
all_pages={urljoin(url,n.attrs.get('href','')) for n in Document(html).root.walk() if n.tag=='a' and parse_qs(urlsplit(urljoin(url,n.attrs.get('href',''))).query).get('day')==['all'] and urlsplit(urljoin(url,n.attrs.get('href',''))).netloc==urlsplit(url).netloc and urlsplit(urljoin(url,n.attrs.get('href',''))).path==urlsplit(url).path}
|
|
if len(all_pages)>1:raise ValueError('Ambiguous all-papers index')
|
|
if all_pages:
|
|
url=all_pages.pop();html=decode_html(fetch(url,cache,32*1024**2,delay=1))
|
|
entries,raw_count=index_entries(html,url)
|
|
records=[]
|
|
for number,entry in enumerate(entries,1):
|
|
if publication(entry['url'])!=(venue,year):raise ValueError('Publication path differs from requested edition')
|
|
record=parse_paper(decode_html(fetch(entry['url'],cache,2*1024**2,delay=1)),entry,source,venue,year,track,url)
|
|
records.append(record)
|
|
if number%25==0:print(json.dumps({'source':source,'venue':venue,'year':year,'track':track,'collected':number,'expected':len(entries)}),flush=True)
|
|
if not records:raise ValueError('Empty publication list')
|
|
path=Path(out)/f'{source}-{venue}{year}-{track}.jsonl';path.parent.mkdir(parents=True,exist_ok=True)
|
|
part=path.with_suffix('.part');part.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records));part.replace(path)
|
|
missing=sum(not r['abstract'] for r in records);missing_authors=sum(not r['authors'] for r in records)
|
|
manifest={'source':source,'parser_version':1,'venue':venue,'year':year,'track':track,
|
|
'count':len(records),'index_link_count':len(entries),'raw_index_entry_count':raw_count,
|
|
'missing_abstract_count':missing,'missing_authors_count':missing_authors,'file':str(path),
|
|
'list_url':url,'status':'partial' if missing or missing_authors else 'collected'}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(manifest),flush=True)
|
|
return manifest
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser();parser.add_argument('--source',choices=ROOTS,required=True)
|
|
parser.add_argument('--cache',required=True);parser.add_argument('--out',required=True)
|
|
parser.add_argument('--year',type=int,action='append');parser.add_argument('--venue',choices=['CVPR','ICCV','ECCV'],action='append')
|
|
args=parser.parse_args();out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
|
|
indexes=discover(args.source,args.cache)
|
|
if args.year and set(args.year)-{r['year'] for r in indexes}:parser.error('Requested year is absent from the official source')
|
|
if args.venue and set(args.venue)-{r['venue'] for r in indexes}:parser.error('Requested venue is absent from this source')
|
|
if not any((not args.year or r['year'] in args.year) and (not args.venue or r['venue'] in args.venue) for r in indexes):parser.error('Requested venue/year combination is absent from the official source')
|
|
path=out/f'{args.source}-index-status.json';catalogue=json.loads(path.read_text()) if path.exists() else {};requested=[]
|
|
(out/f'{args.source}-discovery.json').write_text(json.dumps({'source':args.source,'root_url':ROOTS[args.source],
|
|
'discovered_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'indexes':[
|
|
{**{k:index[k] for k in ('venue','year','track','index_url')},
|
|
'index_link_count':len(index['entries']) if 'entries' in index else None} for index in indexes]},indent=2))
|
|
for index in indexes:
|
|
if args.year and index['year'] not in args.year or args.venue and index['venue'] not in args.venue:continue
|
|
key=f'{index["venue"]}:{index["year"]}:{index["track"]}';requested.append(key)
|
|
try:
|
|
manifest=collect_index(args.source,index,args.cache,args.out);catalogue[key]={'status':manifest['status'],'files':[manifest]}
|
|
except Exception as exc:
|
|
catalogue[key]={'status':'blocked','year':index['year'],'venue':index['venue'],'track':index['track'],'index_url':index['index_url'],'error':str(exc)}
|
|
print(json.dumps(catalogue[key]),flush=True)
|
|
part=path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(path)
|
|
(out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year,'venue_filter':args.venue},indent=2))
|
|
if any(catalogue[key]['status']=='blocked' for key in requested):raise SystemExit(1)
|
|
|
|
|
|
if __name__=='__main__':main()
|