263 lines
15 KiB
Python
263 lines
15 KiB
Python
"""Collect CVF/ECVA public titles, abstracts and bibliographic identifiers."""
|
|
import argparse
|
|
from collections import defaultdict
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import time
|
|
from datetime import date
|
|
from urllib.error import HTTPError
|
|
from urllib.parse import parse_qs, unquote, urljoin, urlsplit
|
|
|
|
from paper_atlas.io.http import fetch
|
|
from paper_atlas.io.html import Document, Node, clean
|
|
|
|
ROOTS={'cvf':'https://openaccess.thecvf.com/','ecva':'https://www.ecva.net/papers.php'}
|
|
HOSTS={'openaccess.thecvf.com','www.ecva.net','ecva.net'}
|
|
|
|
|
|
def decode_html(data):
|
|
try:
|
|
return data.decode('utf-8-sig')
|
|
except UnicodeDecodeError:
|
|
charset=re.search(br'charset\s*=\s*["\x27]?([\w-]+)',data[:4096],re.I)
|
|
if not charset:
|
|
raise
|
|
return data.decode(charset[1].decode('ascii'))
|
|
|
|
|
|
def publication(url):
|
|
parsed=urlsplit(url)
|
|
match=re.search(r'(CVPR|ICCV|ECCV)[_/]?(20\d{2})',parsed.path,re.I)
|
|
if parsed.netloc in HOSTS and '/html/' in parsed.path and parsed.path.endswith(('.html','.php')) and match:
|
|
return match[1].upper(),int(match[2])
|
|
|
|
|
|
def doi_link(url):
|
|
parsed=urlsplit(url)
|
|
value=None
|
|
if parsed.netloc in ('doi.org','dx.doi.org'):
|
|
value=unquote(parsed.path.lstrip('/'))
|
|
elif parsed.netloc=='link.springer.com' and parsed.path.startswith('/chapter/'):
|
|
value=unquote(parsed.path.removeprefix('/chapter/'))
|
|
return value.lower() if value and re.fullmatch(r'10\.\d{4,9}/\S+',value) else None
|
|
|
|
|
|
def index_entries(html,index_url):
|
|
records={}
|
|
raw_count=0
|
|
for dl in (n for n in Document(html).root.walk() if n.tag=='dl'):
|
|
current=None
|
|
for node in (n for n in dl.children if isinstance(n,Node)):
|
|
if node.tag=='dt':
|
|
current=None
|
|
links=[urljoin(index_url,n.attrs['href']) for n in node.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))]
|
|
if not links:
|
|
continue
|
|
if len(set(links))!=1:
|
|
raise ValueError('Ambiguous publication links in one index title')
|
|
url=links[0];raw_count+=1
|
|
if url in records and records[url]['index_title']!=clean(node.text()):
|
|
raise ValueError('Different index titles share a publication URL')
|
|
current=records.setdefault(url,{'url':url,'index_title':clean(node.text()),'index_authors':None,'doi':None})
|
|
elif node.tag=='dd' and current is not None:
|
|
text=clean(node.text())
|
|
# Author rows precede download/bibliography rows.
|
|
if current['index_authors'] is None and text and not text.startswith('['):
|
|
current['index_authors']=text
|
|
for link in (n for n in node.walk() if n.tag=='a' and clean(n.text()).casefold()=='doi'):
|
|
value=doi_link(urljoin(index_url,link.attrs.get('href','')))
|
|
if value:
|
|
if current['doi'] and current['doi']!=value:
|
|
raise ValueError('Conflicting DOI links in one index entry')
|
|
current['doi']=value
|
|
if not records:
|
|
raise ValueError('No official paper entries found')
|
|
all_links={urljoin(index_url,n.attrs['href']) for n in Document(html).root.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))}
|
|
if all_links!=set(records):raise ValueError('Paper links exist outside parsed title entries')
|
|
return list(records.values()),raw_count
|
|
|
|
|
|
def discover(source,cache):
|
|
root=ROOTS[source]
|
|
html=decode_html(fetch(root,cache,32*1024**2,delay=1))
|
|
if source=='ecva':
|
|
entries,_=index_entries(html,root)
|
|
groups=defaultdict(list)
|
|
for entry in entries:
|
|
venue,year=publication(entry['url'])
|
|
if venue=='ECCV':groups[(venue,year)].append(entry)
|
|
return [{'venue':venue,'year':year,'track':'main','index_url':root,'entries':entries}
|
|
for (venue,year),entries in sorted(groups.items(),key=lambda item:item[0][1])]
|
|
indexes={}
|
|
for node in Document(html).root.walk():
|
|
if node.tag!='a':continue
|
|
url=urljoin(root,node.attrs.get('href',''));parsed=urlsplit(url)
|
|
match=re.fullmatch(r'/(CVPR|ICCV)(20\d{2})(_findings)?(?:\.py)?/?',parsed.path)
|
|
if parsed.netloc=='openaccess.thecvf.com' and match:
|
|
indexes[url]={'venue':match[1],'year':int(match[2]),'track':'findings' if match[3] else 'main','index_url':url}
|
|
if not indexes:raise ValueError('No official CVPR/ICCV indexes found')
|
|
return sorted(indexes.values(),key=lambda row:(row['year'],row['venue'],row['track']!='main',row['track']))
|
|
|
|
|
|
def author_names(raw):
|
|
raw=(raw or '').split(';',1)[0]
|
|
if ' and ' in raw and all(',' in name for name in raw.split(' and ')):
|
|
names=[' '.join(reversed([part.strip() for part in name.split(',',1)])) for name in raw.split(' and ')]
|
|
else:names=raw.split(',')
|
|
return [clean(name).removesuffix('*').strip() for name in names if clean(name).removesuffix('*').strip()]
|
|
|
|
|
|
def parse_paper(html,entry,source,venue,year,track,index_url):
|
|
nodes=list(Document(html).root.walk());meta=defaultdict(list)
|
|
for node in nodes:
|
|
if node.tag=='meta':meta[node.attrs.get('name','').lower()].append(node.attrs.get('content',''))
|
|
title=next(iter(meta['citation_title']),None) or next((clean(n.text()) for n in nodes if n.attrs.get('id')=='papertitle'),None)
|
|
if not title:raise ValueError('Missing official paper title: '+entry['url'])
|
|
abstract=next((clean(n.text()) or None for n in nodes if n.attrs.get('id')=='abstract'),None)
|
|
authors=[]
|
|
for author in meta['citation_author']:
|
|
if ',' in author:
|
|
family,given=author.split(',',1);author=given.strip()+' '+family.strip()
|
|
if clean(author):authors.append(clean(author))
|
|
if not authors:
|
|
raw=next((clean(n.text()) for n in nodes if n.attrs.get('id')=='authors'),'') or entry.get('index_authors') or ''
|
|
authors=author_names(raw)
|
|
dois={value for doi in meta['citation_doi'] if doi and (value:=doi_link(doi) or doi.strip().lower()) and re.fullmatch(r'10\.\d{4,9}/\S+',value)}
|
|
if entry.get('doi'):dois.add(entry['doi'])
|
|
if len(dois)>1:raise ValueError('Citation DOI differs from the official index')
|
|
identifiers={'doi':dois.pop()} if dois else {}
|
|
# Abstract/reference links are never interpreted as this paper's identifiers.
|
|
return {'source':source,'external_id':urlsplit(entry['url']).path,'url':entry['url'],
|
|
'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,
|
|
'venue':venue,'year':year,'track':track,'status':'accepted',
|
|
'raw_decision':'listed in official proceedings','list_url':index_url,
|
|
'metadata':{'citation':dict(meta),'index_title':entry['index_title'],'index_authors':entry.get('index_authors'),
|
|
'index_urls':entry.get('index_urls',[index_url])}}
|
|
|
|
|
|
def index_identity(url):
|
|
parsed=urlsplit(url)
|
|
match=re.fullmatch(r'/(CVPR|ICCV)(20\d{2})(_findings)?(?:\.py)?/?',parsed.path)
|
|
return match.groups() if parsed.netloc=='openaccess.thecvf.com' and match else None
|
|
|
|
|
|
def read_index(index,cache):
|
|
url=index['index_url'];html=decode_html(fetch(url,cache,32*1024**2,delay=1))
|
|
links=set()
|
|
for node in Document(html).root.walk():
|
|
if node.tag!='a':continue
|
|
link=urljoin(url,node.attrs.get('href',''));day=parse_qs(urlsplit(link).query).get('day',[])
|
|
if index_identity(link)!=index_identity(url) or index_identity(url) is None or len(day)!=1:continue
|
|
if day[0]!='all':
|
|
try:valid=date.fromisoformat(day[0]).year==index['year']
|
|
except ValueError:valid=False
|
|
if not valid:continue
|
|
links.add(link)
|
|
all_pages=sorted(link for link in links if parse_qs(urlsplit(link).query)['day']==['all'])
|
|
day_pages=sorted(links-set(all_pages));index_errors=[]
|
|
if len(all_pages)>1:raise ValueError('Ambiguous all-papers index')
|
|
pages=[]
|
|
if all_pages:
|
|
try:pages=[(all_pages[0],decode_html(fetch(all_pages[0],cache,32*1024**2,delay=1)))]
|
|
except HTTPError as error:
|
|
if error.code!=404 or not day_pages:
|
|
error.close();raise
|
|
index_errors.append({'url':all_pages[0],'status':404});error.close()
|
|
if not pages and day_pages:
|
|
pages=[(page,decode_html(fetch(page,cache,32*1024**2,delay=1))) for page in day_pages]
|
|
if not pages:pages=[(url,html)]
|
|
entries={};raw_count=0
|
|
for page,document in pages:
|
|
parsed,count=index_entries(document,page);raw_count+=count
|
|
for entry in parsed:
|
|
entry['index_urls']=[page];old=entries.get(entry['url'])
|
|
if old:
|
|
for key in ('index_title','index_authors','doi'):
|
|
if old.get(key) and entry.get(key) and old[key]!=entry[key]:raise ValueError('Conflicting metadata across official day indexes')
|
|
if not old.get(key):old[key]=entry.get(key)
|
|
old['index_urls'].append(page)
|
|
else:entries[entry['url']]=entry
|
|
return list(entries.values()),raw_count,[page for page,_ in pages],index_errors
|
|
|
|
|
|
def index_only_record(entry,source,venue,year,track,index_url,error):
|
|
if not entry['index_title'].strip():raise ValueError('Missing title in official index')
|
|
return {'source':source,'external_id':urlsplit(entry['url']).path,'url':entry['url'],
|
|
'title':entry['index_title'],'abstract':None,'authors':author_names(entry.get('index_authors')),
|
|
'identifiers':{'doi':entry['doi']} if entry.get('doi') else {},'venue':venue,'year':year,'track':track,
|
|
'status':'accepted','raw_decision':'listed in official proceedings index',
|
|
'evidence_url':entry['index_urls'][0],'list_url':index_url,
|
|
'metadata':{'index_title':entry['index_title'],'index_authors':entry.get('index_authors'),
|
|
'index_urls':entry['index_urls'],'abstract_fetch_error':{'status':error.code,'url':error.url,'message':str(error)}}}
|
|
|
|
|
|
def collect_index(source,index,cache,out):
|
|
url=index['index_url'];venue=index['venue'];year=index['year'];track=index['track']
|
|
entries=index.get('entries')
|
|
raw_count=len(entries) if entries is not None else None
|
|
index_urls=[url];index_errors=[];missing_pages=[]
|
|
if entries is None:
|
|
entries,raw_count,index_urls,index_errors=read_index(index,cache)
|
|
if len(index_urls)==1:url=index_urls[0]
|
|
records=[]
|
|
for number,entry in enumerate(entries,1):
|
|
if publication(entry['url'])!=(venue,year):raise ValueError('Publication path differs from requested edition')
|
|
try:record=parse_paper(decode_html(fetch(entry['url'],cache,2*1024**2,delay=1)),entry,source,venue,year,track,url)
|
|
except HTTPError as error:
|
|
if error.code!=404:
|
|
error.close();raise
|
|
entry.setdefault('index_urls',[url])
|
|
record=index_only_record(entry,source,venue,year,track,url,error)
|
|
missing_pages.append(entry['url']);error.close()
|
|
records.append(record)
|
|
if number%25==0:print(json.dumps({'source':source,'venue':venue,'year':year,'track':track,'collected':number,'expected':len(entries)}),flush=True)
|
|
if not records:raise ValueError('Empty publication list')
|
|
path=Path(out)/f'{source}-{venue}{year}-{track}.jsonl';path.parent.mkdir(parents=True,exist_ok=True)
|
|
part=path.with_suffix('.part');part.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records));part.replace(path)
|
|
missing=sum(not r['abstract'] for r in records);missing_authors=sum(not r['authors'] for r in records)
|
|
manifest={'source':source,'parser_version':2,'venue':venue,'year':year,'track':track,
|
|
'count':len(records),'index_link_count':len(entries),'raw_index_entry_count':raw_count,
|
|
'missing_abstract_count':missing,'missing_authors_count':missing_authors,'file':str(path),
|
|
'index_urls':index_urls,'index_errors':index_errors,'missing_abstract_pages':missing_pages,
|
|
'list_url':url,'status':'partial' if missing or missing_authors or index_errors else 'collected'}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(manifest),flush=True)
|
|
return manifest
|
|
|
|
|
|
def main(argv=None, *, prog=None, source=None):
|
|
parser=argparse.ArgumentParser(prog=prog)
|
|
if source is None:parser.add_argument('--source',choices=ROOTS,required=True)
|
|
else:parser.set_defaults(source=source)
|
|
parser.add_argument('--cache',required=True);parser.add_argument('--out',required=True)
|
|
parser.add_argument('--year',type=int,action='append');parser.add_argument('--venue',choices=['CVPR','ICCV','ECCV'],action='append')
|
|
parser.add_argument('--retry-catalogue',help='Select only blocked editions from an existing catalogue; write results to --out')
|
|
args=parser.parse_args(argv);out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
|
|
indexes=discover(args.source,args.cache)
|
|
if args.retry_catalogue:
|
|
retry=json.loads(Path(args.retry_catalogue).read_text())
|
|
indexes=[index for index in indexes if retry.get(f'{index["venue"]}:{index["year"]}:{index["track"]}',{}).get('status')=='blocked']
|
|
if not indexes:parser.error('Retry catalogue has no blocked editions for this source')
|
|
if args.year and set(args.year)-{r['year'] for r in indexes}:parser.error('Requested year is absent from the official source')
|
|
if args.venue and set(args.venue)-{r['venue'] for r in indexes}:parser.error('Requested venue is absent from this source')
|
|
if not any((not args.year or r['year'] in args.year) and (not args.venue or r['venue'] in args.venue) for r in indexes):parser.error('Requested venue/year combination is absent from the official source')
|
|
path=out/f'{args.source}-index-status.json';catalogue=json.loads(path.read_text()) if path.exists() else {};requested=[]
|
|
(out/f'{args.source}-discovery.json').write_text(json.dumps({'source':args.source,'root_url':ROOTS[args.source],
|
|
'discovered_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'indexes':[
|
|
{**{k:index[k] for k in ('venue','year','track','index_url')},
|
|
'index_link_count':len(index['entries']) if 'entries' in index else None} for index in indexes]},indent=2))
|
|
for index in indexes:
|
|
if args.year and index['year'] not in args.year or args.venue and index['venue'] not in args.venue:continue
|
|
key=f'{index["venue"]}:{index["year"]}:{index["track"]}';requested.append(key)
|
|
try:
|
|
manifest=collect_index(args.source,index,args.cache,args.out);catalogue[key]={'status':manifest['status'],'files':[manifest]}
|
|
except Exception as exc:
|
|
catalogue[key]={'status':'blocked','year':index['year'],'venue':index['venue'],'track':index['track'],'index_url':index['index_url'],'error':str(exc),'error_url':getattr(exc,'url',None)}
|
|
print(json.dumps(catalogue[key]),flush=True)
|
|
part=path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(path)
|
|
(out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year,'venue_filter':args.venue},indent=2))
|
|
if any(catalogue[key]['status']=='blocked' for key in requested):raise SystemExit(1)
|
|
|
|
|
|
if __name__=='__main__':main()
|