Files
paper-atlas/collect_vision.py
T
codex-bot 396b7dbaf2
Metadata integrity checks / metadata (push) Successful in 14s
Collect CVPR ICCV and ECCV official abstract metadata
2026-11-06 13:15:20 +08:00

190 lines
12 KiB
Python

"""Collect CVF/ECVA public titles, abstracts and bibliographic identifiers."""
import argparse
from collections import defaultdict
import json
from pathlib import Path
import re
import time
from urllib.parse import parse_qs, unquote, urljoin, urlsplit
from collect_pmlr import fetch
from collect_proceedings import Document, Node, clean
ROOTS={'cvf':'https://openaccess.thecvf.com/','ecva':'https://www.ecva.net/papers.php'}
HOSTS={'openaccess.thecvf.com','www.ecva.net','ecva.net'}
def decode_html(data):
try:
return data.decode('utf-8-sig')
except UnicodeDecodeError:
charset=re.search(br'charset\s*=\s*["\x27]?([\w-]+)',data[:4096],re.I)
if not charset:
raise
return data.decode(charset[1].decode('ascii'))
def publication(url):
parsed=urlsplit(url)
match=re.search(r'(CVPR|ICCV|ECCV)[_/]?(20\d{2})',parsed.path,re.I)
if parsed.netloc in HOSTS and '/html/' in parsed.path and parsed.path.endswith(('.html','.php')) and match:
return match[1].upper(),int(match[2])
def doi_link(url):
parsed=urlsplit(url)
value=None
if parsed.netloc in ('doi.org','dx.doi.org'):
value=unquote(parsed.path.lstrip('/'))
elif parsed.netloc=='link.springer.com' and parsed.path.startswith('/chapter/'):
value=unquote(parsed.path.removeprefix('/chapter/'))
return value.lower() if value and re.fullmatch(r'10\.\d{4,9}/\S+',value) else None
def index_entries(html,index_url):
records={}
raw_count=0
for dl in (n for n in Document(html).root.walk() if n.tag=='dl'):
current=None
for node in (n for n in dl.children if isinstance(n,Node)):
if node.tag=='dt':
current=None
links=[urljoin(index_url,n.attrs['href']) for n in node.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))]
if not links:
continue
if len(set(links))!=1:
raise ValueError('Ambiguous publication links in one index title')
url=links[0];raw_count+=1
if url in records and records[url]['index_title']!=clean(node.text()):
raise ValueError('Different index titles share a publication URL')
current=records.setdefault(url,{'url':url,'index_title':clean(node.text()),'index_authors':None,'doi':None})
elif node.tag=='dd' and current is not None:
text=clean(node.text())
# Author rows precede download/bibliography rows.
if current['index_authors'] is None and text and not text.startswith('['):
current['index_authors']=text
for link in (n for n in node.walk() if n.tag=='a' and clean(n.text()).casefold()=='doi'):
value=doi_link(urljoin(index_url,link.attrs.get('href','')))
if value:
if current['doi'] and current['doi']!=value:
raise ValueError('Conflicting DOI links in one index entry')
current['doi']=value
if not records:
raise ValueError('No official paper entries found')
all_links={urljoin(index_url,n.attrs['href']) for n in Document(html).root.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))}
if all_links!=set(records):raise ValueError('Paper links exist outside parsed title entries')
return list(records.values()),raw_count
def discover(source,cache):
root=ROOTS[source]
html=decode_html(fetch(root,cache,32*1024**2,delay=1))
if source=='ecva':
entries,_=index_entries(html,root)
groups=defaultdict(list)
for entry in entries:
venue,year=publication(entry['url'])
if venue=='ECCV':groups[(venue,year)].append(entry)
return [{'venue':venue,'year':year,'track':'main','index_url':root,'entries':entries}
for (venue,year),entries in sorted(groups.items(),key=lambda item:item[0][1])]
indexes={}
for node in Document(html).root.walk():
if node.tag!='a':continue
url=urljoin(root,node.attrs.get('href',''));parsed=urlsplit(url)
match=re.fullmatch(r'/(CVPR|ICCV)(20\d{2})(_findings)?(?:\.py)?/?',parsed.path)
if parsed.netloc=='openaccess.thecvf.com' and match:
indexes[url]={'venue':match[1],'year':int(match[2]),'track':'findings' if match[3] else 'main','index_url':url}
if not indexes:raise ValueError('No official CVPR/ICCV indexes found')
return sorted(indexes.values(),key=lambda row:(row['year'],row['venue'],row['track']!='main',row['track']))
def parse_paper(html,entry,source,venue,year,track,index_url):
nodes=list(Document(html).root.walk());meta=defaultdict(list)
for node in nodes:
if node.tag=='meta':meta[node.attrs.get('name','').lower()].append(node.attrs.get('content',''))
title=next(iter(meta['citation_title']),None) or next((clean(n.text()) for n in nodes if n.attrs.get('id')=='papertitle'),None)
if not title:raise ValueError('Missing official paper title: '+entry['url'])
abstract=next((clean(n.text()) or None for n in nodes if n.attrs.get('id')=='abstract'),None)
authors=[]
for author in meta['citation_author']:
if ',' in author:
family,given=author.split(',',1);author=given.strip()+' '+family.strip()
if clean(author):authors.append(clean(author))
if not authors:
raw=next((clean(n.text()) for n in nodes if n.attrs.get('id')=='authors'),'') or entry.get('index_authors') or ''
raw=raw.split(';',1)[0]
if ' and ' in raw and all(',' in name for name in raw.split(' and ')):
names=[' '.join(reversed([part.strip() for part in name.split(',',1)])) for name in raw.split(' and ')]
else:names=raw.split(',')
authors=[clean(name).removesuffix('*').strip() for name in names if clean(name).removesuffix('*').strip()]
dois={value for doi in meta['citation_doi'] if doi and (value:=doi_link(doi) or doi.strip().lower()) and re.fullmatch(r'10\.\d{4,9}/\S+',value)}
if entry.get('doi'):dois.add(entry['doi'])
if len(dois)>1:raise ValueError('Citation DOI differs from the official index')
identifiers={'doi':dois.pop()} if dois else {}
# Abstract/reference links are never interpreted as this paper's identifiers.
return {'source':source,'external_id':urlsplit(entry['url']).path,'url':entry['url'],
'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,
'venue':venue,'year':year,'track':track,'status':'accepted',
'raw_decision':'listed in official proceedings','list_url':index_url,
'metadata':{'citation':dict(meta),'index_title':entry['index_title'],'index_authors':entry.get('index_authors')}}
def collect_index(source,index,cache,out):
url=index['index_url'];venue=index['venue'];year=index['year'];track=index['track']
entries=index.get('entries')
raw_count=len(entries) if entries is not None else None
if entries is None:
html=decode_html(fetch(url,cache,32*1024**2,delay=1))
# Some defaults contain only day one; prefer the explicitly linked all-papers page.
all_pages={urljoin(url,n.attrs.get('href','')) for n in Document(html).root.walk() if n.tag=='a' and parse_qs(urlsplit(urljoin(url,n.attrs.get('href',''))).query).get('day')==['all'] and urlsplit(urljoin(url,n.attrs.get('href',''))).netloc==urlsplit(url).netloc and urlsplit(urljoin(url,n.attrs.get('href',''))).path==urlsplit(url).path}
if len(all_pages)>1:raise ValueError('Ambiguous all-papers index')
if all_pages:
url=all_pages.pop();html=decode_html(fetch(url,cache,32*1024**2,delay=1))
entries,raw_count=index_entries(html,url)
records=[]
for number,entry in enumerate(entries,1):
if publication(entry['url'])!=(venue,year):raise ValueError('Publication path differs from requested edition')
record=parse_paper(decode_html(fetch(entry['url'],cache,2*1024**2,delay=1)),entry,source,venue,year,track,url)
records.append(record)
if number%25==0:print(json.dumps({'source':source,'venue':venue,'year':year,'track':track,'collected':number,'expected':len(entries)}),flush=True)
if not records:raise ValueError('Empty publication list')
path=Path(out)/f'{source}-{venue}{year}-{track}.jsonl';path.parent.mkdir(parents=True,exist_ok=True)
part=path.with_suffix('.part');part.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records));part.replace(path)
missing=sum(not r['abstract'] for r in records);missing_authors=sum(not r['authors'] for r in records)
manifest={'source':source,'parser_version':1,'venue':venue,'year':year,'track':track,
'count':len(records),'index_link_count':len(entries),'raw_index_entry_count':raw_count,
'missing_abstract_count':missing,'missing_authors_count':missing_authors,'file':str(path),
'list_url':url,'status':'partial' if missing or missing_authors else 'collected'}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(manifest),flush=True)
return manifest
def main():
parser=argparse.ArgumentParser();parser.add_argument('--source',choices=ROOTS,required=True)
parser.add_argument('--cache',required=True);parser.add_argument('--out',required=True)
parser.add_argument('--year',type=int,action='append');parser.add_argument('--venue',choices=['CVPR','ICCV','ECCV'],action='append')
args=parser.parse_args();out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
indexes=discover(args.source,args.cache)
if args.year and set(args.year)-{r['year'] for r in indexes}:parser.error('Requested year is absent from the official source')
if args.venue and set(args.venue)-{r['venue'] for r in indexes}:parser.error('Requested venue is absent from this source')
if not any((not args.year or r['year'] in args.year) and (not args.venue or r['venue'] in args.venue) for r in indexes):parser.error('Requested venue/year combination is absent from the official source')
path=out/f'{args.source}-index-status.json';catalogue=json.loads(path.read_text()) if path.exists() else {};requested=[]
(out/f'{args.source}-discovery.json').write_text(json.dumps({'source':args.source,'root_url':ROOTS[args.source],
'discovered_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'indexes':[
{**{k:index[k] for k in ('venue','year','track','index_url')},
'index_link_count':len(index['entries']) if 'entries' in index else None} for index in indexes]},indent=2))
for index in indexes:
if args.year and index['year'] not in args.year or args.venue and index['venue'] not in args.venue:continue
key=f'{index["venue"]}:{index["year"]}:{index["track"]}';requested.append(key)
try:
manifest=collect_index(args.source,index,args.cache,args.out);catalogue[key]={'status':manifest['status'],'files':[manifest]}
except Exception as exc:
catalogue[key]={'status':'blocked','year':index['year'],'venue':index['venue'],'track':index['track'],'index_url':index['index_url'],'error':str(exc)}
print(json.dumps(catalogue[key]),flush=True)
part=path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(path)
(out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year,'venue_filter':args.venue},indent=2))
if any(catalogue[key]['status']=='blocked' for key in requested):raise SystemExit(1)
if __name__=='__main__':main()