diff --git a/README.md b/README.md index 4d3796a..af7cb53 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 目标包括:完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。 -**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology、ICLR 官方公开程序元数据采集、带清单核验的会议导入和只读查询。PeerRead 提供公开 ICLR 2017 明确决定的补充来源。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。其他会议适配器、历史缺失届次、完整投稿历史、跨平台疑似重复复核和覆盖率核验仍待完成。回归检查在远端执行,已配置 ces-control:host 对应的 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。 +**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology、ICLR 官方公开程序、CVF/ECVA 视觉会议元数据采集、带清单核验的会议导入和只读查询。PeerRead 提供公开 ICLR 2017 明确决定的补充来源。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。其他会议适配器、历史缺失届次、完整投稿历史、跨平台疑似重复复核和覆盖率核验仍待完成。回归检查在远端执行,已配置 ces-control:host 对应的 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。 ## 数据范围与来源 @@ -180,6 +180,21 @@ python3 -B query.py --db /data/paper-index/papers.sqlite \ 主题、关键词使用 `iclr_program:topic`、`iclr_program:keyword` 命名空间。通用导入器按源记录刷新标签证据,保留同一论文其他源记录及 `custom:*` 标签。每年的 JSONL、manifest、`iclr-program-year-status.json` 和 `iclr-program-last-run.json` 保存实际范围、独立展示数、主会展示数、论文数、缺失摘要与缺失 forum 数。录用程序不能提供拒稿全集,公开投稿及拒稿补充仍需独立来源。 +### CVF / ECVA 视觉会议元数据 + +```sh +python3 -B collect_vision.py --source cvf \ + --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected +python3 -B collect_vision.py --source ecva \ + --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected +``` + +CVF 从官方根目录发现 CVPR、ICCV 主会与 Findings,分别保存 track;不把 workshop 或 WACV 算入这些会议。优先跟随页面明确提供的 All Papers 链接,避免只收录默认显示的第一天。ECVA 从官方总目录按论文 URL 的会议与年份分组,当前总目录列有 ECCV 2018、2020、2022、2024。不存在于来源的 `--venue`、`--year` 或二者组合会报错,不补造双年会议届次,也不能因此推断旧届次未举办。 + +只访问论文索引与摘要 HTML/PHP 页面。作者优先取 citation 元数据,旧页面尊重声明编码;无 citation 作者时取索引中的作者行,排除论文集出版信息。DOI 只取该论文条目所属的 DOI 链接或 citation 字段;冲突会失败,不从摘要引用中抽取当前论文 ID。不请求 PDF、补充材料、视频或代码。 + +每份清单记录索引入口数、论文数、缺失摘要与作者数,缺失内容保留为空并标记 partial。存在未被标题条目解析的论文链接时会失败,防止静默漏项。逐来源的 discovery、index-status、last-run 文件分别保存发现目录、逐届结果和实际请求范围。可重复使用 `--year` 或 `--venue` 限定重试;其他届次状态会保留。官方开放目录之外的早期届次、尚未刊出的新届次和独立出版方历史数据仍需后续补齐,采集器完成不代表全历史覆盖已完成。 + ## 长期维护与更新 1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发,遵守来源限流、重试与人机验证要求。 @@ -194,6 +209,6 @@ python3 -B query.py --db /data/paper-index/papers.sqlite \ 2026-11-06 从目标服务器测试:Kaggle 元数据 GET 可下载;OpenReview 会议 group 接口可读,但 notes 接口返回 `ChallengeRequiredError`。拒稿记录导入尚受此限制,不能用录用名单替代全部投稿,也不能声称已覆盖全部拒稿。需通过平台支持的访问方式或有合法来源的公开数据导出继续补全。 -已验证的阶段性结果:ICML 2025 的 3,330 条记录和 PeerRead ICLR 2017 的 427 条记录已合并主库,合计 3,502 条 accepted 决定、255 条 rejected 决定;arXiv 曾在 529,000 条断点暂停合并,已验证恢复。这些是阶段性值,不是全量交付数据量。 +已核验的采集文件计数:ICLR 2020–2026 主会 15,534 条、COLM 2024–2026 共 1,573 条,摘要均已取得;候选范围内的 PMLR 文件为 31,428 条,已剔除两份重复出版元数据。ACL Anthology 已选文件为 63,297 条,其中 15,147 条源记录缺摘要。这些是采集文件的来源记录数,不能相加当作去重论文数,也不能代替主库实时统计;实际入库量使用 query.py --stats 查看。 -代码在 `codex/metadata-index` 工作分支交付,[Draft PR #1](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/1) 尚未合并,等待全量就绪后用户手动合并。已推送的前三个开发提交均有成功的远程 CI;后续提交仍需检查其各自 CI。全量导入、其他会议适配器、去重复核和覆盖率核验继续进行,应随实际结果更新此说明。 +代码在 `codex/metadata-index` 工作分支交付,[Draft PR #1](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/1) 尚未合并,等待全量就绪后用户手动合并。各提交分别由远程 CI 验证,结果以对应提交的 Gitea Actions 检查为准。全量导入、其他会议适配器、去重复核和覆盖率核验继续进行,应随实际结果更新此说明。 diff --git a/collect_vision.py b/collect_vision.py new file mode 100644 index 0000000..9709222 --- /dev/null +++ b/collect_vision.py @@ -0,0 +1,189 @@ +"""Collect CVF/ECVA public titles, abstracts and bibliographic identifiers.""" +import argparse +from collections import defaultdict +import json +from pathlib import Path +import re +import time +from urllib.parse import parse_qs, unquote, urljoin, urlsplit + +from collect_pmlr import fetch +from collect_proceedings import Document, Node, clean + +ROOTS={'cvf':'https://openaccess.thecvf.com/','ecva':'https://www.ecva.net/papers.php'} +HOSTS={'openaccess.thecvf.com','www.ecva.net','ecva.net'} + + +def decode_html(data): + try: + return data.decode('utf-8-sig') + except UnicodeDecodeError: + charset=re.search(br'charset\s*=\s*["\x27]?([\w-]+)',data[:4096],re.I) + if not charset: + raise + return data.decode(charset[1].decode('ascii')) + + +def publication(url): + parsed=urlsplit(url) + match=re.search(r'(CVPR|ICCV|ECCV)[_/]?(20\d{2})',parsed.path,re.I) + if parsed.netloc in HOSTS and '/html/' in parsed.path and parsed.path.endswith(('.html','.php')) and match: + return match[1].upper(),int(match[2]) + + +def doi_link(url): + parsed=urlsplit(url) + value=None + if parsed.netloc in ('doi.org','dx.doi.org'): + value=unquote(parsed.path.lstrip('/')) + elif parsed.netloc=='link.springer.com' and parsed.path.startswith('/chapter/'): + value=unquote(parsed.path.removeprefix('/chapter/')) + return value.lower() if value and re.fullmatch(r'10\.\d{4,9}/\S+',value) else None + + +def index_entries(html,index_url): + records={} + raw_count=0 + for dl in (n for n in Document(html).root.walk() if n.tag=='dl'): + current=None + for node in (n for n in dl.children if isinstance(n,Node)): + if node.tag=='dt': + current=None + links=[urljoin(index_url,n.attrs['href']) for n in node.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))] + if not links: + continue + if len(set(links))!=1: + raise ValueError('Ambiguous publication links in one index title') + url=links[0];raw_count+=1 + if url in records and records[url]['index_title']!=clean(node.text()): + raise ValueError('Different index titles share a publication URL') + current=records.setdefault(url,{'url':url,'index_title':clean(node.text()),'index_authors':None,'doi':None}) + elif node.tag=='dd' and current is not None: + text=clean(node.text()) + # Author rows precede download/bibliography rows. + if current['index_authors'] is None and text and not text.startswith('['): + current['index_authors']=text + for link in (n for n in node.walk() if n.tag=='a' and clean(n.text()).casefold()=='doi'): + value=doi_link(urljoin(index_url,link.attrs.get('href',''))) + if value: + if current['doi'] and current['doi']!=value: + raise ValueError('Conflicting DOI links in one index entry') + current['doi']=value + if not records: + raise ValueError('No official paper entries found') + all_links={urljoin(index_url,n.attrs['href']) for n in Document(html).root.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))} + if all_links!=set(records):raise ValueError('Paper links exist outside parsed title entries') + return list(records.values()),raw_count + + +def discover(source,cache): + root=ROOTS[source] + html=decode_html(fetch(root,cache,32*1024**2,delay=1)) + if source=='ecva': + entries,_=index_entries(html,root) + groups=defaultdict(list) + for entry in entries: + venue,year=publication(entry['url']) + if venue=='ECCV':groups[(venue,year)].append(entry) + return [{'venue':venue,'year':year,'track':'main','index_url':root,'entries':entries} + for (venue,year),entries in sorted(groups.items(),key=lambda item:item[0][1])] + indexes={} + for node in Document(html).root.walk(): + if node.tag!='a':continue + url=urljoin(root,node.attrs.get('href',''));parsed=urlsplit(url) + match=re.fullmatch(r'/(CVPR|ICCV)(20\d{2})(_findings)?(?:\.py)?/?',parsed.path) + if parsed.netloc=='openaccess.thecvf.com' and match: + indexes[url]={'venue':match[1],'year':int(match[2]),'track':'findings' if match[3] else 'main','index_url':url} + if not indexes:raise ValueError('No official CVPR/ICCV indexes found') + return sorted(indexes.values(),key=lambda row:(row['year'],row['venue'],row['track']!='main',row['track'])) + + +def parse_paper(html,entry,source,venue,year,track,index_url): + nodes=list(Document(html).root.walk());meta=defaultdict(list) + for node in nodes: + if node.tag=='meta':meta[node.attrs.get('name','').lower()].append(node.attrs.get('content','')) + title=next(iter(meta['citation_title']),None) or next((clean(n.text()) for n in nodes if n.attrs.get('id')=='papertitle'),None) + if not title:raise ValueError('Missing official paper title: '+entry['url']) + abstract=next((clean(n.text()) or None for n in nodes if n.attrs.get('id')=='abstract'),None) + authors=[] + for author in meta['citation_author']: + if ',' in author: + family,given=author.split(',',1);author=given.strip()+' '+family.strip() + if clean(author):authors.append(clean(author)) + if not authors: + raw=next((clean(n.text()) for n in nodes if n.attrs.get('id')=='authors'),'') or entry.get('index_authors') or '' + raw=raw.split(';',1)[0] + if ' and ' in raw and all(',' in name for name in raw.split(' and ')): + names=[' '.join(reversed([part.strip() for part in name.split(',',1)])) for name in raw.split(' and ')] + else:names=raw.split(',') + authors=[clean(name).removesuffix('*').strip() for name in names if clean(name).removesuffix('*').strip()] + dois={value for doi in meta['citation_doi'] if doi and (value:=doi_link(doi) or doi.strip().lower()) and re.fullmatch(r'10\.\d{4,9}/\S+',value)} + if entry.get('doi'):dois.add(entry['doi']) + if len(dois)>1:raise ValueError('Citation DOI differs from the official index') + identifiers={'doi':dois.pop()} if dois else {} + # Abstract/reference links are never interpreted as this paper's identifiers. + return {'source':source,'external_id':urlsplit(entry['url']).path,'url':entry['url'], + 'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers, + 'venue':venue,'year':year,'track':track,'status':'accepted', + 'raw_decision':'listed in official proceedings','list_url':index_url, + 'metadata':{'citation':dict(meta),'index_title':entry['index_title'],'index_authors':entry.get('index_authors')}} + + +def collect_index(source,index,cache,out): + url=index['index_url'];venue=index['venue'];year=index['year'];track=index['track'] + entries=index.get('entries') + raw_count=len(entries) if entries is not None else None + if entries is None: + html=decode_html(fetch(url,cache,32*1024**2,delay=1)) + # Some defaults contain only day one; prefer the explicitly linked all-papers page. + all_pages={urljoin(url,n.attrs.get('href','')) for n in Document(html).root.walk() if n.tag=='a' and parse_qs(urlsplit(urljoin(url,n.attrs.get('href',''))).query).get('day')==['all'] and urlsplit(urljoin(url,n.attrs.get('href',''))).netloc==urlsplit(url).netloc and urlsplit(urljoin(url,n.attrs.get('href',''))).path==urlsplit(url).path} + if len(all_pages)>1:raise ValueError('Ambiguous all-papers index') + if all_pages: + url=all_pages.pop();html=decode_html(fetch(url,cache,32*1024**2,delay=1)) + entries,raw_count=index_entries(html,url) + records=[] + for number,entry in enumerate(entries,1): + if publication(entry['url'])!=(venue,year):raise ValueError('Publication path differs from requested edition') + record=parse_paper(decode_html(fetch(entry['url'],cache,2*1024**2,delay=1)),entry,source,venue,year,track,url) + records.append(record) + if number%25==0:print(json.dumps({'source':source,'venue':venue,'year':year,'track':track,'collected':number,'expected':len(entries)}),flush=True) + if not records:raise ValueError('Empty publication list') + path=Path(out)/f'{source}-{venue}{year}-{track}.jsonl';path.parent.mkdir(parents=True,exist_ok=True) + part=path.with_suffix('.part');part.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records));part.replace(path) + missing=sum(not r['abstract'] for r in records);missing_authors=sum(not r['authors'] for r in records) + manifest={'source':source,'parser_version':1,'venue':venue,'year':year,'track':track, + 'count':len(records),'index_link_count':len(entries),'raw_index_entry_count':raw_count, + 'missing_abstract_count':missing,'missing_authors_count':missing_authors,'file':str(path), + 'list_url':url,'status':'partial' if missing or missing_authors else 'collected'} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(manifest),flush=True) + return manifest + + +def main(): + parser=argparse.ArgumentParser();parser.add_argument('--source',choices=ROOTS,required=True) + parser.add_argument('--cache',required=True);parser.add_argument('--out',required=True) + parser.add_argument('--year',type=int,action='append');parser.add_argument('--venue',choices=['CVPR','ICCV','ECCV'],action='append') + args=parser.parse_args();out=Path(args.out);out.mkdir(parents=True,exist_ok=True) + indexes=discover(args.source,args.cache) + if args.year and set(args.year)-{r['year'] for r in indexes}:parser.error('Requested year is absent from the official source') + if args.venue and set(args.venue)-{r['venue'] for r in indexes}:parser.error('Requested venue is absent from this source') + if not any((not args.year or r['year'] in args.year) and (not args.venue or r['venue'] in args.venue) for r in indexes):parser.error('Requested venue/year combination is absent from the official source') + path=out/f'{args.source}-index-status.json';catalogue=json.loads(path.read_text()) if path.exists() else {};requested=[] + (out/f'{args.source}-discovery.json').write_text(json.dumps({'source':args.source,'root_url':ROOTS[args.source], + 'discovered_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'indexes':[ + {**{k:index[k] for k in ('venue','year','track','index_url')}, + 'index_link_count':len(index['entries']) if 'entries' in index else None} for index in indexes]},indent=2)) + for index in indexes: + if args.year and index['year'] not in args.year or args.venue and index['venue'] not in args.venue:continue + key=f'{index["venue"]}:{index["year"]}:{index["track"]}';requested.append(key) + try: + manifest=collect_index(args.source,index,args.cache,args.out);catalogue[key]={'status':manifest['status'],'files':[manifest]} + except Exception as exc: + catalogue[key]={'status':'blocked','year':index['year'],'venue':index['venue'],'track':index['track'],'index_url':index['index_url'],'error':str(exc)} + print(json.dumps(catalogue[key]),flush=True) + part=path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(path) + (out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year,'venue_filter':args.venue},indent=2)) + if any(catalogue[key]['status']=='blocked' for key in requested):raise SystemExit(1) + + +if __name__=='__main__':main() diff --git a/test_vision.py b/test_vision.py new file mode 100644 index 0000000..e236bc4 --- /dev/null +++ b/test_vision.py @@ -0,0 +1,72 @@ +import tempfile +import unittest +from unittest.mock import patch + +from collect_vision import collect_index, decode_html, discover, index_entries, parse_paper + + +class VisionTests(unittest.TestCase): + def entry(self): + return {'url':'https://www.ecva.net/papers/eccv_2024/papers_ECCV/html/1_ECCV_2024_paper.php','index_title':'Paper','index_authors':'Jane Doe*, John Smith','doi':'10.1007/example_1'} + + def test_ecva_doi_belongs_to_its_own_title_entry(self): + html='
First paper
Jane Doe*, John Smith
DOI
Second paper
Other Author
DOI
' + entries,count=index_entries(html,'https://www.ecva.net/papers.php') + self.assertEqual(count,2) + self.assertEqual([e['doi'] for e in entries],['10.1007/example_1','10.1007/example_2']) + html='
Paper
Abstract citing another paper and a reference.
' + row=parse_paper(html,entries[0],'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php') + self.assertEqual(row['identifiers'],{'doi':'10.1007/example_1'}) + self.assertEqual(row['authors'],['Jane Doe','John Smith']) + + def test_cvf_meta_authors_and_missing_abstract_never_capture_footer(self): + html='
Do not include the proceedings citation

Copyright footer

' + entry=dict(self.entry(),doi=None) + row=parse_paper(html,entry,'cvf','CVPR',2013,'main','https://openaccess.thecvf.com/CVPR2013') + self.assertEqual(row['authors'],['Jane Doe']) + self.assertIsNone(row['abstract']) + self.assertEqual(row['title'],'A vision paper') + + def test_conflicting_self_doi_is_rejected(self): + html='' + with self.assertRaisesRegex(ValueError,'DOI differs'): + parse_paper(html,self.entry(),'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php') + + def test_legacy_ecva_author_formats_do_not_become_extra_people(self): + entry=dict(self.entry(),index_authors='Doe, Jane and Smith, John') + page='
Paper
Jane Doe, John Smith; The European Conference on Computer Vision (ECCV), 2018, pp. 1-12
' + row=parse_paper(page,entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php') + self.assertEqual(row['authors'],['Jane Doe','John Smith']) + row=parse_paper('
Paper
',entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php') + self.assertEqual(row['authors'],['Jane Doe','John Smith']) + + def test_cvpr_day_one_is_replaced_by_explicit_all_papers_index(self): + root='https://openaccess.thecvf.com/CVPR2026';all_url=root+'?day=all' + first='https://openaccess.thecvf.com/content/CVPR2026/html/First_CVPR_2026_paper.html' + second='https://openaccess.thecvf.com/content/CVPR2026/html/Second_CVPR_2026_paper.html' + def item(url):return f'
Paper
Jane Doe, John Smith
[pdf]
' + pages={root:f'All papers
{item(first)}
',all_url:f'
{item(first)}{item(second)}
',first:'
First paper
First abstract
',second:'
Second paper

Footer without an abstract

'} + with tempfile.TemporaryDirectory() as tmp,patch('collect_vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch: + manifest=collect_index('cvf',{'venue':'CVPR','year':2026,'track':'main','index_url':root},tmp,tmp) + self.assertEqual(manifest['count'],2) + self.assertEqual(manifest['list_url'],all_url) + self.assertEqual(manifest['status'],'partial') + self.assertEqual(manifest['missing_abstract_count'],1) + self.assertEqual({call.args[0] for call in fetch.call_args_list},{root,all_url,first,second}) + + def test_discovery_keeps_findings_separate_and_excludes_workshops(self): + html='MainFindingsMainLegacy mainWorkshopsWACV' + with patch('collect_vision.fetch',return_value=html.encode()):indexes=discover('cvf','unused') + self.assertEqual({(x['venue'],x['year'],x['track']) for x in indexes},{('CVPR',2013,'main'),('CVPR',2026,'main'),('CVPR',2026,'findings'),('ICCV',2025,'main')}) + + def test_unparsed_publication_link_does_not_silently_reduce_coverage(self): + html='Other paper
Paper
' + with self.assertRaisesRegex(ValueError,'outside parsed'): + index_entries(html,'https://openaccess.thecvf.com/') + + def test_legacy_declared_encoding_is_respected(self): + html='
André
' + self.assertIn('André',decode_html(html.encode('iso-8859-1'))) + + +if __name__=='__main__':unittest.main() diff --git a/venues.json b/venues.json index ca25bc6..1784f8a 100644 --- a/venues.json +++ b/venues.json @@ -235,8 +235,7 @@ "venue_type": "conference", "discovery_sources": [ "cvf", - "ecva", - "openreview" + "dblp" ], "scope": "candidate_pending_scope_review", "years": "all_publicly_available", @@ -251,8 +250,7 @@ "venue_type": "conference", "discovery_sources": [ "cvf", - "ecva", - "openreview" + "dblp" ], "scope": "candidate_pending_scope_review", "years": "all_publicly_available", @@ -266,9 +264,8 @@ "field": "vision", "venue_type": "conference", "discovery_sources": [ - "cvf", "ecva", - "openreview" + "dblp" ], "scope": "candidate_pending_scope_review", "years": "all_publicly_available",