From 66b98fa3759ec9ae20ee276b968f119b2f4e412e Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Fri, 6 Nov 2026 03:19:30 +0800 Subject: [PATCH] Fetch old PMLR metadata without entire PDF archives --- README.md | 4 ++-- collect_pmlr.py | 61 +++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e61530e..3da3ff5 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ python3 -B query.py --db /data/paper-index/papers.sqlite --venue COLM --year 202 ## 官方会议元数据采集 ```sh -# 单卷示例：ICML 2025，读取官方 gh-pages 元数据归档，不下载 PDF +# 单卷示例：ICML 2025，读取官方 gh-pages 元数据，不下载论文正文 python3 -B collect_pmlr.py --cache /data/paper-index/raw/http-cache \ --out /data/paper-index/collected --volume 267 @@ -110,7 +110,7 @@ python3 -B import_records.py --db /data/paper-index/papers.sqlite \ /data/paper-index/collected/pmlr-v267.jsonl ``` -采集输出 JSONL 和对应 manifest，只有显式官方录用记录才产生 accepted 决定。清单数目不一致时整个文件导入回滚，记录失败批次，不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。缓存默认有效 24 小时；同一次批次可复用缓存，后续更新会重新读取动态入口。 +采集输出 JSONL 和对应 manifest，只有显式官方录用记录才产生 accepted 决定。清单数目不一致时整个文件导入回滚，记录失败批次，不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。旧卷可能把 PDF 放入归档；采集器对前 50 卷使用 Git partial clone 和 sparse checkout，仅取 `_posts` 和卷配置，不执行出版方项目代码。较新归档若超出 64 MiB 限额，也转为此方式。已对原本超限的 AISTATS 2009（第 5 卷）验证采集 84 条且工作区无 PDF。缓存默认有效 24 小时；同一次批次可复用缓存，后续更新会重新读取动态入口。 PeerRead 采集仅选择作者公开发布的 ICLR 2017 投稿元数据，固定到源仓库 commit。只依据明确的布尔 `accepted` 标签形成 accepted/rejected；空标签为 unknown，不使用其他数据分区中的 probably-rejected 推断。不会保存 reviews 或 parsed_pdfs 正文。 diff --git a/collect_pmlr.py b/collect_pmlr.py index 09910b0..51ea9e9 100644 --- a/collect_pmlr.py +++ b/collect_pmlr.py @@ -4,6 +4,8 @@ import hashlib import io import json import re +import os +import subprocess import tarfile import time import urllib.error @@ -25,6 +27,8 @@ def fetch(url,cache,limit=64*1024**2): with urllib.request.urlopen(request,timeout=90) as response: if response.status != 200: raise RuntimeError('Expected complete HTTP 200 response') + if int(response.headers.get('Content-Length','0'))>limit: + raise RuntimeError('Metadata response exceeds byte limit') data = response.read(limit+1) if len(data)>limit: raise RuntimeError('Metadata response exceeds byte limit') @@ -40,6 +44,49 @@ def fetch(url,cache,limit=64*1024**2): time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1'))))) +def metadata_texts(volume,cache): + archive = None + if volume>50: + try: + archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache) + except RuntimeError as exc: + if 'exceeds byte limit' not in str(exc): + raise + if archive is not None: + with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar: + for member in tar: + if member.isfile() and '/_posts/' in member.name and member.name.endswith(('.md','.markdown','.html')): + if member.size>8*1024**2: + raise RuntimeError('Unexpectedly large metadata member') + yield member.name,tar.extractfile(member).read().decode('utf-8-sig') + return + # Old volumes may contain PDFs. Partial clone plus sparse checkout retrieves + # trees and metadata blobs only, without checking out or executing publisher code. + checkout = Path(cache)/'metadata-checkouts'/f'pmlr-v{volume}' + checkout.parent.mkdir(parents=True,exist_ok=True) + env = dict(os.environ,GIT_TERMINAL_PROMPT='0') + def git(*args): + result = subprocess.run(['git','-c','credential.helper=',*args],env=env,capture_output=True,text=True,timeout=180) + if result.returncode: + raise RuntimeError('Public metadata Git operation failed: '+result.stderr[-500:]) + return result.stdout + if not checkout.exists(): + git('clone','--filter=blob:none','--depth','1','--single-branch','--branch','gh-pages','--no-checkout',f'https://github.com/mlresearch/v{volume}.git',str(checkout)) + git('-C',str(checkout),'sparse-checkout','set','--no-cone','/_posts/','/_config.yml') + git('-C',str(checkout),'checkout','gh-pages') + else: + if git('-C',str(checkout),'status','--porcelain').strip(): + raise RuntimeError('Metadata checkout has local changes; refusing to overwrite') + git('-C',str(checkout),'fetch','--filter=blob:none','--depth','1','origin','gh-pages') + git('-C',str(checkout),'checkout','--detach','FETCH_HEAD') + posts = checkout/'_posts' + for path in sorted(posts.rglob('*')): + if path.is_file() and path.suffix in ('.md','.markdown','.html'): + if path.stat().st_size>8*1024**2: + raise RuntimeError('Unexpectedly large metadata member') + yield str(path.relative_to(checkout)),path.read_text(encoding='utf-8-sig') + + def collect_volume(volume,cache,out,allowed=None): config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml' config = yaml.safe_load(fetch(config_url,cache)) @@ -49,25 +96,19 @@ def collect_volume(volume,cache,out,allowed=None): year = int(config['year']) title = str(config.get('booktitle','')) track = 'workshop' if 'workshop' in title.lower() else 'main' - archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache) path = Path(out)/f'pmlr-v{volume}.jsonl' path.parent.mkdir(parents=True,exist_ok=True) part = path.with_suffix('.part') count = 0 - with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar,part.open('w') as output: - for member in tar: - if not member.isfile() or '/_posts/' not in member.name or not member.name.endswith(('.md','.markdown','.html')): - continue - if member.size>8*1024**2: - raise RuntimeError('Unexpectedly large metadata member') - content = tar.extractfile(member).read().decode('utf-8-sig') + with part.open('w') as output: + for member_name,content in metadata_texts(volume,cache): parts = content.split('---',2) if len(parts)<3 or parts[0].strip(): - raise RuntimeError('Missing YAML frontmatter: '+member.name) + raise RuntimeError('Missing YAML frontmatter: '+member_name) row = yaml.safe_load(parts[1]) if row.get('layout') not in ('inproceedings','article') or not row.get('title'): continue - external = f"v{volume}/{row.get('id') or Path(member.name).stem}" + external = f"v{volume}/{row.get('id') or Path(member_name).stem}" ids = {} for key in ('doi','openreview','arxiv'): value = row.get(key)