Fetch old PMLR metadata without entire PDF archives

README.md

@@ -101,7 +101,7 @@ python3 -B query.py --db /data/paper-index/papers.sqlite --venue COLM --year 202
 ## 官方会议元数据采集
 
 ```sh
-# 单卷示例：ICML 2025，读取官方 gh-pages 元数据归档，不下载 PDF
+# 单卷示例：ICML 2025，读取官方 gh-pages 元数据，不下载论文正文
 python3 -B collect_pmlr.py --cache /data/paper-index/raw/http-cache \
   --out /data/paper-index/collected --volume 267
 
@@ -110,7 +110,7 @@ python3 -B import_records.py --db /data/paper-index/papers.sqlite \
   /data/paper-index/collected/pmlr-v267.jsonl
 ```
 
-采集输出 JSONL 和对应 manifest，只有显式官方录用记录才产生 accepted 决定。清单数目不一致时整个文件导入回滚，记录失败批次，不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。缓存默认有效 24 小时；同一次批次可复用缓存，后续更新会重新读取动态入口。
+采集输出 JSONL 和对应 manifest，只有显式官方录用记录才产生 accepted 决定。清单数目不一致时整个文件导入回滚，记录失败批次，不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。旧卷可能把 PDF 放入归档；采集器对前 50 卷使用 Git partial clone 和 sparse checkout，仅取 `_posts` 和卷配置，不执行出版方项目代码。较新归档若超出 64 MiB 限额，也转为此方式。已对原本超限的 AISTATS 2009（第 5 卷）验证采集 84 条且工作区无 PDF。缓存默认有效 24 小时；同一次批次可复用缓存，后续更新会重新读取动态入口。
 
 PeerRead 采集仅选择作者公开发布的 ICLR 2017 投稿元数据，固定到源仓库 commit。只依据明确的布尔 `accepted` 标签形成 accepted/rejected；空标签为 unknown，不使用其他数据分区中的 probably-rejected 推断。不会保存 reviews 或 parsed_pdfs 正文。
 

collect_pmlr.py

@@ -4,6 +4,8 @@ import hashlib
 import io
 import json
 import re
+import os
+import subprocess
 import tarfile
 import time
 import urllib.error
@@ -25,6 +27,8 @@ def fetch(url,cache,limit=64*1024**2):
             with urllib.request.urlopen(request,timeout=90) as response:
                 if response.status != 200:
                     raise RuntimeError('Expected complete HTTP 200 response')
+                if int(response.headers.get('Content-Length','0'))>limit:
+                    raise RuntimeError('Metadata response exceeds byte limit')
                 data = response.read(limit+1)
                 if len(data)>limit:
                     raise RuntimeError('Metadata response exceeds byte limit')
@@ -40,6 +44,49 @@ def fetch(url,cache,limit=64*1024**2):
             time.sleep(min(30,max(2**attempt,int(exc.headers.get('Retry-After','1')))))
 
 
+def metadata_texts(volume,cache):
+    archive = None
+    if volume>50:
+        try:
+            archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
+        except RuntimeError as exc:
+            if 'exceeds byte limit' not in str(exc):
+                raise
+    if archive is not None:
+        with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
+            for member in tar:
+                if member.isfile() and '/_posts/' in member.name and member.name.endswith(('.md','.markdown','.html')):
+                    if member.size>8*1024**2:
+                        raise RuntimeError('Unexpectedly large metadata member')
+                    yield member.name,tar.extractfile(member).read().decode('utf-8-sig')
+        return
+    # Old volumes may contain PDFs. Partial clone plus sparse checkout retrieves
+    # trees and metadata blobs only, without checking out or executing publisher code.
+    checkout = Path(cache)/'metadata-checkouts'/f'pmlr-v{volume}'
+    checkout.parent.mkdir(parents=True,exist_ok=True)
+    env = dict(os.environ,GIT_TERMINAL_PROMPT='0')
+    def git(*args):
+        result = subprocess.run(['git','-c','credential.helper=',*args],env=env,capture_output=True,text=True,timeout=180)
+        if result.returncode:
+            raise RuntimeError('Public metadata Git operation failed: '+result.stderr[-500:])
+        return result.stdout
+    if not checkout.exists():
+        git('clone','--filter=blob:none','--depth','1','--single-branch','--branch','gh-pages','--no-checkout',f'https://github.com/mlresearch/v{volume}.git',str(checkout))
+        git('-C',str(checkout),'sparse-checkout','set','--no-cone','/_posts/','/_config.yml')
+        git('-C',str(checkout),'checkout','gh-pages')
+    else:
+        if git('-C',str(checkout),'status','--porcelain').strip():
+            raise RuntimeError('Metadata checkout has local changes; refusing to overwrite')
+        git('-C',str(checkout),'fetch','--filter=blob:none','--depth','1','origin','gh-pages')
+        git('-C',str(checkout),'checkout','--detach','FETCH_HEAD')
+    posts = checkout/'_posts'
+    for path in sorted(posts.rglob('*')):
+        if path.is_file() and path.suffix in ('.md','.markdown','.html'):
+            if path.stat().st_size>8*1024**2:
+                raise RuntimeError('Unexpectedly large metadata member')
+            yield str(path.relative_to(checkout)),path.read_text(encoding='utf-8-sig')
+
+
 def collect_volume(volume,cache,out,allowed=None):
     config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml'
     config = yaml.safe_load(fetch(config_url,cache))
@@ -49,25 +96,19 @@ def collect_volume(volume,cache,out,allowed=None):
     year = int(config['year'])
     title = str(config.get('booktitle',''))
     track = 'workshop' if 'workshop' in title.lower() else 'main'
-    archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache)
     path = Path(out)/f'pmlr-v{volume}.jsonl'
     path.parent.mkdir(parents=True,exist_ok=True)
     part = path.with_suffix('.part')
     count = 0
-    with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar,part.open('w') as output:
-        for member in tar:
-            if not member.isfile() or '/_posts/' not in member.name or not member.name.endswith(('.md','.markdown','.html')):
-                continue
-            if member.size>8*1024**2:
-                raise RuntimeError('Unexpectedly large metadata member')
-            content = tar.extractfile(member).read().decode('utf-8-sig')
+    with part.open('w') as output:
+        for member_name,content in metadata_texts(volume,cache):
             parts = content.split('---',2)
             if len(parts)<3 or parts[0].strip():
-                raise RuntimeError('Missing YAML frontmatter: '+member.name)
+                raise RuntimeError('Missing YAML frontmatter: '+member_name)
             row = yaml.safe_load(parts[1])
             if row.get('layout') not in ('inproceedings','article') or not row.get('title'):
                 continue
-            external = f"v{volume}/{row.get('id') or Path(member.name).stem}"
+            external = f"v{volume}/{row.get('id') or Path(member_name).stem}"
             ids = {}
             for key in ('doi','openreview','arxiv'):
                 value = row.get(key)