Inventory pinned DBLP conference metadata before scope classification
Metadata integrity checks / metadata (push) Successful in 8s

This commit is contained in:
2026-11-06 16:43:18 +08:00
parent a40efaa77c
commit 4077cb4666
3 changed files with 207 additions and 0 deletions
+17
View File
@@ -162,6 +162,23 @@ python3 -B download_snapshot.py \
[官方版本与校验值](https://doi.org/10.4230/dblp.xml.2026-11-01)。下载器另外计算 SHA-256,记录来源 URL、大小和核验时间。DBLP 没有提供的摘要不能编造,出版目录也不能作为拒稿证据。
DBLP 目录盘点先核验固定快照与[官方 DTD](https://doi.org/10.4230/dblp.xml.dtd.2023-06-28),再流式读取 XML,直至 gzip EOF 检查通过:
```sh
python3 -B download_snapshot.py \
--url https://drops.dagstuhl.de/storage/artifacts/dblp/xml/2023/dblp-2023-06-28.dtd \
--md5 96e4283e71886dec2173718957293648 \
--output /data/paper-index/raw/dblp-2023-06-28.dtd
python3 -B catalogue_dblp.py \
--archive /data/paper-index/raw/dblp-2026-11-01.xml.gz \
--dtd /data/paper-index/raw/dblp-2023-06-28.dtd \
--out /data/paper-index/collected/dblp
```
`dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本;`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。候选范围包含历史名称与 workshop,尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识,才能生成入库文件;不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF,也不更改论文主库。
XML 只允许校验过的本地 `dblp.dtd`,拒绝其他外部实体和内部 DTD 子集,不访问 XML 声明的任意网络地址。临时目录库或 gzip 检查失败时不发布正式候选文件,需先调查残留的 `.part` 状态,再决定是否重新运行。
公开 ICLR 投稿快照的固定 Release 供后续筛选投稿和明确决定;它是第三方保存的 OpenReview 历史副本,不等于最新官方全量数据:
```sh
+136
View File
@@ -0,0 +1,136 @@
"""Inventory a pinned DBLP snapshot and stage candidate metadata for scope review.
This does not label candidates accepted or import them into the paper database.
Only the checksum-verified local DTD is allowed; XML cannot fetch remote entities.
"""
import argparse
from collections import Counter,deque
import gzip
import json
from pathlib import Path
import sqlite3
from xml.parsers import expat
from download_snapshot import digest_file
from submission_evidence import timestamp
XML_SHA='0376ad019b216553faff0a14e1757b2d4c7a3b84c1beacf0511ef83695211b18'
DTD_MD5='96e4283e71886dec2173718957293648'
RELEASE='https://doi.org/10.4230/dblp.xml.2026-11-01'
# Broad discovery candidates, including historical names; not a venue mapping.
FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps kr atal aamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec'.split())
def records(archive,dtd_bytes):
parser=expat.ParserCreate();parser.buffer_text=True
pending=deque();depth=0;row=None;field=None;text=[];doctype=False
def document_type(name,system_id,public_id,internal):
nonlocal doctype
if name!='dblp' or system_id!='dblp.dtd' or public_id or internal:
raise ValueError('Only the pinned external DBLP DTD is allowed')
doctype=True
def external(context,base,system_id,public_id):
if context is not None or system_id!='dblp.dtd' or public_id:
raise ValueError('Unexpected external XML entity')
child=parser.ExternalEntityParserCreate(context)
child.ExternalEntityRefHandler=lambda *args: (_ for _ in ()).throw(ValueError('Nested external entity'))
child.Parse(dtd_bytes,True);return 1
def start(name,attrs):
nonlocal depth,row,field,text
depth+=1
if depth==1 and name!='dblp':raise ValueError('Unexpected XML root')
if depth==2 and name in ('inproceedings','proceedings'):
row={'record_type':name,'key':attrs.get('key'),'publtype':attrs.get('publtype'),'fields':{}}
elif depth==3 and row is not None:
field=name;text=[]
def content(value):
if row is not None and field is not None:text.append(value)
def end(name):
nonlocal depth,row,field,text
if depth==3 and row is not None and field is not None:
row['fields'].setdefault(field,[]).append(' '.join(''.join(text).split()))
field=None;text=[]
if depth==2 and row is not None:
pending.append(row);row=None
depth-=1
parser.StartDoctypeDeclHandler=document_type;parser.ExternalEntityRefHandler=external
parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS)
parser.StartElementHandler=start;parser.EndElementHandler=end;parser.CharacterDataHandler=content
with gzip.open(archive,'rb') as stream:
while chunk:=stream.read(128*1024):
parser.Parse(chunk,False)
while pending:yield pending.popleft()
parser.Parse(b'',True)
while pending:yield pending.popleft()
if not doctype:raise ValueError('DBLP DTD declaration missing')
def family(key):
parts=(key or '').split('/')
return parts[1] if len(parts)>=3 and parts[0]=='conf' else ''
def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5):
archive=Path(archive);out=Path(out);dtd=Path(dtd)
identity=digest_file(archive);dtd_identity=digest_file(dtd)
if identity['sha256']!=expected_sha:raise ValueError('DBLP snapshot checksum mismatch')
if dtd_identity['md5']!=expected_dtd_md5:raise ValueError('DBLP DTD checksum mismatch')
if dtd.stat().st_size>128*1024:raise ValueError('Unexpectedly large DTD')
out.mkdir(parents=True,exist_ok=True)
target=out/'dblp-catalogue.sqlite';stage=out/'dblp-catalogue.sqlite.part'
candidate=out/'dblp-ai-candidates.jsonl';part=candidate.with_suffix('.part')
if stage.exists():raise ValueError('Unfinished catalogue exists; inspect it before another run')
c=sqlite3.connect(stage)
c.executescript('PRAGMA journal_mode=DELETE; PRAGMA synchronous=NORMAL; CREATE TABLE proceedings(key TEXT PRIMARY KEY,family TEXT,metadata_json TEXT); CREATE INDEX proceedings_family ON proceedings(family); CREATE TABLE volume_counts(crossref TEXT,year TEXT,booktitle TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(crossref,year,booktitle,record_type)); CREATE TABLE family_counts(family TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(family,record_type)); CREATE TABLE family_samples(family TEXT,seq INTEGER,metadata_json TEXT,PRIMARY KEY(family,seq));')
totals=Counter();families=Counter();volumes=Counter();samples=Counter();staged=Counter();seen=0
def flush():
c.executemany('INSERT INTO volume_counts VALUES(?,?,?,?,?) ON CONFLICT(crossref,year,booktitle,record_type) DO UPDATE SET count=count+excluded.count',((*key,count) for key,count in volumes.items()))
volumes.clear();c.commit()
try:
with part.open('w') as output:
for row in records(archive,dtd.read_bytes()):
seen+=1;kind=row['record_type'];key=row['key'];fields=row['fields'];prefix=family(key)
if not key:raise ValueError('Bibliographic record lacks a key')
totals[kind]+=1
if kind=='proceedings':
c.execute('INSERT INTO proceedings VALUES(?,?,?)',(key,prefix,json.dumps(row,ensure_ascii=False)))
if prefix:
families[(prefix,kind)]+=1
if kind=='inproceedings':
crossrefs=fields.get('crossref') or [''];years=fields.get('year') or [''];books=fields.get('booktitle') or ['']
if len(crossrefs)!=1 or len(years)!=1 or len(books)!=1:raise ValueError('Ambiguous publication volume metadata')
volumes[(crossrefs[0],years[0],books[0],kind)]+=1
if samples[prefix]<2:
c.execute('INSERT INTO family_samples VALUES(?,?,?)',(prefix,samples[prefix],json.dumps(row,ensure_ascii=False)));samples[prefix]+=1
if prefix in FAMILIES:
output.write(json.dumps(row,ensure_ascii=False)+'\n');staged[prefix]+=1
if seen%50000==0:
flush();print(json.dumps({'source':'dblp','records_scanned':seen,'candidate_records':sum(staged.values())}),flush=True)
flush()
c.executemany('INSERT INTO family_counts VALUES(?,?,?)',((*key,count) for key,count in families.items()));c.commit()
if c.execute('PRAGMA quick_check').fetchall()!=[('ok',)]:raise ValueError('Catalogue integrity failed')
c.close();c=None
manifest={'source':'dblp','status':'catalogued_candidates_pending_scope_review','source_release':RELEASE,'snapshot_sha256':identity['sha256'],
'dtd_md5':dtd_identity['md5'],'finished_at':timestamp(),'records_scanned':seen,'record_types':dict(totals),
'candidate_records':sum(staged.values()),'candidate_family_counts':dict(staged),'candidate_sha256':digest_file(part)['sha256'],
'file':str(candidate),'catalogue':str(target),'note':'Candidate families include workshops and historical names. No acceptance labels or normalized venue assignments have been inferred.'}
stage.replace(target);part.replace(candidate)
manifest_path=candidate.with_suffix('.manifest.json');manifest_part=manifest_path.with_suffix('.part')
manifest_part.write_text(json.dumps(manifest,indent=2));manifest_part.replace(manifest_path)
print(json.dumps(manifest),flush=True);return manifest
finally:
if c is not None:c.close()
if __name__=='__main__':
parser=argparse.ArgumentParser();parser.add_argument('--archive',required=True);parser.add_argument('--dtd',required=True);parser.add_argument('--out',required=True)
args=parser.parse_args();catalogue(args.archive,args.dtd,args.out)
+54
View File
@@ -0,0 +1,54 @@
import gzip
import hashlib
import json
from pathlib import Path
import sqlite3
import tempfile
import unittest
from catalogue_dblp import catalogue,records
class DblpCatalogueTests(unittest.TestCase):
def setUp(self):
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup);self.root=Path(self.tmp.name)
self.dtd=self.root/'dblp.dtd';self.dtd.write_bytes(b'<!ENTITY eacute "&#233;">')
def archive(self,body,doctype='<!DOCTYPE dblp SYSTEM "dblp.dtd">'):
path=self.root/'dblp.xml.gz'
with gzip.open(path,'wb') as stream:stream.write(('<?xml version="1.0"?>'+doctype+'<dblp>'+body+'</dblp>').encode())
return path
def test_streamed_entities_nested_titles_and_candidate_scope_remain_explicit(self):
path=self.archive('<proceedings key="conf/aaai/2025"><title>AAAI Workshop proceedings</title><year>2025</year></proceedings><inproceedings key="conf/aaai/paper"><author>Ren&eacute;</author><title>Learn <i>deep</i> models</title><year>2025</year><booktitle>AAAI Workshop</booktitle><crossref>conf/aaai/2025</crossref><ee>https://doi.org/10.1234/example</ee></inproceedings><inproceedings key="conf/unrelated/paper"><title>Other</title><year>2020</year></inproceedings>')
manifest=catalogue(path,self.dtd,self.root/'out',hashlib.sha256(path.read_bytes()).hexdigest(),hashlib.md5(self.dtd.read_bytes()).hexdigest())
self.assertEqual(manifest['records_scanned'],3);self.assertEqual(manifest['candidate_records'],1)
row=json.loads(Path(manifest['file']).read_text())
self.assertEqual(row['fields']['author'],['René']);self.assertEqual(row['fields']['title'],['Learn deep models'])
self.assertNotIn('status',row);self.assertNotIn('venue',row)
with sqlite3.connect(manifest['catalogue']) as connection:
self.assertEqual(connection.execute('SELECT sum(count) FROM volume_counts').fetchone()[0],2)
self.assertEqual(connection.execute('SELECT count(*) FROM proceedings').fetchone()[0],1)
def test_remote_external_dtd_is_rejected_without_fetching(self):
path=self.archive('',doctype='<!DOCTYPE dblp SYSTEM "https://example.org/evil.dtd">')
with self.assertRaisesRegex(ValueError,'pinned external'):list(records(path,self.dtd.read_bytes()))
def test_internal_entities_are_rejected(self):
path=self.archive('',doctype='<!DOCTYPE dblp SYSTEM "dblp.dtd" [<!ENTITY x "expanded">]>')
with self.assertRaisesRegex(ValueError,'pinned external'):list(records(path,self.dtd.read_bytes()))
def test_checksum_failure_preserves_existing_output(self):
path=self.archive('');out=self.root/'out';out.mkdir();target=out/'dblp-ai-candidates.jsonl';target.write_text('previous')
with self.assertRaisesRegex(ValueError,'snapshot checksum'):catalogue(path,self.dtd,out,'0'*64)
self.assertEqual(target.read_text(),'previous')
def test_truncated_gzip_cannot_publish_a_catalogue(self):
path=self.archive('<inproceedings key="conf/aaai/one"><title>One</title></inproceedings>')
path.write_bytes(path.read_bytes()[:-8]);out=self.root/'out'
with self.assertRaises((EOFError,OSError)):
catalogue(path,self.dtd,out,hashlib.sha256(path.read_bytes()).hexdigest(),hashlib.md5(self.dtd.read_bytes()).hexdigest())
self.assertFalse((out/'dblp-ai-candidates.jsonl').exists())
if __name__=='__main__':unittest.main()