Resolve the official dated DBLP DTD locally and preserve source encoding
Metadata integrity checks / metadata (push) Successful in 6s

This commit is contained in:
2026-11-06 16:47:08 +08:00
parent 4077cb4666
commit f9bdab49f2
3 changed files with 10 additions and 3 deletions
+1 -1
View File
@@ -177,7 +177,7 @@ python3 -B catalogue_dblp.py \
`dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本;`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。候选范围包含历史名称与 workshop,尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识,才能生成入库文件;不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF,也不更改论文主库。
XML 只允许校验过的本地 `dblp.dtd`,拒绝其他外部实体和内部 DTD 子集,不访问 XML 声明的任意网络地址。临时目录库或 gzip 检查失败时不发布正式候选文件,需先调查残留的 `.part` 状态,再决定是否重新运行。
XML 只允许校验过的本地 DTD;官方通用名 `dblp.dtd` 与月度快照使用的 `dblp-2023-06-28.dtd` 均解析到同一份已校验文件。拒绝其他外部实体和内部 DTD 子集,不访问 XML 声明的任意网络地址,并遵循源 XML 的字符编码。临时目录库或 gzip 检查失败时不发布正式候选文件,需先调查残留的 `.part` 状态,再决定是否重新运行。
公开 ICLR 投稿快照的固定 Release 供后续筛选投稿和明确决定;它是第三方保存的 OpenReview 历史副本,不等于最新官方全量数据:
+3 -2
View File
@@ -17,6 +17,7 @@ from submission_evidence import timestamp
XML_SHA='0376ad019b216553faff0a14e1757b2d4c7a3b84c1beacf0511ef83695211b18'
DTD_MD5='96e4283e71886dec2173718957293648'
DTD_NAMES={'dblp.dtd','dblp-2023-06-28.dtd'}
RELEASE='https://doi.org/10.4230/dblp.xml.2026-11-01'
# Broad discovery candidates, including historical names; not a venue mapping.
FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps kr atal aamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec'.split())
@@ -28,12 +29,12 @@ def records(archive,dtd_bytes):
def document_type(name,system_id,public_id,internal):
nonlocal doctype
if name!='dblp' or system_id!='dblp.dtd' or public_id or internal:
if name!='dblp' or system_id not in DTD_NAMES or public_id or internal:
raise ValueError('Only the pinned external DBLP DTD is allowed')
doctype=True
def external(context,base,system_id,public_id):
if context is not None or system_id!='dblp.dtd' or public_id:
if context is not None or system_id not in DTD_NAMES or public_id:
raise ValueError('Unexpected external XML entity')
child=parser.ExternalEntityParserCreate(context)
child.ExternalEntityRefHandler=lambda *args: (_ for _ in ()).throw(ValueError('Nested external entity'))
+6
View File
@@ -34,6 +34,12 @@ class DblpCatalogueTests(unittest.TestCase):
path=self.archive('',doctype='<!DOCTYPE dblp SYSTEM "https://example.org/evil.dtd">')
with self.assertRaisesRegex(ValueError,'pinned external'):list(records(path,self.dtd.read_bytes()))
def test_real_monthly_dtd_name_and_latin1_encoding(self):
path=self.root/'monthly.xml.gz'
xml='<?xml version="1.0" encoding="ISO-8859-1"?><!DOCTYPE dblp SYSTEM "dblp-2023-06-28.dtd"><dblp><inproceedings key="conf/aaai/one"><author>René</author><title>Metadata</title></inproceedings></dblp>'
with gzip.open(path,'wb') as stream:stream.write(xml.encode('latin1'))
self.assertEqual(list(records(path,self.dtd.read_bytes()))[0]['fields']['author'],['René'])
def test_internal_entities_are_rejected(self):
path=self.archive('',doctype='<!DOCTYPE dblp SYSTEM "dblp.dtd" [<!ENTITY x "expanded">]>')
with self.assertRaisesRegex(ValueError,'pinned external'):list(records(path,self.dtd.read_bytes()))