Preserve multi-volume DBLP metadata without inflating record counts
Metadata integrity checks / metadata (push) Successful in 15s

This commit is contained in:
2026-11-06 16:53:31 +08:00
parent f9bdab49f2
commit 827ba17a99
3 changed files with 17 additions and 5 deletions
+1 -1
View File
@@ -175,7 +175,7 @@ python3 -B catalogue_dblp.py \
--out /data/paper-index/collected/dblp
```
`dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本;`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。候选范围包含历史名称与 workshop,尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识,才能生成入库文件;不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF,也不更改论文主库。
`dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本;`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。存在多个出版卷字段的记录另存 `volume_anomalies`,保留全部值供归属核验;统计中仍只计一条,不任取第一个字段或展开成多篇。候选范围包含历史名称与 workshop,尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识,才能生成入库文件;不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF,也不更改论文主库。
XML 只允许校验过的本地 DTD;官方通用名 `dblp.dtd` 与月度快照使用的 `dblp-2023-06-28.dtd` 均解析到同一份已校验文件。拒绝其他外部实体和内部 DTD 子集,不访问 XML 声明的任意网络地址,并遵循源 XML 的字符编码。临时目录库或 gzip 检查失败时不发布正式候选文件,需先调查残留的 `.part` 状态,再决定是否重新运行。
+8 -4
View File
@@ -89,7 +89,7 @@ def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5):
candidate=out/'dblp-ai-candidates.jsonl';part=candidate.with_suffix('.part')
if stage.exists():raise ValueError('Unfinished catalogue exists; inspect it before another run')
c=sqlite3.connect(stage)
c.executescript('PRAGMA journal_mode=DELETE; PRAGMA synchronous=NORMAL; CREATE TABLE proceedings(key TEXT PRIMARY KEY,family TEXT,metadata_json TEXT); CREATE INDEX proceedings_family ON proceedings(family); CREATE TABLE volume_counts(crossref TEXT,year TEXT,booktitle TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(crossref,year,booktitle,record_type)); CREATE TABLE family_counts(family TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(family,record_type)); CREATE TABLE family_samples(family TEXT,seq INTEGER,metadata_json TEXT,PRIMARY KEY(family,seq));')
c.executescript('PRAGMA journal_mode=DELETE; PRAGMA synchronous=NORMAL; CREATE TABLE proceedings(key TEXT PRIMARY KEY,family TEXT,metadata_json TEXT); CREATE INDEX proceedings_family ON proceedings(family); CREATE TABLE volume_counts(crossref TEXT,year TEXT,booktitle TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(crossref,year,booktitle,record_type)); CREATE TABLE family_counts(family TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(family,record_type)); CREATE TABLE family_samples(family TEXT,seq INTEGER,metadata_json TEXT,PRIMARY KEY(family,seq)); CREATE TABLE volume_anomalies(key TEXT PRIMARY KEY,metadata_json TEXT);')
totals=Counter();families=Counter();volumes=Counter();samples=Counter();staged=Counter();seen=0
def flush():
@@ -108,8 +108,11 @@ def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5):
families[(prefix,kind)]+=1
if kind=='inproceedings':
crossrefs=fields.get('crossref') or [''];years=fields.get('year') or [''];books=fields.get('booktitle') or ['']
if len(crossrefs)!=1 or len(years)!=1 or len(books)!=1:raise ValueError('Ambiguous publication volume metadata')
volumes[(crossrefs[0],years[0],books[0],kind)]+=1
if any(len(values)!=1 for values in (crossrefs,years,books)):
c.execute('INSERT INTO volume_anomalies VALUES(?,?)',(key,json.dumps(row,ensure_ascii=False)))
totals['multiple_volume_fields']+=1
group=tuple(values[0] if len(values)==1 else '@multiple:'+json.dumps(values,ensure_ascii=False) for values in (crossrefs,years,books))
volumes[(*group,kind)]+=1
if samples[prefix]<2:
c.execute('INSERT INTO family_samples VALUES(?,?,?)',(prefix,samples[prefix],json.dumps(row,ensure_ascii=False)));samples[prefix]+=1
if prefix in FAMILIES:
@@ -121,7 +124,8 @@ def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5):
if c.execute('PRAGMA quick_check').fetchall()!=[('ok',)]:raise ValueError('Catalogue integrity failed')
c.close();c=None
manifest={'source':'dblp','status':'catalogued_candidates_pending_scope_review','source_release':RELEASE,'snapshot_sha256':identity['sha256'],
'dtd_md5':dtd_identity['md5'],'finished_at':timestamp(),'records_scanned':seen,'record_types':dict(totals),
'dtd_md5':dtd_identity['md5'],'finished_at':timestamp(),'records_scanned':seen,'record_types':{kind:totals[kind] for kind in ('proceedings','inproceedings')},
'multiple_volume_field_records':totals['multiple_volume_fields'],
'candidate_records':sum(staged.values()),'candidate_family_counts':dict(staged),'candidate_sha256':digest_file(part)['sha256'],
'file':str(candidate),'catalogue':str(target),'note':'Candidate families include workshops and historical names. No acceptance labels or normalized venue assignments have been inferred.'}
stage.replace(target);part.replace(candidate)
+8
View File
@@ -40,6 +40,14 @@ class DblpCatalogueTests(unittest.TestCase):
with gzip.open(path,'wb') as stream:stream.write(xml.encode('latin1'))
self.assertEqual(list(records(path,self.dtd.read_bytes()))[0]['fields']['author'],['René'])
def test_multiple_volume_fields_remain_reviewable_without_double_counting(self):
path=self.archive('<inproceedings key="conf/aaai/joint"><title>Joint publication</title><year>2024</year><booktitle>First meeting</booktitle><booktitle>Second meeting</booktitle><crossref>conf/aaai/joint2024</crossref></inproceedings>')
manifest=catalogue(path,self.dtd,self.root/'out',hashlib.sha256(path.read_bytes()).hexdigest(),hashlib.md5(self.dtd.read_bytes()).hexdigest())
with sqlite3.connect(manifest['catalogue']) as connection:
raw=json.loads(connection.execute('SELECT metadata_json FROM volume_anomalies').fetchone()[0])
self.assertEqual(raw['fields']['booktitle'],['First meeting','Second meeting'])
self.assertEqual(connection.execute('SELECT sum(count) FROM volume_counts').fetchone()[0],1)
def test_internal_entities_are_rejected(self):
path=self.archive('',doctype='<!DOCTYPE dblp SYSTEM "dblp.dtd" [<!ENTITY x "expanded">]>')
with self.assertRaisesRegex(ValueError,'pinned external'):list(records(path,self.dtd.read_bytes()))