From 827ba17a998a084df9176c2b948aabc5c5f46e94 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Fri, 6 Nov 2026 16:53:31 +0800 Subject: [PATCH] Preserve multi-volume DBLP metadata without inflating record counts --- README.md | 2 +- catalogue_dblp.py | 12 ++++++++---- test_dblp_catalogue.py | 8 ++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5ad83b4..498ed4f 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ python3 -B catalogue_dblp.py \ --out /data/paper-index/collected/dblp ``` -`dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本;`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。候选范围包含历史名称与 workshop,尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识,才能生成入库文件;不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF,也不更改论文主库。 +`dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本;`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。存在多个出版卷字段的记录另存 `volume_anomalies`,保留全部值供归属核验;统计中仍只计一条,不任取第一个字段或展开成多篇。候选范围包含历史名称与 workshop,尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识,才能生成入库文件;不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF,也不更改论文主库。 XML 只允许校验过的本地 DTD;官方通用名 `dblp.dtd` 与月度快照使用的 `dblp-2023-06-28.dtd` 均解析到同一份已校验文件。拒绝其他外部实体和内部 DTD 子集,不访问 XML 声明的任意网络地址,并遵循源 XML 的字符编码。临时目录库或 gzip 检查失败时不发布正式候选文件,需先调查残留的 `.part` 状态,再决定是否重新运行。 diff --git a/catalogue_dblp.py b/catalogue_dblp.py index e1c947c..6d7bb16 100644 --- a/catalogue_dblp.py +++ b/catalogue_dblp.py @@ -89,7 +89,7 @@ def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5): candidate=out/'dblp-ai-candidates.jsonl';part=candidate.with_suffix('.part') if stage.exists():raise ValueError('Unfinished catalogue exists; inspect it before another run') c=sqlite3.connect(stage) - c.executescript('PRAGMA journal_mode=DELETE; PRAGMA synchronous=NORMAL; CREATE TABLE proceedings(key TEXT PRIMARY KEY,family TEXT,metadata_json TEXT); CREATE INDEX proceedings_family ON proceedings(family); CREATE TABLE volume_counts(crossref TEXT,year TEXT,booktitle TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(crossref,year,booktitle,record_type)); CREATE TABLE family_counts(family TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(family,record_type)); CREATE TABLE family_samples(family TEXT,seq INTEGER,metadata_json TEXT,PRIMARY KEY(family,seq));') + c.executescript('PRAGMA journal_mode=DELETE; PRAGMA synchronous=NORMAL; CREATE TABLE proceedings(key TEXT PRIMARY KEY,family TEXT,metadata_json TEXT); CREATE INDEX proceedings_family ON proceedings(family); CREATE TABLE volume_counts(crossref TEXT,year TEXT,booktitle TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(crossref,year,booktitle,record_type)); CREATE TABLE family_counts(family TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(family,record_type)); CREATE TABLE family_samples(family TEXT,seq INTEGER,metadata_json TEXT,PRIMARY KEY(family,seq)); CREATE TABLE volume_anomalies(key TEXT PRIMARY KEY,metadata_json TEXT);') totals=Counter();families=Counter();volumes=Counter();samples=Counter();staged=Counter();seen=0 def flush(): @@ -108,8 +108,11 @@ def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5): families[(prefix,kind)]+=1 if kind=='inproceedings': crossrefs=fields.get('crossref') or [''];years=fields.get('year') or [''];books=fields.get('booktitle') or [''] - if len(crossrefs)!=1 or len(years)!=1 or len(books)!=1:raise ValueError('Ambiguous publication volume metadata') - volumes[(crossrefs[0],years[0],books[0],kind)]+=1 + if any(len(values)!=1 for values in (crossrefs,years,books)): + c.execute('INSERT INTO volume_anomalies VALUES(?,?)',(key,json.dumps(row,ensure_ascii=False))) + totals['multiple_volume_fields']+=1 + group=tuple(values[0] if len(values)==1 else '@multiple:'+json.dumps(values,ensure_ascii=False) for values in (crossrefs,years,books)) + volumes[(*group,kind)]+=1 if samples[prefix]<2: c.execute('INSERT INTO family_samples VALUES(?,?,?)',(prefix,samples[prefix],json.dumps(row,ensure_ascii=False)));samples[prefix]+=1 if prefix in FAMILIES: @@ -121,7 +124,8 @@ def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5): if c.execute('PRAGMA quick_check').fetchall()!=[('ok',)]:raise ValueError('Catalogue integrity failed') c.close();c=None manifest={'source':'dblp','status':'catalogued_candidates_pending_scope_review','source_release':RELEASE,'snapshot_sha256':identity['sha256'], - 'dtd_md5':dtd_identity['md5'],'finished_at':timestamp(),'records_scanned':seen,'record_types':dict(totals), + 'dtd_md5':dtd_identity['md5'],'finished_at':timestamp(),'records_scanned':seen,'record_types':{kind:totals[kind] for kind in ('proceedings','inproceedings')}, + 'multiple_volume_field_records':totals['multiple_volume_fields'], 'candidate_records':sum(staged.values()),'candidate_family_counts':dict(staged),'candidate_sha256':digest_file(part)['sha256'], 'file':str(candidate),'catalogue':str(target),'note':'Candidate families include workshops and historical names. No acceptance labels or normalized venue assignments have been inferred.'} stage.replace(target);part.replace(candidate) diff --git a/test_dblp_catalogue.py b/test_dblp_catalogue.py index bfb5ee0..67bc23a 100644 --- a/test_dblp_catalogue.py +++ b/test_dblp_catalogue.py @@ -40,6 +40,14 @@ class DblpCatalogueTests(unittest.TestCase): with gzip.open(path,'wb') as stream:stream.write(xml.encode('latin1')) self.assertEqual(list(records(path,self.dtd.read_bytes()))[0]['fields']['author'],['René']) + def test_multiple_volume_fields_remain_reviewable_without_double_counting(self): + path=self.archive('Joint publication2024First meetingSecond meetingconf/aaai/joint2024') + manifest=catalogue(path,self.dtd,self.root/'out',hashlib.sha256(path.read_bytes()).hexdigest(),hashlib.md5(self.dtd.read_bytes()).hexdigest()) + with sqlite3.connect(manifest['catalogue']) as connection: + raw=json.loads(connection.execute('SELECT metadata_json FROM volume_anomalies').fetchone()[0]) + self.assertEqual(raw['fields']['booktitle'],['First meeting','Second meeting']) + self.assertEqual(connection.execute('SELECT sum(count) FROM volume_counts').fetchone()[0],1) + def test_internal_entities_are_rejected(self): path=self.archive('',doctype=']>') with self.assertRaisesRegex(ValueError,'pinned external'):list(records(path,self.dtd.read_bytes()))