Preserve multi-volume DBLP metadata without inflating record counts
Metadata integrity checks / metadata (push) Successful in 15s
Metadata integrity checks / metadata (push) Successful in 15s
This commit is contained in:
@@ -175,7 +175,7 @@ python3 -B catalogue_dblp.py \
|
||||
--out /data/paper-index/collected/dblp
|
||||
```
|
||||
|
||||
`dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本;`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。候选范围包含历史名称与 workshop,尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识,才能生成入库文件;不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF,也不更改论文主库。
|
||||
`dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本;`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。存在多个出版卷字段的记录另存 `volume_anomalies`,保留全部值供归属核验;统计中仍只计一条,不任取第一个字段或展开成多篇。候选范围包含历史名称与 workshop,尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识,才能生成入库文件;不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF,也不更改论文主库。
|
||||
|
||||
XML 只允许校验过的本地 DTD;官方通用名 `dblp.dtd` 与月度快照使用的 `dblp-2023-06-28.dtd` 均解析到同一份已校验文件。拒绝其他外部实体和内部 DTD 子集,不访问 XML 声明的任意网络地址,并遵循源 XML 的字符编码。临时目录库或 gzip 检查失败时不发布正式候选文件,需先调查残留的 `.part` 状态,再决定是否重新运行。
|
||||
|
||||
|
||||
+8
-4
@@ -89,7 +89,7 @@ def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5):
|
||||
candidate=out/'dblp-ai-candidates.jsonl';part=candidate.with_suffix('.part')
|
||||
if stage.exists():raise ValueError('Unfinished catalogue exists; inspect it before another run')
|
||||
c=sqlite3.connect(stage)
|
||||
c.executescript('PRAGMA journal_mode=DELETE; PRAGMA synchronous=NORMAL; CREATE TABLE proceedings(key TEXT PRIMARY KEY,family TEXT,metadata_json TEXT); CREATE INDEX proceedings_family ON proceedings(family); CREATE TABLE volume_counts(crossref TEXT,year TEXT,booktitle TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(crossref,year,booktitle,record_type)); CREATE TABLE family_counts(family TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(family,record_type)); CREATE TABLE family_samples(family TEXT,seq INTEGER,metadata_json TEXT,PRIMARY KEY(family,seq));')
|
||||
c.executescript('PRAGMA journal_mode=DELETE; PRAGMA synchronous=NORMAL; CREATE TABLE proceedings(key TEXT PRIMARY KEY,family TEXT,metadata_json TEXT); CREATE INDEX proceedings_family ON proceedings(family); CREATE TABLE volume_counts(crossref TEXT,year TEXT,booktitle TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(crossref,year,booktitle,record_type)); CREATE TABLE family_counts(family TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(family,record_type)); CREATE TABLE family_samples(family TEXT,seq INTEGER,metadata_json TEXT,PRIMARY KEY(family,seq)); CREATE TABLE volume_anomalies(key TEXT PRIMARY KEY,metadata_json TEXT);')
|
||||
totals=Counter();families=Counter();volumes=Counter();samples=Counter();staged=Counter();seen=0
|
||||
|
||||
def flush():
|
||||
@@ -108,8 +108,11 @@ def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5):
|
||||
families[(prefix,kind)]+=1
|
||||
if kind=='inproceedings':
|
||||
crossrefs=fields.get('crossref') or [''];years=fields.get('year') or [''];books=fields.get('booktitle') or ['']
|
||||
if len(crossrefs)!=1 or len(years)!=1 or len(books)!=1:raise ValueError('Ambiguous publication volume metadata')
|
||||
volumes[(crossrefs[0],years[0],books[0],kind)]+=1
|
||||
if any(len(values)!=1 for values in (crossrefs,years,books)):
|
||||
c.execute('INSERT INTO volume_anomalies VALUES(?,?)',(key,json.dumps(row,ensure_ascii=False)))
|
||||
totals['multiple_volume_fields']+=1
|
||||
group=tuple(values[0] if len(values)==1 else '@multiple:'+json.dumps(values,ensure_ascii=False) for values in (crossrefs,years,books))
|
||||
volumes[(*group,kind)]+=1
|
||||
if samples[prefix]<2:
|
||||
c.execute('INSERT INTO family_samples VALUES(?,?,?)',(prefix,samples[prefix],json.dumps(row,ensure_ascii=False)));samples[prefix]+=1
|
||||
if prefix in FAMILIES:
|
||||
@@ -121,7 +124,8 @@ def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5):
|
||||
if c.execute('PRAGMA quick_check').fetchall()!=[('ok',)]:raise ValueError('Catalogue integrity failed')
|
||||
c.close();c=None
|
||||
manifest={'source':'dblp','status':'catalogued_candidates_pending_scope_review','source_release':RELEASE,'snapshot_sha256':identity['sha256'],
|
||||
'dtd_md5':dtd_identity['md5'],'finished_at':timestamp(),'records_scanned':seen,'record_types':dict(totals),
|
||||
'dtd_md5':dtd_identity['md5'],'finished_at':timestamp(),'records_scanned':seen,'record_types':{kind:totals[kind] for kind in ('proceedings','inproceedings')},
|
||||
'multiple_volume_field_records':totals['multiple_volume_fields'],
|
||||
'candidate_records':sum(staged.values()),'candidate_family_counts':dict(staged),'candidate_sha256':digest_file(part)['sha256'],
|
||||
'file':str(candidate),'catalogue':str(target),'note':'Candidate families include workshops and historical names. No acceptance labels or normalized venue assignments have been inferred.'}
|
||||
stage.replace(target);part.replace(candidate)
|
||||
|
||||
@@ -40,6 +40,14 @@ class DblpCatalogueTests(unittest.TestCase):
|
||||
with gzip.open(path,'wb') as stream:stream.write(xml.encode('latin1'))
|
||||
self.assertEqual(list(records(path,self.dtd.read_bytes()))[0]['fields']['author'],['René'])
|
||||
|
||||
def test_multiple_volume_fields_remain_reviewable_without_double_counting(self):
|
||||
path=self.archive('<inproceedings key="conf/aaai/joint"><title>Joint publication</title><year>2024</year><booktitle>First meeting</booktitle><booktitle>Second meeting</booktitle><crossref>conf/aaai/joint2024</crossref></inproceedings>')
|
||||
manifest=catalogue(path,self.dtd,self.root/'out',hashlib.sha256(path.read_bytes()).hexdigest(),hashlib.md5(self.dtd.read_bytes()).hexdigest())
|
||||
with sqlite3.connect(manifest['catalogue']) as connection:
|
||||
raw=json.loads(connection.execute('SELECT metadata_json FROM volume_anomalies').fetchone()[0])
|
||||
self.assertEqual(raw['fields']['booktitle'],['First meeting','Second meeting'])
|
||||
self.assertEqual(connection.execute('SELECT sum(count) FROM volume_counts').fetchone()[0],1)
|
||||
|
||||
def test_internal_entities_are_rejected(self):
|
||||
path=self.archive('',doctype='<!DOCTYPE dblp SYSTEM "dblp.dtd" [<!ENTITY x "expanded">]>')
|
||||
with self.assertRaisesRegex(ValueError,'pinned external'):list(records(path,self.dtd.read_bytes()))
|
||||
|
||||
Reference in New Issue
Block a user