排除 DBLP 论文目录中的会务与前后页材料 #6

Merged
Zayden-Jung merged 1 commits from codex/dblp-frontmatter into main 2026-11-06 11:12:48 +00:00
3 changed files with 35 additions and 4 deletions
+2
View File
@@ -130,6 +130,8 @@ paper-atlas collect dblp \
部分早期 SIGIR、CIKM、IJCAI 出版卷使用两位年份键;只有论文集标题明确给出唯一完整年份且与短键相符时才采用,不推测世纪。WWW 特殊兴趣、Alternate track 单列;同卷混合 Applied Data Science、Demo、Industrial 且不能逐篇区分时使用 `proceedings_unspecified`,不把整卷归为某一个具体 track。
书目中的前后页、目录组合和明确会务标题不产生论文录用记录;缺作者的赞助、委员会、活动汇总等标题单独排除。WWW 2019 companion 的匿名合成活动条目归入待核查材料,原始快照仍保留。范围报告分别统计这些原因;单纯缺作者并不排除真实研究论文,也不补造作者。
XML 只允许校验过的本地 DTD;官方通用名 `dblp.dtd` 与月度快照使用的 `dblp-2023-06-28.dtd` 均解析到同一份已校验文件。拒绝其他外部实体和内部 DTD 子集,不访问 XML 声明的任意网络地址,并遵循源 XML 的字符编码。临时目录库或 gzip 检查失败时不发布正式候选文件,需先调查残留的 `.part` 状态,再决定是否重新运行。
公开 ICLR 投稿快照的固定 Release 供后续筛选投稿和明确决定;它是第三方保存的 OpenReview 历史副本,不等于最新官方全量数据:
+17 -3
View File
@@ -19,7 +19,20 @@ from paper_atlas.db.submissions import timestamp
from paper_atlas.io.snapshots import digest_file
AS_OF='2026-11-01T00:00:00.000Z'
FRONTMATTER=re.compile(r'(?:preface|foreword|editorial|welcome(?: message)?|organizing committee|program committee|title page|table of contents)[.!:]?',re.I)
MATERIAL_PART=r'(?:title(?: page)?|preface|foreword|editorial|front\s?matter|back\s?matter|(?:table of )?contents|(?:author |subject )?index|contributors|(?:conference |program |organizing |steering )?committee|conference organization|reviewers)'
FRONTMATTER=re.compile(MATERIAL_PART+r'(?:\s*[,;&]\s*'+MATERIAL_PART+r')*[.!:]?',re.I)
HEADINGS=re.compile(r'(?:(?:IJCAI|AAAI)(?:[- ]\d{2,4})? |(?:19|20)\d{2} )?(?:conference )?(?:organization|organizers and sponsors|sponsorship|sponsors|(?:non-PC )?reviewers|awards and distinguished papers|keynote and invited speakers|area chairs and program committee|tutorials|panel session|invited talks|workshops|workshop summaries|special track on continuous planning)[.!:]?',re.I)
def material_reason(title,authors,key,year,track):
if FRONTMATTER.fullmatch(title):return 'frontmatter'
if re.search(r"\bchairs['’]? welcome(?:\s*&\s*organization listing)?[.!:]?$",title,re.I):return 'frontmatter'
if not authors:
if HEADINGS.fullmatch(title) or re.fullmatch(r'(?:past IJCAI conferences|events colocated with IJCAI-\d{2}|welcome message from the conference chairs|welcome to (?:ICDM|IJCAI|AAAI) \d{4})[.!:]?',title,re.I):return 'proceedings_heading'
# These anonymous synthetic entries name whole workshop events, rather
# than identifiable papers. Hold them without discarding the raw source.
if year==2019 and track=='companion' and re.fullmatch(r'conf/www/X19[a-z]?',key):return 'unverified_proceedings_material'
return None
def identifiers(fields):
@@ -61,9 +74,10 @@ def normalize_row(row,volumes,targets,include_workshops=False,through_year=2026)
titles=fields.get('title',[])
if len(titles)!=1 or not titles[0].strip():return None,'missing_or_multiple_titles'
title=' '.join(titles[0].split())
if FRONTMATTER.fullmatch(title):return None,'frontmatter'
own,ambiguous=identifiers(fields)
authors=[re.sub(r'\s+\d{4}$','',name).strip() for name in fields.get('author',[]) if name.strip()]
reason=material_reason(title,authors,row['key'],edition['year'],edition['track'])
if reason:return None,reason
own,ambiguous=identifiers(fields)
metadata={'snapshot_sha256':XML_SHA,'snapshot_release':RELEASE,'observation_date_precision':'day',
'proceedings_key':refs[0],'proceedings_title':edition['proceedings_title'],'bibliographic_year':edition['bibliographic_year'],
'edition_year_basis':edition['edition_year_basis'],
+16 -1
View File
@@ -9,7 +9,7 @@ import unittest
from unittest.mock import patch
from paper_atlas.collectors.dblp import XML_SHA
from paper_atlas.collectors.dblp_records import collect,identifiers,normalize_row
from paper_atlas.collectors.dblp_records import collect,identifiers,normalize_row,material_reason
from paper_atlas.collectors.dblp_scope import scope
from paper_atlas.db.store import Store
from paper_atlas.ingestion.records import ingest_file
@@ -77,6 +77,21 @@ class DblpRecordTests(unittest.TestCase):
own,ambiguous=identifiers({'ee':['https://doi.org/10.1234/a','https://doi.org/10.1234/b','https://openreview.net/forum?id=2026-Oral--99-placeholder']})
self.assertEqual(own,{});self.assertEqual(len(ambiguous['doi']),2)
def test_frontmatter_and_anonymous_event_headers_are_not_paper_acceptances(self):
for title in ('Title, Preface.','Title, Conference Committee, Table of Contents, Author Index.','Front Matter.','Backmatter.','Foreword.'):
self.assertEqual(material_reason(title,['An Editor'],'conf/sigir/C78',1978,'main'),'frontmatter')
for title in ('Sponsors.','IJCAI-09 Awards and Distinguished Papers.','2013 Conference Organization.','Area Chairs and Program Committee.','Welcome to ICDM 2005.','Workshop Summaries.'):
self.assertIsNotNone(material_reason(title,[],'conf/ijcai/X09',2009,'main'))
self.assertEqual(material_reason("Wiki Workshop Chairs' Welcome.",['An Organizer'],'conf/www/X19t',2019,'companion'),'frontmatter')
self.assertEqual(material_reason('Attention - from Neuroscience to the Web and Wellbeing.',[],'conf/www/X19b',2019,'companion'),'unverified_proceedings_material')
row=self.row();row['fields']['title']=['Sponsors.'];row['fields']['author']=[]
self.assertEqual(normalize_row(row,{'conf/aaai/2024':self.volume('aaai',2024)},{'AAAI'})[1],'proceedings_heading')
def test_missing_authors_alone_do_not_remove_real_research_records(self):
self.assertIsNone(material_reason('The Onomastica interlanguage pronunciation lexicon.',[],'conf/interspeech/X95',1995,'main'))
self.assertIsNone(material_reason('Learning Models of Relational Stochastic Processes.',[],'conf/ecml/SanghaiDW05',2005,'main'))
self.assertIsNone(material_reason('Sponsors.', ['A Researcher'],'conf/aaai/research',2024,'main'))
def fixture(self,root,rows):
catalogue=root/'dblp-catalogue.sqlite';volume=self.volume('aaai',2024)
with closing(sqlite3.connect(catalogue)) as c, c: