排除 DBLP 论文目录中的会务与前后页材料 #6
@@ -130,6 +130,8 @@ paper-atlas collect dblp \
|
||||
|
||||
部分早期 SIGIR、CIKM、IJCAI 出版卷使用两位年份键;只有论文集标题明确给出唯一完整年份且与短键相符时才采用,不推测世纪。WWW 特殊兴趣、Alternate track 单列;同卷混合 Applied Data Science、Demo、Industrial 且不能逐篇区分时使用 `proceedings_unspecified`,不把整卷归为某一个具体 track。
|
||||
|
||||
书目中的前后页、目录组合和明确会务标题不产生论文录用记录;缺作者的赞助、委员会、活动汇总等标题单独排除。WWW 2019 companion 的匿名合成活动条目归入待核查材料,原始快照仍保留。范围报告分别统计这些原因;单纯缺作者并不排除真实研究论文,也不补造作者。
|
||||
|
||||
XML 只允许校验过的本地 DTD;官方通用名 `dblp.dtd` 与月度快照使用的 `dblp-2023-06-28.dtd` 均解析到同一份已校验文件。拒绝其他外部实体和内部 DTD 子集,不访问 XML 声明的任意网络地址,并遵循源 XML 的字符编码。临时目录库或 gzip 检查失败时不发布正式候选文件,需先调查残留的 `.part` 状态,再决定是否重新运行。
|
||||
|
||||
公开 ICLR 投稿快照的固定 Release 供后续筛选投稿和明确决定;它是第三方保存的 OpenReview 历史副本,不等于最新官方全量数据:
|
||||
|
||||
@@ -19,7 +19,20 @@ from paper_atlas.db.submissions import timestamp
|
||||
from paper_atlas.io.snapshots import digest_file
|
||||
|
||||
AS_OF='2026-11-01T00:00:00.000Z'
|
||||
FRONTMATTER=re.compile(r'(?:preface|foreword|editorial|welcome(?: message)?|organizing committee|program committee|title page|table of contents)[.!:]?',re.I)
|
||||
MATERIAL_PART=r'(?:title(?: page)?|preface|foreword|editorial|front\s?matter|back\s?matter|(?:table of )?contents|(?:author |subject )?index|contributors|(?:conference |program |organizing |steering )?committee|conference organization|reviewers)'
|
||||
FRONTMATTER=re.compile(MATERIAL_PART+r'(?:\s*[,;&]\s*'+MATERIAL_PART+r')*[.!:]?',re.I)
|
||||
HEADINGS=re.compile(r'(?:(?:IJCAI|AAAI)(?:[- ]\d{2,4})? |(?:19|20)\d{2} )?(?:conference )?(?:organization|organizers and sponsors|sponsorship|sponsors|(?:non-PC )?reviewers|awards and distinguished papers|keynote and invited speakers|area chairs and program committee|tutorials|panel session|invited talks|workshops|workshop summaries|special track on continuous planning)[.!:]?',re.I)
|
||||
|
||||
|
||||
def material_reason(title,authors,key,year,track):
|
||||
if FRONTMATTER.fullmatch(title):return 'frontmatter'
|
||||
if re.search(r"\bchairs['’]? welcome(?:\s*&\s*organization listing)?[.!:]?$",title,re.I):return 'frontmatter'
|
||||
if not authors:
|
||||
if HEADINGS.fullmatch(title) or re.fullmatch(r'(?:past IJCAI conferences|events colocated with IJCAI-\d{2}|welcome message from the conference chairs|welcome to (?:ICDM|IJCAI|AAAI) \d{4})[.!:]?',title,re.I):return 'proceedings_heading'
|
||||
# These anonymous synthetic entries name whole workshop events, rather
|
||||
# than identifiable papers. Hold them without discarding the raw source.
|
||||
if year==2019 and track=='companion' and re.fullmatch(r'conf/www/X19[a-z]?',key):return 'unverified_proceedings_material'
|
||||
return None
|
||||
|
||||
|
||||
def identifiers(fields):
|
||||
@@ -61,9 +74,10 @@ def normalize_row(row,volumes,targets,include_workshops=False,through_year=2026)
|
||||
titles=fields.get('title',[])
|
||||
if len(titles)!=1 or not titles[0].strip():return None,'missing_or_multiple_titles'
|
||||
title=' '.join(titles[0].split())
|
||||
if FRONTMATTER.fullmatch(title):return None,'frontmatter'
|
||||
own,ambiguous=identifiers(fields)
|
||||
authors=[re.sub(r'\s+\d{4}$','',name).strip() for name in fields.get('author',[]) if name.strip()]
|
||||
reason=material_reason(title,authors,row['key'],edition['year'],edition['track'])
|
||||
if reason:return None,reason
|
||||
own,ambiguous=identifiers(fields)
|
||||
metadata={'snapshot_sha256':XML_SHA,'snapshot_release':RELEASE,'observation_date_precision':'day',
|
||||
'proceedings_key':refs[0],'proceedings_title':edition['proceedings_title'],'bibliographic_year':edition['bibliographic_year'],
|
||||
'edition_year_basis':edition['edition_year_basis'],
|
||||
|
||||
@@ -9,7 +9,7 @@ import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from paper_atlas.collectors.dblp import XML_SHA
|
||||
from paper_atlas.collectors.dblp_records import collect,identifiers,normalize_row
|
||||
from paper_atlas.collectors.dblp_records import collect,identifiers,normalize_row,material_reason
|
||||
from paper_atlas.collectors.dblp_scope import scope
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.records import ingest_file
|
||||
@@ -77,6 +77,21 @@ class DblpRecordTests(unittest.TestCase):
|
||||
own,ambiguous=identifiers({'ee':['https://doi.org/10.1234/a','https://doi.org/10.1234/b','https://openreview.net/forum?id=2026-Oral--99-placeholder']})
|
||||
self.assertEqual(own,{});self.assertEqual(len(ambiguous['doi']),2)
|
||||
|
||||
def test_frontmatter_and_anonymous_event_headers_are_not_paper_acceptances(self):
|
||||
for title in ('Title, Preface.','Title, Conference Committee, Table of Contents, Author Index.','Front Matter.','Backmatter.','Foreword.'):
|
||||
self.assertEqual(material_reason(title,['An Editor'],'conf/sigir/C78',1978,'main'),'frontmatter')
|
||||
for title in ('Sponsors.','IJCAI-09 Awards and Distinguished Papers.','2013 Conference Organization.','Area Chairs and Program Committee.','Welcome to ICDM 2005.','Workshop Summaries.'):
|
||||
self.assertIsNotNone(material_reason(title,[],'conf/ijcai/X09',2009,'main'))
|
||||
self.assertEqual(material_reason("Wiki Workshop Chairs' Welcome.",['An Organizer'],'conf/www/X19t',2019,'companion'),'frontmatter')
|
||||
self.assertEqual(material_reason('Attention - from Neuroscience to the Web and Wellbeing.',[],'conf/www/X19b',2019,'companion'),'unverified_proceedings_material')
|
||||
row=self.row();row['fields']['title']=['Sponsors.'];row['fields']['author']=[]
|
||||
self.assertEqual(normalize_row(row,{'conf/aaai/2024':self.volume('aaai',2024)},{'AAAI'})[1],'proceedings_heading')
|
||||
|
||||
def test_missing_authors_alone_do_not_remove_real_research_records(self):
|
||||
self.assertIsNone(material_reason('The Onomastica interlanguage pronunciation lexicon.',[],'conf/interspeech/X95',1995,'main'))
|
||||
self.assertIsNone(material_reason('Learning Models of Relational Stochastic Processes.',[],'conf/ecml/SanghaiDW05',2005,'main'))
|
||||
self.assertIsNone(material_reason('Sponsors.', ['A Researcher'],'conf/aaai/research',2024,'main'))
|
||||
|
||||
def fixture(self,root,rows):
|
||||
catalogue=root/'dblp-catalogue.sqlite';volume=self.volume('aaai',2024)
|
||||
with closing(sqlite3.connect(catalogue)) as c, c:
|
||||
|
||||
Reference in New Issue
Block a user