From b644de428b1965682f31738e23c8d0dfe601734b Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Fri, 6 Nov 2026 18:34:13 +0800 Subject: [PATCH 1/2] Normalize verified DBLP conference editions and publication metadata --- docs/data-sources.md | 15 +++ paper_atlas/cli.py | 1 + paper_atlas/collectors/dblp_records.py | 172 +++++++++++++++++++++++++ paper_atlas/collectors/dblp_scope.py | 85 ++++++++++++ paper_atlas/db/store.py | 5 +- paper_atlas/ingestion/backfill.py | 2 +- tests/test_dblp_records.py | 116 +++++++++++++++++ 7 files changed, 393 insertions(+), 3 deletions(-) create mode 100644 paper_atlas/collectors/dblp_records.py create mode 100644 paper_atlas/collectors/dblp_scope.py create mode 100644 tests/test_dblp_records.py diff --git a/docs/data-sources.md b/docs/data-sources.md index e949f2e..d2aa829 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -113,6 +113,21 @@ python3 -B -m paper_atlas catalogue dblp \ `dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本；`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。存在多个出版卷字段的记录另存 `volume_anomalies`，保留全部值供归属核验；统计中仍只计一条，不任取第一个字段或展开成多篇。候选范围包含历史名称与 workshop，尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识，才能生成入库文件；不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF，也不更改论文主库。 +目录盘点完成后，可按已核对的书目名称整理指定会议的分届文件。例如先补齐尚无主库记录的会议： + +```sh +paper-atlas collect dblp \ + --catalogue /data/paper-index/collected/dblp/dblp-catalogue.sqlite \ + --archive /data/paper-index/raw/dblp-2026-11-01.xml.gz \ + --dtd /data/paper-index/raw/dblp-2023-06-28.dtd \ + --venue AAAI --venue IJCAI --venue KDD \ + --out /data/paper-index/collected/dblp-normalized +``` + +也可用 `--candidates` 指定已通过清单 SHA-256 核验的候选文件；完整 XML 模式则可包含初次候选族之外的实际目录，例如 CLeaR 的 `clear2`。规则按书目名、crossref 论文集和会议年核对：不把另一个 `CLEAR` 或 Adaptive Agents 会议误标为 CLeaR/AAMAS；联合会议保留名称别名，出版年与会议年分别保存。workshop 默认单独留待审核，可显式使用 `--include-workshops`；其他已识别的官方附属 track 单列。 + +输出按会议/年份/track 分文件，附 manifest、`dblp-normalization.json` 范围审查统计和带哈希的 `dblp-import-queue.json`。不能识别的书目、年份或归属会逐类登记，不默认算作主会。DBLP 不含摘要，因此输出摘要为空、清单为 partial；需要通过有证据的跨来源关联补齐论文实体摘要。书目发表证据与原始审稿决定分别保留，不依据书目缺席推断拒稿。已存在其他出版来源的届次，还须先回填其权威身份并核验镜像记录，不能直接累加投稿数。 + XML 只允许校验过的本地 DTD；官方通用名 `dblp.dtd` 与月度快照使用的 `dblp-2023-06-28.dtd` 均解析到同一份已校验文件。拒绝其他外部实体和内部 DTD 子集，不访问 XML 声明的任意网络地址，并遵循源 XML 的字符编码。临时目录库或 gzip 检查失败时不发布正式候选文件，需先调查残留的 `.part` 状态，再决定是否重新运行。 公开 ICLR 投稿快照的固定 Release 供后续筛选投稿和明确决定；它是第三方保存的 OpenReview 历史副本，不等于最新官方全量数据： diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 8c4519b..4f1ce89 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -18,6 +18,7 @@ COMMANDS = { ('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'), + ('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'), ('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'), ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'), ('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'), diff --git a/paper_atlas/collectors/dblp_records.py b/paper_atlas/collectors/dblp_records.py new file mode 100644 index 0000000..f5cfe64 --- /dev/null +++ b/paper_atlas/collectors/dblp_records.py @@ -0,0 +1,172 @@ +"""Normalize pinned DBLP publication metadata into audited edition files.""" +import argparse +from collections import Counter,OrderedDict +from contextlib import closing +import fcntl +import hashlib +import json +from pathlib import Path +import re +import shutil +import sqlite3 +import tempfile +from urllib.parse import parse_qs,unquote,urlsplit + +from paper_atlas.config import DEFAULT_REGISTRY +from paper_atlas.collectors.dblp import XML_SHA,DTD_MD5,RELEASE,records +from paper_atlas.collectors.dblp_scope import scope +from paper_atlas.db.submissions import timestamp +from paper_atlas.io.snapshots import digest_file + +AS_OF='2026-11-01T00:00:00.000Z' +FRONTMATTER=re.compile(r'(?:preface|foreword|editorial|welcome(?: message)?|organizing committee|program committee|title page|table of contents)[.!:]?',re.I) + + +def identifiers(fields): + dois=set();arxiv=set();forums=set() + for link in fields.get('ee',[]): + parsed=urlsplit(link);path=unquote(parsed.path) + doi=None + if parsed.netloc in ('doi.org','dx.doi.org','doi.ieeecomputersociety.org'):doi=path.lstrip('/') + elif parsed.netloc=='link.springer.com' and path.startswith('/chapter/'):doi=path.removeprefix('/chapter/') + elif parsed.netloc=='dl.acm.org': + match=re.match(r'/doi/(?:abs/|full/|pdf/)?(10\..+)',path) + if match:doi=match[1] + if doi and re.fullmatch(r'10\.\d{4,9}/\S+',doi):dois.add(doi.lower()) + if parsed.netloc in ('arxiv.org','www.arxiv.org'): + match=re.fullmatch(r'/(?:abs|pdf)/((?:\d{4}\.\d{4,5}|[a-zA-Z.-]+/\d{7})(?:v\d+)?)(?:\.pdf)?',path) + if match:arxiv.add(re.sub(r'v\d+$','',match[1])) + if parsed.netloc=='openreview.net' and parsed.path in ('/forum','/pdf'): + values=parse_qs(parsed.query).get('id',[]) + if len(values)==1 and re.fullmatch(r'[A-Za-z0-9_-]+',values[0]) and not re.match(r'20\d{2}-Oral-',values[0]):forums.add(values[0]) + for doi in dois: + match=re.fullmatch(r'10\.48550/arxiv\.((?:\d{4}\.\d{4,5}|[a-z.-]+/\d{7})(?:v\d+)?)',doi) + if match:arxiv.add(re.sub(r'v\d+$','',match[1])) + publications={doi for doi in dois if not doi.startswith('10.48550/arxiv.')} + own={} + for scheme,values in (('doi',publications),('arxiv',arxiv),('openreview',forums)): + if len(values)==1:own[scheme]=next(iter(values)) + ambiguous={scheme:sorted(values) for scheme,values in (('doi',publications),('arxiv',arxiv),('openreview',forums)) if len(values)>1} + return own,ambiguous + + +def normalize_row(row,volumes,targets,include_workshops=False,through_year=2026): + if row['record_type']!='inproceedings':return None,'not_a_paper' + fields=row['fields'];refs=fields.get('crossref',[]) + if len(refs)!=1:return None,'ambiguous_or_missing_crossref' + edition,reason=scope(row,volumes.get(refs[0]),through_year) + if not edition:return None,reason + if not ({edition['venue']}|set(edition['venue_aliases']))&targets:return None,'outside_requested_venues' + if edition['track']=='workshop' and not include_workshops:return None,'separate_workshop' + titles=fields.get('title',[]) + if len(titles)!=1 or not titles[0].strip():return None,'missing_or_multiple_titles' + title=' '.join(titles[0].split()) + if FRONTMATTER.fullmatch(title):return None,'frontmatter' + own,ambiguous=identifiers(fields) + authors=[re.sub(r'\s+\d{4}$','',name).strip() for name in fields.get('author',[]) if name.strip()] + metadata={'snapshot_sha256':XML_SHA,'snapshot_release':RELEASE,'observation_date_precision':'day', + 'proceedings_key':refs[0],'proceedings_title':edition['proceedings_title'],'bibliographic_year':edition['bibliographic_year'], + 'edition_year_basis':'proceedings key, checked against bibliographic year or explicit event year in proceedings title', + 'scope_rule':edition['rule'],'author_names_raw':fields.get('author',[]),'publtype':row.get('publtype'), + 'bibliography':{key:value for key,value in fields.items() if key in ('booktitle','year','ee','url','pages','volume','number','publisher','series','isbn')}, + 'ambiguous_identifiers':ambiguous,'publication_evidence':'inproceedings bibliography; not an original decision note'} + flagged=bool(re.search(r'\b(?:withdrawn|retracted)\b',row.get('publtype') or '',re.I)) + result={'source':'dblp','external_id':row['key'],'url':'https://dblp.org/rec/'+row['key']+'.html', + 'title':title,'abstract':None,'authors':authors,'identifiers':own, + **{key:edition[key] for key in ('venue','venue_aliases','year','track')}, + 'status':'unknown' if flagged else 'accepted','observed_at':AS_OF, + 'raw_decision':'listed in pinned DBLP inproceedings bibliography','metadata':metadata, + 'tags':[{'namespace':'dblp:booktitle','value':fields['booktitle'][0]}]} + if flagged:result['decision_events']=[] + return result,None + + +def collect(catalogue,out,venues=None,candidates=None,archive=None,dtd=None,include_workshops=False,through_year=2026): + if bool(candidates)==bool(archive):raise ValueError('Choose exactly one candidates file or original archive') + catalogue=Path(catalogue);manifest=json.loads(catalogue.with_name('dblp-ai-candidates.manifest.json').read_text()) + if manifest['snapshot_sha256']!=XML_SHA:raise ValueError('Unsupported snapshot identity') + if candidates: + if digest_file(Path(candidates))['sha256']!=manifest['candidate_sha256']:raise ValueError('Candidate checksum mismatch') + else: + if not dtd or digest_file(Path(dtd))['md5']!=DTD_MD5:raise ValueError('DTD checksum mismatch') + if digest_file(Path(archive))['sha256']!=XML_SHA:raise ValueError('Archive checksum mismatch') + with closing(sqlite3.connect(catalogue.resolve().as_uri()+'?mode=ro',uri=True)) as c: + volumes={key:json.loads(payload) for key,payload in c.execute('SELECT key,metadata_json FROM proceedings')} + if len(volumes)!=manifest['record_types']['proceedings']:raise ValueError('Catalogue inventory mismatch') + allowed={row['acronym'].casefold():row['acronym'] for row in json.loads(DEFAULT_REGISTRY.read_text())['venues']} + if venues and set(name.casefold() for name in venues)-allowed.keys():raise ValueError('Unknown registry venue') + targets={allowed[name.casefold()] for name in venues} if venues else set(allowed.values()) + out=Path(out);out.mkdir(parents=True,exist_ok=True) + with (out/'dblp.normalize.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + return _collect_rows(volumes,out,targets,candidates,archive,dtd,include_workshops,through_year,manifest) + + +def _collect_rows(volumes,out,targets,candidates,archive,dtd,include_workshops,through_year,source_manifest): + counts=Counter();excluded=Counter();examples={};groups={};seen=set();handles=OrderedDict();scanned=0 + def rows(): + if candidates: + with Path(candidates).open() as stream: + for line in stream:yield json.loads(line) + else:yield from records(archive,Path(dtd).read_bytes()) + if shutil.disk_usage(out).free<8*1024**3:raise RuntimeError('Less than 8 GiB free before normalization') + with tempfile.TemporaryDirectory(prefix='.dblp-normalize-',dir=out) as temporary: + stage=Path(temporary) + try: + for row in rows(): + scanned+=1 + record,reason=normalize_row(row,volumes,targets,include_workshops,through_year) + if record is None: + excluded[reason]+=1 + label=reason+'|'+'/'.join(row.get('key','').split('/')[:2])+'|'+';'.join(row['fields'].get('booktitle',[])) + if label not in examples:examples[label]={'count':0,'example_key':row.get('key')} + examples[label]['count']+=1 + else: + key=record['external_id'] + if key in seen:raise ValueError('Repeated DBLP publication key') + seen.add(key);group=(record['venue'],record['year'],record['track']) + filename='dblp-'+re.sub(r'[^a-z0-9]+','-',group[0].casefold())+'-'+str(group[1])+'-'+group[2]+'.jsonl' + if group not in groups:groups[group]={'filename':filename,'count':0,'unknown':0,'ambiguous_identifiers':0} + info=groups[group];info['count']+=1;info['unknown']+=record['status']=='unknown';info['ambiguous_identifiers']+=bool(record['metadata']['ambiguous_identifiers']) + if group not in handles: + if len(handles)>=32:handles.popitem(last=False)[1].close() + handles[group]=(stage/filename).open('a') + handles.move_to_end(group);handles[group].write(json.dumps(record,ensure_ascii=False)+'\n');counts[record['venue']]+=1 + if scanned%50000==0: + if shutil.disk_usage(out).free<8*1024**3:raise RuntimeError('Less than 8 GiB free during normalization') + print(json.dumps({'source':'dblp','scanned':scanned,'normalized':sum(counts.values())}),flush=True) + finally: + for handle in handles.values():handle.close() + expected=source_manifest['candidate_records'] if candidates else sum(source_manifest['record_types'].values()) + if scanned!=expected:raise ValueError('Input count differs from verified catalogue') + if not seen:raise ValueError('No target publications were verified') + files=[] + for (venue,year,track),info in sorted(groups.items()): + path=out/info['filename'];checksum=digest_file(stage/info['filename'])['sha256'] + item={'source':'dblp','parser_version':1,'venue':venue,'year':year,'track':track,'count':info['count'], + 'missing_abstract_count':info['count'],'unknown_status_count':info['unknown'],'ambiguous_identifier_count':info['ambiguous_identifiers'], + 'list_url':RELEASE,'status':'partial','file':str(path),'file_sha256':checksum,'snapshot_sha256':XML_SHA, + 'abstract_note':'DBLP provides no abstract; canonical abstracts must come from independently linked sources.'} + (stage/info['filename']).replace(path) + meta=path.with_suffix('.manifest.json');part=meta.with_suffix('.part');part.write_text(json.dumps(item,indent=2));part.replace(meta);files.append(item) + result={'source':'dblp','finished_at':timestamp(),'snapshot_sha256':XML_SHA,'input_mode':'candidates' if candidates else 'original_archive', + 'requested_venues':sorted(targets),'include_workshops':include_workshops,'through_year':through_year,'scanned_records':scanned, + 'normalized_records':len(seen),'records_by_venue':dict(counts),'excluded_by_reason':dict(excluded),'scope_review':examples,'files':files, + 'input_candidate_sha256':source_manifest['candidate_sha256'] if candidates else None, + 'note':'Classification covers requested recognizable bibliography entries, not an independent proof of complete venue histories. Unknown scopes remain in scope_review.'} + target=out/'dblp-normalization.json';part=target.with_suffix('.part');part.write_text(json.dumps(result,ensure_ascii=False,indent=2));part.replace(target) + queue={'files':[item['file'] for item in files],'file_sha256':{item['file']:item['file_sha256'] for item in files}} + target=out/'dblp-import-queue.json';part=target.with_suffix('.part');part.write_text(json.dumps(queue,indent=2));part.replace(target) + print(json.dumps({key:result[key] for key in ('scanned_records','normalized_records','records_by_venue','excluded_by_reason')}),flush=True) + return result + + +def main(argv=None,*,prog=None): + parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--catalogue',required=True);parser.add_argument('--out',required=True) + inputs=parser.add_mutually_exclusive_group(required=True);inputs.add_argument('--candidates');inputs.add_argument('--archive') + parser.add_argument('--dtd');parser.add_argument('--venue',action='append');parser.add_argument('--include-workshops',action='store_true') + parser.add_argument('--through-year',type=int,default=2026);args=parser.parse_args(argv) + collect(args.catalogue,args.out,args.venue,args.candidates,args.archive,args.dtd,args.include_workshops,args.through_year) + + +if __name__=='__main__':main() diff --git a/paper_atlas/collectors/dblp_scope.py b/paper_atlas/collectors/dblp_scope.py new file mode 100644 index 0000000..4adb654 --- /dev/null +++ b/paper_atlas/collectors/dblp_scope.py @@ -0,0 +1,85 @@ +"""Explicit bibliographic aliases; a directory family alone is not venue evidence.""" +import re + + +# Ordered long/joint names precede their shorter forms. +BOOKS = { + 'aaai': [('AAAI/IAAI','AAAI+IAAI',['AAAI','IAAI']),('AAAI','AAAI',[])], + 'ijcai': [('IJCAI/ECAI','ECAI+IJCAI',['ECAI','IJCAI']),('IJCAI','IJCAI',[])], + 'ecai': [('ECAI','ECAI',[]),('AISB/GI (ECAI)','ECAI',[]),('AISB (ECAI)','ECAI',[])], + 'nips': [('NeurIPS Datasets and Benchmarks','NeurIPS',[]),('NeurIPS','NeurIPS',[]),('NIPS','NeurIPS',[])], + 'icml': [('ICML','ICML',[]),('ML','ICML',[])], 'iclr':[('ICLR','ICLR',[])], + 'colm':[('COLM','COLM',[])], 'colt':[('COLT/EuroCOLT','COLT',[]),('COLT','COLT',[])], + 'aistats':[('AISTATS','AISTATS',[])], 'uai':[('UAI','UAI',[])], + 'clear2':[('CLeaR','CLeaR',[])], + 'acl':[('COLING-ACL','ACL+COLING',['ACL','COLING']),('ACL/IJCNLP','ACL',[]),('ACL','ACL',[])], + 'emnlp':[('EMNLP-CoNLL','CoNLL+EMNLP',['CoNLL','EMNLP']),('EMNLP/IJCNLP','EMNLP',[]),('EMNLP','EMNLP',[])], + 'naacl':[('HLT/EMNLP','EMNLP',[]),('NAACL-HLT','NAACL',[]),('HLT-NAACL','NAACL',[]),('NAACL','NAACL',[]),('HLT','NAACL',[])], + 'eacl':[('EACL','EACL',[])], + 'coling':[('LREC/COLING','COLING+LREC',['COLING','LREC']),('COLING-ACL','ACL+COLING',['ACL','COLING']),('COLING','COLING',[])], + 'lrec':[('LREC/COLING','COLING+LREC',['COLING','LREC'])], + 'conll':[('CoNLL/LLL','CoNLL',[]),('CoNLL','CoNLL',[])], + 'cvpr':[('CVPR','CVPR',[])], 'iccv':[('ICCVW','ICCV',[]),('ICCV','ICCV',[])], + 'eccv':[('ECCV','ECCV',[])], 'kdd':[('KDD','KDD',[])], 'sigir':[('SIGIR','SIGIR',[])], + 'www':[('WWW','WWW',[])], 'wsdm':[('WSDM','WSDM',[])], 'recsys':[('RecSys','RecSys',[])], + 'cikm':[('CIKM','CIKM',[])], 'icdm':[('ICDM','ICDM',[])], 'sdm':[('SDM','SDM',[])], + 'ecml':[('ECML','ECML',['ECML-PKDD']),('EWSL','EWSL',['ECML-PKDD'])], + 'pkdd':[('ECML/PKDD','ECML-PKDD',[]),('PKDD','PKDD',['ECML-PKDD'])], + 'icra':[('ICRA','ICRA',[])], 'iros':[('IROS','IROS',[])], + 'rss':[('Robotics: Science and Systems','RSS',[])], 'corl':[('CoRL','CoRL',[])], + 'aips':[('ICAPS','ICAPS',[]),('AIPS','AIPS',['ICAPS'])], 'icaps':[('ICAPS','ICAPS',[])], + 'kr':[('KR','KR',[])], 'atal':[('AAMAS','AAMAS',[])], + 'icassp':[('ICASSP','ICASSP',[])], + 'interspeech':[('INTERSPEECH','INTERSPEECH',[]),('EUROSPEECH','EUROSPEECH',['INTERSPEECH']),('ICSLP','ICSLP',['INTERSPEECH']),('ECST','ECST',['INTERSPEECH'])], + 'ismir':[('ISMIR','ISMIR',[])], 'miccai':[('MICCAI','MICCAI',[])], + 'fat':[('FAccT','FAccT',[]),('FAT*','FAccT',[]),('FAT','FAccT',[])], +} + + +def scope(row,volume,through_year=2026): + fields=row['fields'];books=fields.get('booktitle',[]) + if len(books)!=1:return None,'ambiguous_booktitle' + book=books[0];key=row.get('key','');parts=key.split('/') + family=parts[1] if len(parts)>=3 and parts[0]=='conf' else '' + if family not in BOOKS:return None,'outside_target_family' + if not volume:return None,'missing_proceedings' + title=' '.join(volume['fields'].get('title',[])) + if family=='clear2' and 'causal learning and reasoning' not in title.casefold():return None,'ambiguous_venue_name' + matched=None + for base,venue,aliases in BOOKS[family]: + match=re.fullmatch(re.escape(base)+r'(?:\s+\((.*)\)|\s+(.*)|,\s*(Vol\.\s*\d+))?',book,re.I) + if match: + matched=(base,venue,aliases,' '.join(value for value in match.groups() if value));break + if not matched:return None,'unclassified_booktitle' + base,venue,aliases,suffix=matched + volume_family=volume['key'].split('/')[1] if volume['key'].startswith('conf/') else '' + if volume_family!=family and not any(item[1]==venue for item in BOOKS.get(volume_family,[])): + return None,'crossref_venue_conflict' + year_match=re.match(r'conf/[^/]+/((?:19|20)\d{2})',volume['key']) + if not year_match:return None,'unverified_edition_year' + year=int(year_match[1]) + if year>through_year:return None,'future_edition' + bibliographic=fields.get('year',[]) + if len(bibliographic)!=1 or not re.fullmatch(r'\d{4}',bibliographic[0]):return None,'ambiguous_bibliographic_year' + if int(bibliographic[0])!=year and not re.search(r'\b'+str(year)+r'\b',title):return None,'unverified_year_difference' + text=(suffix+' '+title).casefold();track='main' + if 'findings' in text:track='findings' + elif 'shared task' in text:track='shared_task' + elif 'student research' in text or 'workshop' in text or 'wokshop' in text or base in ('ICCVW','EWSL'):track='workshop' + elif 'demonstration' in text or re.search(r'\bdemos?\b',text):track='demo' + elif 'tutorial' in text:track='tutorial' + elif 'industry' in text or 'industrial track' in text or 'applied data science' in text:track='industry' + elif 'position paper' in text:track='position' + elif 'companion' in text:track='companion' + elif 'late-breaking' in text or 'late breaking' in text:track='late_breaking' + elif 'doctoral' in text:track='doctoral' + elif 'challenge' in text or 'competition' in text:track='competition' + elif 'bridge program' in text:track='bridge' + elif base=='NeurIPS Datasets and Benchmarks':track='datasets_benchmarks' + elif suffix and not (re.fullmatch(r'(?:\d+|Vol\.\s*\d+|Volume \d+: (?:Long|Short) Papers|Long Papers|Short Papers|Selected Papers|Fall)',suffix,re.I) + or suffix.casefold() in ('poster','posters','oral','spotlight')):return None,'unclassified_track' + elif suffix.casefold() in ('poster','posters') and venue not in ('ICLR','ACL','EMNLP','NAACL','EACL','COLING','CoNLL'):track='poster' + if venue=='AAAI+IAAI':track='proceedings_unspecified' + if venue=='AAMAS' and year<2002:return None,'predecessor_not_aamas' + return {'venue':venue,'venue_aliases':aliases,'year':year,'track':track,'bibliographic_year':int(bibliographic[0]), + 'rule':family+':'+base,'proceedings_title':title},None diff --git a/paper_atlas/db/store.py b/paper_atlas/db/store.py index 11a8ca0..dd15e23 100644 --- a/paper_atlas/db/store.py +++ b/paper_atlas/db/store.py @@ -137,8 +137,9 @@ class Store: def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None,identities=None,observed_at=None,decision_date=None,source_decision_id='',submitted_at=None): c = self.conn - c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified')) - venue_id = c.execute('SELECT id FROM venues WHERE acronym=?',(venue,)).fetchone()[0] + existing=c.execute('SELECT id FROM venues WHERE acronym=? COLLATE NOCASE',(venue,)).fetchone() + if existing:venue_id=existing[0] + else:venue_id=c.execute("INSERT INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified')).lastrowid for alias in aliases or []: c.execute('INSERT OR IGNORE INTO venue_aliases VALUES(?,?)',(venue_id,alias)) c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)',(venue_id,int(year),track)) diff --git a/paper_atlas/ingestion/backfill.py b/paper_atlas/ingestion/backfill.py index bee0ebf..b943b4e 100644 --- a/paper_atlas/ingestion/backfill.py +++ b/paper_atlas/ingestion/backfill.py @@ -36,7 +36,7 @@ def backfill(db,files): if any(row.get(key)!=manifest.get(key) for key in ('source','venue','year','track')): raise ValueError('Evidence row differs from edition manifest') record=c.execute('SELECT r.id,r.paper_id,r.metadata_json FROM source_records r JOIN sources s ON s.id=r.source_id WHERE s.name=? AND r.external_id=?',(row['source'],row['external_id'])).fetchone() - edition=c.execute('SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? AND e.year=? AND e.track=?',(row['venue'],row['year'],row['track'])).fetchone() + edition=c.execute('SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? COLLATE NOCASE AND e.year=? AND e.track=?',(row['venue'],row['year'],row['track'])).fetchone() if not record or not edition:raise ValueError('Evidence backfill requires an already imported record and edition') if not c.execute('SELECT 1 FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id WHERE ss.source_record_id=? AND ss.source_submission_id=? AND s.edition_id=?',(record[0],row['external_id'],edition[0])).fetchone(): raise ValueError('No existing submission to backfill') diff --git a/tests/test_dblp_records.py b/tests/test_dblp_records.py new file mode 100644 index 0000000..5893c96 --- /dev/null +++ b/tests/test_dblp_records.py @@ -0,0 +1,116 @@ +import hashlib +import json +from contextlib import closing +from pathlib import Path +import sqlite3 +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from paper_atlas.collectors.dblp import XML_SHA +from paper_atlas.collectors.dblp_records import collect,identifiers,normalize_row +from paper_atlas.collectors.dblp_scope import scope +from paper_atlas.db.store import Store +from paper_atlas.ingestion.records import ingest_file + + +class DblpRecordTests(unittest.TestCase): + def setUp(self): + # Tiny fixtures may live on a small tmpfs; production reserve stays 8 GiB. + budget=patch('paper_atlas.collectors.dblp_records.shutil.disk_usage',return_value=SimpleNamespace(free=16*1024**3)) + budget.start();self.addCleanup(budget.stop) + + def row(self,family='aaai',book='AAAI',year=2024,key='one'): + return {'record_type':'inproceedings','key':f'conf/{family}/{key}','publtype':None, + 'fields':{'title':['A research publication'],'author':['Jane Doe 0001','John Smith'], + 'booktitle':[book],'year':[str(year)],'crossref':[f'conf/{family}/{year}'],'ee':['https://doi.org/10.1234/'+key]}} + + def volume(self,family,year,title=None): + return {'record_type':'proceedings','key':f'conf/{family}/{year}','fields':{'title':[title or f'{family.upper()} {year}'],'year':[str(year)]}} + + def test_real_directory_aliases_do_not_confuse_other_conferences(self): + row=self.row('clear2','CLeaR');volume=self.volume('clear2',2024,'Causal Learning and Reasoning, 2024') + self.assertEqual(scope(row,volume)[0]['venue'],'CLeaR') + self.assertIsNone(scope(self.row('clear','CLEAR'),self.volume('clear',2024))[0]) + self.assertIsNone(scope(self.row('aamas','Adaptive Agents and Multi-Agent Systems'),self.volume('aamas',2024))[0]) + self.assertEqual(scope(self.row('atal','AAMAS'),self.volume('atal',2024))[0]['venue'],'AAMAS') + self.assertEqual(scope(self.row('aips','ICAPS'),self.volume('aips',2024))[0]['venue'],'ICAPS') + + def test_conference_year_is_distinct_from_publication_year(self): + row=self.row('nips','NIPS',1988);row['fields']['crossref']=['conf/nips/1987'] + volume=self.volume('nips',1987,'NIPS Conference, Denver, 1987; published 1988') + result,_=scope(row,volume);self.assertEqual(result['year'],1987);self.assertEqual(result['bibliographic_year'],1988) + volume['fields']['title']=['NIPS collected papers'] + self.assertEqual(scope(row,volume)[1],'unverified_year_difference') + + def test_joint_events_and_tracks_are_explicit(self): + result,_=scope(self.row('ijcai','IJCAI/ECAI',2022),self.volume('ijcai',2022)) + self.assertEqual(result['venue'],'ECAI+IJCAI');self.assertEqual(result['venue_aliases'],['ECAI','IJCAI']) + result,_=scope(self.row('coling','LREC/COLING'),self.volume('coling',2024)) + self.assertEqual(result['venue'],'COLING+LREC') + for book,track in [('ACL (Findings)','findings'),('ACL (Student Research Workshop)','workshop'),('ACL (System Demonstrations)','demo')]: + self.assertEqual(scope(self.row('acl',book),self.volume('acl',2024))[0]['track'],track) + + def test_crossref_mismatch_and_future_editions_are_held_for_review(self): + self.assertEqual(scope(self.row(),self.volume('kdd',2024))[1],'crossref_venue_conflict') + self.assertEqual(scope(self.row(year=2027),self.volume('aaai',2027))[1],'future_edition') + + def test_only_own_identifier_links_are_used_and_ambiguity_is_preserved(self): + own,ambiguous=identifiers({'ee':['https://doi.org/10.1234/published','https://doi.org/10.48550/arXiv.2401.00001','https://openreview.net/forum?id=realForum'], + 'cite':['https://doi.org/10.9999/reference']}) + self.assertEqual(own,{'doi':'10.1234/published','arxiv':'2401.00001','openreview':'realForum'});self.assertFalse(ambiguous) + own,ambiguous=identifiers({'ee':['https://doi.org/10.1234/a','https://doi.org/10.1234/b','https://openreview.net/forum?id=2026-Oral--99-placeholder']}) + self.assertEqual(own,{});self.assertEqual(len(ambiguous['doi']),2) + + def fixture(self,root,rows): + catalogue=root/'dblp-catalogue.sqlite';volume=self.volume('aaai',2024) + with closing(sqlite3.connect(catalogue)) as c, c: + c.execute('CREATE TABLE proceedings(key TEXT PRIMARY KEY,metadata_json TEXT)') + c.execute('INSERT INTO proceedings VALUES(?,?)',(volume['key'],json.dumps(volume))) + candidates=root/'candidates.jsonl';candidates.write_text(''.join(json.dumps(row)+'\n' for row in rows)) + manifest={'snapshot_sha256':XML_SHA,'candidate_sha256':hashlib.sha256(candidates.read_bytes()).hexdigest(), + 'candidate_records':len(rows),'record_types':{'proceedings':1,'inproceedings':len(rows)}} + catalogue.with_name('dblp-ai-candidates.manifest.json').write_text(json.dumps(manifest)) + return catalogue,candidates + + def test_normalized_files_import_with_unknown_abstracts_and_no_invented_retracted_acceptance(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);withdrawn=self.row(key='two');withdrawn['publtype']='withdrawn' + catalogue,candidates=self.fixture(root,[self.row(),withdrawn,self.row(book='AAAI Workshops',key='workshop')]) + result=collect(catalogue,root/'out',['AAAI'],candidates=candidates) + self.assertEqual(result['normalized_records'],2);self.assertEqual(result['excluded_by_reason']['separate_workshop'],1) + manifest=result['files'][0];self.assertEqual(manifest['missing_abstract_count'],2) + rows=[json.loads(line) for line in Path(manifest['file']).read_text().splitlines()] + self.assertEqual(rows[0]['authors'],['Jane Doe','John Smith']);self.assertIsNone(rows[0]['abstract']) + self.assertEqual(rows[1]['status'],'unknown');self.assertEqual(rows[1]['decision_events'],[]) + db=root/'papers.sqlite';ingest_file(db,manifest['file']) + with closing(sqlite3.connect(db)) as c:self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],1) + + def test_checksum_failure_and_duplicate_keys_do_not_publish_files(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);catalogue,candidates=self.fixture(root,[self.row(),self.row()]) + with self.assertRaisesRegex(ValueError,'Repeated DBLP'):collect(catalogue,root/'out',['AAAI'],candidates=candidates) + self.assertFalse(list((root/'out').glob('*.jsonl'))) + candidates.write_text('tampered') + with self.assertRaisesRegex(ValueError,'checksum mismatch'):collect(catalogue,root/'out',['AAAI'],candidates=candidates) + + def test_case_variants_use_one_conference_edition_and_submission(self): + with tempfile.TemporaryDirectory() as tmp: + store=Store(Path(tmp)/'papers.sqlite');self.addCleanup(store.conn.close) + paper,one=store.record('pmlr','one','https://example.org/one','A paper','Abstract',[],{'doi':'10.1234/one'}) + store.decision(paper,one,'CORL',2024,'main','one','https://example.org/one','accepted','Accept',identities=[('publication_doi','10.1234/one')]) + same,two=store.record('dblp','two','https://example.org/two','A paper',None,[],{'doi':'10.1234/one'}) + store.decision(same,two,'CoRL',2024,'main','two','https://example.org/two','accepted','Published',identities=[('publication_doi','10.1234/one')]) + self.assertEqual(store.conn.execute('SELECT count(*) FROM venues').fetchone()[0],1) + self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + + def test_production_disk_reserve_stops_before_publishing(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);catalogue,candidates=self.fixture(root,[self.row()]) + with patch('paper_atlas.collectors.dblp_records.shutil.disk_usage',return_value=SimpleNamespace(free=1024)): + with self.assertRaisesRegex(RuntimeError,'8 GiB'):collect(catalogue,root/'out',['AAAI'],candidates=candidates) + self.assertFalse(list((root/'out').glob('*.jsonl'))) + + +if __name__=='__main__':unittest.main() -- 2.52.0 From e5381d0e70abb5f313a89cb981171e9011e0e6b4 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Fri, 6 Nov 2026 18:38:53 +0800 Subject: [PATCH 2/2] Verify short historical years and preserve mixed publication tracks --- docs/data-sources.md | 2 ++ paper_atlas/collectors/dblp_records.py | 2 +- paper_atlas/collectors/dblp_scope.py | 19 +++++++++++++++---- tests/test_dblp_records.py | 14 ++++++++++++++ 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/docs/data-sources.md b/docs/data-sources.md index d2aa829..7e365e9 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -128,6 +128,8 @@ paper-atlas collect dblp \ 输出按会议/年份/track 分文件，附 manifest、`dblp-normalization.json` 范围审查统计和带哈希的 `dblp-import-queue.json`。不能识别的书目、年份或归属会逐类登记，不默认算作主会。DBLP 不含摘要，因此输出摘要为空、清单为 partial；需要通过有证据的跨来源关联补齐论文实体摘要。书目发表证据与原始审稿决定分别保留，不依据书目缺席推断拒稿。已存在其他出版来源的届次，还须先回填其权威身份并核验镜像记录，不能直接累加投稿数。 +部分早期 SIGIR、CIKM、IJCAI 出版卷使用两位年份键；只有论文集标题明确给出唯一完整年份且与短键相符时才采用，不推测世纪。WWW 特殊兴趣、Alternate track 单列；同卷混合 Applied Data Science、Demo、Industrial 且不能逐篇区分时使用 `proceedings_unspecified`，不把整卷归为某一个具体 track。 + XML 只允许校验过的本地 DTD；官方通用名 `dblp.dtd` 与月度快照使用的 `dblp-2023-06-28.dtd` 均解析到同一份已校验文件。拒绝其他外部实体和内部 DTD 子集，不访问 XML 声明的任意网络地址，并遵循源 XML 的字符编码。临时目录库或 gzip 检查失败时不发布正式候选文件，需先调查残留的 `.part` 状态，再决定是否重新运行。 公开 ICLR 投稿快照的固定 Release 供后续筛选投稿和明确决定；它是第三方保存的 OpenReview 历史副本，不等于最新官方全量数据： diff --git a/paper_atlas/collectors/dblp_records.py b/paper_atlas/collectors/dblp_records.py index f5cfe64..17032e8 100644 --- a/paper_atlas/collectors/dblp_records.py +++ b/paper_atlas/collectors/dblp_records.py @@ -66,7 +66,7 @@ def normalize_row(row,volumes,targets,include_workshops=False,through_year=2026) authors=[re.sub(r'\s+\d{4}$','',name).strip() for name in fields.get('author',[]) if name.strip()] metadata={'snapshot_sha256':XML_SHA,'snapshot_release':RELEASE,'observation_date_precision':'day', 'proceedings_key':refs[0],'proceedings_title':edition['proceedings_title'],'bibliographic_year':edition['bibliographic_year'], - 'edition_year_basis':'proceedings key, checked against bibliographic year or explicit event year in proceedings title', + 'edition_year_basis':edition['edition_year_basis'], 'scope_rule':edition['rule'],'author_names_raw':fields.get('author',[]),'publtype':row.get('publtype'), 'bibliography':{key:value for key,value in fields.items() if key in ('booktitle','year','ee','url','pages','volume','number','publisher','series','isbn')}, 'ambiguous_identifiers':ambiguous,'publication_evidence':'inproceedings bibliography; not an original decision note'} diff --git a/paper_atlas/collectors/dblp_scope.py b/paper_atlas/collectors/dblp_scope.py index 4adb654..33c4b00 100644 --- a/paper_atlas/collectors/dblp_scope.py +++ b/paper_atlas/collectors/dblp_scope.py @@ -24,7 +24,7 @@ BOOKS = { 'www':[('WWW','WWW',[])], 'wsdm':[('WSDM','WSDM',[])], 'recsys':[('RecSys','RecSys',[])], 'cikm':[('CIKM','CIKM',[])], 'icdm':[('ICDM','ICDM',[])], 'sdm':[('SDM','SDM',[])], 'ecml':[('ECML','ECML',['ECML-PKDD']),('EWSL','EWSL',['ECML-PKDD'])], - 'pkdd':[('ECML/PKDD','ECML-PKDD',[]),('PKDD','PKDD',['ECML-PKDD'])], + 'pkdd':[('ECML/PKDD','ECML-PKDD',[]),('ECML PKDD','ECML-PKDD',[]),('PKDD','PKDD',['ECML-PKDD'])], 'icra':[('ICRA','ICRA',[])], 'iros':[('IROS','IROS',[])], 'rss':[('Robotics: Science and Systems','RSS',[])], 'corl':[('CoRL','CoRL',[])], 'aips':[('ICAPS','ICAPS',[]),('AIPS','AIPS',['ICAPS'])], 'icaps':[('ICAPS','ICAPS',[])], @@ -56,8 +56,16 @@ def scope(row,volume,through_year=2026): if volume_family!=family and not any(item[1]==venue for item in BOOKS.get(volume_family,[])): return None,'crossref_venue_conflict' year_match=re.match(r'conf/[^/]+/((?:19|20)\d{2})',volume['key']) - if not year_match:return None,'unverified_edition_year' - year=int(year_match[1]) + year_basis='proceedings key, checked against bibliographic year or explicit event year in proceedings title' + if year_match:year=int(year_match[1]) + else: + # Old SIGIR/CIKM/IJCAI keys use two digits. Never guess their century. + short=re.fullmatch(r'conf/[^/]+/(\d{2})',volume['key']) + years=set(re.findall(r'\b(?:19|20)\d{2}\b',title)) + if not short or len(years)!=1:return None,'unverified_edition_year' + year=int(next(iter(years))) + if year%100!=int(short[1]):return None,'unverified_edition_year' + year_basis='unique explicit event year in proceedings title, corroborated by two-digit proceedings key' if year>through_year:return None,'future_edition' bibliographic=fields.get('year',[]) if len(bibliographic)!=1 or not re.fullmatch(r'\d{4}',bibliographic[0]):return None,'ambiguous_bibliographic_year' @@ -66,6 +74,7 @@ def scope(row,volume,through_year=2026): if 'findings' in text:track='findings' elif 'shared task' in text:track='shared_task' elif 'student research' in text or 'workshop' in text or 'wokshop' in text or base in ('ICCVW','EWSL'):track='workshop' + elif ('demo track' in text and ('industrial track' in text or 'applied data science track' in text)):track='proceedings_unspecified' elif 'demonstration' in text or re.search(r'\bdemos?\b',text):track='demo' elif 'tutorial' in text:track='tutorial' elif 'industry' in text or 'industrial track' in text or 'applied data science' in text:track='industry' @@ -75,6 +84,8 @@ def scope(row,volume,through_year=2026): elif 'doctoral' in text:track='doctoral' elif 'challenge' in text or 'competition' in text:track='competition' elif 'bridge program' in text:track='bridge' + elif 'special interest tracks' in text:track='special_interest' + elif 'alternate track' in text:track='alternate' elif base=='NeurIPS Datasets and Benchmarks':track='datasets_benchmarks' elif suffix and not (re.fullmatch(r'(?:\d+|Vol\.\s*\d+|Volume \d+: (?:Long|Short) Papers|Long Papers|Short Papers|Selected Papers|Fall)',suffix,re.I) or suffix.casefold() in ('poster','posters','oral','spotlight')):return None,'unclassified_track' @@ -82,4 +93,4 @@ def scope(row,volume,through_year=2026): if venue=='AAAI+IAAI':track='proceedings_unspecified' if venue=='AAMAS' and year<2002:return None,'predecessor_not_aamas' return {'venue':venue,'venue_aliases':aliases,'year':year,'track':track,'bibliographic_year':int(bibliographic[0]), - 'rule':family+':'+base,'proceedings_title':title},None + 'rule':family+':'+base,'proceedings_title':title,'edition_year_basis':year_basis},None diff --git a/tests/test_dblp_records.py b/tests/test_dblp_records.py index 5893c96..fc2305b 100644 --- a/tests/test_dblp_records.py +++ b/tests/test_dblp_records.py @@ -51,6 +51,20 @@ class DblpRecordTests(unittest.TestCase): self.assertEqual(result['venue'],'COLING+LREC') for book,track in [('ACL (Findings)','findings'),('ACL (Student Research Workshop)','workshop'),('ACL (System Demonstrations)','demo')]: self.assertEqual(scope(self.row('acl',book),self.volume('acl',2024))[0]['track'],track) + for suffix,track in [('Special interest tracks and posters','special_interest'),('Alternate Track Papers & Posters','alternate')]: + self.assertEqual(scope(self.row('www','WWW ('+suffix+')'),self.volume('www',2024))[0]['track'],track) + volume=self.volume('pkdd',2026,'ECML PKDD 2026, Applied Data Science Track, Demo Track and Industrial Track') + self.assertEqual(scope(self.row('pkdd','ECML PKDD (10)',2026),volume)[0]['track'],'proceedings_unspecified') + + def test_short_year_keys_require_a_unique_explicit_year_not_a_guessed_century(self): + row=self.row('sigir','SIGIR',1980);volume=self.volume('sigir',1980,'Information Retrieval Research, Cambridge, June 1980') + volume['key']='conf/sigir/80';volume['fields']['year']=['1981'];row['fields']['crossref']=[volume['key']] + edition,_=scope(row,volume);self.assertEqual(edition['year'],1980) + self.assertIn('two-digit',edition['edition_year_basis']) + volume['fields']['title']=['SIGIR 1980, published 1981'] + self.assertEqual(scope(row,volume)[1],'unverified_edition_year') + volume['fields']['title']=['SIGIR 1989'] + self.assertEqual(scope(row,volume)[1],'unverified_edition_year') def test_crossref_mismatch_and_future_editions_are_held_for_review(self): self.assertEqual(scope(self.row(),self.volume('kdd',2024))[1],'crossref_venue_conflict') -- 2.52.0