diff --git a/docs/data-sources.md b/docs/data-sources.md index e717789..21cc0f0 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -52,9 +52,9 @@ paper-atlas collect mlsys --cache /data/paper-index/raw/http-cache \ 每年清单核对官方声明数量，缺摘要标为 partial；未知现代轨道、空目录或读取失败不能记为成功。`SysML 2019` 归于同一 MLSys 系列并保留别名。2018 年官方说明接受的投稿为 non-archival，论文集从 2019 年开始；不拼造 2018 年归档卷，也不把未出现在论文集中的投稿判为拒稿。参见 [2018 年官方说明](https://mlsys.org/Conferences/2018/)。 -MLSys 老式 `/paper//hash/…` 与新式 `/paper_files/paper//hash/…` 链接规范为同一个 `publication_mlsys` 标识；DBLP 和官方摘要源共用它，避免同一发表记录成为两个论文实体或两次投稿。只从自身出版社链接提取，不从摘要参考文献提取，也不把发表标识用于合并某次拒稿尝试。 +MLSys 老式 `/paper//hash/…` 与新式 `/paper_files/paper//hash/…` 链接规范为同一个 `publication_mlsys` 标识；DBLP 和官方摘要源共用它，避免同一发表记录成为两个论文实体或两次投稿。只从自身出版社链接提取，不从摘要参考文献提取，也不把发表标识用于合并某次拒稿尝试。共享投稿身份入口要求 MLSys 源具有有效的同年出版社身份，缺失或错年的文件不能生成投稿事件。 -ACM MM 使用固定 DBLP 原始快照和已核验的 proceedings crossref 盘点历史，主会 `ACM Multimedia`、历史编号分卷与同系列 workshop 分开。旧版候选 JSONL 没有 `mm` / `mlsys` 家族，不能用它假装覆盖新增会议；应从已经下载并校验的原始 XML 重新筛选这两个会议，保留完整扫描数量、卷证据与遗漏原因。DBLP 没有摘要，后续须通过 DOI、真实 forum 等独立标识关联官方或公开元数据，不能将书目导入成功表述为摘要已齐全。 +ACM MM 使用固定 DBLP 原始快照和已核验的 proceedings crossref 盘点历史，主会 `ACM Multimedia`、历史编号分卷与同系列 workshop 分开。主会卷也包含 Doctoral Symposium、Reproducibility Companion Papers、Tutorials 等章节（参见 [2025 年目录](https://dblp.org/db/conf/mm/mm2025.html)），而原始 XML 不含逐篇章节，所以这类记录暂标 `proceedings_unspecified`，不能整卷当作 regular 主轨道；后续需逐篇目录证据补齐。旧版候选 JSONL 没有 `mm` / `mlsys` 家族，不能用它假装覆盖新增会议；应从已经下载并校验的原始 XML 重新筛选这两个会议，保留完整扫描数量、卷证据与遗漏原因。DBLP 没有摘要，后续须通过 DOI、真实 forum 等独立标识关联官方或公开元数据，不能将书目导入成功表述为摘要已齐全。 公开拒稿镜像的 `acmmm` 家族映射到 `MM`，只接受明确的负面状态，继续保留镜像与原始签名决定的区别。现有固定 OpenReview 镜像只含 ACM MM 2024 的录用记录，没有据此证明其他年份没有公开拒稿；[2025 年官方 CFP](https://acmmm2025.org/call-for-papers/) 仅说明审稿期间不公开，不能外推为会后永不公开。 diff --git a/paper_atlas/collectors/dblp_records.py b/paper_atlas/collectors/dblp_records.py index 4c520ab..4f5685e 100644 --- a/paper_atlas/collectors/dblp_records.py +++ b/paper_atlas/collectors/dblp_records.py @@ -88,6 +88,8 @@ def normalize_row(row,volumes,targets,include_workshops=False,through_year=2026) 'scope_rule':edition['rule'],'author_names_raw':fields.get('author',[]),'publtype':row.get('publtype'), 'bibliography':{key:value for key,value in fields.items() if key in ('booktitle','year','ee','url','pages','volume','number','publisher','series','isbn')}, 'ambiguous_identifiers':ambiguous,'publication_evidence':'inproceedings bibliography; not an original decision note'} + if edition.get('track_scope_note'): + metadata['track_scope_note']=edition['track_scope_note'] flagged=bool(re.search(r'\b(?:withdrawn|retracted)\b',row.get('publtype') or '',re.I)) result={'source':'dblp','external_id':row['key'],'url':'https://dblp.org/rec/'+row['key']+'.html', 'title':title,'abstract':None,'authors':authors,'identifiers':own, diff --git a/paper_atlas/collectors/dblp_scope.py b/paper_atlas/collectors/dblp_scope.py index 2689b61..08a550d 100644 --- a/paper_atlas/collectors/dblp_scope.py +++ b/paper_atlas/collectors/dblp_scope.py @@ -96,5 +96,11 @@ def scope(row,volume,through_year=2026): elif suffix.casefold() in ('poster','posters') and venue not in ('ICLR','ACL','EMNLP','NAACL','EACL','COLING','CoNLL'):track='poster' if venue=='AAAI+IAAI':track='proceedings_unspecified' if venue=='AAMAS' and year<2002:return None,'predecessor_not_aamas' - return {'venue':venue,'venue_aliases':aliases,'year':year,'track':track,'bibliographic_year':int(bibliographic[0]), - 'rule':family+':'+base,'proceedings_title':title,'edition_year_basis':year_basis},None + result = {'venue':venue,'venue_aliases':aliases,'year':year,'track':track,'bibliographic_year':int(bibliographic[0]), + 'rule':family+':'+base,'proceedings_title':title,'edition_year_basis':year_basis} + if venue=='MM' and track=='main': + # A main MM volume also includes doctoral, tutorials, companion papers + # and other sections. DBLP XML omits the per-paper TOC section. + result['track']='proceedings_unspecified' + result['track_scope_note']='Main proceedings volume; per-paper section is absent from the bibliography. Do not infer a regular research track.' + return result,None diff --git a/paper_atlas/db/submissions.py b/paper_atlas/db/submissions.py index bdf0e4a..99f29ec 100644 --- a/paper_atlas/db/submissions.py +++ b/paper_atlas/db/submissions.py @@ -15,6 +15,11 @@ def timestamp(value=None): def identities_for_row(row): identities=list(row.get('submission_identifiers') or []) own=row.get('identifiers') or {} + if row['source']=='mlsys': + value=own.get('publication_mlsys') + if (not isinstance(value,str) or not re.fullmatch(str(row['year'])+r'/[a-f0-9]{32}',value) + or row['venue']!='MLSys'): + raise ValueError('Invalid authoritative MLSys publisher identity') if own.get('openreview'): identities.append({'scheme':'openreview','value':own['openreview']}) if row['source'] in PUBLICATIONS and row.get('status')=='accepted' and own.get('doi'): diff --git a/paper_atlas/ingestion/backfill.py b/paper_atlas/ingestion/backfill.py index 7f4b0c4..b943b4e 100644 --- a/paper_atlas/ingestion/backfill.py +++ b/paper_atlas/ingestion/backfill.py @@ -21,7 +21,7 @@ def queue_files(path): def backfill(db,files): for filename in files: manifest=json.loads(Path(filename).with_suffix('.manifest.json').read_text()) - if manifest['source'] in ('colm','neurips','mlsys','iclr_program') and manifest.get('parser_version',0)<2: + if manifest['source'] in ('colm','neurips','iclr_program') and manifest.get('parser_version',0)<2: raise ValueError('Recollect old parser output before submission evidence backfill') store=Store(db);c=store.conn;count=0 before=c.execute('SELECT count(*) FROM submissions').fetchone()[0] diff --git a/paper_atlas/ingestion/records.py b/paper_atlas/ingestion/records.py index 05172c7..a06545d 100644 --- a/paper_atlas/ingestion/records.py +++ b/paper_atlas/ingestion/records.py @@ -12,7 +12,7 @@ def ingest_file(db,path): path = Path(path) manifest_path = path.with_suffix('.manifest.json') manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {} - if manifest.get('source') in ('colm','neurips','mlsys') and manifest.get('parser_version',0)<2: + if manifest.get('source') in ('colm','neurips') and manifest.get('parser_version',0)<2: raise ValueError('Recollect proceedings with parser_version>=2 before identity matching') if manifest.get('source')=='iclr_program' and manifest.get('parser_version',0)<2: raise ValueError('Recollect ICLR with parser_version>=2 before submission identity matching') diff --git a/tests/test_mlsys_multimedia.py b/tests/test_mlsys_multimedia.py index 7a45bc1..78769c0 100644 --- a/tests/test_mlsys_multimedia.py +++ b/tests/test_mlsys_multimedia.py @@ -69,7 +69,8 @@ class MlsysMultimediaTests(unittest.TestCase): def test_mm_main_numbered_parts_and_workshops_remain_separate(self): for book in ['ACM Multimedia', 'ACM Multimedia (1)', 'ACM Multimedia (2)']: result, reason = scope(raw_row(book=book), volume()) - self.assertIsNone(reason); self.assertEqual(result['venue'], 'MM'); self.assertEqual(result['track'], 'main') + self.assertIsNone(reason); self.assertEqual(result['venue'], 'MM'); self.assertEqual(result['track'], 'proceedings_unspecified') + self.assertIn('per-paper section', result['track_scope_note']) self.assertIn('ACMMM', result['venue_aliases']) self.assertEqual(scope(raw_row(book='ACM Multimedia Workshops'), volume())[0]['track'], 'workshop') self.assertEqual(scope(raw_row(book='ACM Multimedia (Thematic Workshops)'), volume())[0]['track'], 'workshop') @@ -107,6 +108,8 @@ class MlsysMultimediaTests(unittest.TestCase): 'identifiers': {'publication_mlsys': '2026/' + HASH}} self.assertEqual(identities_for_row(row), [('publication_mlsys', '2026/' + HASH)]) self.assertEqual(identities_for_row({**row, 'status': 'rejected'}), []) + with self.assertRaisesRegex(ValueError, 'Invalid authoritative'): + identities_for_row({**row, 'identifiers': {}}) for changed in [dict(year=2025), dict(venue='MM'), dict(source='curated')]: bad = {**row, **changed, 'submission_identifiers': [{'scheme': 'publication_mlsys', 'value': '2026/' + HASH}]} with self.assertRaisesRegex(ValueError, 'Invalid authoritative'):