diff --git a/docs/data-sources.md b/docs/data-sources.md index 21cc0f0..bc8558a 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -52,7 +52,7 @@ paper-atlas collect mlsys --cache /data/paper-index/raw/http-cache \ 每年清单核对官方声明数量,缺摘要标为 partial;未知现代轨道、空目录或读取失败不能记为成功。`SysML 2019` 归于同一 MLSys 系列并保留别名。2018 年官方说明接受的投稿为 non-archival,论文集从 2019 年开始;不拼造 2018 年归档卷,也不把未出现在论文集中的投稿判为拒稿。参见 [2018 年官方说明](https://mlsys.org/Conferences/2018/)。 -MLSys 老式 `/paper//hash/…` 与新式 `/paper_files/paper//hash/…` 链接规范为同一个 `publication_mlsys` 标识;DBLP 和官方摘要源共用它,避免同一发表记录成为两个论文实体或两次投稿。只从自身出版社链接提取,不从摘要参考文献提取,也不把发表标识用于合并某次拒稿尝试。共享投稿身份入口要求 MLSys 源具有有效的同年出版社身份,缺失或错年的文件不能生成投稿事件。 +MLSys 老式 `/paper//hash/…` 与新式 `/paper_files/paper//hash/…` 链接规范为同一个 `publication_mlsys` 标识,也支持 2023 年实际使用的 `-Abstract-mlsys2023.html` 及对应年份标签;标签年份必须与目录年份一致。DBLP 和官方摘要源共用它,避免同一发表记录成为两个论文实体或两次投稿。只从自身出版社链接提取,不从摘要参考文献提取,也不把发表标识用于合并某次拒稿尝试。共享投稿身份入口要求 MLSys 源具有有效的同年出版社身份,缺失或错年的文件不能生成投稿事件;旧版 DBLP 整理文件可利用已保留的唯一自身出版社链接补得此身份,不需要丢弃来源或依靠标题猜测。 ACM MM 使用固定 DBLP 原始快照和已核验的 proceedings crossref 盘点历史,主会 `ACM Multimedia`、历史编号分卷与同系列 workshop 分开。主会卷也包含 Doctoral Symposium、Reproducibility Companion Papers、Tutorials 等章节(参见 [2025 年目录](https://dblp.org/db/conf/mm/mm2025.html)),而原始 XML 不含逐篇章节,所以这类记录暂标 `proceedings_unspecified`,不能整卷当作 regular 主轨道;后续需逐篇目录证据补齐。旧版候选 JSONL 没有 `mm` / `mlsys` 家族,不能用它假装覆盖新增会议;应从已经下载并校验的原始 XML 重新筛选这两个会议,保留完整扫描数量、卷证据与遗漏原因。DBLP 没有摘要,后续须通过 DOI、真实 forum 等独立标识关联官方或公开元数据,不能将书目导入成功表述为摘要已齐全。 diff --git a/paper_atlas/collectors/proceedings.py b/paper_atlas/collectors/proceedings.py index 649012e..116e848 100644 --- a/paper_atlas/collectors/proceedings.py +++ b/paper_atlas/collectors/proceedings.py @@ -84,7 +84,8 @@ def parse_paper(html,url,source,year,index_url): identifiers['publication_mlsys'] = own if track_text: explicit = {'conference': 'main', 'main track': 'main', 'research track': 'main', - 'industry track': 'industry', 'industrial track': 'industry'} + 'industry track': 'industry', 'industrial track': 'industry', + 'mlsys' + str(year): 'main'} if track_text.casefold() not in explicit: raise ValueError('Unclassified official MLSys track: ' + track_text) track = explicit[track_text.casefold()] diff --git a/paper_atlas/db/submissions.py b/paper_atlas/db/submissions.py index 99f29ec..2e017f4 100644 --- a/paper_atlas/db/submissions.py +++ b/paper_atlas/db/submissions.py @@ -1,6 +1,7 @@ """Canonical submission events with independent source and decision evidence.""" from datetime import datetime, timezone import re +from paper_atlas.io.publication_ids import mlsys_id PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp'} @@ -25,8 +26,16 @@ def identities_for_row(row): if row['source'] in PUBLICATIONS and row.get('status')=='accepted' and own.get('doi'): doi=re.sub(r'^https?://(?:dx\.)?doi\.org/','',str(own['doi']),flags=re.I).strip().lower() if re.fullmatch(r'10\.\d{4,9}/\S+',doi):identities.append({'scheme':'publication_doi','value':doi}) - if row['source'] in PUBLICATIONS and row.get('status')=='accepted' and own.get('publication_mlsys'): - identities.append({'scheme':'publication_mlsys','value':own['publication_mlsys']}) + publication_mlsys = own.get('publication_mlsys') + if not publication_mlsys and row['source']=='dblp' and row['venue']=='MLSys': + # Older normalized DBLP files retain their own publisher links even + # when a newly supported historical URL label was not yet parsed. + links = (row.get('metadata') or {}).get('bibliography', {}).get('ee', []) + candidates = {identity for link in links if (identity := mlsys_id(link))} + if len(candidates)==1: + publication_mlsys = candidates.pop() + if row['source'] in PUBLICATIONS and row.get('status')=='accepted' and publication_mlsys: + identities.append({'scheme':'publication_mlsys','value':publication_mlsys}) meta=row.get('metadata') or {} if row['source']=='iclr_program' and row['venue']=='ICLR' and meta.get('program_identity_type')=='submission' and meta.get('program_submission_number_verified') is True: identities.append({'scheme':'iclr_number','value':str(meta['program_source_id'])}) diff --git a/paper_atlas/io/publication_ids.py b/paper_atlas/io/publication_ids.py index 798e43e..63d686e 100644 --- a/paper_atlas/io/publication_ids.py +++ b/paper_atlas/io/publication_ids.py @@ -7,5 +7,7 @@ def mlsys_id(url): parsed = urlsplit(url) if parsed.scheme not in ('https', 'http') or parsed.netloc != 'proceedings.mlsys.org': return None - match = re.fullmatch(r'/paper(?:_files/paper)?/((?:19|20)\d{2})/hash/([a-f0-9]{32})-Abstract(?:-Conference)?\.html', parsed.path) - return match[1] + '/' + match[2] if match else None + match = re.fullmatch(r'/paper(?:_files/paper)?/((?:19|20)\d{2})/hash/([a-f0-9]{32})-Abstract(?:-(?:Conference|mlsys(20\d{2})))?\.html', parsed.path) + if not match or match[3] and match[3] != match[1]: + return None + return match[1] + '/' + match[2] diff --git a/tests/test_mlsys_multimedia.py b/tests/test_mlsys_multimedia.py index 78769c0..b22bc7f 100644 --- a/tests/test_mlsys_multimedia.py +++ b/tests/test_mlsys_multimedia.py @@ -55,6 +55,28 @@ class MlsysMultimediaTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, 'non-archival'): parse_paper(html, page(2018), 'mlsys', 2018, ROOT) + def test_year_label_and_retained_bibliography_links_bridge_the_2023_format(self): + url = page(2023).replace('-Conference.html', '-mlsys2023.html') + self.assertEqual(mlsys_id(url), '2023/' + HASH) + self.assertIsNone(mlsys_id(url.replace('mlsys2023.html', 'mlsys2022.html'))) + official = parse_paper('

Updated title

mlsys2023

Official abstract

', url, 'mlsys', 2023, ROOT) + self.assertEqual(official['track'], 'main') + legacy = {'source': 'dblp', 'external_id': 'conf/mlsys/paper23', 'url': 'https://dblp.org/rec/conf/mlsys/paper23', + 'venue': 'MLSys', 'year': 2023, 'track': 'main', 'status': 'accepted', 'title': 'Bibliographic title', + 'abstract': None, 'authors': [], 'identifiers': {}, 'metadata': {'bibliography': {'ee': [url]}}} + self.assertEqual(identities_for_row(legacy), [('publication_mlsys', '2023/' + HASH)]) + self.assertEqual(identities_for_row({**legacy, 'status': 'unknown'}), []) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); db = root / 'papers.sqlite' + for row in [legacy, official]: + path = root / (row['source'] + '.jsonl'); path.write_text(json.dumps(row) + '\n') + path.with_suffix('.manifest.json').write_text(json.dumps({'source': row['source'], 'parser_version': 2, 'list_url': ROOT, 'count': 1})) + ingest_file(db, path) + with closing(sqlite3.connect(db)) as c: + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 1) + self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 1) + self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0], 2) + def test_legacy_mlsys_and_sysml_alias_preserve_same_publication_identity(self): self.assertEqual(mlsys_id(page(2019, False)), mlsys_id(page(2019))) self.assertIsNone(mlsys_id(page().replace('proceedings.mlsys.org', 'example.org')))