From 7c6f4c9f1d585300ee90905de00da09235bd5ee4 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Mon, 9 Nov 2026 02:39:18 +0800 Subject: [PATCH] Recognize evidenced early ICML and IROS primary editions --- README.md | 2 + .../historical-primary-series-2026-11-09.md | 33 +++++ paper_atlas/collectors/dblp_scope.py | 27 ++++ tests/test_historical_primary_series.py | 115 ++++++++++++++++++ 4 files changed, 177 insertions(+) create mode 100644 docs/coverage/historical-primary-series-2026-11-09.md create mode 100644 tests/test_historical_primary_series.py diff --git a/README.md b/README.md index 234e8d0..57ca0cc 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \ --out /data/paper-index/collected ``` +早期主系列的Workshop名称按[ICML/IROS精确届次证据](docs/coverage/historical-primary-series-2026-11-09.md)识别，保留原名与范围证据；代码支持不代表已补齐生产数据。 + `paper-atlas collect --help` 查看来源；`paper-atlas import --help` 查看导入方式。完整批次步骤见[使用指南](docs/usage.md)。 共享实例名为 `foundry-postgres`，数据位于 `foundry_data` 库的 `research` schema；其他项目可按权限复用同一组表。库名通过 libpq service 配置、schema 通过 search_path 或 `--pg-schema` 指定，不写死在查询中。PostgreSQL 凭据放在服务器权限为 `0600` 的 libpq service 文件中；命令行只使用服务名称。`paper-atlas maintain postgres --help` 提供迁移入口：先完整核验 21 张业务表，再发布目标 schema。SQLite FTS5 内部表由 PostgreSQL GIN 索引替代。切换后 SQLite 为只读备份；`import records`、`import arxiv` 和 `maintain backfill` 使用 `--pg-service foundry-research` 写入主库，整个写入会话只允许一个协调写入者，会议文件失败时整体回滚并保留失败批次记录。 diff --git a/docs/coverage/historical-primary-series-2026-11-09.md b/docs/coverage/historical-primary-series-2026-11-09.md new file mode 100644 index 0000000..52a6bb1 --- /dev/null +++ b/docs/coverage/historical-primary-series-2026-11-09.md @@ -0,0 +1,33 @@ +# ICML 与 IROS 早期主系列的来源范围 + +本轮修正采集阶段的届次识别，不修改任何现有生产投稿轨道或历史。正式主系列曾以Workshop命名，不等于附属workshop；也不能反过来把所有旧workshop升为主会。 + +## 已核实的精确范围 + +| 会议 | 论文集键 | 依据与行为 | +| --- | --- | --- | +| ICML | `conf/icml/1988`、`1989`、`1990`、`1991`、`1992` | [官方历史](https://icml.cc/Conferences/2008/past_icmls.shtml.html)将其列为第5–9届，年份与举办地对应固定目录中的完整标题。1989/1991/1992的Workshop名称不再导致主系列被默认排除；1988/1990保留main并增加来源范围证据 | +| IROS | `conf/iros/1988`、`1989`、`1990` | [IEEE RAS官方历届目录](https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/)列出三届与论文集链接，地点及日期对应固定目录标题；这些是IROS主系列 | + +规则同时要求family、booktitle基础别名（ML或IROS）、空轨道后缀、完整精确论文集键、匹配的事件年份及固定完整标题。仍执行原有书目年份核查：与事件年不同且标题无明确事件年时拒绝推断。ML91原始标题没有四位年份，使用精确1991卷键、完整标题、逐篇书目年和官方第8届记录，不从两位标题代码猜世纪。 + +`1991w`/`1988w`等卷键、booktitle中的Workshops后缀、不同届次/地点/完整标题、未列入的年份均不会获得上述主系列豁免。AISTATS/COLT既有精确规则保留；真正的附属workshop仍受默认不采集workshop的边界约束。 + +## 原始字段、决定与后续导入 + +采集器仍保存原始proceedings标题（包括Workshop）、书目年、原始作者名、完整来源书目字段及稳定标识；新增 `track_scope_note` 与官方 `track_scope_evidence_url`。接受状态仍只源自该论文的inproceedings书目证据；不从历史目录生成占位论文或拒稿，也不重新导入已经存在的记录。 + +新规则的输出必须重新做原始来源/标题/作者/稳定标识与现有库冲突预检，再按增量导入。此前被隔离的ICML1988/1990记录不能仅凭代码规则而直接解除全部冲突。当前正在运行的历史批次使用原固定版本和文件，不能在运行中替换其源码或输入。 + +## 验证与证据 + +新增回归覆盖：ICML三届Workshop名称、IROS最初三届、ICML原Conference名称的两届、错误卷键/后缀/标题/未知年份，以及归一化记录经过原生文件导入后保留轨道证据。构建与测试仅在已授权的ces-control远程runner执行，本机仅做语法和文本/JSON一致性检查；结果以相应commit的CI回执为准。 + +官方HTML在工作证据目录 `paper-index/historical-primary-series-evidence-v1/`，附响应状态、字节数、时间与SHA256；仅取元数据网页，没有下载论文全文。 + +| 来源 | UTC采样时间 | HTTP | 字节数 | SHA256 | +| --- | --- | ---: | ---: | --- | +| [icml-history](https://icml.cc/Conferences/2008/past_icmls.shtml.html) | 2026-11-08T18:36:43.043135+00:00 | 200 | 11993 | `80cce886fe985551339a52b13c4077acf827e8fcc01555f854dc24077bdebdf0` | +| [iros-history](https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/) | 2026-11-08T18:36:45.687171+00:00 | 200 | 466059 | `fdb4594621e68567e1099f5c88aae53de835b7b8146025148c3b1968df990697` | + +本次不关闭历史年度缺口，也不替代全部导入后的全库去重、来源/年度/track覆盖、摘要和实际查询验收。 diff --git a/paper_atlas/collectors/dblp_scope.py b/paper_atlas/collectors/dblp_scope.py index fa7892b..5106db4 100644 --- a/paper_atlas/collectors/dblp_scope.py +++ b/paper_atlas/collectors/dblp_scope.py @@ -54,7 +54,34 @@ HISTORICAL_WORKSHOP_TITLES = { } +# Exact pinned volume titles corroborated by official series histories. The +# original title (including Workshop) remains in source metadata. In particular +# ML91 has no four-digit year in its title; its exact key/title and bibliographic +# year are checked independently, rather than guessing a title year. +HISTORICAL_PRIMARY_EVIDENCE = { + 'icml': ('ML', 'https://icml.cc/Conferences/2008/past_icmls.shtml.html'), + 'iros': ('IROS', 'https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/'), +} +HISTORICAL_PRIMARY_TITLES = { + ('icml', 1988): 'Machine Learning, Proceedings of the Fifth International Conference on Machine Learning, Ann Arbor, Michigan, USA, June 12-14, 1988', + ('icml', 1989): 'Proceedings of the Sixth International Workshop on Machine Learning (ML 1989), Cornell University, Ithaca, New York, USA, June 26-27, 1989', + ('icml', 1990): 'Machine Learning, Proceedings of the Seventh International Conference on Machine Learning, Austin, Texas, USA, June 21-23, 1990', + ('icml', 1991): 'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA', + ('icml', 1992): 'Proceedings of the Ninth International Workshop on Machine Learning (ML 1992), Aberdeen, Scotland, UK, July 1-3, 1992', + ('iros', 1988): "IEEE International Workshop on Intelligent Robots and Systems '88, Proceedings. IROS 1988, Tokyo, Japan, October 31 - November 2, 1988", + ('iros', 1989): "IEEE International Workshop on Intelligent Robots and Systems '89, Proceedings. IROS 1989, September 4-6, 1989, Tsukuba, Japan", + ('iros', 1990): "IEEE International Workshop on Intelligent Robots and Systems '90, Towards a New Frontier of Applications, Proceedings. IROS 1990, July 3-6, 1990, Ibaraki, Japan", +} + + def historical_track(family, base, suffix, volume_key, year, title): + primary = HISTORICAL_PRIMARY_EVIDENCE.get(family) + if (primary and not suffix and base.casefold() == primary[0].casefold() + and volume_key == f'conf/{family}/{year}' + and title == HISTORICAL_PRIMARY_TITLES.get((family, year))): + return ('main', + 'Historical primary series; exact proceedings key, complete title and bibliographic/event year corroborated by official series history. Workshop in this original title is not an affiliated track.', + primary[1]) if suffix or base.casefold() != family or volume_key != f'conf/{family}/{year}': return None prefix = HISTORICAL_WORKSHOP_TITLES.get((family, year)) diff --git a/tests/test_historical_primary_series.py b/tests/test_historical_primary_series.py new file mode 100644 index 0000000..e872f91 --- /dev/null +++ b/tests/test_historical_primary_series.py @@ -0,0 +1,115 @@ +"""Regression coverage for primary series once named Workshop, with strict scope boundaries.""" +from contextlib import closing +import hashlib +import json +from pathlib import Path +import sqlite3 +import tempfile +import unittest + +from paper_atlas.collectors.dblp_records import normalize_row +from paper_atlas.collectors.dblp_scope import scope +from paper_atlas.ingestion.records import ingest_file + +ICML_HISTORY = 'https://icml.cc/Conferences/2008/past_icmls.shtml.html' +IROS_HISTORY = 'https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/' + + +class HistoricalPrimarySeriesTests(unittest.TestCase): + def example(self, family, year, title, book=None, key=None): + key = key or f'conf/{family}/{year}' + book = book or ('ML' if family == 'icml' else 'IROS') + row = {'record_type':'inproceedings', 'key':f'conf/{family}/Paper{year}', 'publtype':None, + 'fields':{'title':['An independently identified historical research paper'], + 'author':['Jane Doe'], 'booktitle':[book], 'year':[str(year)], + 'crossref':[key], 'ee':[f'https://doi.org/10.1234/{family}.{year}']}} + volume = {'record_type':'proceedings', 'key':key, + 'fields':{'title':[title], 'year':[str(year)]}} + return row, volume + + def test_icml_workshop_primary_editions_are_not_silently_filtered(self): + examples = { + 1989:'Proceedings of the Sixth International Workshop on Machine Learning (ML 1989), Cornell University, Ithaca, New York, USA, June 26-27, 1989', + 1991:'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA', + 1992:'Proceedings of the Ninth International Workshop on Machine Learning (ML 1992), Aberdeen, Scotland, UK, July 1-3, 1992', + } + for year, title in examples.items(): + with self.subTest(year=year): + row, volume = self.example('icml', year, title) + record, reason = normalize_row(row, {volume['key']:volume}, {'ICML'}) + self.assertIsNone(reason) + self.assertEqual((record['venue'],record['year'],record['track'],record['status']), ('ICML',year,'main','accepted')) + self.assertEqual(record['metadata']['proceedings_title'], title) + self.assertEqual(record['metadata']['track_scope_evidence_url'], ICML_HISTORY) + self.assertEqual(record['metadata']['bibliographic_year'], year) + + def test_iros_first_three_editions_are_primary_series(self): + examples = { + 1988:"IEEE International Workshop on Intelligent Robots and Systems '88, Proceedings. IROS 1988, Tokyo, Japan, October 31 - November 2, 1988", + 1989:"IEEE International Workshop on Intelligent Robots and Systems '89, Proceedings. IROS 1989, September 4-6, 1989, Tsukuba, Japan", + 1990:"IEEE International Workshop on Intelligent Robots and Systems '90, Towards a New Frontier of Applications, Proceedings. IROS 1990, July 3-6, 1990, Ibaraki, Japan", + } + for year, title in examples.items(): + with self.subTest(year=year): + row, volume = self.example('iros', year, title) + record, reason = normalize_row(row, {volume['key']:volume}, {'IROS'}) + self.assertIsNone(reason) + self.assertEqual((record['venue'],record['year'],record['track']), ('IROS',year,'main')) + self.assertEqual(record['metadata']['proceedings_title'], title) + self.assertEqual(record['metadata']['track_scope_evidence_url'], IROS_HISTORY) + + def test_main_named_icml_predecessors_also_carry_primary_scope_provenance(self): + examples = { + 1988:'Machine Learning, Proceedings of the Fifth International Conference on Machine Learning, Ann Arbor, Michigan, USA, June 12-14, 1988', + 1990:'Machine Learning, Proceedings of the Seventh International Conference on Machine Learning, Austin, Texas, USA, June 21-23, 1990', + } + for year, title in examples.items(): + row, volume = self.example('icml', year, title) + edition, reason = scope(row, volume) + self.assertIsNone(reason) + self.assertEqual(edition['track'], 'main') + self.assertEqual(edition['track_scope_evidence_url'], ICML_HISTORY) + + def test_wrong_keys_titles_suffixes_and_unknown_years_keep_workshop_boundary(self): + title = 'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA' + for year, changed, book, key in [ + (1991,title,'ML','conf/icml/1991w'), + (1991,title,'ML (Workshops)','conf/icml/1991'), + (1991,title.replace('Eighth','Ninth'),'ML','conf/icml/1991'), + (1991,title + ', Affiliated Workshop','ML','conf/icml/1991'), + (1987,'Machine Learning Workshop 1987, Irvine','ML','conf/icml/1987'), + ]: + with self.subTest(year=year, book=book, key=key, title=changed): + row, volume = self.example('icml',year,changed,book,key) + record, reason = normalize_row(row,{volume['key']:volume},{'ICML'}) + self.assertIsNone(record) + self.assertEqual(reason,'separate_workshop') + self.assertNotIn('track_scope_evidence_url',scope(row,volume)[0]) + title = "IEEE International Workshop on Intelligent Robots and Systems '88, Proceedings. IROS 1988, Tokyo, Japan, October 31 - November 2, 1988" + for changed, book, key in [ + (title,'IROS','conf/iros/1988w'), + (title,'IROS (Workshops)','conf/iros/1988'), + (title.replace('Tokyo','Another city'),'IROS','conf/iros/1988'), + ]: + row, volume = self.example('iros',1988,changed,book,key) + self.assertEqual(normalize_row(row,{volume['key']:volume},{'IROS'})[1],'separate_workshop') + + def test_primary_scope_proof_survives_native_file_ingestion(self): + title = 'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA' + row, volume = self.example('icml',1991,title) + record, reason = normalize_row(row,{volume['key']:volume},{'ICML'}) + self.assertIsNone(reason) + with tempfile.TemporaryDirectory() as temporary: + root=Path(temporary);path=root/'records.jsonl';path.write_text(json.dumps(record)+'\n') + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','count':1,'list_url':ICML_HISTORY,'file_sha256':hashlib.sha256(path.read_bytes()).hexdigest()})) + ingest_file(root/'test.sqlite',path) + with closing(sqlite3.connect(root/'test.sqlite')) as connection: + self.assertEqual(connection.execute('SELECT track FROM editions').fetchone()[0],'main') + self.assertEqual(connection.execute('SELECT status FROM decisions').fetchone()[0],'accepted') + metadata=json.loads(connection.execute('SELECT metadata_json FROM source_records').fetchone()[0]) + self.assertEqual(metadata['proceedings_title'],title) + self.assertEqual(metadata['track_scope_evidence_url'],ICML_HISTORY) + + +if __name__ == '__main__': + unittest.main() -- 2.52.0