Hold untitled public assertions and retain explicit withdrawals

Reviewed-on: #16

docs/data-sources.md

@@ -93,6 +93,8 @@ paper-atlas audit public-openreview --db /data/paper-index/papers.sqlite \
 
 输出按会议、届次及 main、position、datasets_benchmarks 分文件。workshop 默认进入单独的 held 清单；有明确 workshop 名称时可通过 `--include-workshops` 单列采集，不合并到主会。Retraction、未映射的轨道或负面标签也保留在 `held-records.jsonl`，不强改为 Reject 或 Withdrawn。当前决定表没有独立 retracted 状态，因此撤回发表记录仍需后续专门建模。
 
+`parser_version=2` 还会将明确负面标签但标题缺失、空白或 null 的记录放入 `held-records.jsonl`，保留原始状态、forum ID、固定来源版本和行号；不制造标题或导入不可识别的论文，也不让这一条缺失阻断其他可用记录。held 文件哈希与各原因数量保存在 inventory；非文本标题、错误身份、冲突届次和哈希不符仍使整批失败。旧年 ICLR 镜像的明确 `Withdraw` 标签归为 `withdrawn`，`Workshop`、`Poster Workshop`、`Oral Workshop` 均不作为拒稿。采集文件与导入主库仍是两个独立步骤。
+
 该来源明确标记为 `curated_public_status_assertion`：保留源 commit、文件哈希、行号、原标签和 forum ID，不声称已核验原始 PC 签名。只保存标题、摘要、公开作者名、关键词、领域、索引与决定来源，不缓存镜像中的个人资料、邮箱、性别、评审内容或 PDF。所有输出清单均为 partial；盘点中的零条记录仅描述这个固定镜像。
 
 入库前审计核验文件哈希、数量、轨道和来源声明，并用真实 forum ID 对比同届同轨道已有投稿。存在不同决定时阻止整批导入，交由来源和时序核查；另一会议的后续录用属于独立投稿历史，不构成该次拒稿的冲突。同一 forum 的镜像不新增投稿事件，但论文实体的跨平台去重仍须在全量结束后核验。审计通过的报告包含带 SHA-256 的导入队列；应在单写入维护锁下及时串行导入，主库变化后重新审计。

paper_atlas/collectors/public_openreview.py

@@ -27,7 +27,7 @@ RAW_ROOT = 'https://raw.githubusercontent.com/papercopilot/openreview/' + REVISI
 TREE_URL = 'https://api.github.com/repos/papercopilot/openreview/git/trees/' + REVISION + '?recursive=1'
 NEGATIVE = {'reject': 'rejected', 'rejected': 'rejected',
             'desk reject': 'desk_rejected', 'desk rejected': 'desk_rejected',
-            'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'}
+            'withdraw': 'withdrawn', 'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'}
 TRACKS = {'main': 'main', 'Position': 'position',
           'Datasets & Benchmarks': 'datasets_benchmarks'}
 WORKSHOP_REJECT_LABELS = {"aim-fm workshop @ neurips'24 reject",
@@ -90,7 +90,9 @@ def normalize(row, family, year, provenance, include_workshops=False):
         return None, 'unmapped_venue'
     title = row.get('title')
     abstract = row.get('abstract')
-    if not isinstance(title, str) or not title.strip():
+    if title is None or (isinstance(title, str) and not title.strip()):
+        return None, 'missing_title'
+    if not isinstance(title, str):
         raise ValueError('Public submission lacks a title')
     if abstract is not None and not isinstance(abstract, str):
         raise ValueError('Abstract must be text or null')
@@ -215,7 +217,7 @@ def _collect_files(out, selected, include_workshops, fetcher):
             name = f'public-openreview-{family}-{year}-{track}.jsonl'
             path = stage / name
             path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in records))
-            manifest = {'source': SOURCE, 'parser_version': 1, 'venue': records[0]['venue'],
+            manifest = {'source': SOURCE, 'parser_version': 2, 'venue': records[0]['venue'],
                         'year': year, 'track': track, 'count': len(records), 'status': 'partial',
                         'file': str(out / name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
                         'list_url': REPOSITORY + '/tree/' + REVISION, 'source_revision': REVISION,

tests/test_public_openreview.py

@@ -45,10 +45,12 @@ class PublicOpenReviewTests(unittest.TestCase):
 
     def test_only_explicit_decisions_and_verified_scope_become_events(self):
         for value in ('Pending', 'Active', 'Submitted to NeurIPS 2025', 'TBD', '',
-                      'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject'):
+                      'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject',
+                      'Workshop', 'Poster Workshop', 'Oral Workshop'):
             self.assertIsNone(negative_status(value), value)
         self.assertEqual(negative_status('Reject'), 'rejected')
         self.assertEqual(negative_status('Desk rejected'), 'desk_rejected')
+        self.assertEqual(negative_status('Withdraw'), 'withdrawn')
         self.assertEqual(negative_status('NeurIPS 2023 Conference Withdrawn Submission'), 'withdrawn')
         for raw, expected in [('main', 'main'), ('Position', 'position'),
                               ('Datasets & Benchmarks', 'datasets_benchmarks')]:
@@ -98,6 +100,46 @@ class PublicOpenReviewTests(unittest.TestCase):
                 self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
                 self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
 
+    def test_missing_title_is_held_without_losing_valid_negative_records(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            result = self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [
+                self.row(id='SJUdkecgx', title=''),
+                self.row(id='NoTitle001', title=None),
+                self.row(id='Whitespace', title='  '),
+                self.row(id='ValidReject'),
+                self.row(id='Withdraw01', status='Withdraw'),
+                self.row(id='Workshop01', status='Workshop'),
+            ]}, families=['iclr'], years=[2017])
+            self.assertEqual(result['input_rows'], 6)
+            self.assertEqual(result['normalized_records'], 2)
+            self.assertEqual(result['source_files'][0]['excluded_or_held'],
+                             {'missing_title': 3, 'not_explicit_negative': 1})
+            held_path = Path(result['held_file'])
+            self.assertEqual(hashlib.sha256(held_path.read_bytes()).hexdigest(),
+                             result['held_file_sha256'])
+            held = [json.loads(line) for line in held_path.read_text().splitlines()]
+            self.assertEqual([row['id'] for row in held], ['SJUdkecgx', 'NoTitle001', 'Whitespace'])
+            self.assertTrue(all(row['reason'] == 'missing_title' and row['status'] == 'Reject'
+                                for row in held))
+            self.assertEqual([row['provenance']['source_line'] for row in held], [1, 2, 3])
+            self.assertNotIn('private@example.org', held_path.read_text())
+            self.assertNotIn('private review', held_path.read_text())
+            manifest = result['files'][0]
+            self.assertEqual((manifest['parser_version'], manifest['status']), (2, 'partial'))
+            rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
+            self.assertEqual([(row['identifiers']['openreview'], row['status']) for row in rows],
+                             [('ValidReject', 'rejected'), ('Withdraw01', 'withdrawn')])
+            for row in held:
+                self.assertEqual(row['provenance']['source_revision'], REVISION)
+
+    def test_nontext_title_still_fails_collection(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            with self.assertRaisesRegex(ValueError, 'lacks a title'):
+                self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [self.row(title=['invalid'])]},
+                             families=['iclr'], years=[2017])
+            self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
+            self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
+
     def test_mirror_reimport_preserves_one_attempt_and_later_acceptance_elsewhere(self):
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)