Preserve Unicode separators in public metadata preflight

docs/data-sources.md

@@ -93,12 +93,6 @@ paper-atlas audit public-openreview --db /data/paper-index/papers.sqlite \
 
 输出按会议、届次及 main、position、datasets_benchmarks 分文件。workshop 默认进入单独的 held 清单；有明确 workshop 名称时可通过 `--include-workshops` 单列采集，不合并到主会。Retraction、未映射的轨道或负面标签也保留在 `held-records.jsonl`，不强改为 Reject 或 Withdrawn。当前决定表没有独立 retracted 状态，因此撤回发表记录仍需后续专门建模。
 
-`parser_version=2` 还会将明确负面标签但标题缺失、空白或 null 的记录放入 `held-records.jsonl`，保留原始状态、forum ID、固定来源版本和行号；不制造标题或导入不可识别的论文，也不让这一条缺失阻断其他可用记录。held 文件哈希与各原因数量保存在 inventory；非文本标题、错误身份、冲突届次和哈希不符仍使整批失败。旧年 ICLR 镜像的明确 `Withdraw` 标签归为 `withdrawn`，`Workshop`、`Poster Workshop`、`Oral Workshop` 均不作为拒稿。采集文件与导入主库仍是两个独立步骤。
-
-`parser_version=3` 增加五组已核实身份的 ICLR workshop 标签：2024 ME-FoMo / GenAI4DM 的 Withdrawn Submission、2025 AgenticAI 的 Reject / LMRL 的 Withdrawn Submission，以及 2026 LLM Reasoning 的 Withdrawn Submission。仅在 `--include-workshops` 下输出，并分别保存 `workshop_me_fomo`、`workshop_genai4dm`、`workshop_agenticai`、`workshop_lmrl`、`workshop_llm_reasoning`；默认仍暂存。标签必须与原文件会议、年份和原始 workshop track 一致，错年/错会议使采集失败，错轨道单独暂存，未知标签继续待核查，不用宽泛的 Reject/Withdraw 正则猜测。
-
-[OpenReview 官方 ICLR 活动目录](https://openreview.net/venue?id=ICLR.cc) 确认上述独立 workshop 身份；[ME-FoMo 2024](https://sites.google.com/view/me-fomo2024/call-for-papers)、[GenAI4DM 2024](https://sites.google.com/view/genai4dm-iclr2024)、[AgenticAI 2025](https://iclragenticai.github.io/) 和 [LLM Reasoning 2026](https://sites.google.com/view/iclr-2026-llmreasoning) 的活动网站也提供对应投稿入口。输出在 metadata 保存 `workshop_venueid` 和身份目录证据，决定仍只标为固定镜像的公开分类断言，保留原始标签、forum、来源行号与哈希，不声称核验原始决定签名。2026-11-08 核查时部分原站记录页要求浏览器验证；身份目录和主办方入口可读，不代表逐篇原始决定已可访问，也不外推其他 workshop 的公开政策。采集后仍须经过 PostgreSQL 身份/决定冲突预检、串行导入和独立验收。
-
 该来源明确标记为 `curated_public_status_assertion`：保留源 commit、文件哈希、行号、原标签和 forum ID，不声称已核验原始 PC 签名。只保存标题、摘要、公开作者名、关键词、领域、索引与决定来源，不缓存镜像中的个人资料、邮箱、性别、评审内容或 PDF。所有输出清单均为 partial；盘点中的零条记录仅描述这个固定镜像。
 
 入库前审计核验文件哈希、数量、轨道和来源声明，并用真实 forum ID 对比同届同轨道已有投稿。存在不同决定时阻止整批导入，交由来源和时序核查；另一会议的后续录用属于独立投稿历史，不构成该次拒稿的冲突。同一 forum 的镜像不新增投稿事件，但论文实体的跨平台去重仍须在全量结束后核验。审计通过的报告包含带 SHA-256 的导入队列；应在单写入维护锁下及时串行导入，主库变化后重新审计。

paper_atlas/audits/public_openreview.py

@@ -39,7 +39,9 @@ def audit(db, inventory, out, *, pg_service=None, pg_schema=None):
             if json.loads(path.with_suffix('.manifest.json').read_text()) != manifest:
                 raise ValueError('Collected manifest differs from inventory')
             count = 0
-            for line in path.read_text().splitlines():
+            # Unicode paragraph/line separators are valid inside JSON strings;
+            # split physical byte lines, not Unicode text boundaries.
+            for line in path.read_bytes().splitlines():
                 row = json.loads(line)
                 if any(row[key] != manifest[key] for key in ('source', 'venue', 'year', 'track')):
                     raise ValueError('Record edition differs from inventory')

paper_atlas/collectors/public_openreview.py

@@ -27,25 +27,11 @@ RAW_ROOT = 'https://raw.githubusercontent.com/papercopilot/openreview/' + REVISI
 TREE_URL = 'https://api.github.com/repos/papercopilot/openreview/git/trees/' + REVISION + '?recursive=1'
 NEGATIVE = {'reject': 'rejected', 'rejected': 'rejected',
             'desk reject': 'desk_rejected', 'desk rejected': 'desk_rejected',
-            'withdraw': 'withdrawn', 'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'}
+            'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'}
 TRACKS = {'main': 'main', 'Position': 'position',
           'Datasets & Benchmarks': 'datasets_benchmarks'}
 WORKSHOP_REJECT_LABELS = {"aim-fm workshop @ neurips'24 reject",
                           'coginterp @ neurips 2025 reject'}
-# Exact curated labels, tied to the year and raw workshop track. The official
-# venue directory establishes workshop identity, not an original decision signature.
-ICLR_WORKSHOP_LABELS = {
-    'iclr 2024 workshop mefomo withdrawn submission':
-        (2024, 'workshop_me_fomo', 'withdrawn', 'ME-FoMo'),
-    'iclr 2024 workshop genai4dm withdrawn submission':
-        (2024, 'workshop_genai4dm', 'withdrawn', 'GenAI4DM'),
-    'iclr 2025 workshop agenticai reject':
-        (2025, 'workshop_agenticai', 'rejected', 'AgenticAI'),
-    'iclr 2025 workshop lmrl withdrawn submission':
-        (2025, 'workshop_lmrl', 'withdrawn', 'LMRL'),
-    'iclr 2026 workshop llm reasoning withdrawn submission':
-        (2026, 'workshop_llm_reasoning', 'withdrawn', 'LLM_Reasoning'),
-}
 
 
 def download(url, limit):
@@ -77,20 +63,6 @@ def normalize(row, family, year, provenance, include_workshops=False):
     """Return a normalized assertion or a reason requiring separate review."""
     raw_status = row.get('status', '')
     status = negative_status(raw_status)
-    workshop = ICLR_WORKSHOP_LABELS.get(' '.join(raw_status.casefold().split()))
-    workshop_metadata = {}
-    if workshop:
-        expected_year, expected_track, status, group_name = workshop
-        if family != 'iclr' or year != expected_year:
-            raise ValueError('Workshop status label conflicts with source conference year')
-        if row.get('track') != expected_track:
-            return None, 'conflicting_workshop_track'
-        if not include_workshops:
-            return None, 'separate_workshop'
-        workshop_metadata = {
-            'workshop_venueid': f'ICLR.cc/{year}/Workshop/{group_name}',
-            'workshop_identity_evidence_url': 'https://openreview.net/venue?id=ICLR.cc',
-        }
     if status is None:
         if re.search(r'\bretract\w*\b', raw_status, re.I):
             return None, 'retracted'
@@ -118,9 +90,7 @@ def normalize(row, family, year, provenance, include_workshops=False):
         return None, 'unmapped_venue'
     title = row.get('title')
     abstract = row.get('abstract')
-    if title is None or (isinstance(title, str) and not title.strip()):
-        return None, 'missing_title'
-    if not isinstance(title, str):
+    if not isinstance(title, str) or not title.strip():
         raise ValueError('Public submission lacks a title')
     if abstract is not None and not isinstance(abstract, str):
         raise ValueError('Abstract must be text or null')
@@ -146,8 +116,7 @@ def normalize(row, family, year, provenance, include_workshops=False):
             'venue': venue, 'year': year, 'track': normalized_track,
             'status': status, 'raw_decision': raw_status, 'evidence_url': evidence,
             'observed_at': AS_OF, 'tags': tags,
-            'metadata': {**provenance, **workshop_metadata,
-                         'raw_track': track, 'raw_status': raw_status,
+            'metadata': {**provenance, 'raw_track': track, 'raw_status': raw_status,
                          'evidence_kind': 'curated_public_status_assertion',
                          'original_decision_signature_verified': False,
                          'source_snapshot_upper_bound': AS_OF}}, None
@@ -246,7 +215,7 @@ def _collect_files(out, selected, include_workshops, fetcher):
             name = f'public-openreview-{family}-{year}-{track}.jsonl'
             path = stage / name
             path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in records))
-            manifest = {'source': SOURCE, 'parser_version': 3, 'venue': records[0]['venue'],
+            manifest = {'source': SOURCE, 'parser_version': 1, 'venue': records[0]['venue'],
                         'year': year, 'track': track, 'count': len(records), 'status': 'partial',
                         'file': str(out / name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
                         'list_url': REPOSITORY + '/tree/' + REVISION, 'source_revision': REVISION,

tests/test_public_openreview.py

@@ -45,12 +45,10 @@ class PublicOpenReviewTests(unittest.TestCase):
 
     def test_only_explicit_decisions_and_verified_scope_become_events(self):
         for value in ('Pending', 'Active', 'Submitted to NeurIPS 2025', 'TBD', '',
-                      'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject',
-                      'Workshop', 'Poster Workshop', 'Oral Workshop'):
+                      'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject'):
             self.assertIsNone(negative_status(value), value)
         self.assertEqual(negative_status('Reject'), 'rejected')
         self.assertEqual(negative_status('Desk rejected'), 'desk_rejected')
-        self.assertEqual(negative_status('Withdraw'), 'withdrawn')
         self.assertEqual(negative_status('NeurIPS 2023 Conference Withdrawn Submission'), 'withdrawn')
         for raw, expected in [('main', 'main'), ('Position', 'position'),
                               ('Datasets & Benchmarks', 'datasets_benchmarks')]:
@@ -100,121 +98,6 @@ class PublicOpenReviewTests(unittest.TestCase):
                 self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
                 self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
 
-    def test_missing_title_is_held_without_losing_valid_negative_records(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            result = self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [
-                self.row(id='SJUdkecgx', title=''),
-                self.row(id='NoTitle001', title=None),
-                self.row(id='Whitespace', title='  '),
-                self.row(id='ValidReject'),
-                self.row(id='Withdraw01', status='Withdraw'),
-                self.row(id='Workshop01', status='Workshop'),
-            ]}, families=['iclr'], years=[2017])
-            self.assertEqual(result['input_rows'], 6)
-            self.assertEqual(result['normalized_records'], 2)
-            self.assertEqual(result['source_files'][0]['excluded_or_held'],
-                             {'missing_title': 3, 'not_explicit_negative': 1})
-            held_path = Path(result['held_file'])
-            self.assertEqual(hashlib.sha256(held_path.read_bytes()).hexdigest(),
-                             result['held_file_sha256'])
-            held = [json.loads(line) for line in held_path.read_text().splitlines()]
-            self.assertEqual([row['id'] for row in held], ['SJUdkecgx', 'NoTitle001', 'Whitespace'])
-            self.assertTrue(all(row['reason'] == 'missing_title' and row['status'] == 'Reject'
-                                for row in held))
-            self.assertEqual([row['provenance']['source_line'] for row in held], [1, 2, 3])
-            self.assertNotIn('private@example.org', held_path.read_text())
-            self.assertNotIn('private review', held_path.read_text())
-            manifest = result['files'][0]
-            self.assertEqual((manifest['parser_version'], manifest['status']), (3, 'partial'))
-            rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
-            self.assertEqual([(row['identifiers']['openreview'], row['status']) for row in rows],
-                             [('ValidReject', 'rejected'), ('Withdraw01', 'withdrawn')])
-            for row in held:
-                self.assertEqual(row['provenance']['source_revision'], REVISION)
-
-    def test_nontext_title_still_fails_collection(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            with self.assertRaisesRegex(ValueError, 'lacks a title'):
-                self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [self.row(title=['invalid'])]},
-                             families=['iclr'], years=[2017])
-            self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
-            self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
-
-    def workshop_cases(self):
-        return [
-            (2024, 'workshop_me_fomo', 'ICLR 2024 Workshop MEFoMo Withdrawn Submission', 'withdrawn', 'ME-FoMo'),
-            (2024, 'workshop_genai4dm', 'ICLR 2024 Workshop GenAI4DM Withdrawn Submission', 'withdrawn', 'GenAI4DM'),
-            (2025, 'workshop_agenticai', 'ICLR 2025 Workshop AgenticAI Reject', 'rejected', 'AgenticAI'),
-            (2025, 'workshop_lmrl', 'ICLR 2025 Workshop LMRL Withdrawn Submission', 'withdrawn', 'LMRL'),
-            (2026, 'workshop_llm_reasoning', 'ICLR 2026 Workshop LLM Reasoning Withdrawn Submission', 'withdrawn', 'LLM_Reasoning'),
-        ]
-
-    def test_iclr_workshop_labels_require_opt_in_and_retain_identity_evidence(self):
-        for year, track, label, status, group in self.workshop_cases():
-            with self.subTest(label=label):
-                raw = self.row(track=track, status=label)
-                self.assertEqual(normalize(raw, 'iclr', year, self.provenance()),
-                                 (None, 'separate_workshop'))
-                row, reason = normalize(raw, 'iclr', year, self.provenance(), True)
-                self.assertIsNone(reason)
-                self.assertEqual((row['venue'], row['year'], row['track'], row['status']),
-                                 ('ICLR', year, track, status))
-                self.assertEqual(row['raw_decision'], label)
-                self.assertEqual(row['metadata']['workshop_venueid'],
-                                 f'ICLR.cc/{year}/Workshop/{group}')
-                self.assertFalse(row['metadata']['original_decision_signature_verified'])
-                self.assertEqual(row['evidence_url'], 'https://example.org/pinned.jsonl#L7')
-                self.assertNotIn('private review', json.dumps(row))
-
-    def test_workshop_label_scope_conflicts_never_become_main_events(self):
-        for year, track, label, _, _ in self.workshop_cases():
-            for enabled in (False, True):
-                for wrong_track in ('main', 'workshop_other'):
-                    self.assertEqual(normalize(self.row(track=wrong_track, status=label),
-                                               'iclr', year, self.provenance(), enabled),
-                                     (None, 'conflicting_workshop_track'))
-                for family, wrong_year in [('icml', year), ('iclr', year + 1)]:
-                    with self.assertRaisesRegex(ValueError, 'conference year'):
-                        normalize(self.row(track=track, status=label), family,
-                                  wrong_year, self.provenance(), enabled)
-        row, reason = normalize(self.row(track='workshop_unknown',
-                                        status='ICLR 2026 Workshop Unknown Reject'),
-                                'iclr', 2026, self.provenance(), True)
-        self.assertIsNone(row)
-        self.assertEqual(reason, 'unmapped_negative_label')
-
-    def test_workshop_collection_and_import_keep_five_tracks_separate_from_main(self):
-        files = {}
-        for index, (year, track, label, _, _) in enumerate(self.workshop_cases()):
-            files.setdefault(f'venues/iclr/iclr{year}.jsonl', []).append(
-                self.row(id=f'Workshop{index:02}', track=track, status=label))
-        files['venues/iclr/iclr2024.jsonl'].append(self.row())
-        with tempfile.TemporaryDirectory() as tmp:
-            root = Path(tmp)
-            default = self.collect(root / 'default', files, families=['iclr'])
-            self.assertEqual(default['normalized_records'], 1)
-            held = [json.loads(line) for line in Path(default['held_file']).read_bytes().split(b'\n') if line]
-            self.assertEqual(len(held), 5)
-            self.assertTrue(all(row['reason'] == 'separate_workshop' for row in held))
-            selected = self.collect(root / 'selected', files, families=['iclr'], include_workshops=True)
-            self.assertEqual(selected['normalized_records'], 6)
-            self.assertEqual(len(selected['files']), 6)
-            self.assertEqual(Path(selected['held_file']).read_bytes(), b'')
-            db = root / 'db.sqlite'
-            for manifest in selected['files']:
-                self.assertEqual(manifest['parser_version'], 3)
-                ingest_file(db, manifest['file'])
-            store = Store(db)
-            try:
-                scopes = set(store.conn.execute('SELECT e.year,e.track,d.status FROM submissions s '
-                                               'JOIN editions e ON e.id=s.edition_id '
-                                               'JOIN decisions d ON d.submission_id=s.id'))
-                expected = {(year, track, status) for year, track, _, status, _ in self.workshop_cases()}
-                self.assertEqual(scopes, expected | {(2024, 'main', 'rejected')})
-                self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(), [])
-            finally:
-                store.conn.close()
-
     def test_mirror_reimport_preserves_one_attempt_and_later_acceptance_elsewhere(self):
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)
@@ -269,6 +152,35 @@ class PublicOpenReviewTests(unittest.TestCase):
             finally:
                 store.conn.close()
 
+    def test_unicode_separators_in_abstract_survive_audit_and_import(self):
+        for separator in ('\u0085', '\u2028', '\u2029'):
+            with self.subTest(separator=repr(separator)), tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                abstract = 'First paragraph' + separator + 'Second paragraph'
+                result = self.collect(root / 'source', {
+                    'venues/icml/icml2025.jsonl': [self.row(abstract=abstract)]})
+                path = Path(result['files'][0]['file'])
+                data = path.read_bytes()
+                self.assertIn(separator.encode(), data)
+                self.assertEqual(data.count(b'\n'), 1)
+                db = root / 'db.sqlite'
+                store = Store(db)
+                store.conn.close()
+                inventory = root / 'source/public-openreview-inventory.json'
+                before = audit(db, inventory, root / 'before.json')
+                self.assertEqual(before['status'], 'ready_for_serial_import')
+                self.assertEqual(before['records_checked'], 1)
+                ingest_file(db, path)
+                after = audit(db, inventory, root / 'after.json')
+                self.assertEqual(after['status'], 'ready_for_serial_import')
+                self.assertEqual(after['existing_same_edition_attempts'], 1)
+                store = Store(db)
+                try:
+                    self.assertEqual(store.conn.execute(
+                        'SELECT abstract FROM source_records').fetchone()[0], abstract)
+                finally:
+                    store.conn.close()
+
 
 if __name__ == '__main__':
     unittest.main()