Collect official ICLR programs with presentation identity checks

README.md

@@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工
 
 目标包括：完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。
 
-**目前仍在开发，不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查；完整 ICML 2025 元数据已采集 3,330 条，尚未作为全量交付。arXiv 元数据 ZIP 已完整下载（1,878,831,586 字节），全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集，其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流，实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围，不代表已完成采集，也不是与 COLM 比较的客观排名。
+**目前仍在开发，不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology、ICLR 官方公开程序元数据采集、带清单核验的会议导入和只读查询。PeerRead 提供公开 ICLR 2017 明确决定的补充来源。arXiv 元数据 ZIP 已完整下载（1,878,831,586 字节），全量导入正在服务器运行。其他会议适配器、历史缺失届次、完整投稿历史、跨平台疑似重复复核和覆盖率核验仍待完成。回归检查在远端执行，已配置 ces-control:host 对应的 CI 工作流，实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围，不代表已完成采集，也不是与 COLM 比较的客观排名。
 
 ## 数据范围与来源
 
@@ -156,6 +156,26 @@ python3 -B collect_acl.py \
 
 `acl-collection-status.json` 记录每个 XML 及其 volume 的处理结果和排除原因；`acl-last-run.json` 记录源 commit 与本批范围。`--collection P17` 可限定一个确实存在的 collection。已验证真实 ACL 2017 长文 195 条、短文 107 条，二者摘要完整；其他 track 按源实际缺失情况记录，不能凭样本宣称整个历史完成。
 
+### ICLR 官方公开程序元数据
+
+```sh
+python3 -B collect_iclr.py --cache /data/paper-index/raw/http-cache \
+  --out /data/paper-index/collected
+# 入库后按官方主题检索；主题值以实际来源为准
+python3 -B query.py --db /data/paper-index/papers.sqlite \
+  --venue ICLR --tag 'iclr_program:topic=Reinforcement Learning->Deep RL'
+```
+
+从 ICLR 官方年份导航登记届次，读取对应 Papers 页面明确引用的 orals-posters/abstracts JSON。核对声明数量、分页状态及唯一 event ID；只下载公开程序元数据，不访问用户资料、PDF、海报、幻灯片或视频。`--year` 可限制年份；清单中的已采集数与数据库已入库数分别统计。
+
+官方程序中的 event 数不是论文数：同一篇文章可以有 poster 和 oral 两场展示。程序还会给负 sourceid 的 oral 展示生成占位 OpenReview 链接，这些链接不能作为真实论文标识。采集器核对原投稿编号、标题与作者后合并展示，优先使用正编号记录的真实论文链接；原展示编号和类型保留在源元数据中。
+
+同一投稿编号的展示标题可能有小幅修订。只有负编号 oral 与正编号原稿对应、至少两位作者完全一致且标题相似度满足严格阈值时，才保留为同稿的标题变体；该规则仅在一届官方程序内部生效，不用于跨平台模糊合并。
+
+只将 Conference 来源的论文视为 ICLR 主会录用，排除 Blog 和期刊转会展示，并在审计清单保存排除计数。2020/2021 旧数据允许依据官方主会 paper program 形成录用证据；无 forum ID 时保留 program 身份和匹配候选，不伪造标识。历史年度返回 404 会登记 unavailable，未来未发表届次登记 not_yet_published，其他访问或解析失败登记 blocked；这些状态都不能宣称该年已覆盖。
+
+主题、关键词使用 `iclr_program:topic`、`iclr_program:keyword` 命名空间。通用导入器按源记录刷新标签证据，保留同一论文其他源记录及 `custom:*` 标签。每年的 JSONL、manifest、`iclr-program-year-status.json` 和 `iclr-program-last-run.json` 保存实际范围、独立展示数、主会展示数、论文数、缺失摘要与缺失 forum 数。录用程序不能提供拒稿全集，公开投稿及拒稿补充仍需独立来源。
+
 ## 长期维护与更新
 
 1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发，遵守来源限流、重试与人机验证要求。

collect_iclr.py

@@ -0,0 +1,246 @@
+"""Collect public ICLR program metadata, excluding blog/journal presentations."""
+import argparse
+from collections import Counter, defaultdict
+from datetime import datetime, timezone
+from difflib import SequenceMatcher
+import json
+from pathlib import Path
+import re
+from urllib.error import HTTPError
+from urllib.parse import parse_qs, urljoin, urlsplit
+
+from collect_pmlr import fetch
+from collect_proceedings import Document, clean
+from index_core import author_keys, normalized
+
+ROOT = 'https://iclr.cc/'
+SOURCE = 'iclr_program'
+
+
+class NotYetPublished(ValueError):
+    pass
+
+
+def discover(html):
+    years = {}
+    for node in Document(html).root.walk():
+        text = clean(node.text())
+        if node.tag == 'a' and re.fullmatch(r'20\d{2}', text):
+            url = urljoin(ROOT, node.attrs.get('href', ''))
+            if urlsplit(url).netloc == 'iclr.cc':
+                years[int(text)] = url
+    if not years:
+        raise ValueError('No ICLR years in the official navigation')
+    return sorted(years.items())
+
+
+def asset_urls(html, page_url, year):
+    refs = re.findall(r'''["']([^"'\s]+\.json)["']''', html)
+    result = {}
+    for kind in ('orals-posters', 'abstracts'):
+        matches = {urljoin(page_url, ref) for ref in refs
+                   if urlsplit(urljoin(page_url, ref)).netloc == 'iclr.cc'
+                   and urlsplit(urljoin(page_url, ref)).path == f'/static/virtual/data/iclr-{year}-{kind}.json'}
+        if len(matches) > 1:
+            raise ValueError('Ambiguous official program asset')
+        if matches:
+            result[kind] = matches.pop()
+    if 'orals-posters' not in result:
+        raise ValueError('Official page has no supported program JSON reference')
+    return result
+
+
+def main_event(row, year):
+    source = row.get('sourceurl')
+    if source:
+        parsed = urlsplit(source)
+        return (parsed.netloc == 'openreview.net' and parsed.path == '/group'
+                and parse_qs(parsed.query).get('id') == [f'ICLR.cc/{year}/Conference'])
+    # These historical official paper lists predate sourceurl/decision fields.
+    return year in (2020, 2021) and row.get('eventtype') in ('Poster', 'Oral', 'Spotlight')
+
+
+def source_number(row):
+    value = row.get('sourceid')
+    return int(value) if re.fullmatch(r'-?[1-9]\d*', str(value)) else None
+
+
+def identifiers(row):
+    # The program creates negative-ID oral events with fabricated forum URLs.
+    if (source_number(row) or 0) < 0:
+        return {}
+    links = [row.get('paper_url'), row.get('paper_pdf_url')]
+    links += [media.get('uri') for media in row.get('eventmedia', [])
+              if str(media.get('name') or '').casefold() in ('openreview', 'paper')]
+    forums = set()
+    for link in links:
+        if not isinstance(link, str):
+            continue
+        parsed = urlsplit(link)
+        if parsed.netloc == 'openreview.net' and parsed.path in ('/forum', '/pdf'):
+            values = parse_qs(parsed.query).get('id', [])
+            if len(values) == 1 and re.fullmatch(r'[\w-]+', values[0]) and not re.match(r'20\d{2}-Oral-', values[0]):
+                forums.add(values[0])
+    if len(forums) > 1:
+        raise ValueError('Conflicting paper self-links in one program event')
+    return {'openreview': forums.pop()} if forums else {}
+
+
+def parse_program(data, abstracts, year, index_url):
+    if not isinstance(data, dict) or not isinstance(data.get('results'), list):
+        raise ValueError('Expected a program results array with a declared count')
+    events = data['results']
+    if data.get('next') or data.get('previous') or data.get('count') != len(events):
+        raise ValueError('Program JSON is paginated or its count differs from events')
+    if not isinstance(abstracts, dict):
+        raise ValueError('Expected an abstract mapping keyed by event ID')
+    event_ids = [str(row['id']) for row in events]
+    if len(event_ids) != len(set(event_ids)):
+        raise ValueError('Duplicate event IDs in program JSON')
+    grouped = defaultdict(list)
+    excluded = Counter()
+    for row in events:
+        if not main_event(row, year):
+            excluded[str(row.get('sourceurl') or 'unclassified')] += 1
+            continue
+        decision = row.get('decision')
+        if decision and not (str(decision).casefold().startswith('accept') or decision in ('Poster', 'Oral', 'Spotlight')):
+            raise ValueError('Unexpected decision in an accepted conference program')
+        number = source_number(row)
+        identity = ('submission', abs(number)) if number else ('forum', identifiers(row).get('openreview'))
+        if identity == ('forum', None):
+            identity = ('event', row['id'])
+        grouped[identity].append(row)
+    records = []
+    missing_identity = 0
+    for identity, group in grouped.items():
+        # Verify the negative oral event really describes its positive counterpart.
+        group.sort(key=lambda r: ((source_number(r) or 0) < 0, str(r['id'])))
+        primary = group[0]
+        title = normalized(primary.get('name', ''))
+        authors = [author_keys([a['fullname'] for a in row.get('authors', [])]) for row in group]
+        title_match = all(normalized(row.get('name', '')) == title or (
+            identity[0] == 'submission' and (source_number(row) or 0) < 0
+            and (source_number(primary) or 0) > 0 and len(authors[0]) >= 2
+            and SequenceMatcher(None, title, normalized(row.get('name', ''))).ratio() >= 0.85)
+            for row in group)
+        if not title or not title_match or any(a != authors[0] for a in authors):
+            raise ValueError('Program submission ID collides across different papers')
+        own_ids = {}
+        for row in group:
+            for scheme, value in identifiers(row).items():
+                if scheme in own_ids and own_ids[scheme] != value:
+                    raise ValueError('Conflicting identifiers across presentation events')
+                own_ids[scheme] = value
+        abstract = next((text for row in group
+                         if (text := row.get('abstract') or abstracts.get(str(row['id'])))), None)
+        if abstract is not None and not isinstance(abstract, str):
+            raise ValueError('Nontext abstract')
+        presentation_ids = [row['id'] for row in group]
+        forum = own_ids.get('openreview')
+        external = f'{year}/{forum}' if forum else f'{year}/program-{identity[1]}'
+        missing_identity += not bool(forum)
+        page = urljoin(ROOT, primary.get('virtualsite_url') or f'/virtual/{year}/{str(primary.get("eventtype", "Poster")).lower()}/{primary["id"]}')
+        topics = sorted({str(row['topic']).strip() for row in group if row.get('topic')})
+        keywords = sorted({str(keyword).strip() for row in group for keyword in row.get('keywords', []) if keyword})
+        records.append({'source': SOURCE, 'external_id': external, 'url': page,
+                        'title': clean(primary['name']), 'abstract': abstract.strip() if abstract else None,
+                        'authors': [a['fullname'] for a in primary.get('authors', [])],
+                        'identifiers': own_ids, 'venue': 'ICLR', 'year': year, 'track': 'main',
+                        'status': 'accepted', 'raw_decision': str(primary.get('decision') or 'listed in official conference paper program'),
+                        'evidence_url': index_url, 'list_url': index_url,
+                        'tags': [{'namespace': SOURCE+':'+kind, 'value': value}
+                                 for kind, values in [('topic', topics), ('keyword', keywords)] for value in values],
+                        'metadata': {'program_event_ids': presentation_ids,
+                                     'presentation_titles': sorted({clean(r['name']) for r in group}),
+                                     'program_source_id': identity[1], 'presentation_types': sorted({str(r.get('event_type') or r.get('eventtype')) for r in group}),
+                                     'raw_decisions': sorted({str(r.get('decision')) for r in group if r.get('decision')}),
+                                     'identity_state': 'forum' if forum else 'program_only'}})
+    # Different source numbers can still identify the same forum. Keep one record.
+    by_id = {}
+    for row in records:
+        previous = by_id.get(row['external_id'])
+        if previous:
+            if normalized(previous['title']) != normalized(row['title']) or author_keys(previous['authors']) != author_keys(row['authors']):
+                raise ValueError('One forum identifies different program papers')
+            previous['metadata']['program_event_ids'] += row['metadata']['program_event_ids']
+            previous['metadata']['presentation_types'] = sorted(set(previous['metadata']['presentation_types'] + row['metadata']['presentation_types']))
+            previous['abstract'] = previous['abstract'] or row['abstract']
+        else:
+            by_id[row['external_id']] = row
+    result = list(by_id.values())
+    if not result:
+        raise ValueError('No verified main-conference papers in program')
+    return result, {'program_event_count': len(events), 'main_event_count': sum(map(len, grouped.values())),
+                    'excluded_events_by_source': dict(excluded), 'missing_forum_count': missing_identity,
+                    'missing_abstract_count': sum(not r['abstract'] for r in result)}
+
+
+def collect_year(year, cache, out):
+    page = f'https://iclr.cc/virtual/{year}/papers.html'
+    assets = asset_urls(fetch(page, cache, 16*1024**2, delay=1).decode(), page, year)
+    data = json.loads(fetch(assets['orals-posters'], cache, 32*1024**2, delay=1))
+    if data.get('count') == 0 and data.get('results') == [] and year > datetime.now(timezone.utc).year:
+        raise NotYetPublished('Official program is empty for a future conference')
+    abstracts = {}
+    abstract_asset_status = 'not_referenced'
+    if 'abstracts' in assets:
+        try:
+            abstracts = json.loads(fetch(assets['abstracts'], cache, 32*1024**2, delay=1))
+            abstract_asset_status = 'available'
+        except HTTPError as exc:
+            if exc.code != 404:
+                raise
+            # Historical programs embed abstracts but retain a stale split-file URL.
+            exc.close()
+            abstract_asset_status = 'unavailable_404'
+    records, audit = parse_program(data, abstracts, year, page)
+    path = Path(out)/f'iclr-program-{year}-main.jsonl'
+    part = path.with_suffix('.part')
+    part.write_text(''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in records))
+    part.replace(path)
+    manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICLR', 'year': year, 'track': 'main',
+                'count': len(records), 'list_url': page, 'asset_urls': assets,
+                'abstract_asset_status': abstract_asset_status, 'file': str(path),
+                'status': 'partial' if audit['missing_abstract_count'] else 'collected', **audit}
+    path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2))
+    return manifest
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cache', required=True)
+    parser.add_argument('--out', required=True)
+    parser.add_argument('--year', type=int, action='append')
+    args = parser.parse_args()
+    out = Path(args.out); out.mkdir(parents=True, exist_ok=True)
+    years = discover(fetch(ROOT, args.cache, 4*1024**2, delay=1).decode())
+    if args.year and set(args.year)-{year for year, _ in years}:
+        parser.error('Requested year is absent from the official navigation')
+    status_path = out/'iclr-program-year-status.json'
+    catalogue = json.loads(status_path.read_text()) if status_path.exists() else {}
+    requested = []
+    for year, conference_url in years:
+        if args.year and year not in args.year:
+            continue
+        requested.append(year)
+        try:
+            result = collect_year(year, args.cache, out)
+        except NotYetPublished as exc:
+            result = {'year': year, 'status': 'not_yet_published', 'error': str(exc)}
+        except HTTPError as exc:
+            result = {'year': year, 'status': 'not_yet_published' if year > datetime.now(timezone.utc).year and exc.code == 404 else 'unavailable' if exc.code == 404 else 'blocked', 'error': str(exc)}
+            exc.close()
+        except Exception as exc:
+            result = {'year': year, 'status': 'blocked', 'error': str(exc)}
+        result['conference_url'] = conference_url
+        catalogue[str(year)] = result
+        part = status_path.with_suffix('.part'); part.write_text(json.dumps(catalogue, indent=2)); part.replace(status_path)
+        print(json.dumps(result), flush=True)
+    (out/'iclr-program-last-run.json').write_text(json.dumps({'requested_years': requested, 'year_filter': args.year}, indent=2))
+    if any(catalogue[str(y)]['status'] == 'blocked' for y in requested):
+        raise SystemExit(1)
+
+
+if __name__ == '__main__':
+    main()

import_records.py

@@ -35,6 +35,8 @@ def ingest_file(db,path):
                     raise ValueError('Duplicate external ID in one collected file: '+external)
                 seen.add(external)
                 paper,record = store.record(source_name,external,row['url'],row['title'],row.get('abstract'),row.get('authors',[]),row.get('identifiers'),row.get('metadata'))
+                if 'tags' in row:
+                    store.record_tags(paper,record,source_name,row['tags'])
                 if row.get('status') in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):
                     edition = store.decision(paper,record,row['venue'],row['year'],row['track'],external,row.get('evidence_url',row['url']),row['status'],row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status']),row.get('venue_aliases'))
                     editions.add(edition)

index_core.py

@@ -109,6 +109,22 @@ class Store:
                 c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(paper,other),max(paper,other),'exact_title_unconfirmed',None,json.dumps({'title':title}), 'pending'))
         return paper,record
 
+    def record_tags(self,paper,record,source,tags):
+        """Refresh this record's platform tags without removing other evidence."""
+        values=set()
+        for tag in tags:
+            namespace,value=tag['namespace'],tag['value']
+            if not isinstance(namespace,str) or not namespace.startswith(source+':') or not isinstance(value,str) or not value.strip():
+                raise ValueError('Platform tag must have its own source namespace and a nonempty text value')
+            values.add((namespace,value.strip()))
+        evidence=f'{source}:record:{record}:tags'
+        c=self.conn
+        c.execute('DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key=?',(record,evidence))
+        for namespace,value in sorted(values):
+            c.execute('INSERT OR IGNORE INTO tags(namespace,value) VALUES(?,?)',(namespace,value))
+            tag=c.execute('SELECT id FROM tags WHERE namespace=? AND value=?',(namespace,value)).fetchone()[0]
+            c.execute('INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,evidence))
+
     def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None):
         c = self.conn
         c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified'))

test_iclr.py

@@ -0,0 +1,98 @@
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+from urllib.error import HTTPError
+
+from collect_iclr import asset_urls, collect_year, discover, parse_program
+from import_records import ingest_file
+from index_core import Store
+
+
+class ICLRTests(unittest.TestCase):
+    def event(self,identifier=1,sourceid=10,**updates):
+        row={'id':identifier,'sourceid':sourceid,'sourceurl':'https://openreview.net/group?id=ICLR.cc/2026/Conference',
+             'name':'A reliable paper on abstract metadata indexing','authors':[{'fullname':'Jane Doe'},{'fullname':'John Smith'}],
+             'eventtype':'Poster','event_type':'Poster','decision':'Accept (Oral)',
+             'paper_url':'https://openreview.net/forum?id=realForum','topic':'Learning->Theory','keywords':[]}
+        return dict(row,**updates)
+
+    def parse(self,events,abstracts=None,year=2026):
+        return parse_program({'count':len(events),'next':None,'previous':None,'results':events},abstracts or {},year,f'https://iclr.cc/virtual/{year}/papers.html')
+
+    def test_oral_placeholder_is_not_an_identifier_or_another_submission(self):
+        poster=self.event()
+        oral=self.event(2,-10,eventtype='Oral',event_type='Oral',paper_url='https://openreview.net/forum?id=2026-Oral--10-placeholder')
+        rows,audit=self.parse([oral,poster],{'1':'The real abstract.'})
+        self.assertEqual(len(rows),1)
+        self.assertEqual(rows[0]['identifiers'],{'openreview':'realForum'})
+        self.assertEqual(rows[0]['abstract'],'The real abstract.')
+        self.assertEqual(rows[0]['metadata']['presentation_types'],['Oral','Poster'])
+        self.assertEqual(audit['main_event_count'],2)
+
+    def test_blog_and_journal_presentations_are_not_main_acceptances(self):
+        rows,audit=self.parse([self.event(),self.event(2,20,sourceurl='https://openreview.net/group?id=ICLR.cc/2026/BlogPosts'),self.event(3,30,sourceurl='2026-TMLR-J2C-Spreadsheet')])
+        self.assertEqual(len(rows),1)
+        self.assertEqual(sum(audit['excluded_events_by_source'].values()),2)
+        self.assertEqual(audit['missing_abstract_count'],1)
+        self.assertIsNone(rows[0]['abstract'])
+
+    def test_title_revision_is_scoped_to_matching_signed_submission_and_authors(self):
+        poster=self.event(name='AlphaEdit: Null-Space Constrained Knowledge Editing for Language Models')
+        oral=self.event(2,-10,eventtype='Oral',name='AlphaEdit: Null-Space Constrained Model Editing for Language Models')
+        rows,_=self.parse([oral,poster])
+        self.assertEqual(len(rows),1)
+        self.assertEqual(rows[0]['title'],poster['name'])
+        self.assertEqual(len(rows[0]['metadata']['presentation_titles']),2)
+        with self.assertRaisesRegex(ValueError,'collides'):
+            self.parse([poster,dict(oral,authors=[{'fullname':'Unrelated Author'}])])
+
+    def test_truncation_and_identity_collision_fail(self):
+        with self.assertRaisesRegex(ValueError,'count differs'):
+            parse_program({'count':2,'results':[self.event()]},{},2026,'https://iclr.cc/')
+        with self.assertRaisesRegex(ValueError,'collides'):
+            self.parse([self.event(),self.event(2,-10,name='A different paper')])
+        with self.assertRaisesRegex(ValueError,'Conflicting identifiers'):
+            self.parse([self.event(),self.event(2,10,paper_url='https://openreview.net/forum?id=otherForum')])
+
+    def test_legacy_embedded_abstracts_and_paper_links(self):
+        row=self.event(sourceurl=None,abstract='Historical abstract',paper_url=None,paper_pdf_url='https://openreview.net/forum?id=historicalForum',decision='Poster')
+        rows,_=self.parse([row],year=2021)
+        self.assertEqual(rows[0]['abstract'],'Historical abstract')
+        self.assertEqual(rows[0]['identifiers'],{'openreview':'historicalForum'})
+        rows,_=self.parse([dict(row,paper_pdf_url=None)],year=2020)
+        self.assertEqual(rows[0]['metadata']['identity_state'],'program_only')
+        self.assertEqual(rows[0]['identifiers'],{})
+
+    def test_only_referenced_official_assets_and_real_navigation_years(self):
+        html='<a href="/archive/2013/">2013</a><a href="/Conferences/2026">2026</a><a href="https://other.example/2025">2025</a>'
+        self.assertEqual([y for y,_ in discover(html)],[2013,2026])
+        html='start("/static/virtual/data/iclr-2026-orals-posters.json",true,"https://other.example/abstracts.json")'
+        self.assertEqual(asset_urls(html,'https://iclr.cc/virtual/2026/papers.html',2026),{'orals-posters':'https://iclr.cc/static/virtual/data/iclr-2026-orals-posters.json'})
+
+    def test_missing_split_abstract_asset_does_not_discard_embedded_abstract(self):
+        html=b'start("/static/virtual/data/iclr-2021-orals-posters.json",true,"/static/virtual/data/iclr-2021-abstracts.json")'
+        event=self.event(sourceurl=None,decision='Poster',abstract='Embedded historical abstract')
+        responses=[html,json.dumps({'count':1,'results':[event]}).encode(),HTTPError('https://iclr.cc/abstracts',404,'Not Found',None,None)]
+        with tempfile.TemporaryDirectory() as tmp,patch('collect_iclr.fetch',side_effect=responses):
+            manifest=collect_year(2021,tmp,tmp)
+            self.assertEqual(manifest['status'],'collected')
+            self.assertEqual(manifest['missing_abstract_count'],0)
+            self.assertEqual(manifest['abstract_asset_status'],'unavailable_404')
+
+    def test_import_repeat_preserves_one_submission_and_queryable_platform_tags(self):
+        rows,_=self.parse([self.event(),self.event(2,-10,eventtype='Oral')],{'1':'A searchable abstract.'})
+        with tempfile.TemporaryDirectory() as tmp:
+            path=Path(tmp)/'iclr.jsonl';db=Path(tmp)/'papers.sqlite'
+            path.write_text(''.join(json.dumps(r)+'\n' for r in rows))
+            path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','list_url':'https://iclr.cc/virtual/2026/papers.html','count':1,'status':'collected'}))
+            ingest_file(db,path);ingest_file(db,path)
+            store=Store(db);self.addCleanup(store.conn.close)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
+            self.assertEqual(store.conn.execute('SELECT count(*) FROM decisions').fetchone()[0],1)
+            self.assertEqual(store.conn.execute('SELECT namespace,value FROM tags').fetchall(),[('iclr_program:topic','Learning->Theory')])
+            self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(),[])
+
+
+if __name__=='__main__':unittest.main()

test_index_core.py

@@ -58,6 +58,19 @@ class StoreTests(unittest.TestCase):
         self.assertEqual(self.store.conn.execute('SELECT title FROM papers').fetchone()[0],'Updated research title')
         self.assertEqual(self.store.conn.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'searchable'").fetchone()[0],1)
 
+    def test_tag_refresh_keeps_other_record_evidence_and_custom_tags(self):
+        paper,first=self.record('iclr_program','2025/a',{'openreview':'sharedForum'})
+        same,second=self.record('iclr_program','2025/b',{'openreview':'sharedForum'})
+        self.store.record_tags(paper,first,'iclr_program',[{'namespace':'iclr_program:topic','value':'Shared'}])
+        self.store.record_tags(same,second,'iclr_program',[{'namespace':'iclr_program:topic','value':'Shared'}])
+        tag=self.store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','Manual')").lastrowid
+        self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)',(paper,tag,None,'manual'))
+        self.store.record_tags(paper,first,'iclr_program',[])
+        self.assertEqual(self.store.conn.execute('SELECT source_record_id FROM paper_tags ORDER BY source_record_id').fetchall(),[(None,),(second,)])
+        with self.assertRaisesRegex(ValueError,'namespace'):
+            self.store.record_tags(paper,second,'iclr_program',[{'namespace':'custom:topic','value':'No overwrite'}])
+        self.assertEqual(self.store.conn.execute('SELECT count(*) FROM paper_tags').fetchone()[0],2)
+
     def test_missing_abstract_does_not_become_complete_coverage(self):
         path=Path(self.tmp.name)/'partial.jsonl'
         path.write_text(json.dumps({'source':'colm','external_id':'paper1','url':'https://example.org/paper1','title':'Paper','abstract':None,'authors':[],'venue':'COLM','year':2024,'track':'main','status':'accepted'})+'\n')

venues.json

@@ -11,7 +11,8 @@
     "aaai": "https://ojs.aaai.org/index.php/AAAI/issue/archive",
     "ijcai": "https://www.ijcai.org/all_proceedings/",
     "dblp": "https://dblp.org/db/conf/",
-    "colm": "https://proceedings.colm.cc/"
+    "colm": "https://proceedings.colm.cc/",
+    "iclr_program": "https://iclr.cc/virtual/2026/papers.html"
   },
   "venues": [
     {
@@ -20,7 +21,6 @@
       "venue_type": "conference",
       "discovery_sources": [
         "openreview",
-        "pmlr",
         "neurips"
       ],
       "scope": "candidate_pending_scope_review",
@@ -36,8 +36,7 @@
       "venue_type": "conference",
       "discovery_sources": [
         "openreview",
-        "pmlr",
-        "neurips"
+        "pmlr"
       ],
       "scope": "candidate_pending_scope_review",
       "years": "all_publicly_available",
@@ -52,8 +51,7 @@
       "venue_type": "conference",
       "discovery_sources": [
         "openreview",
-        "pmlr",
-        "neurips"
+        "iclr_program"
       ],
       "scope": "candidate_pending_scope_review",
       "years": "all_publicly_available",
@@ -68,8 +66,7 @@
       "venue_type": "conference",
       "discovery_sources": [
         "openreview",
-        "pmlr",
-        "neurips"
+        "colm"
       ],
       "scope": "candidate_pending_scope_review",
       "years": "all_publicly_available",