diff --git a/README.md b/README.md
index 861582c..ab57d1c 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工
目标包括:完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。
-**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集,其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
+**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology、ICLR 官方公开程序元数据采集、带清单核验的会议导入和只读查询。PeerRead 提供公开 ICLR 2017 明确决定的补充来源。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。其他会议适配器、历史缺失届次、完整投稿历史、跨平台疑似重复复核和覆盖率核验仍待完成。回归检查在远端执行,已配置 ces-control:host 对应的 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。
## 数据范围与来源
@@ -156,6 +156,26 @@ python3 -B collect_acl.py \
`acl-collection-status.json` 记录每个 XML 及其 volume 的处理结果和排除原因;`acl-last-run.json` 记录源 commit 与本批范围。`--collection P17` 可限定一个确实存在的 collection。已验证真实 ACL 2017 长文 195 条、短文 107 条,二者摘要完整;其他 track 按源实际缺失情况记录,不能凭样本宣称整个历史完成。
+### ICLR 官方公开程序元数据
+
+```sh
+python3 -B collect_iclr.py --cache /data/paper-index/raw/http-cache \
+ --out /data/paper-index/collected
+# 入库后按官方主题检索;主题值以实际来源为准
+python3 -B query.py --db /data/paper-index/papers.sqlite \
+ --venue ICLR --tag 'iclr_program:topic=Reinforcement Learning->Deep RL'
+```
+
+从 ICLR 官方年份导航登记届次,读取对应 Papers 页面明确引用的 orals-posters/abstracts JSON。核对声明数量、分页状态及唯一 event ID;只下载公开程序元数据,不访问用户资料、PDF、海报、幻灯片或视频。`--year` 可限制年份;清单中的已采集数与数据库已入库数分别统计。
+
+官方程序中的 event 数不是论文数:同一篇文章可以有 poster 和 oral 两场展示。程序还会给负 sourceid 的 oral 展示生成占位 OpenReview 链接,这些链接不能作为真实论文标识。采集器核对原投稿编号、标题与作者后合并展示,优先使用正编号记录的真实论文链接;原展示编号和类型保留在源元数据中。
+
+同一投稿编号的展示标题可能有小幅修订。只有负编号 oral 与正编号原稿对应、至少两位作者完全一致且标题相似度满足严格阈值时,才保留为同稿的标题变体;该规则仅在一届官方程序内部生效,不用于跨平台模糊合并。
+
+只将 Conference 来源的论文视为 ICLR 主会录用,排除 Blog 和期刊转会展示,并在审计清单保存排除计数。2020/2021 旧数据允许依据官方主会 paper program 形成录用证据;无 forum ID 时保留 program 身份和匹配候选,不伪造标识。历史年度返回 404 会登记 unavailable,未来未发表届次登记 not_yet_published,其他访问或解析失败登记 blocked;这些状态都不能宣称该年已覆盖。
+
+主题、关键词使用 `iclr_program:topic`、`iclr_program:keyword` 命名空间。通用导入器按源记录刷新标签证据,保留同一论文其他源记录及 `custom:*` 标签。每年的 JSONL、manifest、`iclr-program-year-status.json` 和 `iclr-program-last-run.json` 保存实际范围、独立展示数、主会展示数、论文数、缺失摘要与缺失 forum 数。录用程序不能提供拒稿全集,公开投稿及拒稿补充仍需独立来源。
+
## 长期维护与更新
1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发,遵守来源限流、重试与人机验证要求。
diff --git a/collect_iclr.py b/collect_iclr.py
new file mode 100644
index 0000000..ef0978f
--- /dev/null
+++ b/collect_iclr.py
@@ -0,0 +1,246 @@
+"""Collect public ICLR program metadata, excluding blog/journal presentations."""
+import argparse
+from collections import Counter, defaultdict
+from datetime import datetime, timezone
+from difflib import SequenceMatcher
+import json
+from pathlib import Path
+import re
+from urllib.error import HTTPError
+from urllib.parse import parse_qs, urljoin, urlsplit
+
+from collect_pmlr import fetch
+from collect_proceedings import Document, clean
+from index_core import author_keys, normalized
+
+ROOT = 'https://iclr.cc/'
+SOURCE = 'iclr_program'
+
+
+class NotYetPublished(ValueError):
+ pass
+
+
+def discover(html):
+ years = {}
+ for node in Document(html).root.walk():
+ text = clean(node.text())
+ if node.tag == 'a' and re.fullmatch(r'20\d{2}', text):
+ url = urljoin(ROOT, node.attrs.get('href', ''))
+ if urlsplit(url).netloc == 'iclr.cc':
+ years[int(text)] = url
+ if not years:
+ raise ValueError('No ICLR years in the official navigation')
+ return sorted(years.items())
+
+
+def asset_urls(html, page_url, year):
+ refs = re.findall(r'''["']([^"'\s]+\.json)["']''', html)
+ result = {}
+ for kind in ('orals-posters', 'abstracts'):
+ matches = {urljoin(page_url, ref) for ref in refs
+ if urlsplit(urljoin(page_url, ref)).netloc == 'iclr.cc'
+ and urlsplit(urljoin(page_url, ref)).path == f'/static/virtual/data/iclr-{year}-{kind}.json'}
+ if len(matches) > 1:
+ raise ValueError('Ambiguous official program asset')
+ if matches:
+ result[kind] = matches.pop()
+ if 'orals-posters' not in result:
+ raise ValueError('Official page has no supported program JSON reference')
+ return result
+
+
+def main_event(row, year):
+ source = row.get('sourceurl')
+ if source:
+ parsed = urlsplit(source)
+ return (parsed.netloc == 'openreview.net' and parsed.path == '/group'
+ and parse_qs(parsed.query).get('id') == [f'ICLR.cc/{year}/Conference'])
+ # These historical official paper lists predate sourceurl/decision fields.
+ return year in (2020, 2021) and row.get('eventtype') in ('Poster', 'Oral', 'Spotlight')
+
+
+def source_number(row):
+ value = row.get('sourceid')
+ return int(value) if re.fullmatch(r'-?[1-9]\d*', str(value)) else None
+
+
+def identifiers(row):
+ # The program creates negative-ID oral events with fabricated forum URLs.
+ if (source_number(row) or 0) < 0:
+ return {}
+ links = [row.get('paper_url'), row.get('paper_pdf_url')]
+ links += [media.get('uri') for media in row.get('eventmedia', [])
+ if str(media.get('name') or '').casefold() in ('openreview', 'paper')]
+ forums = set()
+ for link in links:
+ if not isinstance(link, str):
+ continue
+ parsed = urlsplit(link)
+ if parsed.netloc == 'openreview.net' and parsed.path in ('/forum', '/pdf'):
+ values = parse_qs(parsed.query).get('id', [])
+ if len(values) == 1 and re.fullmatch(r'[\w-]+', values[0]) and not re.match(r'20\d{2}-Oral-', values[0]):
+ forums.add(values[0])
+ if len(forums) > 1:
+ raise ValueError('Conflicting paper self-links in one program event')
+ return {'openreview': forums.pop()} if forums else {}
+
+
+def parse_program(data, abstracts, year, index_url):
+ if not isinstance(data, dict) or not isinstance(data.get('results'), list):
+ raise ValueError('Expected a program results array with a declared count')
+ events = data['results']
+ if data.get('next') or data.get('previous') or data.get('count') != len(events):
+ raise ValueError('Program JSON is paginated or its count differs from events')
+ if not isinstance(abstracts, dict):
+ raise ValueError('Expected an abstract mapping keyed by event ID')
+ event_ids = [str(row['id']) for row in events]
+ if len(event_ids) != len(set(event_ids)):
+ raise ValueError('Duplicate event IDs in program JSON')
+ grouped = defaultdict(list)
+ excluded = Counter()
+ for row in events:
+ if not main_event(row, year):
+ excluded[str(row.get('sourceurl') or 'unclassified')] += 1
+ continue
+ decision = row.get('decision')
+ if decision and not (str(decision).casefold().startswith('accept') or decision in ('Poster', 'Oral', 'Spotlight')):
+ raise ValueError('Unexpected decision in an accepted conference program')
+ number = source_number(row)
+ identity = ('submission', abs(number)) if number else ('forum', identifiers(row).get('openreview'))
+ if identity == ('forum', None):
+ identity = ('event', row['id'])
+ grouped[identity].append(row)
+ records = []
+ missing_identity = 0
+ for identity, group in grouped.items():
+ # Verify the negative oral event really describes its positive counterpart.
+ group.sort(key=lambda r: ((source_number(r) or 0) < 0, str(r['id'])))
+ primary = group[0]
+ title = normalized(primary.get('name', ''))
+ authors = [author_keys([a['fullname'] for a in row.get('authors', [])]) for row in group]
+ title_match = all(normalized(row.get('name', '')) == title or (
+ identity[0] == 'submission' and (source_number(row) or 0) < 0
+ and (source_number(primary) or 0) > 0 and len(authors[0]) >= 2
+ and SequenceMatcher(None, title, normalized(row.get('name', ''))).ratio() >= 0.85)
+ for row in group)
+ if not title or not title_match or any(a != authors[0] for a in authors):
+ raise ValueError('Program submission ID collides across different papers')
+ own_ids = {}
+ for row in group:
+ for scheme, value in identifiers(row).items():
+ if scheme in own_ids and own_ids[scheme] != value:
+ raise ValueError('Conflicting identifiers across presentation events')
+ own_ids[scheme] = value
+ abstract = next((text for row in group
+ if (text := row.get('abstract') or abstracts.get(str(row['id'])))), None)
+ if abstract is not None and not isinstance(abstract, str):
+ raise ValueError('Nontext abstract')
+ presentation_ids = [row['id'] for row in group]
+ forum = own_ids.get('openreview')
+ external = f'{year}/{forum}' if forum else f'{year}/program-{identity[1]}'
+ missing_identity += not bool(forum)
+ page = urljoin(ROOT, primary.get('virtualsite_url') or f'/virtual/{year}/{str(primary.get("eventtype", "Poster")).lower()}/{primary["id"]}')
+ topics = sorted({str(row['topic']).strip() for row in group if row.get('topic')})
+ keywords = sorted({str(keyword).strip() for row in group for keyword in row.get('keywords', []) if keyword})
+ records.append({'source': SOURCE, 'external_id': external, 'url': page,
+ 'title': clean(primary['name']), 'abstract': abstract.strip() if abstract else None,
+ 'authors': [a['fullname'] for a in primary.get('authors', [])],
+ 'identifiers': own_ids, 'venue': 'ICLR', 'year': year, 'track': 'main',
+ 'status': 'accepted', 'raw_decision': str(primary.get('decision') or 'listed in official conference paper program'),
+ 'evidence_url': index_url, 'list_url': index_url,
+ 'tags': [{'namespace': SOURCE+':'+kind, 'value': value}
+ for kind, values in [('topic', topics), ('keyword', keywords)] for value in values],
+ 'metadata': {'program_event_ids': presentation_ids,
+ 'presentation_titles': sorted({clean(r['name']) for r in group}),
+ 'program_source_id': identity[1], 'presentation_types': sorted({str(r.get('event_type') or r.get('eventtype')) for r in group}),
+ 'raw_decisions': sorted({str(r.get('decision')) for r in group if r.get('decision')}),
+ 'identity_state': 'forum' if forum else 'program_only'}})
+ # Different source numbers can still identify the same forum. Keep one record.
+ by_id = {}
+ for row in records:
+ previous = by_id.get(row['external_id'])
+ if previous:
+ if normalized(previous['title']) != normalized(row['title']) or author_keys(previous['authors']) != author_keys(row['authors']):
+ raise ValueError('One forum identifies different program papers')
+ previous['metadata']['program_event_ids'] += row['metadata']['program_event_ids']
+ previous['metadata']['presentation_types'] = sorted(set(previous['metadata']['presentation_types'] + row['metadata']['presentation_types']))
+ previous['abstract'] = previous['abstract'] or row['abstract']
+ else:
+ by_id[row['external_id']] = row
+ result = list(by_id.values())
+ if not result:
+ raise ValueError('No verified main-conference papers in program')
+ return result, {'program_event_count': len(events), 'main_event_count': sum(map(len, grouped.values())),
+ 'excluded_events_by_source': dict(excluded), 'missing_forum_count': missing_identity,
+ 'missing_abstract_count': sum(not r['abstract'] for r in result)}
+
+
+def collect_year(year, cache, out):
+ page = f'https://iclr.cc/virtual/{year}/papers.html'
+ assets = asset_urls(fetch(page, cache, 16*1024**2, delay=1).decode(), page, year)
+ data = json.loads(fetch(assets['orals-posters'], cache, 32*1024**2, delay=1))
+ if data.get('count') == 0 and data.get('results') == [] and year > datetime.now(timezone.utc).year:
+ raise NotYetPublished('Official program is empty for a future conference')
+ abstracts = {}
+ abstract_asset_status = 'not_referenced'
+ if 'abstracts' in assets:
+ try:
+ abstracts = json.loads(fetch(assets['abstracts'], cache, 32*1024**2, delay=1))
+ abstract_asset_status = 'available'
+ except HTTPError as exc:
+ if exc.code != 404:
+ raise
+ # Historical programs embed abstracts but retain a stale split-file URL.
+ exc.close()
+ abstract_asset_status = 'unavailable_404'
+ records, audit = parse_program(data, abstracts, year, page)
+ path = Path(out)/f'iclr-program-{year}-main.jsonl'
+ part = path.with_suffix('.part')
+ part.write_text(''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in records))
+ part.replace(path)
+ manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICLR', 'year': year, 'track': 'main',
+ 'count': len(records), 'list_url': page, 'asset_urls': assets,
+ 'abstract_asset_status': abstract_asset_status, 'file': str(path),
+ 'status': 'partial' if audit['missing_abstract_count'] else 'collected', **audit}
+ path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2))
+ return manifest
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--cache', required=True)
+ parser.add_argument('--out', required=True)
+ parser.add_argument('--year', type=int, action='append')
+ args = parser.parse_args()
+ out = Path(args.out); out.mkdir(parents=True, exist_ok=True)
+ years = discover(fetch(ROOT, args.cache, 4*1024**2, delay=1).decode())
+ if args.year and set(args.year)-{year for year, _ in years}:
+ parser.error('Requested year is absent from the official navigation')
+ status_path = out/'iclr-program-year-status.json'
+ catalogue = json.loads(status_path.read_text()) if status_path.exists() else {}
+ requested = []
+ for year, conference_url in years:
+ if args.year and year not in args.year:
+ continue
+ requested.append(year)
+ try:
+ result = collect_year(year, args.cache, out)
+ except NotYetPublished as exc:
+ result = {'year': year, 'status': 'not_yet_published', 'error': str(exc)}
+ except HTTPError as exc:
+ result = {'year': year, 'status': 'not_yet_published' if year > datetime.now(timezone.utc).year and exc.code == 404 else 'unavailable' if exc.code == 404 else 'blocked', 'error': str(exc)}
+ exc.close()
+ except Exception as exc:
+ result = {'year': year, 'status': 'blocked', 'error': str(exc)}
+ result['conference_url'] = conference_url
+ catalogue[str(year)] = result
+ part = status_path.with_suffix('.part'); part.write_text(json.dumps(catalogue, indent=2)); part.replace(status_path)
+ print(json.dumps(result), flush=True)
+ (out/'iclr-program-last-run.json').write_text(json.dumps({'requested_years': requested, 'year_filter': args.year}, indent=2))
+ if any(catalogue[str(y)]['status'] == 'blocked' for y in requested):
+ raise SystemExit(1)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/import_records.py b/import_records.py
index 665ea13..d287557 100644
--- a/import_records.py
+++ b/import_records.py
@@ -35,6 +35,8 @@ def ingest_file(db,path):
raise ValueError('Duplicate external ID in one collected file: '+external)
seen.add(external)
paper,record = store.record(source_name,external,row['url'],row['title'],row.get('abstract'),row.get('authors',[]),row.get('identifiers'),row.get('metadata'))
+ if 'tags' in row:
+ store.record_tags(paper,record,source_name,row['tags'])
if row.get('status') in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):
edition = store.decision(paper,record,row['venue'],row['year'],row['track'],external,row.get('evidence_url',row['url']),row['status'],row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status']),row.get('venue_aliases'))
editions.add(edition)
diff --git a/index_core.py b/index_core.py
index 5c052d8..7227a30 100644
--- a/index_core.py
+++ b/index_core.py
@@ -109,6 +109,22 @@ class Store:
c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(paper,other),max(paper,other),'exact_title_unconfirmed',None,json.dumps({'title':title}), 'pending'))
return paper,record
+ def record_tags(self,paper,record,source,tags):
+ """Refresh this record's platform tags without removing other evidence."""
+ values=set()
+ for tag in tags:
+ namespace,value=tag['namespace'],tag['value']
+ if not isinstance(namespace,str) or not namespace.startswith(source+':') or not isinstance(value,str) or not value.strip():
+ raise ValueError('Platform tag must have its own source namespace and a nonempty text value')
+ values.add((namespace,value.strip()))
+ evidence=f'{source}:record:{record}:tags'
+ c=self.conn
+ c.execute('DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key=?',(record,evidence))
+ for namespace,value in sorted(values):
+ c.execute('INSERT OR IGNORE INTO tags(namespace,value) VALUES(?,?)',(namespace,value))
+ tag=c.execute('SELECT id FROM tags WHERE namespace=? AND value=?',(namespace,value)).fetchone()[0]
+ c.execute('INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,evidence))
+
def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None):
c = self.conn
c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified'))
diff --git a/test_iclr.py b/test_iclr.py
new file mode 100644
index 0000000..d9f577e
--- /dev/null
+++ b/test_iclr.py
@@ -0,0 +1,98 @@
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+from urllib.error import HTTPError
+
+from collect_iclr import asset_urls, collect_year, discover, parse_program
+from import_records import ingest_file
+from index_core import Store
+
+
+class ICLRTests(unittest.TestCase):
+ def event(self,identifier=1,sourceid=10,**updates):
+ row={'id':identifier,'sourceid':sourceid,'sourceurl':'https://openreview.net/group?id=ICLR.cc/2026/Conference',
+ 'name':'A reliable paper on abstract metadata indexing','authors':[{'fullname':'Jane Doe'},{'fullname':'John Smith'}],
+ 'eventtype':'Poster','event_type':'Poster','decision':'Accept (Oral)',
+ 'paper_url':'https://openreview.net/forum?id=realForum','topic':'Learning->Theory','keywords':[]}
+ return dict(row,**updates)
+
+ def parse(self,events,abstracts=None,year=2026):
+ return parse_program({'count':len(events),'next':None,'previous':None,'results':events},abstracts or {},year,f'https://iclr.cc/virtual/{year}/papers.html')
+
+ def test_oral_placeholder_is_not_an_identifier_or_another_submission(self):
+ poster=self.event()
+ oral=self.event(2,-10,eventtype='Oral',event_type='Oral',paper_url='https://openreview.net/forum?id=2026-Oral--10-placeholder')
+ rows,audit=self.parse([oral,poster],{'1':'The real abstract.'})
+ self.assertEqual(len(rows),1)
+ self.assertEqual(rows[0]['identifiers'],{'openreview':'realForum'})
+ self.assertEqual(rows[0]['abstract'],'The real abstract.')
+ self.assertEqual(rows[0]['metadata']['presentation_types'],['Oral','Poster'])
+ self.assertEqual(audit['main_event_count'],2)
+
+ def test_blog_and_journal_presentations_are_not_main_acceptances(self):
+ rows,audit=self.parse([self.event(),self.event(2,20,sourceurl='https://openreview.net/group?id=ICLR.cc/2026/BlogPosts'),self.event(3,30,sourceurl='2026-TMLR-J2C-Spreadsheet')])
+ self.assertEqual(len(rows),1)
+ self.assertEqual(sum(audit['excluded_events_by_source'].values()),2)
+ self.assertEqual(audit['missing_abstract_count'],1)
+ self.assertIsNone(rows[0]['abstract'])
+
+ def test_title_revision_is_scoped_to_matching_signed_submission_and_authors(self):
+ poster=self.event(name='AlphaEdit: Null-Space Constrained Knowledge Editing for Language Models')
+ oral=self.event(2,-10,eventtype='Oral',name='AlphaEdit: Null-Space Constrained Model Editing for Language Models')
+ rows,_=self.parse([oral,poster])
+ self.assertEqual(len(rows),1)
+ self.assertEqual(rows[0]['title'],poster['name'])
+ self.assertEqual(len(rows[0]['metadata']['presentation_titles']),2)
+ with self.assertRaisesRegex(ValueError,'collides'):
+ self.parse([poster,dict(oral,authors=[{'fullname':'Unrelated Author'}])])
+
+ def test_truncation_and_identity_collision_fail(self):
+ with self.assertRaisesRegex(ValueError,'count differs'):
+ parse_program({'count':2,'results':[self.event()]},{},2026,'https://iclr.cc/')
+ with self.assertRaisesRegex(ValueError,'collides'):
+ self.parse([self.event(),self.event(2,-10,name='A different paper')])
+ with self.assertRaisesRegex(ValueError,'Conflicting identifiers'):
+ self.parse([self.event(),self.event(2,10,paper_url='https://openreview.net/forum?id=otherForum')])
+
+ def test_legacy_embedded_abstracts_and_paper_links(self):
+ row=self.event(sourceurl=None,abstract='Historical abstract',paper_url=None,paper_pdf_url='https://openreview.net/forum?id=historicalForum',decision='Poster')
+ rows,_=self.parse([row],year=2021)
+ self.assertEqual(rows[0]['abstract'],'Historical abstract')
+ self.assertEqual(rows[0]['identifiers'],{'openreview':'historicalForum'})
+ rows,_=self.parse([dict(row,paper_pdf_url=None)],year=2020)
+ self.assertEqual(rows[0]['metadata']['identity_state'],'program_only')
+ self.assertEqual(rows[0]['identifiers'],{})
+
+ def test_only_referenced_official_assets_and_real_navigation_years(self):
+ html='201320262025'
+ self.assertEqual([y for y,_ in discover(html)],[2013,2026])
+ html='start("/static/virtual/data/iclr-2026-orals-posters.json",true,"https://other.example/abstracts.json")'
+ self.assertEqual(asset_urls(html,'https://iclr.cc/virtual/2026/papers.html',2026),{'orals-posters':'https://iclr.cc/static/virtual/data/iclr-2026-orals-posters.json'})
+
+ def test_missing_split_abstract_asset_does_not_discard_embedded_abstract(self):
+ html=b'start("/static/virtual/data/iclr-2021-orals-posters.json",true,"/static/virtual/data/iclr-2021-abstracts.json")'
+ event=self.event(sourceurl=None,decision='Poster',abstract='Embedded historical abstract')
+ responses=[html,json.dumps({'count':1,'results':[event]}).encode(),HTTPError('https://iclr.cc/abstracts',404,'Not Found',None,None)]
+ with tempfile.TemporaryDirectory() as tmp,patch('collect_iclr.fetch',side_effect=responses):
+ manifest=collect_year(2021,tmp,tmp)
+ self.assertEqual(manifest['status'],'collected')
+ self.assertEqual(manifest['missing_abstract_count'],0)
+ self.assertEqual(manifest['abstract_asset_status'],'unavailable_404')
+
+ def test_import_repeat_preserves_one_submission_and_queryable_platform_tags(self):
+ rows,_=self.parse([self.event(),self.event(2,-10,eventtype='Oral')],{'1':'A searchable abstract.'})
+ with tempfile.TemporaryDirectory() as tmp:
+ path=Path(tmp)/'iclr.jsonl';db=Path(tmp)/'papers.sqlite'
+ path.write_text(''.join(json.dumps(r)+'\n' for r in rows))
+ path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','list_url':'https://iclr.cc/virtual/2026/papers.html','count':1,'status':'collected'}))
+ ingest_file(db,path);ingest_file(db,path)
+ store=Store(db);self.addCleanup(store.conn.close)
+ self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
+ self.assertEqual(store.conn.execute('SELECT count(*) FROM decisions').fetchone()[0],1)
+ self.assertEqual(store.conn.execute('SELECT namespace,value FROM tags').fetchall(),[('iclr_program:topic','Learning->Theory')])
+ self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(),[])
+
+
+if __name__=='__main__':unittest.main()
diff --git a/test_index_core.py b/test_index_core.py
index 87e1fea..aa08bf9 100644
--- a/test_index_core.py
+++ b/test_index_core.py
@@ -58,6 +58,19 @@ class StoreTests(unittest.TestCase):
self.assertEqual(self.store.conn.execute('SELECT title FROM papers').fetchone()[0],'Updated research title')
self.assertEqual(self.store.conn.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'searchable'").fetchone()[0],1)
+ def test_tag_refresh_keeps_other_record_evidence_and_custom_tags(self):
+ paper,first=self.record('iclr_program','2025/a',{'openreview':'sharedForum'})
+ same,second=self.record('iclr_program','2025/b',{'openreview':'sharedForum'})
+ self.store.record_tags(paper,first,'iclr_program',[{'namespace':'iclr_program:topic','value':'Shared'}])
+ self.store.record_tags(same,second,'iclr_program',[{'namespace':'iclr_program:topic','value':'Shared'}])
+ tag=self.store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','Manual')").lastrowid
+ self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)',(paper,tag,None,'manual'))
+ self.store.record_tags(paper,first,'iclr_program',[])
+ self.assertEqual(self.store.conn.execute('SELECT source_record_id FROM paper_tags ORDER BY source_record_id').fetchall(),[(None,),(second,)])
+ with self.assertRaisesRegex(ValueError,'namespace'):
+ self.store.record_tags(paper,second,'iclr_program',[{'namespace':'custom:topic','value':'No overwrite'}])
+ self.assertEqual(self.store.conn.execute('SELECT count(*) FROM paper_tags').fetchone()[0],2)
+
def test_missing_abstract_does_not_become_complete_coverage(self):
path=Path(self.tmp.name)/'partial.jsonl'
path.write_text(json.dumps({'source':'colm','external_id':'paper1','url':'https://example.org/paper1','title':'Paper','abstract':None,'authors':[],'venue':'COLM','year':2024,'track':'main','status':'accepted'})+'\n')
diff --git a/venues.json b/venues.json
index 5f7adc6..ca25bc6 100644
--- a/venues.json
+++ b/venues.json
@@ -11,7 +11,8 @@
"aaai": "https://ojs.aaai.org/index.php/AAAI/issue/archive",
"ijcai": "https://www.ijcai.org/all_proceedings/",
"dblp": "https://dblp.org/db/conf/",
- "colm": "https://proceedings.colm.cc/"
+ "colm": "https://proceedings.colm.cc/",
+ "iclr_program": "https://iclr.cc/virtual/2026/papers.html"
},
"venues": [
{
@@ -20,7 +21,6 @@
"venue_type": "conference",
"discovery_sources": [
"openreview",
- "pmlr",
"neurips"
],
"scope": "candidate_pending_scope_review",
@@ -36,8 +36,7 @@
"venue_type": "conference",
"discovery_sources": [
"openreview",
- "pmlr",
- "neurips"
+ "pmlr"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
@@ -52,8 +51,7 @@
"venue_type": "conference",
"discovery_sources": [
"openreview",
- "pmlr",
- "neurips"
+ "iclr_program"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",
@@ -68,8 +66,7 @@
"venue_type": "conference",
"discovery_sources": [
"openreview",
- "pmlr",
- "neurips"
+ "colm"
],
"scope": "candidate_pending_scope_review",
"years": "all_publicly_available",