From dba943f03e41e9fbc5bc2764f58869c9768b7abe Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Fri, 6 Nov 2026 12:58:02 +0800 Subject: [PATCH] Collect official ICLR programs with presentation identity checks --- README.md | 22 +++- collect_iclr.py | 246 +++++++++++++++++++++++++++++++++++++++++++++ import_records.py | 2 + index_core.py | 16 +++ test_iclr.py | 98 ++++++++++++++++++ test_index_core.py | 13 +++ venues.json | 13 +-- 7 files changed, 401 insertions(+), 9 deletions(-) create mode 100644 collect_iclr.py create mode 100644 test_iclr.py diff --git a/README.md b/README.md index 861582c..ab57d1c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 目标包括：完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。 -**目前仍在开发，不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查；完整 ICML 2025 元数据已采集 3,330 条，尚未作为全量交付。arXiv 元数据 ZIP 已完整下载（1,878,831,586 字节），全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集，其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流，实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围，不代表已完成采集，也不是与 COLM 比较的客观排名。 +**目前仍在开发，不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology、ICLR 官方公开程序元数据采集、带清单核验的会议导入和只读查询。PeerRead 提供公开 ICLR 2017 明确决定的补充来源。arXiv 元数据 ZIP 已完整下载（1,878,831,586 字节），全量导入正在服务器运行。其他会议适配器、历史缺失届次、完整投稿历史、跨平台疑似重复复核和覆盖率核验仍待完成。回归检查在远端执行，已配置 ces-control:host 对应的 CI 工作流，实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围，不代表已完成采集，也不是与 COLM 比较的客观排名。 ## 数据范围与来源 @@ -156,6 +156,26 @@ python3 -B collect_acl.py \ `acl-collection-status.json` 记录每个 XML 及其 volume 的处理结果和排除原因；`acl-last-run.json` 记录源 commit 与本批范围。`--collection P17` 可限定一个确实存在的 collection。已验证真实 ACL 2017 长文 195 条、短文 107 条，二者摘要完整；其他 track 按源实际缺失情况记录，不能凭样本宣称整个历史完成。 +### ICLR 官方公开程序元数据 + +```sh +python3 -B collect_iclr.py --cache /data/paper-index/raw/http-cache \ + --out /data/paper-index/collected +# 入库后按官方主题检索；主题值以实际来源为准 +python3 -B query.py --db /data/paper-index/papers.sqlite \ + --venue ICLR --tag 'iclr_program:topic=Reinforcement Learning->Deep RL' +``` + +从 ICLR 官方年份导航登记届次，读取对应 Papers 页面明确引用的 orals-posters/abstracts JSON。核对声明数量、分页状态及唯一 event ID；只下载公开程序元数据，不访问用户资料、PDF、海报、幻灯片或视频。`--year` 可限制年份；清单中的已采集数与数据库已入库数分别统计。 + +官方程序中的 event 数不是论文数：同一篇文章可以有 poster 和 oral 两场展示。程序还会给负 sourceid 的 oral 展示生成占位 OpenReview 链接，这些链接不能作为真实论文标识。采集器核对原投稿编号、标题与作者后合并展示，优先使用正编号记录的真实论文链接；原展示编号和类型保留在源元数据中。 + +同一投稿编号的展示标题可能有小幅修订。只有负编号 oral 与正编号原稿对应、至少两位作者完全一致且标题相似度满足严格阈值时，才保留为同稿的标题变体；该规则仅在一届官方程序内部生效，不用于跨平台模糊合并。 + +只将 Conference 来源的论文视为 ICLR 主会录用，排除 Blog 和期刊转会展示，并在审计清单保存排除计数。2020/2021 旧数据允许依据官方主会 paper program 形成录用证据；无 forum ID 时保留 program 身份和匹配候选，不伪造标识。历史年度返回 404 会登记 unavailable，未来未发表届次登记 not_yet_published，其他访问或解析失败登记 blocked；这些状态都不能宣称该年已覆盖。 + +主题、关键词使用 `iclr_program:topic`、`iclr_program:keyword` 命名空间。通用导入器按源记录刷新标签证据，保留同一论文其他源记录及 `custom:*` 标签。每年的 JSONL、manifest、`iclr-program-year-status.json` 和 `iclr-program-last-run.json` 保存实际范围、独立展示数、主会展示数、论文数、缺失摘要与缺失 forum 数。录用程序不能提供拒稿全集，公开投稿及拒稿补充仍需独立来源。 + ## 长期维护与更新 1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发，遵守来源限流、重试与人机验证要求。 diff --git a/collect_iclr.py b/collect_iclr.py new file mode 100644 index 0000000..ef0978f --- /dev/null +++ b/collect_iclr.py @@ -0,0 +1,246 @@ +"""Collect public ICLR program metadata, excluding blog/journal presentations.""" +import argparse +from collections import Counter, defaultdict +from datetime import datetime, timezone +from difflib import SequenceMatcher +import json +from pathlib import Path +import re +from urllib.error import HTTPError +from urllib.parse import parse_qs, urljoin, urlsplit + +from collect_pmlr import fetch +from collect_proceedings import Document, clean +from index_core import author_keys, normalized + +ROOT = 'https://iclr.cc/' +SOURCE = 'iclr_program' + + +class NotYetPublished(ValueError): + pass + + +def discover(html): + years = {} + for node in Document(html).root.walk(): + text = clean(node.text()) + if node.tag == 'a' and re.fullmatch(r'20\d{2}', text): + url = urljoin(ROOT, node.attrs.get('href', '')) + if urlsplit(url).netloc == 'iclr.cc': + years[int(text)] = url + if not years: + raise ValueError('No ICLR years in the official navigation') + return sorted(years.items()) + + +def asset_urls(html, page_url, year): + refs = re.findall(r'''["']([^"'\s]+\.json)["']''', html) + result = {} + for kind in ('orals-posters', 'abstracts'): + matches = {urljoin(page_url, ref) for ref in refs + if urlsplit(urljoin(page_url, ref)).netloc == 'iclr.cc' + and urlsplit(urljoin(page_url, ref)).path == f'/static/virtual/data/iclr-{year}-{kind}.json'} + if len(matches) > 1: + raise ValueError('Ambiguous official program asset') + if matches: + result[kind] = matches.pop() + if 'orals-posters' not in result: + raise ValueError('Official page has no supported program JSON reference') + return result + + +def main_event(row, year): + source = row.get('sourceurl') + if source: + parsed = urlsplit(source) + return (parsed.netloc == 'openreview.net' and parsed.path == '/group' + and parse_qs(parsed.query).get('id') == [f'ICLR.cc/{year}/Conference']) + # These historical official paper lists predate sourceurl/decision fields. + return year in (2020, 2021) and row.get('eventtype') in ('Poster', 'Oral', 'Spotlight') + + +def source_number(row): + value = row.get('sourceid') + return int(value) if re.fullmatch(r'-?[1-9]\d*', str(value)) else None + + +def identifiers(row): + # The program creates negative-ID oral events with fabricated forum URLs. + if (source_number(row) or 0) < 0: + return {} + links = [row.get('paper_url'), row.get('paper_pdf_url')] + links += [media.get('uri') for media in row.get('eventmedia', []) + if str(media.get('name') or '').casefold() in ('openreview', 'paper')] + forums = set() + for link in links: + if not isinstance(link, str): + continue + parsed = urlsplit(link) + if parsed.netloc == 'openreview.net' and parsed.path in ('/forum', '/pdf'): + values = parse_qs(parsed.query).get('id', []) + if len(values) == 1 and re.fullmatch(r'[\w-]+', values[0]) and not re.match(r'20\d{2}-Oral-', values[0]): + forums.add(values[0]) + if len(forums) > 1: + raise ValueError('Conflicting paper self-links in one program event') + return {'openreview': forums.pop()} if forums else {} + + +def parse_program(data, abstracts, year, index_url): + if not isinstance(data, dict) or not isinstance(data.get('results'), list): + raise ValueError('Expected a program results array with a declared count') + events = data['results'] + if data.get('next') or data.get('previous') or data.get('count') != len(events): + raise ValueError('Program JSON is paginated or its count differs from events') + if not isinstance(abstracts, dict): + raise ValueError('Expected an abstract mapping keyed by event ID') + event_ids = [str(row['id']) for row in events] + if len(event_ids) != len(set(event_ids)): + raise ValueError('Duplicate event IDs in program JSON') + grouped = defaultdict(list) + excluded = Counter() + for row in events: + if not main_event(row, year): + excluded[str(row.get('sourceurl') or 'unclassified')] += 1 + continue + decision = row.get('decision') + if decision and not (str(decision).casefold().startswith('accept') or decision in ('Poster', 'Oral', 'Spotlight')): + raise ValueError('Unexpected decision in an accepted conference program') + number = source_number(row) + identity = ('submission', abs(number)) if number else ('forum', identifiers(row).get('openreview')) + if identity == ('forum', None): + identity = ('event', row['id']) + grouped[identity].append(row) + records = [] + missing_identity = 0 + for identity, group in grouped.items(): + # Verify the negative oral event really describes its positive counterpart. + group.sort(key=lambda r: ((source_number(r) or 0) < 0, str(r['id']))) + primary = group[0] + title = normalized(primary.get('name', '')) + authors = [author_keys([a['fullname'] for a in row.get('authors', [])]) for row in group] + title_match = all(normalized(row.get('name', '')) == title or ( + identity[0] == 'submission' and (source_number(row) or 0) < 0 + and (source_number(primary) or 0) > 0 and len(authors[0]) >= 2 + and SequenceMatcher(None, title, normalized(row.get('name', ''))).ratio() >= 0.85) + for row in group) + if not title or not title_match or any(a != authors[0] for a in authors): + raise ValueError('Program submission ID collides across different papers') + own_ids = {} + for row in group: + for scheme, value in identifiers(row).items(): + if scheme in own_ids and own_ids[scheme] != value: + raise ValueError('Conflicting identifiers across presentation events') + own_ids[scheme] = value + abstract = next((text for row in group + if (text := row.get('abstract') or abstracts.get(str(row['id'])))), None) + if abstract is not None and not isinstance(abstract, str): + raise ValueError('Nontext abstract') + presentation_ids = [row['id'] for row in group] + forum = own_ids.get('openreview') + external = f'{year}/{forum}' if forum else f'{year}/program-{identity[1]}' + missing_identity += not bool(forum) + page = urljoin(ROOT, primary.get('virtualsite_url') or f'/virtual/{year}/{str(primary.get("eventtype", "Poster")).lower()}/{primary["id"]}') + topics = sorted({str(row['topic']).strip() for row in group if row.get('topic')}) + keywords = sorted({str(keyword).strip() for row in group for keyword in row.get('keywords', []) if keyword}) + records.append({'source': SOURCE, 'external_id': external, 'url': page, + 'title': clean(primary['name']), 'abstract': abstract.strip() if abstract else None, + 'authors': [a['fullname'] for a in primary.get('authors', [])], + 'identifiers': own_ids, 'venue': 'ICLR', 'year': year, 'track': 'main', + 'status': 'accepted', 'raw_decision': str(primary.get('decision') or 'listed in official conference paper program'), + 'evidence_url': index_url, 'list_url': index_url, + 'tags': [{'namespace': SOURCE+':'+kind, 'value': value} + for kind, values in [('topic', topics), ('keyword', keywords)] for value in values], + 'metadata': {'program_event_ids': presentation_ids, + 'presentation_titles': sorted({clean(r['name']) for r in group}), + 'program_source_id': identity[1], 'presentation_types': sorted({str(r.get('event_type') or r.get('eventtype')) for r in group}), + 'raw_decisions': sorted({str(r.get('decision')) for r in group if r.get('decision')}), + 'identity_state': 'forum' if forum else 'program_only'}}) + # Different source numbers can still identify the same forum. Keep one record. + by_id = {} + for row in records: + previous = by_id.get(row['external_id']) + if previous: + if normalized(previous['title']) != normalized(row['title']) or author_keys(previous['authors']) != author_keys(row['authors']): + raise ValueError('One forum identifies different program papers') + previous['metadata']['program_event_ids'] += row['metadata']['program_event_ids'] + previous['metadata']['presentation_types'] = sorted(set(previous['metadata']['presentation_types'] + row['metadata']['presentation_types'])) + previous['abstract'] = previous['abstract'] or row['abstract'] + else: + by_id[row['external_id']] = row + result = list(by_id.values()) + if not result: + raise ValueError('No verified main-conference papers in program') + return result, {'program_event_count': len(events), 'main_event_count': sum(map(len, grouped.values())), + 'excluded_events_by_source': dict(excluded), 'missing_forum_count': missing_identity, + 'missing_abstract_count': sum(not r['abstract'] for r in result)} + + +def collect_year(year, cache, out): + page = f'https://iclr.cc/virtual/{year}/papers.html' + assets = asset_urls(fetch(page, cache, 16*1024**2, delay=1).decode(), page, year) + data = json.loads(fetch(assets['orals-posters'], cache, 32*1024**2, delay=1)) + if data.get('count') == 0 and data.get('results') == [] and year > datetime.now(timezone.utc).year: + raise NotYetPublished('Official program is empty for a future conference') + abstracts = {} + abstract_asset_status = 'not_referenced' + if 'abstracts' in assets: + try: + abstracts = json.loads(fetch(assets['abstracts'], cache, 32*1024**2, delay=1)) + abstract_asset_status = 'available' + except HTTPError as exc: + if exc.code != 404: + raise + # Historical programs embed abstracts but retain a stale split-file URL. + exc.close() + abstract_asset_status = 'unavailable_404' + records, audit = parse_program(data, abstracts, year, page) + path = Path(out)/f'iclr-program-{year}-main.jsonl' + part = path.with_suffix('.part') + part.write_text(''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in records)) + part.replace(path) + manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICLR', 'year': year, 'track': 'main', + 'count': len(records), 'list_url': page, 'asset_urls': assets, + 'abstract_asset_status': abstract_asset_status, 'file': str(path), + 'status': 'partial' if audit['missing_abstract_count'] else 'collected', **audit} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)) + return manifest + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--cache', required=True) + parser.add_argument('--out', required=True) + parser.add_argument('--year', type=int, action='append') + args = parser.parse_args() + out = Path(args.out); out.mkdir(parents=True, exist_ok=True) + years = discover(fetch(ROOT, args.cache, 4*1024**2, delay=1).decode()) + if args.year and set(args.year)-{year for year, _ in years}: + parser.error('Requested year is absent from the official navigation') + status_path = out/'iclr-program-year-status.json' + catalogue = json.loads(status_path.read_text()) if status_path.exists() else {} + requested = [] + for year, conference_url in years: + if args.year and year not in args.year: + continue + requested.append(year) + try: + result = collect_year(year, args.cache, out) + except NotYetPublished as exc: + result = {'year': year, 'status': 'not_yet_published', 'error': str(exc)} + except HTTPError as exc: + result = {'year': year, 'status': 'not_yet_published' if year > datetime.now(timezone.utc).year and exc.code == 404 else 'unavailable' if exc.code == 404 else 'blocked', 'error': str(exc)} + exc.close() + except Exception as exc: + result = {'year': year, 'status': 'blocked', 'error': str(exc)} + result['conference_url'] = conference_url + catalogue[str(year)] = result + part = status_path.with_suffix('.part'); part.write_text(json.dumps(catalogue, indent=2)); part.replace(status_path) + print(json.dumps(result), flush=True) + (out/'iclr-program-last-run.json').write_text(json.dumps({'requested_years': requested, 'year_filter': args.year}, indent=2)) + if any(catalogue[str(y)]['status'] == 'blocked' for y in requested): + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/import_records.py b/import_records.py index 665ea13..d287557 100644 --- a/import_records.py +++ b/import_records.py @@ -35,6 +35,8 @@ def ingest_file(db,path): raise ValueError('Duplicate external ID in one collected file: '+external) seen.add(external) paper,record = store.record(source_name,external,row['url'],row['title'],row.get('abstract'),row.get('authors',[]),row.get('identifiers'),row.get('metadata')) + if 'tags' in row: + store.record_tags(paper,record,source_name,row['tags']) if row.get('status') in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'): edition = store.decision(paper,record,row['venue'],row['year'],row['track'],external,row.get('evidence_url',row['url']),row['status'],row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status']),row.get('venue_aliases')) editions.add(edition) diff --git a/index_core.py b/index_core.py index 5c052d8..7227a30 100644 --- a/index_core.py +++ b/index_core.py @@ -109,6 +109,22 @@ class Store: c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(paper,other),max(paper,other),'exact_title_unconfirmed',None,json.dumps({'title':title}), 'pending')) return paper,record + def record_tags(self,paper,record,source,tags): + """Refresh this record's platform tags without removing other evidence.""" + values=set() + for tag in tags: + namespace,value=tag['namespace'],tag['value'] + if not isinstance(namespace,str) or not namespace.startswith(source+':') or not isinstance(value,str) or not value.strip(): + raise ValueError('Platform tag must have its own source namespace and a nonempty text value') + values.add((namespace,value.strip())) + evidence=f'{source}:record:{record}:tags' + c=self.conn + c.execute('DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key=?',(record,evidence)) + for namespace,value in sorted(values): + c.execute('INSERT OR IGNORE INTO tags(namespace,value) VALUES(?,?)',(namespace,value)) + tag=c.execute('SELECT id FROM tags WHERE namespace=? AND value=?',(namespace,value)).fetchone()[0] + c.execute('INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,evidence)) + def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None): c = self.conn c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified')) diff --git a/test_iclr.py b/test_iclr.py new file mode 100644 index 0000000..d9f577e --- /dev/null +++ b/test_iclr.py @@ -0,0 +1,98 @@ +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from urllib.error import HTTPError + +from collect_iclr import asset_urls, collect_year, discover, parse_program +from import_records import ingest_file +from index_core import Store + + +class ICLRTests(unittest.TestCase): + def event(self,identifier=1,sourceid=10,**updates): + row={'id':identifier,'sourceid':sourceid,'sourceurl':'https://openreview.net/group?id=ICLR.cc/2026/Conference', + 'name':'A reliable paper on abstract metadata indexing','authors':[{'fullname':'Jane Doe'},{'fullname':'John Smith'}], + 'eventtype':'Poster','event_type':'Poster','decision':'Accept (Oral)', + 'paper_url':'https://openreview.net/forum?id=realForum','topic':'Learning->Theory','keywords':[]} + return dict(row,**updates) + + def parse(self,events,abstracts=None,year=2026): + return parse_program({'count':len(events),'next':None,'previous':None,'results':events},abstracts or {},year,f'https://iclr.cc/virtual/{year}/papers.html') + + def test_oral_placeholder_is_not_an_identifier_or_another_submission(self): + poster=self.event() + oral=self.event(2,-10,eventtype='Oral',event_type='Oral',paper_url='https://openreview.net/forum?id=2026-Oral--10-placeholder') + rows,audit=self.parse([oral,poster],{'1':'The real abstract.'}) + self.assertEqual(len(rows),1) + self.assertEqual(rows[0]['identifiers'],{'openreview':'realForum'}) + self.assertEqual(rows[0]['abstract'],'The real abstract.') + self.assertEqual(rows[0]['metadata']['presentation_types'],['Oral','Poster']) + self.assertEqual(audit['main_event_count'],2) + + def test_blog_and_journal_presentations_are_not_main_acceptances(self): + rows,audit=self.parse([self.event(),self.event(2,20,sourceurl='https://openreview.net/group?id=ICLR.cc/2026/BlogPosts'),self.event(3,30,sourceurl='2026-TMLR-J2C-Spreadsheet')]) + self.assertEqual(len(rows),1) + self.assertEqual(sum(audit['excluded_events_by_source'].values()),2) + self.assertEqual(audit['missing_abstract_count'],1) + self.assertIsNone(rows[0]['abstract']) + + def test_title_revision_is_scoped_to_matching_signed_submission_and_authors(self): + poster=self.event(name='AlphaEdit: Null-Space Constrained Knowledge Editing for Language Models') + oral=self.event(2,-10,eventtype='Oral',name='AlphaEdit: Null-Space Constrained Model Editing for Language Models') + rows,_=self.parse([oral,poster]) + self.assertEqual(len(rows),1) + self.assertEqual(rows[0]['title'],poster['name']) + self.assertEqual(len(rows[0]['metadata']['presentation_titles']),2) + with self.assertRaisesRegex(ValueError,'collides'): + self.parse([poster,dict(oral,authors=[{'fullname':'Unrelated Author'}])]) + + def test_truncation_and_identity_collision_fail(self): + with self.assertRaisesRegex(ValueError,'count differs'): + parse_program({'count':2,'results':[self.event()]},{},2026,'https://iclr.cc/') + with self.assertRaisesRegex(ValueError,'collides'): + self.parse([self.event(),self.event(2,-10,name='A different paper')]) + with self.assertRaisesRegex(ValueError,'Conflicting identifiers'): + self.parse([self.event(),self.event(2,10,paper_url='https://openreview.net/forum?id=otherForum')]) + + def test_legacy_embedded_abstracts_and_paper_links(self): + row=self.event(sourceurl=None,abstract='Historical abstract',paper_url=None,paper_pdf_url='https://openreview.net/forum?id=historicalForum',decision='Poster') + rows,_=self.parse([row],year=2021) + self.assertEqual(rows[0]['abstract'],'Historical abstract') + self.assertEqual(rows[0]['identifiers'],{'openreview':'historicalForum'}) + rows,_=self.parse([dict(row,paper_pdf_url=None)],year=2020) + self.assertEqual(rows[0]['metadata']['identity_state'],'program_only') + self.assertEqual(rows[0]['identifiers'],{}) + + def test_only_referenced_official_assets_and_real_navigation_years(self): + html='201320262025' + self.assertEqual([y for y,_ in discover(html)],[2013,2026]) + html='start("/static/virtual/data/iclr-2026-orals-posters.json",true,"https://other.example/abstracts.json")' + self.assertEqual(asset_urls(html,'https://iclr.cc/virtual/2026/papers.html',2026),{'orals-posters':'https://iclr.cc/static/virtual/data/iclr-2026-orals-posters.json'}) + + def test_missing_split_abstract_asset_does_not_discard_embedded_abstract(self): + html=b'start("/static/virtual/data/iclr-2021-orals-posters.json",true,"/static/virtual/data/iclr-2021-abstracts.json")' + event=self.event(sourceurl=None,decision='Poster',abstract='Embedded historical abstract') + responses=[html,json.dumps({'count':1,'results':[event]}).encode(),HTTPError('https://iclr.cc/abstracts',404,'Not Found',None,None)] + with tempfile.TemporaryDirectory() as tmp,patch('collect_iclr.fetch',side_effect=responses): + manifest=collect_year(2021,tmp,tmp) + self.assertEqual(manifest['status'],'collected') + self.assertEqual(manifest['missing_abstract_count'],0) + self.assertEqual(manifest['abstract_asset_status'],'unavailable_404') + + def test_import_repeat_preserves_one_submission_and_queryable_platform_tags(self): + rows,_=self.parse([self.event(),self.event(2,-10,eventtype='Oral')],{'1':'A searchable abstract.'}) + with tempfile.TemporaryDirectory() as tmp: + path=Path(tmp)/'iclr.jsonl';db=Path(tmp)/'papers.sqlite' + path.write_text(''.join(json.dumps(r)+'\n' for r in rows)) + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','list_url':'https://iclr.cc/virtual/2026/papers.html','count':1,'status':'collected'})) + ingest_file(db,path);ingest_file(db,path) + store=Store(db);self.addCleanup(store.conn.close) + self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + self.assertEqual(store.conn.execute('SELECT count(*) FROM decisions').fetchone()[0],1) + self.assertEqual(store.conn.execute('SELECT namespace,value FROM tags').fetchall(),[('iclr_program:topic','Learning->Theory')]) + self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(),[]) + + +if __name__=='__main__':unittest.main() diff --git a/test_index_core.py b/test_index_core.py index 87e1fea..aa08bf9 100644 --- a/test_index_core.py +++ b/test_index_core.py @@ -58,6 +58,19 @@ class StoreTests(unittest.TestCase): self.assertEqual(self.store.conn.execute('SELECT title FROM papers').fetchone()[0],'Updated research title') self.assertEqual(self.store.conn.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'searchable'").fetchone()[0],1) + def test_tag_refresh_keeps_other_record_evidence_and_custom_tags(self): + paper,first=self.record('iclr_program','2025/a',{'openreview':'sharedForum'}) + same,second=self.record('iclr_program','2025/b',{'openreview':'sharedForum'}) + self.store.record_tags(paper,first,'iclr_program',[{'namespace':'iclr_program:topic','value':'Shared'}]) + self.store.record_tags(same,second,'iclr_program',[{'namespace':'iclr_program:topic','value':'Shared'}]) + tag=self.store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','Manual')").lastrowid + self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)',(paper,tag,None,'manual')) + self.store.record_tags(paper,first,'iclr_program',[]) + self.assertEqual(self.store.conn.execute('SELECT source_record_id FROM paper_tags ORDER BY source_record_id').fetchall(),[(None,),(second,)]) + with self.assertRaisesRegex(ValueError,'namespace'): + self.store.record_tags(paper,second,'iclr_program',[{'namespace':'custom:topic','value':'No overwrite'}]) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM paper_tags').fetchone()[0],2) + def test_missing_abstract_does_not_become_complete_coverage(self): path=Path(self.tmp.name)/'partial.jsonl' path.write_text(json.dumps({'source':'colm','external_id':'paper1','url':'https://example.org/paper1','title':'Paper','abstract':None,'authors':[],'venue':'COLM','year':2024,'track':'main','status':'accepted'})+'\n') diff --git a/venues.json b/venues.json index 5f7adc6..ca25bc6 100644 --- a/venues.json +++ b/venues.json @@ -11,7 +11,8 @@ "aaai": "https://ojs.aaai.org/index.php/AAAI/issue/archive", "ijcai": "https://www.ijcai.org/all_proceedings/", "dblp": "https://dblp.org/db/conf/", - "colm": "https://proceedings.colm.cc/" + "colm": "https://proceedings.colm.cc/", + "iclr_program": "https://iclr.cc/virtual/2026/papers.html" }, "venues": [ { @@ -20,7 +21,6 @@ "venue_type": "conference", "discovery_sources": [ "openreview", - "pmlr", "neurips" ], "scope": "candidate_pending_scope_review", @@ -36,8 +36,7 @@ "venue_type": "conference", "discovery_sources": [ "openreview", - "pmlr", - "neurips" + "pmlr" ], "scope": "candidate_pending_scope_review", "years": "all_publicly_available", @@ -52,8 +51,7 @@ "venue_type": "conference", "discovery_sources": [ "openreview", - "pmlr", - "neurips" + "iclr_program" ], "scope": "candidate_pending_scope_review", "years": "all_publicly_available", @@ -68,8 +66,7 @@ "venue_type": "conference", "discovery_sources": [ "openreview", - "pmlr", - "neurips" + "colm" ], "scope": "candidate_pending_scope_review", "years": "all_publicly_available",