From 348ebe06959618147c3c8bb09a52c49a9fd04a8e Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Mon, 9 Nov 2026 16:09:13 +0800 Subject: [PATCH 1/3] Collect archived ICLR accepted lists with track and presentation evidence --- .gitea/workflows/check.yml | 1 + README.md | 2 + docs/coverage/iclr-archive-2014-2016.md | 44 +++++ paper_atlas/cli.py | 1 + paper_atlas/collectors/iclr_archive.py | 222 ++++++++++++++++++++++++ tests/test_iclr_archive.py | 144 +++++++++++++++ 6 files changed, 414 insertions(+) create mode 100644 docs/coverage/iclr-archive-2014-2016.md create mode 100644 paper_atlas/collectors/iclr_archive.py create mode 100644 tests/test_iclr_archive.py diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 9ff112c..38fe7b3 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -50,6 +50,7 @@ jobs: subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True) diff --git a/README.md b/README.md index c696d97..d9c201a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追 项目仍在建设中。候选范围包含 43 个会议（含 ACM MM、MLSys），尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准，而不是仓库文件数量。 +历史 ICLR 2014–2016 官方录用页由 `collect iclr-archive` 适配，保留主会/workshop及重复展示证据；运行和身份核验边界见 [历史档案说明](docs/coverage/iclr-archive-2014-2016.md)。 + ## 项目结构 ```text diff --git a/docs/coverage/iclr-archive-2014-2016.md b/docs/coverage/iclr-archive-2014-2016.md new file mode 100644 index 0000000..a097080 --- /dev/null +++ b/docs/coverage/iclr-archive-2014-2016.md @@ -0,0 +1,44 @@ +# ICLR 2014–2016 官方历史录用档案 + +此适配器是长期维护工具的一部分，按需运行。只识别已经独立核对的三种页面布局；后续页面发生变化必须重新核查，不能放松计数或轨道边界以使采集成功。 + +| 官方页面 | 页面链接次数 | 整理后的不同记录 | +|---|---:|---:| +| [2014 Conference Proceedings](https://iclr.cc/archive/2014/conference-proceedings/) | 35 | main 35 | +| [2015 accepted-main](https://iclr.cc/archive/www/doku.php%3Fid=iclr2015:accepted-main.html) | 116 | main 31、workshop 74 | +| [2016 accepted-main](https://iclr.cc/archive/www/doku.php%3Fid=iclr2016:accepted-main.html) | 80 | main 80 | + +2015 的 11 篇 oral 全部再次出现在 poster 段落中。相同年份、轨道及 arXiv 标识的重复展示只生成一条来源记录，但每次展示的标题、作者显示字符串、链接、版本和段落均保留在 `source_variants`。两处标题或解析作者冲突时停止采集、等待人工核查。不同年份或不同轨道不合并投稿。 + +2014 的作者以分号列出，一条有末尾分号，一条标题不在普通段落中；2016 的代码链接不属于作者。保留原文，解析结果不替代来源证据。oral、poster 作为 `iclr_archive:presentation` 标签，不改成独立投稿轨道。arXiv 版本留在来源变体中，稳定标识使用去版本号的编号。 + +## 来源快照与运行 + +2026-11-09 已冻结的官方响应 SHA256： + +- 2014：`a33858176fbffae0066ed3c10310f7dd98a66d12a36334b6339ef1f915ef07af` +- 2015：`131491663742cb084662a4804009b8318e0fa7bc184d4a9044d16dd28f05b888` +- 2016：`7320b8652431793b6ea0743f871ee446bf08b6361ccd4e421b6b53449e71f4f6` + +在服务器运行，输出新目录，不下载论文 PDF： + +```sh +paper-atlas collect iclr-archive --year 2015 \ + --snapshots /data/paper-index/collected/historical-next-source-evidence-v1 \ + --out /data/paper-index/collected/iclr-archive-source-v1/2015 + +# 或重新下载官方 HTML，并保留 HTTP 缓存与收据 +paper-atlas collect iclr-archive --year 2016 \ + --cache /data/paper-index/cache/iclr-archive \ + --out /data/paper-index/collected/iclr-archive-source-v1/2016 +``` + +冻结输入目录必须包含 `receipts.json`，每项有精确 `url`、HTTP `status: 200`、相对 `file`、`bytes`、`sha256` 和带时区的 `finished_at`。读取时重新核对尺寸、哈希和路径；拒绝越界及指向目录外的软链接。已完成输出不可覆盖，应检查报告后另建明确版本目录。解析或计数失败时不生成完成报告，也不导入数据库。 + +输出分轨道 JSONL、manifest、原始 HTML 证据和年度采集报告。仅支持这三年已审查的页面。2014 workshop、其他未列轨道、逐篇摘要及公开拒稿范围尚未通过这些页面核验，不能据此声称完整。缺席不代表 rejected、withdrawn 或从未被录用。 + +## 导入前后的边界 + +这三页合计 220 条不同来源记录，不等于新增 220 篇论文；同一工作可能已存在于 arXiv 或其他会议。先核验来源字段与原始页面，再执行 PostgreSQL 只读预检，比较稳定标识、标题、作者和原始来源。标题/作者差异、版本关联及冲突必须保留证据并明确评估；不能仅靠标题猜测身份，也不能覆盖原有摘要。 + +采集器不连接数据库。正式导入使用既有 `import records`，必须有新的生产保护基线和独立验收；全部导入结束后仍需全库去重和历史覆盖审计。来自官方录用清单的 accepted 是清单观察，不是原始审稿决定签名或决定日期。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 6475087..43248da 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -13,6 +13,7 @@ COMMANDS = { ('download',): ('paper_atlas.io.snapshots', {}, '下载并校验固定版本快照'), ('collect','pmlr'): ('paper_atlas.collectors.pmlr', {}, 'PMLR 官方元数据'), ('collect','acl'): ('paper_atlas.collectors.acl', {}, 'ACL Anthology XML'), + ('collect','iclr-archive'): ('paper_atlas.collectors.iclr_archive', {}, '核对ICLR 2014–2016官方历史录用档案'), ('collect','iclr-program'): ('paper_atlas.collectors.iclr', {}, 'ICLR 官方公开程序'), ('collect','iclr-snapshot'): ('paper_atlas.collectors.openreview', {}, 'ICLR 公开投稿与决定快照'), ('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'), diff --git a/paper_atlas/collectors/iclr_archive.py b/paper_atlas/collectors/iclr_archive.py new file mode 100644 index 0000000..9901bc9 --- /dev/null +++ b/paper_atlas/collectors/iclr_archive.py @@ -0,0 +1,222 @@ +"""Frozen official ICLR 2014–2016 lists; presentation evidence is not a track.""" +import argparse +from collections import Counter, defaultdict +import fcntl +import hashlib +import json +from pathlib import Path +import re +import tempfile +from urllib.parse import urlsplit + +from paper_atlas.db.submissions import timestamp +from paper_atlas.io.html import Document, Node, clean +from paper_atlas.io.http import fetch + +SOURCE = 'iclr_archive' +PAGES = { + 2014: ('https://iclr.cc/archive/2014/conference-proceedings/', {'main': 35}), + 2015: ('https://iclr.cc/archive/www/doku.php%3Fid=iclr2015:accepted-main.html', {'main': 31, 'workshop': 74}), + 2016: ('https://iclr.cc/archive/www/doku.php%3Fid=iclr2016:accepted-main.html', {'main': 80}), +} +SECTIONS = { + 2015: {'Main Conference - Oral Presentations': ('main', 'oral'), + 'Main Conference - Poster Presentations': ('main', 'poster'), + 'Workshop Papers': ('workshop', 'workshop')}, + 2016: {'Oral Presentations': ('main', 'oral'), 'Poster Presentations': ('main', 'poster')}, +} + + +def arxiv_link(url): + parsed = urlsplit(url) + if parsed.hostname != 'arxiv.org': + return None + match = re.fullmatch(r'/(?:abs|pdf)/(\d{4}\.\d{4,5})(v[1-9]\d*)?(?:\.pdf)?', parsed.path) + if (parsed.scheme not in ('http', 'https') or parsed.netloc != 'arxiv.org' + or parsed.query or parsed.fragment or not match): + raise ValueError('Unexpected official arXiv link') + return match[1], match[2] or None + + +def without_links(node): + if node.tag in ('a', 'script', 'style'): + return '' + value = ''.join(without_links(n) if isinstance(n, Node) else n for n in node.children) + return ' '+value+' ' if node.tag in ('div', 'li', 'p', 'br') else value + + +def authors(display, year): + parts = display.rstrip(';').split(';') if year == 2014 else re.split(r',\s*(?:and\s+)?|\s+and\s+', display) + names = [clean(p) for p in parts] + if not names or any(not name for name in names): + raise ValueError('Missing author or unsupported author delimiter') + return names + + +def parse_page(html, year, evidence): + if year not in PAGES: + raise ValueError('Unreviewed ICLR archive year') + document = Document(html) + nodes = list(document.root.walk()) + headings = [clean(n.text()) for n in nodes if n.tag in ('h1', 'h2', 'h3')] + if year == 2014: + if 'Conference Proceedings' not in headings or 'conference papers accepted' not in clean(document.root.text()): + raise ValueError('Unexpected ICLR 2014 accepted list') + elif (set(SECTIONS[year]) - set(headings) + or (year == 2016 and 'Accepted Papers (Conference Track)' not in headings)): + raise ValueError('Unexpected ICLR accepted sections') + entries = [] + section = None + for index, node in enumerate(nodes): + if node.tag in ('h1', 'h2', 'h3'): + section = clean(node.text()) + if year == 2014: + if node.tag != 'a' or not arxiv_link(node.attrs.get('href', '')): + continue + link = node + # This archived Google Sites page has an entry outside a paragraph. + # Author spans are explicit italic text, before the next paper link. + display = None + for following in nodes[index+sum(1 for _ in link.walk()):]: + if following.tag == 'a' and arxiv_link(following.attrs.get('href', '')): + break + if (following.tag in ('i', 'em') or 'font-style:italic' in following.attrs.get('style', '').replace(' ', '')): + text = clean(following.text()) + if text: + display = text + break + if not display: + raise ValueError('Missing explicit ICLR 2014 author span') + track, presentation = 'main', 'conference' + else: + if node.tag != 'li': + continue + links = [n for n in node.walk() if n.tag == 'a' and arxiv_link(n.attrs.get('href', ''))] + if not links: + continue + if len(links) != 1 or section not in SECTIONS[year]: + raise ValueError('Ambiguous paper link or unknown accepted section') + link, = links + display = clean(without_links(node)).lstrip(',').strip() + track, presentation = SECTIONS[year][section] + arxiv, version = arxiv_link(link.attrs['href']) + title = clean(link.text()) + if not title: + raise ValueError('Empty official title') + entries.append({'arxiv': arxiv, 'version': version, 'title': title, + 'authors': authors(display, year), 'authors_display': display, + 'paper_url': link.attrs['href'], 'track': track, + 'section': section, 'presentation': presentation}) + # Check every arXiv occurrence, including links outside the expected layout. + occurrences = sum(n.tag == 'a' and bool(arxiv_link(n.attrs.get('href', ''))) for n in nodes) + if len(entries) != occurrences or not entries: + raise ValueError('Some official paper links were not parsed') + grouped = defaultdict(list) + for entry in entries: + grouped[(entry['track'], entry['arxiv'])].append(entry) + result = [] + for (track, arxiv), variants in grouped.items(): + first = variants[0] + if any((v['title'], v['authors']) != (first['title'], first['authors']) for v in variants): + raise ValueError('Repeated presentation has conflicting title or authors; review required') + presentations = sorted({v['presentation'] for v in variants}) + result.append({'source': SOURCE, 'external_id': f'{year}/{track}/{arxiv}', + 'url': PAGES[year][0], 'title': first['title'], 'abstract': None, + 'authors': first['authors'], 'identifiers': {'arxiv': arxiv}, + 'venue': 'ICLR', 'venue_aliases': [], 'year': year, 'track': track, + 'status': 'accepted', 'observed_at': evidence['observed_at'], + 'raw_decision': 'listed in official ICLR accepted '+track+' list', + 'metadata': {'source_evidence': evidence, 'source_variants': variants, + 'presentations': presentations, 'decision_date_precision': 'not_supplied', + 'identity_note': 'Official arXiv link, not title matching; validate canonical identity before import.', + 'decision_evidence': 'Archived accepted list, not an original decision note.'}, + 'tags': [{'namespace': SOURCE+':presentation', 'value': value} for value in presentations]}) + return result + + +def read_page(year, *, cache=None, snapshots=None): + if bool(cache) == bool(snapshots): + raise ValueError('Choose exactly one cache or frozen snapshot directory') + url = PAGES[year][0] + if snapshots: + root = Path(snapshots).resolve() + receipts = json.loads((root/'receipts.json').read_bytes())['receipts'] + matches = [r for r in receipts if r['url'] == url] + if len(matches) != 1: + raise ValueError('Expected one exact official response receipt') + receipt, = matches + filename = Path(receipt['file']) + path = root/filename + if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root) + or receipt['status'] != 200): + raise ValueError('Unsafe or unsuccessful archive snapshot') + data = path.read_bytes() + if (len(data) > 8*1024**2 or len(data) != receipt['bytes'] + or hashlib.sha256(data).hexdigest() != receipt['sha256']): + raise ValueError('Archive snapshot checksum or size differs') + observed = timestamp(receipt['finished_at']) + else: + data = fetch(url, cache, 8*1024**2, delay=1) + receipt = json.loads((Path(cache)/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_bytes()) + observed = timestamp(receipt['retrieved_at']) + evidence = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data), + 'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html', + 'observed_at': observed, 'http_status': 200, 'response_receipt': receipt} + return data, evidence + + +def collect(year, out, *, cache=None, snapshots=None): + if year not in PAGES: + raise ValueError('Only independently reviewed ICLR archive years are supported') + out = Path(out) + out.mkdir(parents=True, exist_ok=True) + with (out/'iclr-archive.collect.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + report = out/f'iclr-archive-{year}.json' + if report.exists(): + raise FileExistsError('Existing completed archive output must be inspected, not overwritten') + data, evidence = read_page(year, cache=cache, snapshots=snapshots) + records = parse_page(data.decode('utf-8'), year, evidence) + counts = Counter(r['track'] for r in records) + if dict(counts) != PAGES[year][1]: + raise ValueError('Distinct archive entries differ from reviewed inventory') + files = [] + with tempfile.TemporaryDirectory(prefix='.iclr-archive-', dir=out) as temporary: + stage = Path(temporary) + (stage/'evidence').mkdir() + (stage/evidence['file']).write_bytes(data) + for track in sorted(counts): + group = [r for r in records if r['track'] == track] + path = stage/f'iclr-archive-{year}-{track}.jsonl' + path.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group)) + manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICLR', 'year': year, + 'track': track, 'count': len(group), 'file': str(out/path.name), + 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(), + 'list_url': PAGES[year][0], 'status': 'partial', 'missing_abstract_count': len(group)} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n') + files.append(manifest) + result = {'source': SOURCE, 'year': year, 'records': len(records), 'files': files, + 'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False, + 'status': 'collected_pending_identity_review', + 'note': 'Accepted list only. Oral/poster are presentation evidence; absent papers are not rejected. ' + 'ICLR2014 workshop and other unlisted tracks remain outside these reviewed pages.'} + (stage/report.name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n') + (out/'evidence').mkdir(exist_ok=True) + for path in (stage/'evidence').iterdir(): + path.replace(out/'evidence'/path.name) + for path in stage.iterdir(): + if path.is_file() and path.name != report.name: + path.replace(out/path.name) + (stage/report.name).replace(report) + return result + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog) + parser.add_argument('--year', type=int, choices=sorted(PAGES), required=True) + parser.add_argument('--out', required=True) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument('--cache') + source.add_argument('--snapshots') + args = parser.parse_args(argv) + print(json.dumps(collect(args.year, args.out, cache=args.cache, snapshots=args.snapshots), ensure_ascii=False)) diff --git a/tests/test_iclr_archive.py b/tests/test_iclr_archive.py new file mode 100644 index 0000000..9375ba6 --- /dev/null +++ b/tests/test_iclr_archive.py @@ -0,0 +1,144 @@ +"""Historical accepted lists, repeated presentations and preserved work histories.""" +import hashlib +import json +from pathlib import Path +import sqlite3 +import tempfile +import unittest +from unittest.mock import patch + +from paper_atlas.collectors import iclr_archive as archive +from paper_atlas.db.store import Store +from paper_atlas.ingestion.records import ingest_file + +OBSERVED = '2026-11-09T00:00:00Z' + + +def entry(identifier, title='A shared research paper', display='Jane Doe and Bob Smith'): + return f'
  • {title}, {display}
    • ' + + +def page2015(): + return ('

    Main Conference - Oral Presentations

      '+entry('1412.0001v2')+'
    ' + '

    Main Conference - Poster Presentations

      '+entry('1412.0001v2')+'
    ' + '

    Workshop Papers

      '+entry('1412.0002', 'A workshop paper')+'
    ') + + +class IclrArchiveTests(unittest.TestCase): + def frozen(self, root, year, html): + root.mkdir() + data = html.encode() + (root/'page.html').write_bytes(data) + receipt = {'url': archive.PAGES[year][0], 'file': 'page.html', 'status': 200, + 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'finished_at': OBSERVED} + (root/'receipts.json').write_text(json.dumps({'receipts': [receipt]})) + return root + + def test_oral_poster_repeat_is_one_attempt_with_all_source_variants(self): + rows = archive.parse_page(page2015(), 2015, {'observed_at': OBSERVED}) + main, workshop = rows + self.assertEqual((main['track'], workshop['track']), ('main', 'workshop')) + self.assertEqual(main['identifiers'], {'arxiv': '1412.0001'}) + self.assertEqual(main['external_id'], '2015/main/1412.0001') + self.assertEqual(main['metadata']['presentations'], ['oral', 'poster']) + self.assertEqual(len(main['metadata']['source_variants']), 2) + self.assertEqual(main['metadata']['source_variants'][0]['version'], 'v2') + self.assertEqual(main['authors'], ['Jane Doe', 'Bob Smith']) + self.assertTrue(all(r['status'] == 'accepted' and r['abstract'] is None for r in rows)) + + def test_2014_author_span_outside_paragraph_and_trailing_separator(self): + html = ('

    Conference Proceedings

    conference papers accepted

    ' + '
    A research title
    ' + '

    Jane Doe; Bob Smith;

    ') + row, = archive.parse_page(html, 2014, {'observed_at': OBSERVED}) + self.assertEqual(row['title'], 'A research title') + self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith']) + self.assertEqual(row['metadata']['source_variants'][0]['authors_display'], 'Jane Doe; Bob Smith;') + self.assertEqual(row['track'], 'main') + + def test_same_arxiv_in_different_tracks_keeps_separate_source_attempts(self): + html = page2015().replace('1412.0002', '1412.0001v3') + rows = archive.parse_page(html, 2015, {'observed_at': OBSERVED}) + self.assertEqual(len(rows), 2) + self.assertEqual({r['external_id'] for r in rows}, {'2015/main/1412.0001', '2015/workshop/1412.0001'}) + self.assertEqual(rows[1]['metadata']['source_variants'][0]['version'], 'v3') + + def test_2016_code_links_and_breaks_do_not_become_authors(self): + html = ('

    Accepted Papers (Conference Track)

    Oral Presentations

      ' + '
    1. A title' + '[code]
      Jane Doe, Bob Smith
    ' + '

    Poster Presentations

    ') + row, = archive.parse_page(html, 2016, {'observed_at': OBSERVED}) + self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith']) + self.assertEqual(row['metadata']['presentations'], ['oral']) + + def test_conflicting_repeat_unknown_sections_and_unparsed_links_fail(self): + variants = [page2015().replace('Poster Presentations', 'Other Presentations'), + page2015().replace('', 'stray', 1), + page2015().replace('Jane Doe and Bob Smith', 'Different Author', 1)] + for html in variants: + with self.subTest(html=html), self.assertRaises(ValueError): + archive.parse_page(html, 2015, {'observed_at': OBSERVED}) + + def test_arxiv_versions_are_preserved_and_unsafe_links_rejected(self): + self.assertEqual(archive.arxiv_link('https://arxiv.org/pdf/1511.00001v3.pdf'), ('1511.00001', 'v3')) + self.assertIsNone(archive.arxiv_link('https://arxiv.org.example.org/abs/1511.00001')) + for url in ['https://arxiv.org/abs/1511.00001?other=1', 'ftp://arxiv.org/abs/1511.00001', + 'https://user@arxiv.org/abs/1511.00001', 'https://arxiv.org/abs/not-a-paper']: + with self.subTest(url=url), self.assertRaises(ValueError): + archive.arxiv_link(url) + + def test_receipt_hash_status_path_and_symlink_cannot_escape_snapshot(self): + for failure in ['hash', 'status', 'path', 'symlink']: + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = self.frozen(root/'source', 2015, page2015()) + file = source/'receipts.json' + data = json.loads(file.read_text()) + receipt = data['receipts'][0] + if failure == 'hash': receipt['sha256'] = '0'*64 + elif failure == 'status': receipt['status'] = 404 + elif failure == 'path': receipt['file'] = '../outside.html' + else: + (root/'outside.html').write_text(page2015()) + (source/'page.html').unlink() + (source/'page.html').symlink_to(root/'outside.html') + file.write_text(json.dumps(data)) + with self.assertRaises(ValueError): + archive.read_page(2015, snapshots=source) + + def test_changed_inventory_leaves_no_completed_output(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = self.frozen(root/'source', 2015, page2015()) + with self.assertRaises(ValueError): + archive.collect(2015, root/'out', snapshots=source) + self.assertFalse((root/'out/iclr-archive-2015.json').exists()) + + def test_native_import_keeps_rejected_to_accepted_history_and_abstract(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = self.frozen(root/'source', 2015, page2015()) + with patch.dict(archive.PAGES, {2015: (archive.PAGES[2015][0], {'main': 1, 'workshop': 1})}): + result = archive.collect(2015, root/'out', snapshots=source) + with self.assertRaises(FileExistsError): + archive.collect(2015, root/'out', snapshots=source) + db = root/'sample.sqlite' + store = Store(db) + pid, rid = store.record('arxiv', '1412.0001', 'https://arxiv.org/abs/1412.0001', + 'A shared research paper', 'Keep this canonical abstract.', + ['Jane Doe', 'Bob Smith'], {'arxiv': '1412.0001'}, {}) + store.decision(pid, rid, 'Earlier Conference', 2014, 'main', 'original-attempt', + 'https://example.org/rejected', 'rejected', 'explicit earlier decision', observed_at=OBSERVED) + store.conn.commit() + store.conn.close() + for manifest in result['files']: + ingest_file(db, manifest['file']) + ingest_file(db, manifest['file']) + with sqlite3.connect(db) as c: + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 2) + self.assertEqual(c.execute('SELECT abstract FROM papers WHERE id=?', (pid,)).fetchone()[0], 'Keep this canonical abstract.') + history = c.execute('SELECT v.acronym,e.year,d.status FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE s.paper_id=? ORDER BY e.year', (pid,)).fetchall() + self.assertEqual(history, [('Earlier Conference', 2014, 'rejected'), ('ICLR', 2015, 'accepted')]) + self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 3) + self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), []) From 81864e48ae69f87ac230fcca306e8f26475a1263 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Mon, 9 Nov 2026 16:19:19 +0800 Subject: [PATCH 2/3] Preserve complete fragmented ICLR 2014 author paragraphs --- docs/coverage/iclr-archive-2014-2016.md | 2 +- paper_atlas/collectors/iclr_archive.py | 7 +++++-- tests/test_iclr_archive.py | 7 ++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/coverage/iclr-archive-2014-2016.md b/docs/coverage/iclr-archive-2014-2016.md index a097080..958a8be 100644 --- a/docs/coverage/iclr-archive-2014-2016.md +++ b/docs/coverage/iclr-archive-2014-2016.md @@ -10,7 +10,7 @@ 2015 的 11 篇 oral 全部再次出现在 poster 段落中。相同年份、轨道及 arXiv 标识的重复展示只生成一条来源记录，但每次展示的标题、作者显示字符串、链接、版本和段落均保留在 `source_variants`。两处标题或解析作者冲突时停止采集、等待人工核查。不同年份或不同轨道不合并投稿。 -2014 的作者以分号列出，一条有末尾分号，一条标题不在普通段落中；2016 的代码链接不属于作者。保留原文，解析结果不替代来源证据。oral、poster 作为 `iclr_archive:presentation` 标签，不改成独立投稿轨道。arXiv 版本留在来源变体中，稳定标识使用去版本号的编号。 +2014 的作者以分号列出，一条有末尾分号，一条标题不在普通段落中，部分作者段落拆成多个格式片段；必须读取完整作者段落，不能只读首个斜体片段。2016 的代码链接不属于作者。保留原文，解析结果不替代来源证据。oral、poster 作为 `iclr_archive:presentation` 标签，不改成独立投稿轨道。arXiv 版本留在来源变体中，稳定标识使用去版本号的编号。 ## 来源快照与运行 diff --git a/paper_atlas/collectors/iclr_archive.py b/paper_atlas/collectors/iclr_archive.py index 9901bc9..62c060e 100644 --- a/paper_atlas/collectors/iclr_archive.py +++ b/paper_atlas/collectors/iclr_archive.py @@ -75,12 +75,15 @@ def parse_page(html, year, evidence): continue link = node # This archived Google Sites page has an entry outside a paragraph. - # Author spans are explicit italic text, before the next paper link. + # Authors can span several differently formatted italic spans. + # Read their whole paragraph, never only the first styled fragment. display = None for following in nodes[index+sum(1 for _ in link.walk()):]: if following.tag == 'a' and arxiv_link(following.attrs.get('href', '')): break - if (following.tag in ('i', 'em') or 'font-style:italic' in following.attrs.get('style', '').replace(' ', '')): + if following.tag == 'p' and any( + n.tag in ('i', 'em') or 'font-style:italic' in n.attrs.get('style', '').replace(' ', '') + for n in following.walk()): text = clean(following.text()) if text: display = text diff --git a/tests/test_iclr_archive.py b/tests/test_iclr_archive.py index 9375ba6..bf26990 100644 --- a/tests/test_iclr_archive.py +++ b/tests/test_iclr_archive.py @@ -49,11 +49,12 @@ class IclrArchiveTests(unittest.TestCase): def test_2014_author_span_outside_paragraph_and_trailing_separator(self): html = ('

    Conference Proceedings

    conference papers accepted

    ' '
    A research title
    ' - '

    Jane Doe; Bob Smith;

    ') + '

    Jane Doe; ' + 'Bob Smith; Carol Wu;

    ') row, = archive.parse_page(html, 2014, {'observed_at': OBSERVED}) self.assertEqual(row['title'], 'A research title') - self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith']) - self.assertEqual(row['metadata']['source_variants'][0]['authors_display'], 'Jane Doe; Bob Smith;') + self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith', 'Carol Wu']) + self.assertEqual(row['metadata']['source_variants'][0]['authors_display'], 'Jane Doe; Bob Smith; Carol Wu;') self.assertEqual(row['track'], 'main') def test_same_arxiv_in_different_tracks_keeps_separate_source_attempts(self): From b5c0618be56aa927f1f6f1146e2d5a08813d36bd Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Mon, 9 Nov 2026 16:24:42 +0800 Subject: [PATCH 3/3] Handle standalone italic author blocks in the historical archive --- docs/coverage/iclr-archive-2014-2016.md | 2 +- paper_atlas/collectors/iclr_archive.py | 4 ++-- tests/test_iclr_archive.py | 7 +++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/coverage/iclr-archive-2014-2016.md b/docs/coverage/iclr-archive-2014-2016.md index 958a8be..36f92f1 100644 --- a/docs/coverage/iclr-archive-2014-2016.md +++ b/docs/coverage/iclr-archive-2014-2016.md @@ -10,7 +10,7 @@ 2015 的 11 篇 oral 全部再次出现在 poster 段落中。相同年份、轨道及 arXiv 标识的重复展示只生成一条来源记录，但每次展示的标题、作者显示字符串、链接、版本和段落均保留在 `source_variants`。两处标题或解析作者冲突时停止采集、等待人工核查。不同年份或不同轨道不合并投稿。 -2014 的作者以分号列出，一条有末尾分号，一条标题不在普通段落中，部分作者段落拆成多个格式片段；必须读取完整作者段落，不能只读首个斜体片段。2016 的代码链接不属于作者。保留原文，解析结果不替代来源证据。oral、poster 作为 `iclr_archive:presentation` 标签，不改成独立投稿轨道。arXiv 版本留在来源变体中，稳定标识使用去版本号的编号。 +2014 的作者以分号列出，一条有末尾分号，一条标题和作者斜体块不在普通段落中，部分作者段落拆成多个格式片段；必须读取完整作者段落或独立斜体块，不能只读首个格式片段。2016 的代码链接不属于作者。保留原文，解析结果不替代来源证据。oral、poster 作为 `iclr_archive:presentation` 标签，不改成独立投稿轨道。arXiv 版本留在来源变体中，稳定标识使用去版本号的编号。 ## 来源快照与运行 diff --git a/paper_atlas/collectors/iclr_archive.py b/paper_atlas/collectors/iclr_archive.py index 62c060e..98326c5 100644 --- a/paper_atlas/collectors/iclr_archive.py +++ b/paper_atlas/collectors/iclr_archive.py @@ -81,9 +81,9 @@ def parse_page(html, year, evidence): for following in nodes[index+sum(1 for _ in link.walk()):]: if following.tag == 'a' and arxiv_link(following.attrs.get('href', '')): break - if following.tag == 'p' and any( + if following.tag in ('i', 'em') or (following.tag == 'p' and any( n.tag in ('i', 'em') or 'font-style:italic' in n.attrs.get('style', '').replace(' ', '') - for n in following.walk()): + for n in following.walk())): text = clean(following.text()) if text: display = text diff --git a/tests/test_iclr_archive.py b/tests/test_iclr_archive.py index bf26990..94e4407 100644 --- a/tests/test_iclr_archive.py +++ b/tests/test_iclr_archive.py @@ -50,12 +50,15 @@ class IclrArchiveTests(unittest.TestCase): html = ('

    Conference Proceedings

    conference papers accepted

    ' '
    A research title
    ' '

    Jane Doe; ' - 'Bob Smith; Carol Wu;

    ') - row, = archive.parse_page(html, 2014, {'observed_at': OBSERVED}) + 'Bob Smith; Carol Wu;

    ' + '
    Another title' + '
    First Author; Second Author
    ') + row, unwrapped = archive.parse_page(html, 2014, {'observed_at': OBSERVED}) self.assertEqual(row['title'], 'A research title') self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith', 'Carol Wu']) self.assertEqual(row['metadata']['source_variants'][0]['authors_display'], 'Jane Doe; Bob Smith; Carol Wu;') self.assertEqual(row['track'], 'main') + self.assertEqual(unwrapped['authors'], ['First Author', 'Second Author']) def test_same_arxiv_in_different_tracks_keeps_separate_source_attempts(self): html = page2015().replace('1412.0002', '1412.0001v3')