diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 06f38e3..5e15e55 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -52,6 +52,16 @@ jobs: subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','dblp-family','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True) + aamas2026 = json.loads(gzip.decompress((src/'tests/fixtures/aamas2026-fixture.json.gz').read_bytes())) + aamas2026_snapshots = root/'aamas2026-snapshots' + aamas2026_snapshots.mkdir() + (aamas2026_snapshots/'contents.html').write_text(aamas2026['contents']) + (aamas2026_snapshots/'receipts.json').write_text(json.dumps(aamas2026['receipts'])) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2026','--snapshots',str(aamas2026_snapshots),'--out',str(root/'installed-aamas2026')],cwd=outside,check=True) + aamas2026_report = json.loads((root/'installed-aamas2026/aamas-proceedings-2026.json').read_text()) + assert aamas2026_report['records'] == 639 and len(aamas2026_report['held_publications']) == 1 + assert aamas2026_report['status_counts'] == {'accepted': 638, 'withdrawn': 1} + assert not aamas2026_report['database_written'] and not aamas2026_report['pdfs_downloaded'] subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True) diff --git a/README.md b/README.md index 4fe12d1..62f947f 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追 当前主动补漏从 **2020 年**起；2000 年起已发现的名单、缺口与待办保存在[补漏恢复清单](docs/coverage/backfill-handoff-2026-11-10.md)，下次从这里接续。已有旧数据保留，范围排除不算补齐。 +AAMAS 2026 的官方 HTML 目录由 `collect aamas-proceedings --year 2026` 适配，区分 track 和发表形式，keynote 排除、撤稿证据单独保留；[采集与恢复说明](docs/aamas-proceedings.md)。采集不写数据库，仍需身份核验和增量导入。 + 历史 ICLR 2014–2016 官方录用页由 `collect iclr-archive` 适配，保留主会/workshop及重复展示证据；运行和身份核验边界见 [历史档案说明](docs/coverage/iclr-archive-2014-2016.md)。 `collect springer-selected` 核验限定的会后选集，分开保存会议年份、出版年份及选集事件身份。目前支持 AISTATS 1993 对应的 1994 年出版选集，原始会议决定仍须单独核查；配置、证据和恢复见 [会后选集说明](docs/springer-selected-volumes.md)。 diff --git a/docs/aamas-proceedings.md b/docs/aamas-proceedings.md index 261737e..04ed984 100644 --- a/docs/aamas-proceedings.md +++ b/docs/aamas-proceedings.md @@ -1,6 +1,31 @@ # AAMAS 官方论文集元数据 -`paper_atlas.collectors.aamas` 是按需运行的采集适配器，通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025** HTML 版式；其他年份需先核对官方目录和逐篇结构，再增加适配，不能仅替换年份。2025 解析单独封装在 `paper_atlas.collectors.aamas2025`。 +`paper_atlas.collectors.aamas` 是按需运行的采集适配器，通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025、2026** HTML 版式；其他年份需先核对官方目录和逐篇结构，再增加适配，不能仅替换年份。2025、2026 解析分别封装在 `paper_atlas.collectors.aamas2025`、`paper_atlas.collectors.aamas2026`。 + + +## AAMAS 2026 + +2026 只读取[官方 HTML 目录](https://www.ifaamas.org/Proceedings/aamas2026/forms/contents.htm)，不下载 PDF。冻结目录 SHA256 `ce2bc7fc5da340930343901506a9ce37c0334d4068f07546431c9deee3f13de9`，643 个条目：keynote 3、research 338、extended abstract 185、AAAI 19、Blue Sky 17、doctoral 34、demo 44、JAAMAS 3。 + +排除 3 个 keynote；research 的 p238 明确标注 `Retracted June 8, 2026`，完整题名、作者、栏目、日期、原文和来源回执保存在采集报告 `held_publications` 中，**不会输出 accepted 或 withdrawn 投稿记录**。当前决定模型没有独立 retracted 状态，后续须单独建模和核验；原始证据不能删除。任何新增或改变的撤稿标记都会停止采集，等待人工核查。 + +另一个 extended abstract p3178 明确标注 `Withdrawn June 8, 2026`，输出 `withdrawn`，在 metadata 保留日期 `2026-06-08` 和日精度；原网页没有时区/时刻，不能伪造决定 timestamp，不补造其此前的接受决定或 PDF 链接。因此 639 个输出条目是 **638 accepted、1 withdrawn**，按六个 track 分组：`proceedings_unspecified` 522、`aaai_transfer` 19、`blue_sky` 17、`doctoral` 34、`demo` 44、`jaamas` 3。全文/扩展摘要是发表形式，不是研究类型；research 栏目不能证明每篇原始 main/special 投稿轨道。JAAMAS 保留期刊展示语义，不制造另一次独立审稿录用。 + +[AAAI 转投详细政策](https://cyprusconferences.org/aamas2026/call-for-papers-aaai-track/) 要求基于 AAAI-26 rejected 稿件；[总览](https://cyprusconferences.org/aamas2026/calls/) 却写 AAAI-25，须保留冲突。采集器保留 AAAI 栏目归属，不据此生成原 AAAI 投稿 ID、年份或逐篇拒稿决定。目录不是原始决定 note，未来关联仍需逐篇证据。 + +```sh +paper-atlas collect aamas-proceedings --year 2026 \ + --snapshots /data/paper-index/collected/aamas2026-primary-evidence-v1 \ + --out /data/paper-index/collected/aamas2026-normalized-v1 +``` + +在线模式将 `--snapshots` 换成 `--cache /data/paper-index/cache/aamas2026`。冻结模式 `receipts.json` 的 `receipts` 至少包含官方目录 URL、相对 `file`、`bytes`、`sha256`、`status=200` 和带时区 `finished_at`。拒绝越界路径、越界符号链接、失败状态、哈希/大小不符的页面。这里只读取目录对应回执，其他已冻结政策网页继续保留。 + +输出六组 JSONL/manifest、原始 HTML 与最后发布的 `aamas-proceedings-2026.json`；统一入口沿用输出目录锁和暂存目录。manifest 全部为 `partial`，639 条 HTML 摘要缺失如实记录。发布编号使用年份和起始页，随机 PDF 文档 ID 单独保留，不能沿用 2025 的 pN.pdf 规则。 + +首次部署先核对来源哈希和生产状态，再在新输出目录运行采集、独立比对 643 条标题/作者/栏目分区，随后做 PostgreSQL 身份预检与增量导入。采集、远程 CI 通过或目录含某个年份均不代表生产已导入、摘要齐全或最终去重完成。较早年份不自动启动补采，当前主动补漏范围按用户要求从 2020 年起。 + +## AAMAS 2025 2025 版式使用[官方首页](https://www.ifaamas.org/Proceedings/aamas2025/)和[出版目录](https://www.ifaamas.org/Proceedings/aamas2025/forms/contents.htm)。原始目录共 479 条：3 keynote、260 research paper、148 extended abstract、12 Blue Sky、25 doctoral、22 Demo 栏目、9 JAAMAS。排除 keynote，输出 476 条；Demo 栏目中的 p3000 是明确标题的 COMP24 竞赛报告，发表年份仍为 2025，格式记 `competition_report`、投稿 track 留作 `proceedings_unspecified`，保留原 Demo 栏目证据。其余输出按 track 分组为 409 unspecified、12 blue_sky、25 doctoral、21 demo、9 jaamas。 diff --git a/paper_atlas/collectors/aamas.py b/paper_atlas/collectors/aamas.py index 039cdd8..e5b5976 100644 --- a/paper_atlas/collectors/aamas.py +++ b/paper_atlas/collectors/aamas.py @@ -136,6 +136,9 @@ def read_pages(cache=None, snapshots=None): def _collect(year, out, *, cache=None, snapshots=None): + if year == 2026: + from paper_atlas.collectors.aamas2026 import collect as collect_2026 + return collect_2026(out, cache=cache, snapshots=snapshots) if year == 2025: from paper_atlas.collectors.aamas2025 import collect as collect_2025 return collect_2025(out, cache=cache, snapshots=snapshots) @@ -193,8 +196,8 @@ def _collect(year, out, *, cache=None, snapshots=None): def collect(year, out, *, cache=None, snapshots=None): - if year not in (2012, 2025): - raise ValueError('Only independently reviewed AAMAS 2012/2025 layouts are supported') + if year not in (2012, 2025, 2026): + raise ValueError('Only independently reviewed AAMAS 2012/2025/2026 layouts are supported') out = Path(out) out.mkdir(parents=True, exist_ok=True) with (out/'aamas.collect.lock').open('a') as lock: @@ -204,7 +207,7 @@ def collect(year, out, *, cache=None, snapshots=None): def main(argv=None, *, prog=None): parser = argparse.ArgumentParser(prog=prog) - parser.add_argument('--year', type=int, choices=[2012, 2025], required=True) + parser.add_argument('--year', type=int, choices=[2012, 2025, 2026], required=True) parser.add_argument('--out', required=True) source = parser.add_mutually_exclusive_group(required=True) source.add_argument('--cache') diff --git a/paper_atlas/collectors/aamas2026.py b/paper_atlas/collectors/aamas2026.py new file mode 100644 index 0000000..9c47b45 --- /dev/null +++ b/paper_atlas/collectors/aamas2026.py @@ -0,0 +1,219 @@ +"""Reviewed AAMAS 2026 HTML metadata; retractions stay outside accepted queues.""" +from collections import Counter, defaultdict +import hashlib +import json +from pathlib import Path +import re +import tempfile + +from paper_atlas.collectors.aamas2025 import one +from paper_atlas.db.submissions import timestamp +from paper_atlas.io.html import Document, Node, clean +from paper_atlas.io.http import fetch + +ROOT = 'https://www.ifaamas.org/Proceedings/aamas2026/' +URL = ROOT+'forms/contents.htm' +TITLE = 'AAMAS 2026: Proceedings of the 2026 International Conference on Autonomous Agents and Multiagent Systems' +SECTIONS = { + 'K': ('Keynote Talks', 'keynote', 'keynote', 3), + 'R': ('Research Paper Track', 'full_paper', 'proceedings_unspecified', 338), + '1': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 185), + 'A': ('AAAI Track', 'full_paper', 'aaai_transfer', 19), + 'B': ('Blue Sky Ideas Track', 'blue_sky_paper', 'blue_sky', 17), + 'C': ('Doctoral Consortium', 'doctoral_abstract', 'doctoral', 34), + 'D': ('Demonstration Track', 'demo', 'demo', 44), + 'J': ('JAAMAS Track', 'journal_presentation', 'jaamas', 3), +} +PDF = re.compile(r'\.\./pdfs/([A-Z]{4}[0-9]{4})\.pdf') +RETRACTION_TITLE = 'Unifying Model-Free Efficiency and Model-Based Representations via Latent Dynamics' +WITHDRAWAL_TITLE = 'Cognitive Heterogeneity and Behavioral Biases in Multi-Stage Supply Chains: Evidence from LLM-Based Agent Simulations' + + +def parse_authors(paragraph): + font = one([n for n in paragraph.walk() if n.tag == 'font' and + n.attrs.get('color', '').lower() == '#222222'], 'ordered author block') + authors, affiliations = [], [] + name, affiliation = '', '' + def flush(): + nonlocal name, affiliation + if clean(name) or clean(affiliation): + if not clean(name): + raise ValueError('Affiliation without author') + authors.append(clean(name)); affiliations.append(clean(affiliation)) + name, affiliation = '', '' + for child in font.children: + if isinstance(child, str): + name += child + elif child.tag == 'br': + flush() + elif child.tag == 'i': + affiliation += child.text() + else: + raise ValueError('Unreviewed author markup: '+child.tag) + flush() + if not authors: + raise ValueError('Empty author list') + return authors, affiliations + + +def parse_contents(html, evidence): + doc = Document(html) + if [clean(n.text()) for n in doc.root.walk() if n.tag == 'title'] != [TITLE]: + raise ValueError('Unexpected AAMAS proceedings title/year') + section = None + headers, records, excluded, held = [], [], [], [] + counts, seen_pages, seen_pdf_ids = Counter(), set(), set() + for node in doc.root.walk(): + if node.tag != 'p' or any(n is not node and n.tag == 'p' for n in node.walk()): + continue + anchors = [n.attrs['name'] for n in node.walk() if n.tag == 'a' and n.attrs.get('name') in SECTIONS] + if anchors: + section = one(anchors, 'section anchor') + if clean(node.text()) != SECTIONS[section][0]: + raise ValueError('Publisher section heading differs from reviewed layout') + headers.append(section) + page_fonts = [n for n in node.walk() if n.tag == 'font' and + n.attrs.get('color', '').lower() == '#444444'] + if not page_fonts: + continue + page_text = clean(one(page_fonts, 'page marker').text()) + page_match = re.fullmatch(r'\(Page ([1-9][0-9]*)\)', page_text) + if not page_match or section not in SECTIONS: + raise ValueError('Invalid page or paper outside reviewed sections') + page = int(page_match[1]) + if page in seen_pages: + raise ValueError('Repeated publisher page ID') + seen_pages.add(page) + title_node = one([n for n in node.walk() if n.tag == 'strong'], 'paper title') + title = clean(title_node.text()) + if not title: + raise ValueError('Empty paper title') + authors, affiliations = parse_authors(node) + link = one([n for n in node.walk() if n.tag == 'a' and 'href' in n.attrs], 'paper link') + href = link.attrs['href'] + title_nodes = {id(n) for n in title_node.walk()} + retractions = [clean(n.text()) for n in link.walk() if n.tag == 'i' and id(n) not in title_nodes] + heading, publication_format, track, _ = SECTIONS[section] + counts[section] += 1 + metadata = {'publisher_page': page, 'section_url': URL, 'section_anchor': section, + 'section_heading': heading, 'publication_format': publication_format, + 'author_affiliations': affiliations, 'source_evidence': evidence, + 'decision_date_precision': 'not_supplied', + 'missing_abstract_reason': 'Publisher HTML supplies no abstract; research PDF not downloaded.', + 'publication_evidence': 'Published proceedings entry, not an original decision note.'} + withdrawn = retractions == ['Withdrawn June 8, 2026'] + if withdrawn: + if (section, page, title, href) != ('1', 3178, WITHDRAWAL_TITLE, ''): + raise ValueError('Unreviewed withdrawal notice') + metadata.update({'publication_state': 'withdrawn', + 'withdrawal_date': '2026-06-08', + 'raw_publication_state': retractions[0], + 'decision_date_precision': 'day', + 'publication_evidence': 'Explicit dated publisher withdrawal notice, not an original review decision note.'}) + elif retractions: + if (section, page, title, href, retractions) != ('R', 238, RETRACTION_TITLE, '', ['Retracted June 8, 2026']): + raise ValueError('Unreviewed retraction or publication status markup') + held.append({'source': 'aamas_proceedings', 'external_id': '2026/p238', + 'title': title, 'authors': authors, 'year': 2026, 'track': track, + 'url': URL+'#R', 'abstract': None, 'identifiers': {}, + 'observed_at': evidence['observed_at'], 'metadata': metadata, + 'publication_state': 'retracted', 'retraction_date': '2026-06-08', + 'raw_publication_state': retractions[0], + 'hold_reason': 'Explicit retraction; not an accepted/withdrawn submission assertion. Dedicated decision modeling and identity review required.'}) + continue + if not withdrawn: + match = PDF.fullmatch(href) + if not match or match[1] in seen_pdf_ids: + raise ValueError('Invalid or repeated publisher PDF ID') + seen_pdf_ids.add(match[1]) + metadata.update({'publisher_document_id': match[1], 'paper_url': ROOT+'pdfs/'+match[1]+'.pdf'}) + if section == 'K': + excluded.append({'publisher_page': page, 'title': title, 'authors': authors, + 'metadata': metadata, 'reason': 'Publisher keynote, not a research submission.'}) + continue + metadata['track_scope_note'] = ('Published format known; individual main/special submission track not established.' + if track == 'proceedings_unspecified' else 'Explicit publisher section.') + if section == 'A': + metadata['prior_submission_note'] = 'AAAI transfer section does not supply original submission IDs or individual prior decision notes; none inferred.' + if section == 'J': + metadata['track_scope_note'] = 'Journal presentation; not another independent journal review or research-track acceptance.' + records.append({'source': 'aamas_proceedings', 'external_id': '2026/p'+str(page), + 'url': URL+'#'+section, 'title': title, 'abstract': None, + 'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [], + 'year': 2026, 'track': track, 'status': 'withdrawn' if withdrawn else 'accepted', + 'observed_at': evidence['observed_at'], + 'raw_decision': retractions[0] if withdrawn else 'listed in official AAMAS 2026 published proceedings '+heading, + 'metadata': metadata, + 'tags': [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}]}) + if headers != list(SECTIONS) or dict(counts) != {k: v[3] for k, v in SECTIONS.items()}: + raise ValueError('Publisher section inventory differs from reviewed 2026 layout') + if (len(records), len(excluded), len(held)) != (639, 3, 1): + raise ValueError('Unexpected accepted/keynote/retraction partition') + if Counter(r['status'] for r in records) != {'accepted': 638, 'withdrawn': 1}: + raise ValueError('Unexpected publisher publication-state partition') + return records, excluded, held + + +def read_page(cache, snapshots): + if bool(cache) == bool(snapshots): + raise ValueError('Choose exactly one cache or frozen snapshot directory') + if snapshots: + base = Path(snapshots).resolve() + frozen = json.loads((base/'receipts.json').read_text())['receipts'] + r = one([r for r in frozen if r['url'] == URL], 'frozen response receipt') + filename = Path(r['file']) + if (filename.is_absolute() or '..' in filename.parts or + not (base/filename).resolve().is_relative_to(base) or r.get('status') != 200): + raise ValueError('Unsafe or unsuccessful publisher response') + data = (base/filename).read_bytes() + if len(data) > 8*1024**2 or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']: + raise ValueError('Frozen publisher response checksum/size mismatch') + observed = r['finished_at'] + else: + data = fetch(URL, cache, 8*1024**2, delay=1) + r = json.loads((Path(cache)/hashlib.sha256(URL.encode()).hexdigest()).with_suffix('.json').read_text()) + observed = r['retrieved_at'] + timestamp(observed) + return data, {'url': URL, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data), + 'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html', + 'observed_at': observed, 'http_status': 200, 'response_receipt': r} + + +def collect(out, *, cache=None, snapshots=None): + data, evidence = read_page(cache, snapshots) + records, excluded, held = parse_contents(data.decode('utf-8'), evidence) + grouped = defaultdict(list) + for row in records: + grouped[row['track']].append(row) + out = Path(out); out.mkdir(parents=True, exist_ok=True); files = [] + with tempfile.TemporaryDirectory(prefix='.aamas2026-', dir=out) as temporary: + stage = Path(temporary); (stage/'evidence').mkdir() + (stage/evidence['file']).write_bytes(data) + for track, group in sorted(grouped.items()): + p = stage/f'aamas-proceedings-2026-{track}.jsonl' + p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group)) + manifest = {'source': 'aamas_proceedings', 'parser_version': 3, 'venue': 'AAMAS', 'year': 2026, + 'track': track, 'count': len(group), 'file': str(out/p.name), + 'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'list_url': URL, + 'status': 'partial', 'missing_abstract_count': len(group), + 'status_counts': dict(Counter(r['status'] for r in group)), + 'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)), + 'note': 'Reviewed TOC partition excluding keynote/retracted entries; identity and full submission history remain unverified.'} + p.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n') + files.append(manifest) + result = {'source': 'aamas_proceedings', 'year': 2026, 'finished_at': timestamp(), + 'status': 'collected_pending_identity_review', 'records': len(records), 'files': files, + 'excluded_publications': excluded, 'held_publications': held, + 'status_counts': dict(Counter(r['status'] for r in records)), + 'source_receipts': [evidence], 'database_written': False, 'pdfs_downloaded': False, + 'note': 'Retraction evidence held separately; publication is not an original decision note, and absence is not rejection.'} + report = 'aamas-proceedings-2026.json' + (stage/report).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n') + (out/'evidence').mkdir(exist_ok=True) + for p in (stage/'evidence').iterdir(): + p.replace(out/'evidence'/p.name) + for p in stage.iterdir(): + if p.is_file() and p.name != report: + p.replace(out/p.name) + (stage/report).replace(out/report) + return result diff --git a/tests/fixtures/aamas2026-fixture.json.gz b/tests/fixtures/aamas2026-fixture.json.gz new file mode 100644 index 0000000..05e678a Binary files /dev/null and b/tests/fixtures/aamas2026-fixture.json.gz differ diff --git a/tests/test_aamas2026.py b/tests/test_aamas2026.py new file mode 100644 index 0000000..6af1d82 --- /dev/null +++ b/tests/test_aamas2026.py @@ -0,0 +1,167 @@ +import gzip +import hashlib +import json +from pathlib import Path +import tempfile +import sqlite3 +import unittest +from unittest.mock import patch + +from paper_atlas.collectors import aamas, aamas2026 +from paper_atlas.ingestion.records import ingest_file + + +FIXTURE = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/aamas2026-fixture.json.gz').read_bytes())) +EVIDENCE = {'url': aamas2026.URL, 'observed_at': FIXTURE['receipts']['receipts'][0]['finished_at']} + + +class Aamas2026Tests(unittest.TestCase): + def parse(self, html=None): + return aamas2026.parse_contents(FIXTURE['contents'] if html is None else html, EVIDENCE) + + def snapshots(self, folder): + folder.mkdir() + (folder/'contents.html').write_text(FIXTURE['contents']) + (folder/'receipts.json').write_text(json.dumps(FIXTURE['receipts'])) + return folder + + def test_every_publisher_title_author_affiliation_and_page_preserved(self): + rows, excluded, held = self.parse() + self.assertEqual((len(rows), len(excluded), len(held)), (639, 3, 1)) + expected = {r['publisher_page']: r for r in FIXTURE['expected_entries']} + self.assertEqual(len(expected), 643) + all_rows = rows+excluded+held + self.assertEqual({r['metadata']['publisher_page'] for r in all_rows}, set(expected)) + for r in all_rows: + e = expected[r['metadata']['publisher_page']] + self.assertEqual(r['title'], e['title']) + self.assertEqual(r['authors'], e['authors']) + self.assertEqual(r['metadata']['author_affiliations'], e['affiliations']) + self.assertEqual(r['metadata']['section_heading'], e['section']) + for r in rows: + self.assertIsNone(r['abstract']) + self.assertEqual(r['identifiers'], {}) + if r['external_id'] != '2026/p3178': + self.assertEqual(r['status'], 'accepted') + self.assertNotIn('decision_date', r) + + def test_explicit_withdrawal_preserves_status_date_and_no_pdf(self): + rows, _, _ = self.parse() + row, = [r for r in rows if r['external_id'] == '2026/p3178'] + self.assertEqual(row['status'], 'withdrawn') + self.assertEqual(row['metadata']['withdrawal_date'], '2026-06-08') + self.assertNotIn('decision_date', row) + self.assertEqual(row['raw_decision'], 'Withdrawn June 8, 2026') + self.assertNotIn('paper_url', row['metadata']) + self.assertEqual(row['metadata']['publication_state'], 'withdrawn') + self.assertEqual(sum(r['status'] == 'accepted' for r in rows), 638) + + def test_withdrawal_ingestion_retains_date_precision_without_invented_acceptance(self): + rows, _, _ = self.parse() + row, = [r for r in rows if r['external_id'] == '2026/p3178'] + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); path = root/'withdrawn.jsonl'; db = root/'sample.sqlite' + path.write_text(json.dumps(row, ensure_ascii=False)+'\n') + path.with_suffix('.manifest.json').write_text(json.dumps({ + 'source': 'aamas_proceedings', 'list_url': aamas2026.URL, + 'venue': 'AAMAS', 'year': 2026, 'track': row['track'], + 'count': 1, 'status': 'partial', 'missing_abstract_count': 1, + 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest()})) + ingest_file(db, path) + with sqlite3.connect(db) as c: + self.assertEqual(c.execute('SELECT status,raw_value,decision_date FROM decisions').fetchall(), + [('withdrawn', 'Withdrawn June 8, 2026', None)]) + metadata = json.loads(c.execute('SELECT metadata_json FROM source_records').fetchone()[0]) + self.assertEqual(metadata['withdrawal_date'], '2026-06-08') + self.assertEqual(metadata['decision_date_precision'], 'day') + self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), []) + + def test_retraction_is_not_accepted_or_withdrawn_and_retains_date(self): + rows, _, held = self.parse() + self.assertNotIn('2026/p238', {r['external_id'] for r in rows}) + h, = held + self.assertEqual(h['publication_state'], 'retracted') + self.assertEqual(h['retraction_date'], '2026-06-08') + self.assertEqual(h['raw_publication_state'], 'Retracted June 8, 2026') + self.assertNotIn('status', h) + self.assertNotIn('paper_url', h['metadata']) + self.assertEqual(h['metadata']['source_evidence'], EVIDENCE) + + def test_tracks_formats_and_prior_decisions_are_not_conflated(self): + rows, _, _ = self.parse() + counts = {t: sum(r['track'] == t for r in rows) for t in {r['track'] for r in rows}} + self.assertEqual(counts, {'proceedings_unspecified': 522, 'aaai_transfer': 19, + 'blue_sky': 17, 'doctoral': 34, 'demo': 44, 'jaamas': 3}) + for row in rows: + self.assertEqual(row['tags'][0]['namespace'], 'aamas_proceedings:publication_format') + if row['track'] == 'aaai_transfer': + self.assertIn('none inferred', row['metadata']['prior_submission_note']) + self.assertNotIn('decision_events', row) + if row['track'] == 'jaamas': + self.assertEqual(row['metadata']['publication_format'], 'journal_presentation') + self.assertIn('not another independent', row['metadata']['track_scope_note']) + + def test_refuses_changed_year_section_page_link_author_and_retraction(self): + html = FIXTURE['contents'] + replacements = [('AAMAS 2026:', 'AAMAS 2027:'), + ('name="R" id="R">Research Paper Track', + 'name="R" id="R">Workshop Track'), + ('(Page 4)', '(Page 1)'), + ('../pdfs/HQQZ1937.pdf', 'https://other.example/paper.pdf'), + ('Michael Wooldridge ', ''), + ('Retracted June 8, 2026', 'Withdrawn June 8, 2026')] + for old, new in replacements: + self.assertIn(old, html) + with self.subTest(change=old), self.assertRaises(ValueError): + self.parse(html.replace(old, new, 1)) + + def test_frozen_collection_is_partial_with_full_held_evidence(self): + with tempfile.TemporaryDirectory() as tmp, patch.object(aamas2026, 'fetch', side_effect=AssertionError('No HTTP')): + root = Path(tmp) + report = aamas.collect(2026, root/'out', snapshots=self.snapshots(root/'source')) + self.assertEqual(report['records'], 639) + self.assertEqual(report['status_counts'], {'accepted': 638, 'withdrawn': 1}) + self.assertEqual(len(report['files']), 6) + self.assertEqual(len(report['held_publications']), 1) + self.assertFalse(report['database_written']) + self.assertFalse(report['pdfs_downloaded']) + for m in report['files']: + self.assertEqual(m['status'], 'partial') + self.assertEqual(m['missing_abstract_count'], m['count']) + self.assertEqual(hashlib.sha256(Path(m['file']).read_bytes()).hexdigest(), m['file_sha256']) + self.assertTrue((root/'out/aamas-proceedings-2026.json').is_file()) + + def test_frozen_response_rejects_tamper_unsafe_path_status_and_symlink(self): + for mode in ('tamper', 'path', 'status', 'symlink'): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); source = self.snapshots(root/'source') + r = json.loads((source/'receipts.json').read_text()) + if mode == 'tamper': + (source/'contents.html').write_text('bad') + elif mode == 'path': + r['receipts'][0]['file'] = '../outside.html' + elif mode == 'status': + r['receipts'][0]['status'] = 403 + else: + (root/'outside.html').write_text(FIXTURE['contents']) + (source/'contents.html').unlink() + (source/'contents.html').symlink_to(root/'outside.html') + (source/'receipts.json').write_text(json.dumps(r)) + with self.assertRaises(ValueError): + aamas.collect(2026, root/'out', snapshots=source) + self.assertFalse((root/'out/aamas-proceedings-2026.json').exists()) + + def test_live_mode_fetches_only_html_and_preserves_receipt(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); cache = root/'cache'; cache.mkdir() + receipt = {'retrieved_at': EVIDENCE['observed_at'], 'url': aamas2026.URL} + (cache/hashlib.sha256(aamas2026.URL.encode()).hexdigest()).with_suffix('.json').write_text(json.dumps(receipt)) + with patch.object(aamas2026, 'fetch', return_value=FIXTURE['contents'].encode()) as fetch: + report = aamas.collect(2026, root/'out', cache=cache) + fetch.assert_called_once_with(aamas2026.URL, cache, 8*1024**2, delay=1) + self.assertEqual(report['source_receipts'][0]['response_receipt'], receipt) + + def test_frozen_fixture_is_original_hashed_response(self): + r, = FIXTURE['receipts']['receipts'] + self.assertEqual(hashlib.sha256(FIXTURE['contents'].encode()).hexdigest(), r['sha256']) + self.assertEqual(r['bytes'], len(FIXTURE['contents'].encode())) diff --git a/tests/test_aamas_proceedings.py b/tests/test_aamas_proceedings.py index 92be220..3b32ff2 100644 --- a/tests/test_aamas_proceedings.py +++ b/tests/test_aamas_proceedings.py @@ -99,7 +99,7 @@ class AamasProceedingsTests(unittest.TestCase): aamas.collect(2012, root/'out', snapshots=source) self.assertFalse((root/'out/aamas-proceedings-2012.json').exists()) with self.assertRaises(ValueError): - aamas.collect(2026, root/'future', snapshots=source) + aamas.collect(2027, root/'future', snapshots=source) self.assertFalse((root/'future').exists()) def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self):