diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 2d4b3f9..9ff112c 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -49,6 +49,7 @@ jobs: subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True) diff --git a/README.md b/README.md index 57ca0cc..c696d97 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \ 早期主系列的Workshop名称按[ICML/IROS精确届次证据](docs/coverage/historical-primary-series-2026-11-09.md)识别，保留原名与范围证据；代码支持不代表已补齐生产数据。 +AAMAS 的[官方论文集采集器](docs/aamas-proceedings.md)目前核验了 2012 版式，可从缓存或已校验快照采集标题、作者及页面提供的摘要；发表形式单独保存，全文/扩展摘要中未明确的投稿 track 保持待明确，不根据会话主题推定主会轨道。 + `paper-atlas collect --help` 查看来源；`paper-atlas import --help` 查看导入方式。完整批次步骤见[使用指南](docs/usage.md)。 共享实例名为 `foundry-postgres`，数据位于 `foundry_data` 库的 `research` schema；其他项目可按权限复用同一组表。库名通过 libpq service 配置、schema 通过 search_path 或 `--pg-schema` 指定，不写死在查询中。PostgreSQL 凭据放在服务器权限为 `0600` 的 libpq service 文件中；命令行只使用服务名称。`paper-atlas maintain postgres --help` 提供迁移入口：先完整核验 21 张业务表，再发布目标 schema。SQLite FTS5 内部表由 PostgreSQL GIN 索引替代。切换后 SQLite 为只读备份；`import records`、`import arxiv` 和 `maintain backfill` 使用 `--pg-service foundry-research` 写入主库，整个写入会话只允许一个协调写入者，会议文件失败时整体回滚并保留失败批次记录。 diff --git a/docs/aamas-proceedings.md b/docs/aamas-proceedings.md new file mode 100644 index 0000000..55a344d --- /dev/null +++ b/docs/aamas-proceedings.md @@ -0,0 +1,43 @@ +# AAMAS 官方论文集元数据 + +`paper_atlas.collectors.aamas` 是按需运行的采集适配器，通过统一入口 `collect aamas-proceedings` 使用。目前只支持已独立核验的 **AAMAS 2012** HTML 版式；其他年份需先核对官方目录和逐篇结构，再增加适配，不能仅替换年份。 + +## 来源与范围 + +[官方论文集介绍](https://www.ifaamas.org/Proceedings/aamas2012/)及其 Full Papers、Extended Abstracts、Demos、Doctorial Consortium Abstracts 四个栏目分别列出 136、146、28、24 条，共 334 条。旧 DBLP 候选卷的 310 条不能替代这四类官方目录，也不能全部标记为主会全文。栏目数量来自 2026-11-09 冻结的原始页面；页面结构或数量发生变化时，采集器停止，留待重新核验。 + +采集标题、逐篇作者、页面实际提供的摘要、官方条目 ID、目录锚点和 PDF 地址。**只保存 PDF 链接，不下载 PDF。** 全文页面的摘要藏在标题链接内部的 `abstract` 元素，作者链接里的 “has N papers” 是界面徽标，二者不会混入标题或姓名。未提供的摘要保存为 `null`，不会用标题、说明文字或其他论文填充。 + +官方介绍说明了 general、robotics、virtual agents、innovative applications 等投稿轨道，但逐篇目录的主题会话不能独立证明某篇论文的投稿轨道。因此全文和扩展摘要均使用 `proceedings_unspecified`；演示、博士栏目使用 `demo`、`doctoral`。发表形式保留在 `metadata.publication_format` 和 `aamas_proceedings:publication_format` 标签中，会话原文单独保存。它们不是研究类型标注，也不会写入自定义 annotation。 + +目录条目仅提供发表录用证据，不是原始决定信。`observed_at` 是页面采集时间，不能当作 2012 年的决定日期；官方介绍中的投稿、撤稿、录用汇总数不能识别逐篇拒稿或撤稿，采集器不会由总量相减生成这些决定。 + +## 运行 + +在已确认的远程运行环境中，使用 HTTP 缓存采集： + +```sh +paper-atlas collect aamas-proceedings --year 2012 \ + --cache /data/paper-index/cache/aamas \ + --out /data/paper-index/collected/aamas-2012 +``` + +也可使用已冻结的官方页面： + +```sh +paper-atlas collect aamas-proceedings --year 2012 \ + --snapshots /data/paper-index/collected/historical-next-source-evidence-v1 \ + --out /data/paper-index/collected/aamas-2012 +``` + +快照目录需含 `receipts.json`，其中每个响应包含原始 `url`、HTTP `status`、相对 `file`、`bytes`、`sha256` 和 `finished_at`。要求五个准确官方 URL 均唯一匹配且返回 200；验证全部字节数和哈希后才解析，不接受目录穿越或绝对文件路径。快照中其他来源的响应不会被当成 AAMAS 页面。 + +输出为按数据库 track 分组的三个 JSONL 文件和对应 manifest、五份原始 HTML 证据，以及最后写出的 `aamas-proceedings-2012.json` 采集报告。锁文件避免同一输出目录并发运行。manifest 始终为 `partial`，记录发表形式数量和缺失摘要数；**采集命令不连接数据库，也不导入。** + +## 复核与恢复 + +每条记录携带对应官方页面的 URL、采集时间、字节数、SHA-256 与输出内证据路径。重新运行前先检查已有采集报告和文件哈希；采集中断不等于主库已导入，报告缺失或哈希不一致的目录不能直接作为导入队列。输出中的原始证据应随文件一起保存。 + +生产导入前仍需单独与主库核验稳定标识、标题、作者及已有投稿，按证据补充 DOI/arXiv 等标识；目录页上的作者计数徽标和会话 ID 不能作为这些标识。需要另建经过核验的增量清单、保护旧历史，完成独立验收后才能计入覆盖。跨来源同一论文不得仅因同题就新增或合并投稿。 + +本功能的代码、测试或采集成功均不代表 AAMAS 2012 已导入生产，也不证明拒稿全集完整。全库最终去重、摘要和来源覆盖验收属于独立后续阶段。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 384c3d8..6475087 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -23,6 +23,7 @@ COMMANDS = { ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'), + ('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'), ('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'), ('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'), ('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'), diff --git a/paper_atlas/collectors/aamas.py b/paper_atlas/collectors/aamas.py new file mode 100644 index 0000000..0c49db4 --- /dev/null +++ b/paper_atlas/collectors/aamas.py @@ -0,0 +1,210 @@ +"""Official AAMAS proceedings metadata, with publication format separate from track.""" +import argparse +from collections import Counter, defaultdict +import fcntl +import hashlib +import json +from pathlib import Path +import re +import tempfile +from urllib.parse import urljoin, urlsplit + +from paper_atlas.db.submissions import timestamp +from paper_atlas.io.html import Document, Node, clean +from paper_atlas.io.http import fetch + +SOURCE = 'aamas_proceedings' +ROOT = 'https://www.ifaamas.org/Proceedings/aamas2012/' +# Counts independently checked against the frozen publisher sections. This +# adapter deliberately refuses other editions/layouts until they are reviewed. +SECTIONS = { + 'fullpapers': ('Full Papers', 'full_paper', 'proceedings_unspecified', 136), + 'extabstracts': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 146), + 'demos': ('Demos', 'demo', 'demo', 28), + 'doctorals': ('Doctorial Consortium Abstracts', 'doctoral_abstract', 'doctoral', 24), +} + + +def classes(node): + return set(node.attrs.get('class', '').split()) + + +def text_without(node, omitted): + if node.tag in ('script', 'style') or classes(node) & omitted: + return '' + parts = [text_without(child, omitted) if isinstance(child, Node) else child + for child in node.children] + value = ''.join(parts) + return ' '+value+' ' if node.tag in ('p', 'div', 'section', 'li', 'br') else value + + +def one(nodes, description): + if len(nodes) != 1: + raise ValueError('Expected one '+description) + return nodes[0] + + +def parse_section(html, section, observed_at, evidence): + heading, publication_format, track, _ = SECTIONS[section] + document = Document(html) + titles = [clean(n.text()) for n in document.root.walk() if n.tag == 'title'] + if titles != [heading+' - AAMAS 2012']: + raise ValueError('Unexpected proceedings edition or section title') + headings = [clean(n.text()) for n in document.root.walk() if n.tag == 'h1'] + if headings != [heading]: + raise ValueError('Unexpected proceedings section heading') + url = ROOT+'resources/'+section+'.html' + records = [] + seen = set() + session = None + for node in document.root.walk(): + if node.tag == 'h2' and 'section' in classes(node): + session = {'id': node.attrs.get('id'), 'title': clean(node.text())} + if node.tag != 'div' or 'paper' not in classes(node): + continue + paper_id = node.attrs.get('id', '') + if not re.fullmatch(r'[A-Za-z0-9]+_[0-9]+', paper_id) or paper_id in seen: + raise ValueError('Missing or repeated publisher paper ID') + seen.add(paper_id) + title_node = one([n for n in node.walk() if n.tag == 'span' and 'title' in classes(n)], 'paper title') + link = one([n for n in title_node.children if isinstance(n, Node) and n.tag == 'a'], 'paper link') + paper_url = urljoin(url, link.attrs.get('href', '')) + if paper_url != ROOT+'papers/'+paper_id+'.pdf': + raise ValueError('Paper link differs from publisher ID or official host') + title = clean(text_without(link, {'abstract'})) + abstract_nodes = [n for n in link.walk() if n.tag == 'span' and 'abstract' in classes(n)] + if len(abstract_nodes) > 1: + raise ValueError('Multiple abstracts for one paper') + abstract = (clean(abstract_nodes[0].text()) or None) if abstract_nodes else None + author_node = one([n for n in node.walk() if n.tag == 'span' and 'authors' in classes(n)], 'author list') + author_display = clean(text_without(author_node, {'amount'})) + authors = [clean(name) for name in author_display.split(',')] + if not title or not author_display or any(not name for name in authors): + raise ValueError('Empty paper title or author') + tags = [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}] + if session: + tags.append({'namespace': 'aamas_proceedings:session', 'value': session['title']}) + records.append({ + 'source': SOURCE, 'external_id': '2012/'+paper_id, + 'url': url+'#'+paper_id, 'title': title, 'abstract': abstract, + 'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [], + 'year': 2012, 'track': track, 'status': 'accepted', + 'observed_at': observed_at, + 'raw_decision': 'listed in official AAMAS 2012 proceedings '+heading, + 'metadata': {'publisher_paper_id': paper_id, 'paper_url': paper_url, + 'section_url': url, 'section_heading': heading, + 'publication_format': publication_format, 'session': session, + 'authors_display': author_display, 'source_evidence': evidence, + 'publication_evidence': 'official published proceedings entry; not an original decision note', + 'track_scope_note': ('Full papers and extended abstracts include general and special submission tracks; ' + 'session topics do not establish an individual submission track.' + if track == 'proceedings_unspecified' else 'Explicit publisher section.'), + 'decision_date_precision': 'not_supplied'}, + 'tags': tags}) + if not records: + raise ValueError('No publisher paper entries found') + return records + + +def read_pages(cache=None, snapshots=None): + if bool(cache) == bool(snapshots): + raise ValueError('Choose exactly one HTTP cache or frozen snapshot directory') + urls = {'introduction': ROOT, **{key: ROOT+'resources/'+key+'.html' for key in SECTIONS}} + pages = {} + receipts = [] + frozen = json.loads((Path(snapshots)/'receipts.json').read_text())['receipts'] if snapshots else [] + for key, url in urls.items(): + if snapshots: + receipt = one([r for r in frozen if r['url'] == url], 'frozen response receipt') + filename = Path(receipt['file']) + if filename.is_absolute() or '..' in filename.parts or receipt.get('status') != 200: + raise ValueError('Unsafe or unsuccessful frozen response') + data = (Path(snapshots)/filename).read_bytes() + if len(data) > 8*1024**2 or hashlib.sha256(data).hexdigest() != receipt['sha256'] or len(data) != receipt['bytes']: + raise ValueError('Frozen publisher response checksum or size mismatch') + observed = receipt['finished_at'] + else: + data = fetch(url, cache, 8*1024**2, delay=1) + receipt = json.loads((Path(cache)/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_text()) + observed = receipt['retrieved_at'] + evidence = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data), + 'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html', 'observed_at': observed, + 'http_status': 200, 'response_receipt': receipt} + pages[key] = (data, evidence) + receipts.append(evidence) + return pages, receipts + + +def _collect(year, out, *, cache=None, snapshots=None): + if year != 2012: + raise ValueError('Only the independently reviewed AAMAS 2012 layout is supported') + pages, receipts = read_pages(cache, snapshots) + intro = Document(pages['introduction'][0].decode('utf-8')) + if [clean(n.text()) for n in intro.root.walk() if n.tag == 'title'] != ['AAMAS 2012']: + raise ValueError('Unexpected proceedings introduction') + records = [] + for section, (_, _, _, expected) in SECTIONS.items(): + data, evidence = pages[section] + group = parse_section(data.decode('utf-8'), section, evidence['observed_at'], evidence) + if len(group) != expected: + raise ValueError('Publisher section count differs from reviewed inventory: '+section) + records.extend(group) + if len({r['external_id'] for r in records}) != len(records): + raise ValueError('Paper IDs collide across publisher sections') + grouped = defaultdict(list) + for record in records: + grouped[record['track']].append(record) + out = Path(out) + out.mkdir(parents=True, exist_ok=True) + files = [] + with tempfile.TemporaryDirectory(prefix='.aamas-', dir=out) as temporary: + stage = Path(temporary) + (stage/'evidence').mkdir() + for data, evidence in pages.values(): + (stage/evidence['file']).write_bytes(data) + for track, group in sorted(grouped.items()): + path = stage/f'aamas-proceedings-2012-{track}.jsonl' + path.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group)) + manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'AAMAS', 'year': 2012, 'track': track, + 'count': len(group), 'file': str(out/path.name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(), + 'list_url': ROOT, 'status': 'partial', 'missing_abstract_count': sum(not r['abstract'] for r in group), + 'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)), + 'note': 'Publisher sections verified; individual full/extended submission track, decision history and independent identities still need review.'} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n') + files.append(manifest) + result = {'source': SOURCE, 'year': year, 'finished_at': timestamp(), 'status': 'collected_pending_identity_review', + 'records': len(records), 'files': files, 'source_receipts': receipts, + 'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in records)), + 'database_written': False, 'pdfs_downloaded': False, + 'note': 'Aggregate submission/withdrawal totals do not identify individual rejected or withdrawn papers.'} + (stage/'aamas-proceedings-2012.json').write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n') + # Validated files first; write the completed collection report last. + (out/'evidence').mkdir(exist_ok=True) + for path in (stage/'evidence').iterdir(): + path.replace(out/'evidence'/path.name) + for path in stage.iterdir(): + if path.is_file() and path.name != 'aamas-proceedings-2012.json': + path.replace(out/path.name) + (stage/'aamas-proceedings-2012.json').replace(out/'aamas-proceedings-2012.json') + return result + + +def collect(year, out, *, cache=None, snapshots=None): + if year != 2012: + raise ValueError('Only the independently reviewed AAMAS 2012 layout is supported') + out = Path(out) + out.mkdir(parents=True, exist_ok=True) + with (out/'aamas.collect.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + return _collect(year, out, cache=cache, snapshots=snapshots) + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog) + parser.add_argument('--year', type=int, choices=[2012], required=True) + parser.add_argument('--out', required=True) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument('--cache') + source.add_argument('--snapshots') + args = parser.parse_args(argv) + print(json.dumps(collect(args.year, args.out, cache=args.cache, snapshots=args.snapshots), ensure_ascii=False)) diff --git a/tests/test_aamas_proceedings.py b/tests/test_aamas_proceedings.py new file mode 100644 index 0000000..d804271 --- /dev/null +++ b/tests/test_aamas_proceedings.py @@ -0,0 +1,151 @@ +"""Publisher evidence, section boundaries and native-ingestion regression checks.""" +import hashlib +import json +import re +from pathlib import Path +import sqlite3 +import tempfile +import unittest +from unittest.mock import patch + +from paper_atlas.collectors import aamas +from paper_atlas.ingestion.records import ingest_file + + +def paper(pid, abstract=True): + summary = 'Research with evidence.
More detail.' if abstract else '' + return f'''
{pid} +

{summary} + A structured & auditable paper {pid} + Jane Doehas 7 papers, Bob Smith +

''' + + +def page(section, pid, abstract=True): + heading = aamas.SECTIONS[section][0] + return (f'{heading} - AAMAS 2012' + f'

{heading}

Session 1A – Robotics I

' + +paper(pid, abstract)+'') + + +class AamasProceedingsTests(unittest.TestCase): + def snapshots(self, root): + root.mkdir() + receipts = [] + items = {'introduction': ('AAMAS 2012

Aggregate withdrawals do not identify papers.

', aamas.ROOT)} + for index, section in enumerate(aamas.SECTIONS): + items[section] = (page(section, f'P{index}_1', section != 'extabstracts'), aamas.ROOT+'resources/'+section+'.html') + for key, (html, url) in items.items(): + data = html.encode() + (root/(key+'.html')).write_bytes(data) + receipts.append({'url': url, 'status': 200, 'file': key+'.html', 'bytes': len(data), + 'sha256': hashlib.sha256(data).hexdigest(), 'finished_at': '2026-11-09T00:00:00+00:00'}) + (root/'receipts.json').write_text(json.dumps({'receipts': receipts})) + return root + + def test_hidden_abstract_and_author_badges_are_not_title_or_authors(self): + rows = aamas.parse_section(page('fullpapers', '1A_1'), 'fullpapers', '2026-11-09T00:00:00Z', {'sha256': 'proof'}) + row, = rows + self.assertEqual(row['title'], 'A structured & auditable paper 1A_1') + self.assertEqual(row['abstract'], 'Research with evidence. More detail.') + self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith']) + self.assertEqual(row['metadata']['authors_display'], 'Jane Doe, Bob Smith') + self.assertEqual(row['url'], aamas.ROOT+'resources/fullpapers.html#1A_1') + self.assertEqual(row['metadata']['paper_url'], aamas.ROOT+'papers/1A_1.pdf') + self.assertEqual(row['metadata']['source_evidence'], {'sha256': 'proof'}) + + def test_publication_formats_and_session_topics_do_not_invent_submission_tracks(self): + for section, (_, fmt, track, _) in aamas.SECTIONS.items(): + row, = aamas.parse_section(page(section, 'P_1', False), section, '2026-11-09T00:00:00Z', {}) + self.assertEqual((row['track'], row['metadata']['publication_format']), (track, fmt)) + self.assertIsNone(row['abstract']) + self.assertEqual(row['status'], 'accepted') + self.assertEqual(row['metadata']['decision_date_precision'], 'not_supplied') + self.assertEqual(row['metadata']['session']['title'], 'Session 1A – Robotics I') + self.assertIn({'namespace': 'aamas_proceedings:publication_format', 'value': fmt}, row['tags']) + self.assertEqual(aamas.SECTIONS['fullpapers'][2], 'proceedings_unspecified') + self.assertEqual(aamas.SECTIONS['extabstracts'][2], 'proceedings_unspecified') + + def test_changed_edition_section_duplicate_id_or_pdf_link_is_rejected(self): + html = page('fullpapers', '1A_1') + for changed in [html.replace('AAMAS 2012', 'AAMAS 2025'), + html.replace('

Full Papers

', '

Rejected Papers

'), + html.replace('', paper('1A_1')+''), + html.replace('../papers/1A_1.pdf', 'https://example.org/1A_1.pdf'), + html.replace('../papers/1A_1.pdf', '../papers/1A_2.pdf')]: + with self.subTest(changed=changed): + with self.assertRaises(ValueError): + aamas.parse_section(changed, 'fullpapers', '2026-11-09T00:00:00Z', {}) + + def test_frozen_responses_require_checksums_success_and_safe_paths(self): + for failure in ['checksum', 'status', 'path']: + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary: + root = self.snapshots(Path(temporary)/'source') + path = root/'receipts.json' + receipts = json.loads(path.read_text()) + item = receipts['receipts'][0] + if failure == 'checksum': item['sha256'] = '0'*64 + elif failure == 'status': item['status'] = 404 + else: item['file'] = '../outside.html' + path.write_text(json.dumps(receipts)) + with self.assertRaises(ValueError): + aamas.read_pages(snapshots=root) + + def test_reviewed_section_counts_and_year_are_required(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = self.snapshots(root/'source') + with self.assertRaises(ValueError): + aamas.collect(2012, root/'out', snapshots=source) + self.assertFalse((root/'out/aamas-proceedings-2012.json').exists()) + with self.assertRaises(ValueError): + aamas.collect(2025, root/'future', snapshots=source) + self.assertFalse((root/'future').exists()) + + def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = self.snapshots(root/'source') + receipts_path = source/'receipts.json' + receipts = json.loads(receipts_path.read_text()) + for item in receipts['receipts'][1:]: + path = source/item['file'] + data = re.sub(r'P[0-9]_1', 'SAME_1', path.read_text()).encode() + path.write_bytes(data) + item.update(bytes=len(data), sha256=hashlib.sha256(data).hexdigest()) + receipts_path.write_text(json.dumps(receipts)) + small = {key: (*value[:3], 1) for key, value in aamas.SECTIONS.items()} + with patch.dict(aamas.SECTIONS, small, clear=True), self.assertRaises(ValueError): + aamas.collect(2012, root/'out', snapshots=source) + self.assertFalse((root/'out/aamas-proceedings-2012.json').exists()) + + def test_collection_and_native_ingestion_preserve_formats_evidence_and_missing_abstracts(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = self.snapshots(root/'source') + small = {key: (*value[:3], 1) for key, value in aamas.SECTIONS.items()} + with patch.dict(aamas.SECTIONS, small, clear=True): + result = aamas.collect(2012, root/'out', snapshots=source) + self.assertEqual(result['records'], 4) + self.assertEqual(len(result['files']), 3) + self.assertFalse(result['database_written']) + self.assertFalse(result['pdfs_downloaded']) + db = root/'sample.sqlite' + for manifest in result['files']: + path = Path(manifest['file']) + self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), manifest['file_sha256']) + for raw in path.read_text().splitlines(): + record = json.loads(raw) + evidence = record['metadata']['source_evidence'] + self.assertEqual(hashlib.sha256((root/'out'/evidence['file']).read_bytes()).hexdigest(), evidence['sha256']) + ingest_file(db, path) + with sqlite3.connect(db) as connection: + self.assertEqual(connection.execute('SELECT count(*) FROM papers').fetchone()[0], 4) + self.assertEqual(connection.execute('SELECT count(*) FROM submissions').fetchone()[0], 4) + self.assertEqual(connection.execute("SELECT count(*) FROM decisions WHERE status='accepted'").fetchone()[0], 4) + self.assertEqual(connection.execute('SELECT count(*) FROM papers WHERE abstract IS NULL').fetchone()[0], 1) + metadata = [json.loads(r[0]) for r in connection.execute('SELECT metadata_json FROM source_records')] + self.assertEqual({r['publication_format'] for r in metadata}, {'full_paper','extended_abstract','demo','doctoral_abstract'}) + namespaces = {r[0] for r in connection.execute('SELECT DISTINCT namespace FROM tags')} + self.assertEqual(namespaces, {'aamas_proceedings:publication_format', 'aamas_proceedings:session'}) + self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(), [])