diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 736a3a3..e5d4118 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -166,6 +166,21 @@ jobs: recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()] gold_keys = recsys_fixture['independent_gold'][0].keys() assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold'] + ismir_fixture = json.loads(gzip.decompress((src/'tests/fixtures/ismir2026-source.json.gz').read_bytes())) + ismir_snapshots = root/'ismir2026-snapshots' + ismir_snapshots.mkdir() + for name,encoded in ismir_fixture['files'].items(): + target = ismir_snapshots/name + target.parent.mkdir(parents=True,exist_ok=True) + target.write_bytes(base64.b64decode(encoded)) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','ismir2026-program','--snapshots',str(ismir_snapshots),'--out',str(root/'installed-ismir2026')],cwd=outside,check=True) + ismir_report = json.loads((root/'installed-ismir2026/collection-report.json').read_text()) + assert (ismir_report['records'],ismir_report['missing_abstract_count'],ismir_report['review_optin_yes'],ismir_report['review_optin_no']) == (140,0,101,39) + assert ismir_report['literal_title_version_differences'] == 2 and ismir_report['author_sequence_version_differences'] == 3 + assert not ismir_report['database_written'] and ismir_report['network_requests'] == 0 and ismir_report['approved_for_import'] == 0 + ismir_rows = [json.loads(line) for line in pathlib.Path(ismir_report['files'][0]['file']).read_text().splitlines()] + assert [(r['external_id'],r['title'],r['authors'],r['abstract']) for r in ismir_rows] == [('2026/'+g['program_id'],g['title'],g['authors'],g['abstract']) for g in ismir_fixture['independent_gold']] + assert all(not r['identifiers'] and not r['submission_identifiers'] and r['metadata']['paper_type'] is None for r in ismir_rows) corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes())) corl_official = root/'corl2026-official' corl_archive = root/'corl2026-archive' diff --git a/README.md b/README.md index 50458f7..6831df8 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,8 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。 +`collect ismir2026-program` 核验 140 项 ISMIR 官方录用节目及原始摘要,保留两处标题、三处作者名单差异和 101 组自愿公开评审文本;不推断原始 OpenReview ID 或拒稿。见 [ISMIR 2026 使用与恢复说明](docs/ismir2026-program.md)。 + `collect icassp2026-registry` 从冻结的 10 页 Crossref 元数据核验 4,582 条论文,明确排除 7 条非论文及 56 条其他会议记录;保留摘要缺失和单字段作者姓名,不自动融合节目表。见 [ICASSP 2026 采集与恢复说明](docs/icassp2026-registry.md)。 `collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。 diff --git a/docs/ismir2026-program.md b/docs/ismir2026-program.md new file mode 100644 index 0000000..900b118 --- /dev/null +++ b/docs/ismir2026-program.md @@ -0,0 +1,50 @@ +# ISMIR 2026 官方节目与摘要采集 + +这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不联网、不执行网页脚本、不下载论文 PDF、不连接数据库。输出仍须经过作品身份与投稿历史审查。 + +## 使用 + +```sh +paper-atlas collect ismir2026-program \ + --snapshots /data/paper-index/collected \ + --out /data/paper-index/collected/ismir2026-normalized- +``` + +输入父目录需要三个冻结目录: + +- `remaining2026-listings-research-v2`:清单、`ismir-accepted.html`、`ismir-call.html`。 +- `ismir2026-script-research-v1`:清单及 `papers.js`,用于证明官方程序明确加载的数据文件和 poster 链接模板;只读取脚本原文。 +- `ismir2026-json-research-v1`:清单及 `papers.json`,提供 140 项完整元数据和公开摘要。 + +固定清单和全部输入原始字节均校验 SHA256、HTTP 回执、官方 URL 及带时区的采集时间。已有输出目录不能覆盖;先检查原任务是否已结束,再选择新目录。失败保留 `failed.json`,只有全批成功才发布 `collection-report.json`。 + +## 来源与版本 + +[正式录用名单](https://ismir2026.ismir.net/accepted-papers)和[官方虚拟平台](https://ismir2026program.ismir.net/papers.html)各有 140 项,经来源内标题对应核查一致;其中 2 处标题有直引号/弯引号文字差异,3 处作者名单有差异,两个来源的原始写法分别保留并标记,不静默修正或拼接作者。 + +作者差异涉及节目 ID `60`、`126`、`433`,包含大小写和姓名完整度问题。正式名单的作者机构可能含括号和分号,采集器只在括号外拆分作者;作者原始字符串也保留。来源内名单对应不代表数据库作品身份已经获准关联。 + +- 原始 JSON 摘要全文保留,包括数学符号、换行和 Unicode,不作扁平化提取。 +- 数字 `id`、`forum` 是 miniconf 节目定位符,不视为原始 OpenReview 投稿 ID;不推断 DOI 或 arXiv ID,投稿标识列表保持空。 +- `accepted` 来自正式录用名单与节目平台,不是原始签名决定。`track=program_unclassified`,不从主题、session 或远程展示方式推断研究方法、workshop 或论文类型。 +- `ismir_program:keyword` 保存来源非空关键词,规范空白并去除同条记录内完全重复值;原始关键词列表(含空项)保留在元数据。 +- `ismir_program:presentation` 保存来源展示方式。Remote/In-person 是展示方式,不是研究类型。 +- poster URL 严格依据冻结的官方 JS 模板生成;本命令不请求 poster 页面。 + +## 公开评审与拒稿边界 + +原始来源中有 **101 组明确选择公开的评审材料**;其余 39 项 `publish_reviews=No`,评审正文为空。采集器验证此偏好和正文的一致性,原文保存在来源元数据,不把评审意见或 meta-review 中的推荐转换成额外的最终决定。 + +这些记录来自录用节目,不能据此当作 101 条公开拒稿记录。公开拒稿导出是否可得仍记为 unknown,需要单独核查。[征稿与评审流程](https://ismir2026.ismir.net/authors/call-for-papers)中的评审规则也不等于具体投稿决定。 + +输出包含一份 140 条 JSONL、部分覆盖清单、七个原始来源文件及报告,批准导入数为零。采集成功不等于会议论文/track/公开负面决定覆盖完整,也不替代最后的全库去重审计。 + +## 恢复与验证 + +`papers.json` SHA256:`864d384cbc928554504e1b39849137bb380143e247a48f2d71563a698cb8e62d`。 + +原始 JSON 和 JS 恢复包:服务器 `/data/paper-index/validation/ismir2026-json-research-recovery-v1.tar.gz`,本地 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件;SHA256 为 `bba7b236cfe4d8b259d6c1230b3fe09a3e9071f5203884c92b016ca0718fb095`。 + +正式名单与政策恢复包:同目录 `remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复三个源目录、核验文件后可重跑上述命令。 + +完整来源和独立 140 项清单在测试夹具中冻结。测试、构建、打包、安装及实际隔离目录 CLI 验证只在用户指定 `ces-control:host` runner 执行。生产规范化后还需独立核验全部记录,并用 PostgreSQL 只读预检检查候选作品和既往投稿;不自动合并来源差异或人物身份。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index a51eced..3cf457e 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -23,6 +23,7 @@ COMMANDS = { ('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'), ('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'), ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), + ('collect','ismir2026-program'): ('paper_atlas.collectors.ismir2026', {}, '核验 ISMIR 2026 公开节目与录用名单并保留原始摘要、来源差异和自愿公开评审'), ('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'), ('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'), ('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'), diff --git a/paper_atlas/collectors/ismir2026.py b/paper_atlas/collectors/ismir2026.py new file mode 100644 index 0000000..2a8842c --- /dev/null +++ b/paper_atlas/collectors/ismir2026.py @@ -0,0 +1,208 @@ +"""Frozen ISMIR 2026 accepted program, abstracts and opt-in review provenance.""" +import argparse +from collections import Counter +from datetime import datetime +import hashlib +import json +from pathlib import Path +import re +import unicodedata + +from paper_atlas.io.html import Document, Node, clean + +SOURCE = 'ismir_program' +TRACK = 'program_unclassified' +PAPERS = 'https://ismir2026program.ismir.net/papers.html' +ACCEPTED = 'https://ismir2026.ismir.net/accepted-papers' +SNAPSHOTS = { + 'remaining2026-listings-research-v2': ('fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9', + 'research_completed_not_coverage_or_import_approval', { + 'ismir-accepted.html': ACCEPTED, + 'ismir-call.html': 'https://ismir2026.ismir.net/authors/call-for-papers'}), + 'ismir2026-script-research-v1': ('5885b4bcfc2fc65374e5dc7a0d74f4eb86a0a36b8689f9fe8edbd10d49a08794', + 'public_script_research_only', {'papers.js': 'https://ismir2026program.ismir.net/static/js/papers.js'}), + 'ismir2026-json-research-v1': ('8be3e23277c199720264d940917118572655d18c0f9ac6db1b9cd30a6561b34d', + 'public_metadata_json_research_only', {'papers.json': 'https://ismir2026program.ismir.net/papers.json'}), +} +REVIEW_KEYS = {'meta_review','publish_reviews','review1','review2','review3','review4'} + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def one(items): + if len(items) != 1: + raise ValueError('Expected exactly one reviewed source field') + return items[0] + + +def title_key(text): + return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum()) + + +def load_snapshots(root): + root = Path(root) + if root.is_symlink(): + raise ValueError('Unsafe snapshot root') + result = {} + for name, (expected, status, urls) in SNAPSHOTS.items(): + directory = root/name; path = directory/'manifest.json' + if directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024: + raise ValueError('Unsafe snapshot manifest') + raw = path.read_bytes() + if digest(raw) != expected: + raise ValueError('Reviewed manifest hash changed') + manifest = json.loads(raw) + if manifest['status'] != status or manifest['database_written'] is not False: + raise ValueError('Unreviewed snapshot scope') + files = {'manifest.json': raw}; receipts = {} + for filename, url in urls.items(): + receipt = one([r for r in manifest['entries'] if r.get('file') == filename]) + path = directory/filename + if path.is_symlink() or path.stat().st_size > 8*1024**2: + raise ValueError('Unsafe snapshot source file') + body = path.read_bytes() + if (receipt['url'] != url or receipt['final_url'] != url or receipt['http_status'] != 200 + or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body) + or datetime.fromisoformat(receipt['at']).utcoffset() is None): + raise ValueError('Source receipt/body disagree') + files[filename] = body; receipts[filename] = receipt + result[name] = (files, receipts) + return result + + +def author_names(literal): + # Institution names can contain semicolons; split only outside parentheses. + parts = []; depth = 0; start = 0 + for index, char in enumerate(literal): + if char == '(': depth += 1 + elif char == ')': + depth -= 1 + if depth < 0: raise ValueError('Unbalanced author affiliation') + elif char == ';' and depth == 0: + parts.append(literal[start:index]); start = index+1 + if depth: + raise ValueError('Unbalanced author affiliation') + parts.append(literal[start:]) + names = [clean(p.split('(', 1)[0]) for p in parts] + if any(not name for name in names): + raise ValueError('Missing accepted-list author') + return names + + +def accepted_listing(raw): + doc = Document(raw.decode('utf-8')); tables = [] + for table in doc.root.walk(): + if table.tag != 'table': continue + rows = [] + for row in table.walk(): + if row.tag == 'tr': + rows.append([clean(n.text()) for n in row.children if isinstance(n, Node) and n.tag in ('th','td')]) + if rows and rows[0] == ['Paper Title', 'Authors']: + tables.append(rows) + rows = one(tables)[1:]; result = {} + if len(rows) != 140: + raise ValueError('Incomplete official accepted-list census') + for row in rows: + if len(row) != 2 or not row[0] or not row[1] or title_key(row[0]) in result: + raise ValueError('Missing or ambiguous accepted-list title') + result[title_key(row[0])] = {'title': row[0], 'authors_literal': row[1], 'author_names': author_names(row[1])} + return result + + +def program_observations(rows, listing): + if not isinstance(rows, list) or len(rows) != 140: + raise ValueError('Incomplete public program census') + observations = []; seen = set(); used = set() + for original in rows: + if not isinstance(original, dict) or not isinstance(original.get('content'), dict): + raise ValueError('Invalid public program record') + pid = original.get('id'); content = original.get('content', {}) + if not isinstance(pid, str) or not re.fullmatch(r'[1-9][0-9]*', pid) or pid in seen or original.get('forum') != pid: + raise ValueError('Missing/repeated numeric program locator, not an OpenReview ID') + seen.add(pid); title = content.get('title'); authors = content.get('authors'); abstract = content.get('abstract') + if (not isinstance(title, str) or not clean(title) or not isinstance(abstract, str) or not abstract.strip() + or not isinstance(authors, list) or not authors or any(not isinstance(a, str) or not clean(a) for a in authors)): + raise ValueError('Missing program title, author sequence or abstract') + key = title_key(title) + if key not in listing or key in used: + raise ValueError('Program/accepted-list title membership mismatch') + used.add(key); accepted = listing[key] + keywords = content.get('keywords'); presentation = content.get('paper_presentation') + session = content.get('session'); reviews = original.get('reviews') + if (not isinstance(keywords, list) or any(not isinstance(k, str) for k in keywords) + or presentation not in ('Remote','In-person','Remote during oral, In-person during nominee') + or not isinstance(session, list) or len(session) != 1 or not isinstance(session[0], str) or session[0] != original.get('session') + or not re.fullmatch(r'(?:[1-9]|1[0-4])', session[0])): + raise ValueError('Unreviewed source keywords, presentation or session') + if (not isinstance(reviews, dict) or set(reviews) != REVIEW_KEYS + or any(not isinstance(v, str) for v in reviews.values()) + or reviews['publish_reviews'] not in ('Yes','No')): + raise ValueError('Unreviewed opt-in review structure') + review_text = [reviews[k] for k in REVIEW_KEYS-{'publish_reviews'}] + if (reviews['publish_reviews'] == 'No' and any(review_text)) or (reviews['publish_reviews'] == 'Yes' and not any(review_text)): + raise ValueError('Review publication preference/body disagree') + flags = [] + if title != accepted['title']: flags.append('program_and_accepted_title_literal_versions_differ') + if authors != accepted['author_names']: flags.append('program_and_accepted_author_sequence_versions_differ') + observations.append({'original': original, 'accepted_listing': accepted, 'review_flags': flags}) + if (used != listing.keys() or sum('program_and_accepted_title_literal_versions_differ' in o['review_flags'] for o in observations) != 2 + or {o['original']['id'] for o in observations if 'program_and_accepted_author_sequence_versions_differ' in o['review_flags']} != {'60','126','433'} + or Counter(o['original']['reviews']['publish_reviews'] for o in observations) != {'Yes':101,'No':39}): + raise ValueError('Unreviewed source-version or opt-in census') + return observations + + +def collect(out, *, snapshots): + out = Path(out) + if out.exists(): raise ValueError('Output exists; inspect previous operation before retrying') + out.mkdir(parents=True) + try: + sources = load_snapshots(snapshots) + official, _ = sources['remaining2026-listings-research-v2']; script, _ = sources['ismir2026-script-research-v1'] + program, receipts = sources['ismir2026-json-research-v1']; receipt = receipts['papers.json'] + if b"d3.json('papers.json')" not in script['papers.js'] or b'href="poster_${openreview.id}.html"' not in script['papers.js']: + raise ValueError('Unreviewed public program data path or poster link template') + observations = program_observations(json.loads(program['papers.json']), accepted_listing(official['ismir-accepted.html'])) + evidence = out/'evidence'; evidence.mkdir() + for name, (files, _) in sources.items(): + directory = evidence/name; directory.mkdir() + for filename, body in files.items(): (directory/filename).write_bytes(body) + rows = [] + for observation in observations: + original = observation['original']; content = original['content']; pid = original['id'] + tags = [{'namespace':SOURCE+':presentation','value':content['paper_presentation']}] + tags += [{'namespace':SOURCE+':keyword','value':value} for value in dict.fromkeys(clean(k) for k in content['keywords'] if clean(k))] + rows.append({'source':SOURCE,'external_id':'2026/'+pid,'url':'https://ismir2026program.ismir.net/poster_'+pid+'.html', + 'title':content['title'],'authors':content['authors'],'abstract':content['abstract'], + 'identifiers':{},'submission_identifiers':[],'venue':'ISMIR','year':2026,'track':TRACK,'status':'accepted', + 'raw_decision':'Present in official ISMIR2026 accepted list and public virtual program; not an original signed decision.', + 'evidence_url':ACCEPTED,'observed_at':receipt['at'],'tags':tags, + 'metadata':{'program_entry':original,'accepted_listing':observation['accepted_listing'],'review_flags':observation['review_flags'], + 'program_receipt':receipt,'source_evidence':{'file':'evidence/ismir2026-json-research-v1/papers.json','sha256':receipt['sha256'],'url':receipt['url']}, + 'original_decision_signature_verified':False,'paper_type':None, + 'id_scope':'Numeric miniconf id/forum fields are program locators, not original OpenReview submission IDs; no inferred DOI or arxiv ID.', + 'track_scope':'Unclassified regular accepted-list program; no workshop or research-method class inferred from topic/session/remote presentation.', + 'review_scope':'101 source-opt-in review sets retained as raw public text;39No contain no review text. Reviewer recommendations/meta-review prose are not separately inferred final negative decisions.', + 'abstract_transform':'Original JSON abstract string retained verbatim, including math, newlines and Unicode; no markup flattening or PDF extraction.', + 'program_listing_scope':'Source versions remain separate.2title literal differences and3author-list differences flagged, not silently fused or resolved.', + 'scope':'Pending work/history review. Poster URL follows frozen official JS template; poster pages not downloaded. No work merge, aliases, negative decision or final coverage/dedup claim.'}}) + path = out/'ismir-program-2026-unclassified.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw) + manifest={'source':SOURCE,'parser_version':1,'venue':'ISMIR','year':2026,'track':TRACK,'count':140,'file':str(path), + 'file_sha256':digest(raw),'list_url':ACCEPTED,'status':'partial','missing_abstract_count':0,'source_expected_count':140} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n') + report={'status':'collected_pending_identity_review','records':140,'files':[manifest],'missing_abstract_count':0, + 'literal_title_version_differences':2,'author_sequence_version_differences':3,'review_optin_yes':101,'review_optin_no':39, + 'database_written':False,'network_requests':0,'pdfs_downloaded':False,'approved_for_import':0, + 'public_rejected_availability':'unknown_accepted_program_reviews_are_not_a_rejected_export', + 'scope':'140accepted observations;101opt-in review sets, not101rejected records. Source keyword/presentation tags are namespace-scoped; no original OpenReview ID assumption or source-version fusion.'} + part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report + except Exception as error: + (out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n') + raise + + +def main(argv=None, *, prog=None): + parser=argparse.ArgumentParser(prog=prog,description=__doc__);parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True) + args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False)) diff --git a/tests/fixtures/ismir2026-source.json.gz b/tests/fixtures/ismir2026-source.json.gz new file mode 100644 index 0000000..3e35c13 Binary files /dev/null and b/tests/fixtures/ismir2026-source.json.gz differ diff --git a/tests/test_ismir2026.py b/tests/test_ismir2026.py new file mode 100644 index 0000000..88bae6f --- /dev/null +++ b/tests/test_ismir2026.py @@ -0,0 +1,92 @@ +import base64 +import copy +import gzip +import hashlib +import json +from pathlib import Path +import tempfile +import unittest + +from paper_atlas.collectors import ismir2026 as c + + +class ISMIR2026Test(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/ismir2026-source.json.gz').read_bytes())) + cls.files={name:base64.b64decode(raw) for name,raw in cls.fixture['files'].items()} + cls.program=json.loads(cls.files['ismir2026-json-research-v1/papers.json']) + cls.accepted=cls.files['remaining2026-listings-research-v2/ismir-accepted.html'] + + def materialize(self,root): + for name,raw in self.files.items(): + p=root/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(raw) + + def test140_source_versions_and_review_optins_against_independent_gold(self): + listing=c.accepted_listing(self.accepted);observations=c.program_observations(self.program,listing) + self.assertEqual(len(observations),140) + for observation,gold in zip(observations,self.fixture['independent_gold']): + r=observation['original'];v=r['content'];official=observation['accepted_listing'] + self.assertEqual((r['id'],r['forum'],v['title'],v['authors'],v['abstract']),(gold['program_id'],gold['forum_literal'],gold['title'],gold['authors'],gold['abstract'])) + self.assertEqual(official,{'title':gold['official_title'],'authors_literal':gold['official_authors_literal'],'author_names':gold['official_author_names']}) + self.assertEqual(v['session'],gold['session_literal']);self.assertEqual(v['paper_presentation'],gold['presentation_literal']);self.assertEqual(v['keywords'],gold['keywords']) + self.assertEqual(sum('program_and_accepted_title_literal_versions_differ' in r['review_flags'] for r in observations),2) + self.assertEqual({r['original']['id'] for r in observations if 'program_and_accepted_author_sequence_versions_differ' in r['review_flags']},{'60','126','433'}) + self.assertEqual(sum(r['original']['reviews']['publish_reviews']=='Yes' for r in observations),101) + + def test_full_collection_keeps_abstract_bytes_review_text_tags_and_no_inferred_ids(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out';report=c.collect(out,snapshots=src) + self.assertEqual((report['records'],report['missing_abstract_count'],report['review_optin_yes'],report['review_optin_no']),(140,0,101,39)) + self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0) + self.assertEqual(report['public_rejected_availability'],'unknown_accepted_program_reviews_are_not_a_rejected_export') + manifest,=report['files'];raw=Path(manifest['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']);self.assertEqual(manifest['status'],'partial') + rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),140) + for row,original,gold in zip(rows,self.program,self.fixture['independent_gold']): + self.assertEqual(row['metadata']['program_entry'],original);self.assertEqual(row['abstract'],gold['abstract']);self.assertEqual(row['title'],gold['title']);self.assertEqual(row['authors'],gold['authors']) + self.assertEqual(row['external_id'],'2026/'+gold['program_id']);self.assertEqual(row['identifiers'],{});self.assertEqual(row['submission_identifiers'],[]) + self.assertEqual(row['status'],'accepted');self.assertEqual(row['track'],'program_unclassified');self.assertIsNone(row['metadata']['paper_type']);self.assertFalse(row['metadata']['original_decision_signature_verified']) + tags=[{'namespace':'ismir_program:presentation','value':gold['presentation_literal']}]+[{'namespace':'ismir_program:keyword','value':v} for v in dict.fromkeys(' '.join(k.split()) for k in gold['keywords'] if k.strip())] + self.assertEqual(row['tags'],tags);self.assertTrue(all(t['value'] for t in row['tags'])) + if original['reviews']['publish_reviews']=='No':self.assertFalse(any(v for k,v in row['metadata']['program_entry']['reviews'].items() if k!='publish_reviews')) + for name,body in self.files.items():self.assertEqual((out/'evidence'/name).read_bytes(),body) + with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src) + + def test_duplicate_and_malformed_locator_or_incomplete_census_rejected(self): + listing=c.accepted_listing(self.accepted) + for kind in ('repeat','forum','non_numeric','missing_row'): + rows=copy.deepcopy(self.program) + if kind=='repeat':rows[1]['id']=rows[0]['id'];rows[1]['forum']=rows[0]['forum'] + if kind=='forum':rows[0]['forum']='OriginalOpenReviewID' + if kind=='non_numeric':rows[0]['id']='abc';rows[0]['forum']='abc' + if kind=='missing_row':rows.pop() + with self.subTest(kind=kind),self.assertRaises(ValueError):c.program_observations(rows,listing) + + def test_empty_fields_foreign_title_and_session_disagree_rejected(self): + listing=c.accepted_listing(self.accepted) + for field,value in [('authors',[]),('abstract',''),('title','A foreign paper'),('session',['99']),('keywords',[42])]: + rows=copy.deepcopy(self.program);rows[0]['content'][field]=value + with self.subTest(field=field),self.assertRaises(ValueError):c.program_observations(rows,listing) + + def test_source_optin_no_cannot_publish_review_text(self): + rows=copy.deepcopy(self.program);r=next(r for r in rows if r['reviews']['publish_reviews']=='No');r['reviews']['meta_review']='Do not infer a negative decision' + with self.assertRaisesRegex(ValueError,'preference'):c.program_observations(rows,c.accepted_listing(self.accepted)) + + def test_parenthesized_affiliation_semicolons_do_not_split_people(self): + self.assertEqual(c.author_names('A Person (University A; Lab (X)); B Person (Institute B)'),['A Person','B Person']) + for raw in ['A Person (Broken','; A Person (Lab)','A Person )']: + with self.assertRaises(ValueError):c.author_names(raw) + + def test_ambiguous_official_title_is_not_fused(self): + source=self.accepted.replace(b'FusID: Modality-Fused Semantic IDs for Generative Music Recommendation',b'Off-Manifold Robustness in Synthesizer Inversion with Joint Distribution Flow Matching',1) + with self.assertRaisesRegex(ValueError,'ambiguous'):c.accepted_listing(source) + + def test_tamper_missing_file_and_symlink_fail_without_success_report(self): + for kind in ('manifest','body','missing','symlink'): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);src=root/'src';self.materialize(src);path=src/'ismir2026-json-research-v1'/('manifest.json' if kind=='manifest' else 'papers.json') + if kind=='missing':path.unlink() + elif kind=='symlink':other=root/'actual.json';path.rename(other);path.symlink_to(other) + else:path.write_bytes(path.read_bytes()+b' ') + with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src) + self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())