diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index f7f87a1..736a3a3 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -151,6 +151,21 @@ jobs: (rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts'])) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True) assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210 + recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes())) + recsys_snapshots = root/'recsys2026-snapshots' + recsys_snapshots.mkdir() + for name,encoded in recsys_fixture['files'].items(): + path = recsys_snapshots/name + path.parent.mkdir(parents=True,exist_ok=True) + path.write_bytes(base64.b64decode(encoded)) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True) + recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text()) + assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0) + assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3 + assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0 + recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()] + gold_keys = recsys_fixture['independent_gold'][0].keys() + assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold'] corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes())) corl_official = root/'corl2026-official' corl_archive = root/'corl2026-archive' diff --git a/README.md b/README.md index 4417f2c..50458f7 100644 --- a/README.md +++ b/README.md @@ -136,3 +136,5 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 `collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。 `collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。 + +`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。 diff --git a/docs/recsys2026-program.md b/docs/recsys2026-program.md new file mode 100644 index 0000000..7edaf03 --- /dev/null +++ b/docs/recsys2026-program.md @@ -0,0 +1,53 @@ +# RecSys 2026 公开录用贡献采集 + +这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不连接数据库、不访问论文链接或下载 PDF。输出须经过作品、DOI 和投稿历史审查才能导入。 + +## 使用 + +```sh +paper-atlas collect recsys2026-program \ + --snapshots /data/paper-index/collected \ + --out /data/paper-index/collected/recsys2026-normalized- +``` + +输入父目录应包含 `remaining2026-listings-research-v2/manifest.json` 与 `recsys-contributions.html`。采集器检查固定清单 SHA256、官方 URL、HTTP 200、采集时区、原始字节哈希、九个 track 的菜单及完整条目数。现有输出目录不可覆盖;失败保留 `failed.json`,成功才发布 `collection-report.json`。再次运行前先检查原任务是否结束。 + +## 数据范围与重复项 + +来源是 [RecSys 官方录用贡献页面](https://recsys.acm.org/recsys26/contributions/),冻结时间为 2026-11-17。九个来源 track: + +| track | 观察数 | +| --- | ---: | +| long | 65 | +| short | 29 | +| past_present_future | 13 | +| reproducibility | 4 | +| resource | 7 | +| industry | 83 | +| research_practice_notes | 34 | +| demo | 18 | +| doctoral_symposium | 10 | + +**263 项来源观察对应 260 个不同 DOI,并非 263 篇不同论文。** 每条记录用官方页面中的 accordion locator 标识,分别保留来源标题、作者原文、摘要和位置,不假设 locator 是原始投稿 ID。 + +以下 DOI 各出现两次,六个观察均显式标记待审查: + +- `10.1145/3773078.3831839`:Dart 两条来源记录的标题、作者和摘要都有差异;不得静默选一个覆盖另一个。 +- `10.1145/3773078.3831819`:标题与作者相同、摘要版本不同。 +- `10.1145/3773078.3831861`:重复展示的文字相同,保留两个来源位置。 + +所有观察都有公开摘要。正文只转换为 HTML 展示文字并规范空白,原始页面完整保留;遇到未审查的数学或图片结构会停止,不把上标扁平化成数字。作者按来源的逗号和最后一个 and 分隔,原始名单同时保留;这不是已验证的作者人物身份或别名。 + +`accepted` 仅指官方页面明示录用,不是原始签名评审决定。DOI 来自官方超链接,尚未独立核查实时出版商版本;归入 `publication_doi`,不复制其他会议的投稿 ID。track 明确来自页面区块;`recsys_program:contribution_type` 仅保存页面原始类型标签(Research、PastPresentFuture、Doctoral Symposium),各 track 的 RES/IND/RPN 等缩写同时保留在来源字段,不统一推断研究方法类型。 + +本页只含录用贡献,不能据此推断未出现的稿件被拒。公开拒稿导出仍为 unknown,征稿页的 desk rejection 政策不等于具体拒稿记录。不得将本批采集成功当成会议全部来源、负面决定或全库去重完成。 + +## 输出与恢复 + +输出九份 JSONL、对应部分覆盖清单、原始 HTML 和清单证据,以及记录观察数和不同 DOI 数的报告;批准导入数为零。 + +清单 SHA256:`fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9`。 + +服务器恢复包 `/data/paper-index/validation/remaining2026-listings-research-v2-recovery.tar.gz`,本地备份 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复原始目录、核对哈希后重跑上述命令。 + +完整来源与独立 263 项基准已保存在测试夹具,所有测试、构建和安装验证交给指定的 `ces-control:host` runner。生产导入前还需独立核验规范化结果,并检查当前数据库已有 DOI 绑定及历次投稿。最终全库去重审计仍是独立必做阶段。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 8808b18..a51eced 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -30,6 +30,7 @@ COMMANDS = { ('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'), ('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'), ('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'), + ('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'), ('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'), diff --git a/paper_atlas/collectors/recsys2026.py b/paper_atlas/collectors/recsys2026.py new file mode 100644 index 0000000..5ec0876 --- /dev/null +++ b/paper_atlas/collectors/recsys2026.py @@ -0,0 +1,172 @@ +"""Frozen RecSys 2026 accepted-contribution observations, preserving source variants.""" +import argparse +from collections import Counter +from datetime import datetime +import hashlib +import json +from pathlib import Path +import re + +from paper_atlas.io.html import Document, Node, clean + +SOURCE = 'recsys_program' +SNAPSHOT = 'remaining2026-listings-research-v2' +MANIFEST_SHA = 'fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9' +URL = 'https://recsys.acm.org/recsys26/contributions/' +TRACKS = [('long', 'Long Papers', 65), ('short', 'Short Papers', 29), + ('past_present_future', 'PPF Papers', 13), ('reproducibility', 'Reproducibility', 4), + ('resource', 'Resource', 7), ('industry', 'Industry', 83), + ('research_practice_notes', 'R&P Notes', 34), ('demo', 'Demos', 18), + ('doctoral_symposium', 'Doctoral', 10)] + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def one(items): + if len(items) != 1: + raise ValueError('Expected exactly one reviewed source field') + return items[0] + + +def load_snapshot(root): + root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json' + if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024: + raise ValueError('Unsafe snapshot manifest') + raw = path.read_bytes() + if digest(raw) != MANIFEST_SHA: + raise ValueError('Reviewed manifest hash changed') + manifest = json.loads(raw) + if manifest['status'] != 'research_completed_not_coverage_or_import_approval' or manifest['database_written'] is not False: + raise ValueError('Unreviewed snapshot scope') + receipt = one([e for e in manifest['entries'] if e['venue'] == 'RecSys' and e['label'] == 'contributions']) + name = receipt['file']; path = directory/name + if Path(name).name != name or path.is_symlink() or path.stat().st_size > 4*1024**2: + raise ValueError('Unsafe snapshot body') + body = path.read_bytes() + if (receipt['url'] != URL or receipt['final_url'] != URL or receipt['http_status'] != 200 + or not receipt['content_type'].startswith('text/html') or receipt['bytes'] != len(body) + or receipt['sha256'] != digest(body) or datetime.fromisoformat(receipt['at']).utcoffset() is None): + raise ValueError('Snapshot receipt/body disagree') + return receipt, {'manifest.json': raw, name: body}, body + + +def body_text(node, doi_url): + if not isinstance(node, Node): + return node + if node.tag in ('script', 'style') or (node.tag == 'a' and node.attrs.get('href') == doi_url): + return '' + text = ''.join(body_text(c, doi_url) for c in node.children) + return ' '+text+' ' if node.tag in ('p','div','li','br','section') else text + + +def parse_program(raw): + doc = Document(raw.decode('utf-8')); nodes = list(doc.root.walk()) + if not any(n.tag == 'title' and 'RecSys 26 - Accepted Contributions' in n.text() for n in nodes): + raise ValueError('Wrong conference edition') + menu = one([n for n in nodes if n.tag == 'ul' and n.attrs.get('id') == 'tab-1']) + labels = [(n.attrs.get('href'), clean(n.text())) for n in menu.walk() if n.tag == 'a'] + if labels != [(f'#content-tab-1-{i}', label) for i, (_,label,_) in enumerate(TRACKS)]: + raise ValueError('Unreviewed track menu') + observations = []; seen = set() + for tab, (track, label, count) in enumerate(TRACKS): + section = one([n for n in nodes if n.tag == 'div' and n.attrs.get('id') == f'content-tab-1-{tab}']) + listing = one([n for n in section.walk() if n.tag == 'ul' and n.attrs.get('id') == f'accordion-{tab+1}']) + rows = [n for n in listing.children if isinstance(n, Node) and n.tag == 'li'] + if len(rows) != count: + raise ValueError('Incomplete reviewed track census') + for position, row in enumerate(rows, 1): + anchor = one([n for n in row.children if isinstance(n, Node) and n.tag == 'a']) + locator = anchor.attrs.get('rel', '').removeprefix('#') + if not re.fullmatch(f'accordion-{tab+1}-slide-[0-9]+', locator) or locator in seen: + raise ValueError('Missing or repeated program locator') + seen.add(locator) + types = [n for n in anchor.walk() if n.tag == 'span' and n.attrs.get('class') == 'paper-type'] + type_node = one(types); type_literal = type_node.attrs.get('title'); abbreviation = clean(type_node.text()) + expected_type = [('Research','RES'),('Research','RES'),('PastPresentFuture','PPF'), + ('Research','REPR'),('Research','RSC'),('Research','IND'),('Research','RPN'),('Research','DEMO'),('Doctoral Symposium','DS')][tab] + if (type_literal,abbreviation) != expected_type: + raise ValueError('Unreviewed source contribution type') + title = clean(''.join(c.text() if isinstance(c, Node) else c for c in anchor.children + if not isinstance(c, Node) or c.tag not in ('span','em','br'))) + authors_literal = clean(one([n for n in anchor.walk() if n.tag == 'em']).text()) + if not authors_literal.startswith('by '): + raise ValueError('Missing program author prefix') + authors_literal = authors_literal[3:] + authors = [n.strip() for n in re.split(r',\s+|\s+and\s+', authors_literal)] + if not title or any(not n or '@' in n for n in authors): + raise ValueError('Missing title or program author') + body = one([n for n in row.children if isinstance(n, Node) and n.tag == 'div']) + doi_url = one([n.attrs.get('href') for n in body.walk() if n.tag == 'a' + and n.attrs.get('href', '').startswith('https://doi.org/')]) + if not re.fullmatch(r'https://doi.org/10\.1145/3773078\.[0-9]+', doi_url): + raise ValueError('Wrong publication DOI family') + abstract = clean(body_text(body, doi_url)) + if len(abstract) < 100 or any(n.tag in ('math','sup','sub','img','svg') for n in body.walk()): + raise ValueError('Missing abstract or unreviewed structured markup') + observations.append({'doi': doi_url.removeprefix('https://doi.org/'), 'tab': tab, 'position': position, + 'locator': locator, 'title': title, 'authors': authors, 'authors_literal': authors_literal, + 'abstract': abstract, 'type_labels': [type_literal], 'source_type_abbreviation': abbreviation, + 'track': track, 'track_label': label}) + counts = Counter(r['doi'] for r in observations) + if len(observations) != 263 or len(counts) != 260 or {d:n for d,n in counts.items() if n>1} != { + '10.1145/3773078.3831839':2,'10.1145/3773078.3831819':2,'10.1145/3773078.3831861':2}: + raise ValueError('Unreviewed DOI observation census') + for item in observations: + item['review_flags'] = ['repeated_program_doi_preserve_versions_requires_review'] if counts[item['doi']] > 1 else [] + return observations + + +def collect(out, *, snapshots): + out = Path(out) + if out.exists(): + raise ValueError('Output exists; inspect previous operation before retrying') + out.mkdir(parents=True) + try: + receipt, files, body = load_snapshot(snapshots); observations = parse_program(body) + evidence = out/'evidence'; evidence.mkdir() + for name, raw in files.items(): + (evidence/name).write_bytes(raw) + outputs = [] + for track,label,count in TRACKS: + rows = [] + for item in observations: + if item['track'] != track: + continue + doi = item['doi']; url = URL+'#'+item['locator'] + rows.append({'source': SOURCE, 'external_id': '2026/'+item['locator'], 'url': url, + 'title': item['title'], 'authors': item['authors'], 'abstract': item['abstract'], + 'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme':'publication_doi','value':doi}], + 'venue':'RecSys', 'year':2026, 'track':track, 'status':'accepted', 'observed_at':receipt['at'], + 'evidence_url':URL, 'raw_decision':'Listed as an accepted contribution in the official RecSys2026 '+label+' section; not a signed review decision.', + 'tags':[{'namespace':SOURCE+':contribution_type','value':item['type_labels'][0]}], + 'metadata':{'program_fields':item, 'program_receipt':receipt, + 'source_evidence':{'file':'evidence/'+receipt['file'],'sha256':receipt['sha256'],'url':URL}, + 'source_manifest':{'file':'evidence/manifest.json','sha256':MANIFEST_SHA}, + 'original_decision_signature_verified':False, + 'author_scope':'Source-reported comma/and-delimited names; original literal retained. Not verified publisher person identities or aliases.', + 'doi_scope':'DOI asserted by official program hyperlink, not independently verified live publisher metadata.', + 'abstract_transform':'HTML display text and whitespace normalization; source retained. No PDF or cross-source fusion.', + 'scope':'Program observation keyed by source locator. Repeated DOI variants retained separately and flagged, not counted as distinct scientific works. Requires DOI/work/history review before import.'}}) + path = out/f'recsys-program-2026-{track}.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw) + manifest={'source':SOURCE,'parser_version':1,'venue':'RecSys','year':2026,'track':track,'count':len(rows), + 'file':str(path),'file_sha256':digest(raw),'list_url':URL,'status':'partial','missing_abstract_count':0, + 'scope':'Official accepted-contribution observations; unique DOI counts differ from observation counts.'} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');outputs.append(manifest) + report={'status':'collected_pending_identity_review','records':263,'unique_dois':260,'files':outputs, + 'track_counts':{track:count for track,_,count in TRACKS},'missing_abstract_count':0, + 'duplicate_doi_observations':6,'repeated_dois':{d:n for d,n in Counter(r['doi'] for r in observations).items() if n>1}, + 'source_manifest_sha256':MANIFEST_SHA,'database_written':False,'network_requests':0,'pdfs_downloaded':False, + 'approved_for_import':0,'public_rejected_availability':'unknown_no_public_export_verified', + 'scope':'263 program observations for260 DOIs, not263 unique papers or final work identities. Track/type labels are source scoped. Not complete public-negative coverage or a final whole-database audit.'} + part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report + except Exception as error: + (out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n') + raise + + +def main(argv=None, *, prog=None): + parser=argparse.ArgumentParser(prog=prog,description=__doc__) + parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True) + args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False)) diff --git a/tests/fixtures/recsys2026-source.json.gz b/tests/fixtures/recsys2026-source.json.gz new file mode 100644 index 0000000..9dd1e74 Binary files /dev/null and b/tests/fixtures/recsys2026-source.json.gz differ diff --git a/tests/test_recsys2026.py b/tests/test_recsys2026.py new file mode 100644 index 0000000..c24f49e --- /dev/null +++ b/tests/test_recsys2026.py @@ -0,0 +1,79 @@ +import base64 +import gzip +import hashlib +import json +from pathlib import Path +import tempfile +import unittest + +from paper_atlas.collectors import recsys2026 as c + + +class RecSys2026Test(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/recsys2026-source.json.gz').read_bytes())) + cls.files={n:base64.b64decode(raw) for n,raw in cls.fixture['files'].items()} + cls.body=cls.files[c.SNAPSHOT+'/recsys-contributions.html'] + + def materialize(self,root): + for name,raw in self.files.items(): + path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(raw) + + def test_all263_observations_against_independent_gold(self): + items=c.parse_program(self.body) + keys=self.fixture['independent_gold'][0].keys() + self.assertEqual([{k:r[k] for k in keys} for r in items],self.fixture['independent_gold']) + self.assertEqual(len(items),263);self.assertEqual(len({r['doi'] for r in items}),260) + self.assertEqual(sum(bool(r['review_flags']) for r in items),6) + variants=[r for r in items if r['doi']=='10.1145/3773078.3831839'] + self.assertNotEqual(variants[0]['title'],variants[1]['title']);self.assertNotEqual(variants[0]['authors'],variants[1]['authors']) + self.assertNotEqual(variants[0]['abstract'],variants[1]['abstract']) + + def test_installed_format_tracks_evidence_and_original_fields(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out' + report=c.collect(out,snapshots=src) + self.assertEqual((report['records'],report['unique_dois'],report['missing_abstract_count']),(263,260,0)) + self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0) + self.assertEqual(report['track_counts'],{k:n for k,_,n in c.TRACKS});self.assertEqual(report['duplicate_doi_observations'],6) + gold={r['locator']:r for r in self.fixture['independent_gold']};seen=set() + for manifest in report['files']: + path=Path(manifest['file']);raw=path.read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']) + rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),manifest['count']);self.assertEqual(manifest['status'],'partial') + for r in rows: + item=r['metadata']['program_fields'];locator=item['locator'];seen.add(locator);expected=gold[locator] + for key in expected:self.assertEqual(item[key],expected[key]) + self.assertEqual(r['external_id'],'2026/'+locator);self.assertEqual(r['title'],expected['title']);self.assertEqual(r['authors'],expected['authors']);self.assertEqual(r['abstract'],expected['abstract']) + self.assertEqual(r['track'],manifest['track']);self.assertEqual(r['identifiers'],{'doi':expected['doi']});self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':expected['doi']}]) + self.assertEqual(r['tags'],[{'namespace':'recsys_program:contribution_type','value':expected['type_labels'][0]}]);self.assertFalse(r['metadata']['original_decision_signature_verified']) + self.assertEqual(seen,set(gold));self.assertEqual((out/'evidence/recsys-contributions.html').read_bytes(),self.body) + with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src) + + def test_edition_and_track_label_drift_rejected(self): + for old,new in [(b'RecSys 26 - Accepted Contributions',b'RecSys 25 - Accepted Contributions'),(b'>Long Papers',b'>Workshops'),(b"title='Research'",b"title='Unreviewed'")]: + with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1)) + + def test_duplicate_locator_missing_author_and_foreign_doi_rejected(self): + for old,new in [(b'accordion-1-slide-2"',b'accordion-1-slide-1"'),(b'by Md Aminul Islam, Kathryn Vasilaky and Elena Zheleva',b'by '),(b'https://doi.org/10.1145/3773078.3831824',b'https://doi.org/10.1145/9999999.3831824')]: + with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1)) + + def test_math_markup_requires_review_and_is_not_flattened(self): + changed=self.body.replace(b'Learning-to-rank (LTR)',b'Learning-to-rank 2 (LTR)',1) + with self.assertRaisesRegex(ValueError,'markup'):c.parse_program(changed) + + def test_corruption_and_missing_receipt_body_preserve_failure(self): + for kind in ('manifest','body','missing'): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'recsys-contributions.html') + if kind=='missing':path.unlink() + else:path.write_bytes(path.read_bytes()+b' ') + with self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src) + self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists()) + + def test_symlink_source_and_root_rejected(self): + for kind in ('source','root'): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/'recsys-contributions.html' if kind=='source' else src + original=path.with_name(path.name+'-original');path.rename(original);path.symlink_to(original,target_is_directory=kind=='root') + with self.assertRaisesRegex(ValueError,'Unsafe'):c.collect(root/'out',snapshots=src)