diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 9726482..34490c0 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -19,8 +19,9 @@ jobs: # Public repository archive: no owner credentials or runner SSH keys. url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz' with urllib.request.urlopen(url, timeout=60) as response: - archive = response.read(16*1024*1024+1) - if len(archive)>16*1024*1024: + # Full publication-source fixtures retain original bibliographic evidence. + archive = response.read(24*1024*1024+1) + if len(archive)>24*1024*1024: raise SystemExit('Unexpectedly large source archive') with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp: root = pathlib.Path(tmp) @@ -66,7 +67,27 @@ jobs: assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256'] assert [c['source_pointer'] for c in row['contexts']] == gold['locations'] assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row + mm_pub_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-publications.json.gz').read_bytes())) + mm_pub_registry = root/'mm2026-publication-registry' + for name,body in mm_pub_fixture['files'].items(): + path=mm_pub_registry/name + path.parent.mkdir(parents=True,exist_ok=True) + path.write_bytes(body.encode('utf-8')) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True) + mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text()) + assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1717,201) + assert mm_pub_report['unmatched_program_count']==417 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0 + mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())} + assert len(mm_pub_rows)==1918 + for gold in mm_pub_fixture['independent_gold']: + row=mm_pub_rows[gold['doi']] + assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track']) + assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256'] + assert row['metadata']['program_match_kind']==gold['kind'] and not row['metadata']['scientific_work_identity_approved'] and not row['metadata']['research_type_classified'] if service_file.is_file(): + subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--files',*[f['file'] for f in mm_pub_report['files']],'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-publication-preflight')],cwd=outside,env=testenv,check=True) + mm_pub_preflight=json.loads((root/'installed-mm2026-publication-preflight/postgresql-preflight.json').read_text()) + assert mm_pub_preflight['records']==1918 and mm_pub_preflight['transaction_read_only'] and not mm_pub_preflight['database_written'] and mm_pub_preflight['approved_for_import']==0 subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True) mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text()) assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28 diff --git a/README.md b/README.md index 914f413..205294f 100644 --- a/README.md +++ b/README.md @@ -142,3 +142,5 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 `collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。 `collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。 + +`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,717 份节目摘要明确标注来源,192 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。 diff --git a/docs/mm2026-publications.md b/docs/mm2026-publications.md new file mode 100644 index 0000000..374d114 --- /dev/null +++ b/docs/mm2026-publications.md @@ -0,0 +1,67 @@ +# MM 2026 出版条目采集与恢复 + +`collect mm2026-publications` 是包内可复用的冻结快照规范化命令,读取完整注册目录和官方节目,不连接数据库、不请求网络、不下载PDF。总体仍为13/17,来源规范化、测试通过或节目匹配都不能替代身份批准、生产入库或最终去重。 + +## 使用 + +在已配置的服务器环境中,`--registry-snapshots` 是包含 `mm2026-container-registry-v2/` 的父目录;`--program-snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录。父论文集正文使用独立固定快照,不能从不存在的文章ISBN字段猜测: + +```sh +paper-atlas collect mm2026-publications \ + --registry-snapshots /data/paper-index/collected \ + --program-snapshots /data/paper-index/collected \ + --parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \ + --out /data/paper-index/collected/mm2026-publications-native-v1 +``` + +输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。 + +## 注册目录与节目分别保留 + +源目录基于 Crossref [官方精确过滤器](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/) 的 `container-title` 和出版者member320,实际捕获8页、1918个不同DOI。每页请求、时间、原始正文、字节哈希、总量与唯一DOI均核验。父DOI为 `10.1145/3767308`,题名为 *Proceedings of the 34th ACM International Conference on Multimedia*,明确事件为MM2026;文章均注册为 `proceedings-article`。 + +ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `query.container-title` 的检索总量也不能当会议篇数。当前1918是这个明确论文集过滤范围内的注册条目数,不是已经去重的科研作品数、所有MM workshop或全部28个来源track的出版全集。 + +每个出版条目都输出,稳定外部标识为真实DOI。身份标识仅为 `doi`,投稿侧标识仅为 `publication_doi`;不推断原始OpenReview/forum/投稿号或arXiv标识。来源名 `mm_registry_program` 明确表示派生的注册/节目材料,不能把其中所有字段都说成Crossref原文。 + +原始注册响应及父记录复制到 `evidence/registry/`,完整节目及四个既有403回执复制到 `evidence/program/`,不重新请求受限入口。每项元数据保留注册正文的文件/哈希、数组位置和规范JSON对象哈希;常用书目字段另作投影,完整引用、资助等原始字段仍在原始响应中,未丢弃也不在每条JSONL重复膨胀。 + +## 分轨、摘要与版本差异 + +按原始题名的Unicode/字母数字规范化与完整作者多重集合比对,1726项有唯一节目对应。源track映射到对应输出分区,并保留原始标签、节目ID、机构和所有展示上下文;不生成全局人物别名。其余122项作者版本差异、70项题名版本差异保留在 `proceedings_unclassified`,不强配分轨或静默排除。 + +| 输出分区 | 注册条目 | +| --- | ---: | +| main | 1533 | +| grand_challenge | 77 | +| interactive_art | 17 | +| datasets | 40 | +| reproducibility | 1 | +| bni | 22 | +| open_source | 11 | +| doctoral_symposium | 2 | +| demo_and_video | 20 | +| industry_talk | 3 | +| proceedings_unclassified | 192 | + +注册摘要1918项全缺失。1717项有明确对应的节目摘要,逐字保留并标明 `abstract_source=official_program_exact_metadata_correspondence`;注册原始摘要仍标为缺失,绝不伪称由出版者提供。其余201项保持缺失,包括9项已对应节目本身没有摘要的条目及192个版本待核条目。没有用模糊匹配把待核摘要强加给论文。 + +8个作者仅有单一姓名字段,保留登记值并标记,不猜测given/family;37个含排版标记或实体的题名保留字面原文并标记,不能损失数学上下标或假定排版版本已经解决。 + +完整2143个节目ID和4326个展示位置仍在匹配证据或417个未匹配节目中,28个原始track均保留。417个节目与192个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。 + +注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`mm_registry_program:crossref:type` 与 `mm_registry_program:mm_program:track` 分别保存注册书目类型和节目 track;共同的派生来源前缀满足写入器的来源校验,两个标签命名空间仍相互独立,原始平台字段与证据继续保留在各自元数据中,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。 + +## 只读数据库预检与导入边界 + +每个分区都有带count/SHA256的manifest,coverage始终为partial。规范化完成后,对实际报告中的全部输出文件运行现有 `audit source-identities --files ... --pg-service foundry-research --pg-schema research --out <新目录>`,不能靠已有66个arXiv/DOI候选代表全目录。预检使用只读一致性快照,并保存来源/标识/历次投稿/逐次决定的证据包;缺失或截断必须单列。 + +`approved_for_import=0`,每项 `scientific_work_identity_approved=false`。DOI绑定可能有误,精确题名/作者匹配不证明科学作品身份,没有匹配也不证明新作品。进入增量导入前仍须审查已有绑定、科学版本与候选历史,做新鲜保护检查、完整基线/备份,再使用原生写入者。不得覆盖旧拒稿、后续录用、标签、原始元数据或人物姓名版本;既有已消费队列不可重放。 + +## 恢复 + +完整注册目录及独立节目比对恢复包:`/data/paper-index/validation/mm2026-container-registry-recovery-v2.tar.gz`,SHA256 `ddb05a8134f6437b3fb40250e45ed6350d55431d953812abda3300871b6dd0ac`。其中含注册源目录、父记录 `validation/mm2026-parent.json`、逐项比对和请求方法证据。安全解包后可把该父记录路径传给 `--parent-snapshot`。 + +节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。 + +实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1726个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 71fb7e4..e4f8590 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -33,6 +33,7 @@ COMMANDS = { ('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'), ('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'), ('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'), + ('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'), ('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'), diff --git a/paper_atlas/collectors/mm_publications2026.py b/paper_atlas/collectors/mm_publications2026.py new file mode 100644 index 0000000..22d8393 --- /dev/null +++ b/paper_atlas/collectors/mm_publications2026.py @@ -0,0 +1,227 @@ +"""Frozen MM 2026 registered publications with separately attributed program evidence. + +All publications survive normalization. A program metadata match is not a +scientific-work identity decision; publisher article type is not research type. +""" +import argparse +from collections import Counter, defaultdict +from datetime import datetime +import hashlib +import json +from pathlib import Path +import re +import unicodedata +from urllib.parse import parse_qs, urlsplit + +from paper_atlas.collectors import mm2026 as program + +SOURCE = 'mm_registry_program' +SNAPSHOT = 'mm2026-container-registry-v2' +MANIFEST_SHA = '128b9d02c72261a66edb9a4537d118bdccab3ed80d49a1e971fefdac399f9855' +PARENT_SHA = 'bc3ce7da2914f9ddd198c928b3acb2734a8370add3348c23c5f9f12135381628' +PARENT_DOI = '10.1145/3767308' +CONTAINER = 'Proceedings of the 34th ACM International Conference on Multimedia' +EVENT = "MM '26: The 34th ACM International Conference on Multimedia" +TRACKS = {'Main Track':'main','Grand Challenge':'grand_challenge', + 'Interactive Art':'interactive_art','Datasets':'datasets','Reproducibility':'reproducibility', + 'BNI':'bni','Open Source':'open_source','Doctoral Symposium':'doctoral_symposium', + 'Demo and Video':'demo_and_video','Industry Talk':'industry_talk'} +BIBLIOGRAPHY = {'DOI','URL','title','original-title','subtitle','short-title','author', + 'publisher','member','type','container-title','short-container-title','event','abstract', + 'page','ISBN','isbn-type','published','published-print','published-online','issued', + 'license','resource','relation','alternative-id'} + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def key(text): + return ''.join(c for c in unicodedata.normalize('NFKD',text).casefold() if c.isalnum()) + + +def author_names(values): + return Counter(key(v) for v in values) + + +def load_registry(root, parent_path): + directory = Path(root)/SNAPSHOT + raw_manifest = program.safe_read(directory/'manifest.json', 128*1024) + raw_parent = program.safe_read(Path(parent_path), 128*1024) + if digest(raw_manifest) != MANIFEST_SHA or digest(raw_parent) != PARENT_SHA: + raise ValueError('Reviewed registry or parent hash changed') + manifest = json.loads(raw_manifest);parent = json.loads(raw_parent)['message'] + if (manifest['status'] != 'complete_exact_container_registry_capture_not_work_or_track_approval' + or manifest['unique_registry_DOIs'] != 1918 or manifest['expected_registry_items'] != 1918 + or len(manifest['pages']) != 8 or manifest['database_written'] is not False + or manifest['approved_for_import'] != 0 or manifest['parent_body_sha256'] != PARENT_SHA + or parent['DOI'] != PARENT_DOI or parent['type'] != 'proceedings' + or parent['title'] != [CONTAINER] or parent['event']['name'] != EVENT + or parent['member'] != '320' or parent['ISBN'] != ['9798400722134']): + raise ValueError('Incomplete reviewed registry or wrong proceedings') + evidence = {'manifest.json':raw_manifest,'parent.json':raw_parent};rows=[];seen=set() + for number,receipt in enumerate(manifest['pages'],1): + name=receipt['file'] + if name != f'page-{number:03}.json': + raise ValueError('Unexpected registry page filename') + body=program.safe_read(directory/name,8*1024**2) + url=urlsplit(receipt['url']);query=parse_qs(url.query) + expected_filter='container-title:'+CONTAINER+',member:320' + if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url'] + or url.scheme != 'https' or url.netloc != 'api.crossref.org' or url.path != '/works' + or set(query) != {'filter','rows','cursor'} or query['filter'] != [expected_filter] + or query['rows'] != ['250'] or not query['cursor'][0] + or datetime.fromisoformat(receipt['at']).utcoffset() is None + or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)): + raise ValueError('Registry receipt and source body disagree') + obj=json.loads(body);message=obj['message'];expected=250 if number<8 else 168 + if (obj['status'] != 'ok' or message['total-results'] != 1918 + or len(message['items']) != expected or receipt['items'] != expected): + raise ValueError('Incomplete registry pagination') + evidence[name]=body + for position,original in enumerate(message['items']): + doi=original.get('DOI') + if not isinstance(doi,str) or not re.fullmatch(r'10\.1145/3767308\.[0-9]+',doi) or doi in seen: + raise ValueError('Missing, foreign or repeated publication DOI') + seen.add(doi) + if (original.get('URL') != 'https://doi.org/'+doi or original.get('type') != 'proceedings-article' + or original.get('container-title') != [CONTAINER] or original.get('event',{}).get('name') != EVENT + or original.get('member') != '320' or original.get('publisher') != 'ACM' + or original.get('published') != {'date-parts':[[2026,11,9]]} or original.get('abstract')): + raise ValueError('Wrong registered publication context or changed abstract availability') + titles=original.get('title');authors=original.get('author') + if (not isinstance(titles,list) or len(titles)!=1 or not isinstance(titles[0],str) or not titles[0].strip() + or not isinstance(authors,list) or not authors): + raise ValueError('Missing publication title or authors') + names=[];flags=[] + for i,author in enumerate(authors,1): + family=author.get('family');given=author.get('given','') + if not isinstance(family,str) or not family.strip() or not isinstance(given,str): + raise ValueError('Invalid registered author fields') + name=' '.join(x.strip() for x in (given,family) if x.strip()) + if '@' in name or name.casefold()=='anonymous': + raise ValueError('Unreviewed author identity') + names.append(name) + if not given.strip():flags.append({'kind':'single_registered_name_field_not_split','author_position':i}) + if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',titles[0]): + flags.append({'kind':'literal_title_markup_or_entity_retained_requires_review'}) + canonical=json.dumps(original,ensure_ascii=False,sort_keys=True,separators=(',',':')).encode() + rows.append({'doi':doi,'title':titles[0],'authors':names,'flags':flags, + 'fields':{k:v for k,v in original.items() if k in BIBLIOGRAPHY}, + 'original_field_names':sorted(original),'original_canonical_json_sha256':digest(canonical), + 'receipt':receipt,'source_position':position}) + if len(rows)!=1918: + raise ValueError('Incomplete registered publication census') + return evidence,rows + + +def correlate(rows, observations): + titles=defaultdict(list) + for row in observations:titles[key(row['title'])].append(row) + matched=set();kinds=Counter() + for row in rows: + candidates=titles[key(row['title'])] + exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])] + if len(exact)==1: + row['kind']='unique_program_title_fullauthor_agreement_not_identity_approval' + row['program']=exact[0];matched.add(exact[0]['paper_id']) + if exact[0]['track'] not in TRACKS: + raise ValueError('New registered program track requires review') + elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold' + elif candidates:row['kind']='program_author_version_requires_review' + else:row['kind']='no_exact_program_title_requires_review' + row['program_title_candidates']=[p['paper_id'] for p in candidates] + row['program_exact_candidates']=[p['paper_id'] for p in exact] + kinds[row['kind']]+=1 + expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726, + 'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70} + if kinds != expected: + raise ValueError('Reviewed program/publication correspondence changed') + return dict(kinds),[p for p in observations if p['paper_id'] not in matched] + + +def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot): + out=Path(out) + if out.exists() or out.is_symlink(): + raise ValueError('Output exists; inspect prior operation, never overwrite') + if any(p.is_symlink() for p in out.parents): + raise ValueError('Unsafe output parent') + out.mkdir(parents=True) + try: + registry_evidence,publications=load_registry(registry_snapshots,parent_snapshot) + program_evidence=program.load_snapshot(program_snapshots) + observations,_=program.parse_program(program_evidence['program-data.json']) + kinds,unmatched=correlate(publications,observations) + for group,files in [('registry',registry_evidence),('program',program_evidence)]: + directory=out/'evidence'/group;directory.mkdir(parents=True) + for name,body in files.items():(directory/name).write_bytes(body) + buckets=defaultdict(list);missing=0;copied=0;known_tracks=Counter() + for item in sorted(publications,key=lambda r:r['doi']): + p=item.get('program');doi=item['doi'];receipt=item['receipt'] + track=TRACKS[p['track']] if p else 'proceedings_unclassified' + abstract=p['abstract'] if p and p['abstract'] else None + missing+=abstract is None;copied+=abstract is not None + if p:known_tracks[p['track']]+=1 + tags=[{'namespace':SOURCE+':crossref:type','value':'proceedings-article'}] + if p:tags.append({'namespace':SOURCE+':mm_program:track','value':p['track']}) + buckets[track].append({'source':SOURCE,'external_id':doi,'url':'https://doi.org/'+doi, + 'title':item['title'],'authors':item['authors'],'abstract':abstract, + 'identifiers':{'doi':doi},'submission_identifiers':[{'scheme':'publication_doi','value':doi}], + 'venue':'MM','year':2026,'track':track,'status':'accepted', + 'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.', + 'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags, + 'metadata':{'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'], + 'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'], + 'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']), + 'canonical_object_json_sha256':item['original_canonical_json_sha256']}, + 'registry_manifest':{'file':'evidence/registry/manifest.json','sha256':MANIFEST_SHA}, + 'parent_evidence':{'file':'evidence/registry/parent.json','sha256':PARENT_SHA}, + 'program_source_evidence':{'file':'evidence/program/program-data.json','sha256':program.BODY_SHA}, + 'program_fields':p,'program_match_kind':item['kind'], + 'program_title_candidates':item['program_title_candidates'], + 'program_exact_candidates':item['program_exact_candidates'], + 'abstract_source':'official_program_exact_metadata_correspondence' if abstract is not None else 'not_available_from_reviewed_sources', + 'registry_abstract_available':False,'original_decision_signature_verified':False, + 'scientific_work_identity_approved':False,'research_type_classified':False, + 'review_flags':item['flags'], + 'scope':'Derived publication/program metadata with separate original evidence. All registered articles retained, including art/industry/demo; bibliographic type is not research type. Program correlation does not approve work identity or person aliases. Publication dates are not decision dates. DOI is publication evidence, not original OpenReview ID.'}}) + outputs=[] + for track,rows in sorted(buckets.items()): + path=out/f'mm-publications-2026-{track}.jsonl' + raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw) + manifest={'source':SOURCE,'parser_version':1,'venue':'MM','year':2026,'track':track,'count':len(rows), + 'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI, + 'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows), + 'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n');outputs.append(manifest) + report={'status':'collected_publications_pending_work_identity_review','records':1918,'unique_registered_dois':1918, + 'files':outputs,'program_ids':2143,'program_observations':4326,'program_match_counts':kinds, + 'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched, + 'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918, + 'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing, + 'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']), + 'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications), + 'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False, + 'public_negative_availability':'unknown_no_negative_decisions_inferred', + 'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.192title/authorvariants remainunclassified,417unmatchedprogram items retained. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'} + if (sum(f['count'] for f in outputs)!=1918 or copied!=1717 or missing!=201 or len(unmatched)!=417): + raise ValueError('Reviewed output partition or abstract census changed') + part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json') + return report + except Exception as error: + (out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n') + raise + + +def main(argv=None, *, prog=None): + parser=argparse.ArgumentParser(prog=prog,description=__doc__) + parser.add_argument('--registry-snapshots',required=True) + parser.add_argument('--program-snapshots',required=True) + parser.add_argument('--parent-snapshot',required=True) + parser.add_argument('--out',required=True) + args=parser.parse_args(argv) + report=collect(args.out,registry_snapshots=args.registry_snapshots, + program_snapshots=args.program_snapshots,parent_snapshot=args.parent_snapshot) + print(json.dumps({k:report[k] for k in ('status','records','files','program_match_counts', + 'abstracts_from_exact_program_correspondence','output_missing_abstract_count', + 'unmatched_program_count','database_written','approved_for_import')},ensure_ascii=False)) diff --git a/tests/fixtures/mm2026-publications.json.gz b/tests/fixtures/mm2026-publications.json.gz new file mode 100644 index 0000000..0d1de78 Binary files /dev/null and b/tests/fixtures/mm2026-publications.json.gz differ diff --git a/tests/test_mm_publications2026.py b/tests/test_mm_publications2026.py new file mode 100644 index 0000000..3614a7e --- /dev/null +++ b/tests/test_mm_publications2026.py @@ -0,0 +1,218 @@ +from contextlib import redirect_stdout +import io +import os +import uuid +import gzip +import hashlib +import json +from pathlib import Path +import re +import tempfile +import unittest + +from paper_atlas.collectors import mm_publications2026 as c + + +class MMPublications2026Tests(unittest.TestCase): + @classmethod + def setUpClass(cls): + fixtures=Path(__file__).parent/'fixtures' + cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes())) + cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes())) + cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name) + cls.materialize(cls.root/'registry',cls.fixture['files']) + cls.materialize(cls.root/'program',cls.program_fixture['files']) + cls.out=cls.root/'out' + cls.report=c.collect(cls.out,registry_snapshots=cls.root/'registry',program_snapshots=cls.root/'program',parent_snapshot=cls.root/'registry/parent.json') + cls.rows=[json.loads(line) for f in cls.report['files'] for line in Path(f['file']).read_bytes().splitlines()] + cls.by_doi={r['external_id']:r for r in cls.rows} + + @classmethod + def tearDownClass(cls): + cls.temp.cleanup() + + @staticmethod + def materialize(root,files): + for name,body in files.items(): + path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode('utf-8')) + + def test_all1918_publications_against_independent_registry_program_gold(self): + self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918) + self.assertEqual(set(self.by_doi),{g['doi'] for g in self.fixture['independent_gold']}) + for g in self.fixture['independent_gold']: + r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields'] + self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track'])) + self.assertEqual(m['program_match_kind'],g['kind']) + self.assertEqual(m['registry_source_evidence']['json_pointer'],'/message/items/'+str(g['source_position'])) + self.assertEqual(m['registry_source_evidence']['file'],'evidence/registry/'+g['source_page']) + self.assertEqual(m['registry_source_evidence']['canonical_object_json_sha256'],g['registry_object_sha256']) + self.assertEqual(p['paper_id'] if p else None,g['program_id']) + self.assertEqual([x['source_pointer'] for x in p['contexts']] if p else [],g['program_context_pointers']) + + def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self): + count=0 + for g in self.fixture['independent_gold']: + r=self.by_doi[g['doi']];m=r['metadata'] + self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract')) + if g['abstract_sha256'] is not None: + count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256']) + self.assertEqual(r['abstract'],m['program_fields']['abstract']) + self.assertEqual(m['abstract_source'],'official_program_exact_metadata_correspondence') + else: + self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources') + self.assertEqual(count,1717);self.assertEqual(self.report['output_missing_abstract_count'],201) + + def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self): + programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']} + for p in self.report['unmatched_program']: + self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p + self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],417) + gold=self.program_fixture['independent_inventory']['inventory'] + self.assertEqual(set(programs),{g['paper_id'] for g in gold}) + for g in gold: + p=programs[g['paper_id']] + self.assertEqual((p['title'],p['authors'],p['track']),(g['title'],g['authors'],g['track_literal'])) + self.assertEqual([x['source_pointer'] for x in p['contexts']],g['locations']) + self.assertEqual(sum(len(p['contexts']) for p in programs.values()),4326) + self.assertEqual(len({p['track'] for p in programs.values()}),28) + + def test_no_person_alias_original_submission_decision_or_research_type_inference(self): + for r in self.rows: + doi=r['external_id'];m=r['metadata'] + self.assertEqual(r['identifiers'],{'doi':doi}) + self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}]) + self.assertEqual((r['source'],r['venue'],r['year']),(c.SOURCE,'MM',2026)) + self.assertEqual(r['status'],'accepted');self.assertIn('not an original signed decision',r['raw_decision']) + self.assertFalse(m['original_decision_signature_verified']);self.assertFalse(m['research_type_classified']) + self.assertFalse(m['scientific_work_identity_approved']) + self.assertTrue({'decided_at','decision_date','source_decision_id','paper_type'}.isdisjoint(r)) + self.assertIn({'namespace':c.SOURCE+':crossref:type','value':'proceedings-article'},r['tags']) + self.assertTrue({x['namespace'] for x in r['tags']} <= {c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'}) + for track in ('interactive_art','industry_talk','demo_and_video'): + self.assertTrue(any(r['track']==track for r in self.rows)) + variants=[r for r in self.rows if r['track']=='proceedings_unclassified'] + self.assertEqual(len(variants),192) + self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants)) + + def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self): + authors_by_page={} + for name,body in self.fixture['files'].items(): + actual=self.out/'evidence/registry'/('parent.json' if name=='parent.json' else Path(name).name) + self.assertEqual(actual.read_bytes(),body.encode()) + if re.fullmatch(r'page-[0-9]+\.json',Path(name).name): + authors_by_page[str(actual.relative_to(self.out))]=[row['author'] for row in json.loads(body)['message']['items']] + for name,body in self.program_fixture['files'].items(): + self.assertEqual((self.out/'evidence/program'/Path(name).name).read_bytes(),body.encode()) + for r in self.rows: + m=r['metadata'];self.assertNotIn('reference',m['registry_fields']) + self.assertEqual(m['registry_fields']['title'],[r['title']]) + self.assertEqual(m['registry_fields']['author'],authors_by_page[m['registry_source_evidence']['file']][int(m['registry_source_evidence']['json_pointer'].rsplit('/',1)[-1])]) + for f in self.report['files']: + raw=Path(f['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),f['file_sha256']) + self.assertEqual(len(raw.splitlines()),f['count']);self.assertEqual(f['status'],'partial') + + def test_eight_single_name_fields_preserved_without_splitting(self): + n=0 + for g in self.fixture['independent_gold']: + r=self.by_doi[g['doi']];flags=r['metadata']['review_flags'] + actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split'] + self.assertEqual(actual,g['single_name_positions']);n+=len(actual) + for i in actual: + self.assertEqual(r['authors'][i-1],r['metadata']['registry_fields']['author'][i-1]['family'].strip()) + self.assertEqual(n,8) + + def test_structured_title_markup_retained_and_flagged_without_flattening(self): + count=0 + for g in self.fixture['independent_gold']: + if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']): + count+=1;r=self.by_doi[g['doi']] + self.assertEqual(r['title'],g['title']) + self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags']) + self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags']) + + def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self): + for kind in ('registry_manifest','registry_page','parent','program'): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);self.materialize(root/'registry',self.fixture['files']);self.materialize(root/'program',self.program_fixture['files']) + path={'registry_manifest':root/'registry'/c.SNAPSHOT/'manifest.json','registry_page':root/'registry'/c.SNAPSHOT/'page-001.json','parent':root/'registry/parent.json','program':root/'program'/c.program.SNAPSHOT/'program-data.json'}[kind] + path.write_bytes(path.read_bytes()+b' ') + with self.subTest(kind=kind),self.assertRaises(ValueError): + c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=root/'program',parent_snapshot=root/'registry/parent.json') + self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists()) + + def test_symlink_source_rejected_and_no_previous_output_overwrite(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);self.materialize(root/'registry',self.fixture['files']) + path=root/'registry/parent.json';original=root/'registry/actual.json';path.rename(original);path.symlink_to(original) + with self.assertRaisesRegex(ValueError,'Unsafe'): + c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=self.root/'program',parent_snapshot=path) + with self.assertRaisesRegex(ValueError,'Output exists'): + c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json') + + def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self): + self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70}) + self.assertEqual(self.report['registry_missing_abstract_count'],1918) + self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1717) + self.assertEqual(sum(f['count'] for f in self.report['files']),1918) + self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written']) + self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0) + self.assertIn('unknown',self.report['public_negative_availability']) + + + def assert_native_ingestion(self, pg_service=None): + # An isolated synthetic database exercises the writer contract, not a + # production identity approval. All source rows/track partitions run. + from paper_atlas.db.store import Store + from paper_atlas.db.backend import open_store + from paper_atlas.ingestion.records import ingest_file + from paper_atlas.postgres.client import connect + from paper_atlas.postgres.migration import migrate + from paper_atlas.postgres.schema import quote + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);db=root/'seed.sqlite';seed=self.rows[0] + store=Store(db) + paper,record=store.record('arxiv','synthetic-mm-anchor','https://example.org/arxiv', + seed['title'],'Existing canonical abstract',seed['authors'],seed['identifiers']) + store.decision(paper,record,'ICLR',2025,'main','synthetic-old-forum', + 'https://example.org/decision','rejected','Reject') + before=tuple(store.conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone()) + store.conn.commit();store.conn.close() + schema='pa_mm_pub_'+uuid.uuid4().hex[:12] if pg_service else None + try: + with redirect_stdout(io.StringIO()): + if pg_service:migrate(db,pg_service,root/'migration.json',schema=schema,min_free_gib=0) + for f in self.report['files']: + ingest_file(None if pg_service else db,f['file'],pg_service=pg_service,pg_schema=schema) + store=open_store(None if pg_service else db,pg_service=pg_service,pg_schema=schema) + try: + conn=store.conn + self.assertEqual(tuple(conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone()),before) + self.assertEqual(conn.execute('SELECT status FROM decisions WHERE submission_id=1').fetchone()[0],'rejected') + actual={r[0]:r for r in conn.execute("SELECT sr.external_id,sr.paper_id,sr.title,sr.abstract,sr.authors_json,sr.metadata_json,sr.id FROM source_records sr JOIN sources s ON s.id=sr.source_id WHERE s.name=?",(c.SOURCE,))} + self.assertEqual(set(actual),set(self.by_doi)) + self.assertEqual(actual[seed['external_id']][1],paper) + for doi,row in self.by_doi.items(): + got=actual[doi];self.assertEqual((got[2],got[3],json.loads(got[4])),(row['title'],row['abstract'],row['authors'])) + meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata']) + tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall() + self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918) + self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1726) + self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'}) + decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall() + self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)]) + self.assertEqual(conn.execute("SELECT count(*) FROM submission_identifiers WHERE scheme='publication_doi'").fetchone()[0],1918) + coverage=conn.execute("SELECT es.status,sum(es.imported_count) FROM edition_sources es JOIN sources s ON s.id=es.source_id WHERE s.name=? GROUP BY es.status",(c.SOURCE,)).fetchall() + self.assertEqual([tuple(r) for r in coverage],[('partial',1918)]) + finally:store.conn.close() + finally: + if pg_service: + with connect(pg_service) as pg: + for name, in pg.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(schema,)).fetchall(): + pg.execute('DROP SCHEMA '+quote(name)+' CASCADE') + + def test_all1918_native_sqlite_source_tags_and_prior_rejection_survive(self): + self.assert_native_ingestion() + + @unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL test service') + def test_all1918_native_postgres_source_tags_and_prior_rejection_survive(self): + self.assert_native_ingestion(os.environ['PAPER_ATLAS_TEST_PG_SERVICE'])