Compare commits

...

1 Commits

Author SHA1 Message Date
codex-bot 13536d6268 Normalize all MM 2026 publications with attributed program evidence
Metadata integrity checks / metadata (push) Successful in 5m4s
2026-11-18 00:43:45 +08:00
7 changed files with 473 additions and 2 deletions
+23 -2
View File
@@ -19,8 +19,9 @@ jobs:
# Public repository archive: no owner credentials or runner SSH keys.
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
with urllib.request.urlopen(url, timeout=60) as response:
archive = response.read(16*1024*1024+1)
if len(archive)>16*1024*1024:
# Full publication-source fixtures retain original bibliographic evidence.
archive = response.read(24*1024*1024+1)
if len(archive)>24*1024*1024:
raise SystemExit('Unexpectedly large source archive')
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
root = pathlib.Path(tmp)
@@ -66,7 +67,27 @@ jobs:
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
mm_pub_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-publications.json.gz').read_bytes()))
mm_pub_registry = root/'mm2026-publication-registry'
for name,body in mm_pub_fixture['files'].items():
path=mm_pub_registry/name
path.parent.mkdir(parents=True,exist_ok=True)
path.write_bytes(body.encode('utf-8'))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True)
mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text())
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1717,201)
assert mm_pub_report['unmatched_program_count']==417 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
assert len(mm_pub_rows)==1918
for gold in mm_pub_fixture['independent_gold']:
row=mm_pub_rows[gold['doi']]
assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track'])
assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256']
assert row['metadata']['program_match_kind']==gold['kind'] and not row['metadata']['scientific_work_identity_approved'] and not row['metadata']['research_type_classified']
if service_file.is_file():
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--files',*[f['file'] for f in mm_pub_report['files']],'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-publication-preflight')],cwd=outside,env=testenv,check=True)
mm_pub_preflight=json.loads((root/'installed-mm2026-publication-preflight/postgresql-preflight.json').read_text())
assert mm_pub_preflight['records']==1918 and mm_pub_preflight['transaction_read_only'] and not mm_pub_preflight['database_written'] and mm_pub_preflight['approved_for_import']==0
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True)
mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text())
assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28
+2
View File
@@ -142,3 +142,5 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,717 份节目摘要明确标注来源,192 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
+65
View File
@@ -0,0 +1,65 @@
# MM 2026 出版条目采集与恢复
`collect mm2026-publications` 是包内可复用的冻结快照规范化命令,读取完整注册目录和官方节目,不连接数据库、不请求网络、不下载PDF。总体仍为13/17,来源规范化、测试通过或节目匹配都不能替代身份批准、生产入库或最终去重。
## 使用
在已配置的服务器环境中,`--registry-snapshots` 是包含 `mm2026-container-registry-v2/` 的父目录;`--program-snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录。父论文集正文使用独立固定快照,不能从不存在的文章ISBN字段猜测:
```sh
paper-atlas collect mm2026-publications \
--registry-snapshots /data/paper-index/collected \
--program-snapshots /data/paper-index/collected \
--parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \
--out /data/paper-index/collected/mm2026-publications-native-v1
```
输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。
## 注册目录与节目分别保留
源目录基于 Crossref [官方精确过滤器](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/) 的 `container-title` 和出版者member320,实际捕获8页、1918个不同DOI。每页请求、时间、原始正文、字节哈希、总量与唯一DOI均核验。父DOI为 `10.1145/3767308`,题名为 *Proceedings of the 34th ACM International Conference on Multimedia*,明确事件为MM2026;文章均注册为 `proceedings-article`。
ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `query.container-title` 的检索总量也不能当会议篇数。当前1918是这个明确论文集过滤范围内的注册条目数,不是已经去重的科研作品数、所有MM workshop或全部28个来源track的出版全集。
每个出版条目都输出,稳定外部标识为真实DOI。身份标识仅为 `doi`,投稿侧标识仅为 `publication_doi`;不推断原始OpenReview/forum/投稿号或arXiv标识。来源名 `mm_registry_program` 明确表示派生的注册/节目材料,不能把其中所有字段都说成Crossref原文。
原始注册响应及父记录复制到 `evidence/registry/`,完整节目及四个既有403回执复制到 `evidence/program/`,不重新请求受限入口。每项元数据保留注册正文的文件/哈希、数组位置和规范JSON对象哈希;常用书目字段另作投影,完整引用、资助等原始字段仍在原始响应中,未丢弃也不在每条JSONL重复膨胀。
## 分轨、摘要与版本差异
按原始题名的Unicode/字母数字规范化与完整作者多重集合比对,1726项有唯一节目对应。源track映射到对应输出分区,并保留原始标签、节目ID、机构和所有展示上下文;不生成全局人物别名。其余122项作者版本差异、70项题名版本差异保留在 `proceedings_unclassified`,不强配分轨或静默排除。
| 输出分区 | 注册条目 |
| --- | ---: |
| main | 1533 |
| grand_challenge | 77 |
| interactive_art | 17 |
| datasets | 40 |
| reproducibility | 1 |
| bni | 22 |
| open_source | 11 |
| doctoral_symposium | 2 |
| demo_and_video | 20 |
| industry_talk | 3 |
| proceedings_unclassified | 192 |
注册摘要1918项全缺失。1717项有明确对应的节目摘要,逐字保留并标明 `abstract_source=official_program_exact_metadata_correspondence`;注册原始摘要仍标为缺失,绝不伪称由出版者提供。其余201项保持缺失,包括9项已对应节目本身没有摘要的条目及192个版本待核条目。没有用模糊匹配把待核摘要强加给论文。
8个作者仅有单一姓名字段,保留登记值并标记,不猜测given/family;37个含排版标记或实体的题名保留字面原文并标记,不能损失数学上下标或假定排版版本已经解决。
完整2143个节目ID和4326个展示位置仍在匹配证据或417个未匹配节目中,28个原始track均保留。417个节目与192个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`crossref:type` 与 `mm_program:track` 分别保留来源命名空间,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
## 只读数据库预检与导入边界
每个分区都有带count/SHA256的manifest,coverage始终为partial。规范化完成后,对实际报告中的全部输出文件运行现有 `audit source-identities --files ... --pg-service foundry-research --pg-schema research --out <新目录>`,不能靠已有66个arXiv/DOI候选代表全目录。预检使用只读一致性快照,并保存来源/标识/历次投稿/逐次决定的证据包;缺失或截断必须单列。
`approved_for_import=0`,每项 `scientific_work_identity_approved=false`。DOI绑定可能有误,精确题名/作者匹配不证明科学作品身份,没有匹配也不证明新作品。进入增量导入前仍须审查已有绑定、科学版本与候选历史,做新鲜保护检查、完整基线/备份,再使用原生写入者。不得覆盖旧拒稿、后续录用、标签、原始元数据或人物姓名版本;既有已消费队列不可重放。
## 恢复
完整注册目录及独立节目比对恢复包:`/data/paper-index/validation/mm2026-container-registry-recovery-v2.tar.gz`,SHA256 `ddb05a8134f6437b3fb40250e45ed6350d55431d953812abda3300871b6dd0ac`。其中含注册源目录、父记录 `validation/mm2026-parent.json`、逐项比对和请求方法证据。安全解包后可把该父记录路径传给 `--parent-snapshot`。
节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。
+1
View File
@@ -33,6 +33,7 @@ COMMANDS = {
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
@@ -0,0 +1,227 @@
"""Frozen MM 2026 registered publications with separately attributed program evidence.
All publications survive normalization. A program metadata match is not a
scientific-work identity decision; publisher article type is not research type.
"""
import argparse
from collections import Counter, defaultdict
from datetime import datetime
import hashlib
import json
from pathlib import Path
import re
import unicodedata
from urllib.parse import parse_qs, urlsplit
from paper_atlas.collectors import mm2026 as program
SOURCE = 'mm_registry_program'
SNAPSHOT = 'mm2026-container-registry-v2'
MANIFEST_SHA = '128b9d02c72261a66edb9a4537d118bdccab3ed80d49a1e971fefdac399f9855'
PARENT_SHA = 'bc3ce7da2914f9ddd198c928b3acb2734a8370add3348c23c5f9f12135381628'
PARENT_DOI = '10.1145/3767308'
CONTAINER = 'Proceedings of the 34th ACM International Conference on Multimedia'
EVENT = "MM '26: The 34th ACM International Conference on Multimedia"
TRACKS = {'Main Track':'main','Grand Challenge':'grand_challenge',
'Interactive Art':'interactive_art','Datasets':'datasets','Reproducibility':'reproducibility',
'BNI':'bni','Open Source':'open_source','Doctoral Symposium':'doctoral_symposium',
'Demo and Video':'demo_and_video','Industry Talk':'industry_talk'}
BIBLIOGRAPHY = {'DOI','URL','title','original-title','subtitle','short-title','author',
'publisher','member','type','container-title','short-container-title','event','abstract',
'page','ISBN','isbn-type','published','published-print','published-online','issued',
'license','resource','relation','alternative-id'}
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def key(text):
return ''.join(c for c in unicodedata.normalize('NFKD',text).casefold() if c.isalnum())
def author_names(values):
return Counter(key(v) for v in values)
def load_registry(root, parent_path):
directory = Path(root)/SNAPSHOT
raw_manifest = program.safe_read(directory/'manifest.json', 128*1024)
raw_parent = program.safe_read(Path(parent_path), 128*1024)
if digest(raw_manifest) != MANIFEST_SHA or digest(raw_parent) != PARENT_SHA:
raise ValueError('Reviewed registry or parent hash changed')
manifest = json.loads(raw_manifest);parent = json.loads(raw_parent)['message']
if (manifest['status'] != 'complete_exact_container_registry_capture_not_work_or_track_approval'
or manifest['unique_registry_DOIs'] != 1918 or manifest['expected_registry_items'] != 1918
or len(manifest['pages']) != 8 or manifest['database_written'] is not False
or manifest['approved_for_import'] != 0 or manifest['parent_body_sha256'] != PARENT_SHA
or parent['DOI'] != PARENT_DOI or parent['type'] != 'proceedings'
or parent['title'] != [CONTAINER] or parent['event']['name'] != EVENT
or parent['member'] != '320' or parent['ISBN'] != ['9798400722134']):
raise ValueError('Incomplete reviewed registry or wrong proceedings')
evidence = {'manifest.json':raw_manifest,'parent.json':raw_parent};rows=[];seen=set()
for number,receipt in enumerate(manifest['pages'],1):
name=receipt['file']
if name != f'page-{number:03}.json':
raise ValueError('Unexpected registry page filename')
body=program.safe_read(directory/name,8*1024**2)
url=urlsplit(receipt['url']);query=parse_qs(url.query)
expected_filter='container-title:'+CONTAINER+',member:320'
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
or url.scheme != 'https' or url.netloc != 'api.crossref.org' or url.path != '/works'
or set(query) != {'filter','rows','cursor'} or query['filter'] != [expected_filter]
or query['rows'] != ['250'] or not query['cursor'][0]
or datetime.fromisoformat(receipt['at']).utcoffset() is None
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
raise ValueError('Registry receipt and source body disagree')
obj=json.loads(body);message=obj['message'];expected=250 if number<8 else 168
if (obj['status'] != 'ok' or message['total-results'] != 1918
or len(message['items']) != expected or receipt['items'] != expected):
raise ValueError('Incomplete registry pagination')
evidence[name]=body
for position,original in enumerate(message['items']):
doi=original.get('DOI')
if not isinstance(doi,str) or not re.fullmatch(r'10\.1145/3767308\.[0-9]+',doi) or doi in seen:
raise ValueError('Missing, foreign or repeated publication DOI')
seen.add(doi)
if (original.get('URL') != 'https://doi.org/'+doi or original.get('type') != 'proceedings-article'
or original.get('container-title') != [CONTAINER] or original.get('event',{}).get('name') != EVENT
or original.get('member') != '320' or original.get('publisher') != 'ACM'
or original.get('published') != {'date-parts':[[2026,11,9]]} or original.get('abstract')):
raise ValueError('Wrong registered publication context or changed abstract availability')
titles=original.get('title');authors=original.get('author')
if (not isinstance(titles,list) or len(titles)!=1 or not isinstance(titles[0],str) or not titles[0].strip()
or not isinstance(authors,list) or not authors):
raise ValueError('Missing publication title or authors')
names=[];flags=[]
for i,author in enumerate(authors,1):
family=author.get('family');given=author.get('given','')
if not isinstance(family,str) or not family.strip() or not isinstance(given,str):
raise ValueError('Invalid registered author fields')
name=' '.join(x.strip() for x in (given,family) if x.strip())
if '@' in name or name.casefold()=='anonymous':
raise ValueError('Unreviewed author identity')
names.append(name)
if not given.strip():flags.append({'kind':'single_registered_name_field_not_split','author_position':i})
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',titles[0]):
flags.append({'kind':'literal_title_markup_or_entity_retained_requires_review'})
canonical=json.dumps(original,ensure_ascii=False,sort_keys=True,separators=(',',':')).encode()
rows.append({'doi':doi,'title':titles[0],'authors':names,'flags':flags,
'fields':{k:v for k,v in original.items() if k in BIBLIOGRAPHY},
'original_field_names':sorted(original),'original_canonical_json_sha256':digest(canonical),
'receipt':receipt,'source_position':position})
if len(rows)!=1918:
raise ValueError('Incomplete registered publication census')
return evidence,rows
def correlate(rows, observations):
titles=defaultdict(list)
for row in observations:titles[key(row['title'])].append(row)
matched=set();kinds=Counter()
for row in rows:
candidates=titles[key(row['title'])]
exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])]
if len(exact)==1:
row['kind']='unique_program_title_fullauthor_agreement_not_identity_approval'
row['program']=exact[0];matched.add(exact[0]['paper_id'])
if exact[0]['track'] not in TRACKS:
raise ValueError('New registered program track requires review')
elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold'
elif candidates:row['kind']='program_author_version_requires_review'
else:row['kind']='no_exact_program_title_requires_review'
row['program_title_candidates']=[p['paper_id'] for p in candidates]
row['program_exact_candidates']=[p['paper_id'] for p in exact]
kinds[row['kind']]+=1
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70}
if kinds != expected:
raise ValueError('Reviewed program/publication correspondence changed')
return dict(kinds),[p for p in observations if p['paper_id'] not in matched]
def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
out=Path(out)
if out.exists() or out.is_symlink():
raise ValueError('Output exists; inspect prior operation, never overwrite')
if any(p.is_symlink() for p in out.parents):
raise ValueError('Unsafe output parent')
out.mkdir(parents=True)
try:
registry_evidence,publications=load_registry(registry_snapshots,parent_snapshot)
program_evidence=program.load_snapshot(program_snapshots)
observations,_=program.parse_program(program_evidence['program-data.json'])
kinds,unmatched=correlate(publications,observations)
for group,files in [('registry',registry_evidence),('program',program_evidence)]:
directory=out/'evidence'/group;directory.mkdir(parents=True)
for name,body in files.items():(directory/name).write_bytes(body)
buckets=defaultdict(list);missing=0;copied=0;known_tracks=Counter()
for item in sorted(publications,key=lambda r:r['doi']):
p=item.get('program');doi=item['doi'];receipt=item['receipt']
track=TRACKS[p['track']] if p else 'proceedings_unclassified'
abstract=p['abstract'] if p and p['abstract'] else None
missing+=abstract is None;copied+=abstract is not None
if p:known_tracks[p['track']]+=1
tags=[{'namespace':'crossref:type','value':'proceedings-article'}]
if p:tags.append({'namespace':'mm_program:track','value':p['track']})
buckets[track].append({'source':SOURCE,'external_id':doi,'url':'https://doi.org/'+doi,
'title':item['title'],'authors':item['authors'],'abstract':abstract,
'identifiers':{'doi':doi},'submission_identifiers':[{'scheme':'publication_doi','value':doi}],
'venue':'MM','year':2026,'track':track,'status':'accepted',
'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.',
'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags,
'metadata':{'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'],
'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']),
'canonical_object_json_sha256':item['original_canonical_json_sha256']},
'registry_manifest':{'file':'evidence/registry/manifest.json','sha256':MANIFEST_SHA},
'parent_evidence':{'file':'evidence/registry/parent.json','sha256':PARENT_SHA},
'program_source_evidence':{'file':'evidence/program/program-data.json','sha256':program.BODY_SHA},
'program_fields':p,'program_match_kind':item['kind'],
'program_title_candidates':item['program_title_candidates'],
'program_exact_candidates':item['program_exact_candidates'],
'abstract_source':'official_program_exact_metadata_correspondence' if abstract is not None else 'not_available_from_reviewed_sources',
'registry_abstract_available':False,'original_decision_signature_verified':False,
'scientific_work_identity_approved':False,'research_type_classified':False,
'review_flags':item['flags'],
'scope':'Derived publication/program metadata with separate original evidence. All registered articles retained, including art/industry/demo; bibliographic type is not research type. Program correlation does not approve work identity or person aliases. Publication dates are not decision dates. DOI is publication evidence, not original OpenReview ID.'}})
outputs=[]
for track,rows in sorted(buckets.items()):
path=out/f'mm-publications-2026-{track}.jsonl'
raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw)
manifest={'source':SOURCE,'parser_version':1,'venue':'MM','year':2026,'track':track,'count':len(rows),
'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI,
'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows),
'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n');outputs.append(manifest)
report={'status':'collected_publications_pending_work_identity_review','records':1918,'unique_registered_dois':1918,
'files':outputs,'program_ids':2143,'program_observations':4326,'program_match_counts':kinds,
'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched,
'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918,
'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing,
'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']),
'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications),
'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False,
'public_negative_availability':'unknown_no_negative_decisions_inferred',
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.192title/authorvariants remainunclassified,417unmatchedprogram items retained. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
if (sum(f['count'] for f in outputs)!=1918 or copied!=1717 or missing!=201 or len(unmatched)!=417):
raise ValueError('Reviewed output partition or abstract census changed')
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
raise
def main(argv=None, *, prog=None):
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
parser.add_argument('--registry-snapshots',required=True)
parser.add_argument('--program-snapshots',required=True)
parser.add_argument('--parent-snapshot',required=True)
parser.add_argument('--out',required=True)
args=parser.parse_args(argv)
report=collect(args.out,registry_snapshots=args.registry_snapshots,
program_snapshots=args.program_snapshots,parent_snapshot=args.parent_snapshot)
print(json.dumps({k:report[k] for k in ('status','records','files','program_match_counts',
'abstracts_from_exact_program_correspondence','output_missing_abstract_count',
'unmatched_program_count','database_written','approved_for_import')},ensure_ascii=False))
Binary file not shown.
+155
View File
@@ -0,0 +1,155 @@
import gzip
import hashlib
import json
from pathlib import Path
import re
import tempfile
import unittest
from paper_atlas.collectors import mm_publications2026 as c
class MMPublications2026Tests(unittest.TestCase):
@classmethod
def setUpClass(cls):
fixtures=Path(__file__).parent/'fixtures'
cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes()))
cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes()))
cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name)
cls.materialize(cls.root/'registry',cls.fixture['files'])
cls.materialize(cls.root/'program',cls.program_fixture['files'])
cls.out=cls.root/'out'
cls.report=c.collect(cls.out,registry_snapshots=cls.root/'registry',program_snapshots=cls.root/'program',parent_snapshot=cls.root/'registry/parent.json')
cls.rows=[json.loads(line) for f in cls.report['files'] for line in Path(f['file']).read_bytes().splitlines()]
cls.by_doi={r['external_id']:r for r in cls.rows}
@classmethod
def tearDownClass(cls):
cls.temp.cleanup()
@staticmethod
def materialize(root,files):
for name,body in files.items():
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode('utf-8'))
def test_all1918_publications_against_independent_registry_program_gold(self):
self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918)
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.fixture['independent_gold']})
for g in self.fixture['independent_gold']:
r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields']
self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track']))
self.assertEqual(m['program_match_kind'],g['kind'])
self.assertEqual(m['registry_source_evidence']['json_pointer'],'/message/items/'+str(g['source_position']))
self.assertEqual(m['registry_source_evidence']['file'],'evidence/registry/'+g['source_page'])
self.assertEqual(m['registry_source_evidence']['canonical_object_json_sha256'],g['registry_object_sha256'])
self.assertEqual(p['paper_id'] if p else None,g['program_id'])
self.assertEqual([x['source_pointer'] for x in p['contexts']] if p else [],g['program_context_pointers'])
def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self):
count=0
for g in self.fixture['independent_gold']:
r=self.by_doi[g['doi']];m=r['metadata']
self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract'))
if g['abstract_sha256'] is not None:
count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256'])
self.assertEqual(r['abstract'],m['program_fields']['abstract'])
self.assertEqual(m['abstract_source'],'official_program_exact_metadata_correspondence')
else:
self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources')
self.assertEqual(count,1717);self.assertEqual(self.report['output_missing_abstract_count'],201)
def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self):
programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']}
for p in self.report['unmatched_program']:
self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],417)
gold=self.program_fixture['independent_inventory']['inventory']
self.assertEqual(set(programs),{g['paper_id'] for g in gold})
for g in gold:
p=programs[g['paper_id']]
self.assertEqual((p['title'],p['authors'],p['track']),(g['title'],g['authors'],g['track_literal']))
self.assertEqual([x['source_pointer'] for x in p['contexts']],g['locations'])
self.assertEqual(sum(len(p['contexts']) for p in programs.values()),4326)
self.assertEqual(len({p['track'] for p in programs.values()}),28)
def test_no_person_alias_original_submission_decision_or_research_type_inference(self):
for r in self.rows:
doi=r['external_id'];m=r['metadata']
self.assertEqual(r['identifiers'],{'doi':doi})
self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
self.assertEqual((r['source'],r['venue'],r['year']),(c.SOURCE,'MM',2026))
self.assertEqual(r['status'],'accepted');self.assertIn('not an original signed decision',r['raw_decision'])
self.assertFalse(m['original_decision_signature_verified']);self.assertFalse(m['research_type_classified'])
self.assertFalse(m['scientific_work_identity_approved'])
self.assertTrue({'decided_at','decision_date','source_decision_id','paper_type'}.isdisjoint(r))
self.assertIn({'namespace':'crossref:type','value':'proceedings-article'},r['tags'])
self.assertTrue({x['namespace'] for x in r['tags']} <= {'crossref:type','mm_program:track'})
for track in ('interactive_art','industry_talk','demo_and_video'):
self.assertTrue(any(r['track']==track for r in self.rows))
variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
self.assertEqual(len(variants),192)
self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants))
def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self):
authors_by_page={}
for name,body in self.fixture['files'].items():
actual=self.out/'evidence/registry'/('parent.json' if name=='parent.json' else Path(name).name)
self.assertEqual(actual.read_bytes(),body.encode())
if re.fullmatch(r'page-[0-9]+\.json',Path(name).name):
authors_by_page[str(actual.relative_to(self.out))]=[row['author'] for row in json.loads(body)['message']['items']]
for name,body in self.program_fixture['files'].items():
self.assertEqual((self.out/'evidence/program'/Path(name).name).read_bytes(),body.encode())
for r in self.rows:
m=r['metadata'];self.assertNotIn('reference',m['registry_fields'])
self.assertEqual(m['registry_fields']['title'],[r['title']])
self.assertEqual(m['registry_fields']['author'],authors_by_page[m['registry_source_evidence']['file']][int(m['registry_source_evidence']['json_pointer'].rsplit('/',1)[-1])])
for f in self.report['files']:
raw=Path(f['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),f['file_sha256'])
self.assertEqual(len(raw.splitlines()),f['count']);self.assertEqual(f['status'],'partial')
def test_eight_single_name_fields_preserved_without_splitting(self):
n=0
for g in self.fixture['independent_gold']:
r=self.by_doi[g['doi']];flags=r['metadata']['review_flags']
actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split']
self.assertEqual(actual,g['single_name_positions']);n+=len(actual)
for i in actual:
self.assertEqual(r['authors'][i-1],r['metadata']['registry_fields']['author'][i-1]['family'].strip())
self.assertEqual(n,8)
def test_structured_title_markup_retained_and_flagged_without_flattening(self):
count=0
for g in self.fixture['independent_gold']:
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']):
count+=1;r=self.by_doi[g['doi']]
self.assertEqual(r['title'],g['title'])
self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags'])
self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags'])
def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self):
for kind in ('registry_manifest','registry_page','parent','program'):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);self.materialize(root/'registry',self.fixture['files']);self.materialize(root/'program',self.program_fixture['files'])
path={'registry_manifest':root/'registry'/c.SNAPSHOT/'manifest.json','registry_page':root/'registry'/c.SNAPSHOT/'page-001.json','parent':root/'registry/parent.json','program':root/'program'/c.program.SNAPSHOT/'program-data.json'}[kind]
path.write_bytes(path.read_bytes()+b' ')
with self.subTest(kind=kind),self.assertRaises(ValueError):
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=root/'program',parent_snapshot=root/'registry/parent.json')
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
def test_symlink_source_rejected_and_no_previous_output_overwrite(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);self.materialize(root/'registry',self.fixture['files'])
path=root/'registry/parent.json';original=root/'registry/actual.json';path.rename(original);path.symlink_to(original)
with self.assertRaisesRegex(ValueError,'Unsafe'):
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=self.root/'program',parent_snapshot=path)
with self.assertRaisesRegex(ValueError,'Output exists'):
c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json')
def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self):
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70})
self.assertEqual(self.report['registry_missing_abstract_count'],1918)
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1717)
self.assertEqual(sum(f['count'] for f in self.report['files']),1918)
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
self.assertIn('unknown',self.report['public_negative_availability'])