130 lines
13 KiB
TypeScript
130 lines
13 KiB
TypeScript
/** Deterministic segmentation + reviewed semantic regions; no generated imagery. */
|
|
import sharp from 'sharp';
|
|
import {readFile,writeFile,mkdir,rm} from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import {createHash} from 'node:crypto';
|
|
import annotations from '../contentasset:annotations.json';
|
|
import {textMergeTarget,unionBoxes} from '../contentasset:group-text';
|
|
import ocr from '../contentasset:ocr-evidence.json';
|
|
const root=path.resolve(import.meta.dirname,'..'),target=path.resolve(process.env.MEDIA_AUTHORING_ROOT||'/tmp/prompt-media-authoring');
|
|
if(target===root||target.startsWith(root+path.sep))throw Error('MEDIA_AUTHORING_ROOT must be outside repository');
|
|
type Box=[number,number,number,number];
|
|
type Element={id:string;kind:'text'|'image'|'decoration';role:string;box:Box;text:string|null;direction:string;reading:'verified'|'unreadable'|'not-text';asset:string;mask?:string;underlay?:string;pixels:number};
|
|
function components(mask:Uint8Array,w:number,h:number){
|
|
const seen=new Uint8Array(mask.length),items:{box:Box;pixels:number[]}[]=[];
|
|
for(let i=0;i<mask.length;i++)if(mask[i]&&!seen[i]){
|
|
const q=[i];let c=0,x0=w,y0=h,x1=0,y1=0;seen[i]=1;
|
|
while(c<q.length){const j=q[c++],x=j%w,y=Math.floor(j/w);x0=Math.min(x0,x);y0=Math.min(y0,y);x1=Math.max(x1,x);y1=Math.max(y1,y);for(const k of [j-1,j+1,j-w,j+w])if(k>=0&&k<mask.length&&!seen[k]&&mask[k]&&Math.abs(k%w-x)<=1){seen[k]=1;q.push(k)}}
|
|
items.push({box:[x0,y0,x1-x0+1,y1-y0+1],pixels:q});
|
|
}return items;
|
|
}
|
|
async function main(){
|
|
const source=await readFile(path.join(target,'source.png')),{data,info}=await sharp(source).removeAlpha().raw().toBuffer({resolveWithObject:true}),W=info.width,H=info.height;
|
|
const bg=[...data.subarray((3*W+W-6)*3,(3*W+W-6)*3+3)],mask=new Uint8Array(W*H);
|
|
for(let i=0;i<mask.length;i++)mask[i]=Math.max(...bg.map((c,k)=>Math.abs(data[i*3+k]-c)))>19?1:0;
|
|
const found=components(mask,W,H).filter(({box:[,,w,h]})=>w>85&&w<125&&h>145&&h<190).map(({box:[x,y,w,h]})=>({x,y,w,h}));
|
|
if(found.length!==27)throw Error(`Expected 27 panels; found ${found.length}`);
|
|
const rows:number[]=[],cols:number[]=[];
|
|
for(const p of [...found].sort((a,b)=>a.y-b.y))if(!rows.some(y=>Math.abs(y-p.y)<8))rows.push(p.y);
|
|
for(const p of [...found].sort((a,b)=>a.x-b.x))if(!cols.some(x=>Math.abs(x-p.x)<8))cols.push(p.x);
|
|
const panels=found.map(p=>({...p,row:rows.findIndex(y=>Math.abs(y-p.y)<8)+1,column:cols.findIndex(x=>Math.abs(x-p.x)<8)+1})).sort((a,b)=>a.row-b.row||a.x-b.x);
|
|
const manifests=[];let totalPixels=0;
|
|
for(const [index,p]of panels.entries()){
|
|
const id='ref-'+String(index+1).padStart(2,'0'),spec=annotations[index],dir=path.join(target,id);await rm(dir,{recursive:true,force:true});await mkdir(dir,{recursive:true});
|
|
const crop=await sharp(source).extract({left:p.x,top:p.y,width:p.w,height:p.h}).removeAlpha().raw().toBuffer(),w=p.w,h=p.h;
|
|
await sharp(crop,{raw:{width:w,height:h,channels:3}}).png().toFile(path.join(dir,'source.png'));
|
|
const owner=new Int16Array(w*h).fill(-1),elements:Element[]=[];
|
|
function entry(kind:Element['kind'],role:string,box:number[],text:string|null=null,direction='horizontal'){
|
|
const prefix={text:'T',image:'I',decoration:'D'}[kind],n=elements.filter(e=>e.kind===kind).length+1;
|
|
let[x,y,bw,bh]=box.map(Math.round);x=Math.max(0,Math.min(w-1,x));y=Math.max(0,Math.min(h-1,y));
|
|
const e:Element={id:prefix+String(n).padStart(2,'0'),kind,role,box:[x,y,Math.max(1,Math.min(w-x,bw)),Math.max(1,Math.min(h-y,bh))],text,direction,reading:kind==='text'?(text===null?'unreadable':'verified'):'not-text',asset:'',pixels:0};elements.push(e);return e;
|
|
}
|
|
function region(box:Box,fn:(i:number)=>boolean){const[x,y,bw,bh]=box;for(let yy=y;yy<y+bh;yy++)for(let xx=x;xx<x+bw;xx++){const i=yy*w+xx;if(owner[i]===-1&&fn(i))owner[i]=elements.length-1}}
|
|
function fillColor(seed:number[]){const[sx,sy]=seed.map(Math.round),hist=new Map<string,{rgb:number[];count:number}>();
|
|
for(let y=Math.max(1,sy-7);y<Math.min(h-1,sy+8);y++)for(let x=Math.max(1,sx-7);x<Math.min(w-1,sx+8);x++){const i=y*w+x,rgb=[...crop.subarray(i*3,i*3+3)];if(Math.max(...rgb)>220||Math.min(...rgb)<65)continue;const key=rgb.map(v=>Math.round(v/8)).join(','),c=hist.get(key);if(c)c.count++;else hist.set(key,{rgb,count:1})}
|
|
const color=[...hist.values()].sort((a,b)=>b.count-a.count)[0]?.rgb;if(!color)throw Error(id+': no fill color');return color;
|
|
}
|
|
const imageColors=spec.images.map(image=>fillColor(image.seed));
|
|
for(const d of spec.decor.filter(d=>d.box[3]<=2&&d.box[2]>8)){const e=entry('decoration',d.role,d.box);region(e.box,i=>Math.max(...crop.subarray(i*3,i*3+3))<225)}
|
|
for(const t of spec.texts){const e=entry('text',t.role,t.box,t.text,t.direction);
|
|
const[x,y,bw,bh]=e.box,hist=new Map<number,number>();for(let yy=y;yy<y+bh;yy++)for(let xx=x;xx<x+bw;xx++){const i=(yy*w+xx)*3,v=Math.round((crop[i]+crop[i+1]+crop[i+2])/24)*8;hist.set(v,(hist.get(v)||0)+1)}
|
|
const mode=[...hist].sort((a,b)=>b[1]-a[1])[0][0];
|
|
region(e.box,i=>{
|
|
const rgb=[...crop.subarray(i*3,i*3+3)],lum=(rgb[0]+rgb[1]+rgb[2])/3;
|
|
if(t.tone==='light'||t.tone==='muted-light')return lum>mode+7;
|
|
if(t.tone==='muted-dark')return lum<mode-12;
|
|
return lum<(t.text===null?Math.min(240,mode-4):Math.min(225,mode-8))&&!imageColors.some((color,n)=>{const[a,b,c,d]=spec.images[n].box,x=i%w,y=Math.floor(i/w);return x>=a&&x<a+c&&y>=b&&y<b+d&&Math.hypot(...color.map((v,k)=>rgb[k]-v))<16});
|
|
})
|
|
}
|
|
|
|
for(const[n,image]of spec.images.entries()){
|
|
const e=entry('image',image.role,image.box),color=imageColors[n],candidate=new Uint8Array(w*h),[a,b,c,d]=image.box;
|
|
for(let y=Math.max(1,b-12);y<Math.min(h-1,b+d+12);y++)for(let x=Math.max(1,a-12);x<Math.min(w-1,a+c+12);x++){const i=y*w+x;if(Math.hypot(...color.map((v,k)=>crop[i*3+k]-v))<=36)candidate[i]=1}
|
|
const[sx,sy]=image.seed,options=components(candidate,w,h).filter(c=>c.pixels.length>=12).sort((u,v)=>{
|
|
const distance=(part:{box:Box})=>{const[x,y,bw,bh]=part.box;return Math.max(x-sx,sx-x-bw,0)**2+Math.max(y-sy,sy-y-bh,0)**2};return distance(u)-distance(v)||v.pixels.length-u.pixels.length;
|
|
});
|
|
const selected=options[0];if(!selected)throw Error(id+': no connected image region '+image.role);
|
|
for(const i of selected.pixels)if(owner[i]===-1)owner[i]=elements.length-1;
|
|
}
|
|
|
|
for(const d of spec.decor.filter(d=>!(d.box[3]<=2&&d.box[2]>8))){const e=entry('decoration',d.role,d.box);region(e.box,i=>id==='ref-04'?Math.min(...crop.subarray(i*3,i*3+3))>218:Math.max(...crop.subarray(i*3,i*3+3))<240)}
|
|
const residual=new Uint8Array(w*h);for(let y=4;y<h-4;y++)for(let x=4;x<w-4;x++){const i=y*w+x;if(owner[i]===-1&&Math.max(...crop.subarray(i*3,i*3+3))<90)residual[i]=1}
|
|
const extra=components(residual,w,h),tiny:number[]=[];
|
|
for(const c of extra){if(c.pixels.length<2)continue;const[,,bw,bh]=c.box;if((bw>=7&&bh<=2)||(bh>=7&&bw<=2)){entry('decoration','自动识别细线',c.box);for(const i of c.pixels)owner[i]=elements.length-1}else tiny.push(...c.pixels)}
|
|
if(tiny.length){
|
|
const joined=new Uint8Array(w*h);for(const i of tiny){const x=i%w,y=Math.floor(i/w);for(let dy=-1;dy<=1;dy++)for(let dx=-2;dx<=2;dx++)if(x+dx>=4&&x+dx<w-4&&y+dy>=4&&y+dy<h-4)joined[(y+dy)*w+x+dx]=1}
|
|
for(const c of components(joined,w,h)){const set=new Set(c.pixels),ink=tiny.filter(i=>set.has(i));if(!ink.length)continue;const candidates=elements.filter(e=>e.kind==='text'),match=textMergeTarget(c.box,candidates);if(match>=0){const existing=candidates[match],n=elements.indexOf(existing);existing.box=unionBoxes(existing.box,c.box);for(const i of ink)owner[i]=n}else{entry('text','未辨识细字（内容待核对）',c.box);for(const i of ink)owner[i]=elements.length-1}}
|
|
}
|
|
for(const[n,e]of elements.entries()){
|
|
let x0=w,y0=h,x1=-1,y1=-1;for(let i=0;i<owner.length;i++)if(owner[i]===n){const x=i%w,y=Math.floor(i/w);x0=Math.min(x0,x);y0=Math.min(y0,y);x1=Math.max(x1,x);y1=Math.max(y1,y)}
|
|
if(x1>=0)e.box=[x0,y0,x1-x0+1,y1-y0+1];
|
|
}
|
|
const result=Buffer.alloc(w*h*3),background=Buffer.alloc(w*h*4);let assigned=0;
|
|
for(let i=0;i<w*h;i++)if(owner[i]===-1){crop.copy(background,i*4,i*3,i*3+3);background[i*4+3]=255;crop.copy(result,i*3,i*3,i*3+3);assigned++}
|
|
for(const[n,e]of elements.entries()){
|
|
const[x,y,bw,bh]=e.box,rgba=Buffer.alloc(bw*bh*4),binary=new Uint8Array(bw*bh);
|
|
for(let yy=y;yy<y+bh;yy++)for(let xx=x;xx<x+bw;xx++){const i=yy*w+xx,j=(yy-y)*bw+xx-x;if(owner[i]!==n)continue;crop.copy(rgba,j*4,i*3,i*3+3);rgba[j*4+3]=255;binary[j]=1;crop.copy(result,i*3,i*3,i*3+3);e.pixels++;assigned++}
|
|
e.asset=`asset:${id}/${e.id}.png`;await sharp(rgba,{raw:{width:bw,height:bh,channels:4}}).png().toFile(path.join(dir,e.id+'.png'));
|
|
if(e.kind==='text'){
|
|
const underlay=Buffer.alloc(bw*bh*4);
|
|
for(let yy=y;yy<y+bh;yy++)for(let xx=x;xx<x+bw;xx++){
|
|
const i=yy*w+xx;if(owner[i]!==n)continue;const j=((yy-y)*bw+xx-x)*4;
|
|
const samples:number[][]=[];
|
|
for(let r=1;r<=8&&samples.length<4;r++)for(let dy=-r;dy<=r;dy++)for(let dx=-r;dx<=r;dx++){
|
|
if(Math.max(Math.abs(dx),Math.abs(dy))!==r||xx+dx<0||xx+dx>=w||yy+dy<0||yy+dy>=h)continue;
|
|
const k=(yy+dy)*w+xx+dx;if(elements[owner[k]]?.kind==='text')continue;samples.push([...crop.subarray(k*3,k*3+3)]);
|
|
}
|
|
for(let c=0;c<3;c++)underlay[j+c]=samples.length?Math.round(samples.reduce((sum,rgb)=>sum+rgb[c],0)/samples.length):245;underlay[j+3]=255;underlay.copy(background,i*4,j,j+4);
|
|
}
|
|
e.underlay=`asset:${id}/${e.id}-underlay.png`;
|
|
await sharp(underlay,{raw:{width:bw,height:bh,channels:4}}).png().toFile(path.join(dir,e.id+'-underlay.png'));
|
|
}
|
|
if(e.kind==='image'){
|
|
const outside=new Uint8Array(binary.length),q:number[]=[];for(let i=0;i<binary.length;i++)if(!binary[i]&&(i<bw||i>=bw*(bh-1)||i%bw===0||i%bw===bw-1)){q.push(i);outside[i]=1}
|
|
for(let c=0;c<q.length;c++){const i=q[c];for(const j of[i-1,i+1,i-bw,i+bw])if(j>=0&&j<binary.length&&!binary[j]&&!outside[j]&&Math.abs(j%bw-i%bw)<=1){outside[j]=1;q.push(j)}}
|
|
const alpha=Buffer.alloc(binary.length*4);for(let i=0;i<binary.length;i++)if(!outside[i])alpha.fill(255,i*4,i*4+4);
|
|
e.mask=`asset:${id}/${e.id}-mask.png`;await sharp(alpha,{raw:{width:bw,height:bh,channels:4}}).png().toFile(path.join(dir,e.id+'-mask.png'));
|
|
}
|
|
}
|
|
await sharp(background,{raw:{width:w,height:h,channels:4}}).png().toFile(path.join(dir,'background.png'));
|
|
for(const kind of ['text','image','decoration'] as const){
|
|
const map=Buffer.alloc(w*h*4,255),color={text:[44,100,216],image:[230,109,34],decoration:[8,135,93]}[kind];
|
|
for(let i=0;i<w*h;i++){const e=elements[owner[i]];if(e?.kind===kind)for(let k=0;k<3;k++)map[i*4+k]=color[k]}
|
|
await sharp(map,{raw:{width:w,height:h,channels:4}}).resize(w*6,h*6,{kernel:'nearest'}).png().toFile(path.join(dir,kind+'-map.png'));
|
|
}
|
|
if(assigned!==w*h||!result.equals(crop))throw Error(id+': pixel reconstruction failed');
|
|
await sharp(result,{raw:{width:w,height:h,channels:3}}).png().toFile(path.join(dir,'reconstructed.png'));
|
|
const kept=elements.filter(e=>e.pixels>0),colors={text:'#2c64d8',image:'#e66d22',decoration:'#08875d'};
|
|
const overlay=`<svg xmlns="http://www.w3.org/2000/svg" width="${w*6}" height="${h*6}" viewBox="0 0 ${w} ${h}">${kept.map(e=>{const[x,y,bw,bh]=e.box;return `<rect x="${x}" y="${y}" width="${bw}" height="${bh}" fill="${colors[e.kind]}" fill-opacity=".08" stroke="${colors[e.kind]}" stroke-width=".45"/><rect x="${x}" y="${y}" width="12" height="5" fill="${colors[e.kind]}"/><text x="${x+.5}" y="${y+3.8}" font-family="Arial" font-size="3.6" fill="white">${e.id}</text>`}).join('')}</svg>`;
|
|
await sharp(crop,{raw:{width:w,height:h,channels:3}}).resize(w*6,h*6,{kernel:'nearest'}).composite([{input:Buffer.from(overlay)}]).png().toFile(path.join(dir,'annotated.png'));
|
|
const manifest={id,name:spec.name,source:{url:'asset:source.png',sha256:createHash('sha256').update(source).digest('hex'),box:[p.x,p.y,w,h],row:p.row,column:p.column},width:w,height:h,reference:`asset:${id}/source.png`,annotated:`asset:${id}/annotated.png`,background:`asset:${id}/background.png`,elements:kept,ocr:ocr.find(r=>r.file===id+'.png')?.lines||[],verification:{pixelIdentical:true,pixels:w*h,unreadableText:kept.filter(e=>e.reading==='unreadable').length}};
|
|
await writeFile(path.join(dir,'manifest.json'),JSON.stringify(manifest,null,2)+'\n');manifests.push(manifest);totalPixels+=w*h;
|
|
}
|
|
const overlay=`<svg xmlns="http://www.w3.org/2000/svg" width="${W}" height="${H}">${manifests.map(m=>{const[x,y,w,h]=m.source.box;return `<rect x="${x}" y="${y}" width="${w}" height="${h}" fill="none" stroke="#ed4237" stroke-width="1.5"/><rect x="${x}" y="${y}" width="35" height="14" fill="#ed4237"/><text x="${x+2}" y="${y+10}" font-family="Arial" font-size="9" fill="white">${m.id}</text>`}).join('')}</svg>`;
|
|
await sharp(source).composite([{input:Buffer.from(overlay)}]).png().toFile(path.join(target,'numbered-source.png'));
|
|
await writeFile(path.join(root,'contentasset:manifest.json'),JSON.stringify(manifests,null,2)+'\n');
|
|
await writeFile(path.join(target,'index.json'),JSON.stringify({version:2,sourceSha256:manifests[0].source.sha256,numbering:'row-major; exclude left title banner',layouts:manifests.map(m=>({id:m.id,name:m.name,source:m.source,manifest:`asset:${m.id}/manifest.json`}))},null,2)+'\n');
|
|
console.log(`Detected ${panels.length} panels; ${manifests.reduce((n,m)=>n+m.elements.length,0)} typed components; ${totalPixels} pixels reconstructed exactly.`);
|
|
}
|
|
main().catch(e=>{console.error(e);process.exitCode=1});
|