Flatten note tags and store processed media as ordinary resource links
wiki CI and deployment / verify-and-deploy (push) Successful in 13m27s

This commit is contained in:
2026-10-26 00:23:02 +08:00
parent 742b58013e
commit 9986846c36
31 changed files with 255 additions and 209 deletions
+33
View File
@@ -0,0 +1,33 @@
import assert from 'node:assert/strict';
import {readFile} from 'node:fs/promises';
import {randomUUID} from 'node:crypto';
import {db} from '../src/lib/db';
export async function verifyFlatNoteMigration(){
const client=await db().connect();const schema='migration_'+randomUUID().replaceAll('-','');
try{
await client.query(`CREATE SCHEMA ${schema}`);await client.query(`SET search_path TO ${schema}`);
for(const name of ['001_library','002_page_snapshots','003_note_descriptions'])await client.query(await readFile(`db/${name}.sql`,'utf8'));
const [video,picture,post,tag,duplicate,parent]=Array.from({length:6},()=>randomUUID());
await client.query(`INSERT INTO assets(id,kind,storage_key,mime_type,byte_size) VALUES($1,'video','original','video/mp4',100),($2,'image','snapshot','image/png',50)`,[video,picture]);
await client.query(`INSERT INTO posts(id,slug,kind,title,page_key) VALUES($1,'old-note','article','保留的笔记','old-code')`,[post]);
await client.query(`INSERT INTO post_assets(post_id,asset_id,position) VALUES($1,$2,0)`,[post,video]);
await client.query(`INSERT INTO page_snapshots(post_id,page_index,asset_id,revision) VALUES($1,1,$2,'old-revision')`,[post,picture]);
await client.query(`INSERT INTO render_jobs(post_id,status,revision) VALUES($1,'pending','old-revision')`,[post]);
await client.query(`INSERT INTO asset_variants(asset_id,role,storage_key,mime_type,byte_size) VALUES($1,'poster','old-poster','image/jpeg',20)`,[video]);
await client.query(`INSERT INTO media_jobs(id,asset_id) VALUES($1,$2)`,[randomUUID(),video]);
await client.query(`INSERT INTO tags(id,slug,name) VALUES($1,'parent','上级'),($2,'same','重复')`,[parent,tag]);
await client.query(`INSERT INTO tags(id,parent_id,slug,name) VALUES($1,$2,'same','重复')`,[duplicate,parent]);
await client.query(`INSERT INTO post_tags(post_id,tag_id) VALUES($1,$2),($1,$3)`,[post,tag,duplicate]);
await client.query(`INSERT INTO asset_tags(asset_id,tag_id) VALUES($1,$2)`,[video,duplicate]);
await client.query(await readFile('db/004_flat_notes.sql','utf8'));
assert.equal((await client.query('SELECT count(*)::integer n FROM post_assets WHERE post_id=$1 AND asset_id=$2',[post,picture])).rows[0].n,1,'Previously generated image remains linked');
const processed=(await client.query(`SELECT a.storage_key,pa.role FROM post_assets pa JOIN assets a ON a.id=pa.asset_id WHERE pa.post_id=$1 AND a.derivation='poster'`,[post])).rows;
assert.deepEqual(processed,[{storage_key:'old-poster',role:'poster'}],'Processed file uses its existing object and a normal resource link');
assert.equal((await client.query('SELECT count(*)::integer n FROM post_tags WHERE post_id=$1',[post])).rows[0].n,1);
assert.equal((await client.query('SELECT count(*)::integer n FROM asset_tags WHERE asset_id=$1',[video])).rows[0].n,1,'Merging equal tags retains asset associations');
assert.equal((await client.query('SELECT count(*)::integer n FROM media_jobs')).rows[0].n,1,'Video processing queue survives');
assert.equal((await client.query(`SELECT count(*)::integer n FROM information_schema.columns WHERE table_schema=$1 AND column_name IN ('parent_id','page_key')`,[schema])).rows[0].n,0);
}finally{
await client.query('ROLLBACK');await client.query('SET search_path TO public');await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`);client.release();
}
}
+17 -15
View File
@@ -11,9 +11,10 @@ import {randomUUID} from 'node:crypto';
import sharp from 'sharp';
import {db} from '../src/lib/db';
import {probe,run} from '../src/lib/probe';
import {verifyFlatNoteMigration} from './flat-note-migration';
import {publishLearningSeries} from '../src/lib/publish-learning-series';
test('real PostgreSQL + S3 library, sharing, taxonomy, uploads, range and video separation', {timeout:300000}, async()=>{
test('real PostgreSQL + S3 notes, flat tags, resource sharing and video processing', {timeout:300000}, async()=>{
assert.equal(process.env.PROMPT_INTEGRATION_TEST,'1','Run only against an explicitly isolated test database');
const socket=createServer();socket.listen(0,'127.0.0.1');await once(socket,'listening');const address=socket.address();assert.ok(address&&typeof address!=='string');const port=address.port;await new Promise<void>(r=>socket.close(()=>r()));
const external=process.env.PROMPT_TEST_BASE_URL;
@@ -24,14 +25,16 @@ test('real PostgreSQL + S3 library, sharing, taxonomy, uploads, range and video
try{
let ready=false;for(let i=0;i<60;i++){try{if((await fetch(base+'/api/health')).ok){ready=true;break;}}catch{}await setTimeout(250);}assert.ok(ready);
for(const route of ['/','/images','/videos','/audio','/articles','/motion','/library/import','/tools/doodle','/tools/skills','/tools/skills/street-people','/tools/skills/shadow-puppet'])assert.equal((await fetch(base+route)).status,200,route);
const root=await request('/api/tags',{name:'线条',slug:'lines'});const child=await request('/api/tags',{name:'白色',slug:'white',parentId:root.id});
await assert.rejects(db().query('UPDATE tags SET parent_id=$1 WHERE id=$2',[child.id,root.id]),/cycle/);
await verifyFlatNoteMigration();
const root=await request('/api/tags',{name:'线条',slug:'lines'});const child=await request('/api/tags',{name:'白色',slug:'white'});
const sameTag=await request('/api/tags',{name:'#线条'});assert.equal(sameTag.id,root.id,'Hashtag creation reuses the globally unique tag');
assert.equal((await db().query("SELECT count(*)::integer AS n FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='tags' AND column_name='parent_id'")).rows[0].n,0);
const a=await sharp({create:{width:160,height:220,channels:3,background:'#b8c9af'}}).png().toBuffer();
const upload=async(kind:string,files:{data:Uint8Array;name:string;mime:string}[])=>{const form=new FormData();form.set('kind',kind);form.set('title','测试收藏 '+kind);form.set('description','可以复用的独立文字');form.set('tags',JSON.stringify([child.id]));files.forEach(f=>form.append('files',new Blob([new Uint8Array(f.data)],{type:f.mime}),f.name));const res=await fetch(base+'/api/upload',{method:'POST',body:form});assert.equal(res.status,201,await res.clone().text());return res.json();};
const first=await upload('image',[{data:a,name:'a.png',mime:'image/png'},{data:a,name:'b.png',mime:'image/png'}]);
const reused=await request('/api/posts',{kind:'image',title:'同一张图的另一种用法',description:'同一素材的独立说明',assets:[{id:first.assets[0]}],tags:[root.id]});
assert.equal((await db().query('SELECT count(*)::integer AS n FROM post_assets WHERE asset_id=$1',[first.assets[0]])).rows[0].n,2);
const filtered=await request('/api/feed?section=images&path=lines');assert.equal(filtered.posts.length,2);assert.equal(filtered.posts.find((p:{id:string})=>p.id===reused.id).description,'同一素材的独立说明');assert.equal((await request('/api/feed?section=images&path=lines/white')).posts.length,1);
const filtered=await request('/api/feed?section=images&path=lines');assert.equal(filtered.posts.length,1);assert.equal(filtered.posts.find((p:{id:string})=>p.id===reused.id).description,'同一素材的独立说明');assert.equal((await request('/api/feed?section=images&path=white')).posts.length,1);
const empty=randomUUID();await db().query("INSERT INTO posts(id,slug,kind,title) VALUES($1,$2,'image','空对象')",[empty,empty]);assert.equal((await request('/api/feed')).posts.length,2);
assert.equal((await request('/api/feed')).contributions.length,0,'Import does not invent contribution rules');
await db().query("INSERT INTO contribution_events(id,kind,subject_type,amount) VALUES($1,'test.import','asset',2),($2,'test.edit','post',3)",[randomUUID(),randomUUID()]);assert.equal((await request('/api/feed')).contributions.reduce((n:number,c:{total:number})=>n+c.total,0),5);
@@ -44,10 +47,13 @@ test('real PostgreSQL + S3 library, sharing, taxonomy, uploads, range and video
const post=await upload('video',[{data:await readFile(path),name:'video.mp4',mime:'video/mp4'}]);let asset;
for(let i=0;i<120;i++){asset=(await db().query('SELECT * FROM assets WHERE id=$1',[post.assets[0]])).rows[0];if(asset.processing_status==='ready'||asset.processing_status==='failed')break;await setTimeout(500);}
assert.equal(asset.processing_status,'ready',asset.processing_error);assert.equal(asset.has_audio,sound);
const variants:string[]=(await db().query('SELECT role FROM asset_variants WHERE asset_id=$1',[asset.id])).rows.map(r=>r.role);assert.ok(variants.includes('silent_video'));assert.ok(variants.includes('poster'));assert.equal(variants.includes('audio_track'),sound);
const variants:string[]=(await db().query('SELECT derivation AS role FROM assets WHERE source_asset_id=$1',[asset.id])).rows.map(r=>r.role);assert.ok(variants.includes('silent_video'));assert.ok(variants.includes('poster'));assert.equal(variants.includes('audio_track'),sound);
const linked=(await request('/api/feed?section=videos')).posts.find((p:{id:string})=>p.id===post.id);assert.ok(linked.assets.some((a:{role:string})=>a.role==='poster'));
const videoNote=await fetch(base+'/explore/'+post.id+'?agent_mode=True').then(r=>r.text());assert.ok(videoNote.includes('静音视频'));
const reusedVideo=await request('/api/posts',{kind:'video',title:'复用已加工的视频',assets:[{id:asset.id}]});assert.ok((await db().query("SELECT 1 FROM post_assets WHERE post_id=$1 AND role='silent_video'",[reusedVideo.id])).rowCount);
const clip:Response=await fetch(base+'/api/media/'+asset.id+'?variant=silent_video');assert.equal(clip.status,200);const derived=join(dir,'derived.mp4');await writeFile(derived,Buffer.from(await clip.arrayBuffer()));assert.equal((await probe(derived)).streams.some(s=>s.codec_type==='audio'),false);
}
assert.equal((await db().query("SELECT count(*)::integer AS n FROM assets WHERE kind='audio'")).rows[0].n,0,'Bound soundtracks are variants, not independent audio assets');
assert.equal((await db().query("SELECT count(*)::integer AS n FROM assets WHERE kind='audio' AND derivation='audio_track'")).rows[0].n,1,'Video soundtracks are ordinary reusable assets');
const audio=join(dir,'audio.wav');await run('ffmpeg',['-y','-v','error','-f','lavfi','-i','sine=frequency=220','-t','0.2',audio]);await upload('audio',[{data:await readFile(audio),name:'audio.wav',mime:'audio/wav'}]);assert.equal((await request('/api/feed?section=audio')).posts.length,1);
for(const removed of ['/library/assets','/library/tags'])assert.equal((await fetch(base+removed)).status,404);
const published=await publishLearningSeries();await publishLearningSeries();
@@ -61,16 +67,12 @@ test('real PostgreSQL + S3 library, sharing, taxonomy, uploads, range and video
const coverInfo=await sharp(Buffer.from(await coverResponse.arrayBuffer())).metadata();assert.equal(coverInfo.width,1080);assert.equal(coverInfo.height,1440);
assert.equal((await request('/api/feed')).contributions.reduce((n:number,c:{total:number})=>n+c.total,0),5,'Editorial publication does not invent contribution events');
for(const slug of published.published){
for(const n of [1,3,8]){const r=await fetch(base+`/p/${slug}/pages/${n}`);assert.equal(r.status,200);const html=await r.text();assert.ok(html.includes('data-render-page'));assert.ok(!html.includes('global-topbar'));}
let status;for(let i=0;i<480;i++){status=await request(`/api/posts/${slug}/snapshots`);if(status.job?.status==='failed')assert.fail(status.job.error);if(status.job?.status==='ready'&&status.pages.length===8)break;await setTimeout(500)}
assert.equal(status.pages.length,8,'Screenshot job state: '+JSON.stringify(status.job));
const r:Response=await fetch(base+'/api/media/'+status.pages[2].asset_id);const meta:{width?:number;height?:number}=await sharp(Buffer.from(await r.arrayBuffer())).metadata();assert.equal(meta.width,1080);assert.equal(meta.height,1440);
assert.equal((await fetch(base+`/api/posts/${slug}/snapshots`)).status,404);
assert.equal((await fetch(base+`/p/${slug}/pages/1`)).status,404);
}
const notes=await request('/api/explore-feed');for(const slug of published.published){const post=(await request('/api/feed?section=articles')).posts.find((p:{slug:string})=>p.slug===slug);assert.equal(notes.entries.find((e:{id:string})=>e.id===post.id).imageCount,8);}
const before=await request(`/api/posts/${published.published[0]}/snapshots`);
const refresh=await fetch(base+`/api/posts/${published.published[0]}/snapshots`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({page:3})});assert.equal(refresh.status,202);
let after;for(let i=0;i<80;i++){after=await request(`/api/posts/${published.published[0]}/snapshots`);if(after.job?.status==='ready'&&after.pages[2].rendered_at!==before.pages[2].rendered_at)break;await setTimeout(500)}
assert.notEqual(after.pages[2].rendered_at,before.pages[2].rendered_at);assert.equal(after.pages[1].rendered_at,before.pages[1].rendered_at,'Single-page refresh preserves other snapshots');
const article=await request('/api/posts',{kind:'article',title:'无代码页面的笔记',text:'普通正文与 [引用](https://example.com)'});
const articlePage=await fetch(base+'/explore/'+article.id).then(r=>r.text());assert.ok(articlePage.includes('note-inline-link'));
const remaining=(await db().query("SELECT to_regclass('asset_variants') AS variants,to_regclass('page_snapshots') AS snapshots,to_regclass('render_jobs') AS renders")).rows[0];assert.deepEqual(remaining,{variants:null,snapshots:null,renders:null});
for(const doc of ['SKILL','style','upload','checks']){const r=await fetch(base+`/skills/street-people/${doc}.md`);assert.equal(r.status,200);assert.ok(r.headers.get('content-type')?.includes('text/markdown'));const text=await r.text();assert.ok(text.startsWith('# '));assert.ok(!text.includes('<html'));}
for(const doc of ['SKILL','workflow','visual-grammar','prompt','delivery']){
const path=`/skills/shadow-puppet/${doc}.md`;const response=await fetch(base+path);assert.equal(response.status,200,path);assert.ok(response.headers.get('content-type')?.includes('text/markdown'));assert.ok((await response.text()).startsWith('# '));