fix(ci): bound navigation recovery and retire review images safely
creative workspace review checks / verify-deployed-canvas (pull_request) Failing after 59s
creative workspace review checks / verify (pull_request) Has been cancelled

This commit is contained in:
2026-11-01 15:35:23 +08:00
parent cc6b376f56
commit 9887a5958a
7 changed files with 204 additions and 6 deletions
+10 -1
View File
@@ -11,6 +11,8 @@ on:
- 'scripts/integration.sh'
- 'scripts/cleanup-integration.sh'
- 'scripts/test-integration-cleanup.py'
- 'scripts/cleanup-review-images.sh'
- 'scripts/test-review-image-cleanup.py'
concurrency:
group: creative-workspace-${{ gitea.event.pull_request.number }}
cancel-in-progress: true
@@ -42,9 +44,11 @@ jobs:
- name: Check patch formatting
run: |
git diff --check HEAD^ HEAD
bash -n scripts/integration.sh scripts/cleanup-integration.sh scripts/verify-studio-production.sh
bash -n scripts/integration.sh scripts/cleanup-integration.sh scripts/verify-studio-production.sh scripts/cleanup-review-images.sh
node --check scripts/verify-studio-visual.cjs
python3 scripts/test-integration-cleanup.py
python3 scripts/test-review-image-cleanup.py
node --test apps/waterfall-wiki/tests/brand-navigation.test.mjs
- name: Verify Studio types, tests, build and smoke
run: docker build -f apps/studio/Dockerfile -t "creative-studio-review:${{ gitea.sha }}" .
- name: Build Wiki integration and runtime images
@@ -63,3 +67,8 @@ jobs:
env:
GITEA_RUN_ID: ${{ gitea.run_id }}
run: bash scripts/cleanup-integration.sh
- name: Remove this run disposable review image tags
if: always()
env:
REVIEW_SHA: ${{ gitea.sha }}
run: bash scripts/cleanup-review-images.sh
@@ -0,0 +1,32 @@
/**
* Recover once from an idempotent document GET's transient proxy failure.
* This never retries hydration, theme assertions, or mutations, and shares one
* 60-second budget across both attempts rather than extending the deadline.
* @param {(timeout: number) => Promise<{ok(): boolean, status(): number} | null>} navigate
* @param {string} url
* @param {() => number} now
*/
export async function navigateBrandDocument(navigate, url, now = Date.now) {
const deadline = now() + 60000;
for (let attempt = 1; attempt <= 2; attempt++) {
const remaining = deadline - now();
if (remaining <= 0) throw new Error(`Document navigation budget exhausted: ${url}`);
try {
const response = await navigate(remaining);
if (response?.ok()) return response;
const status = response?.status();
if (attempt === 1 && [500, 502, 503, 504].includes(status ?? 0)) {
console.warn('Brand navigation proxy recovery', JSON.stringify({url, status, attempt}));
continue;
}
throw new Error(`Document navigation failed: ${url}, HTTP ${status ?? 'no response'}`);
} catch (error) {
if (attempt === 1 && /ECONNRESET|net::ERR_(?:CONNECTION_RESET|CONNECTION_CLOSED|EMPTY_RESPONSE)/.test(String(error))) {
console.warn('Brand navigation connection recovery', JSON.stringify({url, error: String(error), attempt}));
continue;
}
throw error;
}
}
throw new Error(`Document navigation failed: ${url}`);
}
@@ -0,0 +1,48 @@
import {test} from 'node:test';
import assert from 'node:assert/strict';
import {navigateBrandDocument} from './brand-navigation.mjs';
const response = status => ({ok: () => status === 200, status: () => status});
test('successful documents are fetched once', async () => {
let calls = 0;
await navigateBrandDocument(async () => { calls++; return response(200); }, '/canvas');
assert.equal(calls, 1);
});
test('one proxy failure can recover within the original deadline', async () => {
const timeouts = []; let elapsed = 0;
await navigateBrandDocument(async timeout => {
timeouts.push(timeout); elapsed = 15000;
return response(timeouts.length === 1 ? 500 : 200);
}, '/canvas', () => elapsed);
assert.deepEqual(timeouts, [60000, 45000]);
});
test('connection reset can recover once', async () => {
let calls = 0;
await navigateBrandDocument(async () => {
if (++calls === 1) throw new Error('net::ERR_CONNECTION_RESET');
return response(200);
}, '/canvas');
assert.equal(calls, 2);
});
test('persistent server failure is not hidden', async () => {
let calls = 0;
await assert.rejects(navigateBrandDocument(async () => { calls++; return response(502); }, '/canvas'), /HTTP 502/);
assert.equal(calls, 2);
});
test('missing routes, authentication failures and ordinary timeouts are not retried', async () => {
for (const status of [401, 403, 404]) {
let calls = 0;
await assert.rejects(navigateBrandDocument(async () => { calls++; return response(status); }, '/canvas'), /HTTP/);
assert.equal(calls, 1);
}
let calls = 0;
await assert.rejects(navigateBrandDocument(async () => { calls++; throw new Error('Timeout 60000ms exceeded'); }, '/canvas'), /Timeout/);
assert.equal(calls, 1);
});
test('recovery cannot extend the total navigation budget', async () => {
let calls = 0, elapsed = 0;
await assert.rejects(navigateBrandDocument(async () => {
calls++; elapsed = 60000; return response(503);
}, '/canvas', () => elapsed), /budget exhausted/);
assert.equal(calls, 1);
});
@@ -1,6 +1,7 @@
/** CI-only checks against the disposable stack, never against personal documents. */
import assert from 'node:assert/strict';
import { chromium } from 'playwright-core';
import { navigateBrandDocument } from './brand-navigation.mjs';
export async function verifyBrandUI(base: string) {
for (const prefix of process.env.VERIFY_CANVAS_UI === '1' ? ['', '/studio'] : ['']) {
const icon = await fetch(`${base}${prefix}/favicon.ico`);
@@ -18,11 +19,20 @@ export async function verifyBrandUI(base: string) {
try {
const page = await browser.newPage({ viewport: { width: 1440, height: 960 } });
page.setDefaultTimeout(60000);page.setDefaultNavigationTimeout(60000);
const failures: string[] = [];
page.on('pageerror', error => failures.push(error.message));
page.on('requestfailed', request => failures.push(new URL(request.url()).pathname + ': ' + request.failure()?.errorText));
const visit = async (path: string, theme: string) => {
await page.goto(base + path, { waitUntil: 'commit' });
await page.locator('button[data-theme-ready="true"]').first().waitFor();
await page.waitForFunction(expected => document.documentElement.dataset.theme === expected, theme);
console.log('Brand UI route ready:', path, theme);
const started = Date.now();
try {
await navigateBrandDocument(timeout => page.goto(base + path, { waitUntil: 'commit', timeout }), base + path);
await page.locator('button[data-theme-ready="true"]').first().waitFor();
await page.waitForFunction(expected => document.documentElement.dataset.theme === expected, theme);
console.log('Brand UI route ready:', path, theme, Date.now() - started);
} catch (error) {
console.error('Brand UI navigation diagnostics', JSON.stringify({path, url:page.url(), elapsedMs:Date.now()-started, failures:failures.slice(-20), body:(await page.locator('body').innerText({timeout:3000}).catch(()=>'unavailable')).slice(0,1200)}));
throw error;
}
};
const background = (selector: string) => page.locator(selector).first().evaluate(el => getComputedStyle(el).backgroundColor);
await visit('/', 'home-light');
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -Eeuo pipefail
: "${REVIEW_SHA:?The exact review commit SHA is required}"
[[ "$REVIEW_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'Refusing invalid review SHA' >&2; exit 2; }
# Only this workflow's disposable tags; production tags, volumes and other projects stay out of scope.
for repository in creative-studio-review creative-wiki-tests creative-wiki-review; do
tag="$repository:$REVIEW_SHA"
referenced=$(docker ps -aq | xargs -r docker inspect --format '{{.Image}}')
if ! image=$(docker image inspect --format '{{.Id}}' "$tag" 2>/dev/null); then
echo "No review image: $tag"
continue
fi
if printf '%s\n' "$referenced" | grep -Fxq "$image"; then
echo "Keeping container-referenced review image: $tag"
continue
fi
docker image rm "$tag"
done
+9 -1
View File
@@ -17,7 +17,15 @@ compose(){ docker compose -p "$COMPOSE_PROJECT_NAME" --env-file "$TEST_ENV" "${C
cleanup(){
code=$?
trap - EXIT INT TERM
if [ "$code" -ne 0 ]; then compose logs --tail 160 waterfall-wiki waterfall-wiki-worker "${STUDIO_SERVICES[@]}" || true; fi
if [ "$code" -ne 0 ]; then
echo 'Runner pressure at integration failure (before disposable stack cleanup)'
uptime || true
free -m || true
df -h / || true
for resource in cpu memory io; do cat "/proc/pressure/$resource" || true; done
timeout 10s docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} memory={{.MemUsage}} io={{.BlockIO}} pids={{.PIDs}}' || true
compose logs --tail 160 waterfall-wiki waterfall-wiki-worker "${STUDIO_SERVICES[@]}" || true
fi
cleanup_code=0
bash scripts/cleanup-integration.sh || cleanup_code=$?
rm -f "$TEST_ENV"
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env python3
"""Check review cleanup scope without Docker or build output."""
import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
SCRIPT = Path(__file__).with_name('cleanup-review-images.sh')
SHA = 'a' * 40
FAKE_DOCKER = '''#!/usr/bin/env python3
import json, os, sys
args=sys.argv[1:]
with open(os.environ['DOCKER_CALLS'], 'a') as output:
output.write(json.dumps(args)+'\\n')
if args[0]=='ps':
if os.environ.get('FAIL_INVENTORY'): sys.exit(1)
if os.environ.get('REFERENCED'): print('container-id')
elif args[0]=='inspect':
print('sha256:review-image')
elif args[:2]==['image','inspect']:
if os.environ.get('MISSING_IMAGE'): sys.exit(1)
print('sha256:review-image')
'''
class ReviewCleanupTests(unittest.TestCase):
def run_cleanup(self, sha=SHA, **extra):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
docker = root / 'docker'
docker.write_text(FAKE_DOCKER)
docker.chmod(0o700)
calls = root / 'calls'
result = subprocess.run(['bash', str(SCRIPT)], capture_output=True, env={
**os.environ, 'PATH': directory + os.pathsep + os.environ['PATH'],
'DOCKER_CALLS': str(calls), 'REVIEW_SHA': sha, **extra})
return result.returncode, [json.loads(line) for line in calls.read_text().splitlines()] if calls.exists() else []
def test_removes_only_exact_review_tags_without_force(self):
code, calls = self.run_cleanup()
self.assertEqual(code, 0)
self.assertEqual([args for args in calls if args[:2] == ['image', 'rm']], [
['image', 'rm', name + ':' + SHA] for name in
['creative-studio-review', 'creative-wiki-tests', 'creative-wiki-review']])
self.assertFalse(any('-f' in args or 'prune' in args or 'volume' in args for args in calls))
def test_invalid_sha_never_contacts_docker(self):
for value in ['', 'main', 'a' * 39, 'a' * 41, 'A' * 40, '*' * 40]:
with self.subTest(value=value):
code, calls = self.run_cleanup(value)
self.assertNotEqual(code, 0)
self.assertEqual(calls, [])
def test_container_referenced_images_are_kept(self):
code, calls = self.run_cleanup(REFERENCED='1')
self.assertEqual(code, 0)
self.assertFalse(any(args[:2] == ['image', 'rm'] for args in calls))
def test_missing_tags_are_a_successful_noop(self):
code, calls = self.run_cleanup(MISSING_IMAGE='1')
self.assertEqual(code, 0)
self.assertFalse(any(args[:2] == ['image', 'rm'] for args in calls))
def test_inventory_failure_prevents_removal(self):
code, calls = self.run_cleanup(FAIL_INVENTORY='1')
self.assertNotEqual(code, 0)
self.assertEqual(len(calls), 1)
if __name__ == '__main__':
unittest.main()