diff --git a/.gitea/workflows/creative-workspace.yml b/.gitea/workflows/creative-workspace.yml index c3c3a70..740a735 100644 --- a/.gitea/workflows/creative-workspace.yml +++ b/.gitea/workflows/creative-workspace.yml @@ -11,6 +11,8 @@ on: - 'scripts/integration.sh' - 'scripts/cleanup-integration.sh' - 'scripts/test-integration-cleanup.py' + - 'scripts/cleanup-review-images.sh' + - 'scripts/test-review-image-cleanup.py' concurrency: group: creative-workspace-${{ gitea.event.pull_request.number }} cancel-in-progress: true @@ -42,9 +44,11 @@ jobs: - name: Check patch formatting run: | git diff --check HEAD^ HEAD - bash -n scripts/integration.sh scripts/cleanup-integration.sh scripts/verify-studio-production.sh + bash -n scripts/integration.sh scripts/cleanup-integration.sh scripts/verify-studio-production.sh scripts/cleanup-review-images.sh node --check scripts/verify-studio-visual.cjs python3 scripts/test-integration-cleanup.py + python3 scripts/test-review-image-cleanup.py + node --test apps/waterfall-wiki/tests/brand-navigation.test.mjs - name: Verify Studio types, tests, build and smoke run: docker build -f apps/studio/Dockerfile -t "creative-studio-review:${{ gitea.sha }}" . - name: Build Wiki integration and runtime images @@ -63,3 +67,8 @@ jobs: env: GITEA_RUN_ID: ${{ gitea.run_id }} run: bash scripts/cleanup-integration.sh + - name: Remove this run disposable review image tags + if: always() + env: + REVIEW_SHA: ${{ gitea.sha }} + run: bash scripts/cleanup-review-images.sh diff --git a/apps/waterfall-wiki/tests/brand-navigation.mjs b/apps/waterfall-wiki/tests/brand-navigation.mjs new file mode 100644 index 0000000..60ac8f5 --- /dev/null +++ b/apps/waterfall-wiki/tests/brand-navigation.mjs @@ -0,0 +1,32 @@ +/** + * Recover once from an idempotent document GET's transient proxy failure. + * This never retries hydration, theme assertions, or mutations, and shares one + * 60-second budget across both attempts rather than extending the deadline. + * @param {(timeout: number) => Promise<{ok(): boolean, status(): number} | null>} navigate + * @param {string} url + * @param {() => number} now + */ +export async function navigateBrandDocument(navigate, url, now = Date.now) { + const deadline = now() + 60000; + for (let attempt = 1; attempt <= 2; attempt++) { + const remaining = deadline - now(); + if (remaining <= 0) throw new Error(`Document navigation budget exhausted: ${url}`); + try { + const response = await navigate(remaining); + if (response?.ok()) return response; + const status = response?.status(); + if (attempt === 1 && [500, 502, 503, 504].includes(status ?? 0)) { + console.warn('Brand navigation proxy recovery', JSON.stringify({url, status, attempt})); + continue; + } + throw new Error(`Document navigation failed: ${url}, HTTP ${status ?? 'no response'}`); + } catch (error) { + if (attempt === 1 && /ECONNRESET|net::ERR_(?:CONNECTION_RESET|CONNECTION_CLOSED|EMPTY_RESPONSE)/.test(String(error))) { + console.warn('Brand navigation connection recovery', JSON.stringify({url, error: String(error), attempt})); + continue; + } + throw error; + } + } + throw new Error(`Document navigation failed: ${url}`); +} diff --git a/apps/waterfall-wiki/tests/brand-navigation.test.mjs b/apps/waterfall-wiki/tests/brand-navigation.test.mjs new file mode 100644 index 0000000..a16eee7 --- /dev/null +++ b/apps/waterfall-wiki/tests/brand-navigation.test.mjs @@ -0,0 +1,48 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {navigateBrandDocument} from './brand-navigation.mjs'; +const response = status => ({ok: () => status === 200, status: () => status}); + +test('successful documents are fetched once', async () => { + let calls = 0; + await navigateBrandDocument(async () => { calls++; return response(200); }, '/canvas'); + assert.equal(calls, 1); +}); +test('one proxy failure can recover within the original deadline', async () => { + const timeouts = []; let elapsed = 0; + await navigateBrandDocument(async timeout => { + timeouts.push(timeout); elapsed = 15000; + return response(timeouts.length === 1 ? 500 : 200); + }, '/canvas', () => elapsed); + assert.deepEqual(timeouts, [60000, 45000]); +}); +test('connection reset can recover once', async () => { + let calls = 0; + await navigateBrandDocument(async () => { + if (++calls === 1) throw new Error('net::ERR_CONNECTION_RESET'); + return response(200); + }, '/canvas'); + assert.equal(calls, 2); +}); +test('persistent server failure is not hidden', async () => { + let calls = 0; + await assert.rejects(navigateBrandDocument(async () => { calls++; return response(502); }, '/canvas'), /HTTP 502/); + assert.equal(calls, 2); +}); +test('missing routes, authentication failures and ordinary timeouts are not retried', async () => { + for (const status of [401, 403, 404]) { + let calls = 0; + await assert.rejects(navigateBrandDocument(async () => { calls++; return response(status); }, '/canvas'), /HTTP/); + assert.equal(calls, 1); + } + let calls = 0; + await assert.rejects(navigateBrandDocument(async () => { calls++; throw new Error('Timeout 60000ms exceeded'); }, '/canvas'), /Timeout/); + assert.equal(calls, 1); +}); +test('recovery cannot extend the total navigation budget', async () => { + let calls = 0, elapsed = 0; + await assert.rejects(navigateBrandDocument(async () => { + calls++; elapsed = 60000; return response(503); + }, '/canvas', () => elapsed), /budget exhausted/); + assert.equal(calls, 1); +}); diff --git a/apps/waterfall-wiki/tests/brand-ui-integration.ts b/apps/waterfall-wiki/tests/brand-ui-integration.ts index d0c3bd1..6f1b7f1 100644 --- a/apps/waterfall-wiki/tests/brand-ui-integration.ts +++ b/apps/waterfall-wiki/tests/brand-ui-integration.ts @@ -1,6 +1,7 @@ /** CI-only checks against the disposable stack, never against personal documents. */ import assert from 'node:assert/strict'; import { chromium } from 'playwright-core'; +import { navigateBrandDocument } from './brand-navigation.mjs'; export async function verifyBrandUI(base: string) { for (const prefix of process.env.VERIFY_CANVAS_UI === '1' ? ['', '/studio'] : ['']) { const icon = await fetch(`${base}${prefix}/favicon.ico`); @@ -18,11 +19,20 @@ export async function verifyBrandUI(base: string) { try { const page = await browser.newPage({ viewport: { width: 1440, height: 960 } }); page.setDefaultTimeout(60000);page.setDefaultNavigationTimeout(60000); + const failures: string[] = []; + page.on('pageerror', error => failures.push(error.message)); + page.on('requestfailed', request => failures.push(new URL(request.url()).pathname + ': ' + request.failure()?.errorText)); const visit = async (path: string, theme: string) => { - await page.goto(base + path, { waitUntil: 'commit' }); - await page.locator('button[data-theme-ready="true"]').first().waitFor(); - await page.waitForFunction(expected => document.documentElement.dataset.theme === expected, theme); - console.log('Brand UI route ready:', path, theme); + const started = Date.now(); + try { + await navigateBrandDocument(timeout => page.goto(base + path, { waitUntil: 'commit', timeout }), base + path); + await page.locator('button[data-theme-ready="true"]').first().waitFor(); + await page.waitForFunction(expected => document.documentElement.dataset.theme === expected, theme); + console.log('Brand UI route ready:', path, theme, Date.now() - started); + } catch (error) { + console.error('Brand UI navigation diagnostics', JSON.stringify({path, url:page.url(), elapsedMs:Date.now()-started, failures:failures.slice(-20), body:(await page.locator('body').innerText({timeout:3000}).catch(()=>'unavailable')).slice(0,1200)})); + throw error; + } }; const background = (selector: string) => page.locator(selector).first().evaluate(el => getComputedStyle(el).backgroundColor); await visit('/', 'home-light'); diff --git a/scripts/cleanup-review-images.sh b/scripts/cleanup-review-images.sh new file mode 100644 index 0000000..db4b1ad --- /dev/null +++ b/scripts/cleanup-review-images.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +: "${REVIEW_SHA:?The exact review commit SHA is required}" +[[ "$REVIEW_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'Refusing invalid review SHA' >&2; exit 2; } +# Only this workflow's disposable tags; production tags, volumes and other projects stay out of scope. +for repository in creative-studio-review creative-wiki-tests creative-wiki-review; do + tag="$repository:$REVIEW_SHA" + referenced=$(docker ps -aq | xargs -r docker inspect --format '{{.Image}}') + if ! image=$(docker image inspect --format '{{.Id}}' "$tag" 2>/dev/null); then + echo "No review image: $tag" + continue + fi + if printf '%s\n' "$referenced" | grep -Fxq "$image"; then + echo "Keeping container-referenced review image: $tag" + continue + fi + docker image rm "$tag" +done diff --git a/scripts/integration.sh b/scripts/integration.sh index feab31e..919f839 100644 --- a/scripts/integration.sh +++ b/scripts/integration.sh @@ -17,7 +17,15 @@ compose(){ docker compose -p "$COMPOSE_PROJECT_NAME" --env-file "$TEST_ENV" "${C cleanup(){ code=$? trap - EXIT INT TERM - if [ "$code" -ne 0 ]; then compose logs --tail 160 waterfall-wiki waterfall-wiki-worker "${STUDIO_SERVICES[@]}" || true; fi + if [ "$code" -ne 0 ]; then + echo 'Runner pressure at integration failure (before disposable stack cleanup)' + uptime || true + free -m || true + df -h / || true + for resource in cpu memory io; do cat "/proc/pressure/$resource" || true; done + timeout 10s docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} memory={{.MemUsage}} io={{.BlockIO}} pids={{.PIDs}}' || true + compose logs --tail 160 waterfall-wiki waterfall-wiki-worker "${STUDIO_SERVICES[@]}" || true + fi cleanup_code=0 bash scripts/cleanup-integration.sh || cleanup_code=$? rm -f "$TEST_ENV" diff --git a/scripts/test-review-image-cleanup.py b/scripts/test-review-image-cleanup.py new file mode 100644 index 0000000..8e91729 --- /dev/null +++ b/scripts/test-review-image-cleanup.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Check review cleanup scope without Docker or build output.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +SCRIPT = Path(__file__).with_name('cleanup-review-images.sh') +SHA = 'a' * 40 +FAKE_DOCKER = '''#!/usr/bin/env python3 +import json, os, sys +args=sys.argv[1:] +with open(os.environ['DOCKER_CALLS'], 'a') as output: + output.write(json.dumps(args)+'\\n') +if args[0]=='ps': + if os.environ.get('FAIL_INVENTORY'): sys.exit(1) + if os.environ.get('REFERENCED'): print('container-id') +elif args[0]=='inspect': + print('sha256:review-image') +elif args[:2]==['image','inspect']: + if os.environ.get('MISSING_IMAGE'): sys.exit(1) + print('sha256:review-image') +''' + + +class ReviewCleanupTests(unittest.TestCase): + def run_cleanup(self, sha=SHA, **extra): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + docker = root / 'docker' + docker.write_text(FAKE_DOCKER) + docker.chmod(0o700) + calls = root / 'calls' + result = subprocess.run(['bash', str(SCRIPT)], capture_output=True, env={ + **os.environ, 'PATH': directory + os.pathsep + os.environ['PATH'], + 'DOCKER_CALLS': str(calls), 'REVIEW_SHA': sha, **extra}) + return result.returncode, [json.loads(line) for line in calls.read_text().splitlines()] if calls.exists() else [] + + def test_removes_only_exact_review_tags_without_force(self): + code, calls = self.run_cleanup() + self.assertEqual(code, 0) + self.assertEqual([args for args in calls if args[:2] == ['image', 'rm']], [ + ['image', 'rm', name + ':' + SHA] for name in + ['creative-studio-review', 'creative-wiki-tests', 'creative-wiki-review']]) + self.assertFalse(any('-f' in args or 'prune' in args or 'volume' in args for args in calls)) + + def test_invalid_sha_never_contacts_docker(self): + for value in ['', 'main', 'a' * 39, 'a' * 41, 'A' * 40, '*' * 40]: + with self.subTest(value=value): + code, calls = self.run_cleanup(value) + self.assertNotEqual(code, 0) + self.assertEqual(calls, []) + + def test_container_referenced_images_are_kept(self): + code, calls = self.run_cleanup(REFERENCED='1') + self.assertEqual(code, 0) + self.assertFalse(any(args[:2] == ['image', 'rm'] for args in calls)) + + def test_missing_tags_are_a_successful_noop(self): + code, calls = self.run_cleanup(MISSING_IMAGE='1') + self.assertEqual(code, 0) + self.assertFalse(any(args[:2] == ['image', 'rm'] for args in calls)) + + def test_inventory_failure_prevents_removal(self): + code, calls = self.run_cleanup(FAIL_INVENTORY='1') + self.assertNotEqual(code, 0) + self.assertEqual(len(calls), 1) + + +if __name__ == '__main__': + unittest.main()