fix(ci): bound navigation recovery and retire review images safely
This commit is contained in:
@@ -11,6 +11,8 @@ on:
|
||||
- 'scripts/integration.sh'
|
||||
- 'scripts/cleanup-integration.sh'
|
||||
- 'scripts/test-integration-cleanup.py'
|
||||
- 'scripts/cleanup-review-images.sh'
|
||||
- 'scripts/test-review-image-cleanup.py'
|
||||
concurrency:
|
||||
group: creative-workspace-${{ gitea.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
@@ -42,9 +44,11 @@ jobs:
|
||||
- name: Check patch formatting
|
||||
run: |
|
||||
git diff --check HEAD^ HEAD
|
||||
bash -n scripts/integration.sh scripts/cleanup-integration.sh scripts/verify-studio-production.sh
|
||||
bash -n scripts/integration.sh scripts/cleanup-integration.sh scripts/verify-studio-production.sh scripts/cleanup-review-images.sh
|
||||
node --check scripts/verify-studio-visual.cjs
|
||||
python3 scripts/test-integration-cleanup.py
|
||||
python3 scripts/test-review-image-cleanup.py
|
||||
node --test apps/waterfall-wiki/tests/brand-navigation.test.mjs
|
||||
- name: Verify Studio types, tests, build and smoke
|
||||
run: docker build -f apps/studio/Dockerfile -t "creative-studio-review:${{ gitea.sha }}" .
|
||||
- name: Build Wiki integration and runtime images
|
||||
@@ -63,3 +67,8 @@ jobs:
|
||||
env:
|
||||
GITEA_RUN_ID: ${{ gitea.run_id }}
|
||||
run: bash scripts/cleanup-integration.sh
|
||||
- name: Remove this run disposable review image tags
|
||||
if: always()
|
||||
env:
|
||||
REVIEW_SHA: ${{ gitea.sha }}
|
||||
run: bash scripts/cleanup-review-images.sh
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* Recover once from an idempotent document GET's transient proxy failure.
|
||||
* This never retries hydration, theme assertions, or mutations, and shares one
|
||||
* 60-second budget across both attempts rather than extending the deadline.
|
||||
* @param {(timeout: number) => Promise<{ok(): boolean, status(): number} | null>} navigate
|
||||
* @param {string} url
|
||||
* @param {() => number} now
|
||||
*/
|
||||
export async function navigateBrandDocument(navigate, url, now = Date.now) {
|
||||
const deadline = now() + 60000;
|
||||
for (let attempt = 1; attempt <= 2; attempt++) {
|
||||
const remaining = deadline - now();
|
||||
if (remaining <= 0) throw new Error(`Document navigation budget exhausted: ${url}`);
|
||||
try {
|
||||
const response = await navigate(remaining);
|
||||
if (response?.ok()) return response;
|
||||
const status = response?.status();
|
||||
if (attempt === 1 && [500, 502, 503, 504].includes(status ?? 0)) {
|
||||
console.warn('Brand navigation proxy recovery', JSON.stringify({url, status, attempt}));
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Document navigation failed: ${url}, HTTP ${status ?? 'no response'}`);
|
||||
} catch (error) {
|
||||
if (attempt === 1 && /ECONNRESET|net::ERR_(?:CONNECTION_RESET|CONNECTION_CLOSED|EMPTY_RESPONSE)/.test(String(error))) {
|
||||
console.warn('Brand navigation connection recovery', JSON.stringify({url, error: String(error), attempt}));
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new Error(`Document navigation failed: ${url}`);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import {test} from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {navigateBrandDocument} from './brand-navigation.mjs';
|
||||
const response = status => ({ok: () => status === 200, status: () => status});
|
||||
|
||||
test('successful documents are fetched once', async () => {
|
||||
let calls = 0;
|
||||
await navigateBrandDocument(async () => { calls++; return response(200); }, '/canvas');
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
test('one proxy failure can recover within the original deadline', async () => {
|
||||
const timeouts = []; let elapsed = 0;
|
||||
await navigateBrandDocument(async timeout => {
|
||||
timeouts.push(timeout); elapsed = 15000;
|
||||
return response(timeouts.length === 1 ? 500 : 200);
|
||||
}, '/canvas', () => elapsed);
|
||||
assert.deepEqual(timeouts, [60000, 45000]);
|
||||
});
|
||||
test('connection reset can recover once', async () => {
|
||||
let calls = 0;
|
||||
await navigateBrandDocument(async () => {
|
||||
if (++calls === 1) throw new Error('net::ERR_CONNECTION_RESET');
|
||||
return response(200);
|
||||
}, '/canvas');
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
test('persistent server failure is not hidden', async () => {
|
||||
let calls = 0;
|
||||
await assert.rejects(navigateBrandDocument(async () => { calls++; return response(502); }, '/canvas'), /HTTP 502/);
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
test('missing routes, authentication failures and ordinary timeouts are not retried', async () => {
|
||||
for (const status of [401, 403, 404]) {
|
||||
let calls = 0;
|
||||
await assert.rejects(navigateBrandDocument(async () => { calls++; return response(status); }, '/canvas'), /HTTP/);
|
||||
assert.equal(calls, 1);
|
||||
}
|
||||
let calls = 0;
|
||||
await assert.rejects(navigateBrandDocument(async () => { calls++; throw new Error('Timeout 60000ms exceeded'); }, '/canvas'), /Timeout/);
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
test('recovery cannot extend the total navigation budget', async () => {
|
||||
let calls = 0, elapsed = 0;
|
||||
await assert.rejects(navigateBrandDocument(async () => {
|
||||
calls++; elapsed = 60000; return response(503);
|
||||
}, '/canvas', () => elapsed), /budget exhausted/);
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
/** CI-only checks against the disposable stack, never against personal documents. */
|
||||
import assert from 'node:assert/strict';
|
||||
import { chromium } from 'playwright-core';
|
||||
import { navigateBrandDocument } from './brand-navigation.mjs';
|
||||
export async function verifyBrandUI(base: string) {
|
||||
for (const prefix of process.env.VERIFY_CANVAS_UI === '1' ? ['', '/studio'] : ['']) {
|
||||
const icon = await fetch(`${base}${prefix}/favicon.ico`);
|
||||
@@ -18,11 +19,20 @@ export async function verifyBrandUI(base: string) {
|
||||
try {
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 960 } });
|
||||
page.setDefaultTimeout(60000);page.setDefaultNavigationTimeout(60000);
|
||||
const failures: string[] = [];
|
||||
page.on('pageerror', error => failures.push(error.message));
|
||||
page.on('requestfailed', request => failures.push(new URL(request.url()).pathname + ': ' + request.failure()?.errorText));
|
||||
const visit = async (path: string, theme: string) => {
|
||||
await page.goto(base + path, { waitUntil: 'commit' });
|
||||
await page.locator('button[data-theme-ready="true"]').first().waitFor();
|
||||
await page.waitForFunction(expected => document.documentElement.dataset.theme === expected, theme);
|
||||
console.log('Brand UI route ready:', path, theme);
|
||||
const started = Date.now();
|
||||
try {
|
||||
await navigateBrandDocument(timeout => page.goto(base + path, { waitUntil: 'commit', timeout }), base + path);
|
||||
await page.locator('button[data-theme-ready="true"]').first().waitFor();
|
||||
await page.waitForFunction(expected => document.documentElement.dataset.theme === expected, theme);
|
||||
console.log('Brand UI route ready:', path, theme, Date.now() - started);
|
||||
} catch (error) {
|
||||
console.error('Brand UI navigation diagnostics', JSON.stringify({path, url:page.url(), elapsedMs:Date.now()-started, failures:failures.slice(-20), body:(await page.locator('body').innerText({timeout:3000}).catch(()=>'unavailable')).slice(0,1200)}));
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const background = (selector: string) => page.locator(selector).first().evaluate(el => getComputedStyle(el).backgroundColor);
|
||||
await visit('/', 'home-light');
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
: "${REVIEW_SHA:?The exact review commit SHA is required}"
|
||||
[[ "$REVIEW_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'Refusing invalid review SHA' >&2; exit 2; }
|
||||
# Only this workflow's disposable tags; production tags, volumes and other projects stay out of scope.
|
||||
for repository in creative-studio-review creative-wiki-tests creative-wiki-review; do
|
||||
tag="$repository:$REVIEW_SHA"
|
||||
referenced=$(docker ps -aq | xargs -r docker inspect --format '{{.Image}}')
|
||||
if ! image=$(docker image inspect --format '{{.Id}}' "$tag" 2>/dev/null); then
|
||||
echo "No review image: $tag"
|
||||
continue
|
||||
fi
|
||||
if printf '%s\n' "$referenced" | grep -Fxq "$image"; then
|
||||
echo "Keeping container-referenced review image: $tag"
|
||||
continue
|
||||
fi
|
||||
docker image rm "$tag"
|
||||
done
|
||||
@@ -17,7 +17,15 @@ compose(){ docker compose -p "$COMPOSE_PROJECT_NAME" --env-file "$TEST_ENV" "${C
|
||||
cleanup(){
|
||||
code=$?
|
||||
trap - EXIT INT TERM
|
||||
if [ "$code" -ne 0 ]; then compose logs --tail 160 waterfall-wiki waterfall-wiki-worker "${STUDIO_SERVICES[@]}" || true; fi
|
||||
if [ "$code" -ne 0 ]; then
|
||||
echo 'Runner pressure at integration failure (before disposable stack cleanup)'
|
||||
uptime || true
|
||||
free -m || true
|
||||
df -h / || true
|
||||
for resource in cpu memory io; do cat "/proc/pressure/$resource" || true; done
|
||||
timeout 10s docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} memory={{.MemUsage}} io={{.BlockIO}} pids={{.PIDs}}' || true
|
||||
compose logs --tail 160 waterfall-wiki waterfall-wiki-worker "${STUDIO_SERVICES[@]}" || true
|
||||
fi
|
||||
cleanup_code=0
|
||||
bash scripts/cleanup-integration.sh || cleanup_code=$?
|
||||
rm -f "$TEST_ENV"
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check review cleanup scope without Docker or build output."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
SCRIPT = Path(__file__).with_name('cleanup-review-images.sh')
|
||||
SHA = 'a' * 40
|
||||
FAKE_DOCKER = '''#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
args=sys.argv[1:]
|
||||
with open(os.environ['DOCKER_CALLS'], 'a') as output:
|
||||
output.write(json.dumps(args)+'\\n')
|
||||
if args[0]=='ps':
|
||||
if os.environ.get('FAIL_INVENTORY'): sys.exit(1)
|
||||
if os.environ.get('REFERENCED'): print('container-id')
|
||||
elif args[0]=='inspect':
|
||||
print('sha256:review-image')
|
||||
elif args[:2]==['image','inspect']:
|
||||
if os.environ.get('MISSING_IMAGE'): sys.exit(1)
|
||||
print('sha256:review-image')
|
||||
'''
|
||||
|
||||
|
||||
class ReviewCleanupTests(unittest.TestCase):
|
||||
def run_cleanup(self, sha=SHA, **extra):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
docker = root / 'docker'
|
||||
docker.write_text(FAKE_DOCKER)
|
||||
docker.chmod(0o700)
|
||||
calls = root / 'calls'
|
||||
result = subprocess.run(['bash', str(SCRIPT)], capture_output=True, env={
|
||||
**os.environ, 'PATH': directory + os.pathsep + os.environ['PATH'],
|
||||
'DOCKER_CALLS': str(calls), 'REVIEW_SHA': sha, **extra})
|
||||
return result.returncode, [json.loads(line) for line in calls.read_text().splitlines()] if calls.exists() else []
|
||||
|
||||
def test_removes_only_exact_review_tags_without_force(self):
|
||||
code, calls = self.run_cleanup()
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual([args for args in calls if args[:2] == ['image', 'rm']], [
|
||||
['image', 'rm', name + ':' + SHA] for name in
|
||||
['creative-studio-review', 'creative-wiki-tests', 'creative-wiki-review']])
|
||||
self.assertFalse(any('-f' in args or 'prune' in args or 'volume' in args for args in calls))
|
||||
|
||||
def test_invalid_sha_never_contacts_docker(self):
|
||||
for value in ['', 'main', 'a' * 39, 'a' * 41, 'A' * 40, '*' * 40]:
|
||||
with self.subTest(value=value):
|
||||
code, calls = self.run_cleanup(value)
|
||||
self.assertNotEqual(code, 0)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_container_referenced_images_are_kept(self):
|
||||
code, calls = self.run_cleanup(REFERENCED='1')
|
||||
self.assertEqual(code, 0)
|
||||
self.assertFalse(any(args[:2] == ['image', 'rm'] for args in calls))
|
||||
|
||||
def test_missing_tags_are_a_successful_noop(self):
|
||||
code, calls = self.run_cleanup(MISSING_IMAGE='1')
|
||||
self.assertEqual(code, 0)
|
||||
self.assertFalse(any(args[:2] == ['image', 'rm'] for args in calls))
|
||||
|
||||
def test_inventory_failure_prevents_removal(self):
|
||||
code, calls = self.run_cleanup(FAIL_INVENTORY='1')
|
||||
self.assertNotEqual(code, 0)
|
||||
self.assertEqual(len(calls), 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user