Plan owned review tags and old build cache using image ancestry

.gitea/workflows/creative-browser-diagnostics.yml

@@ -1,41 +1,50 @@
-name: creative disk recovery and inventory
+name: creative image ownership and physical disk plan
 on:
   push:
     branches: [codex/creative-browser-diagnosis]
 jobs:
   diagnose:
     runs-on: ces-control
-    timeout-minutes: 15
+    timeout-minutes: 20
     steps:
-      - name: Recover only verified terminal review tags and measure disk use
+      - name: Plan only ended review tags and old unreferenced build cache
         run: |
-          set -euo pipefail
-          df -h /
-          for sha in 2cc6f1f5433331758e825cc5cf3f5246d6101475 40c1023a86b3db7f6dd87743074b85e47e9225d2 a589f51698dad4a6dfb36ded3a107e1d537f1fe3 ab5ea7876b9d0eb0d7b56a02daf106fa2d0ccdde; do
-            for repository in creative-studio-review creative-wiki-tests creative-wiki-review; do
-              tag="$repository:$sha"
-              if ! image=$(docker image inspect --format '{{.Id}}' "$tag" 2>/dev/null); then continue; fi
-              docker image inspect --format '{{.Id}} size={{.Size}} tags={{json .RepoTags}} project={{with index .Config "Labels"}}{{index . "io.entroia.project"}}{{end}} created={{.Created}}' "$tag"
-              referenced=$(docker ps -aq | xargs -r docker inspect --format '{{.Image}}' | sort -u)
-              if printf '%s\n' "$referenced" | grep -Fxq "$image"; then
-                echo "KEEP: container reference $tag"
-              else
-                docker image rm "$tag"
-              fi
-            done
+          python3 - <<'PY'
+          import json,subprocess,datetime
+
+          def output(args):return subprocess.check_output(args,text=True)
+          allowed=set(['041620e2a563ce95339f95fb178793444675a54a', '0c8cf7c4cd0b8f7d904349184efc9da22b9dffb9', '15542e977ad8a768583631c2ef5e43b17355364d', '265cd7444b67ffc1301a2082294b862533eead36', '2bcd5992ebeecf7c7c5b5042c3d2bfdc7e9adcd6', '2cc6f1f5433331758e825cc5cf3f5246d6101475', '3d48f5042da0ce418b89f066126adbae8d843bac', '40a607696132818d6de0e6223723dce89d830fd6', '40c1023a86b3db7f6dd87743074b85e47e9225d2', '429e78c7222207854b0db5a0eddf9cec69406054', '4791729b65c7bb5ac7c4e37d503aaaccde3f3256', '4faef8b766a87017dfb5d838c6ed89eff132fad8', '53a4103308ba69411a26382ce8a5028b7ee444e2', '5a0554b3b1a86ebbae61d2fcd36352f519bba365', '5ab0ca78b23247a94008856f65c8004188d80fa2', '67dea99cd10e54d76c95106f9747d62afb520b39', '71da1173356dd5df7e5b5bda4ee0773ec8e18ff2', '7f7fc28e5797751b44e9df0148431a81e614d650', '8944e59eadc2d33f03f0c30a541384ecef4f62c2', '8ab67def5f95174e4afee0d8fb44c2543da144b9', '924724c5e4fec97913fe39c73cdca4bf53b7f9df', '94193d485bf2a6cf4667e2466063ba8098e63100', '95673d5c543e9aeb0686a37ff1bde21c4d52b7a8', 'a589f51698dad4a6dfb36ded3a107e1d537f1fe3', 'ab5ea7876b9d0eb0d7b56a02daf106fa2d0ccdde', 'af4e9b13edbb5378a0678607843f3036d4361367', 'b1e60839960fef369abaa7a657085066fbd376ff', 'bd8ccbb4a14be51e4515dc332f722f6bf6227bad', 'cad84f8a6730f7780cc8c4d851e1c6f9814cb355', 'cdcd4c4b94d445829d624800193c66955ff09704', 'cfc668690469acd32ef7b378458c682070d04479', 'd2eaabe0da98d1461408a270e8a1ac6b1d8c2cd8', 'd362e1a975492bd4d06290028cf421e80081d12e', 'd8c06d6cc66b2b60813940332a54fc4a1344827f', 'de851085ddf186fed31b1e5b6ea03b2c94e63fde', 'eed5d43e210ff38ee24f10778a8becc99cfb0f62', 'f1c794654e5f8c80c3d1a56519f9accbfa626505', 'f4f0d67306a4daa3f28b16e06aee001ae95bf101', 'fb6f29db760abbc6acd3f5cd05f161413799d0c4', 'fcc0aba8100c0aa1d4874c8e9b8c2dd31570069b', 'fd9b8640b71114c8a947ccd27e3a6323b3f8b564'])
+          ids=sorted(set(output(['docker','image','ls','-aq','--no-trunc']).split()))
+          containers=output(['docker','ps','-aq']).split()
+          refs=set(output(['docker','inspect','--format','{{.Image}}',*containers]).split()) if containers else set()
+          images={}
+          for offset in range(0,len(ids),40):
+           for image in json.loads(output(['docker','image','inspect',*ids[offset:offset+40]])):
+            labels=image.get('Config',{}).get('Labels') or {}
+            images[image['Id']]={'id':image['Id'],'parent':image.get('Parent',''),'created':image['Created'],'size':image['Size'],'tags':image.get('RepoTags') or [],'project':labels.get('io.entroia.project'),'build':labels.get('io.entroia.wiki.build')}
+          review=[]
+          for image in images.values():
+           for tag in image['tags']:
+            name,_,sha=tag.rpartition(':')
+            if name in ('creative-studio-review','creative-wiki-tests','creative-wiki-review') and sha in allowed and image['id'] not in refs:
+             review.append({'tag':tag,'id':image['id']})
+          candidate_tags={item['tag'] for item in review}
+          protected=set(refs)|{image['id'] for image in images.values() if any(tag not in candidate_tags for tag in image['tags'])}
+          queue=list(protected)
+          while queue:
+           parent=images.get(queue.pop(),{}).get('parent')
+           if parent and parent not in protected:
+            protected.add(parent);queue.append(parent)
+          now=datetime.datetime.now(datetime.timezone.utc)
+          cache=[]
+          for image in images.values():
+           age=(now-datetime.datetime.fromisoformat(image['created'].replace('Z','+00:00'))).total_seconds()
+           if not image['tags'] and image['project']=='waterfall-wiki' and image['build']=='true' and image['id'] not in protected and age>=86400:
+            cache.append(image)
+          cache.sort(key=lambda image:image['created'],reverse=True)
+          print('DISKPLAN '+json.dumps({'reviewTags':review,'buildCache':cache,'totalImages':len(images),'containerReferencedImages':len(refs),'protectedImageAndAncestorCount':len(protected)}))
+          PY
+          for directory in /root/.cache/act /opt /var/log /var/lib/containerd/io.containerd.snapshotter.v1.overlayfs; do
+            timeout 240s du -x -s -B 1 "$directory" 2>/dev/null || true
           done
           df -h /
-          docker info --format 'docker-root={{.DockerRootDir}} storage={{.Driver}}'
-          timeout 240s du -x -h -d 1 /var/lib/docker /var/lib/containerd /root/.cache/act /opt /var/log 2>/dev/null | sort -h || true
-          echo 'VOLUME OWNERSHIP AND SIZE (metadata only)'
-          for volume in $(docker volume ls -q); do
-            docker volume inspect --format '{{.Name}} path={{.Mountpoint}} project={{with index . "Labels"}}{{index . "com.docker.compose.project"}}{{end}}' "$volume"
-            mount=$(docker volume inspect --format '{{.Mountpoint}}' "$volume")
-            timeout 20s du -s -B 1 "$mount" 2>/dev/null || true
-          done
-          echo 'CONTAINER OWNERSHIP AND LOG SIZE (metadata only)'
-          for container in $(docker ps -aq); do
-            docker inspect --format '{{.Id}} {{.Name}} status={{.State.Status}} project={{with index .Config "Labels"}}{{index . "com.docker.compose.project"}}{{end}} build={{with index .Config "Labels"}}{{index . "io.entroia.wiki.build"}}{{end}} owner={{with index .Config "Labels"}}{{index . "io.entroia.project"}}{{end}}' "$container"
-            log=$(docker inspect --format '{{.LogPath}}' "$container")
-            if [ -n "$log" ]; then du -s -B 1 "$log" 2>/dev/null || true; fi
-          done