Merge remote-tracking branch 'origin/main' into fapolicyd-tmpfs

.gitea/workflows/registry-gc.yaml

@@ -0,0 +1,190 @@
+# .gitea/workflows/registry-gc.yaml
+#
+# Garbage-collect ORPHANED container images: dev/branch builds whose branch no
+# longer exists. Reconciliation model (NOT event-driven) — it lists the live
+# branches and every container version, then prunes the mismatch. So it catches
+# branches deleted any way (UI, `git push --delete`, API) and never depends on a
+# `delete` webhook the runner might not support or might miss.
+#
+# ┌─ SAFETY — what it will NEVER delete ───────────────────────────────────────┐
+# │ * `latest`.                                                                 │
+# │ * ANY release / version tag — semver `1.2.3` / `v1.2.3`, spdk `v24.09`, the │
+# │   dnsmasq `main-…` line, etc. The guard: only tags ending in a `-<7 hex>`   │
+# │   commit-sha suffix are ever CANDIDATES, and ONLY dev/branch builds get     │
+# │   that suffix (`<branch>-<sha7>`, spdk `<ver>-<branch>-<sha7>`). Release    │
+# │   tags never carry it, so they are structurally out of scope.               │
+# │ * Any image whose (sanitized) branch is still LIVE, and anything on `main`. │
+# │ * Anything it cannot confidently classify (fail-safe = KEEP).               │
+# │ * Everything, if the live-branch list can't be fetched (aborts first).      │
+# └────────────────────────────────────────────────────────────────────────────┘
+#
+# DRY RUN by default. `workflow_dispatch` previews unless dispatched with
+# dry_run=false. SCHEDULED runs also dry-run until you ARM them by setting the
+# repo variable REGISTRY_GC_ENABLE=true — so you can eyeball a scheduled dry-run
+# first, then flip the variable to let it prune automatically.
+#
+# TOKENS (two, by capability — a package token can't read branches, hence the
+# original 403): branch reads use the automatic Actions token (github.token,
+# repo read); package list + delete use PACKAGE_TOKEN (write:package, which the
+# registry-push token already has). Only touches images THIS repo builds; the
+# fleet-kiosk image is owned by the kiosk-app submodule's own CI.
+
+name: registry-gc
+run-name: prune orphan container images (dry_run default; ${{ github.event_name }})
+
+on:
+  schedule:
+    - cron: "17 4 * * *"        # nightly ~04:17 UTC
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Preview only (true) or actually delete (false)"
+        type: boolean
+        default: true
+
+permissions:
+  contents: read                 # branch listing via the auto Actions token
+
+concurrency:
+  group: registry-gc
+  cancel-in-progress: false
+
+jobs:
+  prune:
+    runs-on: ubuntu-latest
+    env:
+      # API base — hardcoded to match the registry host baked into the other
+      # workflows' image names (git.bitasylum.org/audiobuzz/*). Keep in sync.
+      SERVER: "https://git.bitasylum.org"
+      OWNER: ${{ github.repository_owner }}
+      REPO: ${{ github.repository }}
+      REPO_TOKEN: ${{ github.token }}            # auto Actions token: repo/branch reads
+      PKG_TOKEN: ${{ secrets.PACKAGE_TOKEN }}    # package list + delete (write:package)
+      EVENT: ${{ github.event_name }}
+      DISPATCH_DRY: ${{ inputs.dry_run }}
+      GC_ENABLE: ${{ vars.REGISTRY_GC_ENABLE }}
+      # Only images built by THIS repo's workflows.
+      IMAGES: "fleet-boot fleet-ipxe fleet-dnsmasq spdk-nvmf"
+    steps:
+      - name: Reconcile registry against live branches
+        # The runner defaults `run:` to sh (dash); this script uses bashisms
+        # (process substitution, arrays) — force bash. (Cf. gitea-main-ci.yaml.)
+        shell: bash
+        run: |
+          set -euo pipefail
+          command -v jq >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y -qq jq; }
+
+          API="${SERVER}/api/v1"
+
+          # Authenticated GET. $1 = which token (repo|pkg), $2 = url. Echoes the
+          # body on 2xx; on anything else prints an actionable error and exits
+          # (so a scope/auth problem can never be mistaken for "no results").
+          get() {
+            local who="$1" url="$2" tok tmp code body
+            case "$who" in repo) tok="$REPO_TOKEN";; pkg) tok="$PKG_TOKEN";; esac
+            tmp="$(mktemp)"
+            code="$(curl -sS --retry 3 -o "$tmp" -w '%{http_code}' \
+                     -H "Authorization: token ${tok}" -H "Accept: application/json" "$url" || echo 000)"
+            body="$(cat "$tmp")"; rm -f "$tmp"
+            case "$code" in
+              2*) printf '%s' "$body"; return 0 ;;
+              401|403)
+                echo "::error::GET ${url} -> HTTP ${code} (token '${who}' lacks scope). branches need repo read (github.token); packages need read/write:package (PACKAGE_TOKEN)." >&2
+                exit 1 ;;
+              *)
+                echo "::error::GET ${url} -> HTTP ${code}" >&2
+                exit 1 ;;
+            esac
+          }
+
+          # ---- dry-run resolution ------------------------------------------
+          if [ "$EVENT" = "workflow_dispatch" ]; then
+            DRY="${DISPATCH_DRY:-true}"
+          elif [ "${GC_ENABLE:-}" = "true" ]; then
+            DRY="false"
+          else
+            DRY="true"
+            echo "::notice::scheduled run is DRY — set repo variable REGISTRY_GC_ENABLE=true to arm real deletion"
+          fi
+          echo "event=$EVENT dry_run=$DRY"
+
+          # Sanitize a branch name EXACTLY as the build workflows do, then
+          # lowercase (registries lowercase tags). MUST stay in sync with the
+          # `safe_branch` sed in {gitea-main-ci,ipxe-image,dnsmasq-image,spdk-image}.yaml.
+          sanitize() { printf '%s' "$1" | sed -e 's#[^A-Za-z0-9._-]#-#g' -e 's#^[.-]*##' | cut -c1-80 | tr 'A-Z' 'a-z'; }
+
+          # ---- 1. live branches (fail-safe: abort if none) -----------------
+          live=" main "        # main is always protected, even if listing is odd
+          total=0 page=1
+          while :; do
+            resp="$(get repo "${API}/repos/${REPO}/branches?limit=50&page=${page}")"
+            cnt="$(printf '%s' "$resp" | jq 'length')"
+            [ "$cnt" -eq 0 ] && break
+            while IFS= read -r b; do
+              [ -n "$b" ] || continue
+              live="${live}$(sanitize "$b") "
+              total=$((total + 1))
+            done < <(printf '%s' "$resp" | jq -r '.[].name')
+            page=$((page + 1))
+          done
+          if [ "$total" -eq 0 ]; then
+            echo "::error::fetched 0 live branches — aborting so we never mass-delete on an API/auth failure"
+            exit 1
+          fi
+          echo "live branches ($total):${live}"
+
+          # ---- 2. reconcile each image -------------------------------------
+          orphans=0 kept=0 deleted=0 failed=0
+          for img in $IMAGES; do
+            page=1
+            while :; do
+              resp="$(get pkg "${API}/packages/${OWNER}?type=container&q=${img}&limit=50&page=${page}")"
+              cnt="$(printf '%s' "$resp" | jq 'length')"
+              [ "$cnt" -eq 0 ] && break
+              while IFS= read -r ver; do
+                [ -n "$ver" ] || continue
+
+                # PROTECT releases + latest: a tag WITHOUT the dev commit-sha
+                # suffix (`-<7 hex>`) is a release/version/`latest`/`main-ver`
+                # tag and is NEVER a candidate. This single rule keeps every real
+                # version tag and latest, exactly as required.
+                if [ "$ver" = "latest" ] || ! printf '%s' "$ver" | grep -Eq -- '-[0-9a-f]{7}$'; then
+                  kept=$((kept + 1)); continue
+                fi
+
+                # It's a dev/commit tag. KEEP iff its branch component is a LIVE
+                # branch (anchored so `<branch>-<sha>` and spdk `<ver>-<branch>-<sha>`
+                # both match). Conservative: a suffix collision only ever causes a
+                # KEEP, never a wrongful delete.
+                is_live="false"
+                for lb in $live; do
+                  lre="$(printf '%s' "$lb" | sed 's/\./\\./g')"   # escape regex dots
+                  if printf '%s' "$ver" | grep -Eq -- "(^|-)${lre}-[0-9a-f]{7}\$"; then
+                    is_live="true"; break
+                  fi
+                done
+                if [ "$is_live" = "true" ]; then kept=$((kept + 1)); continue; fi
+
+                # Orphan: dev tag whose branch is gone.
+                orphans=$((orphans + 1))
+                echo "ORPHAN  ${img}:${ver}"
+                if [ "$DRY" != "true" ]; then
+                  code="$(curl -sS -o /dev/null -w '%{http_code}' -X DELETE \
+                           -H "Authorization: token ${PKG_TOKEN}" \
+                           "${API}/packages/${OWNER}/container/${img}/${ver}" || echo 000)"
+                  case "$code" in
+                    2*) deleted=$((deleted + 1)); echo "  deleted ${img}:${ver}" ;;
+                    *)  failed=$((failed + 1)); echo "::warning::delete ${img}:${ver} returned HTTP ${code}" ;;
+                  esac
+                fi
+              done < <(printf '%s' "$resp" | jq -r --arg n "$img" '.[] | select(.name == $n) | .version')
+              page=$((page + 1))
+            done
+          done
+
+          echo "----------------------------------------------------------------"
+          echo "kept=${kept} orphans=${orphans} deleted=${deleted} failed=${failed} dry_run=${DRY}"
+          if [ "$DRY" = "true" ] && [ "$orphans" -gt 0 ]; then
+            echo "::notice::${orphans} orphan version(s) would be pruned. Dispatch with dry_run=false, or set REGISTRY_GC_ENABLE=true for scheduled runs, to actually delete."
+          fi
+          [ "$failed" -eq 0 ] || { echo "::error::${failed} deletion(s) failed"; exit 1; }

CLAUDE.md

@@ -115,9 +115,13 @@ instead of working around it.
    `rd.dot1x.nvindex=` / `fleet-enrol -handle -nvindex`.)
 1. **The hook ordering in `dot1x-macsec-up.sh` is load-bearing.**
    NV-cert extraction → supplicant start → wait for `Authorized` + macsec0 →
-   DHCP **on macsec0** → hostnqn derivation. Never DHCP before auth
+   DHCP **on macsec0** → hostnqn derivation → **targeted NVMe/TCP connect**
+   (`nvme connect -n $rd.nvmf.nqn`, step 6). Never DHCP before auth
    completes (pre-auth leases are invalid after VLAN assignment); never
-   connect NVMe before macsec0 is addressed.
+   connect NVMe before macsec0 is addressed. The connect is **targeted to this
+   version's subsystem** (`rd.nvmf.nqn`, baked per-version into the signed
+   cmdline) — never a discovery `connect-all`, which attaches every served
+   version and defeats the per-version drain/reap gate (see TODO #6).
 1. **The supplicant must survive switch-root.** It is launched with
    `exec -a @wpa_supplicant` (systemd root-storage-daemon convention) and
    its state lives under `/run`. Do not “clean up” the `@`, move state off

diskless.pkr.hcl

@@ -296,6 +296,16 @@ build {
       # talks to the TPM directly via go-tpm. So this list stays at the baseline.)
       "dnf -y install wpa_supplicant tpm2-tools tpm2-tss nvme-cli cryptsetup dracut-network NetworkManager zstd openssl",
 
+      # Locale trim: @^minimal-environment pulls glibc-all-langpacks — ~228 MB of
+      # locale data for EVERY language (measured 2026-07-03, 3rd-largest package in
+      # the whole image). This fleet is English-only (we also strip
+      # /usr/share/locale in 2d). Swap it for the English langpack + the C/POSIX
+      # minimal set → reclaims ~220 MB of the read-only erofs. Install the small
+      # replacements first, then drop the monolith (nothing Requires it; strict so
+      # a surprise dep surfaces in CI rather than silently bloating the image).
+      "dnf -y install glibc-minimal-langpack glibc-langpack-en",
+      "dnf -y remove glibc-all-langpacks",
+
       # EFI / Secure Boot management tools (efivar, mokutil, efibootmgr — all base
       # repos). Used over SSH on a workstation: `mokutil --sb-state/--pk/--kek/--db`
       # reads Secure Boot state + the enrolled PK/KEK/db; efibootmgr manages boot
@@ -396,6 +406,39 @@ build {
       "install -D -m0755 /tmp/fleet-enrol /usr/sbin/fleet-enrol",
       "restorecon /usr/sbin/fleet-enrol 2>/dev/null || true",
 
+      # ---- Firmware slimming (BEFORE dracut so the initramfs shrinks too) ---
+      # linux-firmware ships ~678 MB of blobs for hardware this fixed fleet will
+      # never have. A live fleet audit (`modinfo -F firmware` over loaded modules +
+      # sysfs/`lspci` driver bindings on the real 820-class hosts AND the VMs,
+      # 2026-07-03) proved the ONLY loadable firmware any host requests is Intel
+      # i915 (GPU); every NIC is firmware-free (e1000e's ethtool "firmware-version"
+      # is on-chip NVM, not a /usr/lib/firmware file; virtio_net loads nothing), so
+      # NO NIC needs firmware. This runs PRE-dracut deliberately: --no-hostonly +
+      # plymouth's `drm` dep instmods a broad DRM set (i915/amdgpu/radeon/…) and
+      # dracut bundles each module's firmware into the initramfs (amdgpu fw is the
+      # classic UKI bloater), so trimming FIRST shrinks BOTH the rootfs erofs AND
+      # the UKI's initramfs. Keep i915 (all Intel gens — the fleet's only
+      # firmware-loading GPU) + intel/ice (the ice server-NIC DDP, the SOLE intel/
+      # blob any host needs — ish/sof/ibt are sensor-hub/audio/bluetooth, none
+      # loaded at boot) + intel-ucode; strip the rest (~600 MB). NOT mellanox/ —
+      # those are mlxsw_spectrum SWITCH-ASIC blobs, not NIC firmware (mlx5 NIC fw is
+      # on-card, never request_firmware'd from /usr/lib/firmware), so they're dead
+      # weight even for an mlx5 build. Safe for the initramfs: the fleet
+      # splash GPU (i915) fw is kept and the build-VM splash uses virtio-gpu/bochs
+      # (no fw) — dracut just warns "firmware not found" for the removed drm modules
+      # and continues. Risk asymmetry: missing NIC fw would brick the fleet (none
+      # needed), missing GPU fw only degrades a display. BENCH-VERIFY the kiosk
+      # renders on the 820. Strip-in-place (rm, not rpm -e): linux-firmware can't be
+      # dropped as a package without also losing i915. See docs/TODO #18.
+      "if [ -d /usr/lib/firmware ]; then find /usr/lib/firmware -mindepth 1 -maxdepth 1 ! -name i915 ! -name intel ! -name intel-ucode -exec rm -rf {} + ; fi",
+      # Narrow intel/ to just ice (NIC DDP); drop ish (sensor hub), sof (audio DSP),
+      # ibt (bluetooth) etc. — none are loaded on this fleet (audit 2026-07-03).
+      "[ -d /usr/lib/firmware/intel ] && find /usr/lib/firmware/intel -mindepth 1 -maxdepth 1 ! -name ice -exec rm -rf {} + || true",
+      # Wi-Fi firmware: wired-only fleet (no iwlwifi). Remove the packages cleanly
+      # so the rpmdb stays honest (the find above dropped their iwlwifi-*.ucode).
+      "dnf remove -y 'iwl*-firmware' 2>/dev/null || true",
+      "echo \"==> firmware after strip: $(du -sh /usr/lib/firmware 2>/dev/null | cut -f1) (was ~678 MB)\"",
+
       # ---- Build the fleet-generic initramfs ------------------------------
       # %%{ escapes the rpm query tags so Packer's HCL template engine treats
       # them as literals (a bare %{...} is parsed as a template directive).
@@ -406,6 +449,22 @@ build {
       # under pipefail the pipeline reports failure even when the file is
       # present (a false negative that wrongly failed the build for many runs).
       "lsinitrd /boot/initramfs-generic.img > /tmp/lsinitrd.txt",
+      # Initramfs footprint (this .img is embedded in the signed UKI and pulled
+      # over PXE/HTTP on every boot). Report the built size + any firmware that
+      # still made it in — post-strip this should be i915 only (pulled by
+      # plymouth's drm dep); if amdgpu/radeon/etc. appear, the strip didn't run
+      # before dracut. See docs/TODO #18.
+      "echo \"==> initramfs size: $(du -h /boot/initramfs-generic.img | cut -f1)\"",
+      "echo '==> firmware bundled in initramfs (expect i915 only):'; grep -i '/firmware/' /tmp/lsinitrd.txt || echo '  (none)'",
+      # Largest files IN the initramfs (in-cpio size, before the outer zstd) — after
+      # the firmware strip the .img is module-bound (the --no-hostonly DRM/net/crypto
+      # set), so this surfaces the next trim targets (e.g. GPU driver kmods). See #18.
+      # Sort into a FILE then head the file — NOT `sort | head`: head closes the pipe
+      # after 25 lines, sort dies of SIGPIPE (exit 141), and step 2's
+      # `set -euxo pipefail` turns that into a build failure (same trap as the
+      # lsinitrd `| grep -q` note above). (The kiosk step's rpm-manifest `| head` is
+      # safe only because that step runs plain `set -e`, no pipefail.)
+      "echo '==> initramfs largest files:'; awk '/^-/{print $5, $NF}' /tmp/lsinitrd.txt | sort -rn > /tmp/initrd-sizes.txt; head -25 /tmp/initrd-sizes.txt; rm -f /tmp/initrd-sizes.txt",
       "grep -E 'macsec|wpa_supplicant|tpm2_nvread|nvme.tcp|erofs|veritysetup' /tmp/lsinitrd.txt",
       # Assert NetworkManager is in the initramfs — the network-manager dracut
       # module is what renews the DHCP lease post-switch-root. A missing/renamed
@@ -459,7 +518,27 @@ build {
       # the heaviest items here and are likely unnecessary given weston
       # --use-pixman + chromium --disable-gpu — candidate to drop once the kiosk
       # is display-verified; see TODO #4.)
-      "dnf install -y --setopt=tsflags=nodocs weston chromium mesa-dri-drivers mesa-libEGL mesa-libgbm dejavu-sans-fonts",
+      #
+      # install_weak_deps=False: a locked-down single-URL kiosk needs none of the
+      # Recommends: this transaction otherwise drags in — flatpak + its ostree/
+      # bubblewrap sandbox runtime, tracker-miners (a filesystem indexer, both
+      # dead weight AND a threat surface), the pipewire/pulseaudio audio stack,
+      # and libproxy-webkitgtk4 → webkit2gtk3-jsc (a whole SECOND browser engine
+      # pulled into a Chromium kiosk just for libproxy's PAC runner). Measured:
+      # the default (weak-deps=on) transaction was 229 pkgs / 1.0 GB. The floor
+      # that survives is chromium + chromium-common + llvm-libs (mesa sw-GL) +
+      # gtk3/adwaita + libicu — all HARD deps. If a genuine runtime dep turns out
+      # to be a weak dep (e.g. a font, or libva for VA-API on the GPU path), add
+      # it to the explicit list below rather than re-enabling weak deps wholesale.
+      "dnf install -y --setopt=tsflags=nodocs --setopt=install_weak_deps=False weston chromium mesa-dri-drivers mesa-libEGL mesa-libgbm dejavu-sans-fonts",
+      # Footprint audit (surfaces in the CI build log): the installed set, largest
+      # first, + a total count. Turns 'seems unnecessary' into data for the next
+      # trim round (e.g. dropping gstreamer/media codecs if the SPA plays no HTML5
+      # media, or mesa-dri-drivers if the GPU path is abandoned). %%{...} is escaped
+      # for Packer's HCL2 template engine (a bare %%{ is parsed as a directive).
+      "echo '==> kiosk package footprint (installed size, largest 40 first):'",
+      "rpm -qa --qf '%%{SIZE}\\t%%{NAME}-%%{VERSION}-%%{RELEASE}\\n' | sort -rn | head -40",
+      "echo \"==> total packages installed after kiosk: $(rpm -qa | wc -l)\"",
       # Unprivileged, passwordless kiosk user. Groups: 'video' (KMS /dev/dri/card0
       # for weston) + 'input' (libinput) as a seat fallback; 'render'
       # (/dev/dri/renderD128) so Chromium's GPU process can do hardware raster on
@@ -912,6 +991,25 @@ build {
       # hard-needs it) and mask as a backstop against future weak-dep reinstalls.
       "dnf remove -y firewalld 2>/dev/null || true",
       "systemctl mask firewalld.service 2>/dev/null || true",
+      # nss-mdns / avahi: DO NOT REMOVE — the EPEL `chromium` package hard-Requires
+      # nss-mdns (Chromium resolves `.local` mDNS hostnames for WebRTC ICE
+      # candidates, to avoid leaking local IPs in peer connections). `dnf remove
+      # nss-mdns` therefore takes CHROMIUM with it as a dependent package, then
+      # reaps its whole orphaned GUI stack (avahi, at-spi2, atk, adwaita,
+      # chromium-common, ...) — a ~300 MB drop that ships a browserless, bricked
+      # kiosk (flashing cursor; kiosk-session dies at `${BROWSER:?}`). This bit us
+      # 2026-07-03: the earlier commit checked what nss-mdns NEEDS (avahi) but not
+      # what NEEDS nss-mdns (chromium). The FINAL GUARD below now asserts the kiosk
+      # stack survived so this can never ship silently again.
+      #
+      # We still don't want the avahi mDNS DAEMON listening on :5353 on a
+      # locked-down kiosk (that was the real attack-surface concern). nss-mdns is
+      # only an NSS module (libnss_mdns.so) — it needs no daemon; if avahi-daemon
+      # isn't running, `.local` lookups just fall through. So keep the PACKAGES
+      # (satisfy chromium's Requires) but mask the listener: no socket, no daemon,
+      # dep satisfied, browser intact. ~1.5 MB kept — negligible vs a working kiosk.
+      "systemctl mask avahi-daemon.service avahi-daemon.socket 2>/dev/null || true",
+      "systemctl disable avahi-daemon.service avahi-daemon.socket 2>/dev/null || true",
       # (No Go toolchain to remove — fleet-hostapi + fleet-enrol are built OUT OF
       # the VM on the CI runner and file-provisioned in as static binaries.)
       # The build-time STIG reports must not ship inside the read-only rootfs.
@@ -928,6 +1026,25 @@ build {
       "if [ -d /usr/lib64/chromium-browser/locales ]; then find /usr/lib64/chromium-browser/locales -type f -name '*.pak' ! -name 'en-US.pak' -delete; fi",
       # App translation catalogs (.mo) for locales this fleet never uses.
       "[ -d /usr/share/locale ] && find /usr/share/locale -mindepth 1 -maxdepth 1 -type d ! -name 'en*' ! -name 'C*' -exec rm -rf {} + || true",
+      # (linux-firmware slimming moved to step 2, BEFORE the dracut build, so the
+      # UKI initramfs shrinks too — see TODO #18. Nothing re-adds firmware after
+      # that point, so the rootfs erofs is already trimmed by the time we get here.)
+      #
+      # ---- FINAL GUARD: assert the kiosk stack survived the whole pipeline ----
+      # 2026-07-03 incident: chromium installed cleanly in the 2b kiosk step (the
+      # footprint audit showed chromium-149 + chromium-common, 523 pkgs), but by
+      # the final image it was GONE (472 pkgs) — `dnf remove -y nss-mdns` in 2d
+      # removed chromium AS A DEPENDENT PACKAGE (EPEL chromium hard-Requires
+      # nss-mdns for WebRTC mDNS ICE candidates), then reaped its orphaned GUI
+      # stack. Result: a ~500 MB image whose kiosk died at the `${BROWSER:?}` guard
+      # in kiosk-session — a flashing-cursor boot with no weston, shipped green.
+      # The nss-mdns removal is fixed above (kept + avahi listener masked), but a
+      # missing browser must fail the BUILD, never brick the fleet: assert every
+      # kiosk-critical package is still installed at the very end of the pipeline
+      # (after remediation AND all 2d removes). This catches ANY future silent drop
+      # — a new hard-Requires cascade, an oscap package_*_removed rule, autoremove.
+      # enable_kiosk=false legitimately ships no browser, so gate on it.
+      "if [ \"${var.enable_kiosk}\" = \"true\" ] && [ -n \"${var.kiosk_url}\" ]; then for p in chromium chromium-common weston mesa-dri-drivers mesa-libEGL mesa-libgbm; do rpm -q \"$p\" >/dev/null 2>&1 || { echo \"ERROR: kiosk-critical package '$p' missing from the FINAL image — a later step removed it after the 2b kiosk install (e.g. a dnf remove whose dependency cascade took chromium, cf. the 2026-07-03 nss-mdns incident). Refusing to ship a bricked kiosk.\" >&2; exit 1; }; done; echo '==> guard: kiosk stack intact (chromium/weston/mesa present after remediation + hygiene)'; fi",
     ]
   }
 

docs/TODO.md

@@ -306,6 +306,19 @@ step 8 and §5). Retire/reset the errata afterward. Needs the design-doc source
 trim, ship via CI `enable_kiosk=true`) and the phone-home/NTP lockdown (done) —
 all captured in the sub-notes below. Secure-path render check is folded into #8.
 
+**Package-footprint trim (2026-07-03):** the kiosk dnf transaction was measured at
+**229 pkgs / 1.0 GB** (weak deps ON). Added `--setopt=install_weak_deps=False` to
+the install in `diskless.pkr.hcl` — drops the flatpak/ostree sandbox runtime,
+tracker-miners (fs indexer = dead weight + threat surface), the pipewire/pulseaudio
+audio stack, and `libproxy-webkitgtk4 → webkit2gtk3-jsc` (an 8.9M *second* browser
+engine). Also added an `rpm -qa --qf %%{SIZE}` size-manifest diagnostic to the build
+log to quantify the result. The immovable floor is chromium + chromium-common +
+llvm-libs (mesa sw-GL) + gtk3/adwaita + libicu — all HARD deps. **BENCH-VERIFY:**
+with weak deps off, confirm the kiosk still renders (no genuine runtime dep was a
+weak dep — watch for a missing font or `libva`). NEXT trim candidates (measure via
+the new manifest): gstreamer/media codecs if the SPA plays no HTML5 media;
+mesa-dri-drivers + `--use-pixman`-only if the GPU path is abandoned.
+
 
 **WORKING / display-verified 2026-06-17** (kiosk renders Chromium fullscreen on
 the configured URL). Build-flagged OFF by default for now (`enable_kiosk=false`)
@@ -675,6 +688,37 @@ must be built around **never restarting the process that holds live connections.
     the current UKI), so auto-reap is safe; removal is still drain-gated +
     grace-debounced. The loader + target share the RPC socket on a named volume
     (`nvmf_tgt -r`); the loader reuses the SPDK image (has python3 + rpc.py).
+  - **BENCH FINDING 2026-07-03 (FIXED) — hosts did `connect-all`, so NO version
+    ever drained → auto-reap could never fire.** Observed: `spdk-release list`
+    showed more controllers than workstations, with every box appearing on *every*
+    served version's subsystem (e.g. 1/3/3/1/3 across 5 versions for ~3 boxes).
+    Root cause: the initramfs connected with **`nvme connect-all`** (test path
+    `nvmf-connect.sh`; secure path delegated to the dracut nvmf module's discovery
+    autoconnect), which attaches EVERY subsystem in the discovery log — the host
+    then mounts only its own by-UUID verity namespaces but leaves a live controller
+    on all the others. So each running host pinned every version above 0 conns, and
+    `spdk-loader`'s "auto-reap once drained (0 conns)" **could never trigger** while
+    the fleet was up — the served-version list grew without bound. This defeats the
+    whole drain/reap model, not just cosmetics. **Fix:** targeted connect. The
+    per-version subsystem NQN (`SUBNQN`, already in `verity.json .subnqn`) is now
+    also baked into the signed UKI cmdline as **`rd.nvmf.nqn=`** (`extract-artifacts.sh`),
+    and both connect paths use **`nvme connect -n <nqn>`** — `nvmf-connect.sh` (test)
+    and a new explicit step 6 in `dot1x-macsec-up.sh` (secure, replacing the module
+    delegation; runs after macsec0 is addressed per invariant #2). Legacy UKIs
+    without `rd.nvmf.nqn` fall back to `connect-all` (with a warning) so they still
+    boot. Now each host = 1 controller on its own subsystem → versions drain to 0
+    when their hosts move off → auto-reap works; connections scale O(hosts) not
+    O(hosts × versions). Requires a rebuild + re-sign (cmdline change, invariant #6).
+    **(a) TEST PATH BENCH-VERIFIED 2026-07-03 (commit `0bfe872`): builds, deploys, and
+    the host mounts ONLY ONE image** (its own version's subsystem) instead of connect-
+    all'ing every served version — the targeted `nvme connect -n rd.nvmf.nqn` works.
+    So each host now = 1 controller on its own subsystem, which is what lets old
+    versions drain to 0 and `spdk-loader` auto-reap. **Still to confirm:** the reap
+    payoff itself (migrate a host, watch its old version drain to 0 and get reaped);
+    (b) SECURE PATH — the new step-6 connect brings up the NVMe root over macsec0
+    (this path is not yet bench-confirmed end-to-end); (c) that the dracut nvmf module
+    isn't ALSO connect-all-ing on the secure path (if it is, suppress it — the targeted
+    connect is meant to be the sole connector).
   - **BENCH FINDING 2026-07-03 (FIXED) — the target WAS still recreated on an
     `IMAGE_VERSION` bump.** Despite the above, the `nvmeof` (spdk-target) service
     still carried a versioned `type: image` bootstrap mount
@@ -1456,3 +1500,73 @@ connection to `:4420` established in the initramfs BEFORE firewalld starts. Veri
 default → should, but UNVERIFIED); a reconnect within `ctrl_loss_tmo` re-establishes
 *through* the firewall; DHCP renew + MKA keepalives unaffected; SSH in; no AVCs; verity
 hash reproducible. If it drops the root, that is the fleet-wide outage risk — gate hard.
+
+## [~] 18. Slim the image — locale + firmware (biggest erofs-size wins)
+
+**Finding (image-size audit via the 2b `rpm -qa --qf '%{SIZE}'` manifest, 2026-07-03):**
+the largest packages in the 534-package image are NOT the kiosk — they're base bloat:
+`linux-firmware` **678 MB**, `glibc-all-langpacks` **228 MB**, then chromium (370) +
+llvm-libs (129). The uncompressed erofs is read over NVMe/TCP on every boot of every
+host, so every dead byte is fleet-wide waste + attack surface.
+
+**[x] DONE — glibc langpacks → en-only** (`diskless.pkr.hcl` step 2). `@^minimal-environment`
+pulls `glibc-all-langpacks` (all languages); swapped for `glibc-minimal-langpack` +
+`glibc-langpack-en`. ~220 MB reclaimed; image is English-only (we also strip
+`/usr/share/locale` in 2d). **BENCH-VERIFY:** locale still `en_US.UTF-8`; kiosk + login OK.
+
+**[x] DONE — linux-firmware strip to a hardware-audited keep-list** (`diskless.pkr.hcl`
+step 2, run **BEFORE the dracut build** so the UKI initramfs shrinks too — not just the
+rootfs. `--no-hostonly` + plymouth's `drm` dep instmods a broad DRM set and dracut bundles
+each module's firmware into the initramfs, amdgpu being the classic UKI bloater; the strip
+runs first so dracut never sees it. A post-build diagnostic reports the `.img` size + any
+firmware still bundled — expected i915 only.)
+Live fleet audit (`modinfo -F firmware` over all loaded modules + sysfs/`lspci` driver
+bindings on the real 820-class hosts AND the VMs) proved the **only** loadable firmware
+any host requests is Intel **i915** (GPU). Every other driver is firmware-free —
+crucially **e1000e** (on-chip NVM, not a fw file) and **virtio_net**, so NO NIC needs
+firmware and the fleet-brick case is off the table. Keep-list: `i915` + `intel/` (ice
+DDP insurance for the `ice`/`mlx5` server-NIC build variant + small Intel blobs) +
+`mellanox` + `intel-ucode`; strip the rest (amdgpu/nvidia/radeon/ath/qcom/rtl/iwlwifi…
+~600 MB). Wi-Fi (`iwl*-firmware`, ~83 MB) removed as packages (wired-only fleet).
+Strip-in-place (`find … -exec rm -rf`, not `rpm -e` — can't drop the package without
+losing i915); a `du -sh /usr/lib/firmware` line reports the result in the build log.
+**BENCH-VERIFIED 2026-07-03 (commit `f59eb79`): built, booted, AND the kiosk SPA RENDERS
+on both the VM (virtio-gpu) and the 820 (i915).** This clears the firmware strip's one
+real risk — the i915 DMC/GuC blobs load on real hardware, so the keep-list
+(i915/intel/mellanox/intel-ucode, pre-dracut) is validated end-to-end (firmware 678→147 MB,
+initramfs bundles only the keep-list). Remaining caveat: if a future host has a non-Intel
+GPU or an `ice`/`mlx5` NIC, widen the keep-list before trusting fleet-wide. **NOT yet in
+`f59eb79`** (bench-verify separately — none touch i915, so render should be safe): the
+`nss-mdns`/avahi drop (`5cc72cc`) and the `omit_drivers` GPU-kmod trim + keep-list
+tightening — dropped `mellanox/`, narrowed `intel/` to `ice` only (`853ae20`). Audited
+models: HP-820-class (i915 + e1000e + iwlwifi-dropped) and virtio VMs.
+
+**Build-verified 2026-07-03:** firmware **678 MB → 147 MB**; the initramfs diagnostic
+confirms only the keep-list is bundled (i915 all-gens + intel/ice + intel/ish + mellanox)
+— proving the pre-dracut ordering works. But the initramfs came out **97 MB**, and it's
+now **kernel-MODULE-bound, not firmware** (~10 MB fw): the `--no-hostonly` + plymouth
+`drm` set pulls every GPU driver (`amdgpu.ko` etc.) even though the fleet is i915-only.
+
+**[x] DONE — follow-on trims (2026-07-03):**
+- **`omit_drivers+=" amdgpu radeon nouveau "`** in `90-diskless.conf` — drops the
+  discrete-GPU kmods plymouth's `drm` dep would instmod (amdgpu.ko is the biggest single
+  module); keeps i915 + virtio-gpu/bochs for the splash. Targets the 97 MB initramfs/UKI.
+- **Firmware keep-list tightened**: dropped `mellanox/` (those are `mlxsw_spectrum`
+  SWITCH-ASIC blobs, NOT NIC fw — mlx5 NIC fw is on-card) and narrowed `intel/` to just
+  `ice` (dropped ish/sof/ibt = sensor-hub/audio/bluetooth, none loaded at boot).
+- **Initramfs top-25-files diagnostic** added after the dracut build (sorts `lsinitrd.txt`
+  by size) so the next build shows what dominates the `.img` — measure before the next cut.
+- **BENCH-VERIFY:** kiosk still renders on the 820 (i915 kept); check the new
+  "initramfs largest files" log line for the post-omit size + next targets.
+
+**Combined:** ~**800 MB** off the uncompressed erofs (langpacks + firmware), dwarfing the
+kiosk weak-deps trim (#4), plus a smaller UKI/initramfs from the DRM-driver omit. NEXT
+candidates from the manifest (measure, don't guess): `iso-codes` (19 MB),
+`samba-client-libs`/`intel-mediasdk`/`gstreamer` (chromium-common HARD deps — only
+removable by dropping HTML5 media, and samba/rabbitmq/rist/srt/zmq are libavformat
+DT_NEEDED so they can't go without breaking future media — confirmed via ldd 2026-07-03),
+`binutils` (25 MB, build-only?), `cracklib-dicts` (9.8 MB, needed for pam pwquality — keep).
+
+**Stack note:** committed on `trim-rpm`. When `optimise-ram` rebases onto the new
+`trim-rpm`, expect a trivial `docs/TODO.md` tail conflict (both branches appended after
+#16 — `optimise-ram` added #17 fapolicyd, this added #18); resolve by keeping both.

files/dracut/95dot1x-macsec/dot1x-macsec-up.sh

@@ -5,8 +5,9 @@
 #   2. start wpa_supplicant (argv[0]=@wpa_supplicant => survives switch-root)
 #   3. wait for EAP success + macsec0
 #   4. DHCP on macsec0 (the pre-auth lease, if any, is invalid post-auth/VLAN)
-#   5. derive hostnqn/hostid from the cert so dracut's nvmf module can
+#   5. derive hostnqn/hostid from the cert
-#      connect the NVMe/TCP root namespace over macsec0
+#   6. TARGETED NVMe/TCP connect over macsec0 (rd.nvmf.nqn -> this version's
+#      subsystem only, not a discovery connect-all of every served version)
 # Idempotent: guarded by a stamp file because initqueue may re-run hooks.
 
 type getarg >/dev/null 2>&1 || . /lib/dracut-lib.sh
@@ -159,5 +160,44 @@ if [ ! -s /etc/nvme/hostnqn ]; then
     info "dot1x-macsec: hostnqn $(cat /etc/nvme/hostnqn)"
 fi
 
+# --- 6. connect the NVMe/TCP root over macsec0 (TARGETED to this version) ----
+# Explicit targeted connect. This hook used to only set /etc/nvme/hostnqn and
+# leave the connect to the dracut nvmf module's discovery mechanism — a
+# `connect-all` that attaches EVERY served version's subsystem to this host,
+# inflating target controllers to O(hosts x versions) and keeping old versions
+# pinned above zero connections so the per-version drain/reap gate never fires
+# (see TODO #6). rd.nvmf.nqn pins THIS build's subsystem (baked per-version into
+# the signed cmdline by extract-artifacts.sh, matching verity.json .subnqn on the
+# target); nvme uses the hostnqn/hostid written in step 5. Load-bearing order
+# (invariant #2): this runs AFTER macsec0 is addressed — never connect NVMe
+# before macsec0 has an IP. Keep-alive tuning matches the test path (-k 30 long
+# KATO, -l -1 retry forever, -c 10) so the root link survives kiosk CPU pressure
+# and blips. BENCH-VERIFY: the secure path is not yet confirmed end-to-end.
+_disc=$(getarg rd.nvmf.discover=)
+_nqn=$(getarg rd.nvmf.nqn=)
+if [ -n "$_disc" ] && [ -n "$_nqn" ] && [ ! -e /run/dot1x/.nvmf-connected ]; then
+    _trtype=${_disc%%,*}; _rest=${_disc#*,}
+    _traddr=${_rest%%,*}; _trsvcid=${_rest##*,}
+    modprobe nvme_tcp 2>/dev/null
+    ply nvme-active
+    i=0
+    while [ "$i" -lt "$TIMEOUT" ]; do
+        nvme connect -t "$_trtype" -a "$_traddr" -s "$_trsvcid" -n "$_nqn" \
+            -k 30 -l -1 -c 10 2>/dev/null
+        for _ns in /dev/nvme[0-9]*n[0-9]*; do
+            if [ -b "$_ns" ]; then
+                : > /run/dot1x/.nvmf-connected
+                ply nvme-done
+                info "dot1x-macsec: NVMe root connected over macsec0 ($_ns)"
+                break 2
+            fi
+        done
+        sleep 1; i=$((i + 1))
+    done
+    [ -e /run/dot1x/.nvmf-connected ] || warn "dot1x-macsec: NVMe/TCP connect to $_nqn timed out after ${TIMEOUT}s"
+elif [ -z "$_nqn" ]; then
+    warn "dot1x-macsec: rd.nvmf.nqn not set (legacy UKI) — leaving NVMe connect to the dracut nvmf module (connect-all)"
+fi
+
 : > "$STAMP"
 exit 0

files/dracut/95dot1x-macsec/nvmf-connect.sh

@@ -26,6 +26,15 @@ _rest=${_disc#*,}
 _traddr=${_rest%%,*}
 _trsvcid=${_rest##*,}
 
+# Targeted connect: rd.nvmf.nqn pins THIS build's subsystem (baked per-version
+# into the signed cmdline by extract-artifacts.sh, matching verity.json .subnqn
+# on the target). This replaces a discovery `connect-all`, which attached EVERY
+# served version's subsystem to every host — inflating target controllers and
+# keeping old versions pinned above zero connections so they never drained/reaped
+# (see TODO #6). If rd.nvmf.nqn is absent (a legacy UKI), fall back to connect-all
+# so the host still boots — with a warning.
+_nqn=$(getarg rd.nvmf.nqn=)
+
 # nvme_tcp is usually auto-loaded; ensure it so /dev/nvme-fabrics exists.
 modprobe nvme_tcp 2>/dev/null
 
@@ -50,11 +59,19 @@ _ctrl_loss_tmo=-1
 _reconnect_delay=10
 
 # Networking (ip=dhcp) is brought up in parallel, so poll-and-connect until a
-# fabrics namespace appears or we time out. connect-all is idempotent.
+# fabrics namespace appears or we time out. Both connect forms are idempotent
+# (re-connecting an already-attached subsystem just errors, harmlessly).
+[ -n "$_nqn" ] || echo "nvmf-connect: rd.nvmf.nqn not set — falling back to connect-all (legacy UKI; attaches every served version)" >&2
 _i=0
 while [ "$_i" -lt 60 ]; do
-    nvme connect-all -t "$_trtype" -a "$_traddr" -s "$_trsvcid" \
+    if [ -n "$_nqn" ]; then
-        -k "$_kato" -l "$_ctrl_loss_tmo" -c "$_reconnect_delay" 2>/dev/null
+        # TARGETED: attach only this build's subsystem (its data + hash namespaces).
+        nvme connect -t "$_trtype" -a "$_traddr" -s "$_trsvcid" -n "$_nqn" \
+            -k "$_kato" -l "$_ctrl_loss_tmo" -c "$_reconnect_delay" 2>/dev/null
+    else
+        nvme connect-all -t "$_trtype" -a "$_traddr" -s "$_trsvcid" \
+            -k "$_kato" -l "$_ctrl_loss_tmo" -c "$_reconnect_delay" 2>/dev/null
+    fi
     for _ns in /dev/nvme[0-9]*n[0-9]*; do
         if [ -b "$_ns" ]; then
             mkdir -p /run/dot1x

files/etc/dracut.conf.d/90-diskless.conf

@@ -25,5 +25,13 @@ add_dracutmodules+=" dot1x-macsec network-manager nvmf systemd-veritysetup "
 add_dracutmodules+=" plymouth "
 filesystems+=" erofs overlay "
 add_drivers+=" macsec nvme-tcp nvme-fabrics dm-verity @NIC_DRIVERS@ "
+# Omit the discrete-GPU DRM drivers plymouth's `drm` dep would otherwise instmod
+# in --no-hostonly mode. This fleet is Intel i915 ONLY (the build VM uses
+# virtio-gpu/bochs), so amdgpu/radeon/nouveau are dead weight — amdgpu.ko alone is
+# the single biggest module in the initramfs. i915 + virtio-gpu/bochs stay for the
+# splash. (Their FIRMWARE is already stripped pre-dracut in diskless.pkr.hcl; this
+# drops the MODULES too.) If a future host has a non-Intel GPU, remove it here.
+# See docs/TODO #18.
+omit_drivers+=" amdgpu radeon nouveau "
 install_items+=" /etc/dot1x/radius-ca.pem /etc/dot1x/wpa_supplicant-dot1x.conf "
 compress="zstd"

scripts/extract-artifacts.sh

@@ -73,9 +73,17 @@ ROOT_DATA_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_DATA_UUID}"
 ROOT_HASH_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_HASH_UUID}"
 # Per-version subsystem NQN (distinct NQN per release) — lets a host attach to
 # exactly one version's subsystem and gives clean per-release drain/allow-listing
-# on the long-lived target. Written to verity.json for the serving side; NOT in
+# on the long-lived target. Written to verity.json for the serving side AND baked
-# the cmdline (host still discovers + connects; the by-UUID verity refs select
+# into the UKI cmdline as rd.nvmf.nqn= (below), so the host does a TARGETED
-# what mounts). An explicit SUBNQN env wins.
+# `nvme connect -n <nqn>` instead of a discovery `connect-all`. connect-all
+# attached EVERY served version's subsystem to EVERY host, which inflated target
+# controllers to O(hosts x versions) and — worse — kept every old version pinned
+# above zero connections so the per-version drain/reap gate could never fire
+# (spdk-loader never reaps; see TODO #6). The NQN is version identity, not per-host
+# state, and the cmdline is already per-version + immutable-per-signed-UKI
+# (roothash, by-id UUIDs), so baking it does NOT violate invariant #1. It MUST
+# match verity.json .subnqn (same var here) so host and target agree. Explicit
+# SUBNQN env wins.
 SUBNQN_BASE=${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs}
 SUBNQN=${SUBNQN:-${SUBNQN_BASE}-${IMAGE_VERSION}}
 
@@ -226,6 +234,7 @@ printf '%s' \
   "rd.dot1x.nvindex=0x01800100 rd.dot1x.handle=0x81000100 rd.dot1x.timeout=60 " \
   "rd.fleet.enrol_url=${ENROL_URL} " \
   "rd.nvmf.discover=tcp,${NVMET_TRADDR},4420 " \
+  "rd.nvmf.nqn=${SUBNQN} " \
   "root=/dev/mapper/root rootfstype=erofs rootflags=ro " \
   "systemd.verity=yes roothash=${ROOT_HASH} " \
   "systemd.verity_root_data=${ROOT_DATA_DEV} " \