Merge remote-tracking branch 'origin/main' into trim-rpm

This commit is contained in:
Audiobuzz
2026-07-03 20:14:51 +09:30
6 changed files with 301 additions and 10 deletions
+190
View File
@@ -0,0 +1,190 @@
# .gitea/workflows/registry-gc.yaml
#
# Garbage-collect ORPHANED container images: dev/branch builds whose branch no
# longer exists. Reconciliation model (NOT event-driven) — it lists the live
# branches and every container version, then prunes the mismatch. So it catches
# branches deleted any way (UI, `git push --delete`, API) and never depends on a
# `delete` webhook the runner might not support or might miss.
#
# ┌─ SAFETY — what it will NEVER delete ───────────────────────────────────────┐
# │ * `latest`. │
# │ * ANY release / version tag — semver `1.2.3` / `v1.2.3`, spdk `v24.09`, the │
# │ dnsmasq `main-…` line, etc. The guard: only tags ending in a `-<7 hex>` │
# │ commit-sha suffix are ever CANDIDATES, and ONLY dev/branch builds get │
# │ that suffix (`<branch>-<sha7>`, spdk `<ver>-<branch>-<sha7>`). Release │
# │ tags never carry it, so they are structurally out of scope. │
# │ * Any image whose (sanitized) branch is still LIVE, and anything on `main`. │
# │ * Anything it cannot confidently classify (fail-safe = KEEP). │
# │ * Everything, if the live-branch list can't be fetched (aborts first). │
# └────────────────────────────────────────────────────────────────────────────┘
#
# DRY RUN by default. `workflow_dispatch` previews unless dispatched with
# dry_run=false. SCHEDULED runs also dry-run until you ARM them by setting the
# repo variable REGISTRY_GC_ENABLE=true — so you can eyeball a scheduled dry-run
# first, then flip the variable to let it prune automatically.
#
# TOKENS (two, by capability — a package token can't read branches, hence the
# original 403): branch reads use the automatic Actions token (github.token,
# repo read); package list + delete use PACKAGE_TOKEN (write:package, which the
# registry-push token already has). Only touches images THIS repo builds; the
# fleet-kiosk image is owned by the kiosk-app submodule's own CI.
name: registry-gc
run-name: prune orphan container images (dry_run default; ${{ github.event_name }})
on:
schedule:
- cron: "17 4 * * *" # nightly ~04:17 UTC
workflow_dispatch:
inputs:
dry_run:
description: "Preview only (true) or actually delete (false)"
type: boolean
default: true
permissions:
contents: read # branch listing via the auto Actions token
concurrency:
group: registry-gc
cancel-in-progress: false
jobs:
prune:
runs-on: ubuntu-latest
env:
# API base — hardcoded to match the registry host baked into the other
# workflows' image names (git.bitasylum.org/audiobuzz/*). Keep in sync.
SERVER: "https://git.bitasylum.org"
OWNER: ${{ github.repository_owner }}
REPO: ${{ github.repository }}
REPO_TOKEN: ${{ github.token }} # auto Actions token: repo/branch reads
PKG_TOKEN: ${{ secrets.PACKAGE_TOKEN }} # package list + delete (write:package)
EVENT: ${{ github.event_name }}
DISPATCH_DRY: ${{ inputs.dry_run }}
GC_ENABLE: ${{ vars.REGISTRY_GC_ENABLE }}
# Only images built by THIS repo's workflows.
IMAGES: "fleet-boot fleet-ipxe fleet-dnsmasq spdk-nvmf"
steps:
- name: Reconcile registry against live branches
# The runner defaults `run:` to sh (dash); this script uses bashisms
# (process substitution, arrays) — force bash. (Cf. gitea-main-ci.yaml.)
shell: bash
run: |
set -euo pipefail
command -v jq >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y -qq jq; }
API="${SERVER}/api/v1"
# Authenticated GET. $1 = which token (repo|pkg), $2 = url. Echoes the
# body on 2xx; on anything else prints an actionable error and exits
# (so a scope/auth problem can never be mistaken for "no results").
get() {
local who="$1" url="$2" tok tmp code body
case "$who" in repo) tok="$REPO_TOKEN";; pkg) tok="$PKG_TOKEN";; esac
tmp="$(mktemp)"
code="$(curl -sS --retry 3 -o "$tmp" -w '%{http_code}' \
-H "Authorization: token ${tok}" -H "Accept: application/json" "$url" || echo 000)"
body="$(cat "$tmp")"; rm -f "$tmp"
case "$code" in
2*) printf '%s' "$body"; return 0 ;;
401|403)
echo "::error::GET ${url} -> HTTP ${code} (token '${who}' lacks scope). branches need repo read (github.token); packages need read/write:package (PACKAGE_TOKEN)." >&2
exit 1 ;;
*)
echo "::error::GET ${url} -> HTTP ${code}" >&2
exit 1 ;;
esac
}
# ---- dry-run resolution ------------------------------------------
if [ "$EVENT" = "workflow_dispatch" ]; then
DRY="${DISPATCH_DRY:-true}"
elif [ "${GC_ENABLE:-}" = "true" ]; then
DRY="false"
else
DRY="true"
echo "::notice::scheduled run is DRY — set repo variable REGISTRY_GC_ENABLE=true to arm real deletion"
fi
echo "event=$EVENT dry_run=$DRY"
# Sanitize a branch name EXACTLY as the build workflows do, then
# lowercase (registries lowercase tags). MUST stay in sync with the
# `safe_branch` sed in {gitea-main-ci,ipxe-image,dnsmasq-image,spdk-image}.yaml.
sanitize() { printf '%s' "$1" | sed -e 's#[^A-Za-z0-9._-]#-#g' -e 's#^[.-]*##' | cut -c1-80 | tr 'A-Z' 'a-z'; }
# ---- 1. live branches (fail-safe: abort if none) -----------------
live=" main " # main is always protected, even if listing is odd
total=0 page=1
while :; do
resp="$(get repo "${API}/repos/${REPO}/branches?limit=50&page=${page}")"
cnt="$(printf '%s' "$resp" | jq 'length')"
[ "$cnt" -eq 0 ] && break
while IFS= read -r b; do
[ -n "$b" ] || continue
live="${live}$(sanitize "$b") "
total=$((total + 1))
done < <(printf '%s' "$resp" | jq -r '.[].name')
page=$((page + 1))
done
if [ "$total" -eq 0 ]; then
echo "::error::fetched 0 live branches — aborting so we never mass-delete on an API/auth failure"
exit 1
fi
echo "live branches ($total):${live}"
# ---- 2. reconcile each image -------------------------------------
orphans=0 kept=0 deleted=0 failed=0
for img in $IMAGES; do
page=1
while :; do
resp="$(get pkg "${API}/packages/${OWNER}?type=container&q=${img}&limit=50&page=${page}")"
cnt="$(printf '%s' "$resp" | jq 'length')"
[ "$cnt" -eq 0 ] && break
while IFS= read -r ver; do
[ -n "$ver" ] || continue
# PROTECT releases + latest: a tag WITHOUT the dev commit-sha
# suffix (`-<7 hex>`) is a release/version/`latest`/`main-ver`
# tag and is NEVER a candidate. This single rule keeps every real
# version tag and latest, exactly as required.
if [ "$ver" = "latest" ] || ! printf '%s' "$ver" | grep -Eq -- '-[0-9a-f]{7}$'; then
kept=$((kept + 1)); continue
fi
# It's a dev/commit tag. KEEP iff its branch component is a LIVE
# branch (anchored so `<branch>-<sha>` and spdk `<ver>-<branch>-<sha>`
# both match). Conservative: a suffix collision only ever causes a
# KEEP, never a wrongful delete.
is_live="false"
for lb in $live; do
lre="$(printf '%s' "$lb" | sed 's/\./\\./g')" # escape regex dots
if printf '%s' "$ver" | grep -Eq -- "(^|-)${lre}-[0-9a-f]{7}\$"; then
is_live="true"; break
fi
done
if [ "$is_live" = "true" ]; then kept=$((kept + 1)); continue; fi
# Orphan: dev tag whose branch is gone.
orphans=$((orphans + 1))
echo "ORPHAN ${img}:${ver}"
if [ "$DRY" != "true" ]; then
code="$(curl -sS -o /dev/null -w '%{http_code}' -X DELETE \
-H "Authorization: token ${PKG_TOKEN}" \
"${API}/packages/${OWNER}/container/${img}/${ver}" || echo 000)"
case "$code" in
2*) deleted=$((deleted + 1)); echo " deleted ${img}:${ver}" ;;
*) failed=$((failed + 1)); echo "::warning::delete ${img}:${ver} returned HTTP ${code}" ;;
esac
fi
done < <(printf '%s' "$resp" | jq -r --arg n "$img" '.[] | select(.name == $n) | .version')
page=$((page + 1))
done
done
echo "----------------------------------------------------------------"
echo "kept=${kept} orphans=${orphans} deleted=${deleted} failed=${failed} dry_run=${DRY}"
if [ "$DRY" = "true" ] && [ "$orphans" -gt 0 ]; then
echo "::notice::${orphans} orphan version(s) would be pruned. Dispatch with dry_run=false, or set REGISTRY_GC_ENABLE=true for scheduled runs, to actually delete."
fi
[ "$failed" -eq 0 ] || { echo "::error::${failed} deletion(s) failed"; exit 1; }
+6 -2
View File
@@ -115,9 +115,13 @@ instead of working around it.
`rd.dot1x.nvindex=` / `fleet-enrol -handle -nvindex`.)
1. **The hook ordering in `dot1x-macsec-up.sh` is load-bearing.**
NV-cert extraction → supplicant start → wait for `Authorized` + macsec0 →
DHCP **on macsec0** → hostnqn derivation. Never DHCP before auth
DHCP **on macsec0** → hostnqn derivation → **targeted NVMe/TCP connect**
(`nvme connect -n $rd.nvmf.nqn`, step 6). Never DHCP before auth
completes (pre-auth leases are invalid after VLAN assignment); never
connect NVMe before macsec0 is addressed.
connect NVMe before macsec0 is addressed. The connect is **targeted to this
version's subsystem** (`rd.nvmf.nqn`, baked per-version into the signed
cmdline) — never a discovery `connect-all`, which attaches every served
version and defeats the per-version drain/reap gate (see TODO #6).
1. **The supplicant must survive switch-root.** It is launched with
`exec -a @wpa_supplicant` (systemd root-storage-daemon convention) and
its state lives under `/run`. Do not “clean up” the `@`, move state off
+31
View File
@@ -688,6 +688,37 @@ must be built around **never restarting the process that holds live connections.
the current UKI), so auto-reap is safe; removal is still drain-gated +
grace-debounced. The loader + target share the RPC socket on a named volume
(`nvmf_tgt -r`); the loader reuses the SPDK image (has python3 + rpc.py).
- **BENCH FINDING 2026-07-03 (FIXED) — hosts did `connect-all`, so NO version
ever drained → auto-reap could never fire.** Observed: `spdk-release list`
showed more controllers than workstations, with every box appearing on *every*
served version's subsystem (e.g. 1/3/3/1/3 across 5 versions for ~3 boxes).
Root cause: the initramfs connected with **`nvme connect-all`** (test path
`nvmf-connect.sh`; secure path delegated to the dracut nvmf module's discovery
autoconnect), which attaches EVERY subsystem in the discovery log — the host
then mounts only its own by-UUID verity namespaces but leaves a live controller
on all the others. So each running host pinned every version above 0 conns, and
`spdk-loader`'s "auto-reap once drained (0 conns)" **could never trigger** while
the fleet was up — the served-version list grew without bound. This defeats the
whole drain/reap model, not just cosmetics. **Fix:** targeted connect. The
per-version subsystem NQN (`SUBNQN`, already in `verity.json .subnqn`) is now
also baked into the signed UKI cmdline as **`rd.nvmf.nqn=`** (`extract-artifacts.sh`),
and both connect paths use **`nvme connect -n <nqn>`** — `nvmf-connect.sh` (test)
and a new explicit step 6 in `dot1x-macsec-up.sh` (secure, replacing the module
delegation; runs after macsec0 is addressed per invariant #2). Legacy UKIs
without `rd.nvmf.nqn` fall back to `connect-all` (with a warning) so they still
boot. Now each host = 1 controller on its own subsystem → versions drain to 0
when their hosts move off → auto-reap works; connections scale O(hosts) not
O(hosts × versions). Requires a rebuild + re-sign (cmdline change, invariant #6).
**(a) TEST PATH BENCH-VERIFIED 2026-07-03 (commit `0bfe872`): builds, deploys, and
the host mounts ONLY ONE image** (its own version's subsystem) instead of connect-
all'ing every served version — the targeted `nvme connect -n rd.nvmf.nqn` works.
So each host now = 1 controller on its own subsystem, which is what lets old
versions drain to 0 and `spdk-loader` auto-reap. **Still to confirm:** the reap
payoff itself (migrate a host, watch its old version drain to 0 and get reaped);
(b) SECURE PATH — the new step-6 connect brings up the NVMe root over macsec0
(this path is not yet bench-confirmed end-to-end); (c) that the dracut nvmf module
isn't ALSO connect-all-ing on the secure path (if it is, suppress it — the targeted
connect is meant to be the sole connector).
- **BENCH FINDING 2026-07-03 (FIXED) — the target WAS still recreated on an
`IMAGE_VERSION` bump.** Despite the above, the `nvmeof` (spdk-target) service
still carried a versioned `type: image` bootstrap mount
+42 -2
View File
@@ -5,8 +5,9 @@
# 2. start wpa_supplicant (argv[0]=@wpa_supplicant => survives switch-root)
# 3. wait for EAP success + macsec0
# 4. DHCP on macsec0 (the pre-auth lease, if any, is invalid post-auth/VLAN)
# 5. derive hostnqn/hostid from the cert so dracut's nvmf module can
# connect the NVMe/TCP root namespace over macsec0
# 5. derive hostnqn/hostid from the cert
# 6. TARGETED NVMe/TCP connect over macsec0 (rd.nvmf.nqn -> this version's
# subsystem only, not a discovery connect-all of every served version)
# Idempotent: guarded by a stamp file because initqueue may re-run hooks.
type getarg >/dev/null 2>&1 || . /lib/dracut-lib.sh
@@ -159,5 +160,44 @@ if [ ! -s /etc/nvme/hostnqn ]; then
info "dot1x-macsec: hostnqn $(cat /etc/nvme/hostnqn)"
fi
# --- 6. connect the NVMe/TCP root over macsec0 (TARGETED to this version) ----
# Explicit targeted connect. This hook used to only set /etc/nvme/hostnqn and
# leave the connect to the dracut nvmf module's discovery mechanism — a
# `connect-all` that attaches EVERY served version's subsystem to this host,
# inflating target controllers to O(hosts x versions) and keeping old versions
# pinned above zero connections so the per-version drain/reap gate never fires
# (see TODO #6). rd.nvmf.nqn pins THIS build's subsystem (baked per-version into
# the signed cmdline by extract-artifacts.sh, matching verity.json .subnqn on the
# target); nvme uses the hostnqn/hostid written in step 5. Load-bearing order
# (invariant #2): this runs AFTER macsec0 is addressed — never connect NVMe
# before macsec0 has an IP. Keep-alive tuning matches the test path (-k 30 long
# KATO, -l -1 retry forever, -c 10) so the root link survives kiosk CPU pressure
# and blips. BENCH-VERIFY: the secure path is not yet confirmed end-to-end.
_disc=$(getarg rd.nvmf.discover=)
_nqn=$(getarg rd.nvmf.nqn=)
if [ -n "$_disc" ] && [ -n "$_nqn" ] && [ ! -e /run/dot1x/.nvmf-connected ]; then
_trtype=${_disc%%,*}; _rest=${_disc#*,}
_traddr=${_rest%%,*}; _trsvcid=${_rest##*,}
modprobe nvme_tcp 2>/dev/null
ply nvme-active
i=0
while [ "$i" -lt "$TIMEOUT" ]; do
nvme connect -t "$_trtype" -a "$_traddr" -s "$_trsvcid" -n "$_nqn" \
-k 30 -l -1 -c 10 2>/dev/null
for _ns in /dev/nvme[0-9]*n[0-9]*; do
if [ -b "$_ns" ]; then
: > /run/dot1x/.nvmf-connected
ply nvme-done
info "dot1x-macsec: NVMe root connected over macsec0 ($_ns)"
break 2
fi
done
sleep 1; i=$((i + 1))
done
[ -e /run/dot1x/.nvmf-connected ] || warn "dot1x-macsec: NVMe/TCP connect to $_nqn timed out after ${TIMEOUT}s"
elif [ -z "$_nqn" ]; then
warn "dot1x-macsec: rd.nvmf.nqn not set (legacy UKI) — leaving NVMe connect to the dracut nvmf module (connect-all)"
fi
: > "$STAMP"
exit 0
+20 -3
View File
@@ -26,6 +26,15 @@ _rest=${_disc#*,}
_traddr=${_rest%%,*}
_trsvcid=${_rest##*,}
# Targeted connect: rd.nvmf.nqn pins THIS build's subsystem (baked per-version
# into the signed cmdline by extract-artifacts.sh, matching verity.json .subnqn
# on the target). This replaces a discovery `connect-all`, which attached EVERY
# served version's subsystem to every host — inflating target controllers and
# keeping old versions pinned above zero connections so they never drained/reaped
# (see TODO #6). If rd.nvmf.nqn is absent (a legacy UKI), fall back to connect-all
# so the host still boots — with a warning.
_nqn=$(getarg rd.nvmf.nqn=)
# nvme_tcp is usually auto-loaded; ensure it so /dev/nvme-fabrics exists.
modprobe nvme_tcp 2>/dev/null
@@ -50,11 +59,19 @@ _ctrl_loss_tmo=-1
_reconnect_delay=10
# Networking (ip=dhcp) is brought up in parallel, so poll-and-connect until a
# fabrics namespace appears or we time out. connect-all is idempotent.
# fabrics namespace appears or we time out. Both connect forms are idempotent
# (re-connecting an already-attached subsystem just errors, harmlessly).
[ -n "$_nqn" ] || echo "nvmf-connect: rd.nvmf.nqn not set — falling back to connect-all (legacy UKI; attaches every served version)" >&2
_i=0
while [ "$_i" -lt 60 ]; do
nvme connect-all -t "$_trtype" -a "$_traddr" -s "$_trsvcid" \
-k "$_kato" -l "$_ctrl_loss_tmo" -c "$_reconnect_delay" 2>/dev/null
if [ -n "$_nqn" ]; then
# TARGETED: attach only this build's subsystem (its data + hash namespaces).
nvme connect -t "$_trtype" -a "$_traddr" -s "$_trsvcid" -n "$_nqn" \
-k "$_kato" -l "$_ctrl_loss_tmo" -c "$_reconnect_delay" 2>/dev/null
else
nvme connect-all -t "$_trtype" -a "$_traddr" -s "$_trsvcid" \
-k "$_kato" -l "$_ctrl_loss_tmo" -c "$_reconnect_delay" 2>/dev/null
fi
for _ns in /dev/nvme[0-9]*n[0-9]*; do
if [ -b "$_ns" ]; then
mkdir -p /run/dot1x
+12 -3
View File
@@ -73,9 +73,17 @@ ROOT_DATA_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_DATA_UUID}"
ROOT_HASH_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_HASH_UUID}"
# Per-version subsystem NQN (distinct NQN per release) — lets a host attach to
# exactly one version's subsystem and gives clean per-release drain/allow-listing
# on the long-lived target. Written to verity.json for the serving side; NOT in
# the cmdline (host still discovers + connects; the by-UUID verity refs select
# what mounts). An explicit SUBNQN env wins.
# on the long-lived target. Written to verity.json for the serving side AND baked
# into the UKI cmdline as rd.nvmf.nqn= (below), so the host does a TARGETED
# `nvme connect -n <nqn>` instead of a discovery `connect-all`. connect-all
# attached EVERY served version's subsystem to EVERY host, which inflated target
# controllers to O(hosts x versions) and — worse — kept every old version pinned
# above zero connections so the per-version drain/reap gate could never fire
# (spdk-loader never reaps; see TODO #6). The NQN is version identity, not per-host
# state, and the cmdline is already per-version + immutable-per-signed-UKI
# (roothash, by-id UUIDs), so baking it does NOT violate invariant #1. It MUST
# match verity.json .subnqn (same var here) so host and target agree. Explicit
# SUBNQN env wins.
SUBNQN_BASE=${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs}
SUBNQN=${SUBNQN:-${SUBNQN_BASE}-${IMAGE_VERSION}}
@@ -226,6 +234,7 @@ printf '%s' \
"rd.dot1x.nvindex=0x01800100 rd.dot1x.handle=0x81000100 rd.dot1x.timeout=60 " \
"rd.fleet.enrol_url=${ENROL_URL} " \
"rd.nvmf.discover=tcp,${NVMET_TRADDR},4420 " \
"rd.nvmf.nqn=${SUBNQN} " \
"root=/dev/mapper/root rootfstype=erofs rootflags=ro " \
"systemd.verity=yes roothash=${ROOT_HASH} " \
"systemd.verity_root_data=${ROOT_DATA_DEV} " \