diff --git a/.gitea/workflows/gitea-main-ci.yaml b/.gitea/workflows/gitea-main-ci.yaml index 76d2a62..21fc77d 100644 --- a/.gitea/workflows/gitea-main-ci.yaml +++ b/.gitea/workflows/gitea-main-ci.yaml @@ -135,6 +135,14 @@ jobs: [ -n "$safe_branch" ] || safe_branch="branch" version="${safe_branch}-${short_sha}" fi + # Lowercase the whole thing. image_version is BOTH the OCI tag AND the + # in-artifact UKI filename (fleet-.efi) + verity.json + subnqn. + # A registry may lowercase the pushed TAG while the artifact keeps the + # original case, so a mixed-case branch (e.g. SPDK-permanence) yields a + # capital fleet-SPDK-…​.efi the lowercase boot URL can't fetch (iPXE + # "no such file"). Forcing lowercase keeps tag, filename, verity.json, + # and the dnsmasq boot URL all in agreement. + version="$(printf '%s' "$version" | tr '[:upper:]' '[:lower:]')" echo "image_version=${version}" >> "$GITHUB_OUTPUT" echo "Building image_version=${version}" diff --git a/.gitmodules b/.gitmodules index 6a9e263..5515941 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "serving/kiosk-app"] path = serving/kiosk-app url = git@git.bitasylum.org:audiobuzz/fleet-kiosk-app.git + branch = restyle diff --git a/CLAUDE.md b/CLAUDE.md index dcf33bc..8f0c7a3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,12 +31,32 @@ files/etc/dracut.conf.d/90-diskless.conf no-hostonly + drivers + modules files/plymouth/themes/fleet/ Custom boot splash (scripted security checklist; driven by hook status tokens) files/kiosk/ weston+Chromium kiosk + chromium lockdown policy + (policy allows PNA to the host-info agent) +files/hostapi/ Go host-info agent (fleet-hostapi): read-only, + loopback :9099 /v1/host for the kiosk SPA + (HOST-API.md contract). Built in 2b-hostapi, + fapolicyd-trusted, golang removed in 2d. scripts/extract-artifacts.sh qcow2 -> kernel/initramfs/erofs+verity/cmdline scripts/build-oci.sh ukify + sbsign + roothash gate + OCI image scripts/build-ipxe-oci.sh sbsign snponly.efi -> FROM-scratch fleet-ipxe image serving/ Lab serving stack: dnsmasq + nginx + SPDK (dnsmasq + fleet-ipxe + SPDK images are CI-built + pulled; see .gitea/workflows/) +serving/spdk/ LONG-LIVED nvmf_tgt: spdk-target.sh + (persistent entrypoint, never restarts + for a version change) + spdk-release.sh + (online add/remove/drain a version, 1 + subsystem/version) + spdk-ctl.sh (host + load/unload/list for manual rolling + updates) + spdk-loader.sh (DECLARATIVE: + compose service that adds ${IMAGE_VERSION} + online then auto-reaps drained old + versions — bump tag + compose up -d). + releases// = staged rootfs+verity.json. + spdk-release parses RPC JSON via python3 + (SPDK image has no jq). spdk-setup.sh + (parent dir) is the legacy single-version + entrypoint (deprecated). serving/kiosk-app/ Svelte+Tailwind spaceship-terminal SPA — GIT SUBMODULE (own repo + CI; builds the fleet-kiosk image the stack pulls). diff --git a/diskless.pkr.hcl b/diskless.pkr.hcl index 6149d90..983441c 100644 --- a/diskless.pkr.hcl +++ b/diskless.pkr.hcl @@ -233,6 +233,13 @@ build { destination = "/tmp/" } + # Workstation host-info API (Go). Source staged to /tmp/hostapi; compiled + + # installed in the 2b-hostapi step. The kiosk SPA reads it at localhost:9099. + provisioner "file" { + source = "files/hostapi" + destination = "/tmp/" + } + # Custom 'fleet' Plymouth theme (scripted security-handshake splash). Staged to # /tmp/plymouth, installed + set default in the package step before dracut runs. provisioner "file" { @@ -343,6 +350,16 @@ build { "truncate -s 0 /etc/hostname", "install -D -m0644 /tmp/etc-stage/NetworkManager/conf.d/20-hostname.conf /etc/NetworkManager/conf.d/20-hostname.conf", "restorecon /etc/NetworkManager/conf.d/20-hostname.conf", + # Subscription-manager has no role on a booted, hostname-neutral, immutable + # appliance — entitlements only matter at BUILD time (dnf). Left running, + # rhsmcertd wakes periodically and tries to write Python __pycache__ into a + # bin_t dir on the read-only/overlay root: SELinux denies it (enforcing), so + # it's a benign but RECURRING AVC that floods the SIEM (fleet_audit rule + # 100120) and can never succeed (tmpfs overlay is wiped each boot). Mask the + # daemon + timer to kill the noise and trim attack surface. (Don't waive the + # AVC in Wazuh — that would also hide future rhsmcertd_t denials.) + "systemctl mask rhsmcertd.service 2>/dev/null || true", + "systemctl disable rhsmcertd.service 2>/dev/null || true", # Read-only erofs/verity root; runtime state on a tmpfs overlay provided # by systemd.volatile=overlay (set in the cmdline by extract-artifacts.sh). # RHEL/Rocky dracut has no 'overlayfs'/'rootovl' module. @@ -408,8 +425,13 @@ build { # --use-pixman + chromium --disable-gpu — candidate to drop once the kiosk # is display-verified; see TODO #4.) "dnf install -y --setopt=tsflags=nodocs weston chromium mesa-dri-drivers mesa-libEGL mesa-libgbm dejavu-sans-fonts", - # Unprivileged, passwordless kiosk user; video/input as a seat fallback. + # Unprivileged, passwordless kiosk user. Groups: 'video' (KMS /dev/dri/card0 + # for weston) + 'input' (libinput) as a seat fallback; 'render' + # (/dev/dri/renderD128) so Chromium's GPU process can do hardware raster on + # real hardware (kiosk-session enables GPU when a real KMS driver is present; + # render nodes aren't logind-seat-ACL'd on EL9, so group membership is needed). "id kiosk >/dev/null 2>&1 || useradd -m -G video,input kiosk", + "getent group render >/dev/null 2>&1 && usermod -aG render kiosk || true", # Install the staged unit, weston config, and session wrapper. "install -D -m0755 /tmp/kiosk/kiosk-session /usr/local/bin/kiosk-session", "install -D -m0644 /tmp/kiosk/kiosk.service /etc/systemd/system/kiosk.service", @@ -442,6 +464,44 @@ build { ] } + # --- 2b-hostapi. Workstation host-info API (Go, loopback, read-only) ------ + # Compiles files/hostapi into /usr/local/bin/fleet-hostapi and enables it as a + # systemd service the kiosk SPA reads at http://localhost:9099/v1/host (the + # HOST-API.md contract). Build-time, image-GENERIC facts (image version + + # weston/chromium versions) are written to /etc/fleet-hostinfo.json; everything + # host-specific (kernel, hostname, IP, MAC, board, uptime) is read at RUNTIME, + # so NO per-host state lands in the image (invariant #1). The Go toolchain is + # installed here and removed in 2d hygiene; it never ships. + provisioner "shell" { + environment_vars = [ + "IMAGE_VERSION=${var.image_version}", + ] + inline = [ + "set -euxo pipefail", + "dnf -y install golang", + # Reproducible, dependency-free static build (the verity hash must be a pure + # function of the inputs — invariant #8). GOPROXY=off + GOTOOLCHAIN=local + # guarantee no network/toolchain fetch; -trimpath + -buildid= strip paths + # and the build id. Stdlib-only, so this needs no modules. NOTE: a golang + # package update changes the binary (hence the hash) — a reproducibility- + # sensitive input, like the STIG semanage store; build twice to confirm. + "install -d -m0755 /tmp/gocache", + "env -C /tmp/hostapi CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOPROXY=off GOTOOLCHAIN=local GOFLAGS=-mod=mod GOCACHE=/tmp/gocache GOPATH=/tmp/gopath go build -trimpath -buildvcs=false -ldflags '-s -w -buildid=' -o /usr/local/bin/fleet-hostapi .", + "chmod 0755 /usr/local/bin/fleet-hostapi", + "install -D -m0644 /tmp/hostapi/fleet-hostapi.service /etc/systemd/system/fleet-hostapi.service", + # Build-time facts file (image-generic; same for every host of a release). + # rpm '%%{...}' is doubled so Packer's HCL2 doesn't parse %{ as a directive. + "WESTON_VER=$(rpm -q --qf '%%{VERSION}' weston 2>/dev/null || echo '')", + "CHROMIUM_VER=$(rpm -q --qf '%%{VERSION}' chromium 2>/dev/null || rpm -q --qf '%%{VERSION}' chromium-browser 2>/dev/null || echo '')", + "printf '{\"image\":\"%s\",\"weston\":\"%s\",\"chromium\":\"%s\"}\\n' \"$IMAGE_VERSION\" \"$WESTON_VER\" \"$CHROMIUM_VER\" > /etc/fleet-hostinfo.json", + "chmod 0644 /etc/fleet-hostinfo.json", + "restorecon -R /usr/local/bin/fleet-hostapi /etc/systemd/system/fleet-hostapi.service /etc/fleet-hostinfo.json 2>/dev/null || true", + "systemctl enable fleet-hostapi.service", + "rm -rf /tmp/hostapi /tmp/gocache /tmp/gopath", + "echo '==> fleet-hostapi enabled (localhost:9099/v1/host)'", + ] + } + # --- 2c. Admin access: sudo-enabled key user + locked root -------------- # Fleet-generic (same user + key on every host => NOT per-host state). The # key comes from var.ssh_authorized_key (CI secret). If no key is supplied we @@ -691,9 +751,9 @@ build { # custom kiosk files: add them to fapolicyd's trust DB and rebuild it. weston # /chromium are RPM-installed so already trusted. (See docs/stig-deviations.md.) "if command -v fapolicyd-cli >/dev/null 2>&1; then", - " for f in /usr/local/bin/kiosk-session /usr/local/bin/fleet-sb-enroll; do { [ -f \"$f\" ] && fapolicyd-cli --file add \"$f\"; } || true; done", + " for f in /usr/local/bin/kiosk-session /usr/local/bin/fleet-sb-enroll /usr/local/bin/fleet-hostapi; do { [ -f \"$f\" ] && fapolicyd-cli --file add \"$f\"; } || true; done", " fapolicyd-cli --update 2>/dev/null || true", - " echo '==> fapolicyd: trusted custom binaries (kiosk-session, fleet-sb-enroll) + rebuilt trust db'", + " echo '==> fapolicyd: trusted custom binaries (kiosk-session, fleet-sb-enroll, fleet-hostapi) + rebuilt trust db'", "fi", ] } @@ -766,6 +826,9 @@ build { # already downloaded to the host) — slims the erofs + cuts attack surface. # Keep auditd (it's part of the hardened runtime). Best-effort. "dnf remove -y openscap-scanner scap-security-guide openscap-utils 2>/dev/null || true", + # The Go toolchain was only needed to compile fleet-hostapi (2b-hostapi); + # the static binary ships, the compiler does not. + "dnf remove -y golang 2>/dev/null || true", # The build-time STIG reports must not ship inside the read-only rootfs. "rm -f /root/stig-report.html /root/stig-arf.xml", "dnf clean all", @@ -792,7 +855,7 @@ build { # home dirs) — we dropped mkfs.erofs --all-root, which used to mask this by # flattening everything to root:root. sudo is a no-op if already root, and # the runner has passwordless sudo (cf. the `sudo apt-get` install step). - "sudo bash -c \"NVMET_TRADDR='${var.nvmet_traddr}' BOOT_SPLASH='${var.boot_splash}' STIG='${var.enable_stig}' bash scripts/extract-artifacts.sh output-qemu/diskless-build.qcow2 ${var.artifact_dir}/${var.image_version} ${var.disable_dot1x}\"", + "sudo bash -c \"IMAGE_VERSION='${var.image_version}' NVMET_TRADDR='${var.nvmet_traddr}' BOOT_SPLASH='${var.boot_splash}' STIG='${var.enable_stig}' bash scripts/extract-artifacts.sh output-qemu/diskless-build.qcow2 ${var.artifact_dir}/${var.image_version} ${var.disable_dot1x}\"", # Hand the OUTPUT FILES back to the build user (their in-erofs ownership is # already baked correctly) so build-oci.sh can write the UKI and push with # the build user's `buildah login`. diff --git a/docs/TODO.md b/docs/TODO.md index 2233906..922c6bc 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -645,15 +645,69 @@ must be built around **never restarting the process that holds live connections. - **Planned SPDK restarts only via drain/failover** (multipath fail-over, or a maintenance window with generous `ctrl_loss_tmo` so hosts recover not die). There is **no zero-impact single-node SPDK restart.** -- **Implementation gap (current lab):** SPDK runs as a container with - `spdk-setup.sh` as entrypoint + the artifact image mounted → effectively - **single-version, configured at container start**, so changing versions = - container restart = drop all connections. The work here is a **long-lived target - + an RPC-driven control path** ("add release / remove release / drain status") - decoupled from container lifecycle, with per-version subsystems. +- **[x] Long-lived target + online release control — IMPLEMENTED 2026-06-27.** + The old gap (SPDK ran `spdk-setup.sh` as entrypoint, single-version, + configured-at-start → changing versions = container restart = drop ALL + connections) is closed. Now in `serving/spdk/`: + - **`spdk-target.sh`** — persistent entrypoint: starts `nvmf_tgt` ONCE + (release-agnostic), creates the TCP transport + discovery listener, then + reconciles every staged release. Never restarted to change versions; on + (re)start re-adds everything under `./releases/` (a serving-host reboot + restores all served versions). + - **`spdk-release.sh`** — JSON-RPC control: `add`/`remove`/`list`/ + `drain-status`/`reconcile`, **one subsystem per version** (distinct NQN, so + adding/removing one never disturbs others' connections), **drain-gated + removal** (refuses while a version has connected controllers unless + `--force`). + - **`spdk-ctl.sh`** — host ops: `load`/`unload`/`list`; stages a release's + artifacts from its OCI image into `./releases//` and drives the live + target. Rolling update = `load` new → reboot hosts → `unload` old once + drained. + - **`spdk-loader.sh`** — DECLARATIVE rolling updates (the "init container that + stays up", compose service `spdk-loader`). On `compose up` after an + `IMAGE_VERSION` bump it stages + adds the new rootfs to the running target + ONLINE, then polls and **auto-reaps** any non-current version once it has + drained (0 conns) for a sustained grace window (`REAP_GRACE_SECONDS`, keep > + host `ctrl_loss_tmo`). So migration = bump `IMAGE_VERSION` in `.env` + + `docker compose up -d` (compose recreates the loader + the http/dnsmasq UKI + servers, but NOT the nvmeof target → no dropped connections). Single served + UKI ⇒ a 0-conn old version is genuinely orphaned (a returning host fetches + the current UKI), so auto-reap is safe; removal is still drain-gated + + grace-debounced. The loader + target share the RPC socket on a named volume + (`nvmf_tgt -r`); the loader reuses the SPDK image (has python3 + rpc.py). + - **No jq dependency:** `spdk-release.sh` parses RPC JSON with **python3** (the + SPDK image ships no jq); legacy artifacts (no per-version UUIDs in + verity.json) fall back to the fixed UUID pair with a warning. + - **Per-version namespace UUIDs** (`scripts/extract-artifacts.sh`): derived + deterministically (UUIDv5 of `image_version`), baked into the UKI cmdline + + `verity.json` (now also carries `subnqn`); the serving side reads them back + from each release's `verity.json`. Required so two versions don't collide on + the same `/dev/disk/by-id/nvme-uuid.` host symlink. + - **Bootstrap unchanged:** `docker compose up` with one version still works — + the `/artifacts` image volume is seeded into `./releases/${IMAGE_VERSION}`. + - **PARTIALLY BENCH-VERIFIED 2026-06-28:** long-lived target boots a host end + to end — UKI served over HTTP → host boots → NVMe/TCP root connects to the + persistent `nvmf_tgt` (single version `spdk-permanence-02ffd7d`). Surfaced + + fixed several real bugs along the way: bootstrap seed must COPY not symlink + (a symlink into the version-pinned `/artifacts` re-pointed a stale release at + new data); `spdk-release` parses RPC JSON via **python3** (SPDK image has no + jq); legacy `verity.json` (no per-version UUIDs) falls back to the fixed pair; + mislabeled-dir guard (dir name vs verity.json `image_version`, case-insensitive); + dropped the redundant discovery-listener re-add (daemon logged "Listener + already exists"). **CI lowercases `image_version`** (gitea-main-ci vars step) — + a mixed-case branch (`SPDK-permanence`) had baked a capital + `fleet-SPDK-…​.efi` the lowercase boot URL couldn't fetch (iPXE "no such + file"); booted via a local retag workaround pending a lowercase rebuild. + - **STILL BENCH-VERIFY:** (a) `load` a 2nd version while a host is connected to + the 1st → 1st's connection survives (no AER/reset on the running host); (b) + `unload`/auto-reap drain-gate blocks while CONNS>0 and reaps after grace on a + LIVE host; (c) serving-host reboot reconciles all staged versions; (d) + per-version UUID build reproducible (two builds, same version → identical + `data_ns_uuid`/`hash_ns_uuid` + `root_hash`); (e) the full fix on a clean + LOWERCASE CI rebuild (no retag). See `serving/spdk/README.md`. - **Capacity:** each concurrently-served version costs target memory/hugepages + file-backed bdev overhead — plan for N live versions (current + previous + - canary) during long rollouts. + canary) during long rollouts (raise `HUGEPAGES`). - **Attach gating + discovery:** with several subsystems advertised, the dracut connect + per-subsystem hostNQN allow-list must ensure a vX host attaches **only** vX's namespace (the by-UUID verity ref protects *what mounts*, but gate the @@ -824,6 +878,10 @@ smoke test. `docs/stig-deviations.md` #6 to reflect encrypted offload. (RELP+TLS via `omrelp`/`imrelp` is the alternative if guaranteed delivery is wanted alongside encryption.) +- **fleet-hostapi: scope CORS to the kiosk origin.** The host-info agent ships + with `FLEET_HOSTAPI_ALLOW_ORIGIN=*` (lab). For prod set it to the exact kiosk + origin (default `http://boot.fleet.lab`) via the unit's commented + `Environment=` line (or a drop-in). See #12. ## [ ] 9. (OPTIONAL / nice-to-have) Convert provisioning to Ansible where it pays off @@ -984,3 +1042,104 @@ non-destructive TPM capability check (device, key persist, NV roundtrip, openssl TPM2 provider, CSR signed by the TPM key) to de-risk the TPM half before any of this. Expected gap: the image installs `tpm2-tss` but no openssl TPM2 **provider** (`tpm2-openssl`) — add it to the build once the self-test confirms. + +## [~] 12. Workstation host-info API (`fleet-hostapi`) for the kiosk + +**IMPLEMENTED 2026-06-28.** A small **Go, read-only, loopback-only** HTTP agent on +the workstation that reports host/OS/hardware facts to the kiosk SPA, per the +[HOST-API.md](../HOST-API.md) contract (`GET http://localhost:9099/v1/host`). +Source: `files/hostapi/` (stdlib-only, reproducible static build). Built + enabled +by provisioner **2b-hostapi**; toolchain removed in 2d; binary fapolicyd-trusted +in 2c-STIG-postfix (register #10). Chromium policy grants Private Network Access +scoped to the kiosk origin (`files/kiosk/chromium-kiosk-policy.json`). + +- **Per-host safety:** host-specific fields (kernel, hostname, IP, MAC, board, + uptime) are read at RUNTIME; only image-generic facts (image version + + weston/chromium versions) are baked, to `/etc/fleet-hostinfo.json` (invariant + #1 preserved). Deliberately does NOT expose DMI serial (root-only/per-host) or + any TPM/cert/identity material. +- **Hardening shipped:** loopback bind enforced in code; `DynamicUser`, no caps, + `ProtectSystem=strict`, `MemoryDenyWriteExecute`, `SystemCallFilter=@system-service`, + `IPAddressDeny=any` (loopback only). Runs unprivileged, reads world-readable + files only. + +**Lab posture LEFT AS-IS (decided 2026-06-28) — revisit for prod:** +- **CORS `Allow-Origin=*`** → scope to the kiosk origin (`http://boot.fleet.lab`). + Consolidated into #8. +- **SELinux:** runs in the default service domain (no custom policy module). + **BENCH-VERIFY** no AVC denials for `fleet-hostapi` on an enforcing booted host + (Wazuh/`fleet_audit` rule 100120 will surface any). If denials appear, decide: + write a confined SELinux policy module vs adjust the sandbox. +- **No auth** (justified: loopback + read-only + single-user appliance). Option if + the threat model tightens: shared token in `kiosk.env` + header check. +- **Info sensitivity:** exposes hostname/IP/MAC/board/versions to the kiosk. Drop + MAC/IP if that's later deemed too much. +- **Reproducibility:** the `golang` toolchain version is a reproducibility-sensitive + input (a package bump changes the binary → verity hash); covered by the + build-twice gate. Pin the go version only if drift becomes a problem. + +**BENCH-VERIFY (functional):** (a) service starts under enforcing SELinux + +fapolicyd (not blocked); (b) the kiosk SPA's System page fetches `/v1/host` +successfully through the PNA allowance (preflight + GET); (c) facts render +correctly on real hardware (board/CPU/GPU from DMI/DRM, not the virtio build VM). + +## [~] 13. Kiosk GPU compositing + Chromium flag ownership + +**GPU FIX APPLIED 2026-06-28.** The 820 (Intel HD620/i915) ran Chromium at ~150% +CPU with periodic crashes: `--disable-gpu` in our `kiosk-session` forced software +raster/compositing. Root cause confirmed from the live cmdline. Fixes in +`files/kiosk/kiosk-session` + `diskless.pkr.hcl`: +- **Runtime GPU probe** (fleet-generic image boots real HW *and* virtio VMs): a + real KMS driver (`i915`/`amdgpu`/…) + `/dev/dri/renderD128` → GPU path + (`--ignore-gpu-blocklist --enable-gpu-rasterization --enable-zero-copy`, weston + GL renderer); else software (`--disable-gpu` + `weston --use-pixman`) so the VMs + still render. No per-host state baked. +- **kiosk user added to `render`** (renderD128 isn't logind-seat-ACL'd on EL9; + `video` only covers card0/KMS for weston). +- **Merged the distro wrapper's `--disable-features` into ours** — Chromium takes + the LAST `--disable-features` (not the union), so ours was silently dropping the + wrapper's Lens/Gemini/Compose disables. + +**BENCH-VERIFY (on the 820):** GPU process alive (`chrome://gpu`), Compositor/ +ThreadPool threads near-idle + CPU drop, `gt_act_freq_mhz` rises under load, no +SELinux AVC for `dri_device_t` from the kiosk domain (Wazuh `fleet_audit` 100120 +will surface any — if present, add a scoped SELinux allow or confirm the kiosk +user's SELinux mapping is unconfined). Confirm VMs still render (software path). + +**DECIDED 2026-06-28 — Option 1 (policy lock), APPLIED.** The distro chromium +wrapper (`/usr/bin/chromium-browser`) layers flags as `$CHROMIUM_FLAGS` +(`/etc/chromium/chromium.conf` → VA-API enables + AI disables; overridable via +`CHROMIUM_USER_FLAGS`) + **hardcoded** `$CHROMIUM_DISTRO_FLAGS` +(`--enable-plugins --enable-extensions --enable-user-scripts --enable-printing +--enable-sync --auto-ssl-client-auth`) + our `"$@"` (last). The hardcoded layer +isn't removable via env/conf — only by bypassing the wrapper. Rather than bypass +(Option 2, which would force us to replicate the wrapper's stdio sanitisation + +own the VA-API feature list across updates), we **neutralise in managed policy** +(policy beats cmdline flags, survives updates): `ExtensionInstallBlocklist=["*"]` +added to `files/kiosk/chromium-kiosk-policy.json` (blocks all extensions → also +kills the user-scripts surface). `--enable-sync` already locked by `SyncDisabled`; +`--auto-ssl-client-auth` is moot (no browser-store client cert; the TPM 802.1X +cert isn't one) — `AutoSelectCertificateForUrls` left unset. VA-API enables +inherited from `chromium.conf` unchanged. Option 2 (full flag ownership, bypass +the wrapper, own the VA-API `--enable-features`: +`AllowQt,WaylandLinuxDrmSyncobj,WaylandPerSurfaceScale,WaylandUiScale,AcceleratedVideoDecodeLinuxGL,AcceleratedVideoDecodeLinuxZeroCopyGL,AcceleratedVideoEncoder`) +remains documented here if a minimal/auditable cmdline is ever wanted. +**BENCH-VERIFY:** `chrome://policy` shows `ExtensionInstallBlocklist=["*"]` active; +extensions can't install. + +## [ ] 14. (DESIGN) Centralised fleet management plane (health/version/usage/maintenance) + +**DESIGN CAPTURED 2026-06-28 — [docs/fleet-management-design.md](fleet-management-design.md).** +Server-authoritative, pull-based, TPM-identified management plane reusing existing +pieces: `fleet-hostapi` as the on-host data source, the audit-collector→Wazuh +pipeline as telemetry transport, the digest-pinned deployment manifest as version +truth, and the SPDK `spdk-loader`/`spdk-release` rolling-update flow (#6) as the +redeploy/drain mechanics. Hosts stay stateless (identity from the TPM cert, no +baked enrollment); control is pull/reconcile (no inbound surface); convergence to +a new version = reboot. **Phased build:** (1) push hostapi snapshots as structured +events into Wazuh + fleet dashboard (health/version-drift/offline/usage/alerting); +(2) metrics push (OTLP/remote-write) → VictoriaMetrics + Grafana; (3) mTLS +desired-state pull (version pin, reboot/drain, kiosk URL) wired to the deployment +manifest + SPDK flow. Build-time decisions (telemetry encoding, metrics/desired- +state store, per-host vs cohort, osquery/FleetDM eval) listed in the design note. +Depends on the step-ca fleet CA (#8) for the mTLS identity. diff --git a/docs/fleet-management-design.md b/docs/fleet-management-design.md new file mode 100644 index 0000000..f115dcf --- /dev/null +++ b/docs/fleet-management-design.md @@ -0,0 +1,193 @@ +# Fleet management plane — design note + +**Status:** DESIGN (not built). Captured 2026-06-28. Build phased — see "Phasing" +and `docs/TODO.md` #14. + +Centralised monitoring + maintenance of the diskless kiosk fleet: **health, +version, usage, and maintenance actions**. This note is the authoritative design; +it deliberately reuses what already exists rather than bolting on a generic MDM, +because the base architecture (immutable UKI + NVMe-root, TPM identity, no +per-host state, segmented network) makes most off-the-shelf kiosk-management tools +a poor fit. + +## Design philosophy (forced by the architecture invariants) + +These are consequences of the CLAUDE.md invariants, not preferences: + +1. **Server-authoritative, hosts are cattle.** The root is immutable erofs + + dm-verity on a tmpfs overlay — nothing survives reboot. So **all durable state + (history, inventory, desired-state) lives server-side**; the on-host agent is + stateless and buffers only in RAM (best-effort). +2. **Identity from the TPM, never baked.** No per-host state in any artifact + (invariant #1). The agent derives identity at runtime from the **TPM-held + cert** (key `0x81010001`, cert NV `0x01800001`) — the same identity used for + 802.1X EAP-TLS and the NVMe hostnqn. No enrollment token in the image. +3. **Pull-based control, no inbound surface.** The boot LAN is locked down and + segmented; we do not open an always-on inbound control port on hosts. Hosts + **poll** a desired-state endpoint and reconcile. This also fits the immutable + + digest-pinned model: *convergence to a new version = reboot*. +4. **Reuse the existing buses.** Telemetry rides the already-built + `auditd → audit-collector relay → Wazuh` pipeline; version truth is the + existing digest-pinned deployment manifest; redeploy/drain mechanics are the + SPDK rolling-update flow (TODO #6). The management plane orchestrates these; it + does not reinvent them. +5. **Minimal, reproducible, sandboxed agent.** Same posture as `fleet-hostapi` + (loopback/least-priv, `DynamicUser`, fapolicyd-trusted, reproducible static + build). + +## Components + +### 1. Identity — mTLS with the TPM cert + +The agent presents the TPM-held client cert over **mTLS** to the management +endpoint; the server authenticates the host by cert CN/SAN, which is the canonical +fleet identity (= NVMe hostnqn = 802.1X identity). Reuses the **step-ca fleet CA** +(`serving/dot1x-bench/`). No baked secret; the host also pins the server cert. + +- **Secure path:** full mTLS, TPM cert. Strong, no secrets in the image. +- **Open/test path (`rd.dot1x=0`):** fall back to dnsmasq fleet-name + MAC as a + weak identity. **Lab only** — flagged as insufficient for production. + +Inventory is keyed by **cert serial / hostnqn**, joined to the dnsmasq starship +name (`fleet-namer`), MAC, last-seen, running/desired version, and health. + +### 2. On-host agent — extend `fleet-hostapi` + +`fleet-hostapi` (`files/hostapi/`) is already the only fleet-authored component on +the hardware and already models os/hardware/net/kiosk facts. Add a **report/push +mode** (or a tiny sibling that reads its loopback `/v1/host` and ships it). Keep it +stateless; buffer in tmpfs only; identity from the TPM. + +Reported facts: + +- **Health:** CPU/load, GPU act-freq, temps, **tmpfs overlay usage** (the + RAM-exhaustion → NVMe-keepalive risk — the single most important kiosk health + signal), NVMe-root link state, 802.1X/MACsec `Authorized`, SELinux enforcing, + chrony sync, `kiosk.service` active. +- **Version:** running image version (`/etc/fleet-image`), rootfs digest, kernel. +- **Usage:** kiosk URL, session uptime, **Chromium reload/crash count** (from the + `kiosk-session` restart loop — the exact signal behind the GPU-bug diagnosis), + input idle/active (liveness only, never content — privacy-aware), display power + state. + +Cadence: heartbeat (~30–60 s) + event-driven (crash, auth loss, version change). + +### 3. Transport + +**Phase 1 — ride the Wazuh pipeline.** Emit snapshots as structured JSON events +through the existing `audit-collector` relay → `wazuh1`. Add `fleet_health` / +`fleet_kiosk` decoders + rules alongside `fleet_audit` (`serving/siem/config/`). +Wazuh then provides search, dashboards, and alerting for free, and it's already +secured-path-aware. + +**Phase 2 — metrics push** for high-resolution time-series (CPU/GPU/mem trends): +**push** via OTLP or Prometheus remote-write to a collector on the mgmt host → +VictoriaMetrics/Prometheus. **Push, never scrape** — scraping would require an +inbound port on every locked host, against the posture. + +### 4. Control / maintenance — pull-based desired-state reconciliation + +Each host periodically fetches **its** desired state (mTLS, keyed by TPM cert): +target version, kiosk URL, `reboot-at`, `drain`. The agent reconciles; because the +root is immutable, **convergence = reboot** (a reboot re-PXEs and picks up the new +UKI). + +- The **deployment manifest is already the desired-state for versions** (one + commit per release, digest-pinned). Extend it to per-host / per-cohort desired + state. Rollback = revert the commit (unchanged model). +- **Maintenance actions** (reboot, drain, restart kiosk, change URL, request logs) + ride the same poll: the server sets the intent; the host obeys on next poll. +- This dovetails with the **SPDK rolling update** (TODO #6): "redeploy a cohort" = + bump the manifest → `spdk-loader` serves the new version online → the management + plane tells those hosts to drain/reboot → it watches `spdk-release list` CONNS + fall → the old version auto-reaps. The management plane *orchestrates* this loop. +- **No always-on inbound control socket.** Admin SSH (key-only ECDSA) stays purely + break-glass for exceptional manual ops. + +### 5. Server side (serving host or a dedicated mgmt node, e.g. alongside `wazuh1`) + +- **Mgmt service** (small Go, mirrors the `fleet-hostapi` style): mTLS endpoint + that accepts host reports and serves per-host desired-state; authenticates hosts + by TPM cert via the fleet CA. +- **Store:** time-series (VictoriaMetrics/Prometheus) for metrics; Wazuh / + OpenSearch for events/logs; git or a tiny DB for desired-state + inventory. +- **Dashboards:** Grafana (metric trends) + the Wazuh dashboard (events/alerts) + + a small fleet-overview board (offline hosts, version drift, crash rates, tmpfs + pressure). Could be a sibling of the `serving/kiosk-app` SPA. +- **Alerting:** host offline > N min; Chromium crash-loop; **running ≠ desired + version**; tmpfs near-full; MACsec/auth down; NVMe-root reconnect storms. + +## Data model (sketch) + +``` +Host (by cert-serial / hostnqn) + ├─ identity: cert serial, hostnqn, MAC, starship-name (dnsmasq) + ├─ version: image_version, rootfs digest, kernel, desired_version + ├─ health: cpu/load, gpu_freq, temps, tmpfs_used%, nvme_root, auth/macsec, + │ selinux, chrony, kiosk_active, last_seen + ├─ usage: kiosk_url, session_uptime, chromium_crashes, input_active, display + └─ desired: target_version, kiosk_url, reboot_at, drain (server-authoritative) +``` + +## Mapping to existing building blocks + +| Need | Reuse | +|---|---| +| Host identity | TPM cert / hostnqn / step-ca fleet CA (`serving/dot1x-bench/`) | +| On-host data | `fleet-hostapi` `/v1/host` (`files/hostapi/`) | +| Telemetry + alerting | audit-collector relay → Wazuh (`serving/siem/`) | +| Friendly names / inventory | dnsmasq `fleet-namer` (MAC → name → DNS) | +| Version truth + rollback | digest-pinned deployment manifest (git) | +| Redeploy / drain mechanics | SPDK `spdk-loader` + `spdk-release` (TODO #6) | +| Build / sign / publish | Gitea CI + OCI images | + +## Phasing + +1. **Phase 1 (mostly-built):** push `fleet-hostapi` snapshots as structured events + into Wazuh; add decoders/rules + a fleet dashboard → health, version-drift, + offline detection, usage, alerting. Reuses the existing pipeline end to end. +2. **Phase 2:** metrics push (OTLP / remote-write) → VictoriaMetrics + Grafana for + trend graphs. +3. **Phase 3:** mTLS desired-state pull (version pin, reboot/drain, kiosk URL), + wired to the deployment manifest + SPDK rolling-update flow. + +## Build vs buy + +Off-the-shelf fits poorly because the base is unusual: + +- **FleetDM + osquery** — closest for the query/health side; worth evaluating. But + osquery enrollment carries per-host state, so it must be run carefully (identity + from TPM, enrollment in tmpfs only) or it breaks invariant #1. +- **Mender / balena / typical kiosk MDM** — assume mutable disk + their own OTA + + enrollment. We already have a *better* version mechanism (UKI + NVMe-root + + digest manifest), so their OTA is redundant and their disk model conflicts. + +**Decision: build the thin mgmt service + reconcile loop; reuse Wazuh/Grafana for +the heavy lifting.** The hard parts (identity, immutable versioning, a telemetry +bus) already exist. + +## Security posture + +- mTLS + TPM cert = strong identity, zero baked secrets, reuses the 802.1X CA. +- Pull-based control = **no new inbound attack surface** on locked hosts. +- Telemetry stays on the mgmt/boot LAN, TLS'd (same gap as the Wazuh-relay-TLS + TODO) — no internet egress, air-gap intact. +- Agent stays minimal/sandboxed/fapolicyd-trusted/reproducible (`fleet-hostapi` + template); usage capture is liveness-only, never content (privacy). +- Fleet-wide inventory is recon-valuable → protect the management plane (auth + + segmentation); never expose its dashboards to the boot LAN. + +## Open questions / decisions for build time + +- **Telemetry encoding:** structured syslog/JSON into Wazuh (Phase 1) vs native + OTLP from the start. Leaning Wazuh-first for reuse. +- **Metrics store:** VictoriaMetrics vs Prometheus (VM is lighter for push + + long retention). +- **Desired-state store:** git (GitOps, aud/rollback for free, matches the + deployment-manifest model) vs a small DB (richer per-host scheduling). +- **Per-host vs per-cohort** desired state granularity (cohorts likely enough; + per-host for canaries). +- **Open-path identity:** MAC/fleet-name is weak — gate the management plane to the + secure path, or accept lab-only for the open path. +- **osquery/FleetDM** evaluation for the read side before building a bespoke + collector. diff --git a/docs/stig-deviations.md b/docs/stig-deviations.md index db28c8e..f6024f4 100644 --- a/docs/stig-deviations.md +++ b/docs/stig-deviations.md @@ -143,10 +143,13 @@ hotkey disabled (e.g. a physically exposed terminal), re-select both rule IDs. - **SELinux enforcing** (`selinux --enforcing`; labels preserved into the erofs). - **fapolicyd** kept **enabled** (NOT waived) — it's the real exec-allowlist guard for the writable RAM overlay, which dm-verity does **not** cover. The only - adjustment: the one non-RPM executable, `/usr/local/bin/kiosk-session`, is added - to fapolicyd's trust DB at build (provisioner 2c-STIG-postfix) so the kiosk runs; - everything else stays RPM-trust-only. (fapolicyd blocking that binary was the - original kiosk-won't-start cause.) + adjustment: the few non-RPM executables we ship are added to fapolicyd's trust DB + at build (provisioner 2c-STIG-postfix) so they run; everything else stays + RPM-trust-only. Trusted custom binaries: `/usr/local/bin/kiosk-session`, + `/usr/local/bin/fleet-sb-enroll`, `/usr/local/bin/fleet-hostapi` (the loopback + host-info agent, built from `files/hostapi`). Each is built into the immutable, + dm-verity-protected root, so trusting it doesn't weaken the overlay exec guard. + (fapolicyd blocking `kiosk-session` was the original kiosk-won't-start cause.) - **auditd** enabled; STIG audit rules applied; logs locally. - **USBGuard** enabled + enforcing (`block` target) with a HID/hub allowlist and `AuditBackend=LinuxAudit` (see #12) — blocks USB mass storage / unknown device diff --git a/files/hostapi/README.md b/files/hostapi/README.md new file mode 100644 index 0000000..0005aab --- /dev/null +++ b/files/hostapi/README.md @@ -0,0 +1,53 @@ +# fleet-hostapi — workstation host-info agent + +A tiny **read-only, loopback-only** Go HTTP service that runs on the diskless +workstation and reports facts about the machine the kiosk renders on. It is the +OS-side implementation of the contract in [`../../HOST-API.md`](../../HOST-API.md) +(authored alongside the kiosk-app SPA that consumes it). + +``` +GET http://localhost:9099/v1/host -> application/json (os, hardware, net, kiosk, agent) +GET http://localhost:9099/healthz -> 200 ok +``` + +## Properties (and why) + +- **Loopback only.** Binds `127.0.0.1`; refuses to start on any routable address. + Host facts must never reach the boot LAN. +- **Read only.** No endpoint mutates anything; it reads `/proc`, `/sys`, + `/etc/os-release`, and the build-time facts file. +- **No per-host state baked in** (invariant #1). Host-specific fields (kernel, + hostname, IP, MAC, board, uptime) are read at **runtime**. The only baked facts + are image-generic — image version + weston/chromium versions — written to + `/etc/fleet-hostinfo.json` by the build provisioner. +- **Stdlib only → reproducible build** (the verity hash is a security property). + Built `CGO_ENABLED=0 -trimpath -buildvcs=false -ldflags=-buildid=`; identical + bytes across builds with the same Go toolchain. +- **CORS + Private Network Access.** The SPA is cross-origin and the fetch is a + Chromium private-network request, so responses carry + `Access-Control-Allow-Private-Network: true`. The matching browser-side + allowance is in `files/kiosk/chromium-kiosk-policy.json` + (`InsecurePrivateNetworkRequestsAllowedForUrls`, scoped to the kiosk origin). + +## Build / install + +Done by the `2b-hostapi` provisioner in `diskless.pkr.hcl`: installs the Go +toolchain, builds the static binary to `/usr/local/bin/fleet-hostapi`, writes the +facts file, installs + enables `fleet-hostapi.service`, and adds the binary to +fapolicyd trust (2c-STIG-postfix). The toolchain is removed in 2d hygiene — only +the static binary ships. + +Local build: + +```bash +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false \ + -ldflags '-s -w -buildid=' -o fleet-hostapi . +``` + +## Config (env, set in the unit) + +| Var | Default | Notes | +|---|---|---| +| `FLEET_HOSTAPI_ADDR` | `127.0.0.1:9099` | must be loopback | +| `FLEET_HOSTAPI_ALLOW_ORIGIN` | `*` | tighten to the kiosk origin in prod | +| `FLEET_HOSTAPI_FACTS` | `/etc/fleet-hostinfo.json` | build-time facts | diff --git a/files/hostapi/fleet-hostapi.service b/files/hostapi/fleet-hostapi.service new file mode 100644 index 0000000..2c41e0e --- /dev/null +++ b/files/hostapi/fleet-hostapi.service @@ -0,0 +1,52 @@ +[Unit] +Description=Fleet workstation host-info API (loopback, read-only) +Documentation=https://git.bitasylum.org/audiobuzz/diskless-workstation/HOST-API.md +# Needs networking up only to read the primary IP/MAC; the listener is loopback. +After=network-online.target +Wants=network-online.target + +[Service] +Type=exec +ExecStart=/usr/local/bin/fleet-hostapi +Environment=FLEET_HOSTAPI_ADDR=127.0.0.1:9099 +# Lab default: any origin may read (it's a loopback, read-only facts service on a +# single-purpose kiosk). Tighten to the exact kiosk origin in production, e.g. +# Environment=FLEET_HOSTAPI_ALLOW_ORIGIN=http://boot.fleet.lab +Environment=FLEET_HOSTAPI_ALLOW_ORIGIN=* +Environment=FLEET_HOSTAPI_FACTS=/etc/fleet-hostinfo.json +Restart=on-failure +RestartSec=2 + +# --- sandboxing (STIG-friendly; the agent only READS /proc, /sys, /etc) ------- +# Runs as a transient unprivileged user with no capabilities; everything it +# reads is world-readable, so it needs no privilege at all. +DynamicUser=yes +CapabilityBoundingSet= +AmbientCapabilities= +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +ProtectProc=default +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +SystemCallArchitectures=native +SystemCallFilter=@system-service +SystemCallFilter=~@privileged @resources +RestrictAddressFamilies=AF_INET AF_INET6 +# Kernel-enforced loopback-only egress/ingress (belt + suspenders to the bind). +IPAddressAllow=localhost +IPAddressDeny=any +UMask=0077 + +[Install] +WantedBy=multi-user.target diff --git a/files/hostapi/go.mod b/files/hostapi/go.mod new file mode 100644 index 0000000..7263174 --- /dev/null +++ b/files/hostapi/go.mod @@ -0,0 +1,7 @@ +// fleet-hostapi: the workstation host-info agent (HOST-API.md contract). +// Stdlib only — no external modules — so the build needs no network and is +// trivially reproducible (a hard requirement: the rootfs/verity hash must be a +// pure function of the inputs). Keep it dependency-free. +module fleet-hostapi + +go 1.21 diff --git a/files/hostapi/main.go b/files/hostapi/main.go new file mode 100644 index 0000000..ad75b3a --- /dev/null +++ b/files/hostapi/main.go @@ -0,0 +1,400 @@ +// fleet-hostapi — read-only workstation host-info agent for the kiosk. +// +// Implements the contract in HOST-API.md: a tiny HTTP service on the diskless +// workstation that reports facts about the machine the kiosk renders on. The +// kiosk SPA (served from the boot server, a different origin) fetches +// +// GET http://localhost:9099/v1/host -> application/json +// +// Design constraints (see CLAUDE.md invariants): +// - LOOPBACK ONLY. Never bind a routable address — this leaks host facts and +// must never reach the boot LAN. +// - READ ONLY. No endpoint mutates anything; the agent only reads /proc, /sys, +// /etc/os-release, and a build-time facts file. +// - NO PER-HOST STATE baked into the image. Everything host-specific (kernel, +// hostname, IP, MAC, board, uptime) is read at RUNTIME. The only baked facts +// are image-generic (image version + weston/chromium versions), written to a +// facts file by the build provisioner — same for every host of a release. +// - Stdlib only -> reproducible build (the verity hash is a security property). +// +// CORS + Private Network Access: the SPA is cross-origin AND the fetch is a +// Chromium "private network" request, so every response (incl. the OPTIONS +// preflight) carries Access-Control-Allow-Private-Network: true. +package main + +import ( + "bufio" + "encoding/json" + "flag" + "log" + "math" + "net" + "net/http" + "os" + "runtime" + "strconv" + "strings" + "syscall" + "time" +) + +// agentVersion is overridable at build time via -ldflags "-X main.agentVersion=…". +var agentVersion = "1.0.0" + +// ---- response shape (HOST-API.md) — all fields optional, omit what's unknown -- + +type osInfo struct { + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Kernel string `json:"kernel,omitempty"` + Image string `json:"image,omitempty"` +} + +type hwInfo struct { + CPU string `json:"cpu,omitempty"` + Cores int `json:"cores,omitempty"` + MemGB int `json:"memGB,omitempty"` + GPU string `json:"gpu,omitempty"` + Board string `json:"board,omitempty"` +} + +type netInfo struct { + Hostname string `json:"hostname,omitempty"` + IP string `json:"ip,omitempty"` + MAC string `json:"mac,omitempty"` +} + +type kioskInfo struct { + Uptime string `json:"uptime,omitempty"` + Weston string `json:"weston,omitempty"` + Chromium string `json:"chromium,omitempty"` +} + +type agentInfo struct { + Version string `json:"version,omitempty"` +} + +type hostResponse struct { + OS osInfo `json:"os"` + Hardware hwInfo `json:"hardware"` + Net netInfo `json:"net"` + Kiosk kioskInfo `json:"kiosk"` + Agent agentInfo `json:"agent"` +} + +// buildFacts is the image-generic, build-time facts file (written by the Packer +// provisioner). Distinct from anything per-host. +type buildFacts struct { + Image string `json:"image"` + Weston string `json:"weston"` + Chromium string `json:"chromium"` +} + +func main() { + addr := flag.String("addr", env("FLEET_HOSTAPI_ADDR", "127.0.0.1:9099"), "listen address (MUST be loopback)") + origin := flag.String("allow-origin", env("FLEET_HOSTAPI_ALLOW_ORIGIN", "*"), "Access-Control-Allow-Origin value") + factsPath := flag.String("facts", env("FLEET_HOSTAPI_FACTS", "/etc/fleet-hostinfo.json"), "build-time facts file") + flag.Parse() + + // Refuse to bind anything that isn't loopback — a hard guardrail against a + // misconfiguration exposing host facts on the boot LAN. + if host, _, err := net.SplitHostPort(*addr); err != nil || !isLoopback(host) { + log.Fatalf("refusing to listen on %q: bind address must be loopback (127.0.0.1/::1)", *addr) + } + + facts := loadFacts(*factsPath) + + mux := http.NewServeMux() + mux.HandleFunc("/v1/host", func(w http.ResponseWriter, r *http.Request) { + cors(w, *origin) + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusNoContent) + return + } + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + _ = enc.Encode(collect(facts)) + }) + // Liveness for the unit / debugging (no host facts). + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("ok\n")) + }) + + srv := &http.Server{ + Addr: *addr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + } + log.Printf("fleet-hostapi %s listening on %s (loopback, read-only)", agentVersion, *addr) + log.Fatal(srv.ListenAndServe()) +} + +func cors(w http.ResponseWriter, origin string) { + // Private Network Access: the kiosk page (boot-server origin, an insecure/ + // public context) fetching localhost is a "private network" request — the + // preflight needs this header or Chromium blocks it. See HOST-API.md. + w.Header().Set("Access-Control-Allow-Origin", origin) + w.Header().Set("Access-Control-Allow-Private-Network", "true") + w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS") + w.Header().Set("Vary", "Origin") +} + +// collect gathers all host facts. Each source is best-effort: a failed read +// leaves its field empty (the SPA renders "—" for missing fields). +func collect(f buildFacts) hostResponse { + var resp hostResponse + + // os + rel := parseKV("/etc/os-release") + resp.OS.Name = firstNonEmpty(rel["NAME"], rel["ID"]) + resp.OS.Version = firstNonEmpty(rel["VERSION_ID"], rel["VERSION"]) + resp.OS.Kernel = kernelRelease() + resp.OS.Image = f.Image + + // hardware + resp.Hardware.CPU = cpuModel() + resp.Hardware.Cores = runtime.NumCPU() + resp.Hardware.MemGB = memGB() + resp.Hardware.GPU = gpuModel() + resp.Hardware.Board = boardModel() + + // net + resp.Net.Hostname, _ = os.Hostname() + resp.Net.IP, resp.Net.MAC = primaryIfaceAddr() + + // kiosk + resp.Kiosk.Uptime = uptimeHuman() + resp.Kiosk.Weston = f.Weston + resp.Kiosk.Chromium = f.Chromium + + resp.Agent.Version = agentVersion + return resp +} + +// ---- collectors ----------------------------------------------------------- + +func kernelRelease() string { + var u syscall.Utsname + if err := syscall.Uname(&u); err != nil { + return "" + } + return charsToString(u.Release[:]) +} + +func cpuModel() string { + f, err := os.Open("/proc/cpuinfo") + if err != nil { + return "" + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + line := sc.Text() + // x86: "model name". arm64: "Hardware"/"Processor" if present. + for _, key := range []string{"model name", "Hardware", "Processor"} { + if strings.HasPrefix(line, key) { + if _, v, ok := strings.Cut(line, ":"); ok { + return strings.TrimSpace(v) + } + } + } + } + _ = sc.Err() // best-effort read; empty result on any error + return "" +} + +func memGB() int { + for _, line := range readLines("/proc/meminfo") { + if strings.HasPrefix(line, "MemTotal:") { + fields := strings.Fields(line) // MemTotal: kB + if len(fields) >= 2 { + if kb, err := strconv.ParseFloat(fields[1], 64); err == nil { + return int(math.Round(kb / 1024.0 / 1024.0)) + } + } + } + } + return 0 +} + +// boardModel: vendor + product from DMI (world-readable sysfs — NOT product_serial, +// which is root-only and per-host; we never read it). +func boardModel() string { + vendor := strings.TrimSpace(readFile("/sys/class/dmi/id/sys_vendor")) + product := strings.TrimSpace(readFile("/sys/class/dmi/id/product_name")) + if vendor == "" && product == "" { + vendor = strings.TrimSpace(readFile("/sys/class/dmi/id/board_vendor")) + product = strings.TrimSpace(readFile("/sys/class/dmi/id/board_name")) + } + return strings.TrimSpace(strings.TrimSpace(vendor + " " + product)) +} + +// gpuModel: best-effort from the first DRM card's PCI ids. No PCI-ID database — +// we map the few known vendors and otherwise emit the raw ids. Often empty on a +// headless/virtio build VM; that's fine (optional field). +func gpuModel() string { + for _, card := range []string{"card0", "card1"} { + base := "/sys/class/drm/" + card + "/device/" + vid := strings.TrimSpace(readFile(base + "vendor")) + did := strings.TrimSpace(readFile(base + "device")) + if vid == "" { + continue + } + name := map[string]string{ + "0x8086": "Intel", + "0x1002": "AMD", + "0x10de": "NVIDIA", + "0x1af4": "Red Hat (virtio-gpu)", + "0x1234": "QEMU", + "0x1a03": "ASPEED", + }[strings.ToLower(vid)] + ids := strings.TrimPrefix(strings.ToLower(vid), "0x") + if did != "" { + ids += ":" + strings.TrimPrefix(strings.ToLower(did), "0x") + } + if name != "" { + return name + " [" + ids + "]" + } + return "[" + ids + "]" + } + return "" +} + +// primaryIfaceAddr returns the preferred outbound IPv4 + the MAC of the +// interface that owns it. Uses a UDP "connect" (no packet is sent) to let the +// kernel pick the route's source address. +func primaryIfaceAddr() (ip string, mac string) { + c, err := net.Dial("udp", "192.0.2.1:9") // TEST-NET-1, never reachable; no traffic on UDP connect + if err != nil { + return "", "" + } + defer c.Close() + la, ok := c.LocalAddr().(*net.UDPAddr) + if !ok || la.IP == nil { + return "", "" + } + ip = la.IP.String() + // find the interface carrying this IP -> its MAC + ifaces, err := net.Interfaces() + if err != nil { + return ip, "" + } + for _, ifc := range ifaces { + addrs, _ := ifc.Addrs() + for _, a := range addrs { + if n, ok := a.(*net.IPNet); ok && n.IP.Equal(la.IP) { + return ip, ifc.HardwareAddr.String() + } + } + } + return ip, "" +} + +// uptimeHuman formats system uptime as "3h 12m". On a kiosk appliance the system +// boots straight into the kiosk, so system uptime ≈ kiosk-session uptime — +// reported here to avoid coupling to (and exec'ing) systemd for the unit time. +func uptimeHuman() string { + var si syscall.Sysinfo_t + if err := syscall.Sysinfo(&si); err != nil { + return "" + } + d := time.Duration(si.Uptime) * time.Second + days := int(d.Hours()) / 24 + hours := int(d.Hours()) % 24 + mins := int(d.Minutes()) % 60 + switch { + case days > 0: + return strconv.Itoa(days) + "d " + strconv.Itoa(hours) + "h " + strconv.Itoa(mins) + "m" + case hours > 0: + return strconv.Itoa(hours) + "h " + strconv.Itoa(mins) + "m" + default: + return strconv.Itoa(mins) + "m" + } +} + +// ---- helpers -------------------------------------------------------------- + +func loadFacts(path string) buildFacts { + var f buildFacts + b, err := os.ReadFile(path) + if err != nil { + return f // file absent at runtime -> image/weston/chromium just omitted + } + _ = json.Unmarshal(b, &f) + return f +} + +func env(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +func isLoopback(host string) bool { + if host == "localhost" { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +func parseKV(path string) map[string]string { + m := map[string]string{} + for _, line := range readLines(path) { + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + m[strings.TrimSpace(k)] = strings.Trim(strings.TrimSpace(v), `"`) + } + return m +} + +func readLines(path string) []string { + b := readFile(path) + if b == "" { + return nil + } + return strings.Split(strings.TrimRight(b, "\n"), "\n") +} + +func readFile(path string) string { + b, err := os.ReadFile(path) + if err != nil { + return "" + } + return string(b) +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} + +// charsToString converts a NUL-terminated Utsname field (int8 or uint8 across +// arches) to a Go string. +func charsToString(ca []int8) string { + b := make([]byte, 0, len(ca)) + for _, c := range ca { + if c == 0 { + break + } + b = append(b, byte(c)) + } + return string(b) +} diff --git a/files/kiosk/chromium-kiosk-policy.json b/files/kiosk/chromium-kiosk-policy.json index 4d03e03..722f97d 100644 --- a/files/kiosk/chromium-kiosk-policy.json +++ b/files/kiosk/chromium-kiosk-policy.json @@ -31,5 +31,14 @@ "AutofillCreditCardEnabled": false, "PasswordManagerEnabled": false, "ShoppingListEnabled": false, - "EnableMediaRouter": false + "EnableMediaRouter": false, + + "_comment_capability_lockdown": "The distro chromium WRAPPER hardcodes --enable-extensions --enable-user-scripts --enable-printing --enable-sync --auto-ssl-client-auth (not removable via env/conf, only by bypassing the wrapper). Neutralise the capability-broadening ones HERE — policy beats cmdline flags and survives Chromium updates. ExtensionInstallBlocklist=['*'] blocks ALL extension installs, which also kills the user-scripts surface (user scripts need an extension/dev-mode). --enable-sync is already locked by SyncDisabled above. --auto-ssl-client-auth is moot: it only auto-presents a BROWSER-store client cert, and the TPM 802.1X cert isn't one — there are none; to govern explicitly, set AutoSelectCertificateForUrls (left unset = no auto-select). See files/kiosk/kiosk-session + TODO #13.", + "ExtensionInstallBlocklist": ["*"], + + "_comment_pna": "Private Network Access: the kiosk page (insecure http boot-server origin) fetches the workstation host-info agent at http://localhost:9099 — a cross-origin PRIVATE-network request Chromium blocks by default. The agent sends Access-Control-Allow-Private-Network: true; this is the matching browser-side allowance, scoped to the kiosk ORIGIN ONLY (not '*'). MUST list the origin of kiosk_url (default http://boot.fleet.lab). The machine talking to itself; no internet egress (see HOST-API.md).", + "InsecurePrivateNetworkRequestsAllowedForUrls": [ + "http://boot.fleet.lab", + "http://localhost" + ] } diff --git a/files/kiosk/kiosk-session b/files/kiosk/kiosk-session index 59e46de..b1a8ec5 100644 --- a/files/kiosk/kiosk-session +++ b/files/kiosk/kiosk-session @@ -26,6 +26,50 @@ SOCK=wayland-kiosk BROWSER=$(command -v chromium-browser || command -v chromium || true) : "${BROWSER:?chromium not found (expected chromium-browser or chromium in PATH)}" +# GPU vs software path. This image is FLEET-GENERIC: it boots both real hardware +# (e.g. the 820 / Intel HD620 -> i915) AND the virtio build/test VMs (no real +# GPU). Probe for a real KMS render node and branch — real GPU gets hardware +# raster/compositing (CPU drops off software raster); VMs keep the software path +# so weston + Chromium still render there. Pure runtime probe; no per-host state +# is baked (invariant #1). +GPU=0 +case "$(basename "$(readlink -f /sys/class/drm/card0/device/driver 2>/dev/null)" 2>/dev/null)" in + i915|amdgpu|radeon|nouveau|xe) [ -e /dev/dri/renderD128 ] && GPU=1 ;; +esac + +# Lockdown flags (what the managed policy can't cover). The distro chromium +# wrapper PREPENDS its own defaults ahead of these — including its own +# --disable-features. Chromium does NOT union duplicate --disable-features (the +# LAST occurrence wins), and ours is last, so we MERGE the wrapper's disables +# (LensOverlay/Gemini/Compose/TabOrganization) into ours — otherwise they are +# silently dropped. We deliberately pass NO --enable-features, so the wrapper's +# VA-API enables (AcceleratedVideoDecodeLinuxGL, ...) stay in effect. +# Phone-home suppression (background networking, component/auto update, domain +# reliability, crash upload, pings, search-engine probe) + the comprehensive +# lockdown live in the managed policy (/etc/chromium/policies/managed/). +COMMON_FLAGS=( + --kiosk --noerrdialogs --disable-infobars --incognito --no-first-run + --disable-session-crashed-bubble --start-maximized + --ozone-platform=wayland + --disable-background-networking --disable-component-update + --disable-sync --disable-domain-reliability --disable-breakpad + --no-pings --no-default-browser-check --metrics-recording-only + --disable-search-engine-collection + "--disable-features=LensOverlay,OptimizationGuideOnDeviceModel,PromptAPIForGeminiNano,Compose,TabOrganization,Translate,OptimizationHints,OptimizationTargetPrediction,MediaRouter,AutofillServerCommunication,InterestFeedContentSuggestions" + --user-data-dir=/tmp/kiosk-profile --disk-cache-dir=/tmp/kiosk-cache +) + +if [ "$GPU" = 1 ]; then + # Real GPU: hardware raster/compositing. HD620 (gen7 Intel) is sometimes + # raster-blocklisted by Chromium, so override the blocklist. The GPU process + # opens /dev/dri/renderD128 -> the kiosk user must be in group 'render' (set + # at build, diskless.pkr.hcl). With --disable-gpu gone, the wrapper's VA-API + # --enable-features finally engage (they need a live GPU process). + GPU_FLAGS=(--ignore-gpu-blocklist --enable-gpu-rasterization --enable-zero-copy) +else + GPU_FLAGS=(--disable-gpu) +fi + # Background: wait (up to ~60s) for weston's socket, then keep Chromium running. ( for _ in $(seq 1 120); do @@ -34,30 +78,17 @@ BROWSER=$(command -v chromium-browser || command -v chromium || true) done export WAYLAND_DISPLAY="$SOCK" while :; do - # --disable-* / --no-pings etc.: stop Chromium phoning home to Google - # (background networking = variations/field-trials + GCM, component & - # auto updates, domain reliability, crash upload, hyperlink pings). The - # --disable-features list kills the Optimization Guide hints fetch - # (optimizationguide-pa.googleapis.com), Translate, Media Router discovery - # and Autofill server calls. The comprehensive lockdown (sign-in, metrics, - # safe browsing, search suggest, ...) is the managed policy at - # /etc/chromium/policies/managed/10-kiosk-lockdown.json; these flags cover - # what policy can't. A fixed-URL kiosk needs none of it. - "$BROWSER" \ - --kiosk --noerrdialogs --disable-infobars --incognito --no-first-run \ - --disable-session-crashed-bubble --start-maximized \ - --ozone-platform=wayland --disable-gpu \ - --disable-background-networking --disable-component-update \ - --disable-sync --disable-domain-reliability --disable-breakpad \ - --no-pings --no-default-browser-check --metrics-recording-only \ - --disable-search-engine-collection \ - --disable-features=Translate,OptimizationHints,OptimizationTargetPrediction,MediaRouter,AutofillServerCommunication,InterestFeedContentSuggestions \ - --user-data-dir=/tmp/kiosk-profile --disk-cache-dir=/tmp/kiosk-cache \ - "$KIOSK_URL" || true + "$BROWSER" "${COMMON_FLAGS[@]}" "${GPU_FLAGS[@]}" "$KIOSK_URL" || true sleep 2 done ) & # weston as the foreground process (owns the seat; service restarts if it dies). -# desktop-shell + explicit socket; --use-pixman = software rendering for VMs. -exec weston --use-pixman --socket="$SOCK" --config=/etc/xdg/weston/weston.ini +# desktop-shell + explicit socket. On real GPU use weston's GL renderer (enables +# the zero-copy dmabuf handoff from Chromium so its frames aren't CPU-copied); on +# VMs force --use-pixman (software) — there's no usable DRM/GL device there. +if [ "$GPU" = 1 ]; then + exec weston --socket="$SOCK" --config=/etc/xdg/weston/weston.ini +else + exec weston --use-pixman --socket="$SOCK" --config=/etc/xdg/weston/weston.ini +fi diff --git a/scripts/extract-artifacts.sh b/scripts/extract-artifacts.sh index f43af81..ec3ea59 100755 --- a/scripts/extract-artifacts.sh +++ b/scripts/extract-artifacts.sh @@ -39,12 +39,41 @@ STIG=${STIG:-true} # cmdline. dm-verity needs TWO namespaces — erofs DATA and the verity HASH tree # — because this distro's systemd-veritysetup does not support the in-file # hash-offset layout (it ignores hash-offset= and fails to load the superblock). -# MUST match ROOT_DATA_UUID / ROOT_HASH_UUID in serving/spdk-setup.sh (assigned -# on the target via add_ns -u). +# The serving side reads these back from verity.json (single source of truth) — +# see serving/spdk/spdk-release.sh, which assigns them on the target via add_ns -u. +# +# PER-VERSION UUIDs: so a LONG-LIVED NVMe-oF target can serve MULTIPLE rootfs +# versions at once (rolling updates, TODO #6) the two namespaces must be globally +# distinct per version — otherwise two versions collide on the same by-id symlink +# (/dev/disk/by-id/nvme-uuid.) on the host. We derive them deterministically +# from IMAGE_VERSION (sha1sum — coreutils only, no uuidgen dependency) and bake +# the result into BOTH the immutable UKI cmdline and verity.json. Determinism +# preserves the reproducible-build invariant (same version -> same UUIDs). The +# value need not be an RFC-v5 UUID — it only has to be well-formed + unique, and +# the cmdline and the serving target consume the SAME value (cmdline from here, +# target from verity.json). An explicit ROOT_DATA_UUID/ROOT_HASH_UUID env wins; +# with no version it falls back to the legacy fixed pair (single-version serving). +IMAGE_VERSION=${IMAGE_VERSION:-$(basename "$OUT")} +sha2uuid() { # arbitrary string -> stable UUID-formatted (8-4-4-4-12) hex + local h + h=$(printf '%s' "$1" | sha1sum | cut -c1-32) + printf '%s-%s-%s-%s-%s' "${h:0:8}" "${h:8:4}" "${h:12:4}" "${h:16:4}" "${h:20:12}" +} +if [ -n "$IMAGE_VERSION" ] && command -v sha1sum >/dev/null 2>&1; then + ROOT_DATA_UUID=${ROOT_DATA_UUID:-$(sha2uuid "fleet-rootfs-data-${IMAGE_VERSION}")} + ROOT_HASH_UUID=${ROOT_HASH_UUID:-$(sha2uuid "fleet-rootfs-hash-${IMAGE_VERSION}")} +fi ROOT_DATA_UUID=${ROOT_DATA_UUID:-f1ee7000-0000-4000-8000-000000000001} ROOT_HASH_UUID=${ROOT_HASH_UUID:-f1ee7000-0000-4000-8000-000000000002} ROOT_DATA_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_DATA_UUID}" ROOT_HASH_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_HASH_UUID}" +# Per-version subsystem NQN (distinct NQN per release) — lets a host attach to +# exactly one version's subsystem and gives clean per-release drain/allow-listing +# on the long-lived target. Written to verity.json for the serving side; NOT in +# the cmdline (host still discovers + connects; the by-UUID verity refs select +# what mounts). An explicit SUBNQN env wins. +SUBNQN_BASE=${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs} +SUBNQN=${SUBNQN:-${SUBNQN_BASE}-${IMAGE_VERSION}} for bin in virt-copy-out virt-ls guestfish mkfs.erofs veritysetup jq getfattr; do command -v "$bin" >/dev/null || { echo "ERROR: $bin not found on build host" >&2; exit 1; } @@ -141,8 +170,10 @@ mv "$WORK/rootfs.hash" "$OUT/rootfs.hash" jq -n --arg hash "$ROOT_HASH" --arg erofs_uuid "$FIXED_UUID" \ --arg data_uuid "$ROOT_DATA_UUID" --arg hash_uuid "$ROOT_HASH_UUID" \ + --arg subnqn "$SUBNQN" --arg version "$IMAGE_VERSION" \ '{root_hash: $hash, erofs_uuid: $erofs_uuid, data_ns_uuid: $data_uuid, hash_ns_uuid: $hash_uuid, + subnqn: $subnqn, image_version: $version, data_block_size: 4096, hash_block_size: 4096}' \ > "$OUT/verity.json" diff --git a/serving/.env.example b/serving/.env.example index 9138b40..2ce0277 100644 --- a/serving/.env.example +++ b/serving/.env.example @@ -74,6 +74,23 @@ NVME_HOST=nvme.fleet.lab AUDIT_HOST=audit.fleet.lab # --- NVMe/TCP target ------------------------------------------------------- +# Per-version subsystem NQN base. The long-lived target serves each rootfs +# version as its OWN subsystem "-" (so a new build is added live +# without disturbing connected hosts). Per release the exact NQN is read from +# that version's verity.json (.subnqn, written by extract-artifacts.sh); this is +# only the fallback/default base and MUST match SUBNQN_BASE in that script. +SUBNQN_BASE=nqn.2026-06.org.example.fleet:rootfs +# Legacy single-version NQN (only the deprecated serving/spdk-setup.sh path). SUBNQN=nqn.2026-06.org.example.fleet:rootfs TARGET_PORT=4420 SPDK_CPUMASK=0x1 # one polling core is plenty for a read-only boot fs + +# --- SPDK version loader / reaper (declarative rolling updates) ------------- +# The spdk-loader service stages + adds ${IMAGE_VERSION} to the long-lived target +# on `compose up`, then auto-removes OLD versions once they drain. Migration = +# bump IMAGE_VERSION above + `docker compose up -d`. +REAP_OLD=true # false = keep old versions until you remove them manually +REAP_POLL_SECONDS=30 # how often to poll connection counts +# Sustained-zero window before an old version is removed. Keep it comfortably +# ABOVE the host's nvme ctrl_loss_tmo so a transient reconnect never gets reaped. +REAP_GRACE_SECONDS=300 diff --git a/serving/docker-compose.yml b/serving/docker-compose.yml index 688ef75..dd31d46 100644 --- a/serving/docker-compose.yml +++ b/serving/docker-compose.yml @@ -250,16 +250,99 @@ services: memlock: -1 volumes: - /dev/hugepages:/dev/hugepages + # LONG-LIVED target + ONLINE release control (no restart on a new build): + # spdk-target = the persistent entrypoint (nvmf_tgt stays up for life) + # spdk-release = JSON-RPC add/remove/list/drain of a version, live + - ./spdk/spdk-target.sh:/usr/local/bin/spdk-target:ro + - ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro + # Shared RPC socket so the spdk-loader container (and `compose exec`) can + # drive this target's JSON-RPC. nvmf_tgt creates the socket here (-r). + - spdk-sock:/var/run/spdk + # Multi-version release store: each ./spdk/releases// holds that + # version's rootfs.erofs + rootfs.hash + verity.json. Staged by the + # spdk-loader (or spdk-ctl.sh on the host); reconciled into the target at + # (re)start and added/removed live thereafter. RW so the bootstrap seed + + # loader can write here. Adding a version does NOT recreate this service. + - ./spdk/releases:/releases + # Bootstrap convenience (optional): a single OCI release mounted as an + # image volume. spdk-target seeds it into /releases as ${IMAGE_VERSION} if + # not already staged — preserves the simple one-version `compose up` flow. + # Remove this block for a pure multi-version (spdk-ctl-managed) deployment. - type: image source: ${OCI_IMAGE}:${IMAGE_VERSION} - target: /artifacts # -> /artifacts/rootfs.erofs for spdk-setup.sh + target: /artifacts image: subpath: artifacts # Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/artifacts:ro - - ./spdk-setup.sh:/spdk-setup.sh:ro + # Legacy single-version entrypoint (configure-at-start, restart-to-change): + # - ./spdk-setup.sh:/spdk-setup.sh:ro environment: - SUBNQN: ${SUBNQN} + IMAGE_VERSION: ${IMAGE_VERSION} # bootstrap-seed label for /artifacts + RELEASES_DIR: /releases + RPC_SOCK: /var/run/spdk/spdk.sock # shared with spdk-loader (-r / -s) + SUBNQN_BASE: ${SUBNQN_BASE} # per-version NQN base (falls back per release to verity.json .subnqn) TARGET_ADDR: ${SERVER_IP} TARGET_PORT: ${TARGET_PORT} SPDK_CPUMASK: ${SPDK_CPUMASK} - entrypoint: ["/bin/bash", "/spdk-setup.sh"] + NVMEOF_SVC: nvmeof + # Healthy once spdk-target has written its readiness stamp (transport up + + # releases reconciled). A FILE check, not a python rpc.py probe: nvmf_tgt does + # a slow DPDK init (~50s observed) and busy-polls its reactor core, so a python + # healthcheck starves/times out and the container sticks at "unhealthy" even + # though the target is up. `test -f` is instant and contention-free. The stamp + # is removed on (re)start + exit, so it never falsely reports a dead target. + healthcheck: + test: ["CMD-SHELL", "test -f /var/run/spdk/ready || exit 1"] + interval: 10s + timeout: 5s + retries: 6 + start_period: 120s + # Persistent target: started ONCE, never restarted for a version change. + # Rolling updates add/remove versions online (spdk-loader, or spdk-ctl.sh). + entrypoint: ["/bin/bash", "/usr/local/bin/spdk-target"] + + # --- SPDK version loader / reaper (declarative rolling updates) ------------- + # The "init container that stays up." On `compose up` after an IMAGE_VERSION + # bump it stages + adds the new rootfs to the running nvmeof target ONLINE + # (no target restart, no dropped connections), then polls and removes old + # versions once they drain (all hosts rebooted onto the new UKI). Migration is + # just: bump IMAGE_VERSION in .env -> `docker compose up -d`. Reuses the SPDK + # image (has python3 + rpc.py); the scripts are bind-mounted like nvmeof's. + spdk-loader: + image: ${SPDK_IMAGE}:${SPDK_VERSION} + network_mode: host + restart: unless-stopped + depends_on: + nvmeof: + condition: service_healthy + volumes: + - ./spdk/spdk-loader.sh:/usr/local/bin/spdk-loader:ro + - ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro + - spdk-sock:/var/run/spdk # shared RPC socket to the target + - ./spdk/releases:/releases # shared release store + # The CURRENT release's artifacts, mounted from its OCI image. The version + # tag is the single migration knob — bump IMAGE_VERSION and `up -d`. + - type: image + source: ${OCI_IMAGE}:${IMAGE_VERSION} + target: /in + image: + subpath: artifacts + # Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/in:ro + environment: + IMAGE_VERSION: ${IMAGE_VERSION} # the CURRENT version (staged + protected from reaping) + RELEASES_DIR: /releases + ARTIFACT_IN: /in + RPC_SOCK: /var/run/spdk/spdk.sock + SUBNQN_BASE: ${SUBNQN_BASE} + TARGET_ADDR: ${SERVER_IP} + TARGET_PORT: ${TARGET_PORT} + REAP_OLD: ${REAP_OLD:-true} # auto-remove drained old versions + REAP_POLL_SECONDS: ${REAP_POLL_SECONDS:-30} + REAP_GRACE_SECONDS: ${REAP_GRACE_SECONDS:-300} # sustained-zero window; keep > host ctrl_loss_tmo + entrypoint: ["/bin/bash", "/usr/local/bin/spdk-loader"] + +# Named volume for the SPDK JSON-RPC unix socket, shared between the long-lived +# nvmeof target (which creates it via nvmf_tgt -r) and the spdk-loader (which +# drives add/remove/drain over it). Keeps the control path off the host FS. +volumes: + spdk-sock: diff --git a/serving/kiosk-app b/serving/kiosk-app index f0d0677..4579d22 160000 --- a/serving/kiosk-app +++ b/serving/kiosk-app @@ -1 +1 @@ -Subproject commit f0d0677950c7f26595693a98e3335190d3f39023 +Subproject commit 4579d227b096368469c7b5439d9a1b0f9858ba67 diff --git a/serving/spdk-setup.sh b/serving/spdk-setup.sh index 006a471..9afa602 100644 --- a/serving/spdk-setup.sh +++ b/serving/spdk-setup.sh @@ -1,4 +1,12 @@ #!/usr/bin/env bash +# DEPRECATED / legacy single-version path. Superseded by the LONG-LIVED target: +# serving/spdk/spdk-target.sh (persistent nvmf_tgt) + spdk-release.sh (online +# add/remove of versions) + spdk-ctl.sh (host ops). This script configures ONE +# version at start, so changing versions means restarting the container — which +# drops EVERY running host's NVMe/TCP root connection (see docs/TODO.md #6). Kept +# for reference / one-shot single-version use. The compose stack now runs +# spdk-target.sh as the nvmeof entrypoint. +# # Configure SPDK nvmf_tgt to serve the diskless root filesystem as TWO READ-ONLY # NVMe/TCP namespaces — erofs DATA and the dm-verity HASH tree (design doc §5, §7). # diff --git a/serving/spdk/README.md b/serving/spdk/README.md new file mode 100644 index 0000000..c71f142 --- /dev/null +++ b/serving/spdk/README.md @@ -0,0 +1,112 @@ +# SPDK NVMe/TCP target — long-lived, multi-version + +This is the fleet's **root disk**: a userspace SPDK `nvmf_tgt` exporting each +rootfs release as **two read-only NVMe/TCP namespaces** (erofs DATA + dm-verity +HASH — the distro's systemd-veritysetup can't do the in-file hash-offset layout, +so they're separate namespaces with distinct UUIDs). + +## Why the target is long-lived + +`nvmf_tgt` sits in the **live data path of every running host's root fs**. +Restarting the process tears down **every** NVMe/TCP association at once +(fleet-wide, not version-scoped) — a host whose root link drops past +`ctrl_loss_tmo` gets root I/O errors and must reboot. So **the process must never +restart for a routine change.** + +A host changes root version **only by rebooting** (no live swap). Therefore: +**version transitions are reboots; SPDK changes are additive/online; the target +process stays up.** Routine rollouts never restart SPDK. + +## How it works + +| Piece | Role | +|---|---| +| [`spdk-target.sh`](spdk-target.sh) | **Persistent entrypoint.** Starts `nvmf_tgt` once, creates the TCP transport + discovery listener, then reconciles every staged release. Never restarted to change versions. On (re)start it re-adds everything under `./releases/`, so a serving-host reboot restores all served versions. | +| [`spdk-release.sh`](spdk-release.sh) | **In-container control** (JSON-RPC). `add`/`remove`/`list`/`drain-status`/`reconcile` a version **live**. One **subsystem per version** (distinct NQN) so adding/removing one never disturbs the subsystem other hosts are attached to. Removal is **drain-gated**. | +| [`spdk-ctl.sh`](spdk-ctl.sh) | **Host-side ops** (manual / cleanup). Stages a release's artifacts from its OCI image into `./releases//` and drives the live target via `docker compose exec`. | +| [`spdk-loader.sh`](spdk-loader.sh) | **Declarative rolling updates** (compose service `spdk-loader`, the "init container that stays up"). On `compose up` after an `IMAGE_VERSION` bump it stages + adds the new rootfs online, then auto-reaps old versions once they drain. | +| `./releases//` | Per-version `rootfs.erofs` + `rootfs.hash` + `verity.json`. `verity.json` is the **single source of truth** for that version's namespace UUIDs + subsystem NQN (written by `scripts/extract-artifacts.sh`), so the host's by-id verity refs and the target's namespace UUIDs always agree. | + +Per-version namespace UUIDs are required: two versions served at once must have +globally-distinct namespace UUIDs or they collide on the same +`/dev/disk/by-id/nvme-uuid.` symlink on the host. `extract-artifacts.sh` +derives them deterministically (UUIDv5 of the version) and bakes them into both +the immutable UKI cmdline and `verity.json`. + +## Rolling update — the declarative way (recommended) + +Bump one tag and bring the stack up: + +```bash +# in serving/.env +IMAGE_VERSION=1.3.0 + +docker compose up -d +``` + +Compose recreates the **`spdk-loader`** and the **http/dnsmasq** UKI servers (their +image volumes changed), but **not** the `nvmeof` target (its definition is +unchanged) — so no connections drop. The loader adds 1.3.0 to the live target, +serves the new UKI to all hosts, and then **auto-reaps** 1.2.0 once every host +has rebooted onto 1.3.0 and 1.2.0 has been drained for `REAP_GRACE_SECONDS`. +Rollback = set `IMAGE_VERSION` back + `up -d`. Set `REAP_OLD=false` to keep old +versions and reap them yourself. + +Why auto-reap is safe here: there is a **single served UKI**, so a powered-off +host that returns boots the *current* UKI — nothing can come back to an old +rootfs once its live connections (CONNS) hit 0. Removal is still drain-gated and +debounced by the grace window (ride out transient nvme-tcp reconnects — keep +`REAP_GRACE_SECONDS` above the host `ctrl_loss_tmo`). + +## Rolling update — the manual way (`spdk-ctl.sh`) + +```bash +cd serving/spdk + +./spdk-ctl.sh load 1.3.0 # serve 1.3.0 ALONGSIDE the running 1.2.0 (live, no restart) +# ... bump the deployment manifest digest; reboot hosts → they come up on 1.3.0 ... +./spdk-ctl.sh list # watch 1.2.0 CONNS drop toward 0 +# VERSION CONNS NQN +# rootfs-1.3.0 7 nqn.2026-06.org.example.fleet:rootfs-1.3.0 +# rootfs-1.2.0 0 nqn.2026-06.org.example.fleet:rootfs-1.2.0 +./spdk-ctl.sh unload 1.2.0 # drain-gated: refuses while CONNS > 0 +``` + +`unload` **refuses to remove a version that still has connected controllers** +(an assigned-but-offline host could still return to it). Drain those hosts +first, or `./spdk-ctl.sh unload 1.2.0 --force` to **evict** them (they lose root +I/O — only for a known-dead version). + +In-container equivalents (what `spdk-ctl.sh` calls): + +```bash +docker compose exec nvmeof spdk-release list +docker compose exec nvmeof spdk-release add 1.3.0 +docker compose exec nvmeof spdk-release drain-status 1.2.0 +docker compose exec nvmeof spdk-release remove 1.2.0 [--force] +``` + +## Capacity & availability + +- Each concurrently-served version costs target memory/hugepages + a file-backed + bdev. Plan for N live versions (current + previous + canary) during long + rollouts; raise `HUGEPAGES` accordingly. +- The target is still a **single-node SPOF**. Even with online add/remove, a node + failure or a planned `nvmf_tgt` restart drops every connection. The structural + fix is **HA + NVMe-oF multipath (ANA)**: ≥2 targets serving byte-identical, + same-UUID, read-only namespaces (replication is coherent precisely because the + root is read-only + content-addressed). Cheap host-side mitigation: generous + `ctrl_loss_tmo`/`reconnect_delay`. See `docs/TODO.md` #6/#7. +- There is **no zero-impact single-node restart** — only drain/failover. + +## Bootstrap (simple single-version) + +`docker compose up -d` still works with one version: the OCI image volume at +`/artifacts` is **seeded** into `./releases/${IMAGE_VERSION}/` and served +automatically. Use `spdk-ctl.sh load` to add more versions on top. + +## Legacy + +[`../spdk-setup.sh`](../spdk-setup.sh) is the original **single-version, +configure-at-start** entrypoint (changing versions = container restart = all +connections dropped). Superseded by the long-lived target; kept for reference. diff --git a/serving/spdk/releases/.gitignore b/serving/spdk/releases/.gitignore new file mode 100644 index 0000000..23c89a8 --- /dev/null +++ b/serving/spdk/releases/.gitignore @@ -0,0 +1,5 @@ +# Staged rootfs releases for the long-lived SPDK target (spdk-ctl.sh load/unload). +# Each / holds large, content-addressed artifacts pulled from the OCI +# release image — never commit them. Keep the directory itself. +* +!.gitignore diff --git a/serving/spdk/spdk-ctl.sh b/serving/spdk/spdk-ctl.sh new file mode 100755 index 0000000..0d5d4c4 --- /dev/null +++ b/serving/spdk/spdk-ctl.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# spdk-ctl.sh — host-side fleet release ops for the LONG-LIVED SPDK target. +# +# Stages a release's artifacts into ./releases// and adds/removes it on +# the running nvmf_tgt ONLINE — no container restart, no dropped connections. +# This is the operator entrypoint for rolling updates and cleanup; it wraps the +# in-container spdk-release control script (driven via `docker compose exec`). +# +# ./spdk-ctl.sh load [oci_ref] # stage + add live (oci_ref default: $OCI_IMAGE:) +# ./spdk-ctl.sh unload [--force] # drain-gated remove + delete staged files +# ./spdk-ctl.sh list # served versions + connected hosts +# +# Typical rollout: +# ./spdk-ctl.sh load 1.3.0 # serve the new version alongside the old +# # ... reboot hosts onto 1.3.0 (deployment manifest digest bump) ... +# ./spdk-ctl.sh list # watch 1.2.0 CONNS fall to 0 +# ./spdk-ctl.sh unload 1.2.0 # cleanup once drained (refuses if hosts remain) +set -euo pipefail + +cd "$(dirname "$0")" +RELEASES_DIR="${RELEASES_DIR:-./releases}" +SVC="${NVMEOF_SVC:-nvmeof}" + +# Pull OCI_IMAGE default from the stack .env if present. +if [ -f ../.env ]; then + set -a; . ../.env; set +a +fi + +# Run compose from the stack dir (one level up, where docker-compose.yml lives). +dc() { ( cd .. && docker compose "$@" ); } + +usage() { + echo "usage: $0 {load [oci_ref] | unload [--force] | list}" >&2 + exit 2 +} + +cmd_load() { + local v=${1:-}; [ -n "$v" ] || usage + local ref=${2:-} + if [ -z "$ref" ]; then + [ -n "${OCI_IMAGE:-}" ] || { echo "ERROR: set OCI_IMAGE in ../.env or pass an oci_ref" >&2; exit 1; } + ref="${OCI_IMAGE}:${v}" + fi + local dest="$RELEASES_DIR/$v" + if [ -s "$dest/verity.json" ]; then + echo "==> release $v already staged in $dest (skipping extract)" + else + echo "==> extracting $ref -> $dest" + mkdir -p "$dest" + local cid + cid=$(docker create "$ref") + # FROM-scratch artifact image: copy the three files the target needs. + docker cp "$cid:/artifacts/rootfs.erofs" "$dest/rootfs.erofs" + docker cp "$cid:/artifacts/rootfs.hash" "$dest/rootfs.hash" + docker cp "$cid:/artifacts/verity.json" "$dest/verity.json" + docker rm "$cid" >/dev/null + fi + echo "==> adding release $v on the live target (no restart)" + dc exec -T "$SVC" spdk-release add "$v" + dc exec -T "$SVC" spdk-release list +} + +cmd_unload() { + local v="" force="" + local a + for a in "$@"; do + case "$a" in + --force) force="--force" ;; + *) v="$a" ;; + esac + done + [ -n "$v" ] || usage + echo "==> removing release $v from the live target" + if dc exec -T "$SVC" spdk-release remove "$v" ${force:+$force}; then + rm -rf "${RELEASES_DIR:?}/$v" + echo "==> staged artifacts for $v deleted from $RELEASES_DIR/$v" + else + echo "remove refused (version still has connected hosts?) — staged files left in place." >&2 + echo "Drain those hosts first, or re-run with --force to evict them." >&2 + exit 1 + fi +} + +case "${1:-}" in + load) shift; cmd_load "$@" ;; + unload|rm) shift; cmd_unload "$@" ;; + list|ls) dc exec -T "$SVC" spdk-release list ;; + *) usage ;; +esac diff --git a/serving/spdk/spdk-loader.sh b/serving/spdk/spdk-loader.sh new file mode 100644 index 0000000..9a9138b --- /dev/null +++ b/serving/spdk/spdk-loader.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# spdk-loader.sh — declarative version manager for the long-lived SPDK target. +# +# Runs as a long-lived compose service alongside the nvmeof target. On (re)start: +# 1. STAGE the CURRENT release (${IMAGE_VERSION}) from the mounted artifact +# image (/in) into the shared release store (/releases//) and ADD it to +# the running target ONLINE (no target restart). +# 2. Then stay up and REAP: periodically poll connection counts and remove any +# OTHER (non-current) version once it has been drained (0 connected +# controllers) for a sustained grace window — i.e. once every host that was +# running it has rebooted onto the new UKI. +# +# Migration is therefore: bump IMAGE_VERSION in .env, `docker compose up -d`. +# Compose recreates THIS loader (its artifact mount + env changed) and the +# http/dnsmasq UKI servers (image volume changed), but NOT the nvmeof target +# (its definition is unchanged) — so live connections are never dropped. The new +# loader adds the new rootfs online and drains the old one as hosts roll over. +# +# SAFETY: with a single served UKI a host can only ever boot the CURRENT version, +# so a non-current version at 0 connections is genuinely orphaned (no powered-off +# host can return to it — on next boot it fetches the current UKI). Removal is +# still drain-gated in spdk-release (refuses >0), and gated here by a sustained- +# zero grace window (>= REAP_GRACE_SECONDS) to ride out transient nvme-tcp +# reconnects. Set REAP_GRACE_SECONDS comfortably above the host ctrl_loss_tmo. +set -euo pipefail + +CURRENT="${IMAGE_VERSION:?spdk-loader needs IMAGE_VERSION}" +RELEASES_DIR="${RELEASES_DIR:-/releases}" +IN="${ARTIFACT_IN:-/in}" +POLL="${REAP_POLL_SECONDS:-30}" +GRACE="${REAP_GRACE_SECONDS:-300}" +REAP="${REAP_OLD:-true}" +RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release) +# spdk-release reads RPC_SOCK from the environment (shared socket to the target). + +echo "==> spdk-loader: current=${CURRENT} reap=${REAP} poll=${POLL}s grace=${GRACE}s" + +# --- 1) stage + add the current release (idempotent) ----------------------- +dest="$RELEASES_DIR/$CURRENT" +mkdir -p "$dest" +if [ -s "$IN/verity.json" ]; then + # Skip the (large) copy if already staged with identical verity.json. + if [ -s "$dest/verity.json" ] && cmp -s "$IN/verity.json" "$dest/verity.json"; then + echo "==> $CURRENT already staged in $dest (verity.json identical) — skipping copy" + else + echo "==> staging $CURRENT from $IN -> $dest" + cp "$IN/rootfs.erofs" "$dest/rootfs.erofs" + cp "$IN/rootfs.hash" "$dest/rootfs.hash" + cp "$IN/verity.json" "$dest/verity.json" + fi +else + echo "WARNING: no $IN/verity.json (artifact image not mounted at $IN?) — relying on pre-staged $dest" >&2 +fi +bash "$RELEASE_CTL" add "$CURRENT" +bash "$RELEASE_CTL" list + +if [ "$REAP" != "true" ]; then + echo "==> REAP_OLD=$REAP — staying up for status only; old versions are NOT auto-removed." +fi + +# --- 2) reap loop: a non-current version drained for >= GRACE is removed ---- +declare -A zero_since +while true; do + sleep "$POLL" + [ "$REAP" = "true" ] || continue + [ -d "$RELEASES_DIR" ] || continue + for d in "$RELEASES_DIR"/*/; do + [ -e "${d}verity.json" ] || continue + v=$(basename "$d") + [ "$v" = "$CURRENT" ] && continue # never reap the current version + n=$(bash "$RELEASE_CTL" count "$v" 2>/dev/null || echo -1) + if [ "$n" = "0" ]; then + ts=${zero_since[$v]:-} + if [ -z "$ts" ]; then + ts=$(date +%s); zero_since[$v]=$ts + echo "==> $v drained (0 conns) — starting ${GRACE}s grace before removal" + fi + age=$(( $(date +%s) - ts )) + if [ "$age" -ge "$GRACE" ]; then + echo "==> reaping $v (drained ${age}s >= ${GRACE}s grace)" + if bash "$RELEASE_CTL" remove "$v"; then + rm -rf "${RELEASES_DIR:?}/$v" + unset 'zero_since[$v]' + echo "==> $v removed + staged files deleted" + else + # A connection reappeared between the poll and the remove — + # spdk-release refused (drain-gated). Reset and retry later. + echo "WARNING: remove $v refused (reconnected?) — grace reset, will retry" >&2 + unset 'zero_since[$v]' + fi + fi + else + # >0 connections, or RPC error (-1): reset the grace timer. + if [ -n "${zero_since[$v]:-}" ]; then + echo "==> $v no longer drained (conns=$n) — grace reset" + unset 'zero_since[$v]' + fi + fi + done +done diff --git a/serving/spdk/spdk-release.sh b/serving/spdk/spdk-release.sh new file mode 100755 index 0000000..d83e50a --- /dev/null +++ b/serving/spdk/spdk-release.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash +# spdk-release.sh — add/remove a rootfs RELEASE on the long-lived nvmf_tgt +# ONLINE, via JSON-RPC, WITHOUT restarting the target. The target process holds +# every running host's NVMe/TCP root connection, so restarting it tears down the +# whole fleet's root at once (see docs/TODO.md #6). Routine rollouts therefore +# never restart SPDK: this script adds the new version's bdevs + subsystem + +# namespaces live (existing connections untouched) and removes an old version +# live once it has drained. +# +# ONE SUBSYSTEM PER VERSION (distinct NQN): adding a release as a NEW subsystem +# never disturbs the subsystem running hosts are attached to (adding a namespace +# to a SHARED subsystem would fire an AER + rescan at every host). It also gives +# clean per-release drain + per-release hostNQN allow-listing. +# +# Each release lives under $RELEASES_DIR// as rootfs.erofs + rootfs.hash +# + verity.json. verity.json is the single source of truth for that version's +# namespace UUIDs + subsystem NQN (written by scripts/extract-artifacts.sh) — so +# the host's by-id verity refs and the target's namespace UUIDs always agree. +# +# JSON is parsed with python3 (always present — rpc.py runs on it), NOT jq, which +# the SPDK image does not ship. +# +# Usage: +# spdk-release add # idempotent; live, no restart +# spdk-release remove [--force] # DRAIN-GATED unless --force +# spdk-release list # served versions + connected hosts +# spdk-release drain-status # connected controller count (human) +# spdk-release count # connected controller count (bare int; -1 if absent/error) +# spdk-release reconcile # add every release under $RELEASES_DIR +set -euo pipefail + +RELEASES_DIR="${RELEASES_DIR:-/releases}" +TADDR="${TARGET_ADDR:-0.0.0.0}" +TPORT="${TARGET_PORT:-4420}" +SUBNQN_BASE="${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs}" +# Legacy fixed namespace UUIDs (pre per-version-UUID builds baked these into the +# UKI cmdline). Used as a fallback when verity.json lacks the per-version pair. +LEGACY_DATA_UUID="f1ee7000-0000-4000-8000-000000000001" +LEGACY_HASH_UUID="f1ee7000-0000-4000-8000-000000000002" +RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py) +PY=$(command -v python3 || echo python3) +# Optional explicit RPC socket (shared between the long-lived target and the +# loader container). Empty -> rpc.py's default socket (single-container case). +RPC_SOCK="${RPC_SOCK:-}" + +die() { echo "ERROR: $*" >&2; exit 1; } +rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; } +# bdev names must be identifier-safe; sanitise the version. +san() { printf '%s' "$1" | tr -c 'A-Za-z0-9' '_'; } + +# Read a top-level field from a release's verity.json (empty if missing/unreadable). +vfield() { + local v=$1 key=${2#.} + "$PY" -c ' +import json,sys +try: + d=json.load(open(sys.argv[1])) + x=d.get(sys.argv[2],"") + sys.stdout.write("" if x is None else str(x)) +except Exception: + pass' "$RELEASES_DIR/$v/verity.json" "$key" 2>/dev/null || true +} + +# Per-version subsystem NQN: prefer the one baked into verity.json; otherwise +# derive it the same way extract-artifacts.sh does (base-). +release_nqn() { + local v=$1 nqn + nqn=$(vfield "$v" .subnqn) + [ -n "$nqn" ] || nqn="${SUBNQN_BASE}-${v}" + printf '%s' "$nqn" +} + +subsys_exists() { + rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c ' +import json,sys +try: + subs=json.load(sys.stdin) +except Exception: + sys.exit(1) +sys.exit(0 if any(s.get("nqn")==sys.argv[1] for s in subs) else 1)' "$1" +} + +# Connected controllers on a subsystem (the drain gate). >=0 when served; -1 if +# the subsystem is absent or the RPC failed (so a poller never mistakes an error +# for "drained" and reaps a live version). +ctrl_count() { + local out + if out=$(rpc nvmf_subsystem_get_controllers "$1" 2>/dev/null); then + printf '%s' "$out" | "$PY" -c ' +import json,sys +try: print(len(json.load(sys.stdin))) +except Exception: print(-1)' + else + printf '%s' "-1" + fi +} + +cmd_add() { + local v=${1:?usage: add } + local dir="$RELEASES_DIR/$v" + # Validation failures here RETURN (not die/exit): cmd_add is called per-dir by + # cmd_reconcile as `cmd_add … || warn`, and an `exit` inside a function aborts + # the WHOLE process (|| only catches a return), so one bad/stale dir would stop + # every OTHER release from being added. Return so reconcile skips just this one. + [ -s "$dir/rootfs.erofs" ] || { echo "ERROR: $dir/rootfs.erofs missing/empty" >&2; return 1; } + [ -s "$dir/rootfs.hash" ] || { echo "ERROR: $dir/rootfs.hash missing/empty" >&2; return 1; } + + # Guard: the dir name must match the version inside verity.json. A mismatch + # means a stale/symlinked or mislabeled release dir (e.g. an old symlink into + # the version-pinned /artifacts that now resolves to a different build) — that + # would serve the wrong data/NQN/UUIDs under this name. Refuse loudly. (Legacy + # artifacts have no image_version field — skip the check for them.) + # Case-insensitive: the dir name comes from the OCI tag while verity.json's + # image_version comes from the build var, which can differ only in case — a + # benign skew. A genuine mismatch (e.g. pxe-… vs spdk-…) differs by far more. + local vv + vv=$(vfield "$v" .image_version) + if [ -n "$vv" ] && [ "${vv,,}" != "${v,,}" ]; then + echo "ERROR: release dir '$v' holds verity.json for version '$vv' (stale symlink / mislabeled dir). Refusing to add — remove '$dir' and re-stage." >&2 + return 1 + fi + + local nqn du hu s + nqn=$(release_nqn "$v") + du=$(vfield "$v" .data_ns_uuid) + hu=$(vfield "$v" .hash_ns_uuid) + # LEGACY FALLBACK: artifacts built before the per-version-UUID change have no + # data_ns_uuid/hash_ns_uuid in verity.json. Such a build's UKI cmdline baked + # the fixed pair below, so serve it with exactly those. Two such LEGACY + # versions can't be co-served (they'd collide on these fixed UUIDs) — rebuild + # with the current extract-artifacts.sh for true multi-version serving. + if [ -z "$du" ] || [ -z "$hu" ]; then + du=${du:-$LEGACY_DATA_UUID} + hu=${hu:-$LEGACY_HASH_UUID} + echo "WARNING: $dir/verity.json lacks per-version namespace UUIDs — using the legacy fixed pair ($du / $hu). OK for a single legacy version; rebuild for multi-version." >&2 + fi + + if subsys_exists "$nqn"; then + echo "release $v already served ($nqn) — nothing to do" + return 0 + fi + + s=$(san "$v") + echo "==> adding release $v as subsystem $nqn (data uuid $du, hash uuid $hu)" + # -r opens the backing file read-only where the SPDK build supports it. + rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096 -r \ + || rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096 + rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096 -r \ + || rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096 + # Serial must be <=20 chars; derive a stable, version-tagged one. + rpc nvmf_create_subsystem "$nqn" -a -s "SPDK$(printf '%s' "$s" | cut -c1-16)" -d "FleetRootfs" + rpc nvmf_subsystem_add_ns "$nqn" "rootfs_$s" -u "$du" + rpc nvmf_subsystem_add_ns "$nqn" "roothash_$s" -u "$hu" + rpc nvmf_subsystem_add_listener "$nqn" -t tcp -a "$TADDR" -s "$TPORT" + # NOTE: we deliberately do NOT (re)add the discovery-subsystem listener here. + # The long-lived target registers it ONCE at startup (spdk-target.sh), and a + # subsystem becomes discoverable simply by having its own listener on the same + # address. Re-adding it would make the nvmf_tgt DAEMON log "Listener already + # exists" to the container stdout on every add — noise the client can't + # suppress (the daemon, not rpc.py, emits it). One registration is enough. + echo "==> release $v ONLINE (existing connections untouched)" +} + +cmd_remove() { + local force=0 v="" + local a + for a in "$@"; do + case "$a" in + --force) force=1 ;; + *) v="$a" ;; + esac + done + [ -n "$v" ] || die "usage: remove [--force]" + + local nqn n + nqn=$(release_nqn "$v") + if ! subsys_exists "$nqn"; then + echo "release $v not served ($nqn) — nothing to remove" + return 0 + fi + n=$(ctrl_count "$nqn") + if [ "$n" -gt 0 ] && [ "$force" -ne 1 ]; then + die "release $v still has $n connected controller(s) — refusing to remove. Drain first (reboot/migrate those hosts), or pass --force to EVICT them (they lose root I/O)." + fi + [ "$n" -gt 0 ] && echo "WARNING: --force removing $v with $n connected controller(s) — those hosts lose root I/O" + + local s + s=$(san "$v") + echo "==> removing release $v ($nqn)" + rpc nvmf_delete_subsystem "$nqn" || true + rpc bdev_aio_delete "rootfs_$s" 2>/dev/null || true + rpc bdev_aio_delete "roothash_$s" 2>/dev/null || true + echo "==> release $v removed (other versions + their connections untouched)" +} + +cmd_list() { + local nqn n ver + printf '%-26s %-7s %s\n' VERSION CONNS NQN + rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c ' +import json,sys +try: subs=json.load(sys.stdin) +except Exception: subs=[] +for s in subs: + n=s.get("nqn","") + if "discovery" not in n: print(n)' \ + | while read -r nqn; do + n=$(ctrl_count "$nqn") + ver="${nqn##*:}" # strip the nqn.*: prefix for readability + printf '%-26s %-7s %s\n' "$ver" "$n" "$nqn" + done +} + +cmd_drain_status() { + local v=${1:?usage: drain-status } + local nqn n + nqn=$(release_nqn "$v") + n=$(ctrl_count "$nqn") + if [ "$n" -lt 0 ]; then + echo "$nqn is not served (or target unreachable)" + else + echo "$n connected controller(s) on $nqn" + fi +} + +cmd_count() { + local v=${1:?usage: count } + ctrl_count "$(release_nqn "$v")" + echo +} + +cmd_reconcile() { + [ -d "$RELEASES_DIR" ] || { echo "no $RELEASES_DIR — nothing to reconcile"; return 0; } + local d any=0 + for d in "$RELEASES_DIR"/*/; do + [ -e "${d}verity.json" ] || continue + any=1 + cmd_add "$(basename "$d")" || echo "WARNING: failed to add release $(basename "$d")" >&2 + done + [ "$any" -eq 1 ] || echo "no releases found under $RELEASES_DIR" +} + +case "${1:-}" in + add) shift; cmd_add "$@" ;; + remove|rm) shift; cmd_remove "$@" ;; + list|ls) shift; cmd_list "$@" ;; + drain-status|status) shift; cmd_drain_status "$@" ;; + count) shift; cmd_count "$@" ;; + reconcile) shift; cmd_reconcile "$@" ;; + *) echo "usage: spdk-release {add |remove [--force]|list|drain-status |count |reconcile}" >&2; exit 2 ;; +esac diff --git a/serving/spdk/spdk-target.sh b/serving/spdk/spdk-target.sh new file mode 100755 index 0000000..4280b9b --- /dev/null +++ b/serving/spdk/spdk-target.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# spdk-target.sh — LONG-LIVED nvmf_tgt entrypoint. +# +# Starts the NVMe-oF target ONCE and keeps it up for the container's entire life. +# It is RELEASE-AGNOSTIC: rootfs versions are added/removed ONLINE via +# spdk-release.sh (JSON-RPC) without ever restarting this process, so a new build +# never drops running hosts' NVMe/TCP root connections (docs/TODO.md #6). This is +# the key difference from the legacy serving/spdk-setup.sh, which configured a +# single version at start and thus required a connection-dropping restart to +# change versions. +# +# On (re)start it reconciles: every release staged under $RELEASES_DIR is added, +# so a serving-host reboot restores all currently-served versions automatically. +set -euo pipefail + +RELEASES_DIR="${RELEASES_DIR:-/releases}" +CPUMASK="${SPDK_CPUMASK:-0x1}" +TADDR="${TARGET_ADDR:-0.0.0.0}" +TPORT="${TARGET_PORT:-4420}" +HUGEPAGES="${HUGEPAGES:-1024}" +DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery" +# Readiness stamp for the container healthcheck. Written ONLY after the transport +# is up and releases are reconciled; removed on (re)start + exit. A file check is +# instant and python-free — the right probe for a box where SPDK busy-polls its +# (often single) reactor core and a python rpc.py healthcheck would starve/time +# out. Lives on the shared /run/spdk volume so it's cleared per container start. +READY_FILE="${READY_FILE:-/var/run/spdk/ready}" + +TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt) +RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py) +RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release) +# Optional explicit RPC socket. Set it (compose does, on a shared volume) so a +# SEPARATE loader container can reach this target's JSON-RPC and add/remove +# versions online. Empty -> nvmf_tgt's default socket (single-container case). +RPC_SOCK="${RPC_SOCK:-}" +rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; } +[ -n "$RPC_SOCK" ] && mkdir -p "$(dirname "$RPC_SOCK")" +mkdir -p "$(dirname "$READY_FILE")" +rm -f "$READY_FILE" # clear any stale readiness from a previous container start + +# Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced, so +# a privileged container writes the host value; re-applied after a host reboot +# (the container restarts). Best-effort: allocation can fail on a fragmented +# host, in which case set vm.nr_hugepages on the host via /etc/sysctl.d. +if [ -w /proc/sys/vm/nr_hugepages ]; then + cur=$(cat /proc/sys/vm/nr_hugepages 2>/dev/null || echo 0) + if [ "${cur:-0}" -lt "$HUGEPAGES" ]; then + echo "==> allocating $HUGEPAGES hugepages (was ${cur:-0})" + echo "$HUGEPAGES" > /proc/sys/vm/nr_hugepages 2>/dev/null || true + fi +fi +free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0) +[ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2 + +echo "==> starting LONG-LIVED nvmf_tgt (cpumask ${CPUMASK})${RPC_SOCK:+ rpc-sock $RPC_SOCK}" +TGT_ARGS=(-m "$CPUMASK") +[ -n "$RPC_SOCK" ] && TGT_ARGS+=(-r "$RPC_SOCK") +"$TGT" "${TGT_ARGS[@]}" & +TGT_PID=$! +# A TERM/INT here stops the target and DROPS EVERY connection — only on container +# stop, never for a version change. Make the intent loud. +trap 'echo "==> stopping nvmf_tgt — this drops ALL host root connections" >&2; rm -f "$READY_FILE"; kill "$TGT_PID" 2>/dev/null || true' TERM INT +trap 'rm -f "$READY_FILE"; kill "$TGT_PID" 2>/dev/null || true' EXIT + +# Wait for the RPC socket. +for _ in $(seq 1 30); do + rpc framework_get_config >/dev/null 2>&1 && break + sleep 1 +done + +echo "==> creating TCP transport" +rpc nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192 \ + || echo "WARNING: nvmf_create_transport failed (already present?)" >&2 + +echo "==> registering discovery listener ${TADDR}:${TPORT}" +rpc nvmf_subsystem_add_listener "$DISCOVERY_NQN" -t tcp -a "$TADDR" -s "$TPORT" >/dev/null 2>&1 || true + +# Bootstrap convenience: if a single OCI artifact image is mounted at /artifacts +# (the simple one-version `docker compose up` flow) and that version isn't yet +# staged under $RELEASES_DIR, stage it so the long-lived target serves it too. +# +# COPY, never symlink: /artifacts is a version-pinned image volume that CHANGES +# when IMAGE_VERSION changes, so a symlink into it would silently re-point an +# already-staged release dir at the new version's data on the next bump. Each +# release dir must hold its own immutable copy. Self-heal: re-copy if the staged +# verity.json differs from the mounted artifact (repairs a dir left stale/symlinked +# by an older build). +if [ -s /artifacts/verity.json ] && [ -n "${IMAGE_VERSION:-}" ]; then + seed="$RELEASES_DIR/$IMAGE_VERSION" + if [ ! -s "$seed/verity.json" ] || ! cmp -s /artifacts/verity.json "$seed/verity.json"; then + echo "==> seeding bootstrap release $IMAGE_VERSION from /artifacts (copy)" + mkdir -p "$seed" + cp -f /artifacts/rootfs.erofs /artifacts/rootfs.hash /artifacts/verity.json "$seed/" \ + || echo "WARNING: failed to seed $IMAGE_VERSION from /artifacts" >&2 + fi +fi + +echo "==> reconciling releases under $RELEASES_DIR" +RELEASES_DIR="$RELEASES_DIR" TARGET_ADDR="$TADDR" TARGET_PORT="$TPORT" RPC_SOCK="$RPC_SOCK" \ + bash "$RELEASE_CTL" reconcile || echo "WARNING: reconcile reported errors" >&2 + +# Signal readiness for the healthcheck — transport is up and releases reconciled. +: > "$READY_FILE" + +echo "==> nvmf_tgt up; releases served online. Add/remove versions with:" +echo " docker compose exec ${NVMEOF_SVC:-nvmeof} spdk-release {add|remove|list} " +echo " or host-side: serving/spdk/spdk-ctl.sh {load|unload|list} " +wait "$TGT_PID"