Merge pull request 'workstation-api' (#9) from workstation-api into main
Reviewed-on: https://git.bitasylum.org/audiobuzz/diskless-workstation/pulls/9
This commit is contained in:
audiobuzz
@@ -135,6 +135,14 @@ jobs:
|
|||||||
[ -n "$safe_branch" ] || safe_branch="branch"
|
[ -n "$safe_branch" ] || safe_branch="branch"
|
||||||
version="${safe_branch}-${short_sha}"
|
version="${safe_branch}-${short_sha}"
|
||||||
fi
|
fi
|
||||||
|
# Lowercase the whole thing. image_version is BOTH the OCI tag AND the
|
||||||
|
# in-artifact UKI filename (fleet-<version>.efi) + verity.json + subnqn.
|
||||||
|
# A registry may lowercase the pushed TAG while the artifact keeps the
|
||||||
|
# original case, so a mixed-case branch (e.g. SPDK-permanence) yields a
|
||||||
|
# capital fleet-SPDK-….efi the lowercase boot URL can't fetch (iPXE
|
||||||
|
# "no such file"). Forcing lowercase keeps tag, filename, verity.json,
|
||||||
|
# and the dnsmasq boot URL all in agreement.
|
||||||
|
version="$(printf '%s' "$version" | tr '[:upper:]' '[:lower:]')"
|
||||||
echo "image_version=${version}" >> "$GITHUB_OUTPUT"
|
echo "image_version=${version}" >> "$GITHUB_OUTPUT"
|
||||||
echo "Building image_version=${version}"
|
echo "Building image_version=${version}"
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
[submodule "serving/kiosk-app"]
|
[submodule "serving/kiosk-app"]
|
||||||
path = serving/kiosk-app
|
path = serving/kiosk-app
|
||||||
url = git@git.bitasylum.org:audiobuzz/fleet-kiosk-app.git
|
url = git@git.bitasylum.org:audiobuzz/fleet-kiosk-app.git
|
||||||
|
branch = restyle
|
||||||
|
|||||||
@@ -31,12 +31,32 @@ files/etc/dracut.conf.d/90-diskless.conf no-hostonly + drivers + modules
|
|||||||
files/plymouth/themes/fleet/ Custom boot splash (scripted security
|
files/plymouth/themes/fleet/ Custom boot splash (scripted security
|
||||||
checklist; driven by hook status tokens)
|
checklist; driven by hook status tokens)
|
||||||
files/kiosk/ weston+Chromium kiosk + chromium lockdown policy
|
files/kiosk/ weston+Chromium kiosk + chromium lockdown policy
|
||||||
|
(policy allows PNA to the host-info agent)
|
||||||
|
files/hostapi/ Go host-info agent (fleet-hostapi): read-only,
|
||||||
|
loopback :9099 /v1/host for the kiosk SPA
|
||||||
|
(HOST-API.md contract). Built in 2b-hostapi,
|
||||||
|
fapolicyd-trusted, golang removed in 2d.
|
||||||
scripts/extract-artifacts.sh qcow2 -> kernel/initramfs/erofs+verity/cmdline
|
scripts/extract-artifacts.sh qcow2 -> kernel/initramfs/erofs+verity/cmdline
|
||||||
scripts/build-oci.sh ukify + sbsign + roothash gate + OCI image
|
scripts/build-oci.sh ukify + sbsign + roothash gate + OCI image
|
||||||
scripts/build-ipxe-oci.sh sbsign snponly.efi -> FROM-scratch fleet-ipxe image
|
scripts/build-ipxe-oci.sh sbsign snponly.efi -> FROM-scratch fleet-ipxe image
|
||||||
serving/ Lab serving stack: dnsmasq + nginx + SPDK
|
serving/ Lab serving stack: dnsmasq + nginx + SPDK
|
||||||
(dnsmasq + fleet-ipxe + SPDK images are
|
(dnsmasq + fleet-ipxe + SPDK images are
|
||||||
CI-built + pulled; see .gitea/workflows/)
|
CI-built + pulled; see .gitea/workflows/)
|
||||||
|
serving/spdk/ LONG-LIVED nvmf_tgt: spdk-target.sh
|
||||||
|
(persistent entrypoint, never restarts
|
||||||
|
for a version change) + spdk-release.sh
|
||||||
|
(online add/remove/drain a version, 1
|
||||||
|
subsystem/version) + spdk-ctl.sh (host
|
||||||
|
load/unload/list for manual rolling
|
||||||
|
updates) + spdk-loader.sh (DECLARATIVE:
|
||||||
|
compose service that adds ${IMAGE_VERSION}
|
||||||
|
online then auto-reaps drained old
|
||||||
|
versions — bump tag + compose up -d).
|
||||||
|
releases/<ver>/ = staged rootfs+verity.json.
|
||||||
|
spdk-release parses RPC JSON via python3
|
||||||
|
(SPDK image has no jq). spdk-setup.sh
|
||||||
|
(parent dir) is the legacy single-version
|
||||||
|
entrypoint (deprecated).
|
||||||
serving/kiosk-app/ Svelte+Tailwind spaceship-terminal SPA
|
serving/kiosk-app/ Svelte+Tailwind spaceship-terminal SPA
|
||||||
— GIT SUBMODULE (own repo + CI; builds
|
— GIT SUBMODULE (own repo + CI; builds
|
||||||
the fleet-kiosk image the stack pulls).
|
the fleet-kiosk image the stack pulls).
|
||||||
|
|||||||
+67
-4
@@ -233,6 +233,13 @@ build {
|
|||||||
destination = "/tmp/"
|
destination = "/tmp/"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Workstation host-info API (Go). Source staged to /tmp/hostapi; compiled +
|
||||||
|
# installed in the 2b-hostapi step. The kiosk SPA reads it at localhost:9099.
|
||||||
|
provisioner "file" {
|
||||||
|
source = "files/hostapi"
|
||||||
|
destination = "/tmp/"
|
||||||
|
}
|
||||||
|
|
||||||
# Custom 'fleet' Plymouth theme (scripted security-handshake splash). Staged to
|
# Custom 'fleet' Plymouth theme (scripted security-handshake splash). Staged to
|
||||||
# /tmp/plymouth, installed + set default in the package step before dracut runs.
|
# /tmp/plymouth, installed + set default in the package step before dracut runs.
|
||||||
provisioner "file" {
|
provisioner "file" {
|
||||||
@@ -343,6 +350,16 @@ build {
|
|||||||
"truncate -s 0 /etc/hostname",
|
"truncate -s 0 /etc/hostname",
|
||||||
"install -D -m0644 /tmp/etc-stage/NetworkManager/conf.d/20-hostname.conf /etc/NetworkManager/conf.d/20-hostname.conf",
|
"install -D -m0644 /tmp/etc-stage/NetworkManager/conf.d/20-hostname.conf /etc/NetworkManager/conf.d/20-hostname.conf",
|
||||||
"restorecon /etc/NetworkManager/conf.d/20-hostname.conf",
|
"restorecon /etc/NetworkManager/conf.d/20-hostname.conf",
|
||||||
|
# Subscription-manager has no role on a booted, hostname-neutral, immutable
|
||||||
|
# appliance — entitlements only matter at BUILD time (dnf). Left running,
|
||||||
|
# rhsmcertd wakes periodically and tries to write Python __pycache__ into a
|
||||||
|
# bin_t dir on the read-only/overlay root: SELinux denies it (enforcing), so
|
||||||
|
# it's a benign but RECURRING AVC that floods the SIEM (fleet_audit rule
|
||||||
|
# 100120) and can never succeed (tmpfs overlay is wiped each boot). Mask the
|
||||||
|
# daemon + timer to kill the noise and trim attack surface. (Don't waive the
|
||||||
|
# AVC in Wazuh — that would also hide future rhsmcertd_t denials.)
|
||||||
|
"systemctl mask rhsmcertd.service 2>/dev/null || true",
|
||||||
|
"systemctl disable rhsmcertd.service 2>/dev/null || true",
|
||||||
# Read-only erofs/verity root; runtime state on a tmpfs overlay provided
|
# Read-only erofs/verity root; runtime state on a tmpfs overlay provided
|
||||||
# by systemd.volatile=overlay (set in the cmdline by extract-artifacts.sh).
|
# by systemd.volatile=overlay (set in the cmdline by extract-artifacts.sh).
|
||||||
# RHEL/Rocky dracut has no 'overlayfs'/'rootovl' module.
|
# RHEL/Rocky dracut has no 'overlayfs'/'rootovl' module.
|
||||||
@@ -408,8 +425,13 @@ build {
|
|||||||
# --use-pixman + chromium --disable-gpu — candidate to drop once the kiosk
|
# --use-pixman + chromium --disable-gpu — candidate to drop once the kiosk
|
||||||
# is display-verified; see TODO #4.)
|
# is display-verified; see TODO #4.)
|
||||||
"dnf install -y --setopt=tsflags=nodocs weston chromium mesa-dri-drivers mesa-libEGL mesa-libgbm dejavu-sans-fonts",
|
"dnf install -y --setopt=tsflags=nodocs weston chromium mesa-dri-drivers mesa-libEGL mesa-libgbm dejavu-sans-fonts",
|
||||||
# Unprivileged, passwordless kiosk user; video/input as a seat fallback.
|
# Unprivileged, passwordless kiosk user. Groups: 'video' (KMS /dev/dri/card0
|
||||||
|
# for weston) + 'input' (libinput) as a seat fallback; 'render'
|
||||||
|
# (/dev/dri/renderD128) so Chromium's GPU process can do hardware raster on
|
||||||
|
# real hardware (kiosk-session enables GPU when a real KMS driver is present;
|
||||||
|
# render nodes aren't logind-seat-ACL'd on EL9, so group membership is needed).
|
||||||
"id kiosk >/dev/null 2>&1 || useradd -m -G video,input kiosk",
|
"id kiosk >/dev/null 2>&1 || useradd -m -G video,input kiosk",
|
||||||
|
"getent group render >/dev/null 2>&1 && usermod -aG render kiosk || true",
|
||||||
# Install the staged unit, weston config, and session wrapper.
|
# Install the staged unit, weston config, and session wrapper.
|
||||||
"install -D -m0755 /tmp/kiosk/kiosk-session /usr/local/bin/kiosk-session",
|
"install -D -m0755 /tmp/kiosk/kiosk-session /usr/local/bin/kiosk-session",
|
||||||
"install -D -m0644 /tmp/kiosk/kiosk.service /etc/systemd/system/kiosk.service",
|
"install -D -m0644 /tmp/kiosk/kiosk.service /etc/systemd/system/kiosk.service",
|
||||||
@@ -442,6 +464,44 @@ build {
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# --- 2b-hostapi. Workstation host-info API (Go, loopback, read-only) ------
|
||||||
|
# Compiles files/hostapi into /usr/local/bin/fleet-hostapi and enables it as a
|
||||||
|
# systemd service the kiosk SPA reads at http://localhost:9099/v1/host (the
|
||||||
|
# HOST-API.md contract). Build-time, image-GENERIC facts (image version +
|
||||||
|
# weston/chromium versions) are written to /etc/fleet-hostinfo.json; everything
|
||||||
|
# host-specific (kernel, hostname, IP, MAC, board, uptime) is read at RUNTIME,
|
||||||
|
# so NO per-host state lands in the image (invariant #1). The Go toolchain is
|
||||||
|
# installed here and removed in 2d hygiene; it never ships.
|
||||||
|
provisioner "shell" {
|
||||||
|
environment_vars = [
|
||||||
|
"IMAGE_VERSION=${var.image_version}",
|
||||||
|
]
|
||||||
|
inline = [
|
||||||
|
"set -euxo pipefail",
|
||||||
|
"dnf -y install golang",
|
||||||
|
# Reproducible, dependency-free static build (the verity hash must be a pure
|
||||||
|
# function of the inputs — invariant #8). GOPROXY=off + GOTOOLCHAIN=local
|
||||||
|
# guarantee no network/toolchain fetch; -trimpath + -buildid= strip paths
|
||||||
|
# and the build id. Stdlib-only, so this needs no modules. NOTE: a golang
|
||||||
|
# package update changes the binary (hence the hash) — a reproducibility-
|
||||||
|
# sensitive input, like the STIG semanage store; build twice to confirm.
|
||||||
|
"install -d -m0755 /tmp/gocache",
|
||||||
|
"env -C /tmp/hostapi CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOPROXY=off GOTOOLCHAIN=local GOFLAGS=-mod=mod GOCACHE=/tmp/gocache GOPATH=/tmp/gopath go build -trimpath -buildvcs=false -ldflags '-s -w -buildid=' -o /usr/local/bin/fleet-hostapi .",
|
||||||
|
"chmod 0755 /usr/local/bin/fleet-hostapi",
|
||||||
|
"install -D -m0644 /tmp/hostapi/fleet-hostapi.service /etc/systemd/system/fleet-hostapi.service",
|
||||||
|
# Build-time facts file (image-generic; same for every host of a release).
|
||||||
|
# rpm '%%{...}' is doubled so Packer's HCL2 doesn't parse %{ as a directive.
|
||||||
|
"WESTON_VER=$(rpm -q --qf '%%{VERSION}' weston 2>/dev/null || echo '')",
|
||||||
|
"CHROMIUM_VER=$(rpm -q --qf '%%{VERSION}' chromium 2>/dev/null || rpm -q --qf '%%{VERSION}' chromium-browser 2>/dev/null || echo '')",
|
||||||
|
"printf '{\"image\":\"%s\",\"weston\":\"%s\",\"chromium\":\"%s\"}\\n' \"$IMAGE_VERSION\" \"$WESTON_VER\" \"$CHROMIUM_VER\" > /etc/fleet-hostinfo.json",
|
||||||
|
"chmod 0644 /etc/fleet-hostinfo.json",
|
||||||
|
"restorecon -R /usr/local/bin/fleet-hostapi /etc/systemd/system/fleet-hostapi.service /etc/fleet-hostinfo.json 2>/dev/null || true",
|
||||||
|
"systemctl enable fleet-hostapi.service",
|
||||||
|
"rm -rf /tmp/hostapi /tmp/gocache /tmp/gopath",
|
||||||
|
"echo '==> fleet-hostapi enabled (localhost:9099/v1/host)'",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
# --- 2c. Admin access: sudo-enabled key user + locked root --------------
|
# --- 2c. Admin access: sudo-enabled key user + locked root --------------
|
||||||
# Fleet-generic (same user + key on every host => NOT per-host state). The
|
# Fleet-generic (same user + key on every host => NOT per-host state). The
|
||||||
# key comes from var.ssh_authorized_key (CI secret). If no key is supplied we
|
# key comes from var.ssh_authorized_key (CI secret). If no key is supplied we
|
||||||
@@ -691,9 +751,9 @@ build {
|
|||||||
# custom kiosk files: add them to fapolicyd's trust DB and rebuild it. weston
|
# custom kiosk files: add them to fapolicyd's trust DB and rebuild it. weston
|
||||||
# /chromium are RPM-installed so already trusted. (See docs/stig-deviations.md.)
|
# /chromium are RPM-installed so already trusted. (See docs/stig-deviations.md.)
|
||||||
"if command -v fapolicyd-cli >/dev/null 2>&1; then",
|
"if command -v fapolicyd-cli >/dev/null 2>&1; then",
|
||||||
" for f in /usr/local/bin/kiosk-session /usr/local/bin/fleet-sb-enroll; do { [ -f \"$f\" ] && fapolicyd-cli --file add \"$f\"; } || true; done",
|
" for f in /usr/local/bin/kiosk-session /usr/local/bin/fleet-sb-enroll /usr/local/bin/fleet-hostapi; do { [ -f \"$f\" ] && fapolicyd-cli --file add \"$f\"; } || true; done",
|
||||||
" fapolicyd-cli --update 2>/dev/null || true",
|
" fapolicyd-cli --update 2>/dev/null || true",
|
||||||
" echo '==> fapolicyd: trusted custom binaries (kiosk-session, fleet-sb-enroll) + rebuilt trust db'",
|
" echo '==> fapolicyd: trusted custom binaries (kiosk-session, fleet-sb-enroll, fleet-hostapi) + rebuilt trust db'",
|
||||||
"fi",
|
"fi",
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -766,6 +826,9 @@ build {
|
|||||||
# already downloaded to the host) — slims the erofs + cuts attack surface.
|
# already downloaded to the host) — slims the erofs + cuts attack surface.
|
||||||
# Keep auditd (it's part of the hardened runtime). Best-effort.
|
# Keep auditd (it's part of the hardened runtime). Best-effort.
|
||||||
"dnf remove -y openscap-scanner scap-security-guide openscap-utils 2>/dev/null || true",
|
"dnf remove -y openscap-scanner scap-security-guide openscap-utils 2>/dev/null || true",
|
||||||
|
# The Go toolchain was only needed to compile fleet-hostapi (2b-hostapi);
|
||||||
|
# the static binary ships, the compiler does not.
|
||||||
|
"dnf remove -y golang 2>/dev/null || true",
|
||||||
# The build-time STIG reports must not ship inside the read-only rootfs.
|
# The build-time STIG reports must not ship inside the read-only rootfs.
|
||||||
"rm -f /root/stig-report.html /root/stig-arf.xml",
|
"rm -f /root/stig-report.html /root/stig-arf.xml",
|
||||||
"dnf clean all",
|
"dnf clean all",
|
||||||
@@ -792,7 +855,7 @@ build {
|
|||||||
# home dirs) — we dropped mkfs.erofs --all-root, which used to mask this by
|
# home dirs) — we dropped mkfs.erofs --all-root, which used to mask this by
|
||||||
# flattening everything to root:root. sudo is a no-op if already root, and
|
# flattening everything to root:root. sudo is a no-op if already root, and
|
||||||
# the runner has passwordless sudo (cf. the `sudo apt-get` install step).
|
# the runner has passwordless sudo (cf. the `sudo apt-get` install step).
|
||||||
"sudo bash -c \"NVMET_TRADDR='${var.nvmet_traddr}' BOOT_SPLASH='${var.boot_splash}' STIG='${var.enable_stig}' bash scripts/extract-artifacts.sh output-qemu/diskless-build.qcow2 ${var.artifact_dir}/${var.image_version} ${var.disable_dot1x}\"",
|
"sudo bash -c \"IMAGE_VERSION='${var.image_version}' NVMET_TRADDR='${var.nvmet_traddr}' BOOT_SPLASH='${var.boot_splash}' STIG='${var.enable_stig}' bash scripts/extract-artifacts.sh output-qemu/diskless-build.qcow2 ${var.artifact_dir}/${var.image_version} ${var.disable_dot1x}\"",
|
||||||
# Hand the OUTPUT FILES back to the build user (their in-erofs ownership is
|
# Hand the OUTPUT FILES back to the build user (their in-erofs ownership is
|
||||||
# already baked correctly) so build-oci.sh can write the UKI and push with
|
# already baked correctly) so build-oci.sh can write the UKI and push with
|
||||||
# the build user's `buildah login`.
|
# the build user's `buildah login`.
|
||||||
|
|||||||
+166
-7
@@ -645,15 +645,69 @@ must be built around **never restarting the process that holds live connections.
|
|||||||
- **Planned SPDK restarts only via drain/failover** (multipath fail-over, or a
|
- **Planned SPDK restarts only via drain/failover** (multipath fail-over, or a
|
||||||
maintenance window with generous `ctrl_loss_tmo` so hosts recover not die). There
|
maintenance window with generous `ctrl_loss_tmo` so hosts recover not die). There
|
||||||
is **no zero-impact single-node SPDK restart.**
|
is **no zero-impact single-node SPDK restart.**
|
||||||
- **Implementation gap (current lab):** SPDK runs as a container with
|
- **[x] Long-lived target + online release control — IMPLEMENTED 2026-06-27.**
|
||||||
`spdk-setup.sh` as entrypoint + the artifact image mounted → effectively
|
The old gap (SPDK ran `spdk-setup.sh` as entrypoint, single-version,
|
||||||
**single-version, configured at container start**, so changing versions =
|
configured-at-start → changing versions = container restart = drop ALL
|
||||||
container restart = drop all connections. The work here is a **long-lived target
|
connections) is closed. Now in `serving/spdk/`:
|
||||||
+ an RPC-driven control path** ("add release / remove release / drain status")
|
- **`spdk-target.sh`** — persistent entrypoint: starts `nvmf_tgt` ONCE
|
||||||
decoupled from container lifecycle, with per-version subsystems.
|
(release-agnostic), creates the TCP transport + discovery listener, then
|
||||||
|
reconciles every staged release. Never restarted to change versions; on
|
||||||
|
(re)start re-adds everything under `./releases/` (a serving-host reboot
|
||||||
|
restores all served versions).
|
||||||
|
- **`spdk-release.sh`** — JSON-RPC control: `add`/`remove`/`list`/
|
||||||
|
`drain-status`/`reconcile`, **one subsystem per version** (distinct NQN, so
|
||||||
|
adding/removing one never disturbs others' connections), **drain-gated
|
||||||
|
removal** (refuses while a version has connected controllers unless
|
||||||
|
`--force`).
|
||||||
|
- **`spdk-ctl.sh`** — host ops: `load`/`unload`/`list`; stages a release's
|
||||||
|
artifacts from its OCI image into `./releases/<ver>/` and drives the live
|
||||||
|
target. Rolling update = `load` new → reboot hosts → `unload` old once
|
||||||
|
drained.
|
||||||
|
- **`spdk-loader.sh`** — DECLARATIVE rolling updates (the "init container that
|
||||||
|
stays up", compose service `spdk-loader`). On `compose up` after an
|
||||||
|
`IMAGE_VERSION` bump it stages + adds the new rootfs to the running target
|
||||||
|
ONLINE, then polls and **auto-reaps** any non-current version once it has
|
||||||
|
drained (0 conns) for a sustained grace window (`REAP_GRACE_SECONDS`, keep >
|
||||||
|
host `ctrl_loss_tmo`). So migration = bump `IMAGE_VERSION` in `.env` +
|
||||||
|
`docker compose up -d` (compose recreates the loader + the http/dnsmasq UKI
|
||||||
|
servers, but NOT the nvmeof target → no dropped connections). Single served
|
||||||
|
UKI ⇒ a 0-conn old version is genuinely orphaned (a returning host fetches
|
||||||
|
the current UKI), so auto-reap is safe; removal is still drain-gated +
|
||||||
|
grace-debounced. The loader + target share the RPC socket on a named volume
|
||||||
|
(`nvmf_tgt -r`); the loader reuses the SPDK image (has python3 + rpc.py).
|
||||||
|
- **No jq dependency:** `spdk-release.sh` parses RPC JSON with **python3** (the
|
||||||
|
SPDK image ships no jq); legacy artifacts (no per-version UUIDs in
|
||||||
|
verity.json) fall back to the fixed UUID pair with a warning.
|
||||||
|
- **Per-version namespace UUIDs** (`scripts/extract-artifacts.sh`): derived
|
||||||
|
deterministically (UUIDv5 of `image_version`), baked into the UKI cmdline +
|
||||||
|
`verity.json` (now also carries `subnqn`); the serving side reads them back
|
||||||
|
from each release's `verity.json`. Required so two versions don't collide on
|
||||||
|
the same `/dev/disk/by-id/nvme-uuid.<uuid>` host symlink.
|
||||||
|
- **Bootstrap unchanged:** `docker compose up` with one version still works —
|
||||||
|
the `/artifacts` image volume is seeded into `./releases/${IMAGE_VERSION}`.
|
||||||
|
- **PARTIALLY BENCH-VERIFIED 2026-06-28:** long-lived target boots a host end
|
||||||
|
to end — UKI served over HTTP → host boots → NVMe/TCP root connects to the
|
||||||
|
persistent `nvmf_tgt` (single version `spdk-permanence-02ffd7d`). Surfaced +
|
||||||
|
fixed several real bugs along the way: bootstrap seed must COPY not symlink
|
||||||
|
(a symlink into the version-pinned `/artifacts` re-pointed a stale release at
|
||||||
|
new data); `spdk-release` parses RPC JSON via **python3** (SPDK image has no
|
||||||
|
jq); legacy `verity.json` (no per-version UUIDs) falls back to the fixed pair;
|
||||||
|
mislabeled-dir guard (dir name vs verity.json `image_version`, case-insensitive);
|
||||||
|
dropped the redundant discovery-listener re-add (daemon logged "Listener
|
||||||
|
already exists"). **CI lowercases `image_version`** (gitea-main-ci vars step) —
|
||||||
|
a mixed-case branch (`SPDK-permanence`) had baked a capital
|
||||||
|
`fleet-SPDK-….efi` the lowercase boot URL couldn't fetch (iPXE "no such
|
||||||
|
file"); booted via a local retag workaround pending a lowercase rebuild.
|
||||||
|
- **STILL BENCH-VERIFY:** (a) `load` a 2nd version while a host is connected to
|
||||||
|
the 1st → 1st's connection survives (no AER/reset on the running host); (b)
|
||||||
|
`unload`/auto-reap drain-gate blocks while CONNS>0 and reaps after grace on a
|
||||||
|
LIVE host; (c) serving-host reboot reconciles all staged versions; (d)
|
||||||
|
per-version UUID build reproducible (two builds, same version → identical
|
||||||
|
`data_ns_uuid`/`hash_ns_uuid` + `root_hash`); (e) the full fix on a clean
|
||||||
|
LOWERCASE CI rebuild (no retag). See `serving/spdk/README.md`.
|
||||||
- **Capacity:** each concurrently-served version costs target memory/hugepages +
|
- **Capacity:** each concurrently-served version costs target memory/hugepages +
|
||||||
file-backed bdev overhead — plan for N live versions (current + previous +
|
file-backed bdev overhead — plan for N live versions (current + previous +
|
||||||
canary) during long rollouts.
|
canary) during long rollouts (raise `HUGEPAGES`).
|
||||||
- **Attach gating + discovery:** with several subsystems advertised, the dracut
|
- **Attach gating + discovery:** with several subsystems advertised, the dracut
|
||||||
connect + per-subsystem hostNQN allow-list must ensure a vX host attaches **only**
|
connect + per-subsystem hostNQN allow-list must ensure a vX host attaches **only**
|
||||||
vX's namespace (the by-UUID verity ref protects *what mounts*, but gate the
|
vX's namespace (the by-UUID verity ref protects *what mounts*, but gate the
|
||||||
@@ -824,6 +878,10 @@ smoke test.
|
|||||||
`docs/stig-deviations.md` #6 to reflect encrypted offload.
|
`docs/stig-deviations.md` #6 to reflect encrypted offload.
|
||||||
(RELP+TLS via `omrelp`/`imrelp` is the alternative if guaranteed delivery is
|
(RELP+TLS via `omrelp`/`imrelp` is the alternative if guaranteed delivery is
|
||||||
wanted alongside encryption.)
|
wanted alongside encryption.)
|
||||||
|
- **fleet-hostapi: scope CORS to the kiosk origin.** The host-info agent ships
|
||||||
|
with `FLEET_HOSTAPI_ALLOW_ORIGIN=*` (lab). For prod set it to the exact kiosk
|
||||||
|
origin (default `http://boot.fleet.lab`) via the unit's commented
|
||||||
|
`Environment=` line (or a drop-in). See #12.
|
||||||
|
|
||||||
## [ ] 9. (OPTIONAL / nice-to-have) Convert provisioning to Ansible where it pays off
|
## [ ] 9. (OPTIONAL / nice-to-have) Convert provisioning to Ansible where it pays off
|
||||||
|
|
||||||
@@ -984,3 +1042,104 @@ non-destructive TPM capability check (device, key persist, NV roundtrip, openssl
|
|||||||
TPM2 provider, CSR signed by the TPM key) to de-risk the TPM half before any of
|
TPM2 provider, CSR signed by the TPM key) to de-risk the TPM half before any of
|
||||||
this. Expected gap: the image installs `tpm2-tss` but no openssl TPM2 **provider**
|
this. Expected gap: the image installs `tpm2-tss` but no openssl TPM2 **provider**
|
||||||
(`tpm2-openssl`) — add it to the build once the self-test confirms.
|
(`tpm2-openssl`) — add it to the build once the self-test confirms.
|
||||||
|
|
||||||
|
## [~] 12. Workstation host-info API (`fleet-hostapi`) for the kiosk
|
||||||
|
|
||||||
|
**IMPLEMENTED 2026-06-28.** A small **Go, read-only, loopback-only** HTTP agent on
|
||||||
|
the workstation that reports host/OS/hardware facts to the kiosk SPA, per the
|
||||||
|
[HOST-API.md](../HOST-API.md) contract (`GET http://localhost:9099/v1/host`).
|
||||||
|
Source: `files/hostapi/` (stdlib-only, reproducible static build). Built + enabled
|
||||||
|
by provisioner **2b-hostapi**; toolchain removed in 2d; binary fapolicyd-trusted
|
||||||
|
in 2c-STIG-postfix (register #10). Chromium policy grants Private Network Access
|
||||||
|
scoped to the kiosk origin (`files/kiosk/chromium-kiosk-policy.json`).
|
||||||
|
|
||||||
|
- **Per-host safety:** host-specific fields (kernel, hostname, IP, MAC, board,
|
||||||
|
uptime) are read at RUNTIME; only image-generic facts (image version +
|
||||||
|
weston/chromium versions) are baked, to `/etc/fleet-hostinfo.json` (invariant
|
||||||
|
#1 preserved). Deliberately does NOT expose DMI serial (root-only/per-host) or
|
||||||
|
any TPM/cert/identity material.
|
||||||
|
- **Hardening shipped:** loopback bind enforced in code; `DynamicUser`, no caps,
|
||||||
|
`ProtectSystem=strict`, `MemoryDenyWriteExecute`, `SystemCallFilter=@system-service`,
|
||||||
|
`IPAddressDeny=any` (loopback only). Runs unprivileged, reads world-readable
|
||||||
|
files only.
|
||||||
|
|
||||||
|
**Lab posture LEFT AS-IS (decided 2026-06-28) — revisit for prod:**
|
||||||
|
- **CORS `Allow-Origin=*`** → scope to the kiosk origin (`http://boot.fleet.lab`).
|
||||||
|
Consolidated into #8.
|
||||||
|
- **SELinux:** runs in the default service domain (no custom policy module).
|
||||||
|
**BENCH-VERIFY** no AVC denials for `fleet-hostapi` on an enforcing booted host
|
||||||
|
(Wazuh/`fleet_audit` rule 100120 will surface any). If denials appear, decide:
|
||||||
|
write a confined SELinux policy module vs adjust the sandbox.
|
||||||
|
- **No auth** (justified: loopback + read-only + single-user appliance). Option if
|
||||||
|
the threat model tightens: shared token in `kiosk.env` + header check.
|
||||||
|
- **Info sensitivity:** exposes hostname/IP/MAC/board/versions to the kiosk. Drop
|
||||||
|
MAC/IP if that's later deemed too much.
|
||||||
|
- **Reproducibility:** the `golang` toolchain version is a reproducibility-sensitive
|
||||||
|
input (a package bump changes the binary → verity hash); covered by the
|
||||||
|
build-twice gate. Pin the go version only if drift becomes a problem.
|
||||||
|
|
||||||
|
**BENCH-VERIFY (functional):** (a) service starts under enforcing SELinux +
|
||||||
|
fapolicyd (not blocked); (b) the kiosk SPA's System page fetches `/v1/host`
|
||||||
|
successfully through the PNA allowance (preflight + GET); (c) facts render
|
||||||
|
correctly on real hardware (board/CPU/GPU from DMI/DRM, not the virtio build VM).
|
||||||
|
|
||||||
|
## [~] 13. Kiosk GPU compositing + Chromium flag ownership
|
||||||
|
|
||||||
|
**GPU FIX APPLIED 2026-06-28.** The 820 (Intel HD620/i915) ran Chromium at ~150%
|
||||||
|
CPU with periodic crashes: `--disable-gpu` in our `kiosk-session` forced software
|
||||||
|
raster/compositing. Root cause confirmed from the live cmdline. Fixes in
|
||||||
|
`files/kiosk/kiosk-session` + `diskless.pkr.hcl`:
|
||||||
|
- **Runtime GPU probe** (fleet-generic image boots real HW *and* virtio VMs): a
|
||||||
|
real KMS driver (`i915`/`amdgpu`/…) + `/dev/dri/renderD128` → GPU path
|
||||||
|
(`--ignore-gpu-blocklist --enable-gpu-rasterization --enable-zero-copy`, weston
|
||||||
|
GL renderer); else software (`--disable-gpu` + `weston --use-pixman`) so the VMs
|
||||||
|
still render. No per-host state baked.
|
||||||
|
- **kiosk user added to `render`** (renderD128 isn't logind-seat-ACL'd on EL9;
|
||||||
|
`video` only covers card0/KMS for weston).
|
||||||
|
- **Merged the distro wrapper's `--disable-features` into ours** — Chromium takes
|
||||||
|
the LAST `--disable-features` (not the union), so ours was silently dropping the
|
||||||
|
wrapper's Lens/Gemini/Compose disables.
|
||||||
|
|
||||||
|
**BENCH-VERIFY (on the 820):** GPU process alive (`chrome://gpu`), Compositor/
|
||||||
|
ThreadPool threads near-idle + CPU drop, `gt_act_freq_mhz` rises under load, no
|
||||||
|
SELinux AVC for `dri_device_t` from the kiosk domain (Wazuh `fleet_audit` 100120
|
||||||
|
will surface any — if present, add a scoped SELinux allow or confirm the kiosk
|
||||||
|
user's SELinux mapping is unconfined). Confirm VMs still render (software path).
|
||||||
|
|
||||||
|
**DECIDED 2026-06-28 — Option 1 (policy lock), APPLIED.** The distro chromium
|
||||||
|
wrapper (`/usr/bin/chromium-browser`) layers flags as `$CHROMIUM_FLAGS`
|
||||||
|
(`/etc/chromium/chromium.conf` → VA-API enables + AI disables; overridable via
|
||||||
|
`CHROMIUM_USER_FLAGS`) + **hardcoded** `$CHROMIUM_DISTRO_FLAGS`
|
||||||
|
(`--enable-plugins --enable-extensions --enable-user-scripts --enable-printing
|
||||||
|
--enable-sync --auto-ssl-client-auth`) + our `"$@"` (last). The hardcoded layer
|
||||||
|
isn't removable via env/conf — only by bypassing the wrapper. Rather than bypass
|
||||||
|
(Option 2, which would force us to replicate the wrapper's stdio sanitisation +
|
||||||
|
own the VA-API feature list across updates), we **neutralise in managed policy**
|
||||||
|
(policy beats cmdline flags, survives updates): `ExtensionInstallBlocklist=["*"]`
|
||||||
|
added to `files/kiosk/chromium-kiosk-policy.json` (blocks all extensions → also
|
||||||
|
kills the user-scripts surface). `--enable-sync` already locked by `SyncDisabled`;
|
||||||
|
`--auto-ssl-client-auth` is moot (no browser-store client cert; the TPM 802.1X
|
||||||
|
cert isn't one) — `AutoSelectCertificateForUrls` left unset. VA-API enables
|
||||||
|
inherited from `chromium.conf` unchanged. Option 2 (full flag ownership, bypass
|
||||||
|
the wrapper, own the VA-API `--enable-features`:
|
||||||
|
`AllowQt,WaylandLinuxDrmSyncobj,WaylandPerSurfaceScale,WaylandUiScale,AcceleratedVideoDecodeLinuxGL,AcceleratedVideoDecodeLinuxZeroCopyGL,AcceleratedVideoEncoder`)
|
||||||
|
remains documented here if a minimal/auditable cmdline is ever wanted.
|
||||||
|
**BENCH-VERIFY:** `chrome://policy` shows `ExtensionInstallBlocklist=["*"]` active;
|
||||||
|
extensions can't install.
|
||||||
|
|
||||||
|
## [ ] 14. (DESIGN) Centralised fleet management plane (health/version/usage/maintenance)
|
||||||
|
|
||||||
|
**DESIGN CAPTURED 2026-06-28 — [docs/fleet-management-design.md](fleet-management-design.md).**
|
||||||
|
Server-authoritative, pull-based, TPM-identified management plane reusing existing
|
||||||
|
pieces: `fleet-hostapi` as the on-host data source, the audit-collector→Wazuh
|
||||||
|
pipeline as telemetry transport, the digest-pinned deployment manifest as version
|
||||||
|
truth, and the SPDK `spdk-loader`/`spdk-release` rolling-update flow (#6) as the
|
||||||
|
redeploy/drain mechanics. Hosts stay stateless (identity from the TPM cert, no
|
||||||
|
baked enrollment); control is pull/reconcile (no inbound surface); convergence to
|
||||||
|
a new version = reboot. **Phased build:** (1) push hostapi snapshots as structured
|
||||||
|
events into Wazuh + fleet dashboard (health/version-drift/offline/usage/alerting);
|
||||||
|
(2) metrics push (OTLP/remote-write) → VictoriaMetrics + Grafana; (3) mTLS
|
||||||
|
desired-state pull (version pin, reboot/drain, kiosk URL) wired to the deployment
|
||||||
|
manifest + SPDK flow. Build-time decisions (telemetry encoding, metrics/desired-
|
||||||
|
state store, per-host vs cohort, osquery/FleetDM eval) listed in the design note.
|
||||||
|
Depends on the step-ca fleet CA (#8) for the mTLS identity.
|
||||||
|
|||||||
@@ -0,0 +1,193 @@
|
|||||||
|
# Fleet management plane — design note
|
||||||
|
|
||||||
|
**Status:** DESIGN (not built). Captured 2026-06-28. Build phased — see "Phasing"
|
||||||
|
and `docs/TODO.md` #14.
|
||||||
|
|
||||||
|
Centralised monitoring + maintenance of the diskless kiosk fleet: **health,
|
||||||
|
version, usage, and maintenance actions**. This note is the authoritative design;
|
||||||
|
it deliberately reuses what already exists rather than bolting on a generic MDM,
|
||||||
|
because the base architecture (immutable UKI + NVMe-root, TPM identity, no
|
||||||
|
per-host state, segmented network) makes most off-the-shelf kiosk-management tools
|
||||||
|
a poor fit.
|
||||||
|
|
||||||
|
## Design philosophy (forced by the architecture invariants)
|
||||||
|
|
||||||
|
These are consequences of the CLAUDE.md invariants, not preferences:
|
||||||
|
|
||||||
|
1. **Server-authoritative, hosts are cattle.** The root is immutable erofs +
|
||||||
|
dm-verity on a tmpfs overlay — nothing survives reboot. So **all durable state
|
||||||
|
(history, inventory, desired-state) lives server-side**; the on-host agent is
|
||||||
|
stateless and buffers only in RAM (best-effort).
|
||||||
|
2. **Identity from the TPM, never baked.** No per-host state in any artifact
|
||||||
|
(invariant #1). The agent derives identity at runtime from the **TPM-held
|
||||||
|
cert** (key `0x81010001`, cert NV `0x01800001`) — the same identity used for
|
||||||
|
802.1X EAP-TLS and the NVMe hostnqn. No enrollment token in the image.
|
||||||
|
3. **Pull-based control, no inbound surface.** The boot LAN is locked down and
|
||||||
|
segmented; we do not open an always-on inbound control port on hosts. Hosts
|
||||||
|
**poll** a desired-state endpoint and reconcile. This also fits the immutable +
|
||||||
|
digest-pinned model: *convergence to a new version = reboot*.
|
||||||
|
4. **Reuse the existing buses.** Telemetry rides the already-built
|
||||||
|
`auditd → audit-collector relay → Wazuh` pipeline; version truth is the
|
||||||
|
existing digest-pinned deployment manifest; redeploy/drain mechanics are the
|
||||||
|
SPDK rolling-update flow (TODO #6). The management plane orchestrates these; it
|
||||||
|
does not reinvent them.
|
||||||
|
5. **Minimal, reproducible, sandboxed agent.** Same posture as `fleet-hostapi`
|
||||||
|
(loopback/least-priv, `DynamicUser`, fapolicyd-trusted, reproducible static
|
||||||
|
build).
|
||||||
|
|
||||||
|
## Components
|
||||||
|
|
||||||
|
### 1. Identity — mTLS with the TPM cert
|
||||||
|
|
||||||
|
The agent presents the TPM-held client cert over **mTLS** to the management
|
||||||
|
endpoint; the server authenticates the host by cert CN/SAN, which is the canonical
|
||||||
|
fleet identity (= NVMe hostnqn = 802.1X identity). Reuses the **step-ca fleet CA**
|
||||||
|
(`serving/dot1x-bench/`). No baked secret; the host also pins the server cert.
|
||||||
|
|
||||||
|
- **Secure path:** full mTLS, TPM cert. Strong, no secrets in the image.
|
||||||
|
- **Open/test path (`rd.dot1x=0`):** fall back to dnsmasq fleet-name + MAC as a
|
||||||
|
weak identity. **Lab only** — flagged as insufficient for production.
|
||||||
|
|
||||||
|
Inventory is keyed by **cert serial / hostnqn**, joined to the dnsmasq starship
|
||||||
|
name (`fleet-namer`), MAC, last-seen, running/desired version, and health.
|
||||||
|
|
||||||
|
### 2. On-host agent — extend `fleet-hostapi`
|
||||||
|
|
||||||
|
`fleet-hostapi` (`files/hostapi/`) is already the only fleet-authored component on
|
||||||
|
the hardware and already models os/hardware/net/kiosk facts. Add a **report/push
|
||||||
|
mode** (or a tiny sibling that reads its loopback `/v1/host` and ships it). Keep it
|
||||||
|
stateless; buffer in tmpfs only; identity from the TPM.
|
||||||
|
|
||||||
|
Reported facts:
|
||||||
|
|
||||||
|
- **Health:** CPU/load, GPU act-freq, temps, **tmpfs overlay usage** (the
|
||||||
|
RAM-exhaustion → NVMe-keepalive risk — the single most important kiosk health
|
||||||
|
signal), NVMe-root link state, 802.1X/MACsec `Authorized`, SELinux enforcing,
|
||||||
|
chrony sync, `kiosk.service` active.
|
||||||
|
- **Version:** running image version (`/etc/fleet-image`), rootfs digest, kernel.
|
||||||
|
- **Usage:** kiosk URL, session uptime, **Chromium reload/crash count** (from the
|
||||||
|
`kiosk-session` restart loop — the exact signal behind the GPU-bug diagnosis),
|
||||||
|
input idle/active (liveness only, never content — privacy-aware), display power
|
||||||
|
state.
|
||||||
|
|
||||||
|
Cadence: heartbeat (~30–60 s) + event-driven (crash, auth loss, version change).
|
||||||
|
|
||||||
|
### 3. Transport
|
||||||
|
|
||||||
|
**Phase 1 — ride the Wazuh pipeline.** Emit snapshots as structured JSON events
|
||||||
|
through the existing `audit-collector` relay → `wazuh1`. Add `fleet_health` /
|
||||||
|
`fleet_kiosk` decoders + rules alongside `fleet_audit` (`serving/siem/config/`).
|
||||||
|
Wazuh then provides search, dashboards, and alerting for free, and it's already
|
||||||
|
secured-path-aware.
|
||||||
|
|
||||||
|
**Phase 2 — metrics push** for high-resolution time-series (CPU/GPU/mem trends):
|
||||||
|
**push** via OTLP or Prometheus remote-write to a collector on the mgmt host →
|
||||||
|
VictoriaMetrics/Prometheus. **Push, never scrape** — scraping would require an
|
||||||
|
inbound port on every locked host, against the posture.
|
||||||
|
|
||||||
|
### 4. Control / maintenance — pull-based desired-state reconciliation
|
||||||
|
|
||||||
|
Each host periodically fetches **its** desired state (mTLS, keyed by TPM cert):
|
||||||
|
target version, kiosk URL, `reboot-at`, `drain`. The agent reconciles; because the
|
||||||
|
root is immutable, **convergence = reboot** (a reboot re-PXEs and picks up the new
|
||||||
|
UKI).
|
||||||
|
|
||||||
|
- The **deployment manifest is already the desired-state for versions** (one
|
||||||
|
commit per release, digest-pinned). Extend it to per-host / per-cohort desired
|
||||||
|
state. Rollback = revert the commit (unchanged model).
|
||||||
|
- **Maintenance actions** (reboot, drain, restart kiosk, change URL, request logs)
|
||||||
|
ride the same poll: the server sets the intent; the host obeys on next poll.
|
||||||
|
- This dovetails with the **SPDK rolling update** (TODO #6): "redeploy a cohort" =
|
||||||
|
bump the manifest → `spdk-loader` serves the new version online → the management
|
||||||
|
plane tells those hosts to drain/reboot → it watches `spdk-release list` CONNS
|
||||||
|
fall → the old version auto-reaps. The management plane *orchestrates* this loop.
|
||||||
|
- **No always-on inbound control socket.** Admin SSH (key-only ECDSA) stays purely
|
||||||
|
break-glass for exceptional manual ops.
|
||||||
|
|
||||||
|
### 5. Server side (serving host or a dedicated mgmt node, e.g. alongside `wazuh1`)
|
||||||
|
|
||||||
|
- **Mgmt service** (small Go, mirrors the `fleet-hostapi` style): mTLS endpoint
|
||||||
|
that accepts host reports and serves per-host desired-state; authenticates hosts
|
||||||
|
by TPM cert via the fleet CA.
|
||||||
|
- **Store:** time-series (VictoriaMetrics/Prometheus) for metrics; Wazuh /
|
||||||
|
OpenSearch for events/logs; git or a tiny DB for desired-state + inventory.
|
||||||
|
- **Dashboards:** Grafana (metric trends) + the Wazuh dashboard (events/alerts) +
|
||||||
|
a small fleet-overview board (offline hosts, version drift, crash rates, tmpfs
|
||||||
|
pressure). Could be a sibling of the `serving/kiosk-app` SPA.
|
||||||
|
- **Alerting:** host offline > N min; Chromium crash-loop; **running ≠ desired
|
||||||
|
version**; tmpfs near-full; MACsec/auth down; NVMe-root reconnect storms.
|
||||||
|
|
||||||
|
## Data model (sketch)
|
||||||
|
|
||||||
|
```
|
||||||
|
Host (by cert-serial / hostnqn)
|
||||||
|
├─ identity: cert serial, hostnqn, MAC, starship-name (dnsmasq)
|
||||||
|
├─ version: image_version, rootfs digest, kernel, desired_version
|
||||||
|
├─ health: cpu/load, gpu_freq, temps, tmpfs_used%, nvme_root, auth/macsec,
|
||||||
|
│ selinux, chrony, kiosk_active, last_seen
|
||||||
|
├─ usage: kiosk_url, session_uptime, chromium_crashes, input_active, display
|
||||||
|
└─ desired: target_version, kiosk_url, reboot_at, drain (server-authoritative)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Mapping to existing building blocks
|
||||||
|
|
||||||
|
| Need | Reuse |
|
||||||
|
|---|---|
|
||||||
|
| Host identity | TPM cert / hostnqn / step-ca fleet CA (`serving/dot1x-bench/`) |
|
||||||
|
| On-host data | `fleet-hostapi` `/v1/host` (`files/hostapi/`) |
|
||||||
|
| Telemetry + alerting | audit-collector relay → Wazuh (`serving/siem/`) |
|
||||||
|
| Friendly names / inventory | dnsmasq `fleet-namer` (MAC → name → DNS) |
|
||||||
|
| Version truth + rollback | digest-pinned deployment manifest (git) |
|
||||||
|
| Redeploy / drain mechanics | SPDK `spdk-loader` + `spdk-release` (TODO #6) |
|
||||||
|
| Build / sign / publish | Gitea CI + OCI images |
|
||||||
|
|
||||||
|
## Phasing
|
||||||
|
|
||||||
|
1. **Phase 1 (mostly-built):** push `fleet-hostapi` snapshots as structured events
|
||||||
|
into Wazuh; add decoders/rules + a fleet dashboard → health, version-drift,
|
||||||
|
offline detection, usage, alerting. Reuses the existing pipeline end to end.
|
||||||
|
2. **Phase 2:** metrics push (OTLP / remote-write) → VictoriaMetrics + Grafana for
|
||||||
|
trend graphs.
|
||||||
|
3. **Phase 3:** mTLS desired-state pull (version pin, reboot/drain, kiosk URL),
|
||||||
|
wired to the deployment manifest + SPDK rolling-update flow.
|
||||||
|
|
||||||
|
## Build vs buy
|
||||||
|
|
||||||
|
Off-the-shelf fits poorly because the base is unusual:
|
||||||
|
|
||||||
|
- **FleetDM + osquery** — closest for the query/health side; worth evaluating. But
|
||||||
|
osquery enrollment carries per-host state, so it must be run carefully (identity
|
||||||
|
from TPM, enrollment in tmpfs only) or it breaks invariant #1.
|
||||||
|
- **Mender / balena / typical kiosk MDM** — assume mutable disk + their own OTA +
|
||||||
|
enrollment. We already have a *better* version mechanism (UKI + NVMe-root +
|
||||||
|
digest manifest), so their OTA is redundant and their disk model conflicts.
|
||||||
|
|
||||||
|
**Decision: build the thin mgmt service + reconcile loop; reuse Wazuh/Grafana for
|
||||||
|
the heavy lifting.** The hard parts (identity, immutable versioning, a telemetry
|
||||||
|
bus) already exist.
|
||||||
|
|
||||||
|
## Security posture
|
||||||
|
|
||||||
|
- mTLS + TPM cert = strong identity, zero baked secrets, reuses the 802.1X CA.
|
||||||
|
- Pull-based control = **no new inbound attack surface** on locked hosts.
|
||||||
|
- Telemetry stays on the mgmt/boot LAN, TLS'd (same gap as the Wazuh-relay-TLS
|
||||||
|
TODO) — no internet egress, air-gap intact.
|
||||||
|
- Agent stays minimal/sandboxed/fapolicyd-trusted/reproducible (`fleet-hostapi`
|
||||||
|
template); usage capture is liveness-only, never content (privacy).
|
||||||
|
- Fleet-wide inventory is recon-valuable → protect the management plane (auth +
|
||||||
|
segmentation); never expose its dashboards to the boot LAN.
|
||||||
|
|
||||||
|
## Open questions / decisions for build time
|
||||||
|
|
||||||
|
- **Telemetry encoding:** structured syslog/JSON into Wazuh (Phase 1) vs native
|
||||||
|
OTLP from the start. Leaning Wazuh-first for reuse.
|
||||||
|
- **Metrics store:** VictoriaMetrics vs Prometheus (VM is lighter for push +
|
||||||
|
long retention).
|
||||||
|
- **Desired-state store:** git (GitOps, aud/rollback for free, matches the
|
||||||
|
deployment-manifest model) vs a small DB (richer per-host scheduling).
|
||||||
|
- **Per-host vs per-cohort** desired state granularity (cohorts likely enough;
|
||||||
|
per-host for canaries).
|
||||||
|
- **Open-path identity:** MAC/fleet-name is weak — gate the management plane to the
|
||||||
|
secure path, or accept lab-only for the open path.
|
||||||
|
- **osquery/FleetDM** evaluation for the read side before building a bespoke
|
||||||
|
collector.
|
||||||
@@ -143,10 +143,13 @@ hotkey disabled (e.g. a physically exposed terminal), re-select both rule IDs.
|
|||||||
- **SELinux enforcing** (`selinux --enforcing`; labels preserved into the erofs).
|
- **SELinux enforcing** (`selinux --enforcing`; labels preserved into the erofs).
|
||||||
- **fapolicyd** kept **enabled** (NOT waived) — it's the real exec-allowlist guard
|
- **fapolicyd** kept **enabled** (NOT waived) — it's the real exec-allowlist guard
|
||||||
for the writable RAM overlay, which dm-verity does **not** cover. The only
|
for the writable RAM overlay, which dm-verity does **not** cover. The only
|
||||||
adjustment: the one non-RPM executable, `/usr/local/bin/kiosk-session`, is added
|
adjustment: the few non-RPM executables we ship are added to fapolicyd's trust DB
|
||||||
to fapolicyd's trust DB at build (provisioner 2c-STIG-postfix) so the kiosk runs;
|
at build (provisioner 2c-STIG-postfix) so they run; everything else stays
|
||||||
everything else stays RPM-trust-only. (fapolicyd blocking that binary was the
|
RPM-trust-only. Trusted custom binaries: `/usr/local/bin/kiosk-session`,
|
||||||
original kiosk-won't-start cause.)
|
`/usr/local/bin/fleet-sb-enroll`, `/usr/local/bin/fleet-hostapi` (the loopback
|
||||||
|
host-info agent, built from `files/hostapi`). Each is built into the immutable,
|
||||||
|
dm-verity-protected root, so trusting it doesn't weaken the overlay exec guard.
|
||||||
|
(fapolicyd blocking `kiosk-session` was the original kiosk-won't-start cause.)
|
||||||
- **auditd** enabled; STIG audit rules applied; logs locally.
|
- **auditd** enabled; STIG audit rules applied; logs locally.
|
||||||
- **USBGuard** enabled + enforcing (`block` target) with a HID/hub allowlist and
|
- **USBGuard** enabled + enforcing (`block` target) with a HID/hub allowlist and
|
||||||
`AuditBackend=LinuxAudit` (see #12) — blocks USB mass storage / unknown device
|
`AuditBackend=LinuxAudit` (see #12) — blocks USB mass storage / unknown device
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
# fleet-hostapi — workstation host-info agent
|
||||||
|
|
||||||
|
A tiny **read-only, loopback-only** Go HTTP service that runs on the diskless
|
||||||
|
workstation and reports facts about the machine the kiosk renders on. It is the
|
||||||
|
OS-side implementation of the contract in [`../../HOST-API.md`](../../HOST-API.md)
|
||||||
|
(authored alongside the kiosk-app SPA that consumes it).
|
||||||
|
|
||||||
|
```
|
||||||
|
GET http://localhost:9099/v1/host -> application/json (os, hardware, net, kiosk, agent)
|
||||||
|
GET http://localhost:9099/healthz -> 200 ok
|
||||||
|
```
|
||||||
|
|
||||||
|
## Properties (and why)
|
||||||
|
|
||||||
|
- **Loopback only.** Binds `127.0.0.1`; refuses to start on any routable address.
|
||||||
|
Host facts must never reach the boot LAN.
|
||||||
|
- **Read only.** No endpoint mutates anything; it reads `/proc`, `/sys`,
|
||||||
|
`/etc/os-release`, and the build-time facts file.
|
||||||
|
- **No per-host state baked in** (invariant #1). Host-specific fields (kernel,
|
||||||
|
hostname, IP, MAC, board, uptime) are read at **runtime**. The only baked facts
|
||||||
|
are image-generic — image version + weston/chromium versions — written to
|
||||||
|
`/etc/fleet-hostinfo.json` by the build provisioner.
|
||||||
|
- **Stdlib only → reproducible build** (the verity hash is a security property).
|
||||||
|
Built `CGO_ENABLED=0 -trimpath -buildvcs=false -ldflags=-buildid=`; identical
|
||||||
|
bytes across builds with the same Go toolchain.
|
||||||
|
- **CORS + Private Network Access.** The SPA is cross-origin and the fetch is a
|
||||||
|
Chromium private-network request, so responses carry
|
||||||
|
`Access-Control-Allow-Private-Network: true`. The matching browser-side
|
||||||
|
allowance is in `files/kiosk/chromium-kiosk-policy.json`
|
||||||
|
(`InsecurePrivateNetworkRequestsAllowedForUrls`, scoped to the kiosk origin).
|
||||||
|
|
||||||
|
## Build / install
|
||||||
|
|
||||||
|
Done by the `2b-hostapi` provisioner in `diskless.pkr.hcl`: installs the Go
|
||||||
|
toolchain, builds the static binary to `/usr/local/bin/fleet-hostapi`, writes the
|
||||||
|
facts file, installs + enables `fleet-hostapi.service`, and adds the binary to
|
||||||
|
fapolicyd trust (2c-STIG-postfix). The toolchain is removed in 2d hygiene — only
|
||||||
|
the static binary ships.
|
||||||
|
|
||||||
|
Local build:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false \
|
||||||
|
-ldflags '-s -w -buildid=' -o fleet-hostapi .
|
||||||
|
```
|
||||||
|
|
||||||
|
## Config (env, set in the unit)
|
||||||
|
|
||||||
|
| Var | Default | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| `FLEET_HOSTAPI_ADDR` | `127.0.0.1:9099` | must be loopback |
|
||||||
|
| `FLEET_HOSTAPI_ALLOW_ORIGIN` | `*` | tighten to the kiosk origin in prod |
|
||||||
|
| `FLEET_HOSTAPI_FACTS` | `/etc/fleet-hostinfo.json` | build-time facts |
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Fleet workstation host-info API (loopback, read-only)
|
||||||
|
Documentation=https://git.bitasylum.org/audiobuzz/diskless-workstation/HOST-API.md
|
||||||
|
# Needs networking up only to read the primary IP/MAC; the listener is loopback.
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=exec
|
||||||
|
ExecStart=/usr/local/bin/fleet-hostapi
|
||||||
|
Environment=FLEET_HOSTAPI_ADDR=127.0.0.1:9099
|
||||||
|
# Lab default: any origin may read (it's a loopback, read-only facts service on a
|
||||||
|
# single-purpose kiosk). Tighten to the exact kiosk origin in production, e.g.
|
||||||
|
# Environment=FLEET_HOSTAPI_ALLOW_ORIGIN=http://boot.fleet.lab
|
||||||
|
Environment=FLEET_HOSTAPI_ALLOW_ORIGIN=*
|
||||||
|
Environment=FLEET_HOSTAPI_FACTS=/etc/fleet-hostinfo.json
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
|
||||||
|
# --- sandboxing (STIG-friendly; the agent only READS /proc, /sys, /etc) -------
|
||||||
|
# Runs as a transient unprivileged user with no capabilities; everything it
|
||||||
|
# reads is world-readable, so it needs no privilege at all.
|
||||||
|
DynamicUser=yes
|
||||||
|
CapabilityBoundingSet=
|
||||||
|
AmbientCapabilities=
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectKernelLogs=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
ProtectClock=yes
|
||||||
|
ProtectHostname=yes
|
||||||
|
ProtectProc=default
|
||||||
|
RestrictNamespaces=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
MemoryDenyWriteExecute=yes
|
||||||
|
SystemCallArchitectures=native
|
||||||
|
SystemCallFilter=@system-service
|
||||||
|
SystemCallFilter=~@privileged @resources
|
||||||
|
RestrictAddressFamilies=AF_INET AF_INET6
|
||||||
|
# Kernel-enforced loopback-only egress/ingress (belt + suspenders to the bind).
|
||||||
|
IPAddressAllow=localhost
|
||||||
|
IPAddressDeny=any
|
||||||
|
UMask=0077
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
// fleet-hostapi: the workstation host-info agent (HOST-API.md contract).
|
||||||
|
// Stdlib only — no external modules — so the build needs no network and is
|
||||||
|
// trivially reproducible (a hard requirement: the rootfs/verity hash must be a
|
||||||
|
// pure function of the inputs). Keep it dependency-free.
|
||||||
|
module fleet-hostapi
|
||||||
|
|
||||||
|
go 1.21
|
||||||
@@ -0,0 +1,400 @@
|
|||||||
|
// fleet-hostapi — read-only workstation host-info agent for the kiosk.
|
||||||
|
//
|
||||||
|
// Implements the contract in HOST-API.md: a tiny HTTP service on the diskless
|
||||||
|
// workstation that reports facts about the machine the kiosk renders on. The
|
||||||
|
// kiosk SPA (served from the boot server, a different origin) fetches
|
||||||
|
//
|
||||||
|
// GET http://localhost:9099/v1/host -> application/json
|
||||||
|
//
|
||||||
|
// Design constraints (see CLAUDE.md invariants):
|
||||||
|
// - LOOPBACK ONLY. Never bind a routable address — this leaks host facts and
|
||||||
|
// must never reach the boot LAN.
|
||||||
|
// - READ ONLY. No endpoint mutates anything; the agent only reads /proc, /sys,
|
||||||
|
// /etc/os-release, and a build-time facts file.
|
||||||
|
// - NO PER-HOST STATE baked into the image. Everything host-specific (kernel,
|
||||||
|
// hostname, IP, MAC, board, uptime) is read at RUNTIME. The only baked facts
|
||||||
|
// are image-generic (image version + weston/chromium versions), written to a
|
||||||
|
// facts file by the build provisioner — same for every host of a release.
|
||||||
|
// - Stdlib only -> reproducible build (the verity hash is a security property).
|
||||||
|
//
|
||||||
|
// CORS + Private Network Access: the SPA is cross-origin AND the fetch is a
|
||||||
|
// Chromium "private network" request, so every response (incl. the OPTIONS
|
||||||
|
// preflight) carries Access-Control-Allow-Private-Network: true.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/json"
|
||||||
|
"flag"
|
||||||
|
"log"
|
||||||
|
"math"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// agentVersion is overridable at build time via -ldflags "-X main.agentVersion=…".
|
||||||
|
var agentVersion = "1.0.0"
|
||||||
|
|
||||||
|
// ---- response shape (HOST-API.md) — all fields optional, omit what's unknown --
|
||||||
|
|
||||||
|
type osInfo struct {
|
||||||
|
Name string `json:"name,omitempty"`
|
||||||
|
Version string `json:"version,omitempty"`
|
||||||
|
Kernel string `json:"kernel,omitempty"`
|
||||||
|
Image string `json:"image,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type hwInfo struct {
|
||||||
|
CPU string `json:"cpu,omitempty"`
|
||||||
|
Cores int `json:"cores,omitempty"`
|
||||||
|
MemGB int `json:"memGB,omitempty"`
|
||||||
|
GPU string `json:"gpu,omitempty"`
|
||||||
|
Board string `json:"board,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type netInfo struct {
|
||||||
|
Hostname string `json:"hostname,omitempty"`
|
||||||
|
IP string `json:"ip,omitempty"`
|
||||||
|
MAC string `json:"mac,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type kioskInfo struct {
|
||||||
|
Uptime string `json:"uptime,omitempty"`
|
||||||
|
Weston string `json:"weston,omitempty"`
|
||||||
|
Chromium string `json:"chromium,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type agentInfo struct {
|
||||||
|
Version string `json:"version,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type hostResponse struct {
|
||||||
|
OS osInfo `json:"os"`
|
||||||
|
Hardware hwInfo `json:"hardware"`
|
||||||
|
Net netInfo `json:"net"`
|
||||||
|
Kiosk kioskInfo `json:"kiosk"`
|
||||||
|
Agent agentInfo `json:"agent"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildFacts is the image-generic, build-time facts file (written by the Packer
|
||||||
|
// provisioner). Distinct from anything per-host.
|
||||||
|
type buildFacts struct {
|
||||||
|
Image string `json:"image"`
|
||||||
|
Weston string `json:"weston"`
|
||||||
|
Chromium string `json:"chromium"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
addr := flag.String("addr", env("FLEET_HOSTAPI_ADDR", "127.0.0.1:9099"), "listen address (MUST be loopback)")
|
||||||
|
origin := flag.String("allow-origin", env("FLEET_HOSTAPI_ALLOW_ORIGIN", "*"), "Access-Control-Allow-Origin value")
|
||||||
|
factsPath := flag.String("facts", env("FLEET_HOSTAPI_FACTS", "/etc/fleet-hostinfo.json"), "build-time facts file")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
// Refuse to bind anything that isn't loopback — a hard guardrail against a
|
||||||
|
// misconfiguration exposing host facts on the boot LAN.
|
||||||
|
if host, _, err := net.SplitHostPort(*addr); err != nil || !isLoopback(host) {
|
||||||
|
log.Fatalf("refusing to listen on %q: bind address must be loopback (127.0.0.1/::1)", *addr)
|
||||||
|
}
|
||||||
|
|
||||||
|
facts := loadFacts(*factsPath)
|
||||||
|
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.HandleFunc("/v1/host", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
cors(w, *origin)
|
||||||
|
if r.Method == http.MethodOptions {
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.Header().Set("Cache-Control", "no-store")
|
||||||
|
enc := json.NewEncoder(w)
|
||||||
|
enc.SetIndent("", " ")
|
||||||
|
_ = enc.Encode(collect(facts))
|
||||||
|
})
|
||||||
|
// Liveness for the unit / debugging (no host facts).
|
||||||
|
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "text/plain")
|
||||||
|
_, _ = w.Write([]byte("ok\n"))
|
||||||
|
})
|
||||||
|
|
||||||
|
srv := &http.Server{
|
||||||
|
Addr: *addr,
|
||||||
|
Handler: mux,
|
||||||
|
ReadHeaderTimeout: 5 * time.Second,
|
||||||
|
ReadTimeout: 10 * time.Second,
|
||||||
|
WriteTimeout: 10 * time.Second,
|
||||||
|
IdleTimeout: 60 * time.Second,
|
||||||
|
}
|
||||||
|
log.Printf("fleet-hostapi %s listening on %s (loopback, read-only)", agentVersion, *addr)
|
||||||
|
log.Fatal(srv.ListenAndServe())
|
||||||
|
}
|
||||||
|
|
||||||
|
func cors(w http.ResponseWriter, origin string) {
|
||||||
|
// Private Network Access: the kiosk page (boot-server origin, an insecure/
|
||||||
|
// public context) fetching localhost is a "private network" request — the
|
||||||
|
// preflight needs this header or Chromium blocks it. See HOST-API.md.
|
||||||
|
w.Header().Set("Access-Control-Allow-Origin", origin)
|
||||||
|
w.Header().Set("Access-Control-Allow-Private-Network", "true")
|
||||||
|
w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS")
|
||||||
|
w.Header().Set("Vary", "Origin")
|
||||||
|
}
|
||||||
|
|
||||||
|
// collect gathers all host facts. Each source is best-effort: a failed read
|
||||||
|
// leaves its field empty (the SPA renders "—" for missing fields).
|
||||||
|
func collect(f buildFacts) hostResponse {
|
||||||
|
var resp hostResponse
|
||||||
|
|
||||||
|
// os
|
||||||
|
rel := parseKV("/etc/os-release")
|
||||||
|
resp.OS.Name = firstNonEmpty(rel["NAME"], rel["ID"])
|
||||||
|
resp.OS.Version = firstNonEmpty(rel["VERSION_ID"], rel["VERSION"])
|
||||||
|
resp.OS.Kernel = kernelRelease()
|
||||||
|
resp.OS.Image = f.Image
|
||||||
|
|
||||||
|
// hardware
|
||||||
|
resp.Hardware.CPU = cpuModel()
|
||||||
|
resp.Hardware.Cores = runtime.NumCPU()
|
||||||
|
resp.Hardware.MemGB = memGB()
|
||||||
|
resp.Hardware.GPU = gpuModel()
|
||||||
|
resp.Hardware.Board = boardModel()
|
||||||
|
|
||||||
|
// net
|
||||||
|
resp.Net.Hostname, _ = os.Hostname()
|
||||||
|
resp.Net.IP, resp.Net.MAC = primaryIfaceAddr()
|
||||||
|
|
||||||
|
// kiosk
|
||||||
|
resp.Kiosk.Uptime = uptimeHuman()
|
||||||
|
resp.Kiosk.Weston = f.Weston
|
||||||
|
resp.Kiosk.Chromium = f.Chromium
|
||||||
|
|
||||||
|
resp.Agent.Version = agentVersion
|
||||||
|
return resp
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- collectors -----------------------------------------------------------
|
||||||
|
|
||||||
|
func kernelRelease() string {
|
||||||
|
var u syscall.Utsname
|
||||||
|
if err := syscall.Uname(&u); err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return charsToString(u.Release[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func cpuModel() string {
|
||||||
|
f, err := os.Open("/proc/cpuinfo")
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
sc := bufio.NewScanner(f)
|
||||||
|
for sc.Scan() {
|
||||||
|
line := sc.Text()
|
||||||
|
// x86: "model name". arm64: "Hardware"/"Processor" if present.
|
||||||
|
for _, key := range []string{"model name", "Hardware", "Processor"} {
|
||||||
|
if strings.HasPrefix(line, key) {
|
||||||
|
if _, v, ok := strings.Cut(line, ":"); ok {
|
||||||
|
return strings.TrimSpace(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = sc.Err() // best-effort read; empty result on any error
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func memGB() int {
|
||||||
|
for _, line := range readLines("/proc/meminfo") {
|
||||||
|
if strings.HasPrefix(line, "MemTotal:") {
|
||||||
|
fields := strings.Fields(line) // MemTotal: <kB> kB
|
||||||
|
if len(fields) >= 2 {
|
||||||
|
if kb, err := strconv.ParseFloat(fields[1], 64); err == nil {
|
||||||
|
return int(math.Round(kb / 1024.0 / 1024.0))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// boardModel: vendor + product from DMI (world-readable sysfs — NOT product_serial,
|
||||||
|
// which is root-only and per-host; we never read it).
|
||||||
|
func boardModel() string {
|
||||||
|
vendor := strings.TrimSpace(readFile("/sys/class/dmi/id/sys_vendor"))
|
||||||
|
product := strings.TrimSpace(readFile("/sys/class/dmi/id/product_name"))
|
||||||
|
if vendor == "" && product == "" {
|
||||||
|
vendor = strings.TrimSpace(readFile("/sys/class/dmi/id/board_vendor"))
|
||||||
|
product = strings.TrimSpace(readFile("/sys/class/dmi/id/board_name"))
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(strings.TrimSpace(vendor + " " + product))
|
||||||
|
}
|
||||||
|
|
||||||
|
// gpuModel: best-effort from the first DRM card's PCI ids. No PCI-ID database —
|
||||||
|
// we map the few known vendors and otherwise emit the raw ids. Often empty on a
|
||||||
|
// headless/virtio build VM; that's fine (optional field).
|
||||||
|
func gpuModel() string {
|
||||||
|
for _, card := range []string{"card0", "card1"} {
|
||||||
|
base := "/sys/class/drm/" + card + "/device/"
|
||||||
|
vid := strings.TrimSpace(readFile(base + "vendor"))
|
||||||
|
did := strings.TrimSpace(readFile(base + "device"))
|
||||||
|
if vid == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := map[string]string{
|
||||||
|
"0x8086": "Intel",
|
||||||
|
"0x1002": "AMD",
|
||||||
|
"0x10de": "NVIDIA",
|
||||||
|
"0x1af4": "Red Hat (virtio-gpu)",
|
||||||
|
"0x1234": "QEMU",
|
||||||
|
"0x1a03": "ASPEED",
|
||||||
|
}[strings.ToLower(vid)]
|
||||||
|
ids := strings.TrimPrefix(strings.ToLower(vid), "0x")
|
||||||
|
if did != "" {
|
||||||
|
ids += ":" + strings.TrimPrefix(strings.ToLower(did), "0x")
|
||||||
|
}
|
||||||
|
if name != "" {
|
||||||
|
return name + " [" + ids + "]"
|
||||||
|
}
|
||||||
|
return "[" + ids + "]"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// primaryIfaceAddr returns the preferred outbound IPv4 + the MAC of the
|
||||||
|
// interface that owns it. Uses a UDP "connect" (no packet is sent) to let the
|
||||||
|
// kernel pick the route's source address.
|
||||||
|
func primaryIfaceAddr() (ip string, mac string) {
|
||||||
|
c, err := net.Dial("udp", "192.0.2.1:9") // TEST-NET-1, never reachable; no traffic on UDP connect
|
||||||
|
if err != nil {
|
||||||
|
return "", ""
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
la, ok := c.LocalAddr().(*net.UDPAddr)
|
||||||
|
if !ok || la.IP == nil {
|
||||||
|
return "", ""
|
||||||
|
}
|
||||||
|
ip = la.IP.String()
|
||||||
|
// find the interface carrying this IP -> its MAC
|
||||||
|
ifaces, err := net.Interfaces()
|
||||||
|
if err != nil {
|
||||||
|
return ip, ""
|
||||||
|
}
|
||||||
|
for _, ifc := range ifaces {
|
||||||
|
addrs, _ := ifc.Addrs()
|
||||||
|
for _, a := range addrs {
|
||||||
|
if n, ok := a.(*net.IPNet); ok && n.IP.Equal(la.IP) {
|
||||||
|
return ip, ifc.HardwareAddr.String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ip, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// uptimeHuman formats system uptime as "3h 12m". On a kiosk appliance the system
|
||||||
|
// boots straight into the kiosk, so system uptime ≈ kiosk-session uptime —
|
||||||
|
// reported here to avoid coupling to (and exec'ing) systemd for the unit time.
|
||||||
|
func uptimeHuman() string {
|
||||||
|
var si syscall.Sysinfo_t
|
||||||
|
if err := syscall.Sysinfo(&si); err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
d := time.Duration(si.Uptime) * time.Second
|
||||||
|
days := int(d.Hours()) / 24
|
||||||
|
hours := int(d.Hours()) % 24
|
||||||
|
mins := int(d.Minutes()) % 60
|
||||||
|
switch {
|
||||||
|
case days > 0:
|
||||||
|
return strconv.Itoa(days) + "d " + strconv.Itoa(hours) + "h " + strconv.Itoa(mins) + "m"
|
||||||
|
case hours > 0:
|
||||||
|
return strconv.Itoa(hours) + "h " + strconv.Itoa(mins) + "m"
|
||||||
|
default:
|
||||||
|
return strconv.Itoa(mins) + "m"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- helpers --------------------------------------------------------------
|
||||||
|
|
||||||
|
func loadFacts(path string) buildFacts {
|
||||||
|
var f buildFacts
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return f // file absent at runtime -> image/weston/chromium just omitted
|
||||||
|
}
|
||||||
|
_ = json.Unmarshal(b, &f)
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func env(k, def string) string {
|
||||||
|
if v := os.Getenv(k); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
|
||||||
|
func isLoopback(host string) bool {
|
||||||
|
if host == "localhost" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
ip := net.ParseIP(host)
|
||||||
|
return ip != nil && ip.IsLoopback()
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseKV(path string) map[string]string {
|
||||||
|
m := map[string]string{}
|
||||||
|
for _, line := range readLines(path) {
|
||||||
|
k, v, ok := strings.Cut(line, "=")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m[strings.TrimSpace(k)] = strings.Trim(strings.TrimSpace(v), `"`)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func readLines(path string) []string {
|
||||||
|
b := readFile(path)
|
||||||
|
if b == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return strings.Split(strings.TrimRight(b, "\n"), "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFile(path string) string {
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstNonEmpty(vals ...string) string {
|
||||||
|
for _, v := range vals {
|
||||||
|
if v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// charsToString converts a NUL-terminated Utsname field (int8 or uint8 across
|
||||||
|
// arches) to a Go string.
|
||||||
|
func charsToString(ca []int8) string {
|
||||||
|
b := make([]byte, 0, len(ca))
|
||||||
|
for _, c := range ca {
|
||||||
|
if c == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
b = append(b, byte(c))
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
@@ -31,5 +31,14 @@
|
|||||||
"AutofillCreditCardEnabled": false,
|
"AutofillCreditCardEnabled": false,
|
||||||
"PasswordManagerEnabled": false,
|
"PasswordManagerEnabled": false,
|
||||||
"ShoppingListEnabled": false,
|
"ShoppingListEnabled": false,
|
||||||
"EnableMediaRouter": false
|
"EnableMediaRouter": false,
|
||||||
|
|
||||||
|
"_comment_capability_lockdown": "The distro chromium WRAPPER hardcodes --enable-extensions --enable-user-scripts --enable-printing --enable-sync --auto-ssl-client-auth (not removable via env/conf, only by bypassing the wrapper). Neutralise the capability-broadening ones HERE — policy beats cmdline flags and survives Chromium updates. ExtensionInstallBlocklist=['*'] blocks ALL extension installs, which also kills the user-scripts surface (user scripts need an extension/dev-mode). --enable-sync is already locked by SyncDisabled above. --auto-ssl-client-auth is moot: it only auto-presents a BROWSER-store client cert, and the TPM 802.1X cert isn't one — there are none; to govern explicitly, set AutoSelectCertificateForUrls (left unset = no auto-select). See files/kiosk/kiosk-session + TODO #13.",
|
||||||
|
"ExtensionInstallBlocklist": ["*"],
|
||||||
|
|
||||||
|
"_comment_pna": "Private Network Access: the kiosk page (insecure http boot-server origin) fetches the workstation host-info agent at http://localhost:9099 — a cross-origin PRIVATE-network request Chromium blocks by default. The agent sends Access-Control-Allow-Private-Network: true; this is the matching browser-side allowance, scoped to the kiosk ORIGIN ONLY (not '*'). MUST list the origin of kiosk_url (default http://boot.fleet.lab). The machine talking to itself; no internet egress (see HOST-API.md).",
|
||||||
|
"InsecurePrivateNetworkRequestsAllowedForUrls": [
|
||||||
|
"http://boot.fleet.lab",
|
||||||
|
"http://localhost"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
+53
-22
@@ -26,6 +26,50 @@ SOCK=wayland-kiosk
|
|||||||
BROWSER=$(command -v chromium-browser || command -v chromium || true)
|
BROWSER=$(command -v chromium-browser || command -v chromium || true)
|
||||||
: "${BROWSER:?chromium not found (expected chromium-browser or chromium in PATH)}"
|
: "${BROWSER:?chromium not found (expected chromium-browser or chromium in PATH)}"
|
||||||
|
|
||||||
|
# GPU vs software path. This image is FLEET-GENERIC: it boots both real hardware
|
||||||
|
# (e.g. the 820 / Intel HD620 -> i915) AND the virtio build/test VMs (no real
|
||||||
|
# GPU). Probe for a real KMS render node and branch — real GPU gets hardware
|
||||||
|
# raster/compositing (CPU drops off software raster); VMs keep the software path
|
||||||
|
# so weston + Chromium still render there. Pure runtime probe; no per-host state
|
||||||
|
# is baked (invariant #1).
|
||||||
|
GPU=0
|
||||||
|
case "$(basename "$(readlink -f /sys/class/drm/card0/device/driver 2>/dev/null)" 2>/dev/null)" in
|
||||||
|
i915|amdgpu|radeon|nouveau|xe) [ -e /dev/dri/renderD128 ] && GPU=1 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# Lockdown flags (what the managed policy can't cover). The distro chromium
|
||||||
|
# wrapper PREPENDS its own defaults ahead of these — including its own
|
||||||
|
# --disable-features. Chromium does NOT union duplicate --disable-features (the
|
||||||
|
# LAST occurrence wins), and ours is last, so we MERGE the wrapper's disables
|
||||||
|
# (LensOverlay/Gemini/Compose/TabOrganization) into ours — otherwise they are
|
||||||
|
# silently dropped. We deliberately pass NO --enable-features, so the wrapper's
|
||||||
|
# VA-API enables (AcceleratedVideoDecodeLinuxGL, ...) stay in effect.
|
||||||
|
# Phone-home suppression (background networking, component/auto update, domain
|
||||||
|
# reliability, crash upload, pings, search-engine probe) + the comprehensive
|
||||||
|
# lockdown live in the managed policy (/etc/chromium/policies/managed/).
|
||||||
|
COMMON_FLAGS=(
|
||||||
|
--kiosk --noerrdialogs --disable-infobars --incognito --no-first-run
|
||||||
|
--disable-session-crashed-bubble --start-maximized
|
||||||
|
--ozone-platform=wayland
|
||||||
|
--disable-background-networking --disable-component-update
|
||||||
|
--disable-sync --disable-domain-reliability --disable-breakpad
|
||||||
|
--no-pings --no-default-browser-check --metrics-recording-only
|
||||||
|
--disable-search-engine-collection
|
||||||
|
"--disable-features=LensOverlay,OptimizationGuideOnDeviceModel,PromptAPIForGeminiNano,Compose,TabOrganization,Translate,OptimizationHints,OptimizationTargetPrediction,MediaRouter,AutofillServerCommunication,InterestFeedContentSuggestions"
|
||||||
|
--user-data-dir=/tmp/kiosk-profile --disk-cache-dir=/tmp/kiosk-cache
|
||||||
|
)
|
||||||
|
|
||||||
|
if [ "$GPU" = 1 ]; then
|
||||||
|
# Real GPU: hardware raster/compositing. HD620 (gen7 Intel) is sometimes
|
||||||
|
# raster-blocklisted by Chromium, so override the blocklist. The GPU process
|
||||||
|
# opens /dev/dri/renderD128 -> the kiosk user must be in group 'render' (set
|
||||||
|
# at build, diskless.pkr.hcl). With --disable-gpu gone, the wrapper's VA-API
|
||||||
|
# --enable-features finally engage (they need a live GPU process).
|
||||||
|
GPU_FLAGS=(--ignore-gpu-blocklist --enable-gpu-rasterization --enable-zero-copy)
|
||||||
|
else
|
||||||
|
GPU_FLAGS=(--disable-gpu)
|
||||||
|
fi
|
||||||
|
|
||||||
# Background: wait (up to ~60s) for weston's socket, then keep Chromium running.
|
# Background: wait (up to ~60s) for weston's socket, then keep Chromium running.
|
||||||
(
|
(
|
||||||
for _ in $(seq 1 120); do
|
for _ in $(seq 1 120); do
|
||||||
@@ -34,30 +78,17 @@ BROWSER=$(command -v chromium-browser || command -v chromium || true)
|
|||||||
done
|
done
|
||||||
export WAYLAND_DISPLAY="$SOCK"
|
export WAYLAND_DISPLAY="$SOCK"
|
||||||
while :; do
|
while :; do
|
||||||
# --disable-* / --no-pings etc.: stop Chromium phoning home to Google
|
"$BROWSER" "${COMMON_FLAGS[@]}" "${GPU_FLAGS[@]}" "$KIOSK_URL" || true
|
||||||
# (background networking = variations/field-trials + GCM, component &
|
|
||||||
# auto updates, domain reliability, crash upload, hyperlink pings). The
|
|
||||||
# --disable-features list kills the Optimization Guide hints fetch
|
|
||||||
# (optimizationguide-pa.googleapis.com), Translate, Media Router discovery
|
|
||||||
# and Autofill server calls. The comprehensive lockdown (sign-in, metrics,
|
|
||||||
# safe browsing, search suggest, ...) is the managed policy at
|
|
||||||
# /etc/chromium/policies/managed/10-kiosk-lockdown.json; these flags cover
|
|
||||||
# what policy can't. A fixed-URL kiosk needs none of it.
|
|
||||||
"$BROWSER" \
|
|
||||||
--kiosk --noerrdialogs --disable-infobars --incognito --no-first-run \
|
|
||||||
--disable-session-crashed-bubble --start-maximized \
|
|
||||||
--ozone-platform=wayland --disable-gpu \
|
|
||||||
--disable-background-networking --disable-component-update \
|
|
||||||
--disable-sync --disable-domain-reliability --disable-breakpad \
|
|
||||||
--no-pings --no-default-browser-check --metrics-recording-only \
|
|
||||||
--disable-search-engine-collection \
|
|
||||||
--disable-features=Translate,OptimizationHints,OptimizationTargetPrediction,MediaRouter,AutofillServerCommunication,InterestFeedContentSuggestions \
|
|
||||||
--user-data-dir=/tmp/kiosk-profile --disk-cache-dir=/tmp/kiosk-cache \
|
|
||||||
"$KIOSK_URL" || true
|
|
||||||
sleep 2
|
sleep 2
|
||||||
done
|
done
|
||||||
) &
|
) &
|
||||||
|
|
||||||
# weston as the foreground process (owns the seat; service restarts if it dies).
|
# weston as the foreground process (owns the seat; service restarts if it dies).
|
||||||
# desktop-shell + explicit socket; --use-pixman = software rendering for VMs.
|
# desktop-shell + explicit socket. On real GPU use weston's GL renderer (enables
|
||||||
exec weston --use-pixman --socket="$SOCK" --config=/etc/xdg/weston/weston.ini
|
# the zero-copy dmabuf handoff from Chromium so its frames aren't CPU-copied); on
|
||||||
|
# VMs force --use-pixman (software) — there's no usable DRM/GL device there.
|
||||||
|
if [ "$GPU" = 1 ]; then
|
||||||
|
exec weston --socket="$SOCK" --config=/etc/xdg/weston/weston.ini
|
||||||
|
else
|
||||||
|
exec weston --use-pixman --socket="$SOCK" --config=/etc/xdg/weston/weston.ini
|
||||||
|
fi
|
||||||
|
|||||||
@@ -39,12 +39,41 @@ STIG=${STIG:-true}
|
|||||||
# cmdline. dm-verity needs TWO namespaces — erofs DATA and the verity HASH tree
|
# cmdline. dm-verity needs TWO namespaces — erofs DATA and the verity HASH tree
|
||||||
# — because this distro's systemd-veritysetup does not support the in-file
|
# — because this distro's systemd-veritysetup does not support the in-file
|
||||||
# hash-offset layout (it ignores hash-offset= and fails to load the superblock).
|
# hash-offset layout (it ignores hash-offset= and fails to load the superblock).
|
||||||
# MUST match ROOT_DATA_UUID / ROOT_HASH_UUID in serving/spdk-setup.sh (assigned
|
# The serving side reads these back from verity.json (single source of truth) —
|
||||||
# on the target via add_ns -u).
|
# see serving/spdk/spdk-release.sh, which assigns them on the target via add_ns -u.
|
||||||
|
#
|
||||||
|
# PER-VERSION UUIDs: so a LONG-LIVED NVMe-oF target can serve MULTIPLE rootfs
|
||||||
|
# versions at once (rolling updates, TODO #6) the two namespaces must be globally
|
||||||
|
# distinct per version — otherwise two versions collide on the same by-id symlink
|
||||||
|
# (/dev/disk/by-id/nvme-uuid.<uuid>) on the host. We derive them deterministically
|
||||||
|
# from IMAGE_VERSION (sha1sum — coreutils only, no uuidgen dependency) and bake
|
||||||
|
# the result into BOTH the immutable UKI cmdline and verity.json. Determinism
|
||||||
|
# preserves the reproducible-build invariant (same version -> same UUIDs). The
|
||||||
|
# value need not be an RFC-v5 UUID — it only has to be well-formed + unique, and
|
||||||
|
# the cmdline and the serving target consume the SAME value (cmdline from here,
|
||||||
|
# target from verity.json). An explicit ROOT_DATA_UUID/ROOT_HASH_UUID env wins;
|
||||||
|
# with no version it falls back to the legacy fixed pair (single-version serving).
|
||||||
|
IMAGE_VERSION=${IMAGE_VERSION:-$(basename "$OUT")}
|
||||||
|
sha2uuid() { # arbitrary string -> stable UUID-formatted (8-4-4-4-12) hex
|
||||||
|
local h
|
||||||
|
h=$(printf '%s' "$1" | sha1sum | cut -c1-32)
|
||||||
|
printf '%s-%s-%s-%s-%s' "${h:0:8}" "${h:8:4}" "${h:12:4}" "${h:16:4}" "${h:20:12}"
|
||||||
|
}
|
||||||
|
if [ -n "$IMAGE_VERSION" ] && command -v sha1sum >/dev/null 2>&1; then
|
||||||
|
ROOT_DATA_UUID=${ROOT_DATA_UUID:-$(sha2uuid "fleet-rootfs-data-${IMAGE_VERSION}")}
|
||||||
|
ROOT_HASH_UUID=${ROOT_HASH_UUID:-$(sha2uuid "fleet-rootfs-hash-${IMAGE_VERSION}")}
|
||||||
|
fi
|
||||||
ROOT_DATA_UUID=${ROOT_DATA_UUID:-f1ee7000-0000-4000-8000-000000000001}
|
ROOT_DATA_UUID=${ROOT_DATA_UUID:-f1ee7000-0000-4000-8000-000000000001}
|
||||||
ROOT_HASH_UUID=${ROOT_HASH_UUID:-f1ee7000-0000-4000-8000-000000000002}
|
ROOT_HASH_UUID=${ROOT_HASH_UUID:-f1ee7000-0000-4000-8000-000000000002}
|
||||||
ROOT_DATA_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_DATA_UUID}"
|
ROOT_DATA_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_DATA_UUID}"
|
||||||
ROOT_HASH_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_HASH_UUID}"
|
ROOT_HASH_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_HASH_UUID}"
|
||||||
|
# Per-version subsystem NQN (distinct NQN per release) — lets a host attach to
|
||||||
|
# exactly one version's subsystem and gives clean per-release drain/allow-listing
|
||||||
|
# on the long-lived target. Written to verity.json for the serving side; NOT in
|
||||||
|
# the cmdline (host still discovers + connects; the by-UUID verity refs select
|
||||||
|
# what mounts). An explicit SUBNQN env wins.
|
||||||
|
SUBNQN_BASE=${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs}
|
||||||
|
SUBNQN=${SUBNQN:-${SUBNQN_BASE}-${IMAGE_VERSION}}
|
||||||
|
|
||||||
for bin in virt-copy-out virt-ls guestfish mkfs.erofs veritysetup jq getfattr; do
|
for bin in virt-copy-out virt-ls guestfish mkfs.erofs veritysetup jq getfattr; do
|
||||||
command -v "$bin" >/dev/null || { echo "ERROR: $bin not found on build host" >&2; exit 1; }
|
command -v "$bin" >/dev/null || { echo "ERROR: $bin not found on build host" >&2; exit 1; }
|
||||||
@@ -141,8 +170,10 @@ mv "$WORK/rootfs.hash" "$OUT/rootfs.hash"
|
|||||||
|
|
||||||
jq -n --arg hash "$ROOT_HASH" --arg erofs_uuid "$FIXED_UUID" \
|
jq -n --arg hash "$ROOT_HASH" --arg erofs_uuid "$FIXED_UUID" \
|
||||||
--arg data_uuid "$ROOT_DATA_UUID" --arg hash_uuid "$ROOT_HASH_UUID" \
|
--arg data_uuid "$ROOT_DATA_UUID" --arg hash_uuid "$ROOT_HASH_UUID" \
|
||||||
|
--arg subnqn "$SUBNQN" --arg version "$IMAGE_VERSION" \
|
||||||
'{root_hash: $hash, erofs_uuid: $erofs_uuid,
|
'{root_hash: $hash, erofs_uuid: $erofs_uuid,
|
||||||
data_ns_uuid: $data_uuid, hash_ns_uuid: $hash_uuid,
|
data_ns_uuid: $data_uuid, hash_ns_uuid: $hash_uuid,
|
||||||
|
subnqn: $subnqn, image_version: $version,
|
||||||
data_block_size: 4096, hash_block_size: 4096}' \
|
data_block_size: 4096, hash_block_size: 4096}' \
|
||||||
> "$OUT/verity.json"
|
> "$OUT/verity.json"
|
||||||
|
|
||||||
|
|||||||
@@ -74,6 +74,23 @@ NVME_HOST=nvme.fleet.lab
|
|||||||
AUDIT_HOST=audit.fleet.lab
|
AUDIT_HOST=audit.fleet.lab
|
||||||
|
|
||||||
# --- NVMe/TCP target -------------------------------------------------------
|
# --- NVMe/TCP target -------------------------------------------------------
|
||||||
|
# Per-version subsystem NQN base. The long-lived target serves each rootfs
|
||||||
|
# version as its OWN subsystem "<base>-<version>" (so a new build is added live
|
||||||
|
# without disturbing connected hosts). Per release the exact NQN is read from
|
||||||
|
# that version's verity.json (.subnqn, written by extract-artifacts.sh); this is
|
||||||
|
# only the fallback/default base and MUST match SUBNQN_BASE in that script.
|
||||||
|
SUBNQN_BASE=nqn.2026-06.org.example.fleet:rootfs
|
||||||
|
# Legacy single-version NQN (only the deprecated serving/spdk-setup.sh path).
|
||||||
SUBNQN=nqn.2026-06.org.example.fleet:rootfs
|
SUBNQN=nqn.2026-06.org.example.fleet:rootfs
|
||||||
TARGET_PORT=4420
|
TARGET_PORT=4420
|
||||||
SPDK_CPUMASK=0x1 # one polling core is plenty for a read-only boot fs
|
SPDK_CPUMASK=0x1 # one polling core is plenty for a read-only boot fs
|
||||||
|
|
||||||
|
# --- SPDK version loader / reaper (declarative rolling updates) -------------
|
||||||
|
# The spdk-loader service stages + adds ${IMAGE_VERSION} to the long-lived target
|
||||||
|
# on `compose up`, then auto-removes OLD versions once they drain. Migration =
|
||||||
|
# bump IMAGE_VERSION above + `docker compose up -d`.
|
||||||
|
REAP_OLD=true # false = keep old versions until you remove them manually
|
||||||
|
REAP_POLL_SECONDS=30 # how often to poll connection counts
|
||||||
|
# Sustained-zero window before an old version is removed. Keep it comfortably
|
||||||
|
# ABOVE the host's nvme ctrl_loss_tmo so a transient reconnect never gets reaped.
|
||||||
|
REAP_GRACE_SECONDS=300
|
||||||
|
|||||||
@@ -250,16 +250,99 @@ services:
|
|||||||
memlock: -1
|
memlock: -1
|
||||||
volumes:
|
volumes:
|
||||||
- /dev/hugepages:/dev/hugepages
|
- /dev/hugepages:/dev/hugepages
|
||||||
|
# LONG-LIVED target + ONLINE release control (no restart on a new build):
|
||||||
|
# spdk-target = the persistent entrypoint (nvmf_tgt stays up for life)
|
||||||
|
# spdk-release = JSON-RPC add/remove/list/drain of a version, live
|
||||||
|
- ./spdk/spdk-target.sh:/usr/local/bin/spdk-target:ro
|
||||||
|
- ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
|
||||||
|
# Shared RPC socket so the spdk-loader container (and `compose exec`) can
|
||||||
|
# drive this target's JSON-RPC. nvmf_tgt creates the socket here (-r).
|
||||||
|
- spdk-sock:/var/run/spdk
|
||||||
|
# Multi-version release store: each ./spdk/releases/<ver>/ holds that
|
||||||
|
# version's rootfs.erofs + rootfs.hash + verity.json. Staged by the
|
||||||
|
# spdk-loader (or spdk-ctl.sh on the host); reconciled into the target at
|
||||||
|
# (re)start and added/removed live thereafter. RW so the bootstrap seed +
|
||||||
|
# loader can write here. Adding a version does NOT recreate this service.
|
||||||
|
- ./spdk/releases:/releases
|
||||||
|
# Bootstrap convenience (optional): a single OCI release mounted as an
|
||||||
|
# image volume. spdk-target seeds it into /releases as ${IMAGE_VERSION} if
|
||||||
|
# not already staged — preserves the simple one-version `compose up` flow.
|
||||||
|
# Remove this block for a pure multi-version (spdk-ctl-managed) deployment.
|
||||||
- type: image
|
- type: image
|
||||||
source: ${OCI_IMAGE}:${IMAGE_VERSION}
|
source: ${OCI_IMAGE}:${IMAGE_VERSION}
|
||||||
target: /artifacts # -> /artifacts/rootfs.erofs for spdk-setup.sh
|
target: /artifacts
|
||||||
image:
|
image:
|
||||||
subpath: artifacts
|
subpath: artifacts
|
||||||
# Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/artifacts:ro
|
# Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/artifacts:ro
|
||||||
- ./spdk-setup.sh:/spdk-setup.sh:ro
|
# Legacy single-version entrypoint (configure-at-start, restart-to-change):
|
||||||
|
# - ./spdk-setup.sh:/spdk-setup.sh:ro
|
||||||
environment:
|
environment:
|
||||||
SUBNQN: ${SUBNQN}
|
IMAGE_VERSION: ${IMAGE_VERSION} # bootstrap-seed label for /artifacts
|
||||||
|
RELEASES_DIR: /releases
|
||||||
|
RPC_SOCK: /var/run/spdk/spdk.sock # shared with spdk-loader (-r / -s)
|
||||||
|
SUBNQN_BASE: ${SUBNQN_BASE} # per-version NQN base (falls back per release to verity.json .subnqn)
|
||||||
TARGET_ADDR: ${SERVER_IP}
|
TARGET_ADDR: ${SERVER_IP}
|
||||||
TARGET_PORT: ${TARGET_PORT}
|
TARGET_PORT: ${TARGET_PORT}
|
||||||
SPDK_CPUMASK: ${SPDK_CPUMASK}
|
SPDK_CPUMASK: ${SPDK_CPUMASK}
|
||||||
entrypoint: ["/bin/bash", "/spdk-setup.sh"]
|
NVMEOF_SVC: nvmeof
|
||||||
|
# Healthy once spdk-target has written its readiness stamp (transport up +
|
||||||
|
# releases reconciled). A FILE check, not a python rpc.py probe: nvmf_tgt does
|
||||||
|
# a slow DPDK init (~50s observed) and busy-polls its reactor core, so a python
|
||||||
|
# healthcheck starves/times out and the container sticks at "unhealthy" even
|
||||||
|
# though the target is up. `test -f` is instant and contention-free. The stamp
|
||||||
|
# is removed on (re)start + exit, so it never falsely reports a dead target.
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "test -f /var/run/spdk/ready || exit 1"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 6
|
||||||
|
start_period: 120s
|
||||||
|
# Persistent target: started ONCE, never restarted for a version change.
|
||||||
|
# Rolling updates add/remove versions online (spdk-loader, or spdk-ctl.sh).
|
||||||
|
entrypoint: ["/bin/bash", "/usr/local/bin/spdk-target"]
|
||||||
|
|
||||||
|
# --- SPDK version loader / reaper (declarative rolling updates) -------------
|
||||||
|
# The "init container that stays up." On `compose up` after an IMAGE_VERSION
|
||||||
|
# bump it stages + adds the new rootfs to the running nvmeof target ONLINE
|
||||||
|
# (no target restart, no dropped connections), then polls and removes old
|
||||||
|
# versions once they drain (all hosts rebooted onto the new UKI). Migration is
|
||||||
|
# just: bump IMAGE_VERSION in .env -> `docker compose up -d`. Reuses the SPDK
|
||||||
|
# image (has python3 + rpc.py); the scripts are bind-mounted like nvmeof's.
|
||||||
|
spdk-loader:
|
||||||
|
image: ${SPDK_IMAGE}:${SPDK_VERSION}
|
||||||
|
network_mode: host
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on:
|
||||||
|
nvmeof:
|
||||||
|
condition: service_healthy
|
||||||
|
volumes:
|
||||||
|
- ./spdk/spdk-loader.sh:/usr/local/bin/spdk-loader:ro
|
||||||
|
- ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
|
||||||
|
- spdk-sock:/var/run/spdk # shared RPC socket to the target
|
||||||
|
- ./spdk/releases:/releases # shared release store
|
||||||
|
# The CURRENT release's artifacts, mounted from its OCI image. The version
|
||||||
|
# tag is the single migration knob — bump IMAGE_VERSION and `up -d`.
|
||||||
|
- type: image
|
||||||
|
source: ${OCI_IMAGE}:${IMAGE_VERSION}
|
||||||
|
target: /in
|
||||||
|
image:
|
||||||
|
subpath: artifacts
|
||||||
|
# Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/in:ro
|
||||||
|
environment:
|
||||||
|
IMAGE_VERSION: ${IMAGE_VERSION} # the CURRENT version (staged + protected from reaping)
|
||||||
|
RELEASES_DIR: /releases
|
||||||
|
ARTIFACT_IN: /in
|
||||||
|
RPC_SOCK: /var/run/spdk/spdk.sock
|
||||||
|
SUBNQN_BASE: ${SUBNQN_BASE}
|
||||||
|
TARGET_ADDR: ${SERVER_IP}
|
||||||
|
TARGET_PORT: ${TARGET_PORT}
|
||||||
|
REAP_OLD: ${REAP_OLD:-true} # auto-remove drained old versions
|
||||||
|
REAP_POLL_SECONDS: ${REAP_POLL_SECONDS:-30}
|
||||||
|
REAP_GRACE_SECONDS: ${REAP_GRACE_SECONDS:-300} # sustained-zero window; keep > host ctrl_loss_tmo
|
||||||
|
entrypoint: ["/bin/bash", "/usr/local/bin/spdk-loader"]
|
||||||
|
|
||||||
|
# Named volume for the SPDK JSON-RPC unix socket, shared between the long-lived
|
||||||
|
# nvmeof target (which creates it via nvmf_tgt -r) and the spdk-loader (which
|
||||||
|
# drives add/remove/drain over it). Keeps the control path off the host FS.
|
||||||
|
volumes:
|
||||||
|
spdk-sock:
|
||||||
|
|||||||
+1
-1
Submodule serving/kiosk-app updated: f0d0677950...4579d227b0
@@ -1,4 +1,12 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
# DEPRECATED / legacy single-version path. Superseded by the LONG-LIVED target:
|
||||||
|
# serving/spdk/spdk-target.sh (persistent nvmf_tgt) + spdk-release.sh (online
|
||||||
|
# add/remove of versions) + spdk-ctl.sh (host ops). This script configures ONE
|
||||||
|
# version at start, so changing versions means restarting the container — which
|
||||||
|
# drops EVERY running host's NVMe/TCP root connection (see docs/TODO.md #6). Kept
|
||||||
|
# for reference / one-shot single-version use. The compose stack now runs
|
||||||
|
# spdk-target.sh as the nvmeof entrypoint.
|
||||||
|
#
|
||||||
# Configure SPDK nvmf_tgt to serve the diskless root filesystem as TWO READ-ONLY
|
# Configure SPDK nvmf_tgt to serve the diskless root filesystem as TWO READ-ONLY
|
||||||
# NVMe/TCP namespaces — erofs DATA and the dm-verity HASH tree (design doc §5, §7).
|
# NVMe/TCP namespaces — erofs DATA and the dm-verity HASH tree (design doc §5, §7).
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -0,0 +1,112 @@
|
|||||||
|
# SPDK NVMe/TCP target — long-lived, multi-version
|
||||||
|
|
||||||
|
This is the fleet's **root disk**: a userspace SPDK `nvmf_tgt` exporting each
|
||||||
|
rootfs release as **two read-only NVMe/TCP namespaces** (erofs DATA + dm-verity
|
||||||
|
HASH — the distro's systemd-veritysetup can't do the in-file hash-offset layout,
|
||||||
|
so they're separate namespaces with distinct UUIDs).
|
||||||
|
|
||||||
|
## Why the target is long-lived
|
||||||
|
|
||||||
|
`nvmf_tgt` sits in the **live data path of every running host's root fs**.
|
||||||
|
Restarting the process tears down **every** NVMe/TCP association at once
|
||||||
|
(fleet-wide, not version-scoped) — a host whose root link drops past
|
||||||
|
`ctrl_loss_tmo` gets root I/O errors and must reboot. So **the process must never
|
||||||
|
restart for a routine change.**
|
||||||
|
|
||||||
|
A host changes root version **only by rebooting** (no live swap). Therefore:
|
||||||
|
**version transitions are reboots; SPDK changes are additive/online; the target
|
||||||
|
process stays up.** Routine rollouts never restart SPDK.
|
||||||
|
|
||||||
|
## How it works
|
||||||
|
|
||||||
|
| Piece | Role |
|
||||||
|
|---|---|
|
||||||
|
| [`spdk-target.sh`](spdk-target.sh) | **Persistent entrypoint.** Starts `nvmf_tgt` once, creates the TCP transport + discovery listener, then reconciles every staged release. Never restarted to change versions. On (re)start it re-adds everything under `./releases/`, so a serving-host reboot restores all served versions. |
|
||||||
|
| [`spdk-release.sh`](spdk-release.sh) | **In-container control** (JSON-RPC). `add`/`remove`/`list`/`drain-status`/`reconcile` a version **live**. One **subsystem per version** (distinct NQN) so adding/removing one never disturbs the subsystem other hosts are attached to. Removal is **drain-gated**. |
|
||||||
|
| [`spdk-ctl.sh`](spdk-ctl.sh) | **Host-side ops** (manual / cleanup). Stages a release's artifacts from its OCI image into `./releases/<ver>/` and drives the live target via `docker compose exec`. |
|
||||||
|
| [`spdk-loader.sh`](spdk-loader.sh) | **Declarative rolling updates** (compose service `spdk-loader`, the "init container that stays up"). On `compose up` after an `IMAGE_VERSION` bump it stages + adds the new rootfs online, then auto-reaps old versions once they drain. |
|
||||||
|
| `./releases/<version>/` | Per-version `rootfs.erofs` + `rootfs.hash` + `verity.json`. `verity.json` is the **single source of truth** for that version's namespace UUIDs + subsystem NQN (written by `scripts/extract-artifacts.sh`), so the host's by-id verity refs and the target's namespace UUIDs always agree. |
|
||||||
|
|
||||||
|
Per-version namespace UUIDs are required: two versions served at once must have
|
||||||
|
globally-distinct namespace UUIDs or they collide on the same
|
||||||
|
`/dev/disk/by-id/nvme-uuid.<uuid>` symlink on the host. `extract-artifacts.sh`
|
||||||
|
derives them deterministically (UUIDv5 of the version) and bakes them into both
|
||||||
|
the immutable UKI cmdline and `verity.json`.
|
||||||
|
|
||||||
|
## Rolling update — the declarative way (recommended)
|
||||||
|
|
||||||
|
Bump one tag and bring the stack up:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# in serving/.env
|
||||||
|
IMAGE_VERSION=1.3.0
|
||||||
|
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Compose recreates the **`spdk-loader`** and the **http/dnsmasq** UKI servers (their
|
||||||
|
image volumes changed), but **not** the `nvmeof` target (its definition is
|
||||||
|
unchanged) — so no connections drop. The loader adds 1.3.0 to the live target,
|
||||||
|
serves the new UKI to all hosts, and then **auto-reaps** 1.2.0 once every host
|
||||||
|
has rebooted onto 1.3.0 and 1.2.0 has been drained for `REAP_GRACE_SECONDS`.
|
||||||
|
Rollback = set `IMAGE_VERSION` back + `up -d`. Set `REAP_OLD=false` to keep old
|
||||||
|
versions and reap them yourself.
|
||||||
|
|
||||||
|
Why auto-reap is safe here: there is a **single served UKI**, so a powered-off
|
||||||
|
host that returns boots the *current* UKI — nothing can come back to an old
|
||||||
|
rootfs once its live connections (CONNS) hit 0. Removal is still drain-gated and
|
||||||
|
debounced by the grace window (ride out transient nvme-tcp reconnects — keep
|
||||||
|
`REAP_GRACE_SECONDS` above the host `ctrl_loss_tmo`).
|
||||||
|
|
||||||
|
## Rolling update — the manual way (`spdk-ctl.sh`)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd serving/spdk
|
||||||
|
|
||||||
|
./spdk-ctl.sh load 1.3.0 # serve 1.3.0 ALONGSIDE the running 1.2.0 (live, no restart)
|
||||||
|
# ... bump the deployment manifest digest; reboot hosts → they come up on 1.3.0 ...
|
||||||
|
./spdk-ctl.sh list # watch 1.2.0 CONNS drop toward 0
|
||||||
|
# VERSION CONNS NQN
|
||||||
|
# rootfs-1.3.0 7 nqn.2026-06.org.example.fleet:rootfs-1.3.0
|
||||||
|
# rootfs-1.2.0 0 nqn.2026-06.org.example.fleet:rootfs-1.2.0
|
||||||
|
./spdk-ctl.sh unload 1.2.0 # drain-gated: refuses while CONNS > 0
|
||||||
|
```
|
||||||
|
|
||||||
|
`unload` **refuses to remove a version that still has connected controllers**
|
||||||
|
(an assigned-but-offline host could still return to it). Drain those hosts
|
||||||
|
first, or `./spdk-ctl.sh unload 1.2.0 --force` to **evict** them (they lose root
|
||||||
|
I/O — only for a known-dead version).
|
||||||
|
|
||||||
|
In-container equivalents (what `spdk-ctl.sh` calls):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose exec nvmeof spdk-release list
|
||||||
|
docker compose exec nvmeof spdk-release add 1.3.0
|
||||||
|
docker compose exec nvmeof spdk-release drain-status 1.2.0
|
||||||
|
docker compose exec nvmeof spdk-release remove 1.2.0 [--force]
|
||||||
|
```
|
||||||
|
|
||||||
|
## Capacity & availability
|
||||||
|
|
||||||
|
- Each concurrently-served version costs target memory/hugepages + a file-backed
|
||||||
|
bdev. Plan for N live versions (current + previous + canary) during long
|
||||||
|
rollouts; raise `HUGEPAGES` accordingly.
|
||||||
|
- The target is still a **single-node SPOF**. Even with online add/remove, a node
|
||||||
|
failure or a planned `nvmf_tgt` restart drops every connection. The structural
|
||||||
|
fix is **HA + NVMe-oF multipath (ANA)**: ≥2 targets serving byte-identical,
|
||||||
|
same-UUID, read-only namespaces (replication is coherent precisely because the
|
||||||
|
root is read-only + content-addressed). Cheap host-side mitigation: generous
|
||||||
|
`ctrl_loss_tmo`/`reconnect_delay`. See `docs/TODO.md` #6/#7.
|
||||||
|
- There is **no zero-impact single-node restart** — only drain/failover.
|
||||||
|
|
||||||
|
## Bootstrap (simple single-version)
|
||||||
|
|
||||||
|
`docker compose up -d` still works with one version: the OCI image volume at
|
||||||
|
`/artifacts` is **seeded** into `./releases/${IMAGE_VERSION}/` and served
|
||||||
|
automatically. Use `spdk-ctl.sh load` to add more versions on top.
|
||||||
|
|
||||||
|
## Legacy
|
||||||
|
|
||||||
|
[`../spdk-setup.sh`](../spdk-setup.sh) is the original **single-version,
|
||||||
|
configure-at-start** entrypoint (changing versions = container restart = all
|
||||||
|
connections dropped). Superseded by the long-lived target; kept for reference.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
# Staged rootfs releases for the long-lived SPDK target (spdk-ctl.sh load/unload).
|
||||||
|
# Each <version>/ holds large, content-addressed artifacts pulled from the OCI
|
||||||
|
# release image — never commit them. Keep the directory itself.
|
||||||
|
*
|
||||||
|
!.gitignore
|
||||||
Executable
+89
@@ -0,0 +1,89 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# spdk-ctl.sh — host-side fleet release ops for the LONG-LIVED SPDK target.
|
||||||
|
#
|
||||||
|
# Stages a release's artifacts into ./releases/<version>/ and adds/removes it on
|
||||||
|
# the running nvmf_tgt ONLINE — no container restart, no dropped connections.
|
||||||
|
# This is the operator entrypoint for rolling updates and cleanup; it wraps the
|
||||||
|
# in-container spdk-release control script (driven via `docker compose exec`).
|
||||||
|
#
|
||||||
|
# ./spdk-ctl.sh load <version> [oci_ref] # stage + add live (oci_ref default: $OCI_IMAGE:<version>)
|
||||||
|
# ./spdk-ctl.sh unload <version> [--force] # drain-gated remove + delete staged files
|
||||||
|
# ./spdk-ctl.sh list # served versions + connected hosts
|
||||||
|
#
|
||||||
|
# Typical rollout:
|
||||||
|
# ./spdk-ctl.sh load 1.3.0 # serve the new version alongside the old
|
||||||
|
# # ... reboot hosts onto 1.3.0 (deployment manifest digest bump) ...
|
||||||
|
# ./spdk-ctl.sh list # watch 1.2.0 CONNS fall to 0
|
||||||
|
# ./spdk-ctl.sh unload 1.2.0 # cleanup once drained (refuses if hosts remain)
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
RELEASES_DIR="${RELEASES_DIR:-./releases}"
|
||||||
|
SVC="${NVMEOF_SVC:-nvmeof}"
|
||||||
|
|
||||||
|
# Pull OCI_IMAGE default from the stack .env if present.
|
||||||
|
if [ -f ../.env ]; then
|
||||||
|
set -a; . ../.env; set +a
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Run compose from the stack dir (one level up, where docker-compose.yml lives).
|
||||||
|
dc() { ( cd .. && docker compose "$@" ); }
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "usage: $0 {load <ver> [oci_ref] | unload <ver> [--force] | list}" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_load() {
|
||||||
|
local v=${1:-}; [ -n "$v" ] || usage
|
||||||
|
local ref=${2:-}
|
||||||
|
if [ -z "$ref" ]; then
|
||||||
|
[ -n "${OCI_IMAGE:-}" ] || { echo "ERROR: set OCI_IMAGE in ../.env or pass an oci_ref" >&2; exit 1; }
|
||||||
|
ref="${OCI_IMAGE}:${v}"
|
||||||
|
fi
|
||||||
|
local dest="$RELEASES_DIR/$v"
|
||||||
|
if [ -s "$dest/verity.json" ]; then
|
||||||
|
echo "==> release $v already staged in $dest (skipping extract)"
|
||||||
|
else
|
||||||
|
echo "==> extracting $ref -> $dest"
|
||||||
|
mkdir -p "$dest"
|
||||||
|
local cid
|
||||||
|
cid=$(docker create "$ref")
|
||||||
|
# FROM-scratch artifact image: copy the three files the target needs.
|
||||||
|
docker cp "$cid:/artifacts/rootfs.erofs" "$dest/rootfs.erofs"
|
||||||
|
docker cp "$cid:/artifacts/rootfs.hash" "$dest/rootfs.hash"
|
||||||
|
docker cp "$cid:/artifacts/verity.json" "$dest/verity.json"
|
||||||
|
docker rm "$cid" >/dev/null
|
||||||
|
fi
|
||||||
|
echo "==> adding release $v on the live target (no restart)"
|
||||||
|
dc exec -T "$SVC" spdk-release add "$v"
|
||||||
|
dc exec -T "$SVC" spdk-release list
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_unload() {
|
||||||
|
local v="" force=""
|
||||||
|
local a
|
||||||
|
for a in "$@"; do
|
||||||
|
case "$a" in
|
||||||
|
--force) force="--force" ;;
|
||||||
|
*) v="$a" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
[ -n "$v" ] || usage
|
||||||
|
echo "==> removing release $v from the live target"
|
||||||
|
if dc exec -T "$SVC" spdk-release remove "$v" ${force:+$force}; then
|
||||||
|
rm -rf "${RELEASES_DIR:?}/$v"
|
||||||
|
echo "==> staged artifacts for $v deleted from $RELEASES_DIR/$v"
|
||||||
|
else
|
||||||
|
echo "remove refused (version still has connected hosts?) — staged files left in place." >&2
|
||||||
|
echo "Drain those hosts first, or re-run with --force to evict them." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
case "${1:-}" in
|
||||||
|
load) shift; cmd_load "$@" ;;
|
||||||
|
unload|rm) shift; cmd_unload "$@" ;;
|
||||||
|
list|ls) dc exec -T "$SVC" spdk-release list ;;
|
||||||
|
*) usage ;;
|
||||||
|
esac
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# spdk-loader.sh — declarative version manager for the long-lived SPDK target.
|
||||||
|
#
|
||||||
|
# Runs as a long-lived compose service alongside the nvmeof target. On (re)start:
|
||||||
|
# 1. STAGE the CURRENT release (${IMAGE_VERSION}) from the mounted artifact
|
||||||
|
# image (/in) into the shared release store (/releases/<ver>/) and ADD it to
|
||||||
|
# the running target ONLINE (no target restart).
|
||||||
|
# 2. Then stay up and REAP: periodically poll connection counts and remove any
|
||||||
|
# OTHER (non-current) version once it has been drained (0 connected
|
||||||
|
# controllers) for a sustained grace window — i.e. once every host that was
|
||||||
|
# running it has rebooted onto the new UKI.
|
||||||
|
#
|
||||||
|
# Migration is therefore: bump IMAGE_VERSION in .env, `docker compose up -d`.
|
||||||
|
# Compose recreates THIS loader (its artifact mount + env changed) and the
|
||||||
|
# http/dnsmasq UKI servers (image volume changed), but NOT the nvmeof target
|
||||||
|
# (its definition is unchanged) — so live connections are never dropped. The new
|
||||||
|
# loader adds the new rootfs online and drains the old one as hosts roll over.
|
||||||
|
#
|
||||||
|
# SAFETY: with a single served UKI a host can only ever boot the CURRENT version,
|
||||||
|
# so a non-current version at 0 connections is genuinely orphaned (no powered-off
|
||||||
|
# host can return to it — on next boot it fetches the current UKI). Removal is
|
||||||
|
# still drain-gated in spdk-release (refuses >0), and gated here by a sustained-
|
||||||
|
# zero grace window (>= REAP_GRACE_SECONDS) to ride out transient nvme-tcp
|
||||||
|
# reconnects. Set REAP_GRACE_SECONDS comfortably above the host ctrl_loss_tmo.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
CURRENT="${IMAGE_VERSION:?spdk-loader needs IMAGE_VERSION}"
|
||||||
|
RELEASES_DIR="${RELEASES_DIR:-/releases}"
|
||||||
|
IN="${ARTIFACT_IN:-/in}"
|
||||||
|
POLL="${REAP_POLL_SECONDS:-30}"
|
||||||
|
GRACE="${REAP_GRACE_SECONDS:-300}"
|
||||||
|
REAP="${REAP_OLD:-true}"
|
||||||
|
RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release)
|
||||||
|
# spdk-release reads RPC_SOCK from the environment (shared socket to the target).
|
||||||
|
|
||||||
|
echo "==> spdk-loader: current=${CURRENT} reap=${REAP} poll=${POLL}s grace=${GRACE}s"
|
||||||
|
|
||||||
|
# --- 1) stage + add the current release (idempotent) -----------------------
|
||||||
|
dest="$RELEASES_DIR/$CURRENT"
|
||||||
|
mkdir -p "$dest"
|
||||||
|
if [ -s "$IN/verity.json" ]; then
|
||||||
|
# Skip the (large) copy if already staged with identical verity.json.
|
||||||
|
if [ -s "$dest/verity.json" ] && cmp -s "$IN/verity.json" "$dest/verity.json"; then
|
||||||
|
echo "==> $CURRENT already staged in $dest (verity.json identical) — skipping copy"
|
||||||
|
else
|
||||||
|
echo "==> staging $CURRENT from $IN -> $dest"
|
||||||
|
cp "$IN/rootfs.erofs" "$dest/rootfs.erofs"
|
||||||
|
cp "$IN/rootfs.hash" "$dest/rootfs.hash"
|
||||||
|
cp "$IN/verity.json" "$dest/verity.json"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "WARNING: no $IN/verity.json (artifact image not mounted at $IN?) — relying on pre-staged $dest" >&2
|
||||||
|
fi
|
||||||
|
bash "$RELEASE_CTL" add "$CURRENT"
|
||||||
|
bash "$RELEASE_CTL" list
|
||||||
|
|
||||||
|
if [ "$REAP" != "true" ]; then
|
||||||
|
echo "==> REAP_OLD=$REAP — staying up for status only; old versions are NOT auto-removed."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 2) reap loop: a non-current version drained for >= GRACE is removed ----
|
||||||
|
declare -A zero_since
|
||||||
|
while true; do
|
||||||
|
sleep "$POLL"
|
||||||
|
[ "$REAP" = "true" ] || continue
|
||||||
|
[ -d "$RELEASES_DIR" ] || continue
|
||||||
|
for d in "$RELEASES_DIR"/*/; do
|
||||||
|
[ -e "${d}verity.json" ] || continue
|
||||||
|
v=$(basename "$d")
|
||||||
|
[ "$v" = "$CURRENT" ] && continue # never reap the current version
|
||||||
|
n=$(bash "$RELEASE_CTL" count "$v" 2>/dev/null || echo -1)
|
||||||
|
if [ "$n" = "0" ]; then
|
||||||
|
ts=${zero_since[$v]:-}
|
||||||
|
if [ -z "$ts" ]; then
|
||||||
|
ts=$(date +%s); zero_since[$v]=$ts
|
||||||
|
echo "==> $v drained (0 conns) — starting ${GRACE}s grace before removal"
|
||||||
|
fi
|
||||||
|
age=$(( $(date +%s) - ts ))
|
||||||
|
if [ "$age" -ge "$GRACE" ]; then
|
||||||
|
echo "==> reaping $v (drained ${age}s >= ${GRACE}s grace)"
|
||||||
|
if bash "$RELEASE_CTL" remove "$v"; then
|
||||||
|
rm -rf "${RELEASES_DIR:?}/$v"
|
||||||
|
unset 'zero_since[$v]'
|
||||||
|
echo "==> $v removed + staged files deleted"
|
||||||
|
else
|
||||||
|
# A connection reappeared between the poll and the remove —
|
||||||
|
# spdk-release refused (drain-gated). Reset and retry later.
|
||||||
|
echo "WARNING: remove $v refused (reconnected?) — grace reset, will retry" >&2
|
||||||
|
unset 'zero_since[$v]'
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# >0 connections, or RPC error (-1): reset the grace timer.
|
||||||
|
if [ -n "${zero_since[$v]:-}" ]; then
|
||||||
|
echo "==> $v no longer drained (conns=$n) — grace reset"
|
||||||
|
unset 'zero_since[$v]'
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
done
|
||||||
Executable
+250
@@ -0,0 +1,250 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# spdk-release.sh — add/remove a rootfs RELEASE on the long-lived nvmf_tgt
|
||||||
|
# ONLINE, via JSON-RPC, WITHOUT restarting the target. The target process holds
|
||||||
|
# every running host's NVMe/TCP root connection, so restarting it tears down the
|
||||||
|
# whole fleet's root at once (see docs/TODO.md #6). Routine rollouts therefore
|
||||||
|
# never restart SPDK: this script adds the new version's bdevs + subsystem +
|
||||||
|
# namespaces live (existing connections untouched) and removes an old version
|
||||||
|
# live once it has drained.
|
||||||
|
#
|
||||||
|
# ONE SUBSYSTEM PER VERSION (distinct NQN): adding a release as a NEW subsystem
|
||||||
|
# never disturbs the subsystem running hosts are attached to (adding a namespace
|
||||||
|
# to a SHARED subsystem would fire an AER + rescan at every host). It also gives
|
||||||
|
# clean per-release drain + per-release hostNQN allow-listing.
|
||||||
|
#
|
||||||
|
# Each release lives under $RELEASES_DIR/<version>/ as rootfs.erofs + rootfs.hash
|
||||||
|
# + verity.json. verity.json is the single source of truth for that version's
|
||||||
|
# namespace UUIDs + subsystem NQN (written by scripts/extract-artifacts.sh) — so
|
||||||
|
# the host's by-id verity refs and the target's namespace UUIDs always agree.
|
||||||
|
#
|
||||||
|
# JSON is parsed with python3 (always present — rpc.py runs on it), NOT jq, which
|
||||||
|
# the SPDK image does not ship.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# spdk-release add <version> # idempotent; live, no restart
|
||||||
|
# spdk-release remove <version> [--force] # DRAIN-GATED unless --force
|
||||||
|
# spdk-release list # served versions + connected hosts
|
||||||
|
# spdk-release drain-status <version> # connected controller count (human)
|
||||||
|
# spdk-release count <version> # connected controller count (bare int; -1 if absent/error)
|
||||||
|
# spdk-release reconcile # add every release under $RELEASES_DIR
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
RELEASES_DIR="${RELEASES_DIR:-/releases}"
|
||||||
|
TADDR="${TARGET_ADDR:-0.0.0.0}"
|
||||||
|
TPORT="${TARGET_PORT:-4420}"
|
||||||
|
SUBNQN_BASE="${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs}"
|
||||||
|
# Legacy fixed namespace UUIDs (pre per-version-UUID builds baked these into the
|
||||||
|
# UKI cmdline). Used as a fallback when verity.json lacks the per-version pair.
|
||||||
|
LEGACY_DATA_UUID="f1ee7000-0000-4000-8000-000000000001"
|
||||||
|
LEGACY_HASH_UUID="f1ee7000-0000-4000-8000-000000000002"
|
||||||
|
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
|
||||||
|
PY=$(command -v python3 || echo python3)
|
||||||
|
# Optional explicit RPC socket (shared between the long-lived target and the
|
||||||
|
# loader container). Empty -> rpc.py's default socket (single-container case).
|
||||||
|
RPC_SOCK="${RPC_SOCK:-}"
|
||||||
|
|
||||||
|
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||||
|
rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; }
|
||||||
|
# bdev names must be identifier-safe; sanitise the version.
|
||||||
|
san() { printf '%s' "$1" | tr -c 'A-Za-z0-9' '_'; }
|
||||||
|
|
||||||
|
# Read a top-level field from a release's verity.json (empty if missing/unreadable).
|
||||||
|
vfield() {
|
||||||
|
local v=$1 key=${2#.}
|
||||||
|
"$PY" -c '
|
||||||
|
import json,sys
|
||||||
|
try:
|
||||||
|
d=json.load(open(sys.argv[1]))
|
||||||
|
x=d.get(sys.argv[2],"")
|
||||||
|
sys.stdout.write("" if x is None else str(x))
|
||||||
|
except Exception:
|
||||||
|
pass' "$RELEASES_DIR/$v/verity.json" "$key" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
# Per-version subsystem NQN: prefer the one baked into verity.json; otherwise
|
||||||
|
# derive it the same way extract-artifacts.sh does (base-<version>).
|
||||||
|
release_nqn() {
|
||||||
|
local v=$1 nqn
|
||||||
|
nqn=$(vfield "$v" .subnqn)
|
||||||
|
[ -n "$nqn" ] || nqn="${SUBNQN_BASE}-${v}"
|
||||||
|
printf '%s' "$nqn"
|
||||||
|
}
|
||||||
|
|
||||||
|
subsys_exists() {
|
||||||
|
rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c '
|
||||||
|
import json,sys
|
||||||
|
try:
|
||||||
|
subs=json.load(sys.stdin)
|
||||||
|
except Exception:
|
||||||
|
sys.exit(1)
|
||||||
|
sys.exit(0 if any(s.get("nqn")==sys.argv[1] for s in subs) else 1)' "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Connected controllers on a subsystem (the drain gate). >=0 when served; -1 if
|
||||||
|
# the subsystem is absent or the RPC failed (so a poller never mistakes an error
|
||||||
|
# for "drained" and reaps a live version).
|
||||||
|
ctrl_count() {
|
||||||
|
local out
|
||||||
|
if out=$(rpc nvmf_subsystem_get_controllers "$1" 2>/dev/null); then
|
||||||
|
printf '%s' "$out" | "$PY" -c '
|
||||||
|
import json,sys
|
||||||
|
try: print(len(json.load(sys.stdin)))
|
||||||
|
except Exception: print(-1)'
|
||||||
|
else
|
||||||
|
printf '%s' "-1"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_add() {
|
||||||
|
local v=${1:?usage: add <version>}
|
||||||
|
local dir="$RELEASES_DIR/$v"
|
||||||
|
# Validation failures here RETURN (not die/exit): cmd_add is called per-dir by
|
||||||
|
# cmd_reconcile as `cmd_add … || warn`, and an `exit` inside a function aborts
|
||||||
|
# the WHOLE process (|| only catches a return), so one bad/stale dir would stop
|
||||||
|
# every OTHER release from being added. Return so reconcile skips just this one.
|
||||||
|
[ -s "$dir/rootfs.erofs" ] || { echo "ERROR: $dir/rootfs.erofs missing/empty" >&2; return 1; }
|
||||||
|
[ -s "$dir/rootfs.hash" ] || { echo "ERROR: $dir/rootfs.hash missing/empty" >&2; return 1; }
|
||||||
|
|
||||||
|
# Guard: the dir name must match the version inside verity.json. A mismatch
|
||||||
|
# means a stale/symlinked or mislabeled release dir (e.g. an old symlink into
|
||||||
|
# the version-pinned /artifacts that now resolves to a different build) — that
|
||||||
|
# would serve the wrong data/NQN/UUIDs under this name. Refuse loudly. (Legacy
|
||||||
|
# artifacts have no image_version field — skip the check for them.)
|
||||||
|
# Case-insensitive: the dir name comes from the OCI tag while verity.json's
|
||||||
|
# image_version comes from the build var, which can differ only in case — a
|
||||||
|
# benign skew. A genuine mismatch (e.g. pxe-… vs spdk-…) differs by far more.
|
||||||
|
local vv
|
||||||
|
vv=$(vfield "$v" .image_version)
|
||||||
|
if [ -n "$vv" ] && [ "${vv,,}" != "${v,,}" ]; then
|
||||||
|
echo "ERROR: release dir '$v' holds verity.json for version '$vv' (stale symlink / mislabeled dir). Refusing to add — remove '$dir' and re-stage." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local nqn du hu s
|
||||||
|
nqn=$(release_nqn "$v")
|
||||||
|
du=$(vfield "$v" .data_ns_uuid)
|
||||||
|
hu=$(vfield "$v" .hash_ns_uuid)
|
||||||
|
# LEGACY FALLBACK: artifacts built before the per-version-UUID change have no
|
||||||
|
# data_ns_uuid/hash_ns_uuid in verity.json. Such a build's UKI cmdline baked
|
||||||
|
# the fixed pair below, so serve it with exactly those. Two such LEGACY
|
||||||
|
# versions can't be co-served (they'd collide on these fixed UUIDs) — rebuild
|
||||||
|
# with the current extract-artifacts.sh for true multi-version serving.
|
||||||
|
if [ -z "$du" ] || [ -z "$hu" ]; then
|
||||||
|
du=${du:-$LEGACY_DATA_UUID}
|
||||||
|
hu=${hu:-$LEGACY_HASH_UUID}
|
||||||
|
echo "WARNING: $dir/verity.json lacks per-version namespace UUIDs — using the legacy fixed pair ($du / $hu). OK for a single legacy version; rebuild for multi-version." >&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
if subsys_exists "$nqn"; then
|
||||||
|
echo "release $v already served ($nqn) — nothing to do"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
s=$(san "$v")
|
||||||
|
echo "==> adding release $v as subsystem $nqn (data uuid $du, hash uuid $hu)"
|
||||||
|
# -r opens the backing file read-only where the SPDK build supports it.
|
||||||
|
rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096 -r \
|
||||||
|
|| rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096
|
||||||
|
rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096 -r \
|
||||||
|
|| rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096
|
||||||
|
# Serial must be <=20 chars; derive a stable, version-tagged one.
|
||||||
|
rpc nvmf_create_subsystem "$nqn" -a -s "SPDK$(printf '%s' "$s" | cut -c1-16)" -d "FleetRootfs"
|
||||||
|
rpc nvmf_subsystem_add_ns "$nqn" "rootfs_$s" -u "$du"
|
||||||
|
rpc nvmf_subsystem_add_ns "$nqn" "roothash_$s" -u "$hu"
|
||||||
|
rpc nvmf_subsystem_add_listener "$nqn" -t tcp -a "$TADDR" -s "$TPORT"
|
||||||
|
# NOTE: we deliberately do NOT (re)add the discovery-subsystem listener here.
|
||||||
|
# The long-lived target registers it ONCE at startup (spdk-target.sh), and a
|
||||||
|
# subsystem becomes discoverable simply by having its own listener on the same
|
||||||
|
# address. Re-adding it would make the nvmf_tgt DAEMON log "Listener already
|
||||||
|
# exists" to the container stdout on every add — noise the client can't
|
||||||
|
# suppress (the daemon, not rpc.py, emits it). One registration is enough.
|
||||||
|
echo "==> release $v ONLINE (existing connections untouched)"
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_remove() {
|
||||||
|
local force=0 v=""
|
||||||
|
local a
|
||||||
|
for a in "$@"; do
|
||||||
|
case "$a" in
|
||||||
|
--force) force=1 ;;
|
||||||
|
*) v="$a" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
[ -n "$v" ] || die "usage: remove <version> [--force]"
|
||||||
|
|
||||||
|
local nqn n
|
||||||
|
nqn=$(release_nqn "$v")
|
||||||
|
if ! subsys_exists "$nqn"; then
|
||||||
|
echo "release $v not served ($nqn) — nothing to remove"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
n=$(ctrl_count "$nqn")
|
||||||
|
if [ "$n" -gt 0 ] && [ "$force" -ne 1 ]; then
|
||||||
|
die "release $v still has $n connected controller(s) — refusing to remove. Drain first (reboot/migrate those hosts), or pass --force to EVICT them (they lose root I/O)."
|
||||||
|
fi
|
||||||
|
[ "$n" -gt 0 ] && echo "WARNING: --force removing $v with $n connected controller(s) — those hosts lose root I/O"
|
||||||
|
|
||||||
|
local s
|
||||||
|
s=$(san "$v")
|
||||||
|
echo "==> removing release $v ($nqn)"
|
||||||
|
rpc nvmf_delete_subsystem "$nqn" || true
|
||||||
|
rpc bdev_aio_delete "rootfs_$s" 2>/dev/null || true
|
||||||
|
rpc bdev_aio_delete "roothash_$s" 2>/dev/null || true
|
||||||
|
echo "==> release $v removed (other versions + their connections untouched)"
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_list() {
|
||||||
|
local nqn n ver
|
||||||
|
printf '%-26s %-7s %s\n' VERSION CONNS NQN
|
||||||
|
rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c '
|
||||||
|
import json,sys
|
||||||
|
try: subs=json.load(sys.stdin)
|
||||||
|
except Exception: subs=[]
|
||||||
|
for s in subs:
|
||||||
|
n=s.get("nqn","")
|
||||||
|
if "discovery" not in n: print(n)' \
|
||||||
|
| while read -r nqn; do
|
||||||
|
n=$(ctrl_count "$nqn")
|
||||||
|
ver="${nqn##*:}" # strip the nqn.*: prefix for readability
|
||||||
|
printf '%-26s %-7s %s\n' "$ver" "$n" "$nqn"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_drain_status() {
|
||||||
|
local v=${1:?usage: drain-status <version>}
|
||||||
|
local nqn n
|
||||||
|
nqn=$(release_nqn "$v")
|
||||||
|
n=$(ctrl_count "$nqn")
|
||||||
|
if [ "$n" -lt 0 ]; then
|
||||||
|
echo "$nqn is not served (or target unreachable)"
|
||||||
|
else
|
||||||
|
echo "$n connected controller(s) on $nqn"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_count() {
|
||||||
|
local v=${1:?usage: count <version>}
|
||||||
|
ctrl_count "$(release_nqn "$v")"
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_reconcile() {
|
||||||
|
[ -d "$RELEASES_DIR" ] || { echo "no $RELEASES_DIR — nothing to reconcile"; return 0; }
|
||||||
|
local d any=0
|
||||||
|
for d in "$RELEASES_DIR"/*/; do
|
||||||
|
[ -e "${d}verity.json" ] || continue
|
||||||
|
any=1
|
||||||
|
cmd_add "$(basename "$d")" || echo "WARNING: failed to add release $(basename "$d")" >&2
|
||||||
|
done
|
||||||
|
[ "$any" -eq 1 ] || echo "no releases found under $RELEASES_DIR"
|
||||||
|
}
|
||||||
|
|
||||||
|
case "${1:-}" in
|
||||||
|
add) shift; cmd_add "$@" ;;
|
||||||
|
remove|rm) shift; cmd_remove "$@" ;;
|
||||||
|
list|ls) shift; cmd_list "$@" ;;
|
||||||
|
drain-status|status) shift; cmd_drain_status "$@" ;;
|
||||||
|
count) shift; cmd_count "$@" ;;
|
||||||
|
reconcile) shift; cmd_reconcile "$@" ;;
|
||||||
|
*) echo "usage: spdk-release {add <ver>|remove <ver> [--force]|list|drain-status <ver>|count <ver>|reconcile}" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
Executable
+108
@@ -0,0 +1,108 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# spdk-target.sh — LONG-LIVED nvmf_tgt entrypoint.
|
||||||
|
#
|
||||||
|
# Starts the NVMe-oF target ONCE and keeps it up for the container's entire life.
|
||||||
|
# It is RELEASE-AGNOSTIC: rootfs versions are added/removed ONLINE via
|
||||||
|
# spdk-release.sh (JSON-RPC) without ever restarting this process, so a new build
|
||||||
|
# never drops running hosts' NVMe/TCP root connections (docs/TODO.md #6). This is
|
||||||
|
# the key difference from the legacy serving/spdk-setup.sh, which configured a
|
||||||
|
# single version at start and thus required a connection-dropping restart to
|
||||||
|
# change versions.
|
||||||
|
#
|
||||||
|
# On (re)start it reconciles: every release staged under $RELEASES_DIR is added,
|
||||||
|
# so a serving-host reboot restores all currently-served versions automatically.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
RELEASES_DIR="${RELEASES_DIR:-/releases}"
|
||||||
|
CPUMASK="${SPDK_CPUMASK:-0x1}"
|
||||||
|
TADDR="${TARGET_ADDR:-0.0.0.0}"
|
||||||
|
TPORT="${TARGET_PORT:-4420}"
|
||||||
|
HUGEPAGES="${HUGEPAGES:-1024}"
|
||||||
|
DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery"
|
||||||
|
# Readiness stamp for the container healthcheck. Written ONLY after the transport
|
||||||
|
# is up and releases are reconciled; removed on (re)start + exit. A file check is
|
||||||
|
# instant and python-free — the right probe for a box where SPDK busy-polls its
|
||||||
|
# (often single) reactor core and a python rpc.py healthcheck would starve/time
|
||||||
|
# out. Lives on the shared /run/spdk volume so it's cleared per container start.
|
||||||
|
READY_FILE="${READY_FILE:-/var/run/spdk/ready}"
|
||||||
|
|
||||||
|
TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt)
|
||||||
|
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
|
||||||
|
RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release)
|
||||||
|
# Optional explicit RPC socket. Set it (compose does, on a shared volume) so a
|
||||||
|
# SEPARATE loader container can reach this target's JSON-RPC and add/remove
|
||||||
|
# versions online. Empty -> nvmf_tgt's default socket (single-container case).
|
||||||
|
RPC_SOCK="${RPC_SOCK:-}"
|
||||||
|
rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; }
|
||||||
|
[ -n "$RPC_SOCK" ] && mkdir -p "$(dirname "$RPC_SOCK")"
|
||||||
|
mkdir -p "$(dirname "$READY_FILE")"
|
||||||
|
rm -f "$READY_FILE" # clear any stale readiness from a previous container start
|
||||||
|
|
||||||
|
# Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced, so
|
||||||
|
# a privileged container writes the host value; re-applied after a host reboot
|
||||||
|
# (the container restarts). Best-effort: allocation can fail on a fragmented
|
||||||
|
# host, in which case set vm.nr_hugepages on the host via /etc/sysctl.d.
|
||||||
|
if [ -w /proc/sys/vm/nr_hugepages ]; then
|
||||||
|
cur=$(cat /proc/sys/vm/nr_hugepages 2>/dev/null || echo 0)
|
||||||
|
if [ "${cur:-0}" -lt "$HUGEPAGES" ]; then
|
||||||
|
echo "==> allocating $HUGEPAGES hugepages (was ${cur:-0})"
|
||||||
|
echo "$HUGEPAGES" > /proc/sys/vm/nr_hugepages 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0)
|
||||||
|
[ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2
|
||||||
|
|
||||||
|
echo "==> starting LONG-LIVED nvmf_tgt (cpumask ${CPUMASK})${RPC_SOCK:+ rpc-sock $RPC_SOCK}"
|
||||||
|
TGT_ARGS=(-m "$CPUMASK")
|
||||||
|
[ -n "$RPC_SOCK" ] && TGT_ARGS+=(-r "$RPC_SOCK")
|
||||||
|
"$TGT" "${TGT_ARGS[@]}" &
|
||||||
|
TGT_PID=$!
|
||||||
|
# A TERM/INT here stops the target and DROPS EVERY connection — only on container
|
||||||
|
# stop, never for a version change. Make the intent loud.
|
||||||
|
trap 'echo "==> stopping nvmf_tgt — this drops ALL host root connections" >&2; rm -f "$READY_FILE"; kill "$TGT_PID" 2>/dev/null || true' TERM INT
|
||||||
|
trap 'rm -f "$READY_FILE"; kill "$TGT_PID" 2>/dev/null || true' EXIT
|
||||||
|
|
||||||
|
# Wait for the RPC socket.
|
||||||
|
for _ in $(seq 1 30); do
|
||||||
|
rpc framework_get_config >/dev/null 2>&1 && break
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "==> creating TCP transport"
|
||||||
|
rpc nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192 \
|
||||||
|
|| echo "WARNING: nvmf_create_transport failed (already present?)" >&2
|
||||||
|
|
||||||
|
echo "==> registering discovery listener ${TADDR}:${TPORT}"
|
||||||
|
rpc nvmf_subsystem_add_listener "$DISCOVERY_NQN" -t tcp -a "$TADDR" -s "$TPORT" >/dev/null 2>&1 || true
|
||||||
|
|
||||||
|
# Bootstrap convenience: if a single OCI artifact image is mounted at /artifacts
|
||||||
|
# (the simple one-version `docker compose up` flow) and that version isn't yet
|
||||||
|
# staged under $RELEASES_DIR, stage it so the long-lived target serves it too.
|
||||||
|
#
|
||||||
|
# COPY, never symlink: /artifacts is a version-pinned image volume that CHANGES
|
||||||
|
# when IMAGE_VERSION changes, so a symlink into it would silently re-point an
|
||||||
|
# already-staged release dir at the new version's data on the next bump. Each
|
||||||
|
# release dir must hold its own immutable copy. Self-heal: re-copy if the staged
|
||||||
|
# verity.json differs from the mounted artifact (repairs a dir left stale/symlinked
|
||||||
|
# by an older build).
|
||||||
|
if [ -s /artifacts/verity.json ] && [ -n "${IMAGE_VERSION:-}" ]; then
|
||||||
|
seed="$RELEASES_DIR/$IMAGE_VERSION"
|
||||||
|
if [ ! -s "$seed/verity.json" ] || ! cmp -s /artifacts/verity.json "$seed/verity.json"; then
|
||||||
|
echo "==> seeding bootstrap release $IMAGE_VERSION from /artifacts (copy)"
|
||||||
|
mkdir -p "$seed"
|
||||||
|
cp -f /artifacts/rootfs.erofs /artifacts/rootfs.hash /artifacts/verity.json "$seed/" \
|
||||||
|
|| echo "WARNING: failed to seed $IMAGE_VERSION from /artifacts" >&2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> reconciling releases under $RELEASES_DIR"
|
||||||
|
RELEASES_DIR="$RELEASES_DIR" TARGET_ADDR="$TADDR" TARGET_PORT="$TPORT" RPC_SOCK="$RPC_SOCK" \
|
||||||
|
bash "$RELEASE_CTL" reconcile || echo "WARNING: reconcile reported errors" >&2
|
||||||
|
|
||||||
|
# Signal readiness for the healthcheck — transport is up and releases reconciled.
|
||||||
|
: > "$READY_FILE"
|
||||||
|
|
||||||
|
echo "==> nvmf_tgt up; releases served online. Add/remove versions with:"
|
||||||
|
echo " docker compose exec ${NVMEOF_SVC:-nvmeof} spdk-release {add|remove|list} <version>"
|
||||||
|
echo " or host-side: serving/spdk/spdk-ctl.sh {load|unload|list} <version>"
|
||||||
|
wait "$TGT_PID"
|
||||||
Reference in New Issue
Block a user