sunjizzle/diskless-workstation
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into tpm-enroll

Browse Source
This commit is contained in:
Audiobuzz
2026-06-28 18:32:39 +09:30
parent 5232b36614 889db569e5
commit 3bcf16e384
23 changed files with 1846 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
serving/.env.example
+17
View File
@@ -74,6 +74,23 @@ NVME_HOST=nvme.fleet.lab
AUDIT_HOST=audit.fleet.lab
# --- NVMe/TCP target -------------------------------------------------------
# Per-version subsystem NQN base. The long-lived target serves each rootfs
# version as its OWN subsystem "<base>-<version>" (so a new build is added live
# without disturbing connected hosts). Per release the exact NQN is read from
# that version's verity.json (.subnqn, written by extract-artifacts.sh); this is
# only the fallback/default base and MUST match SUBNQN_BASE in that script.
SUBNQN_BASE=nqn.2026-06.org.example.fleet:rootfs
# Legacy single-version NQN (only the deprecated serving/spdk-setup.sh path).
SUBNQN=nqn.2026-06.org.example.fleet:rootfs
TARGET_PORT=4420
SPDK_CPUMASK=0x1 # one polling core is plenty for a read-only boot fs
# --- SPDK version loader / reaper (declarative rolling updates) -------------
# The spdk-loader service stages + adds ${IMAGE_VERSION} to the long-lived target
# on `compose up`, then auto-removes OLD versions once they drain. Migration =
# bump IMAGE_VERSION above + `docker compose up -d`.
REAP_OLD=true # false = keep old versions until you remove them manually
REAP_POLL_SECONDS=30 # how often to poll connection counts
# Sustained-zero window before an old version is removed. Keep it comfortably
# ABOVE the host's nvme ctrl_loss_tmo so a transient reconnect never gets reaped.
REAP_GRACE_SECONDS=300
serving/docker-compose.yml
+87 -4
View File
@@ -250,16 +250,99 @@ services:
memlock: -1
volumes:
- /dev/hugepages:/dev/hugepages
# LONG-LIVED target + ONLINE release control (no restart on a new build):
# spdk-target = the persistent entrypoint (nvmf_tgt stays up for life)
# spdk-release = JSON-RPC add/remove/list/drain of a version, live
- ./spdk/spdk-target.sh:/usr/local/bin/spdk-target:ro
- ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
# Shared RPC socket so the spdk-loader container (and `compose exec`) can
# drive this target's JSON-RPC. nvmf_tgt creates the socket here (-r).
- spdk-sock:/var/run/spdk
# Multi-version release store: each ./spdk/releases/<ver>/ holds that
# version's rootfs.erofs + rootfs.hash + verity.json. Staged by the
# spdk-loader (or spdk-ctl.sh on the host); reconciled into the target at
# (re)start and added/removed live thereafter. RW so the bootstrap seed +
# loader can write here. Adding a version does NOT recreate this service.
- ./spdk/releases:/releases
# Bootstrap convenience (optional): a single OCI release mounted as an
# image volume. spdk-target seeds it into /releases as ${IMAGE_VERSION} if
# not already staged — preserves the simple one-version `compose up` flow.
# Remove this block for a pure multi-version (spdk-ctl-managed) deployment.
- type: image
source: ${OCI_IMAGE}:${IMAGE_VERSION}
target: /artifacts # -> /artifacts/rootfs.erofs for spdk-setup.sh
target: /artifacts
image:
subpath: artifacts
# Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/artifacts:ro
- ./spdk-setup.sh:/spdk-setup.sh:ro
# Legacy single-version entrypoint (configure-at-start, restart-to-change):
# - ./spdk-setup.sh:/spdk-setup.sh:ro
environment:
SUBNQN: ${SUBNQN}
IMAGE_VERSION: ${IMAGE_VERSION} # bootstrap-seed label for /artifacts
RELEASES_DIR: /releases
RPC_SOCK: /var/run/spdk/spdk.sock # shared with spdk-loader (-r / -s)
SUBNQN_BASE: ${SUBNQN_BASE} # per-version NQN base (falls back per release to verity.json .subnqn)
TARGET_ADDR: ${SERVER_IP}
TARGET_PORT: ${TARGET_PORT}
SPDK_CPUMASK: ${SPDK_CPUMASK}
entrypoint: ["/bin/bash", "/spdk-setup.sh"]
NVMEOF_SVC: nvmeof
# Healthy once spdk-target has written its readiness stamp (transport up +
# releases reconciled). A FILE check, not a python rpc.py probe: nvmf_tgt does
# a slow DPDK init (~50s observed) and busy-polls its reactor core, so a python
# healthcheck starves/times out and the container sticks at "unhealthy" even
# though the target is up. `test -f` is instant and contention-free. The stamp
# is removed on (re)start + exit, so it never falsely reports a dead target.
healthcheck:
test: ["CMD-SHELL", "test -f /var/run/spdk/ready || exit 1"]
interval: 10s
timeout: 5s
retries: 6
start_period: 120s
# Persistent target: started ONCE, never restarted for a version change.
# Rolling updates add/remove versions online (spdk-loader, or spdk-ctl.sh).
entrypoint: ["/bin/bash", "/usr/local/bin/spdk-target"]
# --- SPDK version loader / reaper (declarative rolling updates) -------------
# The "init container that stays up." On `compose up` after an IMAGE_VERSION
# bump it stages + adds the new rootfs to the running nvmeof target ONLINE
# (no target restart, no dropped connections), then polls and removes old
# versions once they drain (all hosts rebooted onto the new UKI). Migration is
# just: bump IMAGE_VERSION in .env -> `docker compose up -d`. Reuses the SPDK
# image (has python3 + rpc.py); the scripts are bind-mounted like nvmeof's.
spdk-loader:
image: ${SPDK_IMAGE}:${SPDK_VERSION}
network_mode: host
restart: unless-stopped
depends_on:
nvmeof:
condition: service_healthy
volumes:
- ./spdk/spdk-loader.sh:/usr/local/bin/spdk-loader:ro
- ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
- spdk-sock:/var/run/spdk # shared RPC socket to the target
- ./spdk/releases:/releases # shared release store
# The CURRENT release's artifacts, mounted from its OCI image. The version
# tag is the single migration knob — bump IMAGE_VERSION and `up -d`.
- type: image
source: ${OCI_IMAGE}:${IMAGE_VERSION}
target: /in
image:
subpath: artifacts
# Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/in:ro
environment:
IMAGE_VERSION: ${IMAGE_VERSION} # the CURRENT version (staged + protected from reaping)
RELEASES_DIR: /releases
ARTIFACT_IN: /in
RPC_SOCK: /var/run/spdk/spdk.sock
SUBNQN_BASE: ${SUBNQN_BASE}
TARGET_ADDR: ${SERVER_IP}
TARGET_PORT: ${TARGET_PORT}
REAP_OLD: ${REAP_OLD:-true} # auto-remove drained old versions
REAP_POLL_SECONDS: ${REAP_POLL_SECONDS:-30}
REAP_GRACE_SECONDS: ${REAP_GRACE_SECONDS:-300} # sustained-zero window; keep > host ctrl_loss_tmo
entrypoint: ["/bin/bash", "/usr/local/bin/spdk-loader"]
# Named volume for the SPDK JSON-RPC unix socket, shared between the long-lived
# nvmeof target (which creates it via nvmf_tgt -r) and the spdk-loader (which
# drives add/remove/drain over it). Keeps the control path off the host FS.
volumes:
spdk-sock:
serving/spdk-setup.sh
+8
View File
@@ -1,4 +1,12 @@
#!/usr/bin/env bash
# DEPRECATED / legacy single-version path. Superseded by the LONG-LIVED target:
# serving/spdk/spdk-target.sh (persistent nvmf_tgt) + spdk-release.sh (online
# add/remove of versions) + spdk-ctl.sh (host ops). This script configures ONE
# version at start, so changing versions means restarting the container — which
# drops EVERY running host's NVMe/TCP root connection (see docs/TODO.md #6). Kept
# for reference / one-shot single-version use. The compose stack now runs
# spdk-target.sh as the nvmeof entrypoint.
#
# Configure SPDK nvmf_tgt to serve the diskless root filesystem as TWO READ-ONLY
# NVMe/TCP namespaces — erofs DATA and the dm-verity HASH tree (design doc §5, §7).
#
serving/spdk/README.md
+112
View File
@@ -0,0 +1,112 @@
# SPDK NVMe/TCP target — long-lived, multi-version
This is the fleet's **root disk**: a userspace SPDK `nvmf_tgt` exporting each
rootfs release as **two read-only NVMe/TCP namespaces** (erofs DATA + dm-verity
HASH — the distro's systemd-veritysetup can't do the in-file hash-offset layout,
so they're separate namespaces with distinct UUIDs).
## Why the target is long-lived
`nvmf_tgt` sits in the **live data path of every running host's root fs**.
Restarting the process tears down **every** NVMe/TCP association at once
(fleet-wide, not version-scoped) — a host whose root link drops past
`ctrl_loss_tmo` gets root I/O errors and must reboot. So **the process must never
restart for a routine change.**
A host changes root version **only by rebooting** (no live swap). Therefore:
**version transitions are reboots; SPDK changes are additive/online; the target
process stays up.** Routine rollouts never restart SPDK.
## How it works
| Piece | Role |
|---|---|
| [`spdk-target.sh`](spdk-target.sh) | **Persistent entrypoint.** Starts `nvmf_tgt` once, creates the TCP transport + discovery listener, then reconciles every staged release. Never restarted to change versions. On (re)start it re-adds everything under `./releases/`, so a serving-host reboot restores all served versions. |
| [`spdk-release.sh`](spdk-release.sh) | **In-container control** (JSON-RPC). `add`/`remove`/`list`/`drain-status`/`reconcile` a version **live**. One **subsystem per version** (distinct NQN) so adding/removing one never disturbs the subsystem other hosts are attached to. Removal is **drain-gated**. |
| [`spdk-ctl.sh`](spdk-ctl.sh) | **Host-side ops** (manual / cleanup). Stages a release's artifacts from its OCI image into `./releases/<ver>/` and drives the live target via `docker compose exec`. |
| [`spdk-loader.sh`](spdk-loader.sh) | **Declarative rolling updates** (compose service `spdk-loader`, the "init container that stays up"). On `compose up` after an `IMAGE_VERSION` bump it stages + adds the new rootfs online, then auto-reaps old versions once they drain. |
| `./releases/<version>/` | Per-version `rootfs.erofs` + `rootfs.hash` + `verity.json`. `verity.json` is the **single source of truth** for that version's namespace UUIDs + subsystem NQN (written by `scripts/extract-artifacts.sh`), so the host's by-id verity refs and the target's namespace UUIDs always agree. |
Per-version namespace UUIDs are required: two versions served at once must have
globally-distinct namespace UUIDs or they collide on the same
`/dev/disk/by-id/nvme-uuid.<uuid>` symlink on the host. `extract-artifacts.sh`
derives them deterministically (UUIDv5 of the version) and bakes them into both
the immutable UKI cmdline and `verity.json`.
## Rolling update — the declarative way (recommended)
Bump one tag and bring the stack up:
```bash
# in serving/.env
IMAGE_VERSION=1.3.0
docker compose up -d
```
Compose recreates the **`spdk-loader`** and the **http/dnsmasq** UKI servers (their
image volumes changed), but **not** the `nvmeof` target (its definition is
unchanged) — so no connections drop. The loader adds 1.3.0 to the live target,
serves the new UKI to all hosts, and then **auto-reaps** 1.2.0 once every host
has rebooted onto 1.3.0 and 1.2.0 has been drained for `REAP_GRACE_SECONDS`.
Rollback = set `IMAGE_VERSION` back + `up -d`. Set `REAP_OLD=false` to keep old
versions and reap them yourself.
Why auto-reap is safe here: there is a **single served UKI**, so a powered-off
host that returns boots the *current* UKI — nothing can come back to an old
rootfs once its live connections (CONNS) hit 0. Removal is still drain-gated and
debounced by the grace window (ride out transient nvme-tcp reconnects — keep
`REAP_GRACE_SECONDS` above the host `ctrl_loss_tmo`).
## Rolling update — the manual way (`spdk-ctl.sh`)
```bash
cd serving/spdk
./spdk-ctl.sh load 1.3.0 # serve 1.3.0 ALONGSIDE the running 1.2.0 (live, no restart)
# ... bump the deployment manifest digest; reboot hosts → they come up on 1.3.0 ...
./spdk-ctl.sh list # watch 1.2.0 CONNS drop toward 0
# VERSION CONNS NQN
# rootfs-1.3.0 7 nqn.2026-06.org.example.fleet:rootfs-1.3.0
# rootfs-1.2.0 0 nqn.2026-06.org.example.fleet:rootfs-1.2.0
./spdk-ctl.sh unload 1.2.0 # drain-gated: refuses while CONNS > 0
```
`unload` **refuses to remove a version that still has connected controllers**
(an assigned-but-offline host could still return to it). Drain those hosts
first, or `./spdk-ctl.sh unload 1.2.0 --force` to **evict** them (they lose root
I/O — only for a known-dead version).
In-container equivalents (what `spdk-ctl.sh` calls):
```bash
docker compose exec nvmeof spdk-release list
docker compose exec nvmeof spdk-release add 1.3.0
docker compose exec nvmeof spdk-release drain-status 1.2.0
docker compose exec nvmeof spdk-release remove 1.2.0 [--force]
```
## Capacity & availability
- Each concurrently-served version costs target memory/hugepages + a file-backed
bdev. Plan for N live versions (current + previous + canary) during long
rollouts; raise `HUGEPAGES` accordingly.
- The target is still a **single-node SPOF**. Even with online add/remove, a node
failure or a planned `nvmf_tgt` restart drops every connection. The structural
fix is **HA + NVMe-oF multipath (ANA)**: ≥2 targets serving byte-identical,
same-UUID, read-only namespaces (replication is coherent precisely because the
root is read-only + content-addressed). Cheap host-side mitigation: generous
`ctrl_loss_tmo`/`reconnect_delay`. See `docs/TODO.md` #6/#7.
- There is **no zero-impact single-node restart** — only drain/failover.
## Bootstrap (simple single-version)
`docker compose up -d` still works with one version: the OCI image volume at
`/artifacts` is **seeded** into `./releases/${IMAGE_VERSION}/` and served
automatically. Use `spdk-ctl.sh load` to add more versions on top.
## Legacy
[`../spdk-setup.sh`](../spdk-setup.sh) is the original **single-version,
configure-at-start** entrypoint (changing versions = container restart = all
connections dropped). Superseded by the long-lived target; kept for reference.
serving/spdk/releases/.gitignore
+5
View File
@@ -0,0 +1,5 @@
# Staged rootfs releases for the long-lived SPDK target (spdk-ctl.sh load/unload).
# Each <version>/ holds large, content-addressed artifacts pulled from the OCI
# release image — never commit them. Keep the directory itself.
*
!.gitignore
serving/spdk/spdk-ctl.sh
Executable
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env bash
# spdk-ctl.sh — host-side fleet release ops for the LONG-LIVED SPDK target.
#
# Stages a release's artifacts into ./releases/<version>/ and adds/removes it on
# the running nvmf_tgt ONLINE — no container restart, no dropped connections.
# This is the operator entrypoint for rolling updates and cleanup; it wraps the
# in-container spdk-release control script (driven via `docker compose exec`).
#
# ./spdk-ctl.sh load <version> [oci_ref] # stage + add live (oci_ref default: $OCI_IMAGE:<version>)
# ./spdk-ctl.sh unload <version> [--force] # drain-gated remove + delete staged files
# ./spdk-ctl.sh list # served versions + connected hosts
#
# Typical rollout:
# ./spdk-ctl.sh load 1.3.0 # serve the new version alongside the old
# # ... reboot hosts onto 1.3.0 (deployment manifest digest bump) ...
# ./spdk-ctl.sh list # watch 1.2.0 CONNS fall to 0
# ./spdk-ctl.sh unload 1.2.0 # cleanup once drained (refuses if hosts remain)
set -euo pipefail
cd "$(dirname "$0")"
RELEASES_DIR="${RELEASES_DIR:-./releases}"
SVC="${NVMEOF_SVC:-nvmeof}"
# Pull OCI_IMAGE default from the stack .env if present.
if [ -f ../.env ]; then
set -a; . ../.env; set +a
fi
# Run compose from the stack dir (one level up, where docker-compose.yml lives).
dc() { ( cd .. && docker compose "$@" ); }
usage() {
echo "usage: $0 {load <ver> [oci_ref] | unload <ver> [--force] | list}" >&2
exit 2
}
cmd_load() {
local v=${1:-}; [ -n "$v" ] || usage
local ref=${2:-}
if [ -z "$ref" ]; then
[ -n "${OCI_IMAGE:-}" ] || { echo "ERROR: set OCI_IMAGE in ../.env or pass an oci_ref" >&2; exit 1; }
ref="${OCI_IMAGE}:${v}"
fi
local dest="$RELEASES_DIR/$v"
if [ -s "$dest/verity.json" ]; then
echo "==> release $v already staged in $dest (skipping extract)"
else
echo "==> extracting $ref -> $dest"
mkdir -p "$dest"
local cid
cid=$(docker create "$ref")
# FROM-scratch artifact image: copy the three files the target needs.
docker cp "$cid:/artifacts/rootfs.erofs" "$dest/rootfs.erofs"
docker cp "$cid:/artifacts/rootfs.hash" "$dest/rootfs.hash"
docker cp "$cid:/artifacts/verity.json" "$dest/verity.json"
docker rm "$cid" >/dev/null
fi
echo "==> adding release $v on the live target (no restart)"
dc exec -T "$SVC" spdk-release add "$v"
dc exec -T "$SVC" spdk-release list
}
cmd_unload() {
local v="" force=""
local a
for a in "$@"; do
case "$a" in
--force) force="--force" ;;
*) v="$a" ;;
esac
done
[ -n "$v" ] || usage
echo "==> removing release $v from the live target"
if dc exec -T "$SVC" spdk-release remove "$v" ${force:+$force}; then
rm -rf "${RELEASES_DIR:?}/$v"
echo "==> staged artifacts for $v deleted from $RELEASES_DIR/$v"
else
echo "remove refused (version still has connected hosts?) — staged files left in place." >&2
echo "Drain those hosts first, or re-run with --force to evict them." >&2
exit 1
fi
}
case "${1:-}" in
load) shift; cmd_load "$@" ;;
unload|rm) shift; cmd_unload "$@" ;;
list|ls) dc exec -T "$SVC" spdk-release list ;;
*) usage ;;
esac
serving/spdk/spdk-loader.sh
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env bash
# spdk-loader.sh — declarative version manager for the long-lived SPDK target.
#
# Runs as a long-lived compose service alongside the nvmeof target. On (re)start:
# 1. STAGE the CURRENT release (${IMAGE_VERSION}) from the mounted artifact
# image (/in) into the shared release store (/releases/<ver>/) and ADD it to
# the running target ONLINE (no target restart).
# 2. Then stay up and REAP: periodically poll connection counts and remove any
# OTHER (non-current) version once it has been drained (0 connected
# controllers) for a sustained grace window — i.e. once every host that was
# running it has rebooted onto the new UKI.
#
# Migration is therefore: bump IMAGE_VERSION in .env, `docker compose up -d`.
# Compose recreates THIS loader (its artifact mount + env changed) and the
# http/dnsmasq UKI servers (image volume changed), but NOT the nvmeof target
# (its definition is unchanged) — so live connections are never dropped. The new
# loader adds the new rootfs online and drains the old one as hosts roll over.
#
# SAFETY: with a single served UKI a host can only ever boot the CURRENT version,
# so a non-current version at 0 connections is genuinely orphaned (no powered-off
# host can return to it — on next boot it fetches the current UKI). Removal is
# still drain-gated in spdk-release (refuses >0), and gated here by a sustained-
# zero grace window (>= REAP_GRACE_SECONDS) to ride out transient nvme-tcp
# reconnects. Set REAP_GRACE_SECONDS comfortably above the host ctrl_loss_tmo.
set -euo pipefail
CURRENT="${IMAGE_VERSION:?spdk-loader needs IMAGE_VERSION}"
RELEASES_DIR="${RELEASES_DIR:-/releases}"
IN="${ARTIFACT_IN:-/in}"
POLL="${REAP_POLL_SECONDS:-30}"
GRACE="${REAP_GRACE_SECONDS:-300}"
REAP="${REAP_OLD:-true}"
RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release)
# spdk-release reads RPC_SOCK from the environment (shared socket to the target).
echo "==> spdk-loader: current=${CURRENT} reap=${REAP} poll=${POLL}s grace=${GRACE}s"
# --- 1) stage + add the current release (idempotent) -----------------------
dest="$RELEASES_DIR/$CURRENT"
mkdir -p "$dest"
if [ -s "$IN/verity.json" ]; then
# Skip the (large) copy if already staged with identical verity.json.
if [ -s "$dest/verity.json" ] && cmp -s "$IN/verity.json" "$dest/verity.json"; then
echo "==> $CURRENT already staged in $dest (verity.json identical) — skipping copy"
else
echo "==> staging $CURRENT from $IN -> $dest"
cp "$IN/rootfs.erofs" "$dest/rootfs.erofs"
cp "$IN/rootfs.hash" "$dest/rootfs.hash"
cp "$IN/verity.json" "$dest/verity.json"
fi
else
echo "WARNING: no $IN/verity.json (artifact image not mounted at $IN?) — relying on pre-staged $dest" >&2
fi
bash "$RELEASE_CTL" add "$CURRENT"
bash "$RELEASE_CTL" list
if [ "$REAP" != "true" ]; then
echo "==> REAP_OLD=$REAP — staying up for status only; old versions are NOT auto-removed."
fi
# --- 2) reap loop: a non-current version drained for >= GRACE is removed ----
declare -A zero_since
while true; do
sleep "$POLL"
[ "$REAP" = "true" ] || continue
[ -d "$RELEASES_DIR" ] || continue
for d in "$RELEASES_DIR"/*/; do
[ -e "${d}verity.json" ] || continue
v=$(basename "$d")
[ "$v" = "$CURRENT" ] && continue # never reap the current version
n=$(bash "$RELEASE_CTL" count "$v" 2>/dev/null || echo -1)
if [ "$n" = "0" ]; then
ts=${zero_since[$v]:-}
if [ -z "$ts" ]; then
ts=$(date +%s); zero_since[$v]=$ts
echo "==> $v drained (0 conns) — starting ${GRACE}s grace before removal"
fi
age=$(( $(date +%s) - ts ))
if [ "$age" -ge "$GRACE" ]; then
echo "==> reaping $v (drained ${age}s >= ${GRACE}s grace)"
if bash "$RELEASE_CTL" remove "$v"; then
rm -rf "${RELEASES_DIR:?}/$v"
unset 'zero_since[$v]'
echo "==> $v removed + staged files deleted"
else
# A connection reappeared between the poll and the remove —
# spdk-release refused (drain-gated). Reset and retry later.
echo "WARNING: remove $v refused (reconnected?) — grace reset, will retry" >&2
unset 'zero_since[$v]'
fi
fi
else
# >0 connections, or RPC error (-1): reset the grace timer.
if [ -n "${zero_since[$v]:-}" ]; then
echo "==> $v no longer drained (conns=$n) — grace reset"
unset 'zero_since[$v]'
fi
fi
done
done
serving/spdk/spdk-release.sh
Executable
+250
View File
@@ -0,0 +1,250 @@
#!/usr/bin/env bash
# spdk-release.sh — add/remove a rootfs RELEASE on the long-lived nvmf_tgt
# ONLINE, via JSON-RPC, WITHOUT restarting the target. The target process holds
# every running host's NVMe/TCP root connection, so restarting it tears down the
# whole fleet's root at once (see docs/TODO.md #6). Routine rollouts therefore
# never restart SPDK: this script adds the new version's bdevs + subsystem +
# namespaces live (existing connections untouched) and removes an old version
# live once it has drained.
#
# ONE SUBSYSTEM PER VERSION (distinct NQN): adding a release as a NEW subsystem
# never disturbs the subsystem running hosts are attached to (adding a namespace
# to a SHARED subsystem would fire an AER + rescan at every host). It also gives
# clean per-release drain + per-release hostNQN allow-listing.
#
# Each release lives under $RELEASES_DIR/<version>/ as rootfs.erofs + rootfs.hash
# + verity.json. verity.json is the single source of truth for that version's
# namespace UUIDs + subsystem NQN (written by scripts/extract-artifacts.sh) — so
# the host's by-id verity refs and the target's namespace UUIDs always agree.
#
# JSON is parsed with python3 (always present — rpc.py runs on it), NOT jq, which
# the SPDK image does not ship.
#
# Usage:
# spdk-release add <version> # idempotent; live, no restart
# spdk-release remove <version> [--force] # DRAIN-GATED unless --force
# spdk-release list # served versions + connected hosts
# spdk-release drain-status <version> # connected controller count (human)
# spdk-release count <version> # connected controller count (bare int; -1 if absent/error)
# spdk-release reconcile # add every release under $RELEASES_DIR
set -euo pipefail
RELEASES_DIR="${RELEASES_DIR:-/releases}"
TADDR="${TARGET_ADDR:-0.0.0.0}"
TPORT="${TARGET_PORT:-4420}"
SUBNQN_BASE="${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs}"
# Legacy fixed namespace UUIDs (pre per-version-UUID builds baked these into the
# UKI cmdline). Used as a fallback when verity.json lacks the per-version pair.
LEGACY_DATA_UUID="f1ee7000-0000-4000-8000-000000000001"
LEGACY_HASH_UUID="f1ee7000-0000-4000-8000-000000000002"
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
PY=$(command -v python3 || echo python3)
# Optional explicit RPC socket (shared between the long-lived target and the
# loader container). Empty -> rpc.py's default socket (single-container case).
RPC_SOCK="${RPC_SOCK:-}"
die() { echo "ERROR: $*" >&2; exit 1; }
rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; }
# bdev names must be identifier-safe; sanitise the version.
san() { printf '%s' "$1" | tr -c 'A-Za-z0-9' '_'; }
# Read a top-level field from a release's verity.json (empty if missing/unreadable).
vfield() {
local v=$1 key=${2#.}
"$PY" -c '
import json,sys
try:
d=json.load(open(sys.argv[1]))
x=d.get(sys.argv[2],"")
sys.stdout.write("" if x is None else str(x))
except Exception:
pass' "$RELEASES_DIR/$v/verity.json" "$key" 2>/dev/null || true
}
# Per-version subsystem NQN: prefer the one baked into verity.json; otherwise
# derive it the same way extract-artifacts.sh does (base-<version>).
release_nqn() {
local v=$1 nqn
nqn=$(vfield "$v" .subnqn)
[ -n "$nqn" ] || nqn="${SUBNQN_BASE}-${v}"
printf '%s' "$nqn"
}
subsys_exists() {
rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c '
import json,sys
try:
subs=json.load(sys.stdin)
except Exception:
sys.exit(1)
sys.exit(0 if any(s.get("nqn")==sys.argv[1] for s in subs) else 1)' "$1"
}
# Connected controllers on a subsystem (the drain gate). >=0 when served; -1 if
# the subsystem is absent or the RPC failed (so a poller never mistakes an error
# for "drained" and reaps a live version).
ctrl_count() {
local out
if out=$(rpc nvmf_subsystem_get_controllers "$1" 2>/dev/null); then
printf '%s' "$out" | "$PY" -c '
import json,sys
try: print(len(json.load(sys.stdin)))
except Exception: print(-1)'
else
printf '%s' "-1"
fi
}
cmd_add() {
local v=${1:?usage: add <version>}
local dir="$RELEASES_DIR/$v"
# Validation failures here RETURN (not die/exit): cmd_add is called per-dir by
# cmd_reconcile as `cmd_add … || warn`, and an `exit` inside a function aborts
# the WHOLE process (|| only catches a return), so one bad/stale dir would stop
# every OTHER release from being added. Return so reconcile skips just this one.
[ -s "$dir/rootfs.erofs" ] || { echo "ERROR: $dir/rootfs.erofs missing/empty" >&2; return 1; }
[ -s "$dir/rootfs.hash" ] || { echo "ERROR: $dir/rootfs.hash missing/empty" >&2; return 1; }
# Guard: the dir name must match the version inside verity.json. A mismatch
# means a stale/symlinked or mislabeled release dir (e.g. an old symlink into
# the version-pinned /artifacts that now resolves to a different build) — that
# would serve the wrong data/NQN/UUIDs under this name. Refuse loudly. (Legacy
# artifacts have no image_version field — skip the check for them.)
# Case-insensitive: the dir name comes from the OCI tag while verity.json's
# image_version comes from the build var, which can differ only in case — a
# benign skew. A genuine mismatch (e.g. pxe-… vs spdk-…) differs by far more.
local vv
vv=$(vfield "$v" .image_version)
if [ -n "$vv" ] && [ "${vv,,}" != "${v,,}" ]; then
echo "ERROR: release dir '$v' holds verity.json for version '$vv' (stale symlink / mislabeled dir). Refusing to add — remove '$dir' and re-stage." >&2
return 1
fi
local nqn du hu s
nqn=$(release_nqn "$v")
du=$(vfield "$v" .data_ns_uuid)
hu=$(vfield "$v" .hash_ns_uuid)
# LEGACY FALLBACK: artifacts built before the per-version-UUID change have no
# data_ns_uuid/hash_ns_uuid in verity.json. Such a build's UKI cmdline baked
# the fixed pair below, so serve it with exactly those. Two such LEGACY
# versions can't be co-served (they'd collide on these fixed UUIDs) — rebuild
# with the current extract-artifacts.sh for true multi-version serving.
if [ -z "$du" ] || [ -z "$hu" ]; then
du=${du:-$LEGACY_DATA_UUID}
hu=${hu:-$LEGACY_HASH_UUID}
echo "WARNING: $dir/verity.json lacks per-version namespace UUIDs — using the legacy fixed pair ($du / $hu). OK for a single legacy version; rebuild for multi-version." >&2
fi
if subsys_exists "$nqn"; then
echo "release $v already served ($nqn) — nothing to do"
return 0
fi
s=$(san "$v")
echo "==> adding release $v as subsystem $nqn (data uuid $du, hash uuid $hu)"
# -r opens the backing file read-only where the SPDK build supports it.
rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096 -r \
|| rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096
rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096 -r \
|| rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096
# Serial must be <=20 chars; derive a stable, version-tagged one.
rpc nvmf_create_subsystem "$nqn" -a -s "SPDK$(printf '%s' "$s" | cut -c1-16)" -d "FleetRootfs"
rpc nvmf_subsystem_add_ns "$nqn" "rootfs_$s" -u "$du"
rpc nvmf_subsystem_add_ns "$nqn" "roothash_$s" -u "$hu"
rpc nvmf_subsystem_add_listener "$nqn" -t tcp -a "$TADDR" -s "$TPORT"
# NOTE: we deliberately do NOT (re)add the discovery-subsystem listener here.
# The long-lived target registers it ONCE at startup (spdk-target.sh), and a
# subsystem becomes discoverable simply by having its own listener on the same
# address. Re-adding it would make the nvmf_tgt DAEMON log "Listener already
# exists" to the container stdout on every add — noise the client can't
# suppress (the daemon, not rpc.py, emits it). One registration is enough.
echo "==> release $v ONLINE (existing connections untouched)"
}
cmd_remove() {
local force=0 v=""
local a
for a in "$@"; do
case "$a" in
--force) force=1 ;;
*) v="$a" ;;
esac
done
[ -n "$v" ] || die "usage: remove <version> [--force]"
local nqn n
nqn=$(release_nqn "$v")
if ! subsys_exists "$nqn"; then
echo "release $v not served ($nqn) — nothing to remove"
return 0
fi
n=$(ctrl_count "$nqn")
if [ "$n" -gt 0 ] && [ "$force" -ne 1 ]; then
die "release $v still has $n connected controller(s) — refusing to remove. Drain first (reboot/migrate those hosts), or pass --force to EVICT them (they lose root I/O)."
fi
[ "$n" -gt 0 ] && echo "WARNING: --force removing $v with $n connected controller(s) — those hosts lose root I/O"
local s
s=$(san "$v")
echo "==> removing release $v ($nqn)"
rpc nvmf_delete_subsystem "$nqn" || true
rpc bdev_aio_delete "rootfs_$s" 2>/dev/null || true
rpc bdev_aio_delete "roothash_$s" 2>/dev/null || true
echo "==> release $v removed (other versions + their connections untouched)"
}
cmd_list() {
local nqn n ver
printf '%-26s %-7s %s\n' VERSION CONNS NQN
rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c '
import json,sys
try: subs=json.load(sys.stdin)
except Exception: subs=[]
for s in subs:
n=s.get("nqn","")
if "discovery" not in n: print(n)' \
| while read -r nqn; do
n=$(ctrl_count "$nqn")
ver="${nqn##*:}" # strip the nqn.*: prefix for readability
printf '%-26s %-7s %s\n' "$ver" "$n" "$nqn"
done
}
cmd_drain_status() {
local v=${1:?usage: drain-status <version>}
local nqn n
nqn=$(release_nqn "$v")
n=$(ctrl_count "$nqn")
if [ "$n" -lt 0 ]; then
echo "$nqn is not served (or target unreachable)"
else
echo "$n connected controller(s) on $nqn"
fi
}
cmd_count() {
local v=${1:?usage: count <version>}
ctrl_count "$(release_nqn "$v")"
echo
}
cmd_reconcile() {
[ -d "$RELEASES_DIR" ] || { echo "no $RELEASES_DIR — nothing to reconcile"; return 0; }
local d any=0
for d in "$RELEASES_DIR"/*/; do
[ -e "${d}verity.json" ] || continue
any=1
cmd_add "$(basename "$d")" || echo "WARNING: failed to add release $(basename "$d")" >&2
done
[ "$any" -eq 1 ] || echo "no releases found under $RELEASES_DIR"
}
case "${1:-}" in
add) shift; cmd_add "$@" ;;
remove|rm) shift; cmd_remove "$@" ;;
list|ls) shift; cmd_list "$@" ;;
drain-status|status) shift; cmd_drain_status "$@" ;;
count) shift; cmd_count "$@" ;;
reconcile) shift; cmd_reconcile "$@" ;;
*) echo "usage: spdk-release {add <ver>|remove <ver> [--force]|list|drain-status <ver>|count <ver>|reconcile}" >&2; exit 2 ;;
esac
serving/spdk/spdk-target.sh
Executable
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env bash
# spdk-target.sh — LONG-LIVED nvmf_tgt entrypoint.
#
# Starts the NVMe-oF target ONCE and keeps it up for the container's entire life.
# It is RELEASE-AGNOSTIC: rootfs versions are added/removed ONLINE via
# spdk-release.sh (JSON-RPC) without ever restarting this process, so a new build
# never drops running hosts' NVMe/TCP root connections (docs/TODO.md #6). This is
# the key difference from the legacy serving/spdk-setup.sh, which configured a
# single version at start and thus required a connection-dropping restart to
# change versions.
#
# On (re)start it reconciles: every release staged under $RELEASES_DIR is added,
# so a serving-host reboot restores all currently-served versions automatically.
set -euo pipefail
RELEASES_DIR="${RELEASES_DIR:-/releases}"
CPUMASK="${SPDK_CPUMASK:-0x1}"
TADDR="${TARGET_ADDR:-0.0.0.0}"
TPORT="${TARGET_PORT:-4420}"
HUGEPAGES="${HUGEPAGES:-1024}"
DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery"
# Readiness stamp for the container healthcheck. Written ONLY after the transport
# is up and releases are reconciled; removed on (re)start + exit. A file check is
# instant and python-free — the right probe for a box where SPDK busy-polls its
# (often single) reactor core and a python rpc.py healthcheck would starve/time
# out. Lives on the shared /run/spdk volume so it's cleared per container start.
READY_FILE="${READY_FILE:-/var/run/spdk/ready}"
TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt)
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release)
# Optional explicit RPC socket. Set it (compose does, on a shared volume) so a
# SEPARATE loader container can reach this target's JSON-RPC and add/remove
# versions online. Empty -> nvmf_tgt's default socket (single-container case).
RPC_SOCK="${RPC_SOCK:-}"
rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; }
[ -n "$RPC_SOCK" ] && mkdir -p "$(dirname "$RPC_SOCK")"
mkdir -p "$(dirname "$READY_FILE")"
rm -f "$READY_FILE" # clear any stale readiness from a previous container start
# Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced, so
# a privileged container writes the host value; re-applied after a host reboot
# (the container restarts). Best-effort: allocation can fail on a fragmented
# host, in which case set vm.nr_hugepages on the host via /etc/sysctl.d.
if [ -w /proc/sys/vm/nr_hugepages ]; then
cur=$(cat /proc/sys/vm/nr_hugepages 2>/dev/null || echo 0)
if [ "${cur:-0}" -lt "$HUGEPAGES" ]; then
echo "==> allocating $HUGEPAGES hugepages (was ${cur:-0})"
echo "$HUGEPAGES" > /proc/sys/vm/nr_hugepages 2>/dev/null || true
fi
fi
free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0)
[ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2
echo "==> starting LONG-LIVED nvmf_tgt (cpumask ${CPUMASK})${RPC_SOCK:+ rpc-sock $RPC_SOCK}"
TGT_ARGS=(-m "$CPUMASK")
[ -n "$RPC_SOCK" ] && TGT_ARGS+=(-r "$RPC_SOCK")
"$TGT" "${TGT_ARGS[@]}" &
TGT_PID=$!
# A TERM/INT here stops the target and DROPS EVERY connection — only on container
# stop, never for a version change. Make the intent loud.
trap 'echo "==> stopping nvmf_tgt — this drops ALL host root connections" >&2; rm -f "$READY_FILE"; kill "$TGT_PID" 2>/dev/null || true' TERM INT
trap 'rm -f "$READY_FILE"; kill "$TGT_PID" 2>/dev/null || true' EXIT
# Wait for the RPC socket.
for _ in $(seq 1 30); do
rpc framework_get_config >/dev/null 2>&1 && break
sleep 1
done
echo "==> creating TCP transport"
rpc nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192 \
|| echo "WARNING: nvmf_create_transport failed (already present?)" >&2
echo "==> registering discovery listener ${TADDR}:${TPORT}"
rpc nvmf_subsystem_add_listener "$DISCOVERY_NQN" -t tcp -a "$TADDR" -s "$TPORT" >/dev/null 2>&1 || true
# Bootstrap convenience: if a single OCI artifact image is mounted at /artifacts
# (the simple one-version `docker compose up` flow) and that version isn't yet
# staged under $RELEASES_DIR, stage it so the long-lived target serves it too.
#
# COPY, never symlink: /artifacts is a version-pinned image volume that CHANGES
# when IMAGE_VERSION changes, so a symlink into it would silently re-point an
# already-staged release dir at the new version's data on the next bump. Each
# release dir must hold its own immutable copy. Self-heal: re-copy if the staged
# verity.json differs from the mounted artifact (repairs a dir left stale/symlinked
# by an older build).
if [ -s /artifacts/verity.json ] && [ -n "${IMAGE_VERSION:-}" ]; then
seed="$RELEASES_DIR/$IMAGE_VERSION"
if [ ! -s "$seed/verity.json" ] || ! cmp -s /artifacts/verity.json "$seed/verity.json"; then
echo "==> seeding bootstrap release $IMAGE_VERSION from /artifacts (copy)"
mkdir -p "$seed"
cp -f /artifacts/rootfs.erofs /artifacts/rootfs.hash /artifacts/verity.json "$seed/" \
|| echo "WARNING: failed to seed $IMAGE_VERSION from /artifacts" >&2
fi
fi
echo "==> reconciling releases under $RELEASES_DIR"
RELEASES_DIR="$RELEASES_DIR" TARGET_ADDR="$TADDR" TARGET_PORT="$TPORT" RPC_SOCK="$RPC_SOCK" \
bash "$RELEASE_CTL" reconcile || echo "WARNING: reconcile reported errors" >&2
# Signal readiness for the healthcheck — transport is up and releases reconciled.
: > "$READY_FILE"
echo "==> nvmf_tgt up; releases served online. Add/remove versions with:"
echo " docker compose exec ${NVMEOF_SVC:-nvmeof} spdk-release {add|remove|list} <version>"
echo " or host-side: serving/spdk/spdk-ctl.sh {load|unload|list} <version>"
wait "$TGT_PID"