From 02ffd7d885f985bdb784cbcaccb3da109993ef57 Mon Sep 17 00:00:00 2001 From: Audiobuzz Date: Sat, 27 Jun 2026 10:44:42 +0000 Subject: [PATCH] Adding spdk loader --- CLAUDE.md | 12 +++-- diskless.pkr.hcl | 10 ++++ docs/TODO.md | 15 ++++++ serving/.env.example | 10 ++++ serving/docker-compose.yml | 67 +++++++++++++++++++++-- serving/spdk/README.md | 30 ++++++++++- serving/spdk/spdk-loader.sh | 100 +++++++++++++++++++++++++++++++++++ serving/spdk/spdk-release.sh | 95 ++++++++++++++++++++++++++------- serving/spdk/spdk-target.sh | 20 ++++--- 9 files changed, 324 insertions(+), 35 deletions(-) create mode 100644 serving/spdk/spdk-loader.sh diff --git a/CLAUDE.md b/CLAUDE.md index 1e0021f..8930229 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,10 +42,16 @@ serving/spdk/ LONG-LIVED nvmf_tgt: spdk-target.sh for a version change) + spdk-release.sh (online add/remove/drain a version, 1 subsystem/version) + spdk-ctl.sh (host - load/unload/list for rolling updates). + load/unload/list for manual rolling + updates) + spdk-loader.sh (DECLARATIVE: + compose service that adds ${IMAGE_VERSION} + online then auto-reaps drained old + versions — bump tag + compose up -d). releases// = staged rootfs+verity.json. - spdk-setup.sh (parent dir) is the legacy - single-version entrypoint (deprecated). + spdk-release parses RPC JSON via python3 + (SPDK image has no jq). spdk-setup.sh + (parent dir) is the legacy single-version + entrypoint (deprecated). serving/kiosk-app/ Svelte+Tailwind spaceship-terminal SPA — GIT SUBMODULE (own repo + CI; builds the fleet-kiosk image the stack pulls). diff --git a/diskless.pkr.hcl b/diskless.pkr.hcl index e8ca5c6..e7c5299 100644 --- a/diskless.pkr.hcl +++ b/diskless.pkr.hcl @@ -343,6 +343,16 @@ build { "truncate -s 0 /etc/hostname", "install -D -m0644 /tmp/etc-stage/NetworkManager/conf.d/20-hostname.conf /etc/NetworkManager/conf.d/20-hostname.conf", "restorecon /etc/NetworkManager/conf.d/20-hostname.conf", + # Subscription-manager has no role on a booted, hostname-neutral, immutable + # appliance — entitlements only matter at BUILD time (dnf). Left running, + # rhsmcertd wakes periodically and tries to write Python __pycache__ into a + # bin_t dir on the read-only/overlay root: SELinux denies it (enforcing), so + # it's a benign but RECURRING AVC that floods the SIEM (fleet_audit rule + # 100120) and can never succeed (tmpfs overlay is wiped each boot). Mask the + # daemon + timer to kill the noise and trim attack surface. (Don't waive the + # AVC in Wazuh — that would also hide future rhsmcertd_t denials.) + "systemctl mask rhsmcertd.service 2>/dev/null || true", + "systemctl disable rhsmcertd.service 2>/dev/null || true", # Read-only erofs/verity root; runtime state on a tmpfs overlay provided # by systemd.volatile=overlay (set in the cmdline by extract-artifacts.sh). # RHEL/Rocky dracut has no 'overlayfs'/'rootovl' module. diff --git a/docs/TODO.md b/docs/TODO.md index 6c00e5a..c87125e 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -663,6 +663,21 @@ must be built around **never restarting the process that holds live connections. artifacts from its OCI image into `./releases//` and drives the live target. Rolling update = `load` new → reboot hosts → `unload` old once drained. + - **`spdk-loader.sh`** — DECLARATIVE rolling updates (the "init container that + stays up", compose service `spdk-loader`). On `compose up` after an + `IMAGE_VERSION` bump it stages + adds the new rootfs to the running target + ONLINE, then polls and **auto-reaps** any non-current version once it has + drained (0 conns) for a sustained grace window (`REAP_GRACE_SECONDS`, keep > + host `ctrl_loss_tmo`). So migration = bump `IMAGE_VERSION` in `.env` + + `docker compose up -d` (compose recreates the loader + the http/dnsmasq UKI + servers, but NOT the nvmeof target → no dropped connections). Single served + UKI ⇒ a 0-conn old version is genuinely orphaned (a returning host fetches + the current UKI), so auto-reap is safe; removal is still drain-gated + + grace-debounced. The loader + target share the RPC socket on a named volume + (`nvmf_tgt -r`); the loader reuses the SPDK image (has python3 + rpc.py). + - **No jq dependency:** `spdk-release.sh` parses RPC JSON with **python3** (the + SPDK image ships no jq); legacy artifacts (no per-version UUIDs in + verity.json) fall back to the fixed UUID pair with a warning. - **Per-version namespace UUIDs** (`scripts/extract-artifacts.sh`): derived deterministically (UUIDv5 of `image_version`), baked into the UKI cmdline + `verity.json` (now also carries `subnqn`); the serving side reads them back diff --git a/serving/.env.example b/serving/.env.example index d157714..2ce0277 100644 --- a/serving/.env.example +++ b/serving/.env.example @@ -84,3 +84,13 @@ SUBNQN_BASE=nqn.2026-06.org.example.fleet:rootfs SUBNQN=nqn.2026-06.org.example.fleet:rootfs TARGET_PORT=4420 SPDK_CPUMASK=0x1 # one polling core is plenty for a read-only boot fs + +# --- SPDK version loader / reaper (declarative rolling updates) ------------- +# The spdk-loader service stages + adds ${IMAGE_VERSION} to the long-lived target +# on `compose up`, then auto-removes OLD versions once they drain. Migration = +# bump IMAGE_VERSION above + `docker compose up -d`. +REAP_OLD=true # false = keep old versions until you remove them manually +REAP_POLL_SECONDS=30 # how often to poll connection counts +# Sustained-zero window before an old version is removed. Keep it comfortably +# ABOVE the host's nvme ctrl_loss_tmo so a transient reconnect never gets reaped. +REAP_GRACE_SECONDS=300 diff --git a/serving/docker-compose.yml b/serving/docker-compose.yml index 1b5d8b8..c48edec 100644 --- a/serving/docker-compose.yml +++ b/serving/docker-compose.yml @@ -255,11 +255,14 @@ services: # spdk-release = JSON-RPC add/remove/list/drain of a version, live - ./spdk/spdk-target.sh:/usr/local/bin/spdk-target:ro - ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro + # Shared RPC socket so the spdk-loader container (and `compose exec`) can + # drive this target's JSON-RPC. nvmf_tgt creates the socket here (-r). + - spdk-sock:/var/run/spdk # Multi-version release store: each ./spdk/releases// holds that - # version's rootfs.erofs + rootfs.hash + verity.json. Staged by - # spdk-ctl.sh on the host; reconciled into the target at (re)start and - # added/removed live thereafter. RW so the /artifacts bootstrap seed and - # spdk-ctl can write here. Adding a version does NOT recreate this service. + # version's rootfs.erofs + rootfs.hash + verity.json. Staged by the + # spdk-loader (or spdk-ctl.sh on the host); reconciled into the target at + # (re)start and added/removed live thereafter. RW so the bootstrap seed + + # loader can write here. Adding a version does NOT recreate this service. - ./spdk/releases:/releases # Bootstrap convenience (optional): a single OCI release mounted as an # image volume. spdk-target seeds it into /releases as ${IMAGE_VERSION} if @@ -276,11 +279,65 @@ services: environment: IMAGE_VERSION: ${IMAGE_VERSION} # bootstrap-seed label for /artifacts RELEASES_DIR: /releases + RPC_SOCK: /var/run/spdk/spdk.sock # shared with spdk-loader (-r / -s) SUBNQN_BASE: ${SUBNQN_BASE} # per-version NQN base (falls back per release to verity.json .subnqn) TARGET_ADDR: ${SERVER_IP} TARGET_PORT: ${TARGET_PORT} SPDK_CPUMASK: ${SPDK_CPUMASK} NVMEOF_SVC: nvmeof + # Healthy once the RPC socket answers — gates the spdk-loader's startup. + healthcheck: + test: ["CMD-SHELL", "rpc.py -s /var/run/spdk/spdk.sock framework_get_config >/dev/null 2>&1 || exit 1"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 10s # Persistent target: started ONCE, never restarted for a version change. - # Rolling updates add/remove versions online (serving/spdk/spdk-ctl.sh). + # Rolling updates add/remove versions online (spdk-loader, or spdk-ctl.sh). entrypoint: ["/bin/bash", "/usr/local/bin/spdk-target"] + + # --- SPDK version loader / reaper (declarative rolling updates) ------------- + # The "init container that stays up." On `compose up` after an IMAGE_VERSION + # bump it stages + adds the new rootfs to the running nvmeof target ONLINE + # (no target restart, no dropped connections), then polls and removes old + # versions once they drain (all hosts rebooted onto the new UKI). Migration is + # just: bump IMAGE_VERSION in .env -> `docker compose up -d`. Reuses the SPDK + # image (has python3 + rpc.py); the scripts are bind-mounted like nvmeof's. + spdk-loader: + image: ${SPDK_IMAGE}:${SPDK_VERSION} + network_mode: host + restart: unless-stopped + depends_on: + nvmeof: + condition: service_healthy + volumes: + - ./spdk/spdk-loader.sh:/usr/local/bin/spdk-loader:ro + - ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro + - spdk-sock:/var/run/spdk # shared RPC socket to the target + - ./spdk/releases:/releases # shared release store + # The CURRENT release's artifacts, mounted from its OCI image. The version + # tag is the single migration knob — bump IMAGE_VERSION and `up -d`. + - type: image + source: ${OCI_IMAGE}:${IMAGE_VERSION} + target: /in + image: + subpath: artifacts + # Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/in:ro + environment: + IMAGE_VERSION: ${IMAGE_VERSION} # the CURRENT version (staged + protected from reaping) + RELEASES_DIR: /releases + ARTIFACT_IN: /in + RPC_SOCK: /var/run/spdk/spdk.sock + SUBNQN_BASE: ${SUBNQN_BASE} + TARGET_ADDR: ${SERVER_IP} + TARGET_PORT: ${TARGET_PORT} + REAP_OLD: ${REAP_OLD:-true} # auto-remove drained old versions + REAP_POLL_SECONDS: ${REAP_POLL_SECONDS:-30} + REAP_GRACE_SECONDS: ${REAP_GRACE_SECONDS:-300} # sustained-zero window; keep > host ctrl_loss_tmo + entrypoint: ["/bin/bash", "/usr/local/bin/spdk-loader"] + +# Named volume for the SPDK JSON-RPC unix socket, shared between the long-lived +# nvmeof target (which creates it via nvmf_tgt -r) and the spdk-loader (which +# drives add/remove/drain over it). Keeps the control path off the host FS. +volumes: + spdk-sock: diff --git a/serving/spdk/README.md b/serving/spdk/README.md index 9caaeb8..c71f142 100644 --- a/serving/spdk/README.md +++ b/serving/spdk/README.md @@ -23,7 +23,8 @@ process stays up.** Routine rollouts never restart SPDK. |---|---| | [`spdk-target.sh`](spdk-target.sh) | **Persistent entrypoint.** Starts `nvmf_tgt` once, creates the TCP transport + discovery listener, then reconciles every staged release. Never restarted to change versions. On (re)start it re-adds everything under `./releases/`, so a serving-host reboot restores all served versions. | | [`spdk-release.sh`](spdk-release.sh) | **In-container control** (JSON-RPC). `add`/`remove`/`list`/`drain-status`/`reconcile` a version **live**. One **subsystem per version** (distinct NQN) so adding/removing one never disturbs the subsystem other hosts are attached to. Removal is **drain-gated**. | -| [`spdk-ctl.sh`](spdk-ctl.sh) | **Host-side ops.** Stages a release's artifacts from its OCI image into `./releases//` and drives the live target via `docker compose exec`. | +| [`spdk-ctl.sh`](spdk-ctl.sh) | **Host-side ops** (manual / cleanup). Stages a release's artifacts from its OCI image into `./releases//` and drives the live target via `docker compose exec`. | +| [`spdk-loader.sh`](spdk-loader.sh) | **Declarative rolling updates** (compose service `spdk-loader`, the "init container that stays up"). On `compose up` after an `IMAGE_VERSION` bump it stages + adds the new rootfs online, then auto-reaps old versions once they drain. | | `./releases//` | Per-version `rootfs.erofs` + `rootfs.hash` + `verity.json`. `verity.json` is the **single source of truth** for that version's namespace UUIDs + subsystem NQN (written by `scripts/extract-artifacts.sh`), so the host's by-id verity refs and the target's namespace UUIDs always agree. | Per-version namespace UUIDs are required: two versions served at once must have @@ -32,7 +33,32 @@ globally-distinct namespace UUIDs or they collide on the same derives them deterministically (UUIDv5 of the version) and bakes them into both the immutable UKI cmdline and `verity.json`. -## Rolling update (add new, drain old, clean up) +## Rolling update — the declarative way (recommended) + +Bump one tag and bring the stack up: + +```bash +# in serving/.env +IMAGE_VERSION=1.3.0 + +docker compose up -d +``` + +Compose recreates the **`spdk-loader`** and the **http/dnsmasq** UKI servers (their +image volumes changed), but **not** the `nvmeof` target (its definition is +unchanged) — so no connections drop. The loader adds 1.3.0 to the live target, +serves the new UKI to all hosts, and then **auto-reaps** 1.2.0 once every host +has rebooted onto 1.3.0 and 1.2.0 has been drained for `REAP_GRACE_SECONDS`. +Rollback = set `IMAGE_VERSION` back + `up -d`. Set `REAP_OLD=false` to keep old +versions and reap them yourself. + +Why auto-reap is safe here: there is a **single served UKI**, so a powered-off +host that returns boots the *current* UKI — nothing can come back to an old +rootfs once its live connections (CONNS) hit 0. Removal is still drain-gated and +debounced by the grace window (ride out transient nvme-tcp reconnects — keep +`REAP_GRACE_SECONDS` above the host `ctrl_loss_tmo`). + +## Rolling update — the manual way (`spdk-ctl.sh`) ```bash cd serving/spdk diff --git a/serving/spdk/spdk-loader.sh b/serving/spdk/spdk-loader.sh new file mode 100644 index 0000000..9a9138b --- /dev/null +++ b/serving/spdk/spdk-loader.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# spdk-loader.sh — declarative version manager for the long-lived SPDK target. +# +# Runs as a long-lived compose service alongside the nvmeof target. On (re)start: +# 1. STAGE the CURRENT release (${IMAGE_VERSION}) from the mounted artifact +# image (/in) into the shared release store (/releases//) and ADD it to +# the running target ONLINE (no target restart). +# 2. Then stay up and REAP: periodically poll connection counts and remove any +# OTHER (non-current) version once it has been drained (0 connected +# controllers) for a sustained grace window — i.e. once every host that was +# running it has rebooted onto the new UKI. +# +# Migration is therefore: bump IMAGE_VERSION in .env, `docker compose up -d`. +# Compose recreates THIS loader (its artifact mount + env changed) and the +# http/dnsmasq UKI servers (image volume changed), but NOT the nvmeof target +# (its definition is unchanged) — so live connections are never dropped. The new +# loader adds the new rootfs online and drains the old one as hosts roll over. +# +# SAFETY: with a single served UKI a host can only ever boot the CURRENT version, +# so a non-current version at 0 connections is genuinely orphaned (no powered-off +# host can return to it — on next boot it fetches the current UKI). Removal is +# still drain-gated in spdk-release (refuses >0), and gated here by a sustained- +# zero grace window (>= REAP_GRACE_SECONDS) to ride out transient nvme-tcp +# reconnects. Set REAP_GRACE_SECONDS comfortably above the host ctrl_loss_tmo. +set -euo pipefail + +CURRENT="${IMAGE_VERSION:?spdk-loader needs IMAGE_VERSION}" +RELEASES_DIR="${RELEASES_DIR:-/releases}" +IN="${ARTIFACT_IN:-/in}" +POLL="${REAP_POLL_SECONDS:-30}" +GRACE="${REAP_GRACE_SECONDS:-300}" +REAP="${REAP_OLD:-true}" +RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release) +# spdk-release reads RPC_SOCK from the environment (shared socket to the target). + +echo "==> spdk-loader: current=${CURRENT} reap=${REAP} poll=${POLL}s grace=${GRACE}s" + +# --- 1) stage + add the current release (idempotent) ----------------------- +dest="$RELEASES_DIR/$CURRENT" +mkdir -p "$dest" +if [ -s "$IN/verity.json" ]; then + # Skip the (large) copy if already staged with identical verity.json. + if [ -s "$dest/verity.json" ] && cmp -s "$IN/verity.json" "$dest/verity.json"; then + echo "==> $CURRENT already staged in $dest (verity.json identical) — skipping copy" + else + echo "==> staging $CURRENT from $IN -> $dest" + cp "$IN/rootfs.erofs" "$dest/rootfs.erofs" + cp "$IN/rootfs.hash" "$dest/rootfs.hash" + cp "$IN/verity.json" "$dest/verity.json" + fi +else + echo "WARNING: no $IN/verity.json (artifact image not mounted at $IN?) — relying on pre-staged $dest" >&2 +fi +bash "$RELEASE_CTL" add "$CURRENT" +bash "$RELEASE_CTL" list + +if [ "$REAP" != "true" ]; then + echo "==> REAP_OLD=$REAP — staying up for status only; old versions are NOT auto-removed." +fi + +# --- 2) reap loop: a non-current version drained for >= GRACE is removed ---- +declare -A zero_since +while true; do + sleep "$POLL" + [ "$REAP" = "true" ] || continue + [ -d "$RELEASES_DIR" ] || continue + for d in "$RELEASES_DIR"/*/; do + [ -e "${d}verity.json" ] || continue + v=$(basename "$d") + [ "$v" = "$CURRENT" ] && continue # never reap the current version + n=$(bash "$RELEASE_CTL" count "$v" 2>/dev/null || echo -1) + if [ "$n" = "0" ]; then + ts=${zero_since[$v]:-} + if [ -z "$ts" ]; then + ts=$(date +%s); zero_since[$v]=$ts + echo "==> $v drained (0 conns) — starting ${GRACE}s grace before removal" + fi + age=$(( $(date +%s) - ts )) + if [ "$age" -ge "$GRACE" ]; then + echo "==> reaping $v (drained ${age}s >= ${GRACE}s grace)" + if bash "$RELEASE_CTL" remove "$v"; then + rm -rf "${RELEASES_DIR:?}/$v" + unset 'zero_since[$v]' + echo "==> $v removed + staged files deleted" + else + # A connection reappeared between the poll and the remove — + # spdk-release refused (drain-gated). Reset and retry later. + echo "WARNING: remove $v refused (reconnected?) — grace reset, will retry" >&2 + unset 'zero_since[$v]' + fi + fi + else + # >0 connections, or RPC error (-1): reset the grace timer. + if [ -n "${zero_since[$v]:-}" ]; then + echo "==> $v no longer drained (conns=$n) — grace reset" + unset 'zero_since[$v]' + fi + fi + done +done diff --git a/serving/spdk/spdk-release.sh b/serving/spdk/spdk-release.sh index cf91434..2ca3cb7 100755 --- a/serving/spdk/spdk-release.sh +++ b/serving/spdk/spdk-release.sh @@ -17,11 +17,15 @@ # namespace UUIDs + subsystem NQN (written by scripts/extract-artifacts.sh) — so # the host's by-id verity refs and the target's namespace UUIDs always agree. # +# JSON is parsed with python3 (always present — rpc.py runs on it), NOT jq, which +# the SPDK image does not ship. +# # Usage: # spdk-release add # idempotent; live, no restart # spdk-release remove [--force] # DRAIN-GATED unless --force # spdk-release list # served versions + connected hosts -# spdk-release drain-status # connected controller count +# spdk-release drain-status # connected controller count (human) +# spdk-release count # connected controller count (bare int; -1 if absent/error) # spdk-release reconcile # add every release under $RELEASES_DIR set -euo pipefail @@ -30,17 +34,32 @@ TADDR="${TARGET_ADDR:-0.0.0.0}" TPORT="${TARGET_PORT:-4420}" SUBNQN_BASE="${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs}" DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery" +# Legacy fixed namespace UUIDs (pre per-version-UUID builds baked these into the +# UKI cmdline). Used as a fallback when verity.json lacks the per-version pair. +LEGACY_DATA_UUID="f1ee7000-0000-4000-8000-000000000001" +LEGACY_HASH_UUID="f1ee7000-0000-4000-8000-000000000002" RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py) +PY=$(command -v python3 || echo python3) +# Optional explicit RPC socket (shared between the long-lived target and the +# loader container). Empty -> rpc.py's default socket (single-container case). +RPC_SOCK="${RPC_SOCK:-}" die() { echo "ERROR: $*" >&2; exit 1; } -rpc() { "$RPC" "$@"; } -# bdev names must be filesystem/identifier-safe; sanitise the version. +rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; } +# bdev names must be identifier-safe; sanitise the version. san() { printf '%s' "$1" | tr -c 'A-Za-z0-9' '_'; } -# Read a field from a release's verity.json (empty if missing). +# Read a top-level field from a release's verity.json (empty if missing/unreadable). vfield() { - local v=$1 path=$2 - jq -r "$path // empty" "$RELEASES_DIR/$v/verity.json" 2>/dev/null || true + local v=$1 key=${2#.} + "$PY" -c ' +import json,sys +try: + d=json.load(open(sys.argv[1])) + x=d.get(sys.argv[2],"") + sys.stdout.write("" if x is None else str(x)) +except Exception: + pass' "$RELEASES_DIR/$v/verity.json" "$key" 2>/dev/null || true } # Per-version subsystem NQN: prefer the one baked into verity.json; otherwise @@ -53,15 +72,28 @@ release_nqn() { } subsys_exists() { - rpc nvmf_get_subsystems 2>/dev/null \ - | jq -e --arg n "$1" 'any(.[]; .nqn==$n)' >/dev/null 2>&1 + rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c ' +import json,sys +try: + subs=json.load(sys.stdin) +except Exception: + sys.exit(1) +sys.exit(0 if any(s.get("nqn")==sys.argv[1] for s in subs) else 1)' "$1" } -# Connected controllers on a subsystem (the drain gate). 0 if absent/none. +# Connected controllers on a subsystem (the drain gate). >=0 when served; -1 if +# the subsystem is absent or the RPC failed (so a poller never mistakes an error +# for "drained" and reaps a live version). ctrl_count() { - local n - n=$(rpc nvmf_subsystem_get_controllers "$1" 2>/dev/null | jq 'length' 2>/dev/null || echo 0) - printf '%s' "${n:-0}" + local out + if out=$(rpc nvmf_subsystem_get_controllers "$1" 2>/dev/null); then + printf '%s' "$out" | "$PY" -c ' +import json,sys +try: print(len(json.load(sys.stdin))) +except Exception: print(-1)' + else + printf '%s' "-1" + fi } cmd_add() { @@ -74,8 +106,16 @@ cmd_add() { nqn=$(release_nqn "$v") du=$(vfield "$v" .data_ns_uuid) hu=$(vfield "$v" .hash_ns_uuid) - [ -n "$du" ] && [ -n "$hu" ] \ - || die "$dir/verity.json lacks data_ns_uuid/hash_ns_uuid (rebuild with the per-version UUID extract-artifacts.sh)" + # LEGACY FALLBACK: artifacts built before the per-version-UUID change have no + # data_ns_uuid/hash_ns_uuid in verity.json. Such a build's UKI cmdline baked + # the fixed pair below, so serve it with exactly those. Two such LEGACY + # versions can't be co-served (they'd collide on these fixed UUIDs) — rebuild + # with the current extract-artifacts.sh for true multi-version serving. + if [ -z "$du" ] || [ -z "$hu" ]; then + du=${du:-$LEGACY_DATA_UUID} + hu=${hu:-$LEGACY_HASH_UUID} + echo "WARNING: $dir/verity.json lacks per-version namespace UUIDs — using the legacy fixed pair ($du / $hu). OK for a single legacy version; rebuild for multi-version." >&2 + fi if subsys_exists "$nqn"; then echo "release $v already served ($nqn) — nothing to do" @@ -135,8 +175,13 @@ cmd_remove() { cmd_list() { local nqn n ver printf '%-26s %-7s %s\n' VERSION CONNS NQN - rpc nvmf_get_subsystems 2>/dev/null \ - | jq -r '.[] | select(.nqn | test("discovery") | not) | .nqn' \ + rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c ' +import json,sys +try: subs=json.load(sys.stdin) +except Exception: subs=[] +for s in subs: + n=s.get("nqn","") + if "discovery" not in n: print(n)' \ | while read -r nqn; do n=$(ctrl_count "$nqn") ver="${nqn##*:}" # strip the nqn.*: prefix for readability @@ -146,9 +191,20 @@ cmd_list() { cmd_drain_status() { local v=${1:?usage: drain-status } - local nqn + local nqn n nqn=$(release_nqn "$v") - echo "$(ctrl_count "$nqn") connected controller(s) on $nqn" + n=$(ctrl_count "$nqn") + if [ "$n" -lt 0 ]; then + echo "$nqn is not served (or target unreachable)" + else + echo "$n connected controller(s) on $nqn" + fi +} + +cmd_count() { + local v=${1:?usage: count } + ctrl_count "$(release_nqn "$v")" + echo } cmd_reconcile() { @@ -167,6 +223,7 @@ case "${1:-}" in remove|rm) shift; cmd_remove "$@" ;; list|ls) shift; cmd_list "$@" ;; drain-status|status) shift; cmd_drain_status "$@" ;; + count) shift; cmd_count "$@" ;; reconcile) shift; cmd_reconcile "$@" ;; - *) echo "usage: spdk-release {add |remove [--force]|list|drain-status |reconcile}" >&2; exit 2 ;; + *) echo "usage: spdk-release {add |remove [--force]|list|drain-status |count |reconcile}" >&2; exit 2 ;; esac diff --git a/serving/spdk/spdk-target.sh b/serving/spdk/spdk-target.sh index baf4443..e448eb5 100755 --- a/serving/spdk/spdk-target.sh +++ b/serving/spdk/spdk-target.sh @@ -23,6 +23,12 @@ DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery" TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt) RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py) RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release) +# Optional explicit RPC socket. Set it (compose does, on a shared volume) so a +# SEPARATE loader container can reach this target's JSON-RPC and add/remove +# versions online. Empty -> nvmf_tgt's default socket (single-container case). +RPC_SOCK="${RPC_SOCK:-}" +rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; } +[ -n "$RPC_SOCK" ] && mkdir -p "$(dirname "$RPC_SOCK")" # Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced, so # a privileged container writes the host value; re-applied after a host reboot @@ -38,8 +44,10 @@ fi free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0) [ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2 -echo "==> starting LONG-LIVED nvmf_tgt (cpumask ${CPUMASK})" -"$TGT" -m "$CPUMASK" & +echo "==> starting LONG-LIVED nvmf_tgt (cpumask ${CPUMASK})${RPC_SOCK:+ rpc-sock $RPC_SOCK}" +TGT_ARGS=(-m "$CPUMASK") +[ -n "$RPC_SOCK" ] && TGT_ARGS+=(-r "$RPC_SOCK") +"$TGT" "${TGT_ARGS[@]}" & TGT_PID=$! # A TERM/INT here stops the target and DROPS EVERY connection — only on container # stop, never for a version change. Make the intent loud. @@ -48,16 +56,16 @@ trap 'kill "$TGT_PID" 2>/dev/null || true' EXIT # Wait for the RPC socket. for _ in $(seq 1 30); do - "$RPC" framework_get_config >/dev/null 2>&1 && break + rpc framework_get_config >/dev/null 2>&1 && break sleep 1 done echo "==> creating TCP transport" -"$RPC" nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192 \ +rpc nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192 \ || echo "WARNING: nvmf_create_transport failed (already present?)" >&2 echo "==> registering discovery listener ${TADDR}:${TPORT}" -"$RPC" nvmf_subsystem_add_listener "$DISCOVERY_NQN" -t tcp -a "$TADDR" -s "$TPORT" 2>/dev/null || true +rpc nvmf_subsystem_add_listener "$DISCOVERY_NQN" -t tcp -a "$TADDR" -s "$TPORT" 2>/dev/null || true # Bootstrap convenience: if a single OCI artifact image is mounted at /artifacts # (the simple one-version `docker compose up` flow) and that version isn't yet @@ -78,7 +86,7 @@ if [ -s /artifacts/verity.json ] && [ -n "${IMAGE_VERSION:-}" ]; then fi echo "==> reconciling releases under $RELEASES_DIR" -RELEASES_DIR="$RELEASES_DIR" TARGET_ADDR="$TADDR" TARGET_PORT="$TPORT" \ +RELEASES_DIR="$RELEASES_DIR" TARGET_ADDR="$TADDR" TARGET_PORT="$TPORT" RPC_SOCK="$RPC_SOCK" \ bash "$RELEASE_CTL" reconcile || echo "WARNING: reconcile reported errors" >&2 echo "==> nvmf_tgt up; releases served online. Add/remove versions with:"