251 lines
11 KiB
Bash
Executable File
251 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# spdk-release.sh — add/remove a rootfs RELEASE on the long-lived nvmf_tgt
|
|
# ONLINE, via JSON-RPC, WITHOUT restarting the target. The target process holds
|
|
# every running host's NVMe/TCP root connection, so restarting it tears down the
|
|
# whole fleet's root at once (see docs/TODO.md #6). Routine rollouts therefore
|
|
# never restart SPDK: this script adds the new version's bdevs + subsystem +
|
|
# namespaces live (existing connections untouched) and removes an old version
|
|
# live once it has drained.
|
|
#
|
|
# ONE SUBSYSTEM PER VERSION (distinct NQN): adding a release as a NEW subsystem
|
|
# never disturbs the subsystem running hosts are attached to (adding a namespace
|
|
# to a SHARED subsystem would fire an AER + rescan at every host). It also gives
|
|
# clean per-release drain + per-release hostNQN allow-listing.
|
|
#
|
|
# Each release lives under $RELEASES_DIR/<version>/ as rootfs.erofs + rootfs.hash
|
|
# + verity.json. verity.json is the single source of truth for that version's
|
|
# namespace UUIDs + subsystem NQN (written by scripts/extract-artifacts.sh) — so
|
|
# the host's by-id verity refs and the target's namespace UUIDs always agree.
|
|
#
|
|
# JSON is parsed with python3 (always present — rpc.py runs on it), NOT jq, which
|
|
# the SPDK image does not ship.
|
|
#
|
|
# Usage:
|
|
# spdk-release add <version> # idempotent; live, no restart
|
|
# spdk-release remove <version> [--force] # DRAIN-GATED unless --force
|
|
# spdk-release list # served versions + connected hosts
|
|
# spdk-release drain-status <version> # connected controller count (human)
|
|
# spdk-release count <version> # connected controller count (bare int; -1 if absent/error)
|
|
# spdk-release reconcile # add every release under $RELEASES_DIR
|
|
set -euo pipefail
|
|
|
|
RELEASES_DIR="${RELEASES_DIR:-/releases}"
|
|
TADDR="${TARGET_ADDR:-0.0.0.0}"
|
|
TPORT="${TARGET_PORT:-4420}"
|
|
SUBNQN_BASE="${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs}"
|
|
# Legacy fixed namespace UUIDs (pre per-version-UUID builds baked these into the
|
|
# UKI cmdline). Used as a fallback when verity.json lacks the per-version pair.
|
|
LEGACY_DATA_UUID="f1ee7000-0000-4000-8000-000000000001"
|
|
LEGACY_HASH_UUID="f1ee7000-0000-4000-8000-000000000002"
|
|
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
|
|
PY=$(command -v python3 || echo python3)
|
|
# Optional explicit RPC socket (shared between the long-lived target and the
|
|
# loader container). Empty -> rpc.py's default socket (single-container case).
|
|
RPC_SOCK="${RPC_SOCK:-}"
|
|
|
|
die() { echo "ERROR: $*" >&2; exit 1; }
|
|
rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; }
|
|
# bdev names must be identifier-safe; sanitise the version.
|
|
san() { printf '%s' "$1" | tr -c 'A-Za-z0-9' '_'; }
|
|
|
|
# Read a top-level field from a release's verity.json (empty if missing/unreadable).
|
|
vfield() {
|
|
local v=$1 key=${2#.}
|
|
"$PY" -c '
|
|
import json,sys
|
|
try:
|
|
d=json.load(open(sys.argv[1]))
|
|
x=d.get(sys.argv[2],"")
|
|
sys.stdout.write("" if x is None else str(x))
|
|
except Exception:
|
|
pass' "$RELEASES_DIR/$v/verity.json" "$key" 2>/dev/null || true
|
|
}
|
|
|
|
# Per-version subsystem NQN: prefer the one baked into verity.json; otherwise
|
|
# derive it the same way extract-artifacts.sh does (base-<version>).
|
|
release_nqn() {
|
|
local v=$1 nqn
|
|
nqn=$(vfield "$v" .subnqn)
|
|
[ -n "$nqn" ] || nqn="${SUBNQN_BASE}-${v}"
|
|
printf '%s' "$nqn"
|
|
}
|
|
|
|
subsys_exists() {
|
|
rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c '
|
|
import json,sys
|
|
try:
|
|
subs=json.load(sys.stdin)
|
|
except Exception:
|
|
sys.exit(1)
|
|
sys.exit(0 if any(s.get("nqn")==sys.argv[1] for s in subs) else 1)' "$1"
|
|
}
|
|
|
|
# Connected controllers on a subsystem (the drain gate). >=0 when served; -1 if
|
|
# the subsystem is absent or the RPC failed (so a poller never mistakes an error
|
|
# for "drained" and reaps a live version).
|
|
ctrl_count() {
|
|
local out
|
|
if out=$(rpc nvmf_subsystem_get_controllers "$1" 2>/dev/null); then
|
|
printf '%s' "$out" | "$PY" -c '
|
|
import json,sys
|
|
try: print(len(json.load(sys.stdin)))
|
|
except Exception: print(-1)'
|
|
else
|
|
printf '%s' "-1"
|
|
fi
|
|
}
|
|
|
|
cmd_add() {
|
|
local v=${1:?usage: add <version>}
|
|
local dir="$RELEASES_DIR/$v"
|
|
# Validation failures here RETURN (not die/exit): cmd_add is called per-dir by
|
|
# cmd_reconcile as `cmd_add … || warn`, and an `exit` inside a function aborts
|
|
# the WHOLE process (|| only catches a return), so one bad/stale dir would stop
|
|
# every OTHER release from being added. Return so reconcile skips just this one.
|
|
[ -s "$dir/rootfs.erofs" ] || { echo "ERROR: $dir/rootfs.erofs missing/empty" >&2; return 1; }
|
|
[ -s "$dir/rootfs.hash" ] || { echo "ERROR: $dir/rootfs.hash missing/empty" >&2; return 1; }
|
|
|
|
# Guard: the dir name must match the version inside verity.json. A mismatch
|
|
# means a stale/symlinked or mislabeled release dir (e.g. an old symlink into
|
|
# the version-pinned /artifacts that now resolves to a different build) — that
|
|
# would serve the wrong data/NQN/UUIDs under this name. Refuse loudly. (Legacy
|
|
# artifacts have no image_version field — skip the check for them.)
|
|
# Case-insensitive: the dir name comes from the OCI tag while verity.json's
|
|
# image_version comes from the build var, which can differ only in case — a
|
|
# benign skew. A genuine mismatch (e.g. pxe-… vs spdk-…) differs by far more.
|
|
local vv
|
|
vv=$(vfield "$v" .image_version)
|
|
if [ -n "$vv" ] && [ "${vv,,}" != "${v,,}" ]; then
|
|
echo "ERROR: release dir '$v' holds verity.json for version '$vv' (stale symlink / mislabeled dir). Refusing to add — remove '$dir' and re-stage." >&2
|
|
return 1
|
|
fi
|
|
|
|
local nqn du hu s
|
|
nqn=$(release_nqn "$v")
|
|
du=$(vfield "$v" .data_ns_uuid)
|
|
hu=$(vfield "$v" .hash_ns_uuid)
|
|
# LEGACY FALLBACK: artifacts built before the per-version-UUID change have no
|
|
# data_ns_uuid/hash_ns_uuid in verity.json. Such a build's UKI cmdline baked
|
|
# the fixed pair below, so serve it with exactly those. Two such LEGACY
|
|
# versions can't be co-served (they'd collide on these fixed UUIDs) — rebuild
|
|
# with the current extract-artifacts.sh for true multi-version serving.
|
|
if [ -z "$du" ] || [ -z "$hu" ]; then
|
|
du=${du:-$LEGACY_DATA_UUID}
|
|
hu=${hu:-$LEGACY_HASH_UUID}
|
|
echo "WARNING: $dir/verity.json lacks per-version namespace UUIDs — using the legacy fixed pair ($du / $hu). OK for a single legacy version; rebuild for multi-version." >&2
|
|
fi
|
|
|
|
if subsys_exists "$nqn"; then
|
|
echo "release $v already served ($nqn) — nothing to do"
|
|
return 0
|
|
fi
|
|
|
|
s=$(san "$v")
|
|
echo "==> adding release $v as subsystem $nqn (data uuid $du, hash uuid $hu)"
|
|
# -r opens the backing file read-only where the SPDK build supports it.
|
|
rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096 -r \
|
|
|| rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096
|
|
rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096 -r \
|
|
|| rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096
|
|
# Serial must be <=20 chars; derive a stable, version-tagged one.
|
|
rpc nvmf_create_subsystem "$nqn" -a -s "SPDK$(printf '%s' "$s" | cut -c1-16)" -d "FleetRootfs"
|
|
rpc nvmf_subsystem_add_ns "$nqn" "rootfs_$s" -u "$du"
|
|
rpc nvmf_subsystem_add_ns "$nqn" "roothash_$s" -u "$hu"
|
|
rpc nvmf_subsystem_add_listener "$nqn" -t tcp -a "$TADDR" -s "$TPORT"
|
|
# NOTE: we deliberately do NOT (re)add the discovery-subsystem listener here.
|
|
# The long-lived target registers it ONCE at startup (spdk-target.sh), and a
|
|
# subsystem becomes discoverable simply by having its own listener on the same
|
|
# address. Re-adding it would make the nvmf_tgt DAEMON log "Listener already
|
|
# exists" to the container stdout on every add — noise the client can't
|
|
# suppress (the daemon, not rpc.py, emits it). One registration is enough.
|
|
echo "==> release $v ONLINE (existing connections untouched)"
|
|
}
|
|
|
|
cmd_remove() {
|
|
local force=0 v=""
|
|
local a
|
|
for a in "$@"; do
|
|
case "$a" in
|
|
--force) force=1 ;;
|
|
*) v="$a" ;;
|
|
esac
|
|
done
|
|
[ -n "$v" ] || die "usage: remove <version> [--force]"
|
|
|
|
local nqn n
|
|
nqn=$(release_nqn "$v")
|
|
if ! subsys_exists "$nqn"; then
|
|
echo "release $v not served ($nqn) — nothing to remove"
|
|
return 0
|
|
fi
|
|
n=$(ctrl_count "$nqn")
|
|
if [ "$n" -gt 0 ] && [ "$force" -ne 1 ]; then
|
|
die "release $v still has $n connected controller(s) — refusing to remove. Drain first (reboot/migrate those hosts), or pass --force to EVICT them (they lose root I/O)."
|
|
fi
|
|
[ "$n" -gt 0 ] && echo "WARNING: --force removing $v with $n connected controller(s) — those hosts lose root I/O"
|
|
|
|
local s
|
|
s=$(san "$v")
|
|
echo "==> removing release $v ($nqn)"
|
|
rpc nvmf_delete_subsystem "$nqn" || true
|
|
rpc bdev_aio_delete "rootfs_$s" 2>/dev/null || true
|
|
rpc bdev_aio_delete "roothash_$s" 2>/dev/null || true
|
|
echo "==> release $v removed (other versions + their connections untouched)"
|
|
}
|
|
|
|
cmd_list() {
|
|
local nqn n ver
|
|
printf '%-26s %-7s %s\n' VERSION CONNS NQN
|
|
rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c '
|
|
import json,sys
|
|
try: subs=json.load(sys.stdin)
|
|
except Exception: subs=[]
|
|
for s in subs:
|
|
n=s.get("nqn","")
|
|
if "discovery" not in n: print(n)' \
|
|
| while read -r nqn; do
|
|
n=$(ctrl_count "$nqn")
|
|
ver="${nqn##*:}" # strip the nqn.*: prefix for readability
|
|
printf '%-26s %-7s %s\n' "$ver" "$n" "$nqn"
|
|
done
|
|
}
|
|
|
|
cmd_drain_status() {
|
|
local v=${1:?usage: drain-status <version>}
|
|
local nqn n
|
|
nqn=$(release_nqn "$v")
|
|
n=$(ctrl_count "$nqn")
|
|
if [ "$n" -lt 0 ]; then
|
|
echo "$nqn is not served (or target unreachable)"
|
|
else
|
|
echo "$n connected controller(s) on $nqn"
|
|
fi
|
|
}
|
|
|
|
cmd_count() {
|
|
local v=${1:?usage: count <version>}
|
|
ctrl_count "$(release_nqn "$v")"
|
|
echo
|
|
}
|
|
|
|
cmd_reconcile() {
|
|
[ -d "$RELEASES_DIR" ] || { echo "no $RELEASES_DIR — nothing to reconcile"; return 0; }
|
|
local d any=0
|
|
for d in "$RELEASES_DIR"/*/; do
|
|
[ -e "${d}verity.json" ] || continue
|
|
any=1
|
|
cmd_add "$(basename "$d")" || echo "WARNING: failed to add release $(basename "$d")" >&2
|
|
done
|
|
[ "$any" -eq 1 ] || echo "no releases found under $RELEASES_DIR"
|
|
}
|
|
|
|
case "${1:-}" in
|
|
add) shift; cmd_add "$@" ;;
|
|
remove|rm) shift; cmd_remove "$@" ;;
|
|
list|ls) shift; cmd_list "$@" ;;
|
|
drain-status|status) shift; cmd_drain_status "$@" ;;
|
|
count) shift; cmd_count "$@" ;;
|
|
reconcile) shift; cmd_reconcile "$@" ;;
|
|
*) echo "usage: spdk-release {add <ver>|remove <ver> [--force]|list|drain-status <ver>|count <ver>|reconcile}" >&2; exit 2 ;;
|
|
esac
|