173 lines
7.0 KiB
Bash
Executable File
173 lines
7.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# spdk-release.sh — add/remove a rootfs RELEASE on the long-lived nvmf_tgt
|
|
# ONLINE, via JSON-RPC, WITHOUT restarting the target. The target process holds
|
|
# every running host's NVMe/TCP root connection, so restarting it tears down the
|
|
# whole fleet's root at once (see docs/TODO.md #6). Routine rollouts therefore
|
|
# never restart SPDK: this script adds the new version's bdevs + subsystem +
|
|
# namespaces live (existing connections untouched) and removes an old version
|
|
# live once it has drained.
|
|
#
|
|
# ONE SUBSYSTEM PER VERSION (distinct NQN): adding a release as a NEW subsystem
|
|
# never disturbs the subsystem running hosts are attached to (adding a namespace
|
|
# to a SHARED subsystem would fire an AER + rescan at every host). It also gives
|
|
# clean per-release drain + per-release hostNQN allow-listing.
|
|
#
|
|
# Each release lives under $RELEASES_DIR/<version>/ as rootfs.erofs + rootfs.hash
|
|
# + verity.json. verity.json is the single source of truth for that version's
|
|
# namespace UUIDs + subsystem NQN (written by scripts/extract-artifacts.sh) — so
|
|
# the host's by-id verity refs and the target's namespace UUIDs always agree.
|
|
#
|
|
# Usage:
|
|
# spdk-release add <version> # idempotent; live, no restart
|
|
# spdk-release remove <version> [--force] # DRAIN-GATED unless --force
|
|
# spdk-release list # served versions + connected hosts
|
|
# spdk-release drain-status <version> # connected controller count
|
|
# spdk-release reconcile # add every release under $RELEASES_DIR
|
|
set -euo pipefail
|
|
|
|
RELEASES_DIR="${RELEASES_DIR:-/releases}"
|
|
TADDR="${TARGET_ADDR:-0.0.0.0}"
|
|
TPORT="${TARGET_PORT:-4420}"
|
|
SUBNQN_BASE="${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs}"
|
|
DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery"
|
|
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
|
|
|
|
die() { echo "ERROR: $*" >&2; exit 1; }
|
|
rpc() { "$RPC" "$@"; }
|
|
# bdev names must be filesystem/identifier-safe; sanitise the version.
|
|
san() { printf '%s' "$1" | tr -c 'A-Za-z0-9' '_'; }
|
|
|
|
# Read a field from a release's verity.json (empty if missing).
|
|
vfield() {
|
|
local v=$1 path=$2
|
|
jq -r "$path // empty" "$RELEASES_DIR/$v/verity.json" 2>/dev/null || true
|
|
}
|
|
|
|
# Per-version subsystem NQN: prefer the one baked into verity.json; otherwise
|
|
# derive it the same way extract-artifacts.sh does (base-<version>).
|
|
release_nqn() {
|
|
local v=$1 nqn
|
|
nqn=$(vfield "$v" .subnqn)
|
|
[ -n "$nqn" ] || nqn="${SUBNQN_BASE}-${v}"
|
|
printf '%s' "$nqn"
|
|
}
|
|
|
|
subsys_exists() {
|
|
rpc nvmf_get_subsystems 2>/dev/null \
|
|
| jq -e --arg n "$1" 'any(.[]; .nqn==$n)' >/dev/null 2>&1
|
|
}
|
|
|
|
# Connected controllers on a subsystem (the drain gate). 0 if absent/none.
|
|
ctrl_count() {
|
|
local n
|
|
n=$(rpc nvmf_subsystem_get_controllers "$1" 2>/dev/null | jq 'length' 2>/dev/null || echo 0)
|
|
printf '%s' "${n:-0}"
|
|
}
|
|
|
|
cmd_add() {
|
|
local v=${1:?usage: add <version>}
|
|
local dir="$RELEASES_DIR/$v"
|
|
[ -s "$dir/rootfs.erofs" ] || die "$dir/rootfs.erofs missing/empty"
|
|
[ -s "$dir/rootfs.hash" ] || die "$dir/rootfs.hash missing/empty"
|
|
|
|
local nqn du hu s
|
|
nqn=$(release_nqn "$v")
|
|
du=$(vfield "$v" .data_ns_uuid)
|
|
hu=$(vfield "$v" .hash_ns_uuid)
|
|
[ -n "$du" ] && [ -n "$hu" ] \
|
|
|| die "$dir/verity.json lacks data_ns_uuid/hash_ns_uuid (rebuild with the per-version UUID extract-artifacts.sh)"
|
|
|
|
if subsys_exists "$nqn"; then
|
|
echo "release $v already served ($nqn) — nothing to do"
|
|
return 0
|
|
fi
|
|
|
|
s=$(san "$v")
|
|
echo "==> adding release $v as subsystem $nqn (data uuid $du, hash uuid $hu)"
|
|
# -r opens the backing file read-only where the SPDK build supports it.
|
|
rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096 -r \
|
|
|| rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096
|
|
rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096 -r \
|
|
|| rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096
|
|
# Serial must be <=20 chars; derive a stable, version-tagged one.
|
|
rpc nvmf_create_subsystem "$nqn" -a -s "SPDK$(printf '%s' "$s" | cut -c1-16)" -d "FleetRootfs"
|
|
rpc nvmf_subsystem_add_ns "$nqn" "rootfs_$s" -u "$du"
|
|
rpc nvmf_subsystem_add_ns "$nqn" "roothash_$s" -u "$hu"
|
|
rpc nvmf_subsystem_add_listener "$nqn" -t tcp -a "$TADDR" -s "$TPORT"
|
|
# Register the listener on the discovery subsystem too (best-effort; older
|
|
# SPDK answers discovery implicitly but logs a deprecation warning).
|
|
rpc nvmf_subsystem_add_listener "$DISCOVERY_NQN" -t tcp -a "$TADDR" -s "$TPORT" 2>/dev/null || true
|
|
echo "==> release $v ONLINE (existing connections untouched)"
|
|
}
|
|
|
|
cmd_remove() {
|
|
local force=0 v=""
|
|
local a
|
|
for a in "$@"; do
|
|
case "$a" in
|
|
--force) force=1 ;;
|
|
*) v="$a" ;;
|
|
esac
|
|
done
|
|
[ -n "$v" ] || die "usage: remove <version> [--force]"
|
|
|
|
local nqn n
|
|
nqn=$(release_nqn "$v")
|
|
if ! subsys_exists "$nqn"; then
|
|
echo "release $v not served ($nqn) — nothing to remove"
|
|
return 0
|
|
fi
|
|
n=$(ctrl_count "$nqn")
|
|
if [ "$n" -gt 0 ] && [ "$force" -ne 1 ]; then
|
|
die "release $v still has $n connected controller(s) — refusing to remove. Drain first (reboot/migrate those hosts), or pass --force to EVICT them (they lose root I/O)."
|
|
fi
|
|
[ "$n" -gt 0 ] && echo "WARNING: --force removing $v with $n connected controller(s) — those hosts lose root I/O"
|
|
|
|
local s
|
|
s=$(san "$v")
|
|
echo "==> removing release $v ($nqn)"
|
|
rpc nvmf_delete_subsystem "$nqn" || true
|
|
rpc bdev_aio_delete "rootfs_$s" 2>/dev/null || true
|
|
rpc bdev_aio_delete "roothash_$s" 2>/dev/null || true
|
|
echo "==> release $v removed (other versions + their connections untouched)"
|
|
}
|
|
|
|
cmd_list() {
|
|
local nqn n ver
|
|
printf '%-26s %-7s %s\n' VERSION CONNS NQN
|
|
rpc nvmf_get_subsystems 2>/dev/null \
|
|
| jq -r '.[] | select(.nqn | test("discovery") | not) | .nqn' \
|
|
| while read -r nqn; do
|
|
n=$(ctrl_count "$nqn")
|
|
ver="${nqn##*:}" # strip the nqn.*: prefix for readability
|
|
printf '%-26s %-7s %s\n' "$ver" "$n" "$nqn"
|
|
done
|
|
}
|
|
|
|
cmd_drain_status() {
|
|
local v=${1:?usage: drain-status <version>}
|
|
local nqn
|
|
nqn=$(release_nqn "$v")
|
|
echo "$(ctrl_count "$nqn") connected controller(s) on $nqn"
|
|
}
|
|
|
|
cmd_reconcile() {
|
|
[ -d "$RELEASES_DIR" ] || { echo "no $RELEASES_DIR — nothing to reconcile"; return 0; }
|
|
local d any=0
|
|
for d in "$RELEASES_DIR"/*/; do
|
|
[ -e "${d}verity.json" ] || continue
|
|
any=1
|
|
cmd_add "$(basename "$d")" || echo "WARNING: failed to add release $(basename "$d")" >&2
|
|
done
|
|
[ "$any" -eq 1 ] || echo "no releases found under $RELEASES_DIR"
|
|
}
|
|
|
|
case "${1:-}" in
|
|
add) shift; cmd_add "$@" ;;
|
|
remove|rm) shift; cmd_remove "$@" ;;
|
|
list|ls) shift; cmd_list "$@" ;;
|
|
drain-status|status) shift; cmd_drain_status "$@" ;;
|
|
reconcile) shift; cmd_reconcile "$@" ;;
|
|
*) echo "usage: spdk-release {add <ver>|remove <ver> [--force]|list|drain-status <ver>|reconcile}" >&2; exit 2 ;;
|
|
esac
|