Adding spdk loader
This commit is contained in:
@@ -84,3 +84,13 @@ SUBNQN_BASE=nqn.2026-06.org.example.fleet:rootfs
|
||||
SUBNQN=nqn.2026-06.org.example.fleet:rootfs
|
||||
TARGET_PORT=4420
|
||||
SPDK_CPUMASK=0x1 # one polling core is plenty for a read-only boot fs
|
||||
|
||||
# --- SPDK version loader / reaper (declarative rolling updates) -------------
|
||||
# The spdk-loader service stages + adds ${IMAGE_VERSION} to the long-lived target
|
||||
# on `compose up`, then auto-removes OLD versions once they drain. Migration =
|
||||
# bump IMAGE_VERSION above + `docker compose up -d`.
|
||||
REAP_OLD=true # false = keep old versions until you remove them manually
|
||||
REAP_POLL_SECONDS=30 # how often to poll connection counts
|
||||
# Sustained-zero window before an old version is removed. Keep it comfortably
|
||||
# ABOVE the host's nvme ctrl_loss_tmo so a transient reconnect never gets reaped.
|
||||
REAP_GRACE_SECONDS=300
|
||||
|
||||
@@ -255,11 +255,14 @@ services:
|
||||
# spdk-release = JSON-RPC add/remove/list/drain of a version, live
|
||||
- ./spdk/spdk-target.sh:/usr/local/bin/spdk-target:ro
|
||||
- ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
|
||||
# Shared RPC socket so the spdk-loader container (and `compose exec`) can
|
||||
# drive this target's JSON-RPC. nvmf_tgt creates the socket here (-r).
|
||||
- spdk-sock:/var/run/spdk
|
||||
# Multi-version release store: each ./spdk/releases/<ver>/ holds that
|
||||
# version's rootfs.erofs + rootfs.hash + verity.json. Staged by
|
||||
# spdk-ctl.sh on the host; reconciled into the target at (re)start and
|
||||
# added/removed live thereafter. RW so the /artifacts bootstrap seed and
|
||||
# spdk-ctl can write here. Adding a version does NOT recreate this service.
|
||||
# version's rootfs.erofs + rootfs.hash + verity.json. Staged by the
|
||||
# spdk-loader (or spdk-ctl.sh on the host); reconciled into the target at
|
||||
# (re)start and added/removed live thereafter. RW so the bootstrap seed +
|
||||
# loader can write here. Adding a version does NOT recreate this service.
|
||||
- ./spdk/releases:/releases
|
||||
# Bootstrap convenience (optional): a single OCI release mounted as an
|
||||
# image volume. spdk-target seeds it into /releases as ${IMAGE_VERSION} if
|
||||
@@ -276,11 +279,65 @@ services:
|
||||
environment:
|
||||
IMAGE_VERSION: ${IMAGE_VERSION} # bootstrap-seed label for /artifacts
|
||||
RELEASES_DIR: /releases
|
||||
RPC_SOCK: /var/run/spdk/spdk.sock # shared with spdk-loader (-r / -s)
|
||||
SUBNQN_BASE: ${SUBNQN_BASE} # per-version NQN base (falls back per release to verity.json .subnqn)
|
||||
TARGET_ADDR: ${SERVER_IP}
|
||||
TARGET_PORT: ${TARGET_PORT}
|
||||
SPDK_CPUMASK: ${SPDK_CPUMASK}
|
||||
NVMEOF_SVC: nvmeof
|
||||
# Healthy once the RPC socket answers — gates the spdk-loader's startup.
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "rpc.py -s /var/run/spdk/spdk.sock framework_get_config >/dev/null 2>&1 || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 24
|
||||
start_period: 10s
|
||||
# Persistent target: started ONCE, never restarted for a version change.
|
||||
# Rolling updates add/remove versions online (serving/spdk/spdk-ctl.sh).
|
||||
# Rolling updates add/remove versions online (spdk-loader, or spdk-ctl.sh).
|
||||
entrypoint: ["/bin/bash", "/usr/local/bin/spdk-target"]
|
||||
|
||||
# --- SPDK version loader / reaper (declarative rolling updates) -------------
|
||||
# The "init container that stays up." On `compose up` after an IMAGE_VERSION
|
||||
# bump it stages + adds the new rootfs to the running nvmeof target ONLINE
|
||||
# (no target restart, no dropped connections), then polls and removes old
|
||||
# versions once they drain (all hosts rebooted onto the new UKI). Migration is
|
||||
# just: bump IMAGE_VERSION in .env -> `docker compose up -d`. Reuses the SPDK
|
||||
# image (has python3 + rpc.py); the scripts are bind-mounted like nvmeof's.
|
||||
spdk-loader:
|
||||
image: ${SPDK_IMAGE}:${SPDK_VERSION}
|
||||
network_mode: host
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
nvmeof:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./spdk/spdk-loader.sh:/usr/local/bin/spdk-loader:ro
|
||||
- ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
|
||||
- spdk-sock:/var/run/spdk # shared RPC socket to the target
|
||||
- ./spdk/releases:/releases # shared release store
|
||||
# The CURRENT release's artifacts, mounted from its OCI image. The version
|
||||
# tag is the single migration knob — bump IMAGE_VERSION and `up -d`.
|
||||
- type: image
|
||||
source: ${OCI_IMAGE}:${IMAGE_VERSION}
|
||||
target: /in
|
||||
image:
|
||||
subpath: artifacts
|
||||
# Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/in:ro
|
||||
environment:
|
||||
IMAGE_VERSION: ${IMAGE_VERSION} # the CURRENT version (staged + protected from reaping)
|
||||
RELEASES_DIR: /releases
|
||||
ARTIFACT_IN: /in
|
||||
RPC_SOCK: /var/run/spdk/spdk.sock
|
||||
SUBNQN_BASE: ${SUBNQN_BASE}
|
||||
TARGET_ADDR: ${SERVER_IP}
|
||||
TARGET_PORT: ${TARGET_PORT}
|
||||
REAP_OLD: ${REAP_OLD:-true} # auto-remove drained old versions
|
||||
REAP_POLL_SECONDS: ${REAP_POLL_SECONDS:-30}
|
||||
REAP_GRACE_SECONDS: ${REAP_GRACE_SECONDS:-300} # sustained-zero window; keep > host ctrl_loss_tmo
|
||||
entrypoint: ["/bin/bash", "/usr/local/bin/spdk-loader"]
|
||||
|
||||
# Named volume for the SPDK JSON-RPC unix socket, shared between the long-lived
|
||||
# nvmeof target (which creates it via nvmf_tgt -r) and the spdk-loader (which
|
||||
# drives add/remove/drain over it). Keeps the control path off the host FS.
|
||||
volumes:
|
||||
spdk-sock:
|
||||
|
||||
+28
-2
@@ -23,7 +23,8 @@ process stays up.** Routine rollouts never restart SPDK.
|
||||
|---|---|
|
||||
| [`spdk-target.sh`](spdk-target.sh) | **Persistent entrypoint.** Starts `nvmf_tgt` once, creates the TCP transport + discovery listener, then reconciles every staged release. Never restarted to change versions. On (re)start it re-adds everything under `./releases/`, so a serving-host reboot restores all served versions. |
|
||||
| [`spdk-release.sh`](spdk-release.sh) | **In-container control** (JSON-RPC). `add`/`remove`/`list`/`drain-status`/`reconcile` a version **live**. One **subsystem per version** (distinct NQN) so adding/removing one never disturbs the subsystem other hosts are attached to. Removal is **drain-gated**. |
|
||||
| [`spdk-ctl.sh`](spdk-ctl.sh) | **Host-side ops.** Stages a release's artifacts from its OCI image into `./releases/<ver>/` and drives the live target via `docker compose exec`. |
|
||||
| [`spdk-ctl.sh`](spdk-ctl.sh) | **Host-side ops** (manual / cleanup). Stages a release's artifacts from its OCI image into `./releases/<ver>/` and drives the live target via `docker compose exec`. |
|
||||
| [`spdk-loader.sh`](spdk-loader.sh) | **Declarative rolling updates** (compose service `spdk-loader`, the "init container that stays up"). On `compose up` after an `IMAGE_VERSION` bump it stages + adds the new rootfs online, then auto-reaps old versions once they drain. |
|
||||
| `./releases/<version>/` | Per-version `rootfs.erofs` + `rootfs.hash` + `verity.json`. `verity.json` is the **single source of truth** for that version's namespace UUIDs + subsystem NQN (written by `scripts/extract-artifacts.sh`), so the host's by-id verity refs and the target's namespace UUIDs always agree. |
|
||||
|
||||
Per-version namespace UUIDs are required: two versions served at once must have
|
||||
@@ -32,7 +33,32 @@ globally-distinct namespace UUIDs or they collide on the same
|
||||
derives them deterministically (UUIDv5 of the version) and bakes them into both
|
||||
the immutable UKI cmdline and `verity.json`.
|
||||
|
||||
## Rolling update (add new, drain old, clean up)
|
||||
## Rolling update — the declarative way (recommended)
|
||||
|
||||
Bump one tag and bring the stack up:
|
||||
|
||||
```bash
|
||||
# in serving/.env
|
||||
IMAGE_VERSION=1.3.0
|
||||
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
Compose recreates the **`spdk-loader`** and the **http/dnsmasq** UKI servers (their
|
||||
image volumes changed), but **not** the `nvmeof` target (its definition is
|
||||
unchanged) — so no connections drop. The loader adds 1.3.0 to the live target,
|
||||
serves the new UKI to all hosts, and then **auto-reaps** 1.2.0 once every host
|
||||
has rebooted onto 1.3.0 and 1.2.0 has been drained for `REAP_GRACE_SECONDS`.
|
||||
Rollback = set `IMAGE_VERSION` back + `up -d`. Set `REAP_OLD=false` to keep old
|
||||
versions and reap them yourself.
|
||||
|
||||
Why auto-reap is safe here: there is a **single served UKI**, so a powered-off
|
||||
host that returns boots the *current* UKI — nothing can come back to an old
|
||||
rootfs once its live connections (CONNS) hit 0. Removal is still drain-gated and
|
||||
debounced by the grace window (ride out transient nvme-tcp reconnects — keep
|
||||
`REAP_GRACE_SECONDS` above the host `ctrl_loss_tmo`).
|
||||
|
||||
## Rolling update — the manual way (`spdk-ctl.sh`)
|
||||
|
||||
```bash
|
||||
cd serving/spdk
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env bash
|
||||
# spdk-loader.sh — declarative version manager for the long-lived SPDK target.
|
||||
#
|
||||
# Runs as a long-lived compose service alongside the nvmeof target. On (re)start:
|
||||
# 1. STAGE the CURRENT release (${IMAGE_VERSION}) from the mounted artifact
|
||||
# image (/in) into the shared release store (/releases/<ver>/) and ADD it to
|
||||
# the running target ONLINE (no target restart).
|
||||
# 2. Then stay up and REAP: periodically poll connection counts and remove any
|
||||
# OTHER (non-current) version once it has been drained (0 connected
|
||||
# controllers) for a sustained grace window — i.e. once every host that was
|
||||
# running it has rebooted onto the new UKI.
|
||||
#
|
||||
# Migration is therefore: bump IMAGE_VERSION in .env, `docker compose up -d`.
|
||||
# Compose recreates THIS loader (its artifact mount + env changed) and the
|
||||
# http/dnsmasq UKI servers (image volume changed), but NOT the nvmeof target
|
||||
# (its definition is unchanged) — so live connections are never dropped. The new
|
||||
# loader adds the new rootfs online and drains the old one as hosts roll over.
|
||||
#
|
||||
# SAFETY: with a single served UKI a host can only ever boot the CURRENT version,
|
||||
# so a non-current version at 0 connections is genuinely orphaned (no powered-off
|
||||
# host can return to it — on next boot it fetches the current UKI). Removal is
|
||||
# still drain-gated in spdk-release (refuses >0), and gated here by a sustained-
|
||||
# zero grace window (>= REAP_GRACE_SECONDS) to ride out transient nvme-tcp
|
||||
# reconnects. Set REAP_GRACE_SECONDS comfortably above the host ctrl_loss_tmo.
|
||||
set -euo pipefail
|
||||
|
||||
CURRENT="${IMAGE_VERSION:?spdk-loader needs IMAGE_VERSION}"
|
||||
RELEASES_DIR="${RELEASES_DIR:-/releases}"
|
||||
IN="${ARTIFACT_IN:-/in}"
|
||||
POLL="${REAP_POLL_SECONDS:-30}"
|
||||
GRACE="${REAP_GRACE_SECONDS:-300}"
|
||||
REAP="${REAP_OLD:-true}"
|
||||
RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release)
|
||||
# spdk-release reads RPC_SOCK from the environment (shared socket to the target).
|
||||
|
||||
echo "==> spdk-loader: current=${CURRENT} reap=${REAP} poll=${POLL}s grace=${GRACE}s"
|
||||
|
||||
# --- 1) stage + add the current release (idempotent) -----------------------
|
||||
dest="$RELEASES_DIR/$CURRENT"
|
||||
mkdir -p "$dest"
|
||||
if [ -s "$IN/verity.json" ]; then
|
||||
# Skip the (large) copy if already staged with identical verity.json.
|
||||
if [ -s "$dest/verity.json" ] && cmp -s "$IN/verity.json" "$dest/verity.json"; then
|
||||
echo "==> $CURRENT already staged in $dest (verity.json identical) — skipping copy"
|
||||
else
|
||||
echo "==> staging $CURRENT from $IN -> $dest"
|
||||
cp "$IN/rootfs.erofs" "$dest/rootfs.erofs"
|
||||
cp "$IN/rootfs.hash" "$dest/rootfs.hash"
|
||||
cp "$IN/verity.json" "$dest/verity.json"
|
||||
fi
|
||||
else
|
||||
echo "WARNING: no $IN/verity.json (artifact image not mounted at $IN?) — relying on pre-staged $dest" >&2
|
||||
fi
|
||||
bash "$RELEASE_CTL" add "$CURRENT"
|
||||
bash "$RELEASE_CTL" list
|
||||
|
||||
if [ "$REAP" != "true" ]; then
|
||||
echo "==> REAP_OLD=$REAP — staying up for status only; old versions are NOT auto-removed."
|
||||
fi
|
||||
|
||||
# --- 2) reap loop: a non-current version drained for >= GRACE is removed ----
|
||||
declare -A zero_since
|
||||
while true; do
|
||||
sleep "$POLL"
|
||||
[ "$REAP" = "true" ] || continue
|
||||
[ -d "$RELEASES_DIR" ] || continue
|
||||
for d in "$RELEASES_DIR"/*/; do
|
||||
[ -e "${d}verity.json" ] || continue
|
||||
v=$(basename "$d")
|
||||
[ "$v" = "$CURRENT" ] && continue # never reap the current version
|
||||
n=$(bash "$RELEASE_CTL" count "$v" 2>/dev/null || echo -1)
|
||||
if [ "$n" = "0" ]; then
|
||||
ts=${zero_since[$v]:-}
|
||||
if [ -z "$ts" ]; then
|
||||
ts=$(date +%s); zero_since[$v]=$ts
|
||||
echo "==> $v drained (0 conns) — starting ${GRACE}s grace before removal"
|
||||
fi
|
||||
age=$(( $(date +%s) - ts ))
|
||||
if [ "$age" -ge "$GRACE" ]; then
|
||||
echo "==> reaping $v (drained ${age}s >= ${GRACE}s grace)"
|
||||
if bash "$RELEASE_CTL" remove "$v"; then
|
||||
rm -rf "${RELEASES_DIR:?}/$v"
|
||||
unset 'zero_since[$v]'
|
||||
echo "==> $v removed + staged files deleted"
|
||||
else
|
||||
# A connection reappeared between the poll and the remove —
|
||||
# spdk-release refused (drain-gated). Reset and retry later.
|
||||
echo "WARNING: remove $v refused (reconnected?) — grace reset, will retry" >&2
|
||||
unset 'zero_since[$v]'
|
||||
fi
|
||||
fi
|
||||
else
|
||||
# >0 connections, or RPC error (-1): reset the grace timer.
|
||||
if [ -n "${zero_since[$v]:-}" ]; then
|
||||
echo "==> $v no longer drained (conns=$n) — grace reset"
|
||||
unset 'zero_since[$v]'
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
@@ -17,11 +17,15 @@
|
||||
# namespace UUIDs + subsystem NQN (written by scripts/extract-artifacts.sh) — so
|
||||
# the host's by-id verity refs and the target's namespace UUIDs always agree.
|
||||
#
|
||||
# JSON is parsed with python3 (always present — rpc.py runs on it), NOT jq, which
|
||||
# the SPDK image does not ship.
|
||||
#
|
||||
# Usage:
|
||||
# spdk-release add <version> # idempotent; live, no restart
|
||||
# spdk-release remove <version> [--force] # DRAIN-GATED unless --force
|
||||
# spdk-release list # served versions + connected hosts
|
||||
# spdk-release drain-status <version> # connected controller count
|
||||
# spdk-release drain-status <version> # connected controller count (human)
|
||||
# spdk-release count <version> # connected controller count (bare int; -1 if absent/error)
|
||||
# spdk-release reconcile # add every release under $RELEASES_DIR
|
||||
set -euo pipefail
|
||||
|
||||
@@ -30,17 +34,32 @@ TADDR="${TARGET_ADDR:-0.0.0.0}"
|
||||
TPORT="${TARGET_PORT:-4420}"
|
||||
SUBNQN_BASE="${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs}"
|
||||
DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery"
|
||||
# Legacy fixed namespace UUIDs (pre per-version-UUID builds baked these into the
|
||||
# UKI cmdline). Used as a fallback when verity.json lacks the per-version pair.
|
||||
LEGACY_DATA_UUID="f1ee7000-0000-4000-8000-000000000001"
|
||||
LEGACY_HASH_UUID="f1ee7000-0000-4000-8000-000000000002"
|
||||
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
|
||||
PY=$(command -v python3 || echo python3)
|
||||
# Optional explicit RPC socket (shared between the long-lived target and the
|
||||
# loader container). Empty -> rpc.py's default socket (single-container case).
|
||||
RPC_SOCK="${RPC_SOCK:-}"
|
||||
|
||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||
rpc() { "$RPC" "$@"; }
|
||||
# bdev names must be filesystem/identifier-safe; sanitise the version.
|
||||
rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; }
|
||||
# bdev names must be identifier-safe; sanitise the version.
|
||||
san() { printf '%s' "$1" | tr -c 'A-Za-z0-9' '_'; }
|
||||
|
||||
# Read a field from a release's verity.json (empty if missing).
|
||||
# Read a top-level field from a release's verity.json (empty if missing/unreadable).
|
||||
vfield() {
|
||||
local v=$1 path=$2
|
||||
jq -r "$path // empty" "$RELEASES_DIR/$v/verity.json" 2>/dev/null || true
|
||||
local v=$1 key=${2#.}
|
||||
"$PY" -c '
|
||||
import json,sys
|
||||
try:
|
||||
d=json.load(open(sys.argv[1]))
|
||||
x=d.get(sys.argv[2],"")
|
||||
sys.stdout.write("" if x is None else str(x))
|
||||
except Exception:
|
||||
pass' "$RELEASES_DIR/$v/verity.json" "$key" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Per-version subsystem NQN: prefer the one baked into verity.json; otherwise
|
||||
@@ -53,15 +72,28 @@ release_nqn() {
|
||||
}
|
||||
|
||||
subsys_exists() {
|
||||
rpc nvmf_get_subsystems 2>/dev/null \
|
||||
| jq -e --arg n "$1" 'any(.[]; .nqn==$n)' >/dev/null 2>&1
|
||||
rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c '
|
||||
import json,sys
|
||||
try:
|
||||
subs=json.load(sys.stdin)
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
sys.exit(0 if any(s.get("nqn")==sys.argv[1] for s in subs) else 1)' "$1"
|
||||
}
|
||||
|
||||
# Connected controllers on a subsystem (the drain gate). 0 if absent/none.
|
||||
# Connected controllers on a subsystem (the drain gate). >=0 when served; -1 if
|
||||
# the subsystem is absent or the RPC failed (so a poller never mistakes an error
|
||||
# for "drained" and reaps a live version).
|
||||
ctrl_count() {
|
||||
local n
|
||||
n=$(rpc nvmf_subsystem_get_controllers "$1" 2>/dev/null | jq 'length' 2>/dev/null || echo 0)
|
||||
printf '%s' "${n:-0}"
|
||||
local out
|
||||
if out=$(rpc nvmf_subsystem_get_controllers "$1" 2>/dev/null); then
|
||||
printf '%s' "$out" | "$PY" -c '
|
||||
import json,sys
|
||||
try: print(len(json.load(sys.stdin)))
|
||||
except Exception: print(-1)'
|
||||
else
|
||||
printf '%s' "-1"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_add() {
|
||||
@@ -74,8 +106,16 @@ cmd_add() {
|
||||
nqn=$(release_nqn "$v")
|
||||
du=$(vfield "$v" .data_ns_uuid)
|
||||
hu=$(vfield "$v" .hash_ns_uuid)
|
||||
[ -n "$du" ] && [ -n "$hu" ] \
|
||||
|| die "$dir/verity.json lacks data_ns_uuid/hash_ns_uuid (rebuild with the per-version UUID extract-artifacts.sh)"
|
||||
# LEGACY FALLBACK: artifacts built before the per-version-UUID change have no
|
||||
# data_ns_uuid/hash_ns_uuid in verity.json. Such a build's UKI cmdline baked
|
||||
# the fixed pair below, so serve it with exactly those. Two such LEGACY
|
||||
# versions can't be co-served (they'd collide on these fixed UUIDs) — rebuild
|
||||
# with the current extract-artifacts.sh for true multi-version serving.
|
||||
if [ -z "$du" ] || [ -z "$hu" ]; then
|
||||
du=${du:-$LEGACY_DATA_UUID}
|
||||
hu=${hu:-$LEGACY_HASH_UUID}
|
||||
echo "WARNING: $dir/verity.json lacks per-version namespace UUIDs — using the legacy fixed pair ($du / $hu). OK for a single legacy version; rebuild for multi-version." >&2
|
||||
fi
|
||||
|
||||
if subsys_exists "$nqn"; then
|
||||
echo "release $v already served ($nqn) — nothing to do"
|
||||
@@ -135,8 +175,13 @@ cmd_remove() {
|
||||
cmd_list() {
|
||||
local nqn n ver
|
||||
printf '%-26s %-7s %s\n' VERSION CONNS NQN
|
||||
rpc nvmf_get_subsystems 2>/dev/null \
|
||||
| jq -r '.[] | select(.nqn | test("discovery") | not) | .nqn' \
|
||||
rpc nvmf_get_subsystems 2>/dev/null | "$PY" -c '
|
||||
import json,sys
|
||||
try: subs=json.load(sys.stdin)
|
||||
except Exception: subs=[]
|
||||
for s in subs:
|
||||
n=s.get("nqn","")
|
||||
if "discovery" not in n: print(n)' \
|
||||
| while read -r nqn; do
|
||||
n=$(ctrl_count "$nqn")
|
||||
ver="${nqn##*:}" # strip the nqn.*: prefix for readability
|
||||
@@ -146,9 +191,20 @@ cmd_list() {
|
||||
|
||||
cmd_drain_status() {
|
||||
local v=${1:?usage: drain-status <version>}
|
||||
local nqn
|
||||
local nqn n
|
||||
nqn=$(release_nqn "$v")
|
||||
echo "$(ctrl_count "$nqn") connected controller(s) on $nqn"
|
||||
n=$(ctrl_count "$nqn")
|
||||
if [ "$n" -lt 0 ]; then
|
||||
echo "$nqn is not served (or target unreachable)"
|
||||
else
|
||||
echo "$n connected controller(s) on $nqn"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_count() {
|
||||
local v=${1:?usage: count <version>}
|
||||
ctrl_count "$(release_nqn "$v")"
|
||||
echo
|
||||
}
|
||||
|
||||
cmd_reconcile() {
|
||||
@@ -167,6 +223,7 @@ case "${1:-}" in
|
||||
remove|rm) shift; cmd_remove "$@" ;;
|
||||
list|ls) shift; cmd_list "$@" ;;
|
||||
drain-status|status) shift; cmd_drain_status "$@" ;;
|
||||
count) shift; cmd_count "$@" ;;
|
||||
reconcile) shift; cmd_reconcile "$@" ;;
|
||||
*) echo "usage: spdk-release {add <ver>|remove <ver> [--force]|list|drain-status <ver>|reconcile}" >&2; exit 2 ;;
|
||||
*) echo "usage: spdk-release {add <ver>|remove <ver> [--force]|list|drain-status <ver>|count <ver>|reconcile}" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
@@ -23,6 +23,12 @@ DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery"
|
||||
TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt)
|
||||
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
|
||||
RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release)
|
||||
# Optional explicit RPC socket. Set it (compose does, on a shared volume) so a
|
||||
# SEPARATE loader container can reach this target's JSON-RPC and add/remove
|
||||
# versions online. Empty -> nvmf_tgt's default socket (single-container case).
|
||||
RPC_SOCK="${RPC_SOCK:-}"
|
||||
rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; }
|
||||
[ -n "$RPC_SOCK" ] && mkdir -p "$(dirname "$RPC_SOCK")"
|
||||
|
||||
# Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced, so
|
||||
# a privileged container writes the host value; re-applied after a host reboot
|
||||
@@ -38,8 +44,10 @@ fi
|
||||
free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0)
|
||||
[ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2
|
||||
|
||||
echo "==> starting LONG-LIVED nvmf_tgt (cpumask ${CPUMASK})"
|
||||
"$TGT" -m "$CPUMASK" &
|
||||
echo "==> starting LONG-LIVED nvmf_tgt (cpumask ${CPUMASK})${RPC_SOCK:+ rpc-sock $RPC_SOCK}"
|
||||
TGT_ARGS=(-m "$CPUMASK")
|
||||
[ -n "$RPC_SOCK" ] && TGT_ARGS+=(-r "$RPC_SOCK")
|
||||
"$TGT" "${TGT_ARGS[@]}" &
|
||||
TGT_PID=$!
|
||||
# A TERM/INT here stops the target and DROPS EVERY connection — only on container
|
||||
# stop, never for a version change. Make the intent loud.
|
||||
@@ -48,16 +56,16 @@ trap 'kill "$TGT_PID" 2>/dev/null || true' EXIT
|
||||
|
||||
# Wait for the RPC socket.
|
||||
for _ in $(seq 1 30); do
|
||||
"$RPC" framework_get_config >/dev/null 2>&1 && break
|
||||
rpc framework_get_config >/dev/null 2>&1 && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "==> creating TCP transport"
|
||||
"$RPC" nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192 \
|
||||
rpc nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192 \
|
||||
|| echo "WARNING: nvmf_create_transport failed (already present?)" >&2
|
||||
|
||||
echo "==> registering discovery listener ${TADDR}:${TPORT}"
|
||||
"$RPC" nvmf_subsystem_add_listener "$DISCOVERY_NQN" -t tcp -a "$TADDR" -s "$TPORT" 2>/dev/null || true
|
||||
rpc nvmf_subsystem_add_listener "$DISCOVERY_NQN" -t tcp -a "$TADDR" -s "$TPORT" 2>/dev/null || true
|
||||
|
||||
# Bootstrap convenience: if a single OCI artifact image is mounted at /artifacts
|
||||
# (the simple one-version `docker compose up` flow) and that version isn't yet
|
||||
@@ -78,7 +86,7 @@ if [ -s /artifacts/verity.json ] && [ -n "${IMAGE_VERSION:-}" ]; then
|
||||
fi
|
||||
|
||||
echo "==> reconciling releases under $RELEASES_DIR"
|
||||
RELEASES_DIR="$RELEASES_DIR" TARGET_ADDR="$TADDR" TARGET_PORT="$TPORT" \
|
||||
RELEASES_DIR="$RELEASES_DIR" TARGET_ADDR="$TADDR" TARGET_PORT="$TPORT" RPC_SOCK="$RPC_SOCK" \
|
||||
bash "$RELEASE_CTL" reconcile || echo "WARNING: reconcile reported errors" >&2
|
||||
|
||||
echo "==> nvmf_tgt up; releases served online. Add/remove versions with:"
|
||||
|
||||
Reference in New Issue
Block a user