fixing final SPDK deployment

serving/docker-compose.yml

@@ -285,13 +285,18 @@ services:
       TARGET_PORT: ${TARGET_PORT}
       SPDK_CPUMASK: ${SPDK_CPUMASK}
       NVMEOF_SVC: nvmeof
-    # Healthy once the RPC socket answers — gates the spdk-loader's startup.
+    # Healthy once spdk-target has written its readiness stamp (transport up +
+    # releases reconciled). A FILE check, not a python rpc.py probe: nvmf_tgt does
+    # a slow DPDK init (~50s observed) and busy-polls its reactor core, so a python
+    # healthcheck starves/times out and the container sticks at "unhealthy" even
+    # though the target is up. `test -f` is instant and contention-free. The stamp
+    # is removed on (re)start + exit, so it never falsely reports a dead target.
     healthcheck:
-      test: ["CMD-SHELL", "rpc.py -s /var/run/spdk/spdk.sock framework_get_config >/dev/null 2>&1 || exit 1"]
-      interval: 5s
-      timeout: 3s
-      retries: 24
-      start_period: 10s
+      test: ["CMD-SHELL", "test -f /var/run/spdk/ready || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 6
+      start_period: 120s
     # Persistent target: started ONCE, never restarted for a version change.
     # Rolling updates add/remove versions online (spdk-loader, or spdk-ctl.sh).
     entrypoint: ["/bin/bash", "/usr/local/bin/spdk-target"]

serving/spdk/spdk-release.sh

@@ -98,8 +98,12 @@ except Exception: print(-1)'
 cmd_add() {
     local v=${1:?usage: add <version>}
     local dir="$RELEASES_DIR/$v"
-    [ -s "$dir/rootfs.erofs" ] || die "$dir/rootfs.erofs missing/empty"
-    [ -s "$dir/rootfs.hash" ]  || die "$dir/rootfs.hash missing/empty"
+    # Validation failures here RETURN (not die/exit): cmd_add is called per-dir by
+    # cmd_reconcile as `cmd_add … || warn`, and an `exit` inside a function aborts
+    # the WHOLE process (|| only catches a return), so one bad/stale dir would stop
+    # every OTHER release from being added. Return so reconcile skips just this one.
+    [ -s "$dir/rootfs.erofs" ] || { echo "ERROR: $dir/rootfs.erofs missing/empty" >&2; return 1; }
+    [ -s "$dir/rootfs.hash" ]  || { echo "ERROR: $dir/rootfs.hash missing/empty" >&2; return 1; }
 
     # Guard: the dir name must match the version inside verity.json. A mismatch
     # means a stale/symlinked or mislabeled release dir (e.g. an old symlink into
@@ -112,7 +116,8 @@ cmd_add() {
     local vv
     vv=$(vfield "$v" .image_version)
     if [ -n "$vv" ] && [ "${vv,,}" != "${v,,}" ]; then
-        die "release dir '$v' holds verity.json for version '$vv' (stale symlink / mislabeled dir). Refusing to add — remove '$dir' and re-stage."
+        echo "ERROR: release dir '$v' holds verity.json for version '$vv' (stale symlink / mislabeled dir). Refusing to add — remove '$dir' and re-stage." >&2
+        return 1
     fi
 
     local nqn du hu s

serving/spdk/spdk-target.sh

@@ -19,6 +19,12 @@ TADDR="${TARGET_ADDR:-0.0.0.0}"
 TPORT="${TARGET_PORT:-4420}"
 HUGEPAGES="${HUGEPAGES:-1024}"
 DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery"
+# Readiness stamp for the container healthcheck. Written ONLY after the transport
+# is up and releases are reconciled; removed on (re)start + exit. A file check is
+# instant and python-free — the right probe for a box where SPDK busy-polls its
+# (often single) reactor core and a python rpc.py healthcheck would starve/time
+# out. Lives on the shared /run/spdk volume so it's cleared per container start.
+READY_FILE="${READY_FILE:-/var/run/spdk/ready}"
 
 TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt)
 RPC=$(command -v rpc.py   || echo /usr/libexec/spdk/scripts/rpc.py)
@@ -29,6 +35,8 @@ RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release)
 RPC_SOCK="${RPC_SOCK:-}"
 rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; }
 [ -n "$RPC_SOCK" ] && mkdir -p "$(dirname "$RPC_SOCK")"
+mkdir -p "$(dirname "$READY_FILE")"
+rm -f "$READY_FILE"   # clear any stale readiness from a previous container start
 
 # Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced, so
 # a privileged container writes the host value; re-applied after a host reboot
@@ -51,8 +59,8 @@ TGT_ARGS=(-m "$CPUMASK")
 TGT_PID=$!
 # A TERM/INT here stops the target and DROPS EVERY connection — only on container
 # stop, never for a version change. Make the intent loud.
-trap 'echo "==> stopping nvmf_tgt — this drops ALL host root connections" >&2; kill "$TGT_PID" 2>/dev/null || true' TERM INT
-trap 'kill "$TGT_PID" 2>/dev/null || true' EXIT
+trap 'echo "==> stopping nvmf_tgt — this drops ALL host root connections" >&2; rm -f "$READY_FILE"; kill "$TGT_PID" 2>/dev/null || true' TERM INT
+trap 'rm -f "$READY_FILE"; kill "$TGT_PID" 2>/dev/null || true' EXIT
 
 # Wait for the RPC socket.
 for _ in $(seq 1 30); do
@@ -91,6 +99,9 @@ echo "==> reconciling releases under $RELEASES_DIR"
 RELEASES_DIR="$RELEASES_DIR" TARGET_ADDR="$TADDR" TARGET_PORT="$TPORT" RPC_SOCK="$RPC_SOCK" \
     bash "$RELEASE_CTL" reconcile || echo "WARNING: reconcile reported errors" >&2
 
+# Signal readiness for the healthcheck — transport is up and releases reconciled.
+: > "$READY_FILE"
+
 echo "==> nvmf_tgt up; releases served online. Add/remove versions with:"
 echo "      docker compose exec ${NVMEOF_SVC:-nvmeof} spdk-release {add|remove|list} <version>"
 echo "    or host-side: serving/spdk/spdk-ctl.sh {load|unload|list} <version>"