Adding spdk loader

serving/docker-compose.yml

@@ -255,11 +255,14 @@ services:
       #   spdk-release = JSON-RPC add/remove/list/drain of a version, live
       - ./spdk/spdk-target.sh:/usr/local/bin/spdk-target:ro
       - ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
+      # Shared RPC socket so the spdk-loader container (and `compose exec`) can
+      # drive this target's JSON-RPC. nvmf_tgt creates the socket here (-r).
+      - spdk-sock:/var/run/spdk
       # Multi-version release store: each ./spdk/releases/<ver>/ holds that
-      # version's rootfs.erofs + rootfs.hash + verity.json. Staged by
-      # spdk-ctl.sh on the host; reconciled into the target at (re)start and
-      # added/removed live thereafter. RW so the /artifacts bootstrap seed and
-      # spdk-ctl can write here. Adding a version does NOT recreate this service.
+      # version's rootfs.erofs + rootfs.hash + verity.json. Staged by the
+      # spdk-loader (or spdk-ctl.sh on the host); reconciled into the target at
+      # (re)start and added/removed live thereafter. RW so the bootstrap seed +
+      # loader can write here. Adding a version does NOT recreate this service.
       - ./spdk/releases:/releases
       # Bootstrap convenience (optional): a single OCI release mounted as an
       # image volume. spdk-target seeds it into /releases as ${IMAGE_VERSION} if
@@ -276,11 +279,65 @@ services:
     environment:
       IMAGE_VERSION: ${IMAGE_VERSION}    # bootstrap-seed label for /artifacts
       RELEASES_DIR: /releases
+      RPC_SOCK: /var/run/spdk/spdk.sock  # shared with spdk-loader (-r / -s)
       SUBNQN_BASE: ${SUBNQN_BASE}        # per-version NQN base (falls back per release to verity.json .subnqn)
       TARGET_ADDR: ${SERVER_IP}
       TARGET_PORT: ${TARGET_PORT}
       SPDK_CPUMASK: ${SPDK_CPUMASK}
       NVMEOF_SVC: nvmeof
+    # Healthy once the RPC socket answers — gates the spdk-loader's startup.
+    healthcheck:
+      test: ["CMD-SHELL", "rpc.py -s /var/run/spdk/spdk.sock framework_get_config >/dev/null 2>&1 || exit 1"]
+      interval: 5s
+      timeout: 3s
+      retries: 24
+      start_period: 10s
     # Persistent target: started ONCE, never restarted for a version change.
-    # Rolling updates add/remove versions online (serving/spdk/spdk-ctl.sh).
+    # Rolling updates add/remove versions online (spdk-loader, or spdk-ctl.sh).
     entrypoint: ["/bin/bash", "/usr/local/bin/spdk-target"]
+
+  # --- SPDK version loader / reaper (declarative rolling updates) -------------
+  # The "init container that stays up." On `compose up` after an IMAGE_VERSION
+  # bump it stages + adds the new rootfs to the running nvmeof target ONLINE
+  # (no target restart, no dropped connections), then polls and removes old
+  # versions once they drain (all hosts rebooted onto the new UKI). Migration is
+  # just: bump IMAGE_VERSION in .env -> `docker compose up -d`. Reuses the SPDK
+  # image (has python3 + rpc.py); the scripts are bind-mounted like nvmeof's.
+  spdk-loader:
+    image: ${SPDK_IMAGE}:${SPDK_VERSION}
+    network_mode: host
+    restart: unless-stopped
+    depends_on:
+      nvmeof:
+        condition: service_healthy
+    volumes:
+      - ./spdk/spdk-loader.sh:/usr/local/bin/spdk-loader:ro
+      - ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
+      - spdk-sock:/var/run/spdk          # shared RPC socket to the target
+      - ./spdk/releases:/releases        # shared release store
+      # The CURRENT release's artifacts, mounted from its OCI image. The version
+      # tag is the single migration knob — bump IMAGE_VERSION and `up -d`.
+      - type: image
+        source: ${OCI_IMAGE}:${IMAGE_VERSION}
+        target: /in
+        image:
+          subpath: artifacts
+      # Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/in:ro
+    environment:
+      IMAGE_VERSION: ${IMAGE_VERSION}    # the CURRENT version (staged + protected from reaping)
+      RELEASES_DIR: /releases
+      ARTIFACT_IN: /in
+      RPC_SOCK: /var/run/spdk/spdk.sock
+      SUBNQN_BASE: ${SUBNQN_BASE}
+      TARGET_ADDR: ${SERVER_IP}
+      TARGET_PORT: ${TARGET_PORT}
+      REAP_OLD: ${REAP_OLD:-true}              # auto-remove drained old versions
+      REAP_POLL_SECONDS: ${REAP_POLL_SECONDS:-30}
+      REAP_GRACE_SECONDS: ${REAP_GRACE_SECONDS:-300}   # sustained-zero window; keep > host ctrl_loss_tmo
+    entrypoint: ["/bin/bash", "/usr/local/bin/spdk-loader"]
+
+# Named volume for the SPDK JSON-RPC unix socket, shared between the long-lived
+# nvmeof target (which creates it via nvmf_tgt -r) and the spdk-loader (which
+# drives add/remove/drain over it). Keeps the control path off the host FS.
+volumes:
+  spdk-sock: