sunjizzle/diskless-workstation
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Permanent SPDK trial

Browse Source
This commit is contained in:
Audiobuzz
2026-06-27 19:29:19 +09:30
parent ce5af959ad
commit e43112b0f1
12 changed files with 554 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
serving/.env.example
+7
View File
@@ -74,6 +74,13 @@ NVME_HOST=nvme.fleet.lab
AUDIT_HOST=audit.fleet.lab
# --- NVMe/TCP target -------------------------------------------------------
# Per-version subsystem NQN base. The long-lived target serves each rootfs
# version as its OWN subsystem "<base>-<version>" (so a new build is added live
# without disturbing connected hosts). Per release the exact NQN is read from
# that version's verity.json (.subnqn, written by extract-artifacts.sh); this is
# only the fallback/default base and MUST match SUBNQN_BASE in that script.
SUBNQN_BASE=nqn.2026-06.org.example.fleet:rootfs
# Legacy single-version NQN (only the deprecated serving/spdk-setup.sh path).
SUBNQN=nqn.2026-06.org.example.fleet:rootfs
TARGET_PORT=4420
SPDK_CPUMASK=0x1 # one polling core is plenty for a read-only boot fs
serving/docker-compose.yml
+25 -4
View File
@@ -250,16 +250,37 @@ services:
memlock: -1
volumes:
- /dev/hugepages:/dev/hugepages
# LONG-LIVED target + ONLINE release control (no restart on a new build):
# spdk-target = the persistent entrypoint (nvmf_tgt stays up for life)
# spdk-release = JSON-RPC add/remove/list/drain of a version, live
- ./spdk/spdk-target.sh:/usr/local/bin/spdk-target:ro
- ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
# Multi-version release store: each ./spdk/releases/<ver>/ holds that
# version's rootfs.erofs + rootfs.hash + verity.json. Staged by
# spdk-ctl.sh on the host; reconciled into the target at (re)start and
# added/removed live thereafter. RW so the /artifacts bootstrap seed and
# spdk-ctl can write here. Adding a version does NOT recreate this service.
- ./spdk/releases:/releases
# Bootstrap convenience (optional): a single OCI release mounted as an
# image volume. spdk-target seeds it into /releases as ${IMAGE_VERSION} if
# not already staged — preserves the simple one-version `compose up` flow.
# Remove this block for a pure multi-version (spdk-ctl-managed) deployment.
- type: image
source: ${OCI_IMAGE}:${IMAGE_VERSION}
target: /artifacts # -> /artifacts/rootfs.erofs for spdk-setup.sh
target: /artifacts
image:
subpath: artifacts
# Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/artifacts:ro
- ./spdk-setup.sh:/spdk-setup.sh:ro
# Legacy single-version entrypoint (configure-at-start, restart-to-change):
# - ./spdk-setup.sh:/spdk-setup.sh:ro
environment:
SUBNQN: ${SUBNQN}
IMAGE_VERSION: ${IMAGE_VERSION} # bootstrap-seed label for /artifacts
RELEASES_DIR: /releases
SUBNQN_BASE: ${SUBNQN_BASE} # per-version NQN base (falls back per release to verity.json .subnqn)
TARGET_ADDR: ${SERVER_IP}
TARGET_PORT: ${TARGET_PORT}
SPDK_CPUMASK: ${SPDK_CPUMASK}
entrypoint: ["/bin/bash", "/spdk-setup.sh"]
NVMEOF_SVC: nvmeof
# Persistent target: started ONCE, never restarted for a version change.
# Rolling updates add/remove versions online (serving/spdk/spdk-ctl.sh).
entrypoint: ["/bin/bash", "/usr/local/bin/spdk-target"]
serving/spdk-setup.sh
+8
View File
@@ -1,4 +1,12 @@
#!/usr/bin/env bash
# DEPRECATED / legacy single-version path. Superseded by the LONG-LIVED target:
# serving/spdk/spdk-target.sh (persistent nvmf_tgt) + spdk-release.sh (online
# add/remove of versions) + spdk-ctl.sh (host ops). This script configures ONE
# version at start, so changing versions means restarting the container — which
# drops EVERY running host's NVMe/TCP root connection (see docs/TODO.md #6). Kept
# for reference / one-shot single-version use. The compose stack now runs
# spdk-target.sh as the nvmeof entrypoint.
#
# Configure SPDK nvmf_tgt to serve the diskless root filesystem as TWO READ-ONLY
# NVMe/TCP namespaces — erofs DATA and the dm-verity HASH tree (design doc §5, §7).
#
serving/spdk/README.md
+86
View File
@@ -0,0 +1,86 @@
# SPDK NVMe/TCP target — long-lived, multi-version
This is the fleet's **root disk**: a userspace SPDK `nvmf_tgt` exporting each
rootfs release as **two read-only NVMe/TCP namespaces** (erofs DATA + dm-verity
HASH — the distro's systemd-veritysetup can't do the in-file hash-offset layout,
so they're separate namespaces with distinct UUIDs).
## Why the target is long-lived
`nvmf_tgt` sits in the **live data path of every running host's root fs**.
Restarting the process tears down **every** NVMe/TCP association at once
(fleet-wide, not version-scoped) — a host whose root link drops past
`ctrl_loss_tmo` gets root I/O errors and must reboot. So **the process must never
restart for a routine change.**
A host changes root version **only by rebooting** (no live swap). Therefore:
**version transitions are reboots; SPDK changes are additive/online; the target
process stays up.** Routine rollouts never restart SPDK.
## How it works
| Piece | Role |
|---|---|
| [`spdk-target.sh`](spdk-target.sh) | **Persistent entrypoint.** Starts `nvmf_tgt` once, creates the TCP transport + discovery listener, then reconciles every staged release. Never restarted to change versions. On (re)start it re-adds everything under `./releases/`, so a serving-host reboot restores all served versions. |
| [`spdk-release.sh`](spdk-release.sh) | **In-container control** (JSON-RPC). `add`/`remove`/`list`/`drain-status`/`reconcile` a version **live**. One **subsystem per version** (distinct NQN) so adding/removing one never disturbs the subsystem other hosts are attached to. Removal is **drain-gated**. |
| [`spdk-ctl.sh`](spdk-ctl.sh) | **Host-side ops.** Stages a release's artifacts from its OCI image into `./releases/<ver>/` and drives the live target via `docker compose exec`. |
| `./releases/<version>/` | Per-version `rootfs.erofs` + `rootfs.hash` + `verity.json`. `verity.json` is the **single source of truth** for that version's namespace UUIDs + subsystem NQN (written by `scripts/extract-artifacts.sh`), so the host's by-id verity refs and the target's namespace UUIDs always agree. |
Per-version namespace UUIDs are required: two versions served at once must have
globally-distinct namespace UUIDs or they collide on the same
`/dev/disk/by-id/nvme-uuid.<uuid>` symlink on the host. `extract-artifacts.sh`
derives them deterministically (UUIDv5 of the version) and bakes them into both
the immutable UKI cmdline and `verity.json`.
## Rolling update (add new, drain old, clean up)
```bash
cd serving/spdk
./spdk-ctl.sh load 1.3.0 # serve 1.3.0 ALONGSIDE the running 1.2.0 (live, no restart)
# ... bump the deployment manifest digest; reboot hosts → they come up on 1.3.0 ...
./spdk-ctl.sh list # watch 1.2.0 CONNS drop toward 0
# VERSION CONNS NQN
# rootfs-1.3.0 7 nqn.2026-06.org.example.fleet:rootfs-1.3.0
# rootfs-1.2.0 0 nqn.2026-06.org.example.fleet:rootfs-1.2.0
./spdk-ctl.sh unload 1.2.0 # drain-gated: refuses while CONNS > 0
```
`unload` **refuses to remove a version that still has connected controllers**
(an assigned-but-offline host could still return to it). Drain those hosts
first, or `./spdk-ctl.sh unload 1.2.0 --force` to **evict** them (they lose root
I/O — only for a known-dead version).
In-container equivalents (what `spdk-ctl.sh` calls):
```bash
docker compose exec nvmeof spdk-release list
docker compose exec nvmeof spdk-release add 1.3.0
docker compose exec nvmeof spdk-release drain-status 1.2.0
docker compose exec nvmeof spdk-release remove 1.2.0 [--force]
```
## Capacity & availability
- Each concurrently-served version costs target memory/hugepages + a file-backed
bdev. Plan for N live versions (current + previous + canary) during long
rollouts; raise `HUGEPAGES` accordingly.
- The target is still a **single-node SPOF**. Even with online add/remove, a node
failure or a planned `nvmf_tgt` restart drops every connection. The structural
fix is **HA + NVMe-oF multipath (ANA)**: ≥2 targets serving byte-identical,
same-UUID, read-only namespaces (replication is coherent precisely because the
root is read-only + content-addressed). Cheap host-side mitigation: generous
`ctrl_loss_tmo`/`reconnect_delay`. See `docs/TODO.md` #6/#7.
- There is **no zero-impact single-node restart** — only drain/failover.
## Bootstrap (simple single-version)
`docker compose up -d` still works with one version: the OCI image volume at
`/artifacts` is **seeded** into `./releases/${IMAGE_VERSION}/` and served
automatically. Use `spdk-ctl.sh load` to add more versions on top.
## Legacy
[`../spdk-setup.sh`](../spdk-setup.sh) is the original **single-version,
configure-at-start** entrypoint (changing versions = container restart = all
connections dropped). Superseded by the long-lived target; kept for reference.
serving/spdk/releases/.gitignore
+5
View File
@@ -0,0 +1,5 @@
# Staged rootfs releases for the long-lived SPDK target (spdk-ctl.sh load/unload).
# Each <version>/ holds large, content-addressed artifacts pulled from the OCI
# release image — never commit them. Keep the directory itself.
*
!.gitignore
serving/spdk/spdk-ctl.sh
Executable
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env bash
# spdk-ctl.sh — host-side fleet release ops for the LONG-LIVED SPDK target.
#
# Stages a release's artifacts into ./releases/<version>/ and adds/removes it on
# the running nvmf_tgt ONLINE — no container restart, no dropped connections.
# This is the operator entrypoint for rolling updates and cleanup; it wraps the
# in-container spdk-release control script (driven via `docker compose exec`).
#
# ./spdk-ctl.sh load <version> [oci_ref] # stage + add live (oci_ref default: $OCI_IMAGE:<version>)
# ./spdk-ctl.sh unload <version> [--force] # drain-gated remove + delete staged files
# ./spdk-ctl.sh list # served versions + connected hosts
#
# Typical rollout:
# ./spdk-ctl.sh load 1.3.0 # serve the new version alongside the old
# # ... reboot hosts onto 1.3.0 (deployment manifest digest bump) ...
# ./spdk-ctl.sh list # watch 1.2.0 CONNS fall to 0
# ./spdk-ctl.sh unload 1.2.0 # cleanup once drained (refuses if hosts remain)
set -euo pipefail
cd "$(dirname "$0")"
RELEASES_DIR="${RELEASES_DIR:-./releases}"
SVC="${NVMEOF_SVC:-nvmeof}"
# Pull OCI_IMAGE default from the stack .env if present.
if [ -f ../.env ]; then
set -a; . ../.env; set +a
fi
# Run compose from the stack dir (one level up, where docker-compose.yml lives).
dc() { ( cd .. && docker compose "$@" ); }
usage() {
echo "usage: $0 {load <ver> [oci_ref] | unload <ver> [--force] | list}" >&2
exit 2
}
cmd_load() {
local v=${1:-}; [ -n "$v" ] || usage
local ref=${2:-}
if [ -z "$ref" ]; then
[ -n "${OCI_IMAGE:-}" ] || { echo "ERROR: set OCI_IMAGE in ../.env or pass an oci_ref" >&2; exit 1; }
ref="${OCI_IMAGE}:${v}"
fi
local dest="$RELEASES_DIR/$v"
if [ -s "$dest/verity.json" ]; then
echo "==> release $v already staged in $dest (skipping extract)"
else
echo "==> extracting $ref -> $dest"
mkdir -p "$dest"
local cid
cid=$(docker create "$ref")
# FROM-scratch artifact image: copy the three files the target needs.
docker cp "$cid:/artifacts/rootfs.erofs" "$dest/rootfs.erofs"
docker cp "$cid:/artifacts/rootfs.hash" "$dest/rootfs.hash"
docker cp "$cid:/artifacts/verity.json" "$dest/verity.json"
docker rm "$cid" >/dev/null
fi
echo "==> adding release $v on the live target (no restart)"
dc exec -T "$SVC" spdk-release add "$v"
dc exec -T "$SVC" spdk-release list
}
cmd_unload() {
local v="" force=""
local a
for a in "$@"; do
case "$a" in
--force) force="--force" ;;
*) v="$a" ;;
esac
done
[ -n "$v" ] || usage
echo "==> removing release $v from the live target"
if dc exec -T "$SVC" spdk-release remove "$v" ${force:+$force}; then
rm -rf "${RELEASES_DIR:?}/$v"
echo "==> staged artifacts for $v deleted from $RELEASES_DIR/$v"
else
echo "remove refused (version still has connected hosts?) — staged files left in place." >&2
echo "Drain those hosts first, or re-run with --force to evict them." >&2
exit 1
fi
}
case "${1:-}" in
load) shift; cmd_load "$@" ;;
unload|rm) shift; cmd_unload "$@" ;;
list|ls) dc exec -T "$SVC" spdk-release list ;;
*) usage ;;
esac
serving/spdk/spdk-release.sh
Executable
+172
View File
@@ -0,0 +1,172 @@
#!/usr/bin/env bash
# spdk-release.sh — add/remove a rootfs RELEASE on the long-lived nvmf_tgt
# ONLINE, via JSON-RPC, WITHOUT restarting the target. The target process holds
# every running host's NVMe/TCP root connection, so restarting it tears down the
# whole fleet's root at once (see docs/TODO.md #6). Routine rollouts therefore
# never restart SPDK: this script adds the new version's bdevs + subsystem +
# namespaces live (existing connections untouched) and removes an old version
# live once it has drained.
#
# ONE SUBSYSTEM PER VERSION (distinct NQN): adding a release as a NEW subsystem
# never disturbs the subsystem running hosts are attached to (adding a namespace
# to a SHARED subsystem would fire an AER + rescan at every host). It also gives
# clean per-release drain + per-release hostNQN allow-listing.
#
# Each release lives under $RELEASES_DIR/<version>/ as rootfs.erofs + rootfs.hash
# + verity.json. verity.json is the single source of truth for that version's
# namespace UUIDs + subsystem NQN (written by scripts/extract-artifacts.sh) — so
# the host's by-id verity refs and the target's namespace UUIDs always agree.
#
# Usage:
# spdk-release add <version> # idempotent; live, no restart
# spdk-release remove <version> [--force] # DRAIN-GATED unless --force
# spdk-release list # served versions + connected hosts
# spdk-release drain-status <version> # connected controller count
# spdk-release reconcile # add every release under $RELEASES_DIR
set -euo pipefail
RELEASES_DIR="${RELEASES_DIR:-/releases}"
TADDR="${TARGET_ADDR:-0.0.0.0}"
TPORT="${TARGET_PORT:-4420}"
SUBNQN_BASE="${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs}"
DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery"
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
die() { echo "ERROR: $*" >&2; exit 1; }
rpc() { "$RPC" "$@"; }
# bdev names must be filesystem/identifier-safe; sanitise the version.
san() { printf '%s' "$1" | tr -c 'A-Za-z0-9' '_'; }
# Read a field from a release's verity.json (empty if missing).
vfield() {
local v=$1 path=$2
jq -r "$path // empty" "$RELEASES_DIR/$v/verity.json" 2>/dev/null || true
}
# Per-version subsystem NQN: prefer the one baked into verity.json; otherwise
# derive it the same way extract-artifacts.sh does (base-<version>).
release_nqn() {
local v=$1 nqn
nqn=$(vfield "$v" .subnqn)
[ -n "$nqn" ] || nqn="${SUBNQN_BASE}-${v}"
printf '%s' "$nqn"
}
subsys_exists() {
rpc nvmf_get_subsystems 2>/dev/null \
| jq -e --arg n "$1" 'any(.[]; .nqn==$n)' >/dev/null 2>&1
}
# Connected controllers on a subsystem (the drain gate). 0 if absent/none.
ctrl_count() {
local n
n=$(rpc nvmf_subsystem_get_controllers "$1" 2>/dev/null | jq 'length' 2>/dev/null || echo 0)
printf '%s' "${n:-0}"
}
cmd_add() {
local v=${1:?usage: add <version>}
local dir="$RELEASES_DIR/$v"
[ -s "$dir/rootfs.erofs" ] || die "$dir/rootfs.erofs missing/empty"
[ -s "$dir/rootfs.hash" ] || die "$dir/rootfs.hash missing/empty"
local nqn du hu s
nqn=$(release_nqn "$v")
du=$(vfield "$v" .data_ns_uuid)
hu=$(vfield "$v" .hash_ns_uuid)
[ -n "$du" ] && [ -n "$hu" ] \
|| die "$dir/verity.json lacks data_ns_uuid/hash_ns_uuid (rebuild with the per-version UUID extract-artifacts.sh)"
if subsys_exists "$nqn"; then
echo "release $v already served ($nqn) — nothing to do"
return 0
fi
s=$(san "$v")
echo "==> adding release $v as subsystem $nqn (data uuid $du, hash uuid $hu)"
# -r opens the backing file read-only where the SPDK build supports it.
rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096 -r \
|| rpc bdev_aio_create "$dir/rootfs.erofs" "rootfs_$s" 4096
rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096 -r \
|| rpc bdev_aio_create "$dir/rootfs.hash" "roothash_$s" 4096
# Serial must be <=20 chars; derive a stable, version-tagged one.
rpc nvmf_create_subsystem "$nqn" -a -s "SPDK$(printf '%s' "$s" | cut -c1-16)" -d "FleetRootfs"
rpc nvmf_subsystem_add_ns "$nqn" "rootfs_$s" -u "$du"
rpc nvmf_subsystem_add_ns "$nqn" "roothash_$s" -u "$hu"
rpc nvmf_subsystem_add_listener "$nqn" -t tcp -a "$TADDR" -s "$TPORT"
# Register the listener on the discovery subsystem too (best-effort; older
# SPDK answers discovery implicitly but logs a deprecation warning).
rpc nvmf_subsystem_add_listener "$DISCOVERY_NQN" -t tcp -a "$TADDR" -s "$TPORT" 2>/dev/null || true
echo "==> release $v ONLINE (existing connections untouched)"
}
cmd_remove() {
local force=0 v=""
local a
for a in "$@"; do
case "$a" in
--force) force=1 ;;
*) v="$a" ;;
esac
done
[ -n "$v" ] || die "usage: remove <version> [--force]"
local nqn n
nqn=$(release_nqn "$v")
if ! subsys_exists "$nqn"; then
echo "release $v not served ($nqn) — nothing to remove"
return 0
fi
n=$(ctrl_count "$nqn")
if [ "$n" -gt 0 ] && [ "$force" -ne 1 ]; then
die "release $v still has $n connected controller(s) — refusing to remove. Drain first (reboot/migrate those hosts), or pass --force to EVICT them (they lose root I/O)."
fi
[ "$n" -gt 0 ] && echo "WARNING: --force removing $v with $n connected controller(s) — those hosts lose root I/O"
local s
s=$(san "$v")
echo "==> removing release $v ($nqn)"
rpc nvmf_delete_subsystem "$nqn" || true
rpc bdev_aio_delete "rootfs_$s" 2>/dev/null || true
rpc bdev_aio_delete "roothash_$s" 2>/dev/null || true
echo "==> release $v removed (other versions + their connections untouched)"
}
cmd_list() {
local nqn n ver
printf '%-26s %-7s %s\n' VERSION CONNS NQN
rpc nvmf_get_subsystems 2>/dev/null \
| jq -r '.[] | select(.nqn | test("discovery") | not) | .nqn' \
| while read -r nqn; do
n=$(ctrl_count "$nqn")
ver="${nqn##*:}" # strip the nqn.*: prefix for readability
printf '%-26s %-7s %s\n' "$ver" "$n" "$nqn"
done
}
cmd_drain_status() {
local v=${1:?usage: drain-status <version>}
local nqn
nqn=$(release_nqn "$v")
echo "$(ctrl_count "$nqn") connected controller(s) on $nqn"
}
cmd_reconcile() {
[ -d "$RELEASES_DIR" ] || { echo "no $RELEASES_DIR — nothing to reconcile"; return 0; }
local d any=0
for d in "$RELEASES_DIR"/*/; do
[ -e "${d}verity.json" ] || continue
any=1
cmd_add "$(basename "$d")" || echo "WARNING: failed to add release $(basename "$d")" >&2
done
[ "$any" -eq 1 ] || echo "no releases found under $RELEASES_DIR"
}
case "${1:-}" in
add) shift; cmd_add "$@" ;;
remove|rm) shift; cmd_remove "$@" ;;
list|ls) shift; cmd_list "$@" ;;
drain-status|status) shift; cmd_drain_status "$@" ;;
reconcile) shift; cmd_reconcile "$@" ;;
*) echo "usage: spdk-release {add <ver>|remove <ver> [--force]|list|drain-status <ver>|reconcile}" >&2; exit 2 ;;
esac
serving/spdk/spdk-target.sh
Executable
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env bash
# spdk-target.sh — LONG-LIVED nvmf_tgt entrypoint.
#
# Starts the NVMe-oF target ONCE and keeps it up for the container's entire life.
# It is RELEASE-AGNOSTIC: rootfs versions are added/removed ONLINE via
# spdk-release.sh (JSON-RPC) without ever restarting this process, so a new build
# never drops running hosts' NVMe/TCP root connections (docs/TODO.md #6). This is
# the key difference from the legacy serving/spdk-setup.sh, which configured a
# single version at start and thus required a connection-dropping restart to
# change versions.
#
# On (re)start it reconciles: every release staged under $RELEASES_DIR is added,
# so a serving-host reboot restores all currently-served versions automatically.
set -euo pipefail
RELEASES_DIR="${RELEASES_DIR:-/releases}"
CPUMASK="${SPDK_CPUMASK:-0x1}"
TADDR="${TARGET_ADDR:-0.0.0.0}"
TPORT="${TARGET_PORT:-4420}"
HUGEPAGES="${HUGEPAGES:-1024}"
DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery"
TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt)
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release)
# Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced, so
# a privileged container writes the host value; re-applied after a host reboot
# (the container restarts). Best-effort: allocation can fail on a fragmented
# host, in which case set vm.nr_hugepages on the host via /etc/sysctl.d.
if [ -w /proc/sys/vm/nr_hugepages ]; then
cur=$(cat /proc/sys/vm/nr_hugepages 2>/dev/null || echo 0)
if [ "${cur:-0}" -lt "$HUGEPAGES" ]; then
echo "==> allocating $HUGEPAGES hugepages (was ${cur:-0})"
echo "$HUGEPAGES" > /proc/sys/vm/nr_hugepages 2>/dev/null || true
fi
fi
free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0)
[ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2
echo "==> starting LONG-LIVED nvmf_tgt (cpumask ${CPUMASK})"
"$TGT" -m "$CPUMASK" &
TGT_PID=$!
# A TERM/INT here stops the target and DROPS EVERY connection — only on container
# stop, never for a version change. Make the intent loud.
trap 'echo "==> stopping nvmf_tgt — this drops ALL host root connections" >&2; kill "$TGT_PID" 2>/dev/null || true' TERM INT
trap 'kill "$TGT_PID" 2>/dev/null || true' EXIT
# Wait for the RPC socket.
for _ in $(seq 1 30); do
"$RPC" framework_get_config >/dev/null 2>&1 && break
sleep 1
done
echo "==> creating TCP transport"
"$RPC" nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192 \
|| echo "WARNING: nvmf_create_transport failed (already present?)" >&2
echo "==> registering discovery listener ${TADDR}:${TPORT}"
"$RPC" nvmf_subsystem_add_listener "$DISCOVERY_NQN" -t tcp -a "$TADDR" -s "$TPORT" 2>/dev/null || true
# Bootstrap convenience: if a single OCI artifact image is mounted at /artifacts
# (the simple one-version `docker compose up` flow) and that version isn't yet
# staged under $RELEASES_DIR, stage it so the long-lived target serves it too.
# Symlinks resolve inside the container (both are container paths); fall back to
# a copy if linking fails.
if [ -s /artifacts/verity.json ] && [ -n "${IMAGE_VERSION:-}" ]; then
seed="$RELEASES_DIR/$IMAGE_VERSION"
if [ ! -e "$seed/verity.json" ]; then
echo "==> seeding bootstrap release $IMAGE_VERSION from /artifacts"
mkdir -p "$seed"
for f in rootfs.erofs rootfs.hash verity.json; do
ln -sf "/artifacts/$f" "$seed/$f" 2>/dev/null \
|| cp "/artifacts/$f" "$seed/$f" 2>/dev/null \
|| echo "WARNING: could not seed /artifacts/$f" >&2
done
fi
fi
echo "==> reconciling releases under $RELEASES_DIR"
RELEASES_DIR="$RELEASES_DIR" TARGET_ADDR="$TADDR" TARGET_PORT="$TPORT" \
bash "$RELEASE_CTL" reconcile || echo "WARNING: reconcile reported errors" >&2
echo "==> nvmf_tgt up; releases served online. Add/remove versions with:"
echo " docker compose exec ${NVMEOF_SVC:-nvmeof} spdk-release {add|remove|list} <version>"
echo " or host-side: serving/spdk/spdk-ctl.sh {load|unload|list} <version>"
wait "$TGT_PID"