109 lines
5.6 KiB
Bash
Executable File
109 lines
5.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# spdk-target.sh — LONG-LIVED nvmf_tgt entrypoint.
|
|
#
|
|
# Starts the NVMe-oF target ONCE and keeps it up for the container's entire life.
|
|
# It is RELEASE-AGNOSTIC: rootfs versions are added/removed ONLINE via
|
|
# spdk-release.sh (JSON-RPC) without ever restarting this process, so a new build
|
|
# never drops running hosts' NVMe/TCP root connections (docs/TODO.md #6). This is
|
|
# the key difference from the legacy serving/spdk-setup.sh, which configured a
|
|
# single version at start and thus required a connection-dropping restart to
|
|
# change versions.
|
|
#
|
|
# On (re)start it reconciles: every release staged under $RELEASES_DIR is added,
|
|
# so a serving-host reboot restores all currently-served versions automatically.
|
|
set -euo pipefail
|
|
|
|
RELEASES_DIR="${RELEASES_DIR:-/releases}"
|
|
CPUMASK="${SPDK_CPUMASK:-0x1}"
|
|
TADDR="${TARGET_ADDR:-0.0.0.0}"
|
|
TPORT="${TARGET_PORT:-4420}"
|
|
HUGEPAGES="${HUGEPAGES:-1024}"
|
|
DISCOVERY_NQN="nqn.2014-08.org.nvmexpress.discovery"
|
|
# Readiness stamp for the container healthcheck. Written ONLY after the transport
|
|
# is up and releases are reconciled; removed on (re)start + exit. A file check is
|
|
# instant and python-free — the right probe for a box where SPDK busy-polls its
|
|
# (often single) reactor core and a python rpc.py healthcheck would starve/time
|
|
# out. Lives on the shared /run/spdk volume so it's cleared per container start.
|
|
READY_FILE="${READY_FILE:-/var/run/spdk/ready}"
|
|
|
|
TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt)
|
|
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
|
|
RELEASE_CTL=$(command -v spdk-release || echo /usr/local/bin/spdk-release)
|
|
# Optional explicit RPC socket. Set it (compose does, on a shared volume) so a
|
|
# SEPARATE loader container can reach this target's JSON-RPC and add/remove
|
|
# versions online. Empty -> nvmf_tgt's default socket (single-container case).
|
|
RPC_SOCK="${RPC_SOCK:-}"
|
|
rpc() { if [ -n "$RPC_SOCK" ]; then "$RPC" -s "$RPC_SOCK" "$@"; else "$RPC" "$@"; fi; }
|
|
[ -n "$RPC_SOCK" ] && mkdir -p "$(dirname "$RPC_SOCK")"
|
|
mkdir -p "$(dirname "$READY_FILE")"
|
|
rm -f "$READY_FILE" # clear any stale readiness from a previous container start
|
|
|
|
# Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced, so
|
|
# a privileged container writes the host value; re-applied after a host reboot
|
|
# (the container restarts). Best-effort: allocation can fail on a fragmented
|
|
# host, in which case set vm.nr_hugepages on the host via /etc/sysctl.d.
|
|
if [ -w /proc/sys/vm/nr_hugepages ]; then
|
|
cur=$(cat /proc/sys/vm/nr_hugepages 2>/dev/null || echo 0)
|
|
if [ "${cur:-0}" -lt "$HUGEPAGES" ]; then
|
|
echo "==> allocating $HUGEPAGES hugepages (was ${cur:-0})"
|
|
echo "$HUGEPAGES" > /proc/sys/vm/nr_hugepages 2>/dev/null || true
|
|
fi
|
|
fi
|
|
free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0)
|
|
[ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2
|
|
|
|
echo "==> starting LONG-LIVED nvmf_tgt (cpumask ${CPUMASK})${RPC_SOCK:+ rpc-sock $RPC_SOCK}"
|
|
TGT_ARGS=(-m "$CPUMASK")
|
|
[ -n "$RPC_SOCK" ] && TGT_ARGS+=(-r "$RPC_SOCK")
|
|
"$TGT" "${TGT_ARGS[@]}" &
|
|
TGT_PID=$!
|
|
# A TERM/INT here stops the target and DROPS EVERY connection — only on container
|
|
# stop, never for a version change. Make the intent loud.
|
|
trap 'echo "==> stopping nvmf_tgt — this drops ALL host root connections" >&2; rm -f "$READY_FILE"; kill "$TGT_PID" 2>/dev/null || true' TERM INT
|
|
trap 'rm -f "$READY_FILE"; kill "$TGT_PID" 2>/dev/null || true' EXIT
|
|
|
|
# Wait for the RPC socket.
|
|
for _ in $(seq 1 30); do
|
|
rpc framework_get_config >/dev/null 2>&1 && break
|
|
sleep 1
|
|
done
|
|
|
|
echo "==> creating TCP transport"
|
|
rpc nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192 \
|
|
|| echo "WARNING: nvmf_create_transport failed (already present?)" >&2
|
|
|
|
echo "==> registering discovery listener ${TADDR}:${TPORT}"
|
|
rpc nvmf_subsystem_add_listener "$DISCOVERY_NQN" -t tcp -a "$TADDR" -s "$TPORT" >/dev/null 2>&1 || true
|
|
|
|
# Bootstrap convenience: if a single OCI artifact image is mounted at /artifacts
|
|
# (the simple one-version `docker compose up` flow) and that version isn't yet
|
|
# staged under $RELEASES_DIR, stage it so the long-lived target serves it too.
|
|
#
|
|
# COPY, never symlink: /artifacts is a version-pinned image volume that CHANGES
|
|
# when IMAGE_VERSION changes, so a symlink into it would silently re-point an
|
|
# already-staged release dir at the new version's data on the next bump. Each
|
|
# release dir must hold its own immutable copy. Self-heal: re-copy if the staged
|
|
# verity.json differs from the mounted artifact (repairs a dir left stale/symlinked
|
|
# by an older build).
|
|
if [ -s /artifacts/verity.json ] && [ -n "${IMAGE_VERSION:-}" ]; then
|
|
seed="$RELEASES_DIR/$IMAGE_VERSION"
|
|
if [ ! -s "$seed/verity.json" ] || ! cmp -s /artifacts/verity.json "$seed/verity.json"; then
|
|
echo "==> seeding bootstrap release $IMAGE_VERSION from /artifacts (copy)"
|
|
mkdir -p "$seed"
|
|
cp -f /artifacts/rootfs.erofs /artifacts/rootfs.hash /artifacts/verity.json "$seed/" \
|
|
|| echo "WARNING: failed to seed $IMAGE_VERSION from /artifacts" >&2
|
|
fi
|
|
fi
|
|
|
|
echo "==> reconciling releases under $RELEASES_DIR"
|
|
RELEASES_DIR="$RELEASES_DIR" TARGET_ADDR="$TADDR" TARGET_PORT="$TPORT" RPC_SOCK="$RPC_SOCK" \
|
|
bash "$RELEASE_CTL" reconcile || echo "WARNING: reconcile reported errors" >&2
|
|
|
|
# Signal readiness for the healthcheck — transport is up and releases reconciled.
|
|
: > "$READY_FILE"
|
|
|
|
echo "==> nvmf_tgt up; releases served online. Add/remove versions with:"
|
|
echo " docker compose exec ${NVMEOF_SVC:-nvmeof} spdk-release {add|remove|list} <version>"
|
|
echo " or host-side: serving/spdk/spdk-ctl.sh {load|unload|list} <version>"
|
|
wait "$TGT_PID"
|