108 lines
5.7 KiB
Bash
108 lines
5.7 KiB
Bash
#!/usr/bin/env bash
|
|
# DEPRECATED / legacy single-version path. Superseded by the LONG-LIVED target:
|
|
# serving/spdk/spdk-target.sh (persistent nvmf_tgt) + spdk-release.sh (online
|
|
# add/remove of versions) + spdk-ctl.sh (host ops). This script configures ONE
|
|
# version at start, so changing versions means restarting the container — which
|
|
# drops EVERY running host's NVMe/TCP root connection (see docs/TODO.md #6). Kept
|
|
# for reference / one-shot single-version use. The compose stack now runs
|
|
# spdk-target.sh as the nvmeof entrypoint.
|
|
#
|
|
# Configure SPDK nvmf_tgt to serve the diskless root filesystem as TWO READ-ONLY
|
|
# NVMe/TCP namespaces — erofs DATA and the dm-verity HASH tree (design doc §5, §7).
|
|
#
|
|
# Two namespaces (not one with an in-file hash-offset): the target Rocky 9
|
|
# systemd-veritysetup does not support the hash-offset layout, so the host needs
|
|
# the verity data and hash on separate block devices. Integrity is enforced
|
|
# end-to-end by dm-verity on the host; the target serves both read-only.
|
|
set -euo pipefail
|
|
|
|
ROOTFS="/artifacts/rootfs.erofs"
|
|
ROOTHASH="/artifacts/rootfs.hash"
|
|
SUBNQN="${SUBNQN:-nqn.2026-06.org.example.fleet:rootfs}"
|
|
# Fixed, fleet-generic namespace UUIDs. The host references each device as
|
|
# /dev/disk/by-id/nvme-uuid.<uuid> on the kernel cmdline, stable regardless of
|
|
# NVMe controller/namespace enumeration order (a transient discovery controller
|
|
# can shift indices — so a hardcoded /dev/nvmeNnM is unsafe). MUST match
|
|
# ROOT_DATA_UUID / ROOT_HASH_UUID in scripts/extract-artifacts.sh, which bake
|
|
# them into the UKI cmdline. NOTE: for multi-version serving (TODO #6) make
|
|
# these version-specific alongside the per-version NQN, so a host can't connect
|
|
# the wrong rootfs.
|
|
ROOT_DATA_UUID="${ROOT_DATA_UUID:-f1ee7000-0000-4000-8000-000000000001}"
|
|
ROOT_HASH_UUID="${ROOT_HASH_UUID:-f1ee7000-0000-4000-8000-000000000002}"
|
|
TADDR="${TARGET_ADDR:-0.0.0.0}"
|
|
TPORT="${TARGET_PORT:-4420}"
|
|
CPUMASK="${SPDK_CPUMASK:-0x1}"
|
|
|
|
[ -s "$ROOTFS" ] || { echo "ERROR: $ROOTFS not found/empty — is ARTIFACT_SRC populated?" >&2; exit 1; }
|
|
[ -s "$ROOTHASH" ] || { echo "ERROR: $ROOTHASH not found/empty — rebuild artifacts with the two-namespace layout." >&2; exit 1; }
|
|
|
|
# Binary locations differ across SPDK images; fall back to the common path.
|
|
TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt)
|
|
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
|
|
|
|
# Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced,
|
|
# so a privileged container writes the host value; this re-applies after a host
|
|
# reboot (the container restarts). Best-effort: allocation can fail on a
|
|
# fragmented host, in which case set vm.nr_hugepages on the host (persistently
|
|
# via /etc/sysctl.d). Override the count with HUGEPAGES.
|
|
HUGEPAGES="${HUGEPAGES:-1024}"
|
|
if [ -w /proc/sys/vm/nr_hugepages ]; then
|
|
cur=$(cat /proc/sys/vm/nr_hugepages 2>/dev/null || echo 0)
|
|
if [ "${cur:-0}" -lt "$HUGEPAGES" ]; then
|
|
echo "==> allocating $HUGEPAGES hugepages (was ${cur:-0})"
|
|
echo "$HUGEPAGES" > /proc/sys/vm/nr_hugepages 2>/dev/null || true
|
|
fi
|
|
fi
|
|
free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0)
|
|
[ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2
|
|
|
|
echo "==> starting nvmf_tgt (cpumask ${CPUMASK})"
|
|
"$TGT" -m "$CPUMASK" &
|
|
TGT_PID=$!
|
|
trap 'kill "$TGT_PID" 2>/dev/null || true' EXIT
|
|
|
|
# Wait for the RPC socket to come up.
|
|
for _ in $(seq 1 30); do
|
|
"$RPC" framework_get_config >/dev/null 2>&1 && break
|
|
sleep 1
|
|
done
|
|
|
|
echo "==> creating TCP transport"
|
|
"$RPC" nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192
|
|
|
|
echo "==> exposing $ROOTFS (data) + $ROOTHASH (verity hash) as bdevs (4 KiB blocks)"
|
|
# -r opens the backing file read-only where supported; harmless if ignored.
|
|
"$RPC" bdev_aio_create "$ROOTFS" rootfs 4096 -r \
|
|
|| "$RPC" bdev_aio_create "$ROOTFS" rootfs 4096
|
|
"$RPC" bdev_aio_create "$ROOTHASH" roothash 4096 -r \
|
|
|| "$RPC" bdev_aio_create "$ROOTHASH" roothash 4096
|
|
|
|
echo "==> creating subsystem $SUBNQN (allow any host — lab/no-auth)"
|
|
"$RPC" nvmf_create_subsystem "$SUBNQN" -a -s SPDKLAB0000000001 -d FleetRootfs
|
|
|
|
# Read-only is enforced by the bdev_aio '-r' open above (this SPDK's add_ns has
|
|
# no --read-only flag), plus the :ro image-volume mount and the client's erofs
|
|
# ro + dm-verity. -u pins each namespace UUID so the host references them as
|
|
# /dev/disk/by-id/nvme-uuid.<uuid> (stable across controller indices).
|
|
echo "==> adding namespace 1: data (uuid $ROOT_DATA_UUID)"
|
|
"$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs -u "$ROOT_DATA_UUID"
|
|
echo "==> adding namespace 2: verity hash (uuid $ROOT_HASH_UUID)"
|
|
"$RPC" nvmf_subsystem_add_ns "$SUBNQN" roothash -u "$ROOT_HASH_UUID"
|
|
|
|
echo "==> adding TCP listener ${TADDR}:${TPORT} to $SUBNQN"
|
|
"$RPC" nvmf_subsystem_add_listener "$SUBNQN" -t tcp -a "$TADDR" -s "$TPORT"
|
|
|
|
# Also add the listener to the well-known discovery subsystem. Without this,
|
|
# SPDK still answers discovery on this listener but logs a deprecation warning
|
|
# ("listener was not added to the discovery subsystem") and the implicit
|
|
# behaviour is slated for removal. Explicitly registering it is the supported
|
|
# way and future-proofs `nvme discover` against an SPDK bump. Best-effort:
|
|
# tolerate older SPDK builds where the discovery NQN can't take a listener.
|
|
echo "==> adding TCP listener ${TADDR}:${TPORT} to the discovery subsystem"
|
|
"$RPC" nvmf_subsystem_add_listener nqn.2014-08.org.nvmexpress.discovery \
|
|
-t tcp -a "$TADDR" -s "$TPORT" \
|
|
|| echo "WARNING: could not add listener to discovery subsystem (older SPDK?) — discovery still works via the deprecated implicit path." >&2
|
|
|
|
echo "==> SPDK serving $SUBNQN over tcp/${TPORT}; rd.nvmf.discover=tcp,${TADDR},${TPORT}"
|
|
wait "$TGT_PID"
|