94 lines
4.7 KiB
Bash
94 lines
4.7 KiB
Bash
#!/usr/bin/env bash
|
|
# Configure SPDK nvmf_tgt to serve the diskless root filesystem as a single
|
|
# READ-ONLY NVMe/TCP namespace (design doc §5, §7).
|
|
#
|
|
# rootfs.erofs is the erofs image with the dm-verity hash tree appended in-file;
|
|
# we expose the WHOLE file as one namespace, and the host splits data/hash with
|
|
# systemd.verity_root_options=hash-offset= from the kernel cmdline. Integrity is
|
|
# enforced end-to-end by dm-verity on the host; the target serves it read-only.
|
|
set -euo pipefail
|
|
|
|
ROOTFS="/artifacts/rootfs.erofs"
|
|
SUBNQN="${SUBNQN:-nqn.2026-06.org.example.fleet:rootfs}"
|
|
# Fixed, fleet-generic namespace UUID. The host references the root device as
|
|
# /dev/disk/by-id/nvme-uuid.$ROOT_NS_UUID on the kernel cmdline, which is stable
|
|
# regardless of NVMe controller enumeration order (the I/O controller can land
|
|
# on nvme0/nvme1/... depending on whether a transient discovery controller took
|
|
# a slot first — so a hardcoded /dev/nvme0n1 is unsafe). MUST match
|
|
# ROOT_NS_UUID in scripts/extract-artifacts.sh, which bakes it into the UKI
|
|
# cmdline. NOTE: for multi-version serving (TODO #6) make this version-specific
|
|
# alongside the per-version NQN, so a host can't connect the wrong rootfs.
|
|
ROOT_NS_UUID="${ROOT_NS_UUID:-f1ee7000-0000-4000-8000-000000000001}"
|
|
TADDR="${TARGET_ADDR:-0.0.0.0}"
|
|
TPORT="${TARGET_PORT:-4420}"
|
|
CPUMASK="${SPDK_CPUMASK:-0x1}"
|
|
|
|
[ -s "$ROOTFS" ] || { echo "ERROR: $ROOTFS not found/empty — is ARTIFACT_SRC populated?" >&2; exit 1; }
|
|
|
|
# Binary locations differ across SPDK images; fall back to the common path.
|
|
TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt)
|
|
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
|
|
|
|
# Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced,
|
|
# so a privileged container writes the host value; this re-applies after a host
|
|
# reboot (the container restarts). Best-effort: allocation can fail on a
|
|
# fragmented host, in which case set vm.nr_hugepages on the host (persistently
|
|
# via /etc/sysctl.d). Override the count with HUGEPAGES.
|
|
HUGEPAGES="${HUGEPAGES:-1024}"
|
|
if [ -w /proc/sys/vm/nr_hugepages ]; then
|
|
cur=$(cat /proc/sys/vm/nr_hugepages 2>/dev/null || echo 0)
|
|
if [ "${cur:-0}" -lt "$HUGEPAGES" ]; then
|
|
echo "==> allocating $HUGEPAGES hugepages (was ${cur:-0})"
|
|
echo "$HUGEPAGES" > /proc/sys/vm/nr_hugepages 2>/dev/null || true
|
|
fi
|
|
fi
|
|
free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0)
|
|
[ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2
|
|
|
|
echo "==> starting nvmf_tgt (cpumask ${CPUMASK})"
|
|
"$TGT" -m "$CPUMASK" &
|
|
TGT_PID=$!
|
|
trap 'kill "$TGT_PID" 2>/dev/null || true' EXIT
|
|
|
|
# Wait for the RPC socket to come up.
|
|
for _ in $(seq 1 30); do
|
|
"$RPC" framework_get_config >/dev/null 2>&1 && break
|
|
sleep 1
|
|
done
|
|
|
|
echo "==> creating TCP transport"
|
|
"$RPC" nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192
|
|
|
|
echo "==> exposing $ROOTFS as bdev 'rootfs' (4 KiB blocks)"
|
|
# -r opens the backing file read-only where supported; harmless if ignored.
|
|
"$RPC" bdev_aio_create "$ROOTFS" rootfs 4096 -r \
|
|
|| "$RPC" bdev_aio_create "$ROOTFS" rootfs 4096
|
|
|
|
echo "==> creating subsystem $SUBNQN (allow any host — lab/no-auth)"
|
|
"$RPC" nvmf_create_subsystem "$SUBNQN" -a -s SPDKLAB0000000001 -d FleetRootfs
|
|
|
|
echo "==> adding namespace 1 (uuid $ROOT_NS_UUID)"
|
|
# Read-only is enforced by the bdev_aio '-r' open above (this SPDK's add_ns has
|
|
# no --read-only flag), plus the :ro image-volume mount and the client's erofs
|
|
# ro + dm-verity.
|
|
# -u pins the namespace UUID so the host can reference it as
|
|
# /dev/disk/by-id/nvme-uuid.$ROOT_NS_UUID (stable across controller indices).
|
|
"$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs -u "$ROOT_NS_UUID"
|
|
|
|
echo "==> adding TCP listener ${TADDR}:${TPORT} to $SUBNQN"
|
|
"$RPC" nvmf_subsystem_add_listener "$SUBNQN" -t tcp -a "$TADDR" -s "$TPORT"
|
|
|
|
# Also add the listener to the well-known discovery subsystem. Without this,
|
|
# SPDK still answers discovery on this listener but logs a deprecation warning
|
|
# ("listener was not added to the discovery subsystem") and the implicit
|
|
# behaviour is slated for removal. Explicitly registering it is the supported
|
|
# way and future-proofs `nvme discover` against an SPDK bump. Best-effort:
|
|
# tolerate older SPDK builds where the discovery NQN can't take a listener.
|
|
echo "==> adding TCP listener ${TADDR}:${TPORT} to the discovery subsystem"
|
|
"$RPC" nvmf_subsystem_add_listener nqn.2014-08.org.nvmexpress.discovery \
|
|
-t tcp -a "$TADDR" -s "$TPORT" \
|
|
|| echo "WARNING: could not add listener to discovery subsystem (older SPDK?) — discovery still works via the deprecated implicit path." >&2
|
|
|
|
echo "==> SPDK serving $SUBNQN over tcp/${TPORT}; rd.nvmf.discover=tcp,${TADDR},${TPORT}"
|
|
wait "$TGT_PID"
|