72 lines
3.1 KiB
Bash
72 lines
3.1 KiB
Bash
#!/usr/bin/env bash
|
|
# Configure SPDK nvmf_tgt to serve the diskless root filesystem as a single
|
|
# READ-ONLY NVMe/TCP namespace (design doc §5, §7).
|
|
#
|
|
# rootfs.erofs is the erofs image with the dm-verity hash tree appended in-file;
|
|
# we expose the WHOLE file as one namespace, and the host splits data/hash with
|
|
# systemd.verity_root_options=hash-offset= from the kernel cmdline. Integrity is
|
|
# enforced end-to-end by dm-verity on the host; the target serves it read-only.
|
|
set -euo pipefail
|
|
|
|
ROOTFS="/artifacts/rootfs.erofs"
|
|
SUBNQN="${SUBNQN:-nqn.2026-06.org.example.fleet:rootfs}"
|
|
TADDR="${TARGET_ADDR:-0.0.0.0}"
|
|
TPORT="${TARGET_PORT:-4420}"
|
|
CPUMASK="${SPDK_CPUMASK:-0x1}"
|
|
|
|
[ -s "$ROOTFS" ] || { echo "ERROR: $ROOTFS not found/empty — is ARTIFACT_SRC populated?" >&2; exit 1; }
|
|
|
|
# Binary locations differ across SPDK images; fall back to the common path.
|
|
TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt)
|
|
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
|
|
|
|
# Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced,
|
|
# so a privileged container writes the host value; this re-applies after a host
|
|
# reboot (the container restarts). Best-effort: allocation can fail on a
|
|
# fragmented host, in which case set vm.nr_hugepages on the host (persistently
|
|
# via /etc/sysctl.d). Override the count with HUGEPAGES.
|
|
HUGEPAGES="${HUGEPAGES:-1024}"
|
|
if [ -w /proc/sys/vm/nr_hugepages ]; then
|
|
cur=$(cat /proc/sys/vm/nr_hugepages 2>/dev/null || echo 0)
|
|
if [ "${cur:-0}" -lt "$HUGEPAGES" ]; then
|
|
echo "==> allocating $HUGEPAGES hugepages (was ${cur:-0})"
|
|
echo "$HUGEPAGES" > /proc/sys/vm/nr_hugepages 2>/dev/null || true
|
|
fi
|
|
fi
|
|
free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0)
|
|
[ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2
|
|
|
|
echo "==> starting nvmf_tgt (cpumask ${CPUMASK})"
|
|
"$TGT" -m "$CPUMASK" &
|
|
TGT_PID=$!
|
|
trap 'kill "$TGT_PID" 2>/dev/null || true' EXIT
|
|
|
|
# Wait for the RPC socket to come up.
|
|
for _ in $(seq 1 30); do
|
|
"$RPC" framework_get_config >/dev/null 2>&1 && break
|
|
sleep 1
|
|
done
|
|
|
|
echo "==> creating TCP transport"
|
|
"$RPC" nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192
|
|
|
|
echo "==> exposing $ROOTFS as bdev 'rootfs' (4 KiB blocks)"
|
|
# -r opens the backing file read-only where supported; harmless if ignored.
|
|
"$RPC" bdev_aio_create "$ROOTFS" rootfs 4096 -r \
|
|
|| "$RPC" bdev_aio_create "$ROOTFS" rootfs 4096
|
|
|
|
echo "==> creating subsystem $SUBNQN (allow any host — lab/no-auth)"
|
|
"$RPC" nvmf_create_subsystem "$SUBNQN" -a -s SPDKLAB0000000001 -d FleetRootfs
|
|
|
|
echo "==> adding namespace 1"
|
|
# Read-only is enforced by the bdev_aio '-r' open above (this SPDK's add_ns has
|
|
# no --read-only flag), plus the :ro image-volume mount and the client's erofs
|
|
# ro + dm-verity.
|
|
"$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs
|
|
|
|
echo "==> adding TCP listener ${TADDR}:${TPORT} (also serves discovery)"
|
|
"$RPC" nvmf_subsystem_add_listener "$SUBNQN" -t tcp -a "$TADDR" -s "$TPORT"
|
|
|
|
echo "==> SPDK serving $SUBNQN over tcp/${TPORT}; rd.nvmf.discover=tcp,${TADDR},${TPORT}"
|
|
wait "$TGT_PID"
|