Fixinf SPDK

docs/TODO.md

@@ -147,3 +147,27 @@ this item builds the rollout machinery around it.
   especially mid-rollout (N versions + reboot bursts).
 - Note: kiosk *content* changes are server-side (the website) and need no fleet
   update — only OS/UKI/rootfs changes require a reboot.
+
+## [ ] 7. Harden / evaluate the NVMe-oF target (serving-node resilience)
+
+There is **no production-grade memory-safe NVMe-oF target** — the Rust NVMe work
+is all driver/initiator-side (vroom, RedLeaf, Rust-for-Linux); the only
+production targets are C (Linux kernel `nvmet`, SPDK). So this is **serving-node
+hardening, not a language swap**. dm-verity already removes the target from the
+clients' integrity TCB (verify-on-read rejects tampered blocks), so the risk is
+RCE/DoS on the node that holds the fleet's disk, not client integrity. The
+export is read-only + single-namespace = inherently small attack surface.
+
+- **Evaluate kernel `nvmet` vs SPDK** for the target: kernel `nvmet` keeps the
+  data path in the hardened/fuzzed kernel with a tiny userspace footprint
+  (arguably a better attack-surface story); SPDK is a userspace C daemon
+  (current choice, needs hugepages + caps). Decide per environment.
+- **Confine the target** (whichever): unprivileged, drop caps, seccomp +
+  SELinux/AppArmor profile, read-only container, minimal mounts; minimal
+  immutable serving image, digest-pinned.
+- **Network-isolate :4420** so only the boot VLAN reaches the target (compose
+  with 802.1X/MACsec segmentation in prod).
+- **Replicas** behind distinct discovery entries (availability is the real risk
+  — it's the fleet's disk). Ties into #6 (rolling updates) availability.
+- Revisit a memory-safe target only if/when an off-the-shelf one matures
+  (e.g. a future Rust-for-Linux `nvmet`).

serving/.env.example

@@ -1,7 +1,7 @@
 # Copy to .env and edit for your test LAN/bridge, then: docker compose up -d
 
 # Release being served. Must match the Packer build (fleet-<version>.efi).
-IMAGE_VERSION=0.1.0
+IMAGE_VERSION=0.0.0-e98d881
 
 # OCI release-artifact image (built by scripts/build-oci.sh). Mounted directly
 # as a Docker image volume — the default artifact source. Pin by digest in
@@ -10,7 +10,7 @@ OCI_IMAGE=git.bitasylum.org/audiobuzz/fleet-boot
 
 # Fallback only (Docker < 28 / offline): local Packer output dir, used by the
 # commented bind-mounts in docker-compose.yml. Ignored when image volumes work.
-ARTIFACT_SRC=../artifacts/0.1.0
+ARTIFACT_SRC=../artifacts/0.0.0-e98d881
 
 # SPDK NVMe/TCP target image — prebuilt + pushed by CI
 # (.gitea/workflows/spdk-image.yaml); the serving host just pulls it.
@@ -18,10 +18,10 @@ SPDK_IMAGE=git.bitasylum.org/audiobuzz/spdk-nvmf
 SPDK_VERSION=v24.09
 
 # --- This serving host on the boot LAN -------------------------------------
-BOOT_IFACE=eth0            # interface facing the booting client / test bridge
+BOOT_IFACE=ens19            # interface facing the booting client / test bridge
 SERVER_IP=192.168.50.10    # this host's address on BOOT_IFACE (DHCP/DNS/HTTP/NVMe)
 GATEWAY_IP=192.168.50.1
-UPSTREAM_DNS=1.1.1.1
+UPSTREAM_DNS=10.42.42.254
 
 # --- DHCP scope ------------------------------------------------------------
 DHCP_RANGE_START=192.168.50.100

serving/docker-compose.yml

@@ -45,6 +45,10 @@ services:
     restart: unless-stopped
     command:
       - --keep-in-foreground
+      # Log to stderr so `docker compose logs` captures output. Without this,
+      # dnsmasq logs to syslog — which doesn't exist in the container — so
+      # --log-dhcp/--log-queries produce nothing even while dnsmasq works fine.
+      - --log-facility=-
       - --log-dhcp
       - --log-queries
       # Bind only to the boot interface so we don't fight the host resolver.

serving/spdk-setup.sh

@@ -20,6 +20,22 @@ CPUMASK="${SPDK_CPUMASK:-0x1}"
 TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt)
 RPC=$(command -v rpc.py   || echo /usr/libexec/spdk/scripts/rpc.py)
 
+# Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced,
+# so a privileged container writes the host value; this re-applies after a host
+# reboot (the container restarts). Best-effort: allocation can fail on a
+# fragmented host, in which case set vm.nr_hugepages on the host (persistently
+# via /etc/sysctl.d). Override the count with HUGEPAGES.
+HUGEPAGES="${HUGEPAGES:-1024}"
+if [ -w /proc/sys/vm/nr_hugepages ]; then
+    cur=$(cat /proc/sys/vm/nr_hugepages 2>/dev/null || echo 0)
+    if [ "${cur:-0}" -lt "$HUGEPAGES" ]; then
+        echo "==> allocating $HUGEPAGES hugepages (was ${cur:-0})"
+        echo "$HUGEPAGES" > /proc/sys/vm/nr_hugepages 2>/dev/null || true
+    fi
+fi
+free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0)
+[ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2
+
 echo "==> starting nvmf_tgt (cpumask ${CPUMASK})"
 "$TGT" -m "$CPUMASK" &
 TGT_PID=$!
@@ -42,10 +58,11 @@ echo "==> exposing $ROOTFS as bdev 'rootfs' (4 KiB blocks)"
 echo "==> creating subsystem $SUBNQN (allow any host — lab/no-auth)"
 "$RPC" nvmf_create_subsystem "$SUBNQN" -a -s SPDKLAB0000000001 -d FleetRootfs
 
-echo "==> adding namespace 1 (read-only)"
-# Newer SPDK accepts --read-only on the namespace; fall back if unsupported.
-"$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs --read-only \
-    || "$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs
+echo "==> adding namespace 1"
+# Read-only is enforced by the bdev_aio '-r' open above (this SPDK's add_ns has
+# no --read-only flag), plus the :ro image-volume mount and the client's erofs
+# ro + dm-verity.
+"$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs
 
 echo "==> adding TCP listener ${TADDR}:${TPORT} (also serves discovery)"
 "$RPC" nvmf_subsystem_add_listener "$SUBNQN" -t tcp -a "$TADDR" -s "$TPORT"

serving/spdk/Dockerfile

@@ -34,7 +34,7 @@ RUN apt-get update \
  && /spdk/scripts/pkgdep.sh \
  && apt-get install -y --no-install-recommends meson ninja-build python3-pyelftools \
  && cd /spdk \
- && ./configure --disable-tests --disable-unit-tests \
+ && ./configure --disable-tests --disable-unit-tests --target-arch=x86-64-v2 \
  && make -j"$(nproc)" \
  && rm -rf /var/lib/apt/lists/*