diff --git a/docs/TODO.md b/docs/TODO.md index add9967..9d10b50 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -147,3 +147,27 @@ this item builds the rollout machinery around it. especially mid-rollout (N versions + reboot bursts). - Note: kiosk *content* changes are server-side (the website) and need no fleet update — only OS/UKI/rootfs changes require a reboot. + +## [ ] 7. Harden / evaluate the NVMe-oF target (serving-node resilience) + +There is **no production-grade memory-safe NVMe-oF target** — the Rust NVMe work +is all driver/initiator-side (vroom, RedLeaf, Rust-for-Linux); the only +production targets are C (Linux kernel `nvmet`, SPDK). So this is **serving-node +hardening, not a language swap**. dm-verity already removes the target from the +clients' integrity TCB (verify-on-read rejects tampered blocks), so the risk is +RCE/DoS on the node that holds the fleet's disk, not client integrity. The +export is read-only + single-namespace = inherently small attack surface. + +- **Evaluate kernel `nvmet` vs SPDK** for the target: kernel `nvmet` keeps the + data path in the hardened/fuzzed kernel with a tiny userspace footprint + (arguably a better attack-surface story); SPDK is a userspace C daemon + (current choice, needs hugepages + caps). Decide per environment. +- **Confine the target** (whichever): unprivileged, drop caps, seccomp + + SELinux/AppArmor profile, read-only container, minimal mounts; minimal + immutable serving image, digest-pinned. +- **Network-isolate :4420** so only the boot VLAN reaches the target (compose + with 802.1X/MACsec segmentation in prod). +- **Replicas** behind distinct discovery entries (availability is the real risk + — it's the fleet's disk). Ties into #6 (rolling updates) availability. +- Revisit a memory-safe target only if/when an off-the-shelf one matures + (e.g. a future Rust-for-Linux `nvmet`). diff --git a/serving/.env.example b/serving/.env.example index 687f70b..e28ef7c 100644 --- a/serving/.env.example +++ b/serving/.env.example @@ -1,7 +1,7 @@ # Copy to .env and edit for your test LAN/bridge, then: docker compose up -d # Release being served. Must match the Packer build (fleet-.efi). -IMAGE_VERSION=0.1.0 +IMAGE_VERSION=0.0.0-e98d881 # OCI release-artifact image (built by scripts/build-oci.sh). Mounted directly # as a Docker image volume — the default artifact source. Pin by digest in @@ -10,7 +10,7 @@ OCI_IMAGE=git.bitasylum.org/audiobuzz/fleet-boot # Fallback only (Docker < 28 / offline): local Packer output dir, used by the # commented bind-mounts in docker-compose.yml. Ignored when image volumes work. -ARTIFACT_SRC=../artifacts/0.1.0 +ARTIFACT_SRC=../artifacts/0.0.0-e98d881 # SPDK NVMe/TCP target image — prebuilt + pushed by CI # (.gitea/workflows/spdk-image.yaml); the serving host just pulls it. @@ -18,10 +18,10 @@ SPDK_IMAGE=git.bitasylum.org/audiobuzz/spdk-nvmf SPDK_VERSION=v24.09 # --- This serving host on the boot LAN ------------------------------------- -BOOT_IFACE=eth0 # interface facing the booting client / test bridge +BOOT_IFACE=ens19 # interface facing the booting client / test bridge SERVER_IP=192.168.50.10 # this host's address on BOOT_IFACE (DHCP/DNS/HTTP/NVMe) GATEWAY_IP=192.168.50.1 -UPSTREAM_DNS=1.1.1.1 +UPSTREAM_DNS=10.42.42.254 # --- DHCP scope ------------------------------------------------------------ DHCP_RANGE_START=192.168.50.100 diff --git a/serving/docker-compose.yml b/serving/docker-compose.yml index 1ee5b18..4bfb85c 100644 --- a/serving/docker-compose.yml +++ b/serving/docker-compose.yml @@ -45,6 +45,10 @@ services: restart: unless-stopped command: - --keep-in-foreground + # Log to stderr so `docker compose logs` captures output. Without this, + # dnsmasq logs to syslog — which doesn't exist in the container — so + # --log-dhcp/--log-queries produce nothing even while dnsmasq works fine. + - --log-facility=- - --log-dhcp - --log-queries # Bind only to the boot interface so we don't fight the host resolver. diff --git a/serving/spdk-setup.sh b/serving/spdk-setup.sh index cc7960e..e054442 100644 --- a/serving/spdk-setup.sh +++ b/serving/spdk-setup.sh @@ -20,6 +20,22 @@ CPUMASK="${SPDK_CPUMASK:-0x1}" TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt) RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py) +# Ensure hugepages exist — DPDK needs them. vm.nr_hugepages isn't namespaced, +# so a privileged container writes the host value; this re-applies after a host +# reboot (the container restarts). Best-effort: allocation can fail on a +# fragmented host, in which case set vm.nr_hugepages on the host (persistently +# via /etc/sysctl.d). Override the count with HUGEPAGES. +HUGEPAGES="${HUGEPAGES:-1024}" +if [ -w /proc/sys/vm/nr_hugepages ]; then + cur=$(cat /proc/sys/vm/nr_hugepages 2>/dev/null || echo 0) + if [ "${cur:-0}" -lt "$HUGEPAGES" ]; then + echo "==> allocating $HUGEPAGES hugepages (was ${cur:-0})" + echo "$HUGEPAGES" > /proc/sys/vm/nr_hugepages 2>/dev/null || true + fi +fi +free=$(awk '/HugePages_Free/{print $2}' /proc/meminfo 2>/dev/null || echo 0) +[ "${free:-0}" -gt 0 ] || echo "WARNING: no free hugepages — SPDK will fail; allocate vm.nr_hugepages on the host." >&2 + echo "==> starting nvmf_tgt (cpumask ${CPUMASK})" "$TGT" -m "$CPUMASK" & TGT_PID=$! @@ -42,10 +58,11 @@ echo "==> exposing $ROOTFS as bdev 'rootfs' (4 KiB blocks)" echo "==> creating subsystem $SUBNQN (allow any host — lab/no-auth)" "$RPC" nvmf_create_subsystem "$SUBNQN" -a -s SPDKLAB0000000001 -d FleetRootfs -echo "==> adding namespace 1 (read-only)" -# Newer SPDK accepts --read-only on the namespace; fall back if unsupported. -"$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs --read-only \ - || "$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs +echo "==> adding namespace 1" +# Read-only is enforced by the bdev_aio '-r' open above (this SPDK's add_ns has +# no --read-only flag), plus the :ro image-volume mount and the client's erofs +# ro + dm-verity. +"$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs echo "==> adding TCP listener ${TADDR}:${TPORT} (also serves discovery)" "$RPC" nvmf_subsystem_add_listener "$SUBNQN" -t tcp -a "$TADDR" -s "$TPORT" diff --git a/serving/spdk/Dockerfile b/serving/spdk/Dockerfile index 68a0fda..b0bd316 100644 --- a/serving/spdk/Dockerfile +++ b/serving/spdk/Dockerfile @@ -34,7 +34,7 @@ RUN apt-get update \ && /spdk/scripts/pkgdep.sh \ && apt-get install -y --no-install-recommends meson ninja-build python3-pyelftools \ && cd /spdk \ - && ./configure --disable-tests --disable-unit-tests \ + && ./configure --disable-tests --disable-unit-tests --target-arch=x86-64-v2 \ && make -j"$(nproc)" \ && rm -rf /var/lib/apt/lists/*