From 2a09476649a6a9585f4d2f49e95f4f8616304723 Mon Sep 17 00:00:00 2001 From: Audiobuzz Date: Sun, 14 Jun 2026 10:10:02 +0000 Subject: [PATCH] Adding pinned uuid for rootfs --- docs/TODO.md | 17 +++++++++++++++++ scripts/extract-artifacts.sh | 12 ++++++++++-- serving/spdk-setup.sh | 28 +++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/docs/TODO.md b/docs/TODO.md index ad4480a..73c023b 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -113,6 +113,23 @@ boot LAN. writable → multi-user. - This validation **gates** item #1 (network-manager port) and item #3 (errata → design v3). +- **Bench findings (boot debug, 2026-06-14):** + - Initramfs has no resolver (dracut writes `resolv.conf` only via its own + DHCP path, not a manual `dhclient`), so the NVMe discovery address must be + an **IP**, not a DNS name — build with `-var nvmet_traddr=192.168.50.10`. + Old UKIs carrying `nvme.fleet.lab` can't connect early in boot. + - NVMe controller indices are **not stable**: a transient discovery + controller can take `nvme0`, pushing the I/O controller to `nvme1`. The + hardcoded `/dev/nvme0n1` in the cmdline was a latent brick. **Fixed:** + namespace UUID pinned on the target (`spdk-setup.sh` `add_ns -u + f1ee7000-0000-4000-8000-000000000001`) and the cmdline now references + `/dev/disk/by-id/nvme-uuid.` (`extract-artifacts.sh`). The two + constants MUST stay in sync; make them version-specific for #6. + - `nvme_tcp`/`nvme_fabrics` must be loaded before `/dev/nvme-fabrics` exists + (dracut's `nvmf` module does this in a normal boot; at `rd.break` modprobe + manually). + - SPDK: add the listener to the discovery subsystem explicitly + (`nqn.2014-08.org.nvmexpress.discovery`) to silence the deprecation warning. ## [ ] 6. Rolling fleet updates (multi-version serve → cohort reboots) diff --git a/scripts/extract-artifacts.sh b/scripts/extract-artifacts.sh index 10e2b74..0ad5ea0 100755 --- a/scripts/extract-artifacts.sh +++ b/scripts/extract-artifacts.sh @@ -23,6 +23,14 @@ DISABLE_DOT1X=${3:-false} # environments. Defaults to the literal placeholder so build-oci.sh's warning # fires if it was never set. NVMET_TRADDR=${NVMET_TRADDR:-NVMET_TRADDR} +# Stable root-namespace UUID. The root device is referenced by-id +# (/dev/disk/by-id/nvme-uuid.$ROOT_NS_UUID) rather than /dev/nvmeNn1 because +# NVMe controller indices are NOT stable: a transient discovery controller can +# take nvme0, pushing the I/O controller to nvme1, so a hardcoded node is a +# latent brick in an immutable signed cmdline. MUST match ROOT_NS_UUID in +# serving/spdk-setup.sh (which assigns it on the target via add_ns -u). +ROOT_NS_UUID=${ROOT_NS_UUID:-f1ee7000-0000-4000-8000-000000000001} +ROOT_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_NS_UUID}" for bin in virt-copy-out virt-tar-out virt-ls mkfs.erofs veritysetup jq; do command -v "$bin" >/dev/null || { echo "ERROR: $bin not found on build host" >&2; exit 1; } @@ -96,8 +104,8 @@ printf '%s' \ "rd.nvmf.discover=tcp,${NVMET_TRADDR},4420 " \ "root=/dev/mapper/root rootfstype=erofs rootflags=ro " \ "systemd.verity=yes roothash=${ROOT_HASH} " \ - "systemd.verity_root_data=/dev/nvme0n1 " \ - "systemd.verity_root_hash=/dev/nvme0n1 " \ + "systemd.verity_root_data=${ROOT_DEV} " \ + "systemd.verity_root_hash=${ROOT_DEV} " \ "systemd.verity_root_options=hash-offset=${HASH_OFFSET}" \ > "$OUT/cmdline.txt" diff --git a/serving/spdk-setup.sh b/serving/spdk-setup.sh index e054442..4e7d3e1 100644 --- a/serving/spdk-setup.sh +++ b/serving/spdk-setup.sh @@ -10,6 +10,15 @@ set -euo pipefail ROOTFS="/artifacts/rootfs.erofs" SUBNQN="${SUBNQN:-nqn.2026-06.org.example.fleet:rootfs}" +# Fixed, fleet-generic namespace UUID. The host references the root device as +# /dev/disk/by-id/nvme-uuid.$ROOT_NS_UUID on the kernel cmdline, which is stable +# regardless of NVMe controller enumeration order (the I/O controller can land +# on nvme0/nvme1/... depending on whether a transient discovery controller took +# a slot first — so a hardcoded /dev/nvme0n1 is unsafe). MUST match +# ROOT_NS_UUID in scripts/extract-artifacts.sh, which bakes it into the UKI +# cmdline. NOTE: for multi-version serving (TODO #6) make this version-specific +# alongside the per-version NQN, so a host can't connect the wrong rootfs. +ROOT_NS_UUID="${ROOT_NS_UUID:-f1ee7000-0000-4000-8000-000000000001}" TADDR="${TARGET_ADDR:-0.0.0.0}" TPORT="${TARGET_PORT:-4420}" CPUMASK="${SPDK_CPUMASK:-0x1}" @@ -58,14 +67,27 @@ echo "==> exposing $ROOTFS as bdev 'rootfs' (4 KiB blocks)" echo "==> creating subsystem $SUBNQN (allow any host — lab/no-auth)" "$RPC" nvmf_create_subsystem "$SUBNQN" -a -s SPDKLAB0000000001 -d FleetRootfs -echo "==> adding namespace 1" +echo "==> adding namespace 1 (uuid $ROOT_NS_UUID)" # Read-only is enforced by the bdev_aio '-r' open above (this SPDK's add_ns has # no --read-only flag), plus the :ro image-volume mount and the client's erofs # ro + dm-verity. -"$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs +# -u pins the namespace UUID so the host can reference it as +# /dev/disk/by-id/nvme-uuid.$ROOT_NS_UUID (stable across controller indices). +"$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs -u "$ROOT_NS_UUID" -echo "==> adding TCP listener ${TADDR}:${TPORT} (also serves discovery)" +echo "==> adding TCP listener ${TADDR}:${TPORT} to $SUBNQN" "$RPC" nvmf_subsystem_add_listener "$SUBNQN" -t tcp -a "$TADDR" -s "$TPORT" +# Also add the listener to the well-known discovery subsystem. Without this, +# SPDK still answers discovery on this listener but logs a deprecation warning +# ("listener was not added to the discovery subsystem") and the implicit +# behaviour is slated for removal. Explicitly registering it is the supported +# way and future-proofs `nvme discover` against an SPDK bump. Best-effort: +# tolerate older SPDK builds where the discovery NQN can't take a listener. +echo "==> adding TCP listener ${TADDR}:${TPORT} to the discovery subsystem" +"$RPC" nvmf_subsystem_add_listener nqn.2014-08.org.nvmexpress.discovery \ + -t tcp -a "$TADDR" -s "$TPORT" \ + || echo "WARNING: could not add listener to discovery subsystem (older SPDK?) — discovery still works via the deprecated implicit path." >&2 + echo "==> SPDK serving $SUBNQN over tcp/${TPORT}; rd.nvmf.discover=tcp,${TADDR},${TPORT}" wait "$TGT_PID"