From 62e49e6b4214820d40683a456988d771ce2131e8 Mon Sep 17 00:00:00 2001 From: Audiobuzz Date: Mon, 15 Jun 2026 08:52:19 +0000 Subject: [PATCH] pivot to a systemd unit for the online hook --- diskless.pkr.hcl | 4 ++ docs/TODO.md | 17 ++++-- files/dracut/95dot1x-macsec/module-setup.sh | 18 ++++-- .../95dot1x-macsec/nvmf-connect.service | 23 ++++++++ files/dracut/95dot1x-macsec/nvmf-connect.sh | 59 +++++++++++-------- serving/.env.example | 4 +- 6 files changed, 88 insertions(+), 37 deletions(-) create mode 100644 files/dracut/95dot1x-macsec/nvmf-connect.service diff --git a/diskless.pkr.hcl b/diskless.pkr.hcl index e423144..6287973 100644 --- a/diskless.pkr.hcl +++ b/diskless.pkr.hcl @@ -216,6 +216,10 @@ build { "KVER=$(rpm -q --qf '%%{VERSION}-%%{RELEASE}.%%{ARCH}\\n' kernel | sort -V | tail -1)", "dracut -f --no-hostonly --kver \"$KVER\" /boot/initramfs-generic.img", "lsinitrd /boot/initramfs-generic.img | grep -E 'macsec|wpa_supplicant|tpm2_nvread|nvme.tcp|erofs|veritysetup' ", + # Assert the test-mode NVMe-oF connect unit made it in (the dracut module + # installs it; this initramfs is systemd-driven so the connect is a + # systemd unit, not a dracut hook). Hard-fail if missing. + "lsinitrd /boot/initramfs-generic.img | grep -q 'nvmf-connect.service' || { echo 'ERROR: nvmf-connect.service missing from initramfs' >&2; exit 1; }", # ---- Build hygiene --------------------------------------------------- "dnf clean all", diff --git a/docs/TODO.md b/docs/TODO.md index 4bb9f97..91386e4 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -146,11 +146,18 @@ boot LAN. `/dev/nvme-fabrics` exists; only the connect trigger is missing. A job placed on the *main* initqueue from a settled hook gets registered but is **never re-executed** once the loop blocks on udev events (verified: job - present, manual `connect-all` works, namespace stays absent). **Fixed:** a - dedicated `initqueue/online` hook (`nvmf-connect.sh`, installed via - `inst_hook initqueue/online 90`) runs `nvme connect-all` when network-legacy - sources online hooks on DHCP-bound — the event-driven trigger the nvmf - module omits. **⚠ The secure path has the same dependency on the + present, manual `connect-all` works, namespace stays absent). An + `initqueue/online` hook also does NOT work: this initramfs is **systemd- + driven** ("A start job is running for /dev/mapper/root") and never sources + dracut online hooks on link-up (verified: hook installs as + `90-nvmf-connect.sh` but never fires — no log, no stamp). **Fixed:** the + connect is now a **systemd unit** (`nvmf-connect.service`, + `ConditionKernelCommandLine=rd.dot1x=0`, `Before=initrd-root-device.target`) + whose ExecStart (`/usr/sbin/nvmf-connect`) polls for the network then runs + `nvme connect-all` until the namespace appears. The dracut module installs + the unit + script and enables it via an `initrd-root-device.target.wants` + symlink; `diskless.pkr.hcl` hard-asserts the unit is in the initramfs. + **⚠ The secure path has the same dependency on the (absent) nvmf autoconnect** — when 802.1X is re-enabled, the macsec0 path must likewise drive `connect-all` (or the network-manager port #1 must wire a working autoconnect). Currently only the test path is fixed. diff --git a/files/dracut/95dot1x-macsec/module-setup.sh b/files/dracut/95dot1x-macsec/module-setup.sh index 5f79641..be7b0a2 100755 --- a/files/dracut/95dot1x-macsec/module-setup.sh +++ b/files/dracut/95dot1x-macsec/module-setup.sh @@ -42,10 +42,20 @@ install() { # Hook: runs from the initqueue once the NIC exists, before netroot inst_hook initqueue/settled 30 "$moddir/dot1x-macsec-up.sh" - # Test-mode (rd.dot1x=0) NVMe-oF connect, driven event-wise on link-up. - # This distro's nvmf module registers no autoconnect, and a main-initqueue - # job isn't re-run once the loop waits on events — so connect here instead. - inst_hook initqueue/online 90 "$moddir/nvmf-connect.sh" + # Test-mode (rd.dot1x=0) NVMe-oF connect as a systemd unit. This initramfs + # is systemd-driven and does NOT source dracut initqueue/online hooks when + # the ip=dhcp link comes up (verified: the hook installs but never fires), + # and the nvmf module registers no autoconnect — so a dracut hook never + # runs. A oneshot unit ordered Before=initrd-root-device.target, with its + # own wait-for-network + connect loop, connects the namespace + # deterministically. ConditionKernelCommandLine=rd.dot1x=0 makes it a no-op + # on the secure path. + inst_simple "$moddir/nvmf-connect.sh" /usr/sbin/nvmf-connect + inst_simple "$moddir/nvmf-connect.service" \ + "$systemdsystemunitdir/nvmf-connect.service" + mkdir -p "${initdir}${systemdsystemunitdir}/initrd-root-device.target.wants" + ln -sf ../nvmf-connect.service \ + "${initdir}${systemdsystemunitdir}/initrd-root-device.target.wants/nvmf-connect.service" dracut_need_initqueue } diff --git a/files/dracut/95dot1x-macsec/nvmf-connect.service b/files/dracut/95dot1x-macsec/nvmf-connect.service new file mode 100644 index 0000000..48db32d --- /dev/null +++ b/files/dracut/95dot1x-macsec/nvmf-connect.service @@ -0,0 +1,23 @@ +[Unit] +Description=Connect NVMe-oF root namespace (test mode, rd.dot1x=0) +Documentation=https://github.com/ (diskless fleet boot — open-network test path) +DefaultDependencies=no +# Open-network test path only; no-op when 802.1X/MACsec is enabled. +ConditionKernelCommandLine=rd.dot1x=0 +After=systemd-udevd.service +# Run before the root block device is required: connecting the namespace makes +# /dev/disk/by-id/nvme-uuid.* appear so systemd-veritysetup can map +# /dev/mapper/root. Networking (ip=dhcp) comes up in parallel; the ExecStart +# script polls for it, so we deliberately do NOT order After the network here +# (avoids depending on a sync point this initramfs may not reach). +Before=initrd-root-device.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/bash /usr/sbin/nvmf-connect +# ExecStart self-limits to ~60s; keep the unit timeout above that. +TimeoutStartSec=120 + +[Install] +WantedBy=initrd-root-device.target diff --git a/files/dracut/95dot1x-macsec/nvmf-connect.sh b/files/dracut/95dot1x-macsec/nvmf-connect.sh index b35b5a0..4e26d61 100644 --- a/files/dracut/95dot1x-macsec/nvmf-connect.sh +++ b/files/dracut/95dot1x-macsec/nvmf-connect.sh @@ -1,42 +1,49 @@ #!/bin/bash -# nvmf-connect.sh — initqueue/online hook (fires when a NIC comes online). +# nvmf-connect — ExecStart for nvmf-connect.service (initramfs, TEST MODE). # -# TEST MODE ONLY (rd.dot1x=0): drive the NVMe-oF root connect here. -# This distro's dracut `nvmf` module parses rd.nvmf.discover into -# /etc/nvme/discovery.conf but registers NO autoconnect hook (initqueue/online/ -# ships empty), and a job placed on the MAIN initqueue is not re-executed once -# the loop blocks waiting on udev events — so `nvme connect-all` otherwise never -# runs, the root namespace never appears, and boot spins on /dev/mapper/root -# forever. initqueue/online is the correct, event-driven place: network-legacy -# sources these hooks once DHCP is bound, so the link is guaranteed up here. -# Idempotent via a stamp; connect-all is itself safe to re-run, and on failure -# we leave the stamp unset so the next online event retries. +# Why a systemd unit and not a dracut hook: this initramfs is systemd-driven and +# does NOT source dracut initqueue/online hooks when the ip=dhcp link comes up +# (verified on the bench — the online hook installs but never fires: no log, no +# connect). dracut's nvmf module also registers no autoconnect, and a job placed +# on the main initqueue is never re-executed once the loop blocks on events. A +# oneshot unit ordered Before=initrd-root-device.target, with its own +# wait-for-network + connect loop, deterministically connects the NVMe-oF root +# namespace so the by-id device appears and systemd-veritysetup can map +# /dev/mapper/root. # -# The secure path (rd.dot1x unset/!=0) connects over macsec0 from its own flow -# in dot1x-macsec-up.sh and must NOT connect over the physical link here. - +# rd.dot1x=0 (open-network test) ONLY — the unit's ConditionKernelCommandLine +# already gates this, but re-check so the script is safe to run directly. The +# secure path connects over macsec0 from dot1x-macsec-up.sh. type getarg >/dev/null 2>&1 || . /lib/dracut-lib.sh [ "$(getarg rd.dot1x=)" = "0" ] || exit 0 [ -e /run/dot1x/.nvmf-connected ] && exit 0 _disc=$(getarg rd.nvmf.discover=) # e.g. tcp,192.168.50.10,4420 -[ -n "$_disc" ] || { warn "nvmf-connect: rd.nvmf.discover not set"; exit 0; } +[ -n "$_disc" ] || { echo "nvmf-connect: rd.nvmf.discover not set" >&2; exit 1; } _trtype=${_disc%%,*} _rest=${_disc#*,} _traddr=${_rest%%,*} _trsvcid=${_rest##*,} -# nvme_tcp is usually auto-loaded, but ensure it so connect-all can't fail on a -# missing /dev/nvme-fabrics. +# nvme_tcp is usually auto-loaded; ensure it so /dev/nvme-fabrics exists. modprobe nvme_tcp 2>/dev/null -info "nvmf-connect: nvme connect-all ${_trtype}/${_traddr}/${_trsvcid}" -if nvme connect-all -t "$_trtype" -a "$_traddr" -s "$_trsvcid"; then - mkdir -p /run/dot1x - : > /run/dot1x/.nvmf-connected - info "nvmf-connect: connected — namespace + by-id link should appear shortly" -else - warn "nvmf-connect: connect-all failed; will retry on next online event" -fi -exit 0 +# Networking (ip=dhcp) is brought up in parallel, so poll-and-connect until a +# fabrics namespace appears or we time out. connect-all is idempotent. +_i=0 +while [ "$_i" -lt 60 ]; do + nvme connect-all -t "$_trtype" -a "$_traddr" -s "$_trsvcid" 2>/dev/null + for _ns in /dev/nvme[0-9]*n[0-9]*; do + if [ -b "$_ns" ]; then + mkdir -p /run/dot1x + : > /run/dot1x/.nvmf-connected + echo "nvmf-connect: connected ${_trtype}/${_traddr}/${_trsvcid} ($_ns)" + exit 0 + fi + done + sleep 1 + _i=$((_i + 1)) +done +echo "nvmf-connect: timed out connecting ${_trtype}/${_traddr}/${_trsvcid}" >&2 +exit 1 diff --git a/serving/.env.example b/serving/.env.example index c5b26f5..1cb629c 100644 --- a/serving/.env.example +++ b/serving/.env.example @@ -1,7 +1,7 @@ # Copy to .env and edit for your test LAN/bridge, then: docker compose up -d # Release being served. Must match the Packer build (fleet-.efi). -IMAGE_VERSION=0.0.0-a508dce +IMAGE_VERSION=0.0.0-2eefd97 # OCI release-artifact image (built by scripts/build-oci.sh). Mounted directly # as a Docker image volume — the default artifact source. Pin by digest in @@ -10,7 +10,7 @@ OCI_IMAGE=git.bitasylum.org/audiobuzz/fleet-boot # Fallback only (Docker < 28 / offline): local Packer output dir, used by the # commented bind-mounts in docker-compose.yml. Ignored when image volumes work. -ARTIFACT_SRC=../artifacts/0.0.0-a508dce +ARTIFACT_SRC=../artifacts/0.0.0-2eefd97 # SPDK NVMe/TCP target image — prebuilt + pushed by CI # (.gitea/workflows/spdk-image.yaml); the serving host just pulls it.