From 62e997297c05a5d364cc80e5e981a22fb59982d6 Mon Sep 17 00:00:00 2001 From: Audiobuzz Date: Sun, 14 Jun 2026 05:58:55 +0000 Subject: [PATCH] Add SPDDK build --- .gitea/workflows/spdk-image.yaml | 77 +++++++++++++++++++++ README.md | 8 ++- diskless.pkr.hcl | 8 ++- docs/TODO.md | 15 ++++ scripts/extract-artifacts.sh | 7 +- serving/.env.example | 5 ++ serving/README.md | 115 +++++++++++++++++++++++++++++++ serving/docker-compose.yml | 5 +- serving/spdk/Dockerfile | 39 +++++++++++ 9 files changed, 273 insertions(+), 6 deletions(-) create mode 100644 .gitea/workflows/spdk-image.yaml create mode 100644 serving/README.md create mode 100644 serving/spdk/Dockerfile diff --git a/.gitea/workflows/spdk-image.yaml b/.gitea/workflows/spdk-image.yaml new file mode 100644 index 0000000..9c54a83 --- /dev/null +++ b/.gitea/workflows/spdk-image.yaml @@ -0,0 +1,77 @@ +# +# .gitea/workflows/spdk-image.yaml +# +# Builds the SPDK NVMe/TCP target image (serving stack) from serving/spdk and +# pushes it to the registry, so serving hosts just `docker compose pull` it. +# There is no official spdk/spdk image to pull, so we compile a pinned release +# once here (slow: ~10-20 min, needs a runner with enough CPU/RAM/disk). +# +# Runs only when the SPDK image inputs change (path filter) or on demand — not +# on every commit. Bump SPDK_VERSION (or use the manual run input) to roll SPDK. +# +# Required Gitea repo secrets: PACKAGE_USER, PACKAGE_TOKEN (registry creds). +# + +name: Build SPDK NVMe-oF image +run-name: ${{ gitea.actor }} builds the SPDK NVMe/TCP target image +on: + push: + branches: + - "main" + paths: + - "serving/spdk/**" + - ".gitea/workflows/spdk-image.yaml" + workflow_dispatch: + inputs: + spdk_version: + description: "SPDK release tag (https://github.com/spdk/spdk/releases)" + required: false + default: "v24.09" + +env: + SPDK_IMAGE: "git.bitasylum.org/audiobuzz/spdk-nvmf" + SPDK_VERSION: "v24.09" # default tag for push-triggered builds + +jobs: + spdk-build: + name: Build & push SPDK image + runs-on: ubuntu-latest + container: + image: git.bitasylum.org/audiobuzz/ubuntu-act-runner-qemu:main + steps: + - + name: Checkout + uses: actions/checkout@v6 + + - + name: Install buildah + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends buildah + + - + name: Registry login + run: | + buildah login \ + --username "${{ secrets.PACKAGE_USER }}" \ + --password "${{ secrets.PACKAGE_TOKEN }}" \ + git.bitasylum.org + + - + name: Build & push + # vfs storage (overlay-on-overlayfs is unsupported in-container) and + # chroot isolation (so the Dockerfile's RUN/compile steps work without + # nested user namespaces). + env: + STORAGE_DRIVER: vfs + BUILDAH_ISOLATION: chroot + run: | + VER="${{ inputs.spdk_version || env.SPDK_VERSION }}" + echo "Building ${SPDK_IMAGE}:${VER}" + buildah bud \ + --build-arg SPDK_VERSION="$VER" \ + -t "${SPDK_IMAGE}:${VER}" \ + -t "${SPDK_IMAGE}:latest" \ + serving/spdk + buildah push "${SPDK_IMAGE}:${VER}" + buildah push "${SPDK_IMAGE}:latest" diff --git a/README.md b/README.md index 24d2c67..e1e5b7b 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,11 @@ packer build \ ``` Omit `sb_key`/`sb_cert` for unsigned lab builds (the pipeline warns). -Substitute the real NVMe target address for `NVMET_TRADDR` in -`scripts/extract-artifacts.sh` (or post-edit `cmdline.txt`) **before** the -OCI step: the cmdline is immutable once signed into the UKI. +Set the NVMe discovery address with `-var nvmet_traddr=` (default +`nvme.fleet.lab`, which the `serving/` stack's dnsmasq resolves); it is baked +into the cmdline and **immutable once signed**, so prefer a per-environment DNS +name over a raw IP. Deploy/boot-test instructions live in +[serving/README.md](serving/README.md). **Test mode (auth disabled).** Add `-var disable_dot1x=true` to bake `rd.dot1x=0` into the cmdline. At boot the hook then skips 802.1X/MACsec and diff --git a/diskless.pkr.hcl b/diskless.pkr.hcl index 217d607..e423144 100644 --- a/diskless.pkr.hcl +++ b/diskless.pkr.hcl @@ -94,6 +94,12 @@ variable "disable_dot1x" { default = false } +variable "nvmet_traddr" { + type = string + description = "NVMe/TCP discovery address baked into the UKI cmdline (rd.nvmf.discover). Use a per-environment DNS name resolved at boot so one UKI works everywhere; the lab serving stack's dnsmasq resolves this name to the serving host. Immutable once the UKI is signed." + default = "nvme.fleet.lab" +} + variable "sb_key" { type = string description = "Secure Boot signing key (PEM) for the UKI. Empty = unsigned lab build. Production: CI/HSM-held; never enters the build VM." @@ -222,7 +228,7 @@ build { # --- 4. Assemble + sign the UKI; build the OCI release-artifact image ---- post-processor "shell-local" { inline = [ - "bash scripts/extract-artifacts.sh output-qemu/diskless-build.qcow2 ${var.artifact_dir}/${var.image_version} ${var.disable_dot1x}", + "NVMET_TRADDR='${var.nvmet_traddr}' bash scripts/extract-artifacts.sh output-qemu/diskless-build.qcow2 ${var.artifact_dir}/${var.image_version} ${var.disable_dot1x}", "bash scripts/build-oci.sh ${var.artifact_dir}/${var.image_version} ${var.oci_image} ${var.image_version} '${var.sb_key}' '${var.sb_cert}' ${var.oci_push}", ] } diff --git a/docs/TODO.md b/docs/TODO.md index 817621f..86f7ba1 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -98,3 +98,18 @@ is the tmpfs overlay), fleet-generic (no per-host state). browser are STIG findings and enlarge the attack surface/image. Reconcile — e.g. a tightly-constrained kiosk session + documented deviation — when hardening lands. + +## [~] 5. Deploy & validate the end-to-end boot path (open-network test) + +In progress. Stand up `serving/` (dnsmasq + nginx + SPDK) per +[serving/README.md](../serving/README.md), rebuild the UKI with +`nvmet_traddr=nvme.fleet.lab` (now a Packer var — older UKIs have the literal +`NVMET_TRADDR` placeholder and won't connect), then boot a UEFI HTTP-boot client +(**Secure Boot OFF**, unsigned UKI; no TPM needed in `rd.dot1x=0` mode) on the +boot LAN. + +- **Pass criteria:** UKI fetched over HTTP → initramfs → DHCP on NIC → NVMe/TCP + namespace connected → dm-verity root mapped → `systemd.volatile=overlay` + writable → multi-user. +- This validation **gates** item #1 (network-manager port) and item #3 + (errata → design v3). diff --git a/scripts/extract-artifacts.sh b/scripts/extract-artifacts.sh index f7288a8..1fc52be 100755 --- a/scripts/extract-artifacts.sh +++ b/scripts/extract-artifacts.sh @@ -18,6 +18,11 @@ OUT=${2:?usage: extract-artifacts.sh [disable_dot1x]} # TEMPORARY test toggle: when "true", bake rd.dot1x=0 into the cmdline so the # UKI boots with 802.1X + MACsec disabled (open-port NVMe-oF boot test). DISABLE_DOT1X=${3:-false} +# NVMe/TCP discovery address baked into the (immutable, signed) UKI cmdline. +# Prefer a per-environment DNS name resolved at boot so one UKI works across +# environments. Defaults to the literal placeholder so build-oci.sh's warning +# fires if it was never set. +NVMET_TRADDR=${NVMET_TRADDR:-NVMET_TRADDR} for bin in virt-copy-out virt-tar-out virt-ls mkfs.erofs veritysetup jq; do command -v "$bin" >/dev/null || { echo "ERROR: $bin not found on build host" >&2; exit 1; } @@ -83,7 +88,7 @@ printf '%s' \ "$DOT1X_FRAGMENT" \ "rd.neednet=1 systemd.volatile=overlay " \ "rd.dot1x.nvindex=0x01800001 rd.dot1x.timeout=60 " \ - "rd.nvmf.discover=tcp,NVMET_TRADDR,4420 " \ + "rd.nvmf.discover=tcp,${NVMET_TRADDR},4420 " \ "root=/dev/mapper/root rootfstype=erofs rootflags=ro " \ "systemd.verity=yes roothash=${ROOT_HASH} " \ "systemd.verity_root_data=/dev/nvme0n1 " \ diff --git a/serving/.env.example b/serving/.env.example index 6b9c6dd..687f70b 100644 --- a/serving/.env.example +++ b/serving/.env.example @@ -12,6 +12,11 @@ OCI_IMAGE=git.bitasylum.org/audiobuzz/fleet-boot # commented bind-mounts in docker-compose.yml. Ignored when image volumes work. ARTIFACT_SRC=../artifacts/0.1.0 +# SPDK NVMe/TCP target image — prebuilt + pushed by CI +# (.gitea/workflows/spdk-image.yaml); the serving host just pulls it. +SPDK_IMAGE=git.bitasylum.org/audiobuzz/spdk-nvmf +SPDK_VERSION=v24.09 + # --- This serving host on the boot LAN ------------------------------------- BOOT_IFACE=eth0 # interface facing the booting client / test bridge SERVER_IP=192.168.50.10 # this host's address on BOOT_IFACE (DHCP/DNS/HTTP/NVMe) diff --git a/serving/README.md b/serving/README.md new file mode 100644 index 0000000..2b29afa --- /dev/null +++ b/serving/README.md @@ -0,0 +1,115 @@ +# Serving stack — deploy & test (lab) + +Stands up DHCP/DNS + HTTP(UKI) + NVMe/TCP(rootfs) so a diskless client can +network-boot the release built by the pipeline, and walks through testing it. +This targets the **open-network test path** (`rd.dot1x=0`, unsigned UKI) — see +the repo README for the secure path. + +## 0. Prerequisites + +- A serving host **on the same L2 as the test client** (the boot VLAN / test + bridge). Note its interface (`BOOT_IFACE`) and address (`SERVER_IP`). +- **Docker Engine ≥ 28** (for the `type: image` artifact volume; otherwise use + the commented bind-mount fallback in `docker-compose.yml`). +- Hugepages for SPDK: `sudo sysctl vm.nr_hugepages=1024`. +- Registry creds (`docker login git.bitasylum.org`) — pulls both the artifact + image and the SPDK target image. +- The **SPDK image is prebuilt by CI** (`.gitea/workflows/spdk-image.yaml` → + `${SPDK_IMAGE}:${SPDK_VERSION}`); the serving host just pulls it. To roll the + SPDK version, bump `SPDK_VERSION` (or run that workflow manually with a tag) + so the image exists in the registry, then update `.env`. (`dnsmasq` is the + only service still built locally — a tiny alpine image.) + +## 1. Build with a real NVMe discovery address ⚠ required + +The UKI's `rd.nvmf.discover` address is **baked in at build time** (immutable +once signed). The default is the DNS name `nvme.fleet.lab`, which this stack's +dnsmasq resolves to `SERVER_IP`. If you built before `nvmet_traddr` existed, the +UKI still contains the literal `NVMET_TRADDR` placeholder and **will not connect +NVMe** — rebuild: + +```bash +packer build \ + -var image_version= \ + -var disable_dot1x=true \ + -var nvmet_traddr=nvme.fleet.lab \ + -var oci_image=git.bitasylum.org/audiobuzz/fleet-boot -var oci_push=true \ + . +``` + +Keep `nvmet_traddr` (UKI) == `NVME_HOST` (this stack's `.env`). + +## 2. Deploy the serving stack + +```bash +cd serving +cp .env.example .env # edit IMAGE_VERSION, BOOT_IFACE, SERVER_IP, DHCP range +docker login git.bitasylum.org +docker compose up -d --build +docker compose ps +``` + +`IMAGE_VERSION` must match the built tag (e.g. a branch build is +`0.0.0-`; a tag push is the clean version). + +**Verify each service:** + +```bash +# HTTP: UKI is served +curl -sI http://${SERVER_IP}/fleet-${IMAGE_VERSION}.efi | head -1 # 200 OK + +# NVMe/TCP: target is listening + discoverable +ss -lntp | grep 4420 +sudo nvme discover -t tcp -a ${SERVER_IP} -s 4420 # lists the subsystem + +# DNS: discovery name resolves to the serving host +dig +short @${SERVER_IP} nvme.fleet.lab # -> ${SERVER_IP} + +# DHCP/boot: watch leases + bootfile handout +docker compose logs -f dnsmasq +``` + +## 3. Boot a test client + +Create a diskless client VM (or bare metal) on the boot LAN: + +- **UEFI firmware (OVMF)** with **HTTP Boot** enabled. +- **Secure Boot DISABLED** — the lab UKI is **unsigned**; firmware will reject it + otherwise. (Re-enable Secure Boot only once you ship a signed UKI.) +- **No disk** (root comes from NVMe/TCP), NIC on the boot bridge. +- No TPM/swtpm needed in this mode (`rd.dot1x=0` skips 802.1X/MACsec). + +qemu example (adjust bridge/OVMF paths): + +```bash +qemu-system-x86_64 -machine q35 -accel kvm -m 4096 \ + -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE.fd \ + -drive if=pflash,format=raw,file=OVMF_VARS.fd \ + -netdev bridge,id=n0,br= -device virtio-net-pci,netdev=n0 \ + -nographic +``` +(In the OVMF menu enable HTTP Boot / pick the NIC if it doesn't auto-boot.) + +## 4. Expected sequence & success criteria + +1. DHCP lease from dnsmasq; firmware gets option 67 → `http://${SERVER_IP}/fleet-.efi`. +2. Firmware fetches the UKI over HTTP and boots it (no GRUB/shim). +3. initramfs: `rd.dot1x=0` bypass → DHCP on the NIC. +4. `nvme discover` to `nvme.fleet.lab:4420` (→ `SERVER_IP`) → connect namespace. +5. dm-verity maps `/dev/mapper/root`; erofs root mounts read-only. +6. `systemd.volatile=overlay` adds the tmpfs overlay; systemd reaches multi-user. + +**Pass:** the client reaches a login/multi-user state with the root served over +NVMe/TCP. Confirm on the client console: `findmnt /` (erofs, ro), +`nvme list` (the connected namespace), `lsblk` (dm-verity `root`). + +## 5. Troubleshooting + +| Symptom | Likely cause | +|---|---| +| Firmware won't load the UKI | Secure Boot still enabled (UKI is unsigned), or wrong boot file URL | +| No DHCP / no boot menu | client not on the same L2 as `BOOT_IFACE`; dnsmasq not bound to that iface | +| Boots, hangs at NVMe | `nvmet_traddr` ≠ `NVME_HOST`, or UKI still has the `NVMET_TRADDR` placeholder (rebuild) | +| `nvme discover` empty | SPDK not listening (check `docker compose logs nvmeof`; hugepages allocated?) | +| SPDK won't start | hugepages not allocated, or `bdev_aio_create -r` unsupported on the pinned SPDK tag | +| verity mapping fails | rootfs.erofs / verity.json mismatch — must be the same release as the UKI (they are, from one build) | diff --git a/serving/docker-compose.yml b/serving/docker-compose.yml index 15f23a3..1ee5b18 100644 --- a/serving/docker-compose.yml +++ b/serving/docker-compose.yml @@ -90,7 +90,10 @@ services: # --- NVMe/TCP target (serves rootfs.erofs, read-only) ---------------------- nvmeof: - image: spdk/spdk:v24.09 # pin to a tag you've tested + # Prebuilt by CI (.gitea/workflows/spdk-image.yaml) and pulled here — no + # official spdk/spdk image exists, so we compile our own once in the + # pipeline. Local-dev alternative: replace this line with `build: ./spdk`. + image: ${SPDK_IMAGE}:${SPDK_VERSION} network_mode: host # NVMe/TCP listener on ${TARGET_PORT} privileged: true # hugepages + IPC_LOCK/SYS_ADMIN for SPDK restart: unless-stopped diff --git a/serving/spdk/Dockerfile b/serving/spdk/Dockerfile new file mode 100644 index 0000000..1b08f82 --- /dev/null +++ b/serving/spdk/Dockerfile @@ -0,0 +1,39 @@ +# Build SPDK's nvmf_tgt from source. +# +# There is no reliable official `spdk/spdk` image to pull (SPDK's guidance is to +# build your own — https://spdk.io/doc/containers.html), so we compile a pinned +# release here. Single-stage: the image is large and the FIRST build is slow +# (~10-20 min — it pulls submodules incl. DPDK and compiles them), but it's +# self-contained, reproducible, and cached thereafter. Bump SPDK_VERSION to a +# tag from https://github.com/spdk/spdk/releases. +FROM ubuntu:24.04 + +ARG SPDK_VERSION=v24.09 +ARG SPDK_REPO=https://github.com/spdk/spdk + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Shallow-clone the pinned release; submodules at their pinned commits. +RUN git clone --branch "${SPDK_VERSION}" --depth 1 "${SPDK_REPO}" /spdk \ + && git -C /spdk submodule update --init --recursive + +# SPDK's own dependency installer (kernel headers, build tools, DPDK deps, +# python for rpc.py, …). Then configure (TCP nvmf transport is in the default +# build; trim the test artifacts) and build. +RUN /spdk/scripts/pkgdep.sh \ + && cd /spdk \ + && ./configure --disable-tests --disable-unit-tests \ + && make -j"$(nproc)" + +# nvmf_tgt lives in build/bin; rpc.py in scripts/ (kept ON PATH so its relative +# imports resolve from the source tree). spdk-setup.sh locates both via `command +# -v`. LD_LIBRARY_PATH is defensive in case of a shared-lib build. +ENV PATH=/spdk/build/bin:/spdk/scripts:$PATH \ + LD_LIBRARY_PATH=/spdk/build/lib:/spdk/dpdk/build/lib + +# spdk-setup.sh is bind-mounted at /spdk-setup.sh and set as the entrypoint by +# docker-compose.yml.