Add SPDDK build

2026-06-14 05:58:55 +00:00
parent e98d8813cf
commit 62e997297c
9 changed files with 273 additions and 6 deletions
@@ -0,0 +1,77 @@
+#
+# .gitea/workflows/spdk-image.yaml
+#
+# Builds the SPDK NVMe/TCP target image (serving stack) from serving/spdk and
+# pushes it to the registry, so serving hosts just `docker compose pull` it.
+# There is no official spdk/spdk image to pull, so we compile a pinned release
+# once here (slow: ~10-20 min, needs a runner with enough CPU/RAM/disk).
+#
+# Runs only when the SPDK image inputs change (path filter) or on demand — not
+# on every commit. Bump SPDK_VERSION (or use the manual run input) to roll SPDK.
+#
+# Required Gitea repo secrets: PACKAGE_USER, PACKAGE_TOKEN (registry creds).
+#
+
+name: Build SPDK NVMe-oF image
+run-name: ${{ gitea.actor }} builds the SPDK NVMe/TCP target image
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - "serving/spdk/**"
+      - ".gitea/workflows/spdk-image.yaml"
+  workflow_dispatch:
+    inputs:
+      spdk_version:
+        description: "SPDK release tag (https://github.com/spdk/spdk/releases)"
+        required: false
+        default: "v24.09"
+
+env:
+  SPDK_IMAGE: "git.bitasylum.org/audiobuzz/spdk-nvmf"
+  SPDK_VERSION: "v24.09"   # default tag for push-triggered builds
+
+jobs:
+  spdk-build:
+    name: Build & push SPDK image
+    runs-on: ubuntu-latest
+    container:
+      image: git.bitasylum.org/audiobuzz/ubuntu-act-runner-qemu:main
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v6
+
+      -
+        name: Install buildah
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends buildah
+
+      -
+        name: Registry login
+        run: |
+          buildah login \
+            --username "${{ secrets.PACKAGE_USER }}" \
+            --password "${{ secrets.PACKAGE_TOKEN }}" \
+            git.bitasylum.org
+
+      -
+        name: Build & push
+        # vfs storage (overlay-on-overlayfs is unsupported in-container) and
+        # chroot isolation (so the Dockerfile's RUN/compile steps work without
+        # nested user namespaces).
+        env:
+          STORAGE_DRIVER: vfs
+          BUILDAH_ISOLATION: chroot
+        run: |
+          VER="${{ inputs.spdk_version || env.SPDK_VERSION }}"
+          echo "Building ${SPDK_IMAGE}:${VER}"
+          buildah bud \
+            --build-arg SPDK_VERSION="$VER" \
+            -t "${SPDK_IMAGE}:${VER}" \
+            -t "${SPDK_IMAGE}:latest" \
+            serving/spdk
+          buildah push "${SPDK_IMAGE}:${VER}"
+          buildah push "${SPDK_IMAGE}:latest"
@@ -67,9 +67,11 @@ packer build \
 ```

 Omit `sb_key`/`sb_cert` for unsigned lab builds (the pipeline warns).
-Substitute the real NVMe target address for `NVMET_TRADDR` in
-`scripts/extract-artifacts.sh` (or post-edit `cmdline.txt`) **before** the
-OCI step: the cmdline is immutable once signed into the UKI.
+Set the NVMe discovery address with `-var nvmet_traddr=<dns-name>` (default
+`nvme.fleet.lab`, which the `serving/` stack's dnsmasq resolves); it is baked
+into the cmdline and **immutable once signed**, so prefer a per-environment DNS
+name over a raw IP. Deploy/boot-test instructions live in
+[serving/README.md](serving/README.md).

 **Test mode (auth disabled).** Add `-var disable_dot1x=true` to bake
 `rd.dot1x=0` into the cmdline. At boot the hook then skips 802.1X/MACsec and
@@ -94,6 +94,12 @@ variable "disable_dot1x" {
  default     = false
 }

+variable "nvmet_traddr" {
+  type        = string
+  description = "NVMe/TCP discovery address baked into the UKI cmdline (rd.nvmf.discover). Use a per-environment DNS name resolved at boot so one UKI works everywhere; the lab serving stack's dnsmasq resolves this name to the serving host. Immutable once the UKI is signed."
+  default     = "nvme.fleet.lab"
+}
+
 variable "sb_key" {
  type        = string
  description = "Secure Boot signing key (PEM) for the UKI. Empty = unsigned lab build. Production: CI/HSM-held; never enters the build VM."
@@ -222,7 +228,7 @@ build {
  # --- 4. Assemble + sign the UKI; build the OCI release-artifact image ----
  post-processor "shell-local" {
    inline = [
-      "bash scripts/extract-artifacts.sh output-qemu/diskless-build.qcow2 ${var.artifact_dir}/${var.image_version} ${var.disable_dot1x}",
+      "NVMET_TRADDR='${var.nvmet_traddr}' bash scripts/extract-artifacts.sh output-qemu/diskless-build.qcow2 ${var.artifact_dir}/${var.image_version} ${var.disable_dot1x}",
      "bash scripts/build-oci.sh ${var.artifact_dir}/${var.image_version} ${var.oci_image} ${var.image_version} '${var.sb_key}' '${var.sb_cert}' ${var.oci_push}",
    ]
  }
@@ -98,3 +98,18 @@ is the tmpfs overlay), fleet-generic (no per-host state).
  browser are STIG findings and enlarge the attack surface/image. Reconcile —
  e.g. a tightly-constrained kiosk session + documented deviation — when
  hardening lands.
+
+## [~] 5. Deploy & validate the end-to-end boot path (open-network test)
+
+In progress. Stand up `serving/` (dnsmasq + nginx + SPDK) per
+[serving/README.md](../serving/README.md), rebuild the UKI with
+`nvmet_traddr=nvme.fleet.lab` (now a Packer var — older UKIs have the literal
+`NVMET_TRADDR` placeholder and won't connect), then boot a UEFI HTTP-boot client
+(**Secure Boot OFF**, unsigned UKI; no TPM needed in `rd.dot1x=0` mode) on the
+boot LAN.
+
+- **Pass criteria:** UKI fetched over HTTP → initramfs → DHCP on NIC → NVMe/TCP
+  namespace connected → dm-verity root mapped → `systemd.volatile=overlay`
+  writable → multi-user.
+- This validation **gates** item #1 (network-manager port) and item #3
+  (errata → design v3).
@@ -18,6 +18,11 @@ OUT=${2:?usage: extract-artifacts.sh <qcow2> <output-dir> [disable_dot1x]}
 # TEMPORARY test toggle: when "true", bake rd.dot1x=0 into the cmdline so the
 # UKI boots with 802.1X + MACsec disabled (open-port NVMe-oF boot test).
 DISABLE_DOT1X=${3:-false}
+# NVMe/TCP discovery address baked into the (immutable, signed) UKI cmdline.
+# Prefer a per-environment DNS name resolved at boot so one UKI works across
+# environments. Defaults to the literal placeholder so build-oci.sh's warning
+# fires if it was never set.
+NVMET_TRADDR=${NVMET_TRADDR:-NVMET_TRADDR}

 for bin in virt-copy-out virt-tar-out virt-ls mkfs.erofs veritysetup jq; do
    command -v "$bin" >/dev/null || { echo "ERROR: $bin not found on build host" >&2; exit 1; }
@@ -83,7 +88,7 @@ printf '%s' \
  "$DOT1X_FRAGMENT" \
  "rd.neednet=1 systemd.volatile=overlay " \
  "rd.dot1x.nvindex=0x01800001 rd.dot1x.timeout=60 " \
-  "rd.nvmf.discover=tcp,NVMET_TRADDR,4420 " \
+  "rd.nvmf.discover=tcp,${NVMET_TRADDR},4420 " \
  "root=/dev/mapper/root rootfstype=erofs rootflags=ro " \
  "systemd.verity=yes roothash=${ROOT_HASH} " \
  "systemd.verity_root_data=/dev/nvme0n1 " \
@@ -12,6 +12,11 @@ OCI_IMAGE=git.bitasylum.org/audiobuzz/fleet-boot
 # commented bind-mounts in docker-compose.yml. Ignored when image volumes work.
 ARTIFACT_SRC=../artifacts/0.1.0

+# SPDK NVMe/TCP target image — prebuilt + pushed by CI
+# (.gitea/workflows/spdk-image.yaml); the serving host just pulls it.
+SPDK_IMAGE=git.bitasylum.org/audiobuzz/spdk-nvmf
+SPDK_VERSION=v24.09
+
 # --- This serving host on the boot LAN -------------------------------------
 BOOT_IFACE=eth0            # interface facing the booting client / test bridge
 SERVER_IP=192.168.50.10    # this host's address on BOOT_IFACE (DHCP/DNS/HTTP/NVMe)
@@ -0,0 +1,115 @@
+# Serving stack — deploy & test (lab)
+
+Stands up DHCP/DNS + HTTP(UKI) + NVMe/TCP(rootfs) so a diskless client can
+network-boot the release built by the pipeline, and walks through testing it.
+This targets the **open-network test path** (`rd.dot1x=0`, unsigned UKI) — see
+the repo README for the secure path.
+
+## 0. Prerequisites
+
+- A serving host **on the same L2 as the test client** (the boot VLAN / test
+  bridge). Note its interface (`BOOT_IFACE`) and address (`SERVER_IP`).
+- **Docker Engine ≥ 28** (for the `type: image` artifact volume; otherwise use
+  the commented bind-mount fallback in `docker-compose.yml`).
+- Hugepages for SPDK: `sudo sysctl vm.nr_hugepages=1024`.
+- Registry creds (`docker login git.bitasylum.org`) — pulls both the artifact
+  image and the SPDK target image.
+- The **SPDK image is prebuilt by CI** (`.gitea/workflows/spdk-image.yaml` →
+  `${SPDK_IMAGE}:${SPDK_VERSION}`); the serving host just pulls it. To roll the
+  SPDK version, bump `SPDK_VERSION` (or run that workflow manually with a tag)
+  so the image exists in the registry, then update `.env`. (`dnsmasq` is the
+  only service still built locally — a tiny alpine image.)
+
+## 1. Build with a real NVMe discovery address  ⚠ required
+
+The UKI's `rd.nvmf.discover` address is **baked in at build time** (immutable
+once signed). The default is the DNS name `nvme.fleet.lab`, which this stack's
+dnsmasq resolves to `SERVER_IP`. If you built before `nvmet_traddr` existed, the
+UKI still contains the literal `NVMET_TRADDR` placeholder and **will not connect
+NVMe** — rebuild:
+
+```bash
+packer build \
+  -var image_version=<ver> \
+  -var disable_dot1x=true \
+  -var nvmet_traddr=nvme.fleet.lab \
+  -var oci_image=git.bitasylum.org/audiobuzz/fleet-boot -var oci_push=true \
+  .
+```
+
+Keep `nvmet_traddr` (UKI) == `NVME_HOST` (this stack's `.env`).
+
+## 2. Deploy the serving stack
+
+```bash
+cd serving
+cp .env.example .env      # edit IMAGE_VERSION, BOOT_IFACE, SERVER_IP, DHCP range
+docker login git.bitasylum.org
+docker compose up -d --build
+docker compose ps
+```
+
+`IMAGE_VERSION` must match the built tag (e.g. a branch build is
+`0.0.0-<short-sha>`; a tag push is the clean version).
+
+**Verify each service:**
+
+```bash
+# HTTP: UKI is served
+curl -sI http://${SERVER_IP}/fleet-${IMAGE_VERSION}.efi | head -1   # 200 OK
+
+# NVMe/TCP: target is listening + discoverable
+ss -lntp | grep 4420
+sudo nvme discover -t tcp -a ${SERVER_IP} -s 4420                   # lists the subsystem
+
+# DNS: discovery name resolves to the serving host
+dig +short @${SERVER_IP} nvme.fleet.lab                             # -> ${SERVER_IP}
+
+# DHCP/boot: watch leases + bootfile handout
+docker compose logs -f dnsmasq
+```
+
+## 3. Boot a test client
+
+Create a diskless client VM (or bare metal) on the boot LAN:
+
+- **UEFI firmware (OVMF)** with **HTTP Boot** enabled.
+- **Secure Boot DISABLED** — the lab UKI is **unsigned**; firmware will reject it
+  otherwise. (Re-enable Secure Boot only once you ship a signed UKI.)
+- **No disk** (root comes from NVMe/TCP), NIC on the boot bridge.
+- No TPM/swtpm needed in this mode (`rd.dot1x=0` skips 802.1X/MACsec).
+
+qemu example (adjust bridge/OVMF paths):
+
+```bash
+qemu-system-x86_64 -machine q35 -accel kvm -m 4096 \
+  -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE.fd \
+  -drive if=pflash,format=raw,file=OVMF_VARS.fd \
+  -netdev bridge,id=n0,br=<boot-bridge> -device virtio-net-pci,netdev=n0 \
+  -nographic
+```
+(In the OVMF menu enable HTTP Boot / pick the NIC if it doesn't auto-boot.)
+
+## 4. Expected sequence & success criteria
+
+1. DHCP lease from dnsmasq; firmware gets option 67 → `http://${SERVER_IP}/fleet-<ver>.efi`.
+2. Firmware fetches the UKI over HTTP and boots it (no GRUB/shim).
+3. initramfs: `rd.dot1x=0` bypass → DHCP on the NIC.
+4. `nvme discover` to `nvme.fleet.lab:4420` (→ `SERVER_IP`) → connect namespace.
+5. dm-verity maps `/dev/mapper/root`; erofs root mounts read-only.
+6. `systemd.volatile=overlay` adds the tmpfs overlay; systemd reaches multi-user.
+
+**Pass:** the client reaches a login/multi-user state with the root served over
+NVMe/TCP. Confirm on the client console: `findmnt /` (erofs, ro),
+`nvme list` (the connected namespace), `lsblk` (dm-verity `root`).
+
+## 5. Troubleshooting
+
+| Symptom | Likely cause |
+|---|---|
+| Firmware won't load the UKI | Secure Boot still enabled (UKI is unsigned), or wrong boot file URL |
+| No DHCP / no boot menu | client not on the same L2 as `BOOT_IFACE`; dnsmasq not bound to that iface |
+| Boots, hangs at NVMe | `nvmet_traddr` ≠ `NVME_HOST`, or UKI still has the `NVMET_TRADDR` placeholder (rebuild) |
+| `nvme discover` empty | SPDK not listening (check `docker compose logs nvmeof`; hugepages allocated?) |
+| SPDK won't start | hugepages not allocated, or `bdev_aio_create -r` unsupported on the pinned SPDK tag |
+| verity mapping fails | rootfs.erofs / verity.json mismatch — must be the same release as the UKI (they are, from one build) |
@@ -90,7 +90,10 @@ services:

  # --- NVMe/TCP target (serves rootfs.erofs, read-only) ----------------------
  nvmeof:
-    image: spdk/spdk:v24.09   # pin to a tag you've tested
+    # Prebuilt by CI (.gitea/workflows/spdk-image.yaml) and pulled here — no
+    # official spdk/spdk image exists, so we compile our own once in the
+    # pipeline. Local-dev alternative: replace this line with `build: ./spdk`.
+    image: ${SPDK_IMAGE}:${SPDK_VERSION}
    network_mode: host        # NVMe/TCP listener on ${TARGET_PORT}
    privileged: true          # hugepages + IPC_LOCK/SYS_ADMIN for SPDK
    restart: unless-stopped
@@ -0,0 +1,39 @@
+# Build SPDK's nvmf_tgt from source.
+#
+# There is no reliable official `spdk/spdk` image to pull (SPDK's guidance is to
+# build your own — https://spdk.io/doc/containers.html), so we compile a pinned
+# release here. Single-stage: the image is large and the FIRST build is slow
+# (~10-20 min — it pulls submodules incl. DPDK and compiles them), but it's
+# self-contained, reproducible, and cached thereafter. Bump SPDK_VERSION to a
+# tag from https://github.com/spdk/spdk/releases.
+FROM ubuntu:24.04
+
+ARG SPDK_VERSION=v24.09
+ARG SPDK_REPO=https://github.com/spdk/spdk
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends git ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+# Shallow-clone the pinned release; submodules at their pinned commits.
+RUN git clone --branch "${SPDK_VERSION}" --depth 1 "${SPDK_REPO}" /spdk \
+ && git -C /spdk submodule update --init --recursive
+
+# SPDK's own dependency installer (kernel headers, build tools, DPDK deps,
+# python for rpc.py, …). Then configure (TCP nvmf transport is in the default
+# build; trim the test artifacts) and build.
+RUN /spdk/scripts/pkgdep.sh \
+ && cd /spdk \
+ && ./configure --disable-tests --disable-unit-tests \
+ && make -j"$(nproc)"
+
+# nvmf_tgt lives in build/bin; rpc.py in scripts/ (kept ON PATH so its relative
+# imports resolve from the source tree). spdk-setup.sh locates both via `command
+# -v`. LD_LIBRARY_PATH is defensive in case of a shared-lib build.
+ENV PATH=/spdk/build/bin:/spdk/scripts:$PATH \
+    LD_LIBRARY_PATH=/spdk/build/lib:/spdk/dpdk/build/lib
+
+# spdk-setup.sh is bind-mounted at /spdk-setup.sh and set as the entrypoint by
+# docker-compose.yml.