Initial commit
This commit is contained in:
Audiobuzz
@@ -0,0 +1,36 @@
|
||||
# Copy to .env and edit for your test LAN/bridge, then: docker compose up -d
|
||||
|
||||
# Release being served. Must match the Packer build (fleet-<version>.efi).
|
||||
IMAGE_VERSION=0.1.0
|
||||
|
||||
# OCI release-artifact image (built by scripts/build-oci.sh). Mounted directly
|
||||
# as a Docker image volume — the default artifact source. Pin by digest in
|
||||
# production, e.g. OCI_IMAGE=git.bitasylum.org/audiobuzz/fleet-boot@sha256:…
|
||||
OCI_IMAGE=git.bitasylum.org/audiobuzz/fleet-boot
|
||||
|
||||
# Fallback only (Docker < 28 / offline): local Packer output dir, used by the
|
||||
# commented bind-mounts in docker-compose.yml. Ignored when image volumes work.
|
||||
ARTIFACT_SRC=../artifacts/0.1.0
|
||||
|
||||
# --- This serving host on the boot LAN -------------------------------------
|
||||
BOOT_IFACE=eth0 # interface facing the booting client / test bridge
|
||||
SERVER_IP=192.168.50.10 # this host's address on BOOT_IFACE (DHCP/DNS/HTTP/NVMe)
|
||||
GATEWAY_IP=192.168.50.1
|
||||
UPSTREAM_DNS=1.1.1.1
|
||||
|
||||
# --- DHCP scope ------------------------------------------------------------
|
||||
DHCP_RANGE_START=192.168.50.100
|
||||
DHCP_RANGE_END=192.168.50.200
|
||||
DHCP_LEASE=12h
|
||||
|
||||
# --- DNS names -------------------------------------------------------------
|
||||
BOOT_DOMAIN=fleet.lab
|
||||
BOOT_HOST=boot.fleet.lab # HTTP boot server name (UKI)
|
||||
# NVMe/TCP discovery name. Put THIS value in the UKI cmdline as NVMET_TRADDR
|
||||
# (extract-artifacts.sh) so rd.nvmf.discover resolves here per environment.
|
||||
NVME_HOST=nvme.fleet.lab
|
||||
|
||||
# --- NVMe/TCP target -------------------------------------------------------
|
||||
SUBNQN=nqn.2026-06.org.example.fleet:rootfs
|
||||
TARGET_PORT=4420
|
||||
SPDK_CPUMASK=0x1 # one polling core is plenty for a read-only boot fs
|
||||
@@ -0,0 +1,6 @@
|
||||
# Minimal, reliable dnsmasq image (Hub dnsmasq images vary; build our own so
|
||||
# the version is pinned and the entrypoint is predictable). Config is passed
|
||||
# entirely as --flags from docker-compose.yml.
|
||||
FROM alpine:3.20
|
||||
RUN apk add --no-cache dnsmasq
|
||||
ENTRYPOINT ["dnsmasq"]
|
||||
@@ -0,0 +1,113 @@
|
||||
# =============================================================================
|
||||
# Diskless fleet serving stack (lab)
|
||||
#
|
||||
# Stands up the three things a diskless host needs to network-boot:
|
||||
# dnsmasq - DHCP (hands UEFI HTTP-Boot clients the UKI URL) + DNS
|
||||
# http - nginx serving the signed/unsigned UKI over HTTP
|
||||
# nvmeof - SPDK nvmf_tgt serving rootfs.erofs as a READ-ONLY NVMe/TCP
|
||||
# namespace on 4420 (design doc §5, §7)
|
||||
#
|
||||
# This matches docs/diskless-fleet-design-v2.pdf, with two LAB simplifications:
|
||||
# * HTTP (not HTTPS) boot - no TlsCaCertificate to enrol into firmware
|
||||
# * 802.1X / MACsec disabled - pairs with the rd.dot1x=0 test UKI
|
||||
# Do NOT use as-is for a secured fleet.
|
||||
#
|
||||
# Networking: all three services use the host network stack so dnsmasq can see
|
||||
# DHCP broadcasts and SPDK gets a clean NVMe/TCP path. The Docker host must sit
|
||||
# on the same L2 as the booting client (e.g. the test bridge / boot VLAN), and
|
||||
# BOOT_IFACE / SERVER_IP must be that interface and its address.
|
||||
#
|
||||
# Artifacts are mounted straight from the FROM-scratch OCI release image using
|
||||
# a Docker "image volume" (type: image) — the compose analog of the k8s
|
||||
# image-volume source in the design doc (§7). The image's /artifacts dir is
|
||||
# mounted read-only; nothing in it executes, so FROM scratch is fine. Pin by
|
||||
# digest (OCI_IMAGE@sha256:…) for immutable, content-addressed releases.
|
||||
# Requires Docker Engine >= 28.0 and a recent Compose. On older Docker, use
|
||||
# the commented bind-mount fallback in each service (ARTIFACT_SRC = local
|
||||
# Packer output dir), or extract once with:
|
||||
# id=$(docker create $OCI_IMAGE:$IMAGE_VERSION); \
|
||||
# docker cp $id:/artifacts/. ./artifacts/$IMAGE_VERSION/; docker rm $id
|
||||
# Private registry: docker login git.bitasylum.org first so the image pulls.
|
||||
#
|
||||
# Usage:
|
||||
# cp .env.example .env # then edit BOOT_IFACE / SERVER_IP / DHCP range
|
||||
# docker compose up -d
|
||||
# =============================================================================
|
||||
|
||||
services:
|
||||
# --- DHCP + DNS ------------------------------------------------------------
|
||||
dnsmasq:
|
||||
build: ./dnsmasq
|
||||
network_mode: host
|
||||
cap_add:
|
||||
- NET_ADMIN # DHCP raw sockets
|
||||
- NET_RAW
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- --keep-in-foreground
|
||||
- --log-dhcp
|
||||
- --log-queries
|
||||
# Bind only to the boot interface so we don't fight the host resolver.
|
||||
- --interface=${BOOT_IFACE}
|
||||
- --bind-interfaces
|
||||
- --except-interface=lo
|
||||
# ---- DNS ----
|
||||
- --domain=${BOOT_DOMAIN}
|
||||
- --no-resolv
|
||||
- --server=${UPSTREAM_DNS}
|
||||
# NVMe/TCP discovery address used in the UKI cmdline (rd.nvmf.discover):
|
||||
# resolve the per-environment DNS name to this serving host.
|
||||
- --address=/${NVME_HOST}/${SERVER_IP}
|
||||
- --address=/${BOOT_HOST}/${SERVER_IP}
|
||||
# ---- DHCP ----
|
||||
- --dhcp-authoritative
|
||||
- --dhcp-range=${DHCP_RANGE_START},${DHCP_RANGE_END},${DHCP_LEASE}
|
||||
- --dhcp-option=option:router,${GATEWAY_IP}
|
||||
- --dhcp-option=option:dns-server,${SERVER_IP}
|
||||
# ---- UEFI HTTP Boot ----
|
||||
# Firmware sends vendor-class "HTTPClient"; we must echo it back (opt 60)
|
||||
# and hand over the UKI URL (opt 67). IP in the URL avoids a DNS round-trip
|
||||
# during boot. client-arch 16 = x86-64 UEFI HTTP.
|
||||
- --dhcp-vendorclass=set:httpboot,HTTPClient
|
||||
- --dhcp-option=tag:httpboot,60,HTTPClient
|
||||
- --dhcp-boot=tag:httpboot,http://${SERVER_IP}/fleet-${IMAGE_VERSION}.efi
|
||||
|
||||
# --- HTTP boot server (serves the UKI) -------------------------------------
|
||||
http:
|
||||
image: nginx:stable
|
||||
network_mode: host # listens on :80 -> http://${SERVER_IP}/fleet-<ver>.efi
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- type: image
|
||||
source: ${OCI_IMAGE}:${IMAGE_VERSION}
|
||||
target: /usr/share/nginx/html
|
||||
image:
|
||||
subpath: artifacts # mount only /artifacts from the image
|
||||
# Fallback (Docker < 28 / offline): bind the local Packer output dir:
|
||||
# - ${ARTIFACT_SRC}:/usr/share/nginx/html:ro
|
||||
# For HTTPS boot (production), terminate TLS here with the boot-infra CA's
|
||||
# cert and switch the dhcp-boot URL above to https://. Left HTTP for the lab.
|
||||
|
||||
# --- NVMe/TCP target (serves rootfs.erofs, read-only) ----------------------
|
||||
nvmeof:
|
||||
image: spdk/spdk:v24.09 # pin to a tag you've tested
|
||||
network_mode: host # NVMe/TCP listener on ${TARGET_PORT}
|
||||
privileged: true # hugepages + IPC_LOCK/SYS_ADMIN for SPDK
|
||||
restart: unless-stopped
|
||||
ulimits:
|
||||
memlock: -1
|
||||
volumes:
|
||||
- /dev/hugepages:/dev/hugepages
|
||||
- type: image
|
||||
source: ${OCI_IMAGE}:${IMAGE_VERSION}
|
||||
target: /artifacts # -> /artifacts/rootfs.erofs for spdk-setup.sh
|
||||
image:
|
||||
subpath: artifacts
|
||||
# Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/artifacts:ro
|
||||
- ./spdk-setup.sh:/spdk-setup.sh:ro
|
||||
environment:
|
||||
SUBNQN: ${SUBNQN}
|
||||
TARGET_ADDR: ${SERVER_IP}
|
||||
TARGET_PORT: ${TARGET_PORT}
|
||||
SPDK_CPUMASK: ${SPDK_CPUMASK}
|
||||
entrypoint: ["/bin/bash", "/spdk-setup.sh"]
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
# Configure SPDK nvmf_tgt to serve the diskless root filesystem as a single
|
||||
# READ-ONLY NVMe/TCP namespace (design doc §5, §7).
|
||||
#
|
||||
# rootfs.erofs is the erofs image with the dm-verity hash tree appended in-file;
|
||||
# we expose the WHOLE file as one namespace, and the host splits data/hash with
|
||||
# systemd.verity_root_options=hash-offset= from the kernel cmdline. Integrity is
|
||||
# enforced end-to-end by dm-verity on the host; the target serves it read-only.
|
||||
set -euo pipefail
|
||||
|
||||
ROOTFS="/artifacts/rootfs.erofs"
|
||||
SUBNQN="${SUBNQN:-nqn.2026-06.org.example.fleet:rootfs}"
|
||||
TADDR="${TARGET_ADDR:-0.0.0.0}"
|
||||
TPORT="${TARGET_PORT:-4420}"
|
||||
CPUMASK="${SPDK_CPUMASK:-0x1}"
|
||||
|
||||
[ -s "$ROOTFS" ] || { echo "ERROR: $ROOTFS not found/empty — is ARTIFACT_SRC populated?" >&2; exit 1; }
|
||||
|
||||
# Binary locations differ across SPDK images; fall back to the common path.
|
||||
TGT=$(command -v nvmf_tgt || echo /usr/libexec/spdk/spdk/build/bin/nvmf_tgt)
|
||||
RPC=$(command -v rpc.py || echo /usr/libexec/spdk/scripts/rpc.py)
|
||||
|
||||
echo "==> starting nvmf_tgt (cpumask ${CPUMASK})"
|
||||
"$TGT" -m "$CPUMASK" &
|
||||
TGT_PID=$!
|
||||
trap 'kill "$TGT_PID" 2>/dev/null || true' EXIT
|
||||
|
||||
# Wait for the RPC socket to come up.
|
||||
for _ in $(seq 1 30); do
|
||||
"$RPC" framework_get_config >/dev/null 2>&1 && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "==> creating TCP transport"
|
||||
"$RPC" nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192
|
||||
|
||||
echo "==> exposing $ROOTFS as bdev 'rootfs' (4 KiB blocks)"
|
||||
# -r opens the backing file read-only where supported; harmless if ignored.
|
||||
"$RPC" bdev_aio_create "$ROOTFS" rootfs 4096 -r \
|
||||
|| "$RPC" bdev_aio_create "$ROOTFS" rootfs 4096
|
||||
|
||||
echo "==> creating subsystem $SUBNQN (allow any host — lab/no-auth)"
|
||||
"$RPC" nvmf_create_subsystem "$SUBNQN" -a -s SPDKLAB0000000001 -d FleetRootfs
|
||||
|
||||
echo "==> adding namespace 1 (read-only)"
|
||||
# Newer SPDK accepts --read-only on the namespace; fall back if unsupported.
|
||||
"$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs --read-only \
|
||||
|| "$RPC" nvmf_subsystem_add_ns "$SUBNQN" rootfs
|
||||
|
||||
echo "==> adding TCP listener ${TADDR}:${TPORT} (also serves discovery)"
|
||||
"$RPC" nvmf_subsystem_add_listener "$SUBNQN" -t tcp -a "$TADDR" -s "$TPORT"
|
||||
|
||||
echo "==> SPDK serving $SUBNQN over tcp/${TPORT}; rd.nvmf.discover=tcp,${TADDR},${TPORT}"
|
||||
wait "$TGT_PID"
|
||||
Reference in New Issue
Block a user