287 lines
16 KiB
YAML
287 lines
16 KiB
YAML
# =============================================================================
|
|
# Diskless fleet serving stack (lab)
|
|
#
|
|
# Stands up the three things a diskless host needs to network-boot:
|
|
# dnsmasq - DHCP (hands UEFI HTTP-Boot clients the UKI URL) + DNS
|
|
# http - nginx serving the signed/unsigned UKI over HTTP
|
|
# nvmeof - SPDK nvmf_tgt serving rootfs.erofs as a READ-ONLY NVMe/TCP
|
|
# namespace on 4420 (design doc §5, §7)
|
|
#
|
|
# This matches docs/diskless-fleet-design-v2.pdf, with two LAB simplifications:
|
|
# * HTTP (not HTTPS) boot - no TlsCaCertificate to enrol into firmware
|
|
# * 802.1X / MACsec disabled - pairs with the rd.dot1x=0 test UKI
|
|
# Do NOT use as-is for a secured fleet.
|
|
#
|
|
# Networking: all three services use the host network stack so dnsmasq can see
|
|
# DHCP broadcasts and SPDK gets a clean NVMe/TCP path. The Docker host must sit
|
|
# on the same L2 as the booting client (e.g. the test bridge / boot VLAN), and
|
|
# BOOT_IFACE / SERVER_IP must be that interface and its address.
|
|
#
|
|
# Artifacts are mounted straight from the FROM-scratch OCI release image using
|
|
# a Docker "image volume" (type: image) — the compose analog of the k8s
|
|
# image-volume source in the design doc (§7). The image's /artifacts dir is
|
|
# mounted read-only; nothing in it executes, so FROM scratch is fine. Pin by
|
|
# digest (OCI_IMAGE@sha256:…) for immutable, content-addressed releases.
|
|
# Requires Docker Engine >= 28.0 and a recent Compose. On older Docker, use
|
|
# the commented bind-mount fallback in each service (ARTIFACT_SRC = local
|
|
# Packer output dir), or extract once with:
|
|
# id=$(docker create $OCI_IMAGE:$IMAGE_VERSION); \
|
|
# docker cp $id:/artifacts/. ./artifacts/$IMAGE_VERSION/; docker rm $id
|
|
# Private registry: docker login git.bitasylum.org first so the image pulls.
|
|
#
|
|
# Usage:
|
|
# cp .env.example .env # then edit BOOT_IFACE / SERVER_IP / DHCP range
|
|
# docker compose up -d
|
|
# =============================================================================
|
|
|
|
services:
|
|
# --- DHCP + DNS ------------------------------------------------------------
|
|
dnsmasq:
|
|
# Prebuilt by CI (.gitea/workflows/dnsmasq-image.yaml) and pulled here, like
|
|
# the SPDK image — the serving host no longer builds it at deploy time. The
|
|
# image is GENERIC (no Secure Boot keys, no iPXE binary): the signed iPXE
|
|
# bootloader rides the separate, per-environment fleet-ipxe image mounted at
|
|
# /ipxe below. Local-dev alternative: replace this line with `build: ./dnsmasq`.
|
|
image: ${DNSMASQ_IMAGE}:${DNSMASQ_VERSION}
|
|
network_mode: host
|
|
cap_add:
|
|
- NET_ADMIN # DHCP raw sockets
|
|
- NET_RAW
|
|
restart: unless-stopped
|
|
volumes:
|
|
# DHCP reservations (MAC -> name [-> IP]); drives DHCP option 12 (hostname)
|
|
# + DNS A/PTR. Auto-populated by fleet-namer (creative names per MAC); add
|
|
# manual lines to pin specific hosts. RW so fleet-namer can append.
|
|
- ./dnsmasq/dhcp-hosts:/etc/dnsmasq-hosts
|
|
# Signed iPXE bootloader for the PXE-only chainload (firmware without UEFI
|
|
# HTTP boot, e.g. EliteBook 820 G4). Mounted from the separate, signed,
|
|
# digest-pinnable fleet-ipxe image (.gitea/workflows/ipxe-image.yaml); the
|
|
# entrypoint copies /ipxe/snponly.efi into /srv/tftp. Kept OUT of the dnsmasq
|
|
# image so the daemon image stays generic/key-agnostic — see serving/tftp/README.
|
|
- type: image
|
|
source: ${IPXE_IMAGE}:${IPXE_VERSION}
|
|
target: /ipxe
|
|
image:
|
|
subpath: ipxe
|
|
# Fallback (Docker < 28 / offline / lab): drop snponly.efi in ./tftp and bind
|
|
# it instead (the entrypoint prefers /ipxe, then falls back to /tftp-src):
|
|
# - ./tftp:/tftp-src:ro
|
|
# The release artifacts (same image volume as the http service), read-only,
|
|
# so the entrypoint can objcopy the UKI's .cmdline section into boot.ipxe
|
|
# (Secure Boot OFF needs the real cmdline fed to iPXE — see entrypoint.sh).
|
|
- type: image
|
|
source: ${OCI_IMAGE}:${IMAGE_VERSION}
|
|
target: /artifacts
|
|
image:
|
|
subpath: artifacts
|
|
# Fallback (Docker < 28 / offline): bind the local Packer output dir instead:
|
|
# - ${ARTIFACT_SRC}:/artifacts:ro
|
|
environment:
|
|
# Consumed by the entrypoint to render /srv/tftp/boot.ipxe (the iPXE chain
|
|
# script that clears load options so the UKI's embedded cmdline survives).
|
|
- SERVER_IP=${SERVER_IP}
|
|
- IMAGE_VERSION=${IMAGE_VERSION}
|
|
command:
|
|
- --keep-in-foreground
|
|
# Log to stderr so `docker compose logs` captures output. Without this,
|
|
# dnsmasq logs to syslog — which doesn't exist in the container — so
|
|
# --log-dhcp/--log-queries produce nothing even while dnsmasq works fine.
|
|
- --log-facility=-
|
|
- --log-dhcp
|
|
- --log-queries
|
|
# Bind only to the boot interface so we don't fight the host resolver.
|
|
- --interface=${BOOT_IFACE}
|
|
- --bind-interfaces
|
|
- --except-interface=lo
|
|
# ---- DNS ----
|
|
- --domain=${BOOT_DOMAIN}
|
|
- --no-resolv
|
|
- --server=${UPSTREAM_DNS}
|
|
# NVMe/TCP discovery address used in the UKI cmdline (rd.nvmf.discover):
|
|
# resolve the per-environment DNS name to this serving host.
|
|
- --address=/${NVME_HOST}/${SERVER_IP}
|
|
- --address=/${BOOT_HOST}/${SERVER_IP}
|
|
# Host-generic remote audit collector name (baked into the image config so
|
|
# every host offloads auditd here; resolved per-environment to this host).
|
|
- --address=/${AUDIT_HOST}/${SERVER_IP}
|
|
# Kiosk is LOCAL-ONLY (loads boot.fleet.lab), so sinkhole Chromium's
|
|
# call-home to 0.0.0.0 — telemetry can't leave the lab even if the browser
|
|
# tries. Defense-in-depth behind the in-image Chromium policy/flags; in
|
|
# production this belongs in the boot-LAN firewall's egress rules. NOTE: the
|
|
# client still *emits* the query (you'll see it in this log answered locally,
|
|
# not "forwarded"); stopping the query itself is the browser policy's job.
|
|
- --address=/google.com/0.0.0.0
|
|
- --address=/googleapis.com/0.0.0.0
|
|
- --address=/gstatic.com/0.0.0.0
|
|
# ---- DHCP ----
|
|
- --dhcp-authoritative
|
|
- --dhcp-range=${DHCP_RANGE_START},${DHCP_RANGE_END},${DHCP_LEASE}
|
|
- --dhcp-option=option:router,${GATEWAY_IP}
|
|
- --dhcp-option=option:dns-server,${SERVER_IP}
|
|
# NTP server (option 42): point diskless hosts at the lab 'ntp' service
|
|
# below so they sync time internally and never reach the public NTP pool.
|
|
# The image's NM dispatcher (10-chrony-dhcp-ntp) feeds this into chrony.
|
|
- --dhcp-option=option:ntp-server,${SERVER_IP}
|
|
# ---- Per-host names (creative starship hostnames, server-assigned) ----
|
|
# The workstation image is hostname-neutral; naming is done HERE. fleet-namer
|
|
# (the --dhcp-script) derives a stable themed name per MAC (e.g.
|
|
# nebula-vanguard-07), appends it to the hostsfile, and SIGHUPs dnsmasq;
|
|
# dnsmasq then sends it as DHCP option 12 (-> NetworkManager sets the
|
|
# hostname) and registers DNS A/PTR under ${BOOT_DOMAIN}. The host picks up
|
|
# its name on the NEXT DHCP transaction (renewal at ~half lease, or reboot)
|
|
# — shorten DHCP_LEASE if you want naming to settle faster on first contact.
|
|
# --user=root so the dhcp-script can write the hostsfile + signal dnsmasq.
|
|
- --user=root
|
|
- --dhcp-hostsfile=/etc/dnsmasq-hosts
|
|
- --dhcp-script=/usr/local/bin/fleet-namer
|
|
# ---- UEFI HTTP Boot ----
|
|
# Firmware sends vendor-class "HTTPClient"; we must echo it back (opt 60)
|
|
# and hand over the UKI URL (opt 67). IP in the URL avoids a DNS round-trip
|
|
# during boot. client-arch 16 = x86-64 UEFI HTTP.
|
|
- --dhcp-vendorclass=set:httpboot,HTTPClient
|
|
- --dhcp-option=tag:httpboot,60,HTTPClient
|
|
- --dhcp-boot=tag:httpboot,http://${SERVER_IP}/fleet-${IMAGE_VERSION}.efi
|
|
# ---- PXE -> iPXE -> HTTP chainload (for firmware WITHOUT UEFI HTTP boot) ----
|
|
# Graceful fallback: HTTP-capable firmware matches 'httpboot' above and gets
|
|
# the UKI directly; PXE-only firmware (PXEClient) gets ipxe.efi over TFTP;
|
|
# once iPXE is running it re-DHCPs with user-class "iPXE" and is handed the
|
|
# SAME UKI HTTP URL, which it chainloads (.efi -> LoadImage). Each client
|
|
# gets the right thing based on what it advertises — no per-host config.
|
|
# snponly.efi = UEFI x64 iPXE that rides the FIRMWARE's NIC driver (UEFI SNP)
|
|
# instead of iPXE's own — works on virtio VMs AND real hardware where the
|
|
# native driver won't bind (e.g. EliteBook 820 G4 / Intel I219-LM, which
|
|
# fails ipxe.efi with "No configurable methods"). Built + signed by CI into
|
|
# the fleet-ipxe image (mounted at /ipxe); see serving/tftp/README.md.
|
|
# The fleet-ipxe image is signed with the fleet db key when CI has the
|
|
# secrets, so this path stays Secure-Boot-enforced end-to-end; an unsigned
|
|
# (lab) fleet-ipxe build needs Secure Boot OFF — see docs/secure-boot-keys.md.
|
|
- --enable-tftp
|
|
- --tftp-root=/srv/tftp
|
|
- --dhcp-vendorclass=set:pxe,PXEClient
|
|
- --dhcp-userclass=set:ipxe,iPXE
|
|
# firmware PXE, not yet iPXE -> hand iPXE (UEFI x64) over TFTP
|
|
- --dhcp-boot=tag:pxe,tag:!ipxe,snponly.efi
|
|
# iPXE running -> hand boot.ipxe (NOT the UKI URL directly). The script
|
|
# imgfetch'es the UKI then `imgargs`-clears the load options before boot, so
|
|
# systemd-stub uses the UKI's EMBEDDED cmdline. Booting the UKI URL directly
|
|
# makes iPXE pass the image name as the cmdline, which (Secure Boot OFF)
|
|
# REPLACES the embedded roothash/nvmf/... args -> dracut emergency shell.
|
|
- --dhcp-boot=tag:ipxe,boot.ipxe
|
|
|
|
# --- HTTP boot server (serves the UKI) -------------------------------------
|
|
http:
|
|
image: harbor.bitasylum.org/dockerhub/library/nginx:stable
|
|
network_mode: host # listens on :80 -> http://${SERVER_IP}/fleet-<ver>.efi
|
|
restart: unless-stopped
|
|
depends_on:
|
|
- kiosk # /kiosk/ is reverse-proxied to the kiosk SPA
|
|
volumes:
|
|
- type: image
|
|
source: ${OCI_IMAGE}:${IMAGE_VERSION}
|
|
target: /usr/share/nginx/html
|
|
image:
|
|
subpath: artifacts # mount only /artifacts from the image
|
|
# Fallback (Docker < 28 / offline): bind the local Packer output dir:
|
|
# - ${ARTIFACT_SRC}:/usr/share/nginx/html:ro
|
|
# Config adds the artifact root + the /kiosk/ reverse-proxy location. The
|
|
# kiosk SPA itself is the separate 'kiosk' service below (not bind-mounted).
|
|
- ./http/default.conf:/etc/nginx/conf.d/default.conf:ro
|
|
# For HTTPS boot (production), terminate TLS here with the boot-infra CA's
|
|
# cert and switch the dhcp-boot URL above to https://. Left HTTP for the lab.
|
|
|
|
# --- Kiosk SPA (spaceship terminal shown by the diskless kiosk) ------------
|
|
# Svelte + Tailwind app built into a static bundle served by its own nginx.
|
|
# Now an INDEPENDENT project — the `kiosk-app` git submodule (its own repo +
|
|
# CI). It builds + pushes its own image (git.../fleet-kiosk); the serving stack
|
|
# pulls it, like dnsmasq/SPDK. Bridge-networked (NOT host) so it doesn't fight
|
|
# :80; published on loopback only, where the host-network 'http' nginx
|
|
# reverse-proxies /kiosk/ to it.
|
|
kiosk:
|
|
image: ${KIOSK_IMAGE}:${KIOSK_VERSION}
|
|
# Local dev from the submodule working tree instead of pulling:
|
|
# build: ./kiosk-app
|
|
restart: unless-stopped
|
|
ports:
|
|
- "127.0.0.1:${KIOSK_APP_PORT:-8088}:80" # loopback only; main nginx proxies here
|
|
environment:
|
|
HASS_URL: ${HASS_URL}
|
|
HASS_TOKEN: ${HASS_TOKEN}
|
|
# --- NTP server (serves time to the boot LAN; handed out via DHCP opt 42) ---
|
|
ntp:
|
|
build: ./chrony
|
|
network_mode: host # serve NTP on udp/123 across the boot LAN
|
|
cap_add:
|
|
- SYS_TIME # discipline the (host-shared) clock it serves
|
|
restart: unless-stopped
|
|
volumes:
|
|
- ./chrony/chrony.conf:/etc/chrony/chrony.conf:ro
|
|
# NOTE: with host networking + SYS_TIME this chronyd disciplines the Docker
|
|
# HOST clock (the clock isn't namespaced) — intended: the serving host is the
|
|
# lab time source. Drop SYS_TIME (and add `chronyd -x`) if the host already
|
|
# runs its own NTP and you only want this to relay.
|
|
|
|
# --- Remote audit / log collector (RECEIVE side) ---------------------------
|
|
# rsyslog receiver: booted diskless hosts (whose RAM-only overlay root loses
|
|
# local logs on reboot) forward auditd via audisp-syslog + syslog here over
|
|
# TCP/514. Records are written per-source-host under
|
|
# ./audit-collector/logs/<sender-ip>/{audit,messages}.log. Host-networked so
|
|
# hosts can reach it at ${AUDIT_HOST} (= this host) on the boot LAN.
|
|
# NOTE: this is the receiver only — the IMAGE must be configured to send
|
|
# (audisp-syslog + an rsyslog forward rule targeting ${AUDIT_HOST}); see
|
|
# audit-collector/README.md. Until then the collector simply waits, idle.
|
|
audit:
|
|
build: ./audit-collector
|
|
network_mode: host # receive from booted hosts on the boot LAN (:514)
|
|
restart: unless-stopped
|
|
volumes:
|
|
- ./audit-collector/logs:/var/log/remote # durable, inspectable on host
|
|
- ./audit-collector/spool:/var/spool/rsyslog # disk-assisted queue
|
|
|
|
# --- NVMe/TCP target (serves rootfs.erofs, read-only) ----------------------
|
|
nvmeof:
|
|
# Prebuilt by CI (.gitea/workflows/spdk-image.yaml) and pulled here — no
|
|
# official spdk/spdk image exists, so we compile our own once in the
|
|
# pipeline. Local-dev alternative: replace this line with `build: ./spdk`.
|
|
image: ${SPDK_IMAGE}:${SPDK_VERSION}
|
|
network_mode: host # NVMe/TCP listener on ${TARGET_PORT}
|
|
privileged: true # hugepages + IPC_LOCK/SYS_ADMIN for SPDK
|
|
restart: unless-stopped
|
|
ulimits:
|
|
memlock: -1
|
|
volumes:
|
|
- /dev/hugepages:/dev/hugepages
|
|
# LONG-LIVED target + ONLINE release control (no restart on a new build):
|
|
# spdk-target = the persistent entrypoint (nvmf_tgt stays up for life)
|
|
# spdk-release = JSON-RPC add/remove/list/drain of a version, live
|
|
- ./spdk/spdk-target.sh:/usr/local/bin/spdk-target:ro
|
|
- ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
|
|
# Multi-version release store: each ./spdk/releases/<ver>/ holds that
|
|
# version's rootfs.erofs + rootfs.hash + verity.json. Staged by
|
|
# spdk-ctl.sh on the host; reconciled into the target at (re)start and
|
|
# added/removed live thereafter. RW so the /artifacts bootstrap seed and
|
|
# spdk-ctl can write here. Adding a version does NOT recreate this service.
|
|
- ./spdk/releases:/releases
|
|
# Bootstrap convenience (optional): a single OCI release mounted as an
|
|
# image volume. spdk-target seeds it into /releases as ${IMAGE_VERSION} if
|
|
# not already staged — preserves the simple one-version `compose up` flow.
|
|
# Remove this block for a pure multi-version (spdk-ctl-managed) deployment.
|
|
- type: image
|
|
source: ${OCI_IMAGE}:${IMAGE_VERSION}
|
|
target: /artifacts
|
|
image:
|
|
subpath: artifacts
|
|
# Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/artifacts:ro
|
|
# Legacy single-version entrypoint (configure-at-start, restart-to-change):
|
|
# - ./spdk-setup.sh:/spdk-setup.sh:ro
|
|
environment:
|
|
IMAGE_VERSION: ${IMAGE_VERSION} # bootstrap-seed label for /artifacts
|
|
RELEASES_DIR: /releases
|
|
SUBNQN_BASE: ${SUBNQN_BASE} # per-version NQN base (falls back per release to verity.json .subnqn)
|
|
TARGET_ADDR: ${SERVER_IP}
|
|
TARGET_PORT: ${TARGET_PORT}
|
|
SPDK_CPUMASK: ${SPDK_CPUMASK}
|
|
NVMEOF_SVC: nvmeof
|
|
# Persistent target: started ONCE, never restarted for a version change.
|
|
# Rolling updates add/remove versions online (serving/spdk/spdk-ctl.sh).
|
|
entrypoint: ["/bin/bash", "/usr/local/bin/spdk-target"]
|