152 lines
7.3 KiB
YAML
152 lines
7.3 KiB
YAML
# =============================================================================
|
|
# Diskless fleet serving stack (lab)
|
|
#
|
|
# Stands up the three things a diskless host needs to network-boot:
|
|
# dnsmasq - DHCP (hands UEFI HTTP-Boot clients the UKI URL) + DNS
|
|
# http - nginx serving the signed/unsigned UKI over HTTP
|
|
# nvmeof - SPDK nvmf_tgt serving rootfs.erofs as a READ-ONLY NVMe/TCP
|
|
# namespace on 4420 (design doc §5, §7)
|
|
#
|
|
# This matches docs/diskless-fleet-design-v2.pdf, with two LAB simplifications:
|
|
# * HTTP (not HTTPS) boot - no TlsCaCertificate to enrol into firmware
|
|
# * 802.1X / MACsec disabled - pairs with the rd.dot1x=0 test UKI
|
|
# Do NOT use as-is for a secured fleet.
|
|
#
|
|
# Networking: all three services use the host network stack so dnsmasq can see
|
|
# DHCP broadcasts and SPDK gets a clean NVMe/TCP path. The Docker host must sit
|
|
# on the same L2 as the booting client (e.g. the test bridge / boot VLAN), and
|
|
# BOOT_IFACE / SERVER_IP must be that interface and its address.
|
|
#
|
|
# Artifacts are mounted straight from the FROM-scratch OCI release image using
|
|
# a Docker "image volume" (type: image) — the compose analog of the k8s
|
|
# image-volume source in the design doc (§7). The image's /artifacts dir is
|
|
# mounted read-only; nothing in it executes, so FROM scratch is fine. Pin by
|
|
# digest (OCI_IMAGE@sha256:…) for immutable, content-addressed releases.
|
|
# Requires Docker Engine >= 28.0 and a recent Compose. On older Docker, use
|
|
# the commented bind-mount fallback in each service (ARTIFACT_SRC = local
|
|
# Packer output dir), or extract once with:
|
|
# id=$(docker create $OCI_IMAGE:$IMAGE_VERSION); \
|
|
# docker cp $id:/artifacts/. ./artifacts/$IMAGE_VERSION/; docker rm $id
|
|
# Private registry: docker login git.bitasylum.org first so the image pulls.
|
|
#
|
|
# Usage:
|
|
# cp .env.example .env # then edit BOOT_IFACE / SERVER_IP / DHCP range
|
|
# docker compose up -d
|
|
# =============================================================================
|
|
|
|
services:
|
|
# --- DHCP + DNS ------------------------------------------------------------
|
|
dnsmasq:
|
|
build: ./dnsmasq
|
|
network_mode: host
|
|
cap_add:
|
|
- NET_ADMIN # DHCP raw sockets
|
|
- NET_RAW
|
|
restart: unless-stopped
|
|
command:
|
|
- --keep-in-foreground
|
|
# Log to stderr so `docker compose logs` captures output. Without this,
|
|
# dnsmasq logs to syslog — which doesn't exist in the container — so
|
|
# --log-dhcp/--log-queries produce nothing even while dnsmasq works fine.
|
|
- --log-facility=-
|
|
- --log-dhcp
|
|
- --log-queries
|
|
# Bind only to the boot interface so we don't fight the host resolver.
|
|
- --interface=${BOOT_IFACE}
|
|
- --bind-interfaces
|
|
- --except-interface=lo
|
|
# ---- DNS ----
|
|
- --domain=${BOOT_DOMAIN}
|
|
- --no-resolv
|
|
- --server=${UPSTREAM_DNS}
|
|
# NVMe/TCP discovery address used in the UKI cmdline (rd.nvmf.discover):
|
|
# resolve the per-environment DNS name to this serving host.
|
|
- --address=/${NVME_HOST}/${SERVER_IP}
|
|
- --address=/${BOOT_HOST}/${SERVER_IP}
|
|
# Kiosk is LOCAL-ONLY (loads boot.fleet.lab), so sinkhole Chromium's
|
|
# call-home to 0.0.0.0 — telemetry can't leave the lab even if the browser
|
|
# tries. Defense-in-depth behind the in-image Chromium policy/flags; in
|
|
# production this belongs in the boot-LAN firewall's egress rules. NOTE: the
|
|
# client still *emits* the query (you'll see it in this log answered locally,
|
|
# not "forwarded"); stopping the query itself is the browser policy's job.
|
|
- --address=/google.com/0.0.0.0
|
|
- --address=/googleapis.com/0.0.0.0
|
|
- --address=/gstatic.com/0.0.0.0
|
|
# ---- DHCP ----
|
|
- --dhcp-authoritative
|
|
- --dhcp-range=${DHCP_RANGE_START},${DHCP_RANGE_END},${DHCP_LEASE}
|
|
- --dhcp-option=option:router,${GATEWAY_IP}
|
|
- --dhcp-option=option:dns-server,${SERVER_IP}
|
|
# NTP server (option 42): point diskless hosts at the lab 'ntp' service
|
|
# below so they sync time internally and never reach the public NTP pool.
|
|
# The image's NM dispatcher (10-chrony-dhcp-ntp) feeds this into chrony.
|
|
- --dhcp-option=option:ntp-server,${SERVER_IP}
|
|
# ---- UEFI HTTP Boot ----
|
|
# Firmware sends vendor-class "HTTPClient"; we must echo it back (opt 60)
|
|
# and hand over the UKI URL (opt 67). IP in the URL avoids a DNS round-trip
|
|
# during boot. client-arch 16 = x86-64 UEFI HTTP.
|
|
- --dhcp-vendorclass=set:httpboot,HTTPClient
|
|
- --dhcp-option=tag:httpboot,60,HTTPClient
|
|
- --dhcp-boot=tag:httpboot,http://${SERVER_IP}/fleet-${IMAGE_VERSION}.efi
|
|
|
|
# --- HTTP boot server (serves the UKI) -------------------------------------
|
|
http:
|
|
image: nginx:stable
|
|
network_mode: host # listens on :80 -> http://${SERVER_IP}/fleet-<ver>.efi
|
|
restart: unless-stopped
|
|
volumes:
|
|
- type: image
|
|
source: ${OCI_IMAGE}:${IMAGE_VERSION}
|
|
target: /usr/share/nginx/html
|
|
image:
|
|
subpath: artifacts # mount only /artifacts from the image
|
|
# Fallback (Docker < 28 / offline): bind the local Packer output dir:
|
|
# - ${ARTIFACT_SRC}:/usr/share/nginx/html:ro
|
|
# Serve the kiosk test page at /kiosk/ alongside the artifacts. Config
|
|
# adds the location; the page is lab content (NOT in the artifact image).
|
|
- ./http/default.conf:/etc/nginx/conf.d/default.conf:ro
|
|
- ./http/kiosk:/usr/share/nginx/kiosk:ro
|
|
# For HTTPS boot (production), terminate TLS here with the boot-infra CA's
|
|
# cert and switch the dhcp-boot URL above to https://. Left HTTP for the lab.
|
|
|
|
# --- NTP server (serves time to the boot LAN; handed out via DHCP opt 42) ---
|
|
ntp:
|
|
build: ./chrony
|
|
network_mode: host # serve NTP on udp/123 across the boot LAN
|
|
cap_add:
|
|
- SYS_TIME # discipline the (host-shared) clock it serves
|
|
restart: unless-stopped
|
|
volumes:
|
|
- ./chrony/chrony.conf:/etc/chrony/chrony.conf:ro
|
|
# NOTE: with host networking + SYS_TIME this chronyd disciplines the Docker
|
|
# HOST clock (the clock isn't namespaced) — intended: the serving host is the
|
|
# lab time source. Drop SYS_TIME (and add `chronyd -x`) if the host already
|
|
# runs its own NTP and you only want this to relay.
|
|
|
|
# --- NVMe/TCP target (serves rootfs.erofs, read-only) ----------------------
|
|
nvmeof:
|
|
# Prebuilt by CI (.gitea/workflows/spdk-image.yaml) and pulled here — no
|
|
# official spdk/spdk image exists, so we compile our own once in the
|
|
# pipeline. Local-dev alternative: replace this line with `build: ./spdk`.
|
|
image: ${SPDK_IMAGE}:${SPDK_VERSION}
|
|
network_mode: host # NVMe/TCP listener on ${TARGET_PORT}
|
|
privileged: true # hugepages + IPC_LOCK/SYS_ADMIN for SPDK
|
|
restart: unless-stopped
|
|
ulimits:
|
|
memlock: -1
|
|
volumes:
|
|
- /dev/hugepages:/dev/hugepages
|
|
- type: image
|
|
source: ${OCI_IMAGE}:${IMAGE_VERSION}
|
|
target: /artifacts # -> /artifacts/rootfs.erofs for spdk-setup.sh
|
|
image:
|
|
subpath: artifacts
|
|
# Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/artifacts:ro
|
|
- ./spdk-setup.sh:/spdk-setup.sh:ro
|
|
environment:
|
|
SUBNQN: ${SUBNQN}
|
|
TARGET_ADDR: ${SERVER_IP}
|
|
TARGET_PORT: ${TARGET_PORT}
|
|
SPDK_CPUMASK: ${SPDK_CPUMASK}
|
|
entrypoint: ["/bin/bash", "/spdk-setup.sh"]
|