# ============================================================================= # Diskless fleet serving stack (lab) # # Stands up the three things a diskless host needs to network-boot: # dnsmasq - DHCP (hands UEFI HTTP-Boot clients the UKI URL) + DNS # http - nginx serving the signed/unsigned UKI over HTTP # nvmeof - SPDK nvmf_tgt serving rootfs.erofs as a READ-ONLY NVMe/TCP # namespace on 4420 (design doc §5, §7) # # This matches docs/diskless-fleet-design-v2.pdf, with two LAB simplifications: # * HTTP (not HTTPS) boot - no TlsCaCertificate to enrol into firmware # * 802.1X / MACsec disabled - pairs with the rd.dot1x=0 test UKI # Do NOT use as-is for a secured fleet. # # Networking: all three services use the host network stack so dnsmasq can see # DHCP broadcasts and SPDK gets a clean NVMe/TCP path. The Docker host must sit # on the same L2 as the booting client (e.g. the test bridge / boot VLAN), and # BOOT_IFACE / SERVER_IP must be that interface and its address. # # Artifacts are mounted straight from the FROM-scratch OCI release image using # a Docker "image volume" (type: image) — the compose analog of the k8s # image-volume source in the design doc (§7). The image's /artifacts dir is # mounted read-only; nothing in it executes, so FROM scratch is fine. Pin by # digest (OCI_IMAGE@sha256:…) for immutable, content-addressed releases. # Requires Docker Engine >= 28.0 and a recent Compose. On older Docker, use # the commented bind-mount fallback in each service (ARTIFACT_SRC = local # Packer output dir), or extract once with: # id=$(docker create $OCI_IMAGE:$IMAGE_VERSION); \ # docker cp $id:/artifacts/. ./artifacts/$IMAGE_VERSION/; docker rm $id # Private registry: docker login git.bitasylum.org first so the image pulls. # # Usage (config = committed defaults.env + your .env overrides, layered): # export COMPOSE_ENV_FILES=defaults.env,.env # or use the Makefile (make up) # printf 'BOOT_IFACE=...\nSERVER_IP=...\n' > .env # per-LAN overrides + secrets # docker compose up -d # ============================================================================= services: # --- DHCP + DNS ------------------------------------------------------------ dnsmasq: # Prebuilt by CI (.gitea/workflows/dnsmasq-image.yaml) and pulled here, like # the SPDK image — the serving host no longer builds it at deploy time. The # image is GENERIC (no Secure Boot keys, no iPXE binary): the signed iPXE # bootloader rides the separate, per-environment fleet-ipxe image mounted at # /ipxe below. Local-dev alternative: replace this line with `build: ./dnsmasq`. image: ${DNSMASQ_IMAGE}:${DNSMASQ_VERSION} network_mode: host cap_add: - NET_ADMIN # DHCP raw sockets - NET_RAW restart: unless-stopped volumes: # DHCP reservations (MAC -> name [-> IP]); drives DHCP option 12 (hostname) # + DNS A/PTR. Auto-populated by fleet-namer (creative names per MAC); add # manual lines to pin specific hosts. RW so fleet-namer can append. - ./dnsmasq/dhcp-hosts:/etc/dnsmasq-hosts # Signed iPXE bootloader for the PXE-only chainload (firmware without UEFI # HTTP boot, e.g. EliteBook 820 G4). Mounted from the separate, signed, # digest-pinnable fleet-ipxe image (.gitea/workflows/ipxe-image.yaml); the # entrypoint copies /ipxe/snponly.efi into /srv/tftp. Kept OUT of the dnsmasq # image so the daemon image stays generic/key-agnostic — see serving/tftp/README. - type: image source: ${IPXE_IMAGE}:${IPXE_VERSION} target: /ipxe image: subpath: ipxe # Fallback (Docker < 28 / offline / lab): drop snponly.efi in ./tftp and bind # it instead (the entrypoint prefers /ipxe, then falls back to /tftp-src): # - ./tftp:/tftp-src:ro # The release artifacts (same image volume as the http service), read-only, # so the entrypoint can objcopy the UKI's .cmdline section into boot.ipxe # (Secure Boot OFF needs the real cmdline fed to iPXE — see entrypoint.sh). - type: image source: ${OCI_IMAGE}:${IMAGE_VERSION} target: /artifacts image: subpath: artifacts # Fallback (Docker < 28 / offline): bind the local Packer output dir instead: # - ${ARTIFACT_SRC}:/artifacts:ro environment: # Consumed by the entrypoint to render /srv/tftp/boot.ipxe (the iPXE chain # script that clears load options so the UKI's embedded cmdline survives). - SERVER_IP=${SERVER_IP} - IMAGE_VERSION=${IMAGE_VERSION} command: - --keep-in-foreground # Log to stderr so `docker compose logs` captures output. Without this, # dnsmasq logs to syslog — which doesn't exist in the container — so # --log-dhcp/--log-queries produce nothing even while dnsmasq works fine. - --log-facility=- - --log-dhcp - --log-queries # Bind only to the boot interface so we don't fight the host resolver. - --interface=${BOOT_IFACE} - --bind-interfaces - --except-interface=lo # ---- DNS ---- - --domain=${BOOT_DOMAIN} - --no-resolv - --server=${UPSTREAM_DNS} # NVMe/TCP discovery address used in the UKI cmdline (rd.nvmf.discover): # resolve the per-environment DNS name to this serving host. - --address=/${NVME_HOST}/${SERVER_IP} - --address=/${BOOT_HOST}/${SERVER_IP} # Host-generic remote audit collector name (baked into the image config so # every host offloads auditd here; resolved per-environment to this host). - --address=/${AUDIT_HOST}/${SERVER_IP} # Fleet-manager AGENT plane (:9600) — baked into the image so enrolled hosts # dial the control plane by name; resolved here like the other fleet names. - --address=/${MGMT_HOST}/${SERVER_IP} # Kiosk is LOCAL-ONLY (loads boot.fleet.lab), so sinkhole Chromium's # call-home to 0.0.0.0 — telemetry can't leave the lab even if the browser # tries. Defense-in-depth behind the in-image Chromium policy/flags; in # production this belongs in the boot-LAN firewall's egress rules. NOTE: the # client still *emits* the query (you'll see it in this log answered locally, # not "forwarded"); stopping the query itself is the browser policy's job. - --address=/google.com/0.0.0.0 - --address=/googleapis.com/0.0.0.0 - --address=/gstatic.com/0.0.0.0 # ---- DHCP ---- - --dhcp-authoritative - --dhcp-range=${DHCP_RANGE_START},${DHCP_RANGE_END},${DHCP_LEASE} - --dhcp-option=option:router,${GATEWAY_IP} - --dhcp-option=option:dns-server,${SERVER_IP} # NTP server (option 42): point diskless hosts at the lab 'ntp' service # below so they sync time internally and never reach the public NTP pool. # The image's NM dispatcher (10-chrony-dhcp-ntp) feeds this into chrony. - --dhcp-option=option:ntp-server,${SERVER_IP} # ---- Per-host names (creative starship hostnames, server-assigned) ---- # The workstation image is hostname-neutral; naming is done HERE. fleet-namer # (the --dhcp-script) derives a stable themed name per MAC (e.g. # nebula-vanguard-07), appends it to the hostsfile, and SIGHUPs dnsmasq; # dnsmasq then sends it as DHCP option 12 (-> NetworkManager sets the # hostname) and registers DNS A/PTR under ${BOOT_DOMAIN}. The host picks up # its name on the NEXT DHCP transaction (renewal at ~half lease, or reboot) # — shorten DHCP_LEASE if you want naming to settle faster on first contact. # --user=root so the dhcp-script can write the hostsfile + signal dnsmasq. - --user=root - --dhcp-hostsfile=/etc/dnsmasq-hosts - --dhcp-script=/usr/local/bin/fleet-namer # ---- UEFI HTTP Boot ---- # Firmware sends vendor-class "HTTPClient"; we must echo it back (opt 60) # and hand over the UKI URL (opt 67). IP in the URL avoids a DNS round-trip # during boot. client-arch 16 = x86-64 UEFI HTTP. - --dhcp-vendorclass=set:httpboot,HTTPClient - --dhcp-option=tag:httpboot,60,HTTPClient - --dhcp-boot=tag:httpboot,http://${SERVER_IP}/fleet-${IMAGE_VERSION}.efi # ---- PXE -> iPXE -> HTTP chainload (for firmware WITHOUT UEFI HTTP boot) ---- # Graceful fallback: HTTP-capable firmware matches 'httpboot' above and gets # the UKI directly; PXE-only firmware (PXEClient) gets ipxe.efi over TFTP; # once iPXE is running it re-DHCPs with user-class "iPXE" and is handed the # SAME UKI HTTP URL, which it chainloads (.efi -> LoadImage). Each client # gets the right thing based on what it advertises — no per-host config. # snponly.efi = UEFI x64 iPXE that rides the FIRMWARE's NIC driver (UEFI SNP) # instead of iPXE's own — works on virtio VMs AND real hardware where the # native driver won't bind (e.g. EliteBook 820 G4 / Intel I219-LM, which # fails ipxe.efi with "No configurable methods"). Built + signed by CI into # the fleet-ipxe image (mounted at /ipxe); see serving/tftp/README.md. # The fleet-ipxe image is signed with the fleet db key when CI has the # secrets, so this path stays Secure-Boot-enforced end-to-end; an unsigned # (lab) fleet-ipxe build needs Secure Boot OFF — see docs/secure-boot-keys.md. - --enable-tftp - --tftp-root=/srv/tftp - --dhcp-vendorclass=set:pxe,PXEClient - --dhcp-userclass=set:ipxe,iPXE # firmware PXE, not yet iPXE -> hand iPXE (UEFI x64) over TFTP - --dhcp-boot=tag:pxe,tag:!ipxe,snponly.efi # iPXE running -> hand boot.ipxe (NOT the UKI URL directly). The script # imgfetch'es the UKI then `imgargs`-clears the load options before boot, so # systemd-stub uses the UKI's EMBEDDED cmdline. Booting the UKI URL directly # makes iPXE pass the image name as the cmdline, which (Secure Boot OFF) # REPLACES the embedded roothash/nvmf/... args -> dracut emergency shell. - --dhcp-boot=tag:ipxe,boot.ipxe # --- HTTP boot server (serves the UKI) ------------------------------------- http: image: harbor.bitasylum.org/dockerhub/library/nginx:stable network_mode: host # listens on :80 -> http://${SERVER_IP}/fleet-.efi restart: unless-stopped depends_on: - kiosk # /kiosk/ is reverse-proxied to the kiosk SPA volumes: - type: image source: ${OCI_IMAGE}:${IMAGE_VERSION} target: /usr/share/nginx/html image: subpath: artifacts # mount only /artifacts from the image # Fallback (Docker < 28 / offline): bind the local Packer output dir: # - ${ARTIFACT_SRC}:/usr/share/nginx/html:ro # Config adds the artifact root + the /kiosk/ reverse-proxy location. The # kiosk SPA itself is the separate 'kiosk' service below (not bind-mounted). - ./http/default.conf:/etc/nginx/conf.d/default.conf:ro # For HTTPS boot (production), terminate TLS here with the boot-infra CA's # cert and switch the dhcp-boot URL above to https://. Left HTTP for the lab. # --- TPM enrolment broker (TEMPORARY home — see docs/TODO.md #11) ----------- # The `http` nginx already reverse-proxies /enrol/ -> 127.0.0.1:9090 (this # broker's public listener), so an empty-NV host can POST its CSR and poll for # the signed cert via the normal serving host. For now the broker runs HERE so # a single `compose up` in serving/ brings the enrolment endpoint up. # # ⚠ PROPER HOME is its own stack co-located with step-ca (the fleet CA), i.e. # serving/dot1x-bench/, since REAL signing needs that CA. SPLIT THIS BACK OUT # once the CA lives beside it. Do NOT run this AND dot1x-bench's enrol-broker on # the same host — they collide on 9090/9443. # # Defaults below are the zero-dependency LAB path: `mock` signer (ephemeral # in-process CA — certs are NOT trusted by RADIUS/NVMe) + plaintext loopback # admin. For REAL enrolment, in .env set ENROL_SIGNER=step + point # ENROL_STEP_CA_* at a reachable step-ca, and set ENROL_ADMIN_INSECURE=false # after generating the admin mTLS PKI (a DEDICATED admin CA, not the fleet root): # ./enrol-broker/admin-pki.sh ./enrol-pki/broker # then import enrol-pki/broker/clients/.p12 and open https://127.0.0.1:9443/ enrol-broker: build: ./enrol-broker network_mode: host # public on 127.0.0.1:9090 (nginx proxies /enrol/) restart: unless-stopped environment: BROKER_PUBLIC_ADDR: "127.0.0.1:9090" # nginx /enrol/ proxies here BROKER_ADMIN_ADDR: "127.0.0.1:9443" # mgmt/loopback ONLY — never the boot LAN BROKER_SIGNER: ${ENROL_SIGNER:-mock} BROKER_PENDING_TTL: ${ENROL_PENDING_TTL:-10m} BROKER_WORDS: ${ENROL_WORDS:-8} HOSTNQN_BASE: ${ENROL_HOSTNQN_BASE:-nqn.2026-06.org.example.fleet:host-} # --- step signer (only consulted when ENROL_SIGNER=step) --- STEP_CA_URL: ${ENROL_STEP_CA_URL:-} STEP_CA_ROOT: ${ENROL_STEP_CA_ROOT:-/pki/root_ca.crt} STEP_PROVISIONER: ${ENROL_STEP_PROVISIONER:-enrol} STEP_PROVISIONER_PASSWORD_FILE: ${ENROL_STEP_PASSWORD_FILE:-/pki/password} STEP_CERT_NOT_AFTER: ${ENROL_CERT_NOT_AFTER:-720h} # --- admin mTLS (set ENROL_ADMIN_INSECURE=false for real mTLS) --- BROKER_ADMIN_INSECURE: ${ENROL_ADMIN_INSECURE:-true} BROKER_ADMIN_CERT: ${ENROL_ADMIN_CERT:-/pki/broker/server.crt} BROKER_ADMIN_KEY: ${ENROL_ADMIN_KEY:-/pki/broker/server.key} BROKER_ADMIN_CLIENT_CA: ${ENROL_ADMIN_CLIENT_CA:-/pki/broker/admin-ca.crt} volumes: # CA root + provisioner password (step signer) and admin mTLS certs live # here; unused/empty for the default mock+insecure path. Gitignored. - ./enrol-pki:/pki:ro # --- Kiosk SPA (spaceship terminal shown by the diskless kiosk) ------------ # Svelte + Tailwind app built into a static bundle served by its own nginx. # Now an INDEPENDENT project — the `kiosk-app` git submodule (its own repo + # CI). It builds + pushes its own image (git.../fleet-kiosk); the serving stack # pulls it, like dnsmasq/SPDK. Bridge-networked (NOT host) so it doesn't fight # :80; published on loopback only, where the host-network 'http' nginx # reverse-proxies /kiosk/ to it. kiosk: image: ${KIOSK_IMAGE}:${KIOSK_VERSION} # Local dev from the submodule working tree instead of pulling: # build: ./kiosk-app restart: unless-stopped ports: - "127.0.0.1:${KIOSK_APP_PORT:-8088}:80" # loopback only; main nginx proxies here environment: HASS_URL: ${HASS_URL} HASS_TOKEN: ${HASS_TOKEN} # --- NTP server (serves time to the boot LAN; handed out via DHCP opt 42) --- ntp: build: ./chrony network_mode: host # serve NTP on udp/123 across the boot LAN cap_add: - SYS_TIME # discipline the (host-shared) clock it serves restart: unless-stopped volumes: - ./chrony/chrony.conf:/etc/chrony/chrony.conf:ro # NOTE: with host networking + SYS_TIME this chronyd disciplines the Docker # HOST clock (the clock isn't namespaced) — intended: the serving host is the # lab time source. Drop SYS_TIME (and add `chronyd -x`) if the host already # runs its own NTP and you only want this to relay. # # FIREWALL (easy to miss — same trap as the audit collector's 514/tcp): host # networking means Docker does NOT open the port; the serving host's own # firewall governs 123/udp. Without a rule, workstations get 192.168.50.10 as # their NTP server (DHCP opt 42) but their packets are dropped — chrony logs # "Added source … / unreachable" and never syncs. Open it on the boot-LAN NIC: # ufw: sudo ufw allow in on to any port 123 proto udp comment 'fleet NTP' # firewalld: sudo firewall-cmd --permanent --add-service=ntp && sudo firewall-cmd --reload # --- Remote audit / log collector (RECEIVE side) --------------------------- # rsyslog receiver: booted diskless hosts (whose RAM-only overlay root loses # local logs on reboot) forward auditd via audisp-syslog + syslog here over # TCP/514. Records are written per-source-host under # ./audit-collector/logs//{audit,messages}.log. Host-networked so # hosts can reach it at ${AUDIT_HOST} (= this host) on the boot LAN. # NOTE: this is the receiver only — the IMAGE must be configured to send # (audisp-syslog + an rsyslog forward rule targeting ${AUDIT_HOST}); see # audit-collector/README.md. Until then the collector simply waits, idle. audit: build: ./audit-collector network_mode: host # receive from booted hosts on the boot LAN (:514) restart: unless-stopped volumes: - ./audit-collector/logs:/var/log/remote # durable, inspectable on host - ./audit-collector/spool:/var/spool/rsyslog # disk-assisted queue # --- NVMe/TCP target (serves rootfs.erofs, read-only) ---------------------- nvmeof: # Prebuilt by CI (.gitea/workflows/spdk-image.yaml) and pulled here — no # official spdk/spdk image exists, so we compile our own once in the # pipeline. Local-dev alternative: replace this line with `build: ./spdk`. image: ${SPDK_IMAGE}:${SPDK_VERSION} network_mode: host # NVMe/TCP listener on ${TARGET_PORT} privileged: true # hugepages + IPC_LOCK/SYS_ADMIN for SPDK restart: unless-stopped ulimits: memlock: -1 volumes: - /dev/hugepages:/dev/hugepages # LONG-LIVED target + ONLINE release control (no restart on a new build): # spdk-target = the persistent entrypoint (nvmf_tgt stays up for life) # spdk-release = JSON-RPC add/remove/list/drain of a version, live - ./spdk/spdk-target.sh:/usr/local/bin/spdk-target:ro - ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro # Shared RPC socket so the spdk-loader container (and `compose exec`) can # drive this target's JSON-RPC. nvmf_tgt creates the socket here (-r). - spdk-sock:/var/run/spdk # Multi-version release store: each ./spdk/releases// holds that # version's rootfs.erofs + rootfs.hash + verity.json. Staged by the # spdk-loader (or spdk-ctl.sh on the host); reconciled into the target at # (re)start and added/removed live thereafter. RW so the spdk-loader can # stage here. Adding a version does NOT recreate this service. - ./spdk/releases:/releases # NOTE: the target is deliberately VERSION-AGNOSTIC. It mounts NO # ${IMAGE_VERSION} artifact image and carries no IMAGE_VERSION env, so a # version bump never changes THIS service's config — `docker compose up -d` # after an IMAGE_VERSION bump recreates the loader/http/dnsmasq but leaves # nvmf_tgt running (recreating it would tear down EVERY host's NVMe/TCP root # at once). Bootstrap AND rolling updates are the spdk-loader's job: it # stages the current release into ./spdk/releases and adds it online via # JSON-RPC. (spdk-target still seeds from /artifacts IF one is manually # mounted + IMAGE_VERSION set — a legacy/standalone convenience, off here.) # Legacy single-version entrypoint (configure-at-start, restart-to-change): # - ./spdk-setup.sh:/spdk-setup.sh:ro environment: RELEASES_DIR: /releases RPC_SOCK: /var/run/spdk/spdk.sock # shared with spdk-loader (-r / -s) SUBNQN_BASE: ${SUBNQN_BASE} # per-version NQN base (falls back per release to verity.json .subnqn) TARGET_ADDR: ${SERVER_IP} TARGET_PORT: ${TARGET_PORT} SPDK_CPUMASK: ${SPDK_CPUMASK} NVMEOF_SVC: nvmeof # Healthy once spdk-target has written its readiness stamp (transport up + # releases reconciled). A FILE check, not a python rpc.py probe: nvmf_tgt does # a slow DPDK init (~50s observed) and busy-polls its reactor core, so a python # healthcheck starves/times out and the container sticks at "unhealthy" even # though the target is up. `test -f` is instant and contention-free. The stamp # is removed on (re)start + exit, so it never falsely reports a dead target. healthcheck: test: ["CMD-SHELL", "test -f /var/run/spdk/ready || exit 1"] interval: 10s timeout: 5s retries: 6 start_period: 120s # Persistent target: started ONCE, never restarted for a version change. # Rolling updates add/remove versions online (spdk-loader, or spdk-ctl.sh). entrypoint: ["/bin/bash", "/usr/local/bin/spdk-target"] # --- SPDK version loader / reaper (declarative rolling updates) ------------- # The "init container that stays up." On `compose up` after an IMAGE_VERSION # bump it stages + adds the new rootfs to the running nvmeof target ONLINE # (no target restart, no dropped connections), then polls and removes old # versions once they drain (all hosts rebooted onto the new UKI). Migration is # just: bump IMAGE_VERSION in .env -> `docker compose up -d`. Reuses the SPDK # image (has python3 + rpc.py); the scripts are bind-mounted like nvmeof's. spdk-loader: image: ${SPDK_IMAGE}:${SPDK_VERSION} network_mode: host restart: unless-stopped depends_on: nvmeof: condition: service_healthy volumes: - ./spdk/spdk-loader.sh:/usr/local/bin/spdk-loader:ro - ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro - spdk-sock:/var/run/spdk # shared RPC socket to the target - ./spdk/releases:/releases # shared release store # The CURRENT release's artifacts, mounted from its OCI image. The version # tag is the single migration knob — bump IMAGE_VERSION and `up -d`. - type: image source: ${OCI_IMAGE}:${IMAGE_VERSION} target: /in image: subpath: artifacts # Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/in:ro environment: IMAGE_VERSION: ${IMAGE_VERSION} # the CURRENT version (staged + protected from reaping) RELEASES_DIR: /releases ARTIFACT_IN: /in RPC_SOCK: /var/run/spdk/spdk.sock SUBNQN_BASE: ${SUBNQN_BASE} TARGET_ADDR: ${SERVER_IP} TARGET_PORT: ${TARGET_PORT} REAP_OLD: ${REAP_OLD:-true} # auto-remove drained old versions REAP_POLL_SECONDS: ${REAP_POLL_SECONDS:-30} REAP_GRACE_SECONDS: ${REAP_GRACE_SECONDS:-300} # sustained-zero window; keep > host ctrl_loss_tmo entrypoint: ["/bin/bash", "/usr/local/bin/spdk-loader"] # Named volume for the SPDK JSON-RPC unix socket, shared between the long-lived # nvmeof target (which creates it via nvmf_tgt -r) and the spdk-loader (which # drives add/remove/drain over it). Keeps the control path off the host FS. volumes: spdk-sock: