diskless-workstation/serving/docker-compose.yml

# =============================================================================
# Diskless fleet serving stack (lab)
#
# Stands up the three things a diskless host needs to network-boot:
#   dnsmasq  - DHCP (hands UEFI HTTP-Boot clients the UKI URL) + DNS
#   http     - nginx serving the signed/unsigned UKI over HTTP
#   nvmeof   - SPDK nvmf_tgt serving rootfs.erofs as a READ-ONLY NVMe/TCP
#              namespace on 4420 (design doc §5, §7)
#
# This matches docs/diskless-fleet-design-v2.pdf, with two LAB simplifications:
#   * HTTP (not HTTPS) boot      - no TlsCaCertificate to enrol into firmware
#   * 802.1X / MACsec disabled   - pairs with the rd.dot1x=0 test UKI
# Do NOT use as-is for a secured fleet.
#
# Networking: all three services use the host network stack so dnsmasq can see
# DHCP broadcasts and SPDK gets a clean NVMe/TCP path. The Docker host must sit
# on the same L2 as the booting client (e.g. the test bridge / boot VLAN), and
# BOOT_IFACE / SERVER_IP must be that interface and its address.
#
# Artifacts are mounted straight from the FROM-scratch OCI release image using
# a Docker "image volume" (type: image) — the compose analog of the k8s
# image-volume source in the design doc (§7). The image's /artifacts dir is
# mounted read-only; nothing in it executes, so FROM scratch is fine. Pin by
# digest (OCI_IMAGE@sha256:…) for immutable, content-addressed releases.
#   Requires Docker Engine >= 28.0 and a recent Compose. On older Docker, use
#   the commented bind-mount fallback in each service (ARTIFACT_SRC = local
#   Packer output dir), or extract once with:
#     id=$(docker create $OCI_IMAGE:$IMAGE_VERSION); \
#     docker cp $id:/artifacts/. ./artifacts/$IMAGE_VERSION/; docker rm $id
#   Private registry: docker login git.bitasylum.org first so the image pulls.
#
# Usage (config = committed defaults.env + your .env overrides, layered):
#   export COMPOSE_ENV_FILES=defaults.env,.env   # or use the Makefile (make up)
#   printf 'BOOT_IFACE=...\nSERVER_IP=...\n' > .env   # per-LAN overrides + secrets
#   docker compose up -d
# =============================================================================

services:
  # --- DHCP + DNS ------------------------------------------------------------
  dnsmasq:
    # Prebuilt by CI (.gitea/workflows/dnsmasq-image.yaml) and pulled here, like
    # the SPDK image — the serving host no longer builds it at deploy time. The
    # image is GENERIC (no Secure Boot keys, no iPXE binary): the signed iPXE
    # bootloader rides the separate, per-environment fleet-ipxe image mounted at
    # /ipxe below. Local-dev alternative: replace this line with `build: ./dnsmasq`.
    image: ${DNSMASQ_IMAGE}:${DNSMASQ_VERSION}
    network_mode: host
    cap_add:
      - NET_ADMIN          # DHCP raw sockets
      - NET_RAW
    restart: unless-stopped
    volumes:
      # DHCP reservations (MAC -> name [-> IP]); drives DHCP option 12 (hostname)
      # + DNS A/PTR. Auto-populated by fleet-namer (creative names per MAC); add
      # manual lines to pin specific hosts. RW so fleet-namer can append.
      - ./dnsmasq/dhcp-hosts:/etc/dnsmasq-hosts
      # Signed iPXE bootloader for the PXE-only chainload (firmware without UEFI
      # HTTP boot, e.g. EliteBook 820 G4). Mounted from the separate, signed,
      # digest-pinnable fleet-ipxe image (.gitea/workflows/ipxe-image.yaml); the
      # entrypoint copies /ipxe/snponly.efi into /srv/tftp. Kept OUT of the dnsmasq
      # image so the daemon image stays generic/key-agnostic — see serving/tftp/README.
      - type: image
        source: ${IPXE_IMAGE}:${IPXE_VERSION}
        target: /ipxe
        image:
          subpath: ipxe
      # Fallback (Docker < 28 / offline / lab): drop snponly.efi in ./tftp and bind
      # it instead (the entrypoint prefers /ipxe, then falls back to /tftp-src):
      # - ./tftp:/tftp-src:ro
      # The release artifacts (same image volume as the http service), read-only,
      # so the entrypoint can objcopy the UKI's .cmdline section into boot.ipxe
      # (Secure Boot OFF needs the real cmdline fed to iPXE — see entrypoint.sh).
      - type: image
        source: ${OCI_IMAGE}:${IMAGE_VERSION}
        target: /artifacts
        image:
          subpath: artifacts
      # Fallback (Docker < 28 / offline): bind the local Packer output dir instead:
      # - ${ARTIFACT_SRC}:/artifacts:ro
    environment:
      # Consumed by the entrypoint to render /srv/tftp/boot.ipxe (the iPXE chain
      # script that clears load options so the UKI's embedded cmdline survives).
      - SERVER_IP=${SERVER_IP}
      - IMAGE_VERSION=${IMAGE_VERSION}
    command:
      - --keep-in-foreground
      # Log to stderr so `docker compose logs` captures output. Without this,
      # dnsmasq logs to syslog — which doesn't exist in the container — so
      # --log-dhcp/--log-queries produce nothing even while dnsmasq works fine.
      - --log-facility=-
      - --log-dhcp
      - --log-queries
      # Bind only to the boot interface so we don't fight the host resolver.
      - --interface=${BOOT_IFACE}
      - --bind-interfaces
      - --except-interface=lo
      # ---- DNS ----
      - --domain=${BOOT_DOMAIN}
      - --no-resolv
      - --server=${UPSTREAM_DNS}
      # NVMe/TCP discovery address used in the UKI cmdline (rd.nvmf.discover):
      # resolve the per-environment DNS name to this serving host.
      - --address=/${NVME_HOST}/${SERVER_IP}
      - --address=/${BOOT_HOST}/${SERVER_IP}
      # Host-generic remote audit collector name (baked into the image config so
      # every host offloads auditd here; resolved per-environment to this host).
      - --address=/${AUDIT_HOST}/${SERVER_IP}
      # Fleet-manager AGENT plane (:9600) — baked into the image so enrolled hosts
      # dial the control plane by name; resolved here like the other fleet names.
      - --address=/${MGMT_HOST}/${SERVER_IP}
      # Kiosk is LOCAL-ONLY (loads boot.fleet.lab), so sinkhole Chromium's
      # call-home to 0.0.0.0 — telemetry can't leave the lab even if the browser
      # tries. Defense-in-depth behind the in-image Chromium policy/flags; in
      # production this belongs in the boot-LAN firewall's egress rules. NOTE: the
      # client still *emits* the query (you'll see it in this log answered locally,
      # not "forwarded"); stopping the query itself is the browser policy's job.
      - --address=/google.com/0.0.0.0
      - --address=/googleapis.com/0.0.0.0
      - --address=/gstatic.com/0.0.0.0
      # ---- DHCP ----
      - --dhcp-authoritative
      - --dhcp-range=${DHCP_RANGE_START},${DHCP_RANGE_END},${DHCP_LEASE}
      - --dhcp-option=option:router,${GATEWAY_IP}
      - --dhcp-option=option:dns-server,${SERVER_IP}
      # NTP server (option 42): point diskless hosts at the lab 'ntp' service
      # below so they sync time internally and never reach the public NTP pool.
      # The image's NM dispatcher (10-chrony-dhcp-ntp) feeds this into chrony.
      - --dhcp-option=option:ntp-server,${SERVER_IP}
      # ---- Per-host names (creative starship hostnames, server-assigned) ----
      # The workstation image is hostname-neutral; naming is done HERE. fleet-namer
      # (the --dhcp-script) derives a stable themed name per MAC (e.g.
      # nebula-vanguard-07), appends it to the hostsfile, and SIGHUPs dnsmasq;
      # dnsmasq then sends it as DHCP option 12 (-> NetworkManager sets the
      # hostname) and registers DNS A/PTR under ${BOOT_DOMAIN}. The host picks up
      # its name on the NEXT DHCP transaction (renewal at ~half lease, or reboot)
      # — shorten DHCP_LEASE if you want naming to settle faster on first contact.
      # --user=root so the dhcp-script can write the hostsfile + signal dnsmasq.
      - --user=root
      - --dhcp-hostsfile=/etc/dnsmasq-hosts
      - --dhcp-script=/usr/local/bin/fleet-namer
      # ---- UEFI HTTP Boot ----
      # Firmware sends vendor-class "HTTPClient"; we must echo it back (opt 60)
      # and hand over the UKI URL (opt 67). IP in the URL avoids a DNS round-trip
      # during boot. client-arch 16 = x86-64 UEFI HTTP.
      - --dhcp-vendorclass=set:httpboot,HTTPClient
      - --dhcp-option=tag:httpboot,60,HTTPClient
      - --dhcp-boot=tag:httpboot,http://${SERVER_IP}/fleet-${IMAGE_VERSION}.efi
      # ---- PXE -> iPXE -> HTTP chainload (for firmware WITHOUT UEFI HTTP boot) ----
      # Graceful fallback: HTTP-capable firmware matches 'httpboot' above and gets
      # the UKI directly; PXE-only firmware (PXEClient) gets ipxe.efi over TFTP;
      # once iPXE is running it re-DHCPs with user-class "iPXE" and is handed the
      # SAME UKI HTTP URL, which it chainloads (.efi -> LoadImage). Each client
      # gets the right thing based on what it advertises — no per-host config.
      # snponly.efi = UEFI x64 iPXE that rides the FIRMWARE's NIC driver (UEFI SNP)
      # instead of iPXE's own — works on virtio VMs AND real hardware where the
      # native driver won't bind (e.g. EliteBook 820 G4 / Intel I219-LM, which
      # fails ipxe.efi with "No configurable methods"). Built + signed by CI into
      # the fleet-ipxe image (mounted at /ipxe); see serving/tftp/README.md.
      # The fleet-ipxe image is signed with the fleet db key when CI has the
      # secrets, so this path stays Secure-Boot-enforced end-to-end; an unsigned
      # (lab) fleet-ipxe build needs Secure Boot OFF — see docs/secure-boot-keys.md.
      - --enable-tftp
      - --tftp-root=/srv/tftp
      - --dhcp-vendorclass=set:pxe,PXEClient
      - --dhcp-userclass=set:ipxe,iPXE
      # firmware PXE, not yet iPXE -> hand iPXE (UEFI x64) over TFTP
      - --dhcp-boot=tag:pxe,tag:!ipxe,snponly.efi
      # iPXE running -> hand boot.ipxe (NOT the UKI URL directly). The script
      # imgfetch'es the UKI then `imgargs`-clears the load options before boot, so
      # systemd-stub uses the UKI's EMBEDDED cmdline. Booting the UKI URL directly
      # makes iPXE pass the image name as the cmdline, which (Secure Boot OFF)
      # REPLACES the embedded roothash/nvmf/... args -> dracut emergency shell.
      - --dhcp-boot=tag:ipxe,boot.ipxe

  # --- HTTP boot server (serves the UKI) -------------------------------------
  http:
    image: harbor.bitasylum.org/dockerhub/library/nginx:stable
    network_mode: host        # listens on :80 -> http://${SERVER_IP}/fleet-<ver>.efi
    restart: unless-stopped
    depends_on:
      - kiosk                 # /kiosk/ is reverse-proxied to the kiosk SPA
    volumes:
      - type: image
        source: ${OCI_IMAGE}:${IMAGE_VERSION}
        target: /usr/share/nginx/html
        image:
          subpath: artifacts   # mount only /artifacts from the image
      # Fallback (Docker < 28 / offline): bind the local Packer output dir:
      # - ${ARTIFACT_SRC}:/usr/share/nginx/html:ro
      # Config adds the artifact root + the /kiosk/ reverse-proxy location. The
      # kiosk SPA itself is the separate 'kiosk' service below (not bind-mounted).
      - ./http/default.conf:/etc/nginx/conf.d/default.conf:ro
    # For HTTPS boot (production), terminate TLS here with the boot-infra CA's
    # cert and switch the dhcp-boot URL above to https://. Left HTTP for the lab.

  # --- TPM enrolment broker — NOT defined here (lives in the CA stack) --------
  # The broker's home is co-located with step-ca (the fleet CA), i.e.
  # serving/dot1x-bench/, since real signing needs that CA. A `mock`-signer
  # convenience copy used to run here too, but two host-networked brokers collide
  # on 9090/9443 and the mock copy issued certs RADIUS/NVMe don't trust — so it was
  # removed (bench 2026-07-11; see docs/ssh-host-certificate-design.md). The `http`
  # nginx still reverse-proxies /enrol/ -> 127.0.0.1:9090; bring up
  # serving/dot1x-bench on the same host (its broker binds that loopback port) to
  # serve enrolment. Do NOT re-add a second broker here.

  # --- Kiosk SPA (spaceship terminal shown by the diskless kiosk) ------------
  # Svelte + Tailwind app built into a static bundle served by its own nginx.
  # Now an INDEPENDENT project — the `kiosk-app` git submodule (its own repo +
  # CI). It builds + pushes its own image (git.../fleet-kiosk); the serving stack
  # pulls it, like dnsmasq/SPDK. Bridge-networked (NOT host) so it doesn't fight
  # :80; published on loopback only, where the host-network 'http' nginx
  # reverse-proxies /kiosk/ to it.
  kiosk:
    image: ${KIOSK_IMAGE}:${KIOSK_VERSION}
    # Local dev from the submodule working tree instead of pulling:
    # build: ./kiosk-app
    restart: unless-stopped
    ports:
      - "127.0.0.1:${KIOSK_APP_PORT:-8088}:80"   # loopback only; main nginx proxies here
    environment:
      HASS_URL: ${HASS_URL}
      HASS_TOKEN: ${HASS_TOKEN}
  # --- NTP server (serves time to the boot LAN; handed out via DHCP opt 42) ---
  ntp:
    build: ./chrony
    network_mode: host        # serve NTP on udp/123 across the boot LAN
    cap_add:
      - SYS_TIME              # discipline the (host-shared) clock it serves
    restart: unless-stopped
    volumes:
      - ./chrony/chrony.conf:/etc/chrony/chrony.conf:ro
    # NOTE: with host networking + SYS_TIME this chronyd disciplines the Docker
    # HOST clock (the clock isn't namespaced) — intended: the serving host is the
    # lab time source. Drop SYS_TIME (and add `chronyd -x`) if the host already
    # runs its own NTP and you only want this to relay.
    #
    # FIREWALL (easy to miss — same trap as the audit collector's 514/tcp): host
    # networking means Docker does NOT open the port; the serving host's own
    # firewall governs 123/udp. Without a rule, workstations get 192.168.50.10 as
    # their NTP server (DHCP opt 42) but their packets are dropped — chrony logs
    # "Added source … / unreachable" and never syncs. Open it on the boot-LAN NIC:
    #   ufw:       sudo ufw allow in on <boot-nic> to any port 123 proto udp comment 'fleet NTP'
    #   firewalld: sudo firewall-cmd --permanent --add-service=ntp && sudo firewall-cmd --reload

  # --- Remote audit / log collector (RECEIVE side) ---------------------------
  # rsyslog receiver: booted diskless hosts (whose RAM-only overlay root loses
  # local logs on reboot) forward auditd via audisp-syslog + syslog here over
  # TCP/514. Records are written per-source-host under
  # ./audit-collector/logs/<sender-ip>/{audit,messages}.log. Host-networked so
  # hosts can reach it at ${AUDIT_HOST} (= this host) on the boot LAN.
  # NOTE: this is the receiver only — the IMAGE must be configured to send
  # (audisp-syslog + an rsyslog forward rule targeting ${AUDIT_HOST}); see
  # audit-collector/README.md. Until then the collector simply waits, idle.
  audit:
    build: ./audit-collector
    network_mode: host        # receive from booted hosts on the boot LAN (:514)
    restart: unless-stopped
    volumes:
      - ./audit-collector/logs:/var/log/remote      # durable, inspectable on host
      - ./audit-collector/spool:/var/spool/rsyslog  # disk-assisted queue

  # --- NVMe/TCP target (serves rootfs.erofs, read-only) ----------------------
  nvmeof:
    # Prebuilt by CI (.gitea/workflows/spdk-image.yaml) and pulled here — no
    # official spdk/spdk image exists, so we compile our own once in the
    # pipeline. Local-dev alternative: replace this line with `build: ./spdk`.
    image: ${SPDK_IMAGE}:${SPDK_VERSION}
    network_mode: host        # NVMe/TCP listener on ${TARGET_PORT}
    privileged: true          # hugepages + IPC_LOCK/SYS_ADMIN for SPDK
    restart: unless-stopped
    ulimits:
      memlock: -1
    volumes:
      - /dev/hugepages:/dev/hugepages
      # LONG-LIVED target + ONLINE release control (no restart on a new build):
      #   spdk-target  = the persistent entrypoint (nvmf_tgt stays up for life)
      #   spdk-release = JSON-RPC add/remove/list/drain of a version, live
      - ./spdk/spdk-target.sh:/usr/local/bin/spdk-target:ro
      - ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
      # Shared RPC socket so the spdk-loader container (and `compose exec`) can
      # drive this target's JSON-RPC. nvmf_tgt creates the socket here (-r).
      - spdk-sock:/var/run/spdk
      # Multi-version release store: each ./spdk/releases/<ver>/ holds that
      # version's rootfs.erofs + rootfs.hash + verity.json. Staged by the
      # spdk-loader (or spdk-ctl.sh on the host); reconciled into the target at
      # (re)start and added/removed live thereafter. RW so the spdk-loader can
      # stage here. Adding a version does NOT recreate this service.
      - ./spdk/releases:/releases
      # NOTE: the target is deliberately VERSION-AGNOSTIC. It mounts NO
      # ${IMAGE_VERSION} artifact image and carries no IMAGE_VERSION env, so a
      # version bump never changes THIS service's config — `docker compose up -d`
      # after an IMAGE_VERSION bump recreates the loader/http/dnsmasq but leaves
      # nvmf_tgt running (recreating it would tear down EVERY host's NVMe/TCP root
      # at once). Bootstrap AND rolling updates are the spdk-loader's job: it
      # stages the current release into ./spdk/releases and adds it online via
      # JSON-RPC. (spdk-target still seeds from /artifacts IF one is manually
      # mounted + IMAGE_VERSION set — a legacy/standalone convenience, off here.)
      # Legacy single-version entrypoint (configure-at-start, restart-to-change):
      # - ./spdk-setup.sh:/spdk-setup.sh:ro
    environment:
      RELEASES_DIR: /releases
      RPC_SOCK: /var/run/spdk/spdk.sock  # shared with spdk-loader (-r / -s)
      SUBNQN_BASE: ${SUBNQN_BASE}        # per-version NQN base (falls back per release to verity.json .subnqn)
      TARGET_ADDR: ${SERVER_IP}
      TARGET_PORT: ${TARGET_PORT}
      SPDK_CPUMASK: ${SPDK_CPUMASK}
      NVMEOF_SVC: nvmeof
    # Healthy once spdk-target has written its readiness stamp (transport up +
    # releases reconciled). A FILE check, not a python rpc.py probe: nvmf_tgt does
    # a slow DPDK init (~50s observed) and busy-polls its reactor core, so a python
    # healthcheck starves/times out and the container sticks at "unhealthy" even
    # though the target is up. `test -f` is instant and contention-free. The stamp
    # is removed on (re)start + exit, so it never falsely reports a dead target.
    healthcheck:
      test: ["CMD-SHELL", "test -f /var/run/spdk/ready || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 6
      start_period: 120s
    # Persistent target: started ONCE, never restarted for a version change.
    # Rolling updates add/remove versions online (spdk-loader, or spdk-ctl.sh).
    entrypoint: ["/bin/bash", "/usr/local/bin/spdk-target"]

  # --- SPDK version loader / reaper (declarative rolling updates) -------------
  # The "init container that stays up." On `compose up` after an IMAGE_VERSION
  # bump it stages + adds the new rootfs to the running nvmeof target ONLINE
  # (no target restart, no dropped connections), then polls and removes old
  # versions once they drain (all hosts rebooted onto the new UKI). Migration is
  # just: bump IMAGE_VERSION in .env -> `docker compose up -d`. Reuses the SPDK
  # image (has python3 + rpc.py); the scripts are bind-mounted like nvmeof's.
  spdk-loader:
    image: ${SPDK_IMAGE}:${SPDK_VERSION}
    network_mode: host
    restart: unless-stopped
    depends_on:
      nvmeof:
        condition: service_healthy
    volumes:
      - ./spdk/spdk-loader.sh:/usr/local/bin/spdk-loader:ro
      - ./spdk/spdk-release.sh:/usr/local/bin/spdk-release:ro
      - spdk-sock:/var/run/spdk          # shared RPC socket to the target
      - ./spdk/releases:/releases        # shared release store
      # The CURRENT release's artifacts, mounted from its OCI image. The version
      # tag is the single migration knob — bump IMAGE_VERSION and `up -d`.
      - type: image
        source: ${OCI_IMAGE}:${IMAGE_VERSION}
        target: /in
        image:
          subpath: artifacts
      # Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/in:ro
    environment:
      IMAGE_VERSION: ${IMAGE_VERSION}    # the CURRENT version (staged + protected from reaping)
      RELEASES_DIR: /releases
      ARTIFACT_IN: /in
      RPC_SOCK: /var/run/spdk/spdk.sock
      SUBNQN_BASE: ${SUBNQN_BASE}
      TARGET_ADDR: ${SERVER_IP}
      TARGET_PORT: ${TARGET_PORT}
      REAP_OLD: ${REAP_OLD:-true}              # auto-remove drained old versions
      REAP_POLL_SECONDS: ${REAP_POLL_SECONDS:-30}
      REAP_GRACE_SECONDS: ${REAP_GRACE_SECONDS:-300}   # sustained-zero window; keep > host ctrl_loss_tmo
    entrypoint: ["/bin/bash", "/usr/local/bin/spdk-loader"]

# Named volume for the SPDK JSON-RPC unix socket, shared between the long-lived
# nvmeof target (which creates it via nvmf_tgt -r) and the spdk-loader (which
# drives add/remove/drain over it). Keeps the control path off the host FS.
volumes:
  spdk-sock: