# ============================================================================= # Diskless fleet serving stack (lab) # # Stands up the three things a diskless host needs to network-boot: # dnsmasq - DHCP (hands UEFI HTTP-Boot clients the UKI URL) + DNS # http - nginx serving the signed/unsigned UKI over HTTP # nvmeof - SPDK nvmf_tgt serving rootfs.erofs as a READ-ONLY NVMe/TCP # namespace on 4420 (design doc §5, §7) # # This matches docs/diskless-fleet-design-v2.pdf, with two LAB simplifications: # * HTTP (not HTTPS) boot - no TlsCaCertificate to enrol into firmware # * 802.1X / MACsec disabled - pairs with the rd.dot1x=0 test UKI # Do NOT use as-is for a secured fleet. # # Networking: all three services use the host network stack so dnsmasq can see # DHCP broadcasts and SPDK gets a clean NVMe/TCP path. The Docker host must sit # on the same L2 as the booting client (e.g. the test bridge / boot VLAN), and # BOOT_IFACE / SERVER_IP must be that interface and its address. # # Artifacts are mounted straight from the FROM-scratch OCI release image using # a Docker "image volume" (type: image) — the compose analog of the k8s # image-volume source in the design doc (§7). The image's /artifacts dir is # mounted read-only; nothing in it executes, so FROM scratch is fine. Pin by # digest (OCI_IMAGE@sha256:…) for immutable, content-addressed releases. # Requires Docker Engine >= 28.0 and a recent Compose. On older Docker, use # the commented bind-mount fallback in each service (ARTIFACT_SRC = local # Packer output dir), or extract once with: # id=$(docker create $OCI_IMAGE:$IMAGE_VERSION); \ # docker cp $id:/artifacts/. ./artifacts/$IMAGE_VERSION/; docker rm $id # Private registry: docker login git.bitasylum.org first so the image pulls. # # Usage: # cp .env.example .env # then edit BOOT_IFACE / SERVER_IP / DHCP range # docker compose up -d # ============================================================================= services: # --- DHCP + DNS ------------------------------------------------------------ dnsmasq: build: ./dnsmasq network_mode: host cap_add: - NET_ADMIN # DHCP raw sockets - NET_RAW restart: unless-stopped volumes: # DHCP reservations (MAC -> name [-> IP]); drives DHCP option 12 (hostname) # + DNS A/PTR. Auto-populated by fleet-namer (creative names per MAC); add # manual lines to pin specific hosts. RW so fleet-namer can append. - ./dnsmasq/dhcp-hosts:/etc/dnsmasq-hosts command: - --keep-in-foreground # Log to stderr so `docker compose logs` captures output. Without this, # dnsmasq logs to syslog — which doesn't exist in the container — so # --log-dhcp/--log-queries produce nothing even while dnsmasq works fine. - --log-facility=- - --log-dhcp - --log-queries # Bind only to the boot interface so we don't fight the host resolver. - --interface=${BOOT_IFACE} - --bind-interfaces - --except-interface=lo # ---- DNS ---- - --domain=${BOOT_DOMAIN} - --no-resolv - --server=${UPSTREAM_DNS} # NVMe/TCP discovery address used in the UKI cmdline (rd.nvmf.discover): # resolve the per-environment DNS name to this serving host. - --address=/${NVME_HOST}/${SERVER_IP} - --address=/${BOOT_HOST}/${SERVER_IP} # Host-generic remote audit collector name (baked into the image config so # every host offloads auditd here; resolved per-environment to this host). - --address=/${AUDIT_HOST}/${SERVER_IP} # Kiosk is LOCAL-ONLY (loads boot.fleet.lab), so sinkhole Chromium's # call-home to 0.0.0.0 — telemetry can't leave the lab even if the browser # tries. Defense-in-depth behind the in-image Chromium policy/flags; in # production this belongs in the boot-LAN firewall's egress rules. NOTE: the # client still *emits* the query (you'll see it in this log answered locally, # not "forwarded"); stopping the query itself is the browser policy's job. - --address=/google.com/0.0.0.0 - --address=/googleapis.com/0.0.0.0 - --address=/gstatic.com/0.0.0.0 # ---- DHCP ---- - --dhcp-authoritative - --dhcp-range=${DHCP_RANGE_START},${DHCP_RANGE_END},${DHCP_LEASE} - --dhcp-option=option:router,${GATEWAY_IP} - --dhcp-option=option:dns-server,${SERVER_IP} # NTP server (option 42): point diskless hosts at the lab 'ntp' service # below so they sync time internally and never reach the public NTP pool. # The image's NM dispatcher (10-chrony-dhcp-ntp) feeds this into chrony. - --dhcp-option=option:ntp-server,${SERVER_IP} # ---- Per-host names (creative starship hostnames, server-assigned) ---- # The workstation image is hostname-neutral; naming is done HERE. fleet-namer # (the --dhcp-script) derives a stable themed name per MAC (e.g. # nebula-vanguard-07), appends it to the hostsfile, and SIGHUPs dnsmasq; # dnsmasq then sends it as DHCP option 12 (-> NetworkManager sets the # hostname) and registers DNS A/PTR under ${BOOT_DOMAIN}. The host picks up # its name on the NEXT DHCP transaction (renewal at ~half lease, or reboot) # — shorten DHCP_LEASE if you want naming to settle faster on first contact. # --user=root so the dhcp-script can write the hostsfile + signal dnsmasq. - --user=root - --dhcp-hostsfile=/etc/dnsmasq-hosts - --dhcp-script=/usr/local/bin/fleet-namer # ---- UEFI HTTP Boot ---- # Firmware sends vendor-class "HTTPClient"; we must echo it back (opt 60) # and hand over the UKI URL (opt 67). IP in the URL avoids a DNS round-trip # during boot. client-arch 16 = x86-64 UEFI HTTP. - --dhcp-vendorclass=set:httpboot,HTTPClient - --dhcp-option=tag:httpboot,60,HTTPClient - --dhcp-boot=tag:httpboot,http://${SERVER_IP}/fleet-${IMAGE_VERSION}.efi # --- HTTP boot server (serves the UKI) ------------------------------------- http: image: harbor.bitasylum.org/dockerhub/library/nginx:stable network_mode: host # listens on :80 -> http://${SERVER_IP}/fleet-.efi restart: unless-stopped depends_on: - kiosk # /kiosk/ is reverse-proxied to the kiosk SPA volumes: - type: image source: ${OCI_IMAGE}:${IMAGE_VERSION} target: /usr/share/nginx/html image: subpath: artifacts # mount only /artifacts from the image # Fallback (Docker < 28 / offline): bind the local Packer output dir: # - ${ARTIFACT_SRC}:/usr/share/nginx/html:ro # Config adds the artifact root + the /kiosk/ reverse-proxy location. The # kiosk SPA itself is the separate 'kiosk' service below (not bind-mounted). - ./http/default.conf:/etc/nginx/conf.d/default.conf:ro # For HTTPS boot (production), terminate TLS here with the boot-infra CA's # cert and switch the dhcp-boot URL above to https://. Left HTTP for the lab. # --- Kiosk SPA (spaceship terminal shown by the diskless kiosk) ------------ # Svelte + Tailwind app built into a static bundle served by its own nginx. # Bridge-networked (NOT host) so it doesn't fight :80; published on loopback # only, where the host-network 'http' nginx reverse-proxies /kiosk/ to it. # Build: docker compose build kiosk kiosk: build: ./kiosk-app restart: unless-stopped ports: - "127.0.0.1:${KIOSK_APP_PORT:-8088}:80" # loopback only; main nginx proxies here # --- NTP server (serves time to the boot LAN; handed out via DHCP opt 42) --- ntp: build: ./chrony network_mode: host # serve NTP on udp/123 across the boot LAN cap_add: - SYS_TIME # discipline the (host-shared) clock it serves restart: unless-stopped volumes: - ./chrony/chrony.conf:/etc/chrony/chrony.conf:ro # NOTE: with host networking + SYS_TIME this chronyd disciplines the Docker # HOST clock (the clock isn't namespaced) — intended: the serving host is the # lab time source. Drop SYS_TIME (and add `chronyd -x`) if the host already # runs its own NTP and you only want this to relay. # --- Remote audit / log collector (RECEIVE side) --------------------------- # rsyslog receiver: booted diskless hosts (whose RAM-only overlay root loses # local logs on reboot) forward auditd via audisp-syslog + syslog here over # TCP/514. Records are written per-source-host under # ./audit-collector/logs//{audit,messages}.log. Host-networked so # hosts can reach it at ${AUDIT_HOST} (= this host) on the boot LAN. # NOTE: this is the receiver only — the IMAGE must be configured to send # (audisp-syslog + an rsyslog forward rule targeting ${AUDIT_HOST}); see # audit-collector/README.md. Until then the collector simply waits, idle. audit: build: ./audit-collector network_mode: host # receive from booted hosts on the boot LAN (:514) restart: unless-stopped volumes: - ./audit-collector/logs:/var/log/remote # durable, inspectable on host - ./audit-collector/spool:/var/spool/rsyslog # disk-assisted queue # --- NVMe/TCP target (serves rootfs.erofs, read-only) ---------------------- nvmeof: # Prebuilt by CI (.gitea/workflows/spdk-image.yaml) and pulled here — no # official spdk/spdk image exists, so we compile our own once in the # pipeline. Local-dev alternative: replace this line with `build: ./spdk`. image: ${SPDK_IMAGE}:${SPDK_VERSION} network_mode: host # NVMe/TCP listener on ${TARGET_PORT} privileged: true # hugepages + IPC_LOCK/SYS_ADMIN for SPDK restart: unless-stopped ulimits: memlock: -1 volumes: - /dev/hugepages:/dev/hugepages - type: image source: ${OCI_IMAGE}:${IMAGE_VERSION} target: /artifacts # -> /artifacts/rootfs.erofs for spdk-setup.sh image: subpath: artifacts # Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/artifacts:ro - ./spdk-setup.sh:/spdk-setup.sh:ro environment: SUBNQN: ${SUBNQN} TARGET_ADDR: ${SERVER_IP} TARGET_PORT: ${TARGET_PORT} SPDK_CPUMASK: ${SPDK_CPUMASK} entrypoint: ["/bin/bash", "/spdk-setup.sh"]