diff --git a/.gitea/workflows/gitea-main-ci.yaml b/.gitea/workflows/gitea-main-ci.yaml index 21fc77d..37a80d8 100644 --- a/.gitea/workflows/gitea-main-ci.yaml +++ b/.gitea/workflows/gitea-main-ci.yaml @@ -189,6 +189,43 @@ jobs: # depend on the runner's shellcheck finding that rc file. shellcheck --exclude=SC1091,SC2154 files/dracut/95dot1x-macsec/*.sh scripts/*.sh + - + name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.26.4' + + - + name: Build host Go binaries out-of-VM (fleet-enrol, fleet-hostapi) + # Built HERE on the runner, NOT inside the Packer VM, so the Rocky build + # image never carries the Go toolchain or the go-tpm dep tree. Packer + # file-provisions the resulting static binaries into the rootfs. Runs + # BEFORE packer validate/build so the file provisioners' sources exist. + # Reproducible: pinned Go (1.26.4) + go.sum-pinned deps + -trimpath/no buildid. + # fleet-enrol fetches go-tpm from the proxy (pinned by go.sum); hostapi is + # stdlib-only (GOPROXY=off). + # shell: bash is explicit — the Gitea/act runner defaults run steps to sh + # (dash), where `set -o pipefail` is an illegal option. `set -e` is + # load-bearing: without it a failed go build/test in the first ( … ) group + # is masked by the trailing `file` success and a broken binary reaches Packer. + shell: bash + run: | + set -euo pipefail + ( cd files/enrol \ + && CGO_ENABLED=0 GOTOOLCHAIN=local \ + go build -trimpath -buildvcs=false -ldflags '-s -w -buildid=' -o bin/fleet-enrol . \ + && CGO_ENABLED=0 go test ./... ) + ( cd files/hostapi \ + && CGO_ENABLED=0 GOPROXY=off GOTOOLCHAIN=local \ + go build -trimpath -buildvcs=false -ldflags '-s -w -buildid=' -o bin/fleet-hostapi . ) + file files/enrol/bin/fleet-enrol files/hostapi/bin/fleet-hostapi + + - + name: TPM enrolment ceremony test (go-tpm software simulator) + # Exercises the real key-gen/sign/NV path against a software TPM (CGO in a + # separate module, so the simulator never enters the shipped enrol build). + run: ( cd files/enrol/simtest && CGO_ENABLED=1 go test ./... ) + - name: packer init run: packer init . diff --git a/.gitignore b/.gitignore index a05b992..36b336a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,8 @@ keys # and the signed one is db-derived/environment-specific — supply per environment. #/serving/tftp/*.efi /serving/tftp/*.kpxe + +# Out-of-VM CI-built host binaries (staged into the image by the Packer file +# provisioner; built by the "Build host Go binaries" CI step). +files/enrol/bin/ +files/hostapi/bin/ diff --git a/CLAUDE.md b/CLAUDE.md index 8f0c7a3..5a1aed0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,6 +63,11 @@ serving/kiosk-app/ Svelte+Tailwind spaceship-terminal SP OS-side kiosk runtime stays in files/kiosk/ serving/dot1x-bench/ Secure-path bench: step-ca + FreeRADIUS + hostapd authenticator + TPM enrolment + (+ enrol-broker service) +serving/enrol-broker/ TPM identity enrolment broker (Go): the one + unauth pre-auth endpoint (POST CSR -> OOB + fingerprint -> admin mTLS approve -> step-ca + sign). docs/tpm-enrolment-design.md serving/siem/ Wazuh SIEM overlay (audit analysis; agentless relay from audit-collector) .gitea/workflows/ Gitea CI: lint -> packer build -> push OCI @@ -98,10 +103,16 @@ instead of working around it. 1. **No per-host state in any artifact, ever.** No hostnames, keys, certs, MACs, or per-host config in the initramfs, rootfs, UKI, or OCI image. - All host identity derives from the TPM at runtime (key at persistent - handle `0x81010001`, cert at NV index `0x01800001`, hostnqn derived from - the cert serial). If a change needs per-host data, it belongs in the - enrolment flow, not the image. + All host identity derives from the TPM at runtime (signing key at persistent + handle `0x81000100` — a **non-reserved owner handle**, deliberately NOT the + EK slot `0x81010001` nor the SRK `0x81000001`, which firmware/swtpm + provisioning already occupy; cert at NV index `0x01800100` — also chosen to + avoid **OEM platform NV**: a Dell 820 firmware squats `0x01800001`–`0x01800004` + + `0x01810000` as `platformcreate|policydelete` indices that no owner/BIOS + clear can remove, so the original `0x01800001` collided; hostnqn derived from + the cert serial). If a change needs per-host data, it belongs in the enrolment + flow, not the image. (Both are overridable: `rd.dot1x.handle=` / + `rd.dot1x.nvindex=` / `fleet-enrol -handle -nvindex`.) 1. **The hook ordering in `dot1x-macsec-up.sh` is load-bearing.** NV-cert extraction → supplicant start → wait for `Authorized` + macsec0 → DHCP **on macsec0** → hostnqn derivation. Never DHCP before auth diff --git a/README.md b/README.md index e1e5b7b..6dfa41f 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ release only changes which artifact image feeds them: ``` rd.neednet=1 systemd.volatile=overlay \ -rd.dot1x.nvindex=0x01800001 rd.dot1x.timeout=60 \ +rd.dot1x.nvindex=0x01800100 rd.dot1x.timeout=60 \ rd.nvmf.discover=tcp,NVMET_TRADDR,4420 \ root=/dev/mapper/root rootfstype=erofs rootflags=ro \ systemd.verity=yes roothash= \ @@ -137,7 +137,7 @@ first non-loopback interface is used. Test builds prepend `rd.dot1x=0`. (needs a reasonably recent systemd; otherwise publish data and hash as two separate namespaces and drop the hash-offset option). 3. **TPM key reference**: confirm wpa_supplicant accepts - `private_key="handle:0x81010001"` via the OpenSSL tpm2 provider; if not, + `private_key="handle:0x81000100"` via the OpenSSL tpm2 provider; if not, switch to the tpm2-pkcs11 option in the supplicant config. 4. **CONFIG_MACSEC** in the shipped wpa_supplicant build. 5. **Pivot survival**: `@wpa_supplicant` survives switch-root; MKA continuity diff --git a/diskless.pkr.hcl b/diskless.pkr.hcl index 983441c..a0855ad 100644 --- a/diskless.pkr.hcl +++ b/diskless.pkr.hcl @@ -100,6 +100,12 @@ variable "nvmet_traddr" { default = "nvme.fleet.lab" } +variable "enrol_url" { + type = string + description = "Enrolment broker base URL baked into the UKI cmdline (rd.fleet.enrol_url) for the inline empty-NV first-boot enrolment (docs/tpm-enrolment-design.md). A fresh host reaches this pre-auth to get its TPM identity signed; hosts that already hold a cert never use it. Per-environment DNS name resolved at boot (the serving nginx reverse-proxies /enrol/ to the broker). Immutable once the UKI is signed." + default = "http://boot.fleet.lab/enrol" +} + variable "ssh_authorized_key" { type = string description = "Single SSH public key authorized for the sudo-enabled admin user. Fleet-generic (same on every host — not per-host state). Baked into the immutable rootfs, so rotating it needs a rebuild. EMPTY = no admin user is created and root is left UNLOCKED (lab convenience, avoids a no-way-in image)." @@ -240,6 +246,19 @@ build { destination = "/tmp/" } + # fleet-enrol + fleet-hostapi static binaries, built OUT OF THE VM on the CI + # runner (the "Build host Go binaries" step) so the Rocky build image never + # carries the Go toolchain / vendored go-tpm. Staged here; installed to + # /usr/sbin (enrol, before the initramfs build) and /usr/local/bin (hostapi, 2b). + provisioner "file" { + source = "files/enrol/bin/fleet-enrol" + destination = "/tmp/fleet-enrol" + } + provisioner "file" { + source = "files/hostapi/bin/fleet-hostapi" + destination = "/tmp/fleet-hostapi" + } + # Custom 'fleet' Plymouth theme (scripted security-handshake splash). Staged to # /tmp/plymouth, installed + set default in the package step before dracut runs. provisioner "file" { @@ -271,6 +290,10 @@ build { # zstd: dracut compresses the initramfs with compress="zstd" # (90-diskless.conf); the minimal ISO lacks the zstd binary, so without it # dracut warns and silently falls back to gzip (a larger initramfs). + # tpm2-tools: tpm2_nvread reads the client cert from NV on the secure path. + # (First-boot ENROLMENT no longer needs tpm2-tools / an openssl tpm2 provider + # / curl in the image — that's the out-of-VM fleet-enrol Go binary, which + # talks to the TPM directly via go-tpm. So this list stays at the baseline.) "dnf -y install wpa_supplicant tpm2-tools tpm2-tss nvme-cli cryptsetup dracut-network NetworkManager zstd openssl", # EFI / Secure Boot management tools (efivar, mokutil, efibootmgr — all base @@ -367,6 +390,12 @@ build { # No local bootloader regeneration wanted on kernel updates inside CI "command -v grub2-mkconfig >/dev/null && echo 'GRUB updates not used on diskless hosts' || true", + # Install the out-of-VM-built enrolment binary BEFORE the initramfs build so + # the dracut module (95dot1x-macsec) can inst it into the initramfs. Non-RPM + # exec => fapolicyd-trusted in 2c-STIG-postfix (like kiosk-session/hostapi). + "install -D -m0755 /tmp/fleet-enrol /usr/sbin/fleet-enrol", + "restorecon /usr/sbin/fleet-enrol 2>/dev/null || true", + # ---- Build the fleet-generic initramfs ------------------------------ # %%{ escapes the rpm query tags so Packer's HCL template engine treats # them as literals (a bare %{...} is parsed as a template directive). @@ -387,6 +416,12 @@ build { # a dracut hook). "grep -q 'nm-unmanaged.sh' /tmp/lsinitrd.txt || { echo 'ERROR: nm-unmanaged.sh hook missing from initramfs' >&2; exit 1; }", "grep -q 'nvmf-connect.service' /tmp/lsinitrd.txt || { echo 'ERROR: nvmf-connect.service missing from initramfs' >&2; exit 1; }", + # Assert the inline empty-NV enrolment path is present: the gate hook + the + # out-of-VM fleet-enrol binary (Go + go-tpm; it does key-gen/CSR/NV directly, + # so no tpm2-tools/openssl-provider/curl are needed in the initramfs). A fresh + # host silently unable to enrol is a bricked host. + "grep -q 'tpm-enrol.sh' /tmp/lsinitrd.txt || { echo 'ERROR: tpm-enrol.sh hook missing from initramfs' >&2; exit 1; }", + "grep -q 'fleet-enrol' /tmp/lsinitrd.txt || { echo 'ERROR: fleet-enrol binary missing from initramfs (out-of-VM build not staged?)' >&2; exit 1; }", # Plymouth must be in the initramfs to splash during the early boot phase. "grep -q 'plymouthd' /tmp/lsinitrd.txt || { echo 'ERROR: plymouth missing from initramfs (plymouth dracut module not installed)' >&2; exit 1; }", @@ -465,29 +500,21 @@ build { } # --- 2b-hostapi. Workstation host-info API (Go, loopback, read-only) ------ - # Compiles files/hostapi into /usr/local/bin/fleet-hostapi and enables it as a - # systemd service the kiosk SPA reads at http://localhost:9099/v1/host (the - # HOST-API.md contract). Build-time, image-GENERIC facts (image version + - # weston/chromium versions) are written to /etc/fleet-hostinfo.json; everything - # host-specific (kernel, hostname, IP, MAC, board, uptime) is read at RUNTIME, - # so NO per-host state lands in the image (invariant #1). The Go toolchain is - # installed here and removed in 2d hygiene; it never ships. + # Installs the OUT-OF-VM-built /usr/local/bin/fleet-hostapi (the "Build host Go + # binaries" CI step; staged to /tmp/fleet-hostapi) and enables it as a systemd + # service the kiosk SPA reads at http://localhost:9099/v1/host (the HOST-API.md + # contract). Build-time, image-GENERIC facts (image version + weston/chromium + # versions) are written to /etc/fleet-hostinfo.json; everything host-specific + # (kernel, hostname, IP, MAC, board, uptime) is read at RUNTIME, so NO per-host + # state lands in the image (invariant #1). No Go toolchain in the VM anymore — + # the reproducible static build happens on the CI runner (like fleet-enrol). provisioner "shell" { environment_vars = [ "IMAGE_VERSION=${var.image_version}", ] inline = [ "set -euxo pipefail", - "dnf -y install golang", - # Reproducible, dependency-free static build (the verity hash must be a pure - # function of the inputs — invariant #8). GOPROXY=off + GOTOOLCHAIN=local - # guarantee no network/toolchain fetch; -trimpath + -buildid= strip paths - # and the build id. Stdlib-only, so this needs no modules. NOTE: a golang - # package update changes the binary (hence the hash) — a reproducibility- - # sensitive input, like the STIG semanage store; build twice to confirm. - "install -d -m0755 /tmp/gocache", - "env -C /tmp/hostapi CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOPROXY=off GOTOOLCHAIN=local GOFLAGS=-mod=mod GOCACHE=/tmp/gocache GOPATH=/tmp/gopath go build -trimpath -buildvcs=false -ldflags '-s -w -buildid=' -o /usr/local/bin/fleet-hostapi .", - "chmod 0755 /usr/local/bin/fleet-hostapi", + "install -D -m0755 /tmp/fleet-hostapi /usr/local/bin/fleet-hostapi", "install -D -m0644 /tmp/hostapi/fleet-hostapi.service /etc/systemd/system/fleet-hostapi.service", # Build-time facts file (image-generic; same for every host of a release). # rpm '%%{...}' is doubled so Packer's HCL2 doesn't parse %{ as a directive. @@ -497,7 +524,7 @@ build { "chmod 0644 /etc/fleet-hostinfo.json", "restorecon -R /usr/local/bin/fleet-hostapi /etc/systemd/system/fleet-hostapi.service /etc/fleet-hostinfo.json 2>/dev/null || true", "systemctl enable fleet-hostapi.service", - "rm -rf /tmp/hostapi /tmp/gocache /tmp/gopath", + "rm -rf /tmp/hostapi /tmp/fleet-hostapi", "echo '==> fleet-hostapi enabled (localhost:9099/v1/host)'", ] } @@ -689,6 +716,20 @@ build { "evaluated=$(grep -oE '(pass|fixed|fail)' /root/stig-arf.xml | wc -l)", "[ \"$evaluated\" -gt 0 ] || { echo 'ERROR: STIG scan evaluated 0 rules (all notapplicable/notselected) — SSG datastream/CPE mismatch for this distro. Hardening did NOT run; aborting.' >&2; exit 1; }", "echo \"==> STIG scan evaluated $evaluated rules (pass/fixed/fail)\"", + # SELinux: allow chronyd to reply to the chronyc our NM dispatcher runs for + # DHCP option-42 NTP (10-chrony-dhcp-ntp -> `chronyc reload sources`). The + # dispatcher's confined domain (NetworkManager_dispatcher_custom_t) labels + # chronyc's reply socket, and the shipped policy denies chronyd_t sendto to + # it -> AVC flood + failed reload under enforcing. Ship a minimal allow + # module (docs/stig-deviations.md). NB: rebuilds the policy store — include + # in the reproducibility check alongside semanage (verity hash stability). + "echo '==> supplemental: SELinux module chrony_dhcp_dispatcher (chronyd reply to NM-dispatcher chronyc)'", + "command -v checkmodule >/dev/null 2>&1 || dnf install -y checkpolicy", + "checkmodule -M -m -o /tmp/chrony_dhcp_dispatcher.mod /tmp/stig/chrony-dhcp-dispatcher.te", + "semodule_package -o /tmp/chrony_dhcp_dispatcher.pp -m /tmp/chrony_dhcp_dispatcher.mod", + "semodule -i /tmp/chrony_dhcp_dispatcher.pp", + "rm -f /tmp/chrony_dhcp_dispatcher.mod /tmp/chrony_dhcp_dispatcher.pp", + "semodule -l | grep -q '^chrony_dhcp_dispatcher' && echo '==> SELinux module chrony_dhcp_dispatcher installed' || { echo 'ERROR: chrony_dhcp_dispatcher module not installed' >&2; exit 1; }", "rm -rf /tmp/stig", "echo '==> STIG remediation complete (report at /root/stig-report.html, ARF at /root/stig-arf.xml)'", ] @@ -751,7 +792,7 @@ build { # custom kiosk files: add them to fapolicyd's trust DB and rebuild it. weston # /chromium are RPM-installed so already trusted. (See docs/stig-deviations.md.) "if command -v fapolicyd-cli >/dev/null 2>&1; then", - " for f in /usr/local/bin/kiosk-session /usr/local/bin/fleet-sb-enroll /usr/local/bin/fleet-hostapi; do { [ -f \"$f\" ] && fapolicyd-cli --file add \"$f\"; } || true; done", + " for f in /usr/local/bin/kiosk-session /usr/local/bin/fleet-sb-enroll /usr/local/bin/fleet-hostapi /usr/sbin/fleet-enrol; do { [ -f \"$f\" ] && fapolicyd-cli --file add \"$f\"; } || true; done", " fapolicyd-cli --update 2>/dev/null || true", " echo '==> fapolicyd: trusted custom binaries (kiosk-session, fleet-sb-enroll, fleet-hostapi) + rebuilt trust db'", "fi", @@ -826,9 +867,19 @@ build { # already downloaded to the host) — slims the erofs + cuts attack surface. # Keep auditd (it's part of the hardened runtime). Best-effort. "dnf remove -y openscap-scanner scap-security-guide openscap-utils 2>/dev/null || true", - # The Go toolchain was only needed to compile fleet-hostapi (2b-hostapi); - # the static binary ships, the compiler does not. - "dnf remove -y golang 2>/dev/null || true", + # firewalld: the kickstart EXCLUDES it (-firewalld, firewall --disabled) — + # this fleet uses network-layer security (802.1X/MACsec + switch); the host + # firewall is a documented OFF item (docs/stig-deviations.md #4). But a + # provisioner dnf pulled it back as a WEAK dep (Recommends) and, once + # installed, it starts by default. Its Python then tries to rewrite + # site-package .pyc bytecode — which the reproducible erofs `-T 0` mtime + # reset makes look stale — so firewalld_t gets write/unlink AVCs on lib_t + # that flood the SIEM (seen 2026-07-03). Remove it (weak dep, nothing + # hard-needs it) and mask as a backstop against future weak-dep reinstalls. + "dnf remove -y firewalld 2>/dev/null || true", + "systemctl mask firewalld.service 2>/dev/null || true", + # (No Go toolchain to remove — fleet-hostapi + fleet-enrol are built OUT OF + # the VM on the CI runner and file-provisioned in as static binaries.) # The build-time STIG reports must not ship inside the read-only rootfs. "rm -f /root/stig-report.html /root/stig-arf.xml", "dnf clean all", @@ -855,7 +906,7 @@ build { # home dirs) — we dropped mkfs.erofs --all-root, which used to mask this by # flattening everything to root:root. sudo is a no-op if already root, and # the runner has passwordless sudo (cf. the `sudo apt-get` install step). - "sudo bash -c \"IMAGE_VERSION='${var.image_version}' NVMET_TRADDR='${var.nvmet_traddr}' BOOT_SPLASH='${var.boot_splash}' STIG='${var.enable_stig}' bash scripts/extract-artifacts.sh output-qemu/diskless-build.qcow2 ${var.artifact_dir}/${var.image_version} ${var.disable_dot1x}\"", + "sudo bash -c \"IMAGE_VERSION='${var.image_version}' NVMET_TRADDR='${var.nvmet_traddr}' ENROL_URL='${var.enrol_url}' BOOT_SPLASH='${var.boot_splash}' STIG='${var.enable_stig}' bash scripts/extract-artifacts.sh output-qemu/diskless-build.qcow2 ${var.artifact_dir}/${var.image_version} ${var.disable_dot1x}\"", # Hand the OUTPUT FILES back to the build user (their in-erofs ownership is # already baked correctly) so build-oci.sh can write the UKI and push with # the build user's `buildah login`. diff --git a/docs/TODO.md b/docs/TODO.md index 922c6bc..a0c251d 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -675,6 +675,23 @@ must be built around **never restarting the process that holds live connections. the current UKI), so auto-reap is safe; removal is still drain-gated + grace-debounced. The loader + target share the RPC socket on a named volume (`nvmf_tgt -r`); the loader reuses the SPDK image (has python3 + rpc.py). + - **BENCH FINDING 2026-07-03 (FIXED) — the target WAS still recreated on an + `IMAGE_VERSION` bump.** Despite the above, the `nvmeof` (spdk-target) service + still carried a versioned `type: image` bootstrap mount + (`source: ${OCI_IMAGE}:${IMAGE_VERSION}` → `/artifacts`) **and** an + `IMAGE_VERSION` env, so bumping the version changed THIS service's config and + `compose up -d` recreated it → `nvmf_tgt` restarted → **every connected host's + NVMe/TCP root dropped** (the exact fleet-wide outage the long-lived-target + design exists to prevent). Fix: **made the target version-agnostic** — removed + the `/artifacts` image-volume + `IMAGE_VERSION` env from `nvmeof` in + `serving/docker-compose.yml`. Bootstrap AND rolling updates are now solely the + `spdk-loader`'s job (stages the current release into `./spdk/releases`, adds it + online); `spdk-target.sh`'s seed block is guarded + (`[ -s /artifacts/verity.json ] && [ -n IMAGE_VERSION ]`) so it no-ops when + unmounted. Verified `nvmeof` no longer references `IMAGE_VERSION` (only + `spdk-loader` does). **BENCH-VERIFY:** bump `IMAGE_VERSION` + `up -d` with a + host connected → `nvmeof` container NOT recreated (`docker ps` ID/uptime + unchanged), host root survives, loader adds the new version online. - **No jq dependency:** `spdk-release.sh` parses RPC JSON with **python3** (the SPDK image ships no jq); legacy artifacts (no per-version UUIDs in verity.json) fall back to the fixed UUID pair with a warning. @@ -766,7 +783,7 @@ smoke test. returns MSK); a `hostapd` wired authenticator does 802.1X + MACsec on an **isolated bridge per workstation** (flat VLAN can't auth — EAPOL dropped + MACsec is P2P). `enrol/tpm-provision.sh` creates the key in the TPM at - `0x81010001` + writes the cert to NV `0x01800001`. Starting solution **signs a + `0x81000100` + writes the cert to NV `0x01800100`. Starting solution **signs a CSR** (key stays in TPM); **TPM attestation** (ACME `device-attest-01`) is the documented upgrade on the same CA. **BENCH-VERIFY** items flagged in the files: hostapd MACsec build, openssl tpm2 provider vs pkcs11 key ref, EAP-TLS version. @@ -775,7 +792,7 @@ smoke test. capability check (run on the booted open-port image, no `disable_dot1x` needed) to de-risk the TPM half before the full path. Likely first finding: no openssl TPM2 **provider** in the image (`tpm2-tss` ≠ the provider) — add `tpm2-openssl`. - - **Enrolment of the per-host identity** (key@`0x81010001` + cert@NV) is its own + - **Enrolment of the per-host identity** (key@`0x81000100` + cert@NV) is its own design — see **#11** (TOFU + OOB fingerprint broker, pre-auth permit-set, provisioning UKI). `tpm-provision.sh` is the manual bench stand-in for it. - **NVMe discovery by DNS:** `nvmet_traddr=nvme.fleet.lab` (CI), resolved at boot @@ -968,7 +985,7 @@ enrolls. every NIC/GPU option ROM is covered by your keys — clearing MS db can break signed **option ROMs** (NIC PXE/HTTP-boot, GPU init), bricking the boot path. - **PCR 7 check:** enrollment changes the Secure Boot measurement in PCR 7; - confirm nothing the fleet seals to the TPM (the EAP-TLS key at `0x81010001`, + confirm nothing the fleet seals to the TPM (the EAP-TLS key at `0x81000100`, NV cert) is PCR-7-policy-bound before relying on this. **Decision before building:** auto-enroll trades assurance for convenience and is @@ -978,8 +995,191 @@ UKI + isolated VLAN + read-back verify.** ## [ ] 11. (DESIGN) TPM 802.1X identity enrolment (TOFU + OOB fingerprint) +**Authoritative design consolidated 2026-06-28: +[tpm-enrolment-design.md](tpm-enrolment-design.md)** — unifies this item with the +fleet-manager (the broker = the fleet-manager's one unauthenticated bootstrap +endpoint; approval births the inventory record) and the full identity lifecycle +(enrol / renew / alert). **v1 decisions (2026-06-28):** fingerprint-only OOB +assurance (TPM attestation via ACME `device-attest-01` is the documented later +upgrade); **inline single-UKI** vehicle for now (empty-NV early-boot branch — keeps +provisioning network-boot-only, no USB; a separate provisioning UKI behind an +`enrol_mode` build flag is the planned later upgrade); enrolment transport is plain +**stdlib HTTP/JSON** (Go, the one unauthenticated edge — gRPC reserved for the +steady-state agent path). The notes below are retained as the originating decisions; +the design note is canonical. + +**Broker BUILT (server side, 2026-06-28): [serving/enrol-broker/](../serving/enrol-broker/)** +— Go/stdlib; public unauth `POST /request`+`GET /status/{id}` (behind the serving +nginx `/enrol/`) + admin **mTLS web UI** (`/`, `/pending`, `/approve|deny/{id}`) +with a **dedicated admin CA** (`admin-pki.sh`); wordlist fingerprint, time-boxed +store, per-IP rate limit; signs via `step` (dev `mock` signer). Ships in +`serving/dot1x-bench/docker-compose.yml`; end-to-end smoke-tested (CSR → mTLS +approve → cert). **BENCH-VERIFY:** run against real step-ca (`bootstrap.sh` root + +`enrol` provisioner password readable by the broker) and confirm an issued cert +authenticates EAP-TLS. + +**HOST SIDE BUILT 2026-07-02 — Go binary (`fleet-enrol`), awaiting bench.** +Pivoted from a shell hook to a static **Go + go-tpm** binary (`files/enrol/`): +it talks to `/dev/tpmrm0` DIRECTLY (via `transport.FromReadWriter` on an +`os.File` — no `linuxtpm`, so go-tpm is the only vendored dep), so the enrolment +path needs **no tpm2-tools key-gen / no openssl tpm2 provider / no curl** in the +image (removes the biggest bench risk — the provider `handle:` key-ref). It +ensures the ECDSA key at `0x81000100`, builds the CSR, POSTs CSR+context to +`rd.fleet.enrol_url`, shows the broker fingerprint on Plymouth, polls `/status`, +and writes the leaf cert to NV `0x01800100`. The dracut hook `tpm-enrol.sh` +(initqueue/settled prio 25) is now a thin GATE: empty-NV check (`tpm2_nvread`, +still needed for the secure-path read) → pre-auth DHCP on `en*` (nmcli override of +the secure-mode unmanaged drop-in) → `exec /usr/sbin/fleet-enrol` → reboot to +converge on exit 0. No-op when a cert is present or under `rd.dot1x=0`. + +**Built OUT OF THE VM (2026-07-02, per decision):** `fleet-enrol` AND +`fleet-hostapi` now compile on the CI runner (`.gitea/workflows/gitea-main-ci.yaml` +"Build host Go binaries" step, Go 1.26.4, `CGO_ENABLED=0`, vendored/stdlib, +`GOPROXY=off`), and Packer **file-provisions** the static binaries in — the Rocky +build image no longer carries the Go toolchain (dropped the in-VM `dnf install +golang` + the 2d removal). `fleet-enrol` lands at `/usr/sbin` before the initramfs +build (dracut `inst_simple`s it; asserted in `lsinitrd`), fapolicyd-trusted. +New `enrol_url` Packer var → `rd.fleet.enrol_url` cmdline. + +**TEST-MODE ENROLMENT ENABLED (2026-07-02, temporary):** the hook no longer +no-ops under `rd.dot1x=0` — enrolment runs on the open/test path too, so it can be +exercised on a host with a TPM + the broker WITHOUT standing up +802.1X/MACsec/RADIUS. In test mode `ip=dhcp` owns the NIC (the hook skips its own +DHCP to avoid the dracut "address already assigned" race) and DNS resolves the +broker via the lease. `rd.fleet.enrol=0` disables it. Production intent is still +secure-path-only; revisit when the secure path is bench-verified. + +**TESTED (2026-07-02):** transaction logic (CSR → mock broker → approve → cert +extract) as stdlib unit tests, AND the **full TPM ceremony (key-gen → persist → +ECDSA sign → CSR verify → chunked NV write → re-enrol) against the go-tpm software +simulator** (separate `files/enrol/simtest/` module so the CGO simulator never +enters the shipped build). Both run in CI. + +**BENCH FINDING 2026-07-02 (FIXED) — the signing-key handle `0x81000100` was the +EK slot; moved to `0x81000100`.** First real-TPM boot (host `cygnus-cruiser-99`, +Proxmox vTPM, `rd.dot1x=0`) failed: `fleet-enrol: ensure key at 0x81010001: key at +handle is not ECC: did not contain eccDetail (selector value was 1)`. `EnsureKey` +died reading an existing RSA object as ECC → enrolment died before the CSR/POST, so +the splash flashed "FIRST-BOOT TPM ENROLMENT" then the test system booted with **no +fingerprint and nothing at the broker**. Root cause (confirmed via `tpm2_readpublic` +under `enforcing=0`): **`0x81000100` is the TCG-standard RSA Endorsement Key handle** +— the object there was the swtpm-provisioned **EK** (`restricted|decrypt|adminwithpolicy`, +RSA-2048/AES128CFB, canonical EK authPolicy `837197674484b3f8…`), with its EK cert at +NV `0x1c00002`. Our design squatted the EK slot fleet-wide (most firmware/swtpm TPMs +have their EK there), so enrolment could never create our key. + - **Fix (commit c574fa4 + follow-up):** the fleet signing key moved to + **`0x81000100`** (non-reserved owner handle; threaded through `fleet-enrol + -handle` default, `rd.dot1x.handle=` cmdline knob baked by `extract-artifacts.sh`, + `wpa_supplicant-dot1x.conf`, the bench scripts, invariant #1, and the design docs). + NV index was `0x01800001` here (owner range); it MOVED to `0x01800100` in a later finding (OEM platform-NV collision on the 820 — see below). + - **EnsureKey hardening:** (i) evicts a genuine *stale signing key* at the handle + and recreates (self-heal for old provisioning leftovers); (ii) **refuses to evict + any restricted-decrypt object (EK/SRK)** as a safety net — never clobber the TPM's + endorsement/storage identity (also protects the future EK/AK attestation). + Regressions: `simtest/TestEnsureKeyEvictsStaleNonECC` + `TestEnsureKeyRefusesToEvictProtectedKey`. + - **IDevID note (future):** if EK/AK attestation (`device-attest-01`, §"Stronger + assurance") lands, the natural home for a TPM device-identity key is the TCG + **IDevID** handle `0x81020001` (LDevID `0x81030001`) — see tpm-enrolment-design.md. + Not adopted for v1 (implies the IDevID template/cert conventions). + - **Also seen — leftover NV suppresses enrolment:** a *different* host (the 820) + booted `tpm-enrol: client cert present at NV 0x01800001 — normal boot` with nothing + enrolled this session — leftover NV state (from an earlier `tpm-provision.sh` / + `tpm-selftest.sh --use-real-handles`) **persists in the hardware TPM** across + reinstalls. Our presence check is content-blind (index defined ⇒ "enrolled"), so + foreign/stale NV silently skips enrolment (→ secure-path auth would then fail). See + the **content-aware NV check** item below. + - **Needs a rebuild + re-boot to confirm** (the sim proves the ceremony, not the + device). ⚠ **Don't hand-run `sudo fleet-enrol` on the booted STIG host** — the + confined admin domain (`sysadm_t`) is SELinux-**denied** write to `/dev/tpmrm0` + (`avc: denied … tpm_device_t`); the faithful path is the **initramfs** run + (reboot, unconfined), or `enforcing=0` for hand-run TPM inspection. + +**FOLLOW-UP HARDENING:** +- **Content-aware NV presence check — BUILT 2026-07-03.** `fleet-enrol -check-only` + (no network / no key-gen) reads NV `0x01800100` via `tpmkey.ReadNVCert`, parses it as + X.509, checks the validity window, and — when a real fleet CA is bundled (`-cacert`, + default `radius-ca.pem`) — verifies it **chains** to it. Only a valid fleet cert counts + as enrolled (rc 0); empty/garbage/foreign/expired → rc 20 (enrol + overwrite; + `WriteNVCert` undefine+redefines). The dracut hook now calls it as the single gate, + replacing the blind `tpm2_nvread` (removed from `tpm-enrol.sh`; the binary's `NVPresent` + stays for cheap existence checks). **Lab fallback:** the shipped `radius-ca.pem` is a + *placeholder* (comments, not a cert) → `loadCAPool` returns nil → chain-less "parseable + + unexpired" acceptance, so the reboot-converge doesn't loop on a bench-CA cert; + **production must replace `radius-ca.pem` with the real fleet CA** for strict chaining. + Tests: `classifyNVCert` unit table (valid/expired/garbage/foreign/no-CA) + `simtest` + ReadNVCert roundtrip across the 512B chunk. **BENCH-VERIFY** on a real boot: a garbage/ + foreign NV now enrols instead of "normal boot". +- **De-enrol / TPM-reset procedure** for re-testing: `tpm2_nvundefine 0x01800100 -C o` + (+ `tpm2_evictcontrol -C o -c 0x81000100` for a stale *app* key — never the EK), or + BIOS "Clear TPM". Document in the bench README. + +**BENCH FINDING 2026-07-03 (FIXED) — NV index `0x01800001` collided with OEM +platform NV; moved to `0x01800100`.** The Dell 820 kept "finding a cert" and +bypassing enrolment even after repeated BIOS TPM clears. `tpm2_nvreadpublic` +showed the truth: `0x01800001` on that hardware is a **firmware/platform** index +(`platformcreate|policydelete|writelocked`, 70 bytes — NOT our cert; siblings at +`0x01800003/4` + `0x01810000` share it), which **no owner/BIOS clear can remove** +(`TPM2_Clear` spares `platformcreate`; `policydelete` needs the platform's secret +policy). So our chosen index was permanently occupied by Dell — the blind check +false-positived on it, and even the content-aware check would have failed at the +NV **write** (can't undefine a platform index). Same class as the EK-handle +collision. Fix: **moved the NV cert index default `0x01800001` → `0x01800100`** +(free on the 820; owner range; overridable via `rd.dot1x.nvindex=`), threaded +through the cmdline (`extract-artifacts.sh`), `fleet-enrol -nvindex`, the dracut +hooks, bench scripts, invariant #1, and docs. Also **hardened `WriteNVCert`** to +refuse a `platformcreate` index with a clear "pick a free owner index" error +instead of a confusing undefine failure (regression `simtest/ +TestWriteNVCertRefusesPlatformIndex`). **BENCH-VERIFY:** 820 now enrols at +`0x01800100`; and check `tpm2_nvreadpublic` on any OTHER hardware model in the +fleet — if a vendor also squats `0x018001xx`, pick a value free on all of them. + +**BENCH-VERIFY (host side, before trusting):** (a) go-tpm against a REAL TPM + +swtpm on the target (the simulator proves the ceremony, not the device) — **the +stale-key fix (above) is the first real-TPM result; re-boot to confirm it now +reaches the broker**; (b) empty-NV create/persist on a fresh TPM with **empty owner +auth** (the code assumes it — a set owner auth needs handling); (c) pre-auth DHCP on `en*` in secure mode +(the `nmcli set managed yes` override of nm-unmanaged.sh actually leases); (d) +fingerprint renders on the `fleet` Plymouth theme + admin can match it; (e) NV +write + **reboot converges** (2nd boot runs the secure path); (f) prod: broker over +**HTTPS** (`fleet-enrol -cacert radius-ca.pem`), currently lab-HTTP. + +**TEMPORARY (2026-06-28): broker also added to the MAIN serving stack** +(`serving/docker-compose.yml`, `enrol-broker` service + `ENROL_*` in +`serving/.env.example`) so a single `compose up` in `serving/` brings the +`/enrol/` endpoint up while the host side is built. Defaults to the +zero-dependency lab path — **`mock` signer** (ephemeral CA; certs NOT trusted by +RADIUS/NVMe) + **plaintext loopback admin** (`ENROL_ADMIN_INSECURE=true`). The +serving nginx already proxies `/enrol/`→`127.0.0.1:9090`. **SPLIT BACK OUT:** the +broker's proper home is a CA-co-located stack (it's still defined in +`dot1x-bench/` next to step-ca); real signing needs the fleet CA. Don't run both +copies on one host (they collide on 9090/9443). To use real signing from the +serving stack meanwhile: set `ENROL_SIGNER=step` + `ENROL_STEP_CA_URL` at a +reachable step-ca, populate `serving/enrol-pki/`, and `ENROL_ADMIN_INSECURE=false`. + +**STEP SIGNER VERIFIED end-to-end 2026-06-28** (broker → real step-ca → signed +cert that chains to the fleet root; hostnqn derived from serial; EKU clientAuth). +**Re-VERIFIED ON THE SERVING HOST 2026-07-02** against step-ca 0.30.2 via +`verify-serving.sh` through the nginx `/enrol/` proxy: issuer = Fleet Bench CA +Intermediate, cert validity exactly 720h (confirms the maxTLSCertDuration claim +raise took effect), EKU clientAuth, `enrol` provisioner extension present, +hostnqn = cert serial. Bring-up gotchas hit + fixed (all in the commits): the CA +wouldn't init (compose used `_PASSWORD_FILE` for a nonexistent file → use +`_PASSWORD`); Docker auto-created `./pki` root-owned (bootstrap self-heals perms + +pre-create the bind-mount dirs); the CLI couldn't resolve `ca.fleet.lab` on the +serving host (connect via `localhost`, which is in the CA SAN). +Verification found + fixed two things the `mock` path had hidden: +- **BUG FIXED (`signer.go`):** `--not-after` was passed to `step ca token` + (validity capped ~1h → "token validity too long") instead of `step ca sign`. + Moved to the sign step. Without it the step signer never issues a cert. +- **CA CLAIM REQUIRED:** a stock step-ca caps certs at 24h, so 720h is rejected + ("authorized maximum certificate duration of 24h"). `dot1x-bench/step-ca/bootstrap.sh` + now raises `authority.claims.maxTLSCertDuration` (+restarts the CA) — this also + unblocks bootstrap's own 720h RADIUS cert. Any other CA needs the same claim. +- **BENCH-VERIFY still open:** issued cert actually authenticates EAP-TLS through + FreeRADIUS (next, before host enrolment). + How a host gets its **802.1X EAP-TLS client identity** the first time — the key at -TPM handle `0x81010001` + cert at NV `0x01800001` that invariant #1 says must come +TPM handle `0x81000100` + cert at NV `0x01800100` that invariant #1 says must come from enrolment, never the image. Distinct from #10 (that enrols *Secure Boot* PK/KEK/db); this enrols the *network identity*. They share the provisioning UKI + isolated-environment vehicle. Pairs with #8 + the CA decision (start **step-ca**, @@ -990,7 +1190,7 @@ on an empty TPM the host generates a key *in* the TPM, builds a CSR, submits it an enrolment **broker**, and shows a **fingerprint** on the Plymouth splash; an admin visually cross-checks that fingerprint against the broker's pending-list and approves; the broker has step-ca sign the CSR and returns the cert; the host writes -it to NV `0x01800001` and proceeds to authenticate. Sound, recognised pattern +it to NV `0x01800100` and proceeds to authenticate. Sound, recognised pattern (SSH/WireGuard/Signal-style OOB verification); good fit for invariant #1. **Key design decisions (settled 2026-06-22):** @@ -1143,3 +1343,116 @@ desired-state pull (version pin, reboot/drain, kiosk URL) wired to the deploymen manifest + SPDK flow. Build-time decisions (telemetry encoding, metrics/desired- state store, per-host vs cohort, osquery/FleetDM eval) listed in the design note. Depends on the step-ca fleet CA (#8) for the mTLS identity. + +## [ ] 15. Scope the root overlay → per-subtree tmpfs/overlay (un-shadowable `/usr` + STIG mount options) + +**DESIGN CAPTURED 2026-07-02.** Today the whole root is one `systemd.volatile=overlay` +tmpfs over `/` (see #5, invariant on the volatile overlay). Replace it with a +**read-only verity root + a handful of scoped writable mounts**, for two payoffs +that land from the same change: + +1. **Security — un-shadowable `/usr`.** dm-verity seals the backing store, but the + whole-root overlay lets any root process write a tmpfs *copy* over a system + binary (e.g. `/usr/sbin/fleet-enrol`) that then executes in preference to the + sealed one until reboot. fapolicyd is the current compensating control (CLAUDE.md: + "the writable-overlay exec guard dm-verity can't provide"). Keeping `/usr` (and + `/`, `/opt`, `/boot`) strictly **read-only** makes the executable set both + verity-sealed *and* un-shadowable — defence in depth, shrinks fapolicyd reliance. + Also: writes outside the sanctioned set fail loudly (`EROFS`) instead of silently + landing in RAM and vanishing at reboot — enforces invariant #1 (no per-host state) + architecturally. + +2. **STIG — converts the partition/mount family from `notapplicable`/waived to real + `pass`.** Quantified against the scan in `reports/stig-arf.xml` (NOT estimates): + the `mount_option_*` rules currently score **`notapplicable`** (no separate mount + to check — the "false-clean via notapplicable" pattern #2 warns about), and + `partition_for_*` are **`notselected`** (waived in `diskless-tailoring.xml` + lines 61-66). Scoped tmpfs mounts with `noexec,nosuid,nodev` + re-selecting the + partition rules convert **~21 rules to genuine `pass`**: + - 15 `mount_option_*` N/A→pass: `/tmp` (nodev,nosuid[,noexec*]), `/var/tmp` + (all 3), `/var/log` (all 3), `/var/log/audit` (all 3), `/var` (nodev), `/home` + (nodev,nosuid). + - 6 `partition_for_*` re-selects→pass: tmp, var, var_log, var_log_audit, var_tmp, + home (+1 easy: `partition_for_dev_shm` — already a separate tmpfs). + - Retires **deviation-register entry #1** (the narrative hand-wave over the whole + partition/mount family) — one fewer waiver to defend at ATO. + - `/dev/shm`'s three option rules **already pass** — no work there. + +**Writable-set + flag matrix (the design):** +- `/` `/usr` `/opt` `/boot` → **read-only** (verity as-is, no overlay). `/usr` keeps + suid for legit setuid bins (sudo/su/passwd). +- `/etc` → overlay (lower=verity `/etc`, upper=tmpfs) — needs defaults + writability + (machine-id, regenerated ssh host keys, NM connections, resolv.conf, chrony). +- `/var` → overlay (NOT bare tmpfs — bare tmpfs hides `/var/lib/rpm`, selinux store); + `nodev`. Nest the noexec leaf mounts on top (order: `/var` first, then children). +- `/var/tmp` `/var/log` `/var/log/audit` → tmpfs `noexec,nosuid,nodev`. +- `/tmp` → tmpfs `nosuid,nodev` (+`noexec` only after the kiosk repoint, below). +- `/home` `/root` `/srv` → tmpfs `nodev,nosuid`. +- Each tmpfs needs the **correct SELinux context** (`tmp_t`, `var_log_t`, + `auditd_log_t`, …) via `context=` or restorecon — this is the login-deny hazard + class (#5); a mislabeled tree denies logins / stops auditd. + +**Prerequisites / caveats that gate the count:** +- `mount_option_tmp_noexec` (1 of the 21) is **conditional**: register entry #1 notes + the Chromium kiosk uses `/tmp` for profile/cache and breaks under `noexec`. Must + **repoint the kiosk scratch** (Chromium `--user-data-dir`/`--disk-cache-dir`) to a + dedicated exec-capable tmpfs (e.g. `/run/kiosk`) first → ties to #13/#4. So ~20 + land immediately, +1 after the repoint. +- **fstab-vs-runtime OVAL — mostly de-risked:** the rules return `notapplicable` + (evaluating the live mountpoint) not `fail` (missing fstab line), evidence they read + the runtime mount table — so systemd mount units / initramfs mounts should score + without fstab entries. Confirm per-rule on the bench; add fstab lines if any rule is + fstab-based (compatible with the empty-fstab invariant — root still comes from + netroot+verity). +- **Does NOT fix red findings** (this family is ~0 fail today) — the value is + evidenced compliance + one fewer deviation + kernel-enforced noexec, not clearing + fails. Framed honestly so it isn't oversold. +- **`partition_for_usr` stays waived** — `/usr` should be verity-sealed, not a + separate writable partition (a *stronger* control than the rule asks). Same for + `encrypt_partitions`, `auditd_..._large_partition`, boot/boot_efi (genuinely N/A). + +**Mechanism:** `systemd.volatile=` has no per-dir knob (it's all-or-var-only), so this +means dropping the global volatile overlay for explicit early mount units (ordered +before `local-fs.target`) or initramfs mounts. Reproducibility unaffected (runtime +mounts, not image content → no verity-hash drift). + +**Load-bearing / bench-gated:** touches the boot path (#5 territory) — full boot-chain +bench (`serving/dot1x-bench/`) + careful SELinux-label verification + a reconciliation +note in `design-v2-errata.md`. **Do AFTER TPM enrolment (#11) is bench-verified.** +Tailoring edits: re-select the 6 `partition_for_*` (+dev_shm), delete/gut register +entry #1, re-run `scripts/stig-deviations.sh` to confirm the new passes. + +## [ ] 16. Enable a locked-down host firewall (firewalld) — defense-in-depth + STIG passes + +**DECISION 2026-07-03: enable a *configured* firewalld.** Currently REMOVED + masked +(`diskless.pkr.hcl` 2d, commit 4346ce7) because an UNCONFIGURED weak-dep firewalld was +running and flooding the SIEM with `lib_t` `.pyc` AVCs. The design deferred the firewall +as "future hardening, currently waived" ([stig-deviations.md](stig-deviations.md) #4) — +NOT incompatible, just unverified during boot-chain bring-up. A diskless kiosk is an +ideal firewall candidate: near-zero legit inbound (admin SSH only; `fleet-hostapi` is +loopback), and the load-bearing paths are outbound (NVMe/TCP, DHCP) or L2 (EAPOL/MACsec). + +**Payoff:** defense-in-depth (limits a compromised same-segment peer) + converts **4 +STIG waivers → passes** (`service_firewalld_enabled`, `package_firewalld_installed`, +`set_firewalld_default_zone`, `configured_firewalld_default_deny` — re-select in the +tailoring; retires deviation-register #4). + +**Must do together:** +1. **Config:** default-DENY inbound zone; allow **SSH** (key-only admin) + + established/related; assign `macsec0` (secure) / `en*` (test) to it. hostapi + (:9099 loopback) + kiosk→hostapi PNA need no rule. +2. **Fix the `.pyc` AVC root cause (REQUIRED — an enabled firewalld re-hits it):** the + reproducible erofs `-T 0` zeroes mtimes, so Python thinks every site-package `.pyc` + is stale and rewrites it → `firewalld_t` (and ANY Python daemon, e.g. `tuned`) + write/unlink AVCs on `lib_t`. Fix once with `python3 -m compileall --invalidation-mode + unchecked-hash` over the Python trees BEFORE the erofs build → mtime-independent + `.pyc`, no rewrites, and MORE reproducible. (Worth doing regardless of firewalld — + it's the true root cause of the firewalld AVCs seen 2026-07-03.) +3. **Tailoring:** re-select the 4 firewalld rules. + +**THE load-bearing bench gate (why #4 deferred it):** the NVMe/TCP root is an OUTBOUND +connection to `:4420` established in the initramfs BEFORE firewalld starts. Verify it +**SURVIVES** firewalld start + `firewall-cmd --reload` (established/related allowed by +default → should, but UNVERIFIED); a reconnect within `ctrl_loss_tmo` re-establishes +*through* the firewall; DHCP renew + MKA keepalives unaffected; SSH in; no AVCs; verity +hash reproducible. If it drops the root, that is the fleet-wide outage risk — gate hard. diff --git a/docs/design-v2-errata.md b/docs/design-v2-errata.md index 6d105c2..9d6a495 100644 --- a/docs/design-v2-errata.md +++ b/docs/design-v2-errata.md @@ -82,10 +82,10 @@ boot→secure dynamic VLAN. Consequences: **Per-host 802.1X identity is created by an enrolment flow, never baked in** (invariant #1). Chosen pattern: **trust-on-first-use with out-of-band fingerprint approval** — on an empty TPM the host generates a key *in* the TPM -(`0x81010001`), submits a CSR to an enrolment **broker** (reachable via the same +(`0x81000100`), submits a CSR to an enrolment **broker** (reachable via the same pre-auth permit-set, e.g. an `/enrol/` location on the serving nginx → broker → step-ca signer), and shows a fingerprint on the Plymouth splash; an admin -cross-checks it and approves; the signed cert is written to NV `0x01800001` and +cross-checks it and approves; the signed cert is written to NV `0x01800100` and authentication proceeds. Enrol triggers on **empty-NV only** (expired cert → automatic renew; auth-failure-with-cert → alert, not re-enrol). The fingerprint authorises *which device*; **TPM attestation** (step-ca ACME `device-attest-01`) diff --git a/docs/fleet-management-design.md b/docs/fleet-management-design.md index f115dcf..63cc389 100644 --- a/docs/fleet-management-design.md +++ b/docs/fleet-management-design.md @@ -20,7 +20,7 @@ These are consequences of the CLAUDE.md invariants, not preferences: stateless and buffers only in RAM (best-effort). 2. **Identity from the TPM, never baked.** No per-host state in any artifact (invariant #1). The agent derives identity at runtime from the **TPM-held - cert** (key `0x81010001`, cert NV `0x01800001`) — the same identity used for + cert** (key `0x81000100`, cert NV `0x01800100`) — the same identity used for 802.1X EAP-TLS and the NVMe hostnqn. No enrollment token in the image. 3. **Pull-based control, no inbound surface.** The boot LAN is locked down and segmented; we do not open an always-on inbound control port on hosts. Hosts @@ -44,6 +44,13 @@ endpoint; the server authenticates the host by cert CN/SAN, which is the canonic fleet identity (= NVMe hostnqn = 802.1X identity). Reuses the **step-ca fleet CA** (`serving/dot1x-bench/`). No baked secret; the host also pins the server cert. +> **Bootstrap (where the cert comes from):** mTLS here assumes the host already +> holds a TPM cert — but an un-enrolled host does not. That chicken-and-egg is +> resolved by **enrolment**, which is the management plane's *one* unauthenticated +> endpoint and the moment the inventory record is born. See +> **[tpm-enrolment-design.md](tpm-enrolment-design.md)** — the enrolment broker is +> designed to fold into this mgmt service as its bootstrap routes. + - **Secure path:** full mTLS, TPM cert. Strong, no secrets in the image. - **Open/test path (`rd.dot1x=0`):** fall back to dnsmasq fleet-name + MAC as a weak identity. **Lab only** — flagged as insufficient for production. @@ -181,6 +188,14 @@ bus) already exist. - **Telemetry encoding:** structured syslog/JSON into Wazuh (Phase 1) vs native OTLP from the start. Leaning Wazuh-first for reuse. +- **Steady-state transport (agent ↔ manager):** the agent is **Go**, the path is + **mTLS + host-dialed** (preserves "no inbound port"). Fork: bespoke **gRPC** + stream (health up / desired-state down) vs reuse **OTLP-gRPC for metrics + Wazuh + for events** + a thin gRPC control stream — the latter gets VictoriaMetrics/ + Grafana/Wazuh for free. `Connect` (connectrpc.com) if one protobuf IDL serving + gRPC + JSON + browser is wanted. Note **enrolment** deliberately stays plain + HTTP/JSON (stdlib, minimal, the one unauth edge) — see + [tpm-enrolment-design.md](tpm-enrolment-design.md) "Implementation". - **Metrics store:** VictoriaMetrics vs Prometheus (VM is lighter for push + long retention). - **Desired-state store:** git (GitOps, aud/rollback for free, matches the diff --git a/docs/ssp-annex-a-ism-residual-risk.md b/docs/ssp-annex-a-ism-residual-risk.md index e2717ce..4d4c725 100644 --- a/docs/ssp-annex-a-ism-residual-risk.md +++ b/docs/ssp-annex-a-ism-residual-risk.md @@ -99,7 +99,7 @@ ISM/PSPF style). Ratings are **pre-treatment** unless stated; the AO accepts the | Topic | ISM (rep.) | Requirement intent | System implementation | Status | Residual | |---|---|---|---|---|---| -| Strong machine authentication | ISM-1546*, ISM-0974* | Mutual, certificate-based auth. | **802.1X EAP-TLS**, client key non-exportable in **TPM** (`0x81010001`), cert in TPM NV. | Partial | R-01 (not bench-verified; disabled in test builds), R-08 (placeholder CA) | +| Strong machine authentication | ISM-1546*, ISM-0974* | Mutual, certificate-based auth. | **802.1X EAP-TLS**, client key non-exportable in **TPM** (`0x81000100`), cert in TPM NV. | Partial | R-01 (not bench-verified; disabled in test builds), R-08 (placeholder CA) | | Privileged access restricted | ISM-1507*, ISM-0445* | Limit & control admin access. | Single key-only `wheel` admin; root login disabled; admin **confined to `staff_u`** with a sudo `sysadm_r`/`sysadm_t` role/type transition (SELinux context elevates on privilege use). | Partial | R-05 | | Secure remote administration | ISM-1506*, ISM-1232* | Hardened admin channel. | SSH **key-only**, password & keyboard-interactive off, hardened ciphers/MACs/KEX (STIG crypto policy; ECDSA/RSA keys — ed25519 excluded by policy). | Implemented | — | | No default/shared credentials | ISM-1304* | No baked credentials. | No per-host secrets in any artifact; identity from TPM at runtime; `machine-id` empty; host keys regenerated at first boot. | Implemented | — | diff --git a/docs/ssp.md b/docs/ssp.md index 050e4e4..fb5e584 100644 --- a/docs/ssp.md +++ b/docs/ssp.md @@ -90,8 +90,8 @@ manifest. One commit = one release = one digest = one manifest entry. hash and STIG kernel parameters). No GRUB. 3. **Initramfs (dracut)** brings up the NIC and performs **802.1X EAP-TLS** network authentication using a **private key held in the TPM** (persistent - handle `0x81010001`); the client certificate is read from TPM NV index - `0x01800001`. + handle `0x81000100`); the client certificate is read from TPM NV index + `0x01800100`. 4. On authorisation, **MACsec** is established on the access link (link-layer encryption/integrity); the host then obtains an IP **on the MACsec interface** via DHCP. diff --git a/docs/stig-deviations.md b/docs/stig-deviations.md index f6024f4..b229d0e 100644 --- a/docs/stig-deviations.md +++ b/docs/stig-deviations.md @@ -138,6 +138,35 @@ down (kiosk autologin to a single locked browser; no getty shell on tty1; `debug_shell` is lab-only and never shipped). If a future deployment wants the hotkey disabled (e.g. a physically exposed terminal), re-select both rule IDs. +## 14. Local SELinux policy module `chrony_dhcp_dispatcher` (added 2026-07-03) + +Not a STIG-rule waiver — a **minimal local `allow` module** shipped to keep a +legitimate function working under enforcing SELinux. Source: +`files/stig/chrony-dhcp-dispatcher.te`; built + `semodule -i`'d in the 2c-STIG +provisioner. + +**The one rule:** `allow chronyd_t NetworkManager_dispatcher_custom_t:unix_dgram_socket sendto;` + +**Why:** the NM dispatcher script `10-chrony-dhcp-ntp` loads DHCP option-42 NTP +servers by running `chronyc reload sources`. That `chronyc` runs in the confined +custom-dispatcher domain (`NetworkManager_dispatcher_custom_t`), so its reply +socket carries that label; the shipped policy has no rule for `chronyd_t` to +`sendto` it. Under enforcing, chronyd processes the reload but its **ack is +denied** — `chronyc` reports failure and the AVC floods the SIEM (seen +2026-07-03). An **`allow`** (not `dontaudit`) is used deliberately: it both stops +the noise and lets the reload ack succeed, so `chronyc` returns cleanly. + +**Scope:** one direction, one class/perm, two well-known types. It grants no +network/file access and nothing beyond the chronyd↔dispatcher reply the DHCP-NTP +path needs. **BENCH-VERIFY:** (a) on an *enforcing* boot the AVC is gone and +`chronyc reload sources` from the dispatcher returns 0 (confirm a DHCP-handed NTP +server appears in `chronyc sources`); (b) no follow-on AVC (extend the `.te` if +one shows); (c) **reproducibility** — `semodule -i` rebuilds the policy store, so +re-check the verity hash is stable across two builds (same concern as the +`semanage` step, #12/sudo elevation). NB: only shipped when `enable_stig=true` +(the module install lives in the STIG-gated provisioner); a non-STIG *enforcing* +build would still see the AVC. + ## Things that are NOT deviations (kept / enforced) - **SELinux enforcing** (`selinux --enforcing`; labels preserved into the erofs). diff --git a/docs/tpm-enrolment-design.md b/docs/tpm-enrolment-design.md new file mode 100644 index 0000000..891c6a9 --- /dev/null +++ b/docs/tpm-enrolment-design.md @@ -0,0 +1,379 @@ +# TPM identity enrolment + the fleet identity lifecycle — design note + +**Status:** DESIGN (consolidated 2026-06-28). Supersedes the scattered decisions +in `docs/TODO.md` #11 (which it folds in) and closes the bootstrap gap left open +by `docs/fleet-management-design.md`. The bench stand-in +(`serving/dot1x-bench/enrol/tpm-provision.sh`) already proves the cryptography +end-to-end *minus* the human-approval queue; the only unbuilt piece is the +**enrolment broker**. + +How a diskless host acquires — and keeps — its **one cryptographic identity**: a +signing key at TPM handle `0x81000100` and a cert at NV index `0x01800100`. +Invariant #1 forbids baking any per-host state into the image, so this identity +**must** be minted at enrolment, at runtime, from the TPM. That single identity is +then reused for everything: **802.1X EAP-TLS**, the **NVMe hostnqn** (derived from +the cert serial), and **mTLS to the fleet-manager**. This note is the authoritative +design for how it is born, used, renewed, and revoked. + +## The gap this closes + +`fleet-management-design.md` authenticates every host by **mTLS keyed on the TPM +cert** — but never says how the cert gets onto the host. That is a chicken-and-egg: +*mTLS needs a cert; the cert needs enrolment; enrolment is the one thing that +cannot itself be mTLS-authenticated* (an empty-NV host has no client identity yet). +Enrolment is therefore the **sole unauthenticated bootstrap** in the whole system, +and the management plane's entire trust model rests on it. Treating the enrolment +**broker as the fleet-manager's one unauthenticated endpoint** — rather than a +separate box — is what makes the two designs one coherent identity story. + +## Design philosophy (forced by the architecture invariants) + +1. **One identity, minted once, reused everywhere.** Key `0x81000100` + cert + NV `0x01800100` = 802.1X identity = NVMe hostnqn (`…:host-`) = + mTLS client identity. There is no second secret. (Invariant #1.) +2. **The private key never leaves the TPM.** Generated *in* the TPM; only a CSR + and the issued cert ever cross the wire — both are public. This is why the + pre-auth, pre-MACsec enrolment link being plaintext is acceptable. +3. **Humans authorise; machines sign.** step-ca can sign but has no notion of + "should I." The authorisation decision is a **human OOB act** (fingerprint + cross-check). No auto-sign of unauthenticated CSRs, ever. +4. **Enrolment is the birth of the inventory record.** The fleet-manager keys + every host by `cert-serial / hostnqn`; that key does not exist until enrolment. + So approval is simultaneously `INSERT host` and the moment a human **binds + physical identity → cryptographic identity** (asset tag / switch port / + location → cert serial). Every downstream signal hangs off this record. +5. **Integrity of the pre-auth path comes from Secure Boot, not the network.** + The UKI fetch and the enrolment exchange both happen pre-auth on an untrusted, + unencrypted L2. The boot is trustworthy because the **UKI is Secure-Boot-signed** + (#8), not because the wire is. Enrolment design and Secure Boot are two halves + of the same "the pre-auth path is hostile" assumption. + +## The identity lifecycle (the load-bearing state machine) + +Triggered by the state of NV `0x01800100` and the cert within it. The three events +are deliberately **split** — conflating them is a security regression: + +``` + EMPTY NV → ENROL human OOB approval; broker→step-ca sign; write NV. + Births the fleet-manager inventory record. + VALID KEY + cert near/at → RENEW automatic, NO human. The host already holds a + expiry CA-trusted key; re-sign the same key, rewrite NV. + AUTH FAILS, cert present → ALERT retry + raise an alert. NEVER auto-re-enrol — + that would mask a revocation and churn keys on a + transient RADIUS/CA outage. + Re-enrol after first time → ADMIN explicit admin action (clear/flag the host); not + reachable from any automatic path. +``` + +- **Renew is safe to automate** precisely because the host proves possession of an + already-trusted key — no human needed, no new trust established. +- **Auth-failure must not re-enrol.** A revoked host that re-enrolled itself would + defeat revocation. Failures alert; they never mint new identity. + +## Components + +### 1. TPM key + NV cert (exists — `tpm-provision.sh`) + +Key generated in the TPM, persisted at `0x81000100`; CSR built with it via the +openssl tpm2 provider (key never leaves the TPM); issued cert written DER to NV +`0x01800100`. At boot, `dot1x-macsec-up.sh` reads the cert from NV, wpa_supplicant +signs EAP-TLS with the handle key, and the hostnqn is derived from the cert serial. +All of this is built and bench-proven; enrolment just replaces the *manual* sign +step with a brokered, human-gated one. + +### 2. The enrolment broker (the only new component) + +A small service that turns "step-ca can sign" into "step-ca signs **only what a +human approved**." Stateless-leaning; pending state is short-lived and server-side. + +**API (HTTPS; image trusts the serving CA):** + +| Route | Auth | Action | +|---|---|---| +| `POST /enrol/request` | none (bootstrap) | accept CSR + self-reported device context; create a time-boxed **PENDING** record; return `enrol-id` + the **fingerprint** to display | +| `GET /enrol/status/{enrol-id}` | none (bootstrap) | host polls → `PENDING` \| `APPROVED`+cert \| `DENIED` \| `EXPIRED` | +| `GET /enrol/pending` | **mTLS, admin** | the approval queue (one entry per pending CSR, with context + fingerprint) | +| `POST /enrol/approve/{enrol-id}` | **mTLS, admin** | bind physical context → record; hand CSR to step-ca; store the issued cert | +| `POST /enrol/deny/{enrol-id}` | **mTLS, admin** | reject + audit | + +The **request/status** routes are the sole unauthenticated surface; **pending/ +approve/deny** are admin-only mTLS — i.e. they are already the fleet-manager's +authenticated plane. This is why the broker *is* the fleet-manager's bootstrap +endpoint, not a separate trust domain. + +**Device context** (what an un-enrolled host can self-report, for the admin to +recognise *what* they are approving): NIC MAC, DMI system UUID/serial, TPM EK +public-key hash, and switch port via LLDP if available. Context is advisory — the +fingerprint is the cryptographic anchor. + +### 3. OOB fingerprint approval (the human gate) — v1 assurance + +**v1 ships fingerprint-only** (TPM attestation is the documented upgrade, § +"Stronger assurance"). The flow: + +- `fingerprint = BIP39-encode( SHA-256( CSR_SubjectPublicKeyInfo_DER || server_nonce )[:N bits] )` + — both broker and host compute it identically from the CSR public key and a + **server-chosen per-`enrol-id` nonce**. +- The host shows it on the **Plymouth splash** (states only — invariant #1 — never + per-host secrets; the fingerprint is public). The admin reads it off the physical + device's screen and matches it to that device's entry in `GET /enrol/pending`. +- **What it proves:** the queue entry the admin approves carries the *same key* as + the device physically in front of them. **What it does not prove:** that the key + actually resides in a TPM (that's attestation, below). + +Hardening, all required for v1: +- **Word-list encoding, not short hex.** Hex is grindable; ~6 BIP39 words (~66 + bits) reads short but resists collision-grinding. +- **Server nonce per `enrol-id`** so an attacker cannot pre-grind a key whose + fingerprint collides with a target device's (the nonce is unknown until after + submission, and is per-request). +- **Rate-limit + time-boxed pending** (expire unapproved requests) to bound queue + flooding. +- **Show device context** beside the fingerprint so the admin approves a *specific* + MAC/serial, not just a matching string. + +### 4. step-ca (exists — `serving/dot1x-bench/step-ca/`) + +The fleet CA. The broker calls it on approval (token-gated `step ca sign`, exactly +as `tpm-provision.sh` does today). Same root that is baked as `radius-ca.pem` and +pinned for mTLS — one trust anchor across 802.1X, NVMe, and management. + +### 5. Relationship to the fleet-manager and host-api + +- **Broker ⊂ fleet-manager.** `/enrol/request` + `/enrol/status` are the + fleet-manager's *only* unauthenticated routes; `/enrol/pending|approve|deny` are + its admin mTLS plane. Approval writes the inventory record the manager keys + everything else on. Build the broker standalone first (fastest path to the state + machine), but design it to fold into the mgmt service with no rewrite — same Go + style, same step-ca anchor, same data model. +- **host-api is not in the enrolment path** (it is loopback/post-boot; the host has + no cert during enrolment). The tie is downstream: the fleet-manager **agent** (the + push sibling of `fleet-hostapi`) authenticates with the *enrolled* cert over mTLS, + and inherits host-api's hardened posture (DynamicUser, fapolicyd-trusted, + reproducible static build). One identity, three stages: **enrol** mints it → + **host-api/agent** uses it → **renew** refreshes it. + +## End-to-end enrolment flow + +``` + empty-NV host (early-boot enrol branch) broker (= fleet-mgr bootstrap) admin step-ca + ─────────────────────────────── ───────────────────────────── ───── ─────── + 1. key in TPM @0x81000100 + 2. build CSR (key never leaves) + 3. POST /enrol/request (CSR+context) ─▶ create PENDING, pick nonce, + compute fingerprint + ◀── enrol-id + fingerprint ───────── + 4. show fingerprint on Plymouth + GET /enrol/pending ◀──────────── 5. read queue (mTLS) + 6. cross-check fp on + the device screen + POST /enrol/approve/{id} ◀────── 7. approve + bind + physical→crypto + bind context → INSERT host, + sign CSR ──────────────────────────────────▶ issue cert + store cert ◀─────────────────────────────── + 8. GET /enrol/status/{id} (poll) ───▶ + ◀── APPROVED + cert ──────────────── + 9. write cert DER → NV 0x01800100 + 10. reboot into production UKI → 802.1X EAP-TLS → mTLS mgmt +``` + +## Implementation: language + transport (decided 2026-06-28) + +**Go for all three fleet-authored components** (broker, `fleet-hostapi`, the +fleet-manager agent) — one language, one build/posture story. Transport is split by +*who the client is* and *where the code runs*, not by preference: + +- **Enrolment path → plain HTTP/1.1 + JSON, stdlib-only. NOT gRPC.** It is two unary + calls + a poll (`POST /enrol/request` → `GET /enrol/status` → cert), so gRPC's + streaming/multiplexing buys nothing. It is the system's *one* unauthenticated, + pre-auth, security-critical edge → the smallest, most auditable parser wins (a + `net/http` handler over a JSON body, not HTTP/2 + protobuf). The **client runs in + the early-boot initramfs** (the production UKI's empty-NV branch for now; a + provisioning UKI later) → keep it stdlib-only and reproducible like `fleet-hostapi` + (no `grpc-go`/protobuf vendoring in the minimal environment). And it + must reverse-proxy through the existing serving nginx `/enrol/` location — a one-line + `proxy_pass` for HTTP/1.1 vs `grpc_pass` + end-to-end HTTP/2 on the pre-auth edge. + CSR/cert ride as base64/PEM in JSON. **Boring is the feature on the bootstrap path.** +- **Admin approve/deny routes → mTLS (HTTP/JSON or gRPC, builder's call).** These are + already the fleet-manager's authenticated plane. +- **Steady-state agent ↔ manager (telemetry + reconciliation) → gRPC over mTLS, + host-dialed.** Streaming fits here, and it stays compatible with "no inbound port on + hosts" because the **host dials out** and holds one long-lived mTLS stream (health + up / desired-state down). OPEN FORK (a fleet-manager build decision, not enrolment): + bespoke gRPC vs reuse **OTLP-gRPC for metrics + Wazuh for events** + a thin gRPC + control stream — the latter gets VictoriaMetrics/Grafana/Wazuh for free. `Connect` + (connectrpc.com) is an option if one protobuf IDL serving gRPC + JSON + browser is + wanted across the authenticated surface. +- **host-api stays HTTP/JSON — locked**, because its consumer is the Chromium kiosk + (a browser, which cannot speak native gRPC). + +## Where it runs: enrolment vehicle (build-flag, two modes) + +Empty-NV and normal boot are **mutually exclusive** — a host with no cert cannot do +802.1X → MACsec → NVMe-root at all, so there is no normal hot-path sequence to +"protect" in the empty-NV case. The choice of vehicle is therefore *not* about +invariant #2 timing; it is about **whether enrolment capability ships in the +production artifact at all**. Two modes behind one build flag (`enrol_mode`, modelled +on the existing `disable_dot1x` / `boot_splash` cmdline toggles): + +- **Inline single-UKI — CHOSEN FOR NOW (decided 2026-06-28).** The production UKI + auto-detects empty-NV and branches into enrolment in early boot. Picked because it + needs **no separate provisioning boot path** — everything stays purely + network-booted (no USB / no provisioning media to manage). The empty-NV gate is the + safety: NV-write at `0x01800100` needs TPM owner auth, so empty-NV is hard to + coerce. Accepted trade-offs: production carries the enrolment + pre-auth broker code + permanently, and there is no PCR-11 discrimination. The empty-NV branch is mutually + exclusive with the secure path, so it does not endanger the load-bearing hook + ordering. It needs a **pre-auth DHCP on the physical NIC** to reach the broker + through the pre-auth permit-set (the same pre-auth network the firmware already used + to fetch the UKI, like the `rd.dot1x=0` test path); on success it writes NV and + **reboots into the normal secure path** (convergence = reboot). +- **Provisioning-UKI variant — PLANNED LATER UPGRADE.** `enrol_mode=true` conditionally + `install()`s the enrolment scripts into the dracut module *and* bakes an immutable + `rd.fleet.enrol=1` cmdline → a **separately signed UKI from the same source tree**. + Production then lacks the code entirely (smaller initramfs, **physically cannot + enrol**), matching the #10 Secure-Boot-enrolment decision and enabling PCR-11 + discrimination. Note the later separate UKI can still be **PXE/network-served** + (dnsmasq `provisioning` tag / unknown-MAC) — it does **not** require USB media; the + reason to defer is simply the extra boot-path + CI sign/artifact, not a media + dependency. Revisit once the broker is built and the secure path is verified. + +Either way, the production *boot* path only ever **reads** an already-present cert; +the human-gated, indefinitely-waiting enrolment lifecycle never interleaves with the +unattended, watchdog-tuned normal boot. + +| | Inline (one UKI) | Provisioning variant (build-flag) | +|---|---|---| +| Ops | simplest — one artifact | +1 CI sign/artifact (same tree) | +| Prod attack surface | carries enrolment + pre-auth broker code | none — code absent from prod | +| "Prod can't enrol" guarantee | empty-NV gate (TPM owner auth) | structural (code/cmdline absent) | +| PCR-11 discrimination | no | yes | +| Consistency with #10 | diverges | matches | + +## Pre-auth network placement + +The firmware cannot do 802.1X, so the UKI fetch (DHCP/DNS/TFTP/HTTP to the serving +host) is **always pre-auth** — the secure model therefore already *requires* a +pre-auth permit-set (low-impact 802.1X / pre-auth dACL / boot→secure dynamic VLAN; +RADIUS returns the secure VLAN on Access-Accept). **Enrolment is just one more entry +in that set:** expose `/enrol/` on the **existing serving nginx**, reverse-proxied +to the broker. No new network mechanism. Pre-auth traffic has **no MACsec** +(plaintext L2) — fine here, because CSR/cert are public and the fingerprint supplies +authenticity, and because boot integrity comes from the SB-signed UKI. + +**Bench caveat:** `hostapd` wired is strictly port-based (no pre-auth per-flow ACL): +before EAP success it passes nothing, so the rig cannot reproduce "boot + enrol +pre-auth on the same port." Model the pre-auth state with a separate provisioning +bridge/VLAN on the bench; validate the in-place low-impact pre-auth permit on the +820's real switch. + +## Security posture / threat model + +- **Sole unauthenticated surface = `/enrol/request|status`.** Scope the pre-auth + permit to *only* these routes + the serving host; everything else is mTLS. +- **No auto-sign.** Every cert traces to a human OOB approval; the broker never + signs on its own. +- **Fingerprint binds queue↔device** (v1). It does **not** prove TPM residency or + prevent a determined insider — it raises the bar to "be physically at the device + and have an admin approve it." +- **Rate-limit + time-box + audit** every request/approve/deny to the collector + (the binding event is forensically important). +- **Revocation is real** only because auth-failure never auto-re-enrols (see + lifecycle). Pair with step-ca CRL/short-lived certs + the renew path. + +## Stronger assurance (documented upgrade, not v1) + +Fingerprint proves "this physical device"; it does not prove "the key is in a TPM." +The residency proof is step-ca's ACME **`device-attest-01`** provisioner (TPM +EK/AK attestation): switch enrolment from `step ca sign` to an ACME exchange with a +TPM attestation provisioner on the same CA, anchored to the TPM vendors' EK roots. +Together — **fingerprint = a human authorised the device, attestation = the key is +genuinely TPM-resident** — this is the strong SSP R-01 story. Deferred from v1 to +keep the first build tractable; the broker's approve step is the natural insertion +point (verify the attestation statement before signing). The CA bootstrap already +carries a commented `acme-attest` provisioner block +(`serving/dot1x-bench/step-ca/bootstrap.sh`). + +### Signing-key handle choice (and the IDevID option) + +The fleet signing key is persisted at **`0x81000100`** — a non-reserved **owner** +persistent handle. It is deliberately **not** the EK slot `0x81010001` / `0x81010002` +nor the SRK `0x81000001` / `0x81000002`: firmware and swtpm provisioning routinely +occupy those (bench-confirmed 2026-07-02 — a Proxmox vTPM had the standard **RSA +Endorsement Key** at `0x81010001`, restricted-decrypt, with the canonical EK +authPolicy `837197…`, and its EK cert at NV `0x1c00002`). `EnsureKey` therefore +also refuses to evict any restricted-decrypt object as a safety net. The handle is +overridable via `rd.dot1x.handle=` / `fleet-enrol -handle`. + +**Future option — IDevID (`0x81020001`):** if/when the EK/AK attestation path above +lands, the natural home for a TPM-resident *device identity* key is the TCG +**IDevID** handle `0x81020001` (with **LDevID** at `0x81030001`), per *TPM 2.0 Keys +for Device Identity and Attestation*. Moving there would align our key with the +IDevID template + EK-anchored certification flow (the key is certified by an AK that +chains to the EK) rather than a bare owner primary. Not adopted for v1 (it implies +the full IDevID template/cert conventions we don't yet follow); revisit together with +`device-attest-01`. + +## Inventory record born at enrolment + +``` +Host (by cert-serial / hostnqn) ← created at /enrol/approve + ├─ identity: cert serial, hostnqn, MAC, TPM EK hash, starship-name + ├─ binding: asset tag, switch port, location ← the human OOB act records this + ├─ enrolled: approver, timestamp, fingerprint, attestation? (later) + └─ (then fleet-manager fills) version / health / usage / desired-state +``` + +This is the join key the rest of `fleet-management-design.md` assumes already +exists. Enrolment is where it comes into being. + +## Phasing + +1. **Built / proven:** TPM key→CSR→sign→NV (manual, `tpm-provision.sh`); + `tpm-selftest.sh` capability check; step-ca + EAP-TLS bench. +2. **Broker — BUILT (2026-06-28): [`serving/enrol-broker/`](../serving/enrol-broker/).** + Go, stdlib-only. Public unauth listener (`POST /request`, `GET /status/{id}`) + behind the serving nginx `/enrol/`; admin **mTLS web UI** (`/`, `/pending`, + `/approve|deny/{id}`) on the mgmt network; wordlist fingerprint = `H(CSR-pubkey ‖ + server-nonce)`; time-boxed in-memory store + per-IP rate limit; signs via + `step ca token`+`step ca sign` (broker holds the provisioner password) with a + dev `mock` signer. **Fingerprint-only assurance.** Admin client CA is a + **dedicated** CA (`admin-pki.sh`), separate from the fleet CA, so host certs + can't authenticate to the admin plane. End-to-end smoke-tested (CSR → mTLS + approve → cert; pubkey match). Ships in `serving/dot1x-bench/docker-compose.yml`. +3. **Host side — BUILT 2026-07-02 (awaiting bench):** the **inline empty-NV + trigger** as a static **Go + go-tpm** binary (`files/enrol/`, `fleet-enrol`) + talking to `/dev/tpmrm0` directly — ensures the key at `0x81000100`, builds the + CSR, POSTs to the broker, shows the Plymouth fingerprint, polls, writes the leaf + cert to NV `0x01800100`. A thin dracut GATE hook (`95dot1x-macsec/tpm-enrol.sh`, + initqueue/settled prio 25) does the empty-NV check + pre-auth DHCP on the + physical NIC + reboot-to-converge. No tpm2-tools key-gen / openssl provider / + curl needed in the image (go-tpm does it all). Built **out of the VM** on the CI + runner + file-provisioned in (like `fleet-hostapi`); `enrol_url` Packer var → + `rd.fleet.enrol_url`. TESTED: transaction (mock broker) + full TPM ceremony vs + the go-tpm simulator (`files/enrol/simtest/`). BENCH-VERIFY list in `docs/TODO.md` #11. +4. **Folds into fleet-manager:** the admin routes become the mgmt service's + authenticated plane; approval writes the shared inventory record. +5. **Provisioning-UKI variant** (`enrol_mode`) — move enrolment out of the production + artifact once the broker + secure path are proven (optional, network-served). +6. **Attestation upgrade:** ACME `device-attest-01` at the approve step. + +## Open questions for build time + +- **openssl TPM2 provider in the enrolling image** (the production image, inline) — + `tpm2-tss` ≠ the provider; + `tpm-selftest.sh` flags whether `tpm2-openssl` must be added (also affects the + wpa_supplicant key-ref form: provider `handle:` vs tpm2-pkcs11 URI). +- **Pending store:** in-memory (simplest, lost on broker restart) vs a tiny + persistent store. Time-boxed pending makes in-memory acceptable for v1. +- **Enrolment-vehicle** — inline single-UKI is **chosen for now** (network-boot only, + no USB); the provisioning-UKI variant is the planned later upgrade behind the + `enrol_mode` build flag (shares the #10 vehicle, not yet built) — see "Where it + runs". Revisit when the broker is built + the secure path verified. +- **Steady-state transport fork** (fleet-manager, not enrolment) — bespoke gRPC vs + OTLP-gRPC metrics + Wazuh events + a thin gRPC control stream. See + [fleet-management-design.md](fleet-management-design.md). +- **Fingerprint length** — exact bit count / word count (entropy vs readability). +- **Open/test path identity** (`rd.dot1x=0`): MAC/fleet-name is a weak stand-in; + gate real enrolment to the secure path, or accept lab-only there. diff --git a/files/dracut/95dot1x-macsec/dot1x-macsec-up.sh b/files/dracut/95dot1x-macsec/dot1x-macsec-up.sh index 3ab4416..5f55cc4 100755 --- a/files/dracut/95dot1x-macsec/dot1x-macsec-up.sh +++ b/files/dracut/95dot1x-macsec/dot1x-macsec-up.sh @@ -31,7 +31,7 @@ if [ -z "$NIC" ]; then NIC=$_name; break done fi -NVINDEX=$(getarg rd.dot1x.nvindex=); [ -n "$NVINDEX" ] || NVINDEX=0x01800001 +NVINDEX=$(getarg rd.dot1x.nvindex=); [ -n "$NVINDEX" ] || NVINDEX=0x01800100 TIMEOUT=$(getarg rd.dot1x.timeout=); [ -n "$TIMEOUT" ] || TIMEOUT=60 info "dot1x-macsec: using interface $NIC, NV index $NVINDEX" diff --git a/files/dracut/95dot1x-macsec/module-setup.sh b/files/dracut/95dot1x-macsec/module-setup.sh index 2651292..84234cc 100755 --- a/files/dracut/95dot1x-macsec/module-setup.sh +++ b/files/dracut/95dot1x-macsec/module-setup.sh @@ -7,6 +7,9 @@ check() { require_binaries wpa_supplicant wpa_cli tpm2_nvread openssl || return 1 + # First-boot ENROLMENT is the out-of-VM fleet-enrol Go binary (bundled in + # install()), which does key-gen/CSR/NV directly via go-tpm — so no tpm2-tools + # key-gen / openssl provider / curl are required in the initramfs here. return 0 } @@ -37,7 +40,20 @@ install() { inst_multiple wpa_supplicant wpa_cli tpm2_nvread openssl ip setsid nmcli \ grep sed cut tr sha256sum sleep mkdir nvme - # TSS libraries that tpm2_nvread dlopens + # Inline empty-NV enrolment: the gate hook (tpm-enrol.sh) drives networking + + # reboot; the actual key-gen/CSR/broker/NV work is the out-of-VM fleet-enrol + # binary (Go + go-tpm, static — talks to /dev/tpmrm0 directly, no tpm2-tools / + # openssl provider / curl needed). reboot: the hook reboots to converge into + # the secure path after writing the cert to NV. + inst_multiple reboot + if [ -x /usr/sbin/fleet-enrol ]; then + inst_simple /usr/sbin/fleet-enrol /usr/sbin/fleet-enrol + else + derror "95dot1x-macsec: /usr/sbin/fleet-enrol missing — build it out-of-VM (CI 'Build host Go binaries') before the initramfs build" + return 1 + fi + + # TSS libraries that tpm2_nvread (secure-path NV read) dlopens. inst_libdir_file "libtss2-*" # Fleet-generic supplicant config + RADIUS CA @@ -51,6 +67,11 @@ install() { # 99-dot1x-unmanaged.conf, which couldn't be mode-aware. inst_hook cmdline 10 "$moddir/nm-unmanaged.sh" + # Inline empty-NV enrolment: runs BEFORE dot1x-macsec-up.sh (priority 25 < 30). + # No-op when a cert is already present (normal boot) or under rd.dot1x=0; on a + # fresh host it enrols an identity into NV then reboots into the secure path. + inst_hook initqueue/settled 25 "$moddir/tpm-enrol.sh" + # Hook: runs from the initqueue once the NIC exists, before netroot inst_hook initqueue/settled 30 "$moddir/dot1x-macsec-up.sh" diff --git a/files/dracut/95dot1x-macsec/tpm-enrol.sh b/files/dracut/95dot1x-macsec/tpm-enrol.sh new file mode 100644 index 0000000..10a8de9 --- /dev/null +++ b/files/dracut/95dot1x-macsec/tpm-enrol.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# tpm-enrol.sh — initqueue/settled hook, runs BEFORE dot1x-macsec-up.sh. +# +# INLINE empty-NV enrolment GATE (docs/tpm-enrolment-design.md). On a secure +# build whose TPM NV 0x01800100 is empty, it brings up a pre-auth network on the +# physical NIC and hands off to the out-of-VM `fleet-enrol` binary (Go + go-tpm), +# which generates the key/CSR, talks to the broker, shows the Plymouth +# fingerprint, and writes the issued cert to NV. On success this hook REBOOTS. +# It is a no-op when a cert is already present (normal boot — dot1x-macsec-up.sh +# reads NV) or when disabled via rd.fleet.enrol=0. +# +# The crypto/HTTP/NV work lives in the binary (testable, TPM-direct); this hook +# owns only the boot-environment concerns: the empty-NV gate, networking, and the +# reboot-to-converge. Empty-NV and the secure path are mutually exclusive, so this +# never reorders the load-bearing boot sequence (invariant #2). +# +# TEST-MODE ENROLMENT (rd.dot1x=0): enrolment ALSO runs on the open/test path so +# the flow can be exercised on a host with a TPM + a reachable broker WITHOUT +# standing up 802.1X/MACsec/RADIUS. In test mode `ip=dhcp` already owns the NIC, +# so this hook must NOT re-DHCP it (that races dracut → "address already +# assigned"); it just uses the existing lease. Production still gates real +# enrolment to the secure path; use rd.fleet.enrol=0 to disable it in either mode. + +type getarg >/dev/null 2>&1 || . /lib/dracut-lib.sh + +STAMP=/run/dot1x/.enrol-done +[ -e "$STAMP" ] && exit 0 +mkdir -p /run/dot1x + +# --- skip conditions -------------------------------------------------------- +if [ "$(getarg rd.fleet.enrol=)" = "0" ]; then + info "tpm-enrol: disabled by rd.fleet.enrol=0"; exit 0 +fi +# Test mode (rd.dot1x=0): enrolment still runs, but ip=dhcp owns the NIC, so we +# must not reconfigure it below. +TESTMODE=0 +[ "$(getarg rd.dot1x=)" = "0" ] && TESTMODE=1 + +# --- knobs (override on the kernel cmdline) --------------------------------- +NVINDEX=$(getarg rd.dot1x.nvindex=); [ -n "$NVINDEX" ] || NVINDEX=0x01800100 +# Persistent handle for the signing key: a NON-reserved owner handle. NOT the EK +# slot 0x81010001 (which most firmware/swtpm provisioning already occupies) nor +# the SRK 0x81000001 — see EnsureKey / CLAUDE.md invariant #1. +HANDLE=$(getarg rd.dot1x.handle=); [ -n "$HANDLE" ] || HANDLE=0x81000100 +ENROL_URL=$(getarg rd.fleet.enrol_url=); [ -n "$ENROL_URL" ] || ENROL_URL=http://boot.fleet.lab/enrol +NETTMO=$(getarg rd.dot1x.timeout=); [ -n "$NETTMO" ] || NETTMO=60 + +# Wait briefly for the TPM resource-manager device so an early settle (before the +# TPM driver is up) doesn't misread NV as empty and wrongly trigger enrolment. +i=0 +while [ ! -e /dev/tpmrm0 ] && [ "$i" -lt 10 ]; do sleep 1; i=$((i + 1)); done +[ -e /dev/tpmrm0 ] || { warn "tpm-enrol: no TPM (/dev/tpmrm0) — leaving to the secure path"; exit 0; } + +# --- already enrolled? then this is a normal boot — do NOT touch the network - +# (Bringing up en* here would fight the secure path, which needs it UNMANAGED for +# wpa_supplicant. So gate on NV BEFORE any network action.) +# CONTENT-AWARE gate (fleet-enrol -check-only, no network/no key-gen): reads NV, +# parses the cert, and — when the fleet CA is bundled — verifies it chains, so a +# leftover/foreign/expired cert (or garbage) does NOT masquerade as enrolled. +# rc: 0 = valid fleet cert (normal boot); 20 = enrolment needed; other = error. +/usr/sbin/fleet-enrol -check-only -nvindex "$NVINDEX" -handle "$HANDLE" +rc=$? +case "$rc" in + 0) info "tpm-enrol: valid fleet cert at NV $NVINDEX — normal boot"; exit 0 ;; + 20) : ;; # fall through to enrolment + *) warn "tpm-enrol: NV check failed (rc=$rc) — leaving to the secure path"; exit 0 ;; +esac + +# Committed to enrolling — one attempt per boot. +: > "$STAMP" +warn "tpm-enrol: no valid fleet cert in NV $NVINDEX — starting FIRST-TIME TPM enrolment" + +# NIC: explicit rd.dot1x.if=, else first non-loopback, non-macsec interface. +NIC=$(getarg rd.dot1x.if=) +if [ -z "$NIC" ]; then + for _dev in /sys/class/net/*; do + _name=${_dev##*/} + case "$_name" in lo|macsec*) continue ;; esac + NIC=$_name; break + done +fi +[ -n "$NIC" ] || { warn "tpm-enrol: no network interface found"; exit 1; } + +# --- network on the PHYSICAL NIC -------------------------------------------- +if [ "$TESTMODE" = "1" ]; then + # Test mode: ip=dhcp already brought the NIC up + wrote resolv.conf. Do NOT + # reconfigure it (a second config races dracut → "address already assigned"). + info "tpm-enrol: test mode (rd.dot1x=0) — using the ip=dhcp NIC as-is" + ip link set "$NIC" up 2>/dev/null || true +else + # Secure mode: nm-unmanaged.sh leaves en* UNMANAGED (wpa_supplicant would own + # it), but there is no supplicant yet — so explicitly hand the NIC to NM for a + # plaintext pre-auth DHCP lease (same permit-set the firmware used for the UKI). + # The runtime `nmcli device set managed yes` overrides the unmanaged drop-in. + ip link set "$NIC" up 2>/dev/null || true + nmcli device set "$NIC" managed yes 2>/dev/null || true + nmcli -w 0 device connect "$NIC" 2>/dev/null || true +fi +i=0 +while [ "$i" -lt "$NETTMO" ]; do + ip -4 addr show "$NIC" 2>/dev/null | grep -q 'inet ' && break + sleep 1; i=$((i + 1)) +done +if [ "$i" -ge "$NETTMO" ]; then + warn "tpm-enrol: no pre-auth DHCP lease on $NIC after ${NETTMO}s" + plymouth message --text="TPM enrolment FAILED (no network)" 2>/dev/null || true + exit 1 +fi +info "tpm-enrol: pre-auth lease on $NIC; handing off to fleet-enrol ($ENROL_URL)" + +# --- hand off to the enrolment binary --------------------------------------- +# It drives the TPM (key/CSR/NV), the broker exchange, and the Plymouth +# fingerprint. Exit 0 = enrolled (reboot to converge); 10 = already enrolled +# (shouldn't reach here after the NV gate, but treat as normal boot); else fail. +/usr/sbin/fleet-enrol -url "$ENROL_URL" -iface "$NIC" -nvindex "$NVINDEX" -handle "$HANDLE" +rc=$? +case "$rc" in + 0) + info "tpm-enrol: enrolled — rebooting into the secure path" + sleep 2 + systemctl --no-block reboot 2>/dev/null || reboot -f 2>/dev/null || \ + { echo 1 > /proc/sys/kernel/sysrq 2>/dev/null; echo b > /proc/sysrq-trigger 2>/dev/null; } + exit 0 ;; + 10) + info "tpm-enrol: fleet-enrol reports cert already present — normal boot" + exit 0 ;; + *) + warn "tpm-enrol: fleet-enrol failed (rc=$rc) — reboot to retry" + exit 1 ;; +esac diff --git a/files/enrol/enrol.go b/files/enrol/enrol.go new file mode 100644 index 0000000..1b28b0e --- /dev/null +++ b/files/enrol/enrol.go @@ -0,0 +1,163 @@ +// enrol.go — the enrolment transaction: build a CSR, submit it to the broker, +// show the returned fingerprint, poll until an admin approves, return the cert. +// TPM-agnostic (takes a crypto.Signer) so it is fully unit-testable with a +// software key against a mock broker; the real signer is TPM-backed (tpm.go). +package main + +import ( + "bytes" + "context" + "crypto" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "net/http" + "time" +) + +// --- broker wire types (mirror serving/enrol-broker) ------------------------ + +type deviceContext struct { + MAC string `json:"mac,omitempty"` + DMIUUID string `json:"dmi_uuid,omitempty"` + DMISerial string `json:"dmi_serial,omitempty"` + EKHash string `json:"ek_hash,omitempty"` +} + +type enrolRequest struct { + CSR string `json:"csr"` + Context deviceContext `json:"context"` +} + +type requestResp struct { + EnrolID string `json:"enrol_id"` + Fingerprint string `json:"fingerprint"` + State string `json:"state"` +} + +type statusResp struct { + State string `json:"state"` + Cert string `json:"cert,omitempty"` + Hostnqn string `json:"hostnqn,omitempty"` + CertSerial string `json:"cert_serial,omitempty"` +} + +// buildCSR produces a PEM CSR signed by signer (the TPM key). The CN is generic +// ("fleet-host") by convention — RADIUS derives real identity from the cert, and +// the per-host NVMe hostnqn is the cert serial; nothing per-host is embedded here. +func buildCSR(signer crypto.Signer, cn string) ([]byte, error) { + tmpl := &x509.CertificateRequest{ + Subject: pkix.Name{CommonName: cn, Organization: []string{"fleet"}}, + } + der, err := x509.CreateCertificateRequest(nil, tmpl, signer) + if err != nil { + return nil, fmt.Errorf("create CSR: %w", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: der}), nil +} + +type brokerClient struct { + http *http.Client + baseURL string // e.g. http://boot.fleet.lab/enrol +} + +func (c *brokerClient) submit(csrPEM []byte, dctx deviceContext) (*requestResp, error) { + body, _ := json.Marshal(enrolRequest{CSR: string(csrPEM), Context: dctx}) + resp, err := c.http.Post(c.baseURL+"/request", "application/json", bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { + return nil, fmt.Errorf("broker /request: %s: %s", resp.Status, bytes.TrimSpace(b)) + } + var r requestResp + if err := json.Unmarshal(b, &r); err != nil { + return nil, fmt.Errorf("broker /request: bad JSON: %w", err) + } + if r.EnrolID == "" { + return nil, fmt.Errorf("broker /request: no enrol_id in response") + } + return &r, nil +} + +func (c *brokerClient) status(id string) (*statusResp, error) { + resp, err := c.http.Get(c.baseURL + "/status/" + id) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("broker /status: %s: %s", resp.Status, bytes.TrimSpace(b)) + } + var s statusResp + if err := json.Unmarshal(b, &s); err != nil { + return nil, fmt.Errorf("broker /status: bad JSON: %w", err) + } + return &s, nil +} + +// enrol runs the full transaction: submit -> show fingerprint -> poll to a +// terminal state. onFingerprint is called once with the code to display; the +// returned statusResp is APPROVED (with cert) or an error for denied/expired/timeout. +func enrol(ctx context.Context, c *brokerClient, csrPEM []byte, dctx deviceContext, + pollEvery time.Duration, onFingerprint func(string)) (*statusResp, error) { + + req, err := c.submit(csrPEM, dctx) + if err != nil { + return nil, err + } + if onFingerprint != nil { + onFingerprint(req.Fingerprint) + } + + tick := time.NewTicker(pollEvery) + defer tick.Stop() + for { + s, err := c.status(req.EnrolID) + if err == nil { + switch s.State { + case "approved": + if s.Cert == "" { + return nil, fmt.Errorf("broker approved but returned no cert") + } + return s, nil + case "denied": + return nil, fmt.Errorf("enrolment denied by admin") + case "expired": + return nil, fmt.Errorf("enrolment request expired") + } + } + // pending (or a transient status error) — keep waiting until the deadline. + select { + case <-ctx.Done(): + return nil, fmt.Errorf("enrolment not approved before timeout") + case <-tick.C: + } + } +} + +// leafDER returns the DER of the FIRST certificate in a (possibly chained) PEM +// bundle — the leaf, which is what wpa_supplicant presents and the hostnqn +// derives from. Also validates it parses. +func leafDER(bundlePEM string) ([]byte, error) { + rest := []byte(bundlePEM) + for { + var blk *pem.Block + blk, rest = pem.Decode(rest) + if blk == nil { + return nil, fmt.Errorf("no CERTIFICATE block in broker response") + } + if blk.Type == "CERTIFICATE" { + if _, err := x509.ParseCertificate(blk.Bytes); err != nil { + return nil, fmt.Errorf("issued cert does not parse: %w", err) + } + return blk.Bytes, nil + } + } +} diff --git a/files/enrol/enrol_test.go b/files/enrol/enrol_test.go new file mode 100644 index 0000000..09e6943 --- /dev/null +++ b/files/enrol/enrol_test.go @@ -0,0 +1,144 @@ +package main + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "math/big" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func bigOne() *big.Int { return big.NewInt(1) } +func bigN(n int64) *big.Int { return big.NewInt(n) } +func pkixCN(cn string) pkix.Name { return pkix.Name{CommonName: cn} } + +// mockBroker mimics serving/enrol-broker: /request returns a fingerprint + +// enrol-id; /status returns pending until approveAfter polls, then approved with +// a cert signed by an in-process CA over the submitted CSR's public key. +func mockBroker(t *testing.T, approveAfterPolls int) *httptest.Server { + t.Helper() + caKey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + caTmpl := &x509.Certificate{SerialNumber: bigOne(), Subject: pkixCN("mock CA"), IsCA: true, BasicConstraintsValid: true} + caDER, _ := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey) + caCert, _ := x509.ParseCertificate(caDER) + + var polls int + var certPEM string + mux := http.NewServeMux() + mux.HandleFunc("/request", func(w http.ResponseWriter, r *http.Request) { + var req enrolRequest + json.NewDecoder(r.Body).Decode(&req) + blk, _ := pem.Decode([]byte(req.CSR)) + if blk == nil { + t.Errorf("submitted CSR is not PEM") + http.Error(w, "bad csr", 400) + return + } + csr, err := x509.ParseCertificateRequest(blk.Bytes) + if err != nil || csr.CheckSignature() != nil { + t.Errorf("CSR signature invalid: %v", err) + http.Error(w, "bad csr sig", 400) + return + } + // sign a leaf over the CSR's public key + leaf := &x509.Certificate{SerialNumber: bigN(0xabc123), Subject: csr.Subject} + der, _ := x509.CreateCertificate(rand.Reader, leaf, caCert, csr.PublicKey, caKey) + certPEM = string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + + string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER})) + json.NewEncoder(w).Encode(requestResp{EnrolID: "e1", Fingerprint: "cedar-onyx-meadow", State: "pending"}) + }) + mux.HandleFunc("/status/", func(w http.ResponseWriter, r *http.Request) { + polls++ + if polls <= approveAfterPolls { + json.NewEncoder(w).Encode(statusResp{State: "pending"}) + return + } + json.NewEncoder(w).Encode(statusResp{ + State: "approved", Cert: certPEM, + Hostnqn: "nqn.2026-06.org.example.fleet:host-abc123", CertSerial: "abc123", + }) + }) + return httptest.NewServer(mux) +} + +func TestEnrolHappyPath(t *testing.T) { + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) // stands in for the TPM signer + csrPEM, err := buildCSR(key, "fleet-host") + if err != nil { + t.Fatalf("buildCSR: %v", err) + } + srv := mockBroker(t, 2) // pending twice, then approved + defer srv.Close() + + c := &brokerClient{http: srv.Client(), baseURL: srv.URL} + var gotFP string + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + st, err := enrol(ctx, c, csrPEM, deviceContext{MAC: "de:ad:be:ef:00:01"}, 10*time.Millisecond, + func(fp string) { gotFP = fp }) + if err != nil { + t.Fatalf("enrol: %v", err) + } + if gotFP != "cedar-onyx-meadow" { + t.Errorf("fingerprint not surfaced: %q", gotFP) + } + der, err := leafDER(st.Cert) + if err != nil { + t.Fatalf("leafDER: %v", err) + } + crt, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse leaf: %v", err) + } + if crt.Subject.CommonName != "fleet-host" { + t.Errorf("leaf CN = %q, want fleet-host", crt.Subject.CommonName) + } +} + +func TestEnrolDenied(t *testing.T) { + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + csrPEM, _ := buildCSR(key, "fleet-host") + mux := http.NewServeMux() + mux.HandleFunc("/request", func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(requestResp{EnrolID: "e1", Fingerprint: "x", State: "pending"}) + }) + mux.HandleFunc("/status/", func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(statusResp{State: "denied"}) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + c := &brokerClient{http: srv.Client(), baseURL: srv.URL} + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, err := enrol(ctx, c, csrPEM, deviceContext{}, time.Millisecond, nil) + if err == nil || !strings.Contains(err.Error(), "denied") { + t.Fatalf("expected denied error, got %v", err) + } +} + +func TestLeafDERChain(t *testing.T) { + // two-cert bundle -> leaf is the first + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + mk := func(cn string) string { + tmpl := &x509.Certificate{SerialNumber: bigOne(), Subject: pkixCN(cn)} + der, _ := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + } + der, err := leafDER(mk("leaf") + mk("inter")) + if err != nil { + t.Fatalf("leafDER: %v", err) + } + crt, _ := x509.ParseCertificate(der) + if crt.Subject.CommonName != "leaf" { + t.Errorf("got %q, want leaf", crt.Subject.CommonName) + } +} diff --git a/files/enrol/go.mod b/files/enrol/go.mod new file mode 100644 index 0000000..6ed0d73 --- /dev/null +++ b/files/enrol/go.mod @@ -0,0 +1,12 @@ +// fleet-enrol — inline empty-NV first-boot TPM enrolment client (initramfs). +// See docs/tpm-enrolment-design.md. Talks to the TPM directly via go-tpm (pure +// Go, no CGO) — one static, reproducible binary (like fleet-hostapi), deps +// VENDORED so the image build runs with GOPROXY=off. The go-tpm software +// simulator (CGO) is confined to the separate simtest/ module. +module fleet-enrol + +go 1.26 + +require github.com/google/go-tpm v0.9.8 + +require golang.org/x/sys v0.8.0 // indirect diff --git a/files/enrol/go.sum b/files/enrol/go.sum new file mode 100644 index 0000000..5416cec --- /dev/null +++ b/files/enrol/go.sum @@ -0,0 +1,6 @@ +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba h1:qJEJcuLzH5KDR0gKc0zcktin6KSAwL7+jWKBYceddTc= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba/go.mod h1:EFYHy8/1y2KfgTAsx7Luu7NGhoxtuVHnNo8jE7FikKc= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/files/enrol/http.go b/files/enrol/http.go new file mode 100644 index 0000000..c03e8f4 --- /dev/null +++ b/files/enrol/http.go @@ -0,0 +1,20 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "net/http" + "time" +) + +// httpClient builds the broker HTTP client. With a non-nil pool it pins the +// fleet CA for an https broker (prod); plain http (lab) ignores it. Short +// timeouts on the individual calls — the long human-approval wait is the poll +// LOOP in enrol(), not a single request. +func httpClient(pool *x509.CertPool) *http.Client { + tr := &http.Transport{} + if pool != nil { + tr.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12} + } + return &http.Client{Timeout: 30 * time.Second, Transport: tr} +} diff --git a/files/enrol/main.go b/files/enrol/main.go new file mode 100644 index 0000000..3bbd3f8 --- /dev/null +++ b/files/enrol/main.go @@ -0,0 +1,211 @@ +// fleet-enrol — inline empty-NV first-boot TPM enrolment client. +// +// Invoked from the dracut hook (95dot1x-macsec/tpm-enrol.sh) ONLY when the +// secure build boots a host whose TPM NV 0x01800100 is empty. It: +// 1. ensures an ECDSA signing key exists at a non-reserved owner handle +// (default 0x81000100 — deliberately NOT the EK slot 0x81010001), +// 2. builds a CSR with it (key never leaves the TPM), +// 3. submits CSR + device context to the enrolment broker, shows the returned +// fingerprint on the Plymouth splash, and polls until an admin approves, +// 4. writes the issued cert (DER) to NV 0x01800100. +// +// The hook then reboots into the normal secure path. Everything host-specific +// (key, cert, hostnqn) is created here at runtime — nothing is baked (invariant #1). +// +// Exit codes (default/enrol mode): 0 = enrolled (cert written to NV; caller +// should reboot); 10 = a VALID fleet cert is already present (normal boot — +// caller proceeds); 1 = failure (caller leaves NV empty -> boot fails -> emergency). +// +// Exit codes (-check-only mode, no network/no key-gen): 0 = a valid fleet cert is +// present (caller: normal boot); 20 = enrolment needed (NV empty/garbage/foreign/ +// expired); 1 = error (e.g. TPM open failed). The dracut hook uses this as the +// single content-aware gate instead of a blind "is the NV index defined" check. +package main + +import ( + "context" + "crypto/x509" + "flag" + "fmt" + "log" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "fleet-enrol/tpmkey" + + "github.com/google/go-tpm/tpm2" + "github.com/google/go-tpm/tpm2/transport" +) + +const ( + exitAlreadyEnrolled = 10 // enrol mode: a valid fleet cert is already present + exitNeedEnrol = 20 // -check-only: no valid fleet cert → enrolment needed +) + +func main() { + var ( + url = flag.String("url", "http://boot.fleet.lab/enrol", "enrolment broker base URL") + tpmPath = flag.String("tpm", "/dev/tpmrm0", "TPM device path") + nvHex = flag.String("nvindex", "0x01800100", "TPM NV index for the client cert") + handleHex = flag.String("handle", "0x81000100", "TPM persistent handle for the signing key (non-reserved owner handle; NOT the EK slot 0x81010001)") + cn = flag.String("cn", "fleet-host", "CSR CommonName (generic; identity is the cert serial)") + iface = flag.String("iface", "", "network interface for the device-context MAC") + caFile = flag.String("cacert", "/etc/dot1x/radius-ca.pem", "fleet CA bundle: verifies an https broker URL AND the NV cert's chain") + waitFor = flag.Duration("timeout", 30*time.Minute, "max wait for admin approval") + pollEvery = flag.Duration("poll", 5*time.Second, "status poll interval") + checkOnly = flag.Bool("check-only", false, "classify the NV cert and exit (no network, no key-gen): 0=valid fleet cert, 20=enrol needed, 1=error") + ) + flag.Parse() + log.SetFlags(0) + log.SetPrefix("fleet-enrol: ") + + nvIndex, err := parseHandle(*nvHex) + if err != nil { + log.Fatalf("bad -nvindex %q: %v", *nvHex, err) + } + keyHandle, err := parseHandle(*handleHex) + if err != nil { + log.Fatalf("bad -handle %q: %v", *handleHex, err) + } + + // /dev/tpmrm0 (the kernel TPM resource manager) is a simple command/response + // char device: write one command, read one complete response. Wrapping the + // *os.File with transport.FromReadWriter (as the simulator tests do) avoids + // the linuxtpm transport and its golang.org/x/sys dependency, keeping the + // vendored, reproducible initramfs binary tiny (go-tpm only). + f, err := os.OpenFile(*tpmPath, os.O_RDWR, 0) + if err != nil { + log.Fatalf("open TPM %s: %v", *tpmPath, err) + } + defer f.Close() + tpm := transport.FromReadWriter(f) + + // Content-aware enrolment decision: read the NV cert, parse it, and (when the + // fleet CA is available) verify it chains — so leftover/foreign/garbage NV does + // NOT masquerade as "enrolled" (a bare "index defined" check let an old + // tpm-provision cert or random bytes silently suppress enrolment). + enrolled, detail := nvEnrolled(tpm, nvIndex, *caFile) + + if *checkOnly { + if enrolled { + log.Printf("check-only: %s — normal boot", detail) + os.Exit(0) + } + log.Printf("check-only: enrolment needed — %s", detail) + os.Exit(exitNeedEnrol) + } + + // Full enrol mode: a valid fleet cert means this is a normal boot. + if enrolled { + log.Printf("%s — normal boot", detail) + os.Exit(exitAlreadyEnrolled) + } + + ply("enrol-active") + plyMsg("TPM enrolment: contacting fleet…") + + signer, created, err := tpmkey.EnsureKey(tpm, keyHandle) + if err != nil { + plyMsg("TPM enrolment FAILED (key)") + log.Fatalf("ensure key at %s: %v", *handleHex, err) + } + if created { + log.Printf("created persistent signing key at %s", *handleHex) + } else { + log.Printf("reusing persistent signing key at %s", *handleHex) + } + + csrPEM, err := buildCSR(signer, *cn) + if err != nil { + plyMsg("TPM enrolment FAILED (CSR)") + log.Fatalf("build CSR: %v", err) + } + + client, err := newBrokerClient(*url, *caFile) + if err != nil { + log.Fatalf("broker client: %v", err) + } + dctx := gatherContext(*iface) + + ctx, cancel := context.WithTimeout(context.Background(), *waitFor) + defer cancel() + log.Printf("submitting CSR to %s (waiting up to %s for approval)", *url, *waitFor) + st, err := enrol(ctx, client, csrPEM, dctx, *pollEvery, func(fp string) { + ply("enrol-wait") + plyMsg("APPROVE THIS DEVICE — code: " + fp) + log.Printf("fingerprint: %s", fp) + }) + if err != nil { + plyMsg("TPM enrolment FAILED — reboot to retry") + log.Fatalf("enrolment: %v", err) + } + + der, err := leafDER(st.Cert) + if err != nil { + plyMsg("TPM enrolment FAILED (bad cert)") + log.Fatalf("issued cert: %v", err) + } + if err := tpmkey.WriteNVCert(tpm, nvIndex, der); err != nil { + plyMsg("TPM enrolment FAILED (NV write)") + log.Fatalf("write cert to NV %s: %v (TPM owner auth?)", *nvHex, err) + } + + ply("enrol-done") + plyMsg("Enrolment complete — rebooting") + log.Printf("ENROLLED: hostnqn %s (serial %s); cert written to NV %s", + st.Hostnqn, st.CertSerial, *nvHex) + // The caller (dracut hook) reboots into the secure path. + os.Exit(0) +} + +func parseHandle(s string) (tpm2.TPMHandle, error) { + v, err := strconv.ParseUint(strings.TrimPrefix(strings.ToLower(s), "0x"), 16, 32) + if err != nil { + return 0, err + } + return tpm2.TPMHandle(v), nil +} + +func newBrokerClient(url, caFile string) (*brokerClient, error) { + c := &brokerClient{baseURL: strings.TrimRight(url, "/"), http: httpClient(nil)} + if strings.HasPrefix(url, "https:") { + pem, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("read CA %s: %w", caFile, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("no certs in %s", caFile) + } + c.http = httpClient(pool) + } + return c, nil +} + +// gatherContext collects advisory device context for the admin to recognise the +// device. Best-effort — every field is optional. +func gatherContext(iface string) deviceContext { + d := deviceContext{ + DMIUUID: strings.TrimSpace(readFile("/sys/class/dmi/id/product_uuid")), + DMISerial: strings.TrimSpace(readFile("/sys/class/dmi/id/product_serial")), + } + if iface != "" { + d.MAC = strings.TrimSpace(readFile("/sys/class/net/" + iface + "/address")) + } + return d +} + +func readFile(p string) string { + b, err := os.ReadFile(p) + if err != nil { + return "" + } + return string(b) +} + +// Plymouth best-effort splash updates (states only; the fingerprint is public). +func ply(status string) { _ = exec.Command("plymouth", "update", "--status="+status).Run() } +func plyMsg(text string) { _ = exec.Command("plymouth", "message", "--text="+text).Run() } diff --git a/files/enrol/nvcheck.go b/files/enrol/nvcheck.go new file mode 100644 index 0000000..df5e674 --- /dev/null +++ b/files/enrol/nvcheck.go @@ -0,0 +1,81 @@ +// nvcheck.go — content-aware validation of the cert stored in TPM NV. The enrol +// trigger must key on "do we hold a VALID FLEET cert", not "is the NV index +// defined": a bare presence check let leftover state (an earlier tpm-provision +// run, another OS) or random bytes masquerade as enrolled, silently suppressing +// enrolment (bench-seen 2026-07-02: a host booted "cert present — normal boot" +// with no real identity). classifyNVCert is TPM-agnostic (takes the DER + a CA +// pool) so it is unit-testable without a TPM. +package main + +import ( + "crypto/x509" + "fmt" + "os" + "time" + + "fleet-enrol/tpmkey" + + "github.com/google/go-tpm/tpm2" + "github.com/google/go-tpm/tpm2/transport" +) + +// loadCAPool builds a cert pool from the fleet CA bundle. Returns nil if the file +// is missing or contains no parseable certificates (e.g. the placeholder +// radius-ca.pem in lab builds) — callers then fall back to a chain-less check. +func loadCAPool(caFile string) *x509.CertPool { + b, err := os.ReadFile(caFile) + if err != nil { + return nil + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(b) { + return nil + } + return pool +} + +// classifyNVCert decides whether the DER in NV is a cert we should treat as an +// enrolled fleet identity. Rules, in order: must parse as X.509; must be within +// its validity window; and — when a fleet CA pool is available — must chain to +// it. With no CA pool (lab/placeholder), a parseable, unexpired cert is accepted +// (chain unverified) rather than forcing a re-enrol loop. Every reject returns a +// human-readable reason for the boot log. +func classifyNVCert(der []byte, caPool *x509.CertPool, now time.Time) (enrolled bool, detail string) { + crt, err := x509.ParseCertificate(der) + if err != nil { + return false, "NV content is not an X.509 certificate" + } + switch { + case now.Before(crt.NotBefore): + return false, "NV cert is not yet valid" + case now.After(crt.NotAfter): + return false, fmt.Sprintf("NV cert expired (%s)", crt.NotAfter.Format(time.RFC3339)) + } + if caPool == nil { + return true, fmt.Sprintf("NV cert present (CN=%q serial %x) — fleet CA unavailable, chain UNVERIFIED", + crt.Subject.CommonName, crt.SerialNumber) + } + // Roots and Intermediates both = the bundle: lets a leaf signed by an + // intermediate verify whether the bundle carries the root, the intermediate, + // or both. ExtKeyUsageAny: we assert membership, not usage (the cert's EKU is + // clientAuth; RADIUS enforces usage at auth time). + if _, err := crt.Verify(x509.VerifyOptions{ + Roots: caPool, + Intermediates: caPool, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + CurrentTime: now, + }); err != nil { + return false, "NV cert does not chain to the fleet CA: " + err.Error() + } + return true, fmt.Sprintf("valid fleet cert in NV (CN=%q serial %x)", crt.Subject.CommonName, crt.SerialNumber) +} + +// nvEnrolled reads the NV cert and classifies it. A read error (undefined/ +// unreadable index) is "not enrolled" — enrol and (over)write. +func nvEnrolled(tpm transport.TPM, nvIndex tpm2.TPMHandle, caFile string) (bool, string) { + der, err := tpmkey.ReadNVCert(tpm, nvIndex) + if err != nil { + return false, "NV empty/undefined/unreadable" + } + return classifyNVCert(der, loadCAPool(caFile), time.Now()) +} diff --git a/files/enrol/nvcheck_test.go b/files/enrol/nvcheck_test.go new file mode 100644 index 0000000..7ebb156 --- /dev/null +++ b/files/enrol/nvcheck_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "testing" + "time" +) + +// mkCA returns a self-signed CA cert + key and a pool trusting it. +func mkCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509.CertPool) { + t.Helper() + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + tmpl := &x509.Certificate{ + SerialNumber: bigOne(), Subject: pkixCN("fleet CA"), + IsCA: true, BasicConstraintsValid: true, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour), + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create CA: %v", err) + } + ca, _ := x509.ParseCertificate(der) + pool := x509.NewCertPool() + pool.AddCert(ca) + return ca, key, pool +} + +// mkLeaf issues a client leaf DER signed by (caCert, caKey), valid over [nb, na]. +func mkLeaf(t *testing.T, caCert *x509.Certificate, caKey *ecdsa.PrivateKey, nb, na time.Time) []byte { + t.Helper() + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + tmpl := &x509.Certificate{ + SerialNumber: bigN(0x1234), Subject: pkixCN("fleet-host"), + NotBefore: nb, NotAfter: na, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &key.PublicKey, caKey) + if err != nil { + t.Fatalf("create leaf: %v", err) + } + return der +} + +func TestClassifyNVCert(t *testing.T) { + now := time.Now() + ca, caKey, pool := mkCA(t) + valid := mkLeaf(t, ca, caKey, now.Add(-time.Hour), now.Add(time.Hour)) + expired := mkLeaf(t, ca, caKey, now.Add(-2*time.Hour), now.Add(-time.Hour)) + + // a leaf from a DIFFERENT CA (foreign / not our fleet) + otherCA, otherKey, _ := mkCA(t) + foreign := mkLeaf(t, otherCA, otherKey, now.Add(-time.Hour), now.Add(time.Hour)) + + cases := []struct { + name string + der []byte + pool *x509.CertPool + enrolled bool + }{ + {"valid fleet cert chains to CA", valid, pool, true}, + {"garbage bytes are not a cert", []byte("not a certificate at all"), pool, false}, + {"expired cert triggers re-enrol", expired, pool, false}, + {"foreign cert does not chain", foreign, pool, false}, + {"no CA pool: parseable+unexpired accepted", valid, nil, true}, + {"no CA pool: expired still rejected", expired, nil, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, detail := classifyNVCert(c.der, c.pool, now) + if got != c.enrolled { + t.Fatalf("enrolled=%v want %v (detail: %s)", got, c.enrolled, detail) + } + }) + } +} diff --git a/files/enrol/simtest/go.mod b/files/enrol/simtest/go.mod new file mode 100644 index 0000000..ed40a2b --- /dev/null +++ b/files/enrol/simtest/go.mod @@ -0,0 +1,19 @@ +// fleet-enrol-simtest — a SEPARATE module so the go-tpm software simulator +// (CGO + the MS TPM reference C, ~8MB) and go-tpm-tools stay OUT of the shipped +// fleet-enrol module's vendored deps. Run from this dir: +// go test ./... (needs CGO + a C toolchain for the simulator) +// It exercises tpmkey's real TPM ceremony (key-gen, sign, NV) against a software +// TPM, so the go-tpm code is proven off-hardware without bloating the image build. +module fleet-enrol-simtest + +go 1.26 + +require ( + fleet-enrol v0.0.0 + github.com/google/go-tpm v0.9.8 + github.com/google/go-tpm-tools v0.4.9 +) + +require golang.org/x/sys v0.38.0 // indirect + +replace fleet-enrol => ../ diff --git a/files/enrol/simtest/go.sum b/files/enrol/simtest/go.sum new file mode 100644 index 0000000..b9e87ef --- /dev/null +++ b/files/enrol/simtest/go.sum @@ -0,0 +1,26 @@ +github.com/GoogleCloudPlatform/confidential-space/server v0.0.0-20260522213940-e5c6d01a3007 h1:DoeEFwEGBdqcawmpiWtSsSVVZ+wk3zpqvcvssO2JLmY= +github.com/GoogleCloudPlatform/confidential-space/server v0.0.0-20260522213940-e5c6d01a3007/go.mod h1:s8F0JYEods/WL03WxZaGsWCnumZeeLD+WKHzspOV9u0= +github.com/google/go-configfs-tsm v0.3.3-0.20240919001351-b4b5b84fdcbc h1:SG12DWUUM5igxm+//YX5Yq4vhdoRnOG9HkCodkOn+YU= +github.com/google/go-configfs-tsm v0.3.3-0.20240919001351-b4b5b84fdcbc/go.mod h1:EL1GTDFMb5PZQWDviGfZV9n87WeGTR/JUg13RfwkgRo= +github.com/google/go-eventlog v0.0.3-0.20260416001248-6807b85eecf0 h1:STyioPkz8nqMMIk3+YlyJ/WyEJZxho1YUZXu99uAbQ0= +github.com/google/go-eventlog v0.0.3-0.20260416001248-6807b85eecf0/go.mod h1:7huE5P8w2NTObSwSJjboHmB7ioBNblkijdzoVa2skfQ= +github.com/google/go-sev-guest v0.14.0 h1:dCb4F3YrHTtrDX3cYIPTifEDz7XagZmXQioxRBW4wOo= +github.com/google/go-sev-guest v0.14.0/go.mod h1:SK9vW+uyfuzYdVN0m8BShL3OQCtXZe/JPF7ZkpD3760= +github.com/google/go-tdx-guest v0.3.2-0.20250814004405-ffb0869e6f4d h1:Ff8goEP/ue2/rZT5qyoRicuySCYDbAXEZS8Cf1fgsUo= +github.com/google/go-tdx-guest v0.3.2-0.20250814004405-ffb0869e6f4d/go.mod h1:uHy3VaNXNXhl0fiPxKqTxieeouqQmW6A0EfLcaeCYBk= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm-tools v0.4.9 h1:jZEhnE4WRFbomSssBH2gWaIViIHU1gjH1jz76+xC9bI= +github.com/google/go-tpm-tools v0.4.9/go.mod h1:Omb8zosA8qY9URn1gsrO2i4b6DFqGp29BqNx18V66c4= +github.com/google/logger v1.1.1 h1:+6Z2geNxc9G+4D4oDO9njjjn2d0wN5d7uOo0vOIW1NQ= +github.com/google/logger v1.1.1/go.mod h1:BkeJZ+1FhQ+/d087r4dzojEg1u2ZX+ZqG1jTUrLM+zQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/files/enrol/simtest/tpmsim_test.go b/files/enrol/simtest/tpmsim_test.go new file mode 100644 index 0000000..919f460 --- /dev/null +++ b/files/enrol/simtest/tpmsim_test.go @@ -0,0 +1,261 @@ +// Package simtest validates tpmkey's TPM ceremony against the go-tpm software +// simulator (the MS TPM 2.0 reference). It lives in its own module so the CGO +// simulator never enters the shipped fleet-enrol build; see go.mod. +package simtest + +import ( + "bytes" + "crypto/ecdsa" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "strings" + "testing" + + "fleet-enrol/tpmkey" + + "github.com/google/go-tpm-tools/simulator" + "github.com/google/go-tpm/tpm2" + "github.com/google/go-tpm/tpm2/transport" +) + +func openSim(t *testing.T) transport.TPM { + t.Helper() + sim, err := simulator.Get() + if err != nil { + t.Fatalf("simulator: %v", err) + } + t.Cleanup(func() { sim.Close() }) + return transport.FromReadWriter(sim) +} + +func TestEnsureKeyAndSignCSR(t *testing.T) { + tpm := openSim(t) + const h = tpm2.TPMHandle(0x81000100) + + signer, created, err := tpmkey.EnsureKey(tpm, h) + if err != nil { + t.Fatalf("EnsureKey: %v", err) + } + if !created { + t.Errorf("first EnsureKey should report created=true") + } + if _, ok := signer.Public().(*ecdsa.PublicKey); !ok { + t.Fatalf("signer public key is not ECDSA") + } + + // idempotent: second call reuses the persistent key (same pubkey). + signer2, created2, err := tpmkey.EnsureKey(tpm, h) + if err != nil || created2 { + t.Fatalf("second EnsureKey: created=%v err=%v", created2, err) + } + if !signer.Public().(*ecdsa.PublicKey).Equal(signer2.Public()) { + t.Errorf("persistent key changed between calls") + } + + // the TPM signer must produce a CSR that verifies — this is the exact + // openssl-tpm2-provider path the Go binary replaces. + tmpl := &x509.CertificateRequest{Subject: pkix.Name{CommonName: "fleet-host"}} + der, err := x509.CreateCertificateRequest(rand.Reader, tmpl, signer) + if err != nil { + t.Fatalf("CreateCertificateRequest with TPM signer: %v", err) + } + csr, err := x509.ParseCertificateRequest(der) + if err != nil { + t.Fatalf("parse CSR: %v", err) + } + if err := csr.CheckSignature(); err != nil { + t.Fatalf("CSR signature (TPM-signed) invalid: %v", err) + } + + // raw signer sanity: sign a digest, verify with the public key. + dig := sha256.Sum256([]byte("hello")) + sig, err := signer.Sign(rand.Reader, dig[:], nil) + if err != nil { + t.Fatalf("sign: %v", err) + } + if !ecdsa.VerifyASN1(signer.Public().(*ecdsa.PublicKey), dig[:], sig) { + t.Fatalf("TPM ECDSA signature did not verify") + } +} + +func TestNVWriteAndPresence(t *testing.T) { + tpm := openSim(t) + const nv = tpm2.TPMHandle(0x01800100) + + if tpmkey.NVPresent(tpm, nv) { + t.Fatalf("NV should be empty on a fresh TPM") + } + if _, err := tpmkey.ReadNVCert(tpm, nv); err == nil { + t.Fatalf("ReadNVCert should error on an undefined index") + } + // realistic cert-sized payload that also crosses the 512B read/write chunk. + want := make([]byte, 700) + rand.Read(want) + if err := tpmkey.WriteNVCert(tpm, nv, want); err != nil { + t.Fatalf("WriteNVCert: %v", err) + } + if !tpmkey.NVPresent(tpm, nv) { + t.Fatalf("NV should be present after write") + } + // ReadNVCert must return exactly what was written (across the chunk boundary). + got, err := tpmkey.ReadNVCert(tpm, nv) + if err != nil { + t.Fatalf("ReadNVCert: %v", err) + } + if !bytes.Equal(got, want) { + t.Fatalf("ReadNVCert roundtrip mismatch: got %d bytes, want %d", len(got), len(want)) + } + // re-write (re-enrolment) must succeed (undefine + redefine). + if err := tpmkey.WriteNVCert(tpm, nv, want[:400]); err != nil { + t.Fatalf("WriteNVCert rewrite: %v", err) + } + if got, _ := tpmkey.ReadNVCert(tpm, nv); !bytes.Equal(got, want[:400]) { + t.Fatalf("ReadNVCert after rewrite mismatch: got %d bytes, want 400", len(got)) + } +} + +// Regression for the bench failure "key at handle is not ECC ... selector value +// was 1": an older provisioning path left an RSA SIGNING key at the enrol handle, +// and EnsureKey fatally failed trying to read it as ECC. It must now evict the +// stale key and create a fresh ECC one. +func TestEnsureKeyEvictsStaleNonECC(t *testing.T) { + tpm := openSim(t) + const h = tpm2.TPMHandle(0x81000100) + + // Pre-populate the handle with an RSA SIGNING key (mimics tpm2-tools/openssl + // default) — SignEncrypt, NOT restricted/decrypt, so it is safe to evict. + rsaTmpl := tpm2.TPMTPublic{ + Type: tpm2.TPMAlgRSA, + NameAlg: tpm2.TPMAlgSHA256, + ObjectAttributes: tpm2.TPMAObject{ + FixedTPM: true, FixedParent: true, SensitiveDataOrigin: true, + UserWithAuth: true, SignEncrypt: true, NoDA: true, + }, + Parameters: tpm2.NewTPMUPublicParms(tpm2.TPMAlgRSA, &tpm2.TPMSRSAParms{ + Symmetric: tpm2.TPMTSymDefObject{Algorithm: tpm2.TPMAlgNull}, + Scheme: tpm2.TPMTRSAScheme{ + Scheme: tpm2.TPMAlgRSASSA, + Details: tpm2.NewTPMUAsymScheme(tpm2.TPMAlgRSASSA, &tpm2.TPMSSigSchemeRSASSA{ + HashAlg: tpm2.TPMAlgSHA256, + }), + }, + KeyBits: 2048, + }), + } + cp, err := tpm2.CreatePrimary{ + PrimaryHandle: tpm2.TPMRHOwner, InPublic: tpm2.New2B(rsaTmpl), + }.Execute(tpm) + if err != nil { + t.Fatalf("create RSA primary: %v", err) + } + if _, err := (tpm2.EvictControl{ + Auth: tpm2.TPMRHOwner, + ObjectHandle: &tpm2.NamedHandle{Handle: cp.ObjectHandle, Name: cp.Name}, + PersistentHandle: h, + }).Execute(tpm); err != nil { + t.Fatalf("persist stale RSA key: %v", err) + } + _, _ = (tpm2.FlushContext{FlushHandle: cp.ObjectHandle}).Execute(tpm) + + // EnsureKey must self-heal: evict the RSA key, create + persist a fresh ECC key. + signer, created, err := tpmkey.EnsureKey(tpm, h) + if err != nil { + t.Fatalf("EnsureKey over stale RSA key: %v", err) + } + if !created { + t.Errorf("EnsureKey should have recreated the key (created=true)") + } + if _, ok := signer.Public().(*ecdsa.PublicKey); !ok { + t.Fatalf("recreated signer is not ECDSA") + } + // and it must be usable for signing (the whole point of the recovery). + dig := sha256.Sum256([]byte("post-heal")) + sig, err := signer.Sign(rand.Reader, dig[:], nil) + if err != nil { + t.Fatalf("sign after heal: %v", err) + } + if !ecdsa.VerifyASN1(signer.Public().(*ecdsa.PublicKey), dig[:], sig) { + t.Fatalf("post-heal ECDSA signature did not verify") + } +} + +// Safety guard: if the handle ever holds a restricted-decrypt object (an EK/SRK — +// e.g. handle misconfigured to the EK slot 0x81010001), EnsureKey must REFUSE to +// evict it and error, never clobber the TPM's endorsement/storage identity. +func TestEnsureKeyRefusesToEvictProtectedKey(t *testing.T) { + tpm := openSim(t) + const h = tpm2.TPMHandle(0x81000100) + + // A restricted decryption primary — the shape of an EK/SRK (storage key). + skTmpl := tpm2.TPMTPublic{ + Type: tpm2.TPMAlgECC, + NameAlg: tpm2.TPMAlgSHA256, + ObjectAttributes: tpm2.TPMAObject{ + FixedTPM: true, FixedParent: true, SensitiveDataOrigin: true, + UserWithAuth: true, Restricted: true, Decrypt: true, NoDA: true, + }, + Parameters: tpm2.NewTPMUPublicParms(tpm2.TPMAlgECC, &tpm2.TPMSECCParms{ + Symmetric: tpm2.TPMTSymDefObject{ + Algorithm: tpm2.TPMAlgAES, + KeyBits: tpm2.NewTPMUSymKeyBits(tpm2.TPMAlgAES, tpm2.TPMKeyBits(128)), + Mode: tpm2.NewTPMUSymMode(tpm2.TPMAlgAES, tpm2.TPMAlgCFB), + }, + Scheme: tpm2.TPMTECCScheme{Scheme: tpm2.TPMAlgNull}, + CurveID: tpm2.TPMECCNistP256, + KDF: tpm2.TPMTKDFScheme{Scheme: tpm2.TPMAlgNull}, + }), + } + cp, err := tpm2.CreatePrimary{ + PrimaryHandle: tpm2.TPMRHOwner, InPublic: tpm2.New2B(skTmpl), + }.Execute(tpm) + if err != nil { + t.Fatalf("create storage-like primary: %v", err) + } + if _, err := (tpm2.EvictControl{ + Auth: tpm2.TPMRHOwner, + ObjectHandle: &tpm2.NamedHandle{Handle: cp.ObjectHandle, Name: cp.Name}, + PersistentHandle: h, + }).Execute(tpm); err != nil { + t.Fatalf("persist storage-like key: %v", err) + } + _, _ = (tpm2.FlushContext{FlushHandle: cp.ObjectHandle}).Execute(tpm) + + // EnsureKey must refuse (error), and must NOT have evicted the object. + if _, _, err := tpmkey.EnsureKey(tpm, h); err == nil { + t.Fatalf("EnsureKey should refuse to evict a restricted-decrypt (EK/SRK) object") + } + if _, err := (tpm2.ReadPublic{ObjectHandle: h}).Execute(tpm); err != nil { + t.Fatalf("protected object was evicted (should have been left intact): %v", err) + } +} + +// Regression for the Dell 820 finding: our NV index collided with an OEM +// PLATFORM-created index (platformcreate|policydelete) that no owner clear can +// remove. WriteNVCert must REFUSE such an index with a clear error (the remedy +// is a different rd.dot1x.nvindex), not a confusing undefine failure. +func TestWriteNVCertRefusesPlatformIndex(t *testing.T) { + tpm := openSim(t) + const nv = tpm2.TPMHandle(0x01800100) + + def := tpm2.NVDefineSpace{ + AuthHandle: tpm2.TPMRHPlatform, + PublicInfo: tpm2.New2B(tpm2.TPMSNVPublic{ + NVIndex: nv, + NameAlg: tpm2.TPMAlgSHA256, + Attributes: tpm2.TPMANV{ + PlatformCreate: true, AuthRead: true, AuthWrite: true, + NoDA: true, NT: tpm2.TPMNTOrdinary, + }, + DataSize: 8, + }), + } + if _, err := def.Execute(tpm); err != nil { + t.Fatalf("define platform NV: %v", err) + } + err := tpmkey.WriteNVCert(tpm, nv, make([]byte, 64)) + if err == nil || !strings.Contains(err.Error(), "platform-created") { + t.Fatalf("WriteNVCert should refuse a platform-created index; got: %v", err) + } +} diff --git a/files/enrol/tpmkey/tpmkey.go b/files/enrol/tpmkey/tpmkey.go new file mode 100644 index 0000000..569f447 --- /dev/null +++ b/files/enrol/tpmkey/tpmkey.go @@ -0,0 +1,280 @@ +// Package tpmkey — TPM 2.0 operations via go-tpm (pure Go, no CGO, no tpm2-tools +// / no openssl tpm2 provider). Provides: ensure a persistent ECDSA signing key +// at a non-reserved owner handle (default 0x81000100 — NOT the EK slot +// 0x81010001, see EnsureKey), a crypto.Signer backed by it (for CSR signing), +// and NV cert read-presence + write at 0x01800100. Talks to any transport.TPM, +// so the same +// code runs against /dev/tpmrm0 (production) and the software simulator (tests). +// +// Exported so the ceremony can be tested against the go-tpm simulator from a +// SEPARATE module (simtest/), keeping this module's vendored deps lean (go-tpm +// only) — the simulator (CGO + the MS TPM reference C) never enters the image build. +package tpmkey + +import ( + "crypto" + "crypto/ecdsa" + "encoding/asn1" + "fmt" + "io" + "math/big" + + "github.com/google/go-tpm/tpm2" + "github.com/google/go-tpm/tpm2/transport" +) + +// eccSigningTemplate: a NON-restricted ECC P-256 ECDSA-SHA256 signing key. +// Non-restricted so it can sign an externally-computed digest (the CSR TBS) with +// a NULL validation ticket; Symmetric/KDF are explicitly NULL (sign-only key). +func eccSigningTemplate() tpm2.TPMTPublic { + return tpm2.TPMTPublic{ + Type: tpm2.TPMAlgECC, + NameAlg: tpm2.TPMAlgSHA256, + ObjectAttributes: tpm2.TPMAObject{ + FixedTPM: true, + FixedParent: true, + SensitiveDataOrigin: true, + UserWithAuth: true, + SignEncrypt: true, + NoDA: true, + }, + Parameters: tpm2.NewTPMUPublicParms(tpm2.TPMAlgECC, &tpm2.TPMSECCParms{ + Symmetric: tpm2.TPMTSymDefObject{Algorithm: tpm2.TPMAlgNull}, + Scheme: tpm2.TPMTECCScheme{ + Scheme: tpm2.TPMAlgECDSA, + Details: tpm2.NewTPMUAsymScheme(tpm2.TPMAlgECDSA, &tpm2.TPMSSigSchemeECDSA{ + HashAlg: tpm2.TPMAlgSHA256, + }), + }, + CurveID: tpm2.TPMECCNistP256, + KDF: tpm2.TPMTKDFScheme{Scheme: tpm2.TPMAlgNull}, + }), + } +} + +// Signer implements crypto.Signer using a persistent TPM key. +type Signer struct { + tpm transport.TPM + handle tpm2.TPMHandle + name tpm2.TPM2BName + pub *ecdsa.PublicKey +} + +func (s *Signer) Public() crypto.PublicKey { return s.pub } + +func (s *Signer) Sign(_ io.Reader, digest []byte, _ crypto.SignerOpts) ([]byte, error) { + rsp, err := tpm2.Sign{ + KeyHandle: tpm2.NamedHandle{Handle: s.handle, Name: s.name}, + Digest: tpm2.TPM2BDigest{Buffer: digest}, + InScheme: tpm2.TPMTSigScheme{ + Scheme: tpm2.TPMAlgECDSA, + Details: tpm2.NewTPMUSigScheme(tpm2.TPMAlgECDSA, &tpm2.TPMSSchemeHash{ + HashAlg: tpm2.TPMAlgSHA256, + }), + }, + Validation: tpm2.TPMTTKHashCheck{Tag: tpm2.TPMSTHashCheck}, + }.Execute(s.tpm) + if err != nil { + return nil, fmt.Errorf("tpm sign: %w", err) + } + ecc, err := rsp.Signature.Signature.ECDSA() + if err != nil { + return nil, fmt.Errorf("tpm sign: not ECDSA: %w", err) + } + r := new(big.Int).SetBytes(ecc.SignatureR.Buffer) + sInt := new(big.Int).SetBytes(ecc.SignatureS.Buffer) + // x509 expects an ASN.1 DER ECDSA-Sig-Value{ r, s }. + return asn1.Marshal(struct{ R, S *big.Int }{r, sInt}) +} + +func signerFromPub(tpm transport.TPM, handle tpm2.TPMHandle, name tpm2.TPM2BName, pub2b tpm2.TPM2BPublic) (*Signer, error) { + pub, err := pub2b.Contents() + if err != nil { + return nil, err + } + det, err := pub.Parameters.ECCDetail() + if err != nil { + return nil, fmt.Errorf("key at handle is not ECC: %w", err) + } + uniq, err := pub.Unique.ECC() + if err != nil { + return nil, err + } + pk, err := tpm2.ECDSAPub(det, uniq) + if err != nil { + return nil, err + } + return &Signer{tpm: tpm, handle: handle, name: name, pub: pk}, nil +} + +// isProtectedObject reports whether a persistent object is a restricted +// decryption key — i.e. an Endorsement Key or Storage Root Key. We must NEVER +// evict one of those (it would destroy the TPM's endorsement/storage identity, +// including the EK the future EK/AK attestation path depends on). Fail-safe: if +// the public area can't be parsed, treat it as protected. +func isProtectedObject(pub2b tpm2.TPM2BPublic) bool { + pub, err := pub2b.Contents() + if err != nil { + return true + } + a := pub.ObjectAttributes + return a.Restricted && a.Decrypt +} + +// ensureKey returns a signer for the persistent key at handle, creating + +// persisting a fresh ECDSA key there if none exists. Returns created=true when +// a new key was generated. handle MUST be a non-reserved owner handle (the fleet +// default is 0x81000100) — NOT the EK (0x81010001/2) or SRK (0x81000001/2) slots. +// +// If the handle already holds an object that is NOT our ECC signing key (e.g. an +// RSA signing key left by an older tpm2-tools/openssl provisioning path — go-tpm +// reports "key at handle is not ECC"), it is treated as STALE and evicted, then a +// fresh ECC key is created. Safe on the enrol path: this only runs on an empty-NV +// host, so no issued cert references the old key. Only a valid ECC key is reused. +// A restricted-decrypt object (EK/SRK) is NEVER evicted — it errors instead, which +// only happens if handle was misconfigured to a reserved slot. +func EnsureKey(tpm transport.TPM, handle tpm2.TPMHandle) (signer *Signer, created bool, err error) { + if rp, e := (tpm2.ReadPublic{ObjectHandle: handle}).Execute(tpm); e == nil { + // Guard FIRST: never touch a restricted-decrypt object (EK/SRK). An ECC + // EK/SRK would otherwise parse as a usable ECC key below and be "reused". + if isProtectedObject(rp.OutPublic) { + return nil, false, fmt.Errorf("refusing to evict a restricted-decrypt object (EK/SRK?) at handle %#x — configure a non-reserved owner handle (e.g. 0x81000100)", uint32(handle)) + } + if s, e2 := signerFromPub(tpm, handle, rp.Name, rp.OutPublic); e2 == nil { + return s, false, nil // our ECC signing key is already there — reuse it + } + // Stale/wrong-type signing key occupying the handle — evict so we recreate. + if _, e2 := (tpm2.EvictControl{ + Auth: tpm2.TPMRHOwner, + ObjectHandle: &tpm2.NamedHandle{Handle: handle, Name: rp.Name}, + PersistentHandle: handle, + }).Execute(tpm); e2 != nil { + return nil, false, fmt.Errorf("evict stale (non-ECC) key at handle: %w", e2) + } + } + + cp, err := tpm2.CreatePrimary{ + PrimaryHandle: tpm2.TPMRHOwner, + InPublic: tpm2.New2B(eccSigningTemplate()), + }.Execute(tpm) + if err != nil { + return nil, false, fmt.Errorf("createprimary: %w", err) + } + // Persist at the well-known handle, then flush the transient object. + _, err = tpm2.EvictControl{ + Auth: tpm2.TPMRHOwner, + ObjectHandle: &tpm2.NamedHandle{Handle: cp.ObjectHandle, Name: cp.Name}, + PersistentHandle: handle, + }.Execute(tpm) + _, _ = (tpm2.FlushContext{FlushHandle: cp.ObjectHandle}).Execute(tpm) + if err != nil { + return nil, false, fmt.Errorf("evictcontrol: %w", err) + } + rp, err := (tpm2.ReadPublic{ObjectHandle: handle}).Execute(tpm) + if err != nil { + return nil, false, fmt.Errorf("readpublic after persist: %w", err) + } + s, err := signerFromPub(tpm, handle, rp.Name, rp.OutPublic) + return s, true, err +} + +// nvPresent reports whether a defined NV index exists. NOTE: "defined" does NOT +// mean "holds our fleet cert" — leftover/foreign NV (e.g. an earlier +// tpm-provision run, or another OS) also reports present. Prefer ReadNVCert + +// content validation for the enrol decision; this stays for cheap existence checks. +func NVPresent(tpm transport.TPM, index tpm2.TPMHandle) bool { + _, err := (tpm2.NVReadPublic{NVIndex: index}).Execute(tpm) + return err == nil +} + +// ReadNVCert reads the full contents of NV index (chunked under the NV buffer +// limit). Returns an error if the index is undefined or unreadable — callers +// treat that as "no cert" (enrol). The bytes are the caller's to parse/validate; +// this does not assume they are a certificate. +func ReadNVCert(tpm transport.TPM, index tpm2.TPMHandle) ([]byte, error) { + rp, err := (tpm2.NVReadPublic{NVIndex: index}).Execute(tpm) + if err != nil { + return nil, fmt.Errorf("nv readpublic: %w", err) + } + pub, err := rp.NVPublic.Contents() + if err != nil { + return nil, err + } + size := int(pub.DataSize) + out := make([]byte, 0, size) + const chunk = 512 // conservatively under TPM_MAX_NV_BUFFER_SIZE + for off := 0; off < size; off += chunk { + end := min(off+chunk, size) + r, err := (tpm2.NVRead{ + AuthHandle: tpm2.AuthHandle{Handle: index, Name: rp.NVName, Auth: tpm2.PasswordAuth(nil)}, + NVIndex: tpm2.NamedHandle{Handle: index, Name: rp.NVName}, + Size: uint16(end - off), + Offset: uint16(off), + }).Execute(tpm) + if err != nil { + return nil, fmt.Errorf("nv read @%d: %w", off, err) + } + out = append(out, r.Data.Buffer...) + } + return out, nil +} + +// nvWriteCert (re)defines the NV index sized to der and writes it under owner +// auth. Chunks the write to stay within the TPM's NV buffer limit. +func WriteNVCert(tpm transport.TPM, index tpm2.TPMHandle, der []byte) error { + // Undefine a stale index first (re-enrolment / resized cert). + if rp, err := (tpm2.NVReadPublic{NVIndex: index}).Execute(tpm); err == nil { + // A PLATFORM-created index at our handle is OEM firmware NV (bench-seen on + // a Dell 820: 0x01800001 was platformcreate|policydelete|writelocked). We + // cannot undefine it with owner auth, and TPM2_Clear/BIOS won't remove it — + // so fail LOUDLY with guidance rather than a confusing "nv undefine" error. + // The remedy is a different, free owner index via rd.dot1x.nvindex. + if pub, e := rp.NVPublic.Contents(); e == nil && pub.Attributes.PlatformCreate { + return fmt.Errorf("NV index %#x is a platform-created (firmware-owned) index — cannot be written by enrolment; pick a free owner index via rd.dot1x.nvindex / -nvindex", uint32(index)) + } + if _, err := (tpm2.NVUndefineSpace{ + AuthHandle: tpm2.TPMRHOwner, + NVIndex: tpm2.NamedHandle{Handle: index, Name: rp.NVName}, + }).Execute(tpm); err != nil { + return fmt.Errorf("nv undefine: %w", err) + } + } + + def := tpm2.NVDefineSpace{ + AuthHandle: tpm2.TPMRHOwner, + PublicInfo: tpm2.New2B(tpm2.TPMSNVPublic{ + NVIndex: index, + NameAlg: tpm2.TPMAlgSHA256, + Attributes: tpm2.TPMANV{ + OwnerWrite: true, + OwnerRead: true, + AuthWrite: true, + AuthRead: true, + NT: tpm2.TPMNTOrdinary, + NoDA: true, + }, + DataSize: uint16(len(der)), + }), + } + if _, err := def.Execute(tpm); err != nil { + return fmt.Errorf("nv define: %w", err) + } + rp, err := (tpm2.NVReadPublic{NVIndex: index}).Execute(tpm) + if err != nil { + return fmt.Errorf("nv readpublic: %w", err) + } + + const chunk = 512 // conservatively under TPM_MAX_NV_BUFFER_SIZE + for off := 0; off < len(der); off += chunk { + end := min(off+chunk, len(der)) + if _, err := (tpm2.NVWrite{ + AuthHandle: tpm2.AuthHandle{Handle: index, Name: rp.NVName, Auth: tpm2.PasswordAuth(nil)}, + NVIndex: tpm2.NamedHandle{Handle: index, Name: rp.NVName}, + Data: tpm2.TPM2BMaxNVBuffer{Buffer: der[off:end]}, + Offset: uint16(off), + }).Execute(tpm); err != nil { + return fmt.Errorf("nv write @%d: %w", off, err) + } + } + return nil +} diff --git a/files/etc/dot1x/wpa_supplicant-dot1x.conf b/files/etc/dot1x/wpa_supplicant-dot1x.conf index 7a39cda..8e0e192 100644 --- a/files/etc/dot1x/wpa_supplicant-dot1x.conf +++ b/files/etc/dot1x/wpa_supplicant-dot1x.conf @@ -14,8 +14,11 @@ network={ client_cert="/run/dot1x/client.pem" # BENCH-VERIFY: TPM key reference method on your wpa_supplicant build. + # Handle is the fleet signing-key handle (non-reserved owner handle; NOT the + # EK slot 0x81010001 — see CLAUDE.md invariant #1). Keep in sync with the + # enrolment -handle / rd.dot1x.handle default. # Option A (OpenSSL tpm2 provider, direct persistent handle): - private_key="handle:0x81010001" + private_key="handle:0x81000100" # Option B (tpm2-pkcs11 with a store regenerated into /run at boot): # private_key="pkcs11:token=fleet;object=dot1x-key;type=private;pin-value=..." diff --git a/files/plymouth/themes/fleet/fleet.script b/files/plymouth/themes/fleet/fleet.script index d197b46..c1c7731 100644 --- a/files/plymouth/themes/fleet/fleet.script +++ b/files/plymouth/themes/fleet/fleet.script @@ -66,11 +66,26 @@ foot = Sprite(foot_img); foot.SetX(sw / 2 - foot_img.GetWidth() / 2); foot.SetY(sh / 2 + 150); +# ---- first-boot TPM enrolment banner (only shown on an empty-NV host) ---- +# tpm-enrol.sh drives enrol-active/enrol-wait/enrol-done; the public fingerprint +# rides the message line (on_message) below. States only, no per-host secret. +enrol_b = Sprite(); enrol_b.SetOpacity(0); +fun enrol_banner(text, r, g, b) { + eimg = Image.Text(text, r, g, b, 1, "Sans Bold 16"); + enrol_b.SetImage(eimg); + enrol_b.SetX(sw / 2 - eimg.GetWidth() / 2); + enrol_b.SetY(sh / 2 - 70); + enrol_b.SetOpacity(1.0); +} + # ---- status tokens from the boot hooks ---- fun set(i, s) { state[i] = s; draw(i); } fun on_status(t) { if (t == "mode-test") { set(0, 3); set(1, 3); set(2, 3); } + if (t == "enrol-active") { set(0, 1); enrol_banner("FIRST-BOOT TPM ENROLMENT", 1.00, 0.72, 0.10); } + if (t == "enrol-wait") enrol_banner("AWAITING ADMIN APPROVAL — verify the code below", 1.00, 0.72, 0.10); + if (t == "enrol-done") { enrol_banner("IDENTITY ENROLLED — rebooting", 0.10, 1.00, 0.50); set(0, 2); } if (t == "tpm-active") set(0, 1); if (t == "tpm-done") set(0, 2); if (t == "dot1x-active") set(1, 1); diff --git a/files/stig/chrony-dhcp-dispatcher.te b/files/stig/chrony-dhcp-dispatcher.te new file mode 100644 index 0000000..33d0a11 --- /dev/null +++ b/files/stig/chrony-dhcp-dispatcher.te @@ -0,0 +1,25 @@ +# Raw module-policy syntax (compiled with `checkmodule -M -m`), NOT the refpolicy +# `policy_module()` m4 macro — that macro is only expanded by the refpolicy +# Makefile (/usr/share/selinux/devel) and checkmodule rejects it ("no module +# specification found"). `module ;` is what checkmodule wants. +module chrony_dhcp_dispatcher 1.0.0; + +######################################## +# Let chronyd reply to the chronyc that our NetworkManager dispatcher script +# runs. files/etc/NetworkManager/dispatcher.d/10-chrony-dhcp-ntp loads DHCP +# option-42 NTP servers with `chronyc reload sources`. That chronyc runs in the +# confined custom-dispatcher domain (NetworkManager_dispatcher_custom_t), so its +# reply socket is labeled with that domain, and the shipped policy has no rule +# for chronyd_t -> that socket. Result (enforcing): chronyd processes the reload +# but its ack `sendto` is DENIED — chronyc reports failure and the AVC floods the +# SIEM. This one allow closes it. Minimal + auditable; justified in +# docs/stig-deviations.md. If a follow-on AVC appears on the bench, extend here. +######################################## + +require { + type chronyd_t; + type NetworkManager_dispatcher_custom_t; + class unix_dgram_socket sendto; +} + +allow chronyd_t NetworkManager_dispatcher_custom_t:unix_dgram_socket sendto; diff --git a/scripts/extract-artifacts.sh b/scripts/extract-artifacts.sh index ec3ea59..c781f8b 100755 --- a/scripts/extract-artifacts.sh +++ b/scripts/extract-artifacts.sh @@ -23,6 +23,10 @@ DISABLE_DOT1X=${3:-false} # environments. Defaults to the literal placeholder so build-oci.sh's warning # fires if it was never set. NVMET_TRADDR=${NVMET_TRADDR:-NVMET_TRADDR} +# Enrolment broker URL baked into the cmdline (rd.fleet.enrol_url) for the inline +# empty-NV first-boot enrolment. Per-environment DNS name (the serving nginx +# reverse-proxies /enrol/ to the broker); only used by a host with no TPM cert. +ENROL_URL=${ENROL_URL:-http://boot.fleet.lab/enrol} # Plymouth boot splash toggle (from the boot_splash Packer var). "true" bakes # "rhgb quiet" so Plymouth shows the graphical splash; anything else bakes # "plymouth.enable=0" for a fully verbose console boot (boot-chain debugging). @@ -134,7 +138,10 @@ if ! getfattr -h -n security.selinux "$WORK/root/usr/bin/bash" >/dev/null 2>&1; echo " An enforcing boot would deny login. Check guestfish/tar xattr flags." >&2 exit 1 fi -echo "==> SELinux labels preserved (bash: $(getfattr -h --only-values -n security.selinux "$WORK/root/usr/bin/bash" 2>/dev/null))" +# tr -d '\0': SELinux xattr values are NUL-terminated; without stripping it bash +# warns "ignored null byte in input" on the command substitution (cosmetic only — +# the label check above already passed). +echo "==> SELinux labels preserved (bash: $(getfattr -h --only-values -n security.selinux "$WORK/root/usr/bin/bash" 2>/dev/null | tr -d '\0'))" echo "==> Building reproducible erofs image" # UNCOMPRESSED on purpose: the target Rocky 9 kernel's erofs is built without @@ -216,7 +223,8 @@ printf '%s' \ "$STIG_FRAGMENT" \ "$DOT1X_FRAGMENT" \ "rd.neednet=1 systemd.volatile=overlay " \ - "rd.dot1x.nvindex=0x01800001 rd.dot1x.timeout=60 " \ + "rd.dot1x.nvindex=0x01800100 rd.dot1x.handle=0x81000100 rd.dot1x.timeout=60 " \ + "rd.fleet.enrol_url=${ENROL_URL} " \ "rd.nvmf.discover=tcp,${NVMET_TRADDR},4420 " \ "root=/dev/mapper/root rootfstype=erofs rootflags=ro " \ "systemd.verity=yes roothash=${ROOT_HASH} " \ diff --git a/serving/.env.example b/serving/.env.example index 2ce0277..d5b256f 100644 --- a/serving/.env.example +++ b/serving/.env.example @@ -85,6 +85,39 @@ SUBNQN=nqn.2026-06.org.example.fleet:rootfs TARGET_PORT=4420 SPDK_CPUMASK=0x1 # one polling core is plenty for a read-only boot fs +# --- TPM enrolment broker (TEMPORARY in this stack — see docs/TODO.md #11) --- +# The broker is the one unauthenticated pre-auth endpoint (nginx proxies /enrol/ +# to it). Defaults run the zero-dependency LAB path: a `mock` signer (ephemeral +# CA; certs NOT trusted by RADIUS/NVMe) + plaintext loopback admin — enough to +# exercise the request/approve/poll flow while the host side is built. +# +# For REAL enrolment: set ENROL_SIGNER=step, point ENROL_STEP_CA_URL at a +# reachable step-ca (e.g. the one in serving/dot1x-bench/), drop its root + +# provisioner password into ./enrol-pki/, generate the admin mTLS PKI +# (./enrol-broker/admin-pki.sh ./enrol-pki/broker ), and set +# ENROL_ADMIN_INSECURE=false. Then it belongs in its own CA-co-located stack. +ENROL_SIGNER=mock # mock (lab, no CA) | step (real, needs a CA) +ENROL_ADMIN_INSECURE=true # true = plaintext loopback admin (LAB ONLY); false = mTLS +ENROL_PENDING_TTL=10m # time-box on unapproved requests +ENROL_WORDS=8 # fingerprint word count +ENROL_HOSTNQN_BASE=nqn.2026-06.org.example.fleet:host- +# step signer (only when ENROL_SIGNER=step) — paths are INSIDE the container +# (./enrol-pki is mounted at /pki): +ENROL_STEP_CA_URL= # e.g. https://ca.fleet.lab:8443 +ENROL_STEP_CA_ROOT=/pki/root_ca.crt +ENROL_STEP_PROVISIONER=enrol +ENROL_STEP_PASSWORD_FILE=/pki/password +# Issued-cert lifetime. NOTE: the step-ca provisioner must ALLOW this — a stock +# step-ca caps certs at 24h, so 720h is rejected ("requested duration ... more +# than the authorized maximum certificate duration of 24h"). The bench +# step-ca/bootstrap.sh now raises authority.claims.maxTLSCertDuration; a +# different CA needs the same. Verified end-to-end against a real step-ca. +ENROL_CERT_NOT_AFTER=720h +# admin mTLS (only when ENROL_ADMIN_INSECURE=false): +ENROL_ADMIN_CERT=/pki/broker/server.crt +ENROL_ADMIN_KEY=/pki/broker/server.key +ENROL_ADMIN_CLIENT_CA=/pki/broker/admin-ca.crt + # --- SPDK version loader / reaper (declarative rolling updates) ------------- # The spdk-loader service stages + adds ${IMAGE_VERSION} to the long-lived target # on `compose up`, then auto-removes OLD versions once they drain. Migration = diff --git a/serving/docker-compose.yml b/serving/docker-compose.yml index dd31d46..84c9ff8 100644 --- a/serving/docker-compose.yml +++ b/serving/docker-compose.yml @@ -189,6 +189,51 @@ services: # For HTTPS boot (production), terminate TLS here with the boot-infra CA's # cert and switch the dhcp-boot URL above to https://. Left HTTP for the lab. + # --- TPM enrolment broker (TEMPORARY home — see docs/TODO.md #11) ----------- + # The `http` nginx already reverse-proxies /enrol/ -> 127.0.0.1:9090 (this + # broker's public listener), so an empty-NV host can POST its CSR and poll for + # the signed cert via the normal serving host. For now the broker runs HERE so + # a single `compose up` in serving/ brings the enrolment endpoint up. + # + # ⚠ PROPER HOME is its own stack co-located with step-ca (the fleet CA), i.e. + # serving/dot1x-bench/, since REAL signing needs that CA. SPLIT THIS BACK OUT + # once the CA lives beside it. Do NOT run this AND dot1x-bench's enrol-broker on + # the same host — they collide on 9090/9443. + # + # Defaults below are the zero-dependency LAB path: `mock` signer (ephemeral + # in-process CA — certs are NOT trusted by RADIUS/NVMe) + plaintext loopback + # admin. For REAL enrolment, in .env set ENROL_SIGNER=step + point + # ENROL_STEP_CA_* at a reachable step-ca, and set ENROL_ADMIN_INSECURE=false + # after generating the admin mTLS PKI (a DEDICATED admin CA, not the fleet root): + # ./enrol-broker/admin-pki.sh ./enrol-pki/broker + # then import enrol-pki/broker/clients/.p12 and open https://127.0.0.1:9443/ + enrol-broker: + build: ./enrol-broker + network_mode: host # public on 127.0.0.1:9090 (nginx proxies /enrol/) + restart: unless-stopped + environment: + BROKER_PUBLIC_ADDR: "127.0.0.1:9090" # nginx /enrol/ proxies here + BROKER_ADMIN_ADDR: "127.0.0.1:9443" # mgmt/loopback ONLY — never the boot LAN + BROKER_SIGNER: ${ENROL_SIGNER:-mock} + BROKER_PENDING_TTL: ${ENROL_PENDING_TTL:-10m} + BROKER_WORDS: ${ENROL_WORDS:-8} + HOSTNQN_BASE: ${ENROL_HOSTNQN_BASE:-nqn.2026-06.org.example.fleet:host-} + # --- step signer (only consulted when ENROL_SIGNER=step) --- + STEP_CA_URL: ${ENROL_STEP_CA_URL:-} + STEP_CA_ROOT: ${ENROL_STEP_CA_ROOT:-/pki/root_ca.crt} + STEP_PROVISIONER: ${ENROL_STEP_PROVISIONER:-enrol} + STEP_PROVISIONER_PASSWORD_FILE: ${ENROL_STEP_PASSWORD_FILE:-/pki/password} + STEP_CERT_NOT_AFTER: ${ENROL_CERT_NOT_AFTER:-720h} + # --- admin mTLS (set ENROL_ADMIN_INSECURE=false for real mTLS) --- + BROKER_ADMIN_INSECURE: ${ENROL_ADMIN_INSECURE:-true} + BROKER_ADMIN_CERT: ${ENROL_ADMIN_CERT:-/pki/broker/server.crt} + BROKER_ADMIN_KEY: ${ENROL_ADMIN_KEY:-/pki/broker/server.key} + BROKER_ADMIN_CLIENT_CA: ${ENROL_ADMIN_CLIENT_CA:-/pki/broker/admin-ca.crt} + volumes: + # CA root + provisioner password (step signer) and admin mTLS certs live + # here; unused/empty for the default mock+insecure path. Gitignored. + - ./enrol-pki:/pki:ro + # --- Kiosk SPA (spaceship terminal shown by the diskless kiosk) ------------ # Svelte + Tailwind app built into a static bundle served by its own nginx. # Now an INDEPENDENT project — the `kiosk-app` git submodule (its own repo + @@ -261,23 +306,21 @@ services: # Multi-version release store: each ./spdk/releases// holds that # version's rootfs.erofs + rootfs.hash + verity.json. Staged by the # spdk-loader (or spdk-ctl.sh on the host); reconciled into the target at - # (re)start and added/removed live thereafter. RW so the bootstrap seed + - # loader can write here. Adding a version does NOT recreate this service. + # (re)start and added/removed live thereafter. RW so the spdk-loader can + # stage here. Adding a version does NOT recreate this service. - ./spdk/releases:/releases - # Bootstrap convenience (optional): a single OCI release mounted as an - # image volume. spdk-target seeds it into /releases as ${IMAGE_VERSION} if - # not already staged — preserves the simple one-version `compose up` flow. - # Remove this block for a pure multi-version (spdk-ctl-managed) deployment. - - type: image - source: ${OCI_IMAGE}:${IMAGE_VERSION} - target: /artifacts - image: - subpath: artifacts - # Fallback (Docker < 28 / offline): - ${ARTIFACT_SRC}:/artifacts:ro + # NOTE: the target is deliberately VERSION-AGNOSTIC. It mounts NO + # ${IMAGE_VERSION} artifact image and carries no IMAGE_VERSION env, so a + # version bump never changes THIS service's config — `docker compose up -d` + # after an IMAGE_VERSION bump recreates the loader/http/dnsmasq but leaves + # nvmf_tgt running (recreating it would tear down EVERY host's NVMe/TCP root + # at once). Bootstrap AND rolling updates are the spdk-loader's job: it + # stages the current release into ./spdk/releases and adds it online via + # JSON-RPC. (spdk-target still seeds from /artifacts IF one is manually + # mounted + IMAGE_VERSION set — a legacy/standalone convenience, off here.) # Legacy single-version entrypoint (configure-at-start, restart-to-change): # - ./spdk-setup.sh:/spdk-setup.sh:ro environment: - IMAGE_VERSION: ${IMAGE_VERSION} # bootstrap-seed label for /artifacts RELEASES_DIR: /releases RPC_SOCK: /var/run/spdk/spdk.sock # shared with spdk-loader (-r / -s) SUBNQN_BASE: ${SUBNQN_BASE} # per-version NQN base (falls back per release to verity.json .subnqn) diff --git a/serving/dot1x-bench/.env.example b/serving/dot1x-bench/.env.example index 737867c..04af68b 100644 --- a/serving/dot1x-bench/.env.example +++ b/serving/dot1x-bench/.env.example @@ -3,3 +3,11 @@ CA_DNS=ca.fleet.lab # DNS name(s) clients reach step-ca on (resolved by CA_PORT=8443 # step-ca listen port (host networking) CA_PASSWORD=changeme-bench # password protecting the CA keys (bench only) RADIUS_SECRET=testing123 # shared secret between the authenticator (hostapd) and FreeRADIUS + +# --- enrol-broker (TPM identity enrolment; docs/tpm-enrolment-design.md) ------- +# Before `docker compose up -d enrol-broker`, issue the admin mTLS PKI (a +# DEDICATED admin CA, separate from the fleet CA) and one client cert per admin: +# ../enrol-broker/admin-pki.sh ./pki/broker alice +# then import pki/broker/clients/alice.p12 (empty password) into the browser and +# open https://127.0.0.1:9443/ . The public /enrol/ endpoint is reached via the +# serving stack's nginx (http://boot.fleet.lab/enrol/). diff --git a/serving/dot1x-bench/README.md b/serving/dot1x-bench/README.md index 4346449..b28c5c5 100644 --- a/serving/dot1x-bench/README.md +++ b/serving/dot1x-bench/README.md @@ -83,8 +83,8 @@ qm set --tpmstate0 :1,version=v2.0 3. **Enrol each workstation's TPM** (one-shot, on a provisioning network that can reach step-ca) — see `enrol/tpm-provision.sh`: generates the key **in** the TPM - at `0x81010001`, builds a CSR with it, has step-ca sign it, writes the cert to - NV `0x01800001`. Each host gets a unique cert → unique hostnqn. + at `0x81000100`, builds a CSR with it, has step-ca sign it, writes the cert to + NV `0x01800100`. Each host gets a unique cert → unique hostnqn. 4. **Set up the authenticator** — `authenticator/setup-link.sh` (bridge + macsec plumbing per link) then run `hostapd authenticator/hostapd-wired-.conf`. diff --git a/serving/dot1x-bench/docker-compose.yml b/serving/dot1x-bench/docker-compose.yml index 6c2987f..75a8a59 100644 --- a/serving/dot1x-bench/docker-compose.yml +++ b/serving/dot1x-bench/docker-compose.yml @@ -25,7 +25,13 @@ services: DOCKER_STEPCA_INIT_DNS_NAMES: "${CA_DNS:-ca.fleet.lab},localhost" DOCKER_STEPCA_INIT_ADDRESS: ":${CA_PORT:-8443}" DOCKER_STEPCA_INIT_PROVISIONER_NAME: "enrol" - DOCKER_STEPCA_INIT_PASSWORD_FILE: "/home/step/secrets/password" + # Password to encrypt the CA keys + 'enrol' provisioner. Must be the VALUE + # (_PASSWORD), not _PASSWORD_FILE: the latter is an INPUT path the entrypoint + # reads FROM (for a mounted docker secret), which doesn't exist at first + # init → `step ca init` fails → no CA, no secrets/password (the "CA not + # initialised" error). With _PASSWORD the entrypoint WRITES it to + # /home/step/secrets/password, where bootstrap.sh + the enrol-broker read it. + DOCKER_STEPCA_INIT_PASSWORD: "${CA_PASSWORD:-changeme-bench}" volumes: - ./pki/stepca:/home/step # CA state (root/intermediate/keys/db) — gitignored - ./pki:/export # bootstrap.sh drops root_ca.crt here for the image/radius @@ -45,3 +51,36 @@ services: - ./freeradius/mods-available/eap:/etc/raddb/mods-available/eap:ro # certs issued by bootstrap.sh: the RADIUS server cert/key + the CA root. - ./pki/radius:/etc/raddb/certs/fleet:ro + + # --- TPM enrolment broker (the ONE unauthenticated, pre-auth endpoint) ------- + # docs/tpm-enrolment-design.md. Host networking so the SERVING stack's nginx + # reaches the public listener on 127.0.0.1:9090 (proxied as /enrol/) and step-ca + # is reachable for signing. The admin web UI (mTLS) listens on 127.0.0.1:9443 — + # bind it to the management network ONLY; never expose it to the boot LAN. + # + # First issue the admin mTLS PKI (dedicated admin CA, NOT the fleet root): + # ../enrol-broker/admin-pki.sh ./pki/broker + enrol-broker: + build: ../enrol-broker + network_mode: host + restart: unless-stopped + depends_on: + - step-ca + environment: + BROKER_PUBLIC_ADDR: "127.0.0.1:9090" + BROKER_ADMIN_ADDR: "127.0.0.1:9443" + BROKER_SIGNER: "step" + STEP_CA_URL: "https://${CA_DNS:-ca.fleet.lab}:${CA_PORT:-8443}" + STEP_CA_ROOT: "/pki/root_ca.crt" + STEP_PROVISIONER: "enrol" + STEP_PROVISIONER_PASSWORD_FILE: "/pki/stepca/secrets/password" + STEP_CERT_NOT_AFTER: "720h" + BROKER_PENDING_TTL: "10m" + BROKER_WORDS: "8" + HOSTNQN_BASE: "nqn.2026-06.org.example.fleet:host-" + # Admin mTLS — server cert/key + the DEDICATED admin client CA (see above). + BROKER_ADMIN_CERT: "/pki/broker/server.crt" + BROKER_ADMIN_KEY: "/pki/broker/server.key" + BROKER_ADMIN_CLIENT_CA: "/pki/broker/admin-ca.crt" + volumes: + - ./pki:/pki:ro diff --git a/serving/dot1x-bench/enrol/tpm-provision.sh b/serving/dot1x-bench/enrol/tpm-provision.sh index 25cfd91..a5a658b 100755 --- a/serving/dot1x-bench/enrol/tpm-provision.sh +++ b/serving/dot1x-bench/enrol/tpm-provision.sh @@ -3,10 +3,11 @@ # # One-shot enrolment for ONE workstation. Models the real flow (invariant #1: # per-host identity is created at enrolment, never baked into the image): -# 1. generate a signing key INSIDE the TPM, persist it at handle 0x81010001 +# 1. generate a signing key INSIDE the TPM, persist it at handle 0x81000100 +# (non-reserved owner handle; NOT the EK slot 0x81010001) # 2. build a CSR using that TPM key (private key never leaves the TPM) # 3. have step-ca sign it (token-gated 'enrol' provisioner) -# 4. write the issued cert (DER) to TPM NV index 0x01800001 +# 4. write the issued cert (DER) to TPM NV index 0x01800100 # At boot the dracut hook reads the cert from NV and wpa_supplicant signs EAP-TLS # with the persistent-handle key. hostnqn is later derived from the cert serial. # @@ -25,8 +26,8 @@ set -euo pipefail HOSTID=${1:?usage: tpm-provision.sh e.g. ws-vm-a} -HANDLE=${HANDLE:-0x81010001} -NVINDEX=${NVINDEX:-0x01800001} +HANDLE=${HANDLE:-0x81000100} +NVINDEX=${NVINDEX:-0x01800100} export TPM2TOOLS_TCTI="${TPM2TOOLS_TCTI:-device:/dev/tpmrm0}" CA_DNS=${CA_DNS:-ca.fleet.lab}; CA_PORT=${CA_PORT:-8443} CA_URL="https://${CA_DNS}:${CA_PORT}" diff --git a/serving/dot1x-bench/enrol/tpm-selftest.sh b/serving/dot1x-bench/enrol/tpm-selftest.sh index bd28059..9463a9b 100755 --- a/serving/dot1x-bench/enrol/tpm-selftest.sh +++ b/serving/dot1x-bench/enrol/tpm-selftest.sh @@ -4,7 +4,7 @@ # # Run as root on a booted workstation (the open-port test image is fine — it # ships the same tooling). Uses SCRATCH handle/NV and cleans them up, so it never -# touches the real 0x81010001 / 0x01800001. Pass --use-real-handles to exercise +# touches the real 0x81000100 / 0x01800100. Pass --use-real-handles to exercise # the actual locations instead (still cleaned up unless --keep). # # It validates, in order: @@ -24,7 +24,7 @@ for a in "$@"; do case "$a" in esac; done export TPM2TOOLS_TCTI="${TPM2TOOLS_TCTI:-device:/dev/tpmrm0}" -if [ "$USE_REAL" = 1 ]; then HANDLE=0x81010001; NV=0x01800001 +if [ "$USE_REAL" = 1 ]; then HANDLE=0x81000100; NV=0x01800100 else HANDLE=0x81008000; NV=0x01500000; fi WORK=$(mktemp -d); PASS=0; FAIL=0; SKIP=0 diff --git a/serving/dot1x-bench/step-ca/bootstrap.sh b/serving/dot1x-bench/step-ca/bootstrap.sh index cc81d44..0fc1a6f 100755 --- a/serving/dot1x-bench/step-ca/bootstrap.sh +++ b/serving/dot1x-bench/step-ca/bootstrap.sh @@ -1,43 +1,117 @@ #!/usr/bin/env bash # bootstrap.sh — one-time setup of the bench CA after `docker compose up -d step-ca`. # -# 1. wait for step-ca, export its root to pki/root_ca.crt (-> image radius-ca.pem) +# 1. export the CA root to pki/root_ca.crt (-> image radius-ca.pem) +# 1b. raise the cert-duration claim (so 720h fleet certs are allowed) # 2. ensure an enrolment provisioner exists (signs each host's TPM CSR) # 3. issue the RADIUS server cert/key into pki/radius/ (FreeRADIUS mounts it) # -# Runs the `step` CLI against the running CA. Flags vary a little across step -# versions — BENCH-VERIFY against your `step version` and adjust. Reads .env. +# `step` runs INSIDE the CA container (docker compose exec). Container paths: +# /home/step = CA state (certs/root_ca.crt, secrets/password, config/ca.json) +# /export = host ./pki (files written here appear on the host) +# Flags vary a little across step versions — BENCH-VERIFY against your `step +# version` and adjust. Reads .env. set -euo pipefail cd "$(dirname "$0")/.." [ -f .env ] && . ./.env CA_DNS=${CA_DNS:-ca.fleet.lab} CA_PORT=${CA_PORT:-8443} -CA_URL="https://${CA_DNS}:${CA_PORT}" +# The CLI must reach the CA over a name that RESOLVES on the serving host AND is +# in the CA cert's SAN. ca.fleet.lab is only served by the fleet dnsmasq to +# booting clients — the serving host itself can't resolve it — so default to +# 'localhost' (the compose adds it to DOCKER_STEPCA_INIT_DNS_NAMES). CA_DNS stays +# the cert/SAN name (and the RADIUS cert CN). Override CA_CONNECT if the CA is +# remote/reachable by another resolvable SAN. +CA_CONNECT=${CA_CONNECT:-localhost} +CA_URL="https://${CA_CONNECT}:${CA_PORT}" PKI=pki -mkdir -p "$PKI/radius" -# The CA's password (auto-init wrote it from this file in the container volume). -PW="$PKI/stepca/secrets/password" -[ -f "$PW" ] || { echo "CA not initialised yet ($PW missing) — is step-ca up?" >&2; exit 1; } +# step-ca initialises its state into ./pki/stepca as uid 1000, but Docker created +# the top-level ./pki as ROOT (it auto-creates a missing bind-mount source), so +# the host user can't write into it ("mkdir: Permission denied"). Self-heal — LAB +# ONLY: make the tree read/write for everyone, so BOTH your host user and the +# uid-1000 CA container can use it. (Pre-creating ./pki owned by you before +# `compose up` avoids this; once fixed it stays fixed.) +# The CA's password (auto-init wrote it here in the container volume). Also the +# 'enrol' provisioner password the broker needs (copy to ../enrol-pki/password). +# Location varies by smallstep image version: newer writes $STEPPATH/password, +# older $STEPPATH/secrets/password — accept whichever exists. +PW="$PKI/stepca/password" +[ -e "$PW" ] || { [ -e "$PKI/stepca/secrets/password" ] && PW="$PKI/stepca/secrets/password"; } +# same file as seen inside the container (/home/step is the ./pki/stepca mount). +CONTAINER_PW="/home/step/${PW#"$PKI/stepca/"}" -step() { docker compose exec -T step-ca step "$@"; } +# Two uid hazards, both self-healed here (LAB ONLY): +# - Docker auto-creates a missing ./pki as ROOT -> host mkdir denied. +# - step-ca writes its state as uid 1000 mode 600 -> host user can't read $PW. +# If we can't create the radius dir OR can't read the password, chmod the tree +# readable/writable for everyone (your user AND the uid-1000 CA container). +if ! mkdir -p "$PKI/radius" 2>/dev/null || { [ -e "$PKI/stepca" ] && [ ! -r "$PW" ]; }; then + echo "==> fixing $PKI perms with sudo (docker/uid-1000 owns it; lab)" + sudo chmod -R a+rwX "$PKI" + mkdir -p "$PKI/radius" +fi + +if [ ! -r "$PW" ]; then + echo "ERROR: CA password $PW missing/unreadable — step-ca did not initialise." >&2 + echo " Check: docker compose logs step-ca (look for an init failure)." >&2 + echo " Common cause: DOCKER_STEPCA_INIT_PASSWORD not set (see compose). To reset:" >&2 + echo " docker compose down && sudo rm -rf $PKI && docker compose up -d step-ca" >&2 + exit 1 +fi + +step() { docker compose exec -T step-ca step "$@"; } +incca() { docker compose exec -T step-ca "$@"; } # --- 1. export the root (becomes files/etc/dot1x/radius-ca.pem) -------------- echo "==> exporting CA root" -docker compose exec -T step-ca cat /home/step/certs/root_ca.crt > "$PKI/root_ca.crt" -FP=$(step certificate fingerprint "$PKI/root_ca.crt" 2>/dev/null || \ - openssl x509 -in "$PKI/root_ca.crt" -noout -fingerprint -sha256 | cut -d= -f2) +incca cp /home/step/certs/root_ca.crt /export/root_ca.crt +FP=$(openssl x509 -in "$PKI/root_ca.crt" -noout -fingerprint -sha256 | cut -d= -f2) echo " root fingerprint: $FP" echo " copy to: files/etc/dot1x/radius-ca.pem (image trust anchor)" +# --- 1b. raise the CA cert-duration claim (idempotent) ---------------------- +# A stock step-ca caps issued certs at 24h. Both the RADIUS server cert below +# AND the enrol-broker (--not-after 720h) need longer-lived fleet certs, so the +# CA rejects them with "requested duration ... more than the authorized maximum +# certificate duration of 24h". Patch authority.claims in the mounted ca.json +# and restart the CA. Set CA_MAX_CERT_DUR / CA_DEFAULT_CERT_DUR in .env to tune. +CA_JSON="$PKI/stepca/config/ca.json" +CA_MAX_CERT_DUR=${CA_MAX_CERT_DUR:-2160h} +CA_DEFAULT_CERT_DUR=${CA_DEFAULT_CERT_DUR:-720h} +if [ -f "$CA_JSON" ]; then + if python3 - "$CA_JSON" "$CA_MAX_CERT_DUR" "$CA_DEFAULT_CERT_DUR" <<'PY' +import json,sys +p,maxd,defd=sys.argv[1],sys.argv[2],sys.argv[3] +c=json.load(open(p)) +claims=c.setdefault("authority",{}).setdefault("claims",{}) +if claims.get("maxTLSCertDuration")==maxd and claims.get("defaultTLSCertDuration")==defd: + sys.exit(1) # already set — no restart needed +claims["maxTLSCertDuration"]=maxd +claims["defaultTLSCertDuration"]=defd +json.dump(c,open(p,"w"),indent=2) +PY + then + echo "==> raised CA claims (max=$CA_MAX_CERT_DUR default=$CA_DEFAULT_CERT_DUR); restarting step-ca" + docker compose restart step-ca >/dev/null + for _ in $(seq 1 50); do + curl -fsS -k "$CA_URL/health" >/dev/null 2>&1 && break; sleep 0.2 + done + else + echo "==> CA claims already set (max=$CA_MAX_CERT_DUR)" + fi +else + echo "WARNING: $CA_JSON not found — cannot raise cert-duration claim; 720h certs will be REJECTED." >&2 +fi + # --- 2. enrolment provisioner (idempotent) ---------------------------------- # JWK provisioner 'enrol' is created by auto-init (DOCKER_STEPCA_INIT_PROVISIONER_NAME). # Enrolment authorizes a CSR with a one-time token: # step ca token --provisioner enrol → passed to `step ca sign` -# That token-gated sign is what enrol/tpm-provision.sh uses. +# That token-gated sign is what the enrol-broker and enrol/tpm-provision.sh use. echo "==> provisioners:" -step ca provisioner list --ca-url "$CA_URL" --root "$PKI/root_ca.crt" \ +step ca provisioner list --ca-url "$CA_URL" --root /export/root_ca.crt \ | grep -E '"(name|type)"' || true # To UPGRADE to TPM attestation later, add an ACME attestation provisioner and @@ -49,13 +123,16 @@ step ca provisioner list --ca-url "$CA_URL" --root "$PKI/root_ca.crt" \ # (then `step ca acme` / an ACME client on the host instead of `step ca sign`.) # --- 3. RADIUS server cert (FreeRADIUS serves this; supplicant validates it) -- +# Written via /export so the files land in the host's pki/radius/ that FreeRADIUS +# mounts. Uses the in-container CA root + provisioner password (no host copies). echo "==> issuing RADIUS server cert (radius.${CA_DNS#ca.})" RADIUS_CN="radius.${CA_DNS#ca.}" +incca mkdir -p /export/radius step ca certificate "$RADIUS_CN" \ - "$PKI/radius/server.pem" "$PKI/radius/server.key" \ - --ca-url "$CA_URL" --root "$PKI/root_ca.crt" \ - --provisioner enrol --provisioner-password-file "$PW" \ + /export/radius/server.pem /export/radius/server.key \ + --ca-url "$CA_URL" --root /export/root_ca.crt \ + --provisioner enrol --provisioner-password-file "$CONTAINER_PW" \ --not-after 720h --force -cp "$PKI/root_ca.crt" "$PKI/radius/ca.pem" +incca cp /export/root_ca.crt /export/radius/ca.pem echo "==> done. FreeRADIUS will read pki/radius/{server.pem,server.key,ca.pem}." echo " Next: cp $PKI/root_ca.crt ../../files/etc/dot1x/radius-ca.pem && rebuild secure-path image." diff --git a/serving/enrol-broker/.gitignore b/serving/enrol-broker/.gitignore new file mode 100644 index 0000000..511f01d --- /dev/null +++ b/serving/enrol-broker/.gitignore @@ -0,0 +1,2 @@ +/fleet-enrol-broker +*.test diff --git a/serving/enrol-broker/Dockerfile b/serving/enrol-broker/Dockerfile new file mode 100644 index 0000000..3fa19dd --- /dev/null +++ b/serving/enrol-broker/Dockerfile @@ -0,0 +1,22 @@ +# fleet-enrol-broker — TPM identity enrolment broker (docs/tpm-enrolment-design.md). +# +# Two stages: a reproducible static Go build, then the smallstep `step` CLI image +# as the runtime base (the production signer shells out to `step ca token` + +# `step ca sign`). The broker is stdlib-only, so the build needs no module +# download (GOPROXY=off) and the verity-style reproducibility flags apply. +# +# Pin + mirror these base tags through your registry (like the rest of the stack) +# if the build/serving host has no egress. +FROM golang:1.23-alpine AS build +WORKDIR /src +COPY go.mod . +COPY *.go wordlist.txt ./ +ENV CGO_ENABLED=0 GOPROXY=off GOTOOLCHAIN=local +RUN go build -trimpath -buildvcs=false -ldflags '-s -w -buildid=' -o /enrol-broker . + +FROM smallstep/step-cli:latest +COPY --from=build /enrol-broker /usr/local/bin/enrol-broker +# Non-root by default (the step-cli image runs as uid 1000 "step"); the broker +# binds loopback ports >1024 and only READS the mounted CA root + provisioner +# password, so no extra privilege is needed. +ENTRYPOINT ["/usr/local/bin/enrol-broker"] diff --git a/serving/enrol-broker/README.md b/serving/enrol-broker/README.md new file mode 100644 index 0000000..48afe58 --- /dev/null +++ b/serving/enrol-broker/README.md @@ -0,0 +1,110 @@ +# fleet-enrol-broker + +The TPM identity **enrolment broker** — the one component that turns "step-ca can +sign" into "step-ca signs **only what a human approved**." It is the system's +single **unauthenticated, pre-auth bootstrap endpoint**: an empty-NV host submits +a CSR, an admin verifies a fingerprint out-of-band and approves, and the broker +returns a step-ca-signed cert the host writes to its TPM NV. + +Authoritative design: [`../../docs/tpm-enrolment-design.md`](../../docs/tpm-enrolment-design.md). +This is **v1** (fingerprint-only OOB assurance; inline empty-NV vehicle on the host +side). TPM attestation and the host-side dracut branch are the next phases. + +## Two listeners, by design + +``` +PUBLIC (plain HTTP, behind the serving nginx /enrol/) UNAUTHENTICATED + POST /request {csr, context} -> {enrol_id, fingerprint, ...} + GET /status/{id} -> pending | approved+cert | denied | expired + GET /healthz + +ADMIN (mTLS, management network / loopback ONLY) admin client cert + GET / the pending-queue web UI (fingerprint + approve/deny) + GET /pending JSON queue + POST /approve/{id} OOB-approve -> step ca sign -> store cert + POST /deny/{id} reject +``` + +The public side is deliberately tiny and stdlib-only — it is the only thing +reachable by un-enrolled devices. The admin side never touches the boot LAN. + +## How identity is gated + +1. Host generates a key **in** the TPM, builds a CSR, `POST /request`. +2. Broker stores a time-boxed PENDING record and returns a **fingerprint** = + `wordlist( SHA-256( CSR-pubkey ‖ server-nonce ) )`. The nonce is server-chosen + per enrol-id, so an attacker can't pre-grind a colliding fingerprint. +3. Host shows the fingerprint on its Plymouth splash. An admin opens the mTLS web + UI, **cross-checks the fingerprint against the device's screen**, confirms the + device context (MAC / DMI / TPM EK), and approves. +4. Broker mints a one-time `step` token (it — not the host — holds the provisioner + password) and `step ca sign`s the CSR; the cert is returned on the host's next + `GET /status/{id}` poll. The hostnqn is derived from the cert serial. + +Lifecycle (from the design): empty-NV → **enrol** (human); near-expiry → **renew** +(automatic, not built here); auth-fail with a cert present → **alert, never +auto-re-enrol**. + +## Run it (bench) + +The broker ships as a service in [`../dot1x-bench/docker-compose.yml`](../dot1x-bench/docker-compose.yml) +alongside step-ca. From `serving/dot1x-bench/`: + +```bash +docker compose up -d step-ca && ./step-ca/bootstrap.sh # CA + root_ca.crt + enrol provisioner +../enrol-broker/admin-pki.sh ./pki/broker alice # admin mTLS PKI (dedicated admin CA) +docker compose up -d enrol-broker +``` + +Then import `pki/broker/clients/alice.p12` (empty password) into your browser and +open `https://127.0.0.1:9443/`. Hosts reach the public side via the serving +stack's nginx: `http://boot.fleet.lab/enrol/`. + +### Admin mTLS PKI — important + +`admin-pki.sh` mints a **dedicated admin CA**, separate from the fleet CA. Do +**not** point `BROKER_ADMIN_CLIENT_CA` at the fleet root: host *enrolment* certs +are signed by that same root, so reusing it would let any enrolled host +authenticate to the admin plane and approve enrolments (privilege escalation). + +## Configuration (env) + +| Var | Default | Notes | +|---|---|---| +| `BROKER_PUBLIC_ADDR` | `127.0.0.1:9090` | unauth listener (behind nginx) | +| `BROKER_ADMIN_ADDR` | `127.0.0.1:9443` | mTLS admin listener (mgmt net only) | +| `BROKER_SIGNER` | `step` | `step` (prod) or `mock` (dev, ephemeral CA) | +| `STEP_CA_URL` / `STEP_CA_ROOT` | — | step-ca endpoint + root | +| `STEP_PROVISIONER` / `STEP_PROVISIONER_PASSWORD_FILE` | `enrol` / … | the broker holds this, not hosts | +| `STEP_CERT_NOT_AFTER` | `720h` | issued-cert validity | +| `BROKER_PENDING_TTL` | `10m` | time-box on unapproved requests | +| `BROKER_WORDS` | `8` | fingerprint word count (64-word list ⇒ 6 bits/word ⇒ 48 bits) | +| `BROKER_MAX_PENDING` | `256` | flooding cap | +| `BROKER_RATE_PER_MIN` | `12` | per-IP `/request` rate limit | +| `HOSTNQN_BASE` | `nqn.2026-06.org.example.fleet:host-` | hostnqn = base + cert-serial | +| `BROKER_ADMIN_CERT` / `_KEY` / `_CLIENT_CA` | — | admin mTLS (required unless insecure) | +| `BROKER_ADMIN_INSECURE` | `false` | **dev only**: plaintext admin, no client-cert auth | + +Swap `wordlist.txt` for the 2048-word BIP39 list (11 bits/word) for more entropy +per word; it just needs to be a power-of-two number of lines. + +## Dev loop without step-ca + +```bash +BROKER_SIGNER=mock BROKER_ADMIN_INSECURE=true \ +BROKER_PUBLIC_ADDR=127.0.0.1:9090 BROKER_ADMIN_ADDR=127.0.0.1:9443 go run . +``` + +`mock` signs from an ephemeral in-process CA (LAB ONLY — those certs are not +trusted by RADIUS/NVMe). For an mTLS dev run instead of `INSECURE`, generate certs +with `admin-pki.sh` and set `BROKER_ADMIN_CERT/_KEY/_CLIENT_CA`. + +## Not built yet (next phases) + +- **Host side:** the dracut empty-NV enrolment branch (generate key/CSR, POST, + show fingerprint on Plymouth, poll, write NV, reboot) + `tpm2-openssl` in the + image. `../dot1x-bench/enrol/tpm-provision.sh` is the manual stand-in. +- **Renew** path (automatic, near-expiry) and **TPM attestation** (ACME + `device-attest-01`) at the approve step. +- **Fold into the fleet-manager:** the admin routes become its authenticated + plane; approval writes the shared inventory record. diff --git a/serving/enrol-broker/admin-pki.sh b/serving/enrol-broker/admin-pki.sh new file mode 100644 index 0000000..45b0f94 --- /dev/null +++ b/serving/enrol-broker/admin-pki.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# admin-pki.sh — issue the broker's admin-plane mTLS PKI. +# +# SECURITY: the admin client CA MUST be SEPARATE from the fleet CA. If you reused +# the fleet root as the client CA, any host's *enrolment* cert (signed by that +# same root) would also authenticate to the admin plane — a privilege escalation +# letting an enrolled host approve enrolments. So this mints a DEDICATED admin CA, +# the broker's TLS server cert, and one client cert (+ .p12 for browser import) +# per named admin. +# +# Usage: admin-pki.sh [admin-name ...] +# e.g. ./admin-pki.sh ../dot1x-bench/pki/broker alice bob +# +# Then point the broker at: +# BROKER_ADMIN_CERT=/server.crt +# BROKER_ADMIN_KEY=/server.key +# BROKER_ADMIN_CLIENT_CA=/admin-ca.crt +# and import clients/.p12 (empty password) into each admin's browser. +set -euo pipefail + +OUT=${1:?usage: admin-pki.sh [admin-name ...]} +shift || true +ADMINS=("$@") +[ ${#ADMINS[@]} -eq 0 ] && ADMINS=(admin) +SAN=${BROKER_ADMIN_SAN:-"IP:127.0.0.1,DNS:localhost"} +DAYS=${BROKER_ADMIN_DAYS:-825} + +mkdir -p "$OUT/clients" +cd "$OUT" + +# --- dedicated admin CA (separate from the fleet CA) ------------------------ +if [ ! -f admin-ca.crt ]; then + echo "==> creating admin CA" + openssl ecparam -name prime256v1 -genkey -noout -out admin-ca.key + openssl req -x509 -new -key admin-ca.key -days "$DAYS" \ + -subj "/CN=fleet enrol-broker admin CA" -out admin-ca.crt +fi + +# --- broker TLS server cert ------------------------------------------------- +echo "==> issuing broker server cert (SAN $SAN)" +openssl ecparam -name prime256v1 -genkey -noout -out server.key +openssl req -new -key server.key -subj "/CN=enrol-broker-admin" -out server.csr +openssl x509 -req -in server.csr -CA admin-ca.crt -CAkey admin-ca.key -CAcreateserial \ + -days "$DAYS" -extfile <(printf "subjectAltName=%s\nextendedKeyUsage=serverAuth" "$SAN") \ + -out server.crt +rm -f server.csr + +# --- one client cert (+ .p12) per admin ------------------------------------ +for a in "${ADMINS[@]}"; do + echo "==> issuing admin client cert: $a" + openssl ecparam -name prime256v1 -genkey -noout -out "clients/$a.key" + openssl req -new -key "clients/$a.key" -subj "/CN=$a" -out "clients/$a.csr" + openssl x509 -req -in "clients/$a.csr" -CA admin-ca.crt -CAkey admin-ca.key -CAcreateserial \ + -days "$DAYS" -extfile <(printf "extendedKeyUsage=clientAuth") -out "clients/$a.crt" + rm -f "clients/$a.csr" + openssl pkcs12 -export -inkey "clients/$a.key" -in "clients/$a.crt" -certfile admin-ca.crt \ + -name "fleet enrol admin: $a" -passout pass: -out "clients/$a.p12" +done + +chmod 600 ./*.key clients/*.key 2>/dev/null || true + +echo +echo "done. broker env:" +echo " BROKER_ADMIN_CERT=$(pwd)/server.crt" +echo " BROKER_ADMIN_KEY=$(pwd)/server.key" +echo " BROKER_ADMIN_CLIENT_CA=$(pwd)/admin-ca.crt" +echo "import clients/.p12 (empty password) into each admin's browser." diff --git a/serving/enrol-broker/admin.go b/serving/enrol-broker/admin.go new file mode 100644 index 0000000..5f4b73e --- /dev/null +++ b/serving/enrol-broker/admin.go @@ -0,0 +1,201 @@ +// admin.go — the mTLS admin plane: the pending-queue web UI + approve/deny. +// +// This listener is the AUTHENTICATED half of the broker and must live on the +// management network / loopback only — NEVER the boot LAN (docs invariant: the +// management plane is recon-valuable). Admins authenticate with a client cert +// issued by the fleet CA; the cert CN is recorded as who approved each enrolment. +// The fingerprint shown here is what the admin cross-checks against the device's +// Plymouth splash before approving (the human OOB gate). +package main + +import ( + "crypto/tls" + "crypto/x509" + "html/template" + "log" + "net/http" + "os" + "time" +) + +func (b *broker) serveAdmin() { + mux := http.NewServeMux() + mux.HandleFunc("GET /", b.handleAdminIndex) + mux.HandleFunc("GET /pending", b.handleAdminPending) + mux.HandleFunc("POST /approve/{id}", b.handleApprove) + mux.HandleFunc("POST /deny/{id}", b.handleDeny) + + srv := &http.Server{ + Addr: b.cfg.adminAddr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 40 * time.Second, // approve blocks on step-ca + IdleTimeout: 60 * time.Second, + } + + if b.cfg.adminInsecure { + log.Printf("WARNING: BROKER_ADMIN_INSECURE=true — admin UI is PLAINTEXT with NO client-cert auth. DEV ONLY. Bind to loopback and never expose it.") + log.Fatalf("admin listener: %v", srv.ListenAndServe()) + return + } + + if b.cfg.adminCert == "" || b.cfg.adminKey == "" || b.cfg.adminClientCA == "" { + log.Fatalf("admin mTLS requires BROKER_ADMIN_CERT, BROKER_ADMIN_KEY and BROKER_ADMIN_CLIENT_CA (or set BROKER_ADMIN_INSECURE=true for dev)") + } + cert, err := tls.LoadX509KeyPair(b.cfg.adminCert, b.cfg.adminKey) + if err != nil { + log.Fatalf("admin server cert: %v", err) + } + caPEM, err := os.ReadFile(b.cfg.adminClientCA) + if err != nil { + log.Fatalf("admin client CA: %v", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + log.Fatalf("admin client CA %s contains no certificates", b.cfg.adminClientCA) + } + srv.TLSConfig = &tls.Config{ + Certificates: []tls.Certificate{cert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: pool, + MinVersion: tls.VersionTLS12, + } + log.Fatalf("admin listener: %v", srv.ListenAndServeTLS("", "")) +} + +func adminCN(r *http.Request) string { + if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 { + if cn := r.TLS.PeerCertificates[0].Subject.CommonName; cn != "" { + return cn + } + return "unknown-admin" + } + return "dev-insecure" +} + +func (b *broker) handleAdminPending(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, b.store.Pending()) +} + +func (b *broker) handleApprove(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + admin := adminCN(r) + csrPEM, ok := b.store.csrFor(id) + if !ok { + http.Error(w, "enrol-id not pending (expired/decided/unknown)", http.StatusConflict) + return + } + csr, err := parseCSR(csrPEM) + if err != nil { + http.Error(w, "stored CSR unparsable: "+err.Error(), http.StatusInternalServerError) + return + } + ctx, cancel := ctxWithTimeout(35 * time.Second) + defer cancel() + certPEM, serial, err := b.signer.Sign(ctx, csrPEM, csr.Subject.CommonName) + if err != nil { + log.Printf("audit approve-FAILED id=%s admin=%q err=%v", id, admin, err) + http.Error(w, "signing failed: "+err.Error(), http.StatusBadGateway) + return + } + e, err := b.store.Approve(id, certPEM, serial, admin) + if err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + log.Printf("audit APPROVE id=%s cn=%q serial=%s hostnqn=%s admin=%q", e.ID, e.CN, e.CertSerial, e.Hostnqn, admin) + redirectOrJSON(w, r) +} + +func (b *broker) handleDeny(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + admin := adminCN(r) + e, err := b.store.Deny(id, admin) + if err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + log.Printf("audit DENY id=%s cn=%q admin=%q", e.ID, e.CN, admin) + redirectOrJSON(w, r) +} + +// redirectOrJSON sends a browser form back to the queue, or a 204 for API callers. +func redirectOrJSON(w http.ResponseWriter, r *http.Request) { + if isBrowser(r) { + http.Redirect(w, r, "/", http.StatusSeeOther) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func isBrowser(r *http.Request) bool { + a := r.Header.Get("Accept") + return len(a) >= 9 && a[:9] == "text/html" || // explicit + r.Header.Get("Sec-Fetch-Mode") == "navigate" +} + +func (b *broker) handleAdminIndex(w http.ResponseWriter, r *http.Request) { + data := struct { + Admin string + Pending []*Enrolment + Now time.Time + }{ + Admin: adminCN(r), + Pending: b.store.Pending(), + Now: time.Now(), + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := adminTmpl.Execute(w, data); err != nil { + log.Printf("admin template: %v", err) + } +} + +var adminTmpl = template.Must(template.New("admin").Funcs(template.FuncMap{ + "age": func(t time.Time) string { return time.Since(t).Round(time.Second).String() }, +}).Parse(adminHTML)) + +const adminHTML = ` + + +fleet enrolment — pending + +

fleet enrolment — pending approvals

+

admin: {{.Admin}} · {{len .Pending}} pending · cross-check each fingerprint against the device's splash before approving

+{{if not .Pending}}
no pending enrolments.
{{end}} +{{range .Pending}} +
+
{{.Fingerprint}}
+
+ enrol-id{{.ID}} + CN{{.CN}} + {{if .Context.MAC}}MAC{{.Context.MAC}}{{end}} + {{if .Context.DMISerial}}DMI serial{{.Context.DMISerial}}{{end}} + {{if .Context.DMIUUID}}DMI uuid{{.Context.DMIUUID}}{{end}} + {{if .Context.EKHash}}TPM EK{{.Context.EKHash}}{{end}} + pubkey{{.PubKeyHash}} + age{{age .CreatedAt}} +
+
+
+
+
+
+{{end}} +` diff --git a/serving/enrol-broker/go.mod b/serving/enrol-broker/go.mod new file mode 100644 index 0000000..88af37f --- /dev/null +++ b/serving/enrol-broker/go.mod @@ -0,0 +1,8 @@ +// fleet-enrol-broker — TPM identity enrolment broker. +// See docs/tpm-enrolment-design.md. Stdlib only (no external modules) so the +// container build is trivially reproducible and the system's one UNAUTHENTICATED +// edge carries a minimal dependency surface. step-ca signing is done by shelling +// out to the bundled `step` CLI, not a Go dependency. +module fleet-enrol-broker + +go 1.22 diff --git a/serving/enrol-broker/main.go b/serving/enrol-broker/main.go new file mode 100644 index 0000000..0d7f215 --- /dev/null +++ b/serving/enrol-broker/main.go @@ -0,0 +1,369 @@ +// fleet-enrol-broker — TPM identity enrolment broker. +// +// The ONE unauthenticated, pre-auth bootstrap endpoint of the fleet. An empty-NV +// host generates a key IN its TPM, builds a CSR, POSTs it here, and shows the +// returned fingerprint on its Plymouth splash. An admin cross-checks that +// fingerprint against the pending queue (a separate mTLS web UI) and approves; +// the broker has step-ca sign the CSR and hands the cert back on the host's next +// status poll. The host writes it to TPM NV and authenticates. +// +// See docs/tpm-enrolment-design.md. Two listeners, by design: +// +// PUBLIC (plain HTTP, behind the serving nginx /enrol/) -- unauthenticated: +// POST /request submit a CSR -> {enrol_id, fingerprint} +// GET /status/{id} poll -> pending | approved+cert | denied | expired +// +// ADMIN (mTLS, management network / loopback only) -- admin client cert: +// GET / the pending-queue web UI +// GET /pending JSON queue +// POST /approve/{id} OOB-approve -> sign -> store cert +// POST /deny/{id} reject +// +// Stdlib only; signing shells out to `step`. Nothing here is per-host baked state. +package main + +import ( + "context" + _ "embed" + "encoding/json" + "log" + "net" + "net/http" + "os" + "strconv" + "strings" + "sync" + "time" +) + +//go:embed wordlist.txt +var wordlistRaw string + +type config struct { + publicAddr string + adminAddr string + ttl time.Duration + retention time.Duration + maxPending int + fpWords int + ratePerMin int + trustXFF bool + hostBase string + + signerKind string + // step signer + stepBin string + caURL string + caRoot string + provisioner string + passwordFile string + notAfter string + // admin mTLS + adminCert string + adminKey string + adminClientCA string + adminInsecure bool +} + +func loadConfig() config { + return config{ + publicAddr: env("BROKER_PUBLIC_ADDR", "127.0.0.1:9090"), + adminAddr: env("BROKER_ADMIN_ADDR", "127.0.0.1:9443"), + ttl: envDur("BROKER_PENDING_TTL", 10*time.Minute), + retention: envDur("BROKER_RETENTION", time.Hour), + maxPending: envInt("BROKER_MAX_PENDING", 256), + fpWords: envInt("BROKER_WORDS", 8), + ratePerMin: envInt("BROKER_RATE_PER_MIN", 12), + trustXFF: envBool("BROKER_TRUST_XFF", true), + hostBase: env("HOSTNQN_BASE", "nqn.2026-06.org.example.fleet:host-"), + signerKind: env("BROKER_SIGNER", "step"), + stepBin: env("STEP_BIN", "step"), + caURL: env("STEP_CA_URL", "https://ca.fleet.lab:8443"), + caRoot: env("STEP_CA_ROOT", "/pki/root_ca.crt"), + provisioner: env("STEP_PROVISIONER", "enrol"), + passwordFile: env("STEP_PROVISIONER_PASSWORD_FILE", "/pki/stepca/secrets/password"), + notAfter: env("STEP_CERT_NOT_AFTER", "720h"), + adminCert: env("BROKER_ADMIN_CERT", ""), + adminKey: env("BROKER_ADMIN_KEY", ""), + adminClientCA: env("BROKER_ADMIN_CLIENT_CA", ""), + adminInsecure: envBool("BROKER_ADMIN_INSECURE", false), + } +} + +type broker struct { + cfg config + store *Store + signer Signer + rl *rateLimiter +} + +func main() { + log.SetFlags(log.LstdFlags | log.LUTC) + cfg := loadConfig() + + words := parseWordlist(wordlistRaw) + store, err := NewStore(cfg.ttl, cfg.retention, cfg.maxPending, words, cfg.fpWords, cfg.hostBase) + if err != nil { + log.Fatalf("store: %v", err) + } + + var signer Signer + switch cfg.signerKind { + case "step": + signer = &stepSigner{ + bin: cfg.stepBin, caURL: cfg.caURL, root: cfg.caRoot, + provisioner: cfg.provisioner, passwordFile: cfg.passwordFile, notAfter: cfg.notAfter, + } + case "mock": + signer, err = newMockSigner() + if err != nil { + log.Fatalf("mock signer: %v", err) + } + default: + log.Fatalf("unknown BROKER_SIGNER %q (want step|mock)", cfg.signerKind) + } + + b := &broker{cfg: cfg, store: store, signer: signer, rl: newRateLimiter(cfg.ratePerMin)} + + // background reaper: expire timed-out requests + evict old decided ones. + go func() { + t := time.NewTicker(30 * time.Second) + defer t.Stop() + for range t.C { + if n := store.Reap(); n > 0 { + log.Printf("audit reap expired_pending=%d", n) + } + b.rl.sweep() + } + }() + + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); b.servePublic() }() + go func() { defer wg.Done(); b.serveAdmin() }() + log.Printf("enrol-broker up: public=%s admin=%s signer=%s words=%d ttl=%s", + cfg.publicAddr, cfg.adminAddr, signer.Kind(), len(words), cfg.ttl) + wg.Wait() +} + +// --- public listener (unauthenticated) ------------------------------------- + +func (b *broker) servePublic() { + mux := http.NewServeMux() + mux.HandleFunc("POST /request", b.handleRequest) + mux.HandleFunc("GET /status/{id}", b.handleStatus) + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + srv := &http.Server{ + Addr: b.cfg.publicAddr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 30 * time.Second, + MaxHeaderBytes: 8 << 10, + } + log.Fatalf("public listener: %v", srv.ListenAndServe()) +} + +type enrolRequest struct { + CSR string `json:"csr"` + Context DeviceContext `json:"context"` +} + +type enrolRequestResp struct { + EnrolID string `json:"enrol_id"` + Fingerprint string `json:"fingerprint"` + State string `json:"state"` + PollAfter int `json:"poll_after_seconds"` + ExpiresIn int `json:"expires_in_seconds"` +} + +type statusResp struct { + State string `json:"state"` + Cert string `json:"cert,omitempty"` + Hostnqn string `json:"hostnqn,omitempty"` + CertSerial string `json:"cert_serial,omitempty"` +} + +func (b *broker) handleRequest(w http.ResponseWriter, r *http.Request) { + ip := b.clientIP(r) + if !b.rl.allow(ip) { + writeJSONErr(w, http.StatusTooManyRequests, "rate limit exceeded") + return + } + r.Body = http.MaxBytesReader(w, r.Body, 16<<10) // a CSR is ~1KB + var req enrolRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSONErr(w, http.StatusBadRequest, "invalid JSON body") + return + } + csr, err := parseCSR(req.CSR) + if err != nil { + writeJSONErr(w, http.StatusBadRequest, "invalid CSR: "+err.Error()) + return + } + e, err := b.store.Add(csr.RawSubjectPublicKeyInfo, csr.Subject.CommonName, req.CSR, req.Context) + if err != nil { + if err == errTooManyPending { + writeJSONErr(w, http.StatusServiceUnavailable, "broker busy: too many pending enrolments") + return + } + writeJSONErr(w, http.StatusInternalServerError, "internal error") + return + } + log.Printf("audit request id=%s cn=%q pubkey=%s mac=%q fp=%q from=%s", + e.ID, e.CN, short(e.PubKeyHash), e.Context.MAC, e.Fingerprint, ip) + writeJSON(w, http.StatusAccepted, enrolRequestResp{ + EnrolID: e.ID, + Fingerprint: e.Fingerprint, + State: string(e.State), + PollAfter: 5, + ExpiresIn: int(time.Until(e.ExpiresAt).Seconds()), + }) +} + +func (b *broker) handleStatus(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + e := b.store.Get(id) + if e == nil { + writeJSONErr(w, http.StatusNotFound, "unknown enrol-id") + return + } + resp := statusResp{State: string(e.State)} + if e.State == StateApproved { + resp.Cert = e.certPEM + resp.Hostnqn = e.Hostnqn + resp.CertSerial = e.CertSerial + } + writeJSON(w, http.StatusOK, resp) +} + +func (b *broker) clientIP(r *http.Request) string { + if b.cfg.trustXFF { + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + return strings.TrimSpace(strings.Split(xff, ",")[0]) + } + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} + +// --- rate limiter (per-IP fixed window) ------------------------------------ + +type rateLimiter struct { + mu sync.Mutex + perMin int + windows map[string]*window +} + +type window struct { + start time.Time + count int +} + +func newRateLimiter(perMin int) *rateLimiter { + return &rateLimiter{perMin: perMin, windows: make(map[string]*window)} +} + +func (rl *rateLimiter) allow(ip string) bool { + if rl.perMin <= 0 { + return true + } + rl.mu.Lock() + defer rl.mu.Unlock() + now := time.Now() + w := rl.windows[ip] + if w == nil || now.Sub(w.start) >= time.Minute { + rl.windows[ip] = &window{start: now, count: 1} + return true + } + if w.count >= rl.perMin { + return false + } + w.count++ + return true +} + +func (rl *rateLimiter) sweep() { + rl.mu.Lock() + defer rl.mu.Unlock() + for ip, w := range rl.windows { + if time.Since(w.start) >= time.Minute { + delete(rl.windows, ip) + } + } +} + +// --- small helpers --------------------------------------------------------- + +func parseWordlist(s string) []string { + var out []string + for _, line := range strings.Split(s, "\n") { + if w := strings.TrimSpace(line); w != "" { + out = append(out, w) + } + } + return out +} + +func writeJSON(w http.ResponseWriter, code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(v) +} + +func writeJSONErr(w http.ResponseWriter, code int, msg string) { + writeJSON(w, code, map[string]string{"error": msg}) +} + +func short(s string) string { + if len(s) > 16 { + return s[:16] + } + return s +} + +func env(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +func envInt(k string, def int) int { + if v := os.Getenv(k); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return def +} + +func envBool(k string, def bool) bool { + if v := os.Getenv(k); v != "" { + b, err := strconv.ParseBool(v) + if err == nil { + return b + } + } + return def +} + +func envDur(k string, def time.Duration) time.Duration { + if v := os.Getenv(k); v != "" { + if d, err := time.ParseDuration(v); err == nil { + return d + } + } + return def +} + +// ctxWithTimeout is used by admin approve to bound the step-ca call. +func ctxWithTimeout(d time.Duration) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), d) +} diff --git a/serving/enrol-broker/signer.go b/serving/enrol-broker/signer.go new file mode 100644 index 0000000..04c9668 --- /dev/null +++ b/serving/enrol-broker/signer.go @@ -0,0 +1,214 @@ +// signer.go — turning an APPROVED CSR into a signed cert. +// +// Production path (stepSigner) shells out to the bundled `step` CLI exactly as +// the bench's tpm-provision.sh does: mint a one-time provisioner token, then +// `step ca sign` the CSR. The broker — not the host — holds the provisioner +// password, so a host can never self-sign; signing happens ONLY after an admin +// approves (the human OOB gate). +// +// Dev path (mockSigner) signs with an ephemeral in-process CA so the whole +// request -> approve -> cert flow is testable without a running step-ca. It is +// LAB-ONLY and announces itself loudly; never select it in production. +package main + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "log" + "math/big" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// Signer issues a cert for an approved CSR. Returns the cert PEM and its serial +// (lowercase hex, the value the hostnqn is derived from). +type Signer interface { + Sign(ctx context.Context, csrPEM, cn string) (certPEM, serial string, err error) + Kind() string +} + +// --- production: step-ca via the `step` CLI --------------------------------- + +type stepSigner struct { + bin string + caURL string + root string + provisioner string + passwordFile string + notAfter string +} + +func (s *stepSigner) Kind() string { return "step" } + +func (s *stepSigner) Sign(ctx context.Context, csrPEM, cn string) (string, string, error) { + if cn == "" { + return "", "", fmt.Errorf("CSR has no CommonName; cannot mint a token subject") + } + dir, err := os.MkdirTemp("", "enrol-*") + if err != nil { + return "", "", err + } + defer os.RemoveAll(dir) + csrPath := filepath.Join(dir, "req.csr") + outPath := filepath.Join(dir, "out.pem") + if err := os.WriteFile(csrPath, []byte(csrPEM), 0o600); err != nil { + return "", "", err + } + + // 1. one-time token (subject MUST equal the CSR CN for the JWK provisioner). + // NOTE: token validity is the short-lived authorization window (a JWK + // provisioner caps it at ~1h), NOT the issued cert's lifetime — do not put + // s.notAfter here or a multi-day value fails "token validity too long". + token, err := s.run(ctx, + "ca", "token", cn, + "--provisioner", s.provisioner, + "--ca-url", s.caURL, + "--root", s.root, + "--provisioner-password-file", s.passwordFile, + ) + if err != nil { + return "", "", fmt.Errorf("step ca token: %w", err) + } + token = strings.TrimSpace(token) + + // 2. sign the CSR with that token. The issued CERT's lifetime is set HERE + // (must be <= the provisioner's maxTLSCertDuration claim, else step-ca + // rejects it — raise that claim on the CA for multi-day fleet certs). + signArgs := []string{ + "ca", "sign", csrPath, outPath, + "--token", token, + "--ca-url", s.caURL, + "--root", s.root, + "--force", + } + if s.notAfter != "" { + signArgs = append(signArgs, "--not-after", s.notAfter) + } + if _, err := s.run(ctx, signArgs...); err != nil { + return "", "", fmt.Errorf("step ca sign: %w", err) + } + certPEM, err := os.ReadFile(outPath) + if err != nil { + return "", "", err + } + serial, err := serialFromPEM(certPEM) + if err != nil { + return "", "", err + } + return string(certPEM), serial, nil +} + +// run executes `step ...`, returning stdout. The token is sensitive but is passed +// as an argv element to the trusted local `step`; we never log argv or the +// password-file contents. +func (s *stepSigner) run(ctx context.Context, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, s.bin, args...) + out, err := cmd.Output() + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) { + return "", fmt.Errorf("%w: %s", err, strings.TrimSpace(string(ee.Stderr))) + } + return "", err + } + return string(out), nil +} + +// --- dev only: ephemeral in-process CA ------------------------------------- + +type mockSigner struct { + caCert *x509.Certificate + caKey *ecdsa.PrivateKey +} + +func newMockSigner() (*mockSigner, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "fleet-enrol-broker DEV mock CA"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return nil, err + } + caCert, err := x509.ParseCertificate(der) + if err != nil { + return nil, err + } + log.Printf("WARNING: BROKER_SIGNER=mock — issuing certs from an EPHEMERAL in-process CA. LAB ONLY; these certs are NOT trusted by RADIUS/NVMe. Set BROKER_SIGNER=step for real enrolment.") + return &mockSigner{caCert: caCert, caKey: key}, nil +} + +func (m *mockSigner) Kind() string { return "mock" } + +func (m *mockSigner) Sign(_ context.Context, csrPEM, cn string) (string, string, error) { + csr, err := parseCSR(csrPEM) + if err != nil { + return "", "", err + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return "", "", err + } + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: csr.Subject, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(720 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, m.caCert, csr.PublicKey, m.caKey) + if err != nil { + return "", "", err + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + return string(certPEM), strings.ToLower(fmt.Sprintf("%x", serial)), nil +} + +// --- shared helpers -------------------------------------------------------- + +func parseCSR(csrPEM string) (*x509.CertificateRequest, error) { + blk, _ := pem.Decode([]byte(csrPEM)) + if blk == nil || blk.Type != "CERTIFICATE REQUEST" { + return nil, fmt.Errorf("not a PEM CERTIFICATE REQUEST") + } + csr, err := x509.ParseCertificateRequest(blk.Bytes) + if err != nil { + return nil, err + } + if err := csr.CheckSignature(); err != nil { + return nil, fmt.Errorf("CSR self-signature invalid: %w", err) + } + return csr, nil +} + +func serialFromPEM(certPEM []byte) (string, error) { + blk, _ := pem.Decode(certPEM) + if blk == nil || blk.Type != "CERTIFICATE" { + return "", fmt.Errorf("issued cert is not PEM") + } + cert, err := x509.ParseCertificate(blk.Bytes) + if err != nil { + return "", err + } + return strings.ToLower(fmt.Sprintf("%x", cert.SerialNumber)), nil +} diff --git a/serving/enrol-broker/store.go b/serving/enrol-broker/store.go new file mode 100644 index 0000000..dba53f2 --- /dev/null +++ b/serving/enrol-broker/store.go @@ -0,0 +1,338 @@ +// store.go — the in-memory enrolment state machine + the OOB fingerprint. +// +// The store holds short-lived PENDING records keyed by a random enrol-id. State +// transitions are the load-bearing lifecycle from docs/tpm-enrolment-design.md: +// +// PENDING --(admin approve)--> APPROVED (cert minted, returned on poll) +// PENDING --(admin deny)-----> DENIED +// PENDING --(ttl elapsed)----> EXPIRED +// +// Records are time-boxed (a flooding bound) and evicted after a retention window. +// Nothing here is durable: a broker restart drops pending requests, which is +// acceptable — an un-enrolled host simply re-requests on its next poll/boot. +package main + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "math/bits" + "strings" + "sync" + "time" +) + +type State string + +const ( + StatePending State = "pending" + StateApproved State = "approved" + StateDenied State = "denied" + StateExpired State = "expired" +) + +// DeviceContext is what an un-enrolled host self-reports so the admin knows WHAT +// they are approving. It is advisory only — the fingerprint is the cryptographic +// anchor that binds the queue entry to the device on the console. +type DeviceContext struct { + MAC string `json:"mac,omitempty"` + DMIUUID string `json:"dmi_uuid,omitempty"` + DMISerial string `json:"dmi_serial,omitempty"` + EKHash string `json:"ek_hash,omitempty"` + Extra map[string]string `json:"extra,omitempty"` +} + +// Enrolment is one pending/decided enrolment. Unexported fields (nonce, csrPEM, +// certPEM) are deliberately NOT serialised into the admin queue view; the cert is +// only ever handed back to the requesting host via the status poll. +type Enrolment struct { + ID string `json:"id"` + State State `json:"state"` + Fingerprint string `json:"fingerprint"` + Context DeviceContext `json:"context"` + PubKeyHash string `json:"pubkey_hash"` + CN string `json:"cn"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at"` + DecidedAt time.Time `json:"decided_at,omitempty"` + DecidedBy string `json:"decided_by,omitempty"` + CertSerial string `json:"cert_serial,omitempty"` + Hostnqn string `json:"hostnqn,omitempty"` + + nonce []byte + csrPEM string + certPEM string +} + +// Age returns how long the record has existed (for the UI). +func (e *Enrolment) Age() time.Duration { return time.Since(e.CreatedAt) } + +type Store struct { + mu sync.Mutex + items map[string]*Enrolment + ttl time.Duration + retention time.Duration + maxPending int + + words []string + fpWords int + hostBase string +} + +func NewStore(ttl, retention time.Duration, maxPending int, words []string, fpWords int, hostBase string) (*Store, error) { + if !isPow2(len(words)) { + return nil, fmt.Errorf("wordlist size %d is not a power of two (needed for whole bits/word)", len(words)) + } + if fpWords < 4 { + return nil, fmt.Errorf("fingerprint word count %d too small", fpWords) + } + return &Store{ + items: make(map[string]*Enrolment), + ttl: ttl, + retention: retention, + maxPending: maxPending, + words: words, + fpWords: fpWords, + hostBase: hostBase, + }, nil +} + +var errTooManyPending = errors.New("too many pending enrolments") + +// Add creates a PENDING record for a parsed CSR's public-key DER + CN + context. +func (s *Store) Add(pubDER []byte, cn string, csrPEM string, ctx DeviceContext) (*Enrolment, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.countPendingLocked() >= s.maxPending { + return nil, errTooManyPending + } + + id, err := randHex(16) + if err != nil { + return nil, err + } + nonce := make([]byte, 16) + if _, err := rand.Read(nonce); err != nil { + return nil, err + } + fp, err := fingerprint(pubDER, nonce, s.words, s.fpWords) + if err != nil { + return nil, err + } + sum := sha256.Sum256(pubDER) + now := time.Now() + e := &Enrolment{ + ID: id, + State: StatePending, + Fingerprint: fp, + Context: ctx, + PubKeyHash: hex.EncodeToString(sum[:]), + CN: cn, + CreatedAt: now, + ExpiresAt: now.Add(s.ttl), + nonce: nonce, + csrPEM: csrPEM, + } + s.items[id] = e + return e.copy(), nil +} + +// Get returns a copy of a record (or nil), refreshing EXPIRED lazily. +func (s *Store) Get(id string) *Enrolment { + s.mu.Lock() + defer s.mu.Unlock() + e := s.items[id] + if e == nil { + return nil + } + s.expireLocked(e) + return e.copy() +} + +// csrFor returns the stored CSR PEM for a still-pending id (for the signer). +func (s *Store) csrFor(id string) (string, bool) { + s.mu.Lock() + defer s.mu.Unlock() + e := s.items[id] + if e == nil { + return "", false + } + s.expireLocked(e) + if e.State != StatePending { + return "", false + } + return e.csrPEM, true +} + +// Approve attaches the issued cert and flips the record to APPROVED. The signer +// runs OUTSIDE the lock (it shells out to step) so we re-check state here. +func (s *Store) Approve(id, certPEM, serial, admin string) (*Enrolment, error) { + s.mu.Lock() + defer s.mu.Unlock() + e := s.items[id] + if e == nil { + return nil, errors.New("unknown enrol-id") + } + s.expireLocked(e) + if e.State != StatePending { + return nil, fmt.Errorf("enrol-id is %s, not pending", e.State) + } + e.State = StateApproved + e.certPEM = certPEM + e.CertSerial = serial + e.Hostnqn = s.hostBase + serial + e.DecidedAt = time.Now() + e.DecidedBy = admin + return e.copy(), nil +} + +// Deny rejects a pending record. +func (s *Store) Deny(id, admin string) (*Enrolment, error) { + s.mu.Lock() + defer s.mu.Unlock() + e := s.items[id] + if e == nil { + return nil, errors.New("unknown enrol-id") + } + s.expireLocked(e) + if e.State != StatePending { + return nil, fmt.Errorf("enrol-id is %s, not pending", e.State) + } + e.State = StateDenied + e.DecidedAt = time.Now() + e.DecidedBy = admin + return e.copy(), nil +} + +// Pending returns copies of the still-pending records, oldest first. +func (s *Store) Pending() []*Enrolment { + s.mu.Lock() + defer s.mu.Unlock() + var out []*Enrolment + for _, e := range s.items { + s.expireLocked(e) + if e.State == StatePending { + out = append(out, e.copy()) + } + } + sortByCreated(out) + return out +} + +// Reap expires timed-out pending records and evicts old decided ones. Returns the +// number expired (for audit logging). +func (s *Store) Reap() int { + s.mu.Lock() + defer s.mu.Unlock() + expired := 0 + for id, e := range s.items { + before := e.State + s.expireLocked(e) + if before == StatePending && e.State == StateExpired { + expired++ + } + if e.State != StatePending && time.Since(e.decidedOrExpired()) > s.retention { + delete(s.items, id) + } + } + return expired +} + +func (s *Store) countPendingLocked() int { + n := 0 + for _, e := range s.items { + if e.State == StatePending && time.Now().Before(e.ExpiresAt) { + n++ + } + } + return n +} + +func (s *Store) expireLocked(e *Enrolment) { + if e.State == StatePending && !time.Now().Before(e.ExpiresAt) { + e.State = StateExpired + e.DecidedAt = e.ExpiresAt + } +} + +func (e *Enrolment) decidedOrExpired() time.Time { + if !e.DecidedAt.IsZero() { + return e.DecidedAt + } + return e.ExpiresAt +} + +func (e *Enrolment) copy() *Enrolment { + c := *e + return &c +} + +// --- fingerprint ----------------------------------------------------------- + +// fingerprint deterministically maps (CSR public key, server nonce) to a short +// word sequence. Both broker and host can compute it identically. The nonce is +// server-chosen per enrol-id so an attacker cannot pre-grind a key whose +// fingerprint collides with a target device's. Words (not hex) keep it short to +// read aloud yet resistant to casual collision. Swap wordlist.txt for the 2048- +// word BIP39 list (11 bits/word) if more entropy per word is wanted. +func fingerprint(pubDER, nonce []byte, words []string, n int) (string, error) { + if !isPow2(len(words)) { + return "", fmt.Errorf("wordlist size %d not a power of two", len(words)) + } + bpw := bits.TrailingZeros(uint(len(words))) + stream := digestStream(pubDER, nonce, (n*bpw+7)/8) + out := make([]string, 0, n) + bitpos := 0 + for i := 0; i < n; i++ { + idx := 0 + for k := 0; k < bpw; k++ { + byteIdx := bitpos / 8 + bit := 7 - uint(bitpos%8) + b := (stream[byteIdx] >> bit) & 1 + idx = (idx << 1) | int(b) + bitpos++ + } + out = append(out, words[idx]) + } + return strings.Join(out, "-"), nil +} + +// digestStream is a SHA-256 counter-mode KDF producing >= n bytes deterministically. +func digestStream(a, b []byte, n int) []byte { + var stream []byte + var ctr uint32 + for len(stream) < n { + h := sha256.New() + h.Write(a) + h.Write(b) + var c [4]byte + binary.BigEndian.PutUint32(c[:], ctr) + h.Write(c[:]) + stream = h.Sum(stream) + ctr++ + } + return stream +} + +func isPow2(n int) bool { return n > 0 && n&(n-1) == 0 } + +func randHex(nbytes int) (string, error) { + b := make([]byte, nbytes) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +// sortByCreated is a tiny insertion sort (oldest first); the queue is small. +func sortByCreated(xs []*Enrolment) { + for i := 1; i < len(xs); i++ { + for j := i; j > 0 && xs[j-1].CreatedAt.After(xs[j].CreatedAt); j-- { + xs[j-1], xs[j] = xs[j], xs[j-1] + } + } +} diff --git a/serving/enrol-broker/verify-serving.sh b/serving/enrol-broker/verify-serving.sh new file mode 100755 index 0000000..58fd045 --- /dev/null +++ b/serving/enrol-broker/verify-serving.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Verify the enrol-broker as deployed in the serving stack. Run ON THE SERVING +# HOST. Exercises the FULL path: public side through the nginx /enrol/ proxy +# (:80) + the admin plane on loopback :9443. +# +# ./verify-serving.sh # admin insecure (ENROL_ADMIN_INSECURE=true) +# ADMIN_P12=enrol-pki/broker/clients/alice.p12 ./verify-serving.sh # admin mTLS +# +# Stand-in for the (not-yet-built) host side: an openssl CSR plays the TPM key. +set -euo pipefail + +PUB_BASE="${PUB_BASE:-http://127.0.0.1/enrol}" # via nginx; use http://127.0.0.1:9090 to hit broker directly +ADMIN_BASE="${ADMIN_BASE:-http://127.0.0.1:9443}" +ADMIN_P12="${ADMIN_P12:-}" # set => admin is mTLS (https + client cert) + +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT +jqget() { python3 -c 'import json,sys;print(json.load(sys.stdin)["'"$1"'"])'; } + +# admin curl: plaintext by default, or mTLS if a .p12 is given. +admin() { + if [ -n "$ADMIN_P12" ]; then + openssl pkcs12 -in "$ADMIN_P12" -clcerts -nokeys -passin pass: -out "$WORK/ac.pem" 2>/dev/null + openssl pkcs12 -in "$ADMIN_P12" -nocerts -nodes -passin pass: -out "$WORK/ak.pem" 2>/dev/null + curl -fsS --cert "$WORK/ac.pem" --key "$WORK/ak.pem" -k "${ADMIN_BASE/http:/https:}$1" "${@:2}" + else + curl -fsS "$ADMIN_BASE$1" "${@:2}" + fi +} + +echo "== healthz (through nginx) ==" +curl -fsS "$PUB_BASE/healthz"; echo + +echo "== generate CSR (stand-in for the TPM key) ==" +openssl ecparam -name prime256v1 -genkey -noout -out "$WORK/h.key" 2>/dev/null +openssl req -new -key "$WORK/h.key" -subj "/CN=fleet-host" -out "$WORK/h.csr" 2>/dev/null +CSR_JSON=$(python3 -c 'import json,sys;print(json.dumps(open(sys.argv[1]).read()))' "$WORK/h.csr") + +echo "== POST /enrol/request ==" +REQ=$(curl -fsS -X POST "$PUB_BASE/request" -H 'Content-Type: application/json' \ + -d "{\"csr\": $CSR_JSON, \"context\": {\"mac\":\"de:ad:be:ef:00:01\",\"dmi_serial\":\"VERIFY-001\"}}") +echo "$REQ" +ID=$(echo "$REQ" | jqget enrol_id) +echo "enrol_id=$ID fingerprint=$(echo "$REQ" | jqget fingerprint)" + +echo "== admin /pending =="; admin /pending; echo +echo "== approve $ID =="; admin "/approve/$ID" -X POST; echo +echo "== /enrol/status/$ID (expect approved + cert) ==" +curl -fsS "$PUB_BASE/status/$ID" | python3 -m json.tool +echo +echo "PASS: request -> approve -> cert through the serving deployment." diff --git a/serving/enrol-broker/wordlist.txt b/serving/enrol-broker/wordlist.txt new file mode 100644 index 0000000..b4852d4 --- /dev/null +++ b/serving/enrol-broker/wordlist.txt @@ -0,0 +1,64 @@ +amber +anchor +apple +arrow +aspen +atlas +basil +beacon +birch +bishop +bramble +bronze +canyon +cedar +cobalt +comet +copper +coral +cosmos +cougar +dahlia +delta +ember +falcon +fjord +garnet +ginger +granite +harbor +hazel +indigo +ivory +jasper +jungle +kayak +kernel +lagoon +lantern +maple +marble +meadow +nectar +nimbus +nutmeg +onyx +orbit +otter +pepper +pewter +quartz +quiver +raven +ridge +saffron +sienna +sparrow +tundra +umber +valley +velvet +walnut +willow +zephyr +zinc diff --git a/serving/enrol-pki/.gitignore b/serving/enrol-pki/.gitignore new file mode 100644 index 0000000..a48118d --- /dev/null +++ b/serving/enrol-pki/.gitignore @@ -0,0 +1,4 @@ +# enrol-broker PKI material (CA root, provisioner password, admin mTLS certs). +# Mounted read-only at /pki. Keep the dir, never commit its contents. +* +!.gitignore diff --git a/serving/http/default.conf b/serving/http/default.conf index 6619536..a7eee73 100644 --- a/serving/http/default.conf +++ b/serving/http/default.conf @@ -20,6 +20,21 @@ server { autoindex on; } + # TPM enrolment broker (docs/tpm-enrolment-design.md) — the ONE unauthenticated, + # pre-auth endpoint. An empty-NV host POSTs its CSR and polls for the signed + # cert; the admin approve/deny plane is a SEPARATE mTLS listener and is NOT + # exposed here. Proxied to the broker (dot1x-bench stack) on loopback; the + # trailing slash strips /enrol/ so the broker sees /request and /status/{id}. + location /enrol/ { + proxy_pass http://127.0.0.1:9090/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Connection ""; + client_max_body_size 16k; # a CSR is ~1KB — cap abuse + proxy_read_timeout 30s; + } + # Kiosk SPA. The diskless kiosk (kiosk_url) points at # http://boot.fleet.lab/kiosk/ — resolved to this host by the stack's dnsmasq # and proxied to the kiosk-app container. diff --git a/serving/kiosk-app b/serving/kiosk-app index 4579d22..0ab2ea1 160000 --- a/serving/kiosk-app +++ b/serving/kiosk-app @@ -1 +1 @@ -Subproject commit 4579d227b096368469c7b5439d9a1b0f9858ba67 +Subproject commit 0ab2ea1371a412b93b0ff45f1fa4fdd2662e76ed