fleet-manager: fix CA install, add register-host, surface agent status

Four fixes from the on-host agent bring-up:

1. Bake the real fleet CA into the image. New `fleet_ca` build var (default =
   the committed placeholder) file-provisions the trust anchor to
   /etc/dot1x/radius-ca.pem — used by wpa_supplicant (RADIUS) AND the fleet-agent
   (verify the manager). CI materializes a FLEET_CA secret to a runner-temp file
   and passes -var fleet_ca; unset => placeholder (lab). No CA material committed.

2. server-pki.sh installs the agent CLIENT CA as root+intermediate (not root
   only). Host TPM certs are leaf-only in NV and signed by the step-ca
   intermediate, so a root-only client CA makes the manager advertise only the
   root in its CertificateRequest -> Go clients send no cert ("tls: certificate
   required") and it couldn't verify a leaf-only cert anyway. bootstrap.sh now
   exports intermediate_ca.crt so FLEET_CA_INTERMEDIATE resolves.

3. New `fleet-manager register-host <serial> [-name] [-hostnqn]` subcommand:
   seeds/updates an inventory record for a host enrolled out of band (bench
   broker, migrated fleet) through the real store, so its agent stream is
   accepted. The manual counterpart to the authorise-approve flow.

4. The agent surfaces the authoritative recv-side gRPC status instead of the
   bare io.EOF gRPC returns on the send side when the server ends the stream —
   so "host is not enrolled" (PermissionDenied) shows up instead of "EOF".

.gitea/workflows/gitea-main-ci.yaml

@@ -280,6 +280,22 @@ jobs:
           echo "SB_CERT_FILE=$d/db.crt" >> "$GITHUB_ENV"
           echo "Secure Boot signing material staged (signed UKI build)."
 
+      -
+        name: Materialize fleet CA (radius-ca.pem trust anchor)
+        # The real fleet step-ca root is baked to /etc/dot1x/radius-ca.pem (RADIUS
+        # EAP-TLS + fleet-agent server-verify). Written to a runner-temp FILE and
+        # passed as a build-host PATH (var.fleet_ca). If FLEET_CA is unset this
+        # no-ops and the build falls back to the committed PLACEHOLDER (auth cannot
+        # succeed — lab only). Never commit real internal CA material (CLAUDE.md).
+        if: ${{ secrets.FLEET_CA != '' }}
+        env:
+          FLEET_CA: ${{ secrets.FLEET_CA }}   # fleet step-ca root PEM (root, or root+intermediate)
+        run: |
+          umask 077
+          printf '%s\n' "$FLEET_CA" > "$RUNNER_TEMP/fleet-ca.pem"
+          echo "FLEET_CA_FILE=$RUNNER_TEMP/fleet-ca.pem" >> "$GITHUB_ENV"
+          echo "Fleet CA staged (real trust anchor baked into radius-ca.pem)."
+
       -
         name: Materialize Secure Boot enrollment certs
         # PUBLIC PK/KEK/db .auth files baked into the image for fleet-sb-enroll.
@@ -316,6 +332,7 @@ jobs:
             -var ssh_authorized_key="${{ secrets.SSH_AUTHORIZED_KEY }}" \
             -var sb_key="${SB_KEY_FILE:-}" \
             -var sb_cert="${SB_CERT_FILE:-}" \
+            -var fleet_ca="${FLEET_CA_FILE:-files/etc/dot1x/radius-ca.pem}" \
             -var kiosk_url="${{ env.KIOSK_URL }}" \
             -var boot_splash=${{ env.BOOT_SPLASH }} \
             -var enable_stig=${{ env.ENABLE_STIG }} \

diskless.pkr.hcl

@@ -173,6 +173,12 @@ variable "sb_cert" {
   default     = ""
 }
 
+variable "fleet_ca" {
+  type        = string
+  description = "Path on the BUILD HOST to the fleet CA PEM baked to /etc/dot1x/radius-ca.pem — the trust anchor wpa_supplicant uses for RADIUS EAP-TLS AND the fleet-agent uses to verify the fleet-manager. Defaults to the COMMITTED PLACEHOLDER (files/etc/dot1x/radius-ca.pem), which is NOT a usable CA (auth/telemetry can't succeed with it). Override per-environment (CI secret -> temp file) with your real step-ca root (or a root+intermediate bundle for robustness). Never commit real internal CA material (CLAUDE.md)."
+  default     = "files/etc/dot1x/radius-ca.pem"
+}
+
 # --------------------------------------------------------------------- source
 
 source "qemu" "diskless" {
@@ -237,6 +243,15 @@ build {
     destination = "/tmp/etc-stage"
   }
 
+  # Fleet CA baked to /etc/dot1x/radius-ca.pem (RADIUS trust anchor + fleet-agent
+  # server-verify CA). var.fleet_ca defaults to the committed placeholder; CI
+  # overrides it with the real per-environment step-ca root (a build-host path).
+  # Installed over the placeholder in the etc-install step below.
+  provisioner "file" {
+    source      = var.fleet_ca
+    destination = "/tmp/fleet-ca.pem"
+  }
+
   # Kiosk unit + session wrapper (installed by the kiosk provisioner below).
   # Dir-without-trailing-slash upload -> /tmp/kiosk/ (mirrors the dracut module
   # upload form, which stages reliably).
@@ -368,7 +383,10 @@ build {
       "rm -rf /tmp/plymouth",
 
       # Stage etc files into place
-      "install -D -m0644 /tmp/etc-stage/dot1x/radius-ca.pem            /etc/dot1x/radius-ca.pem",
+      # radius-ca.pem comes from var.fleet_ca (placeholder by default; real fleet CA
+      # per-environment via CI) — NOT the etc-stage copy, so one build input controls
+      # the trust anchor for both RADIUS EAP-TLS and the fleet-agent.
+      "install -D -m0644 /tmp/fleet-ca.pem                             /etc/dot1x/radius-ca.pem",
       "install -D -m0644 /tmp/etc-stage/dot1x/wpa_supplicant-dot1x.conf /etc/dot1x/wpa_supplicant-dot1x.conf",
       "install -D -m0644 /tmp/etc-stage/dracut.conf.d/90-diskless.conf  /etc/dracut.conf.d/90-diskless.conf",
       # NB: the old static /etc/NetworkManager/conf.d/99-dot1x-unmanaged.conf is

files/agent/main.go

@@ -15,6 +15,7 @@ package main
 
 import (
 	"context"
+	"io"
 	"log"
 	"math/rand"
 	"os"
@@ -117,18 +118,16 @@ func runSession(ctx context.Context, client fleetpb.FleetAgentClient, cfg config
 		return err
 	}
 
-	// Hello first — what we're running, so the manager reconciles version drift.
-	facts := fetchFacts(sctx, cfg.hostapiURL)
-	if err := stream.Send(&fleetpb.AgentUp{Msg: &fleetpb.AgentUp_Hello{Hello: buildHello(facts, readKioskURL(cfg.kioskEnv))}}); err != nil {
-		return err
-	}
-	log.Printf("connected to %s — streaming health every %s", cfg.manager, cfg.interval)
-
-	// Receive loop: v1 LOGS desired-state/commands, does not act on them.
+	// Receive loop. Its error is the AUTHORITATIVE session status — when the manager
+	// ends the stream (e.g. PermissionDenied "host is not enrolled"), gRPC only
+	// returns a bare io.EOF on the SEND side, so a raw Send error is uninformative.
+	// We surface the recv-side status instead. v1 LOGS desired-state/commands.
+	recvErr := make(chan error, 1)
 	go func() {
 		for {
 			down, err := stream.Recv()
 			if err != nil {
+				recvErr <- err
 				cancel()
 				return
 			}
@@ -136,24 +135,53 @@ func runSession(ctx context.Context, client fleetpb.FleetAgentClient, cfg config
 		}
 	}()
 
+	// Hello first — what we're running, so the manager reconciles version drift.
+	facts := fetchFacts(sctx, cfg.hostapiURL)
+	if err := stream.Send(&fleetpb.AgentUp{Msg: &fleetpb.AgentUp_Hello{Hello: buildHello(facts, readKioskURL(cfg.kioskEnv))}}); err != nil {
+		return statusFrom(err, recvErr)
+	}
+	log.Printf("connected to %s — streaming health every %s", cfg.manager, cfg.interval)
+
 	// Send one snapshot immediately, then on the interval.
 	t := time.NewTicker(cfg.interval)
 	defer t.Stop()
 	if err := stream.Send(&fleetpb.AgentUp{Msg: &fleetpb.AgentUp_Snapshot{Snapshot: p.snapshot(sctx)}}); err != nil {
-		return err
+		return statusFrom(err, recvErr)
 	}
 	for {
 		select {
+		case err := <-recvErr:
+			return err
 		case <-sctx.Done():
-			return sctx.Err()
+			// Prefer the recv-side status if that's what cancelled us.
+			select {
+			case err := <-recvErr:
+				return err
+			default:
+				return sctx.Err()
+			}
 		case <-t.C:
 			if err := stream.Send(&fleetpb.AgentUp{Msg: &fleetpb.AgentUp_Snapshot{Snapshot: p.snapshot(sctx)}}); err != nil {
-				return err
+				return statusFrom(err, recvErr)
 			}
 		}
 	}
 }
 
+// statusFrom turns a Send-side io.EOF (which gRPC returns when the server has
+// closed the stream) into the authoritative recv-side status — the real reason
+// the manager ended the session — falling back to the original error.
+func statusFrom(err error, recvErr chan error) error {
+	if err == io.EOF {
+		select {
+		case re := <-recvErr:
+			return re
+		case <-time.After(2 * time.Second):
+		}
+	}
+	return err
+}
+
 // logDown records a manager->host message. TELEMETRY-ONLY v1: received, logged,
 // NOT applied. The reconcile actuators land in a follow-up (see docs/TODO.md #14).
 func logDown(down *fleetpb.ManagerDown) {

serving/dot1x-bench/step-ca/bootstrap.sh

@@ -71,6 +71,14 @@ FP=$(openssl x509 -in "$PKI/root_ca.crt" -noout -fingerprint -sha256 | cut -d= -
 echo "    root fingerprint: $FP"
 echo "    copy to: files/etc/dot1x/radius-ca.pem  (image trust anchor)"
 
+# Also export the INTERMEDIATE: leaf certs (host TPM identities, RADIUS, the
+# fleet-manager server cert) are signed by it, so the fleet-manager agent-plane
+# CLIENT CA must be root+intermediate (server-pki.sh reads it via
+# FLEET_CA_INTERMEDIATE). Best-effort — a single-tier CA has none.
+incca cp /home/step/certs/intermediate_ca.crt /export/intermediate_ca.crt 2>/dev/null \
+    && echo "    exported intermediate_ca.crt (-> server-pki.sh fleet-ca.crt bundle)" \
+    || echo "    (no intermediate_ca.crt — single-tier CA)"
+
 # --- 1b. raise the CA cert-duration claim (idempotent) ----------------------
 # A stock step-ca caps issued certs at 24h. Both the RADIUS server cert below
 # AND the enrol-broker (--not-after 720h) need longer-lived fleet certs, so the

serving/fleet-manager/cli.go

@@ -163,3 +163,81 @@ func runRegisterAdmin(args []string) int {
 	fmt.Printf("Enrolment link for %q (valid %s) — open on the mgmt network, then tap the YubiKey:\n\n  %s/#/enrol?token=%s\n\n", name, ttl.String(), base, tok)
 	return 0
 }
+
+// runRegisterHost implements:
+//
+//	fleet-manager register-host [-name S] [-hostnqn N] [-inventory F] <cert-serial-hex>
+//
+// It seeds or updates an inventory record for a host enrolled OUT OF BAND (the
+// standalone bench broker, or a manager inheriting an existing fleet), so the
+// agent stream — which the manager refuses for an unknown serial (agent.go
+// "host is not enrolled") — is accepted. Normally the record is born by the
+// authorise-approve flow; this is the manual import path. It writes through the
+// real jsonStore (atomic save), so it must point at the SAME inventory file the
+// service uses (FLEET_MGR_INVENTORY, default /data in the container). Run it via
+// `docker compose run --rm fleet-manager register-host ...`.
+func runRegisterHost(args []string) int {
+	fs := flag.NewFlagSet("register-host", flag.ContinueOnError)
+	name := fs.String("name", "", "starship name (dnsmasq fleet-namer join; advisory)")
+	inv := fs.String("inventory", env("FLEET_MGR_INVENTORY", "/var/lib/fleet-manager/inventory.json"), "inventory file (must match the service's FLEET_MGR_INVENTORY)")
+	nqn := fs.String("hostnqn", "", "host NQN (default: $HOSTNQN_BASE + serial)")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	serial := strings.ToLower(strings.TrimSpace(fs.Arg(0)))
+	if !isHexSerial(serial) {
+		fmt.Fprintln(os.Stderr, "usage: fleet-manager register-host [-name S] [-hostnqn N] [-inventory F] <cert-serial-hex>")
+		return 2
+	}
+
+	store, err := newJSONStore(*inv)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "inventory %s: %v\n", *inv, err)
+		return 1
+	}
+	hostnqn := *nqn
+	if hostnqn == "" {
+		hostnqn = env("HOSTNQN_BASE", "nqn.2026-06.org.example.fleet:host-") + serial
+	}
+	now := time.Now().UTC()
+
+	h, existed := store.Get(serial)
+	if !existed {
+		h = &Host{Identity: Identity{CertSerial: serial}, FirstSeen: now}
+	}
+	h.Identity.CertSerial = serial
+	h.Identity.Hostnqn = hostnqn
+	if *name != "" {
+		h.Identity.StarshipName = *name
+	}
+	if h.Enrolled.At.IsZero() {
+		h.Enrolled = Enrolled{Approver: "register-host CLI", At: now}
+	}
+	h.LastSeen = now
+
+	if err := store.Upsert(h); err != nil {
+		fmt.Fprintf(os.Stderr, "write inventory %s: %v\n", *inv, err)
+		return 1
+	}
+	verb := "registered"
+	if existed {
+		verb = "updated"
+	}
+	fmt.Printf("%s host %s (starship=%q hostnqn=%s) in %s\n", verb, serial, h.Identity.StarshipName, hostnqn, *inv)
+	return 0
+}
+
+// isHexSerial reports whether s is a non-empty lowercase-hex cert serial (the
+// inventory key form). Serials can be odd-length hex (big-int %x), so we validate
+// characters rather than decoding.
+func isHexSerial(s string) bool {
+	if s == "" {
+		return false
+	}
+	for _, c := range s {
+		if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) {
+			return false
+		}
+	}
+	return true
+}

serving/fleet-manager/main.go

@@ -112,6 +112,13 @@ func main() {
 	if len(os.Args) > 1 && os.Args[1] == "register-admin" {
 		os.Exit(runRegisterAdmin(os.Args[2:]))
 	}
+	// Subcommand: `fleet-manager register-host <serial>` seeds/updates an inventory
+	// record for a host enrolled OUT OF BAND (bench broker, migrated fleet) so its
+	// agent stream is accepted — the manual counterpart to the authorise-approve
+	// flow that normally births the record.
+	if len(os.Args) > 1 && os.Args[1] == "register-host" {
+		os.Exit(runRegisterHost(os.Args[2:]))
+	}
 
 	log.SetFlags(log.LstdFlags | log.LUTC)
 	cfg := loadConfig()

serving/fleet-manager/server-pki.sh

@@ -29,6 +29,7 @@ OUT=${OUT:-./pki/agent}
 AGENT_SANS=${AGENT_SANS:-diskless-server.bitasylum.org}
 FLEET_CA_URL=${FLEET_CA_URL:-https://ca.fleet.lab:8443}
 FLEET_CA_ROOT=${FLEET_CA_ROOT:-../dot1x-bench/pki/root_ca.crt}
+FLEET_CA_INTERMEDIATE=${FLEET_CA_INTERMEDIATE:-../dot1x-bench/pki/intermediate_ca.crt}
 STEP_PROVISIONER=${STEP_PROVISIONER:-enrol}
 STEP_PROVISIONER_PASSWORD_FILE=${STEP_PROVISIONER_PASSWORD_FILE:-../dot1x-bench/pki/stepca/secrets/password}
 CERT_NOT_AFTER=${CERT_NOT_AFTER:-8760h}
@@ -39,9 +40,6 @@ command -v step >/dev/null || { echo "error: 'step' CLI not found (smallstep/cli
 
 mkdir -p "$OUT"
 
-# The fleet root doubles as the agent-plane CLIENT CA (verifies host TPM certs).
-cp "$FLEET_CA_ROOT" "$OUT/fleet-ca.crt"
-
 # First SAN is the CN; every SAN (DNS name / IP) hosts might dial goes in --san.
 primary=${AGENT_SANS%%,*}
 san_args=()
@@ -60,6 +58,30 @@ step ca certificate "$primary" "$OUT/server.crt" "$OUT/server.key" \
 
 chmod 600 "$OUT/server.key"
 
+# fleet-ca.crt = the agent-plane CLIENT CA that verifies host TPM identity certs.
+# Host certs are stored LEAF-ONLY in TPM NV and are signed by the step-ca
+# INTERMEDIATE, so this MUST be root + intermediate. Root-only fails two ways:
+# (1) the manager can't build leaf->intermediate->root, and (2) it advertises only
+# the root in its TLS CertificateRequest, so a Go client whose leaf is issued by
+# the intermediate sends NO cert at all ("remote error: tls: certificate
+# required"). Prefer an explicit intermediate file; else pull it from the chain
+# step wrote into the just-issued server.crt (every cert after the leaf).
+intermediate=""
+if [ -f "$FLEET_CA_INTERMEDIATE" ]; then
+    intermediate=$(cat "$FLEET_CA_INTERMEDIATE")
+else
+    intermediate=$(awk 'BEGIN{c=0} /-----BEGIN CERTIFICATE-----/{c++} c>1' "$OUT/server.crt")
+fi
+{
+    [ -n "$intermediate" ] && printf '%s\n' "$intermediate"
+    cat "$FLEET_CA_ROOT"
+} > "$OUT/fleet-ca.crt"
+if [ -z "$intermediate" ]; then
+    echo "WARNING: no intermediate CA found (set FLEET_CA_INTERMEDIATE, or the CA has none) —" >&2
+    echo "         fleet-ca.crt is ROOT-ONLY; leaf-only host certs issued by an intermediate" >&2
+    echo "         will fail mTLS with 'tls: certificate required'." >&2
+fi
+
 echo
 echo "done. staged in $OUT :"
 echo "  server.crt / server.key   -> FLEET_MGR_AGENT_CERT / _KEY"