fleet-manager: front the admin UI with Traefik (443 + Route 53 ACME), retire acme side-car

Management-plane ingress: a host-networked Traefik terminates the LE cert on :443
(Route 53 DNS-01, public-resolver propagation check) and forwards plaintext to
the manager running FLEET_MGR_ADMIN_BEHIND_PROXY=true on loopback :9500. Only the
admin plane rides Traefik — the agent gRPC/mTLS plane (:9600) stays direct (Traefik
can't terminate fleet-CA mTLS; hosts reach it via OPNsense L3 routing). This is the
k8s ingress-controller + cert-manager role in miniature.

- traefik/dynamic/fleet-manager.yml: file-provider router (Host rule -> manager),
  passHostHeader so the WebAuthn RP origin matches.
- compose: traefik service (static config as CLI flags so ${ACME_EMAIL} interpolates)
  + traefik-acme volume; manager switched to behind-proxy defaults, RP_ORIGIN now
  the port-less https origin.
- Retire the acme side-car (Traefik owns ACME now): remove the service + acme-state
  volume; renew-loop.sh kept + relabelled as the self-TLS-mode cert helper.
- :443 only (admin UI is https-only) so it can't collide with a boot-LAN nginx :80.

serving/fleet-manager/acme/renew-loop.sh

@@ -1,5 +1,10 @@
 #!/bin/sh
-# renew-loop.sh — admin-UI cert issue/renew loop for the `acme` side-car.
+# renew-loop.sh — admin-UI cert issue/renew loop (self-TLS alternative).
+#
+# NOTE: the default deployment fronts the admin UI with Traefik (docker-compose.yml),
+# which owns ACME itself — this script is NOT used then. It remains as the cert
+# helper for the SELF-TLS admin mode (FLEET_MGR_ADMIN_BEHIND_PROXY=false, no
+# Traefik): run it under a `goacme/lego` container to write ./pki/admin/.
 #
 # Runs inside the goacme/lego image (alpine-based). Obtains the fleet-manager
 # admin-UI cert from Let's Encrypt via DNS-01 over AWS Route 53, and writes it

serving/fleet-manager/docker-compose.yml

@@ -71,13 +71,17 @@ services:
       influxdb:
         condition: service_healthy
     environment:
-      # admin plane (HTTPS + passkey). Bind loopback; the mgmt proxy (holding the
+      # admin plane — passkey auth behind the Traefik mgmt ingress: Traefik
-      # LE cert) fronts it, or set the cert paths below + expose directly.
+      # terminates the LE cert on :443 and forwards plaintext here on loopback,
+      # so the listener MUST stay private (127.0.0.1) — only Traefik reaches it.
       FLEET_MGR_ADMIN_ADDR: "${FLEET_MGR_ADMIN_ADDR:-127.0.0.1:9500}"
+      FLEET_MGR_ADMIN_BEHIND_PROXY: "${FLEET_MGR_ADMIN_BEHIND_PROXY:-true}"
+      FLEET_MGR_ADMIN_INSECURE: "${FLEET_MGR_ADMIN_INSECURE:-false}"
+      # Only used for self-TLS mode (BEHIND_PROXY=false + a cert, no Traefik):
       FLEET_MGR_ADMIN_CERT: "${FLEET_MGR_ADMIN_CERT:-/pki/admin/fullchain.pem}"
       FLEET_MGR_ADMIN_KEY: "${FLEET_MGR_ADMIN_KEY:-/pki/admin/privkey.pem}"
-      FLEET_MGR_ADMIN_INSECURE: "${FLEET_MGR_ADMIN_INSECURE:-false}"
+      # WebAuthn RP — MUST be the UI's real origin (origin-bound passkeys). Behind
-      # WebAuthn RP — MUST be the UI's real origin (origin-bound passkeys).
+      # Traefik on :443 this is the PORT-LESS https origin.
       FLEET_MGR_RP_ID: "${FLEET_MGR_RP_ID:?set FLEET_MGR_RP_ID in .env}"
       FLEET_MGR_RP_ORIGIN: "${FLEET_MGR_RP_ORIGIN:?set FLEET_MGR_RP_ORIGIN in .env}"
       FLEET_MGR_RP_NAME: "${FLEET_MGR_RP_NAME:-Fleet Command}"
@@ -114,43 +118,55 @@ services:
       # The admin LE cert under ./pki/admin/ is produced by the `acme` side-car.
       - ./pki:/pki:ro
 
-  # --- admin-UI cert renewal side-car (Let's Encrypt DNS-01 via Route 53) -----
+  # --- management-plane ingress (Traefik) — admin UI on :443 ------------------
-  # Long-running "compose cron": issues the UI cert on first start, then renews
+  # Terminates the Let's Encrypt cert (Route 53 DNS-01) on :443 and forwards
-  # within 30 days of expiry. Writes into ./pki/admin/ — the SAME dir the manager
+  # plaintext to the manager (behind-proxy mode, loopback :9500). Host networking
-  # reads (./pki:/pki:ro) — so the manager hot-reloads it with no restart. Needs
+  # so it binds :443 on the mgmt interface and reaches the host-networked manager
-  # outbound reach to Let's Encrypt + the AWS Route 53 API (bridge NAT).
+  # on 127.0.0.1:9500. Needs outbound reach to Let's Encrypt + the AWS Route 53
-  # Bring this up BEFORE the manager (or the manager crash-loops until the cert
+  # API. ONLY the admin plane rides Traefik — the agent gRPC/mTLS plane (:9600) is
-  # exists, then self-heals). See DEPLOY.md §2c.
+  # direct (Traefik can't terminate fleet-CA client-cert mTLS; hosts reach it via
-  acme:
+  # OPNsense L3 routing). This is the k8s ingress-controller + cert-manager role
-    image: goacme/lego:v4.17.4       # mirror via harbor if no dockerhub egress
+  # in miniature — the router/ACME model maps to Traefik Ingress + cert-manager.
+  #
+  # NOTE: only :443 (the admin UI is https-only; passkeys require it) so it can't
+  # collide with a boot-LAN nginx :80 on a single-host lab. Route 53 DNS-01 needs
+  # no :80. Set the Host() rule in traefik/dynamic/fleet-manager.yml to your UI host.
+  traefik:
+    image: traefik:v3.1              # mirror via harbor if no dockerhub egress
     restart: unless-stopped
-    entrypoint: ["/bin/sh", "/renew-loop.sh"]
+    network_mode: host
+    command:
+      - "--global.checknewversion=false"
+      - "--global.sendanonymoususage=false"
+      - "--entrypoints.websecure.address=:443"
+      - "--providers.file.directory=/etc/traefik/dynamic"
+      - "--providers.file.watch=true"
+      - "--certificatesresolvers.le.acme.email=${ACME_EMAIL:?set ACME_EMAIL in .env}"
+      - "--certificatesresolvers.le.acme.storage=/acme/acme.json"
+      - "--certificatesresolvers.le.acme.dnschallenge.provider=route53"
+      # Public resolvers for the propagation check (split-horizon DNS won't serve
+      # the public _acme-challenge). First dry-run vs LE staging: also add
+      #   --certificatesresolvers.le.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory
+      - "--certificatesresolvers.le.acme.dnschallenge.resolvers=8.8.8.8:53,1.1.1.1:53"
+      - "--log.level=INFO"
     environment:
-      ACME_DOMAIN: "${FLEET_MGR_RP_ID:?set FLEET_MGR_RP_ID in .env}"  # = the UI hostname
+      # Route 53 DNS-01 creds (or attach an instance role + omit). IAM policy in
-      ACME_EMAIL: "${ACME_EMAIL:?set ACME_EMAIL in .env}"
+      # DEPLOY.md §2c; AWS_HOSTED_ZONE_ID optional (pins the zone).
-      ACME_STAGING: "${ACME_STAGING:-}"                               # set 1 for a dry-run
-      # Public resolver(s) for the propagation check — the internal/split-horizon
-      # DNS doesn't serve the public _acme-challenge record. Comma-separated host:port.
-      ACME_RESOLVERS: "${ACME_RESOLVERS:-8.8.8.8:53}"
-      # AWS creds for the Route 53 DNS-01 challenge (or omit + use an instance role).
       AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID:-}"
       AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY:-}"
       AWS_REGION: "${AWS_REGION:-us-east-1}"
-      # Optional: pin the zone (skips the ListHostedZonesByName lookup — needed if
-      # a public + private zone share the domain; lets you tighten the IAM policy).
       AWS_HOSTED_ZONE_ID: "${AWS_HOSTED_ZONE_ID:-}"
     volumes:
-      - ./acme/renew-loop.sh:/renew-loop.sh:ro
+      - ./traefik/dynamic:/etc/traefik/dynamic:ro
-      - acme-state:/le               # lego account + cert store — PERSIST
+      - traefik-acme:/acme            # acme.json (ACME account + issued certs) — PERSIST
-      - ./pki/admin:/out             # shared with the manager's ./pki/admin
+    depends_on:
-    networks:
+      - fleet-manager
-      - fleet-mgmt
 
 volumes:
   influx-data:
   influx-config:
   fleet-data:
-  acme-state:
+  traefik-acme:
 
 networks:
   fleet-mgmt:

serving/fleet-manager/traefik/dynamic/fleet-manager.yml

@@ -0,0 +1,26 @@
+# Traefik dynamic config (file provider) — route the admin UI hostname to the
+# fleet-manager, which runs in behind-proxy mode (plaintext on loopback :9500,
+# passkey auth ON; Traefik terminates the LE cert).
+#
+# ONLY the admin plane rides Traefik. The agent gRPC/mTLS plane (:9600) is NOT
+# here — Traefik can't terminate the fleet-CA client-cert mTLS; hosts reach it
+# directly (host VLAN -> OPNsense L3 -> manager:9600). See DEPLOY.md.
+#
+# Change the Host() rule to your admin-UI hostname (== FLEET_MGR_RP_ID). File
+# config isn't env-interpolated, so this is set per-deployment.
+http:
+  routers:
+    fleet-admin:
+      rule: "Host(`diskless-server.bitasylum.org`)"
+      entryPoints:
+        - websecure
+      service: fleet-admin
+      tls:
+        certResolver: le
+
+  services:
+    fleet-admin:
+      loadBalancer:
+        servers:
+          - url: "http://127.0.0.1:9500"   # manager, behind-proxy plaintext (loopback)
+        passHostHeader: true               # preserve Host so WebAuthn RP origin matches