diskless-workstation/serving/fleet-manager/agent.go

// agent.go — the gRPC agent plane: the long-lived bidi stream each host holds.
//
// mTLS is verified against the FLEET CA (the host's reused TPM identity cert),
// distinct from the admin plane's dedicated admin CA. The host's cert SERIAL is
// the inventory key, taken from the peer cert — never self-asserted — so a host
// can only report/receive its own state.
//
// Concurrency model per Session: one receive loop (host->manager) in the handler
// goroutine + one send loop draining a per-connection channel (manager->host).
// The channel decouples admin-initiated pushes (SetDesired, Command) from the
// stream's Send, which must not be called concurrently. Inventory is the source
// of truth; the live stream is only the low-latency delivery path — a host that
// is offline or whose send buffer is full simply gets the current desired-state
// on its next (re)connect.
package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/keepalive"
	"google.golang.org/grpc/peer"
	"google.golang.org/grpc/status"

	fleetpb "fleet-manager/proto"
)

// MetricsSink receives each health snapshot for time-series storage. The no-op
// sink is the default; the InfluxDB sink lands in checkpoint 4.
type MetricsSink interface {
	WriteSnapshot(serial, hostnqn, starship string, s *fleetpb.HealthSnapshot, at time.Time)
}

type noopSink struct{}

func (noopSink) WriteSnapshot(string, string, string, *fleetpb.HealthSnapshot, time.Time) {}

// agentConn is one live host stream's downstream (manager->host) side.
type agentConn struct {
	serial string
	send   chan *fleetpb.ManagerDown
	done   chan struct{}
	once   sync.Once
}

func (c *agentConn) close() { c.once.Do(func() { close(c.done) }) }

// agentHub implements the FleetAgent gRPC service and holds the live-stream
// registry that admin actions push through.
type agentHub struct {
	fleetpb.UnimplementedFleetAgentServer
	store Store
	sink  MetricsSink

	mu    sync.Mutex
	conns map[string]*agentConn // serial -> live connection
}

func newAgentHub(store Store, sink MetricsSink) *agentHub {
	if sink == nil {
		sink = noopSink{}
	}
	return &agentHub{store: store, sink: sink, conns: map[string]*agentConn{}}
}

// register installs a connection, evicting any prior stream for the same serial
// (a reconnect supersedes a lingering old one).
func (h *agentHub) register(c *agentConn) {
	h.mu.Lock()
	if old := h.conns[c.serial]; old != nil {
		old.close()
	}
	h.conns[c.serial] = c
	h.mu.Unlock()
}

func (h *agentHub) deregister(c *agentConn) {
	h.mu.Lock()
	if h.conns[c.serial] == c { // don't clobber a newer reconnect
		delete(h.conns, c.serial)
	}
	h.mu.Unlock()
	c.close()
}

// pushDown delivers a message to a host's live stream if connected. Returns false
// if the host is offline or its buffer is backed up (it will reconcile on its
// next connect from persisted desired-state — no message is "lost" that matters).
func (h *agentHub) pushDown(serial string, m *fleetpb.ManagerDown) bool {
	h.mu.Lock()
	c := h.conns[serial]
	h.mu.Unlock()
	if c == nil {
		return false
	}
	select {
	case c.send <- m:
		return true
	case <-c.done:
		return false
	default:
		return false // slow consumer; drop — desired-state is durable in inventory
	}
}

// pushDesired sends a host its current desired-state over the live stream.
func (h *agentHub) pushDesired(serial string, d Desired) bool {
	return h.pushDown(serial, desiredMsg(d))
}

// Session is the bidirectional stream RPC.
func (h *agentHub) Session(stream fleetpb.FleetAgent_SessionServer) error {
	serial, err := peerSerial(stream.Context())
	if err != nil {
		return status.Error(codes.Unauthenticated, err.Error())
	}
	// A host must be enrolled (inventory record exists) before it can stream.
	host, ok := h.store.Get(serial)
	if !ok {
		return status.Errorf(codes.PermissionDenied, "host %s is not enrolled", serial)
	}

	c := &agentConn{serial: serial, send: make(chan *fleetpb.ManagerDown, 16), done: make(chan struct{})}
	h.register(c)
	h.store.SetOnline(serial, true, time.Now().UTC())
	log.Printf("agent connected: %s (%s)", serial, host.Identity.StarshipName)
	defer func() {
		h.deregister(c)
		h.store.SetOnline(serial, false, time.Now().UTC())
		log.Printf("agent disconnected: %s", serial)
	}()

	// Deliver current desired-state immediately on connect.
	c.send <- desiredMsg(host.Desired)

	// Sender goroutine: the ONLY caller of stream.Send.
	sendErr := make(chan error, 1)
	go func() {
		for {
			select {
			case m := <-c.send:
				if err := stream.Send(m); err != nil {
					sendErr <- err
					return
				}
			case <-c.done:
				return
			}
		}
	}()

	// Receive loop (this goroutine).
	for {
		select {
		case err := <-sendErr:
			return err
		default:
		}
		up, err := stream.Recv()
		if err != nil {
			return err // EOF / transport error -> disconnect (defer cleans up)
		}
		h.handleUp(serial, up)
	}
}

func (h *agentHub) handleUp(serial string, up *fleetpb.AgentUp) {
	switch m := up.GetMsg().(type) {
	case *fleetpb.AgentUp_Hello:
		hello := m.Hello
		host, ok := h.store.Get(serial)
		if !ok {
			return
		}
		_, _, _ = h.store.ReportState(serial,
			Version{ImageVersion: hello.GetImageVersion(), RootfsDigest: hello.GetRootfsDigest(), Kernel: hello.GetKernel()},
			host.Health, // unchanged until first snapshot
			Usage{KioskURL: hello.GetKioskUrl(), Display: host.Usage.Display},
			time.Now().UTC())
	case *fleetpb.AgentUp_Snapshot:
		snap := m.Snapshot
		at := time.Now().UTC()
		if snap.GetAtUnix() > 0 {
			at = time.Unix(snap.GetAtUnix(), 0).UTC()
		}
		host, ok := h.store.Get(serial)
		if !ok {
			return
		}
		h.store.ReportState(serial, host.Version, snapshotToHealth(snap), snapshotToUsage(snap, host.Usage), at)
		h.sink.WriteSnapshot(serial, host.Identity.Hostnqn, host.Identity.StarshipName, snap, at)
	case *fleetpb.AgentUp_Result:
		res := m.Result
		log.Printf("agent %s command %s result: ok=%v %s", serial, res.GetCommandId(), res.GetOk(), res.GetDetail())
	}
}

// ---- conversions ----------------------------------------------------------

func desiredMsg(d Desired) *fleetpb.ManagerDown {
	var rebootAt int64
	if d.RebootAt != nil {
		rebootAt = d.RebootAt.Unix()
	}
	return &fleetpb.ManagerDown{Msg: &fleetpb.ManagerDown_Desired{Desired: &fleetpb.DesiredState{
		TargetVersion: d.TargetVersion,
		KioskUrl:      d.KioskURL,
		RebootAtUnix:  rebootAt,
		Drain:         d.Drain,
	}}}
}

func snapshotToHealth(s *fleetpb.HealthSnapshot) Health {
	return Health{
		LoadAvg:       s.GetLoadAvg(),
		CPUPct:        s.GetCpuPct(),
		TempC:         s.GetTempC(),
		MemUsedPct:    s.GetMemUsedPct(),
		TmpfsUsedPct:  s.GetTmpfsUsedPct(),
		NVMeRoot:      s.GetNvmeRoot(),
		Auth:          s.GetAuth(),
		SELinux:       s.GetSelinux(),
		Chrony:        s.GetChrony(),
		ChronyOffset:  s.GetChronyOffsetSeconds(),
		ChronySource:  s.GetChronySource(),
		ChronyStratum: s.GetChronyStratum(),
		KioskActive:   s.GetKioskActive(),
		UptimeSeconds: s.GetUptimeSeconds(),
	}
}

func snapshotToUsage(s *fleetpb.HealthSnapshot, prev Usage) Usage {
	u := prev
	u.SessionUptime = s.GetSessionUptime()
	u.ChromiumCrashes = int(s.GetChromiumCrashes())
	if s.GetDisplay() != "" {
		u.Display = s.GetDisplay()
	}
	return u
}

// ---- mTLS identity --------------------------------------------------------

// peerSerial extracts the lowercase-hex serial of the client's mTLS leaf cert —
// the inventory key. It is trusted because RequireAndVerifyClientCert already
// verified the chain to the fleet CA.
func peerSerial(ctx context.Context) (string, error) {
	p, ok := peer.FromContext(ctx)
	if !ok {
		return "", fmt.Errorf("no peer in context")
	}
	tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo)
	if !ok || len(tlsInfo.State.PeerCertificates) == 0 {
		return "", fmt.Errorf("no client certificate")
	}
	return serialHex(tlsInfo.State.PeerCertificates[0]), nil
}

// serialHex renders a cert serial as lowercase hex — the same form the broker
// stores (store.go) so the value matches the inventory key.
func serialHex(cert *x509.Certificate) string {
	return fmt.Sprintf("%x", cert.SerialNumber)
}

// ---- gRPC server wiring ---------------------------------------------------

// agentServer builds the configured gRPC server (mTLS against the fleet CA +
// keepalive) with the hub registered. Split from the listen/serve so tests can
// drive it on an ephemeral listener.
func agentServer(cfg config, hub *agentHub) (*grpc.Server, error) {
	// GetCertificate (not a fixed Certificates slice) so the agent-cert-renewer
	// sidecar can rotate server.crt/key on disk and the manager picks it up on the
	// next handshake without a restart. See certreload.go.
	reloader, err := newCertReloader(cfg.agentCert, cfg.agentKey)
	if err != nil {
		return nil, fmt.Errorf("agent server cert: %w", err)
	}
	caPEM, err := os.ReadFile(cfg.agentClientCA)
	if err != nil {
		return nil, fmt.Errorf("fleet CA: %w", err)
	}
	pool := x509.NewCertPool()
	if !pool.AppendCertsFromPEM(caPEM) {
		return nil, fmt.Errorf("no certificates parsed from FLEET_MGR_AGENT_CLIENT_CA")
	}
	creds := credentials.NewTLS(&tls.Config{
		MinVersion:     tls.VersionTLS12,
		GetCertificate: reloader.GetCertificate,
		ClientAuth:     tls.RequireAndVerifyClientCert,
		ClientCAs:      pool,
	})
	gs := grpc.NewServer(
		grpc.Creds(creds),
		grpc.KeepaliveParams(keepalive.ServerParameters{Time: 30 * time.Second, Timeout: 10 * time.Second}),
		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{MinTime: 10 * time.Second, PermitWithoutStream: true}),
	)
	fleetpb.RegisterFleetAgentServer(gs, hub)
	return gs, nil
}

// serveAgent starts the gRPC agent listener with mTLS against the fleet CA.
func serveAgent(cfg config, hub *agentHub) error {
	gs, err := agentServer(cfg, hub)
	if err != nil {
		return err
	}
	lis, err := net.Listen("tcp", cfg.agentAddr)
	if err != nil {
		return err
	}
	log.Printf("fleet-manager agent (gRPC/mTLS) listening on %s", cfg.agentAddr)
	return gs.Serve(lis)
}