diskless-workstation/files/agent/snapshot.go

// snapshot.go — read-only host health probes → *fleetpb.HealthSnapshot, plus the
// hostapi facts fetch used to build the connect-time Hello.
//
// Every probe degrades gracefully: on any error the field is left at its zero
// value ("" / 0) rather than failing. A probe must NEVER kill the stream — a
// missing thermal zone or an unmounted selinuxfs is normal on some targets/VMs.
// Health METRICS (load/cpu/mem/temp/nvme/auth/selinux) are read straight from
// /proc + /sys; host FACTS (image/kernel/hostname/kiosk) are reused from
// fleet-hostapi (127.0.0.1:9099) so we don't re-probe DMI/GPU.
package main

import (
	"context"
	"encoding/json"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
	"time"

	fleetpb "fleet-agent/proto"
)

// hostFacts is the subset of fleet-hostapi's /v1/host response the agent uses.
type hostFacts struct {
	OS struct {
		Kernel string `json:"kernel"`
		Image  string `json:"image"`
	} `json:"os"`
	Net struct {
		Hostname string `json:"hostname"`
	} `json:"net"`
	Kiosk struct {
		Uptime   string `json:"uptime"`
		Chromium string `json:"chromium"`
	} `json:"kiosk"`
}

// fetchFacts GETs fleet-hostapi over loopback. Best-effort: returns a zero struct
// on any error (the agent still streams; Hello/snapshot fields just stay empty).
func fetchFacts(ctx context.Context, url string) hostFacts {
	var f hostFacts
	if url == "" {
		return f
	}
	rctx, cancel := context.WithTimeout(ctx, 3*time.Second)
	defer cancel()
	req, err := http.NewRequestWithContext(rctx, http.MethodGet, url, nil)
	if err != nil {
		return f
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return f
	}
	defer resp.Body.Close()
	_ = json.NewDecoder(resp.Body).Decode(&f)
	return f
}

// buildHello assembles the connect-time Hello from hostapi facts + the kiosk URL
// + the verity root hash (best-effort). starship_name is advisory (the manager
// already knows it from inventory).
func buildHello(f hostFacts, kioskURL string) *fleetpb.Hello {
	return &fleetpb.Hello{
		ImageVersion: f.OS.Image,
		RootfsDigest: rootfsDigest(),
		Kernel:       f.OS.Kernel,
		KioskUrl:     kioskURL,
		StarshipName: f.Net.Hostname,
	}
}

// prober carries the small state needed for rate metrics (the CPU busy delta
// between two /proc/stat samples). One prober is reused across the connection.
type prober struct {
	hostapiURL string
	prevTotal  uint64
	prevIdle   uint64
	hasPrev    bool
}

// snapshot gathers one HealthSnapshot. at_unix is left 0 so the manager stamps
// receive-time (matches agent.go's snapshotToHealth path).
func (p *prober) snapshot(ctx context.Context) *fleetpb.HealthSnapshot {
	f := fetchFacts(ctx, p.hostapiURL)
	ch := chronyTracking()
	s := &fleetpb.HealthSnapshot{
		LoadAvg:             loadAvg(),
		CpuPct:              p.cpuPct(),
		TempC:               tempC(),
		MemUsedPct:          memUsedPct(),
		TmpfsUsedPct:        fsUsedPct("/"),
		NvmeRoot:            nvmeRoot(),
		Auth:                authState(),
		Selinux:             selinuxState(),
		Chrony:              ch.state,
		ChronyOffsetSeconds: ch.offset,
		ChronySource:        ch.source,
		ChronyStratum:       ch.stratum,
		KioskActive:         unitActive("kiosk.service"),
		SessionUptime:       f.Kiosk.Uptime,
		UptimeSeconds:       uptimeSeconds(),
	}
	return s
}

// uptimeSeconds returns host uptime (seconds since boot) from /proc/uptime. The
// manager derives boot_time = receive-time - this to tell whether a scheduled
// reboot has been actioned yet.
func uptimeSeconds() int64 {
	b, err := os.ReadFile("/proc/uptime")
	if err != nil {
		return 0
	}
	return parseUptime(b)
}

func parseUptime(b []byte) int64 {
	fields := strings.Fields(string(b))
	if len(fields) == 0 {
		return 0
	}
	f, _ := strconv.ParseFloat(fields[0], 64)
	if f < 0 {
		return 0
	}
	return int64(f)
}

// ---- individual probes (all best-effort) ----------------------------------

func loadAvg() float64 {
	b, err := os.ReadFile("/proc/loadavg")
	if err != nil {
		return 0
	}
	return parseLoadAvg(b)
}

// parseLoadAvg extracts the 1-minute load from /proc/loadavg content.
func parseLoadAvg(b []byte) float64 {
	fields := strings.Fields(string(b))
	if len(fields) == 0 {
		return 0
	}
	v, _ := strconv.ParseFloat(fields[0], 64)
	return v
}

// cpuPct returns the busy percentage since the previous sample. The first call
// (no prior sample) returns 0 and primes the delta.
func (p *prober) cpuPct() float64 {
	total, idle, ok := readCPU()
	if !ok {
		return 0
	}
	defer func() { p.prevTotal, p.prevIdle, p.hasPrev = total, idle, true }()
	if !p.hasPrev {
		return 0
	}
	return cpuBusyPct(p.prevTotal, p.prevIdle, total, idle)
}

// cpuBusyPct computes the busy fraction between two /proc/stat samples.
func cpuBusyPct(prevTotal, prevIdle, total, idle uint64) float64 {
	if total <= prevTotal {
		return 0
	}
	dTotal := total - prevTotal
	dIdle := min(idle-prevIdle, dTotal)
	return float64(dTotal-dIdle) / float64(dTotal) * 100
}

// readCPU parses the aggregate "cpu" line of /proc/stat into total + idle jiffies.
func readCPU() (total, idle uint64, ok bool) {
	b, err := os.ReadFile("/proc/stat")
	if err != nil {
		return 0, 0, false
	}
	return parseCPUStat(b)
}

func parseCPUStat(b []byte) (total, idle uint64, ok bool) {
	for line := range strings.SplitSeq(string(b), "\n") {
		if !strings.HasPrefix(line, "cpu ") {
			continue
		}
		fields := strings.Fields(line)[1:] // user nice system idle iowait irq softirq steal ...
		for i, f := range fields {
			v, err := strconv.ParseUint(f, 10, 64)
			if err != nil {
				continue
			}
			total += v
			if i == 3 || i == 4 { // idle + iowait
				idle += v
			}
		}
		return total, idle, true
	}
	return 0, 0, false
}

func memUsedPct() float64 {
	b, err := os.ReadFile("/proc/meminfo")
	if err != nil {
		return 0
	}
	return parseMemUsedPct(b)
}

func parseMemUsedPct(b []byte) float64 {
	var totalKB, availKB float64
	for line := range strings.SplitSeq(string(b), "\n") {
		fields := strings.Fields(line)
		if len(fields) < 2 {
			continue
		}
		v, _ := strconv.ParseFloat(fields[1], 64)
		switch fields[0] {
		case "MemTotal:":
			totalKB = v
		case "MemAvailable:":
			availKB = v
		}
	}
	if totalKB == 0 {
		return 0
	}
	return (totalKB - availKB) / totalKB * 100
}

// fsUsedPct returns used-space percentage of the filesystem at path (the root
// overlay = tmpfs RAM pressure).
func fsUsedPct(path string) float64 {
	var st syscall.Statfs_t
	if err := syscall.Statfs(path, &st); err != nil || st.Blocks == 0 {
		return 0
	}
	used := st.Blocks - st.Bfree
	return float64(used) / float64(st.Blocks) * 100
}

// tempC returns the first readable thermal zone in Celsius (sysfs reports milli-C).
func tempC() float64 {
	for i := range 16 {
		b, err := os.ReadFile("/sys/class/thermal/thermal_zone" + strconv.Itoa(i) + "/temp")
		if err != nil {
			if i == 0 {
				continue // zone0 may be absent; keep scanning a few
			}
			break
		}
		milli, err := strconv.ParseFloat(strings.TrimSpace(string(b)), 64)
		if err == nil && milli > 0 {
			return milli / 1000
		}
	}
	return 0
}

// nvmeRoot reports the verity root device health: ok if /dev/mapper/root exists,
// down otherwise. (A finer degraded state needs nvme-cli; v1 keeps it binary.)
func nvmeRoot() string {
	if _, err := os.Stat("/dev/mapper/root"); err == nil {
		return "ok"
	}
	return "down"
}

// authState reports the 802.1X/MACsec link: up if macsec0 has carrier (secure
// path), "open" when there is no macsec0 (test/open-network path rd.dot1x=0).
func authState() string {
	if _, err := os.Stat("/sys/class/net/macsec0"); err != nil {
		return "open"
	}
	if b, err := os.ReadFile("/sys/class/net/macsec0/carrier"); err == nil && strings.TrimSpace(string(b)) == "1" {
		return "up"
	}
	return "down"
}

// selinuxState reads the enforce node; absent selinuxfs => "" (unknown).
func selinuxState() string {
	b, err := os.ReadFile("/sys/fs/selinux/enforce")
	if err != nil {
		return ""
	}
	if strings.TrimSpace(string(b)) == "1" {
		return "enforcing"
	}
	return "permissive"
}

// chrony holds the parsed `chronyc tracking` fields relative to the NTP source.
type chrony struct {
	state   string  // synced|unsynced|"" (unavailable)
	offset  float64 // system time offset from NTP, seconds (signed: fast=+, slow=-)
	source  string  // reference source name/IP (which server)
	stratum int32   // hops from a reference clock
}

// chronyTracking runs `chronyc tracking` once and parses it. Best-effort: a zero
// chrony{} (state "") if chronyc is unavailable — never fails the snapshot.
//
// -h 127.0.0.1 forces the LOOPBACK UDP command socket instead of chronyd's unix
// socket. Under the agent's hardened unit (DynamicUser — not in the chrony group;
// ProtectSystem=strict — /run read-only) the unix socket at /run/chrony is
// unreachable (can't open it, nor create chronyc's reply socket there). UDP needs
// no fs write or group and rides the unit's already-allowed AF_INET. The image
// binds this socket via `bindcmdaddress 127.0.0.1` in chrony.conf.
func chronyTracking() chrony {
	out, err := runShort("chronyc", "-n", "-h", "127.0.0.1", "tracking")
	if err != nil {
		return chrony{}
	}
	return parseChrony(out)
}

// parseChrony extracts the source/offset/stratum/leap fields from `chronyc
// tracking` key:value output. Pure (unit-testable).
func parseChrony(out string) chrony {
	c := chrony{state: "unsynced"}
	for line := range strings.SplitSeq(out, "\n") {
		key, val, ok := strings.Cut(line, ":")
		if !ok {
			continue
		}
		key = strings.TrimSpace(key)
		val = strings.TrimSpace(val)
		switch key {
		case "Leap status":
			if strings.Contains(val, "Normal") {
				c.state = "synced"
			}
		case "Stratum":
			if n, err := strconv.Atoi(strings.Fields(val)[0]); err == nil {
				c.stratum = int32(n)
			}
		case "Reference ID":
			// "C0A8320A (192.168.50.10)" — prefer the parenthesised name/IP.
			c.source = val
			if i := strings.IndexByte(val, '('); i >= 0 {
				if j := strings.IndexByte(val[i:], ')'); j > 1 {
					c.source = val[i+1 : i+j]
				}
			}
		case "System time":
			// "0.000012345 seconds slow of NTP time" — magnitude + direction.
			fields := strings.Fields(val)
			if len(fields) >= 3 {
				if v, err := strconv.ParseFloat(fields[0], 64); err == nil {
					if fields[2] == "slow" { // system behind NTP -> negative
						v = -v
					}
					c.offset = v
				}
			}
		}
	}
	return c
}

// unitActive reports whether a systemd unit is active (best-effort).
func unitActive(unit string) bool {
	out, err := runShort("systemctl", "is-active", unit)
	return err == nil && strings.TrimSpace(out) == "active"
}

// rootfsDigest best-effort: the dm-verity root hash from the kernel cmdline (the
// content-address of the running root). Empty if not present in this layout.
func rootfsDigest() string {
	b, err := os.ReadFile("/proc/cmdline")
	if err != nil {
		return ""
	}
	for tok := range strings.FieldsSeq(string(b)) {
		for _, k := range []string{"systemd.verity_root_hash=", "roothash="} {
			if after, ok := strings.CutPrefix(tok, k); ok {
				return after
			}
		}
	}
	return ""
}

// runShort runs a command with a short timeout and returns its stdout. Used only
// for the best-effort chrony/systemd probes; a missing binary is a soft failure.
func runShort(name string, args ...string) (string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
	defer cancel()
	out, err := exec.CommandContext(ctx, name, args...).Output()
	return string(out), err
}