// snapshot.go — read-only host health probes → *fleetpb.HealthSnapshot, plus the // hostapi facts fetch used to build the connect-time Hello. // // Every probe degrades gracefully: on any error the field is left at its zero // value ("" / 0) rather than failing. A probe must NEVER kill the stream — a // missing thermal zone or an unmounted selinuxfs is normal on some targets/VMs. // Health METRICS (load/cpu/mem/temp/nvme/auth/selinux) are read straight from // /proc + /sys; host FACTS (image/kernel/hostname/kiosk) are reused from // fleet-hostapi (127.0.0.1:9099) so we don't re-probe DMI/GPU. package main import ( "context" "encoding/json" "net/http" "os" "os/exec" "strconv" "strings" "syscall" "time" fleetpb "fleet-agent/proto" ) // hostFacts is the subset of fleet-hostapi's /v1/host response the agent uses. type hostFacts struct { OS struct { Kernel string `json:"kernel"` Image string `json:"image"` } `json:"os"` Net struct { Hostname string `json:"hostname"` } `json:"net"` Kiosk struct { Uptime string `json:"uptime"` Chromium string `json:"chromium"` } `json:"kiosk"` } // fetchFacts GETs fleet-hostapi over loopback. Best-effort: returns a zero struct // on any error (the agent still streams; Hello/snapshot fields just stay empty). func fetchFacts(ctx context.Context, url string) hostFacts { var f hostFacts if url == "" { return f } rctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() req, err := http.NewRequestWithContext(rctx, http.MethodGet, url, nil) if err != nil { return f } resp, err := http.DefaultClient.Do(req) if err != nil { return f } defer resp.Body.Close() _ = json.NewDecoder(resp.Body).Decode(&f) return f } // buildHello assembles the connect-time Hello from hostapi facts + the kiosk URL // + the verity root hash (best-effort). starship_name is advisory (the manager // already knows it from inventory). func buildHello(f hostFacts, kioskURL string) *fleetpb.Hello { return &fleetpb.Hello{ ImageVersion: f.OS.Image, RootfsDigest: rootfsDigest(), Kernel: f.OS.Kernel, KioskUrl: kioskURL, StarshipName: f.Net.Hostname, } } // prober carries the small state needed for rate metrics (the CPU busy delta // between two /proc/stat samples). One prober is reused across the connection. type prober struct { hostapiURL string prevTotal uint64 prevIdle uint64 hasPrev bool } // snapshot gathers one HealthSnapshot. at_unix is left 0 so the manager stamps // receive-time (matches agent.go's snapshotToHealth path). func (p *prober) snapshot(ctx context.Context) *fleetpb.HealthSnapshot { f := fetchFacts(ctx, p.hostapiURL) ch := chronyTracking() s := &fleetpb.HealthSnapshot{ LoadAvg: loadAvg(), CpuPct: p.cpuPct(), TempC: tempC(), MemUsedPct: memUsedPct(), TmpfsUsedPct: fsUsedPct("/"), NvmeRoot: nvmeRoot(), Auth: authState(), Selinux: selinuxState(), Chrony: ch.state, ChronyOffsetSeconds: ch.offset, ChronySource: ch.source, ChronyStratum: ch.stratum, KioskActive: unitActive("kiosk.service"), SessionUptime: f.Kiosk.Uptime, UptimeSeconds: uptimeSeconds(), } return s } // uptimeSeconds returns host uptime (seconds since boot) from /proc/uptime. The // manager derives boot_time = receive-time - this to tell whether a scheduled // reboot has been actioned yet. func uptimeSeconds() int64 { b, err := os.ReadFile("/proc/uptime") if err != nil { return 0 } return parseUptime(b) } func parseUptime(b []byte) int64 { fields := strings.Fields(string(b)) if len(fields) == 0 { return 0 } f, _ := strconv.ParseFloat(fields[0], 64) if f < 0 { return 0 } return int64(f) } // ---- individual probes (all best-effort) ---------------------------------- func loadAvg() float64 { b, err := os.ReadFile("/proc/loadavg") if err != nil { return 0 } return parseLoadAvg(b) } // parseLoadAvg extracts the 1-minute load from /proc/loadavg content. func parseLoadAvg(b []byte) float64 { fields := strings.Fields(string(b)) if len(fields) == 0 { return 0 } v, _ := strconv.ParseFloat(fields[0], 64) return v } // cpuPct returns the busy percentage since the previous sample. The first call // (no prior sample) returns 0 and primes the delta. func (p *prober) cpuPct() float64 { total, idle, ok := readCPU() if !ok { return 0 } defer func() { p.prevTotal, p.prevIdle, p.hasPrev = total, idle, true }() if !p.hasPrev { return 0 } return cpuBusyPct(p.prevTotal, p.prevIdle, total, idle) } // cpuBusyPct computes the busy fraction between two /proc/stat samples. func cpuBusyPct(prevTotal, prevIdle, total, idle uint64) float64 { if total <= prevTotal { return 0 } dTotal := total - prevTotal dIdle := min(idle-prevIdle, dTotal) return float64(dTotal-dIdle) / float64(dTotal) * 100 } // readCPU parses the aggregate "cpu" line of /proc/stat into total + idle jiffies. func readCPU() (total, idle uint64, ok bool) { b, err := os.ReadFile("/proc/stat") if err != nil { return 0, 0, false } return parseCPUStat(b) } func parseCPUStat(b []byte) (total, idle uint64, ok bool) { for line := range strings.SplitSeq(string(b), "\n") { if !strings.HasPrefix(line, "cpu ") { continue } fields := strings.Fields(line)[1:] // user nice system idle iowait irq softirq steal ... for i, f := range fields { v, err := strconv.ParseUint(f, 10, 64) if err != nil { continue } total += v if i == 3 || i == 4 { // idle + iowait idle += v } } return total, idle, true } return 0, 0, false } func memUsedPct() float64 { b, err := os.ReadFile("/proc/meminfo") if err != nil { return 0 } return parseMemUsedPct(b) } func parseMemUsedPct(b []byte) float64 { var totalKB, availKB float64 for line := range strings.SplitSeq(string(b), "\n") { fields := strings.Fields(line) if len(fields) < 2 { continue } v, _ := strconv.ParseFloat(fields[1], 64) switch fields[0] { case "MemTotal:": totalKB = v case "MemAvailable:": availKB = v } } if totalKB == 0 { return 0 } return (totalKB - availKB) / totalKB * 100 } // fsUsedPct returns used-space percentage of the filesystem at path (the root // overlay = tmpfs RAM pressure). func fsUsedPct(path string) float64 { var st syscall.Statfs_t if err := syscall.Statfs(path, &st); err != nil || st.Blocks == 0 { return 0 } used := st.Blocks - st.Bfree return float64(used) / float64(st.Blocks) * 100 } // tempC returns the first readable thermal zone in Celsius (sysfs reports milli-C). func tempC() float64 { for i := range 16 { b, err := os.ReadFile("/sys/class/thermal/thermal_zone" + strconv.Itoa(i) + "/temp") if err != nil { if i == 0 { continue // zone0 may be absent; keep scanning a few } break } milli, err := strconv.ParseFloat(strings.TrimSpace(string(b)), 64) if err == nil && milli > 0 { return milli / 1000 } } return 0 } // nvmeRoot reports the verity root device health: ok if /dev/mapper/root exists, // down otherwise. (A finer degraded state needs nvme-cli; v1 keeps it binary.) func nvmeRoot() string { if _, err := os.Stat("/dev/mapper/root"); err == nil { return "ok" } return "down" } // authState reports the 802.1X/MACsec link: up if macsec0 has carrier (secure // path), "open" when there is no macsec0 (test/open-network path rd.dot1x=0). func authState() string { if _, err := os.Stat("/sys/class/net/macsec0"); err != nil { return "open" } if b, err := os.ReadFile("/sys/class/net/macsec0/carrier"); err == nil && strings.TrimSpace(string(b)) == "1" { return "up" } return "down" } // selinuxState reads the enforce node; absent selinuxfs => "" (unknown). func selinuxState() string { b, err := os.ReadFile("/sys/fs/selinux/enforce") if err != nil { return "" } if strings.TrimSpace(string(b)) == "1" { return "enforcing" } return "permissive" } // chrony holds the parsed chrony tracking fields relative to the NTP source. type chrony struct { state string // synced|unsynced|"" (unavailable) offset float64 // system time offset from NTP, seconds (signed: fast=+, slow=-) source string // reference source name/IP (which server) stratum int32 // hops from a reference clock } // chronyLogPath is chrony's tracking log — enabled by `log tracking` in // chrony.conf; chronyd appends one line per clock update. const chronyLogPath = "/var/log/chrony/tracking.log" // chronyTracking reads the LAST line of chrony's tracking.log and parses the // source/stratum/offset. Best-effort: a zero chrony{} if the log is // absent/unreadable/empty — never fails the snapshot. // // We read a FILE rather than query chronyd's command socket by design: that unix // socket is owner-write-only (0755) so the hardened non-root agent (DynamicUser, // not the chrony user) can't connect, and STIG closes the UDP command port // (cmdport 0). `log tracking` writes the same fields to a world-readable log the // agent can read under ProtectSystem=strict — no port, no socket, no group, no // SELinux domain transition. Tradeoff: the offset is as fresh as chrony's last // update (up to ~maxpoll old), which is fine for a slow-moving health metric. func chronyTracking() chrony { b, err := os.ReadFile(chronyLogPath) if err != nil { return chrony{} } return parseTrackingLog(b) } // parseTrackingLog extracts source/stratum/offset/sync-state from the LAST data // line of chrony's tracking.log. Pure (unit-testable). // // Column layout (chrony `log tracking`; header written by logbanner): // // Date Time IP-Address St Freq Skew Offset L Co Offset-sd Rem-corr Root-delay Root-disp // f[0] f[1] f[2] f[3] f[4] f[5] f[6] f[7] ... // // Offset (f[6]) carries chrony's sign convention (ahead/fast = +), matching our // chrony_offset_seconds field. L (f[7]) is the leap/sync flag: N/+/- = synchronised, // '?' = not synchronised. Header/banner lines (first field not a digit) are skipped; // an unsynchronised reference logs 0.0.0.0, reported as an empty source. // BENCH-VERIFY the offset sign against `chronyc tracking` on a synced host. func parseTrackingLog(data []byte) chrony { lines := strings.Split(string(data), "\n") for i := len(lines) - 1; i >= 0; i-- { f := strings.Fields(lines[i]) if len(f) < 8 || f[0] == "" || f[0][0] < '0' || f[0][0] > '9' { continue // blank, header ("Date ..."), or banner ("=====") } c := chrony{state: "unsynced"} if f[7] != "?" { c.state = "synced" } if f[2] != "0.0.0.0" { c.source = f[2] } if n, err := strconv.Atoi(f[3]); err == nil { c.stratum = int32(n) } if v, err := strconv.ParseFloat(f[6], 64); err == nil { c.offset = v } return c } return chrony{} } // unitActive reports whether a systemd unit is active (best-effort). func unitActive(unit string) bool { out, err := runShort("systemctl", "is-active", unit) return err == nil && strings.TrimSpace(out) == "active" } // rootfsDigest best-effort: the dm-verity root hash from the kernel cmdline (the // content-address of the running root). Empty if not present in this layout. func rootfsDigest() string { b, err := os.ReadFile("/proc/cmdline") if err != nil { return "" } for tok := range strings.FieldsSeq(string(b)) { for _, k := range []string{"systemd.verity_root_hash=", "roothash="} { if after, ok := strings.CutPrefix(tok, k); ok { return after } } } return "" } // runShort runs a command with a short timeout and returns its stdout. Used only // for the best-effort chrony/systemd probes; a missing binary is a soft failure. func runShort(name string, args ...string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() out, err := exec.CommandContext(ctx, name, args...).Output() return string(out), err }