diskless-workstation/files/agent/snapshot.go

// snapshot.go — read-only host health probes → *fleetpb.HealthSnapshot, plus the
// hostapi facts fetch used to build the connect-time Hello.
//
// Every probe degrades gracefully: on any error the field is left at its zero
// value ("" / 0) rather than failing. A probe must NEVER kill the stream — a
// missing thermal zone or an unmounted selinuxfs is normal on some targets/VMs.
// Health METRICS (load/cpu/mem/temp/nvme/auth/selinux) are read straight from
// /proc + /sys; host FACTS (image/kernel/hostname/kiosk) are reused from
// fleet-hostapi (127.0.0.1:9099) so we don't re-probe DMI/GPU.
package main

import (
	"context"
	"encoding/json"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
	"time"

	fleetpb "fleet-agent/proto"
)

// hostFacts is the subset of fleet-hostapi's /v1/host response the agent uses.
type hostFacts struct {
	OS struct {
		Kernel string `json:"kernel"`
		Image  string `json:"image"`
	} `json:"os"`
	Net struct {
		Hostname string `json:"hostname"`
	} `json:"net"`
	Kiosk struct {
		Uptime   string `json:"uptime"`
		Chromium string `json:"chromium"`
	} `json:"kiosk"`
}

// fetchFacts GETs fleet-hostapi over loopback. Best-effort: returns a zero struct
// on any error (the agent still streams; Hello/snapshot fields just stay empty).
func fetchFacts(ctx context.Context, url string) hostFacts {
	var f hostFacts
	if url == "" {
		return f
	}
	rctx, cancel := context.WithTimeout(ctx, 3*time.Second)
	defer cancel()
	req, err := http.NewRequestWithContext(rctx, http.MethodGet, url, nil)
	if err != nil {
		return f
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return f
	}
	defer resp.Body.Close()
	_ = json.NewDecoder(resp.Body).Decode(&f)
	return f
}

// buildHello assembles the connect-time Hello from hostapi facts + the kiosk URL
// + the verity root hash (best-effort). starship_name is advisory (the manager
// already knows it from inventory).
func buildHello(f hostFacts, kioskURL string) *fleetpb.Hello {
	return &fleetpb.Hello{
		ImageVersion: f.OS.Image,
		RootfsDigest: rootfsDigest(),
		Kernel:       f.OS.Kernel,
		KioskUrl:     kioskURL,
		StarshipName: f.Net.Hostname,
	}
}

// prober carries the small state needed for rate metrics (the CPU busy delta
// between two /proc/stat samples). One prober is reused across the connection.
type prober struct {
	hostapiURL string
	prevTotal  uint64
	prevIdle   uint64
	hasPrev    bool
}

// snapshot gathers one HealthSnapshot. at_unix is left 0 so the manager stamps
// receive-time (matches agent.go's snapshotToHealth path).
func (p *prober) snapshot(ctx context.Context) *fleetpb.HealthSnapshot {
	f := fetchFacts(ctx, p.hostapiURL)
	ch := chronyTracking()
	s := &fleetpb.HealthSnapshot{
		LoadAvg:             loadAvg(),
		CpuPct:              p.cpuPct(),
		TempC:               tempC(),
		MemUsedPct:          memUsedPct(),
		TmpfsUsedPct:        fsUsedPct("/"),
		NvmeRoot:            nvmeRoot(),
		Auth:                authState(),
		Selinux:             selinuxState(),
		Chrony:              ch.state,
		ChronyOffsetSeconds: ch.offset,
		ChronySource:        ch.source,
		ChronyStratum:       ch.stratum,
		KioskActive:         unitActive("kiosk.service"),
		SessionUptime:       f.Kiosk.Uptime,
		UptimeSeconds:       uptimeSeconds(),
	}
	return s
}

// uptimeSeconds returns host uptime (seconds since boot) from /proc/uptime. The
// manager derives boot_time = receive-time - this to tell whether a scheduled
// reboot has been actioned yet.
func uptimeSeconds() int64 {
	b, err := os.ReadFile("/proc/uptime")
	if err != nil {
		return 0
	}
	return parseUptime(b)
}

func parseUptime(b []byte) int64 {
	fields := strings.Fields(string(b))
	if len(fields) == 0 {
		return 0
	}
	f, _ := strconv.ParseFloat(fields[0], 64)
	if f < 0 {
		return 0
	}
	return int64(f)
}

// ---- individual probes (all best-effort) ----------------------------------

func loadAvg() float64 {
	b, err := os.ReadFile("/proc/loadavg")
	if err != nil {
		return 0
	}
	return parseLoadAvg(b)
}

// parseLoadAvg extracts the 1-minute load from /proc/loadavg content.
func parseLoadAvg(b []byte) float64 {
	fields := strings.Fields(string(b))
	if len(fields) == 0 {
		return 0
	}
	v, _ := strconv.ParseFloat(fields[0], 64)
	return v
}

// cpuPct returns the busy percentage since the previous sample. The first call
// (no prior sample) returns 0 and primes the delta.
func (p *prober) cpuPct() float64 {
	total, idle, ok := readCPU()
	if !ok {
		return 0
	}
	defer func() { p.prevTotal, p.prevIdle, p.hasPrev = total, idle, true }()
	if !p.hasPrev {
		return 0
	}
	return cpuBusyPct(p.prevTotal, p.prevIdle, total, idle)
}

// cpuBusyPct computes the busy fraction between two /proc/stat samples.
func cpuBusyPct(prevTotal, prevIdle, total, idle uint64) float64 {
	if total <= prevTotal {
		return 0
	}
	dTotal := total - prevTotal
	dIdle := min(idle-prevIdle, dTotal)
	return float64(dTotal-dIdle) / float64(dTotal) * 100
}

// readCPU parses the aggregate "cpu" line of /proc/stat into total + idle jiffies.
func readCPU() (total, idle uint64, ok bool) {
	b, err := os.ReadFile("/proc/stat")
	if err != nil {
		return 0, 0, false
	}
	return parseCPUStat(b)
}

func parseCPUStat(b []byte) (total, idle uint64, ok bool) {
	for line := range strings.SplitSeq(string(b), "\n") {
		if !strings.HasPrefix(line, "cpu ") {
			continue
		}
		fields := strings.Fields(line)[1:] // user nice system idle iowait irq softirq steal ...
		for i, f := range fields {
			v, err := strconv.ParseUint(f, 10, 64)
			if err != nil {
				continue
			}
			total += v
			if i == 3 || i == 4 { // idle + iowait
				idle += v
			}
		}
		return total, idle, true
	}
	return 0, 0, false
}

func memUsedPct() float64 {
	b, err := os.ReadFile("/proc/meminfo")
	if err != nil {
		return 0
	}
	return parseMemUsedPct(b)
}

func parseMemUsedPct(b []byte) float64 {
	var totalKB, availKB float64
	for line := range strings.SplitSeq(string(b), "\n") {
		fields := strings.Fields(line)
		if len(fields) < 2 {
			continue
		}
		v, _ := strconv.ParseFloat(fields[1], 64)
		switch fields[0] {
		case "MemTotal:":
			totalKB = v
		case "MemAvailable:":
			availKB = v
		}
	}
	if totalKB == 0 {
		return 0
	}
	return (totalKB - availKB) / totalKB * 100
}

// fsUsedPct returns used-space percentage of the filesystem at path (the root
// overlay = tmpfs RAM pressure).
func fsUsedPct(path string) float64 {
	var st syscall.Statfs_t
	if err := syscall.Statfs(path, &st); err != nil || st.Blocks == 0 {
		return 0
	}
	used := st.Blocks - st.Bfree
	return float64(used) / float64(st.Blocks) * 100
}

// tempC returns the first readable thermal zone in Celsius (sysfs reports milli-C).
func tempC() float64 {
	for i := range 16 {
		b, err := os.ReadFile("/sys/class/thermal/thermal_zone" + strconv.Itoa(i) + "/temp")
		if err != nil {
			if i == 0 {
				continue // zone0 may be absent; keep scanning a few
			}
			break
		}
		milli, err := strconv.ParseFloat(strings.TrimSpace(string(b)), 64)
		if err == nil && milli > 0 {
			return milli / 1000
		}
	}
	return 0
}

// nvmeRoot reports the verity root device health: ok if /dev/mapper/root exists,
// down otherwise. (A finer degraded state needs nvme-cli; v1 keeps it binary.)
func nvmeRoot() string {
	if _, err := os.Stat("/dev/mapper/root"); err == nil {
		return "ok"
	}
	return "down"
}

// authState reports the 802.1X/MACsec link: up if macsec0 has carrier (secure
// path), "open" when there is no macsec0 (test/open-network path rd.dot1x=0).
func authState() string {
	if _, err := os.Stat("/sys/class/net/macsec0"); err != nil {
		return "open"
	}
	if b, err := os.ReadFile("/sys/class/net/macsec0/carrier"); err == nil && strings.TrimSpace(string(b)) == "1" {
		return "up"
	}
	return "down"
}

// selinuxState reads the enforce node; absent selinuxfs => "" (unknown).
func selinuxState() string {
	b, err := os.ReadFile("/sys/fs/selinux/enforce")
	if err != nil {
		return ""
	}
	if strings.TrimSpace(string(b)) == "1" {
		return "enforcing"
	}
	return "permissive"
}

// chrony holds the parsed chrony tracking fields relative to the NTP source.
type chrony struct {
	state   string  // synced|unsynced|"" (unavailable)
	offset  float64 // system time offset from NTP, seconds (signed: fast=+, slow=-)
	source  string  // reference source name/IP (which server)
	stratum int32   // hops from a reference clock
}

// chronyLogPath is chrony's tracking log — enabled by `log tracking` in
// chrony.conf; chronyd appends one line per clock update.
const chronyLogPath = "/var/log/chrony/tracking.log"

// chronyTracking reads the LAST line of chrony's tracking.log and parses the
// source/stratum/offset. Best-effort: a zero chrony{} if the log is
// absent/unreadable/empty — never fails the snapshot.
//
// We read a FILE rather than query chronyd's command socket by design: that unix
// socket is owner-write-only (0755) so the hardened non-root agent (DynamicUser,
// not the chrony user) can't connect, and STIG closes the UDP command port
// (cmdport 0). `log tracking` writes the same fields to a world-readable log the
// agent can read under ProtectSystem=strict — no port, no socket, no group, no
// SELinux domain transition. Tradeoff: the offset is as fresh as chrony's last
// update (up to ~maxpoll old), which is fine for a slow-moving health metric.
func chronyTracking() chrony {
	b, err := os.ReadFile(chronyLogPath)
	if err != nil {
		return chrony{}
	}
	return parseTrackingLog(b)
}

// parseTrackingLog extracts source/stratum/offset/sync-state from the LAST data
// line of chrony's tracking.log. Pure (unit-testable).
//
// Column layout (chrony `log tracking`; header written by logbanner):
//
//	Date Time IP-Address St Freq Skew Offset L Co Offset-sd Rem-corr Root-delay Root-disp
//	f[0] f[1]   f[2]    f[3] f[4] f[5]  f[6] f[7] ...
//
// Offset (f[6]) carries chrony's sign convention (ahead/fast = +), matching our
// chrony_offset_seconds field. L (f[7]) is the leap/sync flag: N/+/- = synchronised,
// '?' = not synchronised. Header/banner lines (first field not a digit) are skipped;
// an unsynchronised reference logs 0.0.0.0, reported as an empty source.
// BENCH-VERIFY the offset sign against `chronyc tracking` on a synced host.
func parseTrackingLog(data []byte) chrony {
	lines := strings.Split(string(data), "\n")
	for i := len(lines) - 1; i >= 0; i-- {
		f := strings.Fields(lines[i])
		if len(f) < 8 || f[0] == "" || f[0][0] < '0' || f[0][0] > '9' {
			continue // blank, header ("Date ..."), or banner ("=====")
		}
		c := chrony{state: "unsynced"}
		if f[7] != "?" {
			c.state = "synced"
		}
		if f[2] != "0.0.0.0" {
			c.source = f[2]
		}
		if n, err := strconv.Atoi(f[3]); err == nil {
			c.stratum = int32(n)
		}
		if v, err := strconv.ParseFloat(f[6], 64); err == nil {
			c.offset = v
		}
		return c
	}
	return chrony{}
}

// unitActive reports whether a systemd unit is active (best-effort).
func unitActive(unit string) bool {
	out, err := runShort("systemctl", "is-active", unit)
	return err == nil && strings.TrimSpace(out) == "active"
}

// rootfsDigest best-effort: the dm-verity root hash from the kernel cmdline (the
// content-address of the running root). Empty if not present in this layout.
func rootfsDigest() string {
	b, err := os.ReadFile("/proc/cmdline")
	if err != nil {
		return ""
	}
	for tok := range strings.FieldsSeq(string(b)) {
		for _, k := range []string{"systemd.verity_root_hash=", "roothash="} {
			if after, ok := strings.CutPrefix(tok, k); ok {
				return after
			}
		}
	}
	return ""
}

// runShort runs a command with a short timeout and returns its stdout. Used only
// for the best-effort chrony/systemd probes; a missing binary is a soft failure.
func runShort(name string, args ...string) (string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
	defer cancel()
	out, err := exec.CommandContext(ctx, name, args...).Output()
	return string(out), err
}