From df7dad81b791fe6304f3ca0d321a9fc31198e0ef Mon Sep 17 00:00:00 2001 From: Audiobuzz Date: Fri, 10 Jul 2026 06:41:09 +0000 Subject: [PATCH] fleet-manager: host uptime metric + reboot-pending status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manager lets an admin schedule a reboot (desired.reboot_at) but never showed whether it had happened. Add host uptime to the health snapshot and derive a reboot status from it. - proto: HealthSnapshot.uptime_seconds (15); regenerated pb.go (manager + agent copy). - agent: probe /proc/uptime -> uptime_seconds (with parser + test). - manager: Health.UptimeSeconds cached in inventory; snapshotToHealth maps it; influx writes + whitelists uptime_seconds as a queryable field. - UI: fmt.js dur() + rebootStatus() — boot_time = last_seen - uptime (manager clock frame, no host/manager sync needed), compared to reboot_at. Monitor host detail shows an "uptime" row and a "reboot" row: pending in X / overdue by X / rebooted. Observability-only — works before the reboot ACTUATOR lands (deferred). Once a host boots after the scheduled time (BMC/manual now, agent later), the badge flips to "rebooted" automatically. --- files/agent/proto/fleet_agent.pb.go | 15 +++++++-- files/agent/proto/fleet_agent.proto | 4 ++- files/agent/snapshot.go | 24 ++++++++++++++ files/agent/snapshot_test.go | 12 +++++++ serving/fleet-manager/agent.go | 21 ++++++------ serving/fleet-manager/influx.go | 2 ++ serving/fleet-manager/inventory.go | 21 ++++++------ serving/fleet-manager/proto/fleet_agent.pb.go | 15 +++++++-- serving/fleet-manager/proto/fleet_agent.proto | 4 ++- serving/fleet-manager/ui/src/lib/fmt.js | 32 +++++++++++++++++++ .../fleet-manager/ui/src/pages/Monitor.svelte | 7 +++- 11 files changed, 128 insertions(+), 29 deletions(-) diff --git a/files/agent/proto/fleet_agent.pb.go b/files/agent/proto/fleet_agent.pb.go index 2da81fb..11f2f3a 100644 --- a/files/agent/proto/fleet_agent.pb.go +++ b/files/agent/proto/fleet_agent.pb.go @@ -274,9 +274,10 @@ type HealthSnapshot struct { Selinux string `protobuf:"bytes,9,opt,name=selinux,proto3" json:"selinux,omitempty"` // enforcing|permissive Chrony string `protobuf:"bytes,10,opt,name=chrony,proto3" json:"chrony,omitempty"` // synced|unsynced KioskActive bool `protobuf:"varint,11,opt,name=kiosk_active,json=kioskActive,proto3" json:"kiosk_active,omitempty"` - SessionUptime string `protobuf:"bytes,12,opt,name=session_uptime,json=sessionUptime,proto3" json:"session_uptime,omitempty"` + SessionUptime string `protobuf:"bytes,12,opt,name=session_uptime,json=sessionUptime,proto3" json:"session_uptime,omitempty"` // KIOSK session uptime (advisory string) ChromiumCrashes int32 `protobuf:"varint,13,opt,name=chromium_crashes,json=chromiumCrashes,proto3" json:"chromium_crashes,omitempty"` Display string `protobuf:"bytes,14,opt,name=display,proto3" json:"display,omitempty"` + UptimeSeconds int64 `protobuf:"varint,15,opt,name=uptime_seconds,json=uptimeSeconds,proto3" json:"uptime_seconds,omitempty"` // HOST uptime (seconds since boot) — boot_time = unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -409,6 +410,13 @@ func (x *HealthSnapshot) GetDisplay() string { return "" } +func (x *HealthSnapshot) GetUptimeSeconds() int64 { + if x != nil { + return x.UptimeSeconds + } + return 0 +} + type CommandResult struct { state protoimpl.MessageState `protogen:"open.v1"` CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"` @@ -688,7 +696,7 @@ const file_proto_fleet_agent_proto_rawDesc = "" + "\rrootfs_digest\x18\x02 \x01(\tR\frootfsDigest\x12\x16\n" + "\x06kernel\x18\x03 \x01(\tR\x06kernel\x12\x1b\n" + "\tkiosk_url\x18\x04 \x01(\tR\bkioskUrl\x12#\n" + - "\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xae\x03\n" + + "\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xd5\x03\n" + "\x0eHealthSnapshot\x12\x17\n" + "\aat_unix\x18\x01 \x01(\x03R\x06atUnix\x12\x19\n" + "\bload_avg\x18\x02 \x01(\x01R\aloadAvg\x12\x17\n" + @@ -705,7 +713,8 @@ const file_proto_fleet_agent_proto_rawDesc = "" + "\fkiosk_active\x18\v \x01(\bR\vkioskActive\x12%\n" + "\x0esession_uptime\x18\f \x01(\tR\rsessionUptime\x12)\n" + "\x10chromium_crashes\x18\r \x01(\x05R\x0fchromiumCrashes\x12\x18\n" + - "\adisplay\x18\x0e \x01(\tR\adisplay\"V\n" + + "\adisplay\x18\x0e \x01(\tR\adisplay\x12%\n" + + "\x0euptime_seconds\x18\x0f \x01(\x03R\ruptimeSeconds\"V\n" + "\rCommandResult\x12\x1d\n" + "\n" + "command_id\x18\x01 \x01(\tR\tcommandId\x12\x0e\n" + diff --git a/files/agent/proto/fleet_agent.proto b/files/agent/proto/fleet_agent.proto index 80f230b..6479e68 100644 --- a/files/agent/proto/fleet_agent.proto +++ b/files/agent/proto/fleet_agent.proto @@ -54,9 +54,11 @@ message HealthSnapshot { string selinux = 9; // enforcing|permissive string chrony = 10; // synced|unsynced bool kiosk_active = 11; - string session_uptime = 12; + string session_uptime = 12; // KIOSK session uptime (advisory string) int32 chromium_crashes = 13; string display = 14; + int64 uptime_seconds = 15; // HOST uptime (seconds since boot) — boot_time = + // manager receive-time - this; drives reboot-pending } message CommandResult { diff --git a/files/agent/snapshot.go b/files/agent/snapshot.go index 6492e2f..efb226a 100644 --- a/files/agent/snapshot.go +++ b/files/agent/snapshot.go @@ -98,10 +98,34 @@ func (p *prober) snapshot(ctx context.Context) *fleetpb.HealthSnapshot { Chrony: chronyState(), KioskActive: unitActive("kiosk.service"), SessionUptime: f.Kiosk.Uptime, + UptimeSeconds: uptimeSeconds(), } return s } +// uptimeSeconds returns host uptime (seconds since boot) from /proc/uptime. The +// manager derives boot_time = receive-time - this to tell whether a scheduled +// reboot has been actioned yet. +func uptimeSeconds() int64 { + b, err := os.ReadFile("/proc/uptime") + if err != nil { + return 0 + } + return parseUptime(b) +} + +func parseUptime(b []byte) int64 { + fields := strings.Fields(string(b)) + if len(fields) == 0 { + return 0 + } + f, _ := strconv.ParseFloat(fields[0], 64) + if f < 0 { + return 0 + } + return int64(f) +} + // ---- individual probes (all best-effort) ---------------------------------- func loadAvg() float64 { diff --git a/files/agent/snapshot_test.go b/files/agent/snapshot_test.go index 0c3b773..341e0d5 100644 --- a/files/agent/snapshot_test.go +++ b/files/agent/snapshot_test.go @@ -56,6 +56,18 @@ func TestParseMemUsedPct(t *testing.T) { } } +func TestParseUptime(t *testing.T) { + if got := parseUptime([]byte("12345.67 98765.43\n")); got != 12345 { + t.Fatalf("uptime = %d, want 12345", got) + } + if got := parseUptime([]byte("")); got != 0 { + t.Fatalf("empty uptime = %d, want 0", got) + } + if got := parseUptime([]byte("garbage")); got != 0 { + t.Fatalf("garbage uptime = %d, want 0", got) + } +} + func TestParseHandle(t *testing.T) { for _, tc := range []struct { in string diff --git a/serving/fleet-manager/agent.go b/serving/fleet-manager/agent.go index b4975b4..31d43eb 100644 --- a/serving/fleet-manager/agent.go +++ b/serving/fleet-manager/agent.go @@ -222,16 +222,17 @@ func desiredMsg(d Desired) *fleetpb.ManagerDown { func snapshotToHealth(s *fleetpb.HealthSnapshot) Health { return Health{ - LoadAvg: s.GetLoadAvg(), - CPUPct: s.GetCpuPct(), - TempC: s.GetTempC(), - MemUsedPct: s.GetMemUsedPct(), - TmpfsUsedPct: s.GetTmpfsUsedPct(), - NVMeRoot: s.GetNvmeRoot(), - Auth: s.GetAuth(), - SELinux: s.GetSelinux(), - Chrony: s.GetChrony(), - KioskActive: s.GetKioskActive(), + LoadAvg: s.GetLoadAvg(), + CPUPct: s.GetCpuPct(), + TempC: s.GetTempC(), + MemUsedPct: s.GetMemUsedPct(), + TmpfsUsedPct: s.GetTmpfsUsedPct(), + NVMeRoot: s.GetNvmeRoot(), + Auth: s.GetAuth(), + SELinux: s.GetSelinux(), + Chrony: s.GetChrony(), + KioskActive: s.GetKioskActive(), + UptimeSeconds: s.GetUptimeSeconds(), } } diff --git a/serving/fleet-manager/influx.go b/serving/fleet-manager/influx.go index 1ee09e9..16332d9 100644 --- a/serving/fleet-manager/influx.go +++ b/serving/fleet-manager/influx.go @@ -26,6 +26,7 @@ const influxMeasurement = "fleet_health" var metricFields = map[string]bool{ "load_avg": true, "cpu_pct": true, "temp_c": true, "mem_used_pct": true, "tmpfs_used_pct": true, "chromium_crashes": true, "kiosk_active": true, + "uptime_seconds": true, } var serialRe = regexp.MustCompile(`^[0-9a-fA-F]+$`) // cert serials are hex @@ -80,6 +81,7 @@ func snapshotPoint(serial, hostnqn, starship string, snap *fleetpb.HealthSnapsho "tmpfs_used_pct": snap.GetTmpfsUsedPct(), "chromium_crashes": int64(snap.GetChromiumCrashes()), "kiosk_active": boolToInt(snap.GetKioskActive()), + "uptime_seconds": snap.GetUptimeSeconds(), // string status fields kept as fields too (handy for annotating charts) "nvme_root": snap.GetNvmeRoot(), "auth": snap.GetAuth(), diff --git a/serving/fleet-manager/inventory.go b/serving/fleet-manager/inventory.go index c39021c..1d9a8a4 100644 --- a/serving/fleet-manager/inventory.go +++ b/serving/fleet-manager/inventory.go @@ -49,16 +49,17 @@ type Version struct { // (written by the manager as agent reports arrive); this cached last value is // what the fleet-overview list renders without a per-host Flux query. type Health struct { - LoadAvg float64 `json:"load_avg,omitempty"` - CPUPct float64 `json:"cpu_pct,omitempty"` - TempC float64 `json:"temp_c,omitempty"` - MemUsedPct float64 `json:"mem_used_pct,omitempty"` - TmpfsUsedPct float64 `json:"tmpfs_used_pct,omitempty"` // overlay RAM pressure (ties to #17) - NVMeRoot string `json:"nvme_root,omitempty"` // ok|degraded|down - Auth string `json:"auth,omitempty"` // 802.1x/macsec: up|down - SELinux string `json:"selinux,omitempty"` // enforcing|permissive - Chrony string `json:"chrony,omitempty"` // synced|unsynced - KioskActive bool `json:"kiosk_active"` + LoadAvg float64 `json:"load_avg,omitempty"` + CPUPct float64 `json:"cpu_pct,omitempty"` + TempC float64 `json:"temp_c,omitempty"` + MemUsedPct float64 `json:"mem_used_pct,omitempty"` + TmpfsUsedPct float64 `json:"tmpfs_used_pct,omitempty"` // overlay RAM pressure (ties to #17) + NVMeRoot string `json:"nvme_root,omitempty"` // ok|degraded|down + Auth string `json:"auth,omitempty"` // 802.1x/macsec: up|down + SELinux string `json:"selinux,omitempty"` // enforcing|permissive + Chrony string `json:"chrony,omitempty"` // synced|unsynced + KioskActive bool `json:"kiosk_active"` + UptimeSeconds int64 `json:"uptime_seconds,omitempty"` // host uptime at last report; boot_time = last_seen - this (reboot-pending) } // Usage — kiosk-session facts (agent-reported). diff --git a/serving/fleet-manager/proto/fleet_agent.pb.go b/serving/fleet-manager/proto/fleet_agent.pb.go index 2da81fb..11f2f3a 100644 --- a/serving/fleet-manager/proto/fleet_agent.pb.go +++ b/serving/fleet-manager/proto/fleet_agent.pb.go @@ -274,9 +274,10 @@ type HealthSnapshot struct { Selinux string `protobuf:"bytes,9,opt,name=selinux,proto3" json:"selinux,omitempty"` // enforcing|permissive Chrony string `protobuf:"bytes,10,opt,name=chrony,proto3" json:"chrony,omitempty"` // synced|unsynced KioskActive bool `protobuf:"varint,11,opt,name=kiosk_active,json=kioskActive,proto3" json:"kiosk_active,omitempty"` - SessionUptime string `protobuf:"bytes,12,opt,name=session_uptime,json=sessionUptime,proto3" json:"session_uptime,omitempty"` + SessionUptime string `protobuf:"bytes,12,opt,name=session_uptime,json=sessionUptime,proto3" json:"session_uptime,omitempty"` // KIOSK session uptime (advisory string) ChromiumCrashes int32 `protobuf:"varint,13,opt,name=chromium_crashes,json=chromiumCrashes,proto3" json:"chromium_crashes,omitempty"` Display string `protobuf:"bytes,14,opt,name=display,proto3" json:"display,omitempty"` + UptimeSeconds int64 `protobuf:"varint,15,opt,name=uptime_seconds,json=uptimeSeconds,proto3" json:"uptime_seconds,omitempty"` // HOST uptime (seconds since boot) — boot_time = unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -409,6 +410,13 @@ func (x *HealthSnapshot) GetDisplay() string { return "" } +func (x *HealthSnapshot) GetUptimeSeconds() int64 { + if x != nil { + return x.UptimeSeconds + } + return 0 +} + type CommandResult struct { state protoimpl.MessageState `protogen:"open.v1"` CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"` @@ -688,7 +696,7 @@ const file_proto_fleet_agent_proto_rawDesc = "" + "\rrootfs_digest\x18\x02 \x01(\tR\frootfsDigest\x12\x16\n" + "\x06kernel\x18\x03 \x01(\tR\x06kernel\x12\x1b\n" + "\tkiosk_url\x18\x04 \x01(\tR\bkioskUrl\x12#\n" + - "\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xae\x03\n" + + "\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xd5\x03\n" + "\x0eHealthSnapshot\x12\x17\n" + "\aat_unix\x18\x01 \x01(\x03R\x06atUnix\x12\x19\n" + "\bload_avg\x18\x02 \x01(\x01R\aloadAvg\x12\x17\n" + @@ -705,7 +713,8 @@ const file_proto_fleet_agent_proto_rawDesc = "" + "\fkiosk_active\x18\v \x01(\bR\vkioskActive\x12%\n" + "\x0esession_uptime\x18\f \x01(\tR\rsessionUptime\x12)\n" + "\x10chromium_crashes\x18\r \x01(\x05R\x0fchromiumCrashes\x12\x18\n" + - "\adisplay\x18\x0e \x01(\tR\adisplay\"V\n" + + "\adisplay\x18\x0e \x01(\tR\adisplay\x12%\n" + + "\x0euptime_seconds\x18\x0f \x01(\x03R\ruptimeSeconds\"V\n" + "\rCommandResult\x12\x1d\n" + "\n" + "command_id\x18\x01 \x01(\tR\tcommandId\x12\x0e\n" + diff --git a/serving/fleet-manager/proto/fleet_agent.proto b/serving/fleet-manager/proto/fleet_agent.proto index 80f230b..6479e68 100644 --- a/serving/fleet-manager/proto/fleet_agent.proto +++ b/serving/fleet-manager/proto/fleet_agent.proto @@ -54,9 +54,11 @@ message HealthSnapshot { string selinux = 9; // enforcing|permissive string chrony = 10; // synced|unsynced bool kiosk_active = 11; - string session_uptime = 12; + string session_uptime = 12; // KIOSK session uptime (advisory string) int32 chromium_crashes = 13; string display = 14; + int64 uptime_seconds = 15; // HOST uptime (seconds since boot) — boot_time = + // manager receive-time - this; drives reboot-pending } message CommandResult { diff --git a/serving/fleet-manager/ui/src/lib/fmt.js b/serving/fleet-manager/ui/src/lib/fmt.js index 7e5a342..92d115e 100644 --- a/serving/fleet-manager/ui/src/lib/fmt.js +++ b/serving/fleet-manager/ui/src/lib/fmt.js @@ -35,3 +35,35 @@ export function healthState(h) { export function shortSerial(s) { return s ? s.slice(0, 8) : '—' } + +// dur formats a span of SECONDS as a compact "3d 4h" / "5h 12m" / "45s". +export function dur(s) { + s = Math.max(0, Math.floor(s || 0)) + const d = Math.floor(s / 86400) + const h = Math.floor((s % 86400) / 3600) + const m = Math.floor((s % 3600) / 60) + const sec = s % 60 + if (d) return `${d}d ${h}h` + if (h) return `${h}h ${m}m` + if (m) return `${m}m ${sec}s` + return `${sec}s` +} + +// rebootStatus reports whether a scheduled reboot (desired.reboot_at) has been +// actioned, by comparing the host's boot time to the requested time. Boot time is +// derived in the manager's clock frame: last_seen (a manager-stamped snapshot +// time) minus the reported host uptime — so it needs no host/manager clock sync. +// Returns null when no reboot is scheduled. state ∈ pending | overdue | done. +export function rebootStatus(host) { + const raIso = host?.desired?.reboot_at + if (!raIso) return null + const ra = new Date(raIso).getTime() + if (Number.isNaN(ra) || ra <= 0) return null // unset / Go zero time + const now = Date.now() + const up = host?.health?.uptime_seconds || 0 + const ls = host?.last_seen ? new Date(host.last_seen).getTime() : NaN + const boot = up > 0 && ls > 0 ? ls - up * 1000 : NaN // manager-frame boot time + if (!Number.isNaN(boot) && boot >= ra) return { state: 'done', label: 'rebooted' } + if (ra > now) return { state: 'pending', label: 'reboot pending in ' + dur((ra - now) / 1000) } + return { state: 'overdue', label: 'reboot overdue by ' + dur((now - ra) / 1000) } +} diff --git a/serving/fleet-manager/ui/src/pages/Monitor.svelte b/serving/fleet-manager/ui/src/pages/Monitor.svelte index be7d2cb..0b59fd3 100644 --- a/serving/fleet-manager/ui/src/pages/Monitor.svelte +++ b/serving/fleet-manager/ui/src/pages/Monitor.svelte @@ -3,7 +3,7 @@ import Panel from '../components/Panel.svelte' import StatePill from '../components/StatePill.svelte' import Chart from '../components/Chart.svelte' - import { relTime, pct, num, healthState, shortSerial } from '../lib/fmt.js' + import { relTime, pct, num, healthState, shortSerial, dur, rebootStatus } from '../lib/fmt.js' let hosts = $state([]) let error = $state('') @@ -105,6 +105,11 @@
selinux
{selected.health?.selinux || '—'}
chrony
{selected.health?.chrony || '—'}
kiosk
{selected.health?.kiosk_active ? 'active' : 'idle'} · {selected.usage?.session_uptime || '—'}
+
uptime
{selected.health?.uptime_seconds ? dur(selected.health.uptime_seconds) : '—'}
+ {#if rebootStatus(selected)} + {@const rb = rebootStatus(selected)} +
reboot
{rb.label}
+ {/if}
last seen
{selected.online ? 'live' : relTime(selected.last_seen)}