sunjizzle/diskless-workstation
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fleet-manager: host uptime metric + reboot-pending status

Browse Source 
The manager lets an admin schedule a reboot (desired.reboot_at) but never showed
whether it had happened. Add host uptime to the health snapshot and derive a
reboot status from it.

- proto: HealthSnapshot.uptime_seconds (15); regenerated pb.go (manager + agent copy).
- agent: probe /proc/uptime -> uptime_seconds (with parser + test).
- manager: Health.UptimeSeconds cached in inventory; snapshotToHealth maps it;
  influx writes + whitelists uptime_seconds as a queryable field.
- UI: fmt.js dur() + rebootStatus() — boot_time = last_seen - uptime (manager clock
  frame, no host/manager sync needed), compared to reboot_at. Monitor host detail
  shows an "uptime" row and a "reboot" row: pending in X / overdue by X / rebooted.

Observability-only — works before the reboot ACTUATOR lands (deferred). Once a host
boots after the scheduled time (BMC/manual now, agent later), the badge flips to
"rebooted" automatically.
This commit is contained in:
Audiobuzz
2026-07-10 06:41:09 +00:00
parent e6ae8c4b23
commit df7dad81b7
11 changed files with 128 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
files/agent/proto/fleet_agent.pb.go
+12 -3
View File
@@ -274,9 +274,10 @@ type HealthSnapshot struct {
Selinux string `protobuf:"bytes,9,opt,name=selinux,proto3" json:"selinux,omitempty"` // enforcing|permissive
Chrony string `protobuf:"bytes,10,opt,name=chrony,proto3" json:"chrony,omitempty"` // synced|unsynced
KioskActive bool `protobuf:"varint,11,opt,name=kiosk_active,json=kioskActive,proto3" json:"kiosk_active,omitempty"`
SessionUptime string `protobuf:"bytes,12,opt,name=session_uptime,json=sessionUptime,proto3" json:"session_uptime,omitempty"`
SessionUptime string `protobuf:"bytes,12,opt,name=session_uptime,json=sessionUptime,proto3" json:"session_uptime,omitempty"` // KIOSK session uptime (advisory string)
ChromiumCrashes int32 `protobuf:"varint,13,opt,name=chromium_crashes,json=chromiumCrashes,proto3" json:"chromium_crashes,omitempty"`
Display string `protobuf:"bytes,14,opt,name=display,proto3" json:"display,omitempty"`
UptimeSeconds int64 `protobuf:"varint,15,opt,name=uptime_seconds,json=uptimeSeconds,proto3" json:"uptime_seconds,omitempty"` // HOST uptime (seconds since boot) — boot_time =
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -409,6 +410,13 @@ func (x *HealthSnapshot) GetDisplay() string {
return ""
}
func (x *HealthSnapshot) GetUptimeSeconds() int64 {
if x != nil {
return x.UptimeSeconds
}
return 0
}
type CommandResult struct {
state protoimpl.MessageState `protogen:"open.v1"`
CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`
@@ -688,7 +696,7 @@ const file_proto_fleet_agent_proto_rawDesc = "" +
"\rrootfs_digest\x18\x02 \x01(\tR\frootfsDigest\x12\x16\n" +
"\x06kernel\x18\x03 \x01(\tR\x06kernel\x12\x1b\n" +
"\tkiosk_url\x18\x04 \x01(\tR\bkioskUrl\x12#\n" +
"\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xae\x03\n" +
"\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xd5\x03\n" +
"\x0eHealthSnapshot\x12\x17\n" +
"\aat_unix\x18\x01 \x01(\x03R\x06atUnix\x12\x19\n" +
"\bload_avg\x18\x02 \x01(\x01R\aloadAvg\x12\x17\n" +
@@ -705,7 +713,8 @@ const file_proto_fleet_agent_proto_rawDesc = "" +
"\fkiosk_active\x18\v \x01(\bR\vkioskActive\x12%\n" +
"\x0esession_uptime\x18\f \x01(\tR\rsessionUptime\x12)\n" +
"\x10chromium_crashes\x18\r \x01(\x05R\x0fchromiumCrashes\x12\x18\n" +
"\adisplay\x18\x0e \x01(\tR\adisplay\"V\n" +
"\adisplay\x18\x0e \x01(\tR\adisplay\x12%\n" +
"\x0euptime_seconds\x18\x0f \x01(\x03R\ruptimeSeconds\"V\n" +
"\rCommandResult\x12\x1d\n" +
"\n" +
"command_id\x18\x01 \x01(\tR\tcommandId\x12\x0e\n" +
files/agent/proto/fleet_agent.proto
+3 -1
View File
@@ -54,9 +54,11 @@ message HealthSnapshot {
string selinux = 9; // enforcing|permissive
string chrony = 10; // synced|unsynced
bool kiosk_active = 11;
string session_uptime = 12;
string session_uptime = 12; // KIOSK session uptime (advisory string)
int32 chromium_crashes = 13;
string display = 14;
int64 uptime_seconds = 15; // HOST uptime (seconds since boot) — boot_time =
// manager receive-time - this; drives reboot-pending
}
message CommandResult {
files/agent/snapshot.go
+24
View File
@@ -98,10 +98,34 @@ func (p *prober) snapshot(ctx context.Context) *fleetpb.HealthSnapshot {
Chrony: chronyState(),
KioskActive: unitActive("kiosk.service"),
SessionUptime: f.Kiosk.Uptime,
UptimeSeconds: uptimeSeconds(),
}
return s
}
// uptimeSeconds returns host uptime (seconds since boot) from /proc/uptime. The
// manager derives boot_time = receive-time - this to tell whether a scheduled
// reboot has been actioned yet.
func uptimeSeconds() int64 {
b, err := os.ReadFile("/proc/uptime")
if err != nil {
return 0
}
return parseUptime(b)
}
func parseUptime(b []byte) int64 {
fields := strings.Fields(string(b))
if len(fields) == 0 {
return 0
}
f, _ := strconv.ParseFloat(fields[0], 64)
if f < 0 {
return 0
}
return int64(f)
}
// ---- individual probes (all best-effort) ----------------------------------
func loadAvg() float64 {
files/agent/snapshot_test.go
+12
View File
@@ -56,6 +56,18 @@ func TestParseMemUsedPct(t *testing.T) {
}
}
func TestParseUptime(t *testing.T) {
if got := parseUptime([]byte("12345.67 98765.43\n")); got != 12345 {
t.Fatalf("uptime = %d, want 12345", got)
}
if got := parseUptime([]byte("")); got != 0 {
t.Fatalf("empty uptime = %d, want 0", got)
}
if got := parseUptime([]byte("garbage")); got != 0 {
t.Fatalf("garbage uptime = %d, want 0", got)
}
}
func TestParseHandle(t *testing.T) {
for _, tc := range []struct {
in string
serving/fleet-manager/agent.go
+11 -10
View File
@@ -222,16 +222,17 @@ func desiredMsg(d Desired) *fleetpb.ManagerDown {
func snapshotToHealth(s *fleetpb.HealthSnapshot) Health {
return Health{
LoadAvg: s.GetLoadAvg(),
CPUPct: s.GetCpuPct(),
TempC: s.GetTempC(),
MemUsedPct: s.GetMemUsedPct(),
TmpfsUsedPct: s.GetTmpfsUsedPct(),
NVMeRoot: s.GetNvmeRoot(),
Auth: s.GetAuth(),
SELinux: s.GetSelinux(),
Chrony: s.GetChrony(),
KioskActive: s.GetKioskActive(),
LoadAvg: s.GetLoadAvg(),
CPUPct: s.GetCpuPct(),
TempC: s.GetTempC(),
MemUsedPct: s.GetMemUsedPct(),
TmpfsUsedPct: s.GetTmpfsUsedPct(),
NVMeRoot: s.GetNvmeRoot(),
Auth: s.GetAuth(),
SELinux: s.GetSelinux(),
Chrony: s.GetChrony(),
KioskActive: s.GetKioskActive(),
UptimeSeconds: s.GetUptimeSeconds(),
}
}
serving/fleet-manager/influx.go
+2
View File
@@ -26,6 +26,7 @@ const influxMeasurement = "fleet_health"
var metricFields = map[string]bool{
"load_avg": true, "cpu_pct": true, "temp_c": true, "mem_used_pct": true,
"tmpfs_used_pct": true, "chromium_crashes": true, "kiosk_active": true,
"uptime_seconds": true,
}
var serialRe = regexp.MustCompile(`^[0-9a-fA-F]+$`) // cert serials are hex
@@ -80,6 +81,7 @@ func snapshotPoint(serial, hostnqn, starship string, snap *fleetpb.HealthSnapsho
"tmpfs_used_pct": snap.GetTmpfsUsedPct(),
"chromium_crashes": int64(snap.GetChromiumCrashes()),
"kiosk_active": boolToInt(snap.GetKioskActive()),
"uptime_seconds": snap.GetUptimeSeconds(),
// string status fields kept as fields too (handy for annotating charts)
"nvme_root": snap.GetNvmeRoot(),
"auth": snap.GetAuth(),
serving/fleet-manager/inventory.go
+11 -10
View File
@@ -49,16 +49,17 @@ type Version struct {
// (written by the manager as agent reports arrive); this cached last value is
// what the fleet-overview list renders without a per-host Flux query.
type Health struct {
LoadAvg float64 `json:"load_avg,omitempty"`
CPUPct float64 `json:"cpu_pct,omitempty"`
TempC float64 `json:"temp_c,omitempty"`
MemUsedPct float64 `json:"mem_used_pct,omitempty"`
TmpfsUsedPct float64 `json:"tmpfs_used_pct,omitempty"` // overlay RAM pressure (ties to #17)
NVMeRoot string `json:"nvme_root,omitempty"` // ok|degraded|down
Auth string `json:"auth,omitempty"` // 802.1x/macsec: up|down
SELinux string `json:"selinux,omitempty"` // enforcing|permissive
Chrony string `json:"chrony,omitempty"` // synced|unsynced
KioskActive bool `json:"kiosk_active"`
LoadAvg float64 `json:"load_avg,omitempty"`
CPUPct float64 `json:"cpu_pct,omitempty"`
TempC float64 `json:"temp_c,omitempty"`
MemUsedPct float64 `json:"mem_used_pct,omitempty"`
TmpfsUsedPct float64 `json:"tmpfs_used_pct,omitempty"` // overlay RAM pressure (ties to #17)
NVMeRoot string `json:"nvme_root,omitempty"` // ok|degraded|down
Auth string `json:"auth,omitempty"` // 802.1x/macsec: up|down
SELinux string `json:"selinux,omitempty"` // enforcing|permissive
Chrony string `json:"chrony,omitempty"` // synced|unsynced
KioskActive bool `json:"kiosk_active"`
UptimeSeconds int64 `json:"uptime_seconds,omitempty"` // host uptime at last report; boot_time = last_seen - this (reboot-pending)
}
// Usage — kiosk-session facts (agent-reported).
serving/fleet-manager/proto/fleet_agent.pb.go
+12 -3
View File
@@ -274,9 +274,10 @@ type HealthSnapshot struct {
Selinux string `protobuf:"bytes,9,opt,name=selinux,proto3" json:"selinux,omitempty"` // enforcing|permissive
Chrony string `protobuf:"bytes,10,opt,name=chrony,proto3" json:"chrony,omitempty"` // synced|unsynced
KioskActive bool `protobuf:"varint,11,opt,name=kiosk_active,json=kioskActive,proto3" json:"kiosk_active,omitempty"`
SessionUptime string `protobuf:"bytes,12,opt,name=session_uptime,json=sessionUptime,proto3" json:"session_uptime,omitempty"`
SessionUptime string `protobuf:"bytes,12,opt,name=session_uptime,json=sessionUptime,proto3" json:"session_uptime,omitempty"` // KIOSK session uptime (advisory string)
ChromiumCrashes int32 `protobuf:"varint,13,opt,name=chromium_crashes,json=chromiumCrashes,proto3" json:"chromium_crashes,omitempty"`
Display string `protobuf:"bytes,14,opt,name=display,proto3" json:"display,omitempty"`
UptimeSeconds int64 `protobuf:"varint,15,opt,name=uptime_seconds,json=uptimeSeconds,proto3" json:"uptime_seconds,omitempty"` // HOST uptime (seconds since boot) — boot_time =
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -409,6 +410,13 @@ func (x *HealthSnapshot) GetDisplay() string {
return ""
}
func (x *HealthSnapshot) GetUptimeSeconds() int64 {
if x != nil {
return x.UptimeSeconds
}
return 0
}
type CommandResult struct {
state protoimpl.MessageState `protogen:"open.v1"`
CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`
@@ -688,7 +696,7 @@ const file_proto_fleet_agent_proto_rawDesc = "" +
"\rrootfs_digest\x18\x02 \x01(\tR\frootfsDigest\x12\x16\n" +
"\x06kernel\x18\x03 \x01(\tR\x06kernel\x12\x1b\n" +
"\tkiosk_url\x18\x04 \x01(\tR\bkioskUrl\x12#\n" +
"\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xae\x03\n" +
"\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xd5\x03\n" +
"\x0eHealthSnapshot\x12\x17\n" +
"\aat_unix\x18\x01 \x01(\x03R\x06atUnix\x12\x19\n" +
"\bload_avg\x18\x02 \x01(\x01R\aloadAvg\x12\x17\n" +
@@ -705,7 +713,8 @@ const file_proto_fleet_agent_proto_rawDesc = "" +
"\fkiosk_active\x18\v \x01(\bR\vkioskActive\x12%\n" +
"\x0esession_uptime\x18\f \x01(\tR\rsessionUptime\x12)\n" +
"\x10chromium_crashes\x18\r \x01(\x05R\x0fchromiumCrashes\x12\x18\n" +
"\adisplay\x18\x0e \x01(\tR\adisplay\"V\n" +
"\adisplay\x18\x0e \x01(\tR\adisplay\x12%\n" +
"\x0euptime_seconds\x18\x0f \x01(\x03R\ruptimeSeconds\"V\n" +
"\rCommandResult\x12\x1d\n" +
"\n" +
"command_id\x18\x01 \x01(\tR\tcommandId\x12\x0e\n" +
serving/fleet-manager/proto/fleet_agent.proto
+3 -1
View File
@@ -54,9 +54,11 @@ message HealthSnapshot {
string selinux = 9; // enforcing|permissive
string chrony = 10; // synced|unsynced
bool kiosk_active = 11;
string session_uptime = 12;
string session_uptime = 12; // KIOSK session uptime (advisory string)
int32 chromium_crashes = 13;
string display = 14;
int64 uptime_seconds = 15; // HOST uptime (seconds since boot) — boot_time =
// manager receive-time - this; drives reboot-pending
}
message CommandResult {
serving/fleet-manager/ui/src/lib/fmt.js
+32
View File
@@ -35,3 +35,35 @@ export function healthState(h) {
export function shortSerial(s) {
return s ? s.slice(0, 8) : '—'
}
// dur formats a span of SECONDS as a compact "3d 4h" / "5h 12m" / "45s".
export function dur(s) {
s = Math.max(0, Math.floor(s || 0))
const d = Math.floor(s / 86400)
const h = Math.floor((s % 86400) / 3600)
const m = Math.floor((s % 3600) / 60)
const sec = s % 60
if (d) return `${d}d ${h}h`
if (h) return `${h}h ${m}m`
if (m) return `${m}m ${sec}s`
return `${sec}s`
}
// rebootStatus reports whether a scheduled reboot (desired.reboot_at) has been
// actioned, by comparing the host's boot time to the requested time. Boot time is
// derived in the manager's clock frame: last_seen (a manager-stamped snapshot
// time) minus the reported host uptime — so it needs no host/manager clock sync.
// Returns null when no reboot is scheduled. state ∈ pending | overdue | done.
export function rebootStatus(host) {
const raIso = host?.desired?.reboot_at
if (!raIso) return null
const ra = new Date(raIso).getTime()
if (Number.isNaN(ra) || ra <= 0) return null // unset / Go zero time
const now = Date.now()
const up = host?.health?.uptime_seconds || 0
const ls = host?.last_seen ? new Date(host.last_seen).getTime() : NaN
const boot = up > 0 && ls > 0 ? ls - up * 1000 : NaN // manager-frame boot time
if (!Number.isNaN(boot) && boot >= ra) return { state: 'done', label: 'rebooted' }
if (ra > now) return { state: 'pending', label: 'reboot pending in ' + dur((ra - now) / 1000) }
return { state: 'overdue', label: 'reboot overdue by ' + dur((now - ra) / 1000) }
}
serving/fleet-manager/ui/src/pages/Monitor.svelte
+6 -1
View File
@@ -3,7 +3,7 @@
import Panel from '../components/Panel.svelte'
import StatePill from '../components/StatePill.svelte'
import Chart from '../components/Chart.svelte'
import { relTime, pct, num, healthState, shortSerial } from '../lib/fmt.js'
import { relTime, pct, num, healthState, shortSerial, dur, rebootStatus } from '../lib/fmt.js'
let hosts = $state([])
let error = $state('')
@@ -105,6 +105,11 @@
<dt class="text-faint">selinux</dt><dd class="text-muted">{selected.health?.selinux || '—'}</dd>
<dt class="text-faint">chrony</dt><dd class="text-muted">{selected.health?.chrony || '—'}</dd>
<dt class="text-faint">kiosk</dt><dd class="text-muted">{selected.health?.kiosk_active ? 'active' : 'idle'} · {selected.usage?.session_uptime || '—'}</dd>
<dt class="text-faint">uptime</dt><dd class="text-muted">{selected.health?.uptime_seconds ? dur(selected.health.uptime_seconds) : '—'}</dd>
{#if rebootStatus(selected)}
{@const rb = rebootStatus(selected)}
<dt class="text-faint">reboot</dt><dd class={rb.state === 'overdue' ? 'text-amber-hot' : rb.state === 'pending' ? 'text-amber' : 'text-teal'}>{rb.label}</dd>
{/if}
<dt class="text-faint">last seen</dt><dd class="text-muted">{selected.online ? 'live' : relTime(selected.last_seen)}</dd>
</dl>
<div class="mt-4 space-y-3">