sunjizzle/diskless-workstation
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fleet-manager: chrony tracking metrics relative to the NTP source

Browse Source 
The agent only reported chrony as synced|unsynced — nothing about how well the
clock tracks. Parse `chronyc tracking` for the source-relative metrics.

- proto: HealthSnapshot.chrony_offset_seconds (16, signed system-time offset,
  fast=+), chrony_source (17, reference name/IP), chrony_stratum (18); regenerated.
- agent: replace chronyState() with chronyTracking()/parseChrony() — one exec,
  parses Leap status + Reference ID + Stratum + System time offset (+ test).
- manager: Health.{ChronyOffset,ChronySource,ChronyStratum}; snapshotToHealth maps
  them; influx writes chrony_offset_seconds/_stratum/_source and whitelists the
  offset + stratum as graphable fields.
- UI: chronySummary() + offsetMs() — the chrony row now shows
  "synced · 192.168.50.10 · s3 · +0.01ms" instead of just "synced".
This commit is contained in:
Audiobuzz
2026-07-10 06:47:27 +00:00
parent df7dad81b7
commit 1c68c37824
11 changed files with 222 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
files/agent/proto/fleet_agent.pb.go
+33 -4
View File
@@ -278,8 +278,13 @@ type HealthSnapshot struct {
ChromiumCrashes int32 `protobuf:"varint,13,opt,name=chromium_crashes,json=chromiumCrashes,proto3" json:"chromium_crashes,omitempty"`
Display string `protobuf:"bytes,14,opt,name=display,proto3" json:"display,omitempty"`
UptimeSeconds int64 `protobuf:"varint,15,opt,name=uptime_seconds,json=uptimeSeconds,proto3" json:"uptime_seconds,omitempty"` // HOST uptime (seconds since boot) — boot_time =
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
// manager receive-time - this; drives reboot-pending
// chrony tracking relative to the NTP source (from `chronyc tracking`):
ChronyOffsetSeconds float64 `protobuf:"fixed64,16,opt,name=chrony_offset_seconds,json=chronyOffsetSeconds,proto3" json:"chrony_offset_seconds,omitempty"` // system time offset from NTP (signed, fast=+)
ChronySource string `protobuf:"bytes,17,opt,name=chrony_source,json=chronySource,proto3" json:"chrony_source,omitempty"` // reference source name/IP (which server)
ChronyStratum int32 `protobuf:"varint,18,opt,name=chrony_stratum,json=chronyStratum,proto3" json:"chrony_stratum,omitempty"` // hops from a reference clock (0 = unsynced)
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *HealthSnapshot) Reset() {
@@ -417,6 +422,27 @@ func (x *HealthSnapshot) GetUptimeSeconds() int64 {
return 0
}
func (x *HealthSnapshot) GetChronyOffsetSeconds() float64 {
if x != nil {
return x.ChronyOffsetSeconds
}
return 0
}
func (x *HealthSnapshot) GetChronySource() string {
if x != nil {
return x.ChronySource
}
return ""
}
func (x *HealthSnapshot) GetChronyStratum() int32 {
if x != nil {
return x.ChronyStratum
}
return 0
}
type CommandResult struct {
state protoimpl.MessageState `protogen:"open.v1"`
CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`
@@ -696,7 +722,7 @@ const file_proto_fleet_agent_proto_rawDesc = "" +
"\rrootfs_digest\x18\x02 \x01(\tR\frootfsDigest\x12\x16\n" +
"\x06kernel\x18\x03 \x01(\tR\x06kernel\x12\x1b\n" +
"\tkiosk_url\x18\x04 \x01(\tR\bkioskUrl\x12#\n" +
"\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xd5\x03\n" +
"\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xd5\x04\n" +
"\x0eHealthSnapshot\x12\x17\n" +
"\aat_unix\x18\x01 \x01(\x03R\x06atUnix\x12\x19\n" +
"\bload_avg\x18\x02 \x01(\x01R\aloadAvg\x12\x17\n" +
@@ -714,7 +740,10 @@ const file_proto_fleet_agent_proto_rawDesc = "" +
"\x0esession_uptime\x18\f \x01(\tR\rsessionUptime\x12)\n" +
"\x10chromium_crashes\x18\r \x01(\x05R\x0fchromiumCrashes\x12\x18\n" +
"\adisplay\x18\x0e \x01(\tR\adisplay\x12%\n" +
"\x0euptime_seconds\x18\x0f \x01(\x03R\ruptimeSeconds\"V\n" +
"\x0euptime_seconds\x18\x0f \x01(\x03R\ruptimeSeconds\x122\n" +
"\x15chrony_offset_seconds\x18\x10 \x01(\x01R\x13chronyOffsetSeconds\x12#\n" +
"\rchrony_source\x18\x11 \x01(\tR\fchronySource\x12%\n" +
"\x0echrony_stratum\x18\x12 \x01(\x05R\rchronyStratum\"V\n" +
"\rCommandResult\x12\x1d\n" +
"\n" +
"command_id\x18\x01 \x01(\tR\tcommandId\x12\x0e\n" +
files/agent/proto/fleet_agent.proto
+4
View File
@@ -59,6 +59,10 @@ message HealthSnapshot {
string display = 14;
int64 uptime_seconds = 15; // HOST uptime (seconds since boot) — boot_time =
// manager receive-time - this; drives reboot-pending
// chrony tracking relative to the NTP source (from `chronyc tracking`):
double chrony_offset_seconds = 16; // system time offset from NTP (signed, fast=+)
string chrony_source = 17; // reference source name/IP (which server)
int32 chrony_stratum = 18; // hops from a reference clock (0 = unsynced)
}
message CommandResult {
files/agent/snapshot.go
+69 -21
View File
@@ -86,19 +86,23 @@ type prober struct {
// receive-time (matches agent.go's snapshotToHealth path).
func (p *prober) snapshot(ctx context.Context) *fleetpb.HealthSnapshot {
f := fetchFacts(ctx, p.hostapiURL)
ch := chronyTracking()
s := &fleetpb.HealthSnapshot{
LoadAvg: loadAvg(),
CpuPct: p.cpuPct(),
TempC: tempC(),
MemUsedPct: memUsedPct(),
TmpfsUsedPct: fsUsedPct("/"),
NvmeRoot: nvmeRoot(),
Auth: authState(),
Selinux: selinuxState(),
Chrony: chronyState(),
KioskActive: unitActive("kiosk.service"),
SessionUptime: f.Kiosk.Uptime,
UptimeSeconds: uptimeSeconds(),
LoadAvg: loadAvg(),
CpuPct: p.cpuPct(),
TempC: tempC(),
MemUsedPct: memUsedPct(),
TmpfsUsedPct: fsUsedPct("/"),
NvmeRoot: nvmeRoot(),
Auth: authState(),
Selinux: selinuxState(),
Chrony: ch.state,
ChronyOffsetSeconds: ch.offset,
ChronySource: ch.source,
ChronyStratum: ch.stratum,
KioskActive: unitActive("kiosk.service"),
SessionUptime: f.Kiosk.Uptime,
UptimeSeconds: uptimeSeconds(),
}
return s
}
@@ -291,22 +295,66 @@ func selinuxState() string {
return "permissive"
}
// chronyState best-effort: "synced" when chronyc tracking reports a Normal leap
// status, "unsynced" otherwise, "" if chronyc is unavailable.
func chronyState() string {
// chrony holds the parsed `chronyc tracking` fields relative to the NTP source.
type chrony struct {
state string // synced|unsynced|"" (unavailable)
offset float64 // system time offset from NTP, seconds (signed: fast=+, slow=-)
source string // reference source name/IP (which server)
stratum int32 // hops from a reference clock
}
// chronyTracking runs `chronyc tracking` once and parses it. Best-effort: a zero
// chrony{} (state "") if chronyc is unavailable — never fails the snapshot.
func chronyTracking() chrony {
out, err := runShort("chronyc", "-n", "tracking")
if err != nil {
return ""
return chrony{}
}
return parseChrony(out)
}
// parseChrony extracts the source/offset/stratum/leap fields from `chronyc
// tracking` key:value output. Pure (unit-testable).
func parseChrony(out string) chrony {
c := chrony{state: "unsynced"}
for line := range strings.SplitSeq(out, "\n") {
if strings.HasPrefix(line, "Leap status") {
if strings.Contains(line, "Normal") {
return "synced"
key, val, ok := strings.Cut(line, ":")
if !ok {
continue
}
key = strings.TrimSpace(key)
val = strings.TrimSpace(val)
switch key {
case "Leap status":
if strings.Contains(val, "Normal") {
c.state = "synced"
}
case "Stratum":
if n, err := strconv.Atoi(strings.Fields(val)[0]); err == nil {
c.stratum = int32(n)
}
case "Reference ID":
// "C0A8320A (192.168.50.10)" — prefer the parenthesised name/IP.
c.source = val
if i := strings.IndexByte(val, '('); i >= 0 {
if j := strings.IndexByte(val[i:], ')'); j > 1 {
c.source = val[i+1 : i+j]
}
}
case "System time":
// "0.000012345 seconds slow of NTP time" — magnitude + direction.
fields := strings.Fields(val)
if len(fields) >= 3 {
if v, err := strconv.ParseFloat(fields[0], 64); err == nil {
if fields[2] == "slow" { // system behind NTP -> negative
v = -v
}
c.offset = v
}
}
return "unsynced"
}
}
return "unsynced"
return c
}
// unitActive reports whether a systemd unit is active (best-effort).
files/agent/snapshot_test.go
+33
View File
@@ -68,6 +68,39 @@ func TestParseUptime(t *testing.T) {
}
}
func TestParseChrony(t *testing.T) {
out := `Reference ID : C0A8320A (192.168.50.10)
Stratum : 3
Ref time (UTC) : Fri Jul 10 06:00:00 2026
System time : 0.000012345 seconds slow of NTP time
Last offset : +0.000001234 seconds
Frequency : 12.345 ppm slow
Leap status : Normal
`
c := parseChrony(out)
if c.state != "synced" {
t.Errorf("state = %q, want synced", c.state)
}
if c.source != "192.168.50.10" {
t.Errorf("source = %q, want 192.168.50.10", c.source)
}
if c.stratum != 3 {
t.Errorf("stratum = %d, want 3", c.stratum)
}
// "slow of NTP time" -> negative offset (sign-sensitive check).
if c.offset >= 0 || math.Abs(c.offset+0.000012345) > 1e-9 {
t.Errorf("offset = %v, want -0.000012345", c.offset)
}
// unsynced when Leap status isn't Normal.
if c := parseChrony("Leap status : Not synchronised\n"); c.state != "unsynced" {
t.Errorf("state = %q, want unsynced", c.state)
}
// empty / unavailable.
if c := parseChrony(""); c.state != "unsynced" || c.source != "" {
t.Errorf("empty parse = %+v", c)
}
}
func TestParseHandle(t *testing.T) {
for _, tc := range []struct {
in string
serving/fleet-manager/agent.go
+3
View File
@@ -231,6 +231,9 @@ func snapshotToHealth(s *fleetpb.HealthSnapshot) Health {
Auth: s.GetAuth(),
SELinux: s.GetSelinux(),
Chrony: s.GetChrony(),
ChronyOffset: s.GetChronyOffsetSeconds(),
ChronySource: s.GetChronySource(),
ChronyStratum: s.GetChronyStratum(),
KioskActive: s.GetKioskActive(),
UptimeSeconds: s.GetUptimeSeconds(),
}
serving/fleet-manager/influx.go
+12 -9
View File
@@ -26,7 +26,7 @@ const influxMeasurement = "fleet_health"
var metricFields = map[string]bool{
"load_avg": true, "cpu_pct": true, "temp_c": true, "mem_used_pct": true,
"tmpfs_used_pct": true, "chromium_crashes": true, "kiosk_active": true,
"uptime_seconds": true,
"uptime_seconds": true, "chrony_offset_seconds": true, "chrony_stratum": true,
}
var serialRe = regexp.MustCompile(`^[0-9a-fA-F]+$`) // cert serials are hex
@@ -74,14 +74,17 @@ func snapshotPoint(serial, hostnqn, starship string, snap *fleetpb.HealthSnapsho
tags["starship"] = starship
}
fields := map[string]any{
"load_avg": snap.GetLoadAvg(),
"cpu_pct": snap.GetCpuPct(),
"temp_c": snap.GetTempC(),
"mem_used_pct": snap.GetMemUsedPct(),
"tmpfs_used_pct": snap.GetTmpfsUsedPct(),
"chromium_crashes": int64(snap.GetChromiumCrashes()),
"kiosk_active": boolToInt(snap.GetKioskActive()),
"uptime_seconds": snap.GetUptimeSeconds(),
"load_avg": snap.GetLoadAvg(),
"cpu_pct": snap.GetCpuPct(),
"temp_c": snap.GetTempC(),
"mem_used_pct": snap.GetMemUsedPct(),
"tmpfs_used_pct": snap.GetTmpfsUsedPct(),
"chromium_crashes": int64(snap.GetChromiumCrashes()),
"kiosk_active": boolToInt(snap.GetKioskActive()),
"uptime_seconds": snap.GetUptimeSeconds(),
"chrony_offset_seconds": snap.GetChronyOffsetSeconds(),
"chrony_stratum": int64(snap.GetChronyStratum()),
"chrony_source": snap.GetChronySource(),
// string status fields kept as fields too (handy for annotating charts)
"nvme_root": snap.GetNvmeRoot(),
"auth": snap.GetAuth(),
serving/fleet-manager/inventory.go
+8 -5
View File
@@ -53,11 +53,14 @@ type Health struct {
CPUPct float64 `json:"cpu_pct,omitempty"`
TempC float64 `json:"temp_c,omitempty"`
MemUsedPct float64 `json:"mem_used_pct,omitempty"`
TmpfsUsedPct float64 `json:"tmpfs_used_pct,omitempty"` // overlay RAM pressure (ties to #17)
NVMeRoot string `json:"nvme_root,omitempty"` // ok|degraded|down
Auth string `json:"auth,omitempty"` // 802.1x/macsec: up|down
SELinux string `json:"selinux,omitempty"` // enforcing|permissive
Chrony string `json:"chrony,omitempty"` // synced|unsynced
TmpfsUsedPct float64 `json:"tmpfs_used_pct,omitempty"` // overlay RAM pressure (ties to #17)
NVMeRoot string `json:"nvme_root,omitempty"` // ok|degraded|down
Auth string `json:"auth,omitempty"` // 802.1x/macsec: up|down
SELinux string `json:"selinux,omitempty"` // enforcing|permissive
Chrony string `json:"chrony,omitempty"` // synced|unsynced
ChronyOffset float64 `json:"chrony_offset_seconds,omitempty"` // system time offset from NTP (signed, fast=+)
ChronySource string `json:"chrony_source,omitempty"` // NTP reference source name/IP
ChronyStratum int32 `json:"chrony_stratum,omitempty"` // hops from a reference clock
KioskActive bool `json:"kiosk_active"`
UptimeSeconds int64 `json:"uptime_seconds,omitempty"` // host uptime at last report; boot_time = last_seen - this (reboot-pending)
}
serving/fleet-manager/proto/fleet_agent.pb.go
+33 -4
View File
@@ -278,8 +278,13 @@ type HealthSnapshot struct {
ChromiumCrashes int32 `protobuf:"varint,13,opt,name=chromium_crashes,json=chromiumCrashes,proto3" json:"chromium_crashes,omitempty"`
Display string `protobuf:"bytes,14,opt,name=display,proto3" json:"display,omitempty"`
UptimeSeconds int64 `protobuf:"varint,15,opt,name=uptime_seconds,json=uptimeSeconds,proto3" json:"uptime_seconds,omitempty"` // HOST uptime (seconds since boot) — boot_time =
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
// manager receive-time - this; drives reboot-pending
// chrony tracking relative to the NTP source (from `chronyc tracking`):
ChronyOffsetSeconds float64 `protobuf:"fixed64,16,opt,name=chrony_offset_seconds,json=chronyOffsetSeconds,proto3" json:"chrony_offset_seconds,omitempty"` // system time offset from NTP (signed, fast=+)
ChronySource string `protobuf:"bytes,17,opt,name=chrony_source,json=chronySource,proto3" json:"chrony_source,omitempty"` // reference source name/IP (which server)
ChronyStratum int32 `protobuf:"varint,18,opt,name=chrony_stratum,json=chronyStratum,proto3" json:"chrony_stratum,omitempty"` // hops from a reference clock (0 = unsynced)
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *HealthSnapshot) Reset() {
@@ -417,6 +422,27 @@ func (x *HealthSnapshot) GetUptimeSeconds() int64 {
return 0
}
func (x *HealthSnapshot) GetChronyOffsetSeconds() float64 {
if x != nil {
return x.ChronyOffsetSeconds
}
return 0
}
func (x *HealthSnapshot) GetChronySource() string {
if x != nil {
return x.ChronySource
}
return ""
}
func (x *HealthSnapshot) GetChronyStratum() int32 {
if x != nil {
return x.ChronyStratum
}
return 0
}
type CommandResult struct {
state protoimpl.MessageState `protogen:"open.v1"`
CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`
@@ -696,7 +722,7 @@ const file_proto_fleet_agent_proto_rawDesc = "" +
"\rrootfs_digest\x18\x02 \x01(\tR\frootfsDigest\x12\x16\n" +
"\x06kernel\x18\x03 \x01(\tR\x06kernel\x12\x1b\n" +
"\tkiosk_url\x18\x04 \x01(\tR\bkioskUrl\x12#\n" +
"\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xd5\x03\n" +
"\rstarship_name\x18\x05 \x01(\tR\fstarshipName\"\xd5\x04\n" +
"\x0eHealthSnapshot\x12\x17\n" +
"\aat_unix\x18\x01 \x01(\x03R\x06atUnix\x12\x19\n" +
"\bload_avg\x18\x02 \x01(\x01R\aloadAvg\x12\x17\n" +
@@ -714,7 +740,10 @@ const file_proto_fleet_agent_proto_rawDesc = "" +
"\x0esession_uptime\x18\f \x01(\tR\rsessionUptime\x12)\n" +
"\x10chromium_crashes\x18\r \x01(\x05R\x0fchromiumCrashes\x12\x18\n" +
"\adisplay\x18\x0e \x01(\tR\adisplay\x12%\n" +
"\x0euptime_seconds\x18\x0f \x01(\x03R\ruptimeSeconds\"V\n" +
"\x0euptime_seconds\x18\x0f \x01(\x03R\ruptimeSeconds\x122\n" +
"\x15chrony_offset_seconds\x18\x10 \x01(\x01R\x13chronyOffsetSeconds\x12#\n" +
"\rchrony_source\x18\x11 \x01(\tR\fchronySource\x12%\n" +
"\x0echrony_stratum\x18\x12 \x01(\x05R\rchronyStratum\"V\n" +
"\rCommandResult\x12\x1d\n" +
"\n" +
"command_id\x18\x01 \x01(\tR\tcommandId\x12\x0e\n" +
serving/fleet-manager/proto/fleet_agent.proto
+4
View File
@@ -59,6 +59,10 @@ message HealthSnapshot {
string display = 14;
int64 uptime_seconds = 15; // HOST uptime (seconds since boot) — boot_time =
// manager receive-time - this; drives reboot-pending
// chrony tracking relative to the NTP source (from `chronyc tracking`):
double chrony_offset_seconds = 16; // system time offset from NTP (signed, fast=+)
string chrony_source = 17; // reference source name/IP (which server)
int32 chrony_stratum = 18; // hops from a reference clock (0 = unsynced)
}
message CommandResult {
serving/fleet-manager/ui/src/lib/fmt.js
+21
View File
@@ -36,6 +36,27 @@ export function shortSerial(s) {
return s ? s.slice(0, 8) : '—'
}
// offsetMs formats a signed seconds offset as milliseconds ("+0.02ms" / "-1.3ms").
export function offsetMs(sec) {
if (sec === undefined || sec === null || Number.isNaN(sec)) return null
const ms = sec * 1000
const digits = Math.abs(ms) < 1 ? 3 : 2
return (ms >= 0 ? '+' : '') + ms.toFixed(digits) + 'ms'
}
// chronySummary renders the chrony health as "synced · <source> · s<stratum> ·
// <offset>" — the sync state plus how it's tracking relative to the NTP source.
export function chronySummary(h) {
if (!h) return '—'
const extra = []
if (h.chrony_source) extra.push(h.chrony_source)
if (h.chrony_stratum) extra.push('s' + h.chrony_stratum)
const off = offsetMs(h.chrony_offset_seconds)
if (off && h.chrony) extra.push(off)
const state = h.chrony || '—'
return extra.length ? `${state} · ${extra.join(' · ')}` : state
}
// dur formats a span of SECONDS as a compact "3d 4h" / "5h 12m" / "45s".
export function dur(s) {
s = Math.max(0, Math.floor(s || 0))
serving/fleet-manager/ui/src/pages/Monitor.svelte
+2 -2
View File
@@ -3,7 +3,7 @@
import Panel from '../components/Panel.svelte'
import StatePill from '../components/StatePill.svelte'
import Chart from '../components/Chart.svelte'
import { relTime, pct, num, healthState, shortSerial, dur, rebootStatus } from '../lib/fmt.js'
import { relTime, pct, num, healthState, shortSerial, dur, rebootStatus, chronySummary } from '../lib/fmt.js'
let hosts = $state([])
let error = $state('')
@@ -103,7 +103,7 @@
<dt class="text-faint">auth</dt><dd class="text-muted">{selected.health?.auth || '—'}</dd>
<dt class="text-faint">nvme root</dt><dd class="text-muted">{selected.health?.nvme_root || '—'}</dd>
<dt class="text-faint">selinux</dt><dd class="text-muted">{selected.health?.selinux || '—'}</dd>
<dt class="text-faint">chrony</dt><dd class="text-muted">{selected.health?.chrony || '—'}</dd>
<dt class="text-faint">chrony</dt><dd class="text-muted">{chronySummary(selected.health)}</dd>
<dt class="text-faint">kiosk</dt><dd class="text-muted">{selected.health?.kiosk_active ? 'active' : 'idle'} · {selected.usage?.session_uptime || '—'}</dd>
<dt class="text-faint">uptime</dt><dd class="text-muted">{selected.health?.uptime_seconds ? dur(selected.health.uptime_seconds) : '—'}</dd>
{#if rebootStatus(selected)}