diff --git a/serving/fleet-manager/DEPLOY.md b/serving/fleet-manager/DEPLOY.md index 7098205..8bf683b 100644 --- a/serving/fleet-manager/DEPLOY.md +++ b/serving/fleet-manager/DEPLOY.md @@ -222,6 +222,19 @@ and the `influx-data` volume. Losing `admins.json` locks every admin out — you re-bootstrap with `register-admin`. Losing `enroll.key` invalidates any outstanding (unused) enrol links. +`./backup.sh [dest-dir]` tars both volumes (read-only, safe to run live) into +timestamped `.tar.gz` files (default `./backups`, keeps the newest +`BACKUP_KEEP=14`). Schedule it with a systemd timer / cron and copy the output +offsite: + +```bash +./backup.sh /srv/backups/fleet-manager +# restore (manager stopped) — see the header of backup.sh +``` + +If you run compose under a non-default project name, pass +`COMPOSE_PROJECT_NAME=` so it finds the `_fleet-data` volume. + ## Troubleshooting - **`permission denied` writing the inventory on approve** — the runtime image is diff --git a/serving/fleet-manager/backup.sh b/serving/fleet-manager/backup.sh new file mode 100755 index 0000000..f992082 --- /dev/null +++ b/serving/fleet-manager/backup.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# backup.sh — snapshot the fleet-manager durable state to timestamped tarballs. +# +# Two docker named volumes hold everything that isn't reproducible from the image: +# - fleet-data : inventory.json + admins.json + enroll.key (CRITICAL — losing +# admins.json/enroll.key locks every admin out of the UI) +# - influx-data : the metrics time-series (history; regenerates going forward) +# +# Runs a throwaway alpine container to tar each volume read-only into $DEST. Safe +# to run live. Schedule it (systemd timer / cron); copy $DEST offsite. +# +# Usage: +# ./backup.sh [dest-dir] # default dest: ./backups +# COMPOSE_PROJECT_NAME=foo ./backup.sh # if you run compose under another project +# +# Restore (manager stopped): +# docker run --rm -v _fleet-data:/dest -v "$PWD/backups":/src alpine \ +# sh -c 'rm -rf /dest/* && tar xzf /src/fleet-data-YYYYmmdd-HHMMSS.tar.gz -C /dest' +set -euo pipefail + +DEST=${1:-./backups} +PROJECT=${COMPOSE_PROJECT_NAME:-fleet-manager} # compose derives volume names as _ +STAMP=$(date -u +%Y%m%d-%H%M%S) +VOLUMES=${BACKUP_VOLUMES:-"fleet-data influx-data"} + +mkdir -p "$DEST" +DEST_ABS=$(cd "$DEST" && pwd) + +for key in $VOLUMES; do + vol="${PROJECT}_${key}" + if ! docker volume inspect "$vol" >/dev/null 2>&1; then + echo "backup: SKIP $vol (no such volume — wrong COMPOSE_PROJECT_NAME?)" + continue + fi + out="${key}-${STAMP}.tar.gz" + echo "backup: $vol -> $DEST_ABS/$out" + docker run --rm \ + -v "${vol}:/src:ro" \ + -v "${DEST_ABS}:/backup" \ + alpine tar czf "/backup/${out}" -C /src . +done + +# Retention: keep the newest $KEEP of each volume's tarballs (0 = keep all). +KEEP=${BACKUP_KEEP:-14} +if [ "$KEEP" -gt 0 ]; then + for key in $VOLUMES; do + # ls -t for mtime order; our filenames are safe (key + UTC timestamp). + # shellcheck disable=SC2012 + ls -1t "$DEST_ABS/${key}-"*.tar.gz 2>/dev/null | tail -n +"$((KEEP + 1))" | while read -r old; do + echo "backup: prune $old" + rm -f "$old" + done + done +fi + +echo "backup: done ($STAMP) — copy $DEST_ABS offsite."