#!/usr/bin/env bash # extract-artifacts.sh # # Pulls the diskless deliverables out of the Packer-built disk image: # vmlinuz -> HTTPS boot server # initramfs.img -> HTTPS boot server (the fleet-generic image) # rootfs.erofs -> nvmet target: READ-ONLY NVMe/TCP DATA namespace (erofs) # rootfs.hash -> nvmet target: READ-ONLY NVMe/TCP dm-verity HASH namespace # verity.json -> root hash + data/hash namespace UUIDs (feed the cmdline) # cmdline.txt -> ready-made kernel cmdline fragment for this build # # Requires on the build host: libguestfs-tools, erofs-utils (mkfs.erofs), # cryptsetup (veritysetup), jq. set -euo pipefail QCOW2=${1:?usage: extract-artifacts.sh [disable_dot1x]} OUT=${2:?usage: extract-artifacts.sh [disable_dot1x]} # TEMPORARY test toggle: when "true", bake rd.dot1x=0 into the cmdline so the # UKI boots with 802.1X + MACsec disabled (open-port NVMe-oF boot test). DISABLE_DOT1X=${3:-false} # NVMe/TCP discovery address baked into the (immutable, signed) UKI cmdline. # Prefer a per-environment DNS name resolved at boot so one UKI works across # environments. Defaults to the literal placeholder so build-oci.sh's warning # fires if it was never set. NVMET_TRADDR=${NVMET_TRADDR:-NVMET_TRADDR} # Plymouth boot splash toggle (from the boot_splash Packer var). "true" bakes # "rhgb quiet" so Plymouth shows the graphical splash; anything else bakes # "plymouth.enable=0" for a fully verbose console boot (boot-chain debugging). BOOT_SPLASH=${BOOT_SPLASH:-true} # STIG hardening toggle (from the enable_stig Packer var). "true" bakes the STIG # kernel params into the (signed) UKI cmdline — the no-GRUB compensating control # for the bootloader-argument rules (see docs/stig-deviations.md). FIPS is NOT # included here (decision: off); add fips=1 when FIPS is turned on. STIG=${STIG:-true} # Stable namespace UUIDs. Devices are referenced by-id # (/dev/disk/by-id/nvme-uuid.) rather than /dev/nvmeNnM because NVMe # controller/namespace indices are NOT stable (a transient discovery controller # can shift them), so a hardcoded node is a latent brick in an immutable signed # cmdline. dm-verity needs TWO namespaces — erofs DATA and the verity HASH tree # — because this distro's systemd-veritysetup does not support the in-file # hash-offset layout (it ignores hash-offset= and fails to load the superblock). # The serving side reads these back from verity.json (single source of truth) — # see serving/spdk/spdk-release.sh, which assigns them on the target via add_ns -u. # # PER-VERSION UUIDs: so a LONG-LIVED NVMe-oF target can serve MULTIPLE rootfs # versions at once (rolling updates, TODO #6) the two namespaces must be globally # distinct per version — otherwise two versions collide on the same by-id symlink # (/dev/disk/by-id/nvme-uuid.) on the host. We derive them deterministically # from IMAGE_VERSION (sha1sum — coreutils only, no uuidgen dependency) and bake # the result into BOTH the immutable UKI cmdline and verity.json. Determinism # preserves the reproducible-build invariant (same version -> same UUIDs). The # value need not be an RFC-v5 UUID — it only has to be well-formed + unique, and # the cmdline and the serving target consume the SAME value (cmdline from here, # target from verity.json). An explicit ROOT_DATA_UUID/ROOT_HASH_UUID env wins; # with no version it falls back to the legacy fixed pair (single-version serving). IMAGE_VERSION=${IMAGE_VERSION:-$(basename "$OUT")} sha2uuid() { # arbitrary string -> stable UUID-formatted (8-4-4-4-12) hex local h h=$(printf '%s' "$1" | sha1sum | cut -c1-32) printf '%s-%s-%s-%s-%s' "${h:0:8}" "${h:8:4}" "${h:12:4}" "${h:16:4}" "${h:20:12}" } if [ -n "$IMAGE_VERSION" ] && command -v sha1sum >/dev/null 2>&1; then ROOT_DATA_UUID=${ROOT_DATA_UUID:-$(sha2uuid "fleet-rootfs-data-${IMAGE_VERSION}")} ROOT_HASH_UUID=${ROOT_HASH_UUID:-$(sha2uuid "fleet-rootfs-hash-${IMAGE_VERSION}")} fi ROOT_DATA_UUID=${ROOT_DATA_UUID:-f1ee7000-0000-4000-8000-000000000001} ROOT_HASH_UUID=${ROOT_HASH_UUID:-f1ee7000-0000-4000-8000-000000000002} ROOT_DATA_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_DATA_UUID}" ROOT_HASH_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_HASH_UUID}" # Per-version subsystem NQN (distinct NQN per release) — lets a host attach to # exactly one version's subsystem and gives clean per-release drain/allow-listing # on the long-lived target. Written to verity.json for the serving side; NOT in # the cmdline (host still discovers + connects; the by-UUID verity refs select # what mounts). An explicit SUBNQN env wins. SUBNQN_BASE=${SUBNQN_BASE:-nqn.2026-06.org.example.fleet:rootfs} SUBNQN=${SUBNQN:-${SUBNQN_BASE}-${IMAGE_VERSION}} for bin in virt-copy-out virt-ls guestfish mkfs.erofs veritysetup jq getfattr; do command -v "$bin" >/dev/null || { echo "ERROR: $bin not found on build host" >&2; exit 1; } done # Must run as root: `tar -x` can only restore the build tree's real ownership # (root:ssh_keys host keys, tty/crontab setgid groups, service-user dirs, user # home dirs) with CAP_CHOWN. Non-root silently flattens ownership to the caller # -> a broken image (sshd refuses, setgid tools misbehave). We removed # mkfs.erofs --all-root, so correct ownership now depends on this. if [ "$(id -u)" -ne 0 ]; then echo "ERROR: run extract-artifacts.sh as root so file ownership is preserved in the rootfs." >&2 exit 1 fi mkdir -p "$OUT" export LIBGUESTFS_BACKEND=direct WORK=$(mktemp -d) trap 'rm -rf "$WORK"' EXIT echo "==> Locating kernel and generic initramfs in $QCOW2" KERNEL=$(virt-ls -a "$QCOW2" /boot | grep '^vmlinuz-' | grep -v rescue | sort -V | tail -1) [ -n "$KERNEL" ] || { echo "ERROR: no kernel found in image" >&2; exit 1; } virt-copy-out -a "$QCOW2" "/boot/$KERNEL" "/boot/initramfs-generic.img" "$OUT/" mv "$OUT/$KERNEL" "$OUT/vmlinuz" mv "$OUT/initramfs-generic.img" "$OUT/initramfs.img" echo "==> Exporting root tree (preserving SELinux labels + xattrs)" # SELinux labels MUST survive into the erofs, or the enforcing target denies the # login session — you authenticate, then logind kills the session, and a # read-only erofs can't autorelabel to fix itself. guestfish tar-out with # selinux:true/xattrs:true captures security.selinux; the extraction restores it # (--xattrs), and mkfs.erofs preserves xattrs into the image. (The old # virt-tar-out + plain `tar -xpf` dropped the labels — that was the login bug.) guestfish --ro -a "$QCOW2" -i \ tar-out / "$WORK/rootfs.tar" numericowner:true xattrs:true selinux:true acls:true mkdir "$WORK/root" tar -C "$WORK/root" --numeric-owner --xattrs --xattrs-include='*' --acls --selinux \ -xpf "$WORK/rootfs.tar" rm -rf "$WORK/root/boot/"* "$WORK/rootfs.tar" # diskless hosts don't need /boot # Strip SSH host keys. They are PER-HOST identity (invariant #1) and must not be # baked: a baked key means every host shares one SSH identity (MITM risk), and # the build VM's key perms/group don't survive the tar->erofs round-trip, so # sshd rejects them ("Permissions 640 ... too open") and refuses to start. # RHEL's sshd-keygen@.service regenerates them with correct perms before sshd # starts, into the writable volatile overlay. NOTE: with the ephemeral overlay # they regenerate EVERY boot, so the host key changes each reboot (SSH clients # will warn) — for a stable identity, persist them or derive from the TPM # (future; ties to the hostnqn/identity model). rm -f "$WORK/root/etc/ssh/"ssh_host_* # Fail loudly if labels didn't make it — shipping an unlabeled enforcing image # boots but denies all logins (the symptom we just chased down). if ! getfattr -h -n security.selinux "$WORK/root/usr/bin/bash" >/dev/null 2>&1; then echo "ERROR: SELinux label missing on /usr/bin/bash in the extracted tree." >&2 echo " An enforcing boot would deny login. Check guestfish/tar xattr flags." >&2 exit 1 fi echo "==> SELinux labels preserved (bash: $(getfattr -h --only-values -n security.selinux "$WORK/root/usr/bin/bash" 2>/dev/null))" echo "==> Building reproducible erofs image" # UNCOMPRESSED on purpose: the target Rocky 9 kernel's erofs is built without # CONFIG_EROFS_FS_ZIP, so mounting a compressed image fails at boot with # "erofs: lz4 algorithm isn't enabled" (verified on the bench — the root mount # never completes). An uncompressed image mounts with base CONFIG_EROFS_FS # alone. Tradeoff: larger namespace + more NVMe/TCP bandwidth per read than the # old `-z lz4hc`. Re-add compression (`-z lz4hc`, or zstd/deflate) ONLY against # a kernel you've confirmed has EROFS_FS_ZIP (grep CONFIG_EROFS /boot/config-*). # -T0 + fixed UUID keep output deterministic => stable dm-verity root hash for # identical input trees. # Do NOT use --all-root: it forces every inode to root:root, discarding group # ownership (ssh_keys on host keys -> sshd "permissions too open"; tty/crontab # setgid binaries) and user ownership (home dirs). Real ownership is preserved # from the tree, which is why this script must run as root (see the euid check). FIXED_UUID="6c10a7f0-1d2e-4f5a-9b3c-000000000001" mkfs.erofs -T 0 -U "$FIXED_UUID" \ "$WORK/rootfs.erofs" "$WORK/root" echo "==> Generating dm-verity hash tree as a SEPARATE file (two-namespace layout)" # Hash tree in its OWN file (served as a 2nd namespace), NOT appended in-file: # this distro's systemd-veritysetup ignores hash-offset= ("unknown option") and # fails to load the verity superblock. With separate data/hash devices the # cmdline needs no offset. Deterministic for identical input trees (fixed erofs # UUID + -T0 => stable data => stable hash tree => stable root hash). VERITY_OUT=$(veritysetup format "$WORK/rootfs.erofs" "$WORK/rootfs.hash" \ --data-block-size=4096 --hash-block-size=4096) ROOT_HASH=$(printf '%s\n' "$VERITY_OUT" | awk '/Root hash:/ {print $3}') [ -n "$ROOT_HASH" ] || { echo "ERROR: veritysetup produced no root hash" >&2; exit 1; } mv "$WORK/rootfs.erofs" "$OUT/rootfs.erofs" mv "$WORK/rootfs.hash" "$OUT/rootfs.hash" jq -n --arg hash "$ROOT_HASH" --arg erofs_uuid "$FIXED_UUID" \ --arg data_uuid "$ROOT_DATA_UUID" --arg hash_uuid "$ROOT_HASH_UUID" \ --arg subnqn "$SUBNQN" --arg version "$IMAGE_VERSION" \ '{root_hash: $hash, erofs_uuid: $erofs_uuid, data_ns_uuid: $data_uuid, hash_ns_uuid: $hash_uuid, subnqn: $subnqn, image_version: $version, data_block_size: 4096, hash_block_size: 4096}' \ > "$OUT/verity.json" # Kernel cmdline fragment for the boot server. Single line: this file is # consumed verbatim by ukify in build-oci.sh and baked into the signed UKI. # The device node assumes the root namespace is the host's first/only # NVMe/TCP namespace; adjust if not. NVMET_TRADDR: set per environment # *before* the UKI build, since the cmdline is immutable once signed. # Plymouth: rhgb (Red Hat graphical boot) + quiet for a clean splash, or # plymouth.enable=0 for a verbose debug boot. Press ESC at boot to see details # behind the splash; or override at runtime via the SMBIOS Type 11 # io.systemd.stub.kernel-cmdline-extra OEM string (adds args without a rebuild). if [ "$BOOT_SPLASH" = "true" ]; then SPLASH_FRAGMENT="rhgb quiet " else SPLASH_FRAGMENT="plymouth.enable=0 " fi # STIG kernel hardening params (no GRUB here, so they ride the signed UKI # cmdline): kernel auditing on from boot, larger audit backlog, slab/page # poisoning, no vsyscall, zero-on-alloc/free. NO fips=1 (FIPS off — decision). STIG_FRAGMENT="" if [ "$STIG" = "true" ]; then STIG_FRAGMENT="audit=1 audit_backlog_limit=8192 slub_debug=P page_poison=1 vsyscall=none init_on_alloc=1 init_on_free=1 " fi DOT1X_FRAGMENT="" if [ "$DISABLE_DOT1X" = "true" ]; then echo "==> WARNING: building TEST cmdline with rd.dot1x=0 (802.1X + MACsec DISABLED)" >&2 echo " open-port NVMe-oF boot test only — do NOT ship this UKI." >&2 # ip=dhcp: with auth disabled, dracut brings the NIC up by DHCP (writes # resolv.conf, satisfies rd.neednet) so the nvmf module can connect. The # cmdline otherwise has no ip=, so nothing configures the NIC. Test-only — # in the secure build the hook owns the link (macsec0); ip=dhcp must NOT be # set there. DOT1X_FRAGMENT="rd.dot1x=0 ip=dhcp " fi printf '%s' \ "$SPLASH_FRAGMENT" \ "$STIG_FRAGMENT" \ "$DOT1X_FRAGMENT" \ "rd.neednet=1 systemd.volatile=overlay " \ "rd.dot1x.nvindex=0x01800001 rd.dot1x.timeout=60 " \ "rd.nvmf.discover=tcp,${NVMET_TRADDR},4420 " \ "root=/dev/mapper/root rootfstype=erofs rootflags=ro " \ "systemd.verity=yes roothash=${ROOT_HASH} " \ "systemd.verity_root_data=${ROOT_DATA_DEV} " \ "systemd.verity_root_hash=${ROOT_HASH_DEV}" \ > "$OUT/cmdline.txt" echo "==> Checksums" ( cd "$OUT" && sha256sum vmlinuz initramfs.img rootfs.erofs rootfs.hash | tee SHA256SUMS ) cat < publish to the HTTPS boot server (versioned dir, flip a symlink for atomic rollout/rollback) rootfs.erofs -> expose as a READ-ONLY nvmet DATA namespace rootfs.hash -> expose as a READ-ONLY nvmet HASH namespace (verity) cmdline.txt -> kernel cmdline for this build. The roothash binds the root fs to the boot artifacts: serve via a signed UKI or signed GRUB config so Secure Boot covers it, or the verity guarantee is advisory. Remember: identity is NOT in these artifacts. Hosts authenticate with the TPM key at the fleet persistent handle and the cert in the fleet NV index; the hostnqn is derived from that cert at boot. EOF