201 lines
11 KiB
Bash
Executable File
201 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# extract-artifacts.sh <qcow2> <output-dir>
|
|
#
|
|
# Pulls the diskless deliverables out of the Packer-built disk image:
|
|
# vmlinuz -> HTTPS boot server
|
|
# initramfs.img -> HTTPS boot server (the fleet-generic image)
|
|
# rootfs.erofs -> nvmet target: READ-ONLY NVMe/TCP DATA namespace (erofs)
|
|
# rootfs.hash -> nvmet target: READ-ONLY NVMe/TCP dm-verity HASH namespace
|
|
# verity.json -> root hash + data/hash namespace UUIDs (feed the cmdline)
|
|
# cmdline.txt -> ready-made kernel cmdline fragment for this build
|
|
#
|
|
# Requires on the build host: libguestfs-tools, erofs-utils (mkfs.erofs),
|
|
# cryptsetup (veritysetup), jq.
|
|
set -euo pipefail
|
|
|
|
QCOW2=${1:?usage: extract-artifacts.sh <qcow2> <output-dir> [disable_dot1x]}
|
|
OUT=${2:?usage: extract-artifacts.sh <qcow2> <output-dir> [disable_dot1x]}
|
|
# TEMPORARY test toggle: when "true", bake rd.dot1x=0 into the cmdline so the
|
|
# UKI boots with 802.1X + MACsec disabled (open-port NVMe-oF boot test).
|
|
DISABLE_DOT1X=${3:-false}
|
|
# NVMe/TCP discovery address baked into the (immutable, signed) UKI cmdline.
|
|
# Prefer a per-environment DNS name resolved at boot so one UKI works across
|
|
# environments. Defaults to the literal placeholder so build-oci.sh's warning
|
|
# fires if it was never set.
|
|
NVMET_TRADDR=${NVMET_TRADDR:-NVMET_TRADDR}
|
|
# Plymouth boot splash toggle (from the boot_splash Packer var). "true" bakes
|
|
# "rhgb quiet" so Plymouth shows the graphical splash; anything else bakes
|
|
# "plymouth.enable=0" for a fully verbose console boot (boot-chain debugging).
|
|
BOOT_SPLASH=${BOOT_SPLASH:-true}
|
|
# Stable namespace UUIDs. Devices are referenced by-id
|
|
# (/dev/disk/by-id/nvme-uuid.<uuid>) rather than /dev/nvmeNnM because NVMe
|
|
# controller/namespace indices are NOT stable (a transient discovery controller
|
|
# can shift them), so a hardcoded node is a latent brick in an immutable signed
|
|
# cmdline. dm-verity needs TWO namespaces — erofs DATA and the verity HASH tree
|
|
# — because this distro's systemd-veritysetup does not support the in-file
|
|
# hash-offset layout (it ignores hash-offset= and fails to load the superblock).
|
|
# MUST match ROOT_DATA_UUID / ROOT_HASH_UUID in serving/spdk-setup.sh (assigned
|
|
# on the target via add_ns -u).
|
|
ROOT_DATA_UUID=${ROOT_DATA_UUID:-f1ee7000-0000-4000-8000-000000000001}
|
|
ROOT_HASH_UUID=${ROOT_HASH_UUID:-f1ee7000-0000-4000-8000-000000000002}
|
|
ROOT_DATA_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_DATA_UUID}"
|
|
ROOT_HASH_DEV="/dev/disk/by-id/nvme-uuid.${ROOT_HASH_UUID}"
|
|
|
|
for bin in virt-copy-out virt-ls guestfish mkfs.erofs veritysetup jq getfattr; do
|
|
command -v "$bin" >/dev/null || { echo "ERROR: $bin not found on build host" >&2; exit 1; }
|
|
done
|
|
|
|
# Must run as root: `tar -x` can only restore the build tree's real ownership
|
|
# (root:ssh_keys host keys, tty/crontab setgid groups, service-user dirs, user
|
|
# home dirs) with CAP_CHOWN. Non-root silently flattens ownership to the caller
|
|
# -> a broken image (sshd refuses, setgid tools misbehave). We removed
|
|
# mkfs.erofs --all-root, so correct ownership now depends on this.
|
|
if [ "$(id -u)" -ne 0 ]; then
|
|
echo "ERROR: run extract-artifacts.sh as root so file ownership is preserved in the rootfs." >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$OUT"
|
|
export LIBGUESTFS_BACKEND=direct
|
|
WORK=$(mktemp -d)
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
echo "==> Locating kernel and generic initramfs in $QCOW2"
|
|
KERNEL=$(virt-ls -a "$QCOW2" /boot | grep '^vmlinuz-' | grep -v rescue | sort -V | tail -1)
|
|
[ -n "$KERNEL" ] || { echo "ERROR: no kernel found in image" >&2; exit 1; }
|
|
|
|
virt-copy-out -a "$QCOW2" "/boot/$KERNEL" "/boot/initramfs-generic.img" "$OUT/"
|
|
mv "$OUT/$KERNEL" "$OUT/vmlinuz"
|
|
mv "$OUT/initramfs-generic.img" "$OUT/initramfs.img"
|
|
|
|
echo "==> Exporting root tree (preserving SELinux labels + xattrs)"
|
|
# SELinux labels MUST survive into the erofs, or the enforcing target denies the
|
|
# login session — you authenticate, then logind kills the session, and a
|
|
# read-only erofs can't autorelabel to fix itself. guestfish tar-out with
|
|
# selinux:true/xattrs:true captures security.selinux; the extraction restores it
|
|
# (--xattrs), and mkfs.erofs preserves xattrs into the image. (The old
|
|
# virt-tar-out + plain `tar -xpf` dropped the labels — that was the login bug.)
|
|
guestfish --ro -a "$QCOW2" -i \
|
|
tar-out / "$WORK/rootfs.tar" numericowner:true xattrs:true selinux:true acls:true
|
|
mkdir "$WORK/root"
|
|
tar -C "$WORK/root" --numeric-owner --xattrs --xattrs-include='*' --acls --selinux \
|
|
-xpf "$WORK/rootfs.tar"
|
|
rm -rf "$WORK/root/boot/"* "$WORK/rootfs.tar" # diskless hosts don't need /boot
|
|
|
|
# Strip SSH host keys. They are PER-HOST identity (invariant #1) and must not be
|
|
# baked: a baked key means every host shares one SSH identity (MITM risk), and
|
|
# the build VM's key perms/group don't survive the tar->erofs round-trip, so
|
|
# sshd rejects them ("Permissions 640 ... too open") and refuses to start.
|
|
# RHEL's sshd-keygen@.service regenerates them with correct perms before sshd
|
|
# starts, into the writable volatile overlay. NOTE: with the ephemeral overlay
|
|
# they regenerate EVERY boot, so the host key changes each reboot (SSH clients
|
|
# will warn) — for a stable identity, persist them or derive from the TPM
|
|
# (future; ties to the hostnqn/identity model).
|
|
rm -f "$WORK/root/etc/ssh/"ssh_host_*
|
|
|
|
# Fail loudly if labels didn't make it — shipping an unlabeled enforcing image
|
|
# boots but denies all logins (the symptom we just chased down).
|
|
if ! getfattr -h -n security.selinux "$WORK/root/usr/bin/bash" >/dev/null 2>&1; then
|
|
echo "ERROR: SELinux label missing on /usr/bin/bash in the extracted tree." >&2
|
|
echo " An enforcing boot would deny login. Check guestfish/tar xattr flags." >&2
|
|
exit 1
|
|
fi
|
|
echo "==> SELinux labels preserved (bash: $(getfattr -h --only-values -n security.selinux "$WORK/root/usr/bin/bash" 2>/dev/null))"
|
|
|
|
echo "==> Building reproducible erofs image"
|
|
# UNCOMPRESSED on purpose: the target Rocky 9 kernel's erofs is built without
|
|
# CONFIG_EROFS_FS_ZIP, so mounting a compressed image fails at boot with
|
|
# "erofs: lz4 algorithm isn't enabled" (verified on the bench — the root mount
|
|
# never completes). An uncompressed image mounts with base CONFIG_EROFS_FS
|
|
# alone. Tradeoff: larger namespace + more NVMe/TCP bandwidth per read than the
|
|
# old `-z lz4hc`. Re-add compression (`-z lz4hc`, or zstd/deflate) ONLY against
|
|
# a kernel you've confirmed has EROFS_FS_ZIP (grep CONFIG_EROFS /boot/config-*).
|
|
# -T0 + fixed UUID keep output deterministic => stable dm-verity root hash for
|
|
# identical input trees.
|
|
# Do NOT use --all-root: it forces every inode to root:root, discarding group
|
|
# ownership (ssh_keys on host keys -> sshd "permissions too open"; tty/crontab
|
|
# setgid binaries) and user ownership (home dirs). Real ownership is preserved
|
|
# from the tree, which is why this script must run as root (see the euid check).
|
|
FIXED_UUID="6c10a7f0-1d2e-4f5a-9b3c-000000000001"
|
|
mkfs.erofs -T 0 -U "$FIXED_UUID" \
|
|
"$WORK/rootfs.erofs" "$WORK/root"
|
|
|
|
echo "==> Generating dm-verity hash tree as a SEPARATE file (two-namespace layout)"
|
|
# Hash tree in its OWN file (served as a 2nd namespace), NOT appended in-file:
|
|
# this distro's systemd-veritysetup ignores hash-offset= ("unknown option") and
|
|
# fails to load the verity superblock. With separate data/hash devices the
|
|
# cmdline needs no offset. Deterministic for identical input trees (fixed erofs
|
|
# UUID + -T0 => stable data => stable hash tree => stable root hash).
|
|
VERITY_OUT=$(veritysetup format "$WORK/rootfs.erofs" "$WORK/rootfs.hash" \
|
|
--data-block-size=4096 --hash-block-size=4096)
|
|
ROOT_HASH=$(printf '%s\n' "$VERITY_OUT" | awk '/Root hash:/ {print $3}')
|
|
[ -n "$ROOT_HASH" ] || { echo "ERROR: veritysetup produced no root hash" >&2; exit 1; }
|
|
|
|
mv "$WORK/rootfs.erofs" "$OUT/rootfs.erofs"
|
|
mv "$WORK/rootfs.hash" "$OUT/rootfs.hash"
|
|
|
|
jq -n --arg hash "$ROOT_HASH" --arg erofs_uuid "$FIXED_UUID" \
|
|
--arg data_uuid "$ROOT_DATA_UUID" --arg hash_uuid "$ROOT_HASH_UUID" \
|
|
'{root_hash: $hash, erofs_uuid: $erofs_uuid,
|
|
data_ns_uuid: $data_uuid, hash_ns_uuid: $hash_uuid,
|
|
data_block_size: 4096, hash_block_size: 4096}' \
|
|
> "$OUT/verity.json"
|
|
|
|
# Kernel cmdline fragment for the boot server. Single line: this file is
|
|
# consumed verbatim by ukify in build-oci.sh and baked into the signed UKI.
|
|
# The device node assumes the root namespace is the host's first/only
|
|
# NVMe/TCP namespace; adjust if not. NVMET_TRADDR: set per environment
|
|
# *before* the UKI build, since the cmdline is immutable once signed.
|
|
# Plymouth: rhgb (Red Hat graphical boot) + quiet for a clean splash, or
|
|
# plymouth.enable=0 for a verbose debug boot. Press ESC at boot to see details
|
|
# behind the splash; or override at runtime via the SMBIOS Type 11
|
|
# io.systemd.stub.kernel-cmdline-extra OEM string (adds args without a rebuild).
|
|
if [ "$BOOT_SPLASH" = "true" ]; then
|
|
SPLASH_FRAGMENT="rhgb quiet "
|
|
else
|
|
SPLASH_FRAGMENT="plymouth.enable=0 "
|
|
fi
|
|
|
|
DOT1X_FRAGMENT=""
|
|
if [ "$DISABLE_DOT1X" = "true" ]; then
|
|
echo "==> WARNING: building TEST cmdline with rd.dot1x=0 (802.1X + MACsec DISABLED)" >&2
|
|
echo " open-port NVMe-oF boot test only — do NOT ship this UKI." >&2
|
|
# ip=dhcp: with auth disabled, dracut brings the NIC up by DHCP (writes
|
|
# resolv.conf, satisfies rd.neednet) so the nvmf module can connect. The
|
|
# cmdline otherwise has no ip=, so nothing configures the NIC. Test-only —
|
|
# in the secure build the hook owns the link (macsec0); ip=dhcp must NOT be
|
|
# set there.
|
|
DOT1X_FRAGMENT="rd.dot1x=0 ip=dhcp "
|
|
fi
|
|
printf '%s' \
|
|
"$SPLASH_FRAGMENT" \
|
|
"$DOT1X_FRAGMENT" \
|
|
"rd.neednet=1 systemd.volatile=overlay " \
|
|
"rd.dot1x.nvindex=0x01800001 rd.dot1x.timeout=60 " \
|
|
"rd.nvmf.discover=tcp,${NVMET_TRADDR},4420 " \
|
|
"root=/dev/mapper/root rootfstype=erofs rootflags=ro " \
|
|
"systemd.verity=yes roothash=${ROOT_HASH} " \
|
|
"systemd.verity_root_data=${ROOT_DATA_DEV} " \
|
|
"systemd.verity_root_hash=${ROOT_HASH_DEV}" \
|
|
> "$OUT/cmdline.txt"
|
|
|
|
echo "==> Checksums"
|
|
( cd "$OUT" && sha256sum vmlinuz initramfs.img rootfs.erofs rootfs.hash | tee SHA256SUMS )
|
|
|
|
cat <<EOF
|
|
|
|
Artifacts in $OUT:
|
|
vmlinuz, initramfs.img -> publish to the HTTPS boot server (versioned dir,
|
|
flip a symlink for atomic rollout/rollback)
|
|
rootfs.erofs -> expose as a READ-ONLY nvmet DATA namespace
|
|
rootfs.hash -> expose as a READ-ONLY nvmet HASH namespace (verity)
|
|
cmdline.txt -> kernel cmdline for this build. The roothash binds
|
|
the root fs to the boot artifacts: serve via a
|
|
signed UKI or signed GRUB config so Secure Boot
|
|
covers it, or the verity guarantee is advisory.
|
|
|
|
Remember: identity is NOT in these artifacts. Hosts authenticate with the
|
|
TPM key at the fleet persistent handle and the cert in the fleet NV index;
|
|
the hostnqn is derived from that cert at boot.
|
|
EOF
|