add Phase 2A host maintenance procedure
This commit is contained in:
parent
13ec935e87
commit
2cf249d6f2
9 changed files with 1119 additions and 3 deletions
|
|
@ -0,0 +1,325 @@
|
|||
#!/usr/bin/env bash
|
||||
# Read-only host inspection plus root-only evidence recording.
|
||||
# Run as a separate process; this script changes no shell options and uses return only.
|
||||
|
||||
add_failure() {
|
||||
printf '%s\n' "$1" >> "$record/failed-mandatory-checks.txt"
|
||||
overall=HARD_STOP
|
||||
return 0
|
||||
}
|
||||
|
||||
mandatory() {
|
||||
label=$1
|
||||
outfile=$2
|
||||
shift 2
|
||||
"$@" > "$record/$outfile" 2>&1
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
add_failure "$label|rc=$rc|file=$outfile"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
mandatory_shell() {
|
||||
label=$1
|
||||
outfile=$2
|
||||
command_text=$3
|
||||
/usr/bin/bash -o pipefail -c "$command_text" > "$record/$outfile" 2>&1
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
add_failure "$label|rc=$rc|file=$outfile"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
optional_shell() {
|
||||
label=$1
|
||||
outfile=$2
|
||||
command_text=$3
|
||||
/usr/bin/bash -o pipefail -c "$command_text" > "$record/$outfile" 2>&1
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
printf '%s|rc=%s|file=%s\n' "$label" "$rc" "$outfile" >> "$record/warnings.txt"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_health_urls() {
|
||||
: > "$record/health-urls.sanitized"
|
||||
if [ -z "${PUBLIC_HEALTH_URLS:-}" ]; then
|
||||
add_failure 'public-health-urls|unset'
|
||||
return 0
|
||||
fi
|
||||
case "$PUBLIC_HEALTH_URLS" in
|
||||
*$'\n'*|*$'\r'*|*$'\t'*)
|
||||
add_failure 'public-health-urls|control-character-rejected'
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
for url in $PUBLIC_HEALTH_URLS; do
|
||||
case "$url" in
|
||||
https://*) ;;
|
||||
*) add_failure 'public-health-url|unsupported-scheme-or-malformed'; continue ;;
|
||||
esac
|
||||
authority=${url#https://}
|
||||
authority=${authority%%/*}
|
||||
case "$url" in
|
||||
*'?'*|*'#'*|*$'\n'*|*$'\r'*|*$'\t'*|*' '*)
|
||||
add_failure 'public-health-url|query-fragment-or-control-character-rejected'
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
case "$authority" in
|
||||
''|*@*) add_failure 'public-health-url|empty-authority-or-userinfo-rejected'; continue ;;
|
||||
esac
|
||||
printf '%s\n' "$url" >> "$record/health-urls.sanitized"
|
||||
done
|
||||
if [ ! -s "$record/health-urls.sanitized" ]; then
|
||||
add_failure 'public-health-urls|no-valid-non-secret-https-url'
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
check_public_health() {
|
||||
: > "$record/public-health.txt"
|
||||
while IFS= read -r url; do
|
||||
[ -n "$url" ] || continue
|
||||
metrics=$(curl --fail --silent --show-error --max-redirs 0 \
|
||||
--connect-timeout 10 --max-time 30 --output /dev/null \
|
||||
--write-out 'http=%{http_code}|remote=%{remote_ip}|time=%{time_total}' \
|
||||
"$url" 2>> "$record/public-health-errors.txt")
|
||||
rc=$?
|
||||
printf '%s|%s\n' "$url" "$metrics" >> "$record/public-health.txt"
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
add_failure "public-health|rc=$rc|url=$url"
|
||||
fi
|
||||
done < "$record/health-urls.sanitized"
|
||||
return 0
|
||||
}
|
||||
|
||||
capture_bootloader() {
|
||||
found=0
|
||||
: > "$record/bootloader-inventory.txt"
|
||||
if [ -d /boot/loader/entries ]; then
|
||||
printf '%s\n' 'type=systemd-boot' >> "$record/bootloader-inventory.txt"
|
||||
bootctl status >> "$record/bootloader-inventory.txt" 2>&1
|
||||
bootctl_rc=$?
|
||||
find /boot/loader/entries -maxdepth 1 -type f -print -exec sed -n '1,160p' {} \; \
|
||||
>> "$record/bootloader-inventory.txt" 2>&1
|
||||
find_rc=$?
|
||||
if [ "$bootctl_rc" -eq 0 ] && [ "$find_rc" -eq 0 ]; then found=1; fi
|
||||
fi
|
||||
if [ -f /boot/grub/grub.cfg ]; then
|
||||
printf '%s\n' 'type=grub' >> "$record/bootloader-inventory.txt"
|
||||
grub-install --version >> "$record/bootloader-inventory.txt" 2>&1
|
||||
grub_rc=$?
|
||||
grep -E '^menuentry |^submenu |linux.*vmlinuz|initrd' /boot/grub/grub.cfg \
|
||||
>> "$record/bootloader-inventory.txt" 2>&1
|
||||
grep_rc=$?
|
||||
if [ "$grub_rc" -eq 0 ] && [ "$grep_rc" -eq 0 ]; then found=1; fi
|
||||
fi
|
||||
if [ "$found" -ne 1 ]; then
|
||||
add_failure 'bootloader-inventory|no-verified-supported-bootloader'
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
capture_unit_baseline() {
|
||||
systemctl list-units --type=service --all --no-legend --no-pager \
|
||||
> "$record/all-service-units.txt" 2>&1
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
add_failure "service-unit-inventory|rc=$rc"
|
||||
return 0
|
||||
fi
|
||||
: > "$record/critical-units-baseline.psv"
|
||||
awk '{print $1}' "$record/all-service-units.txt" | while IFS= read -r unit; do
|
||||
case "$unit" in
|
||||
docker.service|nginx.service|php-fpm.service|mariadb.service|mysqld.service|postgresql.service|postfix.service|dovecot.service|rspamd.service|forgejo.service|sshd.service)
|
||||
;;
|
||||
*) continue ;;
|
||||
esac
|
||||
load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null)
|
||||
active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null)
|
||||
sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null)
|
||||
printf '%s|%s|%s|%s\n' "$unit" "$load" "$active" "$sub"
|
||||
done > "$record/critical-units-baseline.psv"
|
||||
for unit in ${EXTRA_CRITICAL_UNITS:-}; do
|
||||
load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null)
|
||||
if [ "$load" = loaded ]; then
|
||||
active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null)
|
||||
sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null)
|
||||
printf '%s|%s|%s|%s\n' "$unit" "$load" "$active" "$sub" \
|
||||
>> "$record/critical-units-baseline.psv"
|
||||
else
|
||||
add_failure "extra-critical-unit|not-loaded|unit=$unit"
|
||||
fi
|
||||
done
|
||||
sort -u -o "$record/critical-units-baseline.psv" "$record/critical-units-baseline.psv"
|
||||
return 0
|
||||
}
|
||||
|
||||
capture_database_baseline() {
|
||||
: > "$record/database-baseline.psv"
|
||||
while IFS='|' read -r unit load active sub; do
|
||||
[ "$active" = active ] || continue
|
||||
case "$unit" in
|
||||
mariadb.service|mysqld.service) printf 'unit|mysql|%s|%s|%s\n' "$unit" "$active" "$sub" ;;
|
||||
postgresql.service) printf 'unit|postgresql|%s|%s|%s\n' "$unit" "$active" "$sub" ;;
|
||||
esac
|
||||
done < "$record/critical-units-baseline.psv" >> "$record/database-baseline.psv"
|
||||
while IFS='|' read -r name image status ports; do
|
||||
case "$status" in Up\ *) ;; *) continue ;; esac
|
||||
lowered=$(printf '%s %s' "$name" "$image" | tr '[:upper:]' '[:lower:]')
|
||||
case "$lowered" in
|
||||
*maria*|*mysql*) printf 'container|mysql|%s|%s|%s\n' "$name" "$status" "$ports" ;;
|
||||
*postgres*) printf 'container|postgresql|%s|%s|%s\n' "$name" "$status" "$ports" ;;
|
||||
esac
|
||||
done < "$record/containers-all.psv" >> "$record/database-baseline.psv"
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_database_baseline() {
|
||||
: > "$record/database-readiness-before.txt"
|
||||
while IFS='|' read -r model engine identity state detail; do
|
||||
[ "$model" = unit ] || continue
|
||||
if [ "$engine" = mysql ]; then
|
||||
mysqladmin ping >> "$record/database-readiness-before.txt" 2>&1
|
||||
rc=$?; [ "$rc" -eq 0 ] || add_failure "mysqladmin-ping|unit=$identity|rc=$rc"
|
||||
elif [ "$engine" = postgresql ]; then
|
||||
pg_isready >> "$record/database-readiness-before.txt" 2>&1
|
||||
rc=$?; [ "$rc" -eq 0 ] || add_failure "pg-isready|unit=$identity|rc=$rc"
|
||||
fi
|
||||
done < "$record/database-baseline.psv"
|
||||
while IFS='|' read -r model engine identity state detail; do
|
||||
[ "$model" = container ] || continue
|
||||
running=$(docker inspect -f '{{.State.Running}}' "$identity" 2>/dev/null)
|
||||
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$identity" 2>/dev/null)
|
||||
printf '%s|%s|running=%s|health=%s\n' "$engine" "$identity" "${running:-missing}" "${health:-unknown}" \
|
||||
>> "$record/database-readiness-before.txt"
|
||||
if [ "$running" != true ] || [ "$health" = unhealthy ]; then
|
||||
add_failure "database-container|name=$identity|running=${running:-missing}|health=${health:-unknown}"
|
||||
fi
|
||||
done < "$record/database-baseline.psv"
|
||||
return 0
|
||||
}
|
||||
|
||||
main() {
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
printf 'HARD_STOP: run as a separate root process so records are root-only\n' >&2
|
||||
return 10
|
||||
fi
|
||||
umask 077
|
||||
install -d -o root -g root -m 0700 /var/log/le-phase2a-preflight
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
printf 'HARD_STOP: cannot create root-only preflight record parent\n' >&2
|
||||
return 11
|
||||
fi
|
||||
record=$(mktemp -d "/var/log/le-phase2a-preflight/preflight.$(date -u +%Y%m%dT%H%M%SZ).XXXXXXXX")
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ] || [ -z "$record" ] || [ ! -d "$record" ]; then
|
||||
printf 'HARD_STOP: cannot atomically create unique preflight record\n' >&2
|
||||
return 12
|
||||
fi
|
||||
chown root:root "$record"
|
||||
chmod 0700 "$record"
|
||||
|
||||
overall=PASS
|
||||
: > "$record/failed-mandatory-checks.txt"
|
||||
: > "$record/warnings.txt"
|
||||
timestamp_epoch=$(date +%s)
|
||||
timestamp_utc=$(date -u +%FT%TZ)
|
||||
hostname_value=$(hostname)
|
||||
boot_id=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null)
|
||||
running_kernel=$(uname -r)
|
||||
[ -n "$hostname_value" ] || add_failure 'hostname|empty'
|
||||
[ -n "$boot_id" ] || add_failure 'boot-id|unavailable'
|
||||
[ -n "$running_kernel" ] || add_failure 'running-kernel|unavailable'
|
||||
[ "${LE_PHASE2A_RECOVERY_READY:-}" = YES ] || add_failure 'recovery-access-attestation|missing'
|
||||
[ "${LE_PHASE2A_BACKUPS_READY:-}" = YES ] || add_failure 'backup-readiness-attestation|missing'
|
||||
|
||||
mandatory 'uptime' uptime.txt uptime
|
||||
mandatory 'logged-in-sessions' logged-in-sessions.txt who -a
|
||||
mandatory 'filesystem-capacity' filesystem-capacity.txt df -hT / /boot /var/cache/pacman/pkg /tmp
|
||||
mandatory 'filesystem-inodes' filesystem-inodes.txt df -hi / /boot /var/cache/pacman/pkg /tmp
|
||||
mandatory 'installed-package-inventory' packages-before.txt pacman -Q
|
||||
mandatory 'failed-unit-inventory' failed-units-before.txt systemctl --failed --no-legend --plain --no-pager
|
||||
if [ -s "$record/failed-units-before.txt" ]; then add_failure 'failed-units|present'; fi
|
||||
mandatory_shell 'kernel-initramfs-inventory' kernel-initramfs-before.txt \
|
||||
"find /usr/lib/modules /boot -maxdepth 2 -type f \\
|
||||
\( -name vmlinuz -o -name 'vmlinuz-*' -o -name 'initramfs*' -o -name 'initrd*' -o -name pkgbase \) \\
|
||||
-printf '%p size=%s mtime=%TY-%Tm-%TdT%TH:%TM:%TS\\n' | sort; test \"\$(find /boot -maxdepth 1 -type f \\
|
||||
\( -name 'vmlinuz-*' -o -name 'initramfs*' -o -name 'initrd*' \) | wc -l)\" -gt 0"
|
||||
capture_bootloader
|
||||
optional_shell 'dkms-status' dkms-before.txt 'command -v dkms >/dev/null && dkms status'
|
||||
mandatory_shell 'docker-container-inventory' containers-all.psv \
|
||||
"docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort"
|
||||
mandatory_shell 'running-container-inventory' running-containers-before.txt \
|
||||
"docker ps --format '{{.Names}}' | sort"
|
||||
capture_unit_baseline
|
||||
capture_database_baseline
|
||||
validate_database_baseline
|
||||
|
||||
validate_health_urls
|
||||
check_public_health
|
||||
mandatory_shell 'network-firewall-listener-inventory' network-firewall-before.txt \
|
||||
'findmnt --real; ip -brief address; ip route show table all; ip -6 route show table all; nft list ruleset; ss -lntup'
|
||||
optional_shell 'wireguard-observation' wireguard-before.txt 'wg show'
|
||||
mandatory 'package-database-consistency' pacman-Dk-before.txt pacman -Dk
|
||||
optional_shell 'packaged-file-consistency-observation' pacman-Qkk-before.txt 'pacman -Qkk'
|
||||
grep -E -i 'missing|could not read|permission denied|unreadable' "$record/pacman-Qkk-before.txt" \
|
||||
> "$record/pacman-Qkk-missing-or-unreadable-before.txt" 2>/dev/null
|
||||
if [ -s "$record/pacman-Qkk-missing-or-unreadable-before.txt" ]; then
|
||||
add_failure 'packaged-files-missing-or-unreadable-before-maintenance'
|
||||
fi
|
||||
mandatory_shell 'pacnew-pacsave-inventory' pacnew-pacsave-before.txt \
|
||||
"find /etc -xdev -type f \( -name '*.pacnew' -o -name '*.pacsave' \) -printf '%p\\n' | sort"
|
||||
mandatory_shell 'migration-git-state' migration-git-state.txt \
|
||||
'git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration status --short --branch; git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration rev-parse HEAD; git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration rev-parse origin/migration/le-app-database'
|
||||
mandatory_shell 'preview-report-integrity' preview-report-checksum.txt \
|
||||
"printf '%s %s\\n' 195d74afe121556294318d0ed49e3fb5504088dbb31a4c0786b8adea44f1369d /tmp/le-phase2-package-decision-20260712.Mn7mIx/phase2-package-decision-report.md | sha256sum -c -"
|
||||
|
||||
inventory_files='packages-before.txt failed-units-before.txt kernel-initramfs-before.txt bootloader-inventory.txt containers-all.psv running-containers-before.txt critical-units-baseline.psv database-baseline.psv database-readiness-before.txt health-urls.sanitized public-health.txt network-firewall-before.txt pacman-Dk-before.txt pacman-Qkk-before.txt pacman-Qkk-missing-or-unreadable-before.txt pacnew-pacsave-before.txt migration-git-state.txt preview-report-checksum.txt'
|
||||
(
|
||||
cd "$record" || return 1
|
||||
sha256sum $inventory_files > inventory-hashes.sha256
|
||||
sha256sum -c inventory-hashes.sha256 > inventory-hash-verification.txt
|
||||
)
|
||||
hash_rc=$?
|
||||
if [ "$hash_rc" -ne 0 ]; then
|
||||
add_failure "inventory-hashes|rc=$hash_rc"
|
||||
fi
|
||||
hashes_hash=$(sha256sum "$record/inventory-hashes.sha256" | awk '{print $1}')
|
||||
|
||||
failure_count=$(wc -l < "$record/failed-mandatory-checks.txt")
|
||||
failed_checks_hash=$(sha256sum "$record/failed-mandatory-checks.txt" | awk '{print $1}')
|
||||
{
|
||||
printf 'format=le-phase2a-preflight-v1\n'
|
||||
printf 'hostname=%s\n' "$hostname_value"
|
||||
printf 'boot_id=%s\n' "$boot_id"
|
||||
printf 'running_kernel=%s\n' "$running_kernel"
|
||||
printf 'timestamp_epoch=%s\n' "$timestamp_epoch"
|
||||
printf 'timestamp_utc=%s\n' "$timestamp_utc"
|
||||
printf 'overall=%s\n' "$overall"
|
||||
printf 'failure_count=%s\n' "$failure_count"
|
||||
printf 'failed_checks_file=failed-mandatory-checks.txt\n'
|
||||
printf 'failed_checks_sha256=%s\n' "$failed_checks_hash"
|
||||
printf 'inventory_hashes_sha256=%s\n' "$hashes_hash"
|
||||
} > "$record/result.manifest"
|
||||
(cd "$record" && sha256sum result.manifest > result.manifest.sha256)
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
overall=HARD_STOP
|
||||
printf '%s\n' "result-manifest-checksum|rc=$rc" >> "$record/failed-mandatory-checks.txt"
|
||||
fi
|
||||
chmod -R go-rwx "$record"
|
||||
printf 'PREFLIGHT_RECORD=%s\n' "$record"
|
||||
printf 'PREFLIGHT_RESULT=%s failures=%s\n' "$overall" "$failure_count"
|
||||
if [ "$overall" != PASS ]; then
|
||||
return 20
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Loading…
Add table
Add a link
Reference in a new issue