#!/usr/bin/env bash # Read-only validation plus root-only evidence recording. Run as a separate process. add_failure() { printf '%s\n' "$1" >> "$post_record/failed-checks.txt" failures=$((failures + 1)) return 0 } check() { label=$1; outfile=$2; shift 2 "$@" > "$post_record/$outfile" 2>&1 rc=$? if [ "$rc" -ne 0 ]; then add_failure "$label|rc=$rc|file=$outfile"; fi return 0 } check_shell() { label=$1; outfile=$2; command_text=$3 /usr/bin/bash -o pipefail -c "$command_text" > "$post_record/$outfile" 2>&1 rc=$? if [ "$rc" -ne 0 ]; then add_failure "$label|rc=$rc|file=$outfile"; fi return 0 } validate_health_url_file() { file=$1 if [ ! -s "$file" ]; then add_failure 'health-urls|missing'; return 0; fi while IFS= read -r url; do case "$url" in https://*) ;; *) add_failure 'health-url|unsupported-scheme'; continue ;; esac authority=${url#https://}; authority=${authority%%/*} case "$url" in *'?'*|*'#'*|*$'\n'*|*$'\r'*|*$'\t'*|*' '*) add_failure 'health-url|unsafe-content'; continue ;; esac case "$authority" in ''|*@*) add_failure 'health-url|userinfo-or-empty-authority'; continue ;; esac done < "$file" return 0 } compare_containers() { : > "$post_record/container-comparison.psv" while IFS= read -r container; do [ -n "$container" ] || continue running=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$container" 2>/dev/null) printf '%s|running=%s|health=%s\n' "$container" "${running:-missing}" "${health:-unknown}" \ >> "$post_record/container-comparison.psv" if [ "$running" != true ] || [ "$health" = unhealthy ]; then add_failure "container-regression|name=$container|running=${running:-missing}|health=${health:-unknown}" fi done < "$baseline/running-containers-before.txt" return 0 } compare_units() { : > "$post_record/unit-comparison.psv" while IFS='|' read -r unit old_load old_active old_sub; do [ -n "$unit" ] || continue new_load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null) new_active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null) new_sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null) printf '%s|before=%s,%s,%s|after=%s,%s,%s\n' \ "$unit" "$old_load" "$old_active" "$old_sub" "${new_load:-missing}" "${new_active:-missing}" "${new_sub:-missing}" \ >> "$post_record/unit-comparison.psv" if [ "$old_load" = loaded ] && [ "$new_load" != loaded ]; then add_failure "critical-unit-missing|unit=$unit" elif [ "$old_active" = active ] && [ "$new_active" != active ]; then add_failure "critical-unit-not-active|unit=$unit|state=${new_active:-missing}" fi done < "$baseline/critical-units-baseline.psv" return 0 } validate_databases() { : > "$post_record/database-validation.txt" while IFS='|' read -r model engine identity state detail; do [ -n "$model" ] || continue if [ "$model" = unit ]; then active=$(systemctl show "$identity" -p ActiveState --value 2>/dev/null) printf '%s|%s|unit=%s|active=%s\n' "$model" "$engine" "$identity" "${active:-missing}" \ >> "$post_record/database-validation.txt" if [ "$active" != active ]; then add_failure "database-unit-not-active|unit=$identity"; fi if [ "$engine" = mysql ]; then mysqladmin ping >> "$post_record/database-validation.txt" 2>&1 rc=$?; [ "$rc" -eq 0 ] || add_failure "mysqladmin-ping-failed|unit=$identity|rc=$rc" elif [ "$engine" = postgresql ]; then pg_isready >> "$post_record/database-validation.txt" 2>&1 rc=$?; [ "$rc" -eq 0 ] || add_failure "pg-isready-failed|unit=$identity|rc=$rc" fi elif [ "$model" = container ]; then running=$(docker inspect -f '{{.State.Running}}' "$identity" 2>/dev/null) health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$identity" 2>/dev/null) printf '%s|%s|container=%s|running=%s|health=%s\n' \ "$model" "$engine" "$identity" "${running:-missing}" "${health:-unknown}" \ >> "$post_record/database-validation.txt" if [ "$running" != true ] || [ "$health" = unhealthy ]; then add_failure "database-container-failed|container=$identity|running=${running:-missing}|health=${health:-unknown}" fi fi done < "$baseline/database-baseline.psv" return 0 } check_package_files() { pacman -Qkk > "$post_record/pacman-Qkk.txt" 2>&1 qkk_rc=$? printf 'pacman-Qkk-return=%s\n' "$qkk_rc" > "$post_record/pacman-Qkk-classification.txt" grep -E -i 'missing|could not read|permission denied|unreadable' "$post_record/pacman-Qkk.txt" \ > "$post_record/pacman-Qkk-missing-or-unreadable.txt" 2>/dev/null missing_rc=$? grep -E -i 'altered|modified' "$post_record/pacman-Qkk.txt" \ > "$post_record/pacman-Qkk-modified.txt" 2>/dev/null if [ "$missing_rc" -eq 0 ] && [ -s "$post_record/pacman-Qkk-missing-or-unreadable.txt" ]; then add_failure 'packaged-files-missing-or-unreadable' fi printf 'Modified configuration/package files require manual classification; no repair performed.\n' \ >> "$post_record/pacman-Qkk-classification.txt" return 0 } main() { failures=0 maintenance=${1:-} if [ "$(id -u)" -ne 0 ]; then printf 'HARD_STOP: run as a separate root process\n' >&2 return 10 fi if [ -z "$maintenance" ] || [ ! -d "$maintenance" ] || [ -L "$maintenance" ]; then printf 'HARD_STOP: supply the exact maintenance record path\n' >&2 return 11 fi owner=$(stat -c %u "$maintenance" 2>/dev/null); mode=$(stat -c %a "$maintenance" 2>/dev/null) if [ "$owner" != 0 ] || [ "$mode" != 700 ]; then printf 'HARD_STOP: maintenance record must be root-owned mode 0700\n' >&2 return 12 fi baseline=$maintenance/preflight-record required='running-containers-before.txt critical-units-baseline.psv database-baseline.psv health-urls.sanitized inventory-hashes.sha256 result.manifest result.manifest.sha256' for file in $required; do if [ ! -f "$baseline/$file" ] || [ -L "$baseline/$file" ]; then printf 'HARD_STOP: missing baseline file %s\n' "$file" >&2 return 13 fi done (cd "$baseline" && sha256sum -c inventory-hashes.sha256 > /dev/null 2>&1) rc=$? if [ "$rc" -ne 0 ]; then printf 'HARD_STOP: copied preflight baseline checksum failure\n' >&2 return 14 fi (cd "$baseline" && sha256sum -c result.manifest.sha256 > /dev/null 2>&1) rc=$? if [ "$rc" -ne 0 ]; then printf 'HARD_STOP: copied preflight result manifest checksum failure\n' >&2 return 14 fi umask 077 install -d -o root -g root -m 0700 /var/log/le-phase2a-post-reboot rc=$?; [ "$rc" -eq 0 ] || { printf 'HARD_STOP: cannot create post-reboot record parent\n' >&2; return 15; } post_record=$(mktemp -d "/var/log/le-phase2a-post-reboot/validation.$(date -u +%Y%m%dT%H%M%SZ).XXXXXXXX") rc=$? if [ "$rc" -ne 0 ] || [ -z "$post_record" ] || [ ! -d "$post_record" ]; then printf 'HARD_STOP: cannot atomically create unique post-reboot record\n' >&2 return 16 fi chown root:root "$post_record"; chmod 0700 "$post_record" : > "$post_record/failed-checks.txt" printf 'maintenance_record=%s\n' "$maintenance" > "$post_record/source.txt" check 'running-kernel' running-kernel.txt uname -a check_shell 'installed-kernels' installed-kernels.txt 'pacman -Q linux linux-headers linux-zen linux-zen-headers' check 'failed-units' failed-units.txt systemctl --failed --no-legend --plain --no-pager if [ -s "$post_record/failed-units.txt" ]; then add_failure 'failed-units-present'; fi check_shell 'ssh-state' ssh-state.txt \ 'systemctl show sshd.service -p LoadState -p ActiveState -p SubState -p ExecMainStatus; journalctl -b -u sshd.service --no-pager' check_shell 'docker-daemon' docker-daemon.txt \ "systemctl show docker.service -p LoadState -p ActiveState -p SubState -p ExecMainStatus; docker info" check_shell 'all-containers' containers-after.psv \ "docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort" compare_containers compare_units validate_databases check_shell 'mounts-and-capacity' mounts-capacity.txt 'findmnt --real; df -hT / /boot /var/cache/pacman/pkg /tmp' check_shell 'network-routes-firewall-ports' network-after.txt \ 'ip -brief address; ip route show table all; ip -6 route show table all; nft list ruleset; ss -lntup' wg show > "$post_record/wireguard-after.txt" 2>&1 wg_rc=$?; if [ "$wg_rc" -ne 0 ]; then printf 'wireguard-observation|rc=%s\n' "$wg_rc" > "$post_record/warnings.txt"; fi validate_health_url_file "$baseline/health-urls.sanitized" : > "$post_record/public-health.txt" while IFS= read -r url; do metrics=$(curl --fail --silent --show-error --max-redirs 0 \ --connect-timeout 10 --max-time 30 --output /dev/null \ --write-out 'http=%{http_code}|remote=%{remote_ip}|time=%{time_total}' \ "$url" 2>> "$post_record/public-health-errors.txt") rc=$? printf '%s|%s\n' "$url" "$metrics" >> "$post_record/public-health.txt" if [ "$rc" -ne 0 ]; then add_failure "public-health-failed|url=$url|rc=$rc"; fi done < "$baseline/health-urls.sanitized" pacman -Dk > "$post_record/pacman-Dk.txt" 2>&1 dk_rc=$? if [ "$dk_rc" -ne 0 ]; then add_failure "pacman-Dk-failed|rc=$dk_rc"; fi check_package_files check_shell 'target-packages' target-packages.txt 'pacman -Q rootlesskit slirp4netns openai-codex' check_shell 'pacnew-pacsave-inventory' pacnew-pacsave.txt \ "find /etc -xdev -type f \( -name '*.pacnew' -o -name '*.pacsave' \) -printf '%p\\n' | sort; pacdiff -o 2>&1" find "$post_record" -maxdepth 1 -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum \ > "$post_record/SHA256SUMS" 2>&1 chmod -R go-rwx "$post_record" printf 'POST_REBOOT_RECORD=%s\n' "$post_record" if [ "$failures" -ne 0 ]; then printf 'POST_REBOOT_RESULT=HARD_STOP failures=%s\n' "$failures" return 20 fi printf 'POST_REBOOT_RESULT=PASS failures=0\n' printf 'Rootless-Docker provisioning remains separately approval-gated.\n' return 0 } main "$@"