add Phase 2A host maintenance procedure
This commit is contained in:
parent
13ec935e87
commit
2cf249d6f2
9 changed files with 1119 additions and 3 deletions
|
|
@ -0,0 +1,222 @@
|
|||
#!/usr/bin/env bash
|
||||
# Read-only validation plus root-only evidence recording. Run as a separate process.
|
||||
|
||||
add_failure() {
|
||||
printf '%s\n' "$1" >> "$post_record/failed-checks.txt"
|
||||
failures=$((failures + 1))
|
||||
return 0
|
||||
}
|
||||
|
||||
check() {
|
||||
label=$1; outfile=$2; shift 2
|
||||
"$@" > "$post_record/$outfile" 2>&1
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then add_failure "$label|rc=$rc|file=$outfile"; fi
|
||||
return 0
|
||||
}
|
||||
|
||||
check_shell() {
|
||||
label=$1; outfile=$2; command_text=$3
|
||||
/usr/bin/bash -o pipefail -c "$command_text" > "$post_record/$outfile" 2>&1
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then add_failure "$label|rc=$rc|file=$outfile"; fi
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_health_url_file() {
|
||||
file=$1
|
||||
if [ ! -s "$file" ]; then add_failure 'health-urls|missing'; return 0; fi
|
||||
while IFS= read -r url; do
|
||||
case "$url" in https://*) ;; *) add_failure 'health-url|unsupported-scheme'; continue ;; esac
|
||||
authority=${url#https://}; authority=${authority%%/*}
|
||||
case "$url" in *'?'*|*'#'*|*$'\n'*|*$'\r'*|*$'\t'*|*' '*) add_failure 'health-url|unsafe-content'; continue ;; esac
|
||||
case "$authority" in ''|*@*) add_failure 'health-url|userinfo-or-empty-authority'; continue ;; esac
|
||||
done < "$file"
|
||||
return 0
|
||||
}
|
||||
|
||||
compare_containers() {
|
||||
: > "$post_record/container-comparison.psv"
|
||||
while IFS= read -r container; do
|
||||
[ -n "$container" ] || continue
|
||||
running=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$container" 2>/dev/null)
|
||||
printf '%s|running=%s|health=%s\n' "$container" "${running:-missing}" "${health:-unknown}" \
|
||||
>> "$post_record/container-comparison.psv"
|
||||
if [ "$running" != true ] || [ "$health" = unhealthy ]; then
|
||||
add_failure "container-regression|name=$container|running=${running:-missing}|health=${health:-unknown}"
|
||||
fi
|
||||
done < "$baseline/running-containers-before.txt"
|
||||
return 0
|
||||
}
|
||||
|
||||
compare_units() {
|
||||
: > "$post_record/unit-comparison.psv"
|
||||
while IFS='|' read -r unit old_load old_active old_sub; do
|
||||
[ -n "$unit" ] || continue
|
||||
new_load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null)
|
||||
new_active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null)
|
||||
new_sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null)
|
||||
printf '%s|before=%s,%s,%s|after=%s,%s,%s\n' \
|
||||
"$unit" "$old_load" "$old_active" "$old_sub" "${new_load:-missing}" "${new_active:-missing}" "${new_sub:-missing}" \
|
||||
>> "$post_record/unit-comparison.psv"
|
||||
if [ "$old_load" = loaded ] && [ "$new_load" != loaded ]; then
|
||||
add_failure "critical-unit-missing|unit=$unit"
|
||||
elif [ "$old_active" = active ] && [ "$new_active" != active ]; then
|
||||
add_failure "critical-unit-not-active|unit=$unit|state=${new_active:-missing}"
|
||||
fi
|
||||
done < "$baseline/critical-units-baseline.psv"
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_databases() {
|
||||
: > "$post_record/database-validation.txt"
|
||||
while IFS='|' read -r model engine identity state detail; do
|
||||
[ -n "$model" ] || continue
|
||||
if [ "$model" = unit ]; then
|
||||
active=$(systemctl show "$identity" -p ActiveState --value 2>/dev/null)
|
||||
printf '%s|%s|unit=%s|active=%s\n' "$model" "$engine" "$identity" "${active:-missing}" \
|
||||
>> "$post_record/database-validation.txt"
|
||||
if [ "$active" != active ]; then add_failure "database-unit-not-active|unit=$identity"; fi
|
||||
if [ "$engine" = mysql ]; then
|
||||
mysqladmin ping >> "$post_record/database-validation.txt" 2>&1
|
||||
rc=$?; [ "$rc" -eq 0 ] || add_failure "mysqladmin-ping-failed|unit=$identity|rc=$rc"
|
||||
elif [ "$engine" = postgresql ]; then
|
||||
pg_isready >> "$post_record/database-validation.txt" 2>&1
|
||||
rc=$?; [ "$rc" -eq 0 ] || add_failure "pg-isready-failed|unit=$identity|rc=$rc"
|
||||
fi
|
||||
elif [ "$model" = container ]; then
|
||||
running=$(docker inspect -f '{{.State.Running}}' "$identity" 2>/dev/null)
|
||||
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$identity" 2>/dev/null)
|
||||
printf '%s|%s|container=%s|running=%s|health=%s\n' \
|
||||
"$model" "$engine" "$identity" "${running:-missing}" "${health:-unknown}" \
|
||||
>> "$post_record/database-validation.txt"
|
||||
if [ "$running" != true ] || [ "$health" = unhealthy ]; then
|
||||
add_failure "database-container-failed|container=$identity|running=${running:-missing}|health=${health:-unknown}"
|
||||
fi
|
||||
fi
|
||||
done < "$baseline/database-baseline.psv"
|
||||
return 0
|
||||
}
|
||||
|
||||
check_package_files() {
|
||||
pacman -Qkk > "$post_record/pacman-Qkk.txt" 2>&1
|
||||
qkk_rc=$?
|
||||
printf 'pacman-Qkk-return=%s\n' "$qkk_rc" > "$post_record/pacman-Qkk-classification.txt"
|
||||
grep -E -i 'missing|could not read|permission denied|unreadable' "$post_record/pacman-Qkk.txt" \
|
||||
> "$post_record/pacman-Qkk-missing-or-unreadable.txt" 2>/dev/null
|
||||
missing_rc=$?
|
||||
grep -E -i 'altered|modified' "$post_record/pacman-Qkk.txt" \
|
||||
> "$post_record/pacman-Qkk-modified.txt" 2>/dev/null
|
||||
if [ "$missing_rc" -eq 0 ] && [ -s "$post_record/pacman-Qkk-missing-or-unreadable.txt" ]; then
|
||||
add_failure 'packaged-files-missing-or-unreadable'
|
||||
fi
|
||||
printf 'Modified configuration/package files require manual classification; no repair performed.\n' \
|
||||
>> "$post_record/pacman-Qkk-classification.txt"
|
||||
return 0
|
||||
}
|
||||
|
||||
main() {
|
||||
failures=0
|
||||
maintenance=${1:-}
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
printf 'HARD_STOP: run as a separate root process\n' >&2
|
||||
return 10
|
||||
fi
|
||||
if [ -z "$maintenance" ] || [ ! -d "$maintenance" ] || [ -L "$maintenance" ]; then
|
||||
printf 'HARD_STOP: supply the exact maintenance record path\n' >&2
|
||||
return 11
|
||||
fi
|
||||
owner=$(stat -c %u "$maintenance" 2>/dev/null); mode=$(stat -c %a "$maintenance" 2>/dev/null)
|
||||
if [ "$owner" != 0 ] || [ "$mode" != 700 ]; then
|
||||
printf 'HARD_STOP: maintenance record must be root-owned mode 0700\n' >&2
|
||||
return 12
|
||||
fi
|
||||
baseline=$maintenance/preflight-record
|
||||
required='running-containers-before.txt critical-units-baseline.psv database-baseline.psv health-urls.sanitized inventory-hashes.sha256 result.manifest result.manifest.sha256'
|
||||
for file in $required; do
|
||||
if [ ! -f "$baseline/$file" ] || [ -L "$baseline/$file" ]; then
|
||||
printf 'HARD_STOP: missing baseline file %s\n' "$file" >&2
|
||||
return 13
|
||||
fi
|
||||
done
|
||||
(cd "$baseline" && sha256sum -c inventory-hashes.sha256 > /dev/null 2>&1)
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
printf 'HARD_STOP: copied preflight baseline checksum failure\n' >&2
|
||||
return 14
|
||||
fi
|
||||
(cd "$baseline" && sha256sum -c result.manifest.sha256 > /dev/null 2>&1)
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
printf 'HARD_STOP: copied preflight result manifest checksum failure\n' >&2
|
||||
return 14
|
||||
fi
|
||||
|
||||
umask 077
|
||||
install -d -o root -g root -m 0700 /var/log/le-phase2a-post-reboot
|
||||
rc=$?; [ "$rc" -eq 0 ] || { printf 'HARD_STOP: cannot create post-reboot record parent\n' >&2; return 15; }
|
||||
post_record=$(mktemp -d "/var/log/le-phase2a-post-reboot/validation.$(date -u +%Y%m%dT%H%M%SZ).XXXXXXXX")
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ] || [ -z "$post_record" ] || [ ! -d "$post_record" ]; then
|
||||
printf 'HARD_STOP: cannot atomically create unique post-reboot record\n' >&2
|
||||
return 16
|
||||
fi
|
||||
chown root:root "$post_record"; chmod 0700 "$post_record"
|
||||
: > "$post_record/failed-checks.txt"
|
||||
printf 'maintenance_record=%s\n' "$maintenance" > "$post_record/source.txt"
|
||||
|
||||
check 'running-kernel' running-kernel.txt uname -a
|
||||
check_shell 'installed-kernels' installed-kernels.txt 'pacman -Q linux linux-headers linux-zen linux-zen-headers'
|
||||
check 'failed-units' failed-units.txt systemctl --failed --no-legend --plain --no-pager
|
||||
if [ -s "$post_record/failed-units.txt" ]; then add_failure 'failed-units-present'; fi
|
||||
check_shell 'ssh-state' ssh-state.txt \
|
||||
'systemctl show sshd.service -p LoadState -p ActiveState -p SubState -p ExecMainStatus; journalctl -b -u sshd.service --no-pager'
|
||||
check_shell 'docker-daemon' docker-daemon.txt \
|
||||
"systemctl show docker.service -p LoadState -p ActiveState -p SubState -p ExecMainStatus; docker info"
|
||||
check_shell 'all-containers' containers-after.psv \
|
||||
"docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort"
|
||||
compare_containers
|
||||
compare_units
|
||||
validate_databases
|
||||
|
||||
check_shell 'mounts-and-capacity' mounts-capacity.txt 'findmnt --real; df -hT / /boot /var/cache/pacman/pkg /tmp'
|
||||
check_shell 'network-routes-firewall-ports' network-after.txt \
|
||||
'ip -brief address; ip route show table all; ip -6 route show table all; nft list ruleset; ss -lntup'
|
||||
wg show > "$post_record/wireguard-after.txt" 2>&1
|
||||
wg_rc=$?; if [ "$wg_rc" -ne 0 ]; then printf 'wireguard-observation|rc=%s\n' "$wg_rc" > "$post_record/warnings.txt"; fi
|
||||
|
||||
validate_health_url_file "$baseline/health-urls.sanitized"
|
||||
: > "$post_record/public-health.txt"
|
||||
while IFS= read -r url; do
|
||||
metrics=$(curl --fail --silent --show-error --max-redirs 0 \
|
||||
--connect-timeout 10 --max-time 30 --output /dev/null \
|
||||
--write-out 'http=%{http_code}|remote=%{remote_ip}|time=%{time_total}' \
|
||||
"$url" 2>> "$post_record/public-health-errors.txt")
|
||||
rc=$?
|
||||
printf '%s|%s\n' "$url" "$metrics" >> "$post_record/public-health.txt"
|
||||
if [ "$rc" -ne 0 ]; then add_failure "public-health-failed|url=$url|rc=$rc"; fi
|
||||
done < "$baseline/health-urls.sanitized"
|
||||
|
||||
pacman -Dk > "$post_record/pacman-Dk.txt" 2>&1
|
||||
dk_rc=$?
|
||||
if [ "$dk_rc" -ne 0 ]; then add_failure "pacman-Dk-failed|rc=$dk_rc"; fi
|
||||
check_package_files
|
||||
check_shell 'target-packages' target-packages.txt 'pacman -Q rootlesskit slirp4netns openai-codex'
|
||||
check_shell 'pacnew-pacsave-inventory' pacnew-pacsave.txt \
|
||||
"find /etc -xdev -type f \( -name '*.pacnew' -o -name '*.pacsave' \) -printf '%p\\n' | sort; pacdiff -o 2>&1"
|
||||
|
||||
find "$post_record" -maxdepth 1 -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum \
|
||||
> "$post_record/SHA256SUMS" 2>&1
|
||||
chmod -R go-rwx "$post_record"
|
||||
printf 'POST_REBOOT_RECORD=%s\n' "$post_record"
|
||||
if [ "$failures" -ne 0 ]; then
|
||||
printf 'POST_REBOOT_RESULT=HARD_STOP failures=%s\n' "$failures"
|
||||
return 20
|
||||
fi
|
||||
printf 'POST_REBOOT_RESULT=PASS failures=0\n'
|
||||
printf 'Rootless-Docker provisioning remains separately approval-gated.\n'
|
||||
return 0
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Loading…
Add table
Add a link
Reference in a new issue