313 lines
15 KiB
Bash
313 lines
15 KiB
Bash
#!/usr/bin/env bash
|
|
# Read-only validation plus root-only evidence recording. Run as a separate process.
|
|
|
|
add_failure() {
|
|
printf '%s\n' "$1" >> "$post_record/failed-checks.txt"
|
|
failures=$((failures + 1))
|
|
return 0
|
|
}
|
|
|
|
check() {
|
|
label=$1; outfile=$2; shift 2
|
|
"$@" > "$post_record/$outfile" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure "$label|rc=$rc|file=$outfile"; fi
|
|
return 0
|
|
}
|
|
|
|
check_shell() {
|
|
label=$1; outfile=$2; command_text=$3
|
|
/usr/bin/bash -o pipefail -c "$command_text" > "$post_record/$outfile" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure "$label|rc=$rc|file=$outfile"; fi
|
|
return 0
|
|
}
|
|
|
|
capture_failed_unit_fingerprint() {
|
|
output=$1
|
|
systemctl --failed --no-legend --plain --no-pager | awk '{print $1}' | LC_ALL=C sort |
|
|
while IFS= read -r unit; do
|
|
[ -n "$unit" ] || continue
|
|
printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
|
|
"$(systemctl show "$unit" -p Id --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p Type --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p LoadState --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p ActiveState --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p SubState --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p Result --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p InvocationID --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p StateChangeTimestamp --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p FragmentPath --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p ExecMainCode --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p ExecMainStatus --value --no-pager)"
|
|
done > "$output"
|
|
return ${PIPESTATUS[0]}
|
|
}
|
|
|
|
validate_failed_unit_disposition() {
|
|
(cd "$script_dir/failed-unit-disposition" && sha256sum -c SHA256SUMS) \
|
|
> "$post_record/failed-unit-allowlist-checksum.txt" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure 'failed-unit-allowlist-checksum'; return 0; fi
|
|
capture_failed_unit_fingerprint "$post_record/failed-units-fingerprint.psv"
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure "failed-unit-fingerprint|rc=$rc"; return 0; fi
|
|
diff -u "$script_dir/failed-unit-disposition/drkonqi-failed-units.allowlist.psv" \
|
|
"$post_record/failed-units-fingerprint.psv" > "$post_record/failed-unit-disposition.diff" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure 'failed-unit-disposition-changed-after-reboot'; fi
|
|
return 0
|
|
}
|
|
|
|
validate_health_url_file() {
|
|
file=$1
|
|
if [ ! -s "$file" ]; then add_failure 'health-urls|missing'; return 0; fi
|
|
while IFS= read -r url; do
|
|
case "$url" in https://*) ;; *) add_failure 'health-url|unsupported-scheme'; continue ;; esac
|
|
authority=${url#https://}; authority=${authority%%/*}
|
|
case "$url" in *'?'*|*'#'*|*$'\n'*|*$'\r'*|*$'\t'*|*' '*) add_failure 'health-url|unsafe-content'; continue ;; esac
|
|
case "$authority" in ''|*@*) add_failure 'health-url|userinfo-or-empty-authority'; continue ;; esac
|
|
done < "$file"
|
|
return 0
|
|
}
|
|
|
|
capture_container_fingerprint() {
|
|
output=$1
|
|
: > "$output"
|
|
docker ps -aq | while IFS= read -r cid; do
|
|
[ -n "$cid" ] || continue
|
|
base=$(docker inspect --format \
|
|
'{{.Name}}|{{.Id}}|{{.Config.Image}}|{{.Image}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}|{{.State.StartedAt}}|{{.HostConfig.NetworkMode}}|{{.HostConfig.RestartPolicy.Name}}|{{index .Config.Labels "com.docker.compose.project"}}' \
|
|
"$cid" 2>/dev/null)
|
|
inspect_rc=$?
|
|
health_hash=$(docker inspect --format '{{json .Config.Healthcheck}}' "$cid" 2>/dev/null | sha256sum | awk '{print $1}')
|
|
hash_rc=$?
|
|
if [ "$inspect_rc" -ne 0 ] || [ "$hash_rc" -ne 0 ] || [ -z "$base" ] || [ -z "$health_hash" ]; then return 1; fi
|
|
printf '%s|healthcheck_sha256=%s\n' "${base#/}" "$health_hash"
|
|
done | LC_ALL=C sort > "$output"
|
|
pipeline_status="${PIPESTATUS[*]}"
|
|
case "$pipeline_status" in '0 0 0') return 0 ;; *) return 1 ;; esac
|
|
}
|
|
|
|
compare_containers() {
|
|
capture_container_fingerprint "$post_record/container-fingerprint-after.psv"
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure "container-fingerprint-after|rc=$rc"; return 0; fi
|
|
: > "$post_record/container-comparison.psv"
|
|
: > "$post_record/restart-policy-no-attention.txt"
|
|
awk -F'|' '{print $1}' "$baseline/container-fingerprint.psv" > "$post_record/container-names-before.txt"
|
|
awk -F'|' '{print $1}' "$post_record/container-fingerprint-after.psv" > "$post_record/container-names-after.txt"
|
|
diff -u "$post_record/container-names-before.txt" "$post_record/container-names-after.txt" \
|
|
> "$post_record/container-name-set.diff" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure 'container-name-set-changed'; fi
|
|
while IFS='|' read -r name old_id old_image_ref old_image_id old_state old_health old_started old_network old_restart old_project old_health_hash; do
|
|
[ -n "$name" ] || continue
|
|
current=$(awk -F'|' -v name="$name" '$1==name {print}' "$post_record/container-fingerprint-after.psv")
|
|
if [ -z "$current" ]; then
|
|
add_failure "container-missing|name=$name"
|
|
[ "$old_restart" = no ] && printf '%s|missing|manual-operator-attention-required\n' "$name" >> "$post_record/restart-policy-no-attention.txt"
|
|
continue
|
|
fi
|
|
IFS='|' read -r new_name new_id new_image_ref new_image_id new_state new_health new_started new_network new_restart new_project new_health_hash <<EOF
|
|
$current
|
|
EOF
|
|
printf '%s|before=%s,%s|after=%s,%s|restart=%s\n' "$name" "$old_state" "$old_health" "$new_state" "$new_health" "$new_restart" \
|
|
>> "$post_record/container-comparison.psv"
|
|
if [ "$old_id" != "$new_id" ] || [ "$old_image_ref" != "$new_image_ref" ] || \
|
|
[ "$old_image_id" != "$new_image_id" ] || [ "$old_network" != "$new_network" ] || \
|
|
[ "$old_restart" != "$new_restart" ] || [ "$old_project" != "$new_project" ] || \
|
|
[ "$old_health_hash" != "$new_health_hash" ]; then
|
|
add_failure "container-identity-or-policy-changed|name=$name"
|
|
continue
|
|
fi
|
|
if [ "$name" = directus ]; then
|
|
[ "$new_state" = exited ] || add_failure "retired-directus-state|expected=exited|actual=$new_state"
|
|
elif [ "$name" = sonarr ]; then
|
|
if [ "$new_state" != running ]; then add_failure "sonarr-not-running|state=$new_state"; fi
|
|
case "$new_health" in unhealthy|healthy) ;; *) add_failure "sonarr-health-worsened|health=$new_health" ;; esac
|
|
elif [ "$old_state" = running ]; then
|
|
if [ "$new_state" != running ]; then
|
|
add_failure "previously-running-container-not-running|name=$name|state=$new_state"
|
|
[ "$old_restart" = no ] && printf '%s|%s|manual-operator-attention-required\n' "$name" "$new_state" >> "$post_record/restart-policy-no-attention.txt"
|
|
elif [ "$old_health" = healthy ] && [ "$new_health" != healthy ]; then
|
|
add_failure "healthy-container-regressed|name=$name|health=$new_health"
|
|
elif [ "$old_health" = no-healthcheck ] && [ "$new_health" != no-healthcheck ]; then
|
|
add_failure "container-healthcheck-identity-changed|name=$name|health=$new_health"
|
|
fi
|
|
fi
|
|
done < "$baseline/container-fingerprint.psv"
|
|
return 0
|
|
}
|
|
|
|
compare_units() {
|
|
: > "$post_record/unit-comparison.psv"
|
|
while IFS='|' read -r unit old_load old_active old_sub; do
|
|
[ -n "$unit" ] || continue
|
|
new_load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null)
|
|
new_active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null)
|
|
new_sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null)
|
|
printf '%s|before=%s,%s,%s|after=%s,%s,%s\n' \
|
|
"$unit" "$old_load" "$old_active" "$old_sub" "${new_load:-missing}" "${new_active:-missing}" "${new_sub:-missing}" \
|
|
>> "$post_record/unit-comparison.psv"
|
|
if [ "$old_load" = loaded ] && [ "$new_load" != loaded ]; then
|
|
add_failure "critical-unit-missing|unit=$unit"
|
|
elif [ "$old_active" = active ] && [ "$new_active" != active ]; then
|
|
add_failure "critical-unit-not-active|unit=$unit|state=${new_active:-missing}"
|
|
fi
|
|
done < "$baseline/critical-units-baseline.psv"
|
|
return 0
|
|
}
|
|
|
|
validate_databases() {
|
|
: > "$post_record/database-validation.txt"
|
|
while IFS='|' read -r model engine identity state detail; do
|
|
[ -n "$model" ] || continue
|
|
if [ "$model" = unit ]; then
|
|
active=$(systemctl show "$identity" -p ActiveState --value 2>/dev/null)
|
|
printf '%s|%s|unit=%s|active=%s\n' "$model" "$engine" "$identity" "${active:-missing}" \
|
|
>> "$post_record/database-validation.txt"
|
|
if [ "$active" != active ]; then add_failure "database-unit-not-active|unit=$identity"; fi
|
|
if [ "$engine" = mysql ]; then
|
|
mysqladmin ping >> "$post_record/database-validation.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || add_failure "mysqladmin-ping-failed|unit=$identity|rc=$rc"
|
|
elif [ "$engine" = postgresql ]; then
|
|
pg_isready >> "$post_record/database-validation.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || add_failure "pg-isready-failed|unit=$identity|rc=$rc"
|
|
fi
|
|
elif [ "$model" = container ]; then
|
|
running=$(docker inspect -f '{{.State.Running}}' "$identity" 2>/dev/null)
|
|
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$identity" 2>/dev/null)
|
|
printf '%s|%s|container=%s|running=%s|health=%s\n' \
|
|
"$model" "$engine" "$identity" "${running:-missing}" "${health:-unknown}" \
|
|
>> "$post_record/database-validation.txt"
|
|
if [ "$running" != true ] || [ "$health" = unhealthy ]; then
|
|
add_failure "database-container-failed|container=$identity|running=${running:-missing}|health=${health:-unknown}"
|
|
fi
|
|
fi
|
|
done < "$baseline/database-baseline.psv"
|
|
return 0
|
|
}
|
|
|
|
check_package_files() {
|
|
pacman -Qkk > "$post_record/pacman-Qkk.txt" 2>&1
|
|
qkk_rc=$?
|
|
printf 'pacman-Qkk-return=%s\n' "$qkk_rc" > "$post_record/pacman-Qkk-classification.txt"
|
|
grep -E -i 'missing|could not read|permission denied|unreadable' "$post_record/pacman-Qkk.txt" \
|
|
> "$post_record/pacman-Qkk-missing-or-unreadable.txt" 2>/dev/null
|
|
missing_rc=$?
|
|
grep -E -i 'altered|modified' "$post_record/pacman-Qkk.txt" \
|
|
> "$post_record/pacman-Qkk-modified.txt" 2>/dev/null
|
|
if [ "$missing_rc" -eq 0 ] && [ -s "$post_record/pacman-Qkk-missing-or-unreadable.txt" ]; then
|
|
add_failure 'packaged-files-missing-or-unreadable'
|
|
fi
|
|
printf 'Modified configuration/package files require manual classification; no repair performed.\n' \
|
|
>> "$post_record/pacman-Qkk-classification.txt"
|
|
return 0
|
|
}
|
|
|
|
main() {
|
|
failures=0
|
|
maintenance=${1:-}
|
|
if [ "$(id -u)" -ne 0 ]; then
|
|
printf 'HARD_STOP: run as a separate root process\n' >&2
|
|
return 10
|
|
fi
|
|
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
|
if [ -z "$maintenance" ] || [ ! -d "$maintenance" ] || [ -L "$maintenance" ]; then
|
|
printf 'HARD_STOP: supply the exact maintenance record path\n' >&2
|
|
return 11
|
|
fi
|
|
owner=$(stat -c %u "$maintenance" 2>/dev/null); mode=$(stat -c %a "$maintenance" 2>/dev/null)
|
|
if [ "$owner" != 0 ] || [ "$mode" != 700 ]; then
|
|
printf 'HARD_STOP: maintenance record must be root-owned mode 0700\n' >&2
|
|
return 12
|
|
fi
|
|
baseline=$maintenance/preflight-record
|
|
required='running-containers-before.txt container-fingerprint.psv directus-stopped.allowlist.psv sonarr-unhealthy.allowlist.psv restart-policy-no.psv critical-units-baseline.psv database-baseline.psv health-urls.sanitized inventory-hashes.sha256 result.manifest result.manifest.sha256'
|
|
for file in $required; do
|
|
if [ ! -f "$baseline/$file" ] || [ -L "$baseline/$file" ]; then
|
|
printf 'HARD_STOP: missing baseline file %s\n' "$file" >&2
|
|
return 13
|
|
fi
|
|
done
|
|
(cd "$baseline" && sha256sum -c inventory-hashes.sha256 > /dev/null 2>&1)
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
printf 'HARD_STOP: copied preflight baseline checksum failure\n' >&2
|
|
return 14
|
|
fi
|
|
(cd "$baseline" && sha256sum -c result.manifest.sha256 > /dev/null 2>&1)
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
printf 'HARD_STOP: copied preflight result manifest checksum failure\n' >&2
|
|
return 14
|
|
fi
|
|
|
|
umask 077
|
|
install -d -o root -g root -m 0700 /var/log/le-phase2a-post-reboot
|
|
rc=$?; [ "$rc" -eq 0 ] || { printf 'HARD_STOP: cannot create post-reboot record parent\n' >&2; return 15; }
|
|
post_record=$(mktemp -d "/var/log/le-phase2a-post-reboot/validation.$(date -u +%Y%m%dT%H%M%SZ).XXXXXXXX")
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ] || [ -z "$post_record" ] || [ ! -d "$post_record" ]; then
|
|
printf 'HARD_STOP: cannot atomically create unique post-reboot record\n' >&2
|
|
return 16
|
|
fi
|
|
chown root:root "$post_record"; chmod 0700 "$post_record"
|
|
: > "$post_record/failed-checks.txt"
|
|
printf 'maintenance_record=%s\n' "$maintenance" > "$post_record/source.txt"
|
|
|
|
check 'running-kernel' running-kernel.txt uname -a
|
|
check_shell 'installed-kernels' installed-kernels.txt 'pacman -Q linux linux-headers linux-zen linux-zen-headers'
|
|
check 'failed-units' failed-units.txt systemctl --failed --no-legend --plain --no-pager
|
|
validate_failed_unit_disposition
|
|
check_shell 'ssh-state' ssh-state.txt \
|
|
'systemctl show sshd.service -p LoadState -p ActiveState -p SubState -p ExecMainStatus; journalctl -b -u sshd.service --no-pager'
|
|
check_shell 'docker-daemon' docker-daemon.txt \
|
|
"systemctl show docker.service -p LoadState -p ActiveState -p SubState -p ExecMainStatus; docker info"
|
|
check_shell 'all-containers' containers-after.psv \
|
|
"docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort"
|
|
compare_containers
|
|
compare_units
|
|
validate_databases
|
|
|
|
check_shell 'mounts-and-capacity' mounts-capacity.txt 'findmnt --real; df -hT / /boot /var/cache/pacman/pkg /tmp'
|
|
check_shell 'network-routes-firewall-ports' network-after.txt \
|
|
'ip -brief address; ip route show table all; ip -6 route show table all; nft list ruleset; ss -lntup'
|
|
wg show > "$post_record/wireguard-after.txt" 2>&1
|
|
wg_rc=$?; if [ "$wg_rc" -ne 0 ]; then printf 'wireguard-observation|rc=%s\n' "$wg_rc" > "$post_record/warnings.txt"; fi
|
|
|
|
validate_health_url_file "$baseline/health-urls.sanitized"
|
|
: > "$post_record/public-health.txt"
|
|
while IFS= read -r url; do
|
|
metrics=$(curl --fail --silent --show-error --max-redirs 0 \
|
|
--connect-timeout 10 --max-time 30 --output /dev/null \
|
|
--write-out 'http=%{http_code}|remote=%{remote_ip}|time=%{time_total}' \
|
|
"$url" 2>> "$post_record/public-health-errors.txt")
|
|
rc=$?
|
|
printf '%s|%s\n' "$url" "$metrics" >> "$post_record/public-health.txt"
|
|
if [ "$rc" -ne 0 ]; then add_failure "public-health-failed|url=$url|rc=$rc"; fi
|
|
done < "$baseline/health-urls.sanitized"
|
|
|
|
pacman -Dk > "$post_record/pacman-Dk.txt" 2>&1
|
|
dk_rc=$?
|
|
if [ "$dk_rc" -ne 0 ]; then add_failure "pacman-Dk-failed|rc=$dk_rc"; fi
|
|
check_package_files
|
|
check_shell 'target-packages' target-packages.txt 'pacman -Q rootlesskit slirp4netns openai-codex'
|
|
check_shell 'pacnew-pacsave-inventory' pacnew-pacsave.txt \
|
|
"find /etc -xdev -type f \( -name '*.pacnew' -o -name '*.pacsave' \) -printf '%p\\n' | sort; pacdiff -o 2>&1"
|
|
|
|
find "$post_record" -maxdepth 1 -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum \
|
|
> "$post_record/SHA256SUMS" 2>&1
|
|
chmod -R go-rwx "$post_record"
|
|
printf 'POST_REBOOT_RECORD=%s\n' "$post_record"
|
|
if [ "$failures" -ne 0 ]; then
|
|
printf 'POST_REBOOT_RESULT=HARD_STOP failures=%s\n' "$failures"
|
|
return 20
|
|
fi
|
|
printf 'POST_REBOOT_RESULT=PASS failures=0\n'
|
|
printf 'Rootless-Docker provisioning remains separately approval-gated.\n'
|
|
return 0
|
|
}
|
|
|
|
main "$@"
|