325 lines
14 KiB
Bash
325 lines
14 KiB
Bash
#!/usr/bin/env bash
|
|
# Read-only host inspection plus root-only evidence recording.
|
|
# Run as a separate process; this script changes no shell options and uses return only.
|
|
|
|
add_failure() {
|
|
printf '%s\n' "$1" >> "$record/failed-mandatory-checks.txt"
|
|
overall=HARD_STOP
|
|
return 0
|
|
}
|
|
|
|
mandatory() {
|
|
label=$1
|
|
outfile=$2
|
|
shift 2
|
|
"$@" > "$record/$outfile" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
add_failure "$label|rc=$rc|file=$outfile"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
mandatory_shell() {
|
|
label=$1
|
|
outfile=$2
|
|
command_text=$3
|
|
/usr/bin/bash -o pipefail -c "$command_text" > "$record/$outfile" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
add_failure "$label|rc=$rc|file=$outfile"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
optional_shell() {
|
|
label=$1
|
|
outfile=$2
|
|
command_text=$3
|
|
/usr/bin/bash -o pipefail -c "$command_text" > "$record/$outfile" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
printf '%s|rc=%s|file=%s\n' "$label" "$rc" "$outfile" >> "$record/warnings.txt"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
validate_health_urls() {
|
|
: > "$record/health-urls.sanitized"
|
|
if [ -z "${PUBLIC_HEALTH_URLS:-}" ]; then
|
|
add_failure 'public-health-urls|unset'
|
|
return 0
|
|
fi
|
|
case "$PUBLIC_HEALTH_URLS" in
|
|
*$'\n'*|*$'\r'*|*$'\t'*)
|
|
add_failure 'public-health-urls|control-character-rejected'
|
|
return 0
|
|
;;
|
|
esac
|
|
for url in $PUBLIC_HEALTH_URLS; do
|
|
case "$url" in
|
|
https://*) ;;
|
|
*) add_failure 'public-health-url|unsupported-scheme-or-malformed'; continue ;;
|
|
esac
|
|
authority=${url#https://}
|
|
authority=${authority%%/*}
|
|
case "$url" in
|
|
*'?'*|*'#'*|*$'\n'*|*$'\r'*|*$'\t'*|*' '*)
|
|
add_failure 'public-health-url|query-fragment-or-control-character-rejected'
|
|
continue
|
|
;;
|
|
esac
|
|
case "$authority" in
|
|
''|*@*) add_failure 'public-health-url|empty-authority-or-userinfo-rejected'; continue ;;
|
|
esac
|
|
printf '%s\n' "$url" >> "$record/health-urls.sanitized"
|
|
done
|
|
if [ ! -s "$record/health-urls.sanitized" ]; then
|
|
add_failure 'public-health-urls|no-valid-non-secret-https-url'
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
check_public_health() {
|
|
: > "$record/public-health.txt"
|
|
while IFS= read -r url; do
|
|
[ -n "$url" ] || continue
|
|
metrics=$(curl --fail --silent --show-error --max-redirs 0 \
|
|
--connect-timeout 10 --max-time 30 --output /dev/null \
|
|
--write-out 'http=%{http_code}|remote=%{remote_ip}|time=%{time_total}' \
|
|
"$url" 2>> "$record/public-health-errors.txt")
|
|
rc=$?
|
|
printf '%s|%s\n' "$url" "$metrics" >> "$record/public-health.txt"
|
|
if [ "$rc" -ne 0 ]; then
|
|
add_failure "public-health|rc=$rc|url=$url"
|
|
fi
|
|
done < "$record/health-urls.sanitized"
|
|
return 0
|
|
}
|
|
|
|
capture_bootloader() {
|
|
found=0
|
|
: > "$record/bootloader-inventory.txt"
|
|
if [ -d /boot/loader/entries ]; then
|
|
printf '%s\n' 'type=systemd-boot' >> "$record/bootloader-inventory.txt"
|
|
bootctl status >> "$record/bootloader-inventory.txt" 2>&1
|
|
bootctl_rc=$?
|
|
find /boot/loader/entries -maxdepth 1 -type f -print -exec sed -n '1,160p' {} \; \
|
|
>> "$record/bootloader-inventory.txt" 2>&1
|
|
find_rc=$?
|
|
if [ "$bootctl_rc" -eq 0 ] && [ "$find_rc" -eq 0 ]; then found=1; fi
|
|
fi
|
|
if [ -f /boot/grub/grub.cfg ]; then
|
|
printf '%s\n' 'type=grub' >> "$record/bootloader-inventory.txt"
|
|
grub-install --version >> "$record/bootloader-inventory.txt" 2>&1
|
|
grub_rc=$?
|
|
grep -E '^menuentry |^submenu |linux.*vmlinuz|initrd' /boot/grub/grub.cfg \
|
|
>> "$record/bootloader-inventory.txt" 2>&1
|
|
grep_rc=$?
|
|
if [ "$grub_rc" -eq 0 ] && [ "$grep_rc" -eq 0 ]; then found=1; fi
|
|
fi
|
|
if [ "$found" -ne 1 ]; then
|
|
add_failure 'bootloader-inventory|no-verified-supported-bootloader'
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
capture_unit_baseline() {
|
|
systemctl list-units --type=service --all --no-legend --no-pager \
|
|
> "$record/all-service-units.txt" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
add_failure "service-unit-inventory|rc=$rc"
|
|
return 0
|
|
fi
|
|
: > "$record/critical-units-baseline.psv"
|
|
awk '{print $1}' "$record/all-service-units.txt" | while IFS= read -r unit; do
|
|
case "$unit" in
|
|
docker.service|nginx.service|php-fpm.service|mariadb.service|mysqld.service|postgresql.service|postfix.service|dovecot.service|rspamd.service|forgejo.service|sshd.service)
|
|
;;
|
|
*) continue ;;
|
|
esac
|
|
load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null)
|
|
active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null)
|
|
sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null)
|
|
printf '%s|%s|%s|%s\n' "$unit" "$load" "$active" "$sub"
|
|
done > "$record/critical-units-baseline.psv"
|
|
for unit in ${EXTRA_CRITICAL_UNITS:-}; do
|
|
load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null)
|
|
if [ "$load" = loaded ]; then
|
|
active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null)
|
|
sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null)
|
|
printf '%s|%s|%s|%s\n' "$unit" "$load" "$active" "$sub" \
|
|
>> "$record/critical-units-baseline.psv"
|
|
else
|
|
add_failure "extra-critical-unit|not-loaded|unit=$unit"
|
|
fi
|
|
done
|
|
sort -u -o "$record/critical-units-baseline.psv" "$record/critical-units-baseline.psv"
|
|
return 0
|
|
}
|
|
|
|
capture_database_baseline() {
|
|
: > "$record/database-baseline.psv"
|
|
while IFS='|' read -r unit load active sub; do
|
|
[ "$active" = active ] || continue
|
|
case "$unit" in
|
|
mariadb.service|mysqld.service) printf 'unit|mysql|%s|%s|%s\n' "$unit" "$active" "$sub" ;;
|
|
postgresql.service) printf 'unit|postgresql|%s|%s|%s\n' "$unit" "$active" "$sub" ;;
|
|
esac
|
|
done < "$record/critical-units-baseline.psv" >> "$record/database-baseline.psv"
|
|
while IFS='|' read -r name image status ports; do
|
|
case "$status" in Up\ *) ;; *) continue ;; esac
|
|
lowered=$(printf '%s %s' "$name" "$image" | tr '[:upper:]' '[:lower:]')
|
|
case "$lowered" in
|
|
*maria*|*mysql*) printf 'container|mysql|%s|%s|%s\n' "$name" "$status" "$ports" ;;
|
|
*postgres*) printf 'container|postgresql|%s|%s|%s\n' "$name" "$status" "$ports" ;;
|
|
esac
|
|
done < "$record/containers-all.psv" >> "$record/database-baseline.psv"
|
|
return 0
|
|
}
|
|
|
|
validate_database_baseline() {
|
|
: > "$record/database-readiness-before.txt"
|
|
while IFS='|' read -r model engine identity state detail; do
|
|
[ "$model" = unit ] || continue
|
|
if [ "$engine" = mysql ]; then
|
|
mysqladmin ping >> "$record/database-readiness-before.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || add_failure "mysqladmin-ping|unit=$identity|rc=$rc"
|
|
elif [ "$engine" = postgresql ]; then
|
|
pg_isready >> "$record/database-readiness-before.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || add_failure "pg-isready|unit=$identity|rc=$rc"
|
|
fi
|
|
done < "$record/database-baseline.psv"
|
|
while IFS='|' read -r model engine identity state detail; do
|
|
[ "$model" = container ] || continue
|
|
running=$(docker inspect -f '{{.State.Running}}' "$identity" 2>/dev/null)
|
|
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$identity" 2>/dev/null)
|
|
printf '%s|%s|running=%s|health=%s\n' "$engine" "$identity" "${running:-missing}" "${health:-unknown}" \
|
|
>> "$record/database-readiness-before.txt"
|
|
if [ "$running" != true ] || [ "$health" = unhealthy ]; then
|
|
add_failure "database-container|name=$identity|running=${running:-missing}|health=${health:-unknown}"
|
|
fi
|
|
done < "$record/database-baseline.psv"
|
|
return 0
|
|
}
|
|
|
|
main() {
|
|
if [ "$(id -u)" -ne 0 ]; then
|
|
printf 'HARD_STOP: run as a separate root process so records are root-only\n' >&2
|
|
return 10
|
|
fi
|
|
umask 077
|
|
install -d -o root -g root -m 0700 /var/log/le-phase2a-preflight
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
printf 'HARD_STOP: cannot create root-only preflight record parent\n' >&2
|
|
return 11
|
|
fi
|
|
record=$(mktemp -d "/var/log/le-phase2a-preflight/preflight.$(date -u +%Y%m%dT%H%M%SZ).XXXXXXXX")
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ] || [ -z "$record" ] || [ ! -d "$record" ]; then
|
|
printf 'HARD_STOP: cannot atomically create unique preflight record\n' >&2
|
|
return 12
|
|
fi
|
|
chown root:root "$record"
|
|
chmod 0700 "$record"
|
|
|
|
overall=PASS
|
|
: > "$record/failed-mandatory-checks.txt"
|
|
: > "$record/warnings.txt"
|
|
timestamp_epoch=$(date +%s)
|
|
timestamp_utc=$(date -u +%FT%TZ)
|
|
hostname_value=$(hostname)
|
|
boot_id=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null)
|
|
running_kernel=$(uname -r)
|
|
[ -n "$hostname_value" ] || add_failure 'hostname|empty'
|
|
[ -n "$boot_id" ] || add_failure 'boot-id|unavailable'
|
|
[ -n "$running_kernel" ] || add_failure 'running-kernel|unavailable'
|
|
[ "${LE_PHASE2A_RECOVERY_READY:-}" = YES ] || add_failure 'recovery-access-attestation|missing'
|
|
[ "${LE_PHASE2A_BACKUPS_READY:-}" = YES ] || add_failure 'backup-readiness-attestation|missing'
|
|
|
|
mandatory 'uptime' uptime.txt uptime
|
|
mandatory 'logged-in-sessions' logged-in-sessions.txt who -a
|
|
mandatory 'filesystem-capacity' filesystem-capacity.txt df -hT / /boot /var/cache/pacman/pkg /tmp
|
|
mandatory 'filesystem-inodes' filesystem-inodes.txt df -hi / /boot /var/cache/pacman/pkg /tmp
|
|
mandatory 'installed-package-inventory' packages-before.txt pacman -Q
|
|
mandatory 'failed-unit-inventory' failed-units-before.txt systemctl --failed --no-legend --plain --no-pager
|
|
if [ -s "$record/failed-units-before.txt" ]; then add_failure 'failed-units|present'; fi
|
|
mandatory_shell 'kernel-initramfs-inventory' kernel-initramfs-before.txt \
|
|
"find /usr/lib/modules /boot -maxdepth 2 -type f \\
|
|
\( -name vmlinuz -o -name 'vmlinuz-*' -o -name 'initramfs*' -o -name 'initrd*' -o -name pkgbase \) \\
|
|
-printf '%p size=%s mtime=%TY-%Tm-%TdT%TH:%TM:%TS\\n' | sort; test \"\$(find /boot -maxdepth 1 -type f \\
|
|
\( -name 'vmlinuz-*' -o -name 'initramfs*' -o -name 'initrd*' \) | wc -l)\" -gt 0"
|
|
capture_bootloader
|
|
optional_shell 'dkms-status' dkms-before.txt 'command -v dkms >/dev/null && dkms status'
|
|
mandatory_shell 'docker-container-inventory' containers-all.psv \
|
|
"docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort"
|
|
mandatory_shell 'running-container-inventory' running-containers-before.txt \
|
|
"docker ps --format '{{.Names}}' | sort"
|
|
capture_unit_baseline
|
|
capture_database_baseline
|
|
validate_database_baseline
|
|
|
|
validate_health_urls
|
|
check_public_health
|
|
mandatory_shell 'network-firewall-listener-inventory' network-firewall-before.txt \
|
|
'findmnt --real; ip -brief address; ip route show table all; ip -6 route show table all; nft list ruleset; ss -lntup'
|
|
optional_shell 'wireguard-observation' wireguard-before.txt 'wg show'
|
|
mandatory 'package-database-consistency' pacman-Dk-before.txt pacman -Dk
|
|
optional_shell 'packaged-file-consistency-observation' pacman-Qkk-before.txt 'pacman -Qkk'
|
|
grep -E -i 'missing|could not read|permission denied|unreadable' "$record/pacman-Qkk-before.txt" \
|
|
> "$record/pacman-Qkk-missing-or-unreadable-before.txt" 2>/dev/null
|
|
if [ -s "$record/pacman-Qkk-missing-or-unreadable-before.txt" ]; then
|
|
add_failure 'packaged-files-missing-or-unreadable-before-maintenance'
|
|
fi
|
|
mandatory_shell 'pacnew-pacsave-inventory' pacnew-pacsave-before.txt \
|
|
"find /etc -xdev -type f \( -name '*.pacnew' -o -name '*.pacsave' \) -printf '%p\\n' | sort"
|
|
mandatory_shell 'migration-git-state' migration-git-state.txt \
|
|
'git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration status --short --branch; git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration rev-parse HEAD; git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration rev-parse origin/migration/le-app-database'
|
|
mandatory_shell 'preview-report-integrity' preview-report-checksum.txt \
|
|
"printf '%s %s\\n' 195d74afe121556294318d0ed49e3fb5504088dbb31a4c0786b8adea44f1369d /tmp/le-phase2-package-decision-20260712.Mn7mIx/phase2-package-decision-report.md | sha256sum -c -"
|
|
|
|
inventory_files='packages-before.txt failed-units-before.txt kernel-initramfs-before.txt bootloader-inventory.txt containers-all.psv running-containers-before.txt critical-units-baseline.psv database-baseline.psv database-readiness-before.txt health-urls.sanitized public-health.txt network-firewall-before.txt pacman-Dk-before.txt pacman-Qkk-before.txt pacman-Qkk-missing-or-unreadable-before.txt pacnew-pacsave-before.txt migration-git-state.txt preview-report-checksum.txt'
|
|
(
|
|
cd "$record" || return 1
|
|
sha256sum $inventory_files > inventory-hashes.sha256
|
|
sha256sum -c inventory-hashes.sha256 > inventory-hash-verification.txt
|
|
)
|
|
hash_rc=$?
|
|
if [ "$hash_rc" -ne 0 ]; then
|
|
add_failure "inventory-hashes|rc=$hash_rc"
|
|
fi
|
|
hashes_hash=$(sha256sum "$record/inventory-hashes.sha256" | awk '{print $1}')
|
|
|
|
failure_count=$(wc -l < "$record/failed-mandatory-checks.txt")
|
|
failed_checks_hash=$(sha256sum "$record/failed-mandatory-checks.txt" | awk '{print $1}')
|
|
{
|
|
printf 'format=le-phase2a-preflight-v1\n'
|
|
printf 'hostname=%s\n' "$hostname_value"
|
|
printf 'boot_id=%s\n' "$boot_id"
|
|
printf 'running_kernel=%s\n' "$running_kernel"
|
|
printf 'timestamp_epoch=%s\n' "$timestamp_epoch"
|
|
printf 'timestamp_utc=%s\n' "$timestamp_utc"
|
|
printf 'overall=%s\n' "$overall"
|
|
printf 'failure_count=%s\n' "$failure_count"
|
|
printf 'failed_checks_file=failed-mandatory-checks.txt\n'
|
|
printf 'failed_checks_sha256=%s\n' "$failed_checks_hash"
|
|
printf 'inventory_hashes_sha256=%s\n' "$hashes_hash"
|
|
} > "$record/result.manifest"
|
|
(cd "$record" && sha256sum result.manifest > result.manifest.sha256)
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
overall=HARD_STOP
|
|
printf '%s\n' "result-manifest-checksum|rc=$rc" >> "$record/failed-mandatory-checks.txt"
|
|
fi
|
|
chmod -R go-rwx "$record"
|
|
printf 'PREFLIGHT_RECORD=%s\n' "$record"
|
|
printf 'PREFLIGHT_RESULT=%s failures=%s\n' "$overall" "$failure_count"
|
|
if [ "$overall" != PASS ]; then
|
|
return 20
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
main "$@"
|