396 lines
17 KiB
Bash
396 lines
17 KiB
Bash
#!/usr/bin/env bash
|
|
# Read-only host inspection plus root-only evidence recording.
|
|
# Run as a separate process; this script changes no shell options and uses return only.
|
|
|
|
add_failure() {
|
|
printf '%s\n' "$1" >> "$record/failed-mandatory-checks.txt"
|
|
overall=HARD_STOP
|
|
return 0
|
|
}
|
|
|
|
mandatory() {
|
|
label=$1
|
|
outfile=$2
|
|
shift 2
|
|
"$@" > "$record/$outfile" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
add_failure "$label|rc=$rc|file=$outfile"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
mandatory_shell() {
|
|
label=$1
|
|
outfile=$2
|
|
command_text=$3
|
|
/usr/bin/bash -o pipefail -c "$command_text" > "$record/$outfile" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
add_failure "$label|rc=$rc|file=$outfile"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
optional_shell() {
|
|
label=$1
|
|
outfile=$2
|
|
command_text=$3
|
|
/usr/bin/bash -o pipefail -c "$command_text" > "$record/$outfile" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
printf '%s|rc=%s|file=%s\n' "$label" "$rc" "$outfile" >> "$record/warnings.txt"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
capture_failed_unit_fingerprint() {
|
|
output=$1
|
|
: > "$output"
|
|
systemctl --failed --no-legend --plain --no-pager | awk '{print $1}' | LC_ALL=C sort |
|
|
while IFS= read -r unit; do
|
|
[ -n "$unit" ] || continue
|
|
id=$(systemctl show "$unit" -p Id --value --no-pager)
|
|
type=$(systemctl show "$unit" -p Type --value --no-pager)
|
|
load=$(systemctl show "$unit" -p LoadState --value --no-pager)
|
|
active=$(systemctl show "$unit" -p ActiveState --value --no-pager)
|
|
sub=$(systemctl show "$unit" -p SubState --value --no-pager)
|
|
result=$(systemctl show "$unit" -p Result --value --no-pager)
|
|
invocation=$(systemctl show "$unit" -p InvocationID --value --no-pager)
|
|
changed=$(systemctl show "$unit" -p StateChangeTimestamp --value --no-pager)
|
|
fragment=$(systemctl show "$unit" -p FragmentPath --value --no-pager)
|
|
code=$(systemctl show "$unit" -p ExecMainCode --value --no-pager)
|
|
status=$(systemctl show "$unit" -p ExecMainStatus --value --no-pager)
|
|
printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
|
|
"$id" "$type" "$load" "$active" "$sub" "$result" "$invocation" \
|
|
"$changed" "$fragment" "$code" "$status"
|
|
done > "$output"
|
|
return ${PIPESTATUS[0]}
|
|
}
|
|
|
|
validate_failed_unit_disposition() {
|
|
allowlist_dir=$script_dir/failed-unit-disposition
|
|
allowlist=$allowlist_dir/drkonqi-failed-units.allowlist.psv
|
|
allowlist_hashes=$allowlist_dir/SHA256SUMS
|
|
(cd "$allowlist_dir" && sha256sum -c SHA256SUMS > "$record/failed-unit-allowlist-checksum.txt" 2>&1)
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure "failed-unit-allowlist-checksum|rc=$rc"; return 0; fi
|
|
capture_failed_unit_fingerprint "$record/failed-units-fingerprint.psv"
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure "failed-unit-fingerprint|rc=$rc"; return 0; fi
|
|
cp "$allowlist" "$allowlist_hashes" "$record/"
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure "failed-unit-allowlist-copy|rc=$rc"; return 0; fi
|
|
diff -u "$allowlist" "$record/failed-units-fingerprint.psv" > "$record/failed-unit-disposition.diff" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
add_failure 'failed-unit-disposition|set-state-or-invocation-fingerprint-changed'
|
|
fi
|
|
count=$(wc -l < "$record/failed-units-fingerprint.psv")
|
|
[ "$count" = 234 ] || add_failure "failed-unit-disposition|expected=234|actual=$count"
|
|
return 0
|
|
}
|
|
|
|
validate_snapper_readiness() {
|
|
btrfs subvolume show /.snapshots > "$record/snapshots-subvolume.txt" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure "snapshots-btrfs-subvolume|rc=$rc"; return 0; fi
|
|
stat -c 'path=%n|uid=%u|gid=%g|mode=%a|type=%F' /.snapshots > "$record/snapshots-stat.txt" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then add_failure "snapshots-stat|rc=$rc"; return 0; fi
|
|
owner_mode=$(stat -c '%u:%g:%a' /.snapshots 2>/dev/null)
|
|
[ "$owner_mode" = 0:0:750 ] || add_failure "snapshots-owner-mode|expected=0:0:750|actual=$owner_mode"
|
|
snapper list --columns number,type,pre-number,date,user,cleanup,description \
|
|
> "$record/snapper-snapshots-before.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || add_failure "snapper-list|rc=$rc"
|
|
btrfs qgroup show / > "$record/btrfs-quota-observation.txt" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
printf 'btrfs-quota-accounting|unavailable-or-disabled|rc=%s|snapshot-validity-not-affected\n' "$rc" \
|
|
>> "$record/warnings.txt"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
validate_health_urls() {
|
|
: > "$record/health-urls.sanitized"
|
|
if [ -z "${PUBLIC_HEALTH_URLS:-}" ]; then
|
|
add_failure 'public-health-urls|unset'
|
|
return 0
|
|
fi
|
|
case "$PUBLIC_HEALTH_URLS" in
|
|
*$'\n'*|*$'\r'*|*$'\t'*)
|
|
add_failure 'public-health-urls|control-character-rejected'
|
|
return 0
|
|
;;
|
|
esac
|
|
for url in $PUBLIC_HEALTH_URLS; do
|
|
case "$url" in
|
|
https://*) ;;
|
|
*) add_failure 'public-health-url|unsupported-scheme-or-malformed'; continue ;;
|
|
esac
|
|
authority=${url#https://}
|
|
authority=${authority%%/*}
|
|
case "$url" in
|
|
*'?'*|*'#'*|*$'\n'*|*$'\r'*|*$'\t'*|*' '*)
|
|
add_failure 'public-health-url|query-fragment-or-control-character-rejected'
|
|
continue
|
|
;;
|
|
esac
|
|
case "$authority" in
|
|
''|*@*) add_failure 'public-health-url|empty-authority-or-userinfo-rejected'; continue ;;
|
|
esac
|
|
printf '%s\n' "$url" >> "$record/health-urls.sanitized"
|
|
done
|
|
if [ ! -s "$record/health-urls.sanitized" ]; then
|
|
add_failure 'public-health-urls|no-valid-non-secret-https-url'
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
check_public_health() {
|
|
: > "$record/public-health.txt"
|
|
while IFS= read -r url; do
|
|
[ -n "$url" ] || continue
|
|
metrics=$(curl --fail --silent --show-error --max-redirs 0 \
|
|
--connect-timeout 10 --max-time 30 --output /dev/null \
|
|
--write-out 'http=%{http_code}|remote=%{remote_ip}|time=%{time_total}' \
|
|
"$url" 2>> "$record/public-health-errors.txt")
|
|
rc=$?
|
|
printf '%s|%s\n' "$url" "$metrics" >> "$record/public-health.txt"
|
|
if [ "$rc" -ne 0 ]; then
|
|
add_failure "public-health|rc=$rc|url=$url"
|
|
fi
|
|
done < "$record/health-urls.sanitized"
|
|
return 0
|
|
}
|
|
|
|
capture_bootloader() {
|
|
found=0
|
|
: > "$record/bootloader-inventory.txt"
|
|
if [ -d /boot/loader/entries ]; then
|
|
printf '%s\n' 'type=systemd-boot' >> "$record/bootloader-inventory.txt"
|
|
bootctl status >> "$record/bootloader-inventory.txt" 2>&1
|
|
bootctl_rc=$?
|
|
find /boot/loader/entries -maxdepth 1 -type f -print -exec sed -n '1,160p' {} \; \
|
|
>> "$record/bootloader-inventory.txt" 2>&1
|
|
find_rc=$?
|
|
if [ "$bootctl_rc" -eq 0 ] && [ "$find_rc" -eq 0 ]; then found=1; fi
|
|
fi
|
|
if [ -f /boot/grub/grub.cfg ]; then
|
|
printf '%s\n' 'type=grub' >> "$record/bootloader-inventory.txt"
|
|
grub-install --version >> "$record/bootloader-inventory.txt" 2>&1
|
|
grub_rc=$?
|
|
grep -E '^menuentry |^submenu |linux.*vmlinuz|initrd' /boot/grub/grub.cfg \
|
|
>> "$record/bootloader-inventory.txt" 2>&1
|
|
grep_rc=$?
|
|
if [ "$grub_rc" -eq 0 ] && [ "$grep_rc" -eq 0 ]; then found=1; fi
|
|
fi
|
|
if [ "$found" -ne 1 ]; then
|
|
add_failure 'bootloader-inventory|no-verified-supported-bootloader'
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
capture_unit_baseline() {
|
|
systemctl list-units --type=service --all --no-legend --no-pager \
|
|
> "$record/all-service-units.txt" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
add_failure "service-unit-inventory|rc=$rc"
|
|
return 0
|
|
fi
|
|
: > "$record/critical-units-baseline.psv"
|
|
awk '{print $1}' "$record/all-service-units.txt" | while IFS= read -r unit; do
|
|
case "$unit" in
|
|
docker.service|nginx.service|php-fpm.service|mariadb.service|mysqld.service|postgresql.service|postfix.service|dovecot.service|rspamd.service|forgejo.service|sshd.service)
|
|
;;
|
|
*) continue ;;
|
|
esac
|
|
load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null)
|
|
active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null)
|
|
sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null)
|
|
printf '%s|%s|%s|%s\n' "$unit" "$load" "$active" "$sub"
|
|
done > "$record/critical-units-baseline.psv"
|
|
for unit in ${EXTRA_CRITICAL_UNITS:-}; do
|
|
load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null)
|
|
if [ "$load" = loaded ]; then
|
|
active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null)
|
|
sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null)
|
|
printf '%s|%s|%s|%s\n' "$unit" "$load" "$active" "$sub" \
|
|
>> "$record/critical-units-baseline.psv"
|
|
else
|
|
add_failure "extra-critical-unit|not-loaded|unit=$unit"
|
|
fi
|
|
done
|
|
sort -u -o "$record/critical-units-baseline.psv" "$record/critical-units-baseline.psv"
|
|
return 0
|
|
}
|
|
|
|
capture_database_baseline() {
|
|
: > "$record/database-baseline.psv"
|
|
while IFS='|' read -r unit load active sub; do
|
|
[ "$active" = active ] || continue
|
|
case "$unit" in
|
|
mariadb.service|mysqld.service) printf 'unit|mysql|%s|%s|%s\n' "$unit" "$active" "$sub" ;;
|
|
postgresql.service) printf 'unit|postgresql|%s|%s|%s\n' "$unit" "$active" "$sub" ;;
|
|
esac
|
|
done < "$record/critical-units-baseline.psv" >> "$record/database-baseline.psv"
|
|
while IFS='|' read -r name image status ports; do
|
|
case "$status" in Up\ *) ;; *) continue ;; esac
|
|
lowered=$(printf '%s %s' "$name" "$image" | tr '[:upper:]' '[:lower:]')
|
|
case "$lowered" in
|
|
*maria*|*mysql*) printf 'container|mysql|%s|%s|%s\n' "$name" "$status" "$ports" ;;
|
|
*postgres*) printf 'container|postgresql|%s|%s|%s\n' "$name" "$status" "$ports" ;;
|
|
esac
|
|
done < "$record/containers-all.psv" >> "$record/database-baseline.psv"
|
|
return 0
|
|
}
|
|
|
|
validate_database_baseline() {
|
|
: > "$record/database-readiness-before.txt"
|
|
while IFS='|' read -r model engine identity state detail; do
|
|
[ "$model" = unit ] || continue
|
|
if [ "$engine" = mysql ]; then
|
|
mysqladmin ping >> "$record/database-readiness-before.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || add_failure "mysqladmin-ping|unit=$identity|rc=$rc"
|
|
elif [ "$engine" = postgresql ]; then
|
|
pg_isready >> "$record/database-readiness-before.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || add_failure "pg-isready|unit=$identity|rc=$rc"
|
|
fi
|
|
done < "$record/database-baseline.psv"
|
|
while IFS='|' read -r model engine identity state detail; do
|
|
[ "$model" = container ] || continue
|
|
running=$(docker inspect -f '{{.State.Running}}' "$identity" 2>/dev/null)
|
|
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$identity" 2>/dev/null)
|
|
printf '%s|%s|running=%s|health=%s\n' "$engine" "$identity" "${running:-missing}" "${health:-unknown}" \
|
|
>> "$record/database-readiness-before.txt"
|
|
if [ "$running" != true ] || [ "$health" = unhealthy ]; then
|
|
add_failure "database-container|name=$identity|running=${running:-missing}|health=${health:-unknown}"
|
|
fi
|
|
done < "$record/database-baseline.psv"
|
|
return 0
|
|
}
|
|
|
|
main() {
|
|
if [ "$(id -u)" -ne 0 ]; then
|
|
printf 'HARD_STOP: run as a separate root process so records are root-only\n' >&2
|
|
return 10
|
|
fi
|
|
umask 077
|
|
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
|
install -d -o root -g root -m 0700 /var/log/le-phase2a-preflight
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
printf 'HARD_STOP: cannot create root-only preflight record parent\n' >&2
|
|
return 11
|
|
fi
|
|
record=$(mktemp -d "/var/log/le-phase2a-preflight/preflight.$(date -u +%Y%m%dT%H%M%SZ).XXXXXXXX")
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ] || [ -z "$record" ] || [ ! -d "$record" ]; then
|
|
printf 'HARD_STOP: cannot atomically create unique preflight record\n' >&2
|
|
return 12
|
|
fi
|
|
chown root:root "$record"
|
|
chmod 0700 "$record"
|
|
|
|
overall=PASS
|
|
: > "$record/failed-mandatory-checks.txt"
|
|
: > "$record/warnings.txt"
|
|
timestamp_epoch=$(date +%s)
|
|
timestamp_utc=$(date -u +%FT%TZ)
|
|
hostname_value=$(hostname)
|
|
boot_id=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null)
|
|
running_kernel=$(uname -r)
|
|
[ -n "$hostname_value" ] || add_failure 'hostname|empty'
|
|
[ -n "$boot_id" ] || add_failure 'boot-id|unavailable'
|
|
[ -n "$running_kernel" ] || add_failure 'running-kernel|unavailable'
|
|
[ "${LE_PHASE2A_RECOVERY_READY:-}" = YES ] || add_failure 'recovery-access-attestation|missing'
|
|
[ "${LE_PHASE2A_BACKUPS_READY:-}" = YES ] || add_failure 'backup-readiness-attestation|missing'
|
|
|
|
mandatory 'uptime' uptime.txt uptime
|
|
mandatory 'logged-in-sessions' logged-in-sessions.txt who -a
|
|
mandatory 'filesystem-capacity' filesystem-capacity.txt df -hT / /boot /var/cache/pacman/pkg /tmp
|
|
mandatory 'filesystem-inodes' filesystem-inodes.txt df -hi / /boot /var/cache/pacman/pkg /tmp
|
|
mandatory 'installed-package-inventory' packages-before.txt pacman -Q
|
|
mandatory 'failed-unit-inventory' failed-units-before.txt systemctl --failed --no-legend --plain --no-pager
|
|
validate_failed_unit_disposition
|
|
validate_snapper_readiness
|
|
mandatory_shell 'kernel-initramfs-inventory' kernel-initramfs-before.txt \
|
|
"find /usr/lib/modules /boot -maxdepth 2 -type f \\
|
|
\( -name vmlinuz -o -name 'vmlinuz-*' -o -name 'initramfs*' -o -name 'initrd*' -o -name pkgbase \) \\
|
|
-printf '%p size=%s mtime=%TY-%Tm-%TdT%TH:%TM:%TS\\n' | sort; test \"\$(find /boot -maxdepth 1 -type f \\
|
|
\( -name 'vmlinuz-*' -o -name 'initramfs*' -o -name 'initrd*' \) | wc -l)\" -gt 0"
|
|
capture_bootloader
|
|
optional_shell 'dkms-status' dkms-before.txt 'command -v dkms >/dev/null && dkms status'
|
|
mandatory_shell 'docker-container-inventory' containers-all.psv \
|
|
"docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort"
|
|
mandatory_shell 'running-container-inventory' running-containers-before.txt \
|
|
"docker ps --format '{{.Names}}' | sort"
|
|
capture_unit_baseline
|
|
capture_database_baseline
|
|
validate_database_baseline
|
|
|
|
validate_health_urls
|
|
check_public_health
|
|
mandatory_shell 'network-firewall-listener-inventory' network-firewall-before.txt \
|
|
'findmnt --real; ip -brief address; ip route show table all; ip -6 route show table all; nft list ruleset; ss -lntup'
|
|
optional_shell 'wireguard-observation' wireguard-before.txt 'wg show'
|
|
mandatory 'package-database-consistency' pacman-Dk-before.txt pacman -Dk
|
|
optional_shell 'packaged-file-consistency-observation' pacman-Qkk-before.txt 'pacman -Qkk'
|
|
grep -E -i 'missing|could not read|permission denied|unreadable' "$record/pacman-Qkk-before.txt" \
|
|
> "$record/pacman-Qkk-missing-or-unreadable-before.txt" 2>/dev/null
|
|
if [ -s "$record/pacman-Qkk-missing-or-unreadable-before.txt" ]; then
|
|
add_failure 'packaged-files-missing-or-unreadable-before-maintenance'
|
|
fi
|
|
mandatory_shell 'pacnew-pacsave-inventory' pacnew-pacsave-before.txt \
|
|
"find /etc -xdev -type f \( -name '*.pacnew' -o -name '*.pacsave' \) -printf '%p\\n' | sort"
|
|
mandatory_shell 'migration-git-state' migration-git-state.txt \
|
|
'git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration status --short --branch; git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration rev-parse HEAD; git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration rev-parse origin/migration/le-app-database'
|
|
mandatory_shell 'approved-transaction-integrity' approved-transaction-checksum.txt \
|
|
"cd '$script_dir/approved-transaction' && sha256sum -c SHA256SUMS"
|
|
|
|
inventory_files='packages-before.txt failed-units-before.txt failed-units-fingerprint.psv failed-unit-disposition.diff failed-unit-allowlist-checksum.txt snapshots-subvolume.txt snapshots-stat.txt snapper-snapshots-before.txt btrfs-quota-observation.txt kernel-initramfs-before.txt bootloader-inventory.txt containers-all.psv running-containers-before.txt critical-units-baseline.psv database-baseline.psv database-readiness-before.txt health-urls.sanitized public-health.txt network-firewall-before.txt pacman-Dk-before.txt pacman-Qkk-before.txt pacman-Qkk-missing-or-unreadable-before.txt pacnew-pacsave-before.txt migration-git-state.txt approved-transaction-checksum.txt'
|
|
(
|
|
cd "$record" || return 1
|
|
sha256sum $inventory_files > inventory-hashes.sha256
|
|
sha256sum -c inventory-hashes.sha256 > inventory-hash-verification.txt
|
|
)
|
|
hash_rc=$?
|
|
if [ "$hash_rc" -ne 0 ]; then
|
|
add_failure "inventory-hashes|rc=$hash_rc"
|
|
fi
|
|
hashes_hash=$(sha256sum "$record/inventory-hashes.sha256" | awk '{print $1}')
|
|
|
|
failure_count=$(wc -l < "$record/failed-mandatory-checks.txt")
|
|
failed_checks_hash=$(sha256sum "$record/failed-mandatory-checks.txt" | awk '{print $1}')
|
|
{
|
|
printf 'format=le-phase2a-preflight-v1\n'
|
|
printf 'hostname=%s\n' "$hostname_value"
|
|
printf 'boot_id=%s\n' "$boot_id"
|
|
printf 'running_kernel=%s\n' "$running_kernel"
|
|
printf 'timestamp_epoch=%s\n' "$timestamp_epoch"
|
|
printf 'timestamp_utc=%s\n' "$timestamp_utc"
|
|
printf 'overall=%s\n' "$overall"
|
|
printf 'failure_count=%s\n' "$failure_count"
|
|
printf 'failed_checks_file=failed-mandatory-checks.txt\n'
|
|
printf 'failed_checks_sha256=%s\n' "$failed_checks_hash"
|
|
printf 'inventory_hashes_sha256=%s\n' "$hashes_hash"
|
|
} > "$record/result.manifest"
|
|
(cd "$record" && sha256sum result.manifest > result.manifest.sha256)
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
overall=HARD_STOP
|
|
printf '%s\n' "result-manifest-checksum|rc=$rc" >> "$record/failed-mandatory-checks.txt"
|
|
fi
|
|
chmod -R go-rwx "$record"
|
|
printf 'PREFLIGHT_RECORD=%s\n' "$record"
|
|
printf 'PREFLIGHT_RESULT=%s failures=%s\n' "$overall" "$failure_count"
|
|
if [ "$overall" != PASS ]; then
|
|
return 20
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
main "$@"
|