#!/usr/bin/env bash # Read-only host inspection plus root-only evidence recording. # Run as a separate process; this script changes no shell options and uses return only. add_failure() { printf '%s\n' "$1" >> "$record/failed-mandatory-checks.txt" overall=HARD_STOP return 0 } mandatory() { label=$1 outfile=$2 shift 2 "$@" > "$record/$outfile" 2>&1 rc=$? if [ "$rc" -ne 0 ]; then add_failure "$label|rc=$rc|file=$outfile" fi return 0 } mandatory_shell() { label=$1 outfile=$2 command_text=$3 /usr/bin/bash -o pipefail -c "$command_text" > "$record/$outfile" 2>&1 rc=$? if [ "$rc" -ne 0 ]; then add_failure "$label|rc=$rc|file=$outfile" fi return 0 } optional_shell() { label=$1 outfile=$2 command_text=$3 /usr/bin/bash -o pipefail -c "$command_text" > "$record/$outfile" 2>&1 rc=$? if [ "$rc" -ne 0 ]; then printf '%s|rc=%s|file=%s\n' "$label" "$rc" "$outfile" >> "$record/warnings.txt" fi return 0 } capture_failed_unit_fingerprint() { output=$1 : > "$output" systemctl --failed --no-legend --plain --no-pager | awk '{print $1}' | LC_ALL=C sort | while IFS= read -r unit; do [ -n "$unit" ] || continue id=$(systemctl show "$unit" -p Id --value --no-pager) type=$(systemctl show "$unit" -p Type --value --no-pager) load=$(systemctl show "$unit" -p LoadState --value --no-pager) active=$(systemctl show "$unit" -p ActiveState --value --no-pager) sub=$(systemctl show "$unit" -p SubState --value --no-pager) result=$(systemctl show "$unit" -p Result --value --no-pager) invocation=$(systemctl show "$unit" -p InvocationID --value --no-pager) changed=$(systemctl show "$unit" -p StateChangeTimestamp --value --no-pager) fragment=$(systemctl show "$unit" -p FragmentPath --value --no-pager) code=$(systemctl show "$unit" -p ExecMainCode --value --no-pager) status=$(systemctl show "$unit" -p ExecMainStatus --value --no-pager) printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \ "$id" "$type" "$load" "$active" "$sub" "$result" "$invocation" \ "$changed" "$fragment" "$code" "$status" done > "$output" return ${PIPESTATUS[0]} } validate_failed_unit_disposition() { allowlist_dir=$script_dir/failed-unit-disposition allowlist=$allowlist_dir/drkonqi-failed-units.allowlist.psv allowlist_hashes=$allowlist_dir/SHA256SUMS (cd "$allowlist_dir" && sha256sum -c SHA256SUMS > "$record/failed-unit-allowlist-checksum.txt" 2>&1) rc=$? if [ "$rc" -ne 0 ]; then add_failure "failed-unit-allowlist-checksum|rc=$rc"; return 0; fi capture_failed_unit_fingerprint "$record/failed-units-fingerprint.psv" rc=$? if [ "$rc" -ne 0 ]; then add_failure "failed-unit-fingerprint|rc=$rc"; return 0; fi cp "$allowlist" "$allowlist_hashes" "$record/" rc=$? if [ "$rc" -ne 0 ]; then add_failure "failed-unit-allowlist-copy|rc=$rc"; return 0; fi diff -u "$allowlist" "$record/failed-units-fingerprint.psv" > "$record/failed-unit-disposition.diff" 2>&1 rc=$? if [ "$rc" -ne 0 ]; then add_failure 'failed-unit-disposition|set-state-or-invocation-fingerprint-changed' fi count=$(wc -l < "$record/failed-units-fingerprint.psv") [ "$count" = 234 ] || add_failure "failed-unit-disposition|expected=234|actual=$count" return 0 } validate_snapper_readiness() { btrfs subvolume show /.snapshots > "$record/snapshots-subvolume.txt" 2>&1 rc=$? if [ "$rc" -ne 0 ]; then add_failure "snapshots-btrfs-subvolume|rc=$rc"; return 0; fi stat -c 'path=%n|uid=%u|gid=%g|mode=%a|type=%F' /.snapshots > "$record/snapshots-stat.txt" 2>&1 rc=$? if [ "$rc" -ne 0 ]; then add_failure "snapshots-stat|rc=$rc"; return 0; fi owner_mode=$(stat -c '%u:%g:%a' /.snapshots 2>/dev/null) [ "$owner_mode" = 0:0:750 ] || add_failure "snapshots-owner-mode|expected=0:0:750|actual=$owner_mode" snapper list --columns number,type,pre-number,date,user,cleanup,description \ > "$record/snapper-snapshots-before.txt" 2>&1 rc=$? [ "$rc" -eq 0 ] || add_failure "snapper-list|rc=$rc" btrfs qgroup show / > "$record/btrfs-quota-observation.txt" 2>&1 rc=$? if [ "$rc" -ne 0 ]; then printf 'btrfs-quota-accounting|unavailable-or-disabled|rc=%s|snapshot-validity-not-affected\n' "$rc" \ >> "$record/warnings.txt" fi return 0 } validate_health_urls() { : > "$record/health-urls.sanitized" if [ -z "${PUBLIC_HEALTH_URLS:-}" ]; then add_failure 'public-health-urls|unset' return 0 fi case "$PUBLIC_HEALTH_URLS" in *$'\n'*|*$'\r'*|*$'\t'*) add_failure 'public-health-urls|control-character-rejected' return 0 ;; esac for url in $PUBLIC_HEALTH_URLS; do case "$url" in https://*) ;; *) add_failure 'public-health-url|unsupported-scheme-or-malformed'; continue ;; esac authority=${url#https://} authority=${authority%%/*} case "$url" in *'?'*|*'#'*|*$'\n'*|*$'\r'*|*$'\t'*|*' '*) add_failure 'public-health-url|query-fragment-or-control-character-rejected' continue ;; esac case "$authority" in ''|*@*) add_failure 'public-health-url|empty-authority-or-userinfo-rejected'; continue ;; esac printf '%s\n' "$url" >> "$record/health-urls.sanitized" done if [ ! -s "$record/health-urls.sanitized" ]; then add_failure 'public-health-urls|no-valid-non-secret-https-url' fi return 0 } check_public_health() { : > "$record/public-health.txt" while IFS= read -r url; do [ -n "$url" ] || continue metrics=$(curl --fail --silent --show-error --max-redirs 0 \ --connect-timeout 10 --max-time 30 --output /dev/null \ --write-out 'http=%{http_code}|remote=%{remote_ip}|time=%{time_total}' \ "$url" 2>> "$record/public-health-errors.txt") rc=$? printf '%s|%s\n' "$url" "$metrics" >> "$record/public-health.txt" if [ "$rc" -ne 0 ]; then add_failure "public-health|rc=$rc|url=$url" fi done < "$record/health-urls.sanitized" return 0 } capture_bootloader() { found=0 : > "$record/bootloader-inventory.txt" if [ -d /boot/loader/entries ]; then printf '%s\n' 'type=systemd-boot' >> "$record/bootloader-inventory.txt" bootctl status >> "$record/bootloader-inventory.txt" 2>&1 bootctl_rc=$? find /boot/loader/entries -maxdepth 1 -type f -print -exec sed -n '1,160p' {} \; \ >> "$record/bootloader-inventory.txt" 2>&1 find_rc=$? if [ "$bootctl_rc" -eq 0 ] && [ "$find_rc" -eq 0 ]; then found=1; fi fi if [ -f /boot/grub/grub.cfg ]; then printf '%s\n' 'type=grub' >> "$record/bootloader-inventory.txt" grub-install --version >> "$record/bootloader-inventory.txt" 2>&1 grub_rc=$? grep -E '^menuentry |^submenu |linux.*vmlinuz|initrd' /boot/grub/grub.cfg \ >> "$record/bootloader-inventory.txt" 2>&1 grep_rc=$? if [ "$grub_rc" -eq 0 ] && [ "$grep_rc" -eq 0 ]; then found=1; fi fi if [ "$found" -ne 1 ]; then add_failure 'bootloader-inventory|no-verified-supported-bootloader' fi return 0 } capture_unit_baseline() { systemctl list-units --type=service --all --no-legend --no-pager \ > "$record/all-service-units.txt" 2>&1 rc=$? if [ "$rc" -ne 0 ]; then add_failure "service-unit-inventory|rc=$rc" return 0 fi : > "$record/critical-units-baseline.psv" awk '{print $1}' "$record/all-service-units.txt" | while IFS= read -r unit; do case "$unit" in docker.service|nginx.service|php-fpm.service|mariadb.service|mysqld.service|postgresql.service|postfix.service|dovecot.service|rspamd.service|forgejo.service|sshd.service) ;; *) continue ;; esac load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null) active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null) sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null) printf '%s|%s|%s|%s\n' "$unit" "$load" "$active" "$sub" done > "$record/critical-units-baseline.psv" for unit in ${EXTRA_CRITICAL_UNITS:-}; do load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null) if [ "$load" = loaded ]; then active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null) sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null) printf '%s|%s|%s|%s\n' "$unit" "$load" "$active" "$sub" \ >> "$record/critical-units-baseline.psv" else add_failure "extra-critical-unit|not-loaded|unit=$unit" fi done sort -u -o "$record/critical-units-baseline.psv" "$record/critical-units-baseline.psv" return 0 } capture_database_baseline() { : > "$record/database-baseline.psv" while IFS='|' read -r unit load active sub; do [ "$active" = active ] || continue case "$unit" in mariadb.service|mysqld.service) printf 'unit|mysql|%s|%s|%s\n' "$unit" "$active" "$sub" ;; postgresql.service) printf 'unit|postgresql|%s|%s|%s\n' "$unit" "$active" "$sub" ;; esac done < "$record/critical-units-baseline.psv" >> "$record/database-baseline.psv" while IFS='|' read -r name image status ports; do case "$status" in Up\ *) ;; *) continue ;; esac lowered=$(printf '%s %s' "$name" "$image" | tr '[:upper:]' '[:lower:]') case "$lowered" in *maria*|*mysql*) printf 'container|mysql|%s|%s|%s\n' "$name" "$status" "$ports" ;; *postgres*) printf 'container|postgresql|%s|%s|%s\n' "$name" "$status" "$ports" ;; esac done < "$record/containers-all.psv" >> "$record/database-baseline.psv" return 0 } validate_database_baseline() { : > "$record/database-readiness-before.txt" while IFS='|' read -r model engine identity state detail; do [ "$model" = unit ] || continue if [ "$engine" = mysql ]; then mysqladmin ping >> "$record/database-readiness-before.txt" 2>&1 rc=$?; [ "$rc" -eq 0 ] || add_failure "mysqladmin-ping|unit=$identity|rc=$rc" elif [ "$engine" = postgresql ]; then pg_isready >> "$record/database-readiness-before.txt" 2>&1 rc=$?; [ "$rc" -eq 0 ] || add_failure "pg-isready|unit=$identity|rc=$rc" fi done < "$record/database-baseline.psv" while IFS='|' read -r model engine identity state detail; do [ "$model" = container ] || continue running=$(docker inspect -f '{{.State.Running}}' "$identity" 2>/dev/null) health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$identity" 2>/dev/null) printf '%s|%s|running=%s|health=%s\n' "$engine" "$identity" "${running:-missing}" "${health:-unknown}" \ >> "$record/database-readiness-before.txt" if [ "$running" != true ] || [ "$health" = unhealthy ]; then add_failure "database-container|name=$identity|running=${running:-missing}|health=${health:-unknown}" fi done < "$record/database-baseline.psv" return 0 } main() { if [ "$(id -u)" -ne 0 ]; then printf 'HARD_STOP: run as a separate root process so records are root-only\n' >&2 return 10 fi umask 077 script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) install -d -o root -g root -m 0700 /var/log/le-phase2a-preflight rc=$? if [ "$rc" -ne 0 ]; then printf 'HARD_STOP: cannot create root-only preflight record parent\n' >&2 return 11 fi record=$(mktemp -d "/var/log/le-phase2a-preflight/preflight.$(date -u +%Y%m%dT%H%M%SZ).XXXXXXXX") rc=$? if [ "$rc" -ne 0 ] || [ -z "$record" ] || [ ! -d "$record" ]; then printf 'HARD_STOP: cannot atomically create unique preflight record\n' >&2 return 12 fi chown root:root "$record" chmod 0700 "$record" overall=PASS : > "$record/failed-mandatory-checks.txt" : > "$record/warnings.txt" timestamp_epoch=$(date +%s) timestamp_utc=$(date -u +%FT%TZ) hostname_value=$(hostname) boot_id=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null) running_kernel=$(uname -r) [ -n "$hostname_value" ] || add_failure 'hostname|empty' [ -n "$boot_id" ] || add_failure 'boot-id|unavailable' [ -n "$running_kernel" ] || add_failure 'running-kernel|unavailable' [ "${LE_PHASE2A_RECOVERY_READY:-}" = YES ] || add_failure 'recovery-access-attestation|missing' [ "${LE_PHASE2A_BACKUPS_READY:-}" = YES ] || add_failure 'backup-readiness-attestation|missing' mandatory 'uptime' uptime.txt uptime mandatory 'logged-in-sessions' logged-in-sessions.txt who -a mandatory 'filesystem-capacity' filesystem-capacity.txt df -hT / /boot /var/cache/pacman/pkg /tmp mandatory 'filesystem-inodes' filesystem-inodes.txt df -hi / /boot /var/cache/pacman/pkg /tmp mandatory 'installed-package-inventory' packages-before.txt pacman -Q mandatory 'failed-unit-inventory' failed-units-before.txt systemctl --failed --no-legend --plain --no-pager validate_failed_unit_disposition validate_snapper_readiness mandatory_shell 'kernel-initramfs-inventory' kernel-initramfs-before.txt \ "find /usr/lib/modules /boot -maxdepth 2 -type f \\ \( -name vmlinuz -o -name 'vmlinuz-*' -o -name 'initramfs*' -o -name 'initrd*' -o -name pkgbase \) \\ -printf '%p size=%s mtime=%TY-%Tm-%TdT%TH:%TM:%TS\\n' | sort; test \"\$(find /boot -maxdepth 1 -type f \\ \( -name 'vmlinuz-*' -o -name 'initramfs*' -o -name 'initrd*' \) | wc -l)\" -gt 0" capture_bootloader optional_shell 'dkms-status' dkms-before.txt 'command -v dkms >/dev/null && dkms status' mandatory_shell 'docker-container-inventory' containers-all.psv \ "docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort" mandatory_shell 'running-container-inventory' running-containers-before.txt \ "docker ps --format '{{.Names}}' | sort" capture_unit_baseline capture_database_baseline validate_database_baseline validate_health_urls check_public_health mandatory_shell 'network-firewall-listener-inventory' network-firewall-before.txt \ 'findmnt --real; ip -brief address; ip route show table all; ip -6 route show table all; nft list ruleset; ss -lntup' optional_shell 'wireguard-observation' wireguard-before.txt 'wg show' mandatory 'package-database-consistency' pacman-Dk-before.txt pacman -Dk optional_shell 'packaged-file-consistency-observation' pacman-Qkk-before.txt 'pacman -Qkk' grep -E -i 'missing|could not read|permission denied|unreadable' "$record/pacman-Qkk-before.txt" \ > "$record/pacman-Qkk-missing-or-unreadable-before.txt" 2>/dev/null if [ -s "$record/pacman-Qkk-missing-or-unreadable-before.txt" ]; then add_failure 'packaged-files-missing-or-unreadable-before-maintenance' fi mandatory_shell 'pacnew-pacsave-inventory' pacnew-pacsave-before.txt \ "find /etc -xdev -type f \( -name '*.pacnew' -o -name '*.pacsave' \) -printf '%p\\n' | sort" mandatory_shell 'migration-git-state' migration-git-state.txt \ 'git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration status --short --branch; git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration rev-parse HEAD; git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration rev-parse origin/migration/le-app-database' mandatory_shell 'approved-transaction-integrity' approved-transaction-checksum.txt \ "cd '$script_dir/approved-transaction' && sha256sum -c SHA256SUMS" inventory_files='packages-before.txt failed-units-before.txt failed-units-fingerprint.psv failed-unit-disposition.diff failed-unit-allowlist-checksum.txt snapshots-subvolume.txt snapshots-stat.txt snapper-snapshots-before.txt btrfs-quota-observation.txt kernel-initramfs-before.txt bootloader-inventory.txt containers-all.psv running-containers-before.txt critical-units-baseline.psv database-baseline.psv database-readiness-before.txt health-urls.sanitized public-health.txt network-firewall-before.txt pacman-Dk-before.txt pacman-Qkk-before.txt pacman-Qkk-missing-or-unreadable-before.txt pacnew-pacsave-before.txt migration-git-state.txt approved-transaction-checksum.txt' ( cd "$record" || return 1 sha256sum $inventory_files > inventory-hashes.sha256 sha256sum -c inventory-hashes.sha256 > inventory-hash-verification.txt ) hash_rc=$? if [ "$hash_rc" -ne 0 ]; then add_failure "inventory-hashes|rc=$hash_rc" fi hashes_hash=$(sha256sum "$record/inventory-hashes.sha256" | awk '{print $1}') failure_count=$(wc -l < "$record/failed-mandatory-checks.txt") failed_checks_hash=$(sha256sum "$record/failed-mandatory-checks.txt" | awk '{print $1}') { printf 'format=le-phase2a-preflight-v1\n' printf 'hostname=%s\n' "$hostname_value" printf 'boot_id=%s\n' "$boot_id" printf 'running_kernel=%s\n' "$running_kernel" printf 'timestamp_epoch=%s\n' "$timestamp_epoch" printf 'timestamp_utc=%s\n' "$timestamp_utc" printf 'overall=%s\n' "$overall" printf 'failure_count=%s\n' "$failure_count" printf 'failed_checks_file=failed-mandatory-checks.txt\n' printf 'failed_checks_sha256=%s\n' "$failed_checks_hash" printf 'inventory_hashes_sha256=%s\n' "$hashes_hash" } > "$record/result.manifest" (cd "$record" && sha256sum result.manifest > result.manifest.sha256) rc=$? if [ "$rc" -ne 0 ]; then overall=HARD_STOP printf '%s\n' "result-manifest-checksum|rc=$rc" >> "$record/failed-mandatory-checks.txt" fi chmod -R go-rwx "$record" printf 'PREFLIGHT_RECORD=%s\n' "$record" printf 'PREFLIGHT_RESULT=%s failures=%s\n' "$overall" "$failure_count" if [ "$overall" != PASS ]; then return 20 fi return 0 } main "$@"