le-app/docs/le-app-database-migration/phase2a-host-maintenance/post-reboot-validate.sh

#!/usr/bin/env bash
# Read-only validation plus root-only evidence recording. Run as a separate process.

add_failure() {
  printf '%s\n' "$1" >> "$post_record/failed-checks.txt"
  failures=$((failures + 1))
  return 0
}

check() {
  label=$1; outfile=$2; shift 2
  "$@" > "$post_record/$outfile" 2>&1
  rc=$?
  if [ "$rc" -ne 0 ]; then add_failure "$label|rc=$rc|file=$outfile"; fi
  return 0
}

check_shell() {
  label=$1; outfile=$2; command_text=$3
  /usr/bin/bash -o pipefail -c "$command_text" > "$post_record/$outfile" 2>&1
  rc=$?
  if [ "$rc" -ne 0 ]; then add_failure "$label|rc=$rc|file=$outfile"; fi
  return 0
}

capture_failed_unit_fingerprint() {
  output=$1
  systemctl --failed --no-legend --plain --no-pager | awk '{print $1}' | LC_ALL=C sort |
    while IFS= read -r unit; do
      [ -n "$unit" ] || continue
      printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
        "$(systemctl show "$unit" -p Id --value --no-pager)" \
        "$(systemctl show "$unit" -p Type --value --no-pager)" \
        "$(systemctl show "$unit" -p LoadState --value --no-pager)" \
        "$(systemctl show "$unit" -p ActiveState --value --no-pager)" \
        "$(systemctl show "$unit" -p SubState --value --no-pager)" \
        "$(systemctl show "$unit" -p Result --value --no-pager)" \
        "$(systemctl show "$unit" -p InvocationID --value --no-pager)" \
        "$(systemctl show "$unit" -p StateChangeTimestamp --value --no-pager)" \
        "$(systemctl show "$unit" -p FragmentPath --value --no-pager)" \
        "$(systemctl show "$unit" -p ExecMainCode --value --no-pager)" \
        "$(systemctl show "$unit" -p ExecMainStatus --value --no-pager)"
    done > "$output"
  return ${PIPESTATUS[0]}
}

validate_failed_unit_disposition() {
  (cd "$script_dir/failed-unit-disposition" && sha256sum -c SHA256SUMS) \
    > "$post_record/failed-unit-allowlist-checksum.txt" 2>&1
  rc=$?
  if [ "$rc" -ne 0 ]; then add_failure 'failed-unit-allowlist-checksum'; return 0; fi
  capture_failed_unit_fingerprint "$post_record/failed-units-fingerprint.psv"
  rc=$?
  if [ "$rc" -ne 0 ]; then add_failure "failed-unit-fingerprint|rc=$rc"; return 0; fi
  diff -u "$script_dir/failed-unit-disposition/drkonqi-failed-units.allowlist.psv" \
    "$post_record/failed-units-fingerprint.psv" > "$post_record/failed-unit-disposition.diff" 2>&1
  rc=$?
  if [ "$rc" -ne 0 ]; then add_failure 'failed-unit-disposition-changed-after-reboot'; fi
  return 0
}

validate_health_url_file() {
  file=$1
  if [ ! -s "$file" ]; then add_failure 'health-urls|missing'; return 0; fi
  while IFS= read -r url; do
    case "$url" in https://*) ;; *) add_failure 'health-url|unsupported-scheme'; continue ;; esac
    authority=${url#https://}; authority=${authority%%/*}
    case "$url" in *'?'*|*'#'*|*$'\n'*|*$'\r'*|*$'\t'*|*' '*) add_failure 'health-url|unsafe-content'; continue ;; esac
    case "$authority" in ''|*@*) add_failure 'health-url|userinfo-or-empty-authority'; continue ;; esac
  done < "$file"
  return 0
}

compare_containers() {
  : > "$post_record/container-comparison.psv"
  while IFS= read -r container; do
    [ -n "$container" ] || continue
    running=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
    health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$container" 2>/dev/null)
    printf '%s|running=%s|health=%s\n' "$container" "${running:-missing}" "${health:-unknown}" \
      >> "$post_record/container-comparison.psv"
    if [ "$running" != true ] || [ "$health" = unhealthy ]; then
      add_failure "container-regression|name=$container|running=${running:-missing}|health=${health:-unknown}"
    fi
  done < "$baseline/running-containers-before.txt"
  return 0
}

compare_units() {
  : > "$post_record/unit-comparison.psv"
  while IFS='|' read -r unit old_load old_active old_sub; do
    [ -n "$unit" ] || continue
    new_load=$(systemctl show "$unit" -p LoadState --value 2>/dev/null)
    new_active=$(systemctl show "$unit" -p ActiveState --value 2>/dev/null)
    new_sub=$(systemctl show "$unit" -p SubState --value 2>/dev/null)
    printf '%s|before=%s,%s,%s|after=%s,%s,%s\n' \
      "$unit" "$old_load" "$old_active" "$old_sub" "${new_load:-missing}" "${new_active:-missing}" "${new_sub:-missing}" \
      >> "$post_record/unit-comparison.psv"
    if [ "$old_load" = loaded ] && [ "$new_load" != loaded ]; then
      add_failure "critical-unit-missing|unit=$unit"
    elif [ "$old_active" = active ] && [ "$new_active" != active ]; then
      add_failure "critical-unit-not-active|unit=$unit|state=${new_active:-missing}"
    fi
  done < "$baseline/critical-units-baseline.psv"
  return 0
}

validate_databases() {
  : > "$post_record/database-validation.txt"
  while IFS='|' read -r model engine identity state detail; do
    [ -n "$model" ] || continue
    if [ "$model" = unit ]; then
      active=$(systemctl show "$identity" -p ActiveState --value 2>/dev/null)
      printf '%s|%s|unit=%s|active=%s\n' "$model" "$engine" "$identity" "${active:-missing}" \
        >> "$post_record/database-validation.txt"
      if [ "$active" != active ]; then add_failure "database-unit-not-active|unit=$identity"; fi
      if [ "$engine" = mysql ]; then
        mysqladmin ping >> "$post_record/database-validation.txt" 2>&1
        rc=$?; [ "$rc" -eq 0 ] || add_failure "mysqladmin-ping-failed|unit=$identity|rc=$rc"
      elif [ "$engine" = postgresql ]; then
        pg_isready >> "$post_record/database-validation.txt" 2>&1
        rc=$?; [ "$rc" -eq 0 ] || add_failure "pg-isready-failed|unit=$identity|rc=$rc"
      fi
    elif [ "$model" = container ]; then
      running=$(docker inspect -f '{{.State.Running}}' "$identity" 2>/dev/null)
      health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$identity" 2>/dev/null)
      printf '%s|%s|container=%s|running=%s|health=%s\n' \
        "$model" "$engine" "$identity" "${running:-missing}" "${health:-unknown}" \
        >> "$post_record/database-validation.txt"
      if [ "$running" != true ] || [ "$health" = unhealthy ]; then
        add_failure "database-container-failed|container=$identity|running=${running:-missing}|health=${health:-unknown}"
      fi
    fi
  done < "$baseline/database-baseline.psv"
  return 0
}

check_package_files() {
  pacman -Qkk > "$post_record/pacman-Qkk.txt" 2>&1
  qkk_rc=$?
  printf 'pacman-Qkk-return=%s\n' "$qkk_rc" > "$post_record/pacman-Qkk-classification.txt"
  grep -E -i 'missing|could not read|permission denied|unreadable' "$post_record/pacman-Qkk.txt" \
    > "$post_record/pacman-Qkk-missing-or-unreadable.txt" 2>/dev/null
  missing_rc=$?
  grep -E -i 'altered|modified' "$post_record/pacman-Qkk.txt" \
    > "$post_record/pacman-Qkk-modified.txt" 2>/dev/null
  if [ "$missing_rc" -eq 0 ] && [ -s "$post_record/pacman-Qkk-missing-or-unreadable.txt" ]; then
    add_failure 'packaged-files-missing-or-unreadable'
  fi
  printf 'Modified configuration/package files require manual classification; no repair performed.\n' \
    >> "$post_record/pacman-Qkk-classification.txt"
  return 0
}

main() {
  failures=0
  maintenance=${1:-}
  if [ "$(id -u)" -ne 0 ]; then
    printf 'HARD_STOP: run as a separate root process\n' >&2
    return 10
  fi
  script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
  if [ -z "$maintenance" ] || [ ! -d "$maintenance" ] || [ -L "$maintenance" ]; then
    printf 'HARD_STOP: supply the exact maintenance record path\n' >&2
    return 11
  fi
  owner=$(stat -c %u "$maintenance" 2>/dev/null); mode=$(stat -c %a "$maintenance" 2>/dev/null)
  if [ "$owner" != 0 ] || [ "$mode" != 700 ]; then
    printf 'HARD_STOP: maintenance record must be root-owned mode 0700\n' >&2
    return 12
  fi
  baseline=$maintenance/preflight-record
  required='running-containers-before.txt critical-units-baseline.psv database-baseline.psv health-urls.sanitized inventory-hashes.sha256 result.manifest result.manifest.sha256'
  for file in $required; do
    if [ ! -f "$baseline/$file" ] || [ -L "$baseline/$file" ]; then
      printf 'HARD_STOP: missing baseline file %s\n' "$file" >&2
      return 13
    fi
  done
  (cd "$baseline" && sha256sum -c inventory-hashes.sha256 > /dev/null 2>&1)
  rc=$?
  if [ "$rc" -ne 0 ]; then
    printf 'HARD_STOP: copied preflight baseline checksum failure\n' >&2
    return 14
  fi
  (cd "$baseline" && sha256sum -c result.manifest.sha256 > /dev/null 2>&1)
  rc=$?
  if [ "$rc" -ne 0 ]; then
    printf 'HARD_STOP: copied preflight result manifest checksum failure\n' >&2
    return 14
  fi

  umask 077
  install -d -o root -g root -m 0700 /var/log/le-phase2a-post-reboot
  rc=$?; [ "$rc" -eq 0 ] || { printf 'HARD_STOP: cannot create post-reboot record parent\n' >&2; return 15; }
  post_record=$(mktemp -d "/var/log/le-phase2a-post-reboot/validation.$(date -u +%Y%m%dT%H%M%SZ).XXXXXXXX")
  rc=$?
  if [ "$rc" -ne 0 ] || [ -z "$post_record" ] || [ ! -d "$post_record" ]; then
    printf 'HARD_STOP: cannot atomically create unique post-reboot record\n' >&2
    return 16
  fi
  chown root:root "$post_record"; chmod 0700 "$post_record"
  : > "$post_record/failed-checks.txt"
  printf 'maintenance_record=%s\n' "$maintenance" > "$post_record/source.txt"

  check 'running-kernel' running-kernel.txt uname -a
  check_shell 'installed-kernels' installed-kernels.txt 'pacman -Q linux linux-headers linux-zen linux-zen-headers'
  check 'failed-units' failed-units.txt systemctl --failed --no-legend --plain --no-pager
  validate_failed_unit_disposition
  check_shell 'ssh-state' ssh-state.txt \
    'systemctl show sshd.service -p LoadState -p ActiveState -p SubState -p ExecMainStatus; journalctl -b -u sshd.service --no-pager'
  check_shell 'docker-daemon' docker-daemon.txt \
    "systemctl show docker.service -p LoadState -p ActiveState -p SubState -p ExecMainStatus; docker info"
  check_shell 'all-containers' containers-after.psv \
    "docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort"
  compare_containers
  compare_units
  validate_databases

  check_shell 'mounts-and-capacity' mounts-capacity.txt 'findmnt --real; df -hT / /boot /var/cache/pacman/pkg /tmp'
  check_shell 'network-routes-firewall-ports' network-after.txt \
    'ip -brief address; ip route show table all; ip -6 route show table all; nft list ruleset; ss -lntup'
  wg show > "$post_record/wireguard-after.txt" 2>&1
  wg_rc=$?; if [ "$wg_rc" -ne 0 ]; then printf 'wireguard-observation|rc=%s\n' "$wg_rc" > "$post_record/warnings.txt"; fi

  validate_health_url_file "$baseline/health-urls.sanitized"
  : > "$post_record/public-health.txt"
  while IFS= read -r url; do
    metrics=$(curl --fail --silent --show-error --max-redirs 0 \
      --connect-timeout 10 --max-time 30 --output /dev/null \
      --write-out 'http=%{http_code}|remote=%{remote_ip}|time=%{time_total}' \
      "$url" 2>> "$post_record/public-health-errors.txt")
    rc=$?
    printf '%s|%s\n' "$url" "$metrics" >> "$post_record/public-health.txt"
    if [ "$rc" -ne 0 ]; then add_failure "public-health-failed|url=$url|rc=$rc"; fi
  done < "$baseline/health-urls.sanitized"

  pacman -Dk > "$post_record/pacman-Dk.txt" 2>&1
  dk_rc=$?
  if [ "$dk_rc" -ne 0 ]; then add_failure "pacman-Dk-failed|rc=$dk_rc"; fi
  check_package_files
  check_shell 'target-packages' target-packages.txt 'pacman -Q rootlesskit slirp4netns openai-codex'
  check_shell 'pacnew-pacsave-inventory' pacnew-pacsave.txt \
    "find /etc -xdev -type f \( -name '*.pacnew' -o -name '*.pacsave' \) -printf '%p\\n' | sort; pacdiff -o 2>&1"

  find "$post_record" -maxdepth 1 -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum \
    > "$post_record/SHA256SUMS" 2>&1
  chmod -R go-rwx "$post_record"
  printf 'POST_REBOOT_RECORD=%s\n' "$post_record"
  if [ "$failures" -ne 0 ]; then
    printf 'POST_REBOOT_RESULT=HARD_STOP failures=%s\n' "$failures"
    return 20
  fi
  printf 'POST_REBOOT_RESULT=PASS failures=0\n'
  printf 'Rootless-Docker provisioning remains separately approval-gated.\n'
  return 0
}

main "$@"