le-app/docs/le-app-database-migration/phase2a-host-maintenance/execute-maintenance.sh

#!/usr/bin/env bash
# GENERATED FOR REVIEW. Run only as a separate process after explicit approval.
# No shell-wide error options and no parent-shell termination commands.

fail() {
  printf 'HARD_STOP: %s\n' "$1" >&2
  return "${2:-1}"
}

manifest_value() {
  key=$1
  file=$2
  value=$(awk -F= -v k="$key" '$1==k {sub(/^[^=]*=/, ""); print; found++} END {if(found!=1) exit 1}' "$file")
  rc=$?
  if [ "$rc" -ne 0 ]; then return "$rc"; fi
  printf '%s\n' "$value"
  return 0
}

validate_preflight() {
  preflight=$1
  if [ -z "$preflight" ] || [ ! -d "$preflight" ] || [ -L "$preflight" ]; then
    fail 'preflight path is missing, not a directory, or a symlink' 30
    return $?
  fi
  owner=$(stat -c %u "$preflight" 2>/dev/null)
  mode=$(stat -c %a "$preflight" 2>/dev/null)
  if [ "$owner" != 0 ] || [ "$mode" != 700 ]; then
    fail 'preflight record must be root-owned mode 0700' 31
    return $?
  fi
  manifest=$preflight/result.manifest
  if [ ! -f "$manifest" ] || [ -L "$manifest" ] || [ ! -f "$preflight/result.manifest.sha256" ]; then
    fail 'preflight result manifest is missing or a symlink' 32
    return $?
  fi
  (cd "$preflight" && sha256sum -c result.manifest.sha256 > /dev/null 2>&1)
  rc=$?
  if [ "$rc" -ne 0 ]; then fail 'preflight result manifest checksum failed' 32; return $?; fi
  format=$(manifest_value format "$manifest") || { fail 'malformed preflight format' 33; return $?; }
  recorded_host=$(manifest_value hostname "$manifest") || { fail 'malformed preflight hostname' 33; return $?; }
  recorded_boot=$(manifest_value boot_id "$manifest") || { fail 'malformed preflight boot ID' 33; return $?; }
  recorded_kernel=$(manifest_value running_kernel "$manifest") || { fail 'malformed preflight kernel' 33; return $?; }
  recorded_time=$(manifest_value timestamp_epoch "$manifest") || { fail 'malformed preflight timestamp' 33; return $?; }
  recorded_overall=$(manifest_value overall "$manifest") || { fail 'malformed preflight result' 33; return $?; }
  recorded_failure_count=$(manifest_value failure_count "$manifest") || { fail 'malformed preflight failure count' 33; return $?; }
  recorded_failed_checks_hash=$(manifest_value failed_checks_sha256 "$manifest") || { fail 'malformed failed-check hash' 33; return $?; }
  recorded_hash=$(manifest_value inventory_hashes_sha256 "$manifest") || { fail 'malformed inventory hash' 33; return $?; }
  [ "$format" = le-phase2a-preflight-v1 ] || { fail 'unsupported preflight format' 34; return $?; }
  [ "$recorded_overall" = PASS ] || { fail 'preflight result is not PASS' 35; return $?; }
  [ "$recorded_failure_count" = 0 ] || { fail 'preflight records mandatory failures' 35; return $?; }
  if [ ! -f "$preflight/failed-mandatory-checks.txt" ] || [ -L "$preflight/failed-mandatory-checks.txt" ]; then
    fail 'failed-check list is missing or a symlink' 35
    return $?
  fi
  actual_failed_checks_hash=$(sha256sum "$preflight/failed-mandatory-checks.txt" 2>/dev/null | awk '{print $1}')
  [ "$actual_failed_checks_hash" = "$recorded_failed_checks_hash" ] || { fail 'failed-check list checksum mismatch' 35; return $?; }
  [ "$recorded_host" = "$(hostname)" ] || { fail 'preflight belongs to another host' 36; return $?; }
  [ "$recorded_boot" = "$(cat /proc/sys/kernel/random/boot_id)" ] || { fail 'preflight belongs to another boot' 37; return $?; }
  [ "$recorded_kernel" = "$(uname -r)" ] || { fail 'running kernel changed since preflight' 37; return $?; }
  case "$recorded_time" in *[!0-9]*|'') fail 'preflight timestamp is invalid' 38; return $? ;; esac
  now=$(date +%s)
  age=$((now - recorded_time))
  if [ "$age" -lt 0 ] || [ "$age" -gt 1800 ]; then
    fail "preflight is stale; age=${age}s limit=1800s" 39
    return $?
  fi
  actual_hash=$(sha256sum "$preflight/inventory-hashes.sha256" 2>/dev/null | awk '{print $1}')
  [ "$actual_hash" = "$recorded_hash" ] || { fail 'preflight inventory-hash manifest checksum mismatch' 40; return $?; }
  (cd "$preflight" && sha256sum -c inventory-hashes.sha256 > /dev/null 2>&1)
  rc=$?
  if [ "$rc" -ne 0 ]; then
    fail 'one or more preflight inventory files failed checksum validation' 41
    return $?
  fi
  validated_preflight=$preflight
  return 0
}

validate_health_url_file() {
  file=$1
  [ -s "$file" ] || { fail 'validated preflight contains no health URLs' 42; return $?; }
  while IFS= read -r url; do
    case "$url" in https://*) ;; *) fail 'unsupported health URL scheme' 42; return $? ;; esac
    authority=${url#https://}; authority=${authority%%/*}
    case "$url" in *'?'*|*'#'*|*$'\n'*|*$'\r'*|*$'\t'*|*' '*) fail 'unsafe health URL content' 42; return $? ;; esac
    case "$authority" in ''|*@*) fail 'health URL user-info or empty authority rejected' 42; return $? ;; esac
  done < "$file"
  return 0
}

rerun_volatile_checks() {
  systemctl --failed --no-legend --plain --no-pager > "$record/volatile-failed-units.txt" 2>&1
  rc=$?
  [ "$rc" -eq 0 ] || { fail 'cannot capture current failed units' "$rc"; return $?; }
  [ ! -s "$record/volatile-failed-units.txt" ] || { fail 'one or more failed units are present' 45; return $?; }
  pacman -Dk > "$record/volatile-pacman-Dk.txt" 2>&1
  rc=$?
  [ "$rc" -eq 0 ] || { fail 'package database consistency failed immediately before execution' "$rc"; return $?; }
  df -hT / /boot /var/cache/pacman/pkg /tmp > "$record/volatile-disk-space.txt" 2>&1
  rc=$?
  [ "$rc" -eq 0 ] || { fail 'cannot verify current disk space' "$rc"; return $?; }
  while IFS= read -r container; do
    [ -n "$container" ] || continue
    running=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
    health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$container" 2>/dev/null)
    printf '%s|running=%s|health=%s\n' "$container" "${running:-missing}" "${health:-unknown}" \
      >> "$record/volatile-container-health.txt"
    if [ "$running" != true ] || [ "$health" = unhealthy ]; then
      fail "previously running container is unavailable: $container" 43
      return $?
    fi
  done < "$validated_preflight/running-containers-before.txt"
  : > "$record/volatile-public-health.txt"
  while IFS= read -r url; do
    metrics=$(curl --fail --silent --show-error --max-redirs 0 \
      --connect-timeout 10 --max-time 30 --output /dev/null \
      --write-out 'http=%{http_code}|remote=%{remote_ip}|time=%{time_total}' \
      "$url" 2>> "$record/volatile-public-health-errors.txt")
    rc=$?
    printf '%s|%s\n' "$url" "$metrics" >> "$record/volatile-public-health.txt"
    if [ "$rc" -ne 0 ]; then fail "public health failed immediately before execution: $url" 44; return $?; fi
  done < "$validated_preflight/health-urls.sanitized"
  return 0
}

normalize_transaction() {
  source_file=$1
  output_file=$2
  downgrade_file=$3
  : > "$output_file"
  : > "$downgrade_file"
  while IFS='|' read -r name new_version repository location download_bytes; do
    [ -n "$name" ] || continue
    old_line=$(pacman -Q "$name" 2>/dev/null)
    if [ -n "$old_line" ]; then
      old_version=${old_line#* }
      comparison=$(vercmp "$new_version" "$old_version")
      if [ "$comparison" -lt 0 ]; then action=downgrade
      elif [ "$comparison" -eq 0 ]; then action=same
      else action=upgrade
      fi
    else
      old_version=-
      action=new
    fi
    printf '%s|%s|%s|%s\n' "$name" "$old_version" "$new_version" "$action" >> "$output_file"
    if [ "$action" = downgrade ]; then
      printf '%s|%s|%s|%s\n' "$name" "$old_version" "$new_version" "$action" >> "$downgrade_file"
    fi
  done < "$source_file"
  sort -o "$output_file" "$output_file"
  return 0
}

main() {
  if [ "$(id -u)" -ne 0 ]; then fail 'run as a separate root process' 10; return $?; fi
  if [ "$1" != --preflight ] || [ -z "${2:-}" ] || [ -n "${3:-}" ]; then
    fail 'usage: execute-maintenance.sh --preflight /var/log/le-phase2a-preflight/preflight.XXXXXXXX' 11
    return $?
  fi
  validate_preflight "$2" || return $?
  validate_health_url_file "$validated_preflight/health-urls.sanitized" || return $?

  umask 077
  install -d -o root -g root -m 0700 /var/log/le-phase2a-maintenance
  rc=$?
  [ "$rc" -eq 0 ] || { fail 'cannot create root-only maintenance record parent' "$rc"; return $?; }
  record=$(mktemp -d "/var/log/le-phase2a-maintenance/maintenance.$(date -u +%Y%m%dT%H%M%SZ).XXXXXXXX")
  rc=$?
  if [ "$rc" -ne 0 ] || [ -z "$record" ] || [ ! -d "$record" ]; then
    fail 'cannot atomically create unique maintenance record' 12
    return $?
  fi
  chown root:root "$record"; chmod 0700 "$record"
  maintenance_start=$(date -u +%FT%TZ)
  printf '%s\n' "$record" > "$record/RECORD_PATH.txt"
  cp -a "$validated_preflight" "$record/preflight-record"
  rc=$?
  [ "$rc" -eq 0 ] || { fail 'cannot copy validated preflight record' "$rc"; return $?; }

  report=/tmp/le-phase2-package-decision-20260712.Mn7mIx/phase2-package-decision-report.md
  report_manifest=/tmp/le-phase2-package-decision-20260712.Mn7mIx/SHA256SUMS.main
  printf '%s  %s\n' 195d74afe121556294318d0ed49e3fb5504088dbb31a4c0786b8adea44f1369d "$report" | sha256sum -c - \
    > "$record/approved-report-checksum.txt" 2>&1
  rc=$?
  [ "$rc" -eq 0 ] || { fail 'approved report checksum failed' "$rc"; return $?; }
  cp "$report" "$report_manifest" "$record/"
  rc=$?
  [ "$rc" -eq 0 ] || { fail 'cannot copy approved report and checksum manifest' "$rc"; return $?; }

  uname -a > "$record/running-kernel-before.txt" 2>&1
  rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture running kernel' "$rc"; return $?; }
  cat /proc/sys/kernel/random/boot_id > "$record/boot-id-before.txt" 2>&1
  rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture boot ID' "$rc"; return $?; }
  pacman -Q > "$record/packages-before.txt" 2>&1
  rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture installed package inventory' "$rc"; return $?; }
  cp "$validated_preflight/bootloader-inventory.txt" "$validated_preflight/kernel-initramfs-before.txt" \
    "$validated_preflight/running-containers-before.txt" "$validated_preflight/containers-all.psv" \
    "$validated_preflight/critical-units-baseline.psv" "$validated_preflight/database-baseline.psv" "$record/"
  rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot copy essential rollback baseline evidence' "$rc"; return $?; }
  systemctl --failed --no-legend --plain --no-pager > "$record/failed-units-before.txt" 2>&1
  rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture failed-unit baseline' "$rc"; return $?; }
  [ ! -s "$record/failed-units-before.txt" ] || { fail 'failed units appeared before transaction preview' 45; return $?; }

  rerun_volatile_checks || return $?

  preview_root=$(mktemp -d /tmp/le-phase2a-execution-preview.XXXXXXXX)
  rc=$?
  if [ "$rc" -ne 0 ] || [ -z "$preview_root" ] || [ ! -d "$preview_root" ]; then
    fail 'cannot create isolated transaction preview directory' 13
    return $?
  fi
  chmod 0700 "$preview_root"; mkdir -m 0700 "$preview_root/db" "$preview_root/cache"
  ln -s /var/lib/pacman/local "$preview_root/db/local"

  fakeroot -- pacman --config /etc/pacman.conf --dbpath "$preview_root/db" \
    --cachedir "$preview_root/cache" --logfile "$preview_root/pacman-preview.log" -Sy \
    > "$record/isolated-sync-refresh.log" 2>&1
  rc=$?
  [ "$rc" -eq 0 ] || { fail 'isolated repository refresh failed; live databases untouched' "$rc"; return $?; }
  fakeroot -- pacman --config /etc/pacman.conf --dbpath "$preview_root/db" \
    --cachedir "$preview_root/cache" --logfile "$preview_root/pacman-preview.log" \
    -Su --needed --print-format '%n|%v|%r|%l|%s' rootlesskit slirp4netns openai-codex \
    > "$record/latest-transaction.psv" 2> "$record/latest-transaction.err"
  rc=$?
  [ "$rc" -eq 0 ] || { fail 'latest complete transaction resolution failed' "$rc"; return $?; }
  normalize_transaction "$record/latest-transaction.psv" "$record/latest-normalized-actions.psv" "$record/latest-downgrades.psv"
  [ ! -s "$record/latest-downgrades.psv" ] || { fail 'latest transaction contains a downgrade' 14; return $?; }

  awk -F'|' '{print $1"|"$2"|"$3"|"$5}' \
    /tmp/le-phase2-package-decision-20260712.Mn7mIx/inventory.psv | sort \
    > "$record/approved-normalized-actions.psv"
  rc=$?
  [ "$rc" -eq 0 ] || { fail 'cannot normalize approved preview' "$rc"; return $?; }
  diff -u "$record/approved-normalized-actions.psv" "$record/latest-normalized-actions.psv" \
    > "$record/approved-vs-latest.diff" 2>&1
  diff_rc=$?
  if [ "$diff_rc" -ne 0 ]; then
    fail 'latest package names, versions, or actions differ from the approved preview' 15
    return $?
  fi
  latest_count=$(wc -l < "$record/latest-normalized-actions.psv")
  approved_count=$(wc -l < "$record/approved-normalized-actions.psv")
  [ "$latest_count" = "$approved_count" ] || { fail 'transaction package counts differ' 15; return $?; }
  for required in rootlesskit slirp4netns openai-codex; do
    grep -q "^${required}|" "$record/latest-normalized-actions.psv"
    rc=$?; [ "$rc" -eq 0 ] || { fail "latest transaction omits $required" 16; return $?; }
  done

  cp "$record/latest-transaction.psv" "$record/latest-normalized-actions.psv" \
    "$record/approved-normalized-actions.psv" "$record/approved-vs-latest.diff" "$record/transaction-for-approval.psv"
  rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot preserve transaction preview evidence' "$rc"; return $?; }
  sha256sum "$record/latest-transaction.psv" "$record/latest-normalized-actions.psv" \
    "$record/approved-normalized-actions.psv" > "$record/transaction-preview-hashes.sha256"

  printf '\nMachine comparison passed: packages=%s, no downgrades, exact match to approved preview.\n' "$latest_count"
  cat "$record/latest-normalized-actions.psv"
  printf '\nType exactly: APPROVE MATCHED COMPLETE HOST UPGRADE\n> '
  IFS= read -r approval_one
  if [ "$approval_one" != 'APPROVE MATCHED COMPLETE HOST UPGRADE' ]; then
    fail 'first exact operator approval not granted' 20
    return $?
  fi

  rerun_volatile_checks || return $?
  printf '\nLIMITATION: the final live -Syu refresh may change after this isolated comparison.\n'
  printf 'Pacman will display its final interactive transaction and ask Y/n. The wrapper cannot machine-compare that post-refresh transaction without an unsafe live -Sy-only staging step.\n'
  printf 'Type exactly: ACKNOWLEDGE FINAL PACMAN TRANSACTION REQUIRES MANUAL REVIEW\n> '
  IFS= read -r approval_two
  if [ "$approval_two" != 'ACKNOWLEDGE FINAL PACMAN TRANSACTION REQUIRES MANUAL REVIEW' ]; then
    fail 'second exact operator acknowledgement not granted' 21
    return $?
  fi

  printf 'At Pacman Y/n, answer no if any package, version, action, count, downgrade status, signature, or repository differs.\n'
  script -q -e -f -c 'pacman -Syu --needed rootlesskit slirp4netns openai-codex' "$record/pacman-complete.log"
  rc=$?
  if [ "$rc" -ne 0 ]; then fail "Pacman returned $rc; stop without ad hoc repair" "$rc"; return $?; fi

  pacman -Q > "$record/packages-after.txt" 2>&1
  rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture post-transaction package inventory' "$rc"; return $?; }
  diff -u "$record/packages-before.txt" "$record/packages-after.txt" > "$record/package-inventory.diff" 2>&1
  diff_rc=$?; [ "$diff_rc" -le 1 ] || { fail 'package inventory comparison failed' "$diff_rc"; return $?; }
  find /etc -xdev -type f \( -name '*.pacnew' -o -name '*.pacsave' \) -printf '%p\n' | sort \
    > "$record/pacnew-pacsave-after.txt" 2>&1
  rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot inventory pacnew/pacsave files' "$rc"; return $?; }
  pacdiff -o > "$record/pacdiff-outstanding.txt" 2>&1
  pacdiff_rc=$?
  if [ "$pacdiff_rc" -ne 0 ] && [ "$pacdiff_rc" -ne 1 ]; then
    printf 'pacdiff-observation|rc=%s\n' "$pacdiff_rc" > "$record/warnings-after.txt"
  fi
  grep -E -i 'hook|restart|reload|dkms|dracut|initramfs|mkinitcpio|grub|pacnew|pacsave|warning|error|failed' \
    "$record/pacman-complete.log" > "$record/hook-restart-summary.txt" 2>&1
  systemctl --failed --no-legend --plain --no-pager > "$record/failed-units-after.txt" 2>&1
  journal_units='sshd.service accounts-daemon.service docker.service'
  while IFS='|' read -r unit _; do journal_units="$journal_units $unit"; done < "$record/critical-units-baseline.psv"
  journalctl --since "$maintenance_start" --no-pager $(for unit in $journal_units; do printf -- '-u %q ' "$unit"; done) \
    > "$record/relevant-service-journal.txt" 2>&1
  dkms status > "$record/dkms-after.txt" 2>&1
  find /boot /usr/lib/modules -maxdepth 2 -type f \
    \( -name 'vmlinuz*' -o -name 'initramfs*' -o -name 'initrd*' -o -name pkgbase \) \
    -printf '%p size=%s mtime=%TY-%Tm-%TdT%TH:%TM:%TS\n' | sort > "$record/kernel-boot-artifacts-after.txt" 2>&1
  docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort > "$record/containers-after.txt" 2>&1
  find "$record" -maxdepth 1 -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum \
    > "$record/SHA256SUMS" 2>&1
  chmod -R go-rwx "$record"

  printf 'PACKAGE TRANSACTION COMPLETED; REBOOT NOT PERFORMED.\n'
  printf 'MAINTENANCE_RECORD=%s\n' "$record"
  printf 'Review all warnings, hooks, services, kernel artifacts, and pacnew/pacsave files.\n'
  printf 'Reboot and rootless-Docker provisioning remain separate approval gates.\n'
  return 0
}

main "$@"