#!/usr/bin/env bash # GENERATED FOR REVIEW. Run only as a separate process after explicit approval. # No shell-wide error options and no parent-shell termination commands. fail() { printf 'HARD_STOP: %s\n' "$1" >&2 return "${2:-1}" } manifest_value() { key=$1 file=$2 value=$(awk -F= -v k="$key" '$1==k {sub(/^[^=]*=/, ""); print; found++} END {if(found!=1) exit 1}' "$file") rc=$? if [ "$rc" -ne 0 ]; then return "$rc"; fi printf '%s\n' "$value" return 0 } validate_preflight() { preflight=$1 if [ -z "$preflight" ] || [ ! -d "$preflight" ] || [ -L "$preflight" ]; then fail 'preflight path is missing, not a directory, or a symlink' 30 return $? fi owner=$(stat -c %u "$preflight" 2>/dev/null) mode=$(stat -c %a "$preflight" 2>/dev/null) if [ "$owner" != 0 ] || [ "$mode" != 700 ]; then fail 'preflight record must be root-owned mode 0700' 31 return $? fi manifest=$preflight/result.manifest if [ ! -f "$manifest" ] || [ -L "$manifest" ] || [ ! -f "$preflight/result.manifest.sha256" ]; then fail 'preflight result manifest is missing or a symlink' 32 return $? fi (cd "$preflight" && sha256sum -c result.manifest.sha256 > /dev/null 2>&1) rc=$? if [ "$rc" -ne 0 ]; then fail 'preflight result manifest checksum failed' 32; return $?; fi format=$(manifest_value format "$manifest") || { fail 'malformed preflight format' 33; return $?; } recorded_host=$(manifest_value hostname "$manifest") || { fail 'malformed preflight hostname' 33; return $?; } recorded_boot=$(manifest_value boot_id "$manifest") || { fail 'malformed preflight boot ID' 33; return $?; } recorded_kernel=$(manifest_value running_kernel "$manifest") || { fail 'malformed preflight kernel' 33; return $?; } recorded_time=$(manifest_value timestamp_epoch "$manifest") || { fail 'malformed preflight timestamp' 33; return $?; } recorded_overall=$(manifest_value overall "$manifest") || { fail 'malformed preflight result' 33; return $?; } recorded_failure_count=$(manifest_value failure_count "$manifest") || { fail 'malformed preflight failure count' 33; return $?; } recorded_failed_checks_hash=$(manifest_value failed_checks_sha256 "$manifest") || { fail 'malformed failed-check hash' 33; return $?; } recorded_hash=$(manifest_value inventory_hashes_sha256 "$manifest") || { fail 'malformed inventory hash' 33; return $?; } [ "$format" = le-phase2a-preflight-v1 ] || { fail 'unsupported preflight format' 34; return $?; } [ "$recorded_overall" = PASS ] || { fail 'preflight result is not PASS' 35; return $?; } [ "$recorded_failure_count" = 0 ] || { fail 'preflight records mandatory failures' 35; return $?; } if [ ! -f "$preflight/failed-mandatory-checks.txt" ] || [ -L "$preflight/failed-mandatory-checks.txt" ]; then fail 'failed-check list is missing or a symlink' 35 return $? fi actual_failed_checks_hash=$(sha256sum "$preflight/failed-mandatory-checks.txt" 2>/dev/null | awk '{print $1}') [ "$actual_failed_checks_hash" = "$recorded_failed_checks_hash" ] || { fail 'failed-check list checksum mismatch' 35; return $?; } [ "$recorded_host" = "$(hostname)" ] || { fail 'preflight belongs to another host' 36; return $?; } [ "$recorded_boot" = "$(cat /proc/sys/kernel/random/boot_id)" ] || { fail 'preflight belongs to another boot' 37; return $?; } [ "$recorded_kernel" = "$(uname -r)" ] || { fail 'running kernel changed since preflight' 37; return $?; } case "$recorded_time" in *[!0-9]*|'') fail 'preflight timestamp is invalid' 38; return $? ;; esac now=$(date +%s) age=$((now - recorded_time)) if [ "$age" -lt 0 ] || [ "$age" -gt 1800 ]; then fail "preflight is stale; age=${age}s limit=1800s" 39 return $? fi actual_hash=$(sha256sum "$preflight/inventory-hashes.sha256" 2>/dev/null | awk '{print $1}') [ "$actual_hash" = "$recorded_hash" ] || { fail 'preflight inventory-hash manifest checksum mismatch' 40; return $?; } (cd "$preflight" && sha256sum -c inventory-hashes.sha256 > /dev/null 2>&1) rc=$? if [ "$rc" -ne 0 ]; then fail 'one or more preflight inventory files failed checksum validation' 41 return $? fi validated_preflight=$preflight return 0 } validate_health_url_file() { file=$1 [ -s "$file" ] || { fail 'validated preflight contains no health URLs' 42; return $?; } while IFS= read -r url; do case "$url" in https://*) ;; *) fail 'unsupported health URL scheme' 42; return $? ;; esac authority=${url#https://}; authority=${authority%%/*} case "$url" in *'?'*|*'#'*|*$'\n'*|*$'\r'*|*$'\t'*|*' '*) fail 'unsafe health URL content' 42; return $? ;; esac case "$authority" in ''|*@*) fail 'health URL user-info or empty authority rejected' 42; return $? ;; esac done < "$file" return 0 } rerun_volatile_checks() { systemctl --failed --no-legend --plain --no-pager > "$record/volatile-failed-units.txt" 2>&1 rc=$? [ "$rc" -eq 0 ] || { fail 'cannot capture current failed units' "$rc"; return $?; } [ ! -s "$record/volatile-failed-units.txt" ] || { fail 'one or more failed units are present' 45; return $?; } pacman -Dk > "$record/volatile-pacman-Dk.txt" 2>&1 rc=$? [ "$rc" -eq 0 ] || { fail 'package database consistency failed immediately before execution' "$rc"; return $?; } df -hT / /boot /var/cache/pacman/pkg /tmp > "$record/volatile-disk-space.txt" 2>&1 rc=$? [ "$rc" -eq 0 ] || { fail 'cannot verify current disk space' "$rc"; return $?; } while IFS= read -r container; do [ -n "$container" ] || continue running=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$container" 2>/dev/null) printf '%s|running=%s|health=%s\n' "$container" "${running:-missing}" "${health:-unknown}" \ >> "$record/volatile-container-health.txt" if [ "$running" != true ] || [ "$health" = unhealthy ]; then fail "previously running container is unavailable: $container" 43 return $? fi done < "$validated_preflight/running-containers-before.txt" : > "$record/volatile-public-health.txt" while IFS= read -r url; do metrics=$(curl --fail --silent --show-error --max-redirs 0 \ --connect-timeout 10 --max-time 30 --output /dev/null \ --write-out 'http=%{http_code}|remote=%{remote_ip}|time=%{time_total}' \ "$url" 2>> "$record/volatile-public-health-errors.txt") rc=$? printf '%s|%s\n' "$url" "$metrics" >> "$record/volatile-public-health.txt" if [ "$rc" -ne 0 ]; then fail "public health failed immediately before execution: $url" 44; return $?; fi done < "$validated_preflight/health-urls.sanitized" return 0 } normalize_transaction() { source_file=$1 output_file=$2 downgrade_file=$3 : > "$output_file" : > "$downgrade_file" while IFS='|' read -r name new_version repository location download_bytes; do [ -n "$name" ] || continue old_line=$(pacman -Q "$name" 2>/dev/null) if [ -n "$old_line" ]; then old_version=${old_line#* } comparison=$(vercmp "$new_version" "$old_version") if [ "$comparison" -lt 0 ]; then action=downgrade elif [ "$comparison" -eq 0 ]; then action=same else action=upgrade fi else old_version=- action=new fi printf '%s|%s|%s|%s\n' "$name" "$old_version" "$new_version" "$action" >> "$output_file" if [ "$action" = downgrade ]; then printf '%s|%s|%s|%s\n' "$name" "$old_version" "$new_version" "$action" >> "$downgrade_file" fi done < "$source_file" sort -o "$output_file" "$output_file" return 0 } main() { if [ "$(id -u)" -ne 0 ]; then fail 'run as a separate root process' 10; return $?; fi if [ "$1" != --preflight ] || [ -z "${2:-}" ] || [ -n "${3:-}" ]; then fail 'usage: execute-maintenance.sh --preflight /var/log/le-phase2a-preflight/preflight.XXXXXXXX' 11 return $? fi validate_preflight "$2" || return $? validate_health_url_file "$validated_preflight/health-urls.sanitized" || return $? umask 077 install -d -o root -g root -m 0700 /var/log/le-phase2a-maintenance rc=$? [ "$rc" -eq 0 ] || { fail 'cannot create root-only maintenance record parent' "$rc"; return $?; } record=$(mktemp -d "/var/log/le-phase2a-maintenance/maintenance.$(date -u +%Y%m%dT%H%M%SZ).XXXXXXXX") rc=$? if [ "$rc" -ne 0 ] || [ -z "$record" ] || [ ! -d "$record" ]; then fail 'cannot atomically create unique maintenance record' 12 return $? fi chown root:root "$record"; chmod 0700 "$record" maintenance_start=$(date -u +%FT%TZ) printf '%s\n' "$record" > "$record/RECORD_PATH.txt" cp -a "$validated_preflight" "$record/preflight-record" rc=$? [ "$rc" -eq 0 ] || { fail 'cannot copy validated preflight record' "$rc"; return $?; } report=/tmp/le-phase2-package-decision-20260712.Mn7mIx/phase2-package-decision-report.md report_manifest=/tmp/le-phase2-package-decision-20260712.Mn7mIx/SHA256SUMS.main printf '%s %s\n' 195d74afe121556294318d0ed49e3fb5504088dbb31a4c0786b8adea44f1369d "$report" | sha256sum -c - \ > "$record/approved-report-checksum.txt" 2>&1 rc=$? [ "$rc" -eq 0 ] || { fail 'approved report checksum failed' "$rc"; return $?; } cp "$report" "$report_manifest" "$record/" rc=$? [ "$rc" -eq 0 ] || { fail 'cannot copy approved report and checksum manifest' "$rc"; return $?; } uname -a > "$record/running-kernel-before.txt" 2>&1 rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture running kernel' "$rc"; return $?; } cat /proc/sys/kernel/random/boot_id > "$record/boot-id-before.txt" 2>&1 rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture boot ID' "$rc"; return $?; } pacman -Q > "$record/packages-before.txt" 2>&1 rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture installed package inventory' "$rc"; return $?; } cp "$validated_preflight/bootloader-inventory.txt" "$validated_preflight/kernel-initramfs-before.txt" \ "$validated_preflight/running-containers-before.txt" "$validated_preflight/containers-all.psv" \ "$validated_preflight/critical-units-baseline.psv" "$validated_preflight/database-baseline.psv" "$record/" rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot copy essential rollback baseline evidence' "$rc"; return $?; } systemctl --failed --no-legend --plain --no-pager > "$record/failed-units-before.txt" 2>&1 rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture failed-unit baseline' "$rc"; return $?; } [ ! -s "$record/failed-units-before.txt" ] || { fail 'failed units appeared before transaction preview' 45; return $?; } rerun_volatile_checks || return $? preview_root=$(mktemp -d /tmp/le-phase2a-execution-preview.XXXXXXXX) rc=$? if [ "$rc" -ne 0 ] || [ -z "$preview_root" ] || [ ! -d "$preview_root" ]; then fail 'cannot create isolated transaction preview directory' 13 return $? fi chmod 0700 "$preview_root"; mkdir -m 0700 "$preview_root/db" "$preview_root/cache" ln -s /var/lib/pacman/local "$preview_root/db/local" fakeroot -- pacman --config /etc/pacman.conf --dbpath "$preview_root/db" \ --cachedir "$preview_root/cache" --logfile "$preview_root/pacman-preview.log" -Sy \ > "$record/isolated-sync-refresh.log" 2>&1 rc=$? [ "$rc" -eq 0 ] || { fail 'isolated repository refresh failed; live databases untouched' "$rc"; return $?; } fakeroot -- pacman --config /etc/pacman.conf --dbpath "$preview_root/db" \ --cachedir "$preview_root/cache" --logfile "$preview_root/pacman-preview.log" \ -Su --needed --print-format '%n|%v|%r|%l|%s' rootlesskit slirp4netns openai-codex \ > "$record/latest-transaction.psv" 2> "$record/latest-transaction.err" rc=$? [ "$rc" -eq 0 ] || { fail 'latest complete transaction resolution failed' "$rc"; return $?; } normalize_transaction "$record/latest-transaction.psv" "$record/latest-normalized-actions.psv" "$record/latest-downgrades.psv" [ ! -s "$record/latest-downgrades.psv" ] || { fail 'latest transaction contains a downgrade' 14; return $?; } awk -F'|' '{print $1"|"$2"|"$3"|"$5}' \ /tmp/le-phase2-package-decision-20260712.Mn7mIx/inventory.psv | sort \ > "$record/approved-normalized-actions.psv" rc=$? [ "$rc" -eq 0 ] || { fail 'cannot normalize approved preview' "$rc"; return $?; } diff -u "$record/approved-normalized-actions.psv" "$record/latest-normalized-actions.psv" \ > "$record/approved-vs-latest.diff" 2>&1 diff_rc=$? if [ "$diff_rc" -ne 0 ]; then fail 'latest package names, versions, or actions differ from the approved preview' 15 return $? fi latest_count=$(wc -l < "$record/latest-normalized-actions.psv") approved_count=$(wc -l < "$record/approved-normalized-actions.psv") [ "$latest_count" = "$approved_count" ] || { fail 'transaction package counts differ' 15; return $?; } for required in rootlesskit slirp4netns openai-codex; do grep -q "^${required}|" "$record/latest-normalized-actions.psv" rc=$?; [ "$rc" -eq 0 ] || { fail "latest transaction omits $required" 16; return $?; } done cp "$record/latest-transaction.psv" "$record/latest-normalized-actions.psv" \ "$record/approved-normalized-actions.psv" "$record/approved-vs-latest.diff" "$record/transaction-for-approval.psv" rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot preserve transaction preview evidence' "$rc"; return $?; } sha256sum "$record/latest-transaction.psv" "$record/latest-normalized-actions.psv" \ "$record/approved-normalized-actions.psv" > "$record/transaction-preview-hashes.sha256" printf '\nMachine comparison passed: packages=%s, no downgrades, exact match to approved preview.\n' "$latest_count" cat "$record/latest-normalized-actions.psv" printf '\nType exactly: APPROVE MATCHED COMPLETE HOST UPGRADE\n> ' IFS= read -r approval_one if [ "$approval_one" != 'APPROVE MATCHED COMPLETE HOST UPGRADE' ]; then fail 'first exact operator approval not granted' 20 return $? fi rerun_volatile_checks || return $? printf '\nLIMITATION: the final live -Syu refresh may change after this isolated comparison.\n' printf 'Pacman will display its final interactive transaction and ask Y/n. The wrapper cannot machine-compare that post-refresh transaction without an unsafe live -Sy-only staging step.\n' printf 'Type exactly: ACKNOWLEDGE FINAL PACMAN TRANSACTION REQUIRES MANUAL REVIEW\n> ' IFS= read -r approval_two if [ "$approval_two" != 'ACKNOWLEDGE FINAL PACMAN TRANSACTION REQUIRES MANUAL REVIEW' ]; then fail 'second exact operator acknowledgement not granted' 21 return $? fi printf 'At Pacman Y/n, answer no if any package, version, action, count, downgrade status, signature, or repository differs.\n' script -q -e -f -c 'pacman -Syu --needed rootlesskit slirp4netns openai-codex' "$record/pacman-complete.log" rc=$? if [ "$rc" -ne 0 ]; then fail "Pacman returned $rc; stop without ad hoc repair" "$rc"; return $?; fi pacman -Q > "$record/packages-after.txt" 2>&1 rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture post-transaction package inventory' "$rc"; return $?; } diff -u "$record/packages-before.txt" "$record/packages-after.txt" > "$record/package-inventory.diff" 2>&1 diff_rc=$?; [ "$diff_rc" -le 1 ] || { fail 'package inventory comparison failed' "$diff_rc"; return $?; } find /etc -xdev -type f \( -name '*.pacnew' -o -name '*.pacsave' \) -printf '%p\n' | sort \ > "$record/pacnew-pacsave-after.txt" 2>&1 rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot inventory pacnew/pacsave files' "$rc"; return $?; } pacdiff -o > "$record/pacdiff-outstanding.txt" 2>&1 pacdiff_rc=$? if [ "$pacdiff_rc" -ne 0 ] && [ "$pacdiff_rc" -ne 1 ]; then printf 'pacdiff-observation|rc=%s\n' "$pacdiff_rc" > "$record/warnings-after.txt" fi grep -E -i 'hook|restart|reload|dkms|dracut|initramfs|mkinitcpio|grub|pacnew|pacsave|warning|error|failed' \ "$record/pacman-complete.log" > "$record/hook-restart-summary.txt" 2>&1 systemctl --failed --no-legend --plain --no-pager > "$record/failed-units-after.txt" 2>&1 journal_units='sshd.service accounts-daemon.service docker.service' while IFS='|' read -r unit _; do journal_units="$journal_units $unit"; done < "$record/critical-units-baseline.psv" journalctl --since "$maintenance_start" --no-pager $(for unit in $journal_units; do printf -- '-u %q ' "$unit"; done) \ > "$record/relevant-service-journal.txt" 2>&1 dkms status > "$record/dkms-after.txt" 2>&1 find /boot /usr/lib/modules -maxdepth 2 -type f \ \( -name 'vmlinuz*' -o -name 'initramfs*' -o -name 'initrd*' -o -name pkgbase \) \ -printf '%p size=%s mtime=%TY-%Tm-%TdT%TH:%TM:%TS\n' | sort > "$record/kernel-boot-artifacts-after.txt" 2>&1 docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort > "$record/containers-after.txt" 2>&1 find "$record" -maxdepth 1 -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum \ > "$record/SHA256SUMS" 2>&1 chmod -R go-rwx "$record" printf 'PACKAGE TRANSACTION COMPLETED; REBOOT NOT PERFORMED.\n' printf 'MAINTENANCE_RECORD=%s\n' "$record" printf 'Review all warnings, hooks, services, kernel artifacts, and pacnew/pacsave files.\n' printf 'Reboot and rootless-Docker provisioning remain separate approval gates.\n' return 0 } main "$@"