471 lines
24 KiB
Bash
471 lines
24 KiB
Bash
#!/usr/bin/env bash
|
|
# GENERATED FOR REVIEW. Run only as a separate process after explicit approval.
|
|
# No shell-wide error options and no parent-shell termination commands.
|
|
|
|
fail() {
|
|
printf 'HARD_STOP: %s\n' "$1" >&2
|
|
return "${2:-1}"
|
|
}
|
|
|
|
manifest_value() {
|
|
key=$1
|
|
file=$2
|
|
count=$(awk -F= -v k="$key" '$1==k {found++} END {print found+0}' "$file")
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ] || [ "$count" != 1 ]; then return 1; fi
|
|
value=$(awk -F= -v k="$key" '$1==k {sub(/^[^=]*=/, ""); print}' "$file")
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then return "$rc"; fi
|
|
printf '%s\n' "$value"
|
|
return 0
|
|
}
|
|
|
|
validate_preflight() {
|
|
preflight=$1
|
|
if [ -z "$preflight" ] || [ ! -d "$preflight" ] || [ -L "$preflight" ]; then
|
|
fail 'preflight path is missing, not a directory, or a symlink' 30
|
|
return $?
|
|
fi
|
|
owner=$(stat -c %u "$preflight" 2>/dev/null)
|
|
mode=$(stat -c %a "$preflight" 2>/dev/null)
|
|
if [ "$owner" != 0 ] || [ "$mode" != 700 ]; then
|
|
fail 'preflight record must be root-owned mode 0700' 31
|
|
return $?
|
|
fi
|
|
manifest=$preflight/result.manifest
|
|
if [ ! -f "$manifest" ] || [ -L "$manifest" ] || [ ! -f "$preflight/result.manifest.sha256" ]; then
|
|
fail 'preflight result manifest is missing or a symlink' 32
|
|
return $?
|
|
fi
|
|
(cd "$preflight" && sha256sum -c result.manifest.sha256 > /dev/null 2>&1)
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then fail 'preflight result manifest checksum failed' 32; return $?; fi
|
|
format=$(manifest_value format "$manifest") || { fail 'malformed preflight format' 33; return $?; }
|
|
recorded_host=$(manifest_value hostname "$manifest") || { fail 'malformed preflight hostname' 33; return $?; }
|
|
recorded_boot=$(manifest_value boot_id "$manifest") || { fail 'malformed preflight boot ID' 33; return $?; }
|
|
recorded_kernel=$(manifest_value running_kernel "$manifest") || { fail 'malformed preflight kernel' 33; return $?; }
|
|
recorded_time=$(manifest_value timestamp_epoch "$manifest") || { fail 'malformed preflight timestamp' 33; return $?; }
|
|
recorded_overall=$(manifest_value overall "$manifest") || { fail 'malformed preflight result' 33; return $?; }
|
|
recorded_failure_count=$(manifest_value failure_count "$manifest") || { fail 'malformed preflight failure count' 33; return $?; }
|
|
recorded_failed_checks_hash=$(manifest_value failed_checks_sha256 "$manifest") || { fail 'malformed failed-check hash' 33; return $?; }
|
|
recorded_hash=$(manifest_value inventory_hashes_sha256 "$manifest") || { fail 'malformed inventory hash' 33; return $?; }
|
|
[ "$format" = le-phase2a-preflight-v1 ] || { fail 'unsupported preflight format' 34; return $?; }
|
|
[ "$recorded_overall" = PASS ] || { fail 'preflight result is not PASS' 35; return $?; }
|
|
[ "$recorded_failure_count" = 0 ] || { fail 'preflight records mandatory failures' 35; return $?; }
|
|
if [ ! -f "$preflight/failed-mandatory-checks.txt" ] || [ -L "$preflight/failed-mandatory-checks.txt" ]; then
|
|
fail 'failed-check list is missing or a symlink' 35
|
|
return $?
|
|
fi
|
|
actual_failed_checks_hash=$(sha256sum "$preflight/failed-mandatory-checks.txt" 2>/dev/null | awk '{print $1}')
|
|
[ "$actual_failed_checks_hash" = "$recorded_failed_checks_hash" ] || { fail 'failed-check list checksum mismatch' 35; return $?; }
|
|
[ "$recorded_host" = "$(hostname)" ] || { fail 'preflight belongs to another host' 36; return $?; }
|
|
[ "$recorded_boot" = "$(cat /proc/sys/kernel/random/boot_id)" ] || { fail 'preflight belongs to another boot' 37; return $?; }
|
|
[ "$recorded_kernel" = "$(uname -r)" ] || { fail 'running kernel changed since preflight' 37; return $?; }
|
|
case "$recorded_time" in *[!0-9]*|'') fail 'preflight timestamp is invalid' 38; return $? ;; esac
|
|
now=$(date +%s)
|
|
age=$((now - recorded_time))
|
|
if [ "$age" -lt 0 ] || [ "$age" -gt 1800 ]; then
|
|
fail "preflight is stale; age=${age}s limit=1800s" 39
|
|
return $?
|
|
fi
|
|
actual_hash=$(sha256sum "$preflight/inventory-hashes.sha256" 2>/dev/null | awk '{print $1}')
|
|
[ "$actual_hash" = "$recorded_hash" ] || { fail 'preflight inventory-hash manifest checksum mismatch' 40; return $?; }
|
|
(cd "$preflight" && sha256sum -c inventory-hashes.sha256 > /dev/null 2>&1)
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
fail 'one or more preflight inventory files failed checksum validation' 41
|
|
return $?
|
|
fi
|
|
validated_preflight=$preflight
|
|
return 0
|
|
}
|
|
|
|
validate_health_url_file() {
|
|
file=$1
|
|
[ -s "$file" ] || { fail 'validated preflight contains no health URLs' 42; return $?; }
|
|
while IFS= read -r url; do
|
|
case "$url" in https://*) ;; *) fail 'unsupported health URL scheme' 42; return $? ;; esac
|
|
authority=${url#https://}; authority=${authority%%/*}
|
|
case "$url" in *'?'*|*'#'*|*$'\n'*|*$'\r'*|*$'\t'*|*' '*) fail 'unsafe health URL content' 42; return $? ;; esac
|
|
case "$authority" in ''|*@*) fail 'health URL user-info or empty authority rejected' 42; return $? ;; esac
|
|
done < "$file"
|
|
return 0
|
|
}
|
|
|
|
capture_container_fingerprint() {
|
|
output=$1
|
|
: > "$output"
|
|
docker ps -aq | while IFS= read -r cid; do
|
|
[ -n "$cid" ] || continue
|
|
base=$(docker inspect --format \
|
|
'{{.Name}}|{{.Id}}|{{.Config.Image}}|{{.Image}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}|{{.State.StartedAt}}|{{.HostConfig.NetworkMode}}|{{.HostConfig.RestartPolicy.Name}}|{{index .Config.Labels "com.docker.compose.project"}}' \
|
|
"$cid" 2>/dev/null)
|
|
inspect_rc=$?
|
|
health_hash=$(docker inspect --format '{{json .Config.Healthcheck}}' "$cid" 2>/dev/null | sha256sum | awk '{print $1}')
|
|
hash_rc=$?
|
|
if [ "$inspect_rc" -ne 0 ] || [ "$hash_rc" -ne 0 ] || [ -z "$base" ] || [ -z "$health_hash" ]; then return 1; fi
|
|
printf '%s|healthcheck_sha256=%s\n' "${base#/}" "$health_hash"
|
|
done | LC_ALL=C sort > "$output"
|
|
pipeline_status="${PIPESTATUS[*]}"
|
|
case "$pipeline_status" in '0 0 0') return 0 ;; *) return 1 ;; esac
|
|
}
|
|
|
|
verify_container_disposition() {
|
|
output=$1
|
|
(cd "$script_dir/container-disposition" && sha256sum -c SHA256SUMS) \
|
|
> "$record/container-disposition-checksum-latest.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'container disposition checksum failed' 43; return $?; }
|
|
capture_container_fingerprint "$output"
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'cannot capture current container fingerprint' 43; return $?; }
|
|
diff -u "$validated_preflight/container-fingerprint.psv" "$output" \
|
|
> "$record/container-disposition-latest.diff" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'container identity, state, health, startup, network, restart policy, or health-check identity changed' 43; return $?; }
|
|
return 0
|
|
}
|
|
|
|
capture_failed_unit_fingerprint() {
|
|
output=$1
|
|
: > "$output"
|
|
systemctl --failed --no-legend --plain --no-pager | awk '{print $1}' | LC_ALL=C sort |
|
|
while IFS= read -r unit; do
|
|
[ -n "$unit" ] || continue
|
|
printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
|
|
"$(systemctl show "$unit" -p Id --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p Type --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p LoadState --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p ActiveState --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p SubState --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p Result --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p InvocationID --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p StateChangeTimestamp --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p FragmentPath --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p ExecMainCode --value --no-pager)" \
|
|
"$(systemctl show "$unit" -p ExecMainStatus --value --no-pager)"
|
|
done > "$output"
|
|
return ${PIPESTATUS[0]}
|
|
}
|
|
|
|
verify_failed_unit_disposition() {
|
|
output=$1
|
|
(cd "$script_dir/failed-unit-disposition" && sha256sum -c SHA256SUMS) \
|
|
> "$record/failed-unit-allowlist-checksum-latest.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'failed-unit allowlist checksum failed' 45; return $?; }
|
|
capture_failed_unit_fingerprint "$output"
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'cannot fingerprint current failed units' "$rc"; return $?; }
|
|
diff -u "$script_dir/failed-unit-disposition/drkonqi-failed-units.allowlist.psv" "$output" \
|
|
> "$record/failed-unit-disposition-latest.diff" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'failed-unit set, state, invocation ID, or fingerprint changed' 45; return $?; }
|
|
count=$(wc -l < "$output")
|
|
[ "$count" = 234 ] || { fail "failed-unit count changed: expected=234 actual=$count" 45; return $?; }
|
|
return 0
|
|
}
|
|
|
|
verify_snapshot_subvolume() {
|
|
number=$1
|
|
description=$2
|
|
snapshot_dir=/.snapshots/$number
|
|
snapshot_subvolume=$snapshot_dir/snapshot
|
|
[ -d "$snapshot_dir" ] || { fail "Snapper numbered directory missing: $snapshot_dir" 50; return $?; }
|
|
btrfs subvolume show "$snapshot_subvolume" > "$record/pre-upgrade-snapshot-subvolume.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail "Snapper snapshot is not a verified Btrfs subvolume: $snapshot_subvolume" 50; return $?; }
|
|
snapper list --columns number,type,pre-number,date,user,cleanup,description \
|
|
> "$record/snapper-snapshots-after-explicit-create.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'cannot verify explicit snapshot through Snapper' "$rc"; return $?; }
|
|
awk -v number="$number" -v description="$description" \
|
|
'$1 == number && index($0, description) {print}' \
|
|
"$record/snapper-snapshots-after-explicit-create.txt" \
|
|
> "$record/pre-upgrade-snapshot-list-match.txt" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ] || [ ! -s "$record/pre-upgrade-snapshot-list-match.txt" ]; then
|
|
fail 'explicit snapshot number and description not found together in Snapper list' 50
|
|
return $?
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
create_explicit_pre_upgrade_snapshot() {
|
|
btrfs subvolume show /.snapshots > "$record/snapshots-subvolume-before-create.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail '/.snapshots is not a verified Btrfs subvolume' "$rc"; return $?; }
|
|
snapper list --columns number > "$record/snapper-numbers-before-explicit.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'cannot capture Snapper number baseline' "$rc"; return $?; }
|
|
grep -E '^[[:space:]]*[0-9]+[[:space:]]*$' "$record/snapper-numbers-before-explicit.txt" | tr -d '[:space:]' | LC_ALL=C sort \
|
|
> "$record/snapper-numbers-before-explicit.normalized"
|
|
description="le-phase2a-pre-upgrade-$(date -u +%Y%m%dT%H%M%SZ)-$(basename "$record")"
|
|
snapshot_output=$(snapper create --type single --cleanup-algorithm number \
|
|
--description "$description" --print-number 2> "$record/pre-upgrade-snapshot-create.err")
|
|
rc=$?
|
|
printf '%s\n' "$snapshot_output" > "$record/pre-upgrade-snapshot-create.out"
|
|
[ "$rc" -eq 0 ] || { fail "explicit pre-upgrade Snapper snapshot failed rc=$rc" 50; return $?; }
|
|
snapshot_number=$(printf '%s' "$snapshot_output" | tr -d '[:space:]')
|
|
case "$snapshot_number" in ''|*[!0-9]*) fail 'Snapper returned an invalid snapshot number' 50; return $? ;; esac
|
|
[ "$snapshot_number" -gt 0 ] || { fail 'Snapper returned nonpositive snapshot number' 50; return $?; }
|
|
verify_snapshot_subvolume "$snapshot_number" "$description" || return $?
|
|
{
|
|
printf 'snapshot_number=%s\n' "$snapshot_number"
|
|
printf 'description=%s\n' "$description"
|
|
printf 'snapshot_directory=/.snapshots/%s\n' "$snapshot_number"
|
|
printf 'snapshot_subvolume=/.snapshots/%s/snapshot\n' "$snapshot_number"
|
|
} > "$record/pre-upgrade-snapshot.manifest"
|
|
btrfs qgroup show / > "$record/btrfs-quota-observation-before-upgrade.txt" 2>&1
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
printf 'btrfs-quota-accounting|unavailable-or-disabled|rc=%s|explicit-snapshot-still-verified\n' "$rc" \
|
|
>> "$record/warnings-before-upgrade.txt"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
verify_snap_pac_snapshots() {
|
|
snapper list --columns number > "$record/snapper-numbers-after-transaction.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'cannot capture post-transaction Snapper numbers' "$rc"; return $?; }
|
|
grep -E '^[[:space:]]*[0-9]+[[:space:]]*$' "$record/snapper-numbers-after-transaction.txt" | tr -d '[:space:]' | LC_ALL=C sort \
|
|
> "$record/snapper-numbers-after-transaction.normalized"
|
|
comm -13 "$record/snapper-numbers-before-explicit.normalized" "$record/snapper-numbers-after-transaction.normalized" \
|
|
> "$record/snapper-new-numbers.txt"
|
|
new_count=$(wc -l < "$record/snapper-new-numbers.txt")
|
|
if [ "$new_count" -lt 3 ]; then
|
|
fail "expected explicit plus snap-pac pre/post snapshots; observed new snapshot count=$new_count" 51
|
|
return $?
|
|
fi
|
|
while IFS= read -r number; do
|
|
case "$number" in ''|*[!0-9]*) fail 'invalid new Snapper snapshot number' 51; return $? ;; esac
|
|
btrfs subvolume show "/.snapshots/$number/snapshot" \
|
|
> "$record/snapper-new-subvolume-$number.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail "new Snapper snapshot $number is not a Btrfs subvolume" 51; return $?; }
|
|
done < "$record/snapper-new-numbers.txt"
|
|
return 0
|
|
}
|
|
|
|
rerun_volatile_checks() {
|
|
verify_failed_unit_disposition "$record/volatile-failed-units-fingerprint.psv" || return $?
|
|
pacman -Dk > "$record/volatile-pacman-Dk.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'package database consistency failed immediately before execution' "$rc"; return $?; }
|
|
df -hT / /boot /var/cache/pacman/pkg /tmp > "$record/volatile-disk-space.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'cannot verify current disk space' "$rc"; return $?; }
|
|
verify_container_disposition "$record/volatile-container-fingerprint.psv" || return $?
|
|
: > "$record/volatile-public-health.txt"
|
|
while IFS= read -r url; do
|
|
metrics=$(curl --fail --silent --show-error --max-redirs 0 \
|
|
--connect-timeout 10 --max-time 30 --output /dev/null \
|
|
--write-out 'http=%{http_code}|remote=%{remote_ip}|time=%{time_total}' \
|
|
"$url" 2>> "$record/volatile-public-health-errors.txt")
|
|
rc=$?
|
|
printf '%s|%s\n' "$url" "$metrics" >> "$record/volatile-public-health.txt"
|
|
if [ "$rc" -ne 0 ]; then fail "public health failed immediately before execution: $url" 44; return $?; fi
|
|
done < "$validated_preflight/health-urls.sanitized"
|
|
return 0
|
|
}
|
|
|
|
normalize_transaction() {
|
|
source_file=$1
|
|
output_file=$2
|
|
downgrade_file=$3
|
|
: > "$output_file"
|
|
: > "$downgrade_file"
|
|
while IFS='|' read -r name new_version repository location download_bytes; do
|
|
[ -n "$name" ] || continue
|
|
old_line=$(pacman -Q "$name" 2>/dev/null)
|
|
if [ -n "$old_line" ]; then
|
|
old_version=${old_line#* }
|
|
comparison=$(vercmp "$new_version" "$old_version")
|
|
if [ "$comparison" -lt 0 ]; then action=downgrade
|
|
elif [ "$comparison" -eq 0 ]; then action=same
|
|
else action=upgrade
|
|
fi
|
|
else
|
|
old_version=-
|
|
action=new
|
|
fi
|
|
printf '%s|%s|%s|%s\n' "$name" "$old_version" "$new_version" "$action" >> "$output_file"
|
|
if [ "$action" = downgrade ]; then
|
|
printf '%s|%s|%s|%s\n' "$name" "$old_version" "$new_version" "$action" >> "$downgrade_file"
|
|
fi
|
|
done < "$source_file"
|
|
sort -o "$output_file" "$output_file"
|
|
return 0
|
|
}
|
|
|
|
main() {
|
|
if [ "$(id -u)" -ne 0 ]; then fail 'run as a separate root process' 10; return $?; fi
|
|
if [ "$1" != --preflight ] || [ -z "${2:-}" ] || [ -n "${3:-}" ]; then
|
|
fail 'usage: execute-maintenance.sh --preflight /var/log/le-phase2a-preflight/preflight.XXXXXXXX' 11
|
|
return $?
|
|
fi
|
|
validate_preflight "$2" || return $?
|
|
validate_health_url_file "$validated_preflight/health-urls.sanitized" || return $?
|
|
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
|
|
|
umask 077
|
|
install -d -o root -g root -m 0700 /var/log/le-phase2a-maintenance
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'cannot create root-only maintenance record parent' "$rc"; return $?; }
|
|
record=$(mktemp -d "/var/log/le-phase2a-maintenance/maintenance.$(date -u +%Y%m%dT%H%M%SZ).XXXXXXXX")
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ] || [ -z "$record" ] || [ ! -d "$record" ]; then
|
|
fail 'cannot atomically create unique maintenance record' 12
|
|
return $?
|
|
fi
|
|
chown root:root "$record"; chmod 0700 "$record"
|
|
maintenance_start=$(date -u +%FT%TZ)
|
|
printf '%s\n' "$record" > "$record/RECORD_PATH.txt"
|
|
cp -a "$validated_preflight" "$record/preflight-record"
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'cannot copy validated preflight record' "$rc"; return $?; }
|
|
|
|
approved_transaction=$script_dir/approved-transaction
|
|
(cd "$approved_transaction" && sha256sum -c SHA256SUMS) \
|
|
> "$record/approved-report-checksum.txt" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'approved transaction checksum failed' "$rc"; return $?; }
|
|
cp -a "$approved_transaction" "$record/approved-transaction"
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'cannot copy approved transaction evidence' "$rc"; return $?; }
|
|
|
|
uname -a > "$record/running-kernel-before.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture running kernel' "$rc"; return $?; }
|
|
cat /proc/sys/kernel/random/boot_id > "$record/boot-id-before.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture boot ID' "$rc"; return $?; }
|
|
pacman -Q > "$record/packages-before.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture installed package inventory' "$rc"; return $?; }
|
|
cp "$validated_preflight/bootloader-inventory.txt" "$validated_preflight/kernel-initramfs-before.txt" \
|
|
"$validated_preflight/running-containers-before.txt" "$validated_preflight/containers-all.psv" \
|
|
"$validated_preflight/container-fingerprint.psv" "$validated_preflight/sonarr-unhealthy.allowlist.psv" \
|
|
"$validated_preflight/directus-stopped.allowlist.psv" "$validated_preflight/restart-policy-no.psv" \
|
|
"$validated_preflight/critical-units-baseline.psv" "$validated_preflight/database-baseline.psv" "$record/"
|
|
rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot copy essential rollback baseline evidence' "$rc"; return $?; }
|
|
verify_failed_unit_disposition "$record/failed-units-before-fingerprint.psv" || return $?
|
|
btrfs subvolume show /.snapshots > "$record/snapshots-subvolume-before-preview.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || { fail '/.snapshots is not a Btrfs subvolume' "$rc"; return $?; }
|
|
|
|
rerun_volatile_checks || return $?
|
|
|
|
preview_root=$(mktemp -d /tmp/le-phase2a-execution-preview.XXXXXXXX)
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ] || [ -z "$preview_root" ] || [ ! -d "$preview_root" ]; then
|
|
fail 'cannot create isolated transaction preview directory' 13
|
|
return $?
|
|
fi
|
|
chmod 0700 "$preview_root"; mkdir -m 0700 "$preview_root/db" "$preview_root/cache"
|
|
ln -s /var/lib/pacman/local "$preview_root/db/local"
|
|
|
|
fakeroot -- pacman --config /etc/pacman.conf --dbpath "$preview_root/db" \
|
|
--cachedir "$preview_root/cache" --logfile "$preview_root/pacman-preview.log" -Sy \
|
|
> "$record/isolated-sync-refresh.log" 2>&1
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'isolated repository refresh failed; live databases untouched' "$rc"; return $?; }
|
|
fakeroot -- pacman --config /etc/pacman.conf --dbpath "$preview_root/db" \
|
|
--cachedir "$preview_root/cache" --logfile "$preview_root/pacman-preview.log" \
|
|
-Su --needed --print-format '%n|%v|%r|%l|%s' rootlesskit slirp4netns openai-codex \
|
|
> "$record/latest-transaction.psv" 2> "$record/latest-transaction.err"
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'latest complete transaction resolution failed' "$rc"; return $?; }
|
|
normalize_transaction "$record/latest-transaction.psv" "$record/latest-normalized-actions.psv" "$record/latest-downgrades.psv"
|
|
[ ! -s "$record/latest-downgrades.psv" ] || { fail 'latest transaction contains a downgrade' 14; return $?; }
|
|
|
|
cp "$approved_transaction/normalized-actions.psv" "$record/approved-normalized-actions.psv"
|
|
rc=$?
|
|
[ "$rc" -eq 0 ] || { fail 'cannot normalize approved preview' "$rc"; return $?; }
|
|
diff -u "$record/approved-normalized-actions.psv" "$record/latest-normalized-actions.psv" \
|
|
> "$record/approved-vs-latest.diff" 2>&1
|
|
diff_rc=$?
|
|
if [ "$diff_rc" -ne 0 ]; then
|
|
fail 'latest package names, versions, or actions differ from the approved preview' 15
|
|
return $?
|
|
fi
|
|
latest_count=$(wc -l < "$record/latest-normalized-actions.psv")
|
|
approved_count=$(wc -l < "$record/approved-normalized-actions.psv")
|
|
[ "$latest_count" = "$approved_count" ] || { fail 'transaction package counts differ' 15; return $?; }
|
|
for required in rootlesskit slirp4netns openai-codex; do
|
|
grep -q "^${required}|" "$record/latest-normalized-actions.psv"
|
|
rc=$?; [ "$rc" -eq 0 ] || { fail "latest transaction omits $required" 16; return $?; }
|
|
done
|
|
|
|
cp "$record/latest-transaction.psv" "$record/latest-normalized-actions.psv" \
|
|
"$record/approved-normalized-actions.psv" "$record/approved-vs-latest.diff" "$record/transaction-for-approval.psv"
|
|
rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot preserve transaction preview evidence' "$rc"; return $?; }
|
|
sha256sum "$record/latest-transaction.psv" "$record/latest-normalized-actions.psv" \
|
|
"$record/approved-normalized-actions.psv" > "$record/transaction-preview-hashes.sha256"
|
|
|
|
printf '\nMachine comparison passed: packages=%s, no downgrades, exact match to approved preview.\n' "$latest_count"
|
|
cat "$record/latest-normalized-actions.psv"
|
|
printf '\nType exactly: APPROVE MATCHED COMPLETE HOST UPGRADE\n> '
|
|
IFS= read -r approval_one
|
|
if [ "$approval_one" != 'APPROVE MATCHED COMPLETE HOST UPGRADE' ]; then
|
|
fail 'first exact operator approval not granted' 20
|
|
return $?
|
|
fi
|
|
|
|
rerun_volatile_checks || return $?
|
|
printf '\nLIMITATION: the final live -Syu refresh may change after this isolated comparison.\n'
|
|
printf 'Pacman will display its final interactive transaction and ask Y/n. The wrapper cannot machine-compare that post-refresh transaction without an unsafe live -Sy-only staging step.\n'
|
|
printf 'Type exactly: ACKNOWLEDGE FINAL PACMAN TRANSACTION REQUIRES MANUAL REVIEW\n> '
|
|
IFS= read -r approval_two
|
|
if [ "$approval_two" != 'ACKNOWLEDGE FINAL PACMAN TRANSACTION REQUIRES MANUAL REVIEW' ]; then
|
|
fail 'second exact operator acknowledgement not granted' 21
|
|
return $?
|
|
fi
|
|
|
|
create_explicit_pre_upgrade_snapshot || return $?
|
|
|
|
printf 'At Pacman Y/n, answer no if any package, version, action, count, downgrade status, signature, or repository differs.\n'
|
|
script -q -e -f -c 'pacman -Syu --needed rootlesskit slirp4netns openai-codex' "$record/pacman-complete.log"
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then fail "Pacman returned $rc; stop without ad hoc repair" "$rc"; return $?; fi
|
|
|
|
verify_snapshot_subvolume "$snapshot_number" "$description" || return $?
|
|
snapper list --columns number,type,pre-number,date,user,cleanup,description \
|
|
> "$record/snapper-snapshots-after-transaction.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot inventory Snapper snapshots after transaction' "$rc"; return $?; }
|
|
verify_snap_pac_snapshots || return $?
|
|
|
|
pacman -Q > "$record/packages-after.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot capture post-transaction package inventory' "$rc"; return $?; }
|
|
diff -u "$record/packages-before.txt" "$record/packages-after.txt" > "$record/package-inventory.diff" 2>&1
|
|
diff_rc=$?; [ "$diff_rc" -le 1 ] || { fail 'package inventory comparison failed' "$diff_rc"; return $?; }
|
|
find /etc -xdev -type f \( -name '*.pacnew' -o -name '*.pacsave' \) -printf '%p\n' | sort \
|
|
> "$record/pacnew-pacsave-after.txt" 2>&1
|
|
rc=$?; [ "$rc" -eq 0 ] || { fail 'cannot inventory pacnew/pacsave files' "$rc"; return $?; }
|
|
pacdiff -o > "$record/pacdiff-outstanding.txt" 2>&1
|
|
pacdiff_rc=$?
|
|
if [ "$pacdiff_rc" -ne 0 ] && [ "$pacdiff_rc" -ne 1 ]; then
|
|
printf 'pacdiff-observation|rc=%s\n' "$pacdiff_rc" > "$record/warnings-after.txt"
|
|
fi
|
|
grep -E -i 'hook|restart|reload|dkms|dracut|initramfs|mkinitcpio|grub|pacnew|pacsave|warning|error|failed' \
|
|
"$record/pacman-complete.log" > "$record/hook-restart-summary.txt" 2>&1
|
|
systemctl --failed --no-legend --plain --no-pager > "$record/failed-units-after.txt" 2>&1
|
|
journal_units='sshd.service accounts-daemon.service docker.service'
|
|
while IFS='|' read -r unit _; do journal_units="$journal_units $unit"; done < "$record/critical-units-baseline.psv"
|
|
journalctl --since "$maintenance_start" --no-pager $(for unit in $journal_units; do printf -- '-u %q ' "$unit"; done) \
|
|
> "$record/relevant-service-journal.txt" 2>&1
|
|
dkms status > "$record/dkms-after.txt" 2>&1
|
|
find /boot /usr/lib/modules -maxdepth 2 -type f \
|
|
\( -name 'vmlinuz*' -o -name 'initramfs*' -o -name 'initrd*' -o -name pkgbase \) \
|
|
-printf '%p size=%s mtime=%TY-%Tm-%TdT%TH:%TM:%TS\n' | sort > "$record/kernel-boot-artifacts-after.txt" 2>&1
|
|
docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort > "$record/containers-after.txt" 2>&1
|
|
find "$record" -maxdepth 1 -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum \
|
|
> "$record/SHA256SUMS" 2>&1
|
|
chmod -R go-rwx "$record"
|
|
|
|
printf 'PACKAGE TRANSACTION COMPLETED; REBOOT NOT PERFORMED.\n'
|
|
printf 'MAINTENANCE_RECORD=%s\n' "$record"
|
|
printf 'PRE_UPGRADE_SNAPPER_SNAPSHOT=%s\n' "$snapshot_number"
|
|
printf 'Review all warnings, hooks, services, kernel artifacts, and pacnew/pacsave files.\n'
|
|
printf 'Reboot and rootless-Docker provisioning remain separate approval gates.\n'
|
|
return 0
|
|
}
|
|
|
|
main "$@"
|