add Phase 2A container and recovery readiness gates
This commit is contained in:
parent
da03be61a9
commit
a9fec0d83d
15 changed files with 629 additions and 30 deletions
|
|
@ -65,7 +65,8 @@ capture_failed_unit_fingerprint() {
|
|||
"$id" "$type" "$load" "$active" "$sub" "$result" "$invocation" \
|
||||
"$changed" "$fragment" "$code" "$status"
|
||||
done > "$output"
|
||||
return ${PIPESTATUS[0]}
|
||||
pipeline_status="${PIPESTATUS[*]}"
|
||||
case "$pipeline_status" in '0 0 0') return 0 ;; *) return 1 ;; esac
|
||||
}
|
||||
|
||||
validate_failed_unit_disposition() {
|
||||
|
|
@ -78,8 +79,9 @@ validate_failed_unit_disposition() {
|
|||
capture_failed_unit_fingerprint "$record/failed-units-fingerprint.psv"
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then add_failure "failed-unit-fingerprint|rc=$rc"; return 0; fi
|
||||
cp "$allowlist" "$allowlist_hashes" "$record/"
|
||||
cp "$allowlist" "$record/drkonqi-failed-units.allowlist.psv"
|
||||
rc=$?
|
||||
if [ "$rc" -eq 0 ]; then cp "$allowlist_hashes" "$record/failed-unit-disposition.SHA256SUMS"; rc=$?; fi
|
||||
if [ "$rc" -ne 0 ]; then add_failure "failed-unit-allowlist-copy|rc=$rc"; return 0; fi
|
||||
diff -u "$allowlist" "$record/failed-units-fingerprint.psv" > "$record/failed-unit-disposition.diff" 2>&1
|
||||
rc=$?
|
||||
|
|
@ -113,6 +115,62 @@ validate_snapper_readiness() {
|
|||
return 0
|
||||
}
|
||||
|
||||
capture_container_fingerprint() {
|
||||
output=$1
|
||||
: > "$output"
|
||||
docker ps -aq | while IFS= read -r cid; do
|
||||
[ -n "$cid" ] || continue
|
||||
base=$(docker inspect --format \
|
||||
'{{.Name}}|{{.Id}}|{{.Config.Image}}|{{.Image}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}|{{.State.StartedAt}}|{{.HostConfig.NetworkMode}}|{{.HostConfig.RestartPolicy.Name}}|{{index .Config.Labels "com.docker.compose.project"}}' \
|
||||
"$cid" 2>/dev/null)
|
||||
inspect_rc=$?
|
||||
health_hash=$(docker inspect --format '{{json .Config.Healthcheck}}' "$cid" 2>/dev/null | sha256sum | awk '{print $1}')
|
||||
hash_rc=$?
|
||||
if [ "$inspect_rc" -ne 0 ] || [ "$hash_rc" -ne 0 ] || [ -z "$base" ] || [ -z "$health_hash" ]; then
|
||||
return 1
|
||||
fi
|
||||
printf '%s|healthcheck_sha256=%s\n' "${base#/}" "$health_hash"
|
||||
done | LC_ALL=C sort > "$output"
|
||||
pipeline_status="${PIPESTATUS[*]}"
|
||||
case "$pipeline_status" in '0 0 0') return 0 ;; *) return 1 ;; esac
|
||||
}
|
||||
|
||||
validate_container_disposition() {
|
||||
disposition_dir=$script_dir/container-disposition
|
||||
(cd "$disposition_dir" && sha256sum -c SHA256SUMS) \
|
||||
> "$record/container-disposition-checksum.txt" 2>&1
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then add_failure "container-disposition-checksum|rc=$rc"; return 0; fi
|
||||
capture_container_fingerprint "$record/container-fingerprint.psv"
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then add_failure "container-fingerprint|rc=$rc"; return 0; fi
|
||||
diff -u "$disposition_dir/all-containers.psv" "$record/container-fingerprint.psv" \
|
||||
> "$record/container-disposition.diff" 2>&1
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
add_failure 'container-disposition|identity-state-health-startup-network-restart-or-healthcheck-changed'
|
||||
fi
|
||||
total=$(wc -l < "$record/container-fingerprint.psv")
|
||||
running=$(awk -F'|' '$5=="running"{n++} END{print n+0}' "$record/container-fingerprint.psv")
|
||||
stopped=$(awk -F'|' '$5!="running"{n++} END{print n+0}' "$record/container-fingerprint.psv")
|
||||
unhealthy=$(awk -F'|' '$6=="unhealthy"{n++} END{print n+0}' "$record/container-fingerprint.psv")
|
||||
[ "$total" = 88 ] || add_failure "container-count|expected=88|actual=$total"
|
||||
[ "$running" = 87 ] || add_failure "running-container-count|expected=87|actual=$running"
|
||||
[ "$stopped" = 1 ] || add_failure "stopped-container-count|expected=1|actual=$stopped"
|
||||
[ "$unhealthy" = 1 ] || add_failure "unhealthy-container-count|expected=1|actual=$unhealthy"
|
||||
awk -F'|' '$6=="unhealthy" && $1!="sonarr" {print}' "$record/container-fingerprint.psv" \
|
||||
> "$record/unexpected-unhealthy-containers.psv"
|
||||
[ ! -s "$record/unexpected-unhealthy-containers.psv" ] || add_failure 'container-health|additional-unhealthy-container'
|
||||
awk -F'|' '$5!="running" && $1!="directus" {print}' "$record/container-fingerprint.psv" \
|
||||
> "$record/unexpected-stopped-containers.psv"
|
||||
[ ! -s "$record/unexpected-stopped-containers.psv" ] || add_failure 'container-state|additional-stopped-container'
|
||||
cp "$disposition_dir"/*.psv "$record/"
|
||||
rc=$?
|
||||
if [ "$rc" -eq 0 ]; then cp "$disposition_dir/SHA256SUMS" "$record/container-disposition.SHA256SUMS"; rc=$?; fi
|
||||
[ "$rc" -eq 0 ] || add_failure "container-disposition-copy|rc=$rc"
|
||||
return 0
|
||||
}
|
||||
|
||||
validate_health_urls() {
|
||||
: > "$record/health-urls.sanitized"
|
||||
if [ -z "${PUBLIC_HEALTH_URLS:-}" ]; then
|
||||
|
|
@ -328,6 +386,7 @@ main() {
|
|||
"docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}' | sort"
|
||||
mandatory_shell 'running-container-inventory' running-containers-before.txt \
|
||||
"docker ps --format '{{.Names}}' | sort"
|
||||
validate_container_disposition
|
||||
capture_unit_baseline
|
||||
capture_database_baseline
|
||||
validate_database_baseline
|
||||
|
|
@ -350,8 +409,10 @@ main() {
|
|||
'git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration status --short --branch; git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration rev-parse HEAD; git -C /srv/codex-work/lasereverything.net.db/.worktrees/payload-migration rev-parse origin/migration/le-app-database'
|
||||
mandatory_shell 'approved-transaction-integrity' approved-transaction-checksum.txt \
|
||||
"cd '$script_dir/approved-transaction' && sha256sum -c SHA256SUMS"
|
||||
mandatory_shell 'protected-directus-archive-integrity' directus-archive-verification.txt \
|
||||
"cd /srv/directus-archive/20260710-175408 && sha256sum --status -c SHA256SUMS && printf 'archive=/srv/directus-archive/20260710-175408|manifest=SHA256SUMS|result=PASS|return_code=0\\n'"
|
||||
|
||||
inventory_files='packages-before.txt failed-units-before.txt failed-units-fingerprint.psv failed-unit-disposition.diff failed-unit-allowlist-checksum.txt snapshots-subvolume.txt snapshots-stat.txt snapper-snapshots-before.txt btrfs-quota-observation.txt kernel-initramfs-before.txt bootloader-inventory.txt containers-all.psv running-containers-before.txt critical-units-baseline.psv database-baseline.psv database-readiness-before.txt health-urls.sanitized public-health.txt network-firewall-before.txt pacman-Dk-before.txt pacman-Qkk-before.txt pacman-Qkk-missing-or-unreadable-before.txt pacnew-pacsave-before.txt migration-git-state.txt approved-transaction-checksum.txt'
|
||||
inventory_files='packages-before.txt failed-units-before.txt failed-units-fingerprint.psv failed-unit-disposition.diff failed-unit-allowlist-checksum.txt drkonqi-failed-units.allowlist.psv failed-unit-disposition.SHA256SUMS snapshots-subvolume.txt snapshots-stat.txt snapper-snapshots-before.txt btrfs-quota-observation.txt kernel-initramfs-before.txt bootloader-inventory.txt containers-all.psv running-containers-before.txt container-fingerprint.psv container-disposition.diff container-disposition-checksum.txt container-disposition.SHA256SUMS unexpected-unhealthy-containers.psv unexpected-stopped-containers.psv all-containers.psv running-containers.psv sonarr-unhealthy.allowlist.psv directus-stopped.allowlist.psv restart-policy-no.psv critical-units-baseline.psv database-baseline.psv database-readiness-before.txt health-urls.sanitized public-health.txt network-firewall-before.txt pacman-Dk-before.txt pacman-Qkk-before.txt pacman-Qkk-missing-or-unreadable-before.txt pacnew-pacsave-before.txt migration-git-state.txt approved-transaction-checksum.txt directus-archive-verification.txt'
|
||||
(
|
||||
cd "$record" || return 1
|
||||
sha256sum $inventory_files > inventory-hashes.sha256
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue