makearmy/le-app
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add Phase 2A container and recovery readiness gates

Browse source
This commit is contained in:
makearmy 2026-07-12 18:59:45 -04:00
parent da03be61a9
commit a9fec0d83d
15 changed files with 629 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

74
docs/le-app-database-migration/phase2a-host-maintenance/post-reboot-validate.sh
View file

@ -71,18 +71,72 @@ validate_health_url_file() {
return 0
}
capture_container_fingerprint() {
output=$1
: > "$output"
docker ps -aq | while IFS= read -r cid; do
[ -n "$cid" ] || continue
base=$(docker inspect --format \
'{{.Name}}|{{.Id}}|{{.Config.Image}}|{{.Image}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}|{{.State.StartedAt}}|{{.HostConfig.NetworkMode}}|{{.HostConfig.RestartPolicy.Name}}|{{index .Config.Labels "com.docker.compose.project"}}' \
"$cid" 2>/dev/null)
inspect_rc=$?
health_hash=$(docker inspect --format '{{json .Config.Healthcheck}}' "$cid" 2>/dev/null | sha256sum | awk '{print $1}')
hash_rc=$?
if [ "$inspect_rc" -ne 0 ] || [ "$hash_rc" -ne 0 ] || [ -z "$base" ] || [ -z "$health_hash" ]; then return 1; fi
printf '%s|healthcheck_sha256=%s\n' "${base#/}" "$health_hash"
done | LC_ALL=C sort > "$output"
pipeline_status="${PIPESTATUS[*]}"
case "$pipeline_status" in '0 0 0') return 0 ;; *) return 1 ;; esac
}
compare_containers() {
capture_container_fingerprint "$post_record/container-fingerprint-after.psv"
rc=$?
if [ "$rc" -ne 0 ]; then add_failure "container-fingerprint-after|rc=$rc"; return 0; fi
: > "$post_record/container-comparison.psv"
while IFS= read -r container; do
[ -n "$container" ] || continue
running=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' "$container" 2>/dev/null)
printf '%s|running=%s|health=%s\n' "$container" "${running:-missing}" "${health:-unknown}" \
>> "$post_record/container-comparison.psv"
if [ "$running" != true ] || [ "$health" = unhealthy ]; then
add_failure "container-regression|name=$container|running=${running:-missing}|health=${health:-unknown}"
: > "$post_record/restart-policy-no-attention.txt"
awk -F'|' '{print $1}' "$baseline/container-fingerprint.psv" > "$post_record/container-names-before.txt"
awk -F'|' '{print $1}' "$post_record/container-fingerprint-after.psv" > "$post_record/container-names-after.txt"
diff -u "$post_record/container-names-before.txt" "$post_record/container-names-after.txt" \
> "$post_record/container-name-set.diff" 2>&1
rc=$?
if [ "$rc" -ne 0 ]; then add_failure 'container-name-set-changed'; fi
while IFS='|' read -r name old_id old_image_ref old_image_id old_state old_health old_started old_network old_restart old_project old_health_hash; do
[ -n "$name" ] || continue
current=$(awk -F'|' -v name="$name" '$1==name {print}' "$post_record/container-fingerprint-after.psv")
if [ -z "$current" ]; then
add_failure "container-missing|name=$name"
[ "$old_restart" = no ] && printf '%s|missing|manual-operator-attention-required\n' "$name" >> "$post_record/restart-policy-no-attention.txt"
continue
fi
done < "$baseline/running-containers-before.txt"
IFS='|' read -r new_name new_id new_image_ref new_image_id new_state new_health new_started new_network new_restart new_project new_health_hash <<EOF
$current
EOF
printf '%s|before=%s,%s|after=%s,%s|restart=%s\n' "$name" "$old_state" "$old_health" "$new_state" "$new_health" "$new_restart" \
>> "$post_record/container-comparison.psv"
if [ "$old_id" != "$new_id" ] || [ "$old_image_ref" != "$new_image_ref" ] || \
[ "$old_image_id" != "$new_image_id" ] || [ "$old_network" != "$new_network" ] || \
[ "$old_restart" != "$new_restart" ] || [ "$old_project" != "$new_project" ] || \
[ "$old_health_hash" != "$new_health_hash" ]; then
add_failure "container-identity-or-policy-changed|name=$name"
continue
fi
if [ "$name" = directus ]; then
[ "$new_state" = exited ] || add_failure "retired-directus-state|expected=exited|actual=$new_state"
elif [ "$name" = sonarr ]; then
if [ "$new_state" != running ]; then add_failure "sonarr-not-running|state=$new_state"; fi
case "$new_health" in unhealthy|healthy) ;; *) add_failure "sonarr-health-worsened|health=$new_health" ;; esac
elif [ "$old_state" = running ]; then
if [ "$new_state" != running ]; then
add_failure "previously-running-container-not-running|name=$name|state=$new_state"
[ "$old_restart" = no ] && printf '%s|%s|manual-operator-attention-required\n' "$name" "$new_state" >> "$post_record/restart-policy-no-attention.txt"
elif [ "$old_health" = healthy ] && [ "$new_health" != healthy ]; then
add_failure "healthy-container-regressed|name=$name|health=$new_health"
elif [ "$old_health" = no-healthcheck ] && [ "$new_health" != no-healthcheck ]; then
add_failure "container-healthcheck-identity-changed|name=$name|health=$new_health"
fi
fi
done < "$baseline/container-fingerprint.psv"
return 0
}
@ -170,7 +224,7 @@ main() {
return 12
fi
baseline=$maintenance/preflight-record
required='running-containers-before.txt critical-units-baseline.psv database-baseline.psv health-urls.sanitized inventory-hashes.sha256 result.manifest result.manifest.sha256'
required='running-containers-before.txt container-fingerprint.psv directus-stopped.allowlist.psv sonarr-unhealthy.allowlist.psv restart-policy-no.psv critical-units-baseline.psv database-baseline.psv health-urls.sanitized inventory-hashes.sha256 result.manifest result.manifest.sha256'
for file in $required; do
if [ ! -f "$baseline/$file" ] || [ -L "$baseline/$file" ]; then
printf 'HARD_STOP: missing baseline file %s\n' "$file" >&2