From 885dff127e327b6a3b8adbda9df7ce876cbb18e7 Mon Sep 17 00:00:00 2001 From: makearmy Date: Mon, 13 Jul 2026 23:34:36 -0400 Subject: [PATCH] add Phase 2B Stage 3 namespace activation --- .../MANIFEST.sha256 | 11 + .../README.md | 56 + .../ROLLBACK.md | 17 + .../tests/50-activate-stage3.sh | 197 +++ .../tests/51-rollback-stage3.sh | 431 +++++++ .../tests/52-run-stage1-inert.sh | 219 ++++ .../tests/53-stage3-static-tests.sh | 123 ++ .../tests/54-stage3-failure-path-tests.sh | 632 ++++++++++ .../tests/docker-config/config.json | 1 + .../tests/inert-bin/rm | 6 + .../tests/inert-bin/touch | 9 + .../tests/stage3-common.sh | 1090 +++++++++++++++++ 12 files changed, 2792 insertions(+) create mode 100644 docs/le-app-database-migration/phase2b-stage3-namespace-activation/MANIFEST.sha256 create mode 100644 docs/le-app-database-migration/phase2b-stage3-namespace-activation/README.md create mode 100644 docs/le-app-database-migration/phase2b-stage3-namespace-activation/ROLLBACK.md create mode 100644 docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/50-activate-stage3.sh create mode 100644 docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/51-rollback-stage3.sh create mode 100644 docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/52-run-stage1-inert.sh create mode 100644 docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/53-stage3-static-tests.sh create mode 100644 docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/54-stage3-failure-path-tests.sh create mode 100644 docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/docker-config/config.json create mode 100755 docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/inert-bin/rm create mode 100755 docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/inert-bin/touch create mode 100644 docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/stage3-common.sh diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/MANIFEST.sha256 b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/MANIFEST.sha256 new file mode 100644 index 0000000..26b8d77 --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/MANIFEST.sha256 @@ -0,0 +1,11 @@ +1e281c255a9a7ba72734c40a947cab4168f70b7e6b733eba7415109a40776a9d README.md +908ab54049c1e810ba5c47812fc51848e9eef8818a9ddc373f841672ab509a57 ROLLBACK.md +136707ac1391bd241efc834c549c0f54d4a849ce832c55457bdeb243cc03800a tests/50-activate-stage3.sh +5c8eceb9b80865ae09bd4ece2a248da4677237abcc7c55d26334020543e8edf8 tests/51-rollback-stage3.sh +9aa43f4b72057ea28ec00309889f75e756a9b36db77e5db758b752990768c0a0 tests/52-run-stage1-inert.sh +de70e85a4e4e6015726dfe3cbad2bdccb2990daae25eeeab272e107d2d68a115 tests/53-stage3-static-tests.sh +ec595556f2842329a8b1b6e88bc291ef8f4d0d98e675baeb4b317da6ae1c8e96 tests/54-stage3-failure-path-tests.sh +ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356 tests/docker-config/config.json +1c64518e42afdf490047358c523a8213e989a84a61f51ff0126439db2b485fe3 tests/inert-bin/rm +034b23c09a023ce94d8b7888f0d6f45aed31ba14dbed2efd30c4b0a6322f657b tests/inert-bin/touch +3a3a84df1021e02cc258edbee621ab8abfb9f83bdc17dfa689fb9823b17c8cfd tests/stage3-common.sh diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/README.md b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/README.md new file mode 100644 index 0000000..5c2dc9e --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/README.md @@ -0,0 +1,56 @@ +# Phase 2B Stage 3 — namespace activation and inert acceptance + +Status: prepared for review only, 2026-07-13. Do not execute the activation or rollback scripts from this uncommitted tree. Stage 3 does not authorize the systemd user manager, lingering, rootless Docker, Codex, source/data/secret copying, application startup, service enablement, or production changes. + +## Entry points + +`tests/50-activate-stage3.sh` is the sole activation entry point. `tests/51-rollback-stage3.sh` is the separate Stage 3-only rollback/recovery entry point. Both require root, a clean committed worktree, a validated inherited or newly acquired exclusive lock on the existing root-owned runtime directory, exact Stage 1/2/3 manifests, the pinned Stage 1 state and all 16 installed files plus all 20 directory records, and unchanged Stage 1/2 trees since Stage 2 commit `909f0a8200c79efc205d92618be47e06ebf764fc`. + +Successful Stage 2 execution is an external operator prerequisite confirmed for TITANSERVER. Git provenance can prove which Stage 2 code is trusted; it cannot by itself prove that the prior operator run occurred. + +## Activation behavior + +Before any mutation, activation requires: + +- Stage 1 source `df6b53e63916880bec2c77219b04b010b8b453f1`, artifact-manifest digest `5aaa91b1e6846eda033961dcee392b9657476ad6fd149420979e9309b484445f`, exact `COMPLETE` state, and 16 matching installed files; +- Stage 2 commit and package-manifest provenance, plus a clean committed Stage 3 package; +- installed policy digest `4289b705dbd786281daccb6d5aa660c27b83441f1a34d0cd18590666124be386` and a root `nft --check` that does not load it; +- current public/NAT IPv4 `74.67.173.56`, the committed 41-network rootful Docker configuration, the expected LAN/WireGuard identities, and an already-enabled host IPv4 forward setting that Stage 3 never changes; every Docker read scrubs context/TLS selector variables and uses an explicit `/run/docker.sock` host plus the committed empty client configuration, so it cannot inherit another context or read operator credentials; +- fresh immediate fingerprints of nonproject nftables, stable nonproject host network state, rootful Docker networks/runtime/ports, host listeners, resolver metadata, and public IPv4; and +- no marker, namespace, project veth/table, user manager, linger, UID-1200 process, or rootless socket. + +Immediately before publication and start, the script revalidates the loaded manager state: `systemd-analyze verify`; exact `systemctl cat` fragment/drop-in surfaces for both units; exact namespace fragment path with no drop-ins; current manager cache; no pending job; exact single `ExecStart`/`ExecStop`; oneshot/`RemainAfterExit`; static inactive state; the approval-marker condition; `Before=user@1200.service`; and the reciprocal user-manager `Requires`/`After`, namespace path, fragment, and exact two-drop-in set. + +The script renders and fsyncs a root-owned mode `0600`, single-link version-2 record before atomically renaming it into the approval-marker path and syncing the parent directory. No hard-link publication window exists, and an interrupted write cannot replace the prior authoritative record. Parsing requires the exact canonical one-delimiter, final-newline-complete byte representation, so trailing fields, truncated final writes, and embedded NUL bytes are rejected. The immutable fields include an activation UUID, machine-ID digest, approved Quad9 pair, Stage 1 source and artifact digest, Stage 2 verification commit, Stage 3 activation commit, UTC timestamp, hostname, boot ID, public IPv4, installed-policy digest, and baseline fingerprints. It starts `le-app-codex-netns.service`, then atomically replaces `PREPARED` with an `ACTIVE` record bound to the service invocation, namespace inode, host-veth ifindex/iflink/MAC, both nft table handles, independent counter-normalized table digests, and the combined project-policy digest. + +Post-start validation requires a non-symlink `nsfs` namespace containing exactly loopback and the namespace veth, a typed host inventory with no host-resident peer, reciprocal host/peer ifindex/iflink identity, exact `/30` addresses/routes, no namespace IPv6 address/route, IPv6-disable sysctls, an empty namespace, no listener/published port, exactly the intended two nftables tables, three sets, two filter chains with 4 and 10 rules, and one postrouting masquerade rule. Rule expressions and order are checked against strict nftables JSON envelopes rather than presentation-dependent text; IPv4 set elements receive an additional text cross-check. Each provenance capture derives both independent table hashes and the combined policy hash from the same two raw canonical tables, brackets them with typed handle inventories, then requires a second complete capture to match. This stable exact snapshot is repeated immediately before publishing `ACTIVE`, after network probes, and as the final activation policy gate. It rejects extra nftables object types, DNAT, redirect, and prerouting publication. Counter-delta tests—not connection failure alone—prove the private/LAN/WireGuard/all-39-Docker-gateway/metadata/reserved/public-IP/SMTP/nonapproved-DNS/other-port denials. Both Quad9 servers must answer classic DNS over UDP and TCP. A Quad9-resolved public address must work over TCP 80 and 443 using `curl --resolve` with every uppercase and lowercase proxy variable removed. + +The installed Stage 1 `10-inert-containment.sh` is executed unchanged through `tests/52-run-stage1-inert.sh`. That reviewed wrapper reproduces the installed drop-in's service containment settings, including the resolver bind and inaccessible paths, and points the transient directly at the already-active namespace instead of duplicating the user-manager unit's `[Unit]` dependency declarations. It removes every uppercase/lowercase proxy variable from the transient process environment. The old test's missing `/srv/le-app-codex/tmp` write probe is narrowly redirected by two read-only bound wrappers to the existing approved `build-artifacts` directory and is identity-checked and removed on every success, error, and handled-signal path. + +The existing test can exercise the systemd mount, device, UID, cgroup, and namespace properties only as a transient unit. Stage 3 therefore treats one activation-UUID-named `systemd-run --service-type=exec --wait --collect` service—and only its short-lived UID-1200 test process tree—as the narrow acceptance-test exception to “start only the namespace service” and the otherwise-inert UID. It is never installed, enabled, or persistent; its name must be unclaimed, and the committed command statically fixes every containment property. At runtime an additional random token binds the exact transient fragment, user/group, namespace path, activation-specific description, environment, single ExecStart, and captured invocation to this launch. HUP/INT/TERM are deferred only across client PID/start-time/ownership capture and replayed immediately afterward; cleanup will signal that client only while its `/proc` start time still matches. Success requires captured ownership, while error/signal cleanup stops only that token-bound identity and requires three consecutive successful `not-found` observations before accepting collection. It revalidates the activation process's inherited exclusive lock, has a 120-second runtime ceiling, and must leave no UID-1200 or `codex`-named process before activation can succeed or rollback can begin. No user manager is started. + +Every unrelated baseline is compared after start and after testing. Every same-boot automatic or explicit runtime cleanup compares against the recorded pre-activation baselines before mutation and again after cleanup, in addition to a fresh immediate rollback before/after comparison. Changed-boot recovery also compares the recorded baselines before and after and normally fails closed if reboot changed a runtime identity. Any network, policy, containment, provenance, or baseline failure exits nonzero without starting the user manager. + +## DNS boundary + +The nftables policy restricts classic DNS on UDP/TCP port 53 to `9.9.9.9` and `149.112.112.112`, and the contained resolver file contains only those two servers. Because public TCP 443 is intentionally allowed, this L3/L4 policy cannot distinguish ordinary HTTPS from DNS-over-HTTPS to another public provider. Stage 3 therefore proves the enforceable classic-DNS boundary; it does not claim application-layer DoH prevention. + +## Explicit exclusions + +Activation never enables a unit, starts `user@1200.service`, enables lingering, starts rootless Docker or Codex, authenticates Codex, invokes Docker mutation APIs, loads an arbitrary nftables file, creates a published port, accesses container environments or secrets, clones/copies application material, changes host sysctls, or repairs/restarts production. Rootful Docker access is read-only and explicitly uses `unix:///run/docker.sock`; safe fingerprints exclude container environments. + +During review, run only `tests/53-stage3-static-tests.sh` and `tests/54-stage3-failure-path-tests.sh`. The latter uses isolated `/tmp` fixtures and source guards; it never invokes the activation, rollback, or inert operational entry points, nor any systemd, networking, nftables, or Docker runtime command. + +## Future operator invocations + +Only after this package has been reviewed, committed, and checked out as a clean worktree on TITANSERVER, run from the repository root: + +```sh +sudo /bin/bash docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/50-activate-stage3.sh +``` + +If activation fails or Stage 3 must be removed, use the separately reviewed retryable rollback entry point: + +```sh +sudo /bin/bash docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/51-rollback-stage3.sh +``` diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/ROLLBACK.md b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/ROLLBACK.md new file mode 100644 index 0000000..2c13d7a --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/ROLLBACK.md @@ -0,0 +1,17 @@ +# Phase 2B Stage 3-only rollback and recovery + +This rollback leaves every Stage 1 file/state object and the Stage 2 systemd reload intact. It never invokes the Stage 1 rollback, reloads systemd, stops a production/rootful-Docker service, or invokes the installed namespace helper's broad `down` action directly. + +Rollback accepts exactly one authoritative record: the approval marker or the retained rollback tombstone. It rejects dual records, symlinks, malformed/duplicate/unknown fields, noncanonical delimiters or incomplete final lines, embedded NUL bytes, path-like or escaping values, wrong ownership/mode/link count, host or machine mismatch, conflicting temporary records, and repository provenance drift. An empty fixed temporary is recognized only as the pre-render interruption state; a complete temporary must share every immutable provenance/baseline field and activation ID, and a same-status record must be byte-identical. If a complete temporary is the only surviving record, it is fsynced and revalidated before atomic publication as the retained authority. A complete `ACTIVE` temporary may upgrade a `PREPARED` tombstone only after the temporary is fsynced and its inode/hash plus the retained tombstone are revalidated immediately before the atomic rename; the stricter recorded live identities are then rechecked before cleanup. It requires the same boot for runtime cleanup, pinned manifests/state/files/directories, the exact loaded unit definitions, no user manager/linger/UID-1200 or `codex`-named process/rootless socket, no inert transient, no namespace PID, and absent-or-exact project resources. + +Before cleanup, rollback proves the current public IP, nonproject nftables, host network, rootful Docker network/runtime/ports/listeners, and resolver state still match the baselines recorded before activation, then captures a fresh immediate rollback baseline. Typed inventories reject command errors and syntactically valid wrong-shape JSON rather than treating them as absence. Both fixed project veth names are excluded from the nonproject baseline, which permits a retry after interruption while the peer is still on the host. Docker reads scrub context/TLS selectors and use only the explicit rootful socket and committed empty client configuration. Only after all gates pass does one same-directory atomic rename move the validated marker to `.OPERATOR-INPUTS-APPROVED.stage3-rollback`. That rename simultaneously revokes approval and preserves the exact authoritative bytes; there is no copy/delete gap, an atomic temporary upgrade never leaves the path without an authority, and the tombstone is never removed by Stage 3. + +An inactive service or successful stop is not cleanup proof. Rollback stops only `le-app-codex-netns.service`; its verified `ExecStop` normally removes the project objects. It then requires successful global inventories. Each surviving table is rechecked immediately by name, handle, and its independent canonical digest; the host veth is rechecked by type, ifindex, iflink, MAC, address, and live peer relationship; the namespace is rechecked by list membership, non-symlink nsfs identity, recorded dev/inode, and a successful empty-PID query. Only the exact survivor is directly deleted, in table → veth → namespace order. Drift or an inventory error preserves the tombstone and every unproven survivor. + +An `ACTIVE` retry accepts a recorded resource that is already absent because an earlier cleanup step succeeded, but a present survivor must retain its recorded identity. A `PREPARED` crash recovery requires a current attributable service invocation while authority is still the live approval marker, plus an exact subset of the helper's ordered construction or cleanup states, including namespace-only, both peers still on the host, and the peer moved into the namespace. Once that authority has been atomically retained as the tombstone and the service has been stopped, a retry may accept inactive/dead state with an empty, garbage-collected invocation ID; it still requires the same retained provenance, exact survivor shapes/identities, and unchanged baselines. A presence-shape gate rejects impossible ordering before the per-resource checks; changed or unproven fixed-name objects are preserved. JSON rule semantics/order, table handles/digests, reciprocal veth peer indices, addresses/routes, and namespace membership are rechecked as applicable. A retry after any partial rollback reads the same retained tombstone and resumes only after all gates pass again. + +After cleanup, rollback proves the service is inactive/dead/static, the approval marker and every project resource are absent, the tombstone remains valid, Stage 1 remains exact and installed, the Stage 2-loaded fragment/drop-ins remain exact, and both the immediate and recorded nonproject/production baselines remain unchanged. It never repairs, restarts, or modifies production. The retained tombstone deliberately blocks accidental repeat activation; removing or archiving it belongs to a separately reviewed later stage. + +On a changed boot ID, ephemeral namespace state should already be gone because the service is static. Rollback still compares every recorded pre-activation baseline before and after its action; a normal reboot will often change runtime identities and therefore fail closed pending a separately reviewed recovery. Only if those recorded comparisons pass does rollback also require a successful typed-inventory proof that the service and every fixed runtime name are absent, no user/Codex runtime exists, and a fresh immediate production baseline can be captured. It then retains/promotes the authoritative tombstone without deleting any runtime object and proves both the immediate and recorded production baselines unchanged afterward. If any identity, resource, or baseline differs, deletion and tombstone mutation are refused. + +Activation's EXIT/signal handler invokes this same rollback under the validated inherited lock. The inert runner owns and cleans only its verified UUID-named transient invocation, defers handled signals through client identity capture, refuses to signal a reused client PID, and requires stable collection; rollback refuses to proceed while any such transient remains. `SIGKILL` or power loss can bypass shell handlers, but the transient has a finite runtime and the standalone rollback can resume from `PREPARED`, `ACTIVE`, or the retained tombstone. Do not use `phase2b-contained-runtime/tests/31-rollback-stage1.sh` after Stage 3 activation: its documented safety contract forbids use after a marker, namespace, or firewall component has ever been activated. diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/50-activate-stage3.sh b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/50-activate-stage3.sh new file mode 100644 index 0000000..b529441 --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/50-activate-stage3.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash + +set -o pipefail +export LC_ALL=C +export PATH=/usr/bin:/bin +export SYSTEMD_COLORS=0 +export SYSTEMD_LOG_COLOR=0 +export SYSTEMD_PAGER=cat + +failures=0 +cleanup_armed=0 +cleanup_running=0 +pass() { printf 'PASS: %s\n' "$1"; } +fail() { printf 'FAIL: %s\n' "$1" >&2; failures=$((failures + 1)); } +source "$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" 2>/dev/null && pwd)/stage3-common.sh" +proxy_free_env=(/usr/bin/env -u ALL_PROXY -u HTTPS_PROXY -u HTTP_PROXY -u NO_PROXY -u all_proxy -u https_proxy -u http_proxy -u no_proxy) + +activation_cleanup() { + local original_rc=$? cleanup_rc=0 + trap - EXIT HUP INT TERM + if [ "$cleanup_armed" -eq 1 ] && [ "$cleanup_running" -eq 0 ]; then + cleanup_running=1 + printf 'Stage 3 activation failed; entering reviewed project-only rollback.\n' >&2 + PHASE2B_STAGE3_LOCK_HELD=1 PHASE2B_STAGE3_LOCK_ID="$PHASE2B_STAGE3_LOCK_ID" /bin/bash "$stage3_root/tests/51-rollback-stage3.sh" --activation-failure || cleanup_rc=1 + fi + if [ "$cleanup_rc" -ne 0 ]; then printf 'FAIL: automatic Stage 3 rollback requires operator review\n' >&2; fi + [ "$original_rc" -ne 0 ] || original_rc=1 + exit "$original_rc" +} + +stage3_abort_if_failed() { + if [ "$failures" -ne 0 ]; then printf 'Phase 2B Stage 3 activation failures=%s\n' "$failures" >&2; exit 1; fi +} + +stage3_verify_project_topology() { + local active sub result enabled service_show service_rc enabled_rc listeners listeners_rc + if service_show=$(/usr/bin/systemctl show --no-pager le-app-codex-netns.service --property=ActiveState --property=SubState --property=Result 2>/dev/null); then service_rc=0; else service_rc=$?; fi + active=$(stage3_show_value "$service_show" ActiveState 2>/dev/null || true) + sub=$(stage3_show_value "$service_show" SubState 2>/dev/null || true) + result=$(stage3_show_value "$service_show" Result 2>/dev/null || true) + if enabled=$(/usr/bin/systemctl is-enabled le-app-codex-netns.service 2>/dev/null); then enabled_rc=0; else enabled_rc=$?; fi + if [ "$service_rc" -eq 0 ] && { [ "$enabled_rc" -eq 0 ] || [ "$enabled_rc" -eq 1 ]; } && [ "$active" = active ] && [ "$sub" = exited ] && [ "$result" = success ] && [ "$enabled" = static ]; then pass 'namespace service active/exited/success/static'; else fail "namespace service query/state rc=$service_rc/$enabled_rc $active/$sub/$result/$enabled"; fi + if stage3_capture_active_topology_identity; then pass 'namespace/nsfs, exact links, reciprocal veth, addresses, routes, empty PID set, and IPv6 state'; else fail 'active namespace/veth topology query or identity'; fi + stage3_inventory_has_table inet le_app_codex && stage3_inventory_has_table ip le_app_codex_nat && pass 'both project nftables tables present in typed inventory' || fail 'project nftables table absence or inventory failure' + listeners=$(/usr/bin/ip netns exec le-app-codex /usr/bin/ss -H -lntup 2>/dev/null); listeners_rc=$? + [ "$listeners_rc" -eq 0 ] && [ -z "$listeners" ] && pass 'namespace has no listeners or published ports' || fail 'namespace listener exists or query failed' +} + +stage3_verify_project_nft() { + if stage3_capture_exact_project_policy_snapshot; then + pass 'project nftables policy has exact JSON structure/rules/sets and two stable canonical snapshots' + else + fail 'project nftables policy query, exactness, or stable-snapshot validation' + return 1 + fi +} + +stage3_rule_counter() { + local chain=$1 index=$2 raw + raw=$(/usr/bin/nft -j list chain inet le_app_codex "$chain" 2>/dev/null) || return 1 + printf '%s' "$raw" | /usr/bin/jq -er --argjson index "$index" '[.nftables[] | .rule? | select(.)] | .[$index].expr | ([.[] | .counter?.packets // empty] | add // 0)' +} + +stage3_expect_blocked_counter() { + local label=$1 chain=$2 index=$3 before after rc + shift 3 + before=$(stage3_rule_counter "$chain" "$index") || { fail "$label pre-counter unavailable"; return 1; } + "$@" >/dev/null 2>&1 + rc=$? + after=$(stage3_rule_counter "$chain" "$index") || { fail "$label post-counter unavailable"; return 1; } + if [ "$rc" -ne 0 ] && [[ "$before" =~ ^[0-9]+$ ]] && [[ "$after" =~ ^[0-9]+$ ]] && [ "$after" -gt "$before" ]; then pass "$label denied with counter delta $before->$after"; else fail "$label denial/counter rc=$rc $before->$after"; return 1; fi +} + +stage3_positive_dns() { + local server=$1 transport=$2 output args=() + [ "$transport" = tcp ] && args+=(+tcp) + output=$(/usr/bin/timeout 10 /usr/bin/ip netns exec le-app-codex /usr/bin/dig +time=3 +tries=2 +short "${args[@]}" "@$server" example.com A 2>/dev/null) + if [ $? -eq 0 ] && printf '%s\n' "$output" | /usr/bin/awk '/^[0-9]+([.][0-9]+){3}$/ { found=1 } END { exit !found }'; then pass "Quad9 $server $transport DNS"; else fail "Quad9 $server $transport DNS"; return 1; fi +} + +stage3_network_matrix() { + local gateway reserved web_ipv4 current_docker + current_docker=$(stage3_docker_network_canonical) || { fail 'Docker gateway inventory query failed'; return 1; } + stage3_materialize_docker_ipv4_gateways "$current_docker" || { fail 'Docker gateway inventory is not exact 39-address set'; return 1; } + stage3_positive_dns 9.9.9.9 udp || return 1 + stage3_positive_dns 9.9.9.9 tcp || return 1 + stage3_positive_dns 149.112.112.112 udp || return 1 + stage3_positive_dns 149.112.112.112 tcp || return 1 + stage3_expect_blocked_counter 'non-Quad9 UDP DNS' forward 9 /usr/bin/timeout 6 /usr/bin/ip netns exec le-app-codex /usr/bin/dig +time=2 +tries=1 @1.1.1.1 example.com A || return 1 + stage3_expect_blocked_counter 'non-Quad9 TCP DNS' forward 9 /usr/bin/timeout 6 /usr/bin/ip netns exec le-app-codex /usr/bin/dig +tcp +time=2 +tries=1 @1.1.1.1 example.com A || return 1 + web_ipv4=$(/usr/bin/timeout 10 /usr/bin/ip netns exec le-app-codex /usr/bin/dig +time=3 +tries=2 +short @9.9.9.9 example.com A 2>/dev/null | /usr/bin/awk '/^[0-9]+([.][0-9]+){3}$/ { print; exit }') + case "$web_ipv4" in ''|0.*|10.*|100.6[4-9].*|100.[7-9][0-9].*|100.1[01][0-9].*|100.12[0-7].*|127.*|169.254.*|172.1[6-9].*|172.2[0-9].*|172.3[01].*|192.168.*|198.18.*|198.19.*|224.*|22[5-9].*|23[0-9].*|24[0-9].*|25[0-5].*|74.67.173.56) fail "unsafe public web test address ${web_ipv4:-missing}"; return 1 ;; esac + /usr/bin/timeout 20 /usr/bin/ip netns exec le-app-codex "${proxy_free_env[@]}" /usr/bin/curl --noproxy '*' -4sS --connect-timeout 5 --max-time 15 --resolve "example.com:80:$web_ipv4" -o /dev/null http://example.com/ && pass 'public TCP/80 works' || { fail 'public TCP/80'; return 1; } + /usr/bin/timeout 20 /usr/bin/ip netns exec le-app-codex "${proxy_free_env[@]}" /usr/bin/curl --noproxy '*' -4fsS --connect-timeout 5 --max-time 15 --resolve "example.com:443:$web_ipv4" -o /dev/null https://example.com/ && pass 'public TCP/443 works' || { fail 'public TCP/443'; return 1; } + stage3_expect_blocked_counter 'namespace peer/host denied' input 3 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z 10.200.120.1 22 || return 1 + stage3_expect_blocked_counter 'host LAN address denied' input 3 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z 192.168.10.151 22 || return 1 + stage3_expect_blocked_counter 'host WireGuard address denied' input 3 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z 10.98.0.2 22 || return 1 + for gateway in "${stage3_docker_gateways[@]}"; do stage3_expect_blocked_counter "Docker gateway $gateway denied" input 3 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z "$gateway" 80 || return 1; done + stage3_expect_blocked_counter 'LAN gateway denied' forward 5 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z 192.168.10.1 53 || return 1 + stage3_expect_blocked_counter 'private remote denied' forward 5 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z 10.123.45.67 80 || return 1 + stage3_expect_blocked_counter 'metadata denied' forward 5 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex "${proxy_free_env[@]}" /usr/bin/curl --noproxy '*' -4sS --connect-timeout 2 --max-time 4 http://169.254.169.254/latest/meta-data/ || return 1 + for reserved in 100.64.0.1 192.0.0.1 192.0.2.1 192.88.99.1 198.18.0.1 198.51.100.1 203.0.113.1 240.0.0.1; do stage3_expect_blocked_counter "reserved $reserved denied" forward 5 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z "$reserved" 80 || return 1; done + /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z 127.0.0.1 80 >/dev/null 2>&1 && { fail 'namespace loopback unexpectedly reachable'; return 1; } || pass 'isolated namespace loopback denied/no listener' + stage3_expect_blocked_counter 'observed public/NAT IPv4 denied' forward 4 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z "$expected_public_ipv4" 80 || return 1 + stage3_expect_blocked_counter 'SMTP/25 denied' forward 7 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z 1.1.1.1 25 || return 1 + stage3_expect_blocked_counter 'SMTP/465 denied' forward 7 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z 1.1.1.1 465 || return 1 + stage3_expect_blocked_counter 'SMTP/587 denied' forward 7 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z 1.1.1.1 587 || return 1 + stage3_expect_blocked_counter 'other public TCP port denied' forward 9 /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex /usr/bin/nc -w2 -z 1.1.1.1 22 || return 1 + stage3_expect_blocked_counter 'public UDP/443 denied' forward 9 /usr/bin/timeout 6 /usr/bin/ip netns exec le-app-codex /usr/bin/dig +time=2 +tries=1 -p 443 @1.1.1.1 example.com A || return 1 + /usr/bin/timeout 5 /usr/bin/ip netns exec le-app-codex "${proxy_free_env[@]}" /usr/bin/curl --noproxy '*' -6sS --connect-timeout 2 --max-time 4 'https://[2606:4700:4700::1111]/' >/dev/null 2>&1 && { fail 'IPv6 connection unexpectedly succeeded'; return 1; } || pass 'IPv6 connection denied' + stage3_capture_exact_project_policy_snapshot || { fail 'project policy exact snapshot query failed after probes'; return 1; } + [ "$stage3_snapshot_inet_handle" = "$stage3_nft_inet_handle" ] && + [ "$stage3_snapshot_nat_handle" = "$stage3_nft_nat_handle" ] && + [ "$stage3_snapshot_inet_sha256" = "$stage3_nft_inet_sha256" ] && + [ "$stage3_snapshot_nat_sha256" = "$stage3_nft_nat_sha256" ] && + [ "$stage3_snapshot_project_sha256" = "$stage3_project_nft_sha256" ] && + pass 'project policy exact stable snapshot unchanged after probes' || { fail 'project policy changed during probes'; return 1; } +} + +stage3_activation_main() { + local command pids pids_rc + trap activation_cleanup EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + + for command in /bin/bash /usr/bin/awk /usr/bin/cat /usr/bin/chmod /usr/bin/chown /usr/bin/cut /usr/bin/date /usr/bin/docker /usr/bin/dig /usr/bin/env /usr/bin/flock /usr/bin/git /usr/bin/grep /usr/bin/hostname /usr/bin/id /usr/bin/install /usr/bin/ip /usr/bin/jq /usr/bin/kill /usr/bin/mv /usr/bin/nc /usr/bin/nft /usr/bin/pgrep /usr/bin/ps /usr/bin/sha256sum /usr/bin/sleep /usr/bin/sort /usr/bin/ss /usr/bin/stat /usr/bin/sync /usr/bin/sysctl /usr/bin/systemctl /usr/bin/systemd-analyze /usr/bin/systemd-run /usr/bin/timeout /usr/bin/unlink /usr/bin/wc; do [ -x "$command" ] || fail "required command missing $command"; done + [ "$(/usr/bin/id -u)" = 0 ] && pass 'running as root' || fail 'must run as root' + stage3_verify_lock_parent && pass 'lock/marker parent exact root-owned directory' || fail 'lock/marker parent metadata' + stage3_acquire_lock && pass 'exclusive Stage 3 lock acquired' || fail 'another Stage 3 operation holds the lock' + stage3_verify_repo_provenance || true + stage3_verify_stage1_state_and_files || true + [ "$(/usr/bin/sha256sum "$installed_policy" 2>/dev/null | /usr/bin/cut -d' ' -f1)" = "$expected_policy_digest" ] && pass 'installed policy digest pinned' || fail 'installed policy digest' + /usr/bin/nft --check -f "$installed_policy" && pass 'installed policy root syntax check without load' || fail 'installed policy root syntax check' + stage3_verify_pre_activation_inert || true + stage3_capture_baseline || true + stage3_verify_pre_activation_inert || true + stage3_verify_loaded_units inactive || true + stage3_abort_if_failed + + stage3_activation_id=$(< /proc/sys/kernel/random/uuid) + stage3_machine_id_sha256=$(/usr/bin/sha256sum /etc/machine-id | /usr/bin/cut -d' ' -f1) + stage3_timestamp=$(/usr/bin/date -u +%Y-%m-%dT%H:%M:%SZ) + stage3_hostname=$(/usr/bin/hostname) + stage3_boot_id=$(< /proc/sys/kernel/random/boot_id) + cleanup_armed=1 + stage3_write_marker PREPARED && pass 'root-owned PREPARED approval marker published atomically' || { fail 'approval marker publication'; stage3_abort_if_failed; } + + /usr/bin/systemctl start le-app-codex-netns.service && pass 'started only persistent Stage 3 namespace service' || { fail 'namespace service start'; stage3_abort_if_failed; } + stage3_verify_project_topology + stage3_verify_project_nft + stage3_verify_no_user_runtime 'post-start' || true + stage3_compare_baseline 'post-start' || true + stage3_abort_if_failed + + stage3_capture_active_topology_identity && pass 'active topology identity recaptured immediately before provenance publication' || fail 'active topology recapture before provenance publication' + stage3_verify_project_nft + stage3_service_invocation_id=$(/usr/bin/systemctl show le-app-codex-netns.service --property=InvocationID --value 2>/dev/null) || fail 'namespace service invocation query' + [[ "$stage3_service_invocation_id" =~ ^[0-9a-f]{32}$ ]] || fail 'namespace service invocation identity' + stage3_namespace_dev_inode=$stage3_active_namespace_dev_inode + stage3_host_veth_ifindex=$stage3_active_host_ifindex + stage3_host_veth_iflink=$stage3_active_host_iflink + stage3_host_veth_mac=$stage3_active_host_mac + stage3_capture_exact_project_policy_snapshot || fail 'exact stable project policy snapshot immediately before ACTIVE publication' + stage3_nft_inet_handle=$stage3_snapshot_inet_handle + stage3_nft_nat_handle=$stage3_snapshot_nat_handle + stage3_nft_inet_sha256=$stage3_snapshot_inet_sha256 + stage3_nft_nat_sha256=$stage3_snapshot_nat_sha256 + stage3_project_nft_sha256=$stage3_snapshot_project_sha256 + stage3_abort_if_failed + stage3_write_marker ACTIVE && pass 'root-owned ACTIVE approval marker published atomically' || { fail 'ACTIVE marker publication'; stage3_abort_if_failed; } + + stage3_network_matrix || { fail 'Stage 3 network matrix returned failure'; stage3_abort_if_failed; } + PHASE2B_STAGE3_LOCK_HELD=1 PHASE2B_STAGE3_LOCK_ID="$PHASE2B_STAGE3_LOCK_ID" /bin/bash "$stage3_root/tests/52-run-stage1-inert.sh" && pass 'existing installed Stage 1 inert containment test' || { fail 'existing Stage 1 inert containment test'; stage3_abort_if_failed; } + + stage3_verify_no_user_runtime 'post-tests' || true + pids=$(/usr/bin/ip netns pids le-app-codex 2>/dev/null); pids_rc=$? + [ "$pids_rc" -eq 0 ] && [ -z "$pids" ] && pass 'namespace empty after transient test' || fail 'namespace PID query failed or retains a process after transient test' + [ ! -e "$inert_probe" ] && [ ! -L "$inert_probe" ] && pass 'inert write probe cleaned' || fail 'inert write probe residue' + stage3_read_marker && [ "${stage3_marker_fields[STATUS]}" = ACTIVE ] && [ "${stage3_marker_fields[STAGE3_ACTIVATION_COMMIT]}" = "$stage3_head" ] && pass 'ACTIVE approval marker reverified' || fail 'ACTIVE approval marker drift' + stage3_compare_baseline 'post-tests' || true + stage3_verify_project_topology + stage3_capture_exact_project_policy_snapshot && + [ "$stage3_snapshot_inet_handle" = "${stage3_marker_fields[NFT_INET_HANDLE]:-}" ] && + [ "$stage3_snapshot_nat_handle" = "${stage3_marker_fields[NFT_NAT_HANDLE]:-}" ] && + [ "$stage3_snapshot_inet_sha256" = "${stage3_marker_fields[NFT_INET_SHA256]:-}" ] && + [ "$stage3_snapshot_nat_sha256" = "${stage3_marker_fields[NFT_NAT_SHA256]:-}" ] && + [ "$stage3_snapshot_project_sha256" = "${stage3_marker_fields[PROJECT_NFT_SHA256]:-}" ] && + pass 'final project nftables policy exactly matches retained stable snapshot' || fail 'final project nftables policy drift, instability, or query failure' + stage3_abort_if_failed + + cleanup_armed=0 + trap - EXIT HUP INT TERM + printf 'Phase 2B Stage 3 activation marker=%s service=active namespace=le-app-codex failures=0\n' "$marker" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then stage3_activation_main "$@"; fi diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/51-rollback-stage3.sh b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/51-rollback-stage3.sh new file mode 100644 index 0000000..a86207e --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/51-rollback-stage3.sh @@ -0,0 +1,431 @@ +#!/usr/bin/env bash + +set -o pipefail +export LC_ALL=C +export PATH=/usr/bin:/bin +export SYSTEMD_COLORS=0 +export SYSTEMD_LOG_COLOR=0 +export SYSTEMD_PAGER=cat + +failures=0 +activation_failure=0 +record_location= +pass() { printf 'PASS: %s\n' "$1"; } +fail() { printf 'FAIL: %s\n' "$1" >&2; failures=$((failures + 1)); } +source "$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" 2>/dev/null && pwd)/stage3-common.sh" + +stage3_resources_absent() { + stage3_capture_resource_inventory || return 2 + if stage3_inventory_has_namespace || stage3_inventory_has_link || stage3_inventory_has_host_ns_peer || stage3_inventory_has_table inet le_app_codex || stage3_inventory_has_table ip le_app_codex_nat || [ -e /run/netns/le-app-codex ] || [ -L /run/netns/le-app-codex ]; then return 1; fi + return 0 +} + +stage3_prepared_table_shape() { + local family=$1 name=$2 raw text + raw=$(stage3_query_nft_table_json "$family" "$name") || return 1 + text=$(stage3_query_nft_table_text "$family" "$name") || return 1 + if [ "$family/$name" = inet/le_app_codex ]; then + stage3_project_inet_snapshot_valid "$raw" "$text" + else + [ "$family/$name" = ip/le_app_codex_nat ] || return 1 + stage3_project_nat_snapshot_valid "$raw" "$text" + fi +} + +stage3_table_identity() { + local family=$1 name=$2 expected_handle expected_hash actual_handle actual_hash + stage3_inventory_has_table "$family" "$name" || return 0 + if [ "${stage3_marker_fields[STATUS]}" = ACTIVE ]; then + if [ "$family" = inet ]; then expected_handle=${stage3_marker_fields[NFT_INET_HANDLE]}; expected_hash=${stage3_marker_fields[NFT_INET_SHA256]}; else expected_handle=${stage3_marker_fields[NFT_NAT_HANDLE]}; expected_hash=${stage3_marker_fields[NFT_NAT_SHA256]}; fi + actual_handle=$(stage3_inventory_table_handle "$family" "$name") || return 1 + actual_hash=$(stage3_nft_table_digest "$family" "$name") || return 1 + [ "$actual_handle" = "$expected_handle" ] && [ "$actual_hash" = "$expected_hash" ] + else + stage3_prepared_table_shape "$family" "$name" + fi +} + +stage3_namespace_identity() { + local pids pids_rc current ns_json + stage3_inventory_has_namespace || { [ ! -e /run/netns/le-app-codex ] && [ ! -L /run/netns/le-app-codex ]; return; } + [ -e /run/netns/le-app-codex ] && [ ! -L /run/netns/le-app-codex ] || return 1 + [ "$(/usr/bin/stat -f -c '%T' /run/netns/le-app-codex 2>/dev/null)" = nsfs ] || return 1 + if [ "${stage3_marker_fields[STATUS]}" = ACTIVE ]; then + current=$(/usr/bin/stat -Lc '%D:%i' /run/netns/le-app-codex 2>/dev/null) || return 1 + [ "$current" = "${stage3_marker_fields[NAMESPACE_DEV_INODE]}" ] || return 1 + fi + pids=$(/usr/bin/ip netns pids le-app-codex 2>/dev/null); pids_rc=$? + [ "$pids_rc" -eq 0 ] && [ -z "$pids" ] || return 1 + ns_json=$(/usr/bin/ip -n le-app-codex -d -j link show 2>/dev/null) || return 1 + printf '%s' "$ns_json" | /usr/bin/jq -e 'any(.[]; .ifname == "lo") and ([.[] | select(.ifname == "lecodex-ns")] | length) <= 1 and ([.[] | select(.ifname != "lo" and .ifname != "lecodex-ns")] | length) == 0' >/dev/null +} + +stage3_veth_identity() { + local link_json addr_json host_ipv6 current_ifindex current_iflink current_mac peer_json peer_addr_json peer_ipv6 peer_ifindex peer_iflink peer_location= ns_links ns_routes4 ns_routes6 ns_ipv6 + if ! stage3_inventory_has_link; then + ! stage3_inventory_has_host_ns_peer || return 1 + if stage3_inventory_has_namespace; then + ns_links=$(/usr/bin/ip -n le-app-codex -d -j link show 2>/dev/null) || return 1 + printf '%s' "$ns_links" | /usr/bin/jq -e 'all(.[]; .ifname != "lecodex-ns")' >/dev/null || return 1 + fi + return 0 + fi + link_json=$(printf '%s' "$stage3_inventory_links" | /usr/bin/jq -ec '[.[] | select(.ifname == "lecodex-host")] | select(length == 1)') || return 1 + printf '%s' "$link_json" | /usr/bin/jq -e '.[0].linkinfo.info_kind == "veth"' >/dev/null || return 1 + current_ifindex=$(< /sys/class/net/lecodex-host/ifindex) || return 1 + current_iflink=$(< /sys/class/net/lecodex-host/iflink) || return 1 + current_mac=$(< /sys/class/net/lecodex-host/address) || return 1 + if [ "${stage3_marker_fields[STATUS]}" = ACTIVE ]; then + stage3_veth_fields_match "${stage3_marker_fields[HOST_VETH_IFINDEX]}" "${stage3_marker_fields[HOST_VETH_IFLINK]}" "${stage3_marker_fields[HOST_VETH_MAC]}" "$current_ifindex" "$current_iflink" "$current_mac" || return 1 + fi + addr_json=$(/usr/bin/ip -j -4 address show dev lecodex-host 2>/dev/null) || return 1 + host_ipv6=$(/usr/bin/ip -j -6 address show dev lecodex-host 2>/dev/null) || return 1 + printf '%s' "$host_ipv6" | /usr/bin/jq -e 'all(.[].addr_info[]?; .family != "inet6" or .scope == "link")' >/dev/null || return 1 + if [ "${stage3_marker_fields[STATUS]}" = ACTIVE ]; then + printf '%s' "$addr_json" | /usr/bin/jq -e '[.[].addr_info[] | select(.family == "inet") | {local,prefixlen}] == [{"local":"10.200.120.1","prefixlen":30}]' >/dev/null || return 1 + else + printf '%s' "$addr_json" | /usr/bin/jq -e '([.[].addr_info[] | select(.family == "inet") | {local,prefixlen}]) as $a | ($a == [] or $a == [{"local":"10.200.120.1","prefixlen":30}])' >/dev/null || return 1 + fi + if stage3_inventory_has_host_ns_peer; then + peer_location=host + peer_json=$(printf '%s' "$stage3_inventory_links" | /usr/bin/jq -ec '[.[] | select(.ifname == "lecodex-ns")] | select(length == 1)') || return 1 + peer_ifindex=$(< /sys/class/net/lecodex-ns/ifindex) || return 1 + peer_iflink=$(< /sys/class/net/lecodex-ns/iflink) || return 1 + peer_addr_json=$(/usr/bin/ip -j -4 address show dev lecodex-ns 2>/dev/null) || return 1 + peer_ipv6=$(/usr/bin/ip -j -6 address show dev lecodex-ns 2>/dev/null) || return 1 + printf '%s' "$peer_ipv6" | /usr/bin/jq -e 'all(.[].addr_info[]?; .family != "inet6" or .scope == "link")' >/dev/null || return 1 + if stage3_inventory_has_namespace; then + ns_links=$(/usr/bin/ip -n le-app-codex -d -j link show 2>/dev/null) || return 1 + printf '%s' "$ns_links" | /usr/bin/jq -e 'all(.[]; .ifname != "lecodex-ns")' >/dev/null || return 1 + fi + elif stage3_inventory_has_namespace; then + peer_location=namespace + ns_links=$(/usr/bin/ip -n le-app-codex -d -j link show 2>/dev/null) || return 1 + peer_json=$(printf '%s' "$ns_links" | /usr/bin/jq -ec '[.[] | select(.ifname == "lecodex-ns")] | select(length == 1)') || return 1 + peer_ifindex=$(/usr/bin/ip netns exec le-app-codex /usr/bin/cat /sys/class/net/lecodex-ns/ifindex 2>/dev/null) || return 1 + peer_iflink=$(/usr/bin/ip netns exec le-app-codex /usr/bin/cat /sys/class/net/lecodex-ns/iflink 2>/dev/null) || return 1 + peer_addr_json=$(/usr/bin/ip -n le-app-codex -j -4 address show dev lecodex-ns 2>/dev/null) || return 1 + else + return 1 + fi + printf '%s' "$peer_json" | /usr/bin/jq -e 'length == 1 and .[0].ifname == "lecodex-ns" and .[0].linkinfo.info_kind == "veth"' >/dev/null || return 1 + [ "$peer_ifindex" = "$current_iflink" ] && [ "$peer_iflink" = "$current_ifindex" ] || return 1 + if [ "${stage3_marker_fields[STATUS]}" = ACTIVE ] && [ "$peer_location" = namespace ]; then + printf '%s' "$peer_addr_json" | /usr/bin/jq -e '[.[].addr_info[] | select(.family == "inet") | {local,prefixlen}] == [{"local":"10.200.120.2","prefixlen":30}]' >/dev/null || return 1 + else + printf '%s' "$peer_addr_json" | /usr/bin/jq -e '([.[].addr_info[] | select(.family == "inet") | {local,prefixlen}]) as $a | ($a == [] or $a == [{"local":"10.200.120.2","prefixlen":30}])' >/dev/null || return 1 + fi + if [ "$peer_location" = namespace ]; then + ns_routes4=$(/usr/bin/ip -n le-app-codex -j -4 route show table main 2>/dev/null) || return 1 + ns_ipv6=$(/usr/bin/ip -n le-app-codex -j -6 address show 2>/dev/null) || return 1 + ns_routes6=$(/usr/bin/ip -n le-app-codex -j -6 route show table main 2>/dev/null) || return 1 + if [ "${stage3_marker_fields[STATUS]}" = ACTIVE ]; then + printf '%s' "$ns_routes4" | /usr/bin/jq -e 'length == 2 and any(.[]; .dst == "10.200.120.0/30" and .dev == "lecodex-ns") and any(.[]; .dst == "default" and .gateway == "10.200.120.1" and .dev == "lecodex-ns")' >/dev/null || return 1 + printf '%s' "$ns_ipv6" | /usr/bin/jq -e '[.[].addr_info[]? | select(.family == "inet6")] | length == 0' >/dev/null || return 1 + printf '%s' "$ns_routes6" | /usr/bin/jq -e 'length == 0' >/dev/null || return 1 + else + printf '%s' "$ns_routes4" | /usr/bin/jq -e 'length <= 2 and all(.[]; (.dst == "10.200.120.0/30" and .dev == "lecodex-ns") or (.dst == "default" and .gateway == "10.200.120.1" and .dev == "lecodex-ns"))' >/dev/null || return 1 + printf '%s' "$ns_ipv6" | /usr/bin/jq -e 'all(.[].addr_info[]?; .family != "inet6" or .scope == "link" or .local == "::1")' >/dev/null || return 1 + printf '%s' "$ns_routes6" | /usr/bin/jq -e 'all(.[]; ((.dst // "") | startswith("fe80:")) or .dst == "::1")' >/dev/null || return 1 + fi + fi +} + +stage3_service_identity() { + local show active sub invocation + show=$(/usr/bin/systemctl show --no-pager le-app-codex-netns.service --property=ActiveState --property=SubState --property=InvocationID 2>/dev/null) || return 1 + active=$(stage3_show_value "$show" ActiveState) || return 1 + sub=$(stage3_show_value "$show" SubState) || return 1 + invocation=$(stage3_show_value "$show" InvocationID) || return 1 + if [ "$active/$sub" = inactive/dead ]; then return 0; fi + [ "$active/$sub" = active/exited ] || [ "$active/$sub" = failed/failed ] || return 1 + [[ "$invocation" =~ ^[0-9a-f]{32}$ ]] || return 1 + if [ "${stage3_marker_fields[STATUS]}" = ACTIVE ]; then [ "$invocation" = "${stage3_marker_fields[SERVICE_INVOCATION_ID]}" ]; else return 0; fi +} + +stage3_resource_presence_shape_valid() { + local namespace=$1 host=$2 peer_host=$3 peer_namespace=$4 inet_table=$5 nat_table=$6 value + for value in "$namespace" "$host" "$peer_host" "$peer_namespace" "$inet_table" "$nat_table"; do [ "$value" = 0 ] || [ "$value" = 1 ] || return 1; done + if [ "$namespace" -eq 0 ]; then [ "$host$peer_host$peer_namespace$inet_table$nat_table" = 00000 ]; return; fi + if [ "$host" -eq 0 ]; then [ "$peer_host$peer_namespace$inet_table$nat_table" = 0000 ]; return; fi + [ $((peer_host + peer_namespace)) -eq 1 ] || return 1 + if [ "$peer_host" -eq 1 ]; then [ "$inet_table$nat_table" = 00 ]; return; fi + return 0 +} + +stage3_prepared_live_invocation_valid() { + local has_resources=$1 invocation=${2:-} + [ "$has_resources" = 0 ] || [ "$has_resources" = 1 ] || return 1 + [ "${stage3_marker_fields[STATUS]}" = PREPARED ] || return 0 + [ "$has_resources" -eq 1 ] || return 0 + if [ "$record_location" = tombstone ]; then return 0; fi + [ "$record_location" = marker ] && [[ "$invocation" =~ ^[0-9a-f]{32}$ ]] +} + +stage3_verify_survivors() { + local namespace_present=0 host_present=0 peer_host_present=0 peer_namespace_present=0 inet_present=0 nat_present=0 ns_links has_resources=0 invocation= + stage3_capture_resource_inventory || return 1 + stage3_inventory_has_namespace && namespace_present=1 + stage3_inventory_has_link && host_present=1 + stage3_inventory_has_host_ns_peer && peer_host_present=1 + stage3_inventory_has_table inet le_app_codex && inet_present=1 + stage3_inventory_has_table ip le_app_codex_nat && nat_present=1 + if [ "$namespace_present" -eq 1 ]; then + ns_links=$(/usr/bin/ip -n le-app-codex -d -j link show 2>/dev/null) || return 1 + printf '%s' "$ns_links" | /usr/bin/jq -e 'type == "array" and all(.[]; type == "object" and (.ifname | type) == "string")' >/dev/null || return 1 + printf '%s' "$ns_links" | /usr/bin/jq -e 'any(.[]; .ifname == "lecodex-ns")' >/dev/null && peer_namespace_present=1 + fi + stage3_resource_presence_shape_valid "$namespace_present" "$host_present" "$peer_host_present" "$peer_namespace_present" "$inet_present" "$nat_present" || return 1 + stage3_service_identity || return 1 + stage3_table_identity inet le_app_codex || return 1 + stage3_table_identity ip le_app_codex_nat || return 1 + stage3_veth_identity || return 1 + stage3_namespace_identity || return 1 + if [ "$namespace_present" -eq 1 ] || [ "$host_present" -eq 1 ] || [ "$inet_present" -eq 1 ] || [ "$nat_present" -eq 1 ]; then has_resources=1; fi + if [ "${stage3_marker_fields[STATUS]}" = PREPARED ] && [ "$has_resources" -eq 1 ] && [ "$record_location" = marker ]; then + invocation=$(/usr/bin/systemctl show le-app-codex-netns.service --property=InvocationID --value 2>/dev/null) || return 1 + fi + stage3_prepared_live_invocation_valid "$has_resources" "$invocation" +} + +stage3_validate_safe_temporary() { + local temporary=$1 authority=$2 expected_uid=${3:-0} expected_gid=${4:-0} require_provenance=${5:-1} stat stat_uid stat_gid stat_mode links size type activation record_status record_common record_hash temp_activation temp_status temp_common temp_hash + [ -e "$temporary" ] || [ -L "$temporary" ] || return 0 + stat=$(/usr/bin/stat -c '%u|%g|%a|%h|%s|%F' "$temporary" 2>/dev/null) || return 1 + IFS='|' read -r stat_uid stat_gid stat_mode links size type <<< "$stat" + [ "$stat_uid" = "$expected_uid" ] && [ "$stat_gid" = "$expected_gid" ] && [ "$stat_mode" = 600 ] && [ "$links" = 1 ] && { [ "$type" = 'regular file' ] || [ "$type" = 'regular empty file' ]; } && [ -f "$temporary" ] && [ ! -L "$temporary" ] || return 1 + [[ "$size" =~ ^[0-9]+$ ]] || return 1 + activation=${stage3_marker_fields[ACTIVATION_ID]} + record_status=${stage3_marker_fields[STATUS]} + record_common=$(stage3_record_common_sha256) || return 1 + record_hash=$(/usr/bin/sha256sum "$authority" 2>/dev/null | /usr/bin/cut -d' ' -f1) || return 1 + [ "$size" -ne 0 ] || return 0 + if ! stage3_parse_record_file "$temporary" "$require_provenance"; then + if [ "$authority" = "$marker" ]; then stage3_read_marker >/dev/null 2>&1 || true; else stage3_read_rollback_record >/dev/null 2>&1 || true; fi + return 1 + fi + temp_activation=${stage3_marker_fields[ACTIVATION_ID]} + temp_status=${stage3_marker_fields[STATUS]} + temp_common=$(stage3_record_common_sha256) || { if [ "$authority" = "$marker" ]; then stage3_read_marker >/dev/null 2>&1 || true; else stage3_read_rollback_record >/dev/null 2>&1 || true; fi; return 1; } + temp_hash=$(/usr/bin/sha256sum "$temporary" 2>/dev/null | /usr/bin/cut -d' ' -f1) || { if [ "$authority" = "$marker" ]; then stage3_read_marker >/dev/null 2>&1 || true; else stage3_read_rollback_record >/dev/null 2>&1 || true; fi; return 1; } + if [ "$authority" = "$marker" ]; then stage3_read_marker || return 1; else stage3_read_rollback_record || return 1; fi + [ "$temp_activation" = "$activation" ] || return 1 + [ "$record_common" = "$temp_common" ] || return 1 + if [ "$record_status/$temp_status" = PREPARED/ACTIVE ]; then + return 0 + elif [ "$record_status" = "$temp_status" ]; then + [ "$record_hash" = "$temp_hash" ] + else + return 1 + fi +} + +stage3_cleanup_safe_temporary() { + local temporary=$1 expected_uid=${2:-0} expected_gid=${3:-0} require_provenance=${4:-1} stat size record_status temp_status temporary_id temporary_hash authority_id authority_hash + [ -e "$temporary" ] || [ -L "$temporary" ] || return 0 + stage3_validate_safe_temporary "$temporary" "$rollback_record" "$expected_uid" "$expected_gid" "$require_provenance" || return 1 + stat=$(/usr/bin/stat -c '%s' "$temporary" 2>/dev/null) || return 1 + [[ "$stat" =~ ^[0-9]+$ ]] || return 1 + size=$stat + if [ "$size" -eq 0 ]; then + /usr/bin/unlink -- "$temporary" && stage3_sync_parent "$temporary" + return + fi + record_status=${stage3_marker_fields[STATUS]} + stage3_parse_record_file "$temporary" "$require_provenance" || return 1 + temp_status=${stage3_marker_fields[STATUS]} + stage3_read_rollback_record || return 1 + if [ "$record_status/$temp_status" = PREPARED/ACTIVE ]; then + temporary_id=$(/usr/bin/stat -c '%D:%i' "$temporary" 2>/dev/null) || return 1 + temporary_hash=$(/usr/bin/sha256sum "$temporary" 2>/dev/null | /usr/bin/cut -d' ' -f1) || return 1 + authority_id=$(/usr/bin/stat -c '%D:%i' "$rollback_record" 2>/dev/null) || return 1 + authority_hash=$(/usr/bin/sha256sum "$rollback_record" 2>/dev/null | /usr/bin/cut -d' ' -f1) || return 1 + /usr/bin/sync -f "$temporary" 2>/dev/null || /usr/bin/sync || return 1 + [ "$temporary_id" = "$(/usr/bin/stat -c '%D:%i' "$temporary" 2>/dev/null)" ] && [ "$temporary_hash" = "$(/usr/bin/sha256sum "$temporary" 2>/dev/null | /usr/bin/cut -d' ' -f1)" ] || return 1 + stage3_validate_safe_temporary "$temporary" "$rollback_record" "$expected_uid" "$expected_gid" "$require_provenance" || return 1 + [ "$authority_id" = "$(/usr/bin/stat -c '%D:%i' "$rollback_record" 2>/dev/null)" ] && [ "$authority_hash" = "$(/usr/bin/sha256sum "$rollback_record" 2>/dev/null | /usr/bin/cut -d' ' -f1)" ] || return 1 + /usr/bin/mv -T -- "$temporary" "$rollback_record" || return 1 + [ "$(/usr/bin/stat -c '%D:%i' "$rollback_record" 2>/dev/null)" = "$temporary_id" ] || return 1 + stage3_sync_parent "$rollback_record" || return 1 + stage3_read_rollback_record && [ "${stage3_marker_fields[STATUS]}" = ACTIVE ] + return + fi + [ "$record_status" = "$temp_status" ] || return 1 + /usr/bin/unlink -- "$temporary" && stage3_sync_parent "$temporary" +} + +stage3_select_authoritative_record() { + local expected_uid=${1:-0} expected_gid=${2:-0} require_provenance=${3:-1} marker_exists=0 tomb_exists=0 temp_stat temp_uid temp_gid temp_mode temp_links temp_size temp_type temporary_id temporary_hash + if [ -e "$marker" ] || [ -L "$marker" ]; then marker_exists=1; fi + if [ -e "$rollback_record" ] || [ -L "$rollback_record" ]; then tomb_exists=1; fi + [ "$marker_exists" -eq 0 ] || [ "$tomb_exists" -eq 0 ] || return 1 + if [ "$tomb_exists" -eq 1 ]; then + stage3_read_rollback_record || return 1 + record_location=tombstone + elif [ "$marker_exists" -eq 1 ]; then + stage3_read_marker || return 1 + record_location=marker + elif [ -e "$marker_temporary" ] || [ -L "$marker_temporary" ]; then + temp_stat=$(/usr/bin/stat -c '%u|%g|%a|%h|%s|%F' "$marker_temporary" 2>/dev/null) || return 1 + IFS='|' read -r temp_uid temp_gid temp_mode temp_links temp_size temp_type <<< "$temp_stat" + [ "$temp_uid" = "$expected_uid" ] && [ "$temp_gid" = "$expected_gid" ] && [ "$temp_mode" = 600 ] && [ "$temp_links" = 1 ] && { [ "$temp_type" = 'regular file' ] || [ "$temp_type" = 'regular empty file' ]; } && [ -f "$marker_temporary" ] && [ ! -L "$marker_temporary" ] || return 1 + [[ "$temp_size" =~ ^[0-9]+$ ]] || return 1 + if ! stage3_parse_record_file "$marker_temporary" "$require_provenance"; then + [ "$temp_size" -eq 0 ] || return 1 + stage3_resources_absent || return 1 + /usr/bin/unlink -- "$marker_temporary" && stage3_sync_parent "$marker_temporary" || return 1 + return 3 + fi + temporary_id=$(/usr/bin/stat -c '%D:%i' "$marker_temporary" 2>/dev/null) || return 1 + temporary_hash=$(/usr/bin/sha256sum "$marker_temporary" 2>/dev/null | /usr/bin/cut -d' ' -f1) || return 1 + /usr/bin/sync -f "$marker_temporary" 2>/dev/null || /usr/bin/sync || return 1 + [ "$temporary_id" = "$(/usr/bin/stat -c '%D:%i' "$marker_temporary" 2>/dev/null)" ] && [ "$temporary_hash" = "$(/usr/bin/sha256sum "$marker_temporary" 2>/dev/null | /usr/bin/cut -d' ' -f1)" ] || return 1 + stage3_parse_record_file "$marker_temporary" "$require_provenance" || return 1 + stage3_atomic_publish_new "$marker_temporary" "$rollback_record" || return 1 + stage3_sync_parent "$rollback_record" || return 1 + stage3_read_rollback_record || return 1 + record_location=tombstone + else + return 3 + fi +} + +stage3_capture_rollback_baseline() { + rollback_before_public=$(stage3_public_ipv4) || return 1 + rollback_before_docker_network=$(stage3_docker_network_canonical | stage3_sha_stream) || return 1 + rollback_before_nft=$(stage3_nonproject_nft_canonical | stage3_sha_stream) || return 1 + rollback_before_host=$(stage3_host_network_canonical | stage3_sha_stream) || return 1 + rollback_before_docker_runtime=$(stage3_docker_runtime_canonical | stage3_sha_stream) || return 1 + [[ "$rollback_before_docker_network$rollback_before_nft$rollback_before_host$rollback_before_docker_runtime" =~ ^[0-9a-f]{256}$ ]] +} + +stage3_compare_rollback_baseline() { + [ "$(stage3_public_ipv4)" = "$rollback_before_public" ] || return 1 + [ "$(stage3_docker_network_canonical | stage3_sha_stream)" = "$rollback_before_docker_network" ] || return 1 + [ "$(stage3_nonproject_nft_canonical | stage3_sha_stream)" = "$rollback_before_nft" ] || return 1 + [ "$(stage3_host_network_canonical | stage3_sha_stream)" = "$rollback_before_host" ] || return 1 + [ "$(stage3_docker_runtime_canonical | stage3_sha_stream)" = "$rollback_before_docker_runtime" ] +} + +stage3_delete_surviving_table() { + local family=$1 name=$2 + stage3_capture_resource_inventory || return 1 + stage3_inventory_has_table "$family" "$name" || return 0 + stage3_table_identity "$family" "$name" || return 1 + if [ "$family/$name" = inet/le_app_codex ]; then /usr/bin/nft delete table inet le_app_codex; else /usr/bin/nft delete table ip le_app_codex_nat; fi +} + +stage3_delete_surviving_veth() { + stage3_capture_resource_inventory || return 1 + stage3_veth_identity || return 1 + stage3_inventory_has_link || return 0 + /usr/bin/ip link delete lecodex-host +} + +stage3_delete_surviving_namespace() { + stage3_capture_resource_inventory || return 1 + stage3_veth_identity || return 1 + ! stage3_inventory_has_link && ! stage3_inventory_has_host_ns_peer || return 1 + stage3_inventory_has_namespace || { [ ! -e /run/netns/le-app-codex ] && [ ! -L /run/netns/le-app-codex ]; return; } + stage3_namespace_identity || return 1 + /usr/bin/ip netns delete le-app-codex +} + +stage3_rollback_main() { + local select_rc current_boot active sub enabled inert_units inert_rc before_gate authority_path service_show service_rc enabled_rc + [ "${1:-}" = --activation-failure ] && activation_failure=1 + if [ "$#" -gt 1 ] || { [ "$#" -eq 1 ] && [ "$activation_failure" -ne 1 ]; }; then printf 'usage: %s [--activation-failure]\n' "$0" >&2; return 64; fi + + [ "$(/usr/bin/id -u)" = 0 ] && pass 'running as root' || fail 'must run as root' + stage3_verify_lock_parent && pass 'lock/marker parent exact' || fail 'lock/marker parent metadata' + stage3_acquire_lock && pass 'exclusive Stage 3 lock acquired' || fail 'another Stage 3 operation holds the lock' + stage3_verify_repo_provenance || true + stage3_verify_stage1_state_and_files || true + stage3_verify_loaded_units runtime || true + [ "$failures" -eq 0 ] || { printf 'Rollback preflight failed; no Stage 3 runtime object was changed.\n' >&2; return 1; } + + stage3_select_authoritative_record; select_rc=$? + if [ "$select_rc" -eq 3 ]; then + stage3_resources_absent; select_rc=$? + if [ "$activation_failure" -eq 1 ] && [ "$select_rc" -eq 0 ]; then stage3_cleanup_probe_exact "$inert_probe" || return 1; pass 'activation failed before authoritative publication and left no resources'; return 0; fi + fail 'no authoritative Stage 3 marker or tombstone' + return 1 + elif [ "$select_rc" -ne 0 ]; then + fail 'authoritative Stage 3 record state is unsafe or ambiguous' + return 1 + fi + pass "valid authoritative Stage 3 $record_location record" + if [ "$record_location" = marker ]; then authority_path=$marker; else authority_path=$rollback_record; fi + stage3_validate_safe_temporary "$marker_temporary" "$authority_path" || { fail 'approval temporary is unsafe, malformed, or mismatched'; return 1; } + + current_boot=$(< /proc/sys/kernel/random/boot_id) + if [ "$current_boot" != "${stage3_marker_fields[BOOT_ID]}" ]; then + stage3_resources_absent; select_rc=$? + if service_show=$(/usr/bin/systemctl show --no-pager le-app-codex-netns.service --property=ActiveState --property=SubState 2>/dev/null); then service_rc=0; else service_rc=$?; fi + active=$(stage3_show_value "$service_show" ActiveState 2>/dev/null || true) + sub=$(stage3_show_value "$service_show" SubState 2>/dev/null || true) + if [ "$select_rc" -eq 0 ] && [ "$service_rc" -eq 0 ] && [ "$active" = inactive ] && [ "$sub" = dead ]; then + before_gate=$failures + stage3_compare_recorded_baseline 'changed-boot pre-rollback' || true + stage3_verify_no_user_runtime 'changed-boot rollback' || true + stage3_capture_rollback_baseline || fail 'changed-boot immediate production baseline capture' + [ "$failures" -eq "$before_gate" ] || { printf 'Changed-boot rollback baseline/runtime gate failed; no Stage 3 state was changed.\n' >&2; return 1; } + if [ "$record_location" = marker ]; then stage3_promote_marker_to_tombstone || return 1; fi + stage3_cleanup_safe_temporary "$marker_temporary" || return 1 + stage3_compare_rollback_baseline || { fail 'changed-boot immediate production baseline drift'; return 1; } + stage3_compare_recorded_baseline 'changed-boot post-rollback' || true + stage3_verify_loaded_units inactive || true + stage3_verify_no_user_runtime 'changed-boot post-rollback' || true + [ "$failures" -eq "$before_gate" ] || return 1 + pass 'changed-boot record retained as tombstone after proving runtime resources absent' + return 0 + fi + fail 'boot changed and project resource absence cannot be proven' + return 1 + fi + + before_gate=$failures + stage3_compare_recorded_baseline 'pre-rollback' || true + stage3_verify_no_user_runtime 'pre-rollback' || true + inert_units=$(/usr/bin/systemctl list-units --all --plain --no-legend "$inert_unit_prefix*.service" 2>/dev/null); inert_rc=$? + [ "$inert_rc" -eq 0 ] && [ -z "$inert_units" ] || fail 'inert transient still exists or query failed' + stage3_verify_survivors || fail 'project survivor provenance/topology drift or inventory failure' + stage3_capture_rollback_baseline || fail 'immediate rollback baseline capture' + [ "$failures" -eq "$before_gate" ] || { printf 'Rollback provenance/baseline gate failed; no service or project resource was changed.\n' >&2; return 1; } + + if [ "$record_location" = marker ]; then stage3_promote_marker_to_tombstone && pass 'approval marker atomically promoted to retained rollback tombstone' || { fail 'rollback tombstone publication'; return 1; }; record_location=tombstone; fi + stage3_cleanup_safe_temporary "$marker_temporary" || { fail 'approval temporary cleanup'; return 1; } + stage3_verify_survivors || { fail 'project survivor changed after tombstone publication'; return 1; } + stage3_cleanup_probe_exact "$inert_probe" || { fail 'inert write-probe cleanup'; return 1; } + + if service_show=$(/usr/bin/systemctl show --no-pager le-app-codex-netns.service --property=ActiveState --property=SubState 2>/dev/null); then service_rc=0; else service_rc=$?; fi + [ "$service_rc" -eq 0 ] || { fail 'namespace service state query before stop'; return 1; } + active=$(stage3_show_value "$service_show" ActiveState 2>/dev/null) || { fail 'namespace service ActiveState before stop'; return 1; } + if [ "$active" != inactive ]; then /usr/bin/systemctl stop le-app-codex-netns.service && pass 'stopped only le-app-codex-netns.service' || fail 'namespace service stop'; else pass 'namespace service already inactive for retry'; fi + [ "$failures" -eq 0 ] || return 1 + + stage3_delete_surviving_table inet le_app_codex || { fail 'verified inet table cleanup'; return 1; } + stage3_delete_surviving_table ip le_app_codex_nat || { fail 'verified NAT table cleanup'; return 1; } + stage3_delete_surviving_veth || { fail 'verified host veth cleanup'; return 1; } + stage3_delete_surviving_namespace || { fail 'verified namespace cleanup'; return 1; } + + stage3_resources_absent; select_rc=$? + [ "$select_rc" -eq 0 ] && pass 'namespace, veth, and project tables absent via successful inventories' || fail "project absence result=$select_rc" + [ ! -e "$marker" ] && [ ! -L "$marker" ] && [ -f "$rollback_record" ] && [ ! -L "$rollback_record" ] && stage3_read_rollback_record && pass 'approval marker absent and authoritative tombstone retained' || fail 'marker/tombstone post-state' + if service_show=$(/usr/bin/systemctl show --no-pager le-app-codex-netns.service --property=ActiveState --property=SubState 2>/dev/null); then service_rc=0; else service_rc=$?; fi + active=$(stage3_show_value "$service_show" ActiveState 2>/dev/null || true) + sub=$(stage3_show_value "$service_show" SubState 2>/dev/null || true) + if enabled=$(/usr/bin/systemctl is-enabled le-app-codex-netns.service 2>/dev/null); then enabled_rc=0; else enabled_rc=$?; fi + [ "$service_rc" -eq 0 ] && { [ "$enabled_rc" -eq 0 ] || [ "$enabled_rc" -eq 1 ]; } && [ "$active" = inactive ] && [ "$sub" = dead ] && [ "$enabled" = static ] && pass 'namespace service inactive/dead/static' || fail "namespace service post-query/state rc=$service_rc/$enabled_rc $active/$sub/$enabled" + + stage3_compare_rollback_baseline && pass 'rollback changed no nonproject/production baseline' || fail 'rollback immediate production baseline drift' + stage3_compare_recorded_baseline 'post-rollback' || true + stage3_verify_stage1_state_and_files || true + stage3_verify_loaded_units inactive || true + stage3_verify_no_user_runtime 'post-rollback' || true + + printf 'Phase 2B Stage 3 rollback marker=absent tombstone=retained service=inactive namespace=absent stage1=installed failures=%s\n' "$failures" + [ "$failures" -eq 0 ] +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then stage3_rollback_main "$@"; fi diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/52-run-stage1-inert.sh b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/52-run-stage1-inert.sh new file mode 100644 index 0000000..f2d62fa --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/52-run-stage1-inert.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash + +set -Eeu -o pipefail +export LC_ALL=C +export PATH=/usr/bin:/bin + +stage3_root=$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." 2>/dev/null && pwd) +source "$stage3_root/tests/stage3-common.sh" +touch_wrapper=$stage3_root/tests/inert-bin/touch +rm_wrapper=$stage3_root/tests/inert-bin/rm +inert_client_pid= +inert_client_starttime= +inert_client_started=0 +inert_owned=0 +inert_invocation= +inert_activation= +inert_token= +inert_unit= +inert_pending_signal=0 +inert_probe_uid=1200 +inert_probe_gid=1200 +inert_probe_mode=644 + +stage3_inert_show() { + /usr/bin/systemctl show --no-pager "$1" --property=LoadState --property=ActiveState --property=Transient --property=FragmentPath --property=User --property=Group --property=NetworkNamespacePath --property=Description --property=Environment --property=UnsetEnvironment --property=ExecStart --property=InvocationID +} + +stage3_inert_stop() { /usr/bin/systemctl stop "$1"; } + +stage3_inert_pid_starttime() { + local pid=$1 + [[ "$pid" =~ ^[0-9]+$ ]] || return 1 + /usr/bin/awk 'NF >= 22 && $22 ~ /^[0-9]+$/ { print $22; found=1 } END { exit !found }' "/proc/$pid/stat" 2>/dev/null +} + +stage3_inert_pid_identity_matches() { + local pid=$1 expected=$2 current + [[ "$expected" =~ ^[0-9]+$ ]] || return 1 + current=$(stage3_inert_pid_starttime "$pid") || return 1 + [ "$current" = "$expected" ] +} + +stage3_inert_note_signal() { + [ "$inert_pending_signal" -ne 0 ] || inert_pending_signal=$1 +} + +stage3_inert_defer_signals() { + trap 'stage3_inert_note_signal 129' HUP + trap 'stage3_inert_note_signal 130' INT + trap 'stage3_inert_note_signal 143' TERM +} + +stage3_inert_arm_signals() { + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM +} + +stage3_inert_replay_pending_signal() { + local pending=$inert_pending_signal + inert_pending_signal=0 + [ "$pending" -eq 0 ] || return "$pending" +} + +stage3_inert_cleanup() { + local original_rc=$? cleanup_rc=0 show load show_rc invocation attempt stop_attempted=0 collected=0 not_found_streak=0 + trap - EXIT HUP INT TERM + if [ -n "$inert_client_pid" ] && /usr/bin/kill -0 "$inert_client_pid" 2>/dev/null; then + if stage3_inert_pid_identity_matches "$inert_client_pid" "$inert_client_starttime"; then + /usr/bin/kill -TERM "$inert_client_pid" 2>/dev/null || cleanup_rc=1 + wait "$inert_client_pid" 2>/dev/null || true + else + cleanup_rc=1 + wait "$inert_client_pid" 2>/dev/null || true + fi + fi + for attempt in {1..100}; do + if show=$(stage3_inert_show "$inert_unit" 2>/dev/null); then show_rc=0; else show_rc=$?; fi + if [ "$show_rc" -ne 0 ]; then not_found_streak=0; /usr/bin/sleep 0.05; continue; fi + load=$(stage3_show_value "$show" LoadState 2>/dev/null || true) + if [ "$load" = not-found ]; then + not_found_streak=$((not_found_streak + 1)) + if [ "$not_found_streak" -ge 3 ]; then collected=1; break; fi + /usr/bin/sleep 0.05 + continue + fi + not_found_streak=0 + if [ "$load" = loaded ] && [ "$inert_client_started" -eq 1 ]; then + invocation=$(stage3_show_value "$show" InvocationID 2>/dev/null || true) + if stage3_inert_unit_identity "$show" "$inert_unit" "${inert_invocation:-}" "$inert_activation" "$inert_token"; then + if [ "$inert_owned" -eq 0 ]; then inert_invocation=$invocation; inert_owned=1; fi + if [ "$stop_attempted" -eq 0 ]; then stage3_inert_stop "$inert_unit" >/dev/null 2>&1 || cleanup_rc=1; stop_attempted=1; fi + else + cleanup_rc=1 + break + fi + else + cleanup_rc=1 + break + fi + /usr/bin/sleep 0.05 + done + [ "$collected" -eq 1 ] || cleanup_rc=1 + stage3_cleanup_probe_exact "$inert_probe" "$inert_probe_uid" "$inert_probe_gid" "$inert_probe_mode" || cleanup_rc=1 + [ "$original_rc" -ne 0 ] || original_rc=$cleanup_rc + exit "$original_rc" +} + +stage3_inert_main() { + local compact_id show load run_rc=1 attempt show_rc capture_rc pending_rc + [ "$#" -eq 0 ] + [ "$(/usr/bin/id -u)" = 0 ] + [ "${PHASE2B_STAGE3_LOCK_HELD:-0}" = 1 ] + stage3_acquire_lock + stage3_read_marker + [ "${stage3_marker_fields[STATUS]}" = ACTIVE ] + inert_activation=${stage3_marker_fields[ACTIVATION_ID]} + compact_id=${inert_activation//-/} + inert_token=$(< /proc/sys/kernel/random/uuid) + inert_token=${inert_token//-/} + [[ "$inert_token" =~ ^[0-9a-f]{32}$ ]] + inert_unit="${inert_unit_prefix}${compact_id}.service" + trap stage3_inert_cleanup EXIT + stage3_inert_arm_signals + + [ "$(/usr/bin/systemctl show user@1200.service --property=ActiveState --value)" = inactive ] + [ "$(/usr/bin/systemctl show user@1200.service --property=SubState --value)" = dead ] + [ ! -e "$inert_probe" ] && [ ! -L "$inert_probe" ] + [ -x "$touch_wrapper" ] && [ -x "$rm_wrapper" ] + show=$(/usr/bin/systemctl show --no-pager "$inert_unit" --property=LoadState --property=ActiveState --property=FragmentPath) + stage3_inert_unit_unclaimed "$show" + + stage3_inert_defer_signals + /usr/bin/systemd-run --service-type=exec --wait --pipe --collect --unit="$inert_unit" --description="Phase 2B Stage 3 inert containment $inert_activation" \ + --uid=1200 --gid=1200 \ + --setenv=DOCKER_HOST=unix:///run/user/1200/docker.sock \ + --setenv="PHASE2B_STAGE3_INERT_TOKEN=$inert_token" \ + --property='UnsetEnvironment=ALL_PROXY HTTPS_PROXY HTTP_PROXY NO_PROXY all_proxy https_proxy http_proxy no_proxy' \ + --property=RuntimeMaxSec=120s --property=TimeoutStopSec=15s --property=KillMode=control-group --property=UMask=0022 \ + --property=CPUQuota=600% --property=MemoryHigh=48G --property=MemoryMax=64G --property=MemorySwapMax=16G --property=TasksMax=4096 \ + --property=NetworkNamespacePath=/run/netns/le-app-codex \ + --property=ProtectSystem=strict \ + --property=PrivateTmp=yes \ + --property=ProtectHome=yes \ + --property=ProtectProc=invisible \ + --property=ProcSubset=all \ + --property=BindReadOnlyPaths=/etc/le-app-codex-runtime/resolv.conf:/etc/resolv.conf \ + --property=TemporaryFileSystem=/srv:ro \ + --property=BindPaths=/srv/le-app-codex:/srv/le-app-codex \ + --property=TemporaryFileSystem=/var:ro \ + --property=TemporaryFileSystem=/mnt:ro \ + --property=TemporaryFileSystem=/media:ro \ + --property=TemporaryFileSystem=/opt:ro \ + --property=InaccessiblePaths=/var/www \ + --property=InaccessiblePaths=/root \ + --property=InaccessiblePaths=/home \ + --property=InaccessiblePaths=/boot \ + --property=InaccessiblePaths=/efi \ + --property=InaccessiblePaths=/run/docker.sock \ + --property=InaccessiblePaths=/var/run/docker.sock \ + --property=InaccessiblePaths=/run/containerd/containerd.sock \ + --property=InaccessiblePaths=/run/podman \ + --property=InaccessiblePaths=/run/dbus/system_bus_socket \ + --property=InaccessiblePaths=/run/systemd/private \ + --property=InaccessiblePaths=/var/lib/docker \ + --property=InaccessiblePaths=/var/lib/containerd \ + --property=InaccessiblePaths=/var/lib/containers \ + --property=InaccessiblePaths=/etc/docker \ + --property=InaccessiblePaths=/etc/containers \ + --property=InaccessiblePaths=/etc/ssh \ + --property=InaccessiblePaths=/etc/NetworkManager/system-connections \ + --property=InaccessiblePaths=/etc/wireguard \ + --property=DevicePolicy=closed \ + --property='DeviceAllow=/dev/fuse rw' \ + --property='DeviceAllow=/dev/net/tun rw' \ + --property="BindReadOnlyPaths=$touch_wrapper:/usr/bin/touch" \ + --property="BindReadOnlyPaths=$rm_wrapper:/usr/bin/rm" \ + /srv/le-app-codex/phase2b-tests/10-inert-containment.sh & + inert_client_pid=$! + inert_client_started=1 + if inert_client_starttime=$(stage3_inert_pid_starttime "$inert_client_pid"); then capture_rc=0; else capture_rc=$?; fi + stage3_inert_arm_signals + if stage3_inert_replay_pending_signal; then pending_rc=0; else pending_rc=$?; fi + [ "$pending_rc" -eq 0 ] || exit "$pending_rc" + [ "$capture_rc" -eq 0 ] + + for attempt in {1..100}; do + show=$(stage3_inert_show "$inert_unit" 2>/dev/null) || show= + load=$(stage3_show_value "$show" LoadState 2>/dev/null || true) + if [ "$load" = loaded ] && stage3_inert_unit_identity "$show" "$inert_unit" '' "$inert_activation" "$inert_token"; then + inert_invocation=$(stage3_show_value "$show" InvocationID) + inert_owned=1 + break + fi + /usr/bin/kill -0 "$inert_client_pid" 2>/dev/null || break + /usr/bin/sleep 0.05 + done + stage3_inert_defer_signals + if wait "$inert_client_pid"; then run_rc=0; else run_rc=$?; fi + stage3_inert_arm_signals + if stage3_inert_replay_pending_signal; then pending_rc=0; else pending_rc=$?; fi + [ "$pending_rc" -eq 0 ] || exit "$pending_rc" + inert_client_pid= + inert_client_starttime= + [ "$run_rc" -eq 0 ] + [ "$inert_owned" -eq 1 ] && [[ "$inert_invocation" =~ ^[0-9a-f]{32}$ ]] + stage3_cleanup_probe_exact "$inert_probe" "$inert_probe_uid" "$inert_probe_gid" "$inert_probe_mode" + for attempt in {1..100}; do + if load=$(/usr/bin/systemctl show "$inert_unit" --property=LoadState --value 2>/dev/null); then show_rc=0; else show_rc=$?; fi + [ "$show_rc" -eq 0 ] || break + [ "$load" = not-found ] && break + /usr/bin/sleep 0.05 + done + [ "$show_rc" -eq 0 ] && [ "$load" = not-found ] + [ "$(/usr/bin/systemctl show user@1200.service --property=ActiveState --value)" = inactive ] + trap - EXIT HUP INT TERM +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then stage3_inert_main "$@"; fi diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/53-stage3-static-tests.sh b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/53-stage3-static-tests.sh new file mode 100644 index 0000000..9b0cbfa --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/53-stage3-static-tests.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +set -o pipefail +export LC_ALL=C + +failures=0 +stage3_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." 2>/dev/null && pwd) +migration_root=$(CDPATH= cd -- "$stage3_root/.." 2>/dev/null && pwd) +stage1_root=$migration_root/phase2b-contained-runtime +stage2_root=$migration_root/phase2b-stage2-verification +activate=$stage3_root/tests/50-activate-stage3.sh +rollback=$stage3_root/tests/51-rollback-stage3.sh +common=$stage3_root/tests/stage3-common.sh +runner=$stage3_root/tests/52-run-stage1-inert.sh +failure_suite=$stage3_root/tests/54-stage3-failure-path-tests.sh +installed_inert=$stage1_root/tests/10-inert-containment.sh + +pass() { printf 'PASS: %s\n' "$1"; } +fail() { printf 'FAIL: %s\n' "$1" >&2; failures=$((failures + 1)); } + +emit_array_lines() { + local file=$1 name=$2 + /usr/bin/awk -v name="$name" ' + $0 ~ "^[[:space:]]*" name "=\\($" { inside=1; next } + inside && $0 ~ "^[[:space:]]*\\)$" { exit } + inside { sub(/^[[:space:]]+/, ""); print } + ' "$file" +} + +verify_working_manifest_coverage() { + local line path file relative records=0 files=0 + declare -A listed=() + while IFS= read -r line || [ -n "$line" ]; do + if [[ "$line" =~ ^[0-9a-f]{64}\ \ (.+)$ ]]; then path=${BASH_REMATCH[1]}; else fail 'Stage 3 malformed manifest record'; continue; fi + if [[ "$path" = /* ]] || [[ "$path" = *'|'* ]] || [[ "$path" = ../* ]] || [[ "$path" = */../* ]] || [[ "$path" = */.. ]] || [ "$path" = . ] || [ "$path" = .. ] || [ "$path" = MANIFEST.sha256 ] || [ -n "${listed[$path]:-}" ]; then fail "Stage 3 unsafe or duplicate manifest record $path"; else listed["$path"]=1; records=$((records + 1)); fi + done < "$stage3_root/MANIFEST.sha256" + while IFS= read -r -d '' file; do + relative=${file#"$stage3_root"/} + [ "$relative" = MANIFEST.sha256 ] && continue + files=$((files + 1)) + [ -n "${listed[$relative]:-}" ] || fail "Stage 3 manifest omits $relative" + done < <(/usr/bin/find "$stage3_root" -type f -print0) + if /usr/bin/find "$stage3_root" -type l -print -quit | /usr/bin/grep -q .; then fail 'Stage 3 package contains a symlink'; fi + if [ "$records" -eq "$files" ] && [ "$records" -gt 0 ]; then pass "Stage 3 manifest covers $records working files"; else fail "Stage 3 manifest coverage records=$records files=$files"; fi +} + +syntax_failures=0 +for file in "$activate" "$rollback" "$common" "$runner" "$stage3_root/tests/53-stage3-static-tests.sh" "$failure_suite"; do /bin/bash -n "$file" || syntax_failures=$((syntax_failures + 1)); done +for file in "$stage3_root"/tests/inert-bin/*; do /bin/sh -n "$file" || syntax_failures=$((syntax_failures + 1)); done +[ "$syntax_failures" -eq 0 ] && pass 'Stage 3 shell syntax' || fail 'Stage 3 shell syntax' +[ -x "$stage3_root/tests/inert-bin/touch" ] && [ -x "$stage3_root/tests/inert-bin/rm" ] && pass 'inert probe wrappers executable' || fail 'inert probe wrapper modes' +(cd "$stage1_root" && /usr/bin/sha256sum -c MANIFEST.sha256 >/dev/null) && pass 'unchanged Stage 1 manifest' || fail 'Stage 1 manifest' +(cd "$stage2_root" && /usr/bin/sha256sum -c MANIFEST.sha256 >/dev/null) && pass 'unchanged Stage 2 manifest' || fail 'Stage 2 manifest' +(cd "$stage3_root" && /usr/bin/sha256sum -c MANIFEST.sha256 >/dev/null) && pass 'Stage 3 manifest' || fail 'Stage 3 manifest' +verify_working_manifest_coverage + +repo_root=$(/usr/bin/git -C "$stage3_root" rev-parse --show-toplevel 2>/dev/null) +stage1_relative=${stage1_root#"$repo_root"/} +stage2_relative=${stage2_root#"$repo_root"/} +if [ -z "$(/usr/bin/git -C "$repo_root" status --porcelain --untracked-files=all -- "$stage1_relative" "$stage2_relative")" ] && /usr/bin/git -C "$repo_root" diff --quiet HEAD -- "$stage1_relative" "$stage2_relative"; then pass 'Stage 1 and Stage 2 trees byte-for-byte unchanged'; else fail 'Stage 1 or Stage 2 tree changed'; fi +[ "$(/usr/bin/sha256sum "$stage1_root/MANIFEST.sha256" | /usr/bin/cut -d' ' -f1)" = 5aaa91b1e6846eda033961dcee392b9657476ad6fd149420979e9309b484445f ] && [ "$(/usr/bin/sha256sum "$stage2_root/MANIFEST.sha256" | /usr/bin/cut -d' ' -f1)" = 4ce1de182dd6fda902d910c1d6f3f169dce55266e61f9abf2e57cd119fad600d ] && pass 'Stage 1/2 provenance digests pinned' || fail 'Stage 1/2 provenance digest drift' + +stage2_operator=$stage2_root/tests/40-verify-stage2.sh +if /usr/bin/diff -u <(emit_array_lines "$stage2_operator" sources) <(emit_array_lines "$common" sources) >/dev/null && /usr/bin/diff -u <(emit_array_lines "$stage2_operator" destinations) <(emit_array_lines "$common" destinations) >/dev/null; then pass 'Stage 3 16-file map exactly matches Stage 2'; else fail 'Stage 3 16-file map differs from Stage 2'; fi + +start_count=$(/usr/bin/grep -Fc '/usr/bin/systemctl start le-app-codex-netns.service' "$activate") +stop_count=$(/usr/bin/grep -Fc '/usr/bin/systemctl stop le-app-codex-netns.service' "$rollback") +nft_check_count=$(/usr/bin/grep -Fc '/usr/bin/nft --check -f "$installed_policy"' "$activate") +[ "$start_count" -eq 1 ] && [ "$stop_count" -eq 1 ] && [ "$nft_check_count" -eq 1 ] && pass 'exact activation/rollback/nft-check command counts' || fail "command counts start=$start_count stop=$stop_count nft-check=$nft_check_count" +[ "$(/usr/bin/grep -hFo '/usr/bin/systemctl start ' "$activate" "$rollback" "$common" "$runner" | /usr/bin/wc -l)" -eq 1 ] && [ "$(/usr/bin/grep -hFo '/usr/bin/systemctl stop ' "$activate" "$rollback" "$common" "$runner" | /usr/bin/wc -l)" -eq 2 ] && [ "$(/usr/bin/grep -Fc 'stage3_inert_stop() { /usr/bin/systemctl stop "$1"; }' "$runner")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_inert_stop "$inert_unit"' "$runner")" -eq 1 ] && pass 'only namespace activation plus token-verified transient cleanup can start/stop units' || fail 'unexpected systemctl start/stop surface' +systemctl_verbs=$(/usr/bin/grep -hEo '/usr/bin/systemctl[[:space:]]+[[:alnum:]-]+' "$activate" "$rollback" "$common" "$runner" | /usr/bin/awk '{ print $2 }' | /usr/bin/sort -u) +[ "$systemctl_verbs" = $'cat\nis-enabled\nlist-units\nshow\nstart\nstop' ] && pass 'systemctl verb surface is a closed read/start/verified-stop allowlist' || fail "unexpected systemctl verb surface: $systemctl_verbs" +if /usr/bin/grep -Eq '^[[:space:]]*/usr/bin/systemctl[[:space:]]+(enable|disable|restart|reenable|preset|mask|unmask)|^[[:space:]]*/usr/bin/systemctl[[:space:]]+start[[:space:]]+user@1200|^[[:space:]]*/usr/bin/loginctl|^[[:space:]]*/usr/bin/nft[[:space:]]+-f|^[[:space:]]*/usr/local/libexec/le-app-codex-netns[[:space:]]+down|^[[:space:]]*/usr/bin/(docker|codex)[[:space:]]+(run|start|login|auth)|git[[:space:]]+clone|^[[:space:]]*/usr/bin/(cp|scp|rsync)[[:space:]]|/srv/[^[:space:]]*secret' "$activate" "$rollback" "$common" "$runner"; then fail 'prohibited activation, service, copy, or secret command literal'; else pass 'no prohibited activation, service, copy, or secret command literal'; fi +if /usr/bin/grep -Eq '/usr/bin/nft[[:space:]]+--numeric|/usr/bin/nft[[:space:]]+-f|/usr/bin/systemctl[[:space:]]+start[[:space:]]+(user@1200|docker|codex)|/usr/bin/systemctl[[:space:]]+enable' "$activate" "$rollback" "$common" "$runner"; then fail 'numeric/load or prohibited service command literal'; else pass 'no numeric nft output, nft load, or prohibited service command literal'; fi +[ "$(/usr/bin/grep -hFo '/usr/bin/nft delete ' "$activate" "$rollback" "$common" "$runner" | /usr/bin/wc -l)" -eq 2 ] && [ "$(/usr/bin/grep -hFo '/usr/bin/ip link delete ' "$activate" "$rollback" "$common" "$runner" | /usr/bin/wc -l)" -eq 1 ] && [ "$(/usr/bin/grep -hFo '/usr/bin/ip netns delete ' "$activate" "$rollback" "$common" "$runner" | /usr/bin/wc -l)" -eq 1 ] && [ "$(/usr/bin/grep -hE '/usr/bin/ip.*[[:space:]]delete[[:space:]]' "$activate" "$rollback" "$common" "$runner" | /usr/bin/wc -l)" -eq 2 ] && ! /usr/bin/grep -Eq '/usr/bin/nft[[:space:]]+([^#;|]*[[:space:]])?(add|insert|replace|flush|reset|create|destroy|rename|import|monitor)[[:space:]]' "$activate" "$rollback" "$common" "$runner" && ! /usr/bin/grep -Eq '/usr/bin/ip[[:space:]]+([^#;|]*[[:space:]])?(add|set|replace|flush)[[:space:]]' "$activate" "$rollback" "$common" "$runner" && pass 'nft/ip mutations are closed to two exact table deletes, one veth delete, and one namespace delete' || fail 'unexpected nft/ip mutation surface' +[ "$(/usr/bin/grep -Fc 'stage3_nft_json_inet_rules_match "$raw"' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_nft_json_nat_rules_match "$raw"' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_project_inet_snapshot_valid "$raw" "$text"' "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_project_nat_snapshot_valid "$raw" "$text"' "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_capture_exact_project_policy_snapshot' "$activate")" -eq 4 ] && ! /usr/bin/grep -q 'stage3_nft_text_chain' "$activate" "$rollback" "$common" && pass 'activation and rollback share exact canonical JSON policy validation' || fail 'nft JSON validation surface' +[ "$(/usr/bin/grep -Fc '/usr/bin/nft delete table inet le_app_codex' "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '/usr/bin/nft delete table ip le_app_codex_nat' "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '/usr/bin/ip link delete lecodex-host' "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '/usr/bin/ip netns delete le-app-codex' "$rollback")" -eq 1 ] && pass 'rollback direct cleanup limited to exact project names' || fail 'rollback direct cleanup literals' +if [ "$(/usr/bin/grep -Fc '/usr/bin/docker --config "$stage3_root/tests/docker-config" --host unix:///run/docker.sock "$@"' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '/usr/bin/docker' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_rootful_docker ' "$common")" -eq 4 ] && [ "$(/usr/bin/grep -Fc 'stage3_rootful_docker network ls --no-trunc -q' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_rootful_docker network inspect --format' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_rootful_docker ps --no-trunc -aq' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_rootful_docker inspect --format' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc -- '-u DOCKER_API_VERSION -u DOCKER_CERT_PATH -u DOCKER_CONTEXT -u DOCKER_HOST -u DOCKER_TLS -u DOCKER_TLS_VERIFY' "$common")" -eq 1 ] && /usr/bin/jq -e 'type == "object" and length == 0' "$stage3_root/tests/docker-config/config.json" >/dev/null; then pass 'all rootful Docker reads use one context-scrubbed explicit-socket read allowlist and empty config'; else fail 'rootful Docker helper/socket/config surface'; fi +[ "$(/usr/bin/grep -Fc '/usr/bin/systemd-run --service-type=exec --wait --pipe --collect --unit="$inert_unit"' "$runner")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '/srv/le-app-codex/phase2b-tests/10-inert-containment.sh' "$runner")" -eq 1 ] && [ "$(/usr/bin/grep -Fc -- '--property=RuntimeMaxSec=120s' "$runner")" -eq 1 ] && pass 'finite collected wrapper executes existing installed inert test' || fail 'inert test wrapper command surface' +[ "$(/usr/bin/grep -Fc -- "--property='UnsetEnvironment=ALL_PROXY HTTPS_PROXY HTTP_PROXY NO_PROXY all_proxy https_proxy http_proxy no_proxy'" "$runner")" -eq 1 ] && ! /usr/bin/grep -Eq -- '--setenv=(ALL_PROXY|HTTPS_PROXY|HTTP_PROXY|NO_PROXY|all_proxy|https_proxy|http_proxy|no_proxy)' "$runner" && pass 'transient inert test removes every proxy variable' || fail 'transient inert proxy environment' +[ "$(/usr/bin/grep -Fc -- '--setenv="PHASE2B_STAGE3_INERT_TOKEN=$inert_token"' "$runner")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '[ "$inert_owned" -eq 1 ] && [[ "$inert_invocation" =~ ^[0-9a-f]{32}$ ]]' "$runner")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_inert_unit_identity "$show" "$inert_unit" "${inert_invocation:-}" "$inert_activation" "$inert_token"' "$runner")" -eq 1 ] && pass 'transient success and cleanup require token-bound captured ownership' || fail 'transient ownership/token gate' +if /usr/bin/awk ' + /^ stage3_inert_defer_signals$/ && state == 0 { state=1; next } + state == 1 && /systemd-run --service-type=exec/ { state=2; next } + state == 2 && /^ inert_client_pid=\$!$/ { state=3; next } + state == 3 && /^ inert_client_started=1$/ { state=4; next } + state == 4 && /inert_client_starttime=.*stage3_inert_pid_starttime/ { state=5; next } + state == 5 && /^ stage3_inert_arm_signals$/ { state=6; next } + state == 6 && /stage3_inert_replay_pending_signal/ { state=7 } + END { exit state != 7 } +' "$runner" && [ "$(/usr/bin/grep -Fc 'if stage3_inert_pid_identity_matches "$inert_client_pid" "$inert_client_starttime"; then' "$runner")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '[ "$not_found_streak" -ge 3 ]' "$runner")" -eq 1 ]; then pass 'inert launch defers signals through token/PID ownership and cleanup requires PID identity plus stable collection'; else fail 'inert launch/cleanup race hardening surface'; fi +[ "$(/usr/bin/grep -Ec '/usr/bin/ip netns exec le-app-codex .* /usr/bin/curl ' "$activate")" -eq 4 ] && ! /usr/bin/grep -E '/usr/bin/ip netns exec le-app-codex .* /usr/bin/curl ' "$activate" | /usr/bin/grep -Fv '"${proxy_free_env[@]}" /usr/bin/curl --noproxy' >/dev/null && [ "$(/usr/bin/grep -Fc -- '-u ALL_PROXY -u HTTPS_PROXY -u HTTP_PROXY -u NO_PROXY -u all_proxy -u https_proxy -u http_proxy -u no_proxy /usr/bin/curl --noproxy' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'expect_denied curl' "$installed_inert")" -ge 1 ] && pass 'all namespace/public-IP curl probes have proxy removal' || fail 'curl proxy-removal coverage' +[ "$(/usr/bin/grep -Fc "printf 'ACTIVATION_ID|%s" "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "printf 'MACHINE_ID_SHA256|%s" "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "printf 'APPROVED_DNS4|%s" "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "printf 'APPROVED_ARTIFACT_MANIFEST_SHA256|%s" "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "printf 'STAGE1_SOURCE_COMMIT|%s" "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "printf 'STAGE2_VERIFICATION_COMMIT|%s" "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "printf 'TIMESTAMP|%s" "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "printf 'HOSTNAME|%s" "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "printf 'NFT_INET_SHA256|%s" "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "printf 'NFT_NAT_SHA256|%s" "$common")" -eq 1 ] && pass 'required marker provenance and per-table fields emitted once' || fail 'required marker provenance fields' +[ "$(/usr/bin/grep -Fc 'stage3_atomic_publish_new "$marker" "$rollback_record"' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '/usr/bin/rm -- "$marker"' "$rollback")" -eq 0 ] && pass 'rollback atomically retains activation provenance' || fail 'rollback tombstone transition' +[ "$(/usr/bin/grep -Fc '[[ "$line" =~ ^([^|]+)\|([^|]+)$ ]]' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '[ "$parsed_size" -eq "$record_size" ]' "$common")" -eq 1 ] && ! /usr/bin/grep -Fq "IFS='|' read -r key value extra" "$common" && pass 'record parser requires canonical one-delimiter newline-complete bytes' || fail 'record parser exact-byte surface' +[ "$(/usr/bin/grep -n '/usr/bin/sync -f "$temporary"' "$rollback" | /usr/bin/cut -d: -f1)" -lt "$(/usr/bin/grep -n '/usr/bin/mv -T -- "$temporary" "$rollback_record"' "$rollback" | /usr/bin/cut -d: -f1)" ] && [ "$(/usr/bin/grep -Fc 'stage3_validate_safe_temporary "$temporary" "$rollback_record" "$expected_uid" "$expected_gid" "$require_provenance"' "$rollback")" -eq 2 ] && pass 'recovered ACTIVE temporary is fsynced and revalidated before tombstone replacement' || fail 'ACTIVE temporary durability/revalidation order' +[ "$(/usr/bin/grep -n '/usr/bin/sync -f "$marker_temporary"' "$rollback" | /usr/bin/cut -d: -f1)" -lt "$(/usr/bin/grep -n 'stage3_atomic_publish_new "$marker_temporary" "$rollback_record"' "$rollback" | /usr/bin/cut -d: -f1)" ] && pass 'temp-only complete record is fsynced before retained-authority publication' || fail 'temp-only authority durability order' +[ "$(/usr/bin/grep -Fc 'stage3_validate_safe_temporary "$marker_temporary" "$authority_path"' "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_record_common_sha256' "$rollback")" -ge 2 ] && pass 'temporary publication state is provenance-compared before mutation' || fail 'temporary record validation surface' +[ "$(/usr/bin/grep -Fc 'PHASE2B_STAGE3_LOCK_ID' "$activate")" -ge 2 ] && [ "$(/usr/bin/grep -Fc 'stage3_validate_inherited_lock 9 /etc/le-app-codex-runtime' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_lock_descriptor_matches 9' "$common")" -eq 2 ] && [ "$(/usr/bin/grep -Fc 'stage3_acquire_lock' "$runner")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_validate_inherited_lock 9 "$root"' "$failure_suite")" -eq 2 ] && pass 'activation, automatic rollback, inert child, and tests validate one inherited lock/token' || fail 'inherited lock validation surface' +[ "$(/usr/bin/grep -Fc 'stage3_verify_loaded_units inactive' "$activate")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'systemctl cat --no-pager user@1200.service' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_write_marker PREPARED' "$activate")" -eq 1 ] && [ "$(/usr/bin/grep -n 'stage3_verify_loaded_units inactive' "$activate" | /usr/bin/cut -d: -f1)" -lt "$(/usr/bin/grep -n 'stage3_write_marker PREPARED' "$activate" | /usr/bin/cut -d: -f1)" ] && pass 'loaded-unit fragment/drop-in/property gate is final before marker/start' || fail 'loaded-unit validation placement or surface' +[ "$(/usr/bin/grep -Fc 'stage3_capture_active_topology_identity' "$activate")" -eq 2 ] && [ "$(/usr/bin/grep -Fc 'stage3_read_exact_project_policy_snapshot' "$common")" -eq 3 ] && [ "$(/usr/bin/grep -Fc 'stage3_project_nft_canonical_from_tables "$inet_canonical" "$nat_canonical"' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_project_nft_sha256=$stage3_snapshot_project_sha256' "$activate")" -eq 1 ] && /usr/bin/grep -Fq '[ "$host_ifindex" = "$peer_iflink" ] && [ "$host_iflink" = "$peer_ifindex" ]' "$common" && pass 'active topology is reciprocal and project provenance comes from two matching exact table snapshots' || fail 'active topology/stable nft snapshot surface' +[ "$(/usr/bin/grep -Fc 'type == "array" and all(.[];' "$common")" -ge 5 ] && [ "$(/usr/bin/grep -Fc 'error("invalid link inventory")' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'error("invalid nft table inventory")' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '.ifname != "lecodex-host" and .ifname != "lecodex-ns"' "$common")" -eq 1 ] && pass 'typed inventories reject wrong shapes and both project peers are baseline-excluded' || fail 'typed inventory/baseline exclusion surface' +[ "$(/usr/bin/grep -Fc 'stage3_resource_presence_shape_valid' "$rollback")" -eq 2 ] && [ "$(/usr/bin/grep -Fc 'stage3_resource_presence_shape_valid' "$failure_suite")" -ge 12 ] && [ "$(/usr/bin/grep -Fc 'stage3_prepared_live_invocation_valid' "$rollback")" -eq 2 ] && [ "$(/usr/bin/grep -Fc 'case_prepared_tombstone_retry_without_live_invocation' "$failure_suite")" -eq 2 ] && pass 'ordered partial survivors and PREPARED tombstone retry attribution are enforced and tested' || fail 'partial survivor/retry surface' +[ "$(/usr/bin/grep -Fc 'stage3_materialize_docker_ipv4_gateways' "$activate")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '"$raw_count" -eq 39' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '"$unique_count" -eq 39' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -n 'stage3_materialize_docker_ipv4_gateways "$current_docker"' "$activate" | /usr/bin/cut -d: -f1)" -lt "$(/usr/bin/grep -n "stage3_expect_blocked_counter 'non-Quad9 UDP DNS'" "$activate" | /usr/bin/cut -d: -f1)" ] && pass 'all 39 unique Docker gateways are required before any denial probe' || fail 'Docker gateway materialization surface/order' +[ "$(/usr/bin/grep -Fc '/usr/bin/ps -eo comm=' "$common")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '$1 == "codex"' "$common")" -eq 1 ] && pass 'global Codex-name inventory complements complete UID-1200 process absence' || fail 'Codex process inventory surface' +[ "$(/usr/bin/grep -Fc "stage3_compare_recorded_baseline 'pre-rollback'" "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "stage3_compare_recorded_baseline 'post-rollback'" "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "stage3_compare_recorded_baseline 'changed-boot pre-rollback'" "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc "stage3_compare_recorded_baseline 'changed-boot post-rollback'" "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_compare_rollback_baseline' "$rollback")" -ge 2 ] && pass 'recorded and immediate baselines gate same-boot and changed-boot rollback/recovery' || fail 'rollback baseline comparison surface' +[ "$(/usr/bin/grep -Fc 'stage3_capture_resource_inventory || return 1' "$rollback")" -ge 4 ] && [ "$(/usr/bin/grep -Fc 'stage3_table_identity "$family" "$name" || return 1' "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'stage3_veth_identity || return 1' "$rollback")" -ge 2 ] && [ "$(/usr/bin/grep -Fc 'stage3_namespace_identity || return 1' "$rollback")" -ge 2 ] && pass 'every direct cleanup path performs fresh inventory and identity checks' || fail 'direct cleanup revalidation surface' +[ "$(/usr/bin/grep -Fc 'if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then stage3_activation_main "$@"; fi' "$activate")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then stage3_rollback_main "$@"; fi' "$rollback")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then stage3_inert_main "$@"; fi' "$runner")" -eq 1 ] && pass 'operational entry points are source-guarded' || fail 'operational source guard' +[ "$(/usr/bin/grep -Fc '/srv/le-app-codex/tmp/phase2b-write-test' "$stage3_root/tests/inert-bin/touch")" -eq 1 ] && [ "$(/usr/bin/grep -Fc '/srv/le-app-codex/tmp/phase2b-write-test' "$stage3_root/tests/inert-bin/rm")" -eq 1 ] && pass 'inert probe wrappers accept only legacy exact path' || fail 'inert probe wrapper path gate' +[ "$(/usr/bin/grep -Fc 'sudo /bin/bash docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/50-activate-stage3.sh' "$stage3_root/README.md")" -eq 1 ] && [ "$(/usr/bin/grep -Fc 'sudo /bin/bash docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/51-rollback-stage3.sh' "$stage3_root/README.md")" -eq 1 ] && /usr/bin/grep -Fq 'Only after this package has been reviewed, committed, and checked out as a clean worktree on TITANSERVER' "$stage3_root/README.md" && pass 'future activation and rollback invocations are exact and review-gated' || fail 'future operator invocation documentation' + +failures_before=$failures +source "$common" +committed_networks=$(stage3_committed_docker_network_canonical) +printf '%s' "$committed_networks" | /usr/bin/jq -e 'length == 41' >/dev/null && pass 'committed Docker canonicalizer yields 41 networks' || fail 'committed Docker canonicalizer' +[ "$failures" -eq "$failures_before" ] || fail 'common library source has side effects' + +/bin/bash "$stage1_root/tests/32-stage1-state-tests.sh" >/dev/null && pass 'existing non-mutating Stage 1 state tests' || fail 'Stage 1 state tests' +/bin/bash "$stage2_root/tests/41-stage2-static-tests.sh" >/dev/null && pass 'existing non-mutating Stage 2 static tests' || fail 'Stage 2 static tests' +/bin/bash "$failure_suite" >/dev/null && pass 'Stage 3 non-mutating failure-path suite' || fail 'Stage 3 failure-path suite' + +printf 'Phase 2B Stage 3 static tests failures=%s\n' "$failures" +[ "$failures" -eq 0 ] diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/54-stage3-failure-path-tests.sh b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/54-stage3-failure-path-tests.sh new file mode 100644 index 0000000..75b3ff1 --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/54-stage3-failure-path-tests.sh @@ -0,0 +1,632 @@ +#!/usr/bin/env bash + +set -o pipefail +export LC_ALL=C + +failures=0 +stage3_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." 2>/dev/null && pwd) +source "$stage3_root/tests/stage3-common.sh" +source "$stage3_root/tests/51-rollback-stage3.sh" + +pass() { printf 'PASS: %s\n' "$1"; } +fail() { printf 'FAIL: %s\n' "$1" >&2; failures=$((failures + 1)); } +expect_success() { local label=$1; shift; if ( "$@" ); then pass "$label"; else fail "$label"; fi; } +expect_failure() { local label=$1; shift; if ( "$@" ); then fail "$label"; else pass "$label"; fi; } + +fixture=$(/usr/bin/mktemp -d /tmp/le-app-codex-stage3-fixture.XXXXXX) +case "$fixture" in /tmp/le-app-codex-stage3-fixture.*) ;; *) printf 'unsafe fixture root\n' >&2; exit 1 ;; esac +trap '/usr/bin/rm -rf -- "$fixture"' EXIT HUP INT TERM +/usr/bin/chmod 0755 "$fixture" + +write_prepared_record() { + local target=$1 + stage3_activation_id=11111111-2222-4333-8444-555555555555 + stage3_machine_id_sha256=$(printf 'a%.0s' {1..64}) + stage3_head=$(printf 'b%.0s' {1..40}) + stage3_timestamp=2026-07-13T12:00:00Z + stage3_hostname=fixture-host + stage3_boot_id=aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee + baseline_public_ipv4=$expected_public_ipv4 + baseline_nft_sha256=$(printf 'c%.0s' {1..64}) + baseline_docker_network_sha256=$(printf 'd%.0s' {1..64}) + baseline_host_network_sha256=$(printf 'e%.0s' {1..64}) + baseline_docker_runtime_sha256=$(printf 'f%.0s' {1..64}) + stage3_render_record PREPARED > "$target" + /usr/bin/chmod 0600 "$target" +} + +write_active_record() { + local target=$1 + write_prepared_record "$target" + stage3_service_invocation_id=$(printf '1%.0s' {1..32}) + stage3_namespace_dev_inode=ab:12345 + stage3_host_veth_ifindex=101 + stage3_host_veth_iflink=102 + stage3_host_veth_mac=02:11:22:33:44:55 + stage3_nft_inet_handle=201 + stage3_nft_nat_handle=202 + stage3_nft_inet_sha256=$(printf '6%.0s' {1..64}) + stage3_nft_nat_sha256=$(printf '7%.0s' {1..64}) + stage3_project_nft_sha256=$(printf '8%.0s' {1..64}) + stage3_render_record ACTIVE > "$target" + /usr/bin/chmod 0600 "$target" +} + +emit_fixture_inert_show() { + local unit=$1 activation=$2 token=$3 invocation=$4 + printf 'LoadState=loaded\n' + printf 'ActiveState=active\n' + printf 'Transient=yes\n' + printf 'FragmentPath=/run/systemd/transient/%s\n' "$unit" + printf 'User=1200\nGroup=1200\n' + printf 'NetworkNamespacePath=/run/netns/le-app-codex\n' + printf 'Description=Phase 2B Stage 3 inert containment %s\n' "$activation" + printf 'Environment=DOCKER_HOST=unix:///run/user/1200/docker.sock PHASE2B_STAGE3_INERT_TOKEN=%s\n' "$token" + printf 'UnsetEnvironment=ALL_PROXY HTTPS_PROXY HTTP_PROXY NO_PROXY all_proxy https_proxy http_proxy no_proxy\n' + printf 'ExecStart={ path=/srv/le-app-codex/phase2b-tests/10-inert-containment.sh ; argv[]=/srv/le-app-codex/phase2b-tests/10-inert-containment.sh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\n' + printf 'InvocationID=%s\n' "$invocation" +} + +case_marker_publication() { + local root=$fixture/marker source target before_hash after_hash + source=$root/new; target=$root/marker + /usr/bin/mkdir "$root" + write_prepared_record "$source" + [ ! -e "$target" ] || return 1 + before_hash=$(/usr/bin/sha256sum "$source" | /usr/bin/cut -d' ' -f1) + stage3_atomic_publish_new "$source" "$target" || return 1 + [ ! -e "$source" ] && [ -f "$target" ] || return 1 + after_hash=$(/usr/bin/sha256sum "$target" | /usr/bin/cut -d' ' -f1) + [ "$before_hash" = "$after_hash" ] && stage3_parse_record_file "$target" 0 +} + +case_marker_no_clobber() { + local root=$fixture/no-clobber source target hash + source=$root/new; target=$root/marker + /usr/bin/mkdir "$root" + write_prepared_record "$source" + printf 'malformed\n' > "$target" + hash=$(/usr/bin/sha256sum "$target" | /usr/bin/cut -d' ' -f1) + ! stage3_atomic_publish_new "$source" "$target" && [ -f "$source" ] && [ "$hash" = "$(/usr/bin/sha256sum "$target" | /usr/bin/cut -d' ' -f1)" ] +} + +case_complete_temporary_becomes_durable_authority() { + local root=$fixture/temp-only uid gid hash + root=$fixture/temp-only; /usr/bin/mkdir "$root" + uid=$(/usr/bin/id -u); gid=$(/usr/bin/id -g) + marker=$root/marker; rollback_record=$root/tomb; marker_temporary=$root/new + write_prepared_record "$marker_temporary" + hash=$(/usr/bin/sha256sum "$marker_temporary" | /usr/bin/cut -d' ' -f1) + stage3_read_marker() { return 1; } + stage3_read_rollback_record() { stage3_parse_record_file "$rollback_record" 0; } + stage3_resources_absent() { return 0; } + stage3_select_authoritative_record "$uid" "$gid" 0 || return 1 + [ "$record_location" = tombstone ] && [ ! -e "$marker_temporary" ] && [ "$hash" = "$(/usr/bin/sha256sum "$rollback_record" | /usr/bin/cut -d' ' -f1)" ] || return 1 + stage3_read_rollback_record && [ "${stage3_marker_fields[STATUS]}" = PREPARED ] +} + +case_interrupted_marker_temporary() { + local root=$fixture/interrupted-temp authority temporary uid gid + root=$fixture/interrupted-temp; authority=$root/marker; temporary=$root/new + /usr/bin/mkdir "$root"; uid=$(/usr/bin/id -u); gid=$(/usr/bin/id -g) + write_prepared_record "$authority" + marker=$authority + stage3_read_marker() { stage3_parse_record_file "$marker" 0; } + stage3_read_rollback_record() { return 1; } + stage3_parse_record_file "$authority" 0 || return 1 + : > "$temporary"; /usr/bin/chmod 0600 "$temporary" + stage3_validate_safe_temporary "$temporary" "$authority" "$uid" "$gid" 0 || return 1 + write_active_record "$temporary" + stage3_parse_record_file "$authority" 0 || return 1 + stage3_validate_safe_temporary "$temporary" "$authority" "$uid" "$gid" 0 || return 1 + /usr/bin/sed -i 's/TIMESTAMP|2026-07-13T12:00:00Z/TIMESTAMP|2026-07-13T12:00:01Z/' "$temporary" + stage3_parse_record_file "$authority" 0 || return 1 + ! stage3_validate_safe_temporary "$temporary" "$authority" "$uid" "$gid" 0 || return 1 + printf 'partial-nonzero' > "$temporary"; /usr/bin/chmod 0600 "$temporary" + stage3_parse_record_file "$authority" 0 || return 1 + ! stage3_validate_safe_temporary "$temporary" "$authority" "$uid" "$gid" 0 +} + +case_conflicting_active_temporary() { + local root=$fixture/conflicting-active authority temporary uid gid + root=$fixture/conflicting-active; authority=$root/tomb; temporary=$root/new + /usr/bin/mkdir "$root"; uid=$(/usr/bin/id -u); gid=$(/usr/bin/id -g) + write_active_record "$authority"; /usr/bin/cp "$authority" "$temporary"; /usr/bin/chmod 0600 "$temporary" + rollback_record=$authority + stage3_read_rollback_record() { stage3_parse_record_file "$rollback_record" 0; } + stage3_read_marker() { return 1; } + stage3_parse_record_file "$authority" 0 || return 1 + stage3_validate_safe_temporary "$temporary" "$authority" "$uid" "$gid" 0 || return 1 + /usr/bin/sed -i 's/NFT_INET_HANDLE|201/NFT_INET_HANDLE|999/' "$temporary" + stage3_parse_record_file "$authority" 0 || return 1 + ! stage3_validate_safe_temporary "$temporary" "$authority" "$uid" "$gid" 0 +} + +case_tombstone_publication() { + local root=$fixture/tomb marker_path tomb_path hash + marker_path=$root/marker; tomb_path=$root/tomb + /usr/bin/mkdir "$root" + write_prepared_record "$marker_path" + hash=$(/usr/bin/sha256sum "$marker_path" | /usr/bin/cut -d' ' -f1) + [ -f "$marker_path" ] && [ ! -e "$tomb_path" ] || return 1 + stage3_atomic_publish_new "$marker_path" "$tomb_path" || return 1 + [ ! -e "$marker_path" ] && [ -f "$tomb_path" ] && [ "$hash" = "$(/usr/bin/sha256sum "$tomb_path" | /usr/bin/cut -d' ' -f1)" ] || return 1 + ! stage3_atomic_publish_new "$marker_path" "$tomb_path" +} + +case_durable_active_temporary_upgrade() { + local root=$fixture/durable-upgrade uid gid active_hash + root=$fixture/durable-upgrade; /usr/bin/mkdir "$root" + marker=$root/marker; rollback_record=$root/tomb; marker_temporary=$root/new + uid=$(/usr/bin/id -u); gid=$(/usr/bin/id -g) + write_prepared_record "$rollback_record" + write_active_record "$marker_temporary" + active_hash=$(/usr/bin/sha256sum "$marker_temporary" | /usr/bin/cut -d' ' -f1) + stage3_read_marker() { return 1; } + stage3_read_rollback_record() { stage3_parse_record_file "$rollback_record" 0; } + stage3_read_rollback_record || return 1 + stage3_cleanup_safe_temporary "$marker_temporary" "$uid" "$gid" 0 || return 1 + [ ! -e "$marker_temporary" ] && [ "$active_hash" = "$(/usr/bin/sha256sum "$rollback_record" | /usr/bin/cut -d' ' -f1)" ] || return 1 + stage3_read_rollback_record && [ "${stage3_marker_fields[STATUS]}" = ACTIVE ] +} + +case_partial_activation_and_rollback_retry() { + local root=$fixture/partial-activation retained_hash + root=$fixture/partial-activation; /usr/bin/mkdir "$root" + marker=$root/marker; rollback_record=$root/tomb; marker_temporary=$root/new + write_prepared_record "$marker" + stage3_read_marker() { stage3_parse_record_file "$marker" 0; } + stage3_read_rollback_record() { stage3_parse_record_file "$rollback_record" 0; } + stage3_select_authoritative_record || return 1 + [ "$record_location" = marker ] || return 1 + stage3_resource_presence_shape_valid 0 0 0 0 0 0 || return 1 + stage3_resource_presence_shape_valid 1 0 0 0 0 0 || return 1 + stage3_resource_presence_shape_valid 1 1 1 0 0 0 || return 1 + stage3_resource_presence_shape_valid 1 1 0 1 0 0 || return 1 + stage3_resource_presence_shape_valid 1 1 0 1 1 1 || return 1 + ! stage3_resource_presence_shape_valid 0 1 1 0 0 0 || return 1 + ! stage3_resource_presence_shape_valid 1 1 1 1 0 0 || return 1 + ! stage3_resource_presence_shape_valid 1 0 0 0 1 0 || return 1 + declare -gA stage3_marker_fields=([STATUS]=PREPARED) + test_table_present=1 test_shape_ok=1 + stage3_inventory_has_table() { [ "$test_table_present" -eq 1 ]; } + stage3_prepared_table_shape() { [ "$test_shape_ok" -eq 1 ]; } + stage3_table_identity inet le_app_codex || return 1 + test_shape_ok=0 + ! stage3_table_identity inet le_app_codex || return 1 + test_table_present=0 + stage3_table_identity inet le_app_codex || return 1 + stage3_atomic_publish_new "$marker" "$rollback_record" || return 1 + retained_hash=$(/usr/bin/sha256sum "$rollback_record" | /usr/bin/cut -d' ' -f1) + stage3_resource_presence_shape_valid 1 0 0 0 0 0 || return 1 + record_location= + stage3_select_authoritative_record || return 1 + [ "$record_location" = tombstone ] && [ ! -e "$marker" ] && [ "$retained_hash" = "$(/usr/bin/sha256sum "$rollback_record" | /usr/bin/cut -d' ' -f1)" ] || return 1 + stage3_resource_presence_shape_valid 0 0 0 0 0 0 +} + +case_prepared_tombstone_retry_without_live_invocation() { + declare -gA stage3_marker_fields=([STATUS]=PREPARED) + record_location=marker + ! stage3_prepared_live_invocation_valid 1 '' || return 1 + stage3_prepared_live_invocation_valid 1 "$(printf 'a%.0s' {1..32})" || return 1 + record_location=tombstone + stage3_prepared_live_invocation_valid 1 '' || return 1 + stage3_prepared_live_invocation_valid 0 '' +} + +case_partial_rollback_retry_and_nft_drift() { + local exact_hash root=$fixture/partial-rollback retained_hash + root=$fixture/partial-rollback; /usr/bin/mkdir "$root" + marker=$root/marker; rollback_record=$root/tomb; marker_temporary=$root/new + write_active_record "$marker" + stage3_read_marker() { stage3_parse_record_file "$marker" 0; } + stage3_read_rollback_record() { stage3_parse_record_file "$rollback_record" 0; } + stage3_select_authoritative_record || return 1 + stage3_atomic_publish_new "$marker" "$rollback_record" || return 1 + retained_hash=$(/usr/bin/sha256sum "$rollback_record" | /usr/bin/cut -d' ' -f1) + exact_hash=$(printf 'inet' | /usr/bin/sha256sum | /usr/bin/cut -d' ' -f1) + declare -gA stage3_marker_fields=([STATUS]=ACTIVE [NFT_INET_HANDLE]=42 [NFT_INET_SHA256]="$exact_hash") + test_table_present=0 + stage3_inventory_has_table() { [ "$test_table_present" -eq 1 ]; } + stage3_inventory_table_handle() { printf '%s' "$test_handle"; } + stage3_nft_table_canonical() { printf '%s' "$test_table_body"; } + stage3_table_identity inet le_app_codex || return 1 + test_table_present=1 test_handle=42 test_table_body=inet + stage3_table_identity inet le_app_codex || return 1 + test_handle=43 + ! stage3_table_identity inet le_app_codex || return 1 + test_handle=42 test_table_body=drift + ! stage3_table_identity inet le_app_codex || return 1 + stage3_resource_presence_shape_valid 1 1 0 1 0 1 || return 1 + stage3_resource_presence_shape_valid 1 1 0 1 0 0 || return 1 + record_location= + stage3_select_authoritative_record || return 1 + [ "$record_location" = tombstone ] && [ "$retained_hash" = "$(/usr/bin/sha256sum "$rollback_record" | /usr/bin/cut -d' ' -f1)" ] || return 1 + stage3_resource_presence_shape_valid 1 0 0 0 0 0 || return 1 + stage3_resource_presence_shape_valid 0 0 0 0 0 0 +} + +case_inventory_failures() { + stage3_inventory_namespaces=stale stage3_inventory_links=stale stage3_inventory_tables=stale + stage3_query_namespaces() { return 1; } + ! stage3_capture_resource_inventory && [ -z "$stage3_inventory_namespaces$stage3_inventory_links$stage3_inventory_tables" ] || return 1 + stage3_query_namespaces() { printf '../escape\n'; } + ! stage3_capture_resource_inventory 2>/dev/null || return 1 + stage3_query_namespaces() { printf ''; } + stage3_query_links_json() { return 1; } + ! stage3_capture_resource_inventory 2>/dev/null || return 1 + stage3_query_links_json() { printf 'not-json'; } + ! stage3_capture_resource_inventory 2>/dev/null || return 1 + stage3_query_links_json() { printf '{}'; } + ! stage3_capture_resource_inventory 2>/dev/null || return 1 + stage3_query_links_json() { printf '[null]'; } + ! stage3_capture_resource_inventory 2>/dev/null || return 1 + stage3_query_links_json() { printf '[{}]'; } + ! stage3_capture_resource_inventory 2>/dev/null || return 1 + stage3_query_links_json() { printf '[{"ifname":"lo","ifindex":1}]'; } + stage3_query_tables_json() { return 1; } + ! stage3_capture_resource_inventory 2>/dev/null || return 1 + stage3_query_tables_json() { printf 'not-json'; } + ! stage3_capture_resource_inventory 2>/dev/null || return 1 + stage3_query_tables_json() { printf '{"nftables":{}}'; } + ! stage3_capture_resource_inventory 2>/dev/null || return 1 + stage3_query_tables_json() { printf '{"nftables":[{"error":"query failed"}]}'; } + ! stage3_capture_resource_inventory 2>/dev/null || return 1 + stage3_query_tables_json() { printf '{"nftables":[{"table":{"family":"inet","name":"x"}}]}'; } + ! stage3_capture_resource_inventory 2>/dev/null || return 1 + stage3_query_tables_json() { printf '{"nftables":[{"metainfo":{}},{"table":{"family":"inet","name":"x","handle":7}}]}'; } + stage3_capture_resource_inventory || return 1 + [ "$stage3_inventory_namespaces" = '' ] && [ "$stage3_inventory_links" = '[{"ifname":"lo","ifindex":1}]' ] && [ "$stage3_inventory_tables" = '[{"family":"inet","name":"x","handle":7}]' ] || return 1 + stage3_capture_resource_inventory() { return 1; } + stage3_resources_absent; [ $? -eq 2 ] +} + +case_host_baseline_project_peer_and_schema() { + local addresses routes rules canonical + addresses='[{"ifname":"enp4s0","ifindex":2,"addr_info":[]},{"ifname":"lecodex-host","ifindex":10,"addr_info":[{"family":"inet","local":"10.200.120.1"}]},{"ifname":"lecodex-ns","ifindex":11,"addr_info":[]}]' + canonical=$(stage3_host_addresses_json_canonical "$addresses") || return 1 + printf '%s' "$canonical" | /usr/bin/jq -e 'length == 1 and .[0].ifname == "enp4s0"' >/dev/null || return 1 + routes='[{"dst":"default","dev":"enp4s0"},{"dst":"10.200.120.0/30","dev":"lecodex-host"},{"dst":"10.200.120.0/30","dev":"lecodex-ns"}]' + canonical=$(stage3_host_routes_json_canonical "$routes") || return 1 + printf '%s' "$canonical" | /usr/bin/jq -e 'length == 1 and .[0].dev == "enp4s0"' >/dev/null || return 1 + rules='[{"priority":0}]' + stage3_host_rules_json_canonical "$rules" >/dev/null || return 1 + ! stage3_host_addresses_json_canonical '{}' >/dev/null 2>&1 || return 1 + ! stage3_host_addresses_json_canonical '[null]' >/dev/null 2>&1 || return 1 + ! stage3_host_routes_json_canonical '{}' >/dev/null 2>&1 || return 1 + ! stage3_host_rules_json_canonical '[null]' >/dev/null 2>&1 +} + +case_nft_json_rule_validation() { + local inet nat drift inet_full nat_full fixture_inet_text fixture_nat_text expected_canonical + inet=$(/usr/bin/jq -nc ' + def mm($key; $op; $right): {"match":{"left":{"meta":{"key":$key}},"op":$op,"right":$right}}; + def pm($protocol; $field; $right): {"match":{"left":{"payload":{"protocol":$protocol,"field":$field}},"op":"==","right":$right}}; + def ct: {"match":{"left":{"ct":{"key":"state"}},"op":"in","right":{"set":["established","related"]}}}; + def counter: {"counter":{"packets":0,"bytes":0}}; + def accept: {"accept":null}; def drop: {"drop":null}; + {"nftables":[ + {"rule":{"chain":"input","expr":[mm("iifname";"==";"lecodex-host"),ct,accept]}}, + {"rule":{"chain":"input","expr":[mm("iifname";"==";"lecodex-host"),pm("ip";"daddr";"@approved_dns4"),pm("udp";"dport";53),accept]}}, + {"rule":{"chain":"input","expr":[mm("iifname";"==";"lecodex-host"),pm("ip";"daddr";"@approved_dns4"),pm("tcp";"dport";53),accept]}}, + {"rule":{"chain":"input","expr":[mm("iifname";"==";"lecodex-host"),counter,drop]}}, + {"rule":{"chain":"forward","expr":[mm("oifname";"==";"lecodex-host"),ct,accept]}}, + {"rule":{"chain":"forward","expr":[mm("oifname";"==";"lecodex-host"),counter,drop]}}, + {"rule":{"chain":"forward","expr":[mm("iifname";"==";"lecodex-host"),pm("ip";"daddr";"@approved_dns4"),pm("udp";"dport";53),accept]}}, + {"rule":{"chain":"forward","expr":[mm("iifname";"==";"lecodex-host"),pm("ip";"daddr";"@approved_dns4"),pm("tcp";"dport";53),accept]}}, + {"rule":{"chain":"forward","expr":[mm("iifname";"==";"lecodex-host"),pm("ip";"daddr";"@operator_host_public4"),counter,drop]}}, + {"rule":{"chain":"forward","expr":[mm("iifname";"==";"lecodex-host"),pm("ip";"daddr";"@denied4"),counter,drop]}}, + {"rule":{"chain":"forward","expr":[mm("iifname";"==";"lecodex-host"),mm("nfproto";"==";"ipv6"),counter,drop]}}, + {"rule":{"chain":"forward","expr":[mm("iifname";"==";"lecodex-host"),pm("tcp";"dport";{"set":[25,465,587]}),counter,drop]}}, + {"rule":{"chain":"forward","expr":[mm("iifname";"==";"lecodex-host"),pm("tcp";"dport";{"set":[80,443]}),accept]}}, + {"rule":{"chain":"forward","expr":[mm("iifname";"==";"lecodex-host"),counter,drop]}} + ]}') || return 1 + nat=$(/usr/bin/jq -nc '{"nftables":[{"rule":{"chain":"postrouting","expr":[{"match":{"left":{"payload":{"protocol":"ip","field":"saddr"}},"op":"==","right":{"prefix":{"addr":"10.200.120.0","len":30}}}},{"match":{"left":{"meta":{"key":"oifname"}},"op":"!=","right":"lecodex-host"}},{"masquerade":null}]}}]}') || return 1 + stage3_nft_json_inet_rules_match "$inet" || return 1 + stage3_nft_json_nat_rules_match "$nat" || return 1 + drift=$(printf '%s' "$inet" | /usr/bin/jq -c '(.nftables[12].rule.expr[1].match.right.set) += [8080]') || return 1 + ! stage3_nft_json_inet_rules_match "$drift" || return 1 + drift=$(printf '%s' "$nat" | /usr/bin/jq -c '(.nftables[0].rule.expr[1].match.right) = "other0"') || return 1 + ! stage3_nft_json_nat_rules_match "$drift" || return 1 + + inet_full=$(printf '%s' "$inet" | /usr/bin/jq -c ' + .nftables as $rules | + {"nftables": ( + [{"metainfo":{"json_schema_version":1}}, + {"table":{"family":"inet","name":"le_app_codex","handle":10}}, + {"set":{"family":"inet","table":"le_app_codex","name":"approved_dns4","type":"ipv4_addr","handle":11,"flags":["interval"],"elem":["9.9.9.9","149.112.112.112"]}}, + {"set":{"family":"inet","table":"le_app_codex","name":"operator_host_public4","type":"ipv4_addr","handle":12,"flags":["interval"],"elem":["74.67.173.56"]}}, + {"set":{"family":"inet","table":"le_app_codex","name":"denied4","type":"ipv4_addr","handle":13,"flags":["interval"],"elem":["0.0.0.0/8"]}}, + {"chain":{"family":"inet","table":"le_app_codex","name":"input","handle":14,"type":"filter","hook":"input","prio":0,"policy":"accept"}}, + {"chain":{"family":"inet","table":"le_app_codex","name":"forward","handle":15,"type":"filter","hook":"forward","prio":0,"policy":"accept"}}] + + ($rules | map(.rule += {"family":"inet","table":"le_app_codex","handle":100}))) }') || return 1 + nat_full=$(printf '%s' "$nat" | /usr/bin/jq -c ' + .nftables as $rules | + {"nftables": ( + [{"metainfo":{"json_schema_version":1}}, + {"table":{"family":"ip","name":"le_app_codex_nat","handle":20}}, + {"chain":{"family":"ip","table":"le_app_codex_nat","name":"postrouting","handle":21,"type":"nat","hook":"postrouting","prio":100,"policy":"accept"}}] + + ($rules | map(.rule += {"family":"ip","table":"le_app_codex_nat","handle":200}))) }') || return 1 + fixture_inet_text=$'table inet le_app_codex {\n set approved_dns4 {\n type ipv4_addr\n flags interval\n elements = { 9.9.9.9, 149.112.112.112 }\n }\n set operator_host_public4 {\n type ipv4_addr\n flags interval\n elements = { 74.67.173.56 }\n }\n set denied4 {\n type ipv4_addr\n flags interval\n elements = { 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.0.0.0/24, 192.0.2.0/24, 192.88.99.0/24, 192.168.0.0/16, 198.18.0.0/15, 198.51.100.0/24, 203.0.113.0/24, 224.0.0.0/4, 240.0.0.0/4 }\n }\n chain input { }\n chain forward { }\n}' + fixture_nat_text=$'table ip le_app_codex_nat {\n chain postrouting { }\n}' + stage3_project_inet_snapshot_valid "$inet_full" "$fixture_inet_text" || return 1 + stage3_project_nat_snapshot_valid "$nat_full" "$fixture_nat_text" || return 1 + stage3_query_nft_table_json() { if [ "$1/$2" = inet/le_app_codex ]; then printf '%s' "$inet_full"; else printf '%s' "$nat_full"; fi; } + stage3_query_nft_table_text() { if [ "$1/$2" = inet/le_app_codex ]; then printf '%s' "$fixture_inet_text"; else printf '%s' "$fixture_nat_text"; fi; } + stage3_capture_resource_inventory() { stage3_inventory_namespaces=; stage3_inventory_links='[]'; stage3_inventory_tables='[{"family":"inet","name":"le_app_codex","handle":10},{"family":"ip","name":"le_app_codex_nat","handle":20}]'; } + stage3_read_exact_project_policy_snapshot || return 1 + [ "$stage3_snapshot_inet_handle/$stage3_snapshot_nat_handle" = 10/20 ] || return 1 + expected_canonical=$(stage3_nft_table_json_canonical "$inet_full" inet le_app_codex) || return 1 + [ "$stage3_snapshot_inet_canonical" = "$expected_canonical" ] && [ "$stage3_snapshot_inet_sha256" = "$(printf '%s' "$expected_canonical" | stage3_sha_stream)" ] || return 1 + expected_canonical=$(stage3_nft_table_json_canonical "$nat_full" ip le_app_codex_nat) || return 1 + [ "$stage3_snapshot_nat_canonical" = "$expected_canonical" ] && [ "$stage3_snapshot_nat_sha256" = "$(printf '%s' "$expected_canonical" | stage3_sha_stream)" ] || return 1 + stage3_prepared_table_shape inet le_app_codex || return 1 + stage3_prepared_table_shape ip le_app_codex_nat || return 1 + drift=$(printf '%s' "$inet_full" | /usr/bin/jq -c '.nftables += [{"chain":{"family":"inet","table":"le_app_codex","name":"unexpected","handle":999}}]') || return 1 + ! stage3_project_inet_snapshot_valid "$drift" "$fixture_inet_text" 2>/dev/null || return 1 + drift=$(printf '%s' "$nat_full" | /usr/bin/jq -c '.nftables[0].unexpected = true') || return 1 + ! stage3_project_nat_snapshot_valid "$drift" "$fixture_nat_text" 2>/dev/null +} + +case_stable_project_policy_snapshot() { + local calls=0 stable=$(printf 'a%.0s' {1..64}) drift=$(printf 'b%.0s' {1..64}) + stage3_read_exact_project_policy_snapshot() { + calls=$((calls + 1)) + stage3_snapshot_inet_handle=10 + stage3_snapshot_nat_handle=20 + stage3_snapshot_inet_canonical=inet + stage3_snapshot_nat_canonical=nat + stage3_snapshot_inet_sha256=$stable + stage3_snapshot_nat_sha256=$stable + stage3_snapshot_project_sha256=$stable + [ "$calls" -ne 2 ] || [ "${test_snapshot_drift:-0}" -eq 0 ] || stage3_snapshot_project_sha256=$drift + } + test_snapshot_drift=0 + stage3_capture_exact_project_policy_snapshot || return 1 + calls=0 test_snapshot_drift=1 + ! stage3_capture_exact_project_policy_snapshot +} + +case_nft_digest_query_failures() { + stage3_nft_table_canonical() { return 1; } + stage3_project_nft_canonical() { return 1; } + ! stage3_nft_table_digest inet le_app_codex && ! stage3_project_nft_digest +} + +case_malformed_and_escaping_records() { + local root=$fixture/records good bad + good=$root/good; bad=$root/bad + /usr/bin/mkdir "$root" + write_prepared_record "$good" + stage3_parse_record_file "$good" 0 || return 1 + /usr/bin/cp "$good" "$bad"; printf 'UNKNOWN|x\n' >> "$bad" + ! stage3_parse_record_file "$bad" 0 || return 1 + /usr/bin/cp "$good" "$bad"; printf 'STATUS|PREPARED\n' >> "$bad" + ! stage3_parse_record_file "$bad" 0 || return 1 + /usr/bin/sed 's/^STATUS|PREPARED$/STATUS|PREPARED|/' "$good" > "$bad" + ! stage3_parse_record_file "$bad" 0 || return 1 + /usr/bin/cp "$good" "$bad"; /usr/bin/truncate -s -1 "$bad" + ! stage3_parse_record_file "$bad" 0 || return 1 + /usr/bin/cp "$good" "$bad"; printf '\0' >> "$bad" + ! stage3_parse_record_file "$bad" 0 || return 1 + /usr/bin/sed 's#HOSTNAME|fixture-host#HOSTNAME|../escape#' "$good" > "$bad" + ! stage3_parse_record_file "$bad" 0 || return 1 + /usr/bin/rm "$bad"; /usr/bin/ln -s good "$bad" + ! stage3_parse_record_file "$bad" 0 || return 1 + ! stage3_read_record "$good" +} + +case_host_and_repository_provenance_mismatch() { + local root=$fixture/provenance good bad head parent original_stage3_root + root=$fixture/provenance; good=$root/good; bad=$root/bad + /usr/bin/mkdir "$root" + head=$(/usr/bin/git -C "$repo_root" rev-parse HEAD) || return 1 + parent=$(/usr/bin/git -C "$repo_root" rev-parse HEAD^) || return 1 + stage3_activation_id=11111111-2222-4333-8444-555555555555 + stage3_machine_id_sha256=$(/usr/bin/sha256sum /etc/machine-id | /usr/bin/cut -d' ' -f1) + stage3_head=$head + stage3_timestamp=2026-07-13T12:00:00Z + stage3_hostname=$(/usr/bin/hostname) + stage3_boot_id=aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee + baseline_public_ipv4=$expected_public_ipv4 + baseline_nft_sha256=$(printf 'c%.0s' {1..64}) + baseline_docker_network_sha256=$(printf 'd%.0s' {1..64}) + baseline_host_network_sha256=$(printf 'e%.0s' {1..64}) + baseline_docker_runtime_sha256=$(printf 'f%.0s' {1..64}) + stage3_render_record PREPARED > "$good"; /usr/bin/chmod 0600 "$good" + stage3_parse_record_file "$good" 1 || return 1 + /usr/bin/sed "s/^HOSTNAME|.*/HOSTNAME|wrong-host/" "$good" > "$bad" + ! stage3_parse_record_file "$bad" 1 || return 1 + /usr/bin/sed "s/^MACHINE_ID_SHA256|.*/MACHINE_ID_SHA256|$(printf '0%.0s' {1..64})/" "$good" > "$bad" + ! stage3_parse_record_file "$bad" 1 || return 1 + /usr/bin/sed "s/^STAGE3_ACTIVATION_COMMIT|.*/STAGE3_ACTIVATION_COMMIT|$(printf '0%.0s' {1..40})/" "$good" > "$bad" + ! stage3_parse_record_file "$bad" 1 || return 1 + original_stage3_root=$stage3_root + stage3_root=$stage2_root + stage3_head=$parent + stage3_render_record PREPARED > "$bad"; /usr/bin/chmod 0600 "$bad" + stage3_head=$head + ! stage3_parse_record_file "$bad" 1 || return 1 + stage3_root=$original_stage3_root +} + +case_lock_mismatch_and_concurrency() { + local root=$fixture/lock wrong=$fixture/wrong uid gid contention token + /usr/bin/mkdir "$root" "$wrong"; /usr/bin/chmod 0755 "$root" "$wrong" + uid=$(/usr/bin/id -u); gid=$(/usr/bin/id -g) + exec 9< "$root" + token=$(/usr/bin/stat -c '%D:%i' "$root") || return 1 + stage3_validate_inherited_lock 9 "$root" "$token" "$uid" "$gid" 755 || return 1 + ! stage3_validate_inherited_lock 9 "$root" 'forged-token' "$uid" "$gid" 755 || return 1 + ! stage3_validate_inherited_lock 9 "$wrong" "$token" "$uid" "$gid" 755 || return 1 + ! stage3_validate_inherited_lock 8 "$root" "$token" "$uid" "$gid" 755 || return 1 + contention=$(ROOT="$root" /bin/bash -c 'exec 7<&-; exec 8< "$ROOT"; /usr/bin/flock -n 8; printf "%s" "$?"') + [ "$contention" -ne 0 ] || return 1 + exec 9<&- +} + +case_transient_collision() { + local free loaded exact unit=le-app-codex-inert-stage3-11111111222243338444555555555555.service activation=11111111-2222-4333-8444-555555555555 token invocation + free=$'LoadState=not-found\nActiveState=inactive\nFragmentPath=' + loaded=$'LoadState=loaded\nActiveState=active\nFragmentPath=/run/systemd/transient/collision.service' + token=$(printf 'a%.0s' {1..32}); invocation=$(printf 'b%.0s' {1..32}) + exact=$(emit_fixture_inert_show "$unit" "$activation" "$token" "$invocation") || return 1 + stage3_inert_unit_unclaimed "$free" && ! stage3_inert_unit_unclaimed "$loaded" || return 1 + stage3_inert_unit_identity "$exact" "$unit" "$invocation" "$activation" "$token" || return 1 + ! stage3_inert_unit_identity "$exact" "$unit" "$invocation" "$activation" "$(printf 'c%.0s' {1..32})" || return 1 + ! stage3_inert_unit_identity "$exact" "$unit" "$(printf 'd%.0s' {1..32})" "$activation" "$token" +} + +case_inert_signal_deferral_and_pid_identity() { + local rc + ( + source "$stage3_root/tests/52-run-stage1-inert.sh" || exit 90 + set +Eeu + inert_pending_signal=0 + stage3_inert_defer_signals + /usr/bin/kill -TERM "$BASHPID" + [ "$inert_pending_signal" -eq 143 ] || exit 91 + stage3_inert_arm_signals + stage3_inert_replay_pending_signal + exit $? + ); rc=$? + [ "$rc" -eq 143 ] || return 1 + ( + source "$stage3_root/tests/52-run-stage1-inert.sh" || exit 90 + set +Eeu + pid=$BASHPID + current=$(stage3_inert_pid_starttime "$pid") || exit 91 + stage3_inert_pid_identity_matches "$pid" "$current" || exit 92 + ! stage3_inert_pid_identity_matches "$pid" "$((current + 1))" + ) +} + +case_inert_probe_cleanup() { + local root=$fixture/probe probe_path uid gid + probe_path=$root/probe + /usr/bin/mkdir "$root"; uid=$(/usr/bin/id -u); gid=$(/usr/bin/id -g) + : > "$probe_path"; /usr/bin/chmod 0644 "$probe_path" + stage3_cleanup_probe_exact "$probe_path" "$uid" "$gid" 644 || return 1 + [ ! -e "$probe_path" ] || return 1 + /usr/bin/ln -s /etc/passwd "$probe_path" + ! stage3_cleanup_probe_exact "$probe_path" "$uid" "$gid" 644 && [ -L "$probe_path" ] +} + +case_inert_probe_failure_trap() { + local root=$fixture/probe-failure probe_path state late uid gid rc + probe_path=$root/probe + state=$root/unit-loaded + late=$root/late-registration + /usr/bin/mkdir "$root"; uid=$(/usr/bin/id -u); gid=$(/usr/bin/id -g) + ( + source "$stage3_root/tests/52-run-stage1-inert.sh" || exit 90 + set +e + inert_probe=$probe_path; inert_probe_uid=$uid; inert_probe_gid=$gid; inert_probe_mode=644 + inert_unit=le-app-codex-inert-stage3-11111111222243338444555555555555.service + inert_activation=11111111-2222-4333-8444-555555555555 + inert_token=$(printf 'a%.0s' {1..32}); inert_invocation=$(printf 'b%.0s' {1..32}) + inert_client_started=1; inert_owned=1; inert_client_pid= + stage3_inert_show() { if [ -e "$late" ]; then /usr/bin/unlink -- "$late"; printf 'LoadState=not-found\n'; elif [ -e "$state" ]; then emit_fixture_inert_show "$inert_unit" "$inert_activation" "$inert_token" "$inert_invocation"; else printf 'LoadState=not-found\n'; fi; } + stage3_inert_stop() { /usr/bin/unlink -- "$state"; } + : > "$state"; : > "$late"; : > "$probe_path"; /usr/bin/chmod 0644 "$probe_path" + trap stage3_inert_cleanup EXIT + exit 37 + ); rc=$? + [ "$rc" -eq 37 ] && [ ! -e "$state" ] && [ ! -e "$probe_path" ] && [ ! -L "$probe_path" ] || return 1 + ( + source "$stage3_root/tests/52-run-stage1-inert.sh" || exit 90 + set +e + inert_probe=$probe_path; inert_probe_uid=$uid; inert_probe_gid=$gid; inert_probe_mode=644 + inert_unit=le-app-codex-inert-stage3-11111111222243338444555555555555.service + inert_activation=11111111-2222-4333-8444-555555555555 + inert_token=$(printf 'a%.0s' {1..32}); inert_invocation=$(printf 'b%.0s' {1..32}) + inert_client_started=1; inert_owned=1; inert_client_pid= + stage3_inert_show() { if [ -e "$state" ]; then emit_fixture_inert_show "$inert_unit" "$inert_activation" "$inert_token" "$inert_invocation"; else printf 'LoadState=not-found\n'; fi; } + stage3_inert_stop() { /usr/bin/unlink -- "$state"; return 1; } + : > "$state"; : > "$probe_path"; /usr/bin/chmod 0644 "$probe_path" + stage3_inert_cleanup + ); rc=$? + [ "$rc" -eq 1 ] && [ ! -e "$state" ] && [ ! -e "$probe_path" ] && [ ! -L "$probe_path" ] +} + +case_docker_gateway_inventory() { + local exact missing extra duplicate + exact=$(stage3_committed_docker_network_canonical) || return 1 + stage3_materialize_docker_ipv4_gateways "$exact" && [ "${#stage3_docker_gateways[@]}" -eq 39 ] || return 1 + missing=$(printf '%s' "$exact" | /usr/bin/jq -c '.[0].IPAM.Config = []') + ! stage3_materialize_docker_ipv4_gateways "$missing" && [ "${#stage3_docker_gateways[@]}" -eq 0 ] || return 1 + extra=$(printf '%s' "$exact" | /usr/bin/jq -c '.[0].IPAM.Config += [{"Subnet":"10.254.0.0/24","Gateway":"10.254.0.1"}]') + ! stage3_materialize_docker_ipv4_gateways "$extra" && [ "${#stage3_docker_gateways[@]}" -eq 0 ] || return 1 + duplicate=$(printf '%s' "$exact" | /usr/bin/jq -c '.[0].IPAM.Config += [.[1].IPAM.Config[0]]') + ! stage3_materialize_docker_ipv4_gateways "$duplicate" && [ "${#stage3_docker_gateways[@]}" -eq 0 ] || return 1 + ! stage3_materialize_docker_ipv4_gateways '[]' && [ "${#stage3_docker_gateways[@]}" -eq 0 ] +} + +case_veth_drift() { + stage3_veth_fields_match 10 11 aa:bb:cc:dd:ee:ff 10 11 aa:bb:cc:dd:ee:ff || return 1 + ! stage3_veth_fields_match 10 11 aa:bb:cc:dd:ee:ff 12 11 aa:bb:cc:dd:ee:ff || return 1 + ! stage3_veth_fields_match 10 11 aa:bb:cc:dd:ee:ff 10 99 aa:bb:cc:dd:ee:ff || return 1 + ! stage3_veth_fields_match 10 11 aa:bb:cc:dd:ee:ff 10 11 00:00:00:00:00:00 +} + +case_direct_cleanup_drift_gates() { + stage3_capture_resource_inventory() { return 0; } + stage3_inventory_has_table() { return 0; } + stage3_table_identity() { return 1; } + ! stage3_delete_surviving_table inet le_app_codex || return 1 + stage3_veth_identity() { return 1; } + ! stage3_delete_surviving_veth || return 1 + ! stage3_delete_surviving_namespace +} + +case_recorded_baseline_mismatch() { + local good=$(printf 'a%.0s' {1..64}) + failures=0 + declare -gA stage3_marker_fields=([CURRENT_PUBLIC_NAT_IPV4]="$expected_public_ipv4" [BASELINE_DOCKER_NETWORK_SHA256]="$good" [BASELINE_NFT_SHA256]="$good" [BASELINE_HOST_NETWORK_SHA256]="$good" [BASELINE_DOCKER_RUNTIME_SHA256]="$good") + stage3_public_ipv4() { printf '%s' "$expected_public_ipv4"; } + stage3_docker_network_canonical() { printf x; } + stage3_nonproject_nft_canonical() { printf x; } + stage3_host_network_canonical() { printf x; } + stage3_docker_runtime_canonical() { printf x; } + stage3_sha_stream() { /usr/bin/sha256sum | /usr/bin/cut -d' ' -f1; } + good=$(printf x | stage3_sha_stream) + stage3_marker_fields[BASELINE_DOCKER_NETWORK_SHA256]=$good + stage3_marker_fields[BASELINE_NFT_SHA256]=$good + stage3_marker_fields[BASELINE_HOST_NETWORK_SHA256]=$good + stage3_marker_fields[BASELINE_DOCKER_RUNTIME_SHA256]=$good + stage3_compare_recorded_baseline fixture >/dev/null || return 1 + before=$failures + stage3_marker_fields[BASELINE_NFT_SHA256]=$(printf '0%.0s' {1..64}) + ! stage3_compare_recorded_baseline fixture-drift >/dev/null 2>&1 && [ "$failures" -gt "$before" ] +} + +expect_success 'interrupted marker publication has atomic before/after states' case_marker_publication +expect_success 'marker publication never clobbers an existing target' case_marker_no_clobber +expect_success 'complete interrupted marker temporary is fsynced into retained authority' case_complete_temporary_becomes_durable_authority +expect_success 'interrupted marker temporary accepts empty/full recovery and rejects partial or mismatched state' case_interrupted_marker_temporary +expect_success 'same-status ACTIVE temporary must exactly match retained authority' case_conflicting_active_temporary +expect_success 'interrupted tombstone publication retains one exact authority' case_tombstone_publication +expect_success 'fsynced ACTIVE temporary atomically upgrades retained PREPARED authority' case_durable_active_temporary_upgrade +expect_success 'PREPARED partial activation states retain tombstone through a retry' case_partial_activation_and_rollback_retry +expect_success 'PREPARED tombstone retry remains valid after inactive unit loses live invocation ID' case_prepared_tombstone_retry_without_live_invocation +expect_success 'ACTIVE partial rollback retains tombstone across a second retry and rejects nft drift' case_partial_rollback_retry_and_nft_drift +expect_success 'inventory command, malformed text, and wrong-shape JSON failures are never absence' case_inventory_failures +expect_success 'host baselines exclude both project peers and reject wrong-shape JSON' case_host_baseline_project_peer_and_schema +expect_success 'canonical nft JSON accepts exact ordered rules and rejects semantic drift' case_nft_json_rule_validation +expect_success 'two-pass project policy snapshot rejects an unstable canonical capture' case_stable_project_policy_snapshot +expect_success 'nft canonical query failures cannot become empty-input digests' case_nft_digest_query_failures +expect_success 'malformed, symlink, duplicate, and escaping records rejected' case_malformed_and_escaping_records +expect_success 'host, machine, commit, and package provenance mismatches rejected' case_host_and_repository_provenance_mismatch +expect_success 'actual inherited lock token mismatch and concurrent invocation rejected' case_lock_mismatch_and_concurrency +expect_success 'transient unit collision and wrong ownership token rejected' case_transient_collision +expect_success 'inert signals are deferred through ownership capture and PID start-time identity is exact' case_inert_signal_deferral_and_pid_identity +expect_success 'inert write probe cleaned exactly and symlink preserved' case_inert_probe_cleanup +expect_success 'actual inert cleanup function removes mocked unit/probe and propagates execution or cleanup failure' case_inert_probe_failure_trap +expect_success 'Docker gateway inventory requires exact 39 unique addresses' case_docker_gateway_inventory +expect_success 'veth identity drift rejected before cleanup' case_veth_drift +expect_success 'table/veth drift aborts each direct cleanup path before deletion' case_direct_cleanup_drift_gates +expect_success 'recorded activation-baseline mismatch fails rollback gate' case_recorded_baseline_mismatch + +printf 'Phase 2B Stage 3 failure-path tests failures=%s\n' "$failures" +[ "$failures" -eq 0 ] diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/docker-config/config.json b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/docker-config/config.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/docker-config/config.json @@ -0,0 +1 @@ +{} diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/inert-bin/rm b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/inert-bin/rm new file mode 100755 index 0000000..0f4ea75 --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/inert-bin/rm @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu + +test "$#" -eq 1 +test "$1" = /srv/le-app-codex/tmp/phase2b-write-test +/usr/bin/unlink /srv/le-app-codex/build-artifacts/.phase2b-stage3-write-test diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/inert-bin/touch b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/inert-bin/touch new file mode 100755 index 0000000..4b60975 --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/inert-bin/touch @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +test "$#" -eq 1 +test "$1" = /srv/le-app-codex/tmp/phase2b-write-test +test ! -e /srv/le-app-codex/build-artifacts/.phase2b-stage3-write-test +test ! -L /srv/le-app-codex/build-artifacts/.phase2b-stage3-write-test +set -C +: > /srv/le-app-codex/build-artifacts/.phase2b-stage3-write-test diff --git a/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/stage3-common.sh b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/stage3-common.sh new file mode 100644 index 0000000..f7ca02d --- /dev/null +++ b/docs/le-app-database-migration/phase2b-stage3-namespace-activation/tests/stage3-common.sh @@ -0,0 +1,1090 @@ +#!/usr/bin/env bash + +stage3_root=$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." 2>/dev/null && pwd) +migration_root=$(CDPATH= cd -- "$stage3_root/.." 2>/dev/null && pwd) +stage1_root=$migration_root/phase2b-contained-runtime +stage2_root=$migration_root/phase2b-stage2-verification +repo_root=$(CDPATH= cd -- "$migration_root/../.." 2>/dev/null && pwd) +payload_root=$stage1_root/payload +state_parent=/var/lib/le-app-codex-phase2b +state_transaction=$state_parent/stage1 +marker=/etc/le-app-codex-runtime/OPERATOR-INPUTS-APPROVED +marker_temporary=/etc/le-app-codex-runtime/.OPERATOR-INPUTS-APPROVED.stage3-new +rollback_record=/etc/le-app-codex-runtime/.OPERATOR-INPUTS-APPROVED.stage3-rollback +installed_policy=/etc/nftables.d/50-le-app-codex.nft +inert_unit_prefix=le-app-codex-inert-stage3- +inert_probe=/srv/le-app-codex/build-artifacts/.phase2b-stage3-write-test +context_hash=1b645e55bd6af77d420732b143423e7a27b12fbb5a71497e5d89c52481b6a535 +context_source=$payload_root/srv/le-app-codex/home/.docker/contexts/meta/$context_hash/meta.json +context_destination=/srv/le-app-codex/home/.docker/contexts/meta/$context_hash/meta.json + +expected_stage1_source_commit=df6b53e63916880bec2c77219b04b010b8b453f1 +expected_stage1_artifact_digest=5aaa91b1e6846eda033961dcee392b9657476ad6fd149420979e9309b484445f +expected_stage2_commit=909f0a8200c79efc205d92618be47e06ebf764fc +expected_stage2_manifest_digest=4ce1de182dd6fda902d910c1d6f3f169dce55266e61f9abf2e57cd119fad600d +expected_policy_digest=4289b705dbd786281daccb6d5aa660c27b83441f1a34d0cd18590666124be386 +expected_public_ipv4=74.67.173.56 +approved_dns4=9.9.9.9,149.112.112.112 + +stage3_relative_path_safe() { + local path=$1 + [ -n "$path" ] && [[ "$path" != /* ]] && [[ "$path" != *'|'* ]] && [[ "$path" != *$'\n'* ]] && [[ "$path" != ../* ]] && [[ "$path" != */../* ]] && [[ "$path" != */.. ]] && [ "$path" != . ] && [ "$path" != .. ] +} + +stage3_verify_committed_manifest() { + local package_root=$1 label=$2 package_relative manifest line expected path actual committed rc tracked relative + local record_count=0 tree_count=0 before=$failures + declare -A listed=() + package_relative=${package_root#"$repo_root"/} + manifest=$package_root/MANIFEST.sha256 + if [ -f "$manifest" ] && [ ! -L "$manifest" ]; then pass "$label manifest is a regular file"; else fail "$label manifest missing or unsafe"; return 1; fi + tracked=$(/usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" ls-files --error-unmatch -- "$package_relative/MANIFEST.sha256" 2>/dev/null) + if [ "$tracked" = "$package_relative/MANIFEST.sha256" ]; then pass "$label manifest is tracked"; else fail "$label manifest is not tracked"; fi + while IFS= read -r line || [ -n "$line" ]; do + if [[ "$line" =~ ^([0-9a-f]{64})\ \ (.+)$ ]]; then expected=${BASH_REMATCH[1]}; path=${BASH_REMATCH[2]}; else fail "$label malformed manifest record"; continue; fi + if ! stage3_relative_path_safe "$path" || [ "$path" = MANIFEST.sha256 ] || [ -n "${listed[$path]:-}" ]; then fail "$label unsafe or duplicate manifest path $path"; continue; fi + listed["$path"]=1 + record_count=$((record_count + 1)) + if [ -f "$package_root/$path" ] && [ ! -L "$package_root/$path" ]; then + actual=$(/usr/bin/sha256sum "$package_root/$path" 2>/dev/null | /usr/bin/cut -d' ' -f1) + [ "$actual" = "$expected" ] && pass "$label working hash $path" || fail "$label working hash mismatch $path" + else + fail "$label working path is missing, non-regular, or a symlink $path" + fi + relative=$package_relative/$path + tracked=$(/usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" ls-files --error-unmatch -- "$relative" 2>/dev/null) + if [ "$tracked" = "$relative" ]; then + committed=$(/usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" show "HEAD:$relative" 2>/dev/null | /usr/bin/sha256sum | /usr/bin/cut -d' ' -f1) + rc=$? + if [ "$rc" -eq 0 ] && [ "$committed" = "$expected" ]; then pass "$label committed hash $path"; else fail "$label committed hash mismatch $path"; fi + else + fail "$label untracked manifest path $path" + fi + done < "$manifest" + while IFS= read -r tracked; do + relative=${tracked#"$package_relative"/} + [ "$relative" = MANIFEST.sha256 ] && continue + tree_count=$((tree_count + 1)) + [ -n "${listed[$relative]:-}" ] || fail "$label manifest omits tracked path $relative" + done < <(/usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" ls-tree -r --name-only HEAD -- "$package_relative" 2>/dev/null) + if [ "$record_count" -gt 0 ] && [ "$record_count" -eq "$tree_count" ]; then pass "$label manifest covers $record_count committed files"; else fail "$label manifest/tree count mismatch records=$record_count tree=$tree_count"; fi + [ "$failures" -eq "$before" ] +} + +stage3_verify_repo_provenance() { + local status_rc stage1_relative stage2_relative historical_stage2_digest before=$failures + stage3_head=$(/usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" rev-parse HEAD 2>/dev/null) + repo_status=$(/usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" status --porcelain --untracked-files=all 2>/dev/null) + status_rc=$? + [[ "$stage3_head" =~ ^[0-9a-f]{40}$ ]] && pass "committed Stage 3 HEAD $stage3_head" || fail 'Stage 3 HEAD unavailable' + [ "$status_rc" -eq 0 ] && [ -z "$repo_status" ] && pass 'repository worktree clean' || fail 'repository worktree dirty or unreadable' + stage3_verify_committed_manifest "$stage1_root" 'Stage 1' || true + stage3_verify_committed_manifest "$stage2_root" 'Stage 2' || true + stage3_verify_committed_manifest "$stage3_root" 'Stage 3' || true + (cd "$stage1_root/inventory" && /usr/bin/sha256sum -c SHA256SUMS >/dev/null) && pass 'Stage 1 inventory checksums' || fail 'Stage 1 inventory checksums' + [ "$(/usr/bin/sha256sum "$stage1_root/MANIFEST.sha256" | /usr/bin/cut -d' ' -f1)" = "$expected_stage1_artifact_digest" ] && pass 'Stage 1 artifact digest pinned' || fail 'Stage 1 artifact digest drift' + [ "$(/usr/bin/sha256sum "$stage2_root/MANIFEST.sha256" | /usr/bin/cut -d' ' -f1)" = "$expected_stage2_manifest_digest" ] && pass 'Stage 2 manifest digest pinned' || fail 'Stage 2 manifest digest drift' + /usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" cat-file -e "$expected_stage2_commit^{commit}" 2>/dev/null && pass 'Stage 2 verification commit exists' || fail 'Stage 2 verification commit missing' + /usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" merge-base --is-ancestor "$expected_stage2_commit" HEAD 2>/dev/null && pass 'Stage 2 verification commit is an ancestor of HEAD' || fail 'Stage 2 verification commit is not an ancestor of HEAD' + stage1_relative=${stage1_root#"$repo_root"/} + stage2_relative=${stage2_root#"$repo_root"/} + /usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" diff --quiet "$expected_stage2_commit" HEAD -- "$stage1_relative" "$stage2_relative" && pass 'Stage 1 and Stage 2 trees unchanged since verification commit' || fail 'Stage 1 or Stage 2 tree changed since verification commit' + historical_stage2_digest=$(/usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" show "$expected_stage2_commit:$stage2_relative/MANIFEST.sha256" 2>/dev/null | /usr/bin/sha256sum | /usr/bin/cut -d' ' -f1) + [ "$historical_stage2_digest" = "$expected_stage2_manifest_digest" ] && pass 'Stage 2 commit binds expected package manifest' || fail 'Stage 2 package provenance mismatch' + [ "$failures" -eq "$before" ] +} + +stage3_load_stage1_specs() { + sources=( + "$payload_root/usr/local/libexec/dockerd-rootless-29.6.1.sh" + "$payload_root/usr/local/libexec/le-app-codex-netns" + "$payload_root/etc/le-app-codex-runtime/resolv.conf" + "$payload_root/etc/nftables.d/50-le-app-codex.nft" + "$payload_root/etc/systemd/system/le-app-codex-netns.service" + "$payload_root/etc/systemd/system/user-1200.slice.d/50-le-app-codex-resources.conf" + "$payload_root/etc/systemd/system/user@1200.service.d/50-le-app-codex-containment.conf" + "$payload_root/srv/le-app-codex/home/.config/systemd/user/docker.service" + "$payload_root/srv/le-app-codex/home/.config/docker/daemon.json" + "$payload_root/srv/le-app-codex/home/.docker/config.json" + "$context_source" + "$stage1_root/tests/00-read-only-preflight.sh" + "$stage1_root/tests/00-root-preinstall-validate.sh" + "$stage1_root/tests/10-inert-containment.sh" + "$stage1_root/tests/20-rootless-docker.sh" + "$stage1_root/tests/run-inert-without-user-manager.sh" + ) + destinations=( + /usr/local/libexec/dockerd-rootless-29.6.1.sh + /usr/local/libexec/le-app-codex-netns + /etc/le-app-codex-runtime/resolv.conf + /etc/nftables.d/50-le-app-codex.nft + /etc/systemd/system/le-app-codex-netns.service + /etc/systemd/system/user-1200.slice.d/50-le-app-codex-resources.conf + /etc/systemd/system/user@1200.service.d/50-le-app-codex-containment.conf + /srv/le-app-codex/home/.config/systemd/user/docker.service + /srv/le-app-codex/home/.config/docker/daemon.json + /srv/le-app-codex/home/.docker/config.json + "$context_destination" + /srv/le-app-codex/phase2b-tests/00-read-only-preflight.sh + /srv/le-app-codex/phase2b-tests/00-root-preinstall-validate.sh + /srv/le-app-codex/phase2b-tests/10-inert-containment.sh + /srv/le-app-codex/phase2b-tests/20-rootless-docker.sh + /srv/le-app-codex/phase2b-tests/run-inert-without-user-manager.sh + ) + uids=(0 0 0 0 0 0 0 1200 1200 1200 1200 1200 1200 1200 1200 1200) + gids=(0 0 0 0 0 0 0 1200 1200 1200 1200 1200 1200 1200 1200 1200) + modes=(0755 0755 0644 0644 0644 0644 0644 0640 0640 0640 0640 0750 0750 0750 0750 0750) + allowed_directories=( + /srv/le-app-codex/home/.docker/contexts/meta/$context_hash /srv/le-app-codex/home/.docker/contexts/meta /srv/le-app-codex/home/.docker/contexts /srv/le-app-codex/home/.docker + /srv/le-app-codex/home/.config/systemd/user /srv/le-app-codex/home/.config/systemd /srv/le-app-codex/home/.config/docker /srv/le-app-codex/home/.config /srv/le-app-codex/phase2b-tests + /etc/systemd/system/user@1200.service.d /etc/systemd/system/user-1200.slice.d /etc/le-app-codex-runtime /etc/nftables.d /usr/local/libexec "$state_parent" + /srv/le-app-codex/home /srv/le-app-codex /usr/local /etc/systemd/system /etc + ) + directory_uids=(1200 1200 1200 1200 1200 1200 1200 1200 1200 0 0 0 0 0 0 1200 0 0 0 0) + directory_gids=(1200 1200 1200 1200 1200 1200 1200 1200 1200 0 0 0 0 0 0 1200 1200 0 0 0) + directory_modes=(0750 0750 0750 0750 0750 0750 0750 0750 0750 0755 0755 0755 0755 0755 0700 0700 0750 0755 0755 0755) + directory_kinds=(project project project project project project project project project project project project shared shared state project-boundary project-boundary shared-base shared-base shared-base) + declare -gA stage1_allowed_file_specs=() stage1_allowed_directory_specs=() stage1_never_created_directories=() + local index source_hash source_size directory + for index in "${!destinations[@]}"; do + source_hash=$(/usr/bin/sha256sum "${sources[$index]}" 2>/dev/null | /usr/bin/cut -d' ' -f1) + source_size=$(/usr/bin/stat -c '%s' "${sources[$index]}" 2>/dev/null) + stage1_allowed_file_specs["${destinations[$index]}"]="$source_hash|$source_size|${uids[$index]}|${gids[$index]}|${modes[$index]}" + done + for index in "${!allowed_directories[@]}"; do stage1_allowed_directory_specs["${allowed_directories[$index]}"]="${directory_uids[$index]}|${directory_gids[$index]}|${directory_modes[$index]}|${directory_kinds[$index]}"; done + for directory in /srv/le-app-codex/home /srv/le-app-codex /usr/local /etc/systemd/system /etc; do stage1_never_created_directories["$directory"]=1; done +} + +stage3_verify_installed_file() { + local path=$1 label=${2:-$1} spec expected_hash expected_size expected_uid expected_gid expected_mode actual_hash actual_size actual_stat links + spec=${stage1_allowed_file_specs[$path]:-} + if [ -z "$spec" ]; then fail "$label has no committed Stage 1 specification"; return 1; fi + IFS='|' read -r expected_hash expected_size expected_uid expected_gid expected_mode <<< "$spec" + actual_hash=$(/usr/bin/sha256sum "$path" 2>/dev/null | /usr/bin/cut -d' ' -f1) + actual_size=$(/usr/bin/stat -c '%s' "$path" 2>/dev/null) + actual_stat=$(/usr/bin/stat -c '%u:%g:%a:%F' "$path" 2>/dev/null) + links=$(/usr/bin/stat -c '%h' "$path" 2>/dev/null) + if [ ! -L "$path" ] && [ "$actual_hash" = "$expected_hash" ] && [ "$actual_size" = "$expected_size" ] && [ "$actual_stat" = "$expected_uid:$expected_gid:${expected_mode#0}:regular file" ] && [ "$links" = 1 ]; then pass "$label hash/size/metadata"; else fail "$label hash, size, metadata, type, or link-count mismatch"; return 1; fi +} + +stage3_verify_installed_directory() { + local path=$1 spec expected_uid expected_gid expected_mode expected_kind actual + spec=${stage1_allowed_directory_specs[$path]:-} + if [ -z "$spec" ]; then fail "$path has no committed Stage 1 directory specification"; return 1; fi + IFS='|' read -r expected_uid expected_gid expected_mode expected_kind <<< "$spec" + actual=$(/usr/bin/stat -c '%u:%g:%a:%F' "$path" 2>/dev/null) + if [ -d "$path" ] && [ ! -L "$path" ] && [ "$actual" = "$expected_uid:$expected_gid:${expected_mode#0}:directory" ]; then pass "$path directory metadata ($expected_kind)"; else fail "$path directory metadata, type, or ownership mismatch"; return 1; fi +} + +stage3_verify_lock_parent() { + [ -d /etc ] && [ ! -L /etc ] && [ "$(/usr/bin/stat -c '%u:%g:%a:%F' /etc 2>/dev/null)" = '0:0:755:directory' ] || return 1 + [ -d /etc/le-app-codex-runtime ] && [ ! -L /etc/le-app-codex-runtime ] && [ "$(/usr/bin/stat -c '%u:%g:%a:%F' /etc/le-app-codex-runtime 2>/dev/null)" = '0:0:755:directory' ] +} + +stage3_verify_stage1_state_and_files() { + local destination directory historical_digest completed_directory_records before=$failures + stage3_load_stage1_specs + if [ "${#stage1_allowed_file_specs[@]}" -eq 16 ] && [ "${#stage1_allowed_directory_specs[@]}" -eq 20 ]; then pass 'exact Stage 1 allowlists'; else fail 'Stage 1 allowlist cardinality'; fi + source "$stage1_root/tests/stage1-state-lib.sh" + stage1_state_resolve_current "$state_transaction" 0 0 0600 && pass 'authoritative Stage 1 state generation' || fail 'authoritative Stage 1 state generation' + if [ "$failures" -eq "$before" ]; then stage1_state_validate_records "$stage1_current_manifest" "$expected_stage1_artifact_digest" "$(/usr/bin/hostname)" 16 && pass 'authoritative Stage 1 state records' || fail 'authoritative Stage 1 state records'; fi + completed_directory_records=$((${#stage1_validated_created_directories[@]} + ${#stage1_validated_existing_directories[@]})) + if [ "${stage1_validated_status:-}" = COMPLETE ] && [ "${stage1_validated_source_commit:-}" = "$expected_stage1_source_commit" ] && [ "${stage1_validated_artifact_digest:-}" = "$expected_stage1_artifact_digest" ] && [ "${#stage1_validated_file_paths[@]}" -eq 16 ] && [ "${#stage1_validated_pending_paths[@]}" -eq 0 ] && [ "${#stage1_validated_pending_directories[@]}" -eq 0 ] && [ "$completed_directory_records" -eq 20 ]; then pass 'Stage 1 state exact COMPLETE and pinned'; else fail 'Stage 1 state is not exact pinned COMPLETE'; fi + historical_digest=$(/usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" show "$expected_stage1_source_commit:${stage1_root#"$repo_root"/}/MANIFEST.sha256" 2>/dev/null | /usr/bin/sha256sum | /usr/bin/cut -d' ' -f1) + [ "$historical_digest" = "$expected_stage1_artifact_digest" ] && pass 'Stage 1 state digest binds source commit' || fail 'Stage 1 state/source commit mismatch' + for destination in "${destinations[@]}"; do stage3_verify_installed_file "$destination" || true; done + for directory in "${allowed_directories[@]}"; do stage3_verify_installed_directory "$directory" || true; done + [ "$failures" -eq "$before" ] +} + +stage3_rootful_docker() { + /usr/bin/env -u DOCKER_API_VERSION -u DOCKER_CERT_PATH -u DOCKER_CONTEXT -u DOCKER_HOST -u DOCKER_TLS -u DOCKER_TLS_VERIFY \ + /usr/bin/docker --config "$stage3_root/tests/docker-config" --host unix:///run/docker.sock "$@" +} + +stage3_docker_network_canonical() { + local ids_text raw id + local -a ids=() + ids_text=$(stage3_rootful_docker network ls --no-trunc -q 2>/dev/null) || return 1 + [ -n "$ids_text" ] || return 1 + while IFS= read -r id; do + [[ "$id" =~ ^[0-9a-f]{64}$ ]] || return 1 + ids+=("$id") + done <<< "$ids_text" + [ "${#ids[@]}" -gt 0 ] || return 1 + raw=$(stage3_rootful_docker network inspect --format '{"Name":{{json .Name}},"Id":{{json .Id}},"Scope":{{json .Scope}},"Driver":{{json .Driver}},"EnableIPv4":{{json .EnableIPv4}},"EnableIPv6":{{json .EnableIPv6}},"IPAM":{{json .IPAM}},"Internal":{{json .Internal}},"Attachable":{{json .Attachable}},"Ingress":{{json .Ingress}}}' "${ids[@]}" 2>/dev/null) || return 1 + printf '%s\n' "$raw" | /usr/bin/jq -esS -c ' + if length > 0 and all(.[]; + type == "object" and (.Name | type) == "string" and (.Id | type) == "string" and + (.Scope | type) == "string" and (.Driver | type) == "string" and (.IPAM | type) == "object" and + (.EnableIPv4 | type) == "boolean" and (.EnableIPv6 | type) == "boolean" and + (.Internal | type) == "boolean" and (.Attachable | type) == "boolean" and (.Ingress | type) == "boolean") + then sort_by(.Id) else error("invalid Docker network inventory") end' +} + +stage3_committed_docker_network_canonical() { + /usr/bin/jq -S -c 'map({Name,Id,Scope,Driver,EnableIPv4,EnableIPv6,IPAM,Internal,Attachable,Ingress}) | sort_by(.Id)' "$stage1_root/inventory/docker-networks.json" +} + +stage3_nonproject_nft_canonical() { + local raw + raw=$(/usr/bin/nft -j list ruleset 2>/dev/null) || return 1 + printf '%s' "$raw" | /usr/bin/jq -eS -c ' + def project_object: + to_entries[0] as $e | ($e.value // {}) as $v | + (($e.key == "table" and $v.family == "inet" and $v.name == "le_app_codex") or + ($e.key == "table" and $v.family == "ip" and $v.name == "le_app_codex_nat") or + ($e.key != "table" and $v.family == "inet" and ($v.table // "") == "le_app_codex") or + ($e.key != "table" and $v.family == "ip" and ($v.table // "") == "le_app_codex_nat")); + if type == "object" and (.nftables | type) == "array" and all(.nftables[]; type == "object") + then .nftables |= map(select((has("metainfo") or project_object) | not)) | + walk(if type == "object" and has("counter") and (.counter | type == "object") then .counter |= del(.packets,.bytes) else . end) + else error("invalid nft ruleset JSON") end' +} + +stage3_query_nft_table_json() { /usr/bin/nft -j list table "$1" "$2" 2>/dev/null; } +stage3_query_nft_table_text() { /usr/bin/nft list table "$1" "$2" 2>/dev/null; } + +stage3_nft_table_json_canonical() { + local raw=$1 family=$2 table=$3 + printf '%s' "$raw" | /usr/bin/jq -eS -c --arg family "$family" --arg table "$table" ' + def exact_outer_object: + type == "object" and (keys | length) == 1 and + if has("metainfo") then (.metainfo | type) == "object" + elif has("table") then + (.table | type) == "object" and .table.family == $family and .table.name == $table + elif has("set") then + (.set | type) == "object" and .set.family == $family and .set.table == $table + elif has("chain") then + (.chain | type) == "object" and .chain.family == $family and .chain.table == $table + elif has("rule") then + (.rule | type) == "object" and .rule.family == $family and .rule.table == $table + else false end; + if type == "object" and keys == ["nftables"] and (.nftables | type) == "array" and + (.nftables | length) > 0 and all(.nftables[]; exact_outer_object) and + ([.nftables[] | select(has("metainfo"))] | length) == 1 and + ([.nftables[] | .table? | select(.)] | length) == 1 + then .nftables |= map(select(has("metainfo") | not)) | + walk(if type == "object" and has("counter") and (.counter | type == "object") then .counter |= del(.packets,.bytes) else . end) | + walk(if type == "object" then del(.handle) else . end) + else error("invalid nft table JSON envelope") end' +} + +stage3_nft_table_canonical() { + local family=$1 table=$2 raw + raw=$(stage3_query_nft_table_json "$family" "$table") || return 1 + stage3_nft_table_json_canonical "$raw" "$family" "$table" +} + +stage3_project_nft_canonical_from_tables() { + local inet=$1 nat=$2 + /usr/bin/jq -enS -c --argjson inet "$inet" --argjson nat "$nat" '[$inet,$nat]' +} + +stage3_project_nft_canonical() { + local inet nat + inet=$(stage3_nft_table_canonical inet le_app_codex) || return 1 + nat=$(stage3_nft_table_canonical ip le_app_codex_nat) || return 1 + stage3_project_nft_canonical_from_tables "$inet" "$nat" +} + +stage3_nft_table_digest() { + local canonical + canonical=$(stage3_nft_table_canonical "$1" "$2") || return 1 + printf '%s' "$canonical" | stage3_sha_stream +} + +stage3_project_nft_digest() { + local canonical + canonical=$(stage3_project_nft_canonical) || return 1 + printf '%s' "$canonical" | stage3_sha_stream +} + +stage3_nft_json_inet_rules_match() { + local raw=$1 + printf '%s' "$raw" | /usr/bin/jq -e ' + def rules($chain): [.nftables[] | .rule? | select(.chain == $chain)]; + def meta_match($key; $op; $right): + .match? as $m | + ($m | type) == "object" and ($m | keys | sort) == ["left","op","right"] and + $m.left == {"meta":{"key":$key}} and $m.op == $op and $m.right == $right; + def payload_match($protocol; $field; $right): + .match? as $m | + ($m | type) == "object" and ($m | keys | sort) == ["left","op","right"] and + $m.left == {"payload":{"protocol":$protocol,"field":$field}} and + ($m.op == "==" or $m.op == "in") and $m.right == $right; + def named_set_match($protocol; $field; $name): payload_match($protocol; $field; "@" + $name); + def anonymous_set_match($protocol; $field; $values): + .match? as $m | + ($m | type) == "object" and ($m | keys | sort) == ["left","op","right"] and + $m.left == {"payload":{"protocol":$protocol,"field":$field}} and + ($m.op == "==" or $m.op == "in") and ($m.right == {"set":$values} or $m.right == $values); + def ct_established_related: + .match? as $m | + ($m | type) == "object" and ($m | keys | sort) == ["left","op","right"] and + $m.left == {"ct":{"key":"state"}} and ($m.op == "==" or $m.op == "in") and + (($m.right.set? // $m.right) | sort) == ["established","related"]; + def counter: + has("counter") and (.counter | type) == "object" and (.counter | keys | sort) == ["bytes","packets"] and + (.counter.bytes | type) == "number" and (.counter.packets | type) == "number"; + def accept: . == {"accept":null}; + def drop: . == {"drop":null}; + (rules("input")) as $i | (rules("forward")) as $f | + ($i | length) == 4 and ($f | length) == 10 and + ($i[0].expr | length) == 3 and ($i[0].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($i[0].expr[1] | ct_established_related) and ($i[0].expr[2] | accept) and + ($i[1].expr | length) == 4 and ($i[1].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($i[1].expr[1] | named_set_match("ip"; "daddr"; "approved_dns4")) and ($i[1].expr[2] | payload_match("udp"; "dport"; 53)) and ($i[1].expr[3] | accept) and + ($i[2].expr | length) == 4 and ($i[2].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($i[2].expr[1] | named_set_match("ip"; "daddr"; "approved_dns4")) and ($i[2].expr[2] | payload_match("tcp"; "dport"; 53)) and ($i[2].expr[3] | accept) and + ($i[3].expr | length) == 3 and ($i[3].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($i[3].expr[1] | counter) and ($i[3].expr[2] | drop) and + ($f[0].expr | length) == 3 and ($f[0].expr[0] | meta_match("oifname"; "=="; "lecodex-host")) and ($f[0].expr[1] | ct_established_related) and ($f[0].expr[2] | accept) and + ($f[1].expr | length) == 3 and ($f[1].expr[0] | meta_match("oifname"; "=="; "lecodex-host")) and ($f[1].expr[1] | counter) and ($f[1].expr[2] | drop) and + ($f[2].expr | length) == 4 and ($f[2].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($f[2].expr[1] | named_set_match("ip"; "daddr"; "approved_dns4")) and ($f[2].expr[2] | payload_match("udp"; "dport"; 53)) and ($f[2].expr[3] | accept) and + ($f[3].expr | length) == 4 and ($f[3].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($f[3].expr[1] | named_set_match("ip"; "daddr"; "approved_dns4")) and ($f[3].expr[2] | payload_match("tcp"; "dport"; 53)) and ($f[3].expr[3] | accept) and + ($f[4].expr | length) == 4 and ($f[4].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($f[4].expr[1] | named_set_match("ip"; "daddr"; "operator_host_public4")) and ($f[4].expr[2] | counter) and ($f[4].expr[3] | drop) and + ($f[5].expr | length) == 4 and ($f[5].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($f[5].expr[1] | named_set_match("ip"; "daddr"; "denied4")) and ($f[5].expr[2] | counter) and ($f[5].expr[3] | drop) and + ($f[6].expr | length) == 4 and ($f[6].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($f[6].expr[1] | meta_match("nfproto"; "=="; "ipv6")) and ($f[6].expr[2] | counter) and ($f[6].expr[3] | drop) and + ($f[7].expr | length) == 4 and ($f[7].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($f[7].expr[1] | anonymous_set_match("tcp"; "dport"; [25,465,587])) and ($f[7].expr[2] | counter) and ($f[7].expr[3] | drop) and + ($f[8].expr | length) == 3 and ($f[8].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($f[8].expr[1] | anonymous_set_match("tcp"; "dport"; [80,443])) and ($f[8].expr[2] | accept) and + ($f[9].expr | length) == 3 and ($f[9].expr[0] | meta_match("iifname"; "=="; "lecodex-host")) and ($f[9].expr[1] | counter) and ($f[9].expr[2] | drop) + ' >/dev/null +} + +stage3_nft_json_nat_rules_match() { + local raw=$1 + printf '%s' "$raw" | /usr/bin/jq -e ' + def payload_prefix_match: + .match? as $m | + ($m | type) == "object" and ($m | keys | sort) == ["left","op","right"] and + $m.left == {"payload":{"protocol":"ip","field":"saddr"}} and ($m.op == "==" or $m.op == "in") and + ($m.right == {"prefix":{"addr":"10.200.120.0","len":30}} or $m.right == "10.200.120.0/30"); + def oif_not_project: + .match? as $m | + ($m | type) == "object" and ($m | keys | sort) == ["left","op","right"] and + $m.left == {"meta":{"key":"oifname"}} and $m.op == "!=" and $m.right == "lecodex-host"; + def masquerade: has("masquerade") and (.masquerade == null or .masquerade == {}); + [.nftables[] | .rule? | select(.chain == "postrouting")] as $r | + ($r | length) == 1 and ($r[0].expr | length) == 3 and + ($r[0].expr[0] | payload_prefix_match) and ($r[0].expr[1] | oif_not_project) and ($r[0].expr[2] | masquerade) + ' >/dev/null +} + +stage3_expected_denied4() { + printf '%s\n' 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.0.0.0/24 192.0.2.0/24 192.88.99.0/24 192.168.0.0/16 198.18.0.0/15 198.51.100.0/24 203.0.113.0/24 224.0.0.0/4 240.0.0.0/4 | /usr/bin/sort -V +} + +stage3_project_inet_snapshot_valid() { + local raw=$1 text=$2 expected actual + stage3_nft_table_json_canonical "$raw" inet le_app_codex >/dev/null || return 1 + printf '%s' "$raw" | /usr/bin/jq -e ' + ([.nftables[] | .table? | select(.)][0] | + (keys - ["family","handle","name"] | length) == 0) and + ([.nftables[] | .set? | select(.)] as $sets | + ($sets | length) == 3 and + ([$sets[].name] | sort) == ["approved_dns4","denied4","operator_host_public4"] and + all($sets[]; + .family == "inet" and .table == "le_app_codex" and .type == "ipv4_addr" and .flags == ["interval"] and + (keys - ["elem","family","flags","handle","name","table","type"] | length) == 0)) and + ([.nftables[] | .chain? | select(.)] as $chains | + ($chains | length) == 2 and ([$chains[].name] | sort) == ["forward","input"] and + all($chains[]; + .family == "inet" and .table == "le_app_codex" and .type == "filter" and .hook == .name and + .prio == 0 and .policy == "accept" and + (keys - ["family","handle","hook","name","policy","prio","table","type"] | length) == 0)) and + ([.nftables[] | .rule? | select(.)] as $rules | + ($rules | length) == 14 and + ([$rules[] | select(.chain == "input")] | length) == 4 and + ([$rules[] | select(.chain == "forward")] | length) == 10 and + all($rules[]; + .family == "inet" and .table == "le_app_codex" and (.chain == "input" or .chain == "forward") and + (.expr | type) == "array" and + (keys - ["chain","expr","family","handle","table"] | length) == 0)) + ' >/dev/null || return 1 + stage3_nft_json_inet_rules_match "$raw" || return 1 + expected=$'9.9.9.9\n149.112.112.112' + actual=$(stage3_nft_text_set_ipv4_elements "$text" approved_dns4) || return 1 + [ "$actual" = "$expected" ] || return 1 + actual=$(stage3_nft_text_set_ipv4_elements "$text" operator_host_public4) || return 1 + [ "$actual" = "$expected_public_ipv4" ] || return 1 + expected=$(stage3_expected_denied4) || return 1 + actual=$(stage3_nft_text_set_ipv4_elements "$text" denied4) || return 1 + [ "$actual" = "$expected" ] || return 1 + if printf '%s\n' "$text" | /usr/bin/grep -Eiq 'dnat|redirect|prerouting'; then return 1; fi +} + +stage3_project_nat_snapshot_valid() { + local raw=$1 text=$2 + stage3_nft_table_json_canonical "$raw" ip le_app_codex_nat >/dev/null || return 1 + printf '%s' "$raw" | /usr/bin/jq -e ' + ([.nftables[] | .table? | select(.)][0] | + (keys - ["family","handle","name"] | length) == 0) and + ([.nftables[] | .set? | select(.)] | length) == 0 and + ([.nftables[] | .chain? | select(.)] as $chains | + ($chains | length) == 1 and $chains[0].family == "ip" and $chains[0].table == "le_app_codex_nat" and + $chains[0].name == "postrouting" and $chains[0].type == "nat" and $chains[0].hook == "postrouting" and + $chains[0].prio == 100 and $chains[0].policy == "accept" and + ($chains[0] | keys - ["family","handle","hook","name","policy","prio","table","type"] | length) == 0) and + ([.nftables[] | .rule? | select(.)] as $rules | + ($rules | length) == 1 and $rules[0].family == "ip" and $rules[0].table == "le_app_codex_nat" and + $rules[0].chain == "postrouting" and ($rules[0].expr | type) == "array" and + ($rules[0] | keys - ["chain","expr","family","handle","table"] | length) == 0) + ' >/dev/null || return 1 + stage3_nft_json_nat_rules_match "$raw" || return 1 + if printf '%s\n' "$text" | /usr/bin/grep -Eiq 'dnat|redirect|prerouting'; then return 1; fi +} + +stage3_query_namespaces() { /usr/bin/ip netns list 2>/dev/null; } +stage3_query_links_json() { /usr/bin/ip -d -j link show 2>/dev/null; } +stage3_query_tables_json() { /usr/bin/nft -a -j list tables 2>/dev/null; } + +stage3_namespace_inventory_valid() { + /usr/bin/awk ' + NF == 0 { next } + $1 !~ /^[[:alnum:]_.-]+$/ { exit 1 } + seen[$1]++ { exit 1 } + NF == 1 { next } + NF == 3 && $2 == "(id:" && $3 ~ /^[0-9]+\)$/ { next } + { exit 1 } + ' <<< "$1" +} + +stage3_capture_resource_inventory() { + local namespaces raw links tables + stage3_inventory_namespaces= stage3_inventory_links= stage3_inventory_tables= + namespaces=$(stage3_query_namespaces) || return 1 + stage3_namespace_inventory_valid "$namespaces" || return 1 + raw=$(stage3_query_links_json) || return 1 + links=$(printf '%s' "$raw" | /usr/bin/jq -e -c ' + if type == "array" and all(.[]; + type == "object" and (.ifname | type) == "string" and (.ifname | length) > 0 and + (.ifindex | type) == "number") + then . else error("invalid link inventory") end') || return 1 + raw=$(stage3_query_tables_json) || return 1 + tables=$(printf '%s' "$raw" | /usr/bin/jq -e -c ' + if type == "object" and (.nftables | type) == "array" and all(.nftables[]; + type == "object" and + (((keys == ["metainfo"]) and (.metainfo | type) == "object") or + ((keys == ["table"]) and (.table | type) == "object" and + (.table.family | type) == "string" and (.table.name | type) == "string" and + (.table.handle | type) == "number"))) + then [.nftables[] | select(has("table")) | .table | {family,name,handle}] + else error("invalid nft table inventory") end') || return 1 + stage3_inventory_namespaces=$namespaces + stage3_inventory_links=$links + stage3_inventory_tables=$tables +} + +stage3_inventory_has_namespace() { printf '%s\n' "$stage3_inventory_namespaces" | /usr/bin/awk '$1 == "le-app-codex" { found=1 } END { exit !found }'; } +stage3_inventory_has_link() { printf '%s' "$stage3_inventory_links" | /usr/bin/jq -e 'any(.[]; .ifname == "lecodex-host")' >/dev/null; } +stage3_inventory_has_host_ns_peer() { printf '%s' "$stage3_inventory_links" | /usr/bin/jq -e 'any(.[]; .ifname == "lecodex-ns")' >/dev/null; } +stage3_inventory_has_table() { printf '%s' "$stage3_inventory_tables" | /usr/bin/jq -e --arg family "$1" --arg name "$2" 'any(.[]; .family == $family and .name == $name)' >/dev/null; } +stage3_inventory_table_handle() { printf '%s' "$stage3_inventory_tables" | /usr/bin/jq -er --arg family "$1" --arg name "$2" '[.[] | select(.family == $family and .name == $name) | .handle] | select(length == 1) | .[0]'; } + +stage3_read_exact_project_policy_snapshot() { + local before_inet_handle before_nat_handle after_inet_handle after_nat_handle inet_raw nat_raw inet_text nat_text inet_canonical nat_canonical combined + stage3_snapshot_inet_handle= stage3_snapshot_nat_handle= stage3_snapshot_inet_sha256= stage3_snapshot_nat_sha256= stage3_snapshot_project_sha256= + stage3_snapshot_inet_canonical= stage3_snapshot_nat_canonical= + stage3_capture_resource_inventory || return 1 + before_inet_handle=$(stage3_inventory_table_handle inet le_app_codex) || return 1 + before_nat_handle=$(stage3_inventory_table_handle ip le_app_codex_nat) || return 1 + inet_raw=$(stage3_query_nft_table_json inet le_app_codex) || return 1 + nat_raw=$(stage3_query_nft_table_json ip le_app_codex_nat) || return 1 + inet_text=$(stage3_query_nft_table_text inet le_app_codex) || return 1 + nat_text=$(stage3_query_nft_table_text ip le_app_codex_nat) || return 1 + stage3_project_inet_snapshot_valid "$inet_raw" "$inet_text" || return 1 + stage3_project_nat_snapshot_valid "$nat_raw" "$nat_text" || return 1 + inet_canonical=$(stage3_nft_table_json_canonical "$inet_raw" inet le_app_codex) || return 1 + nat_canonical=$(stage3_nft_table_json_canonical "$nat_raw" ip le_app_codex_nat) || return 1 + combined=$(stage3_project_nft_canonical_from_tables "$inet_canonical" "$nat_canonical") || return 1 + stage3_capture_resource_inventory || return 1 + after_inet_handle=$(stage3_inventory_table_handle inet le_app_codex) || return 1 + after_nat_handle=$(stage3_inventory_table_handle ip le_app_codex_nat) || return 1 + [ "$before_inet_handle" = "$after_inet_handle" ] && [ "$before_nat_handle" = "$after_nat_handle" ] || return 1 + stage3_snapshot_inet_handle=$after_inet_handle + stage3_snapshot_nat_handle=$after_nat_handle + stage3_snapshot_inet_canonical=$inet_canonical + stage3_snapshot_nat_canonical=$nat_canonical + stage3_snapshot_inet_sha256=$(printf '%s' "$inet_canonical" | stage3_sha_stream) || return 1 + stage3_snapshot_nat_sha256=$(printf '%s' "$nat_canonical" | stage3_sha_stream) || return 1 + stage3_snapshot_project_sha256=$(printf '%s' "$combined" | stage3_sha_stream) || return 1 + [[ "$stage3_snapshot_inet_handle$stage3_snapshot_nat_handle" =~ ^[0-9]+[0-9]+$ ]] || return 1 + [[ "$stage3_snapshot_inet_sha256$stage3_snapshot_nat_sha256$stage3_snapshot_project_sha256" =~ ^[0-9a-f]{192}$ ]] +} + +stage3_capture_exact_project_policy_snapshot() { + local first_inet_handle first_nat_handle first_inet_canonical first_nat_canonical first_inet_sha256 first_nat_sha256 first_project_sha256 + stage3_read_exact_project_policy_snapshot || return 1 + first_inet_handle=$stage3_snapshot_inet_handle + first_nat_handle=$stage3_snapshot_nat_handle + first_inet_canonical=$stage3_snapshot_inet_canonical + first_nat_canonical=$stage3_snapshot_nat_canonical + first_inet_sha256=$stage3_snapshot_inet_sha256 + first_nat_sha256=$stage3_snapshot_nat_sha256 + first_project_sha256=$stage3_snapshot_project_sha256 + stage3_read_exact_project_policy_snapshot || return 1 + [ "$stage3_snapshot_inet_handle" = "$first_inet_handle" ] && + [ "$stage3_snapshot_nat_handle" = "$first_nat_handle" ] && + [ "$stage3_snapshot_inet_canonical" = "$first_inet_canonical" ] && + [ "$stage3_snapshot_nat_canonical" = "$first_nat_canonical" ] && + [ "$stage3_snapshot_inet_sha256" = "$first_inet_sha256" ] && + [ "$stage3_snapshot_nat_sha256" = "$first_nat_sha256" ] && + [ "$stage3_snapshot_project_sha256" = "$first_project_sha256" ] +} + +stage3_capture_active_topology_identity() { + local namespace_count namespace_stat pids pids_rc host_json ns_json host_addr ns_addr ns_ipv6 routes4 routes6 + local host_ifindex host_iflink host_mac peer_ifindex peer_iflink all_ipv6 default_ipv6 + stage3_active_namespace_dev_inode= stage3_active_host_ifindex= stage3_active_host_iflink= stage3_active_host_mac= + stage3_capture_resource_inventory || return 1 + namespace_count=$(printf '%s\n' "$stage3_inventory_namespaces" | /usr/bin/awk '$1 == "le-app-codex" { count++ } END { print count+0 }') || return 1 + [ "$namespace_count" -eq 1 ] && stage3_inventory_has_link && ! stage3_inventory_has_host_ns_peer || return 1 + [ -e /run/netns/le-app-codex ] && [ ! -L /run/netns/le-app-codex ] || return 1 + [ "$(/usr/bin/stat -f -c '%T' /run/netns/le-app-codex 2>/dev/null)" = nsfs ] || return 1 + namespace_stat=$(/usr/bin/stat -Lc '%D:%i' /run/netns/le-app-codex 2>/dev/null) || return 1 + [[ "$namespace_stat" =~ ^[0-9a-f]+:[0-9]+$ ]] || return 1 + pids=$(/usr/bin/ip netns pids le-app-codex 2>/dev/null); pids_rc=$? + [ "$pids_rc" -eq 0 ] && [ -z "$pids" ] || return 1 + + host_json=$(printf '%s' "$stage3_inventory_links" | /usr/bin/jq -ec '[.[] | select(.ifname == "lecodex-host")] | select(length == 1)') || return 1 + printf '%s' "$host_json" | /usr/bin/jq -e '.[0].operstate == "UP" and .[0].linkinfo.info_kind == "veth"' >/dev/null || return 1 + ns_json=$(/usr/bin/ip -n le-app-codex -d -j link show 2>/dev/null) || return 1 + printf '%s' "$ns_json" | /usr/bin/jq -e ' + type == "array" and length == 2 and all(.[]; type == "object" and (.ifname | type) == "string" and (.ifindex | type) == "number") and + ([.[].ifname] | sort) == ["lecodex-ns","lo"] and + ([.[] | select(.ifname == "lecodex-ns")] | length) == 1 and + ([.[] | select(.ifname == "lecodex-ns")][0] | .operstate == "UP" and .linkinfo.info_kind == "veth")' >/dev/null || return 1 + + host_ifindex=$(< /sys/class/net/lecodex-host/ifindex) || return 1 + host_iflink=$(< /sys/class/net/lecodex-host/iflink) || return 1 + host_mac=$(< /sys/class/net/lecodex-host/address) || return 1 + peer_ifindex=$(/usr/bin/ip netns exec le-app-codex /usr/bin/cat /sys/class/net/lecodex-ns/ifindex 2>/dev/null) || return 1 + peer_iflink=$(/usr/bin/ip netns exec le-app-codex /usr/bin/cat /sys/class/net/lecodex-ns/iflink 2>/dev/null) || return 1 + [[ "$host_ifindex" =~ ^[0-9]+$ ]] && [[ "$host_iflink" =~ ^[0-9]+$ ]] && [[ "$peer_ifindex" =~ ^[0-9]+$ ]] && [[ "$peer_iflink" =~ ^[0-9]+$ ]] || return 1 + [[ "$host_mac" =~ ^([0-9a-f]{2}:){5}[0-9a-f]{2}$ ]] || return 1 + [ "$host_ifindex" = "$peer_iflink" ] && [ "$host_iflink" = "$peer_ifindex" ] || return 1 + + host_addr=$(/usr/bin/ip -j -4 address show dev lecodex-host 2>/dev/null) || return 1 + ns_addr=$(/usr/bin/ip -n le-app-codex -j -4 address show dev lecodex-ns 2>/dev/null) || return 1 + printf '%s' "$host_addr" | /usr/bin/jq -e 'type == "array" and length == 1 and ([.[].addr_info[] | select(.family == "inet") | {local,prefixlen}]) == [{"local":"10.200.120.1","prefixlen":30}]' >/dev/null || return 1 + printf '%s' "$ns_addr" | /usr/bin/jq -e 'type == "array" and length == 1 and ([.[].addr_info[] | select(.family == "inet") | {local,prefixlen}]) == [{"local":"10.200.120.2","prefixlen":30}]' >/dev/null || return 1 + routes4=$(/usr/bin/ip -n le-app-codex -j -4 route show table main 2>/dev/null) || return 1 + printf '%s' "$routes4" | /usr/bin/jq -e 'type == "array" and length == 2 and any(.[]; .dst == "10.200.120.0/30" and .dev == "lecodex-ns") and any(.[]; .dst == "default" and .gateway == "10.200.120.1" and .dev == "lecodex-ns")' >/dev/null || return 1 + ns_ipv6=$(/usr/bin/ip -n le-app-codex -j -6 address show 2>/dev/null) || return 1 + routes6=$(/usr/bin/ip -n le-app-codex -j -6 route show table main 2>/dev/null) || return 1 + printf '%s' "$ns_ipv6" | /usr/bin/jq -e 'type == "array" and all(.[]; type == "object") and ([.[].addr_info[]? | select(.family == "inet6")] | length) == 0' >/dev/null || return 1 + printf '%s' "$routes6" | /usr/bin/jq -e 'type == "array" and length == 0' >/dev/null || return 1 + all_ipv6=$(/usr/bin/ip netns exec le-app-codex /usr/bin/sysctl -n net.ipv6.conf.all.disable_ipv6 2>/dev/null) || return 1 + default_ipv6=$(/usr/bin/ip netns exec le-app-codex /usr/bin/sysctl -n net.ipv6.conf.default.disable_ipv6 2>/dev/null) || return 1 + [ "$all_ipv6" = 1 ] && [ "$default_ipv6" = 1 ] || return 1 + + stage3_active_namespace_dev_inode=$namespace_stat + stage3_active_host_ifindex=$host_ifindex + stage3_active_host_iflink=$host_iflink + stage3_active_host_mac=$host_mac +} + +stage3_nft_text_set_ipv4_elements() { + local table_text=$1 set_name=$2 + printf '%s\n' "$table_text" | /usr/bin/awk -v set_name="$set_name" ' + $1 == "set" && $2 == set_name && $3 == "{" { inside=1; next } + inside && $0 ~ /^[[:space:]]*}/ { exit } + inside { print } + ' | /usr/bin/grep -Eo '([0-9]{1,3}[.]){3}[0-9]{1,3}(/[0-9]{1,2})?' | /usr/bin/sort -V +} + +stage3_host_addresses_json_canonical() { + printf '%s' "$1" | /usr/bin/jq -eS -c ' + if type == "array" and all(.[]; + type == "object" and (.ifname | type) == "string" and (.addr_info | type) == "array") + then map(select(.ifname != "lecodex-host" and .ifname != "lecodex-ns")) | + walk(if type == "object" then del(.stats,.stats64,.valid_life_time,.preferred_life_time) else . end) | + sort_by(.ifname) + else error("invalid address inventory") end' +} + +stage3_host_routes_json_canonical() { + printf '%s' "$1" | /usr/bin/jq -eS -c ' + if type == "array" and all(.[]; type == "object") + then map(select((.dev // "") != "lecodex-host" and (.dev // "") != "lecodex-ns")) | + walk(if type == "object" then del(.expires,.used) else . end) | sort_by(tostring) + else error("invalid route inventory") end' +} + +stage3_host_rules_json_canonical() { + printf '%s' "$1" | /usr/bin/jq -eS -c ' + if type == "array" and all(.[]; type == "object") then sort_by(tostring) + else error("invalid rule inventory") end' +} + +stage3_host_network_canonical() { + local raw addresses routes4 routes6 rules4 rules6 namespaces resolver resolver_stat resolver_hash + raw=$(/usr/bin/ip -j address show 2>/dev/null) || return 1 + addresses=$(stage3_host_addresses_json_canonical "$raw") || return 1 + raw=$(/usr/bin/ip -j -4 route show table all 2>/dev/null) || return 1 + routes4=$(stage3_host_routes_json_canonical "$raw") || return 1 + raw=$(/usr/bin/ip -j -6 route show table all 2>/dev/null) || return 1 + routes6=$(stage3_host_routes_json_canonical "$raw") || return 1 + raw=$(/usr/bin/ip -j -4 rule show 2>/dev/null) || return 1 + rules4=$(stage3_host_rules_json_canonical "$raw") || return 1 + raw=$(/usr/bin/ip -j -6 rule show 2>/dev/null) || return 1 + rules6=$(stage3_host_rules_json_canonical "$raw") || return 1 + raw=$(/usr/bin/ip netns list 2>/dev/null) || return 1 + stage3_namespace_inventory_valid "$raw" || return 1 + namespaces=$(printf '%s\n' "$raw" | /usr/bin/awk '$1 != "le-app-codex" { print }' | /usr/bin/sort) || return 1 + resolver_stat=$(/usr/bin/stat -c '%u:%g:%a:%F:%D:%i' /etc/resolv.conf 2>/dev/null) || return 1 + resolver_hash=$(/usr/bin/sha256sum /etc/resolv.conf 2>/dev/null | /usr/bin/cut -d' ' -f1) || return 1 + [[ "$resolver_hash" =~ ^[0-9a-f]{64}$ ]] || return 1 + resolver=$resolver_stat:$resolver_hash + /usr/bin/printf '%s\n%s\n%s\n%s\n%s\n%s\n%s\n' "$addresses" "$routes4" "$routes6" "$rules4" "$rules6" "$namespaces" "$resolver" +} + +stage3_docker_runtime_canonical() { + local ids_text raw containers units socket listeners id + local -a ids=() + ids_text=$(stage3_rootful_docker ps --no-trunc -aq 2>/dev/null) || return 1 + if [ -n "$ids_text" ]; then + while IFS= read -r id; do + [[ "$id" =~ ^[0-9a-f]{64}$ ]] || return 1 + ids+=("$id") + done <<< "$ids_text" + raw=$(stage3_rootful_docker inspect --format '{"Id":{{json .Id}},"Name":{{json .Name}},"ImageID":{{json .Image}},"Image":{{json .Config.Image}},"State":{"Status":{{json .State.Status}},"Running":{{json .State.Running}},"Paused":{{json .State.Paused}},"Restarting":{{json .State.Restarting}},"OOMKilled":{{json .State.OOMKilled}},"Dead":{{json .State.Dead}},"Pid":{{json .State.Pid}},"ExitCode":{{json .State.ExitCode}},"StartedAt":{{json .State.StartedAt}},"Health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}},"NetworkMode":{{json .HostConfig.NetworkMode}},"RestartPolicy":{{json .HostConfig.RestartPolicy}},"PortBindings":{{json .HostConfig.PortBindings}},"Ports":{{json .NetworkSettings.Ports}},"Networks":{{json .NetworkSettings.Networks}}}' "${ids[@]}" 2>/dev/null) || return 1 + containers=$(printf '%s\n' "$raw" | /usr/bin/jq -esS -c ' + if length > 0 and all(.[]; + type == "object" and (.Id | type) == "string" and (.Name | type) == "string" and + (.State | type) == "object" and (.Networks | type) == "object") + then sort_by(.Id) else error("invalid Docker runtime inventory") end') || return 1 + else + containers='[]' + fi + units=$(/usr/bin/systemctl show docker.service containerd.service --property=Id --property=LoadState --property=ActiveState --property=SubState --property=MainPID --property=InvocationID --no-pager 2>/dev/null) || return 1 + socket=$(/usr/bin/stat -Lc '%D:%i:%u:%g:%a:%F' /run/docker.sock 2>/dev/null) || return 1 + raw=$(/usr/bin/ss -H -lntup 2>/dev/null) || return 1 + listeners=$(printf '%s\n' "$raw" | /usr/bin/awk 'NF { print $1 "|" $2 "|" $5 "|" $7 }' | /usr/bin/sort) || return 1 + /usr/bin/printf '%s\n%s\n%s\n%s\n' "$containers" "$units" "$socket" "$listeners" +} + +stage3_sha_stream() { /usr/bin/sha256sum | /usr/bin/cut -d' ' -f1; } + +stage3_public_ipv4() { + /usr/bin/env -u ALL_PROXY -u HTTPS_PROXY -u HTTP_PROXY -u NO_PROXY -u all_proxy -u https_proxy -u http_proxy -u no_proxy /usr/bin/curl --noproxy '*' -4fsS --max-time 10 https://api.ipify.org 2>/dev/null +} + +stage3_materialize_docker_ipv4_gateways() { + local current_json=${1:-} committed_json current_list committed_list raw_count unique_count + stage3_docker_gateways=() + [ -n "$current_json" ] || current_json=$(stage3_docker_network_canonical) || return 1 + committed_json=$(stage3_committed_docker_network_canonical) || return 1 + current_list=$(printf '%s' "$current_json" | /usr/bin/jq -er '[.[].IPAM.Config[]? | .Gateway // empty | select(test(":" ) | not)] | sort | .[]') || return 1 + committed_list=$(printf '%s' "$committed_json" | /usr/bin/jq -er '[.[].IPAM.Config[]? | .Gateway // empty | select(test(":" ) | not)] | sort | .[]') || return 1 + raw_count=$(printf '%s\n' "$current_list" | /usr/bin/awk 'NF { count++ } END { print count+0 }') + unique_count=$(printf '%s\n' "$current_list" | /usr/bin/sort -u | /usr/bin/awk 'NF { count++ } END { print count+0 }') + [ "$raw_count" -eq 39 ] && [ "$unique_count" -eq 39 ] && [ "$current_list" = "$committed_list" ] || return 1 + mapfile -t stage3_docker_gateways <<< "$current_list" + [ "${#stage3_docker_gateways[@]}" -eq 39 ] +} + +stage3_capture_baseline() { + local committed_docker current_docker before=$failures + baseline_public_ipv4=$(stage3_public_ipv4) + [ "$baseline_public_ipv4" = "$expected_public_ipv4" ] && pass 'current public/NAT IPv4 pinned' || fail "public/NAT IPv4 drift ${baseline_public_ipv4:-unavailable}" + current_docker=$(stage3_docker_network_canonical) || current_docker= + committed_docker=$(stage3_committed_docker_network_canonical) || committed_docker= + if [ -n "$current_docker" ] && [ "$current_docker" = "$committed_docker" ] && [ "$(printf '%s' "$current_docker" | /usr/bin/jq 'length')" -eq 41 ]; then pass 'current rootful Docker network inventory matches committed 41-network inventory'; else fail 'current rootful Docker network inventory drift'; fi + baseline_docker_network_sha256=$(printf '%s' "$current_docker" | stage3_sha_stream) + baseline_nft_sha256=$(stage3_nonproject_nft_canonical | stage3_sha_stream) || baseline_nft_sha256= + baseline_host_network_sha256=$(stage3_host_network_canonical | stage3_sha_stream) || baseline_host_network_sha256= + baseline_docker_runtime_sha256=$(stage3_docker_runtime_canonical | stage3_sha_stream) || baseline_docker_runtime_sha256= + [[ "$baseline_nft_sha256$baseline_host_network_sha256$baseline_docker_runtime_sha256" =~ ^[0-9a-f]{192}$ ]] && pass 'fresh host/nft/rootful-Docker baselines captured' || fail 'fresh baseline capture failed' + [ "$(/usr/bin/sysctl -n net.ipv4.ip_forward 2>/dev/null)" = 1 ] && pass 'host IPv4 forwarding already enabled' || fail 'host IPv4 forwarding is not already enabled' + /usr/bin/ip -4 -o address show dev enp4s0 2>/dev/null | /usr/bin/grep -qw '192.168.10.151/24' && pass 'current LAN identity' || fail 'current LAN identity drift' + /usr/bin/ip -4 -o address show dev wg0 2>/dev/null | /usr/bin/grep -qw '10.98.0.2/24' && pass 'current WireGuard identity' || fail 'current WireGuard identity drift' + [ "$failures" -eq "$before" ] +} + +stage3_compare_baseline() { + local label=$1 public current before=$failures + public=$(stage3_public_ipv4) + [ "$public" = "$baseline_public_ipv4" ] && pass "$label public/NAT IPv4 unchanged" || fail "$label public/NAT IPv4 changed" + current=$(stage3_docker_network_canonical | stage3_sha_stream) && [ "$current" = "$baseline_docker_network_sha256" ] && pass "$label rootful Docker networks unchanged" || fail "$label rootful Docker network drift" + current=$(stage3_nonproject_nft_canonical | stage3_sha_stream) && [ "$current" = "$baseline_nft_sha256" ] && pass "$label nonproject nftables unchanged" || fail "$label nonproject nftables drift" + current=$(stage3_host_network_canonical | stage3_sha_stream) && [ "$current" = "$baseline_host_network_sha256" ] && pass "$label nonproject host network unchanged" || fail "$label nonproject host network drift" + current=$(stage3_docker_runtime_canonical | stage3_sha_stream) && [ "$current" = "$baseline_docker_runtime_sha256" ] && pass "$label rootful Docker runtime/listeners unchanged" || fail "$label rootful Docker runtime/listener drift" + [ "$failures" -eq "$before" ] +} + +stage3_compare_recorded_baseline() { + local label=$1 public current before=$failures + public=$(stage3_public_ipv4) + [ "$public" = "${stage3_marker_fields[CURRENT_PUBLIC_NAT_IPV4]:-}" ] && pass "$label recorded public/NAT IPv4 unchanged" || fail "$label recorded public/NAT IPv4 drift" + current=$(stage3_docker_network_canonical | stage3_sha_stream) && [ "$current" = "${stage3_marker_fields[BASELINE_DOCKER_NETWORK_SHA256]:-}" ] && pass "$label recorded rootful Docker networks unchanged" || fail "$label recorded rootful Docker network drift" + current=$(stage3_nonproject_nft_canonical | stage3_sha_stream) && [ "$current" = "${stage3_marker_fields[BASELINE_NFT_SHA256]:-}" ] && pass "$label recorded nonproject nftables unchanged" || fail "$label recorded nonproject nftables drift" + current=$(stage3_host_network_canonical | stage3_sha_stream) && [ "$current" = "${stage3_marker_fields[BASELINE_HOST_NETWORK_SHA256]:-}" ] && pass "$label recorded nonproject host network unchanged" || fail "$label recorded nonproject host network drift" + current=$(stage3_docker_runtime_canonical | stage3_sha_stream) && [ "$current" = "${stage3_marker_fields[BASELINE_DOCKER_RUNTIME_SHA256]:-}" ] && pass "$label recorded rootful Docker runtime/listeners unchanged" || fail "$label recorded rootful Docker runtime/listener drift" + [ "$failures" -eq "$before" ] +} + +stage3_verify_no_user_runtime() { + local label=$1 active sub user_show user_rc processes process_rc comms comm_rc codex_count before=$failures + if user_show=$(/usr/bin/systemctl show --no-pager user@1200.service --property=ActiveState --property=SubState 2>/dev/null); then user_rc=0; else user_rc=$?; fi + active=$(stage3_show_value "$user_show" ActiveState 2>/dev/null || true) + sub=$(stage3_show_value "$user_show" SubState 2>/dev/null || true) + [ "$user_rc" -eq 0 ] && [ "$active" = inactive ] && [ "$sub" = dead ] && pass "$label user@1200 inactive/dead" || fail "$label user@1200 query/state rc=$user_rc ${active:-unknown}/${sub:-unknown}" + if [ ! -e /var/lib/systemd/linger/le_app_codex ] && [ ! -L /var/lib/systemd/linger/le_app_codex ] && [ ! -e /var/lib/systemd/linger/1200 ] && [ ! -L /var/lib/systemd/linger/1200 ]; then pass "$label lingering absent"; else fail "$label lingering present"; fi + processes=$(/usr/bin/pgrep -u 1200 2>/dev/null); process_rc=$? + [ "$process_rc" -eq 1 ] && [ -z "$processes" ] && pass "$label UID 1200 processes absent" || fail "$label UID 1200 process query/presence rc=$process_rc ${processes:-}" + comms=$(/usr/bin/ps -eo comm= 2>/dev/null); comm_rc=$? + codex_count=$(printf '%s\n' "$comms" | /usr/bin/awk '$1 == "codex" { count++ } END { print count+0 }') + [ "$comm_rc" -eq 0 ] && [ "$codex_count" -eq 0 ] && pass "$label Codex-named processes absent" || fail "$label Codex process inventory failed or found count=$codex_count" + [ ! -e /run/user/1200/docker.sock ] && [ ! -L /run/user/1200/docker.sock ] && pass "$label rootless Docker socket absent" || fail "$label rootless Docker socket present" + [ "$failures" -eq "$before" ] +} + +stage3_show_value() { + local output=$1 property=$2 line value= count=0 + while IFS= read -r line; do + if [[ "$line" = "$property="* ]]; then value=${line#*=}; count=$((count + 1)); fi + done <<< "$output" + [ "$count" -eq 1 ] || return 1 + printf '%s' "$value" +} + +stage3_show_has_word() { + local output=$1 property=$2 expected=$3 value word + value=$(stage3_show_value "$output" "$property") || return 1 + for word in $value; do [ "$word" = "$expected" ] && return 0; done + return 1 +} + +stage3_show_exact() { + local output=$1 property=$2 expected=$3 value + value=$(stage3_show_value "$output" "$property") || return 1 + [ "$value" = "$expected" ] +} + +stage3_show_words_exact() { + local output=$1 property=$2 value actual expected + shift 2 + value=$(stage3_show_value "$output" "$property") || return 1 + actual=$(for word in $value; do printf '%s\n' "$word"; done | /usr/bin/sort) + expected=$(printf '%s\n' "$@" | /usr/bin/sort) + [ "$actual" = "$expected" ] +} + +stage3_inert_unit_identity() { + local output=$1 unit=$2 invocation=${3:-} activation=${4:-} token=${5:-} value + [[ "$token" =~ ^[0-9a-f]{32}$ ]] || return 1 + [ "$(stage3_show_value "$output" LoadState 2>/dev/null)" = loaded ] || return 1 + [ "$(stage3_show_value "$output" Transient 2>/dev/null)" = yes ] || return 1 + [ "$(stage3_show_value "$output" FragmentPath 2>/dev/null)" = "/run/systemd/transient/$unit" ] || return 1 + [ "$(stage3_show_value "$output" User 2>/dev/null)" = 1200 ] && [ "$(stage3_show_value "$output" Group 2>/dev/null)" = 1200 ] || return 1 + [ "$(stage3_show_value "$output" NetworkNamespacePath 2>/dev/null)" = /run/netns/le-app-codex ] || return 1 + [ -z "$activation" ] || [ "$(stage3_show_value "$output" Description 2>/dev/null)" = "Phase 2B Stage 3 inert containment $activation" ] || return 1 + stage3_show_words_exact "$output" UnsetEnvironment ALL_PROXY HTTPS_PROXY HTTP_PROXY NO_PROXY all_proxy https_proxy http_proxy no_proxy || return 1 + stage3_show_words_exact "$output" Environment "DOCKER_HOST=unix:///run/user/1200/docker.sock" "PHASE2B_STAGE3_INERT_TOKEN=$token" || return 1 + value=$(stage3_show_value "$output" ExecStart 2>/dev/null) || return 1 + [[ "$value" = \{\ path=/srv/le-app-codex/phase2b-tests/10-inert-containment.sh\ \;\ argv\[\]=/srv/le-app-codex/phase2b-tests/10-inert-containment.sh\ \;\ ignore_errors=no\ \;* ]] || return 1 + [[ "$value" != *'} ; {'* ]] && [ "$(printf '%s' "$value" | /usr/bin/grep -o 'path=' | /usr/bin/wc -l)" -eq 1 ] || return 1 + value=$(stage3_show_value "$output" InvocationID 2>/dev/null) || return 1 + [[ "$value" =~ ^[0-9a-f]{32}$ ]] || return 1 + [ -z "$invocation" ] || [ "$value" = "$invocation" ] +} + +stage3_inert_unit_unclaimed() { + local output=$1 + stage3_show_exact "$output" LoadState not-found && stage3_show_exact "$output" ActiveState inactive && stage3_show_exact "$output" FragmentPath '' +} + +stage3_veth_fields_match() { + [ "$1" = "$4" ] && [ "$2" = "$5" ] && [ "$3" = "$6" ] +} + +stage3_cleanup_probe_exact() { + local path=$1 expected_uid=${2:-1200} expected_gid=${3:-1200} expected_mode=${4:-644} actual + if [ ! -e "$path" ] && [ ! -L "$path" ]; then return 0; fi + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + actual=$(/usr/bin/stat -c '%u:%g:%a:%h:%s' "$path" 2>/dev/null) || return 1 + [ "$actual" = "$expected_uid:$expected_gid:$expected_mode:1:0" ] || return 1 + /usr/bin/unlink -- "$path" +} + +stage3_verify_loaded_units() { + local mode=${1:-inactive} analyze_output analyze_rc cat_output cat_rc headers netns_show netns_rc user_cat_output user_cat_rc user_headers user_header_set user_show user_rc value active sub before=$failures + [ "$mode" = inactive ] || [ "$mode" = runtime ] || return 1 + analyze_output=$(/usr/bin/systemd-analyze verify --recursive-errors=no le-app-codex-netns.service user@1200.service user-1200.slice 2>&1) + analyze_rc=$? + [ -z "$analyze_output" ] || printf '%s\n' "$analyze_output" + [ "$analyze_rc" -eq 0 ] && [ -z "$analyze_output" ] && pass 'loaded namespace/user units verify with no diagnostics' || fail "loaded namespace/user unit verification rc=$analyze_rc or emitted diagnostics" + + cat_output=$(/usr/bin/systemctl cat --no-pager le-app-codex-netns.service 2>&1); cat_rc=$? + headers=$(printf '%s\n' "$cat_output" | /usr/bin/grep -c '^# /' || true) + if [ "$cat_rc" -eq 0 ] && [ "$headers" -eq 1 ] && printf '%s\n' "$cat_output" | /usr/bin/grep -Fqx '# /etc/systemd/system/le-app-codex-netns.service'; then pass 'namespace unit has exact loaded fragment and no concatenated drop-in'; else fail 'namespace systemctl cat fragment/drop-in surface'; fi + printf '%s\n' "$cat_output" | /usr/bin/grep -Fqx 'ConditionPathExists=/etc/le-app-codex-runtime/OPERATOR-INPUTS-APPROVED' && pass 'namespace unit approval-marker condition' || fail 'namespace unit approval-marker condition' + printf '%s\n' "$cat_output" | /usr/bin/grep -Fqx 'Before=user@1200.service' && pass 'namespace unit ordering declaration' || fail 'namespace unit ordering declaration' + printf '%s\n' "$cat_output" | /usr/bin/grep -Fqx 'ExecStart=/usr/local/libexec/le-app-codex-netns up' && printf '%s\n' "$cat_output" | /usr/bin/grep -Fqx 'ExecStop=/usr/local/libexec/le-app-codex-netns down' && pass 'namespace unit declared start/stop commands' || fail 'namespace unit declared start/stop commands' + + netns_show=$(/usr/bin/systemctl show --no-pager le-app-codex-netns.service --property=Id --property=LoadState --property=ActiveState --property=SubState --property=UnitFileState --property=FragmentPath --property=DropInPaths --property=NeedDaemonReload --property=Job --property=Type --property=RemainAfterExit --property=ExecStart --property=ExecStop --property=Before 2>&1); netns_rc=$? + [ "$netns_rc" -eq 0 ] || fail "namespace systemctl show rc=$netns_rc" + active=$(stage3_show_value "$netns_show" ActiveState 2>/dev/null); sub=$(stage3_show_value "$netns_show" SubState 2>/dev/null) + if [ "$mode" = inactive ]; then + [ "$(stage3_show_value "$netns_show" Id 2>/dev/null)" = le-app-codex-netns.service ] && [ "$(stage3_show_value "$netns_show" LoadState 2>/dev/null)" = loaded ] && [ "$active" = inactive ] && [ "$sub" = dead ] && [ "$(stage3_show_value "$netns_show" UnitFileState 2>/dev/null)" = static ] && pass 'namespace unit loaded/inactive/dead/static' || fail 'namespace loaded state' + else + [ "$(stage3_show_value "$netns_show" Id 2>/dev/null)" = le-app-codex-netns.service ] && [ "$(stage3_show_value "$netns_show" LoadState 2>/dev/null)" = loaded ] && { [ "$active/$sub" = active/exited ] || [ "$active/$sub" = inactive/dead ] || [ "$active" = activating ] || [ "$active" = failed ]; } && [ "$(stage3_show_value "$netns_show" UnitFileState 2>/dev/null)" = static ] && pass "namespace unit loaded runtime state $active/$sub/static" || fail 'namespace runtime loaded state' + fi + stage3_show_exact "$netns_show" FragmentPath /etc/systemd/system/le-app-codex-netns.service && stage3_show_exact "$netns_show" DropInPaths '' && pass 'namespace loaded fragment exact with no drop-ins' || fail 'namespace fragment or unexpected drop-in' + if [ "$mode" = inactive ]; then + stage3_show_exact "$netns_show" NeedDaemonReload no && stage3_show_exact "$netns_show" Job '' && pass 'namespace manager cache current with no pending job' || fail 'namespace manager cache or pending job' + else + stage3_show_exact "$netns_show" NeedDaemonReload no && stage3_show_exact "$netns_show" Job '' && pass 'namespace manager cache current with no pending job' || fail 'namespace manager cache stale or pending job' + fi + [ "$(stage3_show_value "$netns_show" Type 2>/dev/null)" = oneshot ] && [ "$(stage3_show_value "$netns_show" RemainAfterExit 2>/dev/null)" = yes ] && pass 'namespace service type/retention exact' || fail 'namespace service type/retention' + value=$(stage3_show_value "$netns_show" ExecStart 2>/dev/null) + [[ "$value" = \{\ path=/usr/local/libexec/le-app-codex-netns\ \;\ argv\[\]=/usr/local/libexec/le-app-codex-netns\ up\ \;\ ignore_errors=no\ \;* ]] && [[ "$value" != *'} ; {'* ]] && [ "$(printf '%s' "$value" | /usr/bin/grep -o 'path=' | /usr/bin/wc -l)" -eq 1 ] && pass 'namespace loaded ExecStart exact' || fail 'namespace loaded ExecStart' + value=$(stage3_show_value "$netns_show" ExecStop 2>/dev/null) + [[ "$value" = \{\ path=/usr/local/libexec/le-app-codex-netns\ \;\ argv\[\]=/usr/local/libexec/le-app-codex-netns\ down\ \;\ ignore_errors=no\ \;* ]] && [[ "$value" != *'} ; {'* ]] && [ "$(printf '%s' "$value" | /usr/bin/grep -o 'path=' | /usr/bin/wc -l)" -eq 1 ] && pass 'namespace loaded ExecStop exact' || fail 'namespace loaded ExecStop' + stage3_show_has_word "$netns_show" Before user@1200.service && pass 'namespace loaded Before includes user manager' || fail 'namespace loaded Before ordering' + + user_cat_output=$(/usr/bin/systemctl cat --no-pager user@1200.service 2>&1); user_cat_rc=$? + user_headers=$(printf '%s\n' "$user_cat_output" | /usr/bin/grep -c '^# /' || true) + user_header_set=$(printf '%s\n' "$user_cat_output" | /usr/bin/awk '/^# \// { sub(/^# /, ""); print }' | /usr/bin/sort) + if [ "$user_cat_rc" -eq 0 ] && [ "$user_headers" -eq 3 ] && [ "$user_header_set" = $'/etc/systemd/system/user@1200.service.d/50-le-app-codex-containment.conf\n/usr/lib/systemd/system/user@.service\n/usr/lib/systemd/system/user@.service.d/10-login-barrier.conf' ]; then pass 'user manager has exact loaded fragment and two-drop-in surface'; else fail 'user manager systemctl cat fragment/drop-in surface'; fi + printf '%s\n' "$user_cat_output" | /usr/bin/grep -Fqx 'Requires=le-app-codex-netns.service' && printf '%s\n' "$user_cat_output" | /usr/bin/grep -Fqx 'After=le-app-codex-netns.service' && printf '%s\n' "$user_cat_output" | /usr/bin/grep -Fqx 'NetworkNamespacePath=/run/netns/le-app-codex' && pass 'user manager declared namespace dependency/order/path' || fail 'user manager declared namespace dependency/order/path' + + user_show=$(/usr/bin/systemctl show --no-pager user@1200.service --property=Id --property=LoadState --property=ActiveState --property=SubState --property=UnitFileState --property=FragmentPath --property=DropInPaths --property=NeedDaemonReload --property=Job --property=Type --property=ExecStart --property=Requires --property=After --property=NetworkNamespacePath 2>&1); user_rc=$? + [ "$user_rc" -eq 0 ] || fail "user manager systemctl show rc=$user_rc" + [ "$(stage3_show_value "$user_show" Id 2>/dev/null)" = user@1200.service ] && [ "$(stage3_show_value "$user_show" LoadState 2>/dev/null)" = loaded ] && [ "$(stage3_show_value "$user_show" ActiveState 2>/dev/null)" = inactive ] && [ "$(stage3_show_value "$user_show" SubState 2>/dev/null)" = dead ] && pass 'user manager loaded/inactive/dead' || fail 'user manager loaded state' + stage3_show_exact "$user_show" FragmentPath /usr/lib/systemd/system/user@.service && stage3_show_words_exact "$user_show" DropInPaths /usr/lib/systemd/system/user@.service.d/10-login-barrier.conf /etc/systemd/system/user@1200.service.d/50-le-app-codex-containment.conf && stage3_show_exact "$user_show" NetworkNamespacePath /run/netns/le-app-codex && pass 'user manager exact fragment/drop-ins/namespace' || fail 'user manager fragment, drop-ins, or namespace path' + stage3_show_exact "$user_show" UnitFileState static && stage3_show_exact "$user_show" NeedDaemonReload no && stage3_show_exact "$user_show" Job '' && stage3_show_exact "$user_show" Type notify-reload && pass 'user manager static/type/cache/job state exact' || fail 'user manager static/type/cache/job state' + value=$(stage3_show_value "$user_show" ExecStart 2>/dev/null) + [[ "$value" = \{\ path=/usr/lib/systemd/systemd\ \;\ argv\[\]=/usr/lib/systemd/systemd\ --user\ \;\ ignore_errors=no\ \;* ]] && [[ "$value" != *'} ; {'* ]] && [ "$(printf '%s' "$value" | /usr/bin/grep -o 'path=' | /usr/bin/wc -l)" -eq 1 ] && pass 'user manager loaded ExecStart exact' || fail 'user manager loaded ExecStart' + stage3_show_has_word "$user_show" Requires le-app-codex-netns.service && stage3_show_has_word "$user_show" After le-app-codex-netns.service && pass 'user manager loaded namespace dependency/order' || fail 'user manager namespace dependency/order' + [ "$failures" -eq "$before" ] +} + +stage3_verify_pre_activation_inert() { + local active sub enabled service_show service_rc enabled_rc inert_units inert_rc before=$failures + [ ! -e "$marker" ] && [ ! -L "$marker" ] && [ ! -e "$marker_temporary" ] && [ ! -L "$marker_temporary" ] && [ ! -e "$rollback_record" ] && [ ! -L "$rollback_record" ] && pass 'approval marker, temporary, and rollback record absent' || fail 'marker transaction path present' + if service_show=$(/usr/bin/systemctl show --no-pager le-app-codex-netns.service --property=ActiveState --property=SubState 2>/dev/null); then service_rc=0; else service_rc=$?; fi + active=$(stage3_show_value "$service_show" ActiveState 2>/dev/null || true) + sub=$(stage3_show_value "$service_show" SubState 2>/dev/null || true) + if enabled=$(/usr/bin/systemctl is-enabled le-app-codex-netns.service 2>/dev/null); then enabled_rc=0; else enabled_rc=$?; fi + [ "$service_rc" -eq 0 ] && { [ "$enabled_rc" -eq 0 ] || [ "$enabled_rc" -eq 1 ]; } && [ "$active" = inactive ] && [ "$sub" = dead ] && [ "$enabled" = static ] && pass 'namespace service inactive/dead/static' || fail "namespace service pre-query/state rc=$service_rc/$enabled_rc $active/$sub/$enabled" + if stage3_capture_resource_inventory; then + ! stage3_inventory_has_namespace && [ ! -e /run/netns/le-app-codex ] && [ ! -L /run/netns/le-app-codex ] && pass 'project namespace absent from successful inventory' || fail 'project namespace present' + ! stage3_inventory_has_link && ! stage3_inventory_has_host_ns_peer && pass 'project veth names absent from successful inventory' || fail 'project veth name present' + ! stage3_inventory_has_table inet le_app_codex && ! stage3_inventory_has_table ip le_app_codex_nat && pass 'project nftables tables absent from successful inventory' || fail 'project nftables table present' + else + fail 'pre-activation namespace/link/nft inventory command failed' + fi + inert_units=$(/usr/bin/systemctl list-units --all --plain --no-legend "$inert_unit_prefix*.service" 2>/dev/null); inert_rc=$? + [ "$inert_rc" -eq 0 ] && [ -z "$inert_units" ] && pass 'inert transient unit namespace is unclaimed' || fail 'inert transient unit collision or query failure' + [ ! -e "$inert_probe" ] && [ ! -L "$inert_probe" ] && pass 'inert write-probe path absent' || fail 'inert write-probe path present' + stage3_verify_no_user_runtime 'pre-activation' || true + [ "$failures" -eq "$before" ] +} + +stage3_atomic_publish_new() { + local source=$1 target=$2 source_id target_id + [ -f "$source" ] && [ ! -L "$source" ] && [ ! -e "$target" ] && [ ! -L "$target" ] || return 1 + [ "$(/usr/bin/stat -c '%D' "$source" 2>/dev/null)" = "$(/usr/bin/stat -c '%D' "$(dirname -- "$target")" 2>/dev/null)" ] || return 1 + source_id=$(/usr/bin/stat -c '%D:%i' "$source" 2>/dev/null) || return 1 + /usr/bin/mv --no-clobber -T -- "$source" "$target" || return 1 + [ ! -e "$source" ] && [ ! -L "$source" ] && [ -f "$target" ] && [ ! -L "$target" ] || return 1 + target_id=$(/usr/bin/stat -c '%D:%i' "$target" 2>/dev/null) || return 1 + [ "$target_id" = "$source_id" ] +} + +stage3_sync_parent() { + local path=$1 + /usr/bin/sync -f "$(dirname -- "$path")" 2>/dev/null || /usr/bin/sync +} + +stage3_render_record() { + local status=$1 + printf 'FORMAT|le-app-codex-phase2b-stage3-v2\n' + printf 'STATUS|%s\n' "$status" + printf 'ACTIVATION_ID|%s\n' "$stage3_activation_id" + printf 'MACHINE_ID_SHA256|%s\n' "$stage3_machine_id_sha256" + printf 'APPROVED_DNS4|%s\n' "$approved_dns4" + printf 'APPROVED_ARTIFACT_MANIFEST_SHA256|%s\n' "$expected_stage1_artifact_digest" + printf 'STAGE1_SOURCE_COMMIT|%s\n' "$expected_stage1_source_commit" + printf 'STAGE2_VERIFICATION_COMMIT|%s\n' "$expected_stage2_commit" + printf 'STAGE3_ACTIVATION_COMMIT|%s\n' "$stage3_head" + printf 'TIMESTAMP|%s\n' "$stage3_timestamp" + printf 'HOSTNAME|%s\n' "$stage3_hostname" + printf 'BOOT_ID|%s\n' "$stage3_boot_id" + printf 'CURRENT_PUBLIC_NAT_IPV4|%s\n' "$baseline_public_ipv4" + printf 'POLICY_SHA256|%s\n' "$expected_policy_digest" + printf 'BASELINE_NFT_SHA256|%s\n' "$baseline_nft_sha256" + printf 'BASELINE_DOCKER_NETWORK_SHA256|%s\n' "$baseline_docker_network_sha256" + printf 'BASELINE_HOST_NETWORK_SHA256|%s\n' "$baseline_host_network_sha256" + printf 'BASELINE_DOCKER_RUNTIME_SHA256|%s\n' "$baseline_docker_runtime_sha256" + if [ "$status" = ACTIVE ]; then + printf 'SERVICE_INVOCATION_ID|%s\n' "$stage3_service_invocation_id" + printf 'NAMESPACE_DEV_INODE|%s\n' "$stage3_namespace_dev_inode" + printf 'HOST_VETH_IFINDEX|%s\n' "$stage3_host_veth_ifindex" + printf 'HOST_VETH_IFLINK|%s\n' "$stage3_host_veth_iflink" + printf 'HOST_VETH_MAC|%s\n' "$stage3_host_veth_mac" + printf 'NFT_INET_HANDLE|%s\n' "$stage3_nft_inet_handle" + printf 'NFT_NAT_HANDLE|%s\n' "$stage3_nft_nat_handle" + printf 'NFT_INET_SHA256|%s\n' "$stage3_nft_inet_sha256" + printf 'NFT_NAT_SHA256|%s\n' "$stage3_nft_nat_sha256" + printf 'PROJECT_NFT_SHA256|%s\n' "$stage3_project_nft_sha256" + fi +} + +stage3_record_common_sha256() { + local key + for key in FORMAT ACTIVATION_ID MACHINE_ID_SHA256 APPROVED_DNS4 APPROVED_ARTIFACT_MANIFEST_SHA256 STAGE1_SOURCE_COMMIT STAGE2_VERIFICATION_COMMIT STAGE3_ACTIVATION_COMMIT TIMESTAMP HOSTNAME BOOT_ID CURRENT_PUBLIC_NAT_IPV4 POLICY_SHA256 BASELINE_NFT_SHA256 BASELINE_DOCKER_NETWORK_SHA256 BASELINE_HOST_NETWORK_SHA256 BASELINE_DOCKER_RUNTIME_SHA256; do + [ -n "${stage3_marker_fields[$key]:-}" ] || return 1 + printf '%s|%s\n' "$key" "${stage3_marker_fields[$key]}" + done | stage3_sha_stream +} + +stage3_parse_record_file() { + local LC_ALL=C record=$1 require_provenance=${2:-1} line key value expected_count marker_commit stage3_relative record_size parsed_size=0 + declare -gA stage3_marker_fields=() + [ -f "$record" ] && [ ! -L "$record" ] || return 1 + record_size=$(/usr/bin/stat -c '%s' -- "$record" 2>/dev/null) || return 1 + [[ "$record_size" =~ ^[0-9]+$ ]] && [ "$record_size" -gt 0 ] || return 1 + while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] && [[ "$line" != *$'\r'* ]] || return 1 + parsed_size=$((parsed_size + ${#line} + 1)) + [[ "$line" =~ ^([^|]+)\|([^|]+)$ ]] || return 1 + key=${BASH_REMATCH[1]} + value=${BASH_REMATCH[2]} + [[ "$value" != */* ]] && [[ "$value" != *'..'* ]] || return 1 + case "$key" in + FORMAT|STATUS|ACTIVATION_ID|MACHINE_ID_SHA256|APPROVED_DNS4|APPROVED_ARTIFACT_MANIFEST_SHA256|STAGE1_SOURCE_COMMIT|STAGE2_VERIFICATION_COMMIT|STAGE3_ACTIVATION_COMMIT|TIMESTAMP|HOSTNAME|BOOT_ID|CURRENT_PUBLIC_NAT_IPV4|POLICY_SHA256|BASELINE_NFT_SHA256|BASELINE_DOCKER_NETWORK_SHA256|BASELINE_HOST_NETWORK_SHA256|BASELINE_DOCKER_RUNTIME_SHA256|SERVICE_INVOCATION_ID|NAMESPACE_DEV_INODE|HOST_VETH_IFINDEX|HOST_VETH_IFLINK|HOST_VETH_MAC|NFT_INET_HANDLE|NFT_NAT_HANDLE|NFT_INET_SHA256|NFT_NAT_SHA256|PROJECT_NFT_SHA256) ;; + *) return 1 ;; + esac + [ -z "${stage3_marker_fields[$key]:-}" ] || return 1 + stage3_marker_fields["$key"]=$value + done < "$record" + [ "$parsed_size" -eq "$record_size" ] || return 1 + [ "${stage3_marker_fields[FORMAT]:-}" = le-app-codex-phase2b-stage3-v2 ] || return 1 + [ "${stage3_marker_fields[STATUS]:-}" = PREPARED ] || [ "${stage3_marker_fields[STATUS]:-}" = ACTIVE ] || return 1 + [[ "${stage3_marker_fields[ACTIVATION_ID]:-}" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] || return 1 + [[ "${stage3_marker_fields[MACHINE_ID_SHA256]:-}" =~ ^[0-9a-f]{64}$ ]] || return 1 + [ "${stage3_marker_fields[APPROVED_DNS4]:-}" = "$approved_dns4" ] || return 1 + [ "${stage3_marker_fields[APPROVED_ARTIFACT_MANIFEST_SHA256]:-}" = "$expected_stage1_artifact_digest" ] || return 1 + [ "${stage3_marker_fields[STAGE1_SOURCE_COMMIT]:-}" = "$expected_stage1_source_commit" ] || return 1 + [ "${stage3_marker_fields[STAGE2_VERIFICATION_COMMIT]:-}" = "$expected_stage2_commit" ] || return 1 + [ "${stage3_marker_fields[POLICY_SHA256]:-}" = "$expected_policy_digest" ] || return 1 + [ "${stage3_marker_fields[CURRENT_PUBLIC_NAT_IPV4]:-}" = "$expected_public_ipv4" ] || return 1 + [[ "${stage3_marker_fields[TIMESTAMP]:-}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || return 1 + [[ "${stage3_marker_fields[STAGE3_ACTIVATION_COMMIT]:-}" =~ ^[0-9a-f]{40}$ ]] || return 1 + [[ "${stage3_marker_fields[BOOT_ID]:-}" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] || return 1 + [[ "${stage3_marker_fields[BASELINE_NFT_SHA256]:-}${stage3_marker_fields[BASELINE_DOCKER_NETWORK_SHA256]:-}${stage3_marker_fields[BASELINE_HOST_NETWORK_SHA256]:-}${stage3_marker_fields[BASELINE_DOCKER_RUNTIME_SHA256]:-}" =~ ^[0-9a-f]{256}$ ]] || return 1 + if [ "${stage3_marker_fields[STATUS]}" = ACTIVE ]; then + expected_count=28 + [[ "${stage3_marker_fields[SERVICE_INVOCATION_ID]:-}" =~ ^[0-9a-f]{32}$ ]] || return 1 + [[ "${stage3_marker_fields[NAMESPACE_DEV_INODE]:-}" =~ ^[0-9a-f]+:[0-9]+$ ]] || return 1 + [[ "${stage3_marker_fields[HOST_VETH_IFINDEX]:-}" =~ ^[0-9]+$ ]] || return 1 + [[ "${stage3_marker_fields[HOST_VETH_IFLINK]:-}" =~ ^[0-9]+$ ]] || return 1 + [[ "${stage3_marker_fields[HOST_VETH_MAC]:-}" =~ ^([0-9a-f]{2}:){5}[0-9a-f]{2}$ ]] || return 1 + [[ "${stage3_marker_fields[NFT_INET_HANDLE]:-}" =~ ^[0-9]+$ ]] || return 1 + [[ "${stage3_marker_fields[NFT_NAT_HANDLE]:-}" =~ ^[0-9]+$ ]] || return 1 + [[ "${stage3_marker_fields[NFT_INET_SHA256]:-}${stage3_marker_fields[NFT_NAT_SHA256]:-}${stage3_marker_fields[PROJECT_NFT_SHA256]:-}" =~ ^[0-9a-f]{192}$ ]] || return 1 + else + expected_count=18 + fi + [ "${#stage3_marker_fields[@]}" -eq "$expected_count" ] || return 1 + [ "$require_provenance" -eq 1 ] || return 0 + [ "${stage3_marker_fields[HOSTNAME]:-}" = "$(/usr/bin/hostname)" ] || return 1 + [ "${stage3_marker_fields[MACHINE_ID_SHA256]}" = "$(/usr/bin/sha256sum /etc/machine-id 2>/dev/null | /usr/bin/cut -d' ' -f1)" ] || return 1 + marker_commit=${stage3_marker_fields[STAGE3_ACTIVATION_COMMIT]} + stage3_relative=${stage3_root#"$repo_root"/} + /usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" cat-file -e "$marker_commit^{commit}" 2>/dev/null || return 1 + /usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" merge-base --is-ancestor "$marker_commit" "${stage3_head:-HEAD}" 2>/dev/null || return 1 + /usr/bin/git -c safe.directory="$repo_root" -C "$repo_root" diff --quiet "$marker_commit" "${stage3_head:-HEAD}" -- "$stage3_relative" || return 1 +} + +stage3_read_record() { + local record=$1 stat links + [ "$record" = "$marker" ] || [ "$record" = "$rollback_record" ] || return 1 + stat=$(/usr/bin/stat -c '%u:%g:%a:%F' "$record" 2>/dev/null) + links=$(/usr/bin/stat -c '%h' "$record" 2>/dev/null) + [ "$stat" = '0:0:600:regular file' ] && [ "$links" = 1 ] || return 1 + stage3_parse_record_file "$record" 1 +} + +stage3_read_marker() { stage3_read_record "$marker"; } +stage3_read_rollback_record() { stage3_read_record "$rollback_record"; } + +stage3_write_marker() { + local status=$1 publish_mode=create old_activation= old_hash temporary_id + [ "$status" = PREPARED ] || [ "$status" = ACTIVE ] || return 1 + stage3_verify_lock_parent || return 1 + [ ! -e "$marker_temporary" ] && [ ! -L "$marker_temporary" ] && [ ! -e "$rollback_record" ] && [ ! -L "$rollback_record" ] || return 1 + if [ "$status" = PREPARED ]; then + [ ! -e "$marker" ] && [ ! -L "$marker" ] || return 1 + else + stage3_read_marker || return 1 + [ "${stage3_marker_fields[STATUS]}" = PREPARED ] || return 1 + old_activation=${stage3_marker_fields[ACTIVATION_ID]} + old_hash=$(/usr/bin/sha256sum "$marker" | /usr/bin/cut -d' ' -f1) || return 1 + publish_mode=replace + fi + /usr/bin/install -o 0 -g 0 -m 0600 -T /dev/null "$marker_temporary" || return 1 + stage3_render_record "$status" > "$marker_temporary" || return 1 + /usr/bin/chown 0:0 "$marker_temporary" && /usr/bin/chmod 0600 "$marker_temporary" || return 1 + [ "$(/usr/bin/stat -c '%u:%g:%a:%F:%h' "$marker_temporary" 2>/dev/null)" = '0:0:600:regular file:1' ] || return 1 + stage3_parse_record_file "$marker_temporary" 1 || return 1 + [ -z "$old_activation" ] || [ "${stage3_marker_fields[ACTIVATION_ID]}" = "$old_activation" ] || return 1 + /usr/bin/sync -f "$marker_temporary" 2>/dev/null || /usr/bin/sync || return 1 + temporary_id=$(/usr/bin/stat -c '%D:%i' "$marker_temporary" 2>/dev/null) || return 1 + if [ "$publish_mode" = create ]; then + stage3_atomic_publish_new "$marker_temporary" "$marker" || return 1 + else + stage3_read_marker || return 1 + [ "${stage3_marker_fields[STATUS]}" = PREPARED ] && [ "${stage3_marker_fields[ACTIVATION_ID]}" = "$old_activation" ] || return 1 + [ "$old_hash" = "$(/usr/bin/sha256sum "$marker" 2>/dev/null | /usr/bin/cut -d' ' -f1)" ] || return 1 + /usr/bin/mv -T -- "$marker_temporary" "$marker" || return 1 + [ ! -e "$marker_temporary" ] && [ "$(/usr/bin/stat -c '%D:%i' "$marker" 2>/dev/null)" = "$temporary_id" ] || return 1 + fi + stage3_sync_parent "$marker" || return 1 + stage3_read_marker && [ "${stage3_marker_fields[STATUS]}" = "$status" ] +} + +stage3_promote_marker_to_tombstone() { + local marker_id + stage3_verify_lock_parent || return 1 + [ ! -e "$rollback_record" ] && [ ! -L "$rollback_record" ] || return 1 + stage3_read_marker || return 1 + marker_id=$(/usr/bin/stat -c '%D:%i' "$marker" 2>/dev/null) || return 1 + stage3_atomic_publish_new "$marker" "$rollback_record" || return 1 + [ "$(/usr/bin/stat -c '%D:%i' "$rollback_record" 2>/dev/null)" = "$marker_id" ] || return 1 + stage3_sync_parent "$rollback_record" || return 1 + stage3_read_rollback_record +} + +stage3_lock_descriptor_matches() { + local fd=$1 path=$2 expected_uid=${3:-0} expected_gid=${4:-0} expected_mode=${5:-755} fd_stat path_stat + [[ "$fd" =~ ^[0-9]+$ ]] && [ -e "/proc/self/fd/$fd" ] || return 1 + fd_stat=$(/usr/bin/stat -Lc '%D:%i:%u:%g:%a:%F' "/proc/self/fd/$fd" 2>/dev/null) || return 1 + path_stat=$(/usr/bin/stat -c '%D:%i:%u:%g:%a:%F' "$path" 2>/dev/null) || return 1 + [ "$fd_stat" = "$path_stat" ] && [[ "$path_stat" = *":$expected_uid:$expected_gid:$expected_mode:directory" ]] +} + +stage3_validate_inherited_lock() { + local fd=$1 path=$2 token=$3 expected_uid=${4:-0} expected_gid=${5:-0} expected_mode=${6:-755} current_id + stage3_lock_descriptor_matches "$fd" "$path" "$expected_uid" "$expected_gid" "$expected_mode" || return 1 + current_id=$(/usr/bin/stat -c '%D:%i' "$path" 2>/dev/null) || return 1 + [ "$token" = "$current_id" ] || return 1 + /usr/bin/flock -n "$fd" || return 1 + stage3_lock_descriptor_matches "$fd" "$path" "$expected_uid" "$expected_gid" "$expected_mode" || return 1 + [ "$current_id" = "$(/usr/bin/stat -c '%D:%i' "$path" 2>/dev/null)" ] +} + +stage3_acquire_lock() { + local current_id + stage3_verify_lock_parent || return 1 + if [ "${PHASE2B_STAGE3_LOCK_HELD:-0}" = 1 ]; then + stage3_validate_inherited_lock 9 /etc/le-app-codex-runtime "${PHASE2B_STAGE3_LOCK_ID:-}" || return 1 + else + exec 9< /etc/le-app-codex-runtime || return 1 + stage3_lock_descriptor_matches 9 /etc/le-app-codex-runtime || return 1 + /usr/bin/flock -n 9 || return 1 + stage3_lock_descriptor_matches 9 /etc/le-app-codex-runtime || return 1 + current_id=$(/usr/bin/stat -c '%D:%i' /etc/le-app-codex-runtime 2>/dev/null) || return 1 + PHASE2B_STAGE3_LOCK_ID=$current_id + fi +}