fix: harden recovery flow and clean diagnostics

This commit is contained in:
Alexander Sellite 2026-09-01 07:44:01 -04:00
parent 0158b0215f
commit ff78710303
64 changed files with 1163 additions and 418 deletions

View file

@ -1042,9 +1042,11 @@ func _valid_json_integer(value: Variant, minimum: int, maximum: int) -> bool:
)
func _make_social_request(name: String, callback: Callable) -> HTTPRequest:
func _make_social_request(
request_name: String, callback: Callable
) -> HTTPRequest:
var request := HTTPRequest.new()
request.name = name
request.name = request_name
request.timeout = REQUEST_TIMEOUT_SECONDS
add_child(request)
request.request_completed.connect(callback)

View file

@ -340,11 +340,12 @@ func _handle_request(peer_id: int, data: Dictionary) -> void:
message["request_id"] = request_id
message["sender_fingerprint"] = data["sender_fingerprint"]
message["sender_signature"] = data["sender_signature"]
ledger[request_id] = (
message.duplicate(true)
if _should_store_host_history()
else true
)
# `true` is the intentional no-history replay sentinel. It records that the
# request was handled without retaining a message payload for replay.
var ledger_entry: Variant = true
if _should_store_host_history():
ledger_entry = message.duplicate(true)
ledger[request_id] = ledger_entry
while ledger.size() > 64:
ledger.erase(ledger.keys().front())
_request_ledgers[peer_id] = ledger

View file

@ -29,6 +29,11 @@ const ANIMATION_REFRESH_INTERVAL: float = 1.0
const MAX_MOVEMENT_INPUT_SEQUENCE: int = 2147483647
const MAX_MOVEMENT_ONE_WAY_TRANSIT_SECONDS: float = 0.25
const SPACE_TRANSITION_SNAPSHOT_GUARD_DISTANCE: float = 12.0
# A reliable teleport travels on a different channel from ordinary movement
# audits. Keep the acceptance radius below Player's hard-reconciliation
# distance so an old audit can never pull a just-relocated local player back
# into the place they were recovered from.
const AUTHORITATIVE_TELEPORT_SNAPSHOT_GUARD_DISTANCE: float = 2.0
const MOVEMENT_FLAG_JUMP: int = 1 << 0
const MOVEMENT_FLAG_SPRINT: int = 1 << 1
@ -77,7 +82,11 @@ signal server_trust_required(
signal peer_identity_observed(peer_id: int, status: String)
signal operator_status_changed(peer_id: int, is_operator: bool)
signal server_lost(message: String)
signal remote_recovery_requested(peer_id: int, entry_position: Vector3)
signal remote_recovery_requested(
peer_id: int,
entry_position: Vector3,
recovery_input_sequence: int,
)
signal remote_recovery_presentation_changed(
peer_id: int,
active: bool,
@ -128,6 +137,9 @@ var _last_local_snapshot_received_msec: int = 0
var _local_space_transition_guard_active: bool = false
var _local_space_transition_guard_position: Vector3 = Vector3.ZERO
var _local_space_transition_minimum_ack: int = 0
var _local_authoritative_teleport_guard_active: bool = false
var _local_authoritative_teleport_guard_position: Vector3 = Vector3.ZERO
var _local_authoritative_teleport_minimum_ack: int = 0
var _animation_refresh_accumulator: float = 0.0
var _last_animation_state_by_peer: Dictionary[int, Dictionary] = {}
var _pending_animation_state_by_peer: Dictionary[int, Dictionary] = {}
@ -601,20 +613,20 @@ func get_last_server_metadata() -> Dictionary:
}
func set_host_world_seed(seed: int) -> bool:
return set_host_world(WorldLayout.GENERATED, seed)
func set_host_world_seed(world_seed: int) -> bool:
return set_host_world(WorldLayout.GENERATED, world_seed)
func set_host_world(world_layout: StringName, seed: int) -> bool:
func set_host_world(world_layout: StringName, world_seed: int) -> bool:
if (
state != State.INACTIVE
or not WorldLayout.is_valid(world_layout)
or seed <= 0
or seed > NetworkProtocol.MAX_WORLD_SEED
or world_seed <= 0
or world_seed > NetworkProtocol.MAX_WORLD_SEED
):
return false
_host_world_layout = world_layout
_host_world_seed = seed
_host_world_seed = world_seed
return true
@ -2088,10 +2100,10 @@ func _maybe_send_local_input() -> void:
if avatar == null:
return
var state_hash: int = avatar.get_network_input_state_hash()
var state_changed: bool = state_hash != _last_input_state_hash
var input_state_changed: bool = state_hash != _last_input_state_hash
if (
avatar.has_active_network_input()
or state_changed
or input_state_changed
or _idle_input_accumulator >= IDLE_INPUT_INTERVAL
):
_send_local_input()
@ -2125,8 +2137,8 @@ func _maybe_send_local_animation_action() -> void:
var avatar: Player = _spawn_service.get_avatar(peer_id)
if avatar == null:
return
var state: Dictionary = avatar.make_network_animation_state()
var action: Dictionary = state.get("action", {})
var animation_state: Dictionary = avatar.make_network_animation_state()
var action: Dictionary = animation_state.get("action", {})
if not NetworkPlayerAnimationProtocol.validate_action_state(action):
return
var signature: Array = [
@ -2182,11 +2194,15 @@ func submit_movement_animation_action(encoded: Array) -> void:
if not is_host() or not _registry.has_peer(sender_id):
return
var avatar: Player = _spawn_service.get_avatar(sender_id)
var state: Dictionary = _decode_movement_animation_action(encoded)
if avatar == null or state.is_empty():
var animation_state: Dictionary = _decode_movement_animation_action(encoded)
if avatar == null or animation_state.is_empty():
return
avatar.apply_authoritative_network_animation_action(state["action"])
avatar.apply_authoritative_network_sitting_state(bool(state["sitting"]))
avatar.apply_authoritative_network_animation_action(
animation_state["action"]
)
avatar.apply_authoritative_network_sitting_state(
bool(animation_state["sitting"])
)
static func _encode_movement_input(data: Dictionary) -> Array:
@ -2386,18 +2402,20 @@ func _broadcast_movement_animation_updates(peer_ids: Array[int]) -> void:
var avatar: Player = _spawn_service.get_avatar(subject_id)
if avatar == null:
continue
var state: Dictionary = avatar.make_network_animation_state()
var animation_state: Dictionary = avatar.make_network_animation_state()
var previous: Dictionary = _last_animation_state_by_peer.get(
subject_id, {}
)
if (
not refresh_all
and _movement_animation_signature(state)
and _movement_animation_signature(animation_state)
== _movement_animation_signature(previous)
):
continue
_last_animation_state_by_peer[subject_id] = state.duplicate(true)
var encoded: Array = _encode_movement_animation(subject_id, state)
_last_animation_state_by_peer[subject_id] = animation_state.duplicate(true)
var encoded: Array = _encode_movement_animation(
subject_id, animation_state
)
if not encoded.is_empty():
updates_by_subject[subject_id] = encoded
if updates_by_subject.is_empty():
@ -2416,13 +2434,13 @@ func _broadcast_movement_animation_updates(peer_ids: Array[int]) -> void:
_movement_animation_states_sent += updates.size()
static func _movement_animation_signature(state: Dictionary) -> Array:
if not NetworkPlayerAnimationProtocol.validate_state(state):
static func _movement_animation_signature(animation_state: Dictionary) -> Array:
if not NetworkPlayerAnimationProtocol.validate_state(animation_state):
return []
var action: Dictionary = state["action"]
var action: Dictionary = animation_state["action"]
return [
str(state["locomotion_id"]),
bool(state["grounded"]),
str(animation_state["locomotion_id"]),
bool(animation_state["grounded"]),
str(action["id"]),
int(action["sequence"]),
bool(action.get("paused", false)),
@ -2505,15 +2523,15 @@ static func _decode_movement_snapshot(value: Variant) -> Dictionary:
static func _encode_movement_animation(
peer_id: int,
state: Dictionary,
animation_state: Dictionary,
) -> Array:
if not NetworkPlayerAnimationProtocol.validate_state(state):
if not NetworkPlayerAnimationProtocol.validate_state(animation_state):
return []
var action: Dictionary = state["action"]
var action: Dictionary = animation_state["action"]
return [
peer_id,
str(state["locomotion_id"]),
bool(state["grounded"]),
str(animation_state["locomotion_id"]),
bool(animation_state["grounded"]),
str(action["id"]),
int(action["sequence"]),
float(action["elapsed"]),
@ -2536,7 +2554,7 @@ static func _decode_movement_animation(value: Variant) -> Dictionary:
or typeof(fields[6]) != TYPE_BOOL
):
return {}
var state: Dictionary = NetworkPlayerAnimationProtocol.make_state(
var animation_state: Dictionary = NetworkPlayerAnimationProtocol.make_state(
StringName(str(fields[1])),
bool(fields[2]),
StringName(str(fields[3])),
@ -2546,10 +2564,10 @@ static func _decode_movement_animation(value: Variant) -> Dictionary:
)
if (
int(fields[0]) <= 0
or not NetworkPlayerAnimationProtocol.validate_state(state)
or not NetworkPlayerAnimationProtocol.validate_state(animation_state)
):
return {}
return {"peer_id": int(fields[0]), "state": state}
return {"peer_id": int(fields[0]), "state": animation_state}
static func _encode_movement_animation_action(
@ -2610,7 +2628,10 @@ func receive_movement_snapshots(encoded_snapshots: Array) -> void:
if avatar == null:
continue
if peer_id == local_peer_id:
if _reject_stale_space_transition_snapshot(snapshot):
if (
_reject_stale_space_transition_snapshot(snapshot)
or _reject_stale_authoritative_teleport_snapshot(snapshot)
):
continue
_discard_acknowledged_movement_inputs(
int(snapshot.get("acknowledged_input", 0))
@ -2656,6 +2677,37 @@ func _reject_stale_space_transition_snapshot(snapshot: Dictionary) -> bool:
return false
func _reject_stale_authoritative_teleport_snapshot(
snapshot: Dictionary,
) -> bool:
if not _local_authoritative_teleport_guard_active:
return false
var position_values: Array = snapshot.get("position", [])
if position_values.size() != 3:
return true
var position := Vector3(
float(position_values[0]),
float(position_values[1]),
float(position_values[2]),
)
var acknowledged_input: int = int(
snapshot.get("acknowledged_input", 0)
)
if acknowledged_input < _local_authoritative_teleport_minimum_ack:
return true
# Movement input and reliable recovery RPCs travel on different channels.
# A higher input acknowledgement alone therefore does not prove that this
# snapshot was authored after the teleport; a held pre-recovery packet can
# arrive at the host late. Only a snapshot that is still near the teleported
# position is allowed to release the barrier.
if position.distance_to(_local_authoritative_teleport_guard_position) > (
AUTHORITATIVE_TELEPORT_SNAPSHOT_GUARD_DISTANCE
):
return true
_local_authoritative_teleport_guard_active = false
return false
func _discard_acknowledged_movement_inputs(acknowledged_sequence: int) -> void:
while (
not _pending_movement_inputs.is_empty()
@ -2823,11 +2875,16 @@ func request_safe_respawn(entry_position: Vector3) -> void:
if not entry_position.is_finite():
return
if is_host():
remote_recovery_requested.emit(1, entry_position)
remote_recovery_requested.emit(1, entry_position, _input_sequence)
elif state == State.JOINED_CLIENT:
submit_safe_respawn_request.rpc_id(
1,
[entry_position.x, entry_position.y, entry_position.z]
[
entry_position.x,
entry_position.y,
entry_position.z,
_input_sequence,
]
)
@ -2837,9 +2894,18 @@ func submit_safe_respawn_request(position_data: Array) -> void:
if (
not is_host()
or not _registry.has_peer(sender_id)
or position_data.size() != 3
or position_data.size() not in [3, 4]
):
return
var recovery_input_sequence: int = -1
if position_data.size() == 4:
if (
typeof(position_data[3]) != TYPE_INT
or int(position_data[3]) < 0
or int(position_data[3]) > MAX_MOVEMENT_INPUT_SEQUENCE
):
return
recovery_input_sequence = int(position_data[3])
var entry_position := Vector3(
float(position_data[0]),
float(position_data[1]),
@ -2847,7 +2913,11 @@ func submit_safe_respawn_request(position_data: Array) -> void:
)
if not entry_position.is_finite():
return
remote_recovery_requested.emit(sender_id, entry_position)
remote_recovery_requested.emit(
sender_id,
entry_position,
recovery_input_sequence,
)
@rpc("authority", "call_remote", "reliable", 0)
@ -2856,8 +2926,38 @@ func receive_authoritative_teleport(snapshot: Dictionary) -> void:
return
var peer_id: int = int(snapshot.get("peer_id", 0))
var avatar: Player = _spawn_service.get_avatar(peer_id)
if avatar != null:
avatar.apply_network_teleport(snapshot)
if avatar == null or not avatar.apply_network_teleport(snapshot):
return
if peer_id != multiplayer.get_unique_id():
return
_begin_local_authoritative_teleport_barrier(
avatar,
int(snapshot.get("acknowledged_input", 0)),
)
func _begin_local_authoritative_teleport_barrier(
avatar: Player,
acknowledged_input: int,
) -> void:
if avatar == null or not avatar.global_position.is_finite():
return
# A recovery teleport is a movement discontinuity, just like a home-space
# transition. Drop pre-teleport client prediction and request an immediate
# neutral input so the first post-teleport audit is unambiguous.
_pending_movement_inputs.clear()
_last_local_snapshot_received_msec = 0
_last_input_state_hash = 0
_idle_input_accumulator = IDLE_INPUT_INTERVAL
avatar.reset_local_prediction_after_authoritative_teleport()
_local_authoritative_teleport_guard_active = true
_local_authoritative_teleport_guard_position = avatar.global_position
_local_authoritative_teleport_minimum_ack = maxi(acknowledged_input, 0)
# Water recovery has already suppressed local movement, so this sends a
# neutral post-relocation input on the next server tick instead of waiting
# for the regular input cadence.
if avatar.is_water_recovery_active():
submit_neutral_local_movement()
func _expire_pending_authentication(now: float) -> void:
@ -2987,6 +3087,9 @@ func _teardown_peer() -> void:
_local_space_transition_guard_active = false
_local_space_transition_guard_position = Vector3.ZERO
_local_space_transition_minimum_ack = 0
_local_authoritative_teleport_guard_active = false
_local_authoritative_teleport_guard_position = Vector3.ZERO
_local_authoritative_teleport_minimum_ack = 0
_animation_refresh_accumulator = 0.0
_last_animation_state_by_peer.clear()
_pending_animation_state_by_peer.clear()