Build 060.28: implement Persistent Checkpoint and Startup Recovery

This commit is contained in:
2026-08-01 20:55:32 +03:00
parent 58e5a12a4d
commit 8c485e32b1
63 changed files with 12430 additions and 92 deletions

View File

@@ -2,6 +2,10 @@ from __future__ import annotations
from typing import Any
import pytest
import src.bootstrap.trade_stream_runtime as runtime_factory
from src.bootstrap.market_data_storage import build_market_data_storage
from src.bootstrap.trade_stream_runtime import (
build_trade_stream_production_runtime,
)
@@ -21,7 +25,16 @@ class RecordingTradeObservationSink:
def __init__(self) -> None:
self.observations: list[Trade] = []
def persist(self, trade: Trade) -> None:
def persist_accepted(
self,
trade: Trade,
*,
expected_trade: Trade | None,
) -> None:
del expected_trade
self.observations.append(trade)
def persist_duplicate(self, trade: Trade) -> None:
self.observations.append(trade)
@@ -29,6 +42,9 @@ def make_settings(
*,
enabled: bool = True,
api_key: str = "api-key",
storage_enabled: bool = False,
subscription_ack_timeout_seconds: float = 12.5,
startup_market_buffer_capacity: int = 1_234,
) -> Settings:
return Settings(
bot_token="test-token",
@@ -58,6 +74,12 @@ def make_settings(
heartbeat_timeout_seconds=31.0,
scheduler_interval_seconds=6.0,
recovery_window_ms=123_456,
subscription_ack_timeout_seconds=(
subscription_ack_timeout_seconds
),
startup_market_buffer_capacity=(
startup_market_buffer_capacity
),
),
db_host="localhost",
db_port=5432,
@@ -65,7 +87,7 @@ def make_settings(
db_user="test",
db_password="test",
market_data_storage=MarketDataStorageSettings(
enabled=False,
enabled=storage_enabled,
pool_min_size=1,
pool_max_size=4,
pool_timeout_seconds=10.0,
@@ -81,6 +103,16 @@ def test_disabled_feature_does_not_build_runtime() -> None:
assert build_trade_stream_production_runtime(settings) is None
def test_enabled_storage_requires_enabled_trade_stream() -> None:
settings = make_settings(
enabled=False,
storage_enabled=True,
)
with pytest.raises(RuntimeError, match="Trade Stream"):
build_trade_stream_production_runtime(settings)
def test_builds_runtime_without_starting_lifecycle() -> None:
runtime = build_trade_stream_production_runtime(
make_settings(),
@@ -120,6 +152,116 @@ def test_uses_one_shared_stateful_dependency_graph() -> None:
assert runtime_graph._runtime_scheduler.runtime_supervisor is (
runtime_graph._runtime_supervisor
)
assert runtime_graph._startup_recovery_coordinator is None
def test_persistent_runtime_reuses_one_storage_graph_without_io() -> None:
settings = make_settings(storage_enabled=True)
storage = build_market_data_storage(settings)
assert storage is not None
runtime = build_trade_stream_production_runtime(
settings,
trade_observation_sink=storage.trade_observation_sink,
checkpoint_storage=storage.trade_repository,
)
assert isinstance(runtime, TradeStreamProductionRuntime)
runtime_graph: Any = runtime
startup_recovery = runtime_graph._startup_recovery_coordinator
live_controller = (
runtime_graph._trade_stream_service._consistency_controller
)
assert startup_recovery is not None
assert (
startup_recovery._state_hydrator._checkpoint_storage
is storage.trade_repository
)
assert (
storage.trade_observation_sink._trade_storage
is storage.trade_repository
)
assert (
storage.trade_observation_sink._checkpoint_storage
is storage.trade_repository
)
assert live_controller._trade_observation_sink is (
storage.trade_observation_sink
)
assert startup_recovery._state_hydrator._state_store is (
live_controller._state_store
)
assert startup_recovery._state_hydrator._venue == "dzengi"
assert storage.trade_observation_sink._venue == "dzengi"
assert storage.connection_pool.is_open is False
assert storage.lifecycle.started is False
assert startup_recovery._hydration_task is None
assert startup_recovery._recovery_task is None
assert runtime_graph._startup_task is None
assert runtime_graph._receive_task is None
assert runtime_graph._scheduler_task is None
@pytest.mark.parametrize(
("include_sink", "include_checkpoint"),
(
(False, False),
(True, False),
(False, True),
),
)
def test_enabled_storage_rejects_incomplete_runtime_graph_before_transport(
monkeypatch: pytest.MonkeyPatch,
include_sink: bool,
include_checkpoint: bool,
) -> None:
settings = make_settings(storage_enabled=True)
storage = build_market_data_storage(settings)
assert storage is not None
def unexpected_transport(**kwargs: object) -> None:
del kwargs
raise AssertionError("Transport graph must not be created.")
monkeypatch.setattr(
runtime_factory,
"DzengiWebSocketTransport",
unexpected_transport,
)
with pytest.raises(RuntimeError, match="requires both"):
build_trade_stream_production_runtime(
settings,
trade_observation_sink=(
storage.trade_observation_sink
if include_sink
else None
),
checkpoint_storage=(
storage.trade_repository
if include_checkpoint
else None
),
)
assert storage.connection_pool.is_open is False
def test_disabled_storage_rejects_checkpoint_dependency() -> None:
settings = make_settings()
persistent_settings = make_settings(storage_enabled=True)
storage = build_market_data_storage(persistent_settings)
assert storage is not None
with pytest.raises(RuntimeError, match="requires enabled"):
build_trade_stream_production_runtime(
settings,
checkpoint_storage=storage.trade_repository,
)
def test_optional_storage_sink_is_shared_by_live_and_recovery() -> None:
@@ -172,6 +314,8 @@ def test_applies_explicit_transport_and_runtime_settings() -> None:
"ETH/USD_LEVERAGE",
)
assert runtime_graph._runtime_scheduler.interval_seconds == 6.0
assert runtime_graph._subscription_ack_timeout_seconds == 12.5
assert runtime_graph._startup_market_buffer_capacity == 1_234
assert (
runtime_graph._runtime_supervisor._heartbeat_monitor.timeout_seconds
== 31.0