Build 060.27: implement Persistent Market Data Storage

This commit is contained in:
2026-08-01 03:22:25 +03:00
parent cb8acfe5fe
commit 58e5a12a4d
54 changed files with 10243 additions and 101 deletions

View File

@@ -46,6 +46,7 @@ def make_settings() -> SimpleNamespace:
exchange_name="dzengi",
default_symbol="BTC/USD_LEVERAGE",
trade_stream=SimpleNamespace(enabled=True),
market_data_storage=SimpleNamespace(enabled=True),
)
@@ -56,6 +57,12 @@ def test_create_app_builds_one_application_composition(
bot = object()
dispatcher = object()
runtime = object()
storage_lifecycle = object()
storage_sink = object()
storage = SimpleNamespace(
lifecycle=storage_lifecycle,
trade_observation_sink=storage_sink,
)
journal = RecordingJournal()
observed_runtime_settings: list[object] = []
registered_bots: list[object] = []
@@ -82,8 +89,23 @@ def test_create_app_builds_one_application_composition(
lambda: journal,
)
def build_runtime(received_settings: object) -> object:
monkeypatch.setattr(
app_factory,
"build_market_data_storage",
lambda received_settings: (
storage
if received_settings is settings
else None
),
)
def build_runtime(
received_settings: object,
*,
trade_observation_sink: object,
) -> object:
observed_runtime_settings.append(received_settings)
assert trade_observation_sink is storage_sink
return runtime
monkeypatch.setattr(
@@ -118,10 +140,18 @@ def test_create_app_builds_one_application_composition(
assert application.bot is bot
assert application.dispatcher is dispatcher
assert application.trade_stream_runtime is runtime
assert (
application.market_data_storage_lifecycle
is storage_lifecycle
)
assert observed_runtime_settings == [settings]
assert registered_bots == [bot]
assert routed_dispatchers == [dispatcher]
assert journal.info_calls[0][2]["trade_stream_enabled"] is True
assert (
journal.info_calls[0][2]["market_data_storage_enabled"]
is True
)
def test_runtime_build_error_is_fatal(
@@ -149,9 +179,18 @@ def test_runtime_build_error_is_fatal(
"JournalService",
RecordingJournal,
)
monkeypatch.setattr(
app_factory,
"build_market_data_storage",
lambda settings: None,
)
def fail_runtime_build(settings: object) -> None:
del settings
def fail_runtime_build(
settings: object,
*,
trade_observation_sink: object,
) -> None:
del settings, trade_observation_sink
raise expected
monkeypatch.setattr(

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import threading
import pytest
@@ -46,12 +47,14 @@ class FakeDispatcher:
*,
return_immediately: bool = False,
error: BaseException | None = None,
calls: list[str] | None = None,
) -> None:
self.started = asyncio.Event()
self.release = asyncio.Event()
self.cancelled = asyncio.Event()
self.return_immediately = return_immediately
self.error = error
self.calls = calls
self.close_bot_session_values: list[bool] = []
async def start_polling(
@@ -61,6 +64,8 @@ class FakeDispatcher:
close_bot_session: bool,
) -> None:
del bot
if self.calls is not None:
self.calls.append("polling.start")
self.close_bot_session_values.append(close_bot_session)
self.started.set()
@@ -82,6 +87,7 @@ class FakeRuntime:
return_immediately: bool = False,
error: BaseException | None = None,
stop_error: BaseException | None = None,
calls: list[str] | None = None,
) -> None:
self.started = asyncio.Event()
self.release = asyncio.Event()
@@ -89,6 +95,7 @@ class FakeRuntime:
self.return_immediately = return_immediately
self.error = error
self.stop_error = stop_error
self.calls = calls
self.stop_calls = 0
@property
@@ -104,6 +111,8 @@ class FakeRuntime:
return self.state is TradeStreamProductionRuntimeState.RUNNING
async def run(self) -> None:
if self.calls is not None:
self.calls.append("runtime.run")
self.started.set()
if not self.return_immediately:
@@ -114,6 +123,8 @@ class FakeRuntime:
async def stop(self) -> None:
self.stop_calls += 1
if self.calls is not None:
self.calls.append("runtime.stop")
self.release.set()
self.stopped.set()
@@ -136,19 +147,289 @@ class BlockingStopRuntime(FakeRuntime):
self.stopped.set()
class FakeStorageLifecycle:
def __init__(
self,
*,
calls: list[str] | None = None,
start_error: BaseException | None = None,
stop_error: BaseException | None = None,
start_release: threading.Event | None = None,
stop_release: threading.Event | None = None,
) -> None:
self.calls = calls
self.start_error = start_error
self.stop_error = stop_error
self.start_release = start_release
self.stop_release = stop_release
self.start_entered = threading.Event()
self.stop_entered = threading.Event()
self.start_calls = 0
self.stop_calls = 0
self.started = False
def start(self) -> None:
self.start_calls += 1
if self.calls is not None:
self.calls.append("storage.start")
self.start_entered.set()
if self.start_release is not None:
if not self.start_release.wait(timeout=2.0):
raise TimeoutError("storage startup was not released")
if self.start_error is not None:
raise self.start_error
self.started = True
def stop(self) -> None:
self.stop_calls += 1
if self.calls is not None:
self.calls.append("storage.stop")
self.started = False
self.stop_entered.set()
if self.stop_release is not None:
if not self.stop_release.wait(timeout=2.0):
raise TimeoutError("storage shutdown was not released")
if self.stop_error is not None:
raise self.stop_error
def make_application(
*,
dispatcher: FakeDispatcher,
runtime: FakeRuntime | None,
bot: FakeBot | None = None,
storage_lifecycle: FakeStorageLifecycle | None = None,
) -> ApplicationComposition:
return ApplicationComposition(
bot=bot or FakeBot(), # type: ignore[arg-type]
dispatcher=dispatcher, # type: ignore[arg-type]
trade_stream_runtime=runtime,
market_data_storage_lifecycle=storage_lifecycle,
)
def test_storage_lifecycle_wraps_root_runtime_tasks() -> None:
async def scenario() -> None:
calls: list[str] = []
storage = FakeStorageLifecycle(calls=calls)
dispatcher = FakeDispatcher(
return_immediately=True,
calls=calls,
)
runtime = FakeRuntime(calls=calls)
await run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
storage_lifecycle=storage,
)
)
assert storage.start_calls == 1
assert storage.stop_calls == 1
assert calls.index("storage.start") < calls.index(
"polling.start"
)
assert calls.index("storage.start") < calls.index(
"runtime.run"
)
assert calls.index("runtime.stop") < calls.index(
"storage.stop"
)
asyncio.run(scenario())
def test_storage_startup_failure_prevents_root_task_start() -> None:
async def scenario() -> None:
expected = RuntimeError("storage startup failed")
storage = FakeStorageLifecycle(start_error=expected)
dispatcher = FakeDispatcher()
runtime = FakeRuntime()
bot = FakeBot()
with pytest.raises(RuntimeError) as error_info:
await run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
bot=bot,
storage_lifecycle=storage,
)
)
assert error_info.value is expected
assert dispatcher.started.is_set() is False
assert runtime.started.is_set() is False
assert runtime.stop_calls == 1
assert storage.stop_calls == 1
assert bot.session.close_calls == 1
asyncio.run(scenario())
def test_cancellation_waits_for_storage_startup_before_cleanup() -> None:
async def scenario() -> None:
start_release = threading.Event()
storage = FakeStorageLifecycle(
start_release=start_release,
)
dispatcher = FakeDispatcher()
runtime = FakeRuntime()
bot = FakeBot()
task = asyncio.create_task(
run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
bot=bot,
storage_lifecycle=storage,
)
)
)
entered = await asyncio.to_thread(
storage.start_entered.wait,
1.0,
)
assert entered is True
task.cancel()
try:
await asyncio.sleep(0)
await asyncio.sleep(0)
assert task.done() is False
finally:
start_release.set()
with pytest.raises(asyncio.CancelledError):
await task
assert dispatcher.started.is_set() is False
assert runtime.started.is_set() is False
assert storage.stop_calls == 1
assert bot.session.close_calls == 1
await asyncio.sleep(0)
assert not {
child.get_name()
for child in asyncio.all_tasks()
if child is not asyncio.current_task()
and not child.done()
and child.get_name()
in {
"application-shutdown",
"market-data-storage-shutdown",
"market-data-storage-startup",
"telegram-polling",
"trade-stream-runtime",
}
}
asyncio.run(scenario())
def test_storage_shutdown_error_is_reported_and_bot_still_closes() -> None:
async def scenario() -> None:
expected = RuntimeError("storage close failed")
storage = FakeStorageLifecycle(stop_error=expected)
dispatcher = FakeDispatcher(return_immediately=True)
bot = FakeBot()
with pytest.raises(RuntimeError) as error_info:
await run_application(
make_application(
dispatcher=dispatcher,
runtime=None,
bot=bot,
storage_lifecycle=storage,
)
)
assert error_info.value is expected
assert storage.stop_calls == 1
assert bot.session.close_calls == 1
asyncio.run(scenario())
def test_storage_shutdown_error_does_not_replace_runtime_error() -> None:
async def scenario() -> None:
runtime_error = RuntimeError("runtime failed")
storage_error = RuntimeError("storage close failed")
storage = FakeStorageLifecycle(stop_error=storage_error)
dispatcher = FakeDispatcher()
runtime = FakeRuntime(
return_immediately=True,
error=runtime_error,
)
with pytest.raises(RuntimeError) as error_info:
await run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
storage_lifecycle=storage,
)
)
assert error_info.value is runtime_error
assert any(
"cleanup also failed" in note
for note in getattr(runtime_error, "__notes__", ())
)
asyncio.run(scenario())
def test_repeated_cancellation_does_not_interrupt_storage_shutdown(
) -> None:
async def scenario() -> None:
stop_release = threading.Event()
storage = FakeStorageLifecycle(
stop_release=stop_release,
)
dispatcher = FakeDispatcher(return_immediately=True)
bot = FakeBot()
task = asyncio.create_task(
run_application(
make_application(
dispatcher=dispatcher,
runtime=None,
bot=bot,
storage_lifecycle=storage,
)
)
)
entered = await asyncio.to_thread(
storage.stop_entered.wait,
1.0,
)
assert entered is True
task.cancel()
task.cancel()
try:
await asyncio.sleep(0)
assert task.done() is False
finally:
stop_release.set()
with pytest.raises(asyncio.CancelledError):
await task
assert storage.stop_calls == 1
assert bot.session.close_calls == 1
asyncio.run(scenario())
def test_disabled_runtime_runs_only_polling_and_closes_bot() -> None:
async def scenario() -> None:
dispatcher = FakeDispatcher(return_immediately=True)

View File

@@ -0,0 +1,232 @@
from __future__ import annotations
from typing import Any, cast
import pytest
from psycopg.conninfo import conninfo_to_dict
from src.bootstrap.market_data_storage import (
MarketDataStorageLifecycle,
MarketDataStorageLifecycleProtocol,
build_market_data_storage,
)
from src.core.config import (
MarketDataStorageSettings,
Settings,
TradeStreamSettings,
)
def make_settings(
*,
storage_enabled: bool = True,
trade_stream_enabled: bool = True,
) -> Settings:
return Settings(
bot_token="test-token",
bot_parse_mode="HTML",
app_env="test",
log_level="INFO",
tz="UTC",
exchange_enabled=True,
exchange_name=" dzengi ",
exchange_base_url="https://rest.example.test",
exchange_ws_url="",
exchange_api_key="",
exchange_api_secret="",
exchange_timeout_sec=10,
exchange_testnet=True,
default_symbol="BTC/USD_LEVERAGE",
trade_stream=TradeStreamSettings(
enabled=trade_stream_enabled,
websocket_url="wss://stream.example.test",
symbols=("BTC/USD_LEVERAGE",),
open_timeout_seconds=10.0,
probe_timeout_seconds=20.0,
close_timeout_seconds=10.0,
heartbeat_timeout_seconds=30.0,
scheduler_interval_seconds=5.0,
recovery_window_ms=3_599_999,
),
db_host="db.example.test",
db_port=5544,
db_name="dzentra",
db_user="market-data",
db_password="p@ss word",
market_data_storage=MarketDataStorageSettings(
enabled=storage_enabled,
pool_min_size=2,
pool_max_size=6,
pool_timeout_seconds=7.5,
),
debug_enabled=False,
journal_debug_enabled=False,
)
class RecordingPool:
def __init__(
self,
*,
close_error: BaseException | None = None,
) -> None:
self.close_error = close_error
self.events: list[str] = []
def open(self) -> None:
self.events.append("pool.open")
def close(self) -> None:
self.events.append("pool.close")
if self.close_error is not None:
raise self.close_error
class RecordingMigrationRunner:
def __init__(
self,
*,
pool: RecordingPool,
error: BaseException | None = None,
) -> None:
self._pool = pool
self._error = error
self.calls = 0
def run(self) -> tuple[int, ...]:
self.calls += 1
self._pool.events.append("migrations.run")
if self._error is not None:
raise self._error
return (1,)
def make_lifecycle(
*,
migration_error: BaseException | None = None,
close_error: BaseException | None = None,
) -> tuple[
MarketDataStorageLifecycle,
RecordingPool,
RecordingMigrationRunner,
]:
pool = RecordingPool(close_error=close_error)
runner = RecordingMigrationRunner(
pool=pool,
error=migration_error,
)
lifecycle = MarketDataStorageLifecycle(
connection_pool=pool,
migration_runner=runner,
)
return lifecycle, pool, runner
def test_disabled_storage_builds_nothing() -> None:
assert build_market_data_storage(
make_settings(storage_enabled=False),
) is None
def test_enabled_storage_requires_enabled_trade_stream() -> None:
with pytest.raises(RuntimeError, match="Trade Stream"):
build_market_data_storage(
make_settings(trade_stream_enabled=False),
)
def test_builds_shared_graph_without_opening_pool() -> None:
composition = build_market_data_storage(make_settings())
assert composition is not None
assert composition.connection_pool.is_open is False
assert composition.lifecycle.started is False
migration_provider = cast(
Any,
composition.migration_runner._connection_provider,
)
repository_provider = cast(
Any,
composition.trade_repository._connection_provider,
)
assert migration_provider.__self__ is composition.connection_pool
assert repository_provider.__self__ is composition.connection_pool
assert (
composition.trade_observation_sink._trade_storage
is composition.trade_repository
)
assert composition.trade_observation_sink._venue == "dzengi"
conninfo = conninfo_to_dict(composition.connection_pool._conninfo)
assert conninfo == {
"host": "db.example.test",
"port": "5544",
"dbname": "dzentra",
"user": "market-data",
"password": "p@ss word",
}
def test_lifecycle_opens_pool_before_migrations_and_is_idempotent() -> None:
lifecycle, pool, runner = make_lifecycle()
lifecycle.start()
lifecycle.start()
assert isinstance(lifecycle, MarketDataStorageLifecycleProtocol)
assert lifecycle.started is True
assert pool.events == [
"pool.open",
"migrations.run",
]
assert runner.calls == 1
lifecycle.stop()
lifecycle.stop()
assert lifecycle.started is False
assert pool.events == [
"pool.open",
"migrations.run",
"pool.close",
"pool.close",
]
def test_migration_failure_closes_pool_and_preserves_error() -> None:
migration_error = RuntimeError("migration failed")
lifecycle, pool, _ = make_lifecycle(
migration_error=migration_error,
)
with pytest.raises(RuntimeError) as error_info:
lifecycle.start()
assert error_info.value is migration_error
assert lifecycle.started is False
assert pool.events == [
"pool.open",
"migrations.run",
"pool.close",
]
def test_migration_error_keeps_cleanup_failure_as_note() -> None:
migration_error = RuntimeError("migration failed")
lifecycle, _, _ = make_lifecycle(
migration_error=migration_error,
close_error=RuntimeError("close failed"),
)
with pytest.raises(RuntimeError) as error_info:
lifecycle.start()
assert error_info.value is migration_error
assert migration_error.__notes__ == [
"Market Data Storage startup cleanup also failed: RuntimeError."
]

View File

@@ -5,9 +5,10 @@ import json
import threading
import time
from collections.abc import Awaitable, Callable
from typing import Any
from typing import Any, cast
import pytest
from aiogram import Bot, Dispatcher
from websockets.protocol import State
import src.bootstrap.trade_stream_runtime as production_factory
@@ -15,7 +16,11 @@ from src.bootstrap.application import (
ApplicationComposition,
run_application,
)
from src.core.config import Settings, TradeStreamSettings
from src.core.config import (
MarketDataStorageSettings,
Settings,
TradeStreamSettings,
)
from src.market_data.acquisition.adapters.dzengi.websocket_transport import (
DzengiWebSocketTransport,
)
@@ -37,6 +42,8 @@ SYMBOL = "BTC/USD_LEVERAGE"
_OWNED_TASK_NAMES = frozenset(
{
"application-shutdown",
"market-data-storage-shutdown",
"market-data-storage-startup",
"telegram-polling",
"trade-stream-receive",
"trade-stream-runtime",
@@ -295,6 +302,12 @@ def make_settings(
db_name="test",
db_user="test",
db_password="test",
market_data_storage=MarketDataStorageSettings(
enabled=False,
pool_min_size=1,
pool_max_size=4,
pool_timeout_seconds=10.0,
),
debug_enabled=False,
journal_debug_enabled=False,
)
@@ -367,8 +380,8 @@ def make_application(
bot: FakeBot,
) -> ApplicationComposition:
return ApplicationComposition(
bot=bot, # type: ignore[arg-type]
dispatcher=dispatcher, # type: ignore[arg-type]
bot=cast(Bot, cast(object, bot)),
dispatcher=cast(Dispatcher, cast(object, dispatcher)),
trade_stream_runtime=runtime,
)
@@ -400,14 +413,23 @@ async def assert_no_owned_tasks() -> None:
def state_store_from(
runtime: TradeStreamProductionRuntime,
) -> TradeStreamStateStore:
runtime_graph: Any = runtime
return (
runtime
runtime_graph
._reconnect_recovery_coordinator
._recovery_coordinator
._state_store
)
def subscription_keys_from(
runtime: TradeStreamProductionRuntime,
) -> tuple[str, ...]:
runtime_graph: Any = runtime
return runtime_graph._subscription_manager.subscription_keys
def test_disabled_feature_runs_only_telegram_without_runtime_graph(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -529,7 +551,7 @@ def test_production_factory_processes_ack_trade_and_shutdown(
assert runtime.state is (
TradeStreamProductionRuntimeState.STOPPED
)
assert runtime._subscription_manager.subscription_keys == ()
assert subscription_keys_from(runtime) == ()
assert bot.session.close_calls == 1
assert len(transports) == 1
await assert_no_owned_tasks()
@@ -579,7 +601,7 @@ def test_connection_startup_failure_is_fatal_and_leaves_no_tasks(
TradeStreamProductionRuntimeState.FAILED
)
assert runtime._session.is_connected is False
assert runtime._subscription_manager.subscription_keys == ()
assert subscription_keys_from(runtime) == ()
assert bot.session.close_calls == 1
await assert_no_owned_tasks()
@@ -633,7 +655,7 @@ def test_subscription_startup_failure_rolls_back_concrete_graph(
TradeStreamProductionRuntimeState.FAILED
)
assert runtime._session.is_connected is False
assert runtime._subscription_manager.subscription_keys == ()
assert subscription_keys_from(runtime) == ()
assert bot.session.close_calls == 1
await assert_no_owned_tasks()

View File

@@ -1,13 +1,28 @@
from __future__ import annotations
from typing import Any
from src.bootstrap.trade_stream_runtime import (
build_trade_stream_production_runtime,
)
from src.core.config import Settings, TradeStreamSettings
from src.core.config import (
MarketDataStorageSettings,
Settings,
TradeStreamSettings,
)
from src.market_data.acquisition.runtime.trade_stream_production_runtime import (
TradeStreamProductionRuntime,
TradeStreamProductionRuntimeState,
)
from src.market_data.acquisition.models.trade import Trade
class RecordingTradeObservationSink:
def __init__(self) -> None:
self.observations: list[Trade] = []
def persist(self, trade: Trade) -> None:
self.observations.append(trade)
def make_settings(
@@ -49,6 +64,12 @@ def make_settings(
db_name="test",
db_user="test",
db_password="test",
market_data_storage=MarketDataStorageSettings(
enabled=False,
pool_min_size=1,
pool_max_size=4,
pool_timeout_seconds=10.0,
),
debug_enabled=False,
journal_debug_enabled=False,
)
@@ -80,33 +101,60 @@ def test_uses_one_shared_stateful_dependency_graph() -> None:
assert isinstance(runtime, TradeStreamProductionRuntime)
transport = runtime._transport
service = runtime._trade_stream_service
reconnect_recovery = runtime._reconnect_recovery_coordinator
runtime_graph: Any = runtime
transport = runtime_graph._transport
service = runtime_graph._trade_stream_service
reconnect_recovery = runtime_graph._reconnect_recovery_coordinator
recovery = reconnect_recovery._recovery_coordinator
assert runtime._session._transport is transport
assert runtime._subscription_manager._transport is transport
assert runtime._runtime_scheduler.liveness_probe is transport
assert runtime_graph._session._transport is transport
assert runtime_graph._subscription_manager._transport is transport
assert runtime_graph._runtime_scheduler.liveness_probe is transport
assert (
service._consistency_controller
is recovery._recovery_controller._consistency_controller
)
assert runtime._live_processing_gate is (
assert runtime_graph._live_processing_gate is (
reconnect_recovery.live_processing_gate
)
assert runtime._runtime_scheduler.runtime_supervisor is (
runtime._runtime_supervisor
assert runtime_graph._runtime_scheduler.runtime_supervisor is (
runtime_graph._runtime_supervisor
)
def test_optional_storage_sink_is_shared_by_live_and_recovery() -> None:
sink = RecordingTradeObservationSink()
runtime = build_trade_stream_production_runtime(
make_settings(),
trade_observation_sink=sink,
)
assert isinstance(runtime, TradeStreamProductionRuntime)
runtime_graph: Any = runtime
live_controller = (
runtime_graph._trade_stream_service._consistency_controller
)
recovery_controller = (
runtime_graph
._reconnect_recovery_coordinator
._recovery_coordinator
._recovery_controller
._consistency_controller
)
assert live_controller is recovery_controller
assert live_controller._trade_observation_sink is sink
def test_applies_explicit_transport_and_runtime_settings() -> None:
settings = make_settings()
runtime = build_trade_stream_production_runtime(settings)
assert isinstance(runtime, TradeStreamProductionRuntime)
transport = runtime._transport
runtime_graph: Any = runtime
transport = runtime_graph._transport
assert transport._url == "wss://stream.example.test/root/connect"
assert transport._headers == {
@@ -119,17 +167,17 @@ def test_applies_explicit_transport_and_runtime_settings() -> None:
assert transport._close_timeout == 9.0
assert transport._ping_interval is None
assert transport._ping_timeout is None
assert runtime._symbols == (
assert runtime_graph._symbols == (
"BTC/USD_LEVERAGE",
"ETH/USD_LEVERAGE",
)
assert runtime._runtime_scheduler.interval_seconds == 6.0
assert runtime_graph._runtime_scheduler.interval_seconds == 6.0
assert (
runtime._runtime_supervisor._heartbeat_monitor.timeout_seconds
runtime_graph._runtime_supervisor._heartbeat_monitor.timeout_seconds
== 31.0
)
assert (
runtime._reconnect_recovery_coordinator
runtime_graph._reconnect_recovery_coordinator
._recovery_coordinator
._window_planner
.max_window_ms
@@ -143,8 +191,9 @@ def test_recovery_rest_client_reuses_settings_snapshot() -> None:
runtime = build_trade_stream_production_runtime(settings)
assert isinstance(runtime, TradeStreamProductionRuntime)
runtime_graph: Any = runtime
document_source = (
runtime._reconnect_recovery_coordinator
runtime_graph._reconnect_recovery_coordinator
._recovery_coordinator
._recovery_controller
._document_source
@@ -154,4 +203,4 @@ def test_recovery_rest_client_reuses_settings_snapshot() -> None:
assert rest_client.settings is settings
assert rest_client.base_url == "https://rest.example.test"
assert rest_client.timeout == 17
assert "X-MBX-APIKEY" not in runtime._transport._headers
assert "X-MBX-APIKEY" not in runtime_graph._transport._headers