Build 060.25: implement Production Runtime Integration

This commit is contained in:
2026-07-31 00:29:36 +03:00
parent c142145361
commit 60bec1eaf9
50 changed files with 14044 additions and 83 deletions

View File

@@ -0,0 +1,166 @@
from __future__ import annotations
from types import SimpleNamespace
import pytest
import src.bootstrap.app_factory as app_factory
from src.bootstrap.application import ApplicationComposition
class RecordingJournal:
def __init__(self) -> None:
self.info_calls: list[
tuple[str, str, dict[str, object]]
] = []
def log_info(
self,
event: str,
message: str,
context: dict[str, object],
) -> None:
self.info_calls.append(
(
event,
message,
context,
)
)
def log_critical(
self,
event: str,
message: str,
context: dict[str, object],
) -> None:
del event, message, context
def make_settings() -> SimpleNamespace:
return SimpleNamespace(
bot_token="test-token",
bot_parse_mode="HTML",
log_level="INFO",
app_env="test",
exchange_name="dzengi",
default_symbol="BTC/USD_LEVERAGE",
trade_stream=SimpleNamespace(enabled=True),
)
def test_create_app_builds_one_application_composition(
monkeypatch: pytest.MonkeyPatch,
) -> None:
settings = make_settings()
bot = object()
dispatcher = object()
runtime = object()
journal = RecordingJournal()
observed_runtime_settings: list[object] = []
registered_bots: list[object] = []
routed_dispatchers: list[object] = []
monkeypatch.setattr(
app_factory,
"load_settings",
lambda: settings,
)
monkeypatch.setattr(
app_factory,
"setup_logging",
lambda level: None,
)
monkeypatch.setattr(
app_factory,
"init_schema",
lambda: None,
)
monkeypatch.setattr(
app_factory,
"JournalService",
lambda: journal,
)
def build_runtime(received_settings: object) -> object:
observed_runtime_settings.append(received_settings)
return runtime
monkeypatch.setattr(
app_factory,
"build_trade_stream_production_runtime",
build_runtime,
)
monkeypatch.setattr(
app_factory,
"Bot",
lambda **kwargs: bot,
)
monkeypatch.setattr(
app_factory,
"Dispatcher",
lambda: dispatcher,
)
monkeypatch.setattr(
app_factory.NotificationTargetRegistry,
"set_bot",
registered_bots.append,
)
monkeypatch.setattr(
app_factory,
"setup_routers",
routed_dispatchers.append,
)
application = app_factory.create_app()
assert isinstance(application, ApplicationComposition)
assert application.bot is bot
assert application.dispatcher is dispatcher
assert application.trade_stream_runtime is runtime
assert observed_runtime_settings == [settings]
assert registered_bots == [bot]
assert routed_dispatchers == [dispatcher]
assert journal.info_calls[0][2]["trade_stream_enabled"] is True
def test_runtime_build_error_is_fatal(
monkeypatch: pytest.MonkeyPatch,
) -> None:
expected = RuntimeError("invalid Trade Stream settings")
monkeypatch.setattr(
app_factory,
"load_settings",
make_settings,
)
monkeypatch.setattr(
app_factory,
"setup_logging",
lambda level: None,
)
monkeypatch.setattr(
app_factory,
"init_schema",
lambda: None,
)
monkeypatch.setattr(
app_factory,
"JournalService",
RecordingJournal,
)
def fail_runtime_build(settings: object) -> None:
del settings
raise expected
monkeypatch.setattr(
app_factory,
"build_trade_stream_production_runtime",
fail_runtime_build,
)
with pytest.raises(RuntimeError) as exc_info:
app_factory.create_app()
assert exc_info.value is expected

View File

@@ -0,0 +1,442 @@
from __future__ import annotations
import asyncio
import pytest
from src.bootstrap.application import (
ApplicationComposition,
run_application,
)
from src.market_data.acquisition.runtime.trade_stream_production_runtime import (
TradeStreamProductionRuntimeState,
)
class FakeBotSession:
def __init__(
self,
*,
close_error: BaseException | None = None,
) -> None:
self.close_calls = 0
self.close_error = close_error
async def close(self) -> None:
self.close_calls += 1
if self.close_error is not None:
raise self.close_error
class FakeBot:
def __init__(
self,
*,
close_error: BaseException | None = None,
) -> None:
self.session = FakeBotSession(
close_error=close_error,
)
class FakeDispatcher:
def __init__(
self,
*,
return_immediately: bool = False,
error: BaseException | None = None,
) -> None:
self.started = asyncio.Event()
self.release = asyncio.Event()
self.cancelled = asyncio.Event()
self.return_immediately = return_immediately
self.error = error
self.close_bot_session_values: list[bool] = []
async def start_polling(
self,
bot: FakeBot,
*,
close_bot_session: bool,
) -> None:
del bot
self.close_bot_session_values.append(close_bot_session)
self.started.set()
try:
if not self.return_immediately:
await self.release.wait()
except asyncio.CancelledError:
self.cancelled.set()
raise
if self.error is not None:
raise self.error
class FakeRuntime:
def __init__(
self,
*,
return_immediately: bool = False,
error: BaseException | None = None,
stop_error: BaseException | None = None,
) -> None:
self.started = asyncio.Event()
self.release = asyncio.Event()
self.stopped = asyncio.Event()
self.return_immediately = return_immediately
self.error = error
self.stop_error = stop_error
self.stop_calls = 0
@property
def state(self) -> TradeStreamProductionRuntimeState:
return (
TradeStreamProductionRuntimeState.RUNNING
if self.started.is_set() and not self.stopped.is_set()
else TradeStreamProductionRuntimeState.STOPPED
)
@property
def running(self) -> bool:
return self.state is TradeStreamProductionRuntimeState.RUNNING
async def run(self) -> None:
self.started.set()
if not self.return_immediately:
await self.release.wait()
if self.error is not None:
raise self.error
async def stop(self) -> None:
self.stop_calls += 1
self.release.set()
self.stopped.set()
if self.stop_error is not None:
raise self.stop_error
class BlockingStopRuntime(FakeRuntime):
def __init__(self) -> None:
super().__init__()
self.stop_entered = asyncio.Event()
self.stop_release = asyncio.Event()
async def stop(self) -> None:
self.stop_calls += 1
self.stop_entered.set()
await self.stop_release.wait()
self.release.set()
self.stopped.set()
def make_application(
*,
dispatcher: FakeDispatcher,
runtime: FakeRuntime | None,
bot: FakeBot | None = None,
) -> ApplicationComposition:
return ApplicationComposition(
bot=bot or FakeBot(), # type: ignore[arg-type]
dispatcher=dispatcher, # type: ignore[arg-type]
trade_stream_runtime=runtime,
)
def test_disabled_runtime_runs_only_polling_and_closes_bot() -> None:
async def scenario() -> None:
dispatcher = FakeDispatcher(return_immediately=True)
bot = FakeBot()
await run_application(
make_application(
dispatcher=dispatcher,
runtime=None,
bot=bot,
)
)
assert dispatcher.close_bot_session_values == [False]
assert bot.session.close_calls == 1
asyncio.run(scenario())
def test_polling_completion_stops_and_awaits_runtime() -> None:
async def scenario() -> None:
dispatcher = FakeDispatcher(return_immediately=True)
runtime = FakeRuntime()
bot = FakeBot()
await run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
bot=bot,
)
)
assert runtime.started.is_set()
assert runtime.stop_calls == 1
assert runtime.stopped.is_set()
assert bot.session.close_calls == 1
asyncio.run(scenario())
def test_runtime_error_is_fatal_and_stops_polling() -> None:
async def scenario() -> None:
expected = RuntimeError("trade stream failed")
dispatcher = FakeDispatcher()
runtime = FakeRuntime(
return_immediately=True,
error=expected,
)
bot = FakeBot()
with pytest.raises(RuntimeError) as exc_info:
await run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
bot=bot,
)
)
assert exc_info.value is expected
assert dispatcher.cancelled.is_set()
assert runtime.stop_calls == 1
assert bot.session.close_calls == 1
asyncio.run(scenario())
def test_normal_runtime_completion_is_fatal() -> None:
async def scenario() -> None:
dispatcher = FakeDispatcher()
runtime = FakeRuntime(return_immediately=True)
with pytest.raises(
RuntimeError,
match="terminated unexpectedly",
):
await run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
)
)
assert dispatcher.cancelled.is_set()
assert runtime.stop_calls == 1
asyncio.run(scenario())
def test_application_cancellation_performs_full_cleanup() -> None:
async def scenario() -> None:
dispatcher = FakeDispatcher()
runtime = FakeRuntime()
bot = FakeBot()
task = asyncio.create_task(
run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
bot=bot,
)
)
)
await dispatcher.started.wait()
await runtime.started.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert dispatcher.cancelled.is_set()
assert runtime.stop_calls == 1
assert runtime.stopped.is_set()
assert bot.session.close_calls == 1
await asyncio.sleep(0)
assert not {
child.get_name()
for child in asyncio.all_tasks()
if child is not asyncio.current_task()
and not child.done()
and child.get_name()
in {
"telegram-polling",
"trade-stream-runtime",
"application-shutdown",
}
}
asyncio.run(scenario())
def test_polling_error_stops_runtime_and_preserves_error() -> None:
async def scenario() -> None:
expected = RuntimeError("polling failed")
dispatcher = FakeDispatcher(
return_immediately=True,
error=expected,
)
runtime = FakeRuntime()
bot = FakeBot()
with pytest.raises(RuntimeError) as exc_info:
await run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
bot=bot,
)
)
assert exc_info.value is expected
assert runtime.stop_calls == 1
assert runtime.stopped.is_set()
assert bot.session.close_calls == 1
asyncio.run(scenario())
def test_simultaneous_root_failures_are_awaited_deterministically() -> None:
async def scenario() -> None:
polling_error = RuntimeError("polling failed")
runtime_error = ValueError("trade stream failed")
dispatcher = FakeDispatcher(
error=polling_error,
)
runtime = FakeRuntime(
error=runtime_error,
)
bot = FakeBot()
task = asyncio.create_task(
run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
bot=bot,
)
)
)
await dispatcher.started.wait()
await runtime.started.wait()
dispatcher.release.set()
runtime.release.set()
with pytest.raises(ValueError) as exc_info:
await task
assert exc_info.value is runtime_error
assert runtime.stop_calls == 1
assert bot.session.close_calls == 1
assert any(
"cleanup also failed" in note
for note in getattr(runtime_error, "__notes__", ())
)
await asyncio.sleep(0)
assert not {
child.get_name()
for child in asyncio.all_tasks()
if child is not asyncio.current_task()
and not child.done()
and child.get_name()
in {
"telegram-polling",
"trade-stream-runtime",
"application-shutdown",
}
}
asyncio.run(scenario())
def test_repeated_cancellation_does_not_interrupt_cleanup() -> None:
async def scenario() -> None:
dispatcher = FakeDispatcher()
runtime = BlockingStopRuntime()
bot = FakeBot()
task = asyncio.create_task(
run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
bot=bot,
)
)
)
await dispatcher.started.wait()
await runtime.started.wait()
task.cancel()
await runtime.stop_entered.wait()
task.cancel()
runtime.stop_release.set()
with pytest.raises(asyncio.CancelledError):
await task
assert task.cancelled() is True
assert dispatcher.cancelled.is_set()
assert runtime.stop_calls == 1
assert runtime.stopped.is_set()
assert bot.session.close_calls == 1
await asyncio.sleep(0)
assert not {
child.get_name()
for child in asyncio.all_tasks()
if child is not asyncio.current_task()
and not child.done()
and child.get_name()
in {
"telegram-polling",
"trade-stream-runtime",
"application-shutdown",
}
}
asyncio.run(scenario())
def test_cleanup_error_does_not_replace_runtime_error() -> None:
async def scenario() -> None:
expected = RuntimeError("trade stream failed")
dispatcher = FakeDispatcher()
runtime = FakeRuntime(
return_immediately=True,
error=expected,
stop_error=RuntimeError("stop failed"),
)
bot = FakeBot(
close_error=RuntimeError("close failed"),
)
with pytest.raises(RuntimeError) as exc_info:
await run_application(
make_application(
dispatcher=dispatcher,
runtime=runtime,
bot=bot,
)
)
assert exc_info.value is expected
assert bot.session.close_calls == 1
assert any(
"cleanup also failed" in note
for note in getattr(expected, "__notes__", ())
)
asyncio.run(scenario())

View File

@@ -0,0 +1,890 @@
from __future__ import annotations
import asyncio
import json
import threading
import time
from collections.abc import Awaitable, Callable
from typing import Any
import pytest
from websockets.protocol import State
import src.bootstrap.trade_stream_runtime as production_factory
from src.bootstrap.application import (
ApplicationComposition,
run_application,
)
from src.core.config import Settings, TradeStreamSettings
from src.market_data.acquisition.adapters.dzengi.websocket_transport import (
DzengiWebSocketTransport,
)
from src.market_data.acquisition.consistency.trade_stream_state_store import (
TradeStreamStateStore,
)
from src.market_data.acquisition.exceptions import (
TradeTransportError,
WebSocketTransportError,
)
from src.market_data.acquisition.runtime.trade_stream_production_runtime import (
TradeStreamProductionRuntime,
TradeStreamProductionRuntimeState,
)
SYMBOL = "BTC/USD_LEVERAGE"
_OWNED_TASK_NAMES = frozenset(
{
"application-shutdown",
"telegram-polling",
"trade-stream-receive",
"trade-stream-runtime",
"trade-stream-runtime-recovery",
"trade-stream-scheduler",
"trade-stream-startup",
}
)
class FakeBotSession:
def __init__(self) -> None:
self.close_calls = 0
async def close(self) -> None:
self.close_calls += 1
class FakeBot:
def __init__(self) -> None:
self.session = FakeBotSession()
class ControlledDispatcher:
def __init__(
self,
*,
return_immediately: bool = False,
) -> None:
self.return_immediately = return_immediately
self.started = asyncio.Event()
self.release = asyncio.Event()
self.cancelled = asyncio.Event()
self.close_bot_session_values: list[bool] = []
async def start_polling(
self,
bot: FakeBot,
*,
close_bot_session: bool,
) -> None:
del bot
self.close_bot_session_values.append(close_bot_session)
self.started.set()
if self.return_immediately:
return
try:
await self.release.wait()
except asyncio.CancelledError:
self.cancelled.set()
raise
class ScriptedConnection:
def __init__(
self,
*,
name: str,
calls: list[str],
send_error: Exception | None = None,
close_on_receive_error: bool = True,
) -> None:
self.name = name
self.calls = calls
self.send_error = send_error
self.close_on_receive_error = close_on_receive_error
self.state = State.OPEN
self.incoming: asyncio.Queue[
str | bytes | BaseException
] = asyncio.Queue()
self.sent_messages: list[str | bytes] = []
self.subscription_sent = asyncio.Event()
self.close_calls = 0
self.ping_calls = 0
async def close(
self,
code: int = 1000,
reason: str = "",
) -> None:
del code, reason
self.calls.append(f"{self.name}.close")
self.close_calls += 1
self.state = State.CLOSED
async def send(
self,
message: str | bytes,
) -> None:
self.calls.append(f"{self.name}.send")
if self.send_error is not None:
self.state = State.CLOSED
raise self.send_error
self.sent_messages.append(message)
self.subscription_sent.set()
if not isinstance(message, str):
return
document = json.loads(message)
if document.get("destination") != "trades.subscribe":
return
self.feed(
json.dumps(
{
"correlationId": document["correlationId"],
"destination": "trades.subscribe",
"status": "OK",
}
)
)
async def recv(self) -> str | bytes:
item = await self.incoming.get()
if isinstance(item, BaseException):
self.calls.append(f"{self.name}.receive_error")
if self.close_on_receive_error:
self.state = State.CLOSED
raise item
self.calls.append(f"{self.name}.receive")
return item
async def ping(self) -> Awaitable[float]:
self.calls.append(f"{self.name}.ping")
self.ping_calls += 1
async def wait_for_pong() -> float:
return 0.001
return wait_for_pong()
def feed(
self,
item: str | bytes | BaseException,
) -> None:
self.incoming.put_nowait(item)
class ScriptedConnector:
def __init__(
self,
*,
calls: list[str],
connections: tuple[ScriptedConnection, ...] = (),
error: Exception | None = None,
) -> None:
self.calls = calls
self.connections = list(connections)
self.error = error
self.options: list[dict[str, Any]] = []
async def __call__(
self,
url: str,
**kwargs: Any,
) -> ScriptedConnection:
self.calls.append("connector.connect")
self.options.append(
{
"url": url,
**kwargs,
}
)
if self.error is not None:
raise self.error
return self.connections.pop(0)
class BlockingRestClient:
def __init__(
self,
*,
calls: list[str],
document: object,
error: Exception | None = None,
) -> None:
self.calls = calls
self.document = document
self.error = error
self.entered = threading.Event()
self.release = threading.Event()
self.requests: list[
tuple[str, dict[str, str] | None]
] = []
def get_payload(
self,
path: str,
params: dict[str, str] | None = None,
headers: dict[str, str] | None = None,
) -> object:
del headers
self.calls.append("recovery.fetch")
self.requests.append(
(
path,
params,
)
)
self.entered.set()
if not self.release.wait(timeout=5):
raise TimeoutError("test did not release REST recovery")
if self.error is not None:
raise self.error
return self.document
def make_settings(
*,
enabled: bool = True,
) -> Settings:
return Settings(
bot_token="test-token",
bot_parse_mode="HTML",
app_env="test",
log_level="INFO",
tz="UTC",
exchange_enabled=True,
exchange_name="dzengi",
exchange_base_url="https://rest.example.test",
exchange_ws_url="wss://legacy.example.test",
exchange_api_key="",
exchange_api_secret="",
exchange_timeout_sec=10,
exchange_testnet=True,
default_symbol="LEGACY",
trade_stream=TradeStreamSettings(
enabled=enabled,
websocket_url="wss://stream.example.test",
symbols=(SYMBOL,),
open_timeout_seconds=1.0,
probe_timeout_seconds=1.0,
close_timeout_seconds=1.0,
heartbeat_timeout_seconds=120.0,
scheduler_interval_seconds=60.0,
recovery_window_ms=3_599_999,
),
db_host="localhost",
db_port=5432,
db_name="test",
db_user="test",
db_password="test",
debug_enabled=False,
journal_debug_enabled=False,
)
def make_trade_document(
*,
trade_id: int,
timestamp: int,
) -> dict[str, object]:
return {
"status": "OK",
"destination": "internal.trade",
"payload": {
"id": trade_id,
"price": "64555.55",
"size": "0.002",
"ts": timestamp,
"symbol": SYMBOL,
"buyer": True,
"orderId": f"order-{trade_id}",
},
}
def make_recovered_trade(
*,
trade_id: int,
timestamp: int,
) -> dict[str, object]:
return {
"a": trade_id,
"p": "64555.56",
"q": "0.003",
"T": timestamp,
"m": False,
}
def install_connector(
monkeypatch: pytest.MonkeyPatch,
connector: ScriptedConnector,
) -> list[DzengiWebSocketTransport]:
transport_type = production_factory.DzengiWebSocketTransport
transports: list[DzengiWebSocketTransport] = []
def build_transport(
**kwargs: Any,
) -> DzengiWebSocketTransport:
transport = transport_type(
connector=connector,
**kwargs,
)
transports.append(transport)
return transport
monkeypatch.setattr(
production_factory,
"DzengiWebSocketTransport",
build_transport,
)
return transports
def make_application(
*,
runtime: TradeStreamProductionRuntime | None,
dispatcher: ControlledDispatcher,
bot: FakeBot,
) -> ApplicationComposition:
return ApplicationComposition(
bot=bot, # type: ignore[arg-type]
dispatcher=dispatcher, # type: ignore[arg-type]
trade_stream_runtime=runtime,
)
async def wait_until(
predicate: Callable[[], bool],
) -> None:
for _ in range(1_000):
if predicate():
return
await asyncio.sleep(0)
raise AssertionError("condition was not reached")
async def assert_no_owned_tasks() -> None:
await asyncio.sleep(0)
assert not {
task.get_name()
for task in asyncio.all_tasks()
if task is not asyncio.current_task()
and not task.done()
and task.get_name() in _OWNED_TASK_NAMES
}
def state_store_from(
runtime: TradeStreamProductionRuntime,
) -> TradeStreamStateStore:
return (
runtime
._reconnect_recovery_coordinator
._recovery_coordinator
._state_store
)
def test_disabled_feature_runs_only_telegram_without_runtime_graph(
monkeypatch: pytest.MonkeyPatch,
) -> None:
def forbidden_dependency(
*args: Any,
**kwargs: Any,
) -> None:
del args, kwargs
raise AssertionError("disabled feature built a dependency")
monkeypatch.setattr(
production_factory,
"DzengiWebSocketTransport",
forbidden_dependency,
)
monkeypatch.setattr(
production_factory,
"ExchangeRestClient",
forbidden_dependency,
)
runtime = (
production_factory.build_trade_stream_production_runtime(
make_settings(enabled=False),
)
)
async def scenario() -> None:
bot = FakeBot()
dispatcher = ControlledDispatcher(
return_immediately=True,
)
await run_application(
make_application(
runtime=runtime,
dispatcher=dispatcher,
bot=bot,
)
)
assert dispatcher.close_bot_session_values == [False]
assert bot.session.close_calls == 1
await assert_no_owned_tasks()
assert runtime is None
asyncio.run(scenario())
def test_production_factory_processes_ack_trade_and_shutdown(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def scenario() -> None:
calls: list[str] = []
connection = ScriptedConnection(
name="connection",
calls=calls,
)
connector = ScriptedConnector(
calls=calls,
connections=(connection,),
)
transports = install_connector(
monkeypatch,
connector,
)
runtime = (
production_factory.build_trade_stream_production_runtime(
make_settings(),
)
)
assert isinstance(runtime, TradeStreamProductionRuntime)
bot = FakeBot()
dispatcher = ControlledDispatcher()
application_task = asyncio.create_task(
run_application(
make_application(
runtime=runtime,
dispatcher=dispatcher,
bot=bot,
)
)
)
await connection.subscription_sent.wait()
timestamp = time.time_ns() // 1_000_000
connection.feed(
json.dumps(
make_trade_document(
trade_id=100,
timestamp=timestamp,
)
)
)
state_store = state_store_from(runtime)
await wait_until(
lambda: (
state_store.contains(SYMBOL)
and state_store.get(SYMBOL).last_trade_id == 100
)
)
dispatcher.release.set()
await application_task
subscription = json.loads(
connection.sent_messages[0],
)
assert subscription["destination"] == "trades.subscribe"
assert subscription["payload"]["symbols"] == [SYMBOL]
assert connector.options[0]["ping_interval"] is None
assert connector.options[0]["ping_timeout"] is None
assert connection.close_calls == 1
assert runtime.state is (
TradeStreamProductionRuntimeState.STOPPED
)
assert runtime._subscription_manager.subscription_keys == ()
assert bot.session.close_calls == 1
assert len(transports) == 1
await assert_no_owned_tasks()
asyncio.run(scenario())
def test_connection_startup_failure_is_fatal_and_leaves_no_tasks(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def scenario() -> None:
calls: list[str] = []
connector = ScriptedConnector(
calls=calls,
error=OSError("connect failed"),
)
install_connector(
monkeypatch,
connector,
)
runtime = (
production_factory.build_trade_stream_production_runtime(
make_settings(),
)
)
assert isinstance(runtime, TradeStreamProductionRuntime)
bot = FakeBot()
dispatcher = ControlledDispatcher()
with pytest.raises(
WebSocketTransportError,
match="connect failed",
) as exc_info:
await run_application(
make_application(
runtime=runtime,
dispatcher=dispatcher,
bot=bot,
)
)
assert isinstance(exc_info.value.__cause__, OSError)
assert dispatcher.cancelled.is_set()
assert runtime.state is (
TradeStreamProductionRuntimeState.FAILED
)
assert runtime._session.is_connected is False
assert runtime._subscription_manager.subscription_keys == ()
assert bot.session.close_calls == 1
await assert_no_owned_tasks()
asyncio.run(scenario())
def test_subscription_startup_failure_rolls_back_concrete_graph(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def scenario() -> None:
calls: list[str] = []
connection = ScriptedConnection(
name="connection",
calls=calls,
send_error=OSError("subscription failed"),
)
connector = ScriptedConnector(
calls=calls,
connections=(connection,),
)
install_connector(
monkeypatch,
connector,
)
runtime = (
production_factory.build_trade_stream_production_runtime(
make_settings(),
)
)
assert isinstance(runtime, TradeStreamProductionRuntime)
bot = FakeBot()
dispatcher = ControlledDispatcher()
with pytest.raises(
WebSocketTransportError,
match="subscription failed",
):
await run_application(
make_application(
runtime=runtime,
dispatcher=dispatcher,
bot=bot,
)
)
assert dispatcher.cancelled.is_set()
assert connection.state is State.CLOSED
assert runtime.state is (
TradeStreamProductionRuntimeState.FAILED
)
assert runtime._session.is_connected is False
assert runtime._subscription_manager.subscription_keys == ()
assert bot.session.close_calls == 1
await assert_no_owned_tasks()
asyncio.run(scenario())
def test_reconnect_restores_recovers_then_processes_buffered_trade(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def scenario() -> None:
calls: list[str] = []
first_connection = ScriptedConnection(
name="first",
calls=calls,
close_on_receive_error=False,
)
second_connection = ScriptedConnection(
name="second",
calls=calls,
)
connector = ScriptedConnector(
calls=calls,
connections=(
first_connection,
second_connection,
),
)
install_connector(
monkeypatch,
connector,
)
live_timestamp = (
time.time_ns() // 1_000_000
) - 10_000
rest_client = BlockingRestClient(
calls=calls,
document=[
make_recovered_trade(
trade_id=101,
timestamp=live_timestamp + 1,
)
],
)
monkeypatch.setattr(
production_factory,
"ExchangeRestClient",
lambda settings: rest_client,
)
runtime = (
production_factory.build_trade_stream_production_runtime(
make_settings(),
)
)
assert isinstance(runtime, TradeStreamProductionRuntime)
bot = FakeBot()
dispatcher = ControlledDispatcher()
application_task = asyncio.create_task(
run_application(
make_application(
runtime=runtime,
dispatcher=dispatcher,
bot=bot,
)
)
)
await first_connection.subscription_sent.wait()
first_connection.feed(
json.dumps(
make_trade_document(
trade_id=100,
timestamp=live_timestamp,
)
)
)
state_store = state_store_from(runtime)
await wait_until(
lambda: (
state_store.contains(SYMBOL)
and state_store.get(SYMBOL).last_trade_id == 100
)
)
first_connection.feed(
OSError("connection dropped"),
)
await wait_until(rest_client.entered.is_set)
assert second_connection.subscription_sent.is_set()
assert state_store.get(SYMBOL).last_trade_id == 100
second_connection.feed(
json.dumps(
make_trade_document(
trade_id=102,
timestamp=live_timestamp + 2,
)
)
)
rest_client.release.set()
await wait_until(
lambda: state_store.get(SYMBOL).last_trade_id == 102
)
dispatcher.release.set()
await application_task
state = state_store.get(SYMBOL)
assert state.last_trade_id == 102
assert state.last_trade is not None
assert state.last_trade.executed_at.timestamp() == pytest.approx(
(live_timestamp + 2) / 1_000,
)
assert len(connector.options) == 2
assert first_connection.close_calls == 1
assert second_connection.close_calls == 1
assert len(first_connection.sent_messages) == 1
assert len(second_connection.sent_messages) == 1
assert calls.index("second.send") < calls.index(
"recovery.fetch"
)
assert rest_client.requests[0][0] == "/api/v1/aggTrades"
request_params = rest_client.requests[0][1]
assert request_params is not None
assert request_params["symbol"] == SYMBOL
assert runtime._reconnect_recovery_coordinator.generation == 1
assert runtime.state is (
TradeStreamProductionRuntimeState.STOPPED
)
assert bot.session.close_calls == 1
await assert_no_owned_tasks()
asyncio.run(scenario())
def test_recovery_failure_rejects_buffered_trade_and_cleans_up(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def scenario() -> None:
calls: list[str] = []
first_connection = ScriptedConnection(
name="first",
calls=calls,
close_on_receive_error=False,
)
second_connection = ScriptedConnection(
name="second",
calls=calls,
)
connector = ScriptedConnector(
calls=calls,
connections=(
first_connection,
second_connection,
),
)
install_connector(
monkeypatch,
connector,
)
live_timestamp = (
time.time_ns() // 1_000_000
) - 10_000
rest_client = BlockingRestClient(
calls=calls,
document=[],
error=OSError("recovery failed"),
)
monkeypatch.setattr(
production_factory,
"ExchangeRestClient",
lambda settings: rest_client,
)
runtime = (
production_factory.build_trade_stream_production_runtime(
make_settings(),
)
)
assert isinstance(runtime, TradeStreamProductionRuntime)
bot = FakeBot()
dispatcher = ControlledDispatcher()
application_task = asyncio.create_task(
run_application(
make_application(
runtime=runtime,
dispatcher=dispatcher,
bot=bot,
)
)
)
await first_connection.subscription_sent.wait()
first_connection.feed(
json.dumps(
make_trade_document(
trade_id=100,
timestamp=live_timestamp,
)
)
)
state_store = state_store_from(runtime)
await wait_until(
lambda: (
state_store.contains(SYMBOL)
and state_store.get(SYMBOL).last_trade_id == 100
)
)
first_connection.feed(
OSError("connection dropped"),
)
await wait_until(rest_client.entered.is_set)
second_connection.feed(
json.dumps(
make_trade_document(
trade_id=102,
timestamp=live_timestamp + 2,
)
)
)
rest_client.release.set()
with pytest.raises(
TradeTransportError,
match="recovery failed",
):
await application_task
assert state_store.get(SYMBOL).last_trade_id == 100
assert runtime._live_processing_gate.failed is True
assert dispatcher.cancelled.is_set()
assert runtime.state is (
TradeStreamProductionRuntimeState.FAILED
)
assert second_connection.close_calls == 1
assert bot.session.close_calls == 1
await assert_no_owned_tasks()
asyncio.run(scenario())

View File

@@ -0,0 +1,157 @@
from __future__ import annotations
from src.bootstrap.trade_stream_runtime import (
build_trade_stream_production_runtime,
)
from src.core.config import Settings, TradeStreamSettings
from src.market_data.acquisition.runtime.trade_stream_production_runtime import (
TradeStreamProductionRuntime,
TradeStreamProductionRuntimeState,
)
def make_settings(
*,
enabled: bool = True,
api_key: str = "api-key",
) -> Settings:
return Settings(
bot_token="test-token",
bot_parse_mode="HTML",
app_env="test",
log_level="INFO",
tz="UTC",
exchange_enabled=False,
exchange_name="dzengi",
exchange_base_url="https://rest.example.test/",
exchange_ws_url="wss://legacy.example.test",
exchange_api_key=api_key,
exchange_api_secret="secret",
exchange_timeout_sec=17,
exchange_testnet=True,
default_symbol="LEGACY",
trade_stream=TradeStreamSettings(
enabled=enabled,
websocket_url="wss://stream.example.test/root",
symbols=(
"ETH/USD_LEVERAGE",
"BTC/USD_LEVERAGE",
),
open_timeout_seconds=11.0,
probe_timeout_seconds=21.0,
close_timeout_seconds=9.0,
heartbeat_timeout_seconds=31.0,
scheduler_interval_seconds=6.0,
recovery_window_ms=123_456,
),
db_host="localhost",
db_port=5432,
db_name="test",
db_user="test",
db_password="test",
debug_enabled=False,
journal_debug_enabled=False,
)
def test_disabled_feature_does_not_build_runtime() -> None:
settings = make_settings(enabled=False)
assert build_trade_stream_production_runtime(settings) is None
def test_builds_runtime_without_starting_lifecycle() -> None:
runtime = build_trade_stream_production_runtime(
make_settings(),
)
assert isinstance(runtime, TradeStreamProductionRuntime)
assert runtime.state is TradeStreamProductionRuntimeState.STOPPED
assert runtime.running is False
assert runtime._startup_task is None
assert runtime._receive_task is None
assert runtime._scheduler_task is None
def test_uses_one_shared_stateful_dependency_graph() -> None:
runtime = build_trade_stream_production_runtime(
make_settings(),
)
assert isinstance(runtime, TradeStreamProductionRuntime)
transport = runtime._transport
service = runtime._trade_stream_service
reconnect_recovery = runtime._reconnect_recovery_coordinator
recovery = reconnect_recovery._recovery_coordinator
assert runtime._session._transport is transport
assert runtime._subscription_manager._transport is transport
assert runtime._runtime_scheduler.liveness_probe is transport
assert (
service._consistency_controller
is recovery._recovery_controller._consistency_controller
)
assert runtime._live_processing_gate is (
reconnect_recovery.live_processing_gate
)
assert runtime._runtime_scheduler.runtime_supervisor is (
runtime._runtime_supervisor
)
def test_applies_explicit_transport_and_runtime_settings() -> None:
settings = make_settings()
runtime = build_trade_stream_production_runtime(settings)
assert isinstance(runtime, TradeStreamProductionRuntime)
transport = runtime._transport
assert transport._url == "wss://stream.example.test/root/connect"
assert transport._headers == {
"Origin": "https://rest.example.test",
"Content-Type": "application/json",
"X-MBX-APIKEY": "api-key",
}
assert transport._open_timeout == 11.0
assert transport._probe_timeout == 21.0
assert transport._close_timeout == 9.0
assert transport._ping_interval is None
assert transport._ping_timeout is None
assert runtime._symbols == (
"BTC/USD_LEVERAGE",
"ETH/USD_LEVERAGE",
)
assert runtime._runtime_scheduler.interval_seconds == 6.0
assert (
runtime._runtime_supervisor._heartbeat_monitor.timeout_seconds
== 31.0
)
assert (
runtime._reconnect_recovery_coordinator
._recovery_coordinator
._window_planner
.max_window_ms
== 123_456
)
def test_recovery_rest_client_reuses_settings_snapshot() -> None:
settings = make_settings(api_key="")
runtime = build_trade_stream_production_runtime(settings)
assert isinstance(runtime, TradeStreamProductionRuntime)
document_source = (
runtime._reconnect_recovery_coordinator
._recovery_coordinator
._recovery_controller
._document_source
)
rest_client = document_source._client
assert rest_client.settings is settings
assert rest_client.base_url == "https://rest.example.test"
assert rest_client.timeout == 17
assert "X-MBX-APIKEY" not in runtime._transport._headers