diff --git a/app/.env.example b/app/.env.example index 14075b6..fa3adc4 100644 --- a/app/.env.example +++ b/app/.env.example @@ -17,4 +17,14 @@ EXCHANGE_API_KEY= EXCHANGE_API_SECRET= EXCHANGE_TIMEOUT_SEC=10 EXCHANGE_TESTNET=true -DEFAULT_SYMBOL=BTC/USD_LEVERAGE \ No newline at end of file +DEFAULT_SYMBOL=BTC/USD_LEVERAGE + +TRADE_STREAM_ENABLED=false +TRADE_STREAM_WS_URL= +TRADE_STREAM_SYMBOLS= +TRADE_STREAM_OPEN_TIMEOUT_SECONDS=10 +TRADE_STREAM_PROBE_TIMEOUT_SECONDS=20 +TRADE_STREAM_CLOSE_TIMEOUT_SECONDS=10 +TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS=30 +TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS=5 +TRADE_STREAM_RECOVERY_WINDOW_MS=3599999 diff --git a/app/src/bootstrap/app_factory.py b/app/src/bootstrap/app_factory.py index b67fa57..d173996 100644 --- a/app/src/bootstrap/app_factory.py +++ b/app/src/bootstrap/app_factory.py @@ -5,7 +5,11 @@ from __future__ import annotations from aiogram import Bot, Dispatcher from aiogram.client.default import DefaultBotProperties +from src.bootstrap.application import ApplicationComposition from src.bootstrap.logging import setup_logging +from src.bootstrap.trade_stream_runtime import ( + build_trade_stream_production_runtime, +) from src.core.config import load_settings from src.notifications.targets import NotificationTargetRegistry from src.storage.schema import init_schema @@ -13,7 +17,7 @@ from src.telegram.routers import setup_routers from src.trading.journal.service import JournalService -def create_app() -> tuple[Bot, Dispatcher]: +def create_app() -> ApplicationComposition: settings = load_settings() setup_logging(settings.log_level) @@ -37,18 +41,9 @@ def create_app() -> tuple[Bot, Dispatcher]: pass raise - try: - journal.log_info( - "app_started", - "Приложение запущено", - { - "env": settings.app_env, - "exchange_name": settings.exchange_name, - "default_symbol": settings.default_symbol, - }, - ) - except Exception: - pass + trade_stream_runtime = ( + build_trade_stream_production_runtime(settings) + ) bot = Bot( token=settings.bot_token, @@ -61,4 +56,24 @@ def create_app() -> tuple[Bot, Dispatcher]: setup_routers(dispatcher) - return bot, dispatcher \ No newline at end of file + try: + journal.log_info( + "app_started", + "Приложение запущено", + { + "env": settings.app_env, + "exchange_name": settings.exchange_name, + "default_symbol": settings.default_symbol, + "trade_stream_enabled": ( + settings.trade_stream.enabled + ), + }, + ) + except Exception: + pass + + return ApplicationComposition( + bot=bot, + dispatcher=dispatcher, + trade_stream_runtime=trade_stream_runtime, + ) diff --git a/app/src/bootstrap/application.py b/app/src/bootstrap/application.py new file mode 100644 index 0000000..622af6d --- /dev/null +++ b/app/src/bootstrap/application.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass + +from aiogram import Bot, Dispatcher + +from src.market_data.acquisition.runtime.trade_stream_production_runtime import ( + TradeStreamProductionRuntimeProtocol, +) + + +@dataclass(frozen=True, slots=True) +class ApplicationComposition: + """Корневые компоненты одного запущенного экземпляра приложения.""" + + bot: Bot + dispatcher: Dispatcher + trade_stream_runtime: ( + TradeStreamProductionRuntimeProtocol | None + ) + + +async def run_application( + application: ApplicationComposition, +) -> None: + """ + Выполнять Telegram polling и опциональный Trade Stream как одно целое. + + Ошибка включённого Trade Stream считается фатальной для всего + приложения. Остановка Telegram, Trade Stream и bot session + выполняется одним владельцем и не оставляет фоновых root tasks. + """ + polling_task = asyncio.create_task( + application.dispatcher.start_polling( + application.bot, + close_bot_session=False, + ), + name="telegram-polling", + ) + runtime_task = ( + asyncio.create_task( + application.trade_stream_runtime.run(), + name="trade-stream-runtime", + ) + if application.trade_stream_runtime is not None + else None + ) + + primary_error: BaseException | None = None + + try: + await _wait_for_root_tasks( + polling_task=polling_task, + runtime_task=runtime_task, + ) + except BaseException as error: + primary_error = error + + cleanup_task = asyncio.create_task( + _shutdown_application( + application=application, + polling_task=polling_task, + runtime_task=runtime_task, + primary_error=primary_error, + ), + name="application-shutdown", + ) + cleanup_error = await _await_cleanup( + cleanup_task, + primary_error=primary_error, + ) + + if primary_error is not None: + if cleanup_error is not None: + primary_error.add_note( + "Application cleanup also failed: " + f"{type(cleanup_error).__name__}." + ) + + raise primary_error.with_traceback( + primary_error.__traceback__, + ) + + if cleanup_error is not None: + raise cleanup_error.with_traceback( + cleanup_error.__traceback__, + ) + + +async def _wait_for_root_tasks( + *, + polling_task: asyncio.Task[None], + runtime_task: asyncio.Task[None] | None, +) -> None: + if runtime_task is None: + await polling_task + return + + done_tasks, _ = await asyncio.wait( + ( + polling_task, + runtime_task, + ), + return_when=asyncio.FIRST_COMPLETED, + ) + + if runtime_task in done_tasks: + if runtime_task.cancelled(): + raise RuntimeError( + "Trade Stream Runtime was cancelled unexpectedly." + ) + + runtime_error = runtime_task.exception() + + if runtime_error is not None: + raise runtime_error.with_traceback( + runtime_error.__traceback__, + ) + + raise RuntimeError( + "Trade Stream Runtime terminated unexpectedly." + ) + + await polling_task + + +async def _shutdown_application( + *, + application: ApplicationComposition, + polling_task: asyncio.Task[None], + runtime_task: asyncio.Task[None] | None, + primary_error: BaseException | None, +) -> BaseException | None: + cleanup_error: BaseException | None = None + polling_cancelled = False + runtime_cancelled = False + + if not polling_task.done(): + polling_cancelled = True + polling_task.cancel() + + runtime = application.trade_stream_runtime + + if runtime is not None: + try: + await runtime.stop() + except BaseException as error: + cleanup_error = _merge_cleanup_error( + cleanup_error, + error, + ) + + if runtime_task is not None and not runtime_task.done(): + runtime_cancelled = True + runtime_task.cancel() + + cleanup_error = await _observe_root_task( + polling_task, + expected_cancellation=polling_cancelled, + primary_error=primary_error, + cleanup_error=cleanup_error, + ) + + if runtime_task is not None: + cleanup_error = await _observe_root_task( + runtime_task, + expected_cancellation=runtime_cancelled, + primary_error=primary_error, + cleanup_error=cleanup_error, + ) + + try: + await application.bot.session.close() + except BaseException as error: + cleanup_error = _merge_cleanup_error( + cleanup_error, + error, + ) + + return cleanup_error + + +async def _observe_root_task( + task: asyncio.Task[None], + *, + expected_cancellation: bool, + primary_error: BaseException | None, + cleanup_error: BaseException | None, +) -> BaseException | None: + try: + await task + except asyncio.CancelledError as error: + if ( + not expected_cancellation + and error is not primary_error + and not isinstance(primary_error, asyncio.CancelledError) + ): + return _merge_cleanup_error( + cleanup_error, + error, + ) + except BaseException as error: + if error is not primary_error: + return _merge_cleanup_error( + cleanup_error, + error, + ) + + return cleanup_error + + +async def _await_cleanup( + cleanup_task: asyncio.Task[BaseException | None], + *, + primary_error: BaseException | None, +) -> BaseException | None: + interrupted_error: asyncio.CancelledError | None = None + + while not cleanup_task.done(): + try: + await asyncio.shield(cleanup_task) + except asyncio.CancelledError as error: + if primary_error is None: + primary_error = error + interrupted_error = error + continue + + cleanup_error = cleanup_task.result() + + if interrupted_error is not None: + if cleanup_error is not None: + interrupted_error.add_note( + "Application cleanup also failed: " + f"{type(cleanup_error).__name__}." + ) + + raise interrupted_error + + return cleanup_error + + +def _merge_cleanup_error( + current_error: BaseException | None, + new_error: BaseException, +) -> BaseException: + if current_error is None: + return new_error + + current_error.add_note( + "Additional application cleanup failure: " + f"{type(new_error).__name__}." + ) + return current_error diff --git a/app/src/bootstrap/trade_stream_runtime.py b/app/src/bootstrap/trade_stream_runtime.py new file mode 100644 index 0000000..55932b3 --- /dev/null +++ b/app/src/bootstrap/trade_stream_runtime.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from src.core.config import Settings +from src.integrations.exchange.rest_client import ExchangeRestClient +from src.market_data.acquisition.adapters.dzengi.rest import ( + DzengiTradesDocumentSource, +) +from src.market_data.acquisition.adapters.dzengi.websocket import ( + DzengiUnifiedWebSocketAdapter, +) +from src.market_data.acquisition.adapters.dzengi.websocket_control_message_handler import ( + DzengiWebSocketControlMessageHandler, +) +from src.market_data.acquisition.adapters.dzengi.websocket_inbound_message_classifier import ( + DzengiWebSocketInboundMessageClassifier, +) +from src.market_data.acquisition.adapters.dzengi.websocket_transport import ( + DzengiWebSocketTransport, +) +from src.market_data.acquisition.runtime.acquisition_runtime_event_logging_consumer import ( + AcquisitionRuntimeEventLoggingConsumer, +) +from src.market_data.acquisition.runtime.acquisition_runtime_event_publisher import ( + AcquisitionRuntimeEventPublisher, +) +from src.market_data.acquisition.runtime.trade_stream_production_runtime import ( + TradeStreamProductionRuntime, +) +from src.market_data.acquisition.runtime.websocket_session import ( + WebSocketSession, +) +from src.market_data.acquisition.runtime.websocket_subscription_manager import ( + WebSocketSubscriptionManager, +) +from src.market_data.acquisition.trade_stream_runtime_composition import ( + build_trade_stream_runtime_composition, +) + + +def build_trade_stream_production_runtime( + settings: Settings, +) -> TradeStreamProductionRuntime | None: + """ + Собрать Production Trade Stream Runtime без запуска lifecycle. + + Выключенный feature flag возвращает ``None`` до создания + WebSocket/REST-зависимостей. Все stateful-компоненты включённого + Runtime создаются один раз и затем передаются по identity. + """ + trade_stream = settings.trade_stream + + if not trade_stream.enabled: + return None + + headers = { + "Origin": settings.exchange_base_url.rstrip("/"), + "Content-Type": "application/json", + } + + if settings.exchange_api_key: + headers["X-MBX-APIKEY"] = settings.exchange_api_key + + transport = DzengiWebSocketTransport( + url=trade_stream.websocket_url, + headers=headers, + open_timeout=trade_stream.open_timeout_seconds, + ping_interval=None, + ping_timeout=None, + probe_timeout=trade_stream.probe_timeout_seconds, + close_timeout=trade_stream.close_timeout_seconds, + ) + session = WebSocketSession(transport) + subscription_manager = WebSocketSubscriptionManager( + transport, + supports_unsubscribe=False, + ) + event_publisher = AcquisitionRuntimeEventPublisher( + ( + AcquisitionRuntimeEventLoggingConsumer(), + ) + ) + recovery_document_source = DzengiTradesDocumentSource( + ExchangeRestClient(settings=settings), + ) + message_adapter = DzengiUnifiedWebSocketAdapter() + + composition = build_trade_stream_runtime_composition( + session=session, + transport=transport, + subscription_manager=subscription_manager, + event_publisher=event_publisher, + message_adapter=message_adapter, + recovery_document_source=recovery_document_source, + symbols=trade_stream.symbols, + heartbeat_timeout_seconds=( + trade_stream.heartbeat_timeout_seconds + ), + scheduler_interval_seconds=( + trade_stream.scheduler_interval_seconds + ), + max_recovery_window_ms=trade_stream.recovery_window_ms, + ) + + return TradeStreamProductionRuntime( + session=session, + transport=transport, + subscription_manager=subscription_manager, + event_publisher=event_publisher, + trade_stream_service=( + composition.trade_stream_acquisition_service + ), + message_classifier=( + DzengiWebSocketInboundMessageClassifier() + ), + control_message_handler=( + DzengiWebSocketControlMessageHandler() + ), + live_processing_gate=composition.live_processing_gate, + reconnect_recovery_coordinator=( + composition.runtime_reconnect_recovery_coordinator + ), + runtime_supervisor=composition.runtime_supervisor, + runtime_scheduler=composition.runtime_scheduler, + symbols=trade_stream.symbols, + ) diff --git a/app/src/core/config.py b/app/src/core/config.py index 7bc5a21..2bcfbbf 100644 --- a/app/src/core/config.py +++ b/app/src/core/config.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math import os from dataclasses import dataclass from pathlib import Path @@ -19,6 +20,21 @@ ENV_FILE = BASE_DIR / ".env" load_dotenv(ENV_FILE) +@dataclass(frozen=True, slots=True) +class TradeStreamSettings: + """Настройки Production Trade Stream Runtime.""" + + enabled: bool + websocket_url: str + symbols: tuple[str, ...] + open_timeout_seconds: float + probe_timeout_seconds: float + close_timeout_seconds: float + heartbeat_timeout_seconds: float + scheduler_interval_seconds: float + recovery_window_ms: int + + @dataclass(slots=True) class Settings: # Telegram @@ -40,6 +56,7 @@ class Settings: exchange_timeout_sec: int exchange_testnet: bool default_symbol: str + trade_stream: TradeStreamSettings # Database db_host: str @@ -75,6 +92,206 @@ def _parse_int(raw_value: str, default: int) -> int: return int(value) +def _parse_strict_bool( + raw_value: str, + *, + name: str, + default: bool, +) -> bool: + value = (raw_value or "").strip().lower() + + if not value: + return default + + if value in {"1", "true", "yes", "on"}: + return True + + if value in {"0", "false", "no", "off"}: + return False + + raise ValueError( + f"{name} must be a boolean value" + ) + + +def _parse_positive_float( + raw_value: str, + *, + name: str, + default: float, +) -> float: + value = (raw_value or "").strip() + + try: + result = ( + float(value) + if value + else float(default) + ) + except ValueError as error: + raise ValueError( + f"{name} must be a number" + ) from error + + if not math.isfinite(result) or result <= 0: + raise ValueError( + f"{name} must be positive and finite" + ) + + return result + + +def _parse_positive_int( + raw_value: str, + *, + name: str, + default: int, +) -> int: + value = (raw_value or "").strip() + + try: + result = ( + int(value) + if value + else int(default) + ) + except ValueError as error: + raise ValueError( + f"{name} must be an integer" + ) from error + + if result <= 0: + raise ValueError( + f"{name} must be positive" + ) + + return result + + +def _parse_trade_stream_symbols( + raw_value: str, +) -> tuple[str, ...]: + value = (raw_value or "").strip() + + if not value: + raise RuntimeError( + "TRADE_STREAM_SYMBOLS is required when " + "TRADE_STREAM_ENABLED is true" + ) + + raw_symbols = value.split(",") + + if any(not symbol.strip() for symbol in raw_symbols): + raise ValueError( + "TRADE_STREAM_SYMBOLS must not contain empty symbols" + ) + + return tuple( + sorted( + { + symbol.strip() + for symbol in raw_symbols + } + ) + ) + + +def _load_trade_stream_settings( + *, + exchange_base_url: str, +) -> TradeStreamSettings: + enabled = _parse_strict_bool( + os.getenv("TRADE_STREAM_ENABLED", "false"), + name="TRADE_STREAM_ENABLED", + default=False, + ) + + if not enabled: + return TradeStreamSettings( + enabled=False, + websocket_url="", + symbols=(), + open_timeout_seconds=10.0, + probe_timeout_seconds=20.0, + close_timeout_seconds=10.0, + heartbeat_timeout_seconds=30.0, + scheduler_interval_seconds=5.0, + recovery_window_ms=3_599_999, + ) + + websocket_url = os.getenv( + "TRADE_STREAM_WS_URL", + "", + ).strip() + + if not websocket_url: + raise RuntimeError( + "TRADE_STREAM_WS_URL is required when " + "TRADE_STREAM_ENABLED is true" + ) + + if not exchange_base_url: + raise RuntimeError( + "EXCHANGE_BASE_URL is required for Trade Stream Recovery" + ) + + return TradeStreamSettings( + enabled=True, + websocket_url=websocket_url, + symbols=_parse_trade_stream_symbols( + os.getenv("TRADE_STREAM_SYMBOLS", ""), + ), + open_timeout_seconds=_parse_positive_float( + os.getenv( + "TRADE_STREAM_OPEN_TIMEOUT_SECONDS", + "10", + ), + name="TRADE_STREAM_OPEN_TIMEOUT_SECONDS", + default=10.0, + ), + probe_timeout_seconds=_parse_positive_float( + os.getenv( + "TRADE_STREAM_PROBE_TIMEOUT_SECONDS", + "20", + ), + name="TRADE_STREAM_PROBE_TIMEOUT_SECONDS", + default=20.0, + ), + close_timeout_seconds=_parse_positive_float( + os.getenv( + "TRADE_STREAM_CLOSE_TIMEOUT_SECONDS", + "10", + ), + name="TRADE_STREAM_CLOSE_TIMEOUT_SECONDS", + default=10.0, + ), + heartbeat_timeout_seconds=_parse_positive_float( + os.getenv( + "TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS", + "30", + ), + name="TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS", + default=30.0, + ), + scheduler_interval_seconds=_parse_positive_float( + os.getenv( + "TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS", + "5", + ), + name="TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS", + default=5.0, + ), + recovery_window_ms=_parse_positive_int( + os.getenv( + "TRADE_STREAM_RECOVERY_WINDOW_MS", + "3599999", + ), + name="TRADE_STREAM_RECOVERY_WINDOW_MS", + default=3_599_999, + ), + ) + + # load all settings def load_settings() -> Settings: bot_token = os.getenv("BOT_TOKEN", "").strip() @@ -82,6 +299,11 @@ def load_settings() -> Settings: if not bot_token: raise RuntimeError("BOT_TOKEN is not set in app/.env") + exchange_base_url = os.getenv( + "EXCHANGE_BASE_URL", + "", + ).strip() + return Settings( # Telegram bot_token=bot_token, @@ -99,7 +321,7 @@ def load_settings() -> Settings: # Exchange exchange_enabled=_parse_bool(os.getenv("EXCHANGE_ENABLED", "false")), exchange_name=os.getenv("EXCHANGE_NAME", "dzengi").strip() or "dzengi", - exchange_base_url=os.getenv("EXCHANGE_BASE_URL", "").strip(), + exchange_base_url=exchange_base_url, exchange_ws_url=os.getenv("EXCHANGE_WS_URL", "").strip(), exchange_api_key=os.getenv("EXCHANGE_API_KEY", "").strip(), exchange_api_secret=os.getenv("EXCHANGE_API_SECRET", "").strip(), @@ -107,6 +329,9 @@ def load_settings() -> Settings: exchange_testnet=_parse_bool(os.getenv("EXCHANGE_TESTNET", "false")), default_symbol=os.getenv("DEFAULT_SYMBOL", "ETH/USD_LEVERAGE").strip() or "ETH/USD_LEVERAGE", + trade_stream=_load_trade_stream_settings( + exchange_base_url=exchange_base_url, + ), # Database db_host=os.getenv("DB_HOST", "localhost").strip() or "localhost", @@ -114,4 +339,4 @@ def load_settings() -> Settings: db_name=os.getenv("DB_NAME", "dzentra_bot").strip() or "dzentra_bot", db_user=os.getenv("DB_USER", "dzentra_bot").strip() or "dzentra_bot", db_password=os.getenv("DB_PASSWORD", "").strip(), - ) \ No newline at end of file + ) diff --git a/app/src/integrations/exchange/rest_client.py b/app/src/integrations/exchange/rest_client.py index b1bccc9..c051460 100644 --- a/app/src/integrations/exchange/rest_client.py +++ b/app/src/integrations/exchange/rest_client.py @@ -7,7 +7,7 @@ from urllib.error import HTTPError, URLError from urllib.parse import urlencode from urllib.request import Request, urlopen -from src.core.config import load_settings +from src.core.config import Settings, load_settings from src.integrations.exchange.exceptions import ( ExchangeConnectionError, ExchangeResponseError, @@ -15,8 +15,15 @@ from src.integrations.exchange.exceptions import ( class ExchangeRestClient: - def __init__(self) -> None: - self.settings = load_settings() + def __init__( + self, + settings: Settings | None = None, + ) -> None: + self.settings = ( + settings + if settings is not None + else load_settings() + ) if not self.settings.exchange_base_url: raise ExchangeConnectionError("EXCHANGE_BASE_URL is empty.") self.base_url = self.settings.exchange_base_url.rstrip("/") @@ -133,4 +140,4 @@ class ExchangeRestClient: if not isinstance(payload, dict): raise ExchangeResponseError("Exchange response is not a JSON object.") - return payload \ No newline at end of file + return payload diff --git a/app/src/main.py b/app/src/main.py index 3c71666..ad1484a 100644 --- a/app/src/main.py +++ b/app/src/main.py @@ -2,26 +2,14 @@ import asyncio +from src.bootstrap.application import run_application from src.bootstrap.app_factory import create_app async def main() -> None: - # создаём bot + dispatcher - bot, dispatcher = create_app() - - # WebSocket stream временно отключён. - # Причина: Dzengi Swagger содержит wss:/api/v1/* endpoints, - # но runtime probe не нашёл endpoint с WebSocket Upgrade 101. - # - # Когда Dzengi подтвердит рабочий WS endpoint, - # можно будет вернуть запуск: - # - # from src.integrations.exchange.market_stream import start_market_stream - # market_stream_task = asyncio.create_task(start_market_stream()) - - # запускаем Telegram polling - await dispatcher.start_polling(bot) + application = create_app() + await run_application(application) if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/app/src/market_data/acquisition/adapters/dzengi/websocket_control_message_handler.py b/app/src/market_data/acquisition/adapters/dzengi/websocket_control_message_handler.py new file mode 100644 index 0000000..a9f35b2 --- /dev/null +++ b/app/src/market_data/acquisition/adapters/dzengi/websocket_control_message_handler.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from src.market_data.acquisition.exceptions import ( + WebSocketControlMessageError, + WebSocketMessageRoutingError, +) + + +_TRADE_SUBSCRIPTION_DESTINATION = "trades.subscribe" +_SUCCESS_STATUS = "OK" + + +class DzengiWebSocketControlMessageHandler: + """ + Provider-specific проверка Dzengi control/ACK сообщений. + + Handler не управляет lifecycle и не хранит pending requests. + Ожидаемый correlation ID передаётся владельцем Runtime. + """ + + __slots__ = () + + def handle( + self, + document: object, + *, + expected_correlation_id: str, + ) -> None: + """Проверить ACK текущей Trade Stream подписки.""" + if not isinstance(document, dict): + raise WebSocketMessageRoutingError( + "Control message Dzengi WebSocket должен быть объектом." + ) + + correlation_id = document.get("correlationId") + + if correlation_id != expected_correlation_id: + raise WebSocketMessageRoutingError( + "Получен control message с неизвестным correlationId." + ) + + status = document.get("status") + + if not isinstance(status, str) or not status.strip(): + raise WebSocketMessageRoutingError( + "Control message Dzengi WebSocket должен содержать " + "непустой строковый status." + ) + + if status != _SUCCESS_STATUS: + raise WebSocketControlMessageError( + "Dzengi отклонил Trade Stream subscription ACK." + ) + + if ( + document.get("destination") + != _TRADE_SUBSCRIPTION_DESTINATION + ): + raise WebSocketMessageRoutingError( + "Получен успешный control message с неизвестным " + "destination." + ) diff --git a/app/src/market_data/acquisition/adapters/dzengi/websocket_inbound_message_classifier.py b/app/src/market_data/acquisition/adapters/dzengi/websocket_inbound_message_classifier.py new file mode 100644 index 0000000..dd923c9 --- /dev/null +++ b/app/src/market_data/acquisition/adapters/dzengi/websocket_inbound_message_classifier.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from src.market_data.acquisition.exceptions import ( + WebSocketMessageRoutingError, +) +from src.market_data.acquisition.runtime.websocket_inbound_message import ( + WebSocketInboundMessageKind, +) + + +_DZENGI_MARKET_DESTINATIONS = frozenset( + { + "internal.trade", + "ohlc.event", + } +) + + +class DzengiWebSocketInboundMessageClassifier: + """ + Классификатор декодированных входящих сообщений Dzengi. + + Market markers имеют приоритет над correlationId, чтобы market + document с дополнительными transport metadata не был ошибочно + принят за control response. + """ + + __slots__ = () + + def classify( + self, + document: object, + ) -> WebSocketInboundMessageKind: + """Разделить market documents и ответы на Runtime-команды.""" + if not isinstance(document, dict): + raise WebSocketMessageRoutingError( + "Сообщение Dzengi WebSocket должно быть объектом." + ) + + destination = document.get("destination") + + if ( + ( + isinstance(destination, str) + and destination in _DZENGI_MARKET_DESTINATIONS + ) + or "Payload" in document + ): + return WebSocketInboundMessageKind.MARKET + + if "correlationId" in document: + self._validate_correlation_id( + document["correlationId"], + ) + return WebSocketInboundMessageKind.CONTROL + + raise WebSocketMessageRoutingError( + "Не удалось определить категорию сообщения " + "Dzengi WebSocket." + ) + + @staticmethod + def _validate_correlation_id( + correlation_id: object, + ) -> None: + if ( + isinstance(correlation_id, bool) + or not isinstance(correlation_id, (str, int)) + ): + raise WebSocketMessageRoutingError( + "correlationId сообщения Dzengi WebSocket должен " + "быть строкой или целым числом." + ) + + if ( + isinstance(correlation_id, str) + and not correlation_id.strip() + ): + raise WebSocketMessageRoutingError( + "correlationId сообщения Dzengi WebSocket " + "не должен быть пустым." + ) diff --git a/app/src/market_data/acquisition/adapters/dzengi/websocket_transport.py b/app/src/market_data/acquisition/adapters/dzengi/websocket_transport.py new file mode 100644 index 0000000..000270f --- /dev/null +++ b/app/src/market_data/acquisition/adapters/dzengi/websocket_transport.py @@ -0,0 +1,445 @@ +from __future__ import annotations + +import asyncio +import math +from collections.abc import Awaitable, Callable, Mapping +from typing import Protocol, cast +from urllib.parse import urlsplit, urlunsplit + +from websockets.asyncio.client import connect as websocket_connect +from websockets.protocol import State +from websockets.typing import Subprotocol + +from src.market_data.acquisition.exceptions import ( + WebSocketTransportError, + WebSocketTransportNotConnectedError, +) + + +class _WebSocketConnectionProtocol(Protocol): + """ + Минимальный контракт соединения. + """ + + @property + def state(self) -> State: + """Вернуть состояние WebSocket-соединения.""" + ... + + async def close( + self, + code: int = 1000, + reason: str = "", + ) -> None: + """Закрыть WebSocket-соединение.""" + ... + + async def send( + self, + message: str | bytes, + ) -> None: + """Отправить одно сообщение.""" + ... + + async def recv(self) -> str | bytes: + """Получить одно сообщение.""" + ... + + async def ping(self) -> Awaitable[float]: + """Отправить Ping и вернуть ожидание соответствующего Pong.""" + ... + + +WebSocketConnector = Callable[ + ..., + Awaitable[_WebSocketConnectionProtocol], +] + + +def build_dzengi_websocket_url( + raw_url: str, +) -> str: + """ + Нормализовать Dzengi HTTP/WebSocket URL до endpoint ``/connect``. + """ + if not isinstance(raw_url, str): + raise TypeError("raw_url must be a string") + + normalized_url = raw_url.strip() + + if not normalized_url: + raise ValueError("raw_url must not be empty") + + parsed = urlsplit(normalized_url) + + scheme = { + "http": "ws", + "https": "wss", + "ws": "ws", + "wss": "wss", + }.get(parsed.scheme.lower()) + + if scheme is None or not parsed.netloc: + raise ValueError( + "raw_url must be an absolute HTTP or WebSocket URL" + ) + + if parsed.fragment: + raise ValueError("raw_url must not contain a fragment") + + path = parsed.path.rstrip("/") + + if not path.endswith("/connect"): + path = f"{path}/connect" + + return urlunsplit( + ( + scheme, + parsed.netloc, + path, + parsed.query, + "", + ) + ) + + +class DzengiWebSocketTransport: + """ + Низкоуровневый WebSocket-транспорт Dzengi. + + Transport управляет только соединением и непрозрачными + ``str``/``bytes`` сообщениями. JSON parsing, subscription routing, + recovery и lifecycle находятся за пределами этого компонента. + """ + + __slots__ = ( + "_url", + "_headers", + "_open_timeout", + "_ping_interval", + "_ping_timeout", + "_probe_timeout", + "_close_timeout", + "_connector", + "_connection", + "_lifecycle_lock", + ) + + def __init__( + self, + *, + url: str, + headers: Mapping[str, str] | None = None, + open_timeout: float | None = 10.0, + ping_interval: float | None = 20.0, + ping_timeout: float | None = 20.0, + probe_timeout: float | None = None, + close_timeout: float | None = 10.0, + connector: WebSocketConnector | None = None, + ) -> None: + self._url = build_dzengi_websocket_url(url) + self._headers = self._validate_headers(headers) + self._open_timeout = self._validate_timeout( + "open_timeout", + open_timeout, + ) + self._ping_interval = self._validate_timeout( + "ping_interval", + ping_interval, + ) + self._ping_timeout = self._validate_timeout( + "ping_timeout", + ping_timeout, + ) + self._probe_timeout = self._validate_probe_timeout( + ( + probe_timeout + if probe_timeout is not None + else ( + ping_timeout + if ping_timeout is not None + else 20.0 + ) + ), + ) + self._close_timeout = self._validate_timeout( + "close_timeout", + close_timeout, + ) + self._connector = ( + connector + if connector is not None + else cast( + WebSocketConnector, + websocket_connect, + ) + ) + self._connection: _WebSocketConnectionProtocol | None = None + self._lifecycle_lock = asyncio.Lock() + + @property + def url(self) -> str: + """Вернуть нормализованный WebSocket URL.""" + return self._url + + @property + def is_connected(self) -> bool: + """Показывает, открыто ли соединение.""" + connection = self._connection + + return ( + connection is not None + and connection.state is State.OPEN + ) + + async def connect(self) -> None: + """ + Идемпотентно открыть WebSocket-соединение Dzengi. + """ + async with self._lifecycle_lock: + if self.is_connected: + return + + self._connection = None + + try: + connection = await self._connector( + self._url, + additional_headers=( + self._headers + if self._headers + else None + ), + subprotocols=( + Subprotocol("json"), + ), + open_timeout=self._open_timeout, + ping_interval=self._ping_interval, + ping_timeout=self._ping_timeout, + close_timeout=self._close_timeout, + ) + except Exception as error: + raise WebSocketTransportError( + "Не удалось открыть Dzengi WebSocket connection: " + f"{error}" + ) from error + + self._connection = connection + + if self.is_connected: + return + + self._connection = None + + try: + await connection.close() + except Exception: + pass + + raise WebSocketTransportError( + "Dzengi WebSocket connection не перешло " + "в состояние OPEN." + ) + + async def disconnect(self) -> None: + """ + Идемпотентно закрыть WebSocket-соединение. + """ + async with self._lifecycle_lock: + connection = self._connection + self._connection = None + + if connection is None: + return + + try: + await connection.close() + except Exception as error: + raise WebSocketTransportError( + "Не удалось закрыть Dzengi WebSocket connection: " + f"{error}" + ) from error + + async def send( + self, + message: str | bytes, + ) -> None: + """Отправить непрозрачное транспортное сообщение.""" + if not isinstance(message, (str, bytes)): + raise TypeError("message must be str or bytes") + + connection = self._require_connection() + + try: + await connection.send(message) + except Exception as error: + self._discard_closed_connection(connection) + + raise WebSocketTransportError( + "Не удалось отправить Dzengi WebSocket message: " + f"{error}" + ) from error + + async def receive(self) -> str | bytes: + """Получить непрозрачное транспортное сообщение.""" + connection = self._require_connection() + + try: + return await connection.recv() + except Exception as error: + self._discard_closed_connection(connection) + + raise WebSocketTransportError( + "Не удалось получить Dzengi WebSocket message: " + f"{error}" + ) from error + + async def probe(self) -> bool: + """ + Подтвердить liveness текущего connection через Ping/Pong. + + Закрытое соединение и отсутствие Pong являются штатным + отрицательным результатом. Cancellation не перехватывается. + """ + connection = self._connection + + if ( + connection is None + or connection.state is not State.OPEN + ): + self._connection = None + return False + + try: + pong_waiter = await connection.ping() + + await asyncio.wait_for( + asyncio.shield( + pong_waiter, + ), + timeout=self._probe_timeout, + ) + except asyncio.TimeoutError: + return False + except asyncio.CancelledError: + raise + except Exception as error: + self._discard_closed_connection(connection) + + if connection.state is not State.OPEN: + return False + + raise WebSocketTransportError( + "Не удалось проверить Dzengi WebSocket liveness: " + f"{error}" + ) from error + + return ( + self._connection is connection + and connection.state is State.OPEN + ) + + def _require_connection( + self, + ) -> _WebSocketConnectionProtocol: + """ + Вернуть открытое соединение. + """ + connection = self._connection + + if ( + connection is None + or connection.state is not State.OPEN + ): + self._connection = None + + raise WebSocketTransportNotConnectedError( + "Dzengi WebSocket transport is not connected." + ) + + return connection + + def _discard_closed_connection( + self, + connection: _WebSocketConnectionProtocol, + ) -> None: + """ + Удалить ссылку на закрытое соединение. + """ + if ( + self._connection is connection + and connection.state is not State.OPEN + ): + self._connection = None + + @staticmethod + def _validate_headers( + headers: Mapping[str, str] | None, + ) -> dict[str, str]: + if headers is None: + return {} + + if not isinstance(headers, Mapping): + raise TypeError("headers must be a mapping") + + normalized_headers: dict[str, str] = {} + + for name, value in headers.items(): + if not isinstance(name, str) or not name.strip(): + raise ValueError( + "header names must be non-empty strings" + ) + + if not isinstance(value, str): + raise TypeError("header values must be strings") + + normalized_headers[name] = value + + return normalized_headers + + @staticmethod + def _validate_timeout( + name: str, + value: float | None, + ) -> float | None: + if value is None: + return None + + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + ): + raise TypeError( + f"{name} must be an integer, float, or None" + ) + + if value <= 0: + raise ValueError( + f"{name} must be positive" + ) + + return float(value) + + @staticmethod + def _validate_probe_timeout( + value: float, + ) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + ): + raise TypeError( + "probe_timeout must be an integer or float" + ) + + normalized_value = float(value) + + if ( + not math.isfinite(normalized_value) + or normalized_value <= 0 + ): + raise ValueError( + "probe_timeout must be positive and finite" + ) + + return normalized_value diff --git a/app/src/market_data/acquisition/exceptions.py b/app/src/market_data/acquisition/exceptions.py index e5fb153..b0ce85f 100644 --- a/app/src/market_data/acquisition/exceptions.py +++ b/app/src/market_data/acquisition/exceptions.py @@ -153,4 +153,31 @@ class TradeFeedRegistryError(MarketDataAcquisitionError): # Ошибка определения типа входящего WebSocket-сообщения # и выбора специализированного адаптера. class WebSocketMessageRoutingError(MarketDataAcquisitionError): - pass \ No newline at end of file + pass + + +# Ошибка декодирования сырого WebSocket-сообщения. +class WebSocketMessageDecodeError(MarketDataAcquisitionError): + pass + + +# Ошибка обработки control/ACK сообщения WebSocket-провайдера. +class WebSocketControlMessageError(MarketDataAcquisitionError): + pass + + +# Ошибка низкоуровневой WebSocket-инфраструктуры Acquisition Runtime. +class WebSocketTransportError(MarketDataAcquisitionError): + pass + + +# Попытка использовать WebSocket-транспорт без открытого соединения. +class WebSocketTransportNotConnectedError(WebSocketTransportError): + pass + + +# WebSocket-провайдер не поддерживает транспортную операцию unsubscribe. +class WebSocketUnsubscribeNotSupportedError( + MarketDataAcquisitionError, +): + pass diff --git a/app/src/market_data/acquisition/runtime/acquisition_runtime_event_logging_consumer.py b/app/src/market_data/acquisition/runtime/acquisition_runtime_event_logging_consumer.py new file mode 100644 index 0000000..1df4c4a --- /dev/null +++ b/app/src/market_data/acquisition/runtime/acquisition_runtime_event_logging_consumer.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import logging + +from src.market_data.acquisition.runtime.runtime_events import ( + ConnectedEvent, + ConnectFailedEvent, + DisconnectedEvent, + HeartbeatTimeoutEvent, + MessageReceivedEvent, + MessageSentEvent, + ReconnectCompletedEvent, + ReconnectFailedEvent, + ReconnectStartedEvent, + TransportMessage, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionRuntimeEvent, +) + + +logger = logging.getLogger(__name__) + + +class AcquisitionRuntimeEventLoggingConsumer: + """ + Безопасно записывает Acquisition Runtime Events в standard logging. + + Содержимое transport payload намеренно не журналируется. Для + MessageReceivedEvent и MessageSentEvent фиксируются только тип + сообщения и размер payload. + """ + + __slots__ = ( + "_logger", + ) + + def __init__( + self, + event_logger: logging.Logger = logger, + ) -> None: + if not isinstance(event_logger, logging.Logger): + raise TypeError( + "event_logger must be a logging.Logger" + ) + + self._logger = event_logger + + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + """Записать одно Acquisition Runtime Event.""" + if isinstance(event, ConnectedEvent): + self._logger.info( + "Acquisition WebSocket connected." + ) + return + + if isinstance(event, DisconnectedEvent): + self._logger.info( + "Acquisition WebSocket disconnected." + ) + return + + if isinstance(event, ConnectFailedEvent): + self._logger.error( + "Acquisition WebSocket connection failed: %s", + event.reason, + ) + return + + if isinstance(event, MessageReceivedEvent): + self._log_transport_message( + direction="received", + message=event.message, + ) + return + + if isinstance(event, MessageSentEvent): + self._log_transport_message( + direction="sent", + message=event.message, + ) + return + + if isinstance(event, ReconnectStartedEvent): + self._logger.info( + "Acquisition WebSocket reconnect started: attempt=%d", + event.attempt, + ) + return + + if isinstance(event, ReconnectCompletedEvent): + self._logger.info( + "Acquisition WebSocket reconnect completed: attempt=%d", + event.attempt, + ) + return + + if isinstance(event, ReconnectFailedEvent): + self._logger.error( + "Acquisition WebSocket reconnect failed: " + "attempt=%d reason=%s", + event.attempt, + event.reason, + ) + return + + if isinstance(event, HeartbeatTimeoutEvent): + self._logger.warning( + "Acquisition WebSocket heartbeat timeout: " + "timeout_seconds=%s", + event.timeout_seconds, + ) + return + + raise TypeError( + "event must be an AcquisitionRuntimeEvent" + ) + + def _log_transport_message( + self, + *, + direction: str, + message: TransportMessage, + ) -> None: + self._logger.debug( + "Acquisition WebSocket message %s: " + "message_type=%s payload_size=%d", + direction, + type(message).__name__, + len(message.payload), + ) diff --git a/app/src/market_data/acquisition/runtime/acquisition_runtime_event_publisher.py b/app/src/market_data/acquisition/runtime/acquisition_runtime_event_publisher.py new file mode 100644 index 0000000..63ccf56 --- /dev/null +++ b/app/src/market_data/acquisition/runtime/acquisition_runtime_event_publisher.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Iterable +from contextvars import ContextVar +from typing import Protocol, runtime_checkable + +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionRuntimeEvent, +) + + +logger = logging.getLogger(__name__) + +_active_publisher_ids: ContextVar[tuple[int, ...]] = ContextVar( + "active_acquisition_runtime_event_publisher_ids", + default=(), +) + + +@runtime_checkable +class AcquisitionRuntimeEventConsumerProtocol(Protocol): + """ + Контракт одного потребителя Acquisition Runtime Events. + + Consumer обрабатывает уже произошедший инфраструктурный факт + и не управляет lifecycle Publisher. + """ + + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + """Обработать одно Acquisition Runtime Event.""" + ... + + +class AcquisitionRuntimeEventPublisher: + """ + Последовательный in-process Publisher Acquisition Runtime Events. + + Один вызов publish() полностью доставляет событие всем Consumer + в порядке их регистрации. Параллельные вызовы сериализуются одним + asyncio.Lock. + + Publisher не создаёт фоновых задач, не хранит очередь и не управляет + lifecycle Consumer. + + Обычная ошибка одного Consumer журналируется и не препятствует + доставке остальным Consumer. Сигналы отмены и другие BaseException + не перехватываются. + """ + + __slots__ = ( + "_consumers", + "_publish_lock", + ) + + def __init__( + self, + consumers: Iterable[ + AcquisitionRuntimeEventConsumerProtocol + ] = (), + ) -> None: + registered_consumers = tuple(consumers) + + for consumer in registered_consumers: + if not isinstance( + consumer, + AcquisitionRuntimeEventConsumerProtocol, + ): + raise TypeError( + "consumer must satisfy " + "AcquisitionRuntimeEventConsumerProtocol" + ) + + self._consumers = registered_consumers + self._publish_lock = asyncio.Lock() + + async def publish( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + """ + Последовательно доставить событие всем Consumer. + + Рекурсивная публикация через тот же Publisher запрещена. + Такое ограничение предотвращает deadlock на publish lock. + """ + if not isinstance(event, AcquisitionRuntimeEvent): + raise TypeError( + "event must be an AcquisitionRuntimeEvent" + ) + + publisher_id = id(self) + active_publisher_ids = _active_publisher_ids.get() + + if publisher_id in active_publisher_ids: + raise RuntimeError( + "Recursive publication through the same " + "AcquisitionRuntimeEventPublisher is not supported." + ) + + active_token = _active_publisher_ids.set( + (*active_publisher_ids, publisher_id) + ) + + try: + async with self._publish_lock: + for consumer in self._consumers: + try: + await consumer.consume(event) + except Exception as error: + self._report_consumer_error( + event=event, + consumer=consumer, + error=error, + ) + finally: + _active_publisher_ids.reset(active_token) + + @staticmethod + def _report_consumer_error( + *, + event: AcquisitionRuntimeEvent, + consumer: AcquisitionRuntimeEventConsumerProtocol, + error: Exception, + ) -> None: + """ + Безопасно зафиксировать ошибку Consumer. + + Текст исключения и traceback намеренно не журналируются: + они могут содержать repr(event) с transport payload. + + Ошибка logging handler не должна прерывать Runtime lifecycle + или доставку события остальным Consumer. + """ + try: + logger.error( + "Acquisition Runtime Event consumer failed: " + "event=%s consumer=%s error_type=%s", + type(event).__name__, + type(consumer).__name__, + type(error).__name__, + ) + except Exception: + return diff --git a/app/src/market_data/acquisition/runtime/live_processing_gate.py b/app/src/market_data/acquisition/runtime/live_processing_gate.py new file mode 100644 index 0000000..ba44178 --- /dev/null +++ b/app/src/market_data/acquisition/runtime/live_processing_gate.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from types import TracebackType +from typing import Protocol, runtime_checkable + +import asyncio + + +@runtime_checkable +class RuntimeLiveProcessingGateProtocol(Protocol): + """ + Контракт общей границы Live processing и Runtime Recovery. + """ + + @property + def locked(self) -> bool: + """Показывает, закрыт ли gate текущей операцией.""" + ... + + @property + def failed(self) -> bool: + """Показывает, запрещено ли продолжать Live processing.""" + ... + + def fail( + self, + error: Exception, + ) -> None: + """Запретить Live processing после terminal Runtime error.""" + ... + + def reset(self) -> None: + """Разрешить Live processing для нового Runtime lifecycle.""" + ... + + async def __aenter__( + self, + ) -> RuntimeLiveProcessingGateProtocol: + """Закрыть gate и дождаться эксклюзивного доступа.""" + ... + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """Открыть gate после завершения защищённой операции.""" + ... + + +class RuntimeLiveProcessingGate: + """ + Единый asyncio gate для Live processing и Recovery. + + Gate не содержит market state и не создаёт фоновые задачи. + """ + + __slots__ = ( + "_lock", + "_failure", + ) + + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._failure: Exception | None = None + + @property + def locked(self) -> bool: + """Показывает, закрыт ли gate текущей операцией.""" + return self._lock.locked() + + @property + def failed(self) -> bool: + """Показывает, запрещено ли продолжать Live processing.""" + return self._failure is not None + + def fail( + self, + error: Exception, + ) -> None: + """Запретить Live processing после terminal Runtime error.""" + if not isinstance(error, Exception): + raise TypeError("error must be an Exception") + + self._failure = error + + def reset(self) -> None: + """Разрешить Live processing для нового Runtime lifecycle.""" + if self._lock.locked(): + raise RuntimeError( + "Live processing gate cannot be reset while locked." + ) + + self._failure = None + + async def __aenter__( + self, + ) -> RuntimeLiveProcessingGate: + """Закрыть gate и дождаться эксклюзивного доступа.""" + await self._lock.acquire() + + failure = self._failure + + if failure is not None: + self._lock.release() + raise failure + + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """Открыть gate после завершения защищённой операции.""" + self._lock.release() diff --git a/app/src/market_data/acquisition/runtime/reconnect.py b/app/src/market_data/acquisition/runtime/reconnect.py index dc8dfb0..f35032d 100644 --- a/app/src/market_data/acquisition/runtime/reconnect.py +++ b/app/src/market_data/acquisition/runtime/reconnect.py @@ -7,6 +7,7 @@ from typing import Protocol, runtime_checkable from src.market_data.acquisition.runtime.runtime_commands import ( ConnectCommand, + DisconnectCommand, ) from src.market_data.acquisition.runtime.runtime_events import ( ReconnectCompletedEvent, @@ -73,7 +74,7 @@ class ReconnectCoordinator: - увеличивает номер попытки; - публикует ReconnectStartedEvent; - - передаёт ConnectCommand в Runtime Dispatcher; + - передаёт DisconnectCommand и ConnectCommand в Runtime Dispatcher; - восстанавливает зарегистрированные подписки; - публикует ReconnectCompletedEvent; - при ошибке публикует ReconnectFailedEvent; @@ -132,6 +133,8 @@ class ReconnectCoordinator: ReconnectStartedEvent ↓ + DisconnectCommand + ↓ ConnectCommand ↓ restore_subscriptions() @@ -155,6 +158,10 @@ class ReconnectCoordinator: ) try: + await self._command_dispatcher.dispatch( + DisconnectCommand() + ) + await self._command_dispatcher.dispatch( ConnectCommand() ) diff --git a/app/src/market_data/acquisition/runtime/runtime_liveness_probe.py b/app/src/market_data/acquisition/runtime/runtime_liveness_probe.py new file mode 100644 index 0000000..7399a33 --- /dev/null +++ b/app/src/market_data/acquisition/runtime/runtime_liveness_probe.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class RuntimeLivenessProbeProtocol(Protocol): + """ + Контракт активной проверки transport liveness. + + Успешная проверка подтверждается transport-level ответом, а не + наличием market events. + """ + + async def probe(self) -> bool: + """ + Проверить активность текущего transport connection. + + Вернуть True только после подтверждённого transport response. + """ + ... diff --git a/app/src/market_data/acquisition/runtime/runtime_reconnect_recovery_coordinator.py b/app/src/market_data/acquisition/runtime/runtime_reconnect_recovery_coordinator.py new file mode 100644 index 0000000..505599e --- /dev/null +++ b/app/src/market_data/acquisition/runtime/runtime_reconnect_recovery_coordinator.py @@ -0,0 +1,401 @@ +from __future__ import annotations + +import asyncio +import time +from collections.abc import Callable +from typing import Protocol, runtime_checkable + +from src.market_data.acquisition.recovery.trade_recovery_result import ( + TradeRecoveryResult, +) +from src.market_data.acquisition.runtime.live_processing_gate import ( + RuntimeLiveProcessingGateProtocol, +) +from src.market_data.acquisition.runtime.reconnect import ( + ReconnectCoordinatorProtocol, + ReconnectState, +) +from src.market_data.acquisition.runtime.runtime_recovery_protocol import ( + RuntimeRecoveryProtocol, +) + + +RuntimeUnixTimeMillisecondsClock = Callable[[], int] + + +def system_unix_time_ms() -> int: + """Вернуть текущее Unix time без промежуточного float.""" + return time.time_ns() // 1_000_000 + + +@runtime_checkable +class RuntimeReconnectRecoveryProtocol( + ReconnectCoordinatorProtocol, + Protocol, +): + """ + Контракт единой reconnect → restore → recovery операции. + """ + + @property + def generation(self) -> int: + """Вернуть поколение последней начатой операции.""" + ... + + @property + def live_processing_gate( + self, + ) -> RuntimeLiveProcessingGateProtocol: + """Вернуть общий gate Live processing и Recovery.""" + ... + + @property + def symbols(self) -> tuple[str, ...]: + """Вернуть нормализованные Recovery symbols.""" + ... + + async def reconnect_after_transport_failure( + self, + *, + observed_generation: int, + ) -> None: + """ + Восстановить connection, если ошибка относится к текущему + поколению. + """ + ... + + +class _ReconnectRecoveryOperation: + __slots__ = ( + "source_generation", + "completion", + ) + + def __init__( + self, + *, + source_generation: int, + completion: asyncio.Future[None], + ) -> None: + self.source_generation = source_generation + self.completion = completion + + +class RuntimeReconnectRecoveryCoordinator: + """ + Single-flight координатор reconnect → restore → recovery. + + Существующий ReconnectCoordinator сохраняет ответственность только + за Disconnect → Connect → restore subscriptions. Этот компонент + закрывает общий Live processing gate и после reconnect выполняет + синхронный Runtime Recovery вне event loop. + """ + + __slots__ = ( + "_reconnect_coordinator", + "_recovery_coordinator", + "_live_processing_gate", + "_symbols", + "_clock", + "_coordination_lock", + "_generation", + "_active_operation", + "_last_operation", + "_recovery_task", + ) + + def __init__( + self, + *, + reconnect_coordinator: ReconnectCoordinatorProtocol, + recovery_coordinator: RuntimeRecoveryProtocol, + live_processing_gate: RuntimeLiveProcessingGateProtocol, + symbols: tuple[str, ...], + clock: RuntimeUnixTimeMillisecondsClock = system_unix_time_ms, + ) -> None: + if not callable(clock): + raise TypeError("clock must be callable") + + self._reconnect_coordinator = reconnect_coordinator + self._recovery_coordinator = recovery_coordinator + self._live_processing_gate = live_processing_gate + self._symbols = self._normalize_symbols(symbols) + self._clock = clock + + self._coordination_lock = asyncio.Lock() + self._generation = 0 + self._active_operation: _ReconnectRecoveryOperation | None = None + self._last_operation: _ReconnectRecoveryOperation | None = None + self._recovery_task: ( + asyncio.Task[tuple[TradeRecoveryResult, ...]] | None + ) = None + + @property + def state(self) -> ReconnectState: + """Вернуть состояние базовой reconnect-операции.""" + return self._reconnect_coordinator.state + + @property + def attempt(self) -> int: + """Вернуть номер последней базовой reconnect-попытки.""" + return self._reconnect_coordinator.attempt + + @property + def generation(self) -> int: + """Вернуть поколение последней начатой операции.""" + return self._generation + + @property + def live_processing_gate( + self, + ) -> RuntimeLiveProcessingGateProtocol: + """Вернуть общий gate Live processing и Recovery.""" + return self._live_processing_gate + + @property + def symbols(self) -> tuple[str, ...]: + """Вернуть нормализованные Recovery symbols.""" + return self._symbols + + async def reconnect(self) -> None: + """ + Выполнить либо присоединиться к текущей single-flight операции. + """ + await self._run_or_join( + observed_generation=None, + ) + + async def reconnect_after_transport_failure( + self, + *, + observed_generation: int, + ) -> None: + """ + Обработать transport error конкретного connection generation. + + Ошибка старого connection присоединяется к уже выполняемой + операции либо использует её завершённый результат и не запускает + второй reconnect. + """ + self._validate_generation(observed_generation) + + await self._run_or_join( + observed_generation=observed_generation, + ) + + async def _run_or_join( + self, + *, + observed_generation: int | None, + ) -> None: + operation, is_leader = await self._select_operation( + observed_generation=observed_generation, + ) + + if operation is None: + return + + if not is_leader: + await asyncio.shield( + operation.completion, + ) + return + + try: + await self._execute_operation() + except asyncio.CancelledError: + operation.completion.cancel() + raise + except BaseException as error: + operation.completion.set_exception(error) + operation.completion.exception() + raise + else: + operation.completion.set_result(None) + finally: + if self._active_operation is operation: + self._last_operation = operation + self._active_operation = None + + async def _select_operation( + self, + *, + observed_generation: int | None, + ) -> tuple[_ReconnectRecoveryOperation | None, bool]: + async with self._coordination_lock: + active_operation = self._active_operation + + if active_operation is not None: + return ( + active_operation, + False, + ) + + if ( + observed_generation is not None + and observed_generation != self._generation + ): + last_operation = self._last_operation + + if ( + last_operation is not None + and last_operation.source_generation + == observed_generation + ): + return ( + last_operation, + False, + ) + + return ( + None, + False, + ) + + source_generation = self._generation + self._generation += 1 + + operation = _ReconnectRecoveryOperation( + source_generation=source_generation, + completion=asyncio.get_running_loop().create_future(), + ) + self._active_operation = operation + + return ( + operation, + True, + ) + + async def _execute_operation(self) -> None: + async with self._live_processing_gate: + try: + await self._reconnect_coordinator.reconnect() + + recovery_end_time = self._clock() + self._validate_recovery_end_time( + recovery_end_time, + ) + + recovery_task = asyncio.create_task( + asyncio.to_thread( + self._recover_symbols, + recovery_end_time, + ), + name="trade-stream-runtime-recovery", + ) + self._recovery_task = recovery_task + + try: + await asyncio.shield( + recovery_task, + ) + except asyncio.CancelledError as cancellation: + try: + await self._wait_for_recovery_completion( + recovery_task, + ) + except BaseException as recovery_error: + cancellation.add_note( + "Trade Stream Runtime Recovery also failed: " + f"{type(recovery_error).__name__}." + ) + + raise + finally: + if self._recovery_task is recovery_task: + self._recovery_task = None + except Exception as error: + self._live_processing_gate.fail(error) + raise + + @staticmethod + async def _wait_for_recovery_completion( + recovery_task: asyncio.Task[ + tuple[TradeRecoveryResult, ...] + ], + ) -> None: + while not recovery_task.done(): + try: + await asyncio.shield( + recovery_task, + ) + except asyncio.CancelledError: + continue + except BaseException: + break + + recovery_task.result() + + def _recover_symbols( + self, + recovery_end_time: int, + ) -> tuple[TradeRecoveryResult, ...]: + return tuple( + self._recovery_coordinator.recover( + symbol=symbol, + recovery_end_time=recovery_end_time, + ) + for symbol in self._symbols + ) + + @staticmethod + def _normalize_symbols( + symbols: tuple[str, ...], + ) -> tuple[str, ...]: + if not isinstance(symbols, tuple): + raise TypeError("symbols must be a tuple") + + normalized_symbols: set[str] = set() + + for symbol in symbols: + if not isinstance(symbol, str): + raise TypeError( + "symbols must contain only strings" + ) + + normalized_symbol = symbol.strip() + + if normalized_symbol: + normalized_symbols.add(normalized_symbol) + + if not normalized_symbols: + raise ValueError( + "symbols must contain at least one non-empty symbol" + ) + + return tuple(sorted(normalized_symbols)) + + @staticmethod + def _validate_generation( + generation: int, + ) -> None: + if isinstance(generation, bool) or not isinstance( + generation, + int, + ): + raise TypeError( + "observed_generation must be an integer" + ) + + if generation < 0: + raise ValueError( + "observed_generation must not be negative" + ) + + @staticmethod + def _validate_recovery_end_time( + recovery_end_time: int, + ) -> None: + if isinstance(recovery_end_time, bool) or not isinstance( + recovery_end_time, + int, + ): + raise TypeError( + "clock must return an integer" + ) + + if recovery_end_time < 0: + raise ValueError( + "clock must not return a negative value" + ) diff --git a/app/src/market_data/acquisition/runtime/scheduler.py b/app/src/market_data/acquisition/runtime/scheduler.py index fce13ca..ad22361 100644 --- a/app/src/market_data/acquisition/runtime/scheduler.py +++ b/app/src/market_data/acquisition/runtime/scheduler.py @@ -9,6 +9,9 @@ from typing import Protocol, runtime_checkable from src.market_data.acquisition.runtime.heartbeat import ( HeartbeatMonitorProtocol, ) +from src.market_data.acquisition.runtime.runtime_liveness_probe import ( + RuntimeLivenessProbeProtocol, +) from src.market_data.acquisition.runtime.supervisor import ( RuntimeSupervisorProtocol, ) @@ -37,7 +40,29 @@ class RuntimeSchedulerProtocol(Protocol): """ ... - async def start(self) -> None: + @property + def liveness_probe(self) -> RuntimeLivenessProbeProtocol: + """Вернуть используемый transport liveness probe.""" + ... + + @property + def runtime_supervisor(self) -> RuntimeSupervisorProtocol: + """Вернуть Supervisor, получающий timeout.""" + ... + + def claim(self, owner: object) -> None: + """Закрепить Scheduler за единственным lifecycle owner.""" + ... + + def release(self, owner: object) -> None: + """Освободить Scheduler после завершения owned task.""" + ... + + async def start( + self, + *, + owner: object | None = None, + ) -> None: """ Запустить scheduler loop. @@ -66,6 +91,12 @@ class RuntimeScheduler: Scheduler отвечает только за время выполнения проверок: + RuntimeLivenessProbe.probe() + │ + ├── True + │ └── RuntimeSupervisor.notify_activity() + │ + ▼ HeartbeatMonitor.check_timeout() │ ├── False @@ -85,20 +116,23 @@ class RuntimeScheduler: - retry policy; - публикацию Runtime Events. - Ошибки Heartbeat Monitor, Runtime Supervisor и sleep-функции - распространяются вызывающему коду без обёртки. + Ошибки liveness probe, Heartbeat Monitor, Runtime Supervisor и + sleep-функции распространяются вызывающему коду без обёртки. """ __slots__ = ( + "_liveness_probe", "_heartbeat_monitor", "_runtime_supervisor", "_interval_seconds", "_sleep", "_running", + "_owner", ) def __init__( self, + liveness_probe: RuntimeLivenessProbeProtocol, heartbeat_monitor: HeartbeatMonitorProtocol, runtime_supervisor: RuntimeSupervisorProtocol, *, @@ -121,11 +155,13 @@ class RuntimeScheduler: if not callable(sleep): raise TypeError("sleep must be callable") + self._liveness_probe = liveness_probe self._heartbeat_monitor = heartbeat_monitor self._runtime_supervisor = runtime_supervisor self._interval_seconds = float(interval_seconds) self._sleep = sleep self._running = False + self._owner: object | None = None @property def running(self) -> bool: @@ -141,7 +177,57 @@ class RuntimeScheduler: """ return self._interval_seconds - async def start(self) -> None: + @property + def liveness_probe(self) -> RuntimeLivenessProbeProtocol: + """Вернуть используемый transport liveness probe.""" + return self._liveness_probe + + @property + def runtime_supervisor(self) -> RuntimeSupervisorProtocol: + """Вернуть Supervisor, получающий timeout.""" + return self._runtime_supervisor + + def claim(self, owner: object) -> None: + """ + Атомарно закрепить Scheduler за lifecycle owner. + + После claim только этот owner может запустить loop. Claim не + создаёт asyncio-задачу и безопасен до длительного startup. + """ + if owner is None: + raise TypeError("owner must not be None") + + if self._owner is owner: + return + + if self._owner is not None or self._running: + raise RuntimeError( + "Runtime Scheduler is already owned or active." + ) + + self._owner = owner + + def release(self, owner: object) -> None: + """ + Освободить Scheduler после остановки owned loop. + """ + if self._owner is not owner: + raise RuntimeError( + "Runtime Scheduler ownership does not match." + ) + + if self._running: + raise RuntimeError( + "Cannot release a running Runtime Scheduler." + ) + + self._owner = None + + async def start( + self, + *, + owner: object | None = None, + ) -> None: """ Запустить периодический scheduler loop. @@ -154,6 +240,16 @@ class RuntimeScheduler: - после ошибки; - после отмены внешней asyncio-задачи. """ + if self._owner is not None and self._owner is not owner: + raise RuntimeError( + "Runtime Scheduler is owned by another lifecycle." + ) + + if self._owner is None and owner is not None: + raise RuntimeError( + "Runtime Scheduler owner must claim it before start." + ) + if self._running: return @@ -183,14 +279,25 @@ class RuntimeScheduler: async def run_once(self) -> bool: """ - Выполнить одну проверку Heartbeat. + Выполнить одну liveness- и Heartbeat-проверку. - Если Heartbeat Monitor подтверждает timeout, Scheduler - передаёт его Runtime Supervisor. + Успешный transport probe обновляет время активности через + Supervisor. Если Heartbeat Monitor подтверждает timeout, + Scheduler передаёт его Runtime Supervisor. Возвращаемое значение отражает только результат проверки Heartbeat Monitor. """ + connection_is_live = await self._liveness_probe.probe() + + if not isinstance(connection_is_live, bool): + raise TypeError( + "liveness probe must return a boolean" + ) + + if connection_is_live: + self._runtime_supervisor.notify_activity() + timed_out = await self._heartbeat_monitor.check_timeout() if timed_out: diff --git a/app/src/market_data/acquisition/runtime/supervisor.py b/app/src/market_data/acquisition/runtime/supervisor.py index 49d71b1..dc07a64 100644 --- a/app/src/market_data/acquisition/runtime/supervisor.py +++ b/app/src/market_data/acquisition/runtime/supervisor.py @@ -24,6 +24,28 @@ class RuntimeSupervisorState(str, Enum): FAILED = "failed" +class RuntimeSupervisorReconnectProtocol( + ReconnectCoordinatorProtocol, + Protocol, +): + """ + Generation-aware reconnect-контракт Runtime Supervisor. + """ + + @property + def generation(self) -> int: + """Вернуть текущее connection generation.""" + ... + + async def reconnect_after_transport_failure( + self, + *, + observed_generation: int, + ) -> None: + """Восстановить только наблюдавшееся поколение connection.""" + ... + + @runtime_checkable class RuntimeSupervisorProtocol(Protocol): """ @@ -100,16 +122,18 @@ class RuntimeSupervisor: __slots__ = ( "_heartbeat_monitor", "_reconnect_coordinator", + "_observed_generation", "_state", ) def __init__( self, heartbeat_monitor: HeartbeatMonitorProtocol, - reconnect_coordinator: ReconnectCoordinatorProtocol, + reconnect_coordinator: RuntimeSupervisorReconnectProtocol, ) -> None: self._heartbeat_monitor = heartbeat_monitor self._reconnect_coordinator = reconnect_coordinator + self._observed_generation = reconnect_coordinator.generation self._state = RuntimeSupervisorState.STOPPED @property @@ -127,6 +151,9 @@ class RuntimeSupervisor: period и сохраняет Supervisor в состоянии RUNNING. """ self._heartbeat_monitor.start() + self._observed_generation = ( + self._reconnect_coordinator.generation + ) self._state = RuntimeSupervisorState.RUNNING def stop(self) -> None: @@ -150,6 +177,9 @@ class RuntimeSupervisor: return self._heartbeat_monitor.record_activity() + self._observed_generation = ( + self._reconnect_coordinator.generation + ) async def handle_heartbeat_timeout(self) -> bool: """ @@ -163,7 +193,7 @@ class RuntimeSupervisor: ↓ RECONNECTING ↓ - ReconnectCoordinator.reconnect() + reconnect текущего connection generation ├── success: │ start Heartbeat │ RUNNING @@ -190,14 +220,23 @@ class RuntimeSupervisor: self._heartbeat_monitor.stop() self._state = RuntimeSupervisorState.RECONNECTING + observed_generation = self._observed_generation try: - await self._reconnect_coordinator.reconnect() + await ( + self._reconnect_coordinator + .reconnect_after_transport_failure( + observed_generation=observed_generation, + ) + ) except Exception: self._state = RuntimeSupervisorState.FAILED raise self._heartbeat_monitor.start() + self._observed_generation = ( + self._reconnect_coordinator.generation + ) self._state = RuntimeSupervisorState.RUNNING return True diff --git a/app/src/market_data/acquisition/runtime/trade_stream_production_runtime.py b/app/src/market_data/acquisition/runtime/trade_stream_production_runtime.py new file mode 100644 index 0000000..01688be --- /dev/null +++ b/app/src/market_data/acquisition/runtime/trade_stream_production_runtime.py @@ -0,0 +1,763 @@ +from __future__ import annotations + +import asyncio +import json +from enum import Enum +from typing import Protocol, runtime_checkable +from uuid import uuid4 + +from src.market_data.acquisition.exceptions import ( + WebSocketMessageDecodeError, + WebSocketTransportError, +) +from src.market_data.acquisition.runtime.live_processing_gate import ( + RuntimeLiveProcessingGateProtocol, +) +from src.market_data.acquisition.runtime.runtime_reconnect_recovery_coordinator import ( + RuntimeReconnectRecoveryProtocol, +) +from src.market_data.acquisition.runtime.scheduler import ( + RuntimeSchedulerProtocol, +) +from src.market_data.acquisition.runtime.supervisor import ( + RuntimeSupervisorProtocol, +) +from src.market_data.acquisition.runtime.runtime_events import ( + ConnectedEvent, + ConnectFailedEvent, + DisconnectedEvent, + MessageReceivedEvent, +) +from src.market_data.acquisition.runtime.transport_messages import ( + TransportBinaryMessage, + TransportTextMessage, +) +from src.market_data.acquisition.runtime.websocket_inbound_message import ( + WebSocketControlMessageHandlerProtocol, + WebSocketInboundMessageClassifierProtocol, + WebSocketInboundMessageKind, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionRuntimeEventPublisherProtocol, + WebSocketSessionProtocol, + WebSocketSubscriptionManagerProtocol, + WebSocketTransportProtocol, +) +from src.market_data.acquisition.trade_stream_acquisition_protocol import ( + TradeStreamAcquisitionServiceProtocol, +) + + +class TradeStreamProductionRuntimeState(str, Enum): + """ + Состояние высокоуровневого lifecycle Trade Stream Runtime. + """ + + STOPPED = "stopped" + STARTING = "starting" + RUNNING = "running" + STOPPING = "stopping" + FAILED = "failed" + + +@runtime_checkable +class TradeStreamProductionRuntimeProtocol(Protocol): + """ + Контракт единственного владельца Trade Stream runtime execution. + """ + + @property + def state(self) -> TradeStreamProductionRuntimeState: + """Вернуть текущее lifecycle-состояние.""" + ... + + @property + def running(self) -> bool: + """Показывает, выполняется ли receive lifecycle.""" + ... + + async def run(self) -> None: + """Запустить Runtime и выполнять его до stop или ошибки.""" + ... + + async def stop(self) -> None: + """Идемпотентно остановить Runtime и дождаться cleanup.""" + ... + + +class TradeStreamProductionRuntime: + """ + Высокоуровневый владелец lifecycle Trade Stream. + + Runtime: + + - запускает WebSocket Session; + - регистрирует Trade subscription; + - создаёт ровно одну receive task; + - декодирует JSON и отделяет control messages от market documents; + - передаёт market documents в TradeStreamAcquisitionService; + - после transport failure выполняет reconnect и Runtime Recovery; + - запускает Runtime Supervisor и одну Scheduler task; + - отменяет и await-ит созданные им startup, Scheduler и receive tasks; + - выполняет полный cleanup при stop, cancellation и ошибке. + + Scheduler останавливается до Supervisor и receive loop. Благодаря + этому shutdown не может запустить новую heartbeat recovery-операцию. + """ + + __slots__ = ( + "_session", + "_transport", + "_subscription_manager", + "_event_publisher", + "_trade_stream_service", + "_message_classifier", + "_control_message_handler", + "_live_processing_gate", + "_reconnect_recovery_coordinator", + "_runtime_supervisor", + "_runtime_scheduler", + "_symbols", + "_state", + "_startup_task", + "_receive_task", + "_scheduler_task", + "_scheduler_claimed", + "_supervisor_started", + "_session_started", + "_subscription_correlation_id", + "_lifecycle_lock", + "_stop_requested", + "_stopped", + ) + + def __init__( + self, + *, + session: WebSocketSessionProtocol, + transport: WebSocketTransportProtocol, + subscription_manager: WebSocketSubscriptionManagerProtocol, + event_publisher: AcquisitionRuntimeEventPublisherProtocol, + trade_stream_service: TradeStreamAcquisitionServiceProtocol, + message_classifier: WebSocketInboundMessageClassifierProtocol, + control_message_handler: WebSocketControlMessageHandlerProtocol, + live_processing_gate: RuntimeLiveProcessingGateProtocol, + reconnect_recovery_coordinator: RuntimeReconnectRecoveryProtocol, + runtime_supervisor: RuntimeSupervisorProtocol, + runtime_scheduler: RuntimeSchedulerProtocol, + symbols: tuple[str, ...], + ) -> None: + self._session = session + self._transport = transport + self._subscription_manager = subscription_manager + self._event_publisher = event_publisher + self._trade_stream_service = trade_stream_service + self._message_classifier = message_classifier + self._control_message_handler = control_message_handler + + normalized_symbols = self._normalize_symbols(symbols) + + if ( + reconnect_recovery_coordinator.live_processing_gate + is not live_processing_gate + ): + raise ValueError( + "Production Runtime and reconnect/recovery coordinator " + "must share one live processing gate." + ) + + if reconnect_recovery_coordinator.symbols != normalized_symbols: + raise ValueError( + "Production Runtime and reconnect/recovery coordinator " + "must use the same symbols." + ) + + if runtime_scheduler.runtime_supervisor is not runtime_supervisor: + raise ValueError( + "Production Runtime and Scheduler must share one " + "Runtime Supervisor." + ) + + if runtime_scheduler.liveness_probe is not transport: + raise ValueError( + "Production Runtime and Scheduler must share one " + "WebSocket Transport." + ) + + self._live_processing_gate = live_processing_gate + self._reconnect_recovery_coordinator = ( + reconnect_recovery_coordinator + ) + self._runtime_supervisor = runtime_supervisor + self._runtime_scheduler = runtime_scheduler + self._symbols = normalized_symbols + + self._state = TradeStreamProductionRuntimeState.STOPPED + self._startup_task: asyncio.Task[None] | None = None + self._receive_task: asyncio.Task[None] | None = None + self._scheduler_task: asyncio.Task[None] | None = None + self._scheduler_claimed = False + self._supervisor_started = False + self._session_started = False + self._subscription_correlation_id: str | None = None + + self._lifecycle_lock = asyncio.Lock() + self._stop_requested = asyncio.Event() + self._stopped = asyncio.Event() + self._stopped.set() + + @property + def state(self) -> TradeStreamProductionRuntimeState: + """Вернуть текущее lifecycle-состояние.""" + return self._state + + @property + def running(self) -> bool: + """Показывает, выполняется ли receive lifecycle.""" + return self._state is TradeStreamProductionRuntimeState.RUNNING + + async def run(self) -> None: + """ + Запустить Runtime и выполнять его до stop или terminal error. + + Метод не создаёт скрытую корневую задачу. Внешний bootstrap + владеет coroutine task ``run()``, а Runtime владеет созданными + внутри startup, Scheduler и receive tasks. + """ + await self._begin_run() + + primary_error: BaseException | None = None + + try: + await self._run_lifecycle() + except BaseException as error: + primary_error = error + + cleanup_error = await self._shutdown( + primary_error=primary_error, + ) + + failed = ( + isinstance(primary_error, Exception) + or isinstance(cleanup_error, Exception) + ) + self._finish_run(failed=failed) + + if primary_error is not None: + if cleanup_error is not None: + primary_error.add_note( + "Trade Stream Runtime cleanup also failed: " + f"{type(cleanup_error).__name__}." + ) + + raise primary_error.with_traceback( + primary_error.__traceback__, + ) + + if cleanup_error is not None: + raise cleanup_error.with_traceback( + cleanup_error.__traceback__, + ) + + async def stop(self) -> None: + """ + Запросить остановку и дождаться полного cleanup. + + stop() не отменяет внешнюю task ``run()``. Он останавливает + только startup, Scheduler и receive tasks, принадлежащие + Production Runtime. + """ + async with self._lifecycle_lock: + if self._state in { + TradeStreamProductionRuntimeState.STOPPED, + TradeStreamProductionRuntimeState.FAILED, + }: + return + + if ( + self._state + is not TradeStreamProductionRuntimeState.STOPPING + ): + self._state = TradeStreamProductionRuntimeState.STOPPING + self._stop_requested.set() + + startup_task = self._startup_task + scheduler_task = self._scheduler_task + + if ( + startup_task is not None + and not startup_task.done() + ): + startup_task.cancel() + + if ( + scheduler_task is not None + and not scheduler_task.done() + ): + self._runtime_scheduler.stop() + scheduler_task.cancel() + + await self._stopped.wait() + + async def _begin_run(self) -> None: + async with self._lifecycle_lock: + if self._state not in { + TradeStreamProductionRuntimeState.STOPPED, + TradeStreamProductionRuntimeState.FAILED, + }: + raise RuntimeError( + "Trade Stream Production Runtime is already active." + ) + + self._runtime_scheduler.claim(self) + self._scheduler_claimed = True + + try: + self._live_processing_gate.reset() + except BaseException: + self._runtime_scheduler.release(self) + self._scheduler_claimed = False + raise + + self._state = TradeStreamProductionRuntimeState.STARTING + self._startup_task = None + self._receive_task = None + self._scheduler_task = None + self._supervisor_started = False + self._session_started = False + self._subscription_correlation_id = None + self._stop_requested.clear() + self._stopped.clear() + + async def _run_lifecycle(self) -> None: + startup_task = asyncio.create_task( + self._start_sequence(), + name="trade-stream-startup", + ) + + async with self._lifecycle_lock: + self._startup_task = startup_task + + if self._stop_requested.is_set(): + startup_task.cancel() + + try: + await startup_task + except asyncio.CancelledError: + if not self._stop_requested.is_set(): + raise + + return + + async with self._lifecycle_lock: + if self._stop_requested.is_set(): + return + + self._supervisor_started = True + self._runtime_supervisor.start() + + receive_task = asyncio.create_task( + self._receive_loop(), + name="trade-stream-receive", + ) + scheduler_task = asyncio.create_task( + self._runtime_scheduler.start( + owner=self, + ), + name="trade-stream-scheduler", + ) + self._receive_task = receive_task + self._scheduler_task = scheduler_task + self._state = TradeStreamProductionRuntimeState.RUNNING + + try: + await self._wait_for_runtime_task( + receive_task=receive_task, + scheduler_task=scheduler_task, + ) + except asyncio.CancelledError: + if not self._stop_requested.is_set(): + raise + + async def _start_sequence(self) -> None: + try: + await self._session.start() + except asyncio.CancelledError: + raise + except Exception as error: + await self._event_publisher.publish( + ConnectFailedEvent( + reason=str(error), + ) + ) + raise + + self._session_started = True + + await self._event_publisher.publish( + ConnectedEvent(), + ) + + if self._stop_requested.is_set(): + return + + correlation_id = str(uuid4()) + self._subscription_correlation_id = correlation_id + + await self._trade_stream_service.subscribe( + self._symbols, + correlation_id=correlation_id, + ) + + async def _receive_loop(self) -> None: + while not self._stop_requested.is_set(): + observed_generation = ( + self._reconnect_recovery_coordinator.generation + ) + + try: + raw_message = await self._transport.receive() + except WebSocketTransportError: + if self._stop_requested.is_set(): + return + + await ( + self._reconnect_recovery_coordinator + .reconnect_after_transport_failure( + observed_generation=observed_generation, + ) + ) + continue + + if self._stop_requested.is_set(): + return + + self._runtime_supervisor.notify_activity() + + transport_message = self._wrap_transport_message( + raw_message, + ) + + await self._event_publisher.publish( + MessageReceivedEvent( + message=transport_message, + ) + ) + + document = self._decode_message( + raw_message, + ) + + async with self._live_processing_gate: + if self._stop_requested.is_set(): + return + + message_kind = self._message_classifier.classify( + document, + ) + + if ( + message_kind + is WebSocketInboundMessageKind.CONTROL + ): + correlation_id = ( + self._subscription_correlation_id + ) + + if correlation_id is None: + raise RuntimeError( + "Control message received without an " + "active subscription correlation ID." + ) + + self._control_message_handler.handle( + document, + expected_correlation_id=correlation_id, + ) + continue + + if ( + message_kind + is not WebSocketInboundMessageKind.MARKET + ): + raise TypeError( + "message classifier must return " + "WebSocketInboundMessageKind" + ) + + self._trade_stream_service.handle_message( + document, + ) + + async def _shutdown( + self, + *, + primary_error: BaseException | None, + ) -> BaseException | None: + cleanup_error: BaseException | None = None + startup_task = self._startup_task + scheduler_task = self._scheduler_task + receive_task = self._receive_task + + self._stop_requested.set() + + if ( + startup_task is not None + and not startup_task.done() + ): + startup_task.cancel() + try: + await startup_task + except asyncio.CancelledError: + pass + except BaseException as error: + cleanup_error = self._merge_cleanup_error( + cleanup_error, + error, + ) + + if scheduler_task is not None: + if not scheduler_task.done(): + try: + self._runtime_scheduler.stop() + except BaseException as error: + cleanup_error = self._merge_cleanup_error( + cleanup_error, + error, + ) + + scheduler_task.cancel() + + try: + await scheduler_task + except asyncio.CancelledError: + pass + except BaseException as error: + if error is not primary_error: + cleanup_error = self._merge_cleanup_error( + cleanup_error, + error, + ) + + if self._scheduler_claimed: + try: + self._runtime_scheduler.release(self) + except BaseException as error: + cleanup_error = self._merge_cleanup_error( + cleanup_error, + error, + ) + else: + self._scheduler_claimed = False + + if self._supervisor_started: + try: + self._runtime_supervisor.stop() + except BaseException as error: + cleanup_error = self._merge_cleanup_error( + cleanup_error, + error, + ) + finally: + self._supervisor_started = False + + if receive_task is not None: + if not receive_task.done(): + receive_task.cancel() + + try: + await receive_task + except asyncio.CancelledError: + pass + except BaseException as error: + if error is not primary_error: + cleanup_error = self._merge_cleanup_error( + cleanup_error, + error, + ) + + session_stopped = False + + try: + await self._session.stop() + session_stopped = True + except BaseException as error: + cleanup_error = self._merge_cleanup_error( + cleanup_error, + error, + ) + + try: + await self._subscription_manager.clear_subscriptions() + except BaseException as error: + cleanup_error = self._merge_cleanup_error( + cleanup_error, + error, + ) + + if self._session_started and session_stopped: + try: + await self._event_publisher.publish( + DisconnectedEvent(), + ) + except BaseException as error: + cleanup_error = self._merge_cleanup_error( + cleanup_error, + error, + ) + + self._startup_task = None + self._receive_task = None + self._scheduler_task = None + self._session_started = False + self._subscription_correlation_id = None + + return cleanup_error + + async def _wait_for_runtime_task( + self, + *, + receive_task: asyncio.Task[None], + scheduler_task: asyncio.Task[None], + ) -> None: + done_tasks, _ = await asyncio.wait( + ( + receive_task, + scheduler_task, + ), + return_when=asyncio.FIRST_COMPLETED, + ) + + task_errors: list[BaseException] = [] + cancelled_task_names: list[str] = [] + + for task in ( + receive_task, + scheduler_task, + ): + if task not in done_tasks: + continue + + if task.cancelled(): + cancelled_task_names.append( + task.get_name(), + ) + continue + + error = task.exception() + + if error is not None: + task_errors.append(error) + + if task_errors: + primary_error = task_errors[0] + + for additional_error in task_errors[1:]: + primary_error.add_note( + "Additional Trade Stream Runtime task failure: " + f"{type(additional_error).__name__}." + ) + + raise primary_error.with_traceback( + primary_error.__traceback__, + ) + + if self._stop_requested.is_set(): + return + + if cancelled_task_names: + raise RuntimeError( + "Trade Stream Runtime task was cancelled unexpectedly: " + f"{', '.join(cancelled_task_names)}." + ) + + completed_task_names = ", ".join( + task.get_name() + for task in done_tasks + ) + raise RuntimeError( + "Trade Stream Runtime task terminated unexpectedly: " + f"{completed_task_names}." + ) + + def _finish_run( + self, + *, + failed: bool, + ) -> None: + self._state = ( + TradeStreamProductionRuntimeState.FAILED + if failed + else TradeStreamProductionRuntimeState.STOPPED + ) + self._stopped.set() + + @staticmethod + def _merge_cleanup_error( + current_error: BaseException | None, + new_error: BaseException, + ) -> BaseException: + if current_error is None: + return new_error + + current_error.add_note( + "Additional Trade Stream Runtime cleanup failure: " + f"{type(new_error).__name__}." + ) + return current_error + + @staticmethod + def _wrap_transport_message( + raw_message: str | bytes, + ) -> TransportTextMessage | TransportBinaryMessage: + if isinstance(raw_message, str): + return TransportTextMessage( + payload=raw_message, + ) + + if isinstance(raw_message, bytes): + return TransportBinaryMessage( + payload=raw_message, + ) + + raise TypeError( + "WebSocket transport receive() must return str or bytes" + ) + + @staticmethod + def _decode_message( + raw_message: str | bytes, + ) -> object: + try: + return json.loads(raw_message) + except (json.JSONDecodeError, UnicodeDecodeError) as error: + raise WebSocketMessageDecodeError( + "Не удалось декодировать JSON из WebSocket message." + ) from error + + @staticmethod + def _normalize_symbols( + symbols: tuple[str, ...], + ) -> tuple[str, ...]: + if not isinstance(symbols, tuple): + raise TypeError("symbols must be a tuple") + + normalized_symbols: set[str] = set() + + for symbol in symbols: + if not isinstance(symbol, str): + raise TypeError( + "symbols must contain only strings" + ) + + normalized_symbol = symbol.strip() + + if normalized_symbol: + normalized_symbols.add(normalized_symbol) + + if not normalized_symbols: + raise ValueError( + "symbols must contain at least one non-empty symbol" + ) + + return tuple(sorted(normalized_symbols)) diff --git a/app/src/market_data/acquisition/runtime/websocket_inbound_message.py b/app/src/market_data/acquisition/runtime/websocket_inbound_message.py new file mode 100644 index 0000000..95c3711 --- /dev/null +++ b/app/src/market_data/acquisition/runtime/websocket_inbound_message.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from enum import Enum +from typing import Protocol, runtime_checkable + + +class WebSocketInboundMessageKind(str, Enum): + """ + Категория декодированного входящего WebSocket-сообщения. + """ + + MARKET = "market" + CONTROL = "control" + + +@runtime_checkable +class WebSocketInboundMessageClassifierProtocol(Protocol): + """ + Контракт классификации декодированного WebSocket-документа. + + Конкретная реализация содержит provider-specific знания о форме + market и control messages. Production Runtime использует только + итоговую категорию и не интерпретирует exchange-specific поля. + """ + + def classify( + self, + document: object, + ) -> WebSocketInboundMessageKind: + """Классифицировать один декодированный документ.""" + ... + + +@runtime_checkable +class WebSocketControlMessageHandlerProtocol(Protocol): + """ + Контракт provider-specific проверки входящего control/ACK. + + Production Runtime передаёт ожидаемый correlation ID, но не + интерпретирует provider-specific поля ответа самостоятельно. + """ + + def handle( + self, + document: object, + *, + expected_correlation_id: str, + ) -> None: + """ + Проверить принадлежность и успешность control/ACK сообщения. + """ + ... diff --git a/app/src/market_data/acquisition/runtime/websocket_session.py b/app/src/market_data/acquisition/runtime/websocket_session.py new file mode 100644 index 0000000..7c50a07 --- /dev/null +++ b/app/src/market_data/acquisition/runtime/websocket_session.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import asyncio +from typing import Protocol + +from src.market_data.acquisition.runtime.websocket_protocol import ( + WebSocketTransportProtocol, +) + + +class _SessionTransportProtocol( + WebSocketTransportProtocol, + Protocol, +): + """ + Transport-контракт для Session lifecycle. + """ + + @property + def is_connected(self) -> bool: + """Показывает состояние Transport.""" + ... + + +class WebSocketSession: + """ + Идемпотентный lifecycle WebSocket-сессии. + + Session сериализует start/stop, делегирует сетевые операции + Transport и не содержит parsing, subscriptions или recovery. + """ + + __slots__ = ( + "_transport", + "_started", + "_lifecycle_lock", + ) + + def __init__( + self, + transport: _SessionTransportProtocol, + ) -> None: + self._transport = transport + self._started = False + self._lifecycle_lock = asyncio.Lock() + + @property + def is_connected(self) -> bool: + """Показывает состояние Session.""" + return ( + self._started + and self._transport.is_connected + ) + + async def start(self) -> None: + """ + Открыть Session без повторного соединения. + """ + async with self._lifecycle_lock: + if self.is_connected: + return + + self._started = False + await self._transport.connect() + self._started = True + + async def stop(self) -> None: + """ + Закрыть Session и сбросить локальное состояние. + """ + async with self._lifecycle_lock: + if ( + not self._started + and not self._transport.is_connected + ): + return + + try: + await self._transport.disconnect() + finally: + self._started = False diff --git a/app/src/market_data/acquisition/runtime/websocket_subscription_manager.py b/app/src/market_data/acquisition/runtime/websocket_subscription_manager.py new file mode 100644 index 0000000..50635ff --- /dev/null +++ b/app/src/market_data/acquisition/runtime/websocket_subscription_manager.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import asyncio + +from src.market_data.acquisition.exceptions import ( + WebSocketUnsubscribeNotSupportedError, +) +from src.market_data.acquisition.runtime.transport_messages import ( + TransportBinaryMessage, + TransportTextMessage, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionSubscriptionMessage, + WebSocketTransportProtocol, +) + + +class WebSocketSubscriptionManager: + """ + Реестр WebSocket-подписок и механизм их восстановления. + + Manager хранит только универсальные transport messages. + Он не разбирает payload и не содержит exchange-specific знаний. + """ + + __slots__ = ( + "_transport", + "_supports_unsubscribe", + "_subscriptions", + "_active_subscription_keys", + "_lock", + ) + + def __init__( + self, + transport: WebSocketTransportProtocol, + *, + supports_unsubscribe: bool = False, + ) -> None: + if not isinstance(supports_unsubscribe, bool): + raise TypeError( + "supports_unsubscribe must be a boolean" + ) + + self._transport = transport + self._supports_unsubscribe = supports_unsubscribe + self._subscriptions: dict[ + str, + AcquisitionSubscriptionMessage, + ] = {} + self._active_subscription_keys: set[str] = set() + self._lock = asyncio.Lock() + + @property + def subscription_keys(self) -> tuple[str, ...]: + """Вернуть ключи в порядке регистрации.""" + return tuple(self._subscriptions) + + async def subscribe( + self, + subscription_key: str, + message: AcquisitionSubscriptionMessage, + ) -> None: + """ + Идемпотентно отправить и зарегистрировать подписку. + + Desired subscription сохраняется до сетевой отправки. + При ошибке она остаётся pending для последующего retry/reconnect. + """ + self._validate_subscription_key(subscription_key) + self._validate_message(message) + + async with self._lock: + if subscription_key in self._active_subscription_keys: + return + + self._subscriptions[subscription_key] = message + + await self._send(message) + self._active_subscription_keys.add( + subscription_key, + ) + + async def unsubscribe( + self, + subscription_key: str, + message: AcquisitionSubscriptionMessage, + ) -> None: + """ + Отправить unsubscribe только при явной поддержке провайдером. + + Dzengi ``trades.unsubscribe`` не поддерживает. Поэтому + ``supports_unsubscribe`` по умолчанию равно False. + """ + self._validate_subscription_key(subscription_key) + self._validate_message(message) + + if not self._supports_unsubscribe: + raise WebSocketUnsubscribeNotSupportedError( + "WebSocket provider does not support unsubscribe." + ) + + async with self._lock: + if subscription_key not in self._subscriptions: + return + + if subscription_key in self._active_subscription_keys: + await self._send(message) + + del self._subscriptions[subscription_key] + self._active_subscription_keys.discard( + subscription_key, + ) + + async def restore_subscriptions(self) -> None: + """ + Последовательно восстановить подписки. + """ + async with self._lock: + self._active_subscription_keys.clear() + + for subscription_key, message in ( + self._subscriptions.items() + ): + await self._send(message) + self._active_subscription_keys.add( + subscription_key, + ) + + async def clear_subscriptions(self) -> None: + """ + Очистить реестр без отправки сообщений. + """ + async with self._lock: + self._subscriptions.clear() + self._active_subscription_keys.clear() + + async def _send( + self, + message: AcquisitionSubscriptionMessage, + ) -> None: + if isinstance(message, TransportTextMessage): + await self._transport.send(message.payload) + return + + await self._transport.send(message.payload) + + @staticmethod + def _validate_subscription_key( + subscription_key: str, + ) -> None: + if not isinstance(subscription_key, str): + raise TypeError( + "subscription_key must be a string" + ) + + if not subscription_key.strip(): + raise ValueError( + "subscription_key must not be empty" + ) + + @staticmethod + def _validate_message( + message: AcquisitionSubscriptionMessage, + ) -> None: + if not isinstance( + message, + ( + TransportTextMessage, + TransportBinaryMessage, + ), + ): + raise TypeError( + "message must be TransportTextMessage " + "or TransportBinaryMessage" + ) diff --git a/app/src/market_data/acquisition/trade_stream_runtime_composition.py b/app/src/market_data/acquisition/trade_stream_runtime_composition.py index 8f903dc..76a2e34 100644 --- a/app/src/market_data/acquisition/trade_stream_runtime_composition.py +++ b/app/src/market_data/acquisition/trade_stream_runtime_composition.py @@ -7,6 +7,7 @@ import asyncio import time from collections.abc import Callable from dataclasses import dataclass +from typing import Protocol from src.market_data.acquisition.adapters.dzengi.rest import ( DzengiTradesDocumentSource, @@ -30,9 +31,20 @@ from src.market_data.acquisition.runtime.acquisition_runtime_service import ( from src.market_data.acquisition.runtime.heartbeat import ( HeartbeatMonitor, ) +from src.market_data.acquisition.runtime.live_processing_gate import ( + RuntimeLiveProcessingGate, +) from src.market_data.acquisition.runtime.reconnect import ( ReconnectCoordinator, ) +from src.market_data.acquisition.runtime.runtime_reconnect_recovery_coordinator import ( + RuntimeReconnectRecoveryCoordinator, + RuntimeUnixTimeMillisecondsClock, + system_unix_time_ms, +) +from src.market_data.acquisition.runtime.runtime_liveness_probe import ( + RuntimeLivenessProbeProtocol, +) from src.market_data.acquisition.runtime.runtime_recovery_coordinator import ( RuntimeRecoveryCoordinator, ) @@ -57,6 +69,14 @@ from src.market_data.acquisition.trade_stream_message_adapter_protocol import ( ) +class _TradeStreamRuntimeTransportProtocol( + WebSocketTransportProtocol, + RuntimeLivenessProbeProtocol, + Protocol, +): + """Общий Transport и liveness-контракт Composition.""" + + @dataclass(frozen=True, slots=True) class TradeStreamRuntimeComposition: """ @@ -80,6 +100,11 @@ class TradeStreamRuntimeComposition: trade_stream_acquisition_service: TradeStreamAcquisitionService reconnect_coordinator: ReconnectCoordinator + live_processing_gate: RuntimeLiveProcessingGate + liveness_probe: RuntimeLivenessProbeProtocol + runtime_reconnect_recovery_coordinator: ( + RuntimeReconnectRecoveryCoordinator + ) heartbeat_monitor: HeartbeatMonitor runtime_supervisor: RuntimeSupervisor runtime_scheduler: RuntimeScheduler @@ -88,15 +113,19 @@ class TradeStreamRuntimeComposition: def build_trade_stream_runtime_composition( *, session: WebSocketSessionProtocol, - transport: WebSocketTransportProtocol, + transport: _TradeStreamRuntimeTransportProtocol, subscription_manager: WebSocketSubscriptionManagerProtocol, event_publisher: AcquisitionRuntimeEventPublisherProtocol, message_adapter: TradeStreamMessageAdapterProtocol, recovery_document_source: DzengiTradesDocumentSource, + symbols: tuple[str, ...], heartbeat_timeout_seconds: float, scheduler_interval_seconds: float, max_recovery_window_ms: int = DEFAULT_TRADE_RECOVERY_WINDOW_MS, heartbeat_clock: Callable[[], float] = time.monotonic, + recovery_end_time_clock: RuntimeUnixTimeMillisecondsClock = ( + system_unix_time_ms + ), scheduler_sleep: RuntimeSleep = asyncio.sleep, ) -> TradeStreamRuntimeComposition: """ @@ -151,6 +180,18 @@ def build_trade_stream_runtime_composition( event_publisher=event_publisher, ) + live_processing_gate = RuntimeLiveProcessingGate() + + runtime_reconnect_recovery_coordinator = ( + RuntimeReconnectRecoveryCoordinator( + reconnect_coordinator=reconnect_coordinator, + recovery_coordinator=runtime_recovery_coordinator, + live_processing_gate=live_processing_gate, + symbols=symbols, + clock=recovery_end_time_clock, + ) + ) + heartbeat_monitor = HeartbeatMonitor( event_publisher=event_publisher, timeout_seconds=heartbeat_timeout_seconds, @@ -159,10 +200,13 @@ def build_trade_stream_runtime_composition( runtime_supervisor = RuntimeSupervisor( heartbeat_monitor=heartbeat_monitor, - reconnect_coordinator=reconnect_coordinator, + reconnect_coordinator=( + runtime_reconnect_recovery_coordinator + ), ) runtime_scheduler = RuntimeScheduler( + liveness_probe=transport, heartbeat_monitor=heartbeat_monitor, runtime_supervisor=runtime_supervisor, interval_seconds=scheduler_interval_seconds, @@ -178,6 +222,11 @@ def build_trade_stream_runtime_composition( acquisition_runtime_service=acquisition_runtime_service, trade_stream_acquisition_service=trade_stream_acquisition_service, reconnect_coordinator=reconnect_coordinator, + live_processing_gate=live_processing_gate, + liveness_probe=transport, + runtime_reconnect_recovery_coordinator=( + runtime_reconnect_recovery_coordinator + ), heartbeat_monitor=heartbeat_monitor, runtime_supervisor=runtime_supervisor, runtime_scheduler=runtime_scheduler, diff --git a/app/tests/unit/bootstrap/test_app_factory.py b/app/tests/unit/bootstrap/test_app_factory.py new file mode 100644 index 0000000..c432935 --- /dev/null +++ b/app/tests/unit/bootstrap/test_app_factory.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +import src.bootstrap.app_factory as app_factory +from src.bootstrap.application import ApplicationComposition + + +class RecordingJournal: + def __init__(self) -> None: + self.info_calls: list[ + tuple[str, str, dict[str, object]] + ] = [] + + def log_info( + self, + event: str, + message: str, + context: dict[str, object], + ) -> None: + self.info_calls.append( + ( + event, + message, + context, + ) + ) + + def log_critical( + self, + event: str, + message: str, + context: dict[str, object], + ) -> None: + del event, message, context + + +def make_settings() -> SimpleNamespace: + return SimpleNamespace( + bot_token="test-token", + bot_parse_mode="HTML", + log_level="INFO", + app_env="test", + exchange_name="dzengi", + default_symbol="BTC/USD_LEVERAGE", + trade_stream=SimpleNamespace(enabled=True), + ) + + +def test_create_app_builds_one_application_composition( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = make_settings() + bot = object() + dispatcher = object() + runtime = object() + journal = RecordingJournal() + observed_runtime_settings: list[object] = [] + registered_bots: list[object] = [] + routed_dispatchers: list[object] = [] + + monkeypatch.setattr( + app_factory, + "load_settings", + lambda: settings, + ) + monkeypatch.setattr( + app_factory, + "setup_logging", + lambda level: None, + ) + monkeypatch.setattr( + app_factory, + "init_schema", + lambda: None, + ) + monkeypatch.setattr( + app_factory, + "JournalService", + lambda: journal, + ) + + def build_runtime(received_settings: object) -> object: + observed_runtime_settings.append(received_settings) + return runtime + + monkeypatch.setattr( + app_factory, + "build_trade_stream_production_runtime", + build_runtime, + ) + monkeypatch.setattr( + app_factory, + "Bot", + lambda **kwargs: bot, + ) + monkeypatch.setattr( + app_factory, + "Dispatcher", + lambda: dispatcher, + ) + monkeypatch.setattr( + app_factory.NotificationTargetRegistry, + "set_bot", + registered_bots.append, + ) + monkeypatch.setattr( + app_factory, + "setup_routers", + routed_dispatchers.append, + ) + + application = app_factory.create_app() + + assert isinstance(application, ApplicationComposition) + assert application.bot is bot + assert application.dispatcher is dispatcher + assert application.trade_stream_runtime is runtime + assert observed_runtime_settings == [settings] + assert registered_bots == [bot] + assert routed_dispatchers == [dispatcher] + assert journal.info_calls[0][2]["trade_stream_enabled"] is True + + +def test_runtime_build_error_is_fatal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + expected = RuntimeError("invalid Trade Stream settings") + + monkeypatch.setattr( + app_factory, + "load_settings", + make_settings, + ) + monkeypatch.setattr( + app_factory, + "setup_logging", + lambda level: None, + ) + monkeypatch.setattr( + app_factory, + "init_schema", + lambda: None, + ) + monkeypatch.setattr( + app_factory, + "JournalService", + RecordingJournal, + ) + + def fail_runtime_build(settings: object) -> None: + del settings + raise expected + + monkeypatch.setattr( + app_factory, + "build_trade_stream_production_runtime", + fail_runtime_build, + ) + + with pytest.raises(RuntimeError) as exc_info: + app_factory.create_app() + + assert exc_info.value is expected diff --git a/app/tests/unit/bootstrap/test_application.py b/app/tests/unit/bootstrap/test_application.py new file mode 100644 index 0000000..c7eb12c --- /dev/null +++ b/app/tests/unit/bootstrap/test_application.py @@ -0,0 +1,442 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from src.bootstrap.application import ( + ApplicationComposition, + run_application, +) +from src.market_data.acquisition.runtime.trade_stream_production_runtime import ( + TradeStreamProductionRuntimeState, +) + + +class FakeBotSession: + def __init__( + self, + *, + close_error: BaseException | None = None, + ) -> None: + self.close_calls = 0 + self.close_error = close_error + + async def close(self) -> None: + self.close_calls += 1 + + if self.close_error is not None: + raise self.close_error + + +class FakeBot: + def __init__( + self, + *, + close_error: BaseException | None = None, + ) -> None: + self.session = FakeBotSession( + close_error=close_error, + ) + + +class FakeDispatcher: + def __init__( + self, + *, + return_immediately: bool = False, + error: BaseException | None = None, + ) -> None: + self.started = asyncio.Event() + self.release = asyncio.Event() + self.cancelled = asyncio.Event() + self.return_immediately = return_immediately + self.error = error + self.close_bot_session_values: list[bool] = [] + + async def start_polling( + self, + bot: FakeBot, + *, + close_bot_session: bool, + ) -> None: + del bot + self.close_bot_session_values.append(close_bot_session) + self.started.set() + + try: + if not self.return_immediately: + await self.release.wait() + except asyncio.CancelledError: + self.cancelled.set() + raise + + if self.error is not None: + raise self.error + + +class FakeRuntime: + def __init__( + self, + *, + return_immediately: bool = False, + error: BaseException | None = None, + stop_error: BaseException | None = None, + ) -> None: + self.started = asyncio.Event() + self.release = asyncio.Event() + self.stopped = asyncio.Event() + self.return_immediately = return_immediately + self.error = error + self.stop_error = stop_error + self.stop_calls = 0 + + @property + def state(self) -> TradeStreamProductionRuntimeState: + return ( + TradeStreamProductionRuntimeState.RUNNING + if self.started.is_set() and not self.stopped.is_set() + else TradeStreamProductionRuntimeState.STOPPED + ) + + @property + def running(self) -> bool: + return self.state is TradeStreamProductionRuntimeState.RUNNING + + async def run(self) -> None: + self.started.set() + + if not self.return_immediately: + await self.release.wait() + + if self.error is not None: + raise self.error + + async def stop(self) -> None: + self.stop_calls += 1 + self.release.set() + self.stopped.set() + + if self.stop_error is not None: + raise self.stop_error + + +class BlockingStopRuntime(FakeRuntime): + def __init__(self) -> None: + super().__init__() + + self.stop_entered = asyncio.Event() + self.stop_release = asyncio.Event() + + async def stop(self) -> None: + self.stop_calls += 1 + self.stop_entered.set() + await self.stop_release.wait() + self.release.set() + self.stopped.set() + + +def make_application( + *, + dispatcher: FakeDispatcher, + runtime: FakeRuntime | None, + bot: FakeBot | None = None, +) -> ApplicationComposition: + return ApplicationComposition( + bot=bot or FakeBot(), # type: ignore[arg-type] + dispatcher=dispatcher, # type: ignore[arg-type] + trade_stream_runtime=runtime, + ) + + +def test_disabled_runtime_runs_only_polling_and_closes_bot() -> None: + async def scenario() -> None: + dispatcher = FakeDispatcher(return_immediately=True) + bot = FakeBot() + + await run_application( + make_application( + dispatcher=dispatcher, + runtime=None, + bot=bot, + ) + ) + + assert dispatcher.close_bot_session_values == [False] + assert bot.session.close_calls == 1 + + asyncio.run(scenario()) + + +def test_polling_completion_stops_and_awaits_runtime() -> None: + async def scenario() -> None: + dispatcher = FakeDispatcher(return_immediately=True) + runtime = FakeRuntime() + bot = FakeBot() + + await run_application( + make_application( + dispatcher=dispatcher, + runtime=runtime, + bot=bot, + ) + ) + + assert runtime.started.is_set() + assert runtime.stop_calls == 1 + assert runtime.stopped.is_set() + assert bot.session.close_calls == 1 + + asyncio.run(scenario()) + + +def test_runtime_error_is_fatal_and_stops_polling() -> None: + async def scenario() -> None: + expected = RuntimeError("trade stream failed") + dispatcher = FakeDispatcher() + runtime = FakeRuntime( + return_immediately=True, + error=expected, + ) + bot = FakeBot() + + with pytest.raises(RuntimeError) as exc_info: + await run_application( + make_application( + dispatcher=dispatcher, + runtime=runtime, + bot=bot, + ) + ) + + assert exc_info.value is expected + assert dispatcher.cancelled.is_set() + assert runtime.stop_calls == 1 + assert bot.session.close_calls == 1 + + asyncio.run(scenario()) + + +def test_normal_runtime_completion_is_fatal() -> None: + async def scenario() -> None: + dispatcher = FakeDispatcher() + runtime = FakeRuntime(return_immediately=True) + + with pytest.raises( + RuntimeError, + match="terminated unexpectedly", + ): + await run_application( + make_application( + dispatcher=dispatcher, + runtime=runtime, + ) + ) + + assert dispatcher.cancelled.is_set() + assert runtime.stop_calls == 1 + + asyncio.run(scenario()) + + +def test_application_cancellation_performs_full_cleanup() -> None: + async def scenario() -> None: + dispatcher = FakeDispatcher() + runtime = FakeRuntime() + bot = FakeBot() + task = asyncio.create_task( + run_application( + make_application( + dispatcher=dispatcher, + runtime=runtime, + bot=bot, + ) + ) + ) + + await dispatcher.started.wait() + await runtime.started.wait() + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + + assert dispatcher.cancelled.is_set() + assert runtime.stop_calls == 1 + assert runtime.stopped.is_set() + assert bot.session.close_calls == 1 + await asyncio.sleep(0) + assert not { + child.get_name() + for child in asyncio.all_tasks() + if child is not asyncio.current_task() + and not child.done() + and child.get_name() + in { + "telegram-polling", + "trade-stream-runtime", + "application-shutdown", + } + } + + asyncio.run(scenario()) + + +def test_polling_error_stops_runtime_and_preserves_error() -> None: + async def scenario() -> None: + expected = RuntimeError("polling failed") + dispatcher = FakeDispatcher( + return_immediately=True, + error=expected, + ) + runtime = FakeRuntime() + bot = FakeBot() + + with pytest.raises(RuntimeError) as exc_info: + await run_application( + make_application( + dispatcher=dispatcher, + runtime=runtime, + bot=bot, + ) + ) + + assert exc_info.value is expected + assert runtime.stop_calls == 1 + assert runtime.stopped.is_set() + assert bot.session.close_calls == 1 + + asyncio.run(scenario()) + + +def test_simultaneous_root_failures_are_awaited_deterministically() -> None: + async def scenario() -> None: + polling_error = RuntimeError("polling failed") + runtime_error = ValueError("trade stream failed") + dispatcher = FakeDispatcher( + error=polling_error, + ) + runtime = FakeRuntime( + error=runtime_error, + ) + bot = FakeBot() + task = asyncio.create_task( + run_application( + make_application( + dispatcher=dispatcher, + runtime=runtime, + bot=bot, + ) + ) + ) + + await dispatcher.started.wait() + await runtime.started.wait() + dispatcher.release.set() + runtime.release.set() + + with pytest.raises(ValueError) as exc_info: + await task + + assert exc_info.value is runtime_error + assert runtime.stop_calls == 1 + assert bot.session.close_calls == 1 + assert any( + "cleanup also failed" in note + for note in getattr(runtime_error, "__notes__", ()) + ) + await asyncio.sleep(0) + assert not { + child.get_name() + for child in asyncio.all_tasks() + if child is not asyncio.current_task() + and not child.done() + and child.get_name() + in { + "telegram-polling", + "trade-stream-runtime", + "application-shutdown", + } + } + + asyncio.run(scenario()) + + +def test_repeated_cancellation_does_not_interrupt_cleanup() -> None: + async def scenario() -> None: + dispatcher = FakeDispatcher() + runtime = BlockingStopRuntime() + bot = FakeBot() + task = asyncio.create_task( + run_application( + make_application( + dispatcher=dispatcher, + runtime=runtime, + bot=bot, + ) + ) + ) + + await dispatcher.started.wait() + await runtime.started.wait() + task.cancel() + await runtime.stop_entered.wait() + + task.cancel() + runtime.stop_release.set() + + with pytest.raises(asyncio.CancelledError): + await task + + assert task.cancelled() is True + assert dispatcher.cancelled.is_set() + assert runtime.stop_calls == 1 + assert runtime.stopped.is_set() + assert bot.session.close_calls == 1 + await asyncio.sleep(0) + assert not { + child.get_name() + for child in asyncio.all_tasks() + if child is not asyncio.current_task() + and not child.done() + and child.get_name() + in { + "telegram-polling", + "trade-stream-runtime", + "application-shutdown", + } + } + + asyncio.run(scenario()) + + +def test_cleanup_error_does_not_replace_runtime_error() -> None: + async def scenario() -> None: + expected = RuntimeError("trade stream failed") + dispatcher = FakeDispatcher() + runtime = FakeRuntime( + return_immediately=True, + error=expected, + stop_error=RuntimeError("stop failed"), + ) + bot = FakeBot( + close_error=RuntimeError("close failed"), + ) + + with pytest.raises(RuntimeError) as exc_info: + await run_application( + make_application( + dispatcher=dispatcher, + runtime=runtime, + bot=bot, + ) + ) + + assert exc_info.value is expected + assert bot.session.close_calls == 1 + assert any( + "cleanup also failed" in note + for note in getattr(expected, "__notes__", ()) + ) + + asyncio.run(scenario()) diff --git a/app/tests/unit/bootstrap/test_trade_stream_production_integration.py b/app/tests/unit/bootstrap/test_trade_stream_production_integration.py new file mode 100644 index 0000000..33b16ba --- /dev/null +++ b/app/tests/unit/bootstrap/test_trade_stream_production_integration.py @@ -0,0 +1,890 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import time +from collections.abc import Awaitable, Callable +from typing import Any + +import pytest +from websockets.protocol import State + +import src.bootstrap.trade_stream_runtime as production_factory +from src.bootstrap.application import ( + ApplicationComposition, + run_application, +) +from src.core.config import Settings, TradeStreamSettings +from src.market_data.acquisition.adapters.dzengi.websocket_transport import ( + DzengiWebSocketTransport, +) +from src.market_data.acquisition.consistency.trade_stream_state_store import ( + TradeStreamStateStore, +) +from src.market_data.acquisition.exceptions import ( + TradeTransportError, + WebSocketTransportError, +) +from src.market_data.acquisition.runtime.trade_stream_production_runtime import ( + TradeStreamProductionRuntime, + TradeStreamProductionRuntimeState, +) + + +SYMBOL = "BTC/USD_LEVERAGE" + +_OWNED_TASK_NAMES = frozenset( + { + "application-shutdown", + "telegram-polling", + "trade-stream-receive", + "trade-stream-runtime", + "trade-stream-runtime-recovery", + "trade-stream-scheduler", + "trade-stream-startup", + } +) + + +class FakeBotSession: + def __init__(self) -> None: + self.close_calls = 0 + + async def close(self) -> None: + self.close_calls += 1 + + +class FakeBot: + def __init__(self) -> None: + self.session = FakeBotSession() + + +class ControlledDispatcher: + def __init__( + self, + *, + return_immediately: bool = False, + ) -> None: + self.return_immediately = return_immediately + self.started = asyncio.Event() + self.release = asyncio.Event() + self.cancelled = asyncio.Event() + self.close_bot_session_values: list[bool] = [] + + async def start_polling( + self, + bot: FakeBot, + *, + close_bot_session: bool, + ) -> None: + del bot + self.close_bot_session_values.append(close_bot_session) + self.started.set() + + if self.return_immediately: + return + + try: + await self.release.wait() + except asyncio.CancelledError: + self.cancelled.set() + raise + + +class ScriptedConnection: + def __init__( + self, + *, + name: str, + calls: list[str], + send_error: Exception | None = None, + close_on_receive_error: bool = True, + ) -> None: + self.name = name + self.calls = calls + self.send_error = send_error + self.close_on_receive_error = close_on_receive_error + + self.state = State.OPEN + self.incoming: asyncio.Queue[ + str | bytes | BaseException + ] = asyncio.Queue() + self.sent_messages: list[str | bytes] = [] + self.subscription_sent = asyncio.Event() + self.close_calls = 0 + self.ping_calls = 0 + + async def close( + self, + code: int = 1000, + reason: str = "", + ) -> None: + del code, reason + self.calls.append(f"{self.name}.close") + self.close_calls += 1 + self.state = State.CLOSED + + async def send( + self, + message: str | bytes, + ) -> None: + self.calls.append(f"{self.name}.send") + + if self.send_error is not None: + self.state = State.CLOSED + raise self.send_error + + self.sent_messages.append(message) + self.subscription_sent.set() + + if not isinstance(message, str): + return + + document = json.loads(message) + + if document.get("destination") != "trades.subscribe": + return + + self.feed( + json.dumps( + { + "correlationId": document["correlationId"], + "destination": "trades.subscribe", + "status": "OK", + } + ) + ) + + async def recv(self) -> str | bytes: + item = await self.incoming.get() + + if isinstance(item, BaseException): + self.calls.append(f"{self.name}.receive_error") + + if self.close_on_receive_error: + self.state = State.CLOSED + + raise item + + self.calls.append(f"{self.name}.receive") + return item + + async def ping(self) -> Awaitable[float]: + self.calls.append(f"{self.name}.ping") + self.ping_calls += 1 + + async def wait_for_pong() -> float: + return 0.001 + + return wait_for_pong() + + def feed( + self, + item: str | bytes | BaseException, + ) -> None: + self.incoming.put_nowait(item) + + +class ScriptedConnector: + def __init__( + self, + *, + calls: list[str], + connections: tuple[ScriptedConnection, ...] = (), + error: Exception | None = None, + ) -> None: + self.calls = calls + self.connections = list(connections) + self.error = error + self.options: list[dict[str, Any]] = [] + + async def __call__( + self, + url: str, + **kwargs: Any, + ) -> ScriptedConnection: + self.calls.append("connector.connect") + self.options.append( + { + "url": url, + **kwargs, + } + ) + + if self.error is not None: + raise self.error + + return self.connections.pop(0) + + +class BlockingRestClient: + def __init__( + self, + *, + calls: list[str], + document: object, + error: Exception | None = None, + ) -> None: + self.calls = calls + self.document = document + self.error = error + self.entered = threading.Event() + self.release = threading.Event() + self.requests: list[ + tuple[str, dict[str, str] | None] + ] = [] + + def get_payload( + self, + path: str, + params: dict[str, str] | None = None, + headers: dict[str, str] | None = None, + ) -> object: + del headers + self.calls.append("recovery.fetch") + self.requests.append( + ( + path, + params, + ) + ) + self.entered.set() + + if not self.release.wait(timeout=5): + raise TimeoutError("test did not release REST recovery") + + if self.error is not None: + raise self.error + + return self.document + + +def make_settings( + *, + enabled: bool = True, +) -> Settings: + return Settings( + bot_token="test-token", + bot_parse_mode="HTML", + app_env="test", + log_level="INFO", + tz="UTC", + exchange_enabled=True, + exchange_name="dzengi", + exchange_base_url="https://rest.example.test", + exchange_ws_url="wss://legacy.example.test", + exchange_api_key="", + exchange_api_secret="", + exchange_timeout_sec=10, + exchange_testnet=True, + default_symbol="LEGACY", + trade_stream=TradeStreamSettings( + enabled=enabled, + websocket_url="wss://stream.example.test", + symbols=(SYMBOL,), + open_timeout_seconds=1.0, + probe_timeout_seconds=1.0, + close_timeout_seconds=1.0, + heartbeat_timeout_seconds=120.0, + scheduler_interval_seconds=60.0, + recovery_window_ms=3_599_999, + ), + db_host="localhost", + db_port=5432, + db_name="test", + db_user="test", + db_password="test", + debug_enabled=False, + journal_debug_enabled=False, + ) + + +def make_trade_document( + *, + trade_id: int, + timestamp: int, +) -> dict[str, object]: + return { + "status": "OK", + "destination": "internal.trade", + "payload": { + "id": trade_id, + "price": "64555.55", + "size": "0.002", + "ts": timestamp, + "symbol": SYMBOL, + "buyer": True, + "orderId": f"order-{trade_id}", + }, + } + + +def make_recovered_trade( + *, + trade_id: int, + timestamp: int, +) -> dict[str, object]: + return { + "a": trade_id, + "p": "64555.56", + "q": "0.003", + "T": timestamp, + "m": False, + } + + +def install_connector( + monkeypatch: pytest.MonkeyPatch, + connector: ScriptedConnector, +) -> list[DzengiWebSocketTransport]: + transport_type = production_factory.DzengiWebSocketTransport + transports: list[DzengiWebSocketTransport] = [] + + def build_transport( + **kwargs: Any, + ) -> DzengiWebSocketTransport: + transport = transport_type( + connector=connector, + **kwargs, + ) + transports.append(transport) + return transport + + monkeypatch.setattr( + production_factory, + "DzengiWebSocketTransport", + build_transport, + ) + + return transports + + +def make_application( + *, + runtime: TradeStreamProductionRuntime | None, + dispatcher: ControlledDispatcher, + bot: FakeBot, +) -> ApplicationComposition: + return ApplicationComposition( + bot=bot, # type: ignore[arg-type] + dispatcher=dispatcher, # type: ignore[arg-type] + trade_stream_runtime=runtime, + ) + + +async def wait_until( + predicate: Callable[[], bool], +) -> None: + for _ in range(1_000): + if predicate(): + return + + await asyncio.sleep(0) + + raise AssertionError("condition was not reached") + + +async def assert_no_owned_tasks() -> None: + await asyncio.sleep(0) + + assert not { + task.get_name() + for task in asyncio.all_tasks() + if task is not asyncio.current_task() + and not task.done() + and task.get_name() in _OWNED_TASK_NAMES + } + + +def state_store_from( + runtime: TradeStreamProductionRuntime, +) -> TradeStreamStateStore: + return ( + runtime + ._reconnect_recovery_coordinator + ._recovery_coordinator + ._state_store + ) + + +def test_disabled_feature_runs_only_telegram_without_runtime_graph( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def forbidden_dependency( + *args: Any, + **kwargs: Any, + ) -> None: + del args, kwargs + raise AssertionError("disabled feature built a dependency") + + monkeypatch.setattr( + production_factory, + "DzengiWebSocketTransport", + forbidden_dependency, + ) + monkeypatch.setattr( + production_factory, + "ExchangeRestClient", + forbidden_dependency, + ) + + runtime = ( + production_factory.build_trade_stream_production_runtime( + make_settings(enabled=False), + ) + ) + + async def scenario() -> None: + bot = FakeBot() + dispatcher = ControlledDispatcher( + return_immediately=True, + ) + + await run_application( + make_application( + runtime=runtime, + dispatcher=dispatcher, + bot=bot, + ) + ) + + assert dispatcher.close_bot_session_values == [False] + assert bot.session.close_calls == 1 + await assert_no_owned_tasks() + + assert runtime is None + asyncio.run(scenario()) + + +def test_production_factory_processes_ack_trade_and_shutdown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def scenario() -> None: + calls: list[str] = [] + connection = ScriptedConnection( + name="connection", + calls=calls, + ) + connector = ScriptedConnector( + calls=calls, + connections=(connection,), + ) + transports = install_connector( + monkeypatch, + connector, + ) + runtime = ( + production_factory.build_trade_stream_production_runtime( + make_settings(), + ) + ) + + assert isinstance(runtime, TradeStreamProductionRuntime) + + bot = FakeBot() + dispatcher = ControlledDispatcher() + application_task = asyncio.create_task( + run_application( + make_application( + runtime=runtime, + dispatcher=dispatcher, + bot=bot, + ) + ) + ) + + await connection.subscription_sent.wait() + + timestamp = time.time_ns() // 1_000_000 + connection.feed( + json.dumps( + make_trade_document( + trade_id=100, + timestamp=timestamp, + ) + ) + ) + + state_store = state_store_from(runtime) + await wait_until( + lambda: ( + state_store.contains(SYMBOL) + and state_store.get(SYMBOL).last_trade_id == 100 + ) + ) + + dispatcher.release.set() + await application_task + + subscription = json.loads( + connection.sent_messages[0], + ) + + assert subscription["destination"] == "trades.subscribe" + assert subscription["payload"]["symbols"] == [SYMBOL] + assert connector.options[0]["ping_interval"] is None + assert connector.options[0]["ping_timeout"] is None + assert connection.close_calls == 1 + assert runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert runtime._subscription_manager.subscription_keys == () + assert bot.session.close_calls == 1 + assert len(transports) == 1 + await assert_no_owned_tasks() + + asyncio.run(scenario()) + + +def test_connection_startup_failure_is_fatal_and_leaves_no_tasks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def scenario() -> None: + calls: list[str] = [] + connector = ScriptedConnector( + calls=calls, + error=OSError("connect failed"), + ) + install_connector( + monkeypatch, + connector, + ) + runtime = ( + production_factory.build_trade_stream_production_runtime( + make_settings(), + ) + ) + + assert isinstance(runtime, TradeStreamProductionRuntime) + + bot = FakeBot() + dispatcher = ControlledDispatcher() + + with pytest.raises( + WebSocketTransportError, + match="connect failed", + ) as exc_info: + await run_application( + make_application( + runtime=runtime, + dispatcher=dispatcher, + bot=bot, + ) + ) + + assert isinstance(exc_info.value.__cause__, OSError) + assert dispatcher.cancelled.is_set() + assert runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + assert runtime._session.is_connected is False + assert runtime._subscription_manager.subscription_keys == () + assert bot.session.close_calls == 1 + await assert_no_owned_tasks() + + asyncio.run(scenario()) + + +def test_subscription_startup_failure_rolls_back_concrete_graph( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def scenario() -> None: + calls: list[str] = [] + connection = ScriptedConnection( + name="connection", + calls=calls, + send_error=OSError("subscription failed"), + ) + connector = ScriptedConnector( + calls=calls, + connections=(connection,), + ) + install_connector( + monkeypatch, + connector, + ) + runtime = ( + production_factory.build_trade_stream_production_runtime( + make_settings(), + ) + ) + + assert isinstance(runtime, TradeStreamProductionRuntime) + + bot = FakeBot() + dispatcher = ControlledDispatcher() + + with pytest.raises( + WebSocketTransportError, + match="subscription failed", + ): + await run_application( + make_application( + runtime=runtime, + dispatcher=dispatcher, + bot=bot, + ) + ) + + assert dispatcher.cancelled.is_set() + assert connection.state is State.CLOSED + assert runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + assert runtime._session.is_connected is False + assert runtime._subscription_manager.subscription_keys == () + assert bot.session.close_calls == 1 + await assert_no_owned_tasks() + + asyncio.run(scenario()) + + +def test_reconnect_restores_recovers_then_processes_buffered_trade( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def scenario() -> None: + calls: list[str] = [] + first_connection = ScriptedConnection( + name="first", + calls=calls, + close_on_receive_error=False, + ) + second_connection = ScriptedConnection( + name="second", + calls=calls, + ) + connector = ScriptedConnector( + calls=calls, + connections=( + first_connection, + second_connection, + ), + ) + install_connector( + monkeypatch, + connector, + ) + + live_timestamp = ( + time.time_ns() // 1_000_000 + ) - 10_000 + rest_client = BlockingRestClient( + calls=calls, + document=[ + make_recovered_trade( + trade_id=101, + timestamp=live_timestamp + 1, + ) + ], + ) + monkeypatch.setattr( + production_factory, + "ExchangeRestClient", + lambda settings: rest_client, + ) + + runtime = ( + production_factory.build_trade_stream_production_runtime( + make_settings(), + ) + ) + + assert isinstance(runtime, TradeStreamProductionRuntime) + + bot = FakeBot() + dispatcher = ControlledDispatcher() + application_task = asyncio.create_task( + run_application( + make_application( + runtime=runtime, + dispatcher=dispatcher, + bot=bot, + ) + ) + ) + + await first_connection.subscription_sent.wait() + first_connection.feed( + json.dumps( + make_trade_document( + trade_id=100, + timestamp=live_timestamp, + ) + ) + ) + + state_store = state_store_from(runtime) + await wait_until( + lambda: ( + state_store.contains(SYMBOL) + and state_store.get(SYMBOL).last_trade_id == 100 + ) + ) + + first_connection.feed( + OSError("connection dropped"), + ) + + await wait_until(rest_client.entered.is_set) + + assert second_connection.subscription_sent.is_set() + assert state_store.get(SYMBOL).last_trade_id == 100 + + second_connection.feed( + json.dumps( + make_trade_document( + trade_id=102, + timestamp=live_timestamp + 2, + ) + ) + ) + rest_client.release.set() + + await wait_until( + lambda: state_store.get(SYMBOL).last_trade_id == 102 + ) + + dispatcher.release.set() + await application_task + + state = state_store.get(SYMBOL) + + assert state.last_trade_id == 102 + assert state.last_trade is not None + assert state.last_trade.executed_at.timestamp() == pytest.approx( + (live_timestamp + 2) / 1_000, + ) + assert len(connector.options) == 2 + assert first_connection.close_calls == 1 + assert second_connection.close_calls == 1 + assert len(first_connection.sent_messages) == 1 + assert len(second_connection.sent_messages) == 1 + assert calls.index("second.send") < calls.index( + "recovery.fetch" + ) + assert rest_client.requests[0][0] == "/api/v1/aggTrades" + request_params = rest_client.requests[0][1] + + assert request_params is not None + assert request_params["symbol"] == SYMBOL + assert runtime._reconnect_recovery_coordinator.generation == 1 + assert runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert bot.session.close_calls == 1 + await assert_no_owned_tasks() + + asyncio.run(scenario()) + + +def test_recovery_failure_rejects_buffered_trade_and_cleans_up( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def scenario() -> None: + calls: list[str] = [] + first_connection = ScriptedConnection( + name="first", + calls=calls, + close_on_receive_error=False, + ) + second_connection = ScriptedConnection( + name="second", + calls=calls, + ) + connector = ScriptedConnector( + calls=calls, + connections=( + first_connection, + second_connection, + ), + ) + install_connector( + monkeypatch, + connector, + ) + + live_timestamp = ( + time.time_ns() // 1_000_000 + ) - 10_000 + rest_client = BlockingRestClient( + calls=calls, + document=[], + error=OSError("recovery failed"), + ) + monkeypatch.setattr( + production_factory, + "ExchangeRestClient", + lambda settings: rest_client, + ) + + runtime = ( + production_factory.build_trade_stream_production_runtime( + make_settings(), + ) + ) + + assert isinstance(runtime, TradeStreamProductionRuntime) + + bot = FakeBot() + dispatcher = ControlledDispatcher() + application_task = asyncio.create_task( + run_application( + make_application( + runtime=runtime, + dispatcher=dispatcher, + bot=bot, + ) + ) + ) + + await first_connection.subscription_sent.wait() + first_connection.feed( + json.dumps( + make_trade_document( + trade_id=100, + timestamp=live_timestamp, + ) + ) + ) + + state_store = state_store_from(runtime) + await wait_until( + lambda: ( + state_store.contains(SYMBOL) + and state_store.get(SYMBOL).last_trade_id == 100 + ) + ) + + first_connection.feed( + OSError("connection dropped"), + ) + await wait_until(rest_client.entered.is_set) + + second_connection.feed( + json.dumps( + make_trade_document( + trade_id=102, + timestamp=live_timestamp + 2, + ) + ) + ) + rest_client.release.set() + + with pytest.raises( + TradeTransportError, + match="recovery failed", + ): + await application_task + + assert state_store.get(SYMBOL).last_trade_id == 100 + assert runtime._live_processing_gate.failed is True + assert dispatcher.cancelled.is_set() + assert runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + assert second_connection.close_calls == 1 + assert bot.session.close_calls == 1 + await assert_no_owned_tasks() + + asyncio.run(scenario()) diff --git a/app/tests/unit/bootstrap/test_trade_stream_runtime.py b/app/tests/unit/bootstrap/test_trade_stream_runtime.py new file mode 100644 index 0000000..0d5b7ab --- /dev/null +++ b/app/tests/unit/bootstrap/test_trade_stream_runtime.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +from src.bootstrap.trade_stream_runtime import ( + build_trade_stream_production_runtime, +) +from src.core.config import Settings, TradeStreamSettings +from src.market_data.acquisition.runtime.trade_stream_production_runtime import ( + TradeStreamProductionRuntime, + TradeStreamProductionRuntimeState, +) + + +def make_settings( + *, + enabled: bool = True, + api_key: str = "api-key", +) -> Settings: + return Settings( + bot_token="test-token", + bot_parse_mode="HTML", + app_env="test", + log_level="INFO", + tz="UTC", + exchange_enabled=False, + exchange_name="dzengi", + exchange_base_url="https://rest.example.test/", + exchange_ws_url="wss://legacy.example.test", + exchange_api_key=api_key, + exchange_api_secret="secret", + exchange_timeout_sec=17, + exchange_testnet=True, + default_symbol="LEGACY", + trade_stream=TradeStreamSettings( + enabled=enabled, + websocket_url="wss://stream.example.test/root", + symbols=( + "ETH/USD_LEVERAGE", + "BTC/USD_LEVERAGE", + ), + open_timeout_seconds=11.0, + probe_timeout_seconds=21.0, + close_timeout_seconds=9.0, + heartbeat_timeout_seconds=31.0, + scheduler_interval_seconds=6.0, + recovery_window_ms=123_456, + ), + db_host="localhost", + db_port=5432, + db_name="test", + db_user="test", + db_password="test", + debug_enabled=False, + journal_debug_enabled=False, + ) + + +def test_disabled_feature_does_not_build_runtime() -> None: + settings = make_settings(enabled=False) + + assert build_trade_stream_production_runtime(settings) is None + + +def test_builds_runtime_without_starting_lifecycle() -> None: + runtime = build_trade_stream_production_runtime( + make_settings(), + ) + + assert isinstance(runtime, TradeStreamProductionRuntime) + assert runtime.state is TradeStreamProductionRuntimeState.STOPPED + assert runtime.running is False + assert runtime._startup_task is None + assert runtime._receive_task is None + assert runtime._scheduler_task is None + + +def test_uses_one_shared_stateful_dependency_graph() -> None: + runtime = build_trade_stream_production_runtime( + make_settings(), + ) + + assert isinstance(runtime, TradeStreamProductionRuntime) + + transport = runtime._transport + service = runtime._trade_stream_service + reconnect_recovery = runtime._reconnect_recovery_coordinator + recovery = reconnect_recovery._recovery_coordinator + + assert runtime._session._transport is transport + assert runtime._subscription_manager._transport is transport + assert runtime._runtime_scheduler.liveness_probe is transport + assert ( + service._consistency_controller + is recovery._recovery_controller._consistency_controller + ) + assert runtime._live_processing_gate is ( + reconnect_recovery.live_processing_gate + ) + assert runtime._runtime_scheduler.runtime_supervisor is ( + runtime._runtime_supervisor + ) + + +def test_applies_explicit_transport_and_runtime_settings() -> None: + settings = make_settings() + + runtime = build_trade_stream_production_runtime(settings) + + assert isinstance(runtime, TradeStreamProductionRuntime) + transport = runtime._transport + + assert transport._url == "wss://stream.example.test/root/connect" + assert transport._headers == { + "Origin": "https://rest.example.test", + "Content-Type": "application/json", + "X-MBX-APIKEY": "api-key", + } + assert transport._open_timeout == 11.0 + assert transport._probe_timeout == 21.0 + assert transport._close_timeout == 9.0 + assert transport._ping_interval is None + assert transport._ping_timeout is None + assert runtime._symbols == ( + "BTC/USD_LEVERAGE", + "ETH/USD_LEVERAGE", + ) + assert runtime._runtime_scheduler.interval_seconds == 6.0 + assert ( + runtime._runtime_supervisor._heartbeat_monitor.timeout_seconds + == 31.0 + ) + assert ( + runtime._reconnect_recovery_coordinator + ._recovery_coordinator + ._window_planner + .max_window_ms + == 123_456 + ) + + +def test_recovery_rest_client_reuses_settings_snapshot() -> None: + settings = make_settings(api_key="") + + runtime = build_trade_stream_production_runtime(settings) + + assert isinstance(runtime, TradeStreamProductionRuntime) + document_source = ( + runtime._reconnect_recovery_coordinator + ._recovery_coordinator + ._recovery_controller + ._document_source + ) + rest_client = document_source._client + + assert rest_client.settings is settings + assert rest_client.base_url == "https://rest.example.test" + assert rest_client.timeout == 17 + assert "X-MBX-APIKEY" not in runtime._transport._headers diff --git a/app/tests/unit/core/test_config.py b/app/tests/unit/core/test_config.py new file mode 100644 index 0000000..31a1aca --- /dev/null +++ b/app/tests/unit/core/test_config.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import pytest + +from src.core.config import load_settings + + +_TRADE_STREAM_VARIABLES = ( + "TRADE_STREAM_ENABLED", + "TRADE_STREAM_WS_URL", + "TRADE_STREAM_SYMBOLS", + "TRADE_STREAM_OPEN_TIMEOUT_SECONDS", + "TRADE_STREAM_PROBE_TIMEOUT_SECONDS", + "TRADE_STREAM_CLOSE_TIMEOUT_SECONDS", + "TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS", + "TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS", + "TRADE_STREAM_RECOVERY_WINDOW_MS", +) + + +def prepare_environment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("BOT_TOKEN", "test-token") + monkeypatch.delenv("EXCHANGE_BASE_URL", raising=False) + monkeypatch.delenv("EXCHANGE_ENABLED", raising=False) + + for variable in _TRADE_STREAM_VARIABLES: + monkeypatch.delenv(variable, raising=False) + + +def enable_trade_stream( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("TRADE_STREAM_ENABLED", "true") + monkeypatch.setenv( + "TRADE_STREAM_WS_URL", + "wss://stream.example.test", + ) + monkeypatch.setenv( + "TRADE_STREAM_SYMBOLS", + "ETH/USD_LEVERAGE,BTC/USD_LEVERAGE", + ) + monkeypatch.setenv( + "EXCHANGE_BASE_URL", + "https://rest.example.test", + ) + + +def test_trade_stream_is_disabled_by_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepare_environment(monkeypatch) + monkeypatch.setenv("EXCHANGE_ENABLED", "true") + + settings = load_settings() + + assert settings.exchange_enabled is True + assert settings.trade_stream.enabled is False + assert settings.trade_stream.websocket_url == "" + assert settings.trade_stream.symbols == () + + +def test_disabled_trade_stream_ignores_dependent_values( + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepare_environment(monkeypatch) + monkeypatch.setenv("TRADE_STREAM_OPEN_TIMEOUT_SECONDS", "invalid") + monkeypatch.setenv("TRADE_STREAM_SYMBOLS", ",") + + settings = load_settings() + + assert settings.trade_stream.enabled is False + + +def test_trade_stream_flag_is_strict( + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepare_environment(monkeypatch) + monkeypatch.setenv("TRADE_STREAM_ENABLED", "sometimes") + + with pytest.raises( + ValueError, + match="TRADE_STREAM_ENABLED", + ): + load_settings() + + +@pytest.mark.parametrize( + ("missing_variable", "message"), + ( + ( + "TRADE_STREAM_WS_URL", + "TRADE_STREAM_WS_URL", + ), + ( + "TRADE_STREAM_SYMBOLS", + "TRADE_STREAM_SYMBOLS", + ), + ( + "EXCHANGE_BASE_URL", + "EXCHANGE_BASE_URL", + ), + ), +) +def test_enabled_trade_stream_requires_explicit_endpoints_and_symbols( + monkeypatch: pytest.MonkeyPatch, + missing_variable: str, + message: str, +) -> None: + prepare_environment(monkeypatch) + enable_trade_stream(monkeypatch) + monkeypatch.setenv( + "EXCHANGE_WS_URL", + "wss://legacy-fallback.example.test", + ) + monkeypatch.setenv( + "DEFAULT_SYMBOL", + "LEGACY_FALLBACK", + ) + monkeypatch.delenv(missing_variable) + + with pytest.raises( + RuntimeError, + match=message, + ): + load_settings() + + +def test_enabled_trade_stream_parses_independent_settings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepare_environment(monkeypatch) + enable_trade_stream(monkeypatch) + monkeypatch.setenv( + "TRADE_STREAM_SYMBOLS", + " ETH/USD_LEVERAGE, BTC/USD_LEVERAGE,ETH/USD_LEVERAGE ", + ) + monkeypatch.setenv("TRADE_STREAM_OPEN_TIMEOUT_SECONDS", "11.5") + monkeypatch.setenv("TRADE_STREAM_PROBE_TIMEOUT_SECONDS", "21") + monkeypatch.setenv("TRADE_STREAM_CLOSE_TIMEOUT_SECONDS", "9") + monkeypatch.setenv("TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS", "31") + monkeypatch.setenv("TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS", "6") + monkeypatch.setenv("TRADE_STREAM_RECOVERY_WINDOW_MS", "123456") + + settings = load_settings() + trade_stream = settings.trade_stream + + assert trade_stream.enabled is True + assert trade_stream.websocket_url == "wss://stream.example.test" + assert trade_stream.symbols == ( + "BTC/USD_LEVERAGE", + "ETH/USD_LEVERAGE", + ) + assert trade_stream.open_timeout_seconds == 11.5 + assert trade_stream.probe_timeout_seconds == 21.0 + assert trade_stream.close_timeout_seconds == 9.0 + assert trade_stream.heartbeat_timeout_seconds == 31.0 + assert trade_stream.scheduler_interval_seconds == 6.0 + assert trade_stream.recovery_window_ms == 123_456 + + +@pytest.mark.parametrize( + ("variable", "value"), + ( + ("TRADE_STREAM_OPEN_TIMEOUT_SECONDS", "0"), + ("TRADE_STREAM_PROBE_TIMEOUT_SECONDS", "-1"), + ("TRADE_STREAM_CLOSE_TIMEOUT_SECONDS", "nan"), + ("TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS", "inf"), + ("TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS", "invalid"), + ("TRADE_STREAM_RECOVERY_WINDOW_MS", "1.5"), + ("TRADE_STREAM_RECOVERY_WINDOW_MS", "0"), + ), +) +def test_enabled_trade_stream_rejects_invalid_numeric_settings( + monkeypatch: pytest.MonkeyPatch, + variable: str, + value: str, +) -> None: + prepare_environment(monkeypatch) + enable_trade_stream(monkeypatch) + monkeypatch.setenv(variable, value) + + with pytest.raises(ValueError, match=variable): + load_settings() + + +def test_symbols_must_not_contain_empty_items( + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepare_environment(monkeypatch) + enable_trade_stream(monkeypatch) + monkeypatch.setenv( + "TRADE_STREAM_SYMBOLS", + "BTC/USD_LEVERAGE,,ETH/USD_LEVERAGE", + ) + + with pytest.raises( + ValueError, + match="empty symbols", + ): + load_settings() diff --git a/app/tests/unit/market_data/acquisition/adapters/dzengi/test_websocket_control_message_handler.py b/app/tests/unit/market_data/acquisition/adapters/dzengi/test_websocket_control_message_handler.py new file mode 100644 index 0000000..1c7d1eb --- /dev/null +++ b/app/tests/unit/market_data/acquisition/adapters/dzengi/test_websocket_control_message_handler.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import pytest + +from src.market_data.acquisition.adapters.dzengi.websocket_control_message_handler import ( + DzengiWebSocketControlMessageHandler, +) +from src.market_data.acquisition.exceptions import ( + WebSocketControlMessageError, + WebSocketMessageRoutingError, +) +from src.market_data.acquisition.runtime.websocket_inbound_message import ( + WebSocketControlMessageHandlerProtocol, +) + + +CORRELATION_ID = "trade-subscription-1" + + +@pytest.fixture +def handler() -> DzengiWebSocketControlMessageHandler: + return DzengiWebSocketControlMessageHandler() + + +def test_implements_public_protocol_and_uses_slots( + handler: DzengiWebSocketControlMessageHandler, +) -> None: + assert isinstance( + handler, + WebSocketControlMessageHandlerProtocol, + ) + assert not hasattr(handler, "__dict__") + + +def test_accepts_matching_successful_subscription_ack( + handler: DzengiWebSocketControlMessageHandler, +) -> None: + handler.handle( + { + "correlationId": CORRELATION_ID, + "destination": "trades.subscribe", + "status": "OK", + }, + expected_correlation_id=CORRELATION_ID, + ) + + +def test_rejects_negative_subscription_ack( + handler: DzengiWebSocketControlMessageHandler, +) -> None: + with pytest.raises( + WebSocketControlMessageError, + match="отклонил", + ): + handler.handle( + { + "correlationId": CORRELATION_ID, + "status": "ERROR", + "payload": { + "errorCode": "BAD_REQUEST", + }, + }, + expected_correlation_id=CORRELATION_ID, + ) + + +@pytest.mark.parametrize( + "document", + [ + None, + {}, + { + "correlationId": "unknown", + "destination": "trades.subscribe", + "status": "OK", + }, + { + "correlationId": CORRELATION_ID, + "destination": "unknown", + "status": "OK", + }, + { + "correlationId": CORRELATION_ID, + "destination": "trades.subscribe", + }, + { + "correlationId": CORRELATION_ID, + "destination": "trades.subscribe", + "status": "", + }, + { + "correlationId": CORRELATION_ID, + "destination": "trades.subscribe", + "status": None, + }, + ], +) +def test_rejects_unknown_or_malformed_control_messages( + handler: DzengiWebSocketControlMessageHandler, + document: object, +) -> None: + with pytest.raises(WebSocketMessageRoutingError): + handler.handle( + document, + expected_correlation_id=CORRELATION_ID, + ) diff --git a/app/tests/unit/market_data/acquisition/adapters/dzengi/test_websocket_inbound_message_classifier.py b/app/tests/unit/market_data/acquisition/adapters/dzengi/test_websocket_inbound_message_classifier.py new file mode 100644 index 0000000..d0155d3 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/adapters/dzengi/test_websocket_inbound_message_classifier.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import pytest + +from src.market_data.acquisition.adapters.dzengi.websocket_inbound_message_classifier import ( + DzengiWebSocketInboundMessageClassifier, +) +from src.market_data.acquisition.exceptions import ( + WebSocketMessageRoutingError, +) +from src.market_data.acquisition.runtime.websocket_inbound_message import ( + WebSocketInboundMessageClassifierProtocol, + WebSocketInboundMessageKind, +) + + +@pytest.fixture +def classifier() -> DzengiWebSocketInboundMessageClassifier: + return DzengiWebSocketInboundMessageClassifier() + + +def test_implements_public_protocol( + classifier: DzengiWebSocketInboundMessageClassifier, +) -> None: + assert isinstance( + classifier, + WebSocketInboundMessageClassifierProtocol, + ) + + +def test_uses_slots( + classifier: DzengiWebSocketInboundMessageClassifier, +) -> None: + assert not hasattr( + classifier, + "__dict__", + ) + + +@pytest.mark.parametrize( + "document", + [ + { + "destination": "internal.trade", + }, + { + "destination": "ohlc.event", + }, + { + "Payload": {}, + }, + ], +) +def test_recognizes_market_documents( + classifier: DzengiWebSocketInboundMessageClassifier, + document: object, +) -> None: + assert ( + classifier.classify(document) + is WebSocketInboundMessageKind.MARKET + ) + + +@pytest.mark.parametrize( + "correlation_id", + [ + "request-1", + 1, + ], +) +def test_recognizes_control_responses( + classifier: DzengiWebSocketInboundMessageClassifier, + correlation_id: str | int, +) -> None: + assert ( + classifier.classify( + { + "correlationId": correlation_id, + "destination": "trades.subscribe", + } + ) + is WebSocketInboundMessageKind.CONTROL + ) + + +def test_market_markers_take_priority_over_correlation_id( + classifier: DzengiWebSocketInboundMessageClassifier, +) -> None: + assert ( + classifier.classify( + { + "destination": "internal.trade", + "correlationId": "request-1", + } + ) + is WebSocketInboundMessageKind.MARKET + ) + + +@pytest.mark.parametrize( + "document", + [ + None, + [], + "message", + {}, + { + "destination": "unknown", + }, + { + "destination": [], + }, + { + "correlationId": "", + }, + { + "correlationId": " ", + }, + { + "correlationId": None, + }, + { + "correlationId": True, + }, + { + "correlationId": 1.5, + }, + ], +) +def test_rejects_invalid_or_unknown_documents( + classifier: DzengiWebSocketInboundMessageClassifier, + document: object, +) -> None: + with pytest.raises( + WebSocketMessageRoutingError, + ): + classifier.classify(document) diff --git a/app/tests/unit/market_data/acquisition/adapters/dzengi/test_websocket_transport.py b/app/tests/unit/market_data/acquisition/adapters/dzengi/test_websocket_transport.py new file mode 100644 index 0000000..16b7332 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/adapters/dzengi/test_websocket_transport.py @@ -0,0 +1,573 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable +from typing import Any + +import pytest +from websockets.protocol import State + +from src.market_data.acquisition.adapters.dzengi.websocket_transport import ( + DzengiWebSocketTransport, + build_dzengi_websocket_url, +) +from src.market_data.acquisition.exceptions import ( + WebSocketTransportError, + WebSocketTransportNotConnectedError, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + WebSocketTransportProtocol, +) +from src.market_data.acquisition.runtime.runtime_liveness_probe import ( + RuntimeLivenessProbeProtocol, +) + + +class FakeConnection: + def __init__( + self, + *, + incoming: tuple[str | bytes, ...] = (), + state: State = State.OPEN, + ) -> None: + self.state = state + self.incoming = list(incoming) + self.sent_messages: list[str | bytes] = [] + self.close_calls = 0 + self.send_error: Exception | None = None + self.receive_error: Exception | None = None + self.ping_error: Exception | None = None + self.pong_error: Exception | None = None + self.pong_gate: asyncio.Event | None = None + self.ping_calls = 0 + self.close_error: Exception | None = None + + async def close( + self, + code: int = 1000, + reason: str = "", + ) -> None: + self.close_calls += 1 + self.state = State.CLOSED + + if self.close_error is not None: + raise self.close_error + + async def send( + self, + message: str | bytes, + ) -> None: + if self.send_error is not None: + self.state = State.CLOSED + raise self.send_error + + self.sent_messages.append(message) + + async def recv(self) -> str | bytes: + if self.receive_error is not None: + self.state = State.CLOSED + raise self.receive_error + + return self.incoming.pop(0) + + async def ping(self) -> Awaitable[float]: + self.ping_calls += 1 + + if self.ping_error is not None: + raise self.ping_error + + async def wait_for_pong() -> float: + if self.pong_gate is not None: + await self.pong_gate.wait() + + if self.pong_error is not None: + raise self.pong_error + + return 0.01 + + return wait_for_pong() + + +class RecordingConnector: + def __init__( + self, + *connections: FakeConnection, + ) -> None: + self.connections = list(connections) + self.calls: list[ + tuple[str, dict[str, Any]] + ] = [] + + async def __call__( + self, + url: str, + **kwargs: Any, + ) -> FakeConnection: + self.calls.append( + ( + url, + kwargs, + ) + ) + + return self.connections.pop(0) + + +def create_transport( + connection: FakeConnection | None = None, +) -> tuple[ + DzengiWebSocketTransport, + FakeConnection, + RecordingConnector, +]: + resolved_connection = connection or FakeConnection() + connector = RecordingConnector( + resolved_connection, + ) + transport = DzengiWebSocketTransport( + url="https://api-adapter.dzengi.com", + headers={ + "Origin": "https://api-adapter.dzengi.com", + "X-MBX-APIKEY": "test-key", + }, + open_timeout=11, + ping_interval=12, + ping_timeout=13, + close_timeout=14, + connector=connector, + ) + + return ( + transport, + resolved_connection, + connector, + ) + + +def test_transport_implements_protocol() -> None: + transport, *_ = create_transport() + + assert isinstance( + transport, + WebSocketTransportProtocol, + ) + assert isinstance( + transport, + RuntimeLivenessProbeProtocol, + ) + + +def test_transport_uses_slots() -> None: + transport, *_ = create_transport() + + assert not hasattr(transport, "__dict__") + + +@pytest.mark.parametrize( + ("raw_url", "expected"), + [ + ( + "https://api-adapter.dzengi.com", + "wss://api-adapter.dzengi.com/connect", + ), + ( + "http://localhost:8080/", + "ws://localhost:8080/connect", + ), + ( + "wss://api-adapter.dzengi.com/connect/", + "wss://api-adapter.dzengi.com/connect", + ), + ( + "ws://localhost:8080/custom?token=abc", + "ws://localhost:8080/custom/connect?token=abc", + ), + ], +) +def test_build_url_normalizes_supported_urls( + raw_url: str, + expected: str, +) -> None: + assert build_dzengi_websocket_url(raw_url) == expected + + +@pytest.mark.parametrize( + "raw_url", + [ + "", + " ", + "ftp://api-adapter.dzengi.com", + "api-adapter.dzengi.com", + "wss:///connect", + "wss://api-adapter.dzengi.com/#fragment", + ], +) +def test_build_url_rejects_invalid_values( + raw_url: str, +) -> None: + with pytest.raises(ValueError): + build_dzengi_websocket_url(raw_url) + + +def test_build_url_rejects_non_string() -> None: + with pytest.raises(TypeError): + build_dzengi_websocket_url(123) # type: ignore[arg-type] + + +def test_connect_forwards_connection_options() -> None: + transport, _, connector = create_transport() + + asyncio.run(transport.connect()) + + assert transport.is_connected is True + assert len(connector.calls) == 1 + + url, options = connector.calls[0] + + assert url == "wss://api-adapter.dzengi.com/connect" + assert options["additional_headers"] == { + "Origin": "https://api-adapter.dzengi.com", + "X-MBX-APIKEY": "test-key", + } + assert tuple(options["subprotocols"]) == ("json",) + assert options["open_timeout"] == 11.0 + assert options["ping_interval"] == 12.0 + assert options["ping_timeout"] == 13.0 + assert options["close_timeout"] == 14.0 + + +def test_connect_is_idempotent() -> None: + transport, _, connector = create_transport() + + async def scenario() -> None: + await transport.connect() + await transport.connect() + + asyncio.run(scenario()) + + assert len(connector.calls) == 1 + + +def test_connect_wraps_connector_error() -> None: + async def broken_connector( + url: str, + **kwargs: Any, + ) -> FakeConnection: + raise OSError("connection failed") + + transport = DzengiWebSocketTransport( + url="wss://api-adapter.dzengi.com/connect", + connector=broken_connector, + ) + + with pytest.raises( + WebSocketTransportError, + match="connection failed", + ) as exc_info: + asyncio.run(transport.connect()) + + assert isinstance(exc_info.value.__cause__, OSError) + assert transport.is_connected is False + + +def test_connect_rejects_non_open_connection() -> None: + connection = FakeConnection( + state=State.CLOSED, + ) + transport, _, _ = create_transport(connection) + + with pytest.raises( + WebSocketTransportError, + match="состояние OPEN", + ): + asyncio.run(transport.connect()) + + assert connection.close_calls == 1 + assert transport.is_connected is False + + +def test_disconnect_closes_connection_and_is_idempotent() -> None: + transport, connection, _ = create_transport() + + async def scenario() -> None: + await transport.connect() + await transport.disconnect() + await transport.disconnect() + + asyncio.run(scenario()) + + assert connection.close_calls == 1 + assert transport.is_connected is False + + +def test_disconnect_clears_connection_after_close_error() -> None: + connection = FakeConnection() + connection.close_error = RuntimeError("close failed") + transport, _, _ = create_transport(connection) + + async def scenario() -> None: + await transport.connect() + await transport.disconnect() + + with pytest.raises( + WebSocketTransportError, + match="close failed", + ): + asyncio.run(scenario()) + + assert transport.is_connected is False + + +def test_send_forwards_text_and_binary_messages() -> None: + transport, connection, _ = create_transport() + + async def scenario() -> None: + await transport.connect() + await transport.send("text") + await transport.send(b"binary") + + asyncio.run(scenario()) + + assert connection.sent_messages == [ + "text", + b"binary", + ] + + +def test_send_requires_open_connection() -> None: + transport, _, _ = create_transport() + + with pytest.raises( + WebSocketTransportNotConnectedError, + ): + asyncio.run(transport.send("message")) + + +def test_send_error_is_wrapped_and_discards_closed_connection() -> None: + connection = FakeConnection() + connection.send_error = RuntimeError("send failed") + transport, _, _ = create_transport(connection) + + async def scenario() -> None: + await transport.connect() + await transport.send("message") + + with pytest.raises( + WebSocketTransportError, + match="send failed", + ): + asyncio.run(scenario()) + + assert transport.is_connected is False + + +def test_receive_returns_text_and_binary_messages() -> None: + connection = FakeConnection( + incoming=( + "text", + b"binary", + ), + ) + transport, _, _ = create_transport(connection) + + async def scenario() -> tuple[str | bytes, str | bytes]: + await transport.connect() + + return ( + await transport.receive(), + await transport.receive(), + ) + + assert asyncio.run(scenario()) == ( + "text", + b"binary", + ) + + +def test_receive_requires_open_connection() -> None: + transport, _, _ = create_transport() + + with pytest.raises( + WebSocketTransportNotConnectedError, + ): + asyncio.run(transport.receive()) + + +def test_receive_error_is_wrapped_and_discards_closed_connection() -> None: + connection = FakeConnection() + connection.receive_error = RuntimeError("receive failed") + transport, _, _ = create_transport(connection) + + async def scenario() -> None: + await transport.connect() + await transport.receive() + + with pytest.raises( + WebSocketTransportError, + match="receive failed", + ): + asyncio.run(scenario()) + + assert transport.is_connected is False + + +def test_probe_returns_true_after_pong() -> None: + transport, connection, _ = create_transport() + + async def scenario() -> bool: + await transport.connect() + return await transport.probe() + + assert asyncio.run(scenario()) is True + assert connection.ping_calls == 1 + + +def test_probe_returns_false_without_open_connection() -> None: + transport, connection, _ = create_transport() + + assert asyncio.run(transport.probe()) is False + assert connection.ping_calls == 0 + + +def test_probe_returns_false_after_pong_timeout() -> None: + connection = FakeConnection() + connection.pong_gate = asyncio.Event() + connector = RecordingConnector(connection) + transport = DzengiWebSocketTransport( + url="wss://api-adapter.dzengi.com", + ping_timeout=0.001, + connector=connector, + ) + + async def scenario() -> bool: + await transport.connect() + return await transport.probe() + + assert asyncio.run(scenario()) is False + assert connection.ping_calls == 1 + assert transport.is_connected is True + + +def test_probe_timeout_remains_finite_when_ping_timeout_is_disabled() -> None: + connection = FakeConnection() + connection.pong_gate = asyncio.Event() + connector = RecordingConnector(connection) + transport = DzengiWebSocketTransport( + url="wss://api-adapter.dzengi.com", + ping_timeout=None, + probe_timeout=0.001, + connector=connector, + ) + + async def scenario() -> bool: + await transport.connect() + return await transport.probe() + + assert asyncio.run(scenario()) is False + assert connection.ping_calls == 1 + assert connector.calls[0][1]["ping_timeout"] is None + + +def test_probe_returns_false_when_connection_closes() -> None: + connection = FakeConnection() + connection.pong_error = RuntimeError("connection closed") + transport, _, _ = create_transport(connection) + + async def scenario() -> bool: + await transport.connect() + connection.state = State.CLOSED + return await transport.probe() + + assert asyncio.run(scenario()) is False + assert transport.is_connected is False + + +def test_probe_wraps_unexpected_ping_error_on_open_connection() -> None: + connection = FakeConnection() + connection.ping_error = RuntimeError("ping failed") + transport, _, _ = create_transport(connection) + + async def scenario() -> None: + await transport.connect() + await transport.probe() + + with pytest.raises( + WebSocketTransportError, + match="ping failed", + ) as error_info: + asyncio.run(scenario()) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert transport.is_connected is True + + +def test_probe_cancellation_is_not_swallowed() -> None: + async def scenario() -> DzengiWebSocketTransport: + connection = FakeConnection() + connection.pong_gate = asyncio.Event() + transport, _, _ = create_transport(connection) + await transport.connect() + + probe_task = asyncio.create_task( + transport.probe(), + ) + + while connection.ping_calls == 0: + await asyncio.sleep(0) + + probe_task.cancel() + + with pytest.raises(asyncio.CancelledError): + await probe_task + + return transport + + assert asyncio.run(scenario()).is_connected is True + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("open_timeout", 0), + ("ping_interval", -1), + ("ping_timeout", True), + ("probe_timeout", float("inf")), + ("close_timeout", "10"), + ], +) +def test_rejects_invalid_timeout( + name: str, + value: object, +) -> None: + options = { + name: value, + } + + with pytest.raises( + (TypeError, ValueError), + ): + DzengiWebSocketTransport( + url="wss://api-adapter.dzengi.com", + **options, # type: ignore[arg-type] + ) + + +def test_copies_headers_from_caller() -> None: + headers = { + "Origin": "https://api-adapter.dzengi.com", + } + connection = FakeConnection() + connector = RecordingConnector(connection) + transport = DzengiWebSocketTransport( + url="wss://api-adapter.dzengi.com", + headers=headers, + connector=connector, + ) + + headers["Origin"] = "changed" + + asyncio.run(transport.connect()) + + assert connector.calls[0][1]["additional_headers"] == { + "Origin": "https://api-adapter.dzengi.com", + } diff --git a/app/tests/unit/market_data/acquisition/runtime/test_acquisition_runtime_event_logging_consumer.py b/app/tests/unit/market_data/acquisition/runtime/test_acquisition_runtime_event_logging_consumer.py new file mode 100644 index 0000000..f5b81ce --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_acquisition_runtime_event_logging_consumer.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +import asyncio +import logging + +import pytest + +from src.market_data.acquisition.runtime.acquisition_runtime_event_logging_consumer import ( + AcquisitionRuntimeEventLoggingConsumer, +) +from src.market_data.acquisition.runtime.acquisition_runtime_event_publisher import ( + AcquisitionRuntimeEventConsumerProtocol, +) +from src.market_data.acquisition.runtime.runtime_events import ( + ConnectedEvent, + ConnectFailedEvent, + DisconnectedEvent, + HeartbeatTimeoutEvent, + MessageReceivedEvent, + MessageSentEvent, + ReconnectCompletedEvent, + ReconnectFailedEvent, + ReconnectStartedEvent, +) +from src.market_data.acquisition.runtime.transport_messages import ( + TransportBinaryMessage, + TransportTextMessage, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionRuntimeEvent, +) + + +TEST_LOGGER_NAME = "tests.acquisition_runtime_events" + + +def create_consumer() -> AcquisitionRuntimeEventLoggingConsumer: + return AcquisitionRuntimeEventLoggingConsumer( + event_logger=logging.getLogger( + TEST_LOGGER_NAME, + ) + ) + + +def test_logging_consumer_satisfies_protocol() -> None: + assert isinstance( + create_consumer(), + AcquisitionRuntimeEventConsumerProtocol, + ) + + +def test_logging_consumer_uses_slots() -> None: + consumer = create_consumer() + + assert not hasattr(consumer, "__dict__") + + +def test_invalid_logger_is_rejected() -> None: + with pytest.raises( + TypeError, + match="logging.Logger", + ): + AcquisitionRuntimeEventLoggingConsumer( + event_logger=object(), # type: ignore[arg-type] + ) + + +@pytest.mark.parametrize( + ( + "event", + "expected_level", + "expected_message", + ), + ( + ( + ConnectedEvent(), + logging.INFO, + "connected", + ), + ( + DisconnectedEvent(), + logging.INFO, + "disconnected", + ), + ( + ConnectFailedEvent( + reason="connection refused", + ), + logging.ERROR, + "connection refused", + ), + ( + ReconnectStartedEvent( + attempt=1, + ), + logging.INFO, + "attempt=1", + ), + ( + ReconnectCompletedEvent( + attempt=2, + ), + logging.INFO, + "attempt=2", + ), + ( + ReconnectFailedEvent( + attempt=3, + reason="timeout", + ), + logging.ERROR, + "attempt=3 reason=timeout", + ), + ( + HeartbeatTimeoutEvent( + timeout_seconds=30.0, + ), + logging.WARNING, + "timeout_seconds=30.0", + ), + ), +) +def test_lifecycle_event_uses_expected_log_level( + caplog: pytest.LogCaptureFixture, + event: AcquisitionRuntimeEvent, + expected_level: int, + expected_message: str, +) -> None: + consumer = create_consumer() + + caplog.set_level( + logging.DEBUG, + logger=TEST_LOGGER_NAME, + ) + + asyncio.run( + consumer.consume(event) + ) + + assert len(caplog.records) == 1 + assert caplog.records[0].levelno == expected_level + assert expected_message in caplog.records[0].getMessage() + + +def test_message_events_log_metadata_without_payload( + caplog: pytest.LogCaptureFixture, +) -> None: + consumer = create_consumer() + text_payload = "secret-subscription-payload" + binary_payload = b"secret-binary-payload" + + caplog.set_level( + logging.DEBUG, + logger=TEST_LOGGER_NAME, + ) + + async def consume_messages() -> None: + await consumer.consume( + MessageReceivedEvent( + message=TransportTextMessage( + payload=text_payload, + ), + ) + ) + await consumer.consume( + MessageSentEvent( + message=TransportBinaryMessage( + payload=binary_payload, + ), + ) + ) + + asyncio.run(consume_messages()) + + assert len(caplog.records) == 2 + assert ( + "message_type=TransportTextMessage " + f"payload_size={len(text_payload)}" + in caplog.records[0].getMessage() + ) + assert ( + "message_type=TransportBinaryMessage " + f"payload_size={len(binary_payload)}" + in caplog.records[1].getMessage() + ) + assert text_payload not in caplog.text + assert binary_payload.decode() not in caplog.text + + +def test_invalid_event_is_rejected() -> None: + consumer = create_consumer() + + with pytest.raises( + TypeError, + match="AcquisitionRuntimeEvent", + ): + asyncio.run( + consumer.consume( + object(), # type: ignore[arg-type] + ) + ) diff --git a/app/tests/unit/market_data/acquisition/runtime/test_acquisition_runtime_event_publisher.py b/app/tests/unit/market_data/acquisition/runtime/test_acquisition_runtime_event_publisher.py new file mode 100644 index 0000000..3bdf950 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_acquisition_runtime_event_publisher.py @@ -0,0 +1,679 @@ +from __future__ import annotations + +import asyncio +import logging + +import pytest + +from src.market_data.acquisition.runtime.acquisition_runtime_event_publisher import ( + AcquisitionRuntimeEventConsumerProtocol, + AcquisitionRuntimeEventPublisher, + logger as publisher_logger, +) +from src.market_data.acquisition.runtime.heartbeat import ( + HeartbeatMonitor, + HeartbeatState, +) +from src.market_data.acquisition.runtime.reconnect import ( + ReconnectCoordinator, + ReconnectState, +) +from src.market_data.acquisition.runtime.runtime_commands import ( + ConnectCommand, + DisconnectCommand, +) +from src.market_data.acquisition.runtime.runtime_events import ( + ConnectedEvent, + DisconnectedEvent, + MessageReceivedEvent, + ReconnectStartedEvent, +) +from src.market_data.acquisition.runtime.transport_messages import ( + TransportTextMessage, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionRuntimeCommand, + AcquisitionRuntimeEvent, + AcquisitionRuntimeEventPublisherProtocol, + AcquisitionSubscriptionMessage, +) + + +PUBLISHER_LOGGER_NAME = ( + "src.market_data.acquisition.runtime." + "acquisition_runtime_event_publisher" +) + + +class RecordingConsumer: + def __init__(self) -> None: + self.events: list[AcquisitionRuntimeEvent] = [] + + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + self.events.append(event) + + +def test_publisher_satisfies_protocol() -> None: + publisher = AcquisitionRuntimeEventPublisher() + + assert isinstance( + publisher, + AcquisitionRuntimeEventPublisherProtocol, + ) + + +def test_consumer_satisfies_protocol() -> None: + assert isinstance( + RecordingConsumer(), + AcquisitionRuntimeEventConsumerProtocol, + ) + + +def test_publisher_uses_slots() -> None: + publisher = AcquisitionRuntimeEventPublisher() + + assert not hasattr(publisher, "__dict__") + + +def test_empty_publisher_accepts_event() -> None: + publisher = AcquisitionRuntimeEventPublisher() + + asyncio.run( + publisher.publish( + ConnectedEvent(), + ) + ) + + +def test_constructor_copies_consumer_collection() -> None: + first_consumer = RecordingConsumer() + consumers = [first_consumer] + publisher = AcquisitionRuntimeEventPublisher(consumers) + + second_consumer = RecordingConsumer() + consumers.append(second_consumer) + + event = ConnectedEvent() + asyncio.run(publisher.publish(event)) + + assert first_consumer.events == [event] + assert second_consumer.events == [] + + +def test_invalid_consumer_is_rejected() -> None: + with pytest.raises( + TypeError, + match="AcquisitionRuntimeEventConsumerProtocol", + ): + AcquisitionRuntimeEventPublisher( + consumers=( + object(), # type: ignore[arg-type] + ) + ) + + +def test_invalid_event_is_rejected() -> None: + publisher = AcquisitionRuntimeEventPublisher() + + with pytest.raises( + TypeError, + match="AcquisitionRuntimeEvent", + ): + asyncio.run( + publisher.publish( + object(), # type: ignore[arg-type] + ) + ) + + +def test_event_is_delivered_to_consumers_in_registration_order() -> None: + calls: list[tuple[str, AcquisitionRuntimeEvent]] = [] + + class NamedConsumer: + def __init__(self, name: str) -> None: + self._name = name + + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + calls.append( + ( + self._name, + event, + ) + ) + + event = ConnectedEvent() + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + NamedConsumer("first"), + NamedConsumer("second"), + ) + ) + + asyncio.run(publisher.publish(event)) + + assert calls == [ + ( + "first", + event, + ), + ( + "second", + event, + ), + ] + assert calls[0][1] is event + assert calls[1][1] is event + + +def test_publish_waits_for_consumer_completion() -> None: + completed = False + + class YieldingConsumer: + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + nonlocal completed + + await asyncio.sleep(0) + completed = True + + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + YieldingConsumer(), + ) + ) + + asyncio.run( + publisher.publish( + ConnectedEvent(), + ) + ) + + assert completed is True + + +def test_concurrent_publications_are_serialized() -> None: + calls: list[tuple[str, int]] = [] + + async def scenario() -> None: + first_started = asyncio.Event() + release_first = asyncio.Event() + + class BlockingConsumer: + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + assert isinstance( + event, + ReconnectStartedEvent, + ) + + calls.append( + ( + "start", + event.attempt, + ) + ) + + if event.attempt == 1: + first_started.set() + await release_first.wait() + + calls.append( + ( + "end", + event.attempt, + ) + ) + + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + BlockingConsumer(), + ) + ) + + first_task = asyncio.create_task( + publisher.publish( + ReconnectStartedEvent( + attempt=1, + ) + ) + ) + + await first_started.wait() + + second_task = asyncio.create_task( + publisher.publish( + ReconnectStartedEvent( + attempt=2, + ) + ) + ) + + await asyncio.sleep(0) + + assert calls == [ + ( + "start", + 1, + ), + ] + + release_first.set() + + await asyncio.gather( + first_task, + second_task, + ) + + asyncio.run(scenario()) + + assert calls == [ + ( + "start", + 1, + ), + ( + "end", + 1, + ), + ( + "start", + 2, + ), + ( + "end", + 2, + ), + ] + + +def test_consumer_error_is_logged_and_delivery_continues( + caplog: pytest.LogCaptureFixture, +) -> None: + class BrokenConsumer: + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + raise RuntimeError( + "sensitive consumer failure detail", + ) + + recording_consumer = RecordingConsumer() + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + BrokenConsumer(), + recording_consumer, + ) + ) + event = ConnectedEvent() + + caplog.set_level( + logging.ERROR, + logger=PUBLISHER_LOGGER_NAME, + ) + + asyncio.run(publisher.publish(event)) + + assert recording_consumer.events == [event] + assert len(caplog.records) == 1 + assert ( + "event=ConnectedEvent consumer=BrokenConsumer " + "error_type=RuntimeError" + in caplog.records[0].getMessage() + ) + assert caplog.records[0].exc_info is None + assert ( + "sensitive consumer failure detail" + not in caplog.text + ) + + +def test_consumer_error_log_does_not_include_transport_payload( + caplog: pytest.LogCaptureFixture, +) -> None: + class PayloadEchoingBrokenConsumer: + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + raise RuntimeError( + f"consumer rejected {event!r}" + ) + + payload = "secret-runtime-payload" + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + PayloadEchoingBrokenConsumer(), + ) + ) + + caplog.set_level( + logging.ERROR, + logger=PUBLISHER_LOGGER_NAME, + ) + + asyncio.run( + publisher.publish( + MessageReceivedEvent( + message=TransportTextMessage( + payload=payload, + ), + ) + ) + ) + + assert payload not in caplog.text + assert "consumer rejected" not in caplog.text + assert "error_type=RuntimeError" in caplog.text + + +def test_logging_handler_error_does_not_escape_or_stop_delivery() -> None: + class BrokenConsumer: + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + raise RuntimeError( + "consumer failed", + ) + + class RaisingHandler(logging.Handler): + def emit( + self, + record: logging.LogRecord, + ) -> None: + raise RuntimeError( + "logging failed", + ) + + handler = RaisingHandler() + recording_consumer = RecordingConsumer() + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + BrokenConsumer(), + recording_consumer, + ) + ) + event = ConnectedEvent() + + publisher_logger.addHandler(handler) + + try: + asyncio.run(publisher.publish(event)) + finally: + publisher_logger.removeHandler(handler) + + assert recording_consumer.events == [event] + + +def test_consumer_cancellation_is_propagated() -> None: + class CancelledConsumer: + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + raise asyncio.CancelledError + + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + CancelledConsumer(), + ) + ) + + with pytest.raises(asyncio.CancelledError): + asyncio.run( + publisher.publish( + ConnectedEvent(), + ) + ) + + +def test_publisher_can_be_reused_after_consumer_cancellation() -> None: + class CancelOnceConsumer: + def __init__(self) -> None: + self.calls = 0 + + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + self.calls += 1 + + if self.calls == 1: + raise asyncio.CancelledError + + cancel_once_consumer = CancelOnceConsumer() + recording_consumer = RecordingConsumer() + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + cancel_once_consumer, + recording_consumer, + ) + ) + second_event = DisconnectedEvent() + + async def scenario() -> None: + with pytest.raises(asyncio.CancelledError): + await publisher.publish( + ConnectedEvent(), + ) + + await publisher.publish(second_event) + + asyncio.run(scenario()) + + assert cancel_once_consumer.calls == 2 + assert recording_consumer.events == [ + second_event, + ] + + +def test_recursive_publication_is_rejected_without_deadlock( + caplog: pytest.LogCaptureFixture, +) -> None: + class RecursiveConsumer: + def __init__(self) -> None: + self.publisher: ( + AcquisitionRuntimeEventPublisher | None + ) = None + + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + assert self.publisher is not None + + await self.publisher.publish( + DisconnectedEvent(), + ) + + recursive_consumer = RecursiveConsumer() + recording_consumer = RecordingConsumer() + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + recursive_consumer, + recording_consumer, + ) + ) + recursive_consumer.publisher = publisher + event = ConnectedEvent() + + caplog.set_level( + logging.ERROR, + logger=PUBLISHER_LOGGER_NAME, + ) + + asyncio.run(publisher.publish(event)) + + assert recording_consumer.events == [event] + assert ( + "event=ConnectedEvent consumer=RecursiveConsumer " + "error_type=RuntimeError" + in caplog.text + ) + + +def test_child_task_recursive_publication_is_rejected() -> None: + class ChildTaskRecursiveConsumer: + def __init__(self) -> None: + self.publisher: ( + AcquisitionRuntimeEventPublisher | None + ) = None + self.rejected = False + + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + assert self.publisher is not None + + if not isinstance(event, ConnectedEvent): + return + + try: + await asyncio.create_task( + self.publisher.publish( + DisconnectedEvent(), + ) + ) + except RuntimeError: + self.rejected = True + + recursive_consumer = ChildTaskRecursiveConsumer() + recording_consumer = RecordingConsumer() + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + recursive_consumer, + recording_consumer, + ) + ) + recursive_consumer.publisher = publisher + event = ConnectedEvent() + + asyncio.run( + asyncio.wait_for( + publisher.publish(event), + timeout=1.0, + ) + ) + + assert recursive_consumer.rejected is True + assert recording_consumer.events == [event] + + +def test_consumer_error_does_not_interrupt_heartbeat_timeout() -> None: + class BrokenConsumer: + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + raise RuntimeError( + "consumer failed", + ) + + current_time = [0.0] + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + BrokenConsumer(), + ) + ) + monitor = HeartbeatMonitor( + event_publisher=publisher, + timeout_seconds=10.0, + clock=lambda: current_time[0], + ) + + monitor.start() + current_time[0] = 10.0 + + result = asyncio.run( + monitor.check_timeout() + ) + + assert result is True + assert monitor.state is HeartbeatState.TIMED_OUT + + +def test_consumer_error_does_not_interrupt_reconnect() -> None: + class BrokenConsumer: + async def consume( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + raise RuntimeError( + "consumer failed", + ) + + class RecordingCommandDispatcher: + def __init__(self) -> None: + self.commands: list[ + AcquisitionRuntimeCommand + ] = [] + + async def dispatch( + self, + command: AcquisitionRuntimeCommand, + ) -> None: + self.commands.append(command) + + class RecordingSubscriptionManager: + def __init__(self) -> None: + self.restore_calls = 0 + + async def subscribe( + self, + subscription_key: str, + message: AcquisitionSubscriptionMessage, + ) -> None: + return None + + async def unsubscribe( + self, + subscription_key: str, + message: AcquisitionSubscriptionMessage, + ) -> None: + return None + + async def restore_subscriptions(self) -> None: + self.restore_calls += 1 + + async def clear_subscriptions(self) -> None: + return None + + dispatcher = RecordingCommandDispatcher() + subscriptions = RecordingSubscriptionManager() + publisher = AcquisitionRuntimeEventPublisher( + consumers=( + BrokenConsumer(), + ) + ) + coordinator = ReconnectCoordinator( + command_dispatcher=dispatcher, + subscription_manager=subscriptions, + event_publisher=publisher, + ) + + asyncio.run( + coordinator.reconnect() + ) + + assert coordinator.state is ReconnectState.CONNECTED + assert coordinator.attempt == 1 + assert [ + type(command) + for command in dispatcher.commands + ] == [ + DisconnectCommand, + ConnectCommand, + ] + assert subscriptions.restore_calls == 1 diff --git a/app/tests/unit/market_data/acquisition/runtime/test_live_processing_gate.py b/app/tests/unit/market_data/acquisition/runtime/test_live_processing_gate.py new file mode 100644 index 0000000..ffdcf7f --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_live_processing_gate.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from src.market_data.acquisition.runtime.live_processing_gate import ( + RuntimeLiveProcessingGate, + RuntimeLiveProcessingGateProtocol, +) + + +def test_implements_protocol_uses_slots_and_starts_open() -> None: + gate = RuntimeLiveProcessingGate() + + assert isinstance( + gate, + RuntimeLiveProcessingGateProtocol, + ) + assert not hasattr(gate, "__dict__") + assert gate.locked is False + assert gate.failed is False + + +def test_serializes_protected_operations() -> None: + async def scenario() -> list[str]: + gate = RuntimeLiveProcessingGate() + order: list[str] = [] + + async def contender() -> None: + async with gate: + order.append("contender") + + async with gate: + order.append("owner") + contender_task = asyncio.create_task( + contender(), + ) + await asyncio.sleep(0) + + assert contender_task.done() is False + assert gate.locked is True + + await contender_task + return order + + assert asyncio.run(scenario()) == [ + "owner", + "contender", + ] + + +def test_releases_gate_after_error() -> None: + async def scenario() -> RuntimeLiveProcessingGate: + gate = RuntimeLiveProcessingGate() + + with pytest.raises( + RuntimeError, + match="protected failure", + ): + async with gate: + raise RuntimeError( + "protected failure", + ) + + return gate + + assert asyncio.run(scenario()).locked is False + + +def test_failed_gate_rejects_waiters_until_reset() -> None: + async def scenario() -> RuntimeLiveProcessingGate: + gate = RuntimeLiveProcessingGate() + failure = RuntimeError("recovery failed") + + gate.fail(failure) + + with pytest.raises( + RuntimeError, + match="recovery failed", + ) as error_info: + async with gate: + raise AssertionError( + "failed gate must not admit live processing" + ) + + assert error_info.value is failure + assert gate.locked is False + assert gate.failed is True + + gate.reset() + + async with gate: + assert gate.locked is True + + return gate + + gate = asyncio.run(scenario()) + + assert gate.locked is False + assert gate.failed is False + + +def test_rejects_reset_while_gate_is_locked() -> None: + async def scenario() -> None: + gate = RuntimeLiveProcessingGate() + + async with gate: + with pytest.raises( + RuntimeError, + match="cannot be reset while locked", + ): + gate.reset() + + asyncio.run(scenario()) diff --git a/app/tests/unit/market_data/acquisition/runtime/test_reconnect_coordinator.py b/app/tests/unit/market_data/acquisition/runtime/test_reconnect_coordinator.py index dd237e7..0c08c1c 100644 --- a/app/tests/unit/market_data/acquisition/runtime/test_reconnect_coordinator.py +++ b/app/tests/unit/market_data/acquisition/runtime/test_reconnect_coordinator.py @@ -14,6 +14,7 @@ from src.market_data.acquisition.runtime.reconnect import ( ) from src.market_data.acquisition.runtime.runtime_commands import ( ConnectCommand, + DisconnectCommand, ) from src.market_data.acquisition.runtime.runtime_events import ( ReconnectCompletedEvent, @@ -115,16 +116,14 @@ def test_initial_state_is_disconnected() -> None: assert coordinator.attempt == 0 -def test_reconnect_dispatches_connect_command() -> None: +def test_reconnect_dispatches_disconnect_then_connect() -> None: coordinator, dispatcher, *_ = create_coordinator() asyncio.run(coordinator.reconnect()) - assert len(dispatcher.commands) == 1 - assert isinstance( - dispatcher.commands[0], - ConnectCommand, - ) + assert len(dispatcher.commands) == 2 + assert isinstance(dispatcher.commands[0], DisconnectCommand) + assert isinstance(dispatcher.commands[1], ConnectCommand) def test_reconnect_restores_subscriptions() -> None: @@ -185,7 +184,9 @@ def test_connect_error_publishes_failed_event() -> None: command: Any, ) -> None: self.commands.append(command) - raise RuntimeError("connection failed") + + if isinstance(command, ConnectCommand): + raise RuntimeError("connection failed") dispatcher = BrokenCommandDispatcher() subscriptions = FakeSubscriptionManager() @@ -218,7 +219,8 @@ def test_connect_error_sets_failed_state() -> None: self, command: Any, ) -> None: - raise RuntimeError("connection failed") + if isinstance(command, ConnectCommand): + raise RuntimeError("connection failed") coordinator = ReconnectCoordinator( command_dispatcher=BrokenCommandDispatcher(), @@ -239,7 +241,8 @@ def test_connect_error_does_not_restore_subscriptions() -> None: self, command: Any, ) -> None: - raise RuntimeError("connection failed") + if isinstance(command, ConnectCommand): + raise RuntimeError("connection failed") subscriptions = FakeSubscriptionManager() diff --git a/app/tests/unit/market_data/acquisition/runtime/test_runtime_reconnect_recovery_coordinator.py b/app/tests/unit/market_data/acquisition/runtime/test_runtime_reconnect_recovery_coordinator.py new file mode 100644 index 0000000..a923cab --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_runtime_reconnect_recovery_coordinator.py @@ -0,0 +1,711 @@ +from __future__ import annotations + +import asyncio +import threading + +import pytest + +from src.market_data.acquisition.recovery.trade_recovery_result import ( + TradeRecoveryResult, +) +from src.market_data.acquisition.runtime.live_processing_gate import ( + RuntimeLiveProcessingGate, +) +from src.market_data.acquisition.runtime.reconnect import ( + ReconnectState, +) +from src.market_data.acquisition.runtime.runtime_reconnect_recovery_coordinator import ( + RuntimeReconnectRecoveryCoordinator, + RuntimeReconnectRecoveryProtocol, +) + + +BTC = "BTC/USD_LEVERAGE" +ETH = "ETH/USD_LEVERAGE" +RECOVERY_END_TIME_MS = 1_785_326_405_123 + + +class FakeReconnectCoordinator: + def __init__( + self, + *, + order: list[str], + gate: RuntimeLiveProcessingGate, + error: Exception | None = None, + entered: asyncio.Event | None = None, + release: asyncio.Event | None = None, + ) -> None: + self._order = order + self._gate = gate + self._error = error + self._entered = entered + self._release = release + self._state = ReconnectState.DISCONNECTED + self._attempt = 0 + + @property + def state(self) -> ReconnectState: + return self._state + + @property + def attempt(self) -> int: + return self._attempt + + async def reconnect(self) -> None: + assert self._gate.locked is True + + self._attempt += 1 + self._state = ReconnectState.CONNECTING + self._order.append("reconnect") + + if self._entered is not None: + self._entered.set() + + if self._release is not None: + await self._release.wait() + + if self._error is not None: + self._state = ReconnectState.FAILED + raise self._error + + self._state = ReconnectState.CONNECTED + + +class FakeRecoveryCoordinator: + def __init__( + self, + *, + order: list[str], + gate: RuntimeLiveProcessingGate, + error: Exception | None = None, + started: threading.Event | None = None, + release: threading.Event | None = None, + ) -> None: + self._order = order + self._gate = gate + self._error = error + self._started = started + self._release = release + self.calls: list[tuple[str, int]] = [] + self.thread_ids: list[int] = [] + + def recover( + self, + *, + symbol: str, + recovery_end_time: int, + ) -> TradeRecoveryResult: + assert self._gate.locked is True + + self.calls.append( + ( + symbol, + recovery_end_time, + ) + ) + self.thread_ids.append( + threading.get_ident(), + ) + self._order.append( + f"recover:{symbol}", + ) + + if self._started is not None: + self._started.set() + + if self._release is not None: + if not self._release.wait(timeout=2.0): + raise AssertionError( + "recovery release was not signalled" + ) + + if self._error is not None: + raise self._error + + return TradeRecoveryResult( + symbol=symbol, + requested_start_time=recovery_end_time, + requested_end_time=recovery_end_time, + recovered_trades=(), + ) + + +class RecordingClock: + def __init__( + self, + *, + order: list[str], + value: object = RECOVERY_END_TIME_MS, + ) -> None: + self._order = order + self._value = value + self.calls = 0 + + def __call__(self) -> int: + self.calls += 1 + self._order.append("clock") + return self._value # type: ignore[return-value] + + +def create_coordinator( + *, + symbols: tuple[str, ...] = (BTC,), + reconnect_error: Exception | None = None, + recovery_error: Exception | None = None, + reconnect_entered: asyncio.Event | None = None, + reconnect_release: asyncio.Event | None = None, + recovery_started: threading.Event | None = None, + recovery_release: threading.Event | None = None, + clock_value: object = RECOVERY_END_TIME_MS, +) -> tuple[ + RuntimeReconnectRecoveryCoordinator, + RuntimeLiveProcessingGate, + FakeReconnectCoordinator, + FakeRecoveryCoordinator, + RecordingClock, + list[str], +]: + order: list[str] = [] + gate = RuntimeLiveProcessingGate() + reconnect = FakeReconnectCoordinator( + order=order, + gate=gate, + error=reconnect_error, + entered=reconnect_entered, + release=reconnect_release, + ) + recovery = FakeRecoveryCoordinator( + order=order, + gate=gate, + error=recovery_error, + started=recovery_started, + release=recovery_release, + ) + clock = RecordingClock( + order=order, + value=clock_value, + ) + coordinator = RuntimeReconnectRecoveryCoordinator( + reconnect_coordinator=reconnect, + recovery_coordinator=recovery, + live_processing_gate=gate, + symbols=symbols, + clock=clock, + ) + + return ( + coordinator, + gate, + reconnect, + recovery, + clock, + order, + ) + + +async def wait_until( + predicate: object, +) -> None: + for _ in range(100): + if callable(predicate) and predicate(): + return + + await asyncio.sleep(0) + + raise AssertionError("condition was not reached") + + +def test_implements_protocol_uses_slots_and_delegates_state() -> None: + coordinator, _, reconnect, *_ = create_coordinator() + + assert isinstance( + coordinator, + RuntimeReconnectRecoveryProtocol, + ) + assert not hasattr(coordinator, "__dict__") + assert coordinator.state is ReconnectState.DISCONNECTED + assert coordinator.attempt == 0 + assert coordinator.generation == 0 + assert coordinator.live_processing_gate is coordinator._live_processing_gate + assert coordinator.symbols == (BTC,) + + asyncio.run(coordinator.reconnect()) + + assert coordinator.state is reconnect.state + assert coordinator.state is ReconnectState.CONNECTED + assert coordinator.attempt == 1 + assert coordinator.generation == 1 + + +@pytest.mark.parametrize( + ("symbols", "error_type"), + [ + ([], TypeError), + ((), ValueError), + (("", " "), ValueError), + ((BTC, 1), TypeError), + ], +) +def test_rejects_invalid_symbols( + symbols: object, + error_type: type[Exception], +) -> None: + with pytest.raises(error_type): + create_coordinator( + symbols=symbols, # type: ignore[arg-type] + ) + + +def test_reconnect_restore_boundary_and_recovery_order() -> None: + ( + coordinator, + gate, + _, + recovery, + clock, + order, + ) = create_coordinator( + symbols=( + f" {ETH} ", + BTC, + ETH, + ), + ) + main_thread_id = threading.get_ident() + + asyncio.run(coordinator.reconnect()) + + assert order == [ + "reconnect", + "clock", + f"recover:{BTC}", + f"recover:{ETH}", + ] + assert recovery.calls == [ + ( + BTC, + RECOVERY_END_TIME_MS, + ), + ( + ETH, + RECOVERY_END_TIME_MS, + ), + ] + assert clock.calls == 1 + assert all( + thread_id != main_thread_id + for thread_id in recovery.thread_ids + ) + assert gate.locked is False + + +def test_live_processing_waits_until_recovery_finishes() -> None: + async def scenario() -> list[str]: + recovery_started = threading.Event() + recovery_release = threading.Event() + ( + coordinator, + gate, + *_, + ) = create_coordinator( + recovery_started=recovery_started, + recovery_release=recovery_release, + ) + order: list[str] = [] + + coordinator_task = asyncio.create_task( + coordinator.reconnect(), + ) + await wait_until( + recovery_started.is_set, + ) + + async def process_live() -> None: + async with gate: + order.append("live") + + live_task = asyncio.create_task( + process_live(), + ) + await asyncio.sleep(0) + + assert live_task.done() is False + assert gate.locked is True + + recovery_release.set() + await coordinator_task + await live_task + + return order + + assert asyncio.run(scenario()) == [ + "live", + ] + + +def test_concurrent_callers_share_one_operation() -> None: + async def scenario() -> tuple[ + RuntimeReconnectRecoveryCoordinator, + FakeReconnectCoordinator, + FakeRecoveryCoordinator, + ]: + reconnect_entered = asyncio.Event() + reconnect_release = asyncio.Event() + ( + coordinator, + _, + reconnect, + recovery, + *_, + ) = create_coordinator( + reconnect_entered=reconnect_entered, + reconnect_release=reconnect_release, + ) + + first = asyncio.create_task( + coordinator.reconnect(), + ) + await reconnect_entered.wait() + + second = asyncio.create_task( + coordinator.reconnect(), + ) + await asyncio.sleep(0) + + assert reconnect.attempt == 1 + + reconnect_release.set() + await asyncio.gather( + first, + second, + ) + + return ( + coordinator, + reconnect, + recovery, + ) + + coordinator, reconnect, recovery = asyncio.run( + scenario() + ) + + assert reconnect.attempt == 1 + assert len(recovery.calls) == 1 + assert coordinator.generation == 1 + + +def test_timeout_and_transport_error_share_one_operation() -> None: + async def scenario() -> tuple[ + RuntimeReconnectRecoveryCoordinator, + FakeReconnectCoordinator, + FakeRecoveryCoordinator, + ]: + reconnect_entered = asyncio.Event() + reconnect_release = asyncio.Event() + ( + coordinator, + _, + reconnect, + recovery, + *_, + ) = create_coordinator( + reconnect_entered=reconnect_entered, + reconnect_release=reconnect_release, + ) + observed_generation = coordinator.generation + + timeout_task = asyncio.create_task( + coordinator.reconnect(), + ) + await reconnect_entered.wait() + + transport_task = asyncio.create_task( + coordinator.reconnect_after_transport_failure( + observed_generation=observed_generation, + ), + ) + await asyncio.sleep(0) + + assert reconnect.attempt == 1 + reconnect_release.set() + + await asyncio.gather( + timeout_task, + transport_task, + ) + + return ( + coordinator, + reconnect, + recovery, + ) + + coordinator, reconnect, recovery = asyncio.run( + scenario() + ) + + assert reconnect.attempt == 1 + assert len(recovery.calls) == 1 + assert coordinator.generation == 1 + + +def test_stale_transport_error_reuses_completed_operation() -> None: + ( + coordinator, + _, + reconnect, + recovery, + *_, + ) = create_coordinator() + + observed_generation = coordinator.generation + + asyncio.run(coordinator.reconnect()) + asyncio.run( + coordinator.reconnect_after_transport_failure( + observed_generation=observed_generation, + ) + ) + + assert reconnect.attempt == 1 + assert len(recovery.calls) == 1 + + +def test_reconnect_error_skips_clock_and_recovery() -> None: + reconnect_error = RuntimeError("reconnect failed") + ( + coordinator, + gate, + _, + recovery, + clock, + _, + ) = create_coordinator( + reconnect_error=reconnect_error, + ) + + with pytest.raises( + RuntimeError, + match="reconnect failed", + ) as error_info: + asyncio.run(coordinator.reconnect()) + + assert error_info.value is reconnect_error + assert recovery.calls == [] + assert clock.calls == 0 + assert gate.locked is False + assert gate.failed is True + + +def test_recovery_error_is_not_wrapped() -> None: + recovery_error = RuntimeError("recovery failed") + ( + coordinator, + gate, + *_, + ) = create_coordinator( + recovery_error=recovery_error, + ) + + with pytest.raises( + RuntimeError, + match="recovery failed", + ) as error_info: + asyncio.run(coordinator.reconnect()) + + assert error_info.value is recovery_error + assert gate.locked is False + assert gate.failed is True + assert coordinator._recovery_task is None + + +def test_recovery_error_rejects_buffered_live_processing() -> None: + async def scenario() -> tuple[ + RuntimeLiveProcessingGate, + list[str], + ]: + recovery_error = RuntimeError("recovery failed") + recovery_started = threading.Event() + recovery_release = threading.Event() + ( + coordinator, + gate, + *_, + ) = create_coordinator( + recovery_error=recovery_error, + recovery_started=recovery_started, + recovery_release=recovery_release, + ) + processed: list[str] = [] + coordinator_task = asyncio.create_task( + coordinator.reconnect(), + ) + await wait_until( + recovery_started.is_set, + ) + + async def process_live() -> None: + async with gate: + processed.append("live") + + live_task = asyncio.create_task( + process_live(), + ) + await asyncio.sleep(0) + + assert live_task.done() is False + recovery_release.set() + + with pytest.raises( + RuntimeError, + match="recovery failed", + ): + await coordinator_task + + with pytest.raises( + RuntimeError, + match="recovery failed", + ): + await live_task + + return ( + gate, + processed, + ) + + gate, processed = asyncio.run(scenario()) + + assert processed == [] + assert gate.locked is False + assert gate.failed is True + + +@pytest.mark.parametrize( + ("clock_value", "error_type"), + [ + (True, TypeError), + (1.5, TypeError), + (-1, ValueError), + ], +) +def test_rejects_invalid_clock_result( + clock_value: object, + error_type: type[Exception], +) -> None: + coordinator, gate, *_ = create_coordinator( + clock_value=clock_value, + ) + + with pytest.raises(error_type): + asyncio.run(coordinator.reconnect()) + + assert gate.locked is False + + +def test_cancellation_waits_for_worker_before_opening_gate() -> None: + async def scenario() -> tuple[ + RuntimeReconnectRecoveryCoordinator, + RuntimeLiveProcessingGate, + ]: + recovery_started = threading.Event() + recovery_release = threading.Event() + ( + coordinator, + gate, + *_, + ) = create_coordinator( + recovery_started=recovery_started, + recovery_release=recovery_release, + ) + coordinator_task = asyncio.create_task( + coordinator.reconnect(), + ) + await wait_until( + recovery_started.is_set, + ) + + coordinator_task.cancel() + await asyncio.sleep(0) + + assert coordinator_task.done() is False + assert gate.locked is True + assert coordinator._recovery_task is not None + + recovery_release.set() + + with pytest.raises(asyncio.CancelledError): + await coordinator_task + + return ( + coordinator, + gate, + ) + + coordinator, gate = asyncio.run(scenario()) + + assert gate.locked is False + assert coordinator._recovery_task is None + + +def test_repeated_cancellation_waits_for_worker_before_opening_gate() -> None: + async def scenario() -> tuple[ + RuntimeReconnectRecoveryCoordinator, + RuntimeLiveProcessingGate, + list[str], + ]: + recovery_started = threading.Event() + recovery_release = threading.Event() + ( + coordinator, + gate, + *_, + ) = create_coordinator( + recovery_started=recovery_started, + recovery_release=recovery_release, + ) + processed: list[str] = [] + coordinator_task = asyncio.create_task( + coordinator.reconnect(), + ) + await wait_until( + recovery_started.is_set, + ) + + async def process_live() -> None: + async with gate: + processed.append("live") + + live_task = asyncio.create_task( + process_live(), + ) + + coordinator_task.cancel() + await asyncio.sleep(0) + coordinator_task.cancel() + await asyncio.sleep(0) + + assert coordinator_task.done() is False + assert live_task.done() is False + assert gate.locked is True + assert coordinator._recovery_task is not None + + recovery_release.set() + + with pytest.raises(asyncio.CancelledError): + await coordinator_task + + await live_task + + return ( + coordinator, + gate, + processed, + ) + + coordinator, gate, processed = asyncio.run(scenario()) + + assert processed == ["live"] + assert gate.locked is False + assert gate.failed is False + assert coordinator._recovery_task is None diff --git a/app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py b/app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py index a349019..60763d4 100644 --- a/app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py +++ b/app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py @@ -184,7 +184,9 @@ class FakeStateStore: self._states.clear() -class RecordingWindowPlanner: +class RecordingWindowPlanner( + TradeRecoveryWindowPlanner, +): """ Planner с заранее заданным результатом. """ @@ -290,7 +292,7 @@ def create_coordinator( coordinator = RuntimeRecoveryCoordinator( state_store=resolved_state_store, - window_planner=resolved_window_planner, # type: ignore[arg-type] + window_planner=resolved_window_planner, recovery_controller=resolved_recovery_controller, ) @@ -924,15 +926,8 @@ def test_controller_error_is_not_swallowed() -> None: def test_planner_error_is_not_swallowed() -> None: class BrokenPlanner( - TradeRecoveryWindowPlanner, + RecordingWindowPlanner, ): - def __init__(self) -> None: - pass - - @property - def max_window_ms(self) -> int: - return 1 - def build_windows( self, *, diff --git a/app/tests/unit/market_data/acquisition/runtime/test_runtime_scheduler.py b/app/tests/unit/market_data/acquisition/runtime/test_runtime_scheduler.py index 3a44853..c255a5e 100644 --- a/app/tests/unit/market_data/acquisition/runtime/test_runtime_scheduler.py +++ b/app/tests/unit/market_data/acquisition/runtime/test_runtime_scheduler.py @@ -11,6 +11,9 @@ import pytest from src.market_data.acquisition.runtime.heartbeat import ( HeartbeatState, ) +from src.market_data.acquisition.runtime.runtime_liveness_probe import ( + RuntimeLivenessProbeProtocol, +) from src.market_data.acquisition.runtime.scheduler import ( RuntimeScheduler, RuntimeSchedulerProtocol, @@ -58,9 +61,37 @@ class FakeHeartbeatMonitor: return self._results.pop(0) +class FakeLivenessProbe: + def __init__( + self, + results: list[object] | None = None, + *, + error: BaseException | None = None, + ) -> None: + self._results = list( + results + if results is not None + else [True] + ) + self._error = error + self.calls = 0 + + async def probe(self) -> bool: + self.calls += 1 + + if self._error is not None: + raise self._error + + if not self._results: + return True + + return self._results.pop(0) # type: ignore[return-value] + + class FakeRuntimeSupervisor: def __init__(self) -> None: self.handle_timeout_calls = 0 + self.notify_activity_calls = 0 @property def state(self) -> RuntimeSupervisorState: @@ -73,7 +104,7 @@ class FakeRuntimeSupervisor: return None def notify_activity(self) -> None: - return None + self.notify_activity_calls += 1 async def handle_heartbeat_timeout(self) -> bool: self.handle_timeout_calls += 1 @@ -101,6 +132,7 @@ class RecordingSleep: def create_scheduler( *, + liveness_results: list[object] | None = None, heartbeat_results: list[bool] | None = None, interval_seconds: float = 1.0, sleep: Callable[[float], Awaitable[None]] | None = None, @@ -115,6 +147,9 @@ def create_scheduler( supervisor = FakeRuntimeSupervisor() scheduler = RuntimeScheduler( + liveness_probe=FakeLivenessProbe( + results=liveness_results, + ), heartbeat_monitor=heartbeat, runtime_supervisor=supervisor, interval_seconds=interval_seconds, @@ -131,6 +166,10 @@ def create_scheduler( def test_scheduler_implements_protocol() -> None: scheduler, *_ = create_scheduler() + assert isinstance( + scheduler._liveness_probe, + RuntimeLivenessProbeProtocol, + ) assert isinstance( scheduler, RuntimeSchedulerProtocol, @@ -149,6 +188,71 @@ def test_initial_state_is_not_running() -> None: assert scheduler.running is False +def test_claim_blocks_start_without_matching_owner() -> None: + scheduler, *_ = create_scheduler() + owner = object() + scheduler.claim(owner) + + with pytest.raises( + RuntimeError, + match="owned by another lifecycle", + ): + asyncio.run( + scheduler.start() + ) + + scheduler.release(owner) + + +def test_claim_rejects_active_scheduler() -> None: + scheduler, *_ = create_scheduler() + scheduler._running = True + + with pytest.raises( + RuntimeError, + match="already owned or active", + ): + scheduler.claim(object()) + + +def test_release_rejects_running_owned_scheduler() -> None: + scheduler, *_ = create_scheduler() + owner = object() + scheduler.claim(owner) + scheduler._running = True + + with pytest.raises( + RuntimeError, + match="Cannot release", + ): + scheduler.release(owner) + + +def test_matching_owner_can_run_and_release_scheduler() -> None: + scheduler_holder: list[RuntimeScheduler] = [] + + async def stop_after_first_iteration( + seconds: float, + ) -> None: + scheduler_holder[0].stop() + + scheduler, *_ = create_scheduler( + sleep=stop_after_first_iteration, + ) + scheduler_holder.append(scheduler) + owner = object() + scheduler.claim(owner) + + asyncio.run( + scheduler.start( + owner=owner, + ) + ) + scheduler.release(owner) + + assert scheduler.running is False + + def test_exposes_interval_seconds() -> None: scheduler, *_ = create_scheduler( interval_seconds=2.5, @@ -187,6 +291,7 @@ def test_rejects_invalid_interval_type( ) -> None: with pytest.raises(TypeError): RuntimeScheduler( + liveness_probe=FakeLivenessProbe(), heartbeat_monitor=FakeHeartbeatMonitor(), runtime_supervisor=FakeRuntimeSupervisor(), interval_seconds=interval_seconds, # type: ignore[arg-type] @@ -206,6 +311,7 @@ def test_rejects_non_positive_interval( ) -> None: with pytest.raises(ValueError): RuntimeScheduler( + liveness_probe=FakeLivenessProbe(), heartbeat_monitor=FakeHeartbeatMonitor(), runtime_supervisor=FakeRuntimeSupervisor(), interval_seconds=interval_seconds, @@ -215,6 +321,7 @@ def test_rejects_non_positive_interval( def test_rejects_non_callable_sleep() -> None: with pytest.raises(TypeError): RuntimeScheduler( + liveness_probe=FakeLivenessProbe(), heartbeat_monitor=FakeHeartbeatMonitor(), runtime_supervisor=FakeRuntimeSupervisor(), interval_seconds=1.0, @@ -233,9 +340,43 @@ def test_run_once_checks_heartbeat() -> None: assert result is False assert heartbeat.check_timeout_calls == 1 + assert supervisor.notify_activity_calls == 1 assert supervisor.handle_timeout_calls == 0 +def test_failed_probe_does_not_record_activity() -> None: + scheduler, heartbeat, supervisor = create_scheduler( + liveness_results=[False], + heartbeat_results=[False], + ) + + result = asyncio.run( + scheduler.run_once() + ) + + assert result is False + assert heartbeat.check_timeout_calls == 1 + assert supervisor.notify_activity_calls == 0 + assert supervisor.handle_timeout_calls == 0 + + +def test_rejects_non_boolean_probe_result() -> None: + scheduler, heartbeat, supervisor = create_scheduler( + liveness_results=["alive"], + ) + + with pytest.raises( + TypeError, + match="must return a boolean", + ): + asyncio.run( + scheduler.run_once() + ) + + assert heartbeat.check_timeout_calls == 0 + assert supervisor.notify_activity_calls == 0 + + def test_run_once_calls_supervisor_on_timeout() -> None: scheduler, heartbeat, supervisor = create_scheduler( heartbeat_results=[True], @@ -402,6 +543,7 @@ def test_stop_during_run_once_prevents_sleep() -> None: heartbeat = StoppingHeartbeatMonitor() scheduler = RuntimeScheduler( + liveness_probe=FakeLivenessProbe(), heartbeat_monitor=heartbeat, runtime_supervisor=supervisor, interval_seconds=1.0, @@ -438,6 +580,7 @@ def test_repeated_start_while_running_does_not_create_second_loop() -> None: scheduler.stop() scheduler = RuntimeScheduler( + liveness_probe=FakeLivenessProbe(), heartbeat_monitor=heartbeat, runtime_supervisor=supervisor, interval_seconds=1.0, @@ -491,6 +634,7 @@ def test_heartbeat_error_is_propagated() -> None: raise RuntimeError("heartbeat failed") scheduler = RuntimeScheduler( + liveness_probe=FakeLivenessProbe(), heartbeat_monitor=BrokenHeartbeatMonitor(), runtime_supervisor=FakeRuntimeSupervisor(), interval_seconds=1.0, @@ -511,6 +655,7 @@ def test_supervisor_error_is_propagated() -> None: raise RuntimeError("supervisor failed") scheduler = RuntimeScheduler( + liveness_probe=FakeLivenessProbe(), heartbeat_monitor=FakeHeartbeatMonitor( results=[True], ), @@ -533,6 +678,7 @@ def test_start_resets_running_after_heartbeat_error() -> None: raise RuntimeError("heartbeat failed") scheduler = RuntimeScheduler( + liveness_probe=FakeLivenessProbe(), heartbeat_monitor=BrokenHeartbeatMonitor(), runtime_supervisor=FakeRuntimeSupervisor(), interval_seconds=1.0, @@ -552,6 +698,7 @@ def test_start_resets_running_after_supervisor_error() -> None: raise RuntimeError("supervisor failed") scheduler = RuntimeScheduler( + liveness_probe=FakeLivenessProbe(), heartbeat_monitor=FakeHeartbeatMonitor( results=[True], ), @@ -588,3 +735,45 @@ def test_sleep_error_is_propagated_and_resets_running() -> None: assert heartbeat.check_timeout_calls == 1 assert scheduler.running is False + + +def test_liveness_error_is_propagated_and_resets_running() -> None: + liveness_error = RuntimeError("liveness failed") + scheduler = RuntimeScheduler( + liveness_probe=FakeLivenessProbe( + error=liveness_error, + ), + heartbeat_monitor=FakeHeartbeatMonitor(), + runtime_supervisor=FakeRuntimeSupervisor(), + interval_seconds=1.0, + ) + + with pytest.raises( + RuntimeError, + match="liveness failed", + ) as error_info: + asyncio.run( + scheduler.start() + ) + + assert error_info.value is liveness_error + assert scheduler.running is False + + +def test_liveness_cancellation_is_propagated() -> None: + cancellation = asyncio.CancelledError() + scheduler = RuntimeScheduler( + liveness_probe=FakeLivenessProbe( + error=cancellation, + ), + heartbeat_monitor=FakeHeartbeatMonitor(), + runtime_supervisor=FakeRuntimeSupervisor(), + interval_seconds=1.0, + ) + + with pytest.raises(asyncio.CancelledError): + asyncio.run( + scheduler.start() + ) + + assert scheduler.running is False diff --git a/app/tests/unit/market_data/acquisition/runtime/test_runtime_supervisor.py b/app/tests/unit/market_data/acquisition/runtime/test_runtime_supervisor.py index 7f4143d..eb3651d 100644 --- a/app/tests/unit/market_data/acquisition/runtime/test_runtime_supervisor.py +++ b/app/tests/unit/market_data/acquisition/runtime/test_runtime_supervisor.py @@ -65,6 +65,7 @@ class FakeReconnectCoordinator: def __init__(self) -> None: self.reconnect_calls = 0 self._attempt = 0 + self._generation = 0 self._state = ReconnectState.DISCONNECTED @property @@ -75,11 +76,26 @@ class FakeReconnectCoordinator: def attempt(self) -> int: return self._attempt + @property + def generation(self) -> int: + return self._generation + async def reconnect(self) -> None: self.reconnect_calls += 1 self._attempt += 1 + self._generation += 1 self._state = ReconnectState.CONNECTED + async def reconnect_after_transport_failure( + self, + *, + observed_generation: int, + ) -> None: + if observed_generation != self._generation: + return + + await self.reconnect() + def create_supervisor() -> tuple[ RuntimeSupervisor, @@ -271,6 +287,41 @@ def test_timeout_runs_single_reconnect_attempt() -> None: assert reconnect.reconnect_calls == 1 +def test_delayed_timeout_reuses_completed_transport_reconnect() -> None: + async def scenario() -> tuple[ + RuntimeSupervisor, + FakeHeartbeatMonitor, + FakeReconnectCoordinator, + bool, + ]: + supervisor, heartbeat, reconnect = create_supervisor() + supervisor.start() + observed_generation = reconnect.generation + + await reconnect.reconnect_after_transport_failure( + observed_generation=observed_generation, + ) + result = await supervisor.handle_heartbeat_timeout() + + return ( + supervisor, + heartbeat, + reconnect, + result, + ) + + supervisor, heartbeat, reconnect, result = asyncio.run( + scenario() + ) + + assert result is True + assert reconnect.reconnect_calls == 1 + assert reconnect.generation == 1 + assert heartbeat.stop_calls == 1 + assert heartbeat.start_calls == 2 + assert supervisor.state is RuntimeSupervisorState.RUNNING + + def test_successful_reconnect_restarts_heartbeat() -> None: supervisor, heartbeat, _ = create_supervisor() diff --git a/app/tests/unit/market_data/acquisition/runtime/test_trade_stream_production_runtime.py b/app/tests/unit/market_data/acquisition/runtime/test_trade_stream_production_runtime.py new file mode 100644 index 0000000..0f960c1 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_trade_stream_production_runtime.py @@ -0,0 +1,2039 @@ +from __future__ import annotations + +import asyncio +import json +from collections import deque +from collections.abc import Awaitable, Callable +from typing import cast + +import pytest + +from src.market_data.acquisition.adapters.dzengi.websocket import ( + DzengiUnifiedWebSocketAdapter, +) +from src.market_data.acquisition.adapters.dzengi.websocket_control_message_handler import ( + DzengiWebSocketControlMessageHandler, +) +from src.market_data.acquisition.adapters.dzengi.websocket_inbound_message_classifier import ( + DzengiWebSocketInboundMessageClassifier, +) +from src.market_data.acquisition.consistency.trade_stream_consistency_controller import ( + TradeStreamConsistencyController, +) +from src.market_data.acquisition.consistency.trade_stream_state_store import ( + TradeStreamStateStore, +) +from src.market_data.acquisition.exceptions import ( + WebSocketControlMessageError, + WebSocketMessageDecodeError, + WebSocketMessageRoutingError, + WebSocketTransportError, +) +from src.market_data.acquisition.models.trade import Trade +from src.market_data.acquisition.runtime.runtime_events import ( + ConnectedEvent, + ConnectFailedEvent, + DisconnectedEvent, + MessageReceivedEvent, +) +from src.market_data.acquisition.runtime.live_processing_gate import ( + RuntimeLiveProcessingGate, +) +from src.market_data.acquisition.runtime.reconnect import ( + ReconnectState, +) +from src.market_data.acquisition.runtime.runtime_liveness_probe import ( + RuntimeLivenessProbeProtocol, +) +from src.market_data.acquisition.runtime.supervisor import ( + RuntimeSupervisorState, +) +from src.market_data.acquisition.runtime.trade_stream_production_runtime import ( + TradeStreamProductionRuntime, + TradeStreamProductionRuntimeProtocol, + TradeStreamProductionRuntimeState, +) +from src.market_data.acquisition.runtime.transport_messages import ( + TransportBinaryMessage, + TransportTextMessage, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionRuntimeCommand, + AcquisitionRuntimeEvent, + AcquisitionSubscriptionMessage, +) +from src.market_data.acquisition.trade_stream_acquisition_service import ( + TradeStreamAcquisitionService, +) + + +SYMBOL = "BTC/USD_LEVERAGE" +ETH_SYMBOL = "ETH/USD_LEVERAGE" +MARKET_DOCUMENT = { + "destination": "internal.trade", + "payload": { + "symbol": SYMBOL, + }, +} +MARKET_MESSAGE = ( + '{"destination":"internal.trade",' + f'"payload":{{"symbol":"{SYMBOL}"}}' + "}" +) + + +def make_control_message( + *, + correlation_id: str, + status: str = "OK", +) -> str: + return json.dumps( + { + "correlationId": correlation_id, + "destination": "trades.subscribe", + "status": status, + }, + separators=(",", ":"), + ) + + +class FakeSession: + def __init__( + self, + *, + calls: list[str], + start_error: Exception | None = None, + stop_error: Exception | None = None, + start_gate: asyncio.Event | None = None, + ) -> None: + self._calls = calls + self._start_error = start_error + self._stop_error = stop_error + self._start_gate = start_gate + self._connected = False + self.start_entered = asyncio.Event() + self.start_calls = 0 + self.stop_calls = 0 + + @property + def is_connected(self) -> bool: + return self._connected + + async def start(self) -> None: + self.start_calls += 1 + self._calls.append("session.start") + self.start_entered.set() + + if self._start_gate is not None: + await self._start_gate.wait() + + if self._start_error is not None: + raise self._start_error + + self._connected = True + + async def stop(self) -> None: + self.stop_calls += 1 + self._calls.append("session.stop") + self._connected = False + + if self._stop_error is not None: + raise self._stop_error + + +class FakeTransport: + def __init__( + self, + *, + calls: list[str], + incoming: tuple[str | bytes | BaseException, ...] = (), + ) -> None: + self._calls = calls + self._incoming = deque(incoming) + self._message_available = asyncio.Event() + self.receive_calls = 0 + self.probe_calls = 0 + self.waiting = asyncio.Event() + + async def connect(self) -> None: + raise AssertionError("Session owns connect") + + async def disconnect(self) -> None: + raise AssertionError("Session owns disconnect") + + async def send( + self, + message: str | bytes, + ) -> None: + raise AssertionError("Subscription Service owns send") + + async def receive(self) -> str | bytes: + self.receive_calls += 1 + self._calls.append("transport.receive") + + try: + while not self._incoming: + self.waiting.set() + await self._message_available.wait() + self._message_available.clear() + except asyncio.CancelledError: + self._calls.append( + "transport.receive.cancelled", + ) + raise + + result = self._incoming.popleft() + + if isinstance(result, BaseException): + raise result + + return result + + def feed( + self, + message: str | bytes | BaseException, + ) -> None: + self._incoming.append(message) + self._message_available.set() + + async def probe(self) -> bool: + self.probe_calls += 1 + return True + + +class FakeSubscriptionManager: + def __init__( + self, + *, + calls: list[str], + clear_error: Exception | None = None, + ) -> None: + self._calls = calls + self._clear_error = clear_error + self.clear_calls = 0 + + async def subscribe( + self, + subscription_key: str, + message: AcquisitionSubscriptionMessage, + ) -> None: + raise AssertionError("Trade Stream Service owns subscribe") + + async def unsubscribe( + self, + subscription_key: str, + message: AcquisitionSubscriptionMessage, + ) -> None: + raise AssertionError("Runtime does not unsubscribe") + + async def restore_subscriptions(self) -> None: + raise AssertionError("Reconnect owns restore") + + async def clear_subscriptions(self) -> None: + self.clear_calls += 1 + self._calls.append("subscriptions.clear") + + if self._clear_error is not None: + raise self._clear_error + + +class RecordingEventPublisher: + def __init__( + self, + *, + calls: list[str], + connected_gate: asyncio.Event | None = None, + ) -> None: + self._calls = calls + self._connected_gate = connected_gate + self.events: list[AcquisitionRuntimeEvent] = [] + self.connected_entered = asyncio.Event() + + async def publish( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + self.events.append(event) + self._calls.append( + f"event.{type(event).__name__}" + ) + + if isinstance(event, ConnectedEvent): + self.connected_entered.set() + + if self._connected_gate is not None: + await self._connected_gate.wait() + + +class RecordingRuntimeCommandDispatcher: + def __init__(self) -> None: + self.commands: list[AcquisitionRuntimeCommand] = [] + + async def dispatch( + self, + command: AcquisitionRuntimeCommand, + ) -> None: + self.commands.append(command) + + +class FakeReconnectRecoveryCoordinator: + def __init__( + self, + *, + live_processing_gate: RuntimeLiveProcessingGate, + error: Exception | None = None, + entered: asyncio.Event | None = None, + release: asyncio.Event | None = None, + symbols: object = (SYMBOL,), + ) -> None: + self._live_processing_gate = live_processing_gate + self._error = error + self._entered = entered + self._release = release + if isinstance(symbols, tuple) and all( + isinstance(symbol, str) + for symbol in symbols + ): + valid_symbols = cast( + tuple[str, ...], + symbols, + ) + self._symbols = tuple( + sorted( + { + symbol.strip() + for symbol in valid_symbols + if symbol.strip() + } + ) + ) + else: + self._symbols = () + self._generation = 0 + self.calls: list[int] = [] + + @property + def state(self) -> ReconnectState: + return ReconnectState.CONNECTED + + @property + def attempt(self) -> int: + return len(self.calls) + + @property + def generation(self) -> int: + return self._generation + + @property + def live_processing_gate( + self, + ) -> RuntimeLiveProcessingGate: + return self._live_processing_gate + + @property + def symbols(self) -> tuple[str, ...]: + return self._symbols + + async def reconnect(self) -> None: + self._generation += 1 + + async with self._live_processing_gate: + if self._entered is not None: + self._entered.set() + + if self._release is not None: + await self._release.wait() + + if self._error is not None: + self._live_processing_gate.fail( + self._error, + ) + raise self._error + + async def reconnect_after_transport_failure( + self, + *, + observed_generation: int, + ) -> None: + self.calls.append(observed_generation) + self._generation += 1 + + if self._entered is not None: + self._entered.set() + + if self._release is not None: + await self._release.wait() + + if self._error is not None: + raise self._error + + +class FakeTradeStreamService: + def __init__( + self, + *, + calls: list[str], + subscribe_error: Exception | None = None, + handle_error: Exception | None = None, + subscribe_gate: asyncio.Event | None = None, + ) -> None: + self._calls = calls + self._subscribe_error = subscribe_error + self._handle_error = handle_error + self._subscribe_gate = subscribe_gate + self.subscribe_calls: list[tuple[str, ...]] = [] + self.subscribe_correlation_ids: list[str | None] = [] + self.documents: list[object] = [] + self.handled = asyncio.Event() + self.subscribe_entered = asyncio.Event() + + async def subscribe( + self, + symbols: tuple[str, ...], + *, + correlation_id: str | None = None, + ) -> None: + self.subscribe_calls.append(symbols) + self.subscribe_correlation_ids.append(correlation_id) + self._calls.append("service.subscribe") + self.subscribe_entered.set() + + if self._subscribe_gate is not None: + await self._subscribe_gate.wait() + + if self._subscribe_error is not None: + raise self._subscribe_error + + def handle_message( + self, + document: object, + ) -> Trade | None: + self.documents.append(document) + self._calls.append("service.handle_message") + self.handled.set() + + if self._handle_error is not None: + raise self._handle_error + + return None + + +class FakeRuntimeSupervisor: + def __init__( + self, + *, + calls: list[str], + start_error: Exception | None = None, + stop_error: Exception | None = None, + ) -> None: + self._calls = calls + self._start_error = start_error + self._stop_error = stop_error + self._state = RuntimeSupervisorState.STOPPED + self.start_calls = 0 + self.stop_calls = 0 + self.notify_activity_calls = 0 + + @property + def state(self) -> RuntimeSupervisorState: + return self._state + + def start(self) -> None: + self.start_calls += 1 + self._calls.append("supervisor.start") + + if self._start_error is not None: + raise self._start_error + + self._state = RuntimeSupervisorState.RUNNING + + def stop(self) -> None: + self.stop_calls += 1 + self._calls.append("supervisor.stop") + self._state = RuntimeSupervisorState.STOPPED + + if self._stop_error is not None: + raise self._stop_error + + def notify_activity(self) -> None: + if self._state is not RuntimeSupervisorState.RUNNING: + return + + self.notify_activity_calls += 1 + + async def handle_heartbeat_timeout(self) -> bool: + raise AssertionError("Scheduler owns timeout handling") + + +class FakeRuntimeScheduler: + def __init__( + self, + *, + calls: list[str], + liveness_probe: RuntimeLivenessProbeProtocol, + runtime_supervisor: FakeRuntimeSupervisor, + error: Exception | None = None, + complete_immediately: bool = False, + on_start: Callable[[], Awaitable[None]] | None = None, + ) -> None: + self._calls = calls + self._liveness_probe = liveness_probe + self._runtime_supervisor = runtime_supervisor + self._error = error + self._complete_immediately = complete_immediately + self._on_start = on_start + self._running = False + self._owner: object | None = None + self._stop_event = asyncio.Event() + self.started = asyncio.Event() + self.claim_calls = 0 + self.release_calls = 0 + self.start_calls = 0 + self.stop_calls = 0 + + @property + def running(self) -> bool: + return self._running + + @property + def interval_seconds(self) -> float: + return 1.0 + + @property + def liveness_probe(self) -> RuntimeLivenessProbeProtocol: + return self._liveness_probe + + @property + def runtime_supervisor(self) -> FakeRuntimeSupervisor: + return self._runtime_supervisor + + def claim(self, owner: object) -> None: + self.claim_calls += 1 + + if self._owner is owner: + return + + if self._owner is not None or self._running: + raise RuntimeError( + "Runtime Scheduler is already owned or active." + ) + + self._owner = owner + + def release(self, owner: object) -> None: + self.release_calls += 1 + + if self._owner is not owner: + raise RuntimeError( + "Runtime Scheduler ownership does not match." + ) + + if self._running: + raise RuntimeError( + "Cannot release a running Runtime Scheduler." + ) + + self._owner = None + + async def start( + self, + *, + owner: object | None = None, + ) -> None: + if self._owner is not None and self._owner is not owner: + raise RuntimeError( + "Runtime Scheduler is owned by another lifecycle." + ) + + if self._owner is None and owner is not None: + raise RuntimeError( + "Runtime Scheduler owner must claim it before start." + ) + + if self._running: + return + + self.start_calls += 1 + self._calls.append("scheduler.start") + self._stop_event.clear() + self._running = True + self.started.set() + + try: + if self._error is not None: + raise self._error + + if self._on_start is not None: + await self._on_start() + + if self._complete_immediately: + return + + await self._stop_event.wait() + finally: + self._running = False + + def stop(self) -> None: + self.stop_calls += 1 + self._calls.append("scheduler.stop") + self._running = False + self._stop_event.set() + + async def run_once(self) -> bool: + return False + + +class RuntimeDependencies: + def __init__( + self, + *, + incoming: tuple[str | bytes | BaseException, ...] = (), + start_error: Exception | None = None, + stop_error: Exception | None = None, + subscribe_error: Exception | None = None, + handle_error: Exception | None = None, + clear_error: Exception | None = None, + start_gate: asyncio.Event | None = None, + connected_publish_gate: asyncio.Event | None = None, + subscribe_gate: asyncio.Event | None = None, + reconnect_error: Exception | None = None, + reconnect_entered: asyncio.Event | None = None, + reconnect_release: asyncio.Event | None = None, + supervisor_start_error: Exception | None = None, + supervisor_stop_error: Exception | None = None, + scheduler_error: Exception | None = None, + scheduler_completes_immediately: bool = False, + scheduler_runs_reconnect: bool = False, + scheduler_uses_different_supervisor: bool = False, + scheduler_uses_different_transport: bool = False, + symbols: tuple[str, ...] = (SYMBOL,), + recovery_symbols: tuple[str, ...] | None = None, + ) -> None: + self.calls: list[str] = [] + self.session = FakeSession( + calls=self.calls, + start_error=start_error, + stop_error=stop_error, + start_gate=start_gate, + ) + self.transport = FakeTransport( + calls=self.calls, + incoming=incoming, + ) + self.subscriptions = FakeSubscriptionManager( + calls=self.calls, + clear_error=clear_error, + ) + self.publisher = RecordingEventPublisher( + calls=self.calls, + connected_gate=connected_publish_gate, + ) + self.service = FakeTradeStreamService( + calls=self.calls, + subscribe_error=subscribe_error, + handle_error=handle_error, + subscribe_gate=subscribe_gate, + ) + self.live_processing_gate = RuntimeLiveProcessingGate() + self.reconnect_recovery = FakeReconnectRecoveryCoordinator( + live_processing_gate=self.live_processing_gate, + error=reconnect_error, + entered=reconnect_entered, + release=reconnect_release, + symbols=( + symbols + if recovery_symbols is None + else recovery_symbols + ), + ) + self.supervisor = FakeRuntimeSupervisor( + calls=self.calls, + start_error=supervisor_start_error, + stop_error=supervisor_stop_error, + ) + self.scheduler = FakeRuntimeScheduler( + calls=self.calls, + liveness_probe=( + FakeTransport( + calls=[], + ) + if scheduler_uses_different_transport + else self.transport + ), + runtime_supervisor=( + FakeRuntimeSupervisor( + calls=[], + ) + if scheduler_uses_different_supervisor + else self.supervisor + ), + error=scheduler_error, + complete_immediately=scheduler_completes_immediately, + on_start=( + self.reconnect_recovery.reconnect + if scheduler_runs_reconnect + else None + ), + ) + self.runtime = TradeStreamProductionRuntime( + session=self.session, + transport=self.transport, + subscription_manager=self.subscriptions, + event_publisher=self.publisher, + trade_stream_service=self.service, + message_classifier=( + DzengiWebSocketInboundMessageClassifier() + ), + control_message_handler=( + DzengiWebSocketControlMessageHandler() + ), + live_processing_gate=self.live_processing_gate, + reconnect_recovery_coordinator=( + self.reconnect_recovery + ), + runtime_supervisor=self.supervisor, + runtime_scheduler=self.scheduler, + symbols=symbols, + ) + + +async def wait_until( + predicate: Callable[[], bool], +) -> None: + for _ in range(100): + if predicate(): + return + + await asyncio.sleep(0) + + raise AssertionError("condition was not reached") + + +def test_implements_public_protocol_and_uses_slots() -> None: + dependencies = RuntimeDependencies() + + assert isinstance( + dependencies.runtime, + TradeStreamProductionRuntimeProtocol, + ) + assert not hasattr( + dependencies.runtime, + "__dict__", + ) + assert ( + dependencies.runtime.state + is TradeStreamProductionRuntimeState.STOPPED + ) + assert dependencies.runtime.running is False + + +@pytest.mark.parametrize( + ("symbols", "error_type"), + [ + ([], TypeError), + ((), ValueError), + (("", " "), ValueError), + ((SYMBOL, 1), TypeError), + ], +) +def test_rejects_invalid_symbols( + symbols: object, + error_type: type[Exception], +) -> None: + with pytest.raises(error_type): + RuntimeDependencies( + symbols=symbols, # type: ignore[arg-type] + ) + + +def test_rejects_symbols_different_from_recovery_coordinator() -> None: + with pytest.raises( + ValueError, + match="must use the same symbols", + ): + RuntimeDependencies( + symbols=(SYMBOL,), + recovery_symbols=(ETH_SYMBOL,), + ) + + +def test_rejects_scheduler_with_different_supervisor() -> None: + with pytest.raises( + ValueError, + match="share one Runtime Supervisor", + ): + RuntimeDependencies( + scheduler_uses_different_supervisor=True, + ) + + +def test_rejects_scheduler_with_different_transport() -> None: + with pytest.raises( + ValueError, + match="share one WebSocket Transport", + ): + RuntimeDependencies( + scheduler_uses_different_transport=True, + ) + + +def test_locked_gate_does_not_leave_runtime_partially_started() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies() + + async with dependencies.live_processing_gate: + with pytest.raises( + RuntimeError, + match="cannot be reset while locked", + ): + await dependencies.runtime.run() + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert dependencies.runtime.running is False + assert dependencies.runtime._stopped.is_set() is True + assert dependencies.session.start_calls == 0 + assert dependencies.scheduler.claim_calls == 1 + assert dependencies.scheduler.release_calls == 1 + assert dependencies.scheduler._owner is None + + +def test_active_external_scheduler_prevents_partial_start() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies() + dependencies.scheduler._running = True + + with pytest.raises( + RuntimeError, + match="Scheduler is already owned or active", + ): + await dependencies.runtime.run() + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert dependencies.runtime._stopped.is_set() is True + assert dependencies.session.start_calls == 0 + assert dependencies.supervisor.start_calls == 0 + + +def test_scheduler_claim_blocks_external_start_during_runtime_startup() -> None: + async def scenario() -> RuntimeDependencies: + start_gate = asyncio.Event() + dependencies = RuntimeDependencies( + start_gate=start_gate, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.session.start_entered.wait() + + with pytest.raises( + RuntimeError, + match="owned by another lifecycle", + ): + await dependencies.scheduler.start() + + start_gate.set() + await dependencies.transport.waiting.wait() + await dependencies.scheduler.started.wait() + await dependencies.runtime.stop() + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.scheduler.claim_calls == 1 + assert dependencies.scheduler.start_calls == 1 + assert dependencies.scheduler.stop_calls == 1 + assert dependencies.scheduler.release_calls == 1 + assert dependencies.scheduler.running is False + assert dependencies.scheduler._owner is None + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + + +def test_startup_receive_and_shutdown_order() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies() + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.transport.waiting.wait() + await dependencies.scheduler.started.wait() + + assert dependencies.runtime.running is True + assert dependencies.session.is_connected is True + assert dependencies.service.subscribe_calls == [ + (SYMBOL,), + ] + assert dependencies.transport.receive_calls == 1 + assert dependencies.supervisor.start_calls == 1 + assert dependencies.scheduler.start_calls == 1 + + await dependencies.runtime.stop() + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.calls == [ + "session.start", + "event.ConnectedEvent", + "service.subscribe", + "supervisor.start", + "transport.receive", + "scheduler.start", + "scheduler.stop", + "supervisor.stop", + "transport.receive.cancelled", + "session.stop", + "subscriptions.clear", + "event.DisconnectedEvent", + ] + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert dependencies.runtime._receive_task is None + assert dependencies.runtime._scheduler_task is None + + +@pytest.mark.parametrize( + ("raw_message", "expected_transport_message"), + [ + ( + MARKET_MESSAGE, + TransportTextMessage( + payload=MARKET_MESSAGE, + ), + ), + ( + MARKET_MESSAGE.encode(), + TransportBinaryMessage( + payload=MARKET_MESSAGE.encode(), + ), + ), + ], +) +def test_decodes_and_forwards_market_messages( + raw_message: str | bytes, + expected_transport_message: ( + TransportTextMessage | TransportBinaryMessage + ), +) -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + incoming=( + raw_message, + ), + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.handled.wait() + await dependencies.runtime.stop() + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.documents == [ + MARKET_DOCUMENT, + ] + assert MessageReceivedEvent( + message=expected_transport_message, + ) in dependencies.publisher.events + assert dependencies.calls.index( + "event.MessageReceivedEvent" + ) < dependencies.calls.index( + "service.handle_message" + ) + assert dependencies.supervisor.notify_activity_calls == 1 + + +def test_matching_success_ack_is_routed_without_market_forwarding() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies() + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.subscribe_entered.wait() + correlation_id = ( + dependencies.service.subscribe_correlation_ids[0] + ) + + assert isinstance(correlation_id, str) + + dependencies.transport.feed( + make_control_message( + correlation_id=correlation_id, + ) + ) + + await wait_until( + lambda: dependencies.transport.receive_calls == 2, + ) + + assert dependencies.runtime.running is True + await dependencies.runtime.stop() + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.documents == [] + assert any( + isinstance(event, MessageReceivedEvent) + for event in dependencies.publisher.events + ) + assert len( + dependencies.service.subscribe_correlation_ids + ) == 1 + + +def test_negative_subscription_ack_is_terminal() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies() + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.subscribe_entered.wait() + correlation_id = ( + dependencies.service.subscribe_correlation_ids[0] + ) + + assert isinstance(correlation_id, str) + + dependencies.transport.feed( + make_control_message( + correlation_id=correlation_id, + status="ERROR", + ) + ) + + with pytest.raises( + WebSocketControlMessageError, + match="отклонил", + ): + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.documents == [] + assert dependencies.session.stop_calls == 1 + assert dependencies.subscriptions.clear_calls == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_unknown_control_correlation_id_is_terminal() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies() + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.subscribe_entered.wait() + dependencies.transport.feed( + make_control_message( + correlation_id="unknown-request", + ) + ) + + with pytest.raises( + WebSocketMessageRoutingError, + match="неизвестным correlationId", + ): + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.documents == [] + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_duplicate_success_ack_is_idempotent() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies() + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.subscribe_entered.wait() + correlation_id = ( + dependencies.service.subscribe_correlation_ids[0] + ) + + assert isinstance(correlation_id, str) + + message = make_control_message( + correlation_id=correlation_id, + ) + dependencies.transport.feed(message) + dependencies.transport.feed(message) + + await wait_until( + lambda: dependencies.transport.receive_calls == 3, + ) + + await dependencies.runtime.stop() + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.documents == [] + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + + +def test_live_trade_reaches_real_consistency_checkpoint() -> None: + trade_document = { + "status": "OK", + "destination": "internal.trade", + "payload": { + "id": 2134857062, + "price": "64497.25", + "size": "0.005", + "ts": 1784218066823, + "symbol": SYMBOL, + "buyer": True, + "orderId": "order-123", + }, + } + + async def scenario() -> tuple[ + TradeStreamStateStore, + RecordingRuntimeCommandDispatcher, + ]: + calls: list[str] = [] + session = FakeSession( + calls=calls, + ) + transport = FakeTransport( + calls=calls, + incoming=( + json.dumps(trade_document), + ), + ) + subscriptions = FakeSubscriptionManager( + calls=calls, + ) + publisher = RecordingEventPublisher( + calls=calls, + ) + command_dispatcher = RecordingRuntimeCommandDispatcher() + state_store = TradeStreamStateStore() + trade_stream_service = TradeStreamAcquisitionService( + runtime_service=command_dispatcher, + adapter=DzengiUnifiedWebSocketAdapter(), + consistency_controller=TradeStreamConsistencyController( + state_store, + ), + ) + runtime_live_processing_gate = RuntimeLiveProcessingGate() + runtime_supervisor = FakeRuntimeSupervisor( + calls=calls, + ) + runtime_scheduler = FakeRuntimeScheduler( + calls=calls, + liveness_probe=transport, + runtime_supervisor=runtime_supervisor, + ) + runtime = TradeStreamProductionRuntime( + session=session, + transport=transport, + subscription_manager=subscriptions, + event_publisher=publisher, + trade_stream_service=trade_stream_service, + message_classifier=( + DzengiWebSocketInboundMessageClassifier() + ), + control_message_handler=( + DzengiWebSocketControlMessageHandler() + ), + live_processing_gate=runtime_live_processing_gate, + reconnect_recovery_coordinator=( + FakeReconnectRecoveryCoordinator( + live_processing_gate=( + runtime_live_processing_gate + ), + ) + ), + runtime_supervisor=runtime_supervisor, + runtime_scheduler=runtime_scheduler, + symbols=(SYMBOL,), + ) + runtime_task = asyncio.create_task( + runtime.run(), + ) + + await wait_until( + lambda: state_store.contains(SYMBOL), + ) + await runtime.stop() + await runtime_task + + return ( + state_store, + command_dispatcher, + ) + + state_store, command_dispatcher = asyncio.run( + scenario() + ) + + assert len(command_dispatcher.commands) == 1 + + checkpoint = state_store.get(SYMBOL).last_trade + + assert checkpoint is not None + assert checkpoint.trade_id == 2134857062 + assert checkpoint.symbol == SYMBOL + + +@pytest.mark.parametrize( + ("raw_message", "error_type"), + [ + ("not-json", WebSocketMessageDecodeError), + ("[]", WebSocketMessageRoutingError), + ( + '{"destination":"unknown"}', + WebSocketMessageRoutingError, + ), + ], +) +def test_invalid_messages_are_terminal( + raw_message: str, + error_type: type[Exception], +) -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + incoming=( + raw_message, + ), + ) + + with pytest.raises(error_type): + await dependencies.runtime.run() + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + assert dependencies.session.stop_calls == 1 + assert dependencies.subscriptions.clear_calls == 1 + + +def test_receive_error_is_terminal_and_not_wrapped() -> None: + receive_error = RuntimeError("receive failed") + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + incoming=( + receive_error, + ), + ) + + with pytest.raises( + RuntimeError, + match="receive failed", + ) as error_info: + await dependencies.runtime.run() + + assert error_info.value is receive_error + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_transport_error_runs_recovery_and_resumes_receive_loop() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + incoming=( + WebSocketTransportError( + "connection dropped", + ), + MARKET_MESSAGE, + ), + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.handled.wait() + + assert dependencies.runtime.running is True + await dependencies.runtime.stop() + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.reconnect_recovery.calls == [ + 0, + ] + assert dependencies.service.documents == [ + MARKET_DOCUMENT, + ] + assert dependencies.transport.receive_calls == 3 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + + +def test_buffered_market_waits_for_reconnect_recovery() -> None: + async def scenario() -> RuntimeDependencies: + reconnect_entered = asyncio.Event() + reconnect_release = asyncio.Event() + dependencies = RuntimeDependencies( + incoming=( + WebSocketTransportError( + "connection dropped", + ), + MARKET_MESSAGE, + ), + reconnect_entered=reconnect_entered, + reconnect_release=reconnect_release, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await reconnect_entered.wait() + await asyncio.sleep(0) + + assert dependencies.transport.receive_calls == 1 + assert dependencies.service.documents == [] + + reconnect_release.set() + await dependencies.service.handled.wait() + await dependencies.runtime.stop() + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.documents == [ + MARKET_DOCUMENT, + ] + + +def test_reconnect_recovery_error_is_terminal_and_not_wrapped() -> None: + recovery_error = RuntimeError( + "runtime recovery failed", + ) + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + incoming=( + WebSocketTransportError( + "connection dropped", + ), + ), + reconnect_error=recovery_error, + ) + + with pytest.raises( + RuntimeError, + match="runtime recovery failed", + ) as error_info: + await dependencies.runtime.run() + + assert error_info.value is recovery_error + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.session.stop_calls == 1 + assert dependencies.subscriptions.clear_calls == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_supervisor_recovery_error_rejects_buffered_market() -> None: + recovery_error = RuntimeError( + "supervisor recovery failed", + ) + + async def scenario() -> RuntimeDependencies: + reconnect_entered = asyncio.Event() + reconnect_release = asyncio.Event() + dependencies = RuntimeDependencies( + reconnect_error=recovery_error, + reconnect_entered=reconnect_entered, + reconnect_release=reconnect_release, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.transport.waiting.wait() + + supervisor_task = asyncio.create_task( + dependencies.reconnect_recovery.reconnect(), + ) + await reconnect_entered.wait() + + dependencies.transport.feed( + MARKET_MESSAGE, + ) + await wait_until( + lambda: any( + isinstance(event, MessageReceivedEvent) + for event in dependencies.publisher.events + ), + ) + + assert dependencies.service.documents == [] + reconnect_release.set() + + with pytest.raises( + RuntimeError, + match="supervisor recovery failed", + ): + await supervisor_task + + with pytest.raises( + RuntimeError, + match="supervisor recovery failed", + ): + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.documents == [] + assert dependencies.live_processing_gate.failed is True + assert dependencies.session.stop_calls == 1 + assert dependencies.subscriptions.clear_calls == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_stop_cancels_active_reconnect_recovery() -> None: + async def scenario() -> tuple[ + RuntimeDependencies, + asyncio.Event, + ]: + reconnect_entered = asyncio.Event() + reconnect_release = asyncio.Event() + dependencies = RuntimeDependencies( + incoming=( + WebSocketTransportError( + "connection dropped", + ), + ), + reconnect_entered=reconnect_entered, + reconnect_release=reconnect_release, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await reconnect_entered.wait() + + await asyncio.wait_for( + dependencies.runtime.stop(), + timeout=1.0, + ) + await runtime_task + + return ( + dependencies, + reconnect_release, + ) + + dependencies, reconnect_release = asyncio.run( + scenario() + ) + + assert reconnect_release.is_set() is False + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert dependencies.runtime._receive_task is None + assert dependencies.runtime._scheduler_task is None + + +def test_stop_waits_for_scheduler_reconnect_before_stopping_supervisor() -> None: + async def scenario() -> tuple[ + RuntimeDependencies, + asyncio.Event, + ]: + reconnect_entered = asyncio.Event() + reconnect_release = asyncio.Event() + dependencies = RuntimeDependencies( + reconnect_entered=reconnect_entered, + reconnect_release=reconnect_release, + scheduler_runs_reconnect=True, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await reconnect_entered.wait() + + await asyncio.wait_for( + dependencies.runtime.stop(), + timeout=1.0, + ) + await runtime_task + + return ( + dependencies, + reconnect_release, + ) + + dependencies, reconnect_release = asyncio.run( + scenario() + ) + + assert reconnect_release.is_set() is False + assert dependencies.calls.index( + "scheduler.stop" + ) < dependencies.calls.index( + "supervisor.stop" + ) + assert dependencies.calls.index( + "supervisor.stop" + ) < dependencies.calls.index( + "transport.receive.cancelled" + ) + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert dependencies.runtime._scheduler_task is None + assert dependencies.runtime._receive_task is None + + +def test_market_handler_error_is_terminal_and_not_wrapped() -> None: + handler_error = RuntimeError("handler failed") + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + incoming=( + MARKET_MESSAGE, + ), + handle_error=handler_error, + ) + + with pytest.raises( + RuntimeError, + match="handler failed", + ) as error_info: + await dependencies.runtime.run() + + assert error_info.value is handler_error + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_connect_failure_publishes_event_and_rolls_back() -> None: + connect_error = RuntimeError("connect failed") + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + start_error=connect_error, + ) + + with pytest.raises( + RuntimeError, + match="connect failed", + ) as error_info: + await dependencies.runtime.run() + + assert error_info.value is connect_error + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.publisher.events == [ + ConnectFailedEvent( + reason="connect failed", + ), + ] + assert dependencies.service.subscribe_calls == [] + assert dependencies.supervisor.start_calls == 0 + assert dependencies.scheduler.start_calls == 0 + assert dependencies.session.stop_calls == 1 + assert dependencies.subscriptions.clear_calls == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_subscribe_failure_closes_session_and_clears_registry() -> None: + subscribe_error = RuntimeError("subscribe failed") + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + subscribe_error=subscribe_error, + ) + + with pytest.raises( + RuntimeError, + match="subscribe failed", + ) as error_info: + await dependencies.runtime.run() + + assert error_info.value is subscribe_error + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.session.stop_calls == 1 + assert dependencies.subscriptions.clear_calls == 1 + assert dependencies.supervisor.start_calls == 0 + assert dependencies.scheduler.start_calls == 0 + assert dependencies.publisher.events == [ + ConnectedEvent(), + DisconnectedEvent(), + ] + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_duplicate_run_is_rejected() -> None: + async def scenario() -> None: + dependencies = RuntimeDependencies() + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.transport.waiting.wait() + + with pytest.raises( + RuntimeError, + match="already active", + ): + await dependencies.runtime.run() + + await dependencies.runtime.stop() + await runtime_task + + asyncio.run(scenario()) + + +def test_concurrent_stop_is_idempotent() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies() + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.transport.waiting.wait() + + await asyncio.gather( + dependencies.runtime.stop(), + dependencies.runtime.stop(), + ) + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.session.stop_calls == 1 + assert dependencies.subscriptions.clear_calls == 1 + assert dependencies.publisher.events.count( + DisconnectedEvent(), + ) == 1 + assert dependencies.scheduler.stop_calls == 1 + assert dependencies.supervisor.stop_calls == 1 + + +def test_stop_before_run_is_no_op() -> None: + dependencies = RuntimeDependencies() + + asyncio.run( + dependencies.runtime.stop() + ) + + assert dependencies.calls == [] + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + + +def test_stop_cancels_hung_session_start_and_rolls_back() -> None: + async def scenario() -> RuntimeDependencies: + start_gate = asyncio.Event() + dependencies = RuntimeDependencies( + start_gate=start_gate, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.session.start_entered.wait() + + await asyncio.wait_for( + dependencies.runtime.stop(), + timeout=1.0, + ) + await runtime_task + + assert start_gate.is_set() is False + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.subscribe_calls == [] + assert dependencies.session.stop_calls == 1 + assert dependencies.subscriptions.clear_calls == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert dependencies.runtime._startup_task is None + + +def test_stop_cancels_hung_connected_event_publish() -> None: + async def scenario() -> RuntimeDependencies: + publish_gate = asyncio.Event() + dependencies = RuntimeDependencies( + connected_publish_gate=publish_gate, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.publisher.connected_entered.wait() + + await asyncio.wait_for( + dependencies.runtime.stop(), + timeout=1.0, + ) + await runtime_task + + assert publish_gate.is_set() is False + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.subscribe_calls == [] + assert dependencies.session.stop_calls == 1 + assert dependencies.subscriptions.clear_calls == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + + +def test_stop_cancels_hung_subscription_send() -> None: + async def scenario() -> RuntimeDependencies: + subscribe_gate = asyncio.Event() + dependencies = RuntimeDependencies( + subscribe_gate=subscribe_gate, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.service.subscribe_entered.wait() + + await asyncio.wait_for( + dependencies.runtime.stop(), + timeout=1.0, + ) + await runtime_task + + assert subscribe_gate.is_set() is False + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.subscribe_calls == [ + (SYMBOL,), + ] + assert dependencies.session.stop_calls == 1 + assert dependencies.subscriptions.clear_calls == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + + +def test_cancelling_root_run_cleans_up_and_propagates() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies() + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.transport.waiting.wait() + + runtime_task.cancel() + + with pytest.raises(asyncio.CancelledError): + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.session.stop_calls == 1 + assert dependencies.subscriptions.clear_calls == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert dependencies.runtime._receive_task is None + + +def test_runtime_can_be_started_again_after_normal_stop() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + symbols=( + f" {SYMBOL} ", + "ETH/USD_LEVERAGE", + SYMBOL, + ), + ) + + first_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.transport.waiting.wait() + await dependencies.scheduler.started.wait() + await dependencies.runtime.stop() + await first_task + + dependencies.transport.waiting.clear() + dependencies.scheduler.started.clear() + + second_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.transport.waiting.wait() + await dependencies.scheduler.started.wait() + await dependencies.runtime.stop() + await second_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.subscribe_calls == [ + ( + SYMBOL, + "ETH/USD_LEVERAGE", + ), + ( + SYMBOL, + "ETH/USD_LEVERAGE", + ), + ] + assert dependencies.session.start_calls == 2 + assert dependencies.session.stop_calls == 2 + assert dependencies.subscriptions.clear_calls == 2 + assert dependencies.supervisor.start_calls == 2 + assert dependencies.supervisor.stop_calls == 2 + assert dependencies.scheduler.start_calls == 2 + assert dependencies.scheduler.stop_calls == 2 + + +def test_runtime_can_be_started_again_after_failure() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + incoming=( + RuntimeError("first receive failed"), + ), + ) + + with pytest.raises( + RuntimeError, + match="first receive failed", + ): + await dependencies.runtime.run() + + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + second_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.transport.waiting.wait() + await dependencies.scheduler.started.wait() + await dependencies.runtime.stop() + await second_task + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.session.start_calls == 2 + assert dependencies.session.stop_calls == 2 + assert dependencies.service.subscribe_calls == [ + (SYMBOL,), + (SYMBOL,), + ] + assert dependencies.supervisor.start_calls == 2 + assert dependencies.supervisor.stop_calls == 2 + assert dependencies.scheduler.start_calls == 2 + assert dependencies.scheduler.stop_calls == 2 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + + +def test_scheduler_error_is_terminal_and_stops_receive() -> None: + scheduler_error = RuntimeError("scheduler failed") + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + scheduler_error=scheduler_error, + ) + + with pytest.raises( + RuntimeError, + match="scheduler failed", + ) as error_info: + await dependencies.runtime.run() + + assert error_info.value is scheduler_error + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.scheduler.start_calls == 1 + assert dependencies.supervisor.start_calls == 1 + assert dependencies.supervisor.stop_calls == 1 + assert "transport.receive.cancelled" in dependencies.calls + assert dependencies.calls.index( + "supervisor.stop" + ) < dependencies.calls.index( + "transport.receive.cancelled" + ) + assert dependencies.session.stop_calls == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_normal_scheduler_completion_is_terminal() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + scheduler_completes_immediately=True, + ) + + with pytest.raises( + RuntimeError, + match="trade-stream-scheduler", + ): + await dependencies.runtime.run() + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.supervisor.stop_calls == 1 + assert dependencies.session.stop_calls == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_supervisor_start_error_rolls_back_without_runtime_tasks() -> None: + supervisor_error = RuntimeError("supervisor failed") + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + supervisor_start_error=supervisor_error, + ) + + with pytest.raises( + RuntimeError, + match="supervisor failed", + ) as error_info: + await dependencies.runtime.run() + + assert error_info.value is supervisor_error + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.supervisor.start_calls == 1 + assert dependencies.supervisor.stop_calls == 1 + assert dependencies.scheduler.start_calls == 0 + assert dependencies.transport.receive_calls == 0 + assert dependencies.session.stop_calls == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_cleanup_continues_after_session_stop_error() -> None: + stop_error = RuntimeError("stop failed") + clear_error = RuntimeError("clear failed") + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + stop_error=stop_error, + clear_error=clear_error, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + await dependencies.transport.waiting.wait() + + stop_task = asyncio.create_task( + dependencies.runtime.stop(), + ) + + with pytest.raises( + RuntimeError, + match="stop failed", + ) as error_info: + await runtime_task + + await stop_task + assert error_info.value is stop_error + assert error_info.value.__notes__ == [ + "Additional Trade Stream Runtime cleanup failure: " + "RuntimeError.", + ] + + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.subscriptions.clear_calls == 1 + assert DisconnectedEvent() not in dependencies.publisher.events + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) diff --git a/app/tests/unit/market_data/acquisition/runtime/test_websocket_runtime_reconnect_integration.py b/app/tests/unit/market_data/acquisition/runtime/test_websocket_runtime_reconnect_integration.py new file mode 100644 index 0000000..623b349 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_websocket_runtime_reconnect_integration.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest +from websockets.protocol import State + +from src.market_data.acquisition.adapters.dzengi.websocket_transport import ( + DzengiWebSocketTransport, +) +from src.market_data.acquisition.exceptions import ( + WebSocketTransportError, +) +from src.market_data.acquisition.runtime.acquisition_runtime_service import ( + AcquisitionRuntimeService, +) +from src.market_data.acquisition.runtime.reconnect import ( + ReconnectCoordinator, + ReconnectState, +) +from src.market_data.acquisition.runtime.runtime_events import ( + ReconnectCompletedEvent, + ReconnectStartedEvent, +) +from src.market_data.acquisition.runtime.transport_messages import ( + TransportTextMessage, +) +from src.market_data.acquisition.runtime.websocket_session import ( + WebSocketSession, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionRuntimeEvent, +) +from src.market_data.acquisition.runtime.websocket_subscription_manager import ( + WebSocketSubscriptionManager, +) + + +SUBSCRIPTION_KEY = "trades:BTC/USD_LEVERAGE" +SUBSCRIPTION_PAYLOAD = '{"destination":"trades.subscribe"}' + + +class FakeConnection: + def __init__( + self, + *, + send_error: Exception | None = None, + ) -> None: + self.state = State.OPEN + self.send_error = send_error + self.sent_messages: list[str | bytes] = [] + self.close_calls = 0 + + async def close( + self, + code: int = 1000, + reason: str = "", + ) -> None: + self.close_calls += 1 + self.state = State.CLOSED + + async def send( + self, + message: str | bytes, + ) -> None: + if self.send_error is not None: + self.state = State.CLOSED + raise self.send_error + + self.sent_messages.append(message) + + async def recv(self) -> str | bytes: + return "" + + +class RecordingConnector: + def __init__( + self, + *connections: FakeConnection, + ) -> None: + self._connections = list(connections) + self.calls: list[ + tuple[str, dict[str, Any]] + ] = [] + + async def __call__( + self, + url: str, + **kwargs: Any, + ) -> FakeConnection: + self.calls.append( + ( + url, + kwargs, + ) + ) + + return self._connections.pop(0) + + +class RecordingEventPublisher: + def __init__(self) -> None: + self.events: list[AcquisitionRuntimeEvent] = [] + + async def publish( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + self.events.append(event) + + +def create_runtime( + *connections: FakeConnection, +) -> tuple[ + WebSocketSession, + WebSocketSubscriptionManager, + ReconnectCoordinator, + RecordingConnector, + RecordingEventPublisher, +]: + connector = RecordingConnector( + *connections, + ) + transport = DzengiWebSocketTransport( + url="wss://api-adapter.dzengi.com", + connector=connector, + ) + session = WebSocketSession( + transport, + ) + subscriptions = WebSocketSubscriptionManager( + transport, + ) + publisher = RecordingEventPublisher() + runtime_service = AcquisitionRuntimeService( + session=session, + transport=transport, + subscription_manager=subscriptions, + event_publisher=publisher, + ) + reconnect = ReconnectCoordinator( + command_dispatcher=runtime_service, + subscription_manager=subscriptions, + event_publisher=publisher, + ) + + return ( + session, + subscriptions, + reconnect, + connector, + publisher, + ) + + +def test_reconnect_replaces_open_connection_before_restore() -> None: + first_connection = FakeConnection() + second_connection = FakeConnection() + ( + session, + subscriptions, + reconnect, + connector, + publisher, + ) = create_runtime( + first_connection, + second_connection, + ) + + async def scenario() -> None: + await session.start() + await subscriptions.subscribe( + SUBSCRIPTION_KEY, + TransportTextMessage( + payload=SUBSCRIPTION_PAYLOAD, + ), + ) + await reconnect.reconnect() + + asyncio.run(scenario()) + + assert len(connector.calls) == 2 + assert first_connection.close_calls == 1 + assert first_connection.state is State.CLOSED + assert first_connection.sent_messages == [ + SUBSCRIPTION_PAYLOAD, + ] + assert second_connection.sent_messages == [ + SUBSCRIPTION_PAYLOAD, + ] + assert session.is_connected is True + assert reconnect.state is ReconnectState.CONNECTED + assert publisher.events == [ + ReconnectStartedEvent(attempt=1), + ReconnectCompletedEvent(attempt=1), + ] + + +def test_reconnect_restores_subscription_after_initial_send_failure() -> None: + first_connection = FakeConnection( + send_error=RuntimeError( + "socket dropped while subscribing", + ), + ) + second_connection = FakeConnection() + ( + session, + subscriptions, + reconnect, + connector, + publisher, + ) = create_runtime( + first_connection, + second_connection, + ) + + async def scenario() -> None: + await session.start() + + with pytest.raises( + WebSocketTransportError, + match="socket dropped while subscribing", + ): + await subscriptions.subscribe( + SUBSCRIPTION_KEY, + TransportTextMessage( + payload=SUBSCRIPTION_PAYLOAD, + ), + ) + + assert subscriptions.subscription_keys == ( + SUBSCRIPTION_KEY, + ) + + await reconnect.reconnect() + + asyncio.run(scenario()) + + assert len(connector.calls) == 2 + assert second_connection.sent_messages == [ + SUBSCRIPTION_PAYLOAD, + ] + assert subscriptions.subscription_keys == ( + SUBSCRIPTION_KEY, + ) + assert session.is_connected is True + assert reconnect.state is ReconnectState.CONNECTED + assert publisher.events == [ + ReconnectStartedEvent(attempt=1), + ReconnectCompletedEvent(attempt=1), + ] diff --git a/app/tests/unit/market_data/acquisition/runtime/test_websocket_session.py b/app/tests/unit/market_data/acquisition/runtime/test_websocket_session.py new file mode 100644 index 0000000..cafac96 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_websocket_session.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from src.market_data.acquisition.runtime.websocket_protocol import ( + WebSocketSessionProtocol, +) +from src.market_data.acquisition.runtime.websocket_session import ( + WebSocketSession, +) + + +class FakeTransport: + def __init__(self) -> None: + self.connected = False + self.connect_calls = 0 + self.disconnect_calls = 0 + self.connect_error: Exception | None = None + self.disconnect_error: Exception | None = None + + @property + def is_connected(self) -> bool: + return self.connected + + async def connect(self) -> None: + self.connect_calls += 1 + + if self.connect_error is not None: + raise self.connect_error + + self.connected = True + + async def disconnect(self) -> None: + self.disconnect_calls += 1 + self.connected = False + + if self.disconnect_error is not None: + raise self.disconnect_error + + async def send( + self, + message: str | bytes, + ) -> None: + return None + + async def receive(self) -> str | bytes: + return "" + + +def create_session() -> tuple[ + WebSocketSession, + FakeTransport, +]: + transport = FakeTransport() + session = WebSocketSession( + transport, + ) + + return ( + session, + transport, + ) + + +def test_session_implements_protocol() -> None: + session, _ = create_session() + + assert isinstance( + session, + WebSocketSessionProtocol, + ) + + +def test_session_uses_slots() -> None: + session, _ = create_session() + + assert not hasattr(session, "__dict__") + + +def test_session_is_initially_disconnected() -> None: + session, _ = create_session() + + assert session.is_connected is False + + +def test_start_connects_transport() -> None: + session, transport = create_session() + + asyncio.run(session.start()) + + assert session.is_connected is True + assert transport.connect_calls == 1 + + +def test_start_is_idempotent() -> None: + session, transport = create_session() + + async def scenario() -> None: + await session.start() + await session.start() + + asyncio.run(scenario()) + + assert session.is_connected is True + assert transport.connect_calls == 1 + + +def test_concurrent_start_creates_one_connection() -> None: + session, transport = create_session() + + async def scenario() -> None: + await asyncio.gather( + session.start(), + session.start(), + ) + + asyncio.run(scenario()) + + assert session.is_connected is True + assert transport.connect_calls == 1 + + +def test_start_error_leaves_session_disconnected() -> None: + session, transport = create_session() + transport.connect_error = RuntimeError("start failed") + + with pytest.raises( + RuntimeError, + match="start failed", + ): + asyncio.run(session.start()) + + assert session.is_connected is False + + +def test_start_reconnects_after_remote_disconnect() -> None: + session, transport = create_session() + + async def scenario() -> None: + await session.start() + transport.connected = False + + assert session.is_connected is False + + await session.start() + + asyncio.run(scenario()) + + assert session.is_connected is True + assert transport.connect_calls == 2 + + +def test_stop_disconnects_transport() -> None: + session, transport = create_session() + + async def scenario() -> None: + await session.start() + await session.stop() + + asyncio.run(scenario()) + + assert session.is_connected is False + assert transport.disconnect_calls == 1 + + +def test_stop_is_idempotent() -> None: + session, transport = create_session() + + async def scenario() -> None: + await session.start() + await session.stop() + await session.stop() + + asyncio.run(scenario()) + + assert transport.disconnect_calls == 1 + assert session.is_connected is False + + +def test_stop_error_still_clears_session_state() -> None: + session, transport = create_session() + transport.disconnect_error = RuntimeError("stop failed") + + async def scenario() -> None: + await session.start() + await session.stop() + + with pytest.raises( + RuntimeError, + match="stop failed", + ): + asyncio.run(scenario()) + + assert session.is_connected is False diff --git a/app/tests/unit/market_data/acquisition/runtime/test_websocket_subscription_manager.py b/app/tests/unit/market_data/acquisition/runtime/test_websocket_subscription_manager.py new file mode 100644 index 0000000..a0c1dad --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_websocket_subscription_manager.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from src.market_data.acquisition.exceptions import ( + WebSocketUnsubscribeNotSupportedError, +) +from src.market_data.acquisition.runtime.transport_messages import ( + TransportBinaryMessage, + TransportTextMessage, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + WebSocketSubscriptionManagerProtocol, +) +from src.market_data.acquisition.runtime.websocket_subscription_manager import ( + WebSocketSubscriptionManager, +) + + +class RecordingTransport: + def __init__(self) -> None: + self.messages: list[str | bytes] = [] + self.send_error: Exception | None = None + + async def connect(self) -> None: + return None + + async def disconnect(self) -> None: + return None + + async def send( + self, + message: str | bytes, + ) -> None: + if self.send_error is not None: + raise self.send_error + + self.messages.append(message) + + async def receive(self) -> str | bytes: + return "" + + +def create_manager( + *, + supports_unsubscribe: bool = False, +) -> tuple[ + WebSocketSubscriptionManager, + RecordingTransport, +]: + transport = RecordingTransport() + manager = WebSocketSubscriptionManager( + transport, + supports_unsubscribe=supports_unsubscribe, + ) + + return ( + manager, + transport, + ) + + +def test_manager_implements_protocol() -> None: + manager, _ = create_manager() + + assert isinstance( + manager, + WebSocketSubscriptionManagerProtocol, + ) + + +def test_manager_uses_slots() -> None: + manager, _ = create_manager() + + assert not hasattr(manager, "__dict__") + + +def test_manager_starts_with_empty_registry() -> None: + manager, _ = create_manager() + + assert manager.subscription_keys == () + + +def test_subscribe_sends_and_registers_text_message() -> None: + manager, transport = create_manager() + message = TransportTextMessage( + payload='{"destination":"trades.subscribe"}', + ) + + asyncio.run( + manager.subscribe( + "trades:BTC", + message, + ) + ) + + assert transport.messages == [ + message.payload, + ] + assert manager.subscription_keys == ( + "trades:BTC", + ) + + +def test_subscribe_sends_binary_message() -> None: + manager, transport = create_manager() + message = TransportBinaryMessage( + payload=b"\x01\x02", + ) + + asyncio.run( + manager.subscribe( + "binary", + message, + ) + ) + + assert transport.messages == [ + b"\x01\x02", + ] + assert manager.subscription_keys == ( + "binary", + ) + + +def test_duplicate_subscription_key_is_idempotent() -> None: + manager, transport = create_manager() + first_message = TransportTextMessage( + payload="first", + ) + second_message = TransportTextMessage( + payload="second", + ) + + async def scenario() -> None: + await manager.subscribe( + "trades:BTC", + first_message, + ) + await manager.subscribe( + "trades:BTC", + second_message, + ) + + asyncio.run(scenario()) + + assert transport.messages == [ + "first", + ] + assert manager.subscription_keys == ( + "trades:BTC", + ) + + +def test_failed_subscribe_remains_registered_as_pending() -> None: + manager, transport = create_manager() + transport.send_error = RuntimeError("send failed") + + with pytest.raises( + RuntimeError, + match="send failed", + ): + asyncio.run( + manager.subscribe( + "trades:BTC", + TransportTextMessage( + payload="subscribe", + ), + ) + ) + + assert manager.subscription_keys == ( + "trades:BTC", + ) + + +def test_pending_subscription_can_be_retried() -> None: + manager, transport = create_manager() + transport.send_error = RuntimeError("send failed") + + async def scenario() -> None: + with pytest.raises( + RuntimeError, + match="send failed", + ): + await manager.subscribe( + "trades:BTC", + TransportTextMessage( + payload="first-attempt", + ), + ) + + transport.send_error = None + + await manager.subscribe( + "trades:BTC", + TransportTextMessage( + payload="second-attempt", + ), + ) + + asyncio.run(scenario()) + + assert transport.messages == [ + "second-attempt", + ] + assert manager.subscription_keys == ( + "trades:BTC", + ) + + +def test_restore_sends_registered_messages_in_order() -> None: + manager, transport = create_manager() + + async def scenario() -> None: + await manager.subscribe( + "first", + TransportTextMessage( + payload="first-message", + ), + ) + await manager.subscribe( + "second", + TransportBinaryMessage( + payload=b"second-message", + ), + ) + + transport.messages.clear() + await manager.restore_subscriptions() + + asyncio.run(scenario()) + + assert transport.messages == [ + "first-message", + b"second-message", + ] + + +def test_restore_error_preserves_registry() -> None: + manager, transport = create_manager() + + async def prepare() -> None: + await manager.subscribe( + "trades:BTC", + TransportTextMessage( + payload="subscribe", + ), + ) + + asyncio.run(prepare()) + transport.send_error = RuntimeError("restore failed") + + with pytest.raises( + RuntimeError, + match="restore failed", + ): + asyncio.run(manager.restore_subscriptions()) + + assert manager.subscription_keys == ( + "trades:BTC", + ) + + +def test_unsubscribe_is_explicitly_unsupported_by_default() -> None: + manager, transport = create_manager() + + async def scenario() -> None: + await manager.subscribe( + "trades:BTC", + TransportTextMessage( + payload="subscribe", + ), + ) + await manager.unsubscribe( + "trades:BTC", + TransportTextMessage( + payload="unsubscribe", + ), + ) + + with pytest.raises( + WebSocketUnsubscribeNotSupportedError, + ): + asyncio.run(scenario()) + + assert transport.messages == [ + "subscribe", + ] + assert manager.subscription_keys == ( + "trades:BTC", + ) + + +def test_supported_unsubscribe_sends_and_removes_subscription() -> None: + manager, transport = create_manager( + supports_unsubscribe=True, + ) + + async def scenario() -> None: + await manager.subscribe( + "generic", + TransportTextMessage( + payload="subscribe", + ), + ) + await manager.unsubscribe( + "generic", + TransportTextMessage( + payload="unsubscribe", + ), + ) + + asyncio.run(scenario()) + + assert transport.messages == [ + "subscribe", + "unsubscribe", + ] + assert manager.subscription_keys == () + + +def test_supported_unsubscribe_is_idempotent_for_missing_key() -> None: + manager, transport = create_manager( + supports_unsubscribe=True, + ) + + asyncio.run( + manager.unsubscribe( + "missing", + TransportTextMessage( + payload="unsubscribe", + ), + ) + ) + + assert transport.messages == [] + assert manager.subscription_keys == () + + +def test_clear_removes_registry_without_sending_messages() -> None: + manager, transport = create_manager() + + async def scenario() -> None: + await manager.subscribe( + "trades:BTC", + TransportTextMessage( + payload="subscribe", + ), + ) + + transport.messages.clear() + await manager.clear_subscriptions() + + asyncio.run(scenario()) + + assert manager.subscription_keys == () + assert transport.messages == [] + + +@pytest.mark.parametrize( + "subscription_key", + [ + "", + " ", + ], +) +def test_rejects_empty_subscription_key( + subscription_key: str, +) -> None: + manager, _ = create_manager() + + with pytest.raises(ValueError): + asyncio.run( + manager.subscribe( + subscription_key, + TransportTextMessage( + payload="subscribe", + ), + ) + ) + + +def test_rejects_non_string_subscription_key() -> None: + manager, _ = create_manager() + + with pytest.raises(TypeError): + asyncio.run( + manager.subscribe( + 123, # type: ignore[arg-type] + TransportTextMessage( + payload="subscribe", + ), + ) + ) + + +def test_rejects_unsupported_message_type() -> None: + manager, _ = create_manager() + + with pytest.raises(TypeError): + asyncio.run( + manager.subscribe( + "trades:BTC", + object(), # type: ignore[arg-type] + ) + ) + + +def test_rejects_non_boolean_unsubscribe_capability() -> None: + transport = RecordingTransport() + + with pytest.raises(TypeError): + WebSocketSubscriptionManager( + transport, + supports_unsubscribe="yes", # type: ignore[arg-type] + ) diff --git a/app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py b/app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py index 240f6ad..369a7b1 100644 --- a/app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py +++ b/app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py @@ -28,6 +28,12 @@ from src.market_data.acquisition.runtime.reconnect import ( ReconnectCoordinatorProtocol, ReconnectState, ) +from src.market_data.acquisition.runtime.runtime_reconnect_recovery_coordinator import ( + RuntimeReconnectRecoveryProtocol, +) +from src.market_data.acquisition.runtime.runtime_liveness_probe import ( + RuntimeLivenessProbeProtocol, +) from src.market_data.acquisition.runtime.runtime_events import ( ReconnectCompletedEvent, ReconnectStartedEvent, @@ -123,11 +129,17 @@ class FakeSession: class FakeTransport: - def __init__(self) -> None: + def __init__( + self, + *, + probe_results: tuple[bool, ...] = (True,), + ) -> None: self.connect_calls = 0 self.disconnect_calls = 0 self.sent_messages: list[str | bytes] = [] self.receive_calls = 0 + self.probe_calls = 0 + self._probe_results = list(probe_results) async def connect(self) -> None: self.connect_calls += 1 @@ -145,6 +157,14 @@ class FakeTransport: self.receive_calls += 1 return "" + async def probe(self) -> bool: + self.probe_calls += 1 + + if not self._probe_results: + return True + + return self._probe_results.pop(0) + class FakeSubscriptionManager: def __init__(self) -> None: @@ -264,6 +284,25 @@ class FakeClock: def __call__(self) -> float: return self.value + def advance( + self, + seconds: float, + ) -> None: + self.value += seconds + + +class FakeUnixTimeClock: + def __init__( + self, + value: int = RECOVERY_END_TIME_MS, + ) -> None: + self.value = value + self.calls = 0 + + def __call__(self) -> int: + self.calls += 1 + return self.value + class RecordingSleep: def __init__(self) -> None: @@ -285,6 +324,7 @@ class CompositionDependencies: message_adapter: FakeMessageAdapter recovery_document_source: StubTradesDocumentSource heartbeat_clock: FakeClock + recovery_end_time_clock: FakeUnixTimeClock scheduler_sleep: RecordingSleep @@ -295,13 +335,16 @@ def create_composition( heartbeat_timeout_seconds: float = 10.0, scheduler_interval_seconds: float = 1.0, max_recovery_window_ms: int = 3_599_999, + probe_results: tuple[bool, ...] = (True,), ) -> tuple[ TradeStreamRuntimeComposition, CompositionDependencies, ]: dependencies = CompositionDependencies( session=FakeSession(), - transport=FakeTransport(), + transport=FakeTransport( + probe_results=probe_results, + ), subscription_manager=FakeSubscriptionManager(), event_publisher=FakeEventPublisher(), message_adapter=FakeMessageAdapter( @@ -311,6 +354,7 @@ def create_composition( recovery_document, ), heartbeat_clock=FakeClock(), + recovery_end_time_clock=FakeUnixTimeClock(), scheduler_sleep=RecordingSleep(), ) @@ -323,10 +367,14 @@ def create_composition( recovery_document_source=( dependencies.recovery_document_source ), + symbols=(SYMBOL,), heartbeat_timeout_seconds=heartbeat_timeout_seconds, scheduler_interval_seconds=scheduler_interval_seconds, max_recovery_window_ms=max_recovery_window_ms, heartbeat_clock=dependencies.heartbeat_clock, + recovery_end_time_clock=( + dependencies.recovery_end_time_clock + ), scheduler_sleep=dependencies.scheduler_sleep, ) @@ -373,6 +421,14 @@ def test_components_implement_public_protocols() -> None: composition.reconnect_coordinator, ReconnectCoordinatorProtocol, ) + assert isinstance( + composition.runtime_reconnect_recovery_coordinator, + RuntimeReconnectRecoveryProtocol, + ) + assert isinstance( + composition.liveness_probe, + RuntimeLivenessProbeProtocol, + ) assert isinstance( composition.heartbeat_monitor, HeartbeatMonitorProtocol, @@ -460,8 +516,27 @@ def test_runtime_components_share_lifecycle_dependencies() -> None: ) assert ( composition.runtime_supervisor._reconnect_coordinator + is composition.runtime_reconnect_recovery_coordinator + ) + assert ( + composition.runtime_reconnect_recovery_coordinator + ._reconnect_coordinator is composition.reconnect_coordinator ) + assert ( + composition.runtime_reconnect_recovery_coordinator + ._recovery_coordinator + is composition.runtime_recovery_coordinator + ) + assert ( + composition.runtime_reconnect_recovery_coordinator + .live_processing_gate + is composition.live_processing_gate + ) + assert ( + composition.runtime_reconnect_recovery_coordinator.symbols + == (SYMBOL,) + ) assert ( composition.runtime_scheduler._heartbeat_monitor is composition.heartbeat_monitor @@ -470,6 +545,14 @@ def test_runtime_components_share_lifecycle_dependencies() -> None: composition.runtime_scheduler._runtime_supervisor is composition.runtime_supervisor ) + assert ( + composition.liveness_probe + is dependencies.transport + ) + assert ( + composition.runtime_scheduler._liveness_probe + is composition.liveness_probe + ) def test_configuration_is_forwarded() -> None: @@ -507,6 +590,7 @@ def test_creation_has_no_runtime_side_effects() -> None: assert dependencies.transport.disconnect_calls == 0 assert dependencies.transport.sent_messages == [] assert dependencies.transport.receive_calls == 0 + assert dependencies.transport.probe_calls == 0 assert dependencies.subscription_manager.subscriptions == [] assert dependencies.subscription_manager.unsubscriptions == [] @@ -515,6 +599,7 @@ def test_creation_has_no_runtime_side_effects() -> None: assert dependencies.event_publisher.events == [] assert dependencies.recovery_document_source.calls == [] + assert dependencies.recovery_end_time_clock.calls == 0 assert composition.heartbeat_monitor.state is HeartbeatState.IDLE assert ( @@ -528,6 +613,103 @@ def test_creation_has_no_runtime_side_effects() -> None: assert composition.runtime_scheduler.running is False +def test_successful_probe_keeps_quiet_connection_alive() -> None: + composition, dependencies = create_composition( + heartbeat_timeout_seconds=10.0, + probe_results=(True,), + ) + composition.runtime_supervisor.start() + dependencies.heartbeat_clock.advance(10.0) + + timed_out = asyncio.run( + composition.runtime_scheduler.run_once() + ) + + assert timed_out is False + assert dependencies.transport.probe_calls == 1 + assert dependencies.session.start_calls == 0 + assert dependencies.session.stop_calls == 0 + assert composition.heartbeat_monitor.last_activity_at == 110.0 + assert ( + composition.runtime_supervisor.state + is RuntimeSupervisorState.RUNNING + ) + + +def test_failed_probe_triggers_reconnect_after_timeout() -> None: + composition, dependencies = create_composition( + heartbeat_timeout_seconds=10.0, + probe_results=(False,), + ) + composition.runtime_supervisor.start() + dependencies.heartbeat_clock.advance(10.0) + + timed_out = asyncio.run( + composition.runtime_scheduler.run_once() + ) + + assert timed_out is True + assert dependencies.transport.probe_calls == 1 + assert dependencies.session.stop_calls == 1 + assert dependencies.session.start_calls == 1 + assert dependencies.subscription_manager.restore_calls == 1 + assert dependencies.recovery_end_time_clock.calls == 1 + assert ( + composition.runtime_supervisor.state + is RuntimeSupervisorState.RUNNING + ) + + +def test_delayed_timeout_does_not_repeat_completed_transport_reconnect() -> None: + async def scenario() -> tuple[ + TradeStreamRuntimeComposition, + CompositionDependencies, + bool, + ]: + composition, dependencies = create_composition( + heartbeat_timeout_seconds=10.0, + probe_results=(False,), + ) + composition.runtime_supervisor.start() + observed_generation = ( + composition.runtime_reconnect_recovery_coordinator.generation + ) + + await ( + composition.runtime_reconnect_recovery_coordinator + .reconnect_after_transport_failure( + observed_generation=observed_generation, + ) + ) + dependencies.heartbeat_clock.advance(10.0) + timed_out = await composition.runtime_scheduler.run_once() + + return ( + composition, + dependencies, + timed_out, + ) + + composition, dependencies, timed_out = asyncio.run( + scenario() + ) + + assert timed_out is True + assert dependencies.transport.probe_calls == 1 + assert dependencies.session.stop_calls == 1 + assert dependencies.session.start_calls == 1 + assert dependencies.subscription_manager.restore_calls == 1 + assert dependencies.recovery_end_time_clock.calls == 1 + assert ( + composition.runtime_reconnect_recovery_coordinator.generation + == 1 + ) + assert ( + composition.runtime_supervisor.state + is RuntimeSupervisorState.RUNNING + ) + + def test_live_checkpoint_is_visible_to_runtime_recovery() -> None: checkpoint_trade = make_trade() @@ -642,6 +824,35 @@ def test_reconnect_uses_composed_runtime_dependencies() -> None: ) +def test_runtime_reconnect_runs_recovery_after_subscription_restore() -> None: + composition, dependencies = create_composition( + recovery_document=[], + ) + + composition.trade_stream_acquisition_service.handle_message( + { + "destination": "internal.trade", + } + ) + + asyncio.run( + composition.runtime_reconnect_recovery_coordinator.reconnect() + ) + + assert dependencies.session.stop_calls == 1 + assert dependencies.session.start_calls == 1 + assert dependencies.subscription_manager.restore_calls == 1 + assert dependencies.recovery_end_time_clock.calls == 1 + assert dependencies.recovery_document_source.calls == [ + ( + SYMBOL, + CHECKPOINT_TIME_MS, + RECOVERY_END_TIME_MS, + None, + ) + ] + + def test_separate_compositions_have_independent_state() -> None: first, _ = create_composition() second, _ = create_composition() @@ -690,6 +901,7 @@ def test_invalid_heartbeat_configuration_is_not_hidden( [], ), heartbeat_clock=FakeClock(), + recovery_end_time_clock=FakeUnixTimeClock(), scheduler_sleep=RecordingSleep(), ) @@ -705,9 +917,13 @@ def test_invalid_heartbeat_configuration_is_not_hidden( recovery_document_source=( dependencies.recovery_document_source ), + symbols=(SYMBOL,), heartbeat_timeout_seconds=heartbeat_timeout_seconds, # type: ignore[arg-type] scheduler_interval_seconds=1.0, heartbeat_clock=dependencies.heartbeat_clock, + recovery_end_time_clock=( + dependencies.recovery_end_time_clock + ), scheduler_sleep=dependencies.scheduler_sleep, ) diff --git a/app/tests/unit/test_main.py b/app/tests/unit/test_main.py new file mode 100644 index 0000000..fa47171 --- /dev/null +++ b/app/tests/unit/test_main.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import asyncio + +import src.main as main_module + + +def test_main_runs_application_composition( + monkeypatch, +) -> None: + application = object() + received_applications: list[object] = [] + + monkeypatch.setattr( + main_module, + "create_app", + lambda: application, + ) + + async def run(received_application: object) -> None: + received_applications.append( + received_application, + ) + + monkeypatch.setattr( + main_module, + "run_application", + run, + ) + + asyncio.run(main_module.main()) + + assert received_applications == [application] diff --git a/docs/architecture/dzentra_target_architecture.md b/docs/architecture/dzentra_target_architecture.md new file mode 100644 index 0000000..c4084a6 --- /dev/null +++ b/docs/architecture/dzentra_target_architecture.md @@ -0,0 +1,557 @@ +# Dzentra Target Architecture + +## Контроль документа + +| Свойство | Значение | +|---|---| +| Тип | Target Architecture | +| Статус | Active Baseline | +| Версия | 1.0 | +| Дата | 2026-07-31 | +| Проект | Dzentra | + +--- + +## 1. Назначение + +Документ фиксирует целевую архитектурную карту Dzentra и связывает: + +- фактическую структуру существующего проекта; +- выполняемую Build-by-Build миграцию; +- будущие подсистемы профессиональной автоматической торговой системы; +- обязательное направление зависимостей между доменами. + +Документ не требует немедленного создания всех перечисленных каталогов +и не заменяет архитектурные спецификации отдельных Build. + +Целевая архитектура отвечает на вопрос: + +> Какие ответственности должны существовать в системе и как они могут +> взаимодействовать? + +Последовательность реализации и текущий статус определяет +`docs/roadmap/master-roadmap.md`. + +--- + +## 2. Текущий контекст + +Dzentra является работающим modular monolith, который постепенно +мигрирует из Telegram trading bot в модульную алгоритмическую торговую +платформу. + +Проект не является greenfield-системой. В нём одновременно существуют: + +- принятые новые архитектурные слои; +- рабочие legacy-компоненты; +- временные compatibility boundaries; +- будущие целевые подсистемы. + +Поэтому целевой каталог нельзя создавать механическим переносом файлов. +Любое изменение ownership или направления зависимостей выполняется +отдельным Build после анализа фактических consumers. + +--- + +## 3. Верхнеуровневая карта + +```text +External Exchanges and Data Sources + │ + ▼ +┌───────────────────────────────────────────────┐ +│ Market Data │ +│ Acquisition → Canonicalization → Quality │ +│ │ │ +│ ├── Live / Operational │ +│ └── Storage → Replay │ +└───────────────────────────────────────────────┘ + │ Canonical Live or Replay Data + ▼ +┌───────────────────────────────────────────────┐ +│ Market Intelligence │ +│ Processing → Features → Market State │ +│ → Scenario Evaluation │ +└───────────────────────────────────────────────┘ + │ Analysis Results + ▼ +┌───────────────────────────────────────────────┐ +│ Trading │ +│ Strategy / Decision → Portfolio / Risk │ +│ → Order Management → Execution │ +│ → Position and Account Reconciliation │ +└───────────────────────────────────────────────┘ + │ Commands and Reports + ▼ + Exchange +``` + +Все уровни обслуживаются общей Platform-плоскостью: + +```text +Runtime and Scheduling +Event Contracts +Configuration and Secrets +Logging, Audit and Observability +Resilience and Recovery +Security +Deployment and Operations +``` + +Logging, Audit, Monitoring и Runtime не являются последними этапами +торгового конвейера. Они действуют на всех уровнях системы. + +--- + +## 4. Market Data + +### 4.1. Market Data Acquisition + +**Назначение:** получение внешних рыночных и справочных данных. + +Целевые семейства Feed: + +- Instrument Reference Data; +- Quotes; +- OHLCV / Candles; +- Trades / Time & Sales; +- Order Book; +- Derivatives Market Data; +- Market Index; +- Exchange Time; +- Exchange and Instrument Status. + +Каждый Feed развивается как самостоятельный vertical slice: + +```text +external document + ↓ +schema validation + ↓ +parser / transport model + ↓ +value validation + ↓ +mapper + ↓ +canonical model + ↓ +handler / feed / service +``` + +REST и WebSocket могут иметь независимые transport pipelines, но до +передачи downstream обязаны сходиться в одну source-independent +Canonical Model. + +**Текущее размещение:** + +```text +app/src/market_data/acquisition/ +``` + +**Текущий активный результат:** Production Runtime для Trades Feed +завершён в Build 060.25. + +### 4.2. Validation, Canonicalization and Data Quality + +**Назначение:** + +- проверять структуру внешних документов; +- проверять допустимость значений; +- нормализовать время, числовые значения и идентификаторы; +- преобразовывать transport data в Canonical Models; +- контролировать дубликаты, порядок и пропуски; +- формировать явные признаки качества и freshness. + +Эта ответственность сейчас распределена между: + +```text +adapters/ +validation/ +models/ +consistency/ +recovery/ +``` + +Build 061.00 должен сначала проверить фактические границы, а уже затем +решить, требуется ли отдельный каталог `normalization`. + +Schema Validation, Value Validation, Mapping, Stream Consistency и +Missing Data Recovery не должны смешиваться в один универсальный +компонент. + +### 4.3. Operational Market State + +Оперативное состояние необходимо для работы активного потока: + +- последний принятый Trade; +- deduplication window; +- состояние подписок; +- connection generation; +- liveness и reconnect state. + +Оно не является долговременной историей рынка. + +`TradeStreamState` принадлежит Consistency Layer, а не общему Market +Data Storage и не WebSocket Transport. + +### 4.4. Persistent Market Data Storage + +**Назначение:** долговременно хранить Canonical Market Data для: + +- восстановления после перезапуска; +- исторических запросов; +- Replay; +- разработки и проверки аналитики; +- backtesting; +- аудита качества данных. + +Хранилище должно определить: + +- raw и canonical retention policy; +- ключи идемпотентности; +- порядок и уникальность событий; +- партиционирование по типу данных, символу и времени; +- правила исправления и повторной загрузки; +- provenance и версию Canonical schema. + +Существующий `app/src/storage/` нельзя автоматически считать готовым +Market Data Storage. Его фактические контракты проверяются в +Build 060.27. + +### 4.5. Historical Access and Replay + +**Назначение:** + +- исторические запросы; +- последовательное воспроизведение Canonical Market Data; +- управляемые виртуальные часы; +- одинаковые контракты данных для production и replay consumers; +- основа backtesting и воспроизводимой диагностики. + +Replay должен воспроизводить порядок событий и не обходить Canonical +Models или validation guarantees. + +--- + +## 5. Market Intelligence + +### 5.1. Market Data Processing + +Формирует устойчивое состояние и базовые рыночные представления: + +- Candle aggregation; +- Order Book reconstruction; +- best bid / ask / mid / spread; +- trade-flow aggregation; +- временные окна; +- подтверждённые market events. + +Processing не принимает торговых решений. + +### 5.2. Feature Engineering + +Формирует производные признаки: + +- returns и momentum; +- volatility; +- relative и abnormal volume; +- trade-flow intensity и delta; +- Order Book imbalance и microprice; +- liquidity; +- time/session features; +- statistical и cross-market features; +- derivatives features. + +Canonical Trade price, quantity и timestamp уже являются исходными +данными и не требуют отдельного дублирующего Trade Facts model без +подтверждённого consumer. + +### 5.3. Market State Estimation + +Интерпретирует признаки и определяет: + +- market structure; +- trend и range; +- momentum и correction; +- volatility regime; +- liquidity regime; +- order-flow state; +- market phase; +- microstructure context. + +**Текущее основное размещение:** + +```text +app/src/trading/market_intelligence/ +``` + +Изменение этого ownership требует отдельного архитектурного Build. + +### 5.4. Scenario Evaluation + +Dzentra рассматривает прогнозирование как оценку нескольких возможных +сценариев: + +- scenario generation; +- probability estimation; +- ranking; +- confidence and uncertainty. + +Scenario Evaluation является опциональной аналитической возможностью. +Strategy, которой отдельный прогноз не требуется, не должна создавать +фиктивную зависимость от Forecasting. + +--- + +## 6. Trading + +### 6.1. Strategy and Trading Decision + +Преобразует рыночный анализ в торговое намерение: + +- enter; +- hold; +- reduce; +- exit; +- do nothing. + +Результат Decision не является биржевым ордером и не должен напрямую +вызывать Transport. + +### 6.2. Portfolio and Risk + +Проверяет и ограничивает торговое намерение: + +- допустимый размер позиции; +- leverage и margin; +- общий капитал под риском; +- концентрацию по символам; +- drawdown и loss limits; +- pre-trade, continuous и emergency risk controls. + +Risk не является только одним последовательным шагом после Strategy. +Он может запрещать вход, изменять размер позиции и останавливать уже +активное исполнение. + +### 6.3. Order Management System + +Отвечает за внутренний жизненный цикл собственных заявок: + +- создание client order identity; +- переходы состояния заявки; +- amend / cancel; +- partial fills; +- deduplication; +- reconciliation с биржей; +- восстановление после reconnect и restart. + +OMS не следует смешивать с Market Trades Feed: рыночная сделка и +исполнение собственной заявки являются разными предметными сущностями. + +### 6.4. Execution Management + +Отвечает за: + +- преобразование одобренного намерения в execution command; +- выбор цены и способа исполнения; +- маршрутизацию на биржу; +- контроль slippage, liquidity и freshness; +- обработку transport и exchange errors. + +**Текущее основное размещение:** + +```text +app/src/trading/execution/ +``` + +### 6.5. Position and Account Reconciliation + +Сопоставляет локальное состояние с биржей: + +- позиции; +- balances; +- fills; +- open orders; +- fees; +- realized и unrealized PnL. + +Биржевое состояние является внешним фактом и должно периодически +сверяться, а не только вычисляться из локальных событий. + +--- + +## 7. Platform and Governance + +### Runtime and Scheduling + +- явные владельцы задач и ресурсов; +- конечные timeouts; +- детерминированные startup и shutdown; +- controlled retry/reconnect; +- отсутствие скрытого fire-and-forget. + +### Event Contracts + +- типизированные и версионируемые события; +- явная граница между market, runtime, execution и audit events; +- контролируемое ordering; +- отсутствие чувствительных transport payload в логах. + +### Configuration and Secrets + +- feature flags безопасны по умолчанию; +- URL, symbols и credentials задаются явно; +- production secrets не попадают в код, документацию и события. + +### Logging, Audit and Observability + +- технические логи; +- domain audit trail; +- Trade Journal; +- metrics, latency и data freshness; +- health и liveness; +- alerting; +- объяснение причин торгового решения. + +### Resilience + +- idempotency; +- reconnect; +- Recovery; +- persistent checkpoint; +- startup reconciliation; +- controlled degradation; +- failure isolation. + +### Backtesting and Simulation + +Backtesting является отдельной capability, но переиспользует: + +- Canonical Market Data; +- Replay; +- deterministic clock; +- production Feature, Decision и Risk contracts; +- exchange/OMS simulator. + +Отдельная копия торговой логики только для backtest не допускается. + +--- + +## 8. Правила зависимостей + +Разрешённое основное направление: + +```text +Market Data + ↓ +Market Intelligence + ↓ +Trading Decision + ↓ +Portfolio / Risk + ↓ +OMS / Execution + ↓ +Exchange Integration +``` + +Обязательные ограничения: + +- Market Data не импортирует Trading; +- Canonical Models не зависят от Dzengi transport schema; +- Market Intelligence не управляет WebSocket lifecycle; +- Strategy не отправляет биржевую заявку напрямую; +- Execution не изменяет правила Market Data Consistency; +- Telegram UI не является владельцем domain state; +- Storage не вызывает торговые решения; +- Backtesting использует те же domain contracts, что production; +- Platform-компоненты не получают лишнюю предметную ответственность. + +Обратная информация от Exchange возвращается через явные execution, +order, account и runtime events, а не через обратные импорты между +слоями. + +--- + +## 9. Политика каталогов + +Логический домен и физический каталог не обязаны иметь одинаковое имя. + +Текущее соответствие: + +| Ответственность | Фактическое размещение | +|---|---| +| Market Data Acquisition | `app/src/market_data/acquisition/` | +| Application Runtime Composition | `app/src/bootstrap/` | +| Runtime Events | `app/src/runtime_events/` и локальные runtime events | +| Storage foundation | `app/src/storage/` | +| Market Intelligence | `app/src/trading/market_intelligence/` | +| Trading Decision | `app/src/trading/decision/` | +| Execution | `app/src/trading/execution/` | +| Position | `app/src/trading/position/` | +| Trade Journal | `app/src/trading/journal/` | +| Exchange Integration | `app/src/integrations/exchange/` | + +Новый каталог создаётся только когда: + +1. ответственность подтверждена фактическими consumers; +2. определён публичный контракт; +3. существующее ownership признано неверным; +4. подготовлен отдельный migration plan; +5. проверено направление импортов. + +Предложенные имена вроде `analytics/features`, +`market_data/normalization` или `market_data/storage` являются +логическими ориентирами, а не заранее утверждённой файловой структурой. + +--- + +## 10. Current-to-target status + +| Область | Состояние | +|---|---| +| Market Data Acquisition foundation | Реализуется Build-by-Build | +| Trades Feed production runtime | Completed — Build 060.25 | +| Runtime integration/stress verification | Next — Build 060.26 | +| Persistent Market Data Storage | Planned — Build 060.27 | +| Persistent Checkpoint / Startup Recovery | Planned — Build 060.28 | +| Historical Access / Replay | Planned — Build 060.29 | +| Validation ownership audit/refactoring | Planned — Build 061.00 | +| Market Intelligence | Существующая активная подсистема | +| Decision / Execution / Position | Существующие подсистемы, требуют будущих boundary audits | +| OMS / Reconciliation | Целевая ответственность, отдельный план не утверждён | +| Backtesting | Целевая capability после Storage и Replay | +| Operations / Deployment | Целевая cross-cutting программа | + +`Completed` означает завершение утверждённого Build, а не окончательную +сертификацию всей подсистемы для любой production-нагрузки. + +--- + +## 11. Управление изменениями + +Этот документ изменяется только когда меняется: + +- ownership домена; +- публичная архитектурная граница; +- разрешённое направление зависимостей; +- целевая карта подсистем. + +Статус конкретного Build изменяется в `master-roadmap.md` и его +migration report. + +Детали реализации, тестовые evidence и локальные решения фиксируются в: + +```text +docs/migrations/build__architecture.md +docs/migrations/build_.md +docs/decisions/ +``` + +При конфликте: + +1. фактический код и тесты определяют текущее поведение; +2. принятый Build architecture document определяет локальное решение; +3. настоящий документ определяет целевую границу; +4. roadmap определяет порядок дальнейшей миграции. diff --git a/docs/migrations/build_060_25.md b/docs/migrations/build_060_25.md new file mode 100644 index 0000000..b5e9116 --- /dev/null +++ b/docs/migrations/build_060_25.md @@ -0,0 +1,419 @@ +# Build 060.25 — Production Runtime Integration + +**Engineering Migration Report** + +--- + +## Контроль документа + +| Свойство | Значение | +|---|---| +| Build | 060.25 | +| Статус | Completed | +| Подсистема | Market Data Acquisition | +| Компонент | Trades Feed / Trade Stream Runtime | +| Дата завершения | 2026-07-31 | +| Версия | 1.0 | + +--- + +## Связанные документы + +- `build_060_24.md` — итог внутренней Runtime Recovery Architecture; +- `build_060_24_architecture.md` — архитектурные границы Recovery Runtime; +- `build_060_25_architecture.md` — полная спецификация Production Runtime + Integration и принятые ADR; +- `dzentra_target_architecture.md` — место Trade Stream в целевой + архитектуре Dzentra; +- `master-roadmap.md` — дальнейшая последовательность Build. + +--- + +## 1. Назначение Build + +Build 060.24 создал внутренний граф компонентов Trade Stream Runtime, +но не подключал его к реальному жизненному циклу приложения. + +Цель Build 060.25 — превратить этот граф в один управляемый production +Runtime: + +```text +Application Bootstrap + │ + ▼ +Production Trade Stream Factory + │ + ▼ +WebSocket connect and subscribe + │ + ▼ +Trade receive loop + │ + ├── ACK / control routing + ├── Canonical Trade pipeline + ├── Consistency checkpoint + └── Runtime Events + │ + ▼ +Heartbeat / Supervisor / Scheduler + │ + ▼ +reconnect → subscription restore → recovery + │ + ▼ +resume buffered live processing +``` + +Build сохраняет границы Transport, Acquisition, Consistency, Recovery +и Runtime. Внешний WebSocket документ становится Canonical Trade только +после прохождения существующего Acquisition pipeline. + +--- + +## 2. Завершённые подэтапы + +| Подэтап | Название | Статус | +|---|---|---| +| 060.25.0 | Static Contract Cleanup | Accepted | +| 060.25.1 | Dzengi WebSocket Transport | Accepted | +| 060.25.2 | WebSocket Session and Subscription Manager | Accepted | +| 060.25.3 | Async Runtime Event Publisher | Accepted | +| 060.25.4 | Trade Stream Production Runtime | Accepted | +| 060.25.5 | Reconnect and Recovery Integration | Accepted | +| 060.25.6 | Heartbeat and Scheduler Integration | Accepted | +| 060.25.7 | Settings, Bootstrap and Graceful Shutdown | Accepted | +| 060.25.8 | Targeted Runtime Verification | Accepted | + +Каждый подэтап прошёл отдельный архитектурный review. Обнаруженные +findings исправлялись до принятия соответствующего этапа. + +--- + +## 3. Итоговая ответственность компонентов + +### DzengiWebSocketTransport + +- открывает и закрывает WebSocket connection; +- отправляет и получает только `str | bytes`; +- выполняет конечный Ping/Pong liveness probe; +- не знает JSON, Trade, подписки и Recovery; +- использует отключённый встроенный WebSocket keepalive. + +### WebSocketSession + +- идемпотентно запускает и останавливает Transport; +- повторный `start()` не заменяет исправное OPEN-соединение; +- принудительная замена выполняется только явным reconnect. + +### WebSocketSubscriptionManager + +- разделяет желаемые и фактически активные подписки; +- сохраняет намерение подписаться после временной ошибки send; +- восстанавливает подписки после reconnect; +- не имитирует неподдерживаемый Dzengi unsubscribe. + +### Async Runtime Event Publisher + +- последовательно и с `await` доставляет событие всем Consumer; +- не создаёт скрытых background tasks; +- изолирует обычную ошибку Consumer от Runtime lifecycle; +- распространяет cancellation; +- защищён от прямой и child-task reentrancy; +- не включает чувствительный payload в аварийную диагностику. + +### TradeStreamProductionRuntime + +- является единственным владельцем startup, receive и Scheduler tasks; +- маршрутизирует control и market сообщения; +- передаёт Trade document в общий Acquisition/Consistency pipeline; +- связывает transport failure с reconnect и Recovery; +- выполняет полный cleanup при остановке, ошибке и cancellation. + +### RuntimeReconnectRecoveryCoordinator + +- обеспечивает single-flight для одного поколения соединения; +- закрывает общий live-processing gate; +- выполняет reconnect; +- восстанавливает подписки; +- запускает REST Recovery вне event loop; +- открывает live processing только после завершения Recovery; +- оставляет gate в failed state после terminal Recovery error. + +### Heartbeat, Supervisor и Scheduler + +- используют состояние Transport, а не частоту рыночных сделок; +- выполняют Ping/Pong с конечным timeout; +- объединяют одновременные heartbeat и receive failures в одну + reconnect/recovery operation; +- создают ровно одну Scheduler task; +- останавливаются в порядке Scheduler → Supervisor → receive loop. + +### Application Runner + +- владеет Telegram polling и опциональным Trade Stream Runtime; +- считает ошибку включённого Trade Stream фатальной; +- ожидает завершение обеих корневых задач; +- не позволяет повторной cancellation прервать cleanup; +- закрывает Bot session ровно один раз. + +--- + +## 4. Production lifecycle + +### Запуск + +```text +load settings + ↓ +feature flag disabled + └── Runtime graph не создаётся + +feature flag enabled + ↓ +build concrete dependency graph + ↓ +start WebSocket session + ↓ +send Trade subscription + ↓ +start Supervisor + ↓ +start receive loop and Scheduler +``` + +Фабрика только собирает граф зависимостей. Сетевые действия начинаются +только из `TradeStreamProductionRuntime.run()`. + +### Reconnect и Recovery + +```text +transport failure or failed liveness probe + ↓ +capture connection generation + ↓ +acquire single-flight operation + ↓ +close live-processing gate + ↓ +disconnect old connection + ↓ +connect new connection + ↓ +restore desired subscriptions + ↓ +recover missing Trades through REST + ↓ +process buffered live Trades +``` + +Recovery и live processing используют один экземпляр Consistency Layer +и не изменяют checkpoint параллельно. + +### Остановка + +```text +stop Scheduler + ↓ +await Scheduler task + ↓ +stop Supervisor + ↓ +cancel and await receive loop + ↓ +stop WebSocket session + ↓ +clear subscriptions + ↓ +close Bot session at Application boundary +``` + +--- + +## 5. Конфигурация + +Trade Stream управляется отдельным безопасным feature flag: + +```text +TRADE_STREAM_ENABLED +``` + +При включённом Runtime обязательны явные: + +- WebSocket URL; +- список символов; +- transport timeouts; +- heartbeat timeout; +- Scheduler interval; +- максимальный размер Recovery Window. + +Legacy WebSocket URL и `default_symbol` не используются как fallback. +Наличие `EXCHANGE_ENABLED` само по себе не включает Trade Stream. + +--- + +## 6. Ключевые архитектурные гарантии + +После Build 060.25 выполняются следующие правила: + +```text +exactly one production receive loop +``` + +```text +exactly one scheduler task +``` + +```text +at most one reconnect/recovery sequence per generation +``` + +```text +reconnect = + disconnect + → connect + → restore subscriptions +``` + +```text +recovery precedes buffered live processing +``` + +```text +disabled Trade Stream creates no Runtime graph +``` + +```text +construction has no network side effects +``` + +```text +all owned tasks are cancelled or completed and awaited +``` + +--- + +## 7. Реализованные файлы + +### Добавленные production-файлы + +```text +app/src/bootstrap/ + application.py + trade_stream_runtime.py + +app/src/market_data/acquisition/adapters/dzengi/ + websocket_control_message_handler.py + websocket_inbound_message_classifier.py + websocket_transport.py + +app/src/market_data/acquisition/runtime/ + acquisition_runtime_event_logging_consumer.py + acquisition_runtime_event_publisher.py + live_processing_gate.py + runtime_liveness_probe.py + runtime_reconnect_recovery_coordinator.py + trade_stream_production_runtime.py + websocket_inbound_message.py + websocket_session.py + websocket_subscription_manager.py +``` + +### Изменённые production-файлы + +```text +app/.env.example +app/src/bootstrap/app_factory.py +app/src/core/config.py +app/src/integrations/exchange/rest_client.py +app/src/main.py +app/src/market_data/acquisition/exceptions.py +app/src/market_data/acquisition/runtime/reconnect.py +app/src/market_data/acquisition/runtime/scheduler.py +app/src/market_data/acquisition/runtime/supervisor.py +app/src/market_data/acquisition/trade_stream_runtime_composition.py +``` + +### Тесты + +Добавлено или расширено покрытие: + +```text +app/tests/unit/bootstrap/ +app/tests/unit/core/ +app/tests/unit/market_data/acquisition/adapters/dzengi/ +app/tests/unit/market_data/acquisition/runtime/ +app/tests/unit/market_data/acquisition/ +app/tests/unit/test_main.py +``` + +Постороннее пользовательское изменение `.gitignore` не относится к +Build 060.25. + +--- + +## 8. Targeted Runtime Verification + +060.25.8 добавил только тестовый код и не менял production-поведение. + +Через реальную production factory и реальные компоненты Runtime graph, +но с управляемыми WebSocket и REST boundaries, проверены: + +1. выключенный feature flag не создаёт Runtime graph; +2. subscribe → ACK → Trade обновляет общий checkpoint; +3. ошибка connect фатальна и не оставляет задач; +4. ошибка первой subscription send полностью откатывает startup; +5. reconnect заменяет connection и восстанавливает подписку до Recovery; +6. recovered Trade обрабатывается раньше buffered live Trade; +7. Recovery error запрещает обработку buffered live Trade; +8. одновременные ошибки корневых задач наблюдаются детерминированно; +9. повторная cancellation не прерывает cleanup; +10. Bot session закрывается один раз, owned tasks всегда await-ятся. + +Итоговый read-only review 060.25.8 новых findings не выявил. + +--- + +## 9. Результаты тестирования + +Повторная проверка выполнена 2026-07-31: + +```text +Expanded Production Runtime target: 445 passed +Full project regression: 1869 passed +``` + +Тесты не используют реальную сеть, production credentials или +недетерминированные внешние задержки. + +--- + +## 10. Что не входит в Build + +Build 060.25 не реализует: + +- длительные live exchange и network fault scenarios; +- production stress certification; +- persistent market data storage; +- persistent checkpoint; +- startup recovery после перезапуска процесса; +- historical query и Replay API; +- аналитические вычисления поверх исторических данных. + +Эти задачи относятся к следующим Build и зафиксированы в +`master-roadmap.md`. + +--- + +## 11. Итог + +Build 060.25 завершён. + +Trade Stream подключён к bootstrap приложения как отдельный, +отключённый по умолчанию production Runtime. Он управляет реальным +WebSocket lifecycle, восстанавливает подписки и пропущенные сделки, +сохраняет порядок Recovery и Live processing и детерминированно +освобождает принадлежащие ему ресурсы. + +Следующий этап — Build 060.26, посвящённый интеграционным, fault, +reconnect, recovery и stress-сценариям за пределами детерминированного +in-process unit harness. diff --git a/docs/migrations/build_060_25_architecture.md b/docs/migrations/build_060_25_architecture.md new file mode 100644 index 0000000..cd9d94b --- /dev/null +++ b/docs/migrations/build_060_25_architecture.md @@ -0,0 +1,1637 @@ +# Build 060.25 — Production Runtime Integration Architecture + +**Статус:** Completed + +**Build:** 060.25 + +**Подсистема:** Market Data Acquisition / Trade Stream Runtime + +**Дата начала:** 2026-07-30 + +**Дата завершения:** 2026-07-31 + +**Версия документа:** 1.0 + +--- + +## 1. Назначение + +Документ фиксирует архитектуру Production Runtime Integration для +Trade Stream. + +Build 060.24 создал внутренний граф Runtime Recovery, но намеренно: + +- не создавал реальное WebSocket-соединение; +- не запускал receive loop; +- не управлял asyncio-задачами; +- не связывал reconnect с Trade Recovery; +- не подключал Runtime к bootstrap приложения. + +Build 060.25 превратил этот граф в управляемый production +lifecycle без нарушения границ Transport, Runtime, Acquisition, +Consistency и Recovery. + +Документ является живой архитектурной спецификацией. + +Он разделяет: + +- принятые и реализованные решения 060.25.0–060.25.8; +- end-to-end, reconnect и stress validation Build 060.26; +- persistent storage и startup recovery Build 060.27–060.28. + +Итоговый migration report зафиксирован в `build_060_25.md`. + +--- + +## 2. Статус подэтапов + +| Подэтап | Название | Статус | +|---|---|---| +| 060.25.0 | Static Contract Cleanup | Accepted | +| 060.25.1 | Dzengi WebSocket Transport | Accepted | +| 060.25.2 | WebSocket Session and Subscription Manager | Accepted | +| 060.25.3 | Async Runtime Event Publisher | Accepted | +| 060.25.4 | Trade Stream Production Runtime | Accepted | +| 060.25.5 | Reconnect and Recovery Integration | Accepted | +| 060.25.6 | Heartbeat and Scheduler Integration | Accepted | +| 060.25.7 | Settings, Bootstrap and Graceful Shutdown | Accepted | +| 060.25.8 | Targeted Runtime Verification | Accepted | + +Статус `Accepted` означает: + +- реализация прошла целевые тесты; +- архитектурные findings устранены; +- пользователь принял результат подэтапа; +- файлы ещё не зафиксированы отдельным Build commit. + +--- + +## 3. Исходная архитектура + +Build 060.24 предоставил: + +```text +TradeStreamRuntimeComposition +├── TradeStreamStateStore +├── TradeStreamConsistencyController +├── TradeRecoveryController +├── TradeRecoveryWindowPlanner +├── RuntimeRecoveryCoordinator +├── AcquisitionRuntimeService +├── TradeStreamAcquisitionService +├── ReconnectCoordinator +├── HeartbeatMonitor +├── RuntimeSupervisor +└── RuntimeScheduler +``` + +Composition создаёт единые экземпляры stateful-компонентов, но не +запускает их. + +Внешними зависимостями оставались: + +```text +WebSocketTransportProtocol +WebSocketSessionProtocol +WebSocketSubscriptionManagerProtocol +AcquisitionRuntimeEventPublisherProtocol +TradeStreamMessageAdapterProtocol +DzengiTradesDocumentSource +``` + +До Build 060.25 production-реализации первых четырёх WebSocket Runtime +контрактов отсутствовали. + +--- + +## 4. Целевая модель + +```text + APPLICATION BOOTSTRAP + │ + ▼ + Trade Stream Production Runtime + │ + ┌───────────────────────┼───────────────────────┐ + │ │ │ + ▼ ▼ ▼ + WebSocket lifecycle Runtime supervision Recovery gate + │ │ │ + ▼ ▼ ▼ + Dzengi Transport Heartbeat / Scheduler Runtime Recovery + │ │ │ + ▼ ▼ ▼ + WebSocket Session Runtime Supervisor REST Recovery + │ │ │ + ▼ ▼ │ + Subscription Manager ───► Reconnect Coordinator │ + │ │ + ▼ │ + receive loop │ + │ │ + ▼ ▼ + TradeStreamAcquisitionService ─► TradeStreamConsistencyController + │ + ▼ + TradeStreamStateStore +``` + +Production Runtime является владельцем выполнения, но не владельцем +бизнес-состояния сделок. + +--- + +## 5. Архитектурные границы + +### 5.1. Dzengi WebSocket Transport + +Transport отвечает только за: + +- нормализацию WebSocket URL; +- открытие и закрытие соединения; +- отправку `str | bytes`; +- получение `str | bytes`; +- transport timeouts и ping/pong параметры библиотеки; +- преобразование инфраструктурных ошибок в WebSocket transport errors. + +Transport не: + +- декодирует JSON; +- различает ACK и market events; +- строит subscription payload; +- изменяет Trade Stream state; +- выполняет reconnect policy; +- выполняет Recovery. + +### 5.2. WebSocket Session + +Session отвечает за: + +- идемпотентные `start()` и `stop()`; +- сериализацию lifecycle через `asyncio.Lock`; +- согласование локального состояния с Transport; +- повторный `start()` после фактического закрытия Transport. + +Session не должна превращать повторный `start()` в неявный force +reconnect. Принудительная замена соединения является явной операцией +ReconnectCoordinator. + +### 5.3. Subscription Manager + +Subscription Manager отвечает за: + +- desired subscription registry; +- active state текущего соединения; +- идемпотентную подписку; +- последовательное восстановление подписок; +- очистку runtime-состояния; +- явную политику unsupported unsubscribe. + +Manager работает с: + +```text +TransportTextMessage +TransportBinaryMessage +``` + +Он не анализирует содержимое payload и не знает о Trade, Quote, Candle +или конкретной JSON destination. + +### 5.4. Production Runtime + +Production Runtime должен отвечать за: + +- запуск и остановку полного графа; +- владение asyncio-задачами; +- единственный receive loop; +- JSON decoding; +- отделение control/ACK сообщений от market events; +- передачу market documents в Acquisition Service; +- координацию reconnect и recovery; +- graceful cancellation и shutdown. + +Production Runtime не должен дублировать: + +- Trade consistency; +- subscription registry; +- heartbeat calculations; +- recovery window planning; +- WebSocket transport logic. + +--- + +## 6. Build 060.25.0 — Static Contract Cleanup + +### 6.1. Исходная проблема + +Тестовый helper `create_coordinator()` принимал: + +```text +RecordingWindowPlanner | None +``` + +Негативный тест передавал: + +```text +BrokenPlanner(TradeRecoveryWindowPlanner) +``` + +Runtime-поведение было корректным, но Pylance обнаруживал несовместимый +тип аргумента. + +### 6.2. Принятое решение + +Тестовая иерархия согласована с реальной: + +```text +TradeRecoveryWindowPlanner + │ + ▼ +RecordingWindowPlanner + │ + ▼ +BrokenPlanner +``` + +Удалён локальный `type: ignore[arg-type]`. + +Production-контракт RuntimeRecoveryCoordinator не изменён. + +--- + +## 7. Build 060.25.1 — Dzengi WebSocket Transport + +### 7.1. Реализация + +Добавлен: + +```text +DzengiWebSocketTransport +``` + +Transport структурно соответствует: + +```text +WebSocketTransportProtocol +``` + +Прямое наследование от Protocol не используется, чтобы сохранить +рабочий `__slots__`. + +### 7.2. URL policy + +Поддерживаемые исходные схемы: + +```text +http → ws +https → wss +ws → ws +wss → wss +``` + +URL нормализуется до endpoint: + +```text +/connect +``` + +Примеры: + +```text +https://api-adapter.dzengi.com + ↓ +wss://api-adapter.dzengi.com/connect +``` + +```text +http://localhost:8080/ + ↓ +ws://localhost:8080/connect +``` + +Пустые, относительные, неподдерживаемые и fragment URL отклоняются до +сетевого вызова. + +### 7.3. Connection options + +Через constructor внедряются: + +```text +headers +open_timeout +ping_interval +ping_timeout +close_timeout +connector +``` + +Инъекция connector позволяет тестировать Transport без реальной сети. + +Transport использует WebSocket subprotocol: + +```text +json +``` + +### 7.4. Lifecycle + +```text +DISCONNECTED + │ connect() + ▼ +OPEN + │ disconnect() + ▼ +DISCONNECTED +``` + +Повторные `connect()` и `disconnect()` идемпотентны. + +Lifecycle сериализован отдельным `asyncio.Lock`. + +Transport считает соединение активным только если: + +```text +connection is not None +and connection.state is State.OPEN +``` + +### 7.5. Error policy + +Введены: + +```text +WebSocketTransportError +WebSocketTransportNotConnectedError +``` + +Connect, disconnect, send и receive errors оборачиваются с сохранением +исходной ошибки в `__cause__`. + +Попытка send или receive без открытого соединения завершается +`WebSocketTransportNotConnectedError`. + +`asyncio.CancelledError` не поглощается обработчиками `Exception`. + +--- + +## 8. Build 060.25.2 — WebSocket Session + +### 8.1. Реализация + +Добавлен: + +```text +WebSocketSession +``` + +Session структурно соответствует: + +```text +WebSocketSessionProtocol +``` + +### 8.2. Идемпотентность + +```text +start() + ├── Session и Transport connected → no-op + └── disconnected → transport.connect() +``` + +```text +stop() + ├── Session и Transport disconnected → no-op + └── active → transport.disconnect() +``` + +Параллельные lifecycle-вызовы сериализуются Session lock. + +После ошибки `start()` Session остаётся disconnected. + +После ошибки `stop()` локальный Session state гарантированно +сбрасывается в `finally`. + +### 8.3. Remote disconnect + +Session не доверяет только локальному `_started`. + +Состояние: + +```text +is_connected = + _started + and transport.is_connected +``` + +Если Transport обнаружил закрытие соединения, следующий `start()` +действительно открывает новое соединение. + +--- + +## 9. Build 060.25.2 — Subscription State + +### 9.1. Два уровня состояния + +Subscription Manager разделяет: + +```text +desired subscriptions +``` + +и: + +```text +active subscription keys +``` + +Desired registry описывает требуемое состояние Runtime и переживает +ошибку текущей сетевой попытки. + +Active keys относятся только к текущему WebSocket connection. + +### 9.2. Subscribe + +```text +validate key and message + │ + ▼ +store desired subscription + │ + ▼ +transport.send() + │ + ├── success → mark active + └── failure → keep pending desired subscription +``` + +Повторный вызов для active key является no-op. + +Повторный вызов для pending key выполняет новую отправку и может +обновить сохранённое transport message. + +### 9.3. Restore + +После нового соединения: + +```text +clear active keys + │ + ▼ +for each desired subscription in registration order: + transport.send() + │ + ▼ + mark key active +``` + +Ошибка восстановления не удаляет desired registry. + +### 9.4. Unsubscribe + +Публичный Dzengi endpoint не поддерживает надёжный +`trades.unsubscribe`. + +Поэтому production-default: + +```text +supports_unsubscribe = False +``` + +Попытка unsubscribe завершается: + +```text +WebSocketUnsubscribeNotSupportedError +``` + +и не изменяет registry. + +Generic supported branch сохраняется для будущих провайдеров. + +### 9.5. Clear + +`clear_subscriptions()` очищает desired и active state без отправки +сетевого сообщения. + +Это локальная lifecycle-операция, а не имитация exchange unsubscribe. + +--- + +## 10. Корректирующее решение Reconnect + +### 10.1. Обнаруженный риск + +Первоначальная реализация Build 060.24 выполняла: + +```text +ConnectCommand + │ + ▼ +restore_subscriptions() +``` + +Новая идемпотентная Session показала скрытую проблему. + +При heartbeat timeout WebSocket мог формально оставаться `OPEN`. +В таком состоянии `Session.start()` являлся no-op, и подписки повторно +отправлялись в старое соединение. + +### 10.2. Принятое решение + +Одна reconnect-попытка теперь выполняет: + +```text +ReconnectStartedEvent + │ + ▼ +DisconnectCommand + │ + ▼ +ConnectCommand + │ + ▼ +restore_subscriptions() + │ + ▼ +ReconnectCompletedEvent +``` + +Это гарантирует замену старого connection даже при формальном +`State.OPEN`. + +Registry подписок не очищается при disconnect, поэтому desired state +доступен для восстановления. + +### 10.3. Failure policy + +Ошибка disconnect, connect или restore: + +```text +state = FAILED + │ + ▼ +ReconnectFailedEvent + │ + ▼ +raise original exception +``` + +Retry loop и backoff по-прежнему находятся вне ReconnectCoordinator. + +--- + +## 11. Concurrency invariants + +### 11.1. Локальные locks + +```text +DzengiWebSocketTransport._lifecycle_lock +``` + +защищает connect/disconnect. + +```text +WebSocketSession._lifecycle_lock +``` + +защищает Session start/stop. + +```text +WebSocketSubscriptionManager._lock +``` + +защищает subscribe, unsubscribe, restore и clear. + +### 11.2. Lock ordering + +Допустимый порядок: + +```text +Session lock + │ + ▼ +Transport lifecycle lock +``` + +```text +Subscription Manager lock + │ + ▼ +Transport send +``` + +Transport не вызывает Session или Subscription Manager обратно. +Обратного lock ordering нет. + +### 11.3. Cross-component coordination + +Локальные locks не заменяют общий lifecycle gate. + +Production Runtime обязан исключить гонку: + +```text +live message processing + X +reconnect / recovery state mutation +``` + +Этот gate относится к 060.25.4–060.25.5. + +--- + +## 12. Build 060.25.3 — Async Runtime Event Publisher + +**Статус:** Accepted + +Реализованы: + +```text +AcquisitionRuntimeEventConsumerProtocol +AcquisitionRuntimeEventPublisher +AcquisitionRuntimeEventLoggingConsumer +``` + +`AcquisitionRuntimeEventPublisher` является production-реализацией: + +```text +AcquisitionRuntimeEventPublisherProtocol +``` + +### 12.1. Delivery model + +Publisher: + +- принимает immutable-набор Consumer через constructor injection; +- последовательно await-ит Consumer в порядке регистрации; +- сериализует конкурентные `publish()` одним `asyncio.Lock`; +- возвращается только после завершения fan-out; +- не создаёт `asyncio.Task`; +- не хранит event queue; +- не имеет собственного `start()` или `stop()`. + +Рекурсивная публикация через тот же Publisher отклоняется до входа +в lock. Для определения прямой и child-task reentrancy используется +`ContextVar`. + +### 12.2. Consumer failure policy + +Обычная ошибка одного Consumer: + +- не прерывает доставку остальным Consumer; +- не прерывает Heartbeat или Reconnect lifecycle; +- фиксируется только безопасными метаданными: + `event type`, `consumer type`, `error type`; +- не добавляет exception message, traceback или `repr(event)` в лог. + +Ошибка logging handler также не выходит из Publisher. + +`CancelledError` и другие `BaseException` не перехватываются. +Publisher освобождает lock и reentrancy context в `finally`, поэтому +может использоваться после отменённого вызова. + +### 12.3. Logging Consumer + +`AcquisitionRuntimeEventLoggingConsumer` использует стандартный +Python logging. + +Lifecycle и failure events получают соответствующие уровни +`INFO`, `WARNING` и `ERROR`. + +Для `MessageReceivedEvent` и `MessageSentEvent` на уровне `DEBUG` +фиксируются только: + +- тип transport message; +- размер payload. + +Содержимое payload не журналируется. + +### 12.4. Legacy Runtime Events + +Существующий `src/runtime_events/publisher.py` не изменён. + +Новый Publisher не зависит от: + +- `NotificationService`; +- Telegram; +- `JournalService`; +- legacy `RuntimeEvent`; +- market models. + +Преобразование Acquisition Runtime Event в legacy Runtime Event +в настоящий Build не добавлено. + +--- + +## 13. Build 060.25.4 — Trade Stream Production Runtime + +**Статус:** Accepted + +Реализован один высокоуровневый владелец lifecycle: + +TradeStreamProductionRuntime + +### 13.1. Задачи + +Production Runtime владеет: + +- startup sequence; +- receive task; +- lifecycle gate; +- cancellation; +- graceful shutdown; +- terminal error propagation. + +Startup, Scheduler и receive выполняются как явно сохранённые owned +tasks. +Внешний bootstrap остаётся владельцем корневой coroutine `run()`. + +Production Runtime синхронно закрепляет Scheduler за собой до первого +await сетевого startup. Поэтому другой lifecycle owner не может +запустить тот же Scheduler во время подключения или подписки. + +### 13.2. Receive loop + +Существует ровно один consumer: + +```text +transport.receive() + │ + ▼ +decode JSON + │ + ├── ACK / control → runtime control handling + ├── market document → TradeStreamAcquisitionService + └── invalid document → explicit error policy +``` + +ACK и control messages нельзя передавать в Unified Market Adapter как +неизвестный market event. + +Production Runtime создаёт correlation ID Trade subscription и +передаёт его через Acquisition Service. Provider-specific handler: + +- принимает только совпадающий correlation ID; +- подтверждает успешный ACK; +- идемпотентно принимает повторный успешный ACK; +- делает отрицательный ACK terminal error; +- отклоняет неизвестный correlation ID и malformed control message. + +### 13.3. Task ownership + +Production Runtime создаёт, хранит, отменяет и await-ит собственные +startup, Scheduler и receive tasks. + +`stop()` может отменить незавершившиеся: + +- Session startup; +- публикацию `ConnectedEvent`; +- отправку Trade subscription. + +Ни Transport, ни Session, ни Heartbeat не создают скрытые background +tasks на уровне приложения. + +--- + +## 14. Build 060.25.5 — Reconnect and Recovery + +**Статус:** Accepted + +### 14.1. Основная последовательность + +```text +connection failure or confirmed liveness timeout + │ + ▼ +acquire single reconnect/recovery gate + │ + ▼ +pause live message processing + │ + ▼ +disconnect old connection + │ + ▼ +connect new connection + │ + ▼ +restore desired subscriptions + │ + ▼ +capture recovery_end_time + │ + ▼ +run RuntimeRecoveryCoordinator + │ + ▼ +resume buffered live messages +``` + +### 14.2. Recovery execution + +`RuntimeRecoveryCoordinator.recover()` является синхронным и использует +блокирующий REST. + +В asyncio Runtime он должен выполняться: + +```text +await asyncio.to_thread(...) +``` + +либо через отдельный async adapter с эквивалентной семантикой. + +### 14.3. Shared state + +Live и Recovery используют один: + +```text +TradeStreamStateStore +TradeStreamConsistencyController +``` + +Поэтому live processing должен быть приостановлен на время Recovery. + +Одновременное изменение state из event loop и recovery thread +запрещено. + +### 14.4. Boundary policy + +После восстановления подписки новые WebSocket frames могут накапливаться +в transport buffer. + +Recovery обрабатывается первым, затем buffered live frames. + +Повтор на общей временной границе устраняется существующим Consistency +Layer. + +### 14.5. Single-flight и поколения connection + +Добавлен: + +```text +RuntimeReconnectRecoveryCoordinator +``` + +Он объединяет базовый `ReconnectCoordinator` и +`RuntimeRecoveryCoordinator` в одну single-flight операцию. + +Одновременные запросы reconnect присоединяются к уже выполняющейся +операции. Ошибка старого connection generation не запускает повторный +reconnect после того, как её поколение уже было обработано. + +### 14.6. Cancellation и ошибки Recovery + +Recovery запускается через `asyncio.to_thread()` при закрытом live gate. + +Если владеющая asyncio-задача отменена, Runtime продолжает ждать +фактического завершения Recovery worker. Повторные запросы отмены не +могут преждевременно открыть gate и оставить worker изменяющим +Consistency state в фоне. + +После ошибки reconnect или Recovery gate переходит в failed state. +Накопленные live messages не передаются в Consistency Layer. Failed +state очищается только при начале нового полного Production Runtime +lifecycle. + +### 14.7. Composition identity + +Composition создаёт один экземпляр live gate и передаёт один +`RuntimeReconnectRecoveryCoordinator` в `RuntimeSupervisor`. + +Production Runtime проверяет: + +- identity общего live gate; +- равенство нормализованного набора symbols для Live и Recovery. + +Несогласованный dependency graph отклоняется до запуска WebSocket +lifecycle. + +--- + +## 15. Build 060.25.6 — Heartbeat and Scheduler + +**Статус:** Accepted + +### 15.1. Liveness + +Отсутствие сделок не означает потерю соединения. + +Для тихого symbol допустима длительная пауза между market events. + +Поэтому reconnect нельзя запускать только по условию: + +```text +no Trade messages +``` + +Liveness подтверждается двумя transport-level источниками: + +- успешным WebSocket Ping/Pong; +- успешно полученным WebSocket message независимо от его market/control + назначения. + +### 15.2. Реализованная граница + +Реализован отдельный контракт: + +```text +RuntimeLivenessProbeProtocol + │ + ▼ +WebSocket ping/pong implementation + │ + ├── success → supervisor.notify_activity() + └── failure → reconnect path +``` + +`DzengiWebSocketTransport.probe()`: + +- возвращает `True` только после соответствующего Pong; +- возвращает `False`, если connection отсутствует, закрыт или Pong не + получен вовремя; +- не перехватывает cancellation; +- распространяет неожиданную ошибку открытого connection как + `WebSocketTransportError`. + +Ожидание Pong всегда имеет конечную границу `probe_timeout`. + +Если отдельное значение не передано, оно берётся из конечного +`ping_timeout`. При отключённом `ping_timeout` применяется безопасное +значение 20 секунд. Неположительное и бесконечное значение запрещено. + +### 15.3. Scheduler + +Scheduler продолжает владеть только временем вызовов. + +Каждая итерация: + +1. выполняет transport probe; +2. при успешном Pong уведомляет Supervisor об активности; +3. проверяет Heartbeat timeout; +4. передаёт подтверждённый timeout Supervisor. + +Он не должен: + +- отправлять WebSocket payload самостоятельно; +- изменять subscription registry; +- выполнять Trade Recovery; +- владеть receive loop. + +### 15.4. Reconnect generation + +Supervisor связывает каждый Heartbeat monitoring period с generation +текущего connection. + +Heartbeat timeout передаётся в +`reconnect_after_transport_failure(observed_generation=...)`. + +Поэтому timeout и receive error одного поколения: + +- присоединяются к одной активной reconnect/recovery operation; +- используют результат уже завершённой operation; +- не запускают повторный reconnect после восстановления того же + connection generation. + +### 15.5. Task ownership и shutdown + +Production Runtime выполняет `Scheduler.claim(owner)` до первого +сетевого await и освобождает claim только после завершения owned task. + +Это исключает запуск Scheduler вторым владельцем во время startup. + +Порядок остановки: + +```text +Scheduler stop / cancel / await + ↓ +Supervisor stop + ↓ +receive task cancel / await + ↓ +Session stop +``` + +Scheduler-led reconnect/recovery достигает безопасной границы до +остановки Supervisor. Receive-led operation безопасно завершается при +последующем cancel/await receive task и до остановки Session. + +--- + +## 16. Build 060.25.7 — Settings and Bootstrap + +**Статус:** Accepted + +### 16.1. Settings + +Добавлена отдельная неизменяемая группа: + +```text +TradeStreamSettings +├── enabled +├── websocket_url +├── symbols +├── open_timeout_seconds +├── probe_timeout_seconds +├── close_timeout_seconds +├── heartbeat_timeout_seconds +├── scheduler_interval_seconds +└── recovery_window_ms +``` + +Production Trade Stream должен иметь отдельный feature flag с безопасным +значением по умолчанию. + +Общий `EXCHANGE_ENABLED` не должен неявно запускать новый Runtime без +явной настройки. + +Используются переменные: + +```text +TRADE_STREAM_ENABLED=false +TRADE_STREAM_WS_URL +TRADE_STREAM_SYMBOLS +TRADE_STREAM_OPEN_TIMEOUT_SECONDS=10 +TRADE_STREAM_PROBE_TIMEOUT_SECONDS=20 +TRADE_STREAM_CLOSE_TIMEOUT_SECONDS=10 +TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS=30 +TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS=5 +TRADE_STREAM_RECOVERY_WINDOW_MS=3599999 +``` + +При выключенном flag зависимые Trade Stream значения не проверяются и +не создают Runtime. При включённом flag обязательны отдельные +`TRADE_STREAM_WS_URL`, `TRADE_STREAM_SYMBOLS` и `EXCHANGE_BASE_URL` для +REST Recovery. + +Fallback с `EXCHANGE_WS_URL` и `DEFAULT_SYMBOL` запрещён. +`EXCHANGE_BASE_URL` и `EXCHANGE_TIMEOUT_SEC` используются только +явным REST-клиентом Recovery. + +Все интервалы должны быть положительными и конечными, recovery window — +положительным целым числом. Символы очищаются от пробелов, +дедуплицируются и сортируются. + +### 16.2. Bootstrap + +Добавлен отдельный production composition root: + +```text +build_trade_stream_production_runtime(settings) + │ + ├── feature flag disabled → None + │ + └── enabled + ├── DzengiWebSocketTransport + ├── WebSocketSession + ├── WebSocketSubscriptionManager + ├── AcquisitionRuntimeEventPublisher + ├── DzengiTradesDocumentSource + ├── TradeStreamRuntimeComposition + └── TradeStreamProductionRuntime +``` + +Factory использует один снимок `Settings`. Создание графа: + +- не открывает WebSocket; +- не выполняет REST; +- не создаёт asyncio-задачи; +- сохраняет identity общих stateful-зависимостей. + +Встроенный keepalive библиотеки `websockets` отключён: + +```text +ping_interval=None +ping_timeout=None +``` + +Единственным владельцем активного Ping/Pong является Runtime Scheduler +через конечный `probe_timeout`. + +Корневой `ApplicationComposition` содержит Telegram Bot, Dispatcher и +опциональный Trade Stream Runtime. `run_application()` создаёт и +контролирует обе root tasks. + +Ошибка или неожиданное нормальное завершение включённого Trade Stream +фатальны для всего процесса. Ошибка Telegram polling симметрично +останавливает Trade Stream. Исходная ошибка сохраняет приоритет над +ошибками cleanup. + +`main.py` не должен содержать детали WebSocket protocol или recovery +алгоритма. + +### 16.3. Shutdown + +Внешний порядок приложения: + +```text +stop accepting Telegram work + │ + ▼ +TradeStreamProductionRuntime.stop() + │ + ▼ +await both root tasks + │ + ▼ +close bot session exactly once +``` + +Внутренний порядок Trade Stream Runtime: + +```text +Scheduler stop / cancel / await + │ + ▼ +Supervisor stop + │ + ▼ +receive task cancel / await + │ + ▼ +Session stop + │ + ▼ +clear subscriptions +``` + +Application Runner передаёт Aiogram параметр +`close_bot_session=False`, поэтому сессию закрывает ровно один корневой +владелец. + +Cancellation не прерывает начатый cleanup: cleanup выполняется в +отдельной owned task и ожидается через shield. Все созданные root tasks +отменяются либо завершаются и обязательно await-ятся. + +--- + +## 17. Build 060.25.8 — Verification + +**Статус:** Accepted + +060.25.8 не добавляет production-поведение. Этап подтверждает +детерминированными in-process тестами, что принятые компоненты работают +как один production graph. + +Добавлен управляемый test harness: + +```text +Production Trade Stream Factory + │ + ├── real DzengiWebSocketTransport + ├── real WebSocketSession + ├── real WebSocketSubscriptionManager + ├── real Runtime Composition + └── real TradeStreamProductionRuntime + │ + ├── controlled fake WebSocket connector + ├── controlled fake REST client + └── fake Telegram boundary +``` + +Harness не использует реальную сеть, реальные credentials или +недетерминированные внешние задержки. + +Подтверждены следующие vertical-slice сценарии: + +1. выключенный feature flag не создаёт Runtime graph; +2. production factory выполняет subscribe → ACK → Trade → checkpoint; +3. ошибка WebSocket connect фатальна и не оставляет задач; +4. ошибка первой subscription send откатывает partial startup; +5. reconnect открывает новое connection и восстанавливает subscription + до запуска REST Recovery; +6. recovered Trade обрабатывается раньше buffered Live Trade; +7. ошибка Recovery оставляет gate в failed state и отклоняет buffered + Trade; +8. одновременные ошибки Telegram и Trade Stream обрабатываются + детерминированно; +9. повторная cancellation не прерывает начатый cleanup; +10. Bot session закрывается один раз, owned tasks всегда await-ятся. + +Production-код в 060.25.8 не изменён. + +В Build 060.26 остаются: + +- live exchange integration; +- длительные reconnect scenarios; +- recovery scenarios с реальными задержками; +- stress testing; +- network fault injection; +- финальная Runtime documentation verification. + +Unit tests по умолчанию не должны обращаться к внешней сети. + +--- + +## 18. Реализованные файлы + +### 18.1. Production code + +Добавлены: + +```text +app/src/market_data/acquisition/adapters/dzengi/ + websocket_control_message_handler.py + websocket_inbound_message_classifier.py + websocket_transport.py + +app/src/market_data/acquisition/runtime/ + acquisition_runtime_event_logging_consumer.py + acquisition_runtime_event_publisher.py + live_processing_gate.py + runtime_liveness_probe.py + runtime_reconnect_recovery_coordinator.py + trade_stream_production_runtime.py + websocket_inbound_message.py + websocket_session.py + websocket_subscription_manager.py + +app/src/bootstrap/ + application.py + trade_stream_runtime.py +``` + +Изменены: + +```text +app/.env.example +app/src/bootstrap/app_factory.py +app/src/core/config.py +app/src/integrations/exchange/rest_client.py +app/src/main.py +app/src/market_data/acquisition/exceptions.py +app/src/market_data/acquisition/runtime/reconnect.py +app/src/market_data/acquisition/runtime/scheduler.py +app/src/market_data/acquisition/runtime/supervisor.py +app/src/market_data/acquisition/trade_stream_runtime_composition.py +``` + +### 18.2. Tests + +Добавлены: + +```text +app/tests/unit/market_data/acquisition/adapters/dzengi/ + test_websocket_control_message_handler.py + test_websocket_inbound_message_classifier.py + test_websocket_transport.py + +app/tests/unit/market_data/acquisition/runtime/ + test_acquisition_runtime_event_logging_consumer.py + test_acquisition_runtime_event_publisher.py + test_live_processing_gate.py + test_runtime_reconnect_recovery_coordinator.py + test_trade_stream_production_runtime.py + test_websocket_session.py + test_websocket_subscription_manager.py + test_websocket_runtime_reconnect_integration.py + +app/tests/unit/bootstrap/ + test_app_factory.py + test_application.py + test_trade_stream_production_integration.py + test_trade_stream_runtime.py + +app/tests/unit/core/ + test_config.py + +app/tests/unit/test_main.py +``` + +Изменены: + +```text +app/tests/unit/market_data/acquisition/runtime/ + test_reconnect_coordinator.py + test_runtime_recovery_coordinator.py + test_runtime_scheduler.py + test_runtime_supervisor.py + +app/tests/unit/market_data/acquisition/ + test_trade_stream_runtime_composition.py +``` + +Состав файлов соответствует итоговому состоянию Build 060.25. + +--- + +## 19. Test evidence + +После реализации 060.25.8 выполнены: + +```text +Settings / Bootstrap tests: 30 passed +New vertical / lifecycle scenarios: 8 passed +Expanded Production Runtime target: 445 passed +Full project regression: 1869 passed +``` + +Расширенный целевой набор включает: + +- Trade Stream settings; +- Application Composition, factory и `main.py`; +- production factory vertical-slice integration; +- весь `market_data/acquisition/runtime`; +- Dzengi WebSocket Transport, classifier и control handler; +- Trade Stream Runtime Composition. + +Отдельно проверены: + +```text +OPEN connection + │ reconnect() + ▼ +old connection closed + │ + ▼ +new connection opened + │ + ▼ +subscriptions restored +``` + +и: + +```text +initial subscribe send failure + │ + ▼ +desired subscription remains pending + │ reconnect() + ▼ +subscription sent through new connection +``` + +Для Async Runtime Event Publisher отдельно подтверждены: + +- последовательная доставка; +- сериализация конкурентных публикаций; +- отсутствие fire-and-forget; +- consumer error isolation; +- отсутствие payload в error log; +- устойчивость к ошибке logging handler; +- распространение cancellation; +- повторное использование после cancellation; +- защита от прямой и child-task reentrancy; +- сохранение Heartbeat и Reconnect lifecycle при ошибке Consumer. + +После read-only review 060.25.8 целевой набор и полная регрессия +повторены 2026-07-31. Новых findings не обнаружено. + +--- + +## 20. Принятые архитектурные решения + +### ADR-060.25-001 — Transport остаётся raw boundary + +**Статус:** Accepted + +DzengiWebSocketTransport работает только с `str | bytes` и не содержит +JSON или market routing. + +### ADR-060.25-002 — Session start является идемпотентным + +**Статус:** Accepted + +Повторный start не заменяет исправное открытое соединение. + +### ADR-060.25-003 — Reconnect всегда выполняет disconnect + +**Статус:** Accepted + +Force reconnect выражен последовательностью Disconnect → Connect, а не +скрытым поведением Session.start(). + +### ADR-060.25-004 — Subscription registry хранит desired state + +**Статус:** Accepted + +Сетевой send failure не удаляет намерение подписаться. Desired и active +state разделены. + +### ADR-060.25-005 — Unsupported unsubscribe является явной ошибкой + +**Статус:** Accepted + +Dzengi unsubscribe не имитируется локальным успешным результатом. + +### ADR-060.25-006 — Production Runtime владеет задачами + +**Статус:** Accepted + +Внешний bootstrap владеет корневой task `run()`. Production Runtime +до сетевого startup закрепляет Scheduler за собой, а затем создаёт, +хранит, отменяет и await-ит собственные startup, Scheduler и receive +tasks. Claim освобождается после завершения Scheduler task. + +### ADR-060.25-007 — Recovery выполняется при закрытом live gate + +**Статус:** Accepted + +REST Recovery и live message processing не изменяют общий Consistency +state параллельно. Отмена ожидает фактического завершения Recovery +worker, а ошибка Recovery блокирует buffered live processing до нового +Runtime lifecycle. + +### ADR-060.25-008 — Heartbeat основан на transport liveness + +**Статус:** Accepted + +Отсутствие Trade events не является достаточным признаком разрыва. +Liveness подтверждается конечным Ping/Pong probe либо успешно полученным +WebSocket message. Timeout и receive error используют одно connection +generation и одну single-flight reconnect/recovery operation. + +### ADR-060.25-009 — Runtime Events доставляются inline + +**Статус:** Accepted + +Publisher последовательно await-ит фиксированный набор Consumer и не +создаёт скрытую очередь или background task. + +### ADR-060.25-010 — Consumer errors не управляют Runtime lifecycle + +**Статус:** Accepted + +Обычная ошибка Consumer изолируется, безопасно диагностируется и не +прерывает доставку остальным Consumer. Cancellation распространяется. + +### ADR-060.25-011 — Trade Stream включается только отдельным flag + +**Статус:** Accepted + +`TRADE_STREAM_ENABLED` по умолчанию выключен. `EXCHANGE_ENABLED` не +запускает новый Runtime, а legacy WebSocket URL и default symbol не +используются как fallback. + +### ADR-060.25-012 — Ошибка включённого Runtime фатальна + +**Статус:** Accepted + +Telegram polling и Trade Stream принадлежат одному Application Runner. +Terminal error любой включённой root task останавливает вторую root +task, после чего обе задачи ожидаются и сессия Bot закрывается ровно +один раз. + +### ADR-060.25-013 — Scheduler является единственным keepalive owner + +**Статус:** Accepted + +Автоматический keepalive `websockets` отключён. Ping/Pong выполняется +только Scheduler через явный transport liveness probe с конечным +timeout. + +--- + +## 21. Инварианты Build + +К завершению 060.25 должны выполняться: + +```text +exactly one production receive loop +``` + +```text +exactly one scheduler task +``` + +```text +at most one reconnect/recovery sequence +``` + +```text +reconnect = + disconnect + → connect + → restore subscriptions +``` + +```text +recovery and live state mutation are not concurrent +``` + +```text +desired subscriptions survive transient send failure +``` + +```text +quiet market does not cause false reconnect +``` + +```text +startup and shutdown are deterministic and awaitable +``` + +```text +Composition construction has no network side effects +``` + +```text +disabled Trade Stream creates no Runtime graph +``` + +```text +application owns and awaits both root tasks +``` + +```text +bot session is closed exactly once +``` + +```text +runtime event delivery is ordered and awaitable +``` + +```text +consumer failure does not interrupt runtime lifecycle +``` + +--- + +## 22. Не входит в Build + +Build 060.25 не реализует: + +- persistent market data storage; +- persistent checkpoint; +- startup recovery после перезапуска процесса; +- historical query API; +- replay API; +- analytics API; +- общий retry framework приложения; +- production stress certification. + +Эти задачи относятся к Build 060.26–060.29. + +--- + +## 23. Критерии завершения + +Build 060.25 получил статус `Completed`, поскольку: + +- все этапы 060.25.0–060.25.8 приняты; +- concrete Runtime dependencies собраны через composition root; +- Trade Stream запускается и останавливается из bootstrap; +- receive loop передаёт Trade messages в общий Consistency Layer; +- ACK/control messages не разрушают market pipeline; +- reconnect действительно заменяет connection; +- desired subscriptions восстанавливаются; +- Recovery выполняется до возобновления live processing; +- heartbeat не зависит только от частоты сделок; +- cancellation не оставляет фоновые задачи; +- целевые тесты и полная регрессия проходят; +- `git diff --check` не содержит новых ошибок; +- создан итоговый `build_060_25.md`; +- настоящий документ обновлён до финального фактического состояния. + +--- + +## 24. Текущий итог + +На момент версии 1.0: + +```text +060.25.0 Accepted +060.25.1 Accepted +060.25.2 Accepted +060.25.3 Accepted +060.25.4 Accepted +060.25.5 Accepted +060.25.6 Accepted +060.25.7 Accepted +060.25.8 Accepted +``` + +Реализован production-ready фундамент WebSocket lifecycle: + +- concrete Dzengi Transport; +- идемпотентная Session; +- desired/active Subscription Manager; +- принудительная замена connection во время reconnect; +- интеграционные unit-сценарии двух критических recovery cases; +- последовательный Async Runtime Event Publisher; +- безопасный Logging Consumer; +- изоляция Consumer errors от Heartbeat и Reconnect lifecycle; +- единый Trade Stream Production Runtime; +- отменяемые и awaitable startup/receive tasks; +- явная маршрутизация и проверка Trade subscription ACK; +- single-flight reconnect → subscription restore → Recovery; +- общий live/recovery gate с блокировкой buffered сообщений после + Recovery failure; +- безопасное ожидание Recovery worker при повторной cancellation; +- проверка единого набора Live и Recovery symbols; +- конечный WebSocket Ping/Pong liveness probe; +- generation-aware объединение heartbeat и receive reconnect; +- единственная owned Scheduler task; +- детерминированная остановка Scheduler → Supervisor → receive loop; +- отдельный безопасный Trade Stream feature flag; +- production composition root без сетевых side effects; +- один Application Runner для Telegram и Trade Stream; +- фатальность terminal error включённого Trade Stream; +- явные WebSocket URL и symbols без legacy fallback; +- отключённый встроенный WebSocket keepalive; +- детерминированное ожидание root tasks и однократное закрытие Bot + session. + +Runtime подключён к bootstrap приложения и прошёл финальную +детерминированную in-process verification. Отдельный read-only review +060.25.8 не выявил findings; целевой набор из 445 тестов и полная +регрессия из 1869 тестов прошли. Build 060.25 завершён. diff --git a/docs/roadmap/master-roadmap.md b/docs/roadmap/master-roadmap.md index 2cb59c6..2f15117 100644 --- a/docs/roadmap/master-roadmap.md +++ b/docs/roadmap/master-roadmap.md @@ -1,12 +1,245 @@ -# Master Roadmap — Dzentra Bot +# Master Roadmap — Dzentra + +## Контроль документа + +| Свойство | Значение | +|---|---| +| Тип | Master Delivery Roadmap | +| Статус | Active | +| Версия | 2.0 | +| Дата актуализации | 2026-07-31 | +| Текущий завершённый Build | 060.25 | +| Следующий Build | 060.26 | + +--- ## Цель проекта -Создать Telegram-бота для: -- ручной торговли; -- мониторинга рынка; -- автоторговли; -- аналитики; -- управления стратегиями. + +Dzentra развивается из работающего Telegram trading bot в модульную +алгоритмическую торговую платформу, которая поддерживает: + +- получение и хранение рыночных данных; +- Market Intelligence и объяснимую аналитику; +- ручные и автоматические торговые решения; +- управление риском, исполнением и позициями; +- аудит и воспроизводимость решений; +- безопасный production runtime; +- исторический Replay и Backtesting. + +Целевая карта доменов и направление зависимостей определены в: + +```text +docs/architecture/dzentra_target_architecture.md +``` + +Настоящий roadmap определяет порядок реализации. Подробные решения и +test evidence находятся в документах конкретных Build. + +--- + +# Текущая контрольная точка + +```text +Market Data Acquisition + ↓ +Trades Feed / Time & Sales + ↓ +Build 060.25 — Production Runtime Integration + ↓ +Completed +``` + +060.25.0–060.25.8 приняты. Trade Stream: + +- подключён к application bootstrap через отдельный feature flag; +- владеет WebSocket lifecycle; +- восстанавливает подписки и пропущенные Trades; +- сохраняет порядок Recovery → buffered Live; +- использует transport Ping/Pong liveness; +- детерминированно останавливает и ожидает owned tasks; +- прошёл 445 целевых тестов и полную регрессию из 1869 тестов. + +Подробности: + +```text +docs/migrations/build_060_25_architecture.md +docs/migrations/build_060_25.md +``` + +--- + +# Активная программа — Market Data Acquisition + +## Завершённая ветка Trades Feed + +| Build | Результат | Статус | +|---|---|---| +| 060.1–060.19 | Canonical Trade, REST/WS pipelines, Consistency и Recovery foundation | Completed | +| 060.20 | Trade Runtime Architecture | Completed | +| 060.20.1 | Trade Stream State Ownership Alignment | Completed | +| 060.21 | Runtime Protocol Integration | Completed | +| 060.22 | Runtime Service Integration | Completed | +| 060.23 | Trade Stream Acquisition Integration | Completed | +| 060.24 | Runtime Recovery Architecture | Completed | +| 060.25 | Production Runtime Integration | Completed | + +## Следующие Build + +### Build 060.26 — Integration and Regression + +**Статус:** Next + +Назначение: + +- live exchange integration checks; +- длительные reconnect scenarios; +- recovery при реальных сетевых задержках; +- network fault injection; +- stress и soak testing; +- проверка отсутствия утечек задач и ресурсов; +- финальная verification Runtime documentation. + +Build не должен превращать сетевые сценарии в обязательную часть +обычного unit suite. + +### Build 060.27 — Persistent Market Data Storage + +**Статус:** Planned + +Назначение: + +- долговременное хранение Canonical Trades; +- подготовка хранения Quotes и Candles; +- ключи идемпотентности и ordering; +- raw/canonical retention policy; +- Storage API; +- партиционирование, retention и data provenance. + +Результат: история рынка сохраняется независимо от торгового цикла и +времени жизни процесса. + +### Build 060.28 — Persistent Checkpoint and Startup Recovery + +**Статус:** Planned + +Назначение: + +- Checkpoint Service; +- сохранение runtime checkpoint; +- восстановление после перезапуска; +- сверка checkpoint с durable Market Data; +- Startup Recovery; +- защита от повторной обработки и пропусков. + +Persistent checkpoint не заменяет Market Data Storage и не должен +считаться более достоверным, чем подтверждённая сохранённая история. + +### Build 060.29 — Market Data Access and Replay + +**Статус:** Planned + +Назначение: + +- Data Access Layer; +- Historical Queries; +- Replay API; +- детерминированные replay-часы; +- одинаковые Canonical contracts для live и replay consumers. + +Полноценные аналитические вычисления не входят автоматически в этот +Build. Они принадлежат Market Data Processing и Feature Engineering и +получат отдельный scope после появления устойчивого Storage/Replay. + +### Build 060.30 — Market Data Acquisition Final Documentation + +**Статус:** Planned + +Назначение: + +- итоговый аудит ветки Trades Feed; +- проверка контрактов и направления импортов; +- сверка фактической структуры файлов; +- эксплуатационная документация Runtime; +- обновление architecture overview и project structure; +- фиксация известных ограничений и следующей Feed-ветки. + +### Build 061.00 — Validation and Canonicalization Boundary Refactoring + +**Статус:** Planned + +Сначала выполняется read-only аудит фактического ownership: + +```text +schema validation +parser +value validation +normalization +mapper +canonical model +consistency +data quality +missing data recovery +``` + +Только после аудита определяется, нужен ли отдельный каталог +`market_data/normalization`. Механический перенос существующих +валидаторов не является целью Build. + +--- + +# Дальнейшие программы + +После 061.00 следующая Feed-ветка выбирается на основании фактических +production consumers и доступных данных Dzengi. + +Кандидаты: + +- Quotes lifecycle consolidation; +- OHLCV/Candles production integration; +- Order Book snapshot and delta reconciliation; +- Derivatives Market Data; +- Market Index; +- Exchange Time; +- Exchange and Instrument Status. + +Нумерация и точный scope этих Build утверждаются только после отдельного +read-only архитектурного анализа. Старый ориентировочный план из +`build_044.md` сохраняет историческую ценность, но больше не является +актуальной нумерацией. + +Следующие верхнеуровневые программы пока не имеют утверждённых Build: + +- Market Data Processing; +- Feature Engineering; +- Market Intelligence consolidation; +- Scenario Evaluation; +- Portfolio and Risk boundary; +- Order Management System; +- Position and Account Reconciliation; +- Backtesting and Simulation; +- Production Deployment and Operations. + +--- + +# Правила ведения roadmap + +1. Один Build изменяет одну подтверждённую архитектурную область. +2. Перед Build выполняется read-only анализ фактического кода и тестов. +3. Дальние номера являются ориентиром до утверждения architecture plan. +4. Каждый Build имеет критерии завершения и проверяемый test evidence. +5. Архитектурные findings исправляются до статуса Accepted. +6. Итоговый migration report создаётся после завершения Build. +7. Master roadmap хранит только последовательность и статус, а не + дублирует подробную архитектурную спецификацию. +8. Посторонние изменения рабочего каталога не включаются в Build. + +--- + +# Исторический roadmap Stage 01–09 + +Раздел ниже сохраняет историю развития первоначального Telegram bot. +Его локальные пометки `в работе`, `не начат` и прежняя нумерация +являются историческим снимком, а не текущим источником следующего шага. --- @@ -135,7 +368,7 @@ ✔ minimal trading layout ✔ duplicate info removal -#### 07.4.3.2 — Engine Decoupling (NEXT) ✅ +#### 07.4.3.2 — Engine Decoupling ✅ ✔ split analysis / UI refresh ✔ fast price polling (1s) ✔ slow UI updates (event-driven / 60s) @@ -1491,5 +1724,12 @@ ## Текущий статус проекта -👉 Завершён: 07.4.3.1 -👉 Следующий шаг: 07.4.3.2 — Engine Decoupling + Price Polling \ No newline at end of file +Этот блок исторического Stage-roadmap больше не определяет текущий +следующий шаг. + +Актуальная контрольная точка: + +```text +Завершён: Build 060.25 — Production Runtime Integration +Следующий: Build 060.26 — Integration and Regression +```