From c14214536109d385598e35a2ae693239508ddae2 Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 30 Jul 2026 00:17:12 +0300 Subject: [PATCH] Build 060.24: implement Runtime Recovery Architecture --- .../consistency/trade_stream_state.py | 6 +- .../recovery/trade_recovery_window.py | 51 + .../recovery/trade_recovery_window_planner.py | 147 +++ .../acquisition/runtime/heartbeat.py | 192 ++++ .../acquisition/runtime/reconnect.py | 187 ++- .../runtime/runtime_recovery_coordinator.py | 295 +++++ .../runtime/runtime_recovery_protocol.py | 72 ++ .../acquisition/runtime/scheduler.py | 199 ++++ .../acquisition/runtime/supervisor.py | 204 +++- .../trade_stream_runtime_composition.py | 184 +++ .../consistency/test_trade_stream_state.py | 169 ++- .../recovery/test_trade_recovery_window.py | 158 +++ .../test_trade_recovery_window_planner.py | 380 +++++++ .../runtime/test_heartbeat_monitor.py | 466 ++++++++ .../runtime/test_reconnect_coordinator.py | 349 ++++++ .../test_runtime_recovery_coordinator.py | 989 ++++++++++++++++ .../runtime/test_runtime_scheduler.py | 590 ++++++++++ .../runtime/test_runtime_supervisor.py | 528 +++++++++ .../test_trade_stream_runtime_composition.py | 752 ++++++++++++ docs/migrations/build_060_24.md | 552 +++++++++ docs/migrations/build_060_24_architecture.md | 1010 +++++++++++++++++ 21 files changed, 7475 insertions(+), 5 deletions(-) create mode 100644 app/src/market_data/acquisition/recovery/trade_recovery_window.py create mode 100644 app/src/market_data/acquisition/recovery/trade_recovery_window_planner.py create mode 100644 app/src/market_data/acquisition/runtime/runtime_recovery_coordinator.py create mode 100644 app/src/market_data/acquisition/runtime/runtime_recovery_protocol.py create mode 100644 app/src/market_data/acquisition/trade_stream_runtime_composition.py create mode 100644 app/tests/unit/market_data/acquisition/recovery/test_trade_recovery_window.py create mode 100644 app/tests/unit/market_data/acquisition/recovery/test_trade_recovery_window_planner.py create mode 100644 app/tests/unit/market_data/acquisition/runtime/test_heartbeat_monitor.py create mode 100644 app/tests/unit/market_data/acquisition/runtime/test_reconnect_coordinator.py create mode 100644 app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py create mode 100644 app/tests/unit/market_data/acquisition/runtime/test_runtime_scheduler.py create mode 100644 app/tests/unit/market_data/acquisition/runtime/test_runtime_supervisor.py create mode 100644 app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py create mode 100644 docs/migrations/build_060_24.md create mode 100644 docs/migrations/build_060_24_architecture.md diff --git a/app/src/market_data/acquisition/consistency/trade_stream_state.py b/app/src/market_data/acquisition/consistency/trade_stream_state.py index cdca0a4..81122e8 100644 --- a/app/src/market_data/acquisition/consistency/trade_stream_state.py +++ b/app/src/market_data/acquisition/consistency/trade_stream_state.py @@ -5,11 +5,11 @@ from __future__ import annotations from collections import deque from dataclasses import dataclass, field -from src.market_data.acquisition.models.trade import Trade from src.market_data.acquisition.consistency.trade_stream_exceptions import ( TradeConsistencyError, TradeOrderingError, ) +from src.market_data.acquisition.models.trade import Trade DEFAULT_DEDUPLICATION_WINDOW_SIZE = 10_000 @@ -25,6 +25,7 @@ class TradeStreamState: deduplication_window_size: int = DEFAULT_DEDUPLICATION_WINDOW_SIZE last_trade_id: int | None = None + last_trade: Trade | None = None _trade_window: deque[int] = field(init=False, repr=False) _trades: dict[int, Trade] = field(init=False, repr=False) @@ -88,6 +89,7 @@ class TradeStreamState: self._append(trade) self.last_trade_id = trade_id + self.last_trade = trade return trade @@ -100,4 +102,4 @@ class TradeStreamState: self._trades.pop(oldest_trade_id, None) self._trade_window.append(trade.trade_id) - self._trades[trade.trade_id] = trade \ No newline at end of file + self._trades[trade.trade_id] = trade diff --git a/app/src/market_data/acquisition/recovery/trade_recovery_window.py b/app/src/market_data/acquisition/recovery/trade_recovery_window.py new file mode 100644 index 0000000..8498774 --- /dev/null +++ b/app/src/market_data/acquisition/recovery/trade_recovery_window.py @@ -0,0 +1,51 @@ +# app/src/market_data/acquisition/recovery/trade_recovery_window.py + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class TradeRecoveryWindow: + """ + Неизменяемое описание одного временного диапазона + восстановления Trade Stream. + + Модель не выполняет Recovery и не содержит ограничений + конкретного REST API. Допустимый максимальный размер окна + контролируется TradeRecoveryWindowPlanner. + """ + + symbol: str + start_time: int + end_time: int + + def __post_init__(self) -> None: + if not isinstance(self.symbol, str): + raise TypeError("symbol must be a string") + + if not self.symbol.strip(): + raise ValueError("symbol must not be empty") + + if not isinstance(self.start_time, int) or isinstance( + self.start_time, + bool, + ): + raise TypeError("start_time must be an integer") + + if not isinstance(self.end_time, int) or isinstance( + self.end_time, + bool, + ): + raise TypeError("end_time must be an integer") + + if self.start_time < 0: + raise ValueError("start_time must not be negative") + + if self.end_time < 0: + raise ValueError("end_time must not be negative") + + if self.start_time > self.end_time: + raise ValueError( + "start_time must not be greater than end_time" + ) diff --git a/app/src/market_data/acquisition/recovery/trade_recovery_window_planner.py b/app/src/market_data/acquisition/recovery/trade_recovery_window_planner.py new file mode 100644 index 0000000..12dce27 --- /dev/null +++ b/app/src/market_data/acquisition/recovery/trade_recovery_window_planner.py @@ -0,0 +1,147 @@ +# app/src/market_data/acquisition/recovery/trade_recovery_window_planner.py + +from __future__ import annotations + +from src.market_data.acquisition.recovery.trade_recovery_window import ( + TradeRecoveryWindow, +) + + +MAX_TRADE_RECOVERY_REQUEST_WINDOW_MS = 3_599_999 +DEFAULT_TRADE_RECOVERY_WINDOW_MS = ( + MAX_TRADE_RECOVERY_REQUEST_WINDOW_MS +) + + +class TradeRecoveryWindowPlanner: + """ + Stateless-планировщик временных окон восстановления Trade Stream. + + Planner разбивает один непрерывный временной диапазон на + последовательность окон, каждое из которых совместимо с ограничением + TradeRecoveryRequest: + + end_time - start_time < 3_600_000 ms + + Planner не выполняет REST-запросы и не создаёт TradeRecoveryRequest. + """ + + __slots__ = ("_max_window_ms",) + + def __init__( + self, + *, + max_window_ms: int = DEFAULT_TRADE_RECOVERY_WINDOW_MS, + ) -> None: + if not isinstance(max_window_ms, int) or isinstance( + max_window_ms, + bool, + ): + raise TypeError("max_window_ms must be an integer") + + if max_window_ms <= 0: + raise ValueError("max_window_ms must be positive") + + if max_window_ms > MAX_TRADE_RECOVERY_REQUEST_WINDOW_MS: + raise ValueError( + "max_window_ms must not exceed " + f"{MAX_TRADE_RECOVERY_REQUEST_WINDOW_MS}" + ) + + self._max_window_ms = max_window_ms + + @property + def max_window_ms(self) -> int: + """ + Максимальная длительность одного создаваемого окна. + """ + return self._max_window_ms + + def build_windows( + self, + *, + symbol: str, + start_time: int, + end_time: int, + ) -> tuple[TradeRecoveryWindow, ...]: + """ + Построить последовательность Recovery Window. + + Если start_time равен end_time, восстановление не требуется + и возвращается пустой tuple. + + Соседние окна имеют общую границу: + + previous.end_time == next.start_time + + Возможный повтор Trade на границе должен быть устранён + существующим Consistency Layer. + """ + self._validate_range( + symbol=symbol, + start_time=start_time, + end_time=end_time, + ) + + if start_time == end_time: + return () + + windows: list[TradeRecoveryWindow] = [] + cursor = start_time + + while cursor < end_time: + window_end = min( + cursor + self._max_window_ms, + end_time, + ) + + windows.append( + TradeRecoveryWindow( + symbol=symbol, + start_time=cursor, + end_time=window_end, + ) + ) + + cursor = window_end + + return tuple(windows) + + @staticmethod + def _validate_range( + *, + symbol: str, + start_time: int, + end_time: int, + ) -> None: + """ + Проверить исходный диапазон до начала разбиения. + """ + if not isinstance(symbol, str): + raise TypeError("symbol must be a string") + + if not symbol.strip(): + raise ValueError("symbol must not be empty") + + if not isinstance(start_time, int) or isinstance( + start_time, + bool, + ): + raise TypeError("start_time must be an integer") + + if not isinstance(end_time, int) or isinstance( + end_time, + bool, + ): + raise TypeError("end_time must be an integer") + + if start_time < 0: + raise ValueError("start_time must not be negative") + + if end_time < 0: + raise ValueError("end_time must not be negative") + + if start_time > end_time: + raise ValueError( + "start_time must not be greater than end_time" + ) diff --git a/app/src/market_data/acquisition/runtime/heartbeat.py b/app/src/market_data/acquisition/runtime/heartbeat.py index e69de29..cfb7c6e 100644 --- a/app/src/market_data/acquisition/runtime/heartbeat.py +++ b/app/src/market_data/acquisition/runtime/heartbeat.py @@ -0,0 +1,192 @@ +# app/src/market_data/acquisition/runtime/heartbeat.py + +from __future__ import annotations + +import time +from enum import Enum +from typing import Callable, Protocol, runtime_checkable + +from src.market_data.acquisition.runtime.runtime_events import ( + HeartbeatTimeoutEvent, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionRuntimeEventPublisherProtocol, +) + + +class HeartbeatState(str, Enum): + """ + Текущее состояние Heartbeat Monitor. + """ + + IDLE = "idle" + MONITORING = "monitoring" + TIMED_OUT = "timed_out" + + +@runtime_checkable +class HeartbeatMonitorProtocol(Protocol): + """ + Контракт пассивного контроля активности Acquisition Runtime. + """ + + @property + def state(self) -> HeartbeatState: + """Вернуть текущее состояние Heartbeat Monitor.""" + ... + + @property + def timeout_seconds(self) -> float: + """Вернуть настроенный интервал timeout.""" + ... + + @property + def last_activity_at(self) -> float | None: + """Вернуть monotonic-время последней активности.""" + ... + + def start(self) -> None: + """Начать мониторинг активности.""" + ... + + def stop(self) -> None: + """Остановить мониторинг и очистить текущее состояние.""" + ... + + def record_activity(self) -> None: + """Зафиксировать текущий момент как последнюю активность.""" + ... + + async def check_timeout(self) -> bool: + """ + Проверить наступление timeout. + + Вернуть True только при первом обнаружении timeout. + """ + ... + + +class HeartbeatMonitor: + """ + Пассивный монитор активности Acquisition Runtime. + + Monitor: + + - хранит monotonic-время последней активности; + - детерминированно проверяет превышение timeout; + - однократно публикует HeartbeatTimeoutEvent; + - не запускает собственные фоновые задачи; + - не выполняет reconnect; + - не управляет WebSocket lifecycle. + + Периодический вызов check_timeout() выполняет Runtime Scheduler. + """ + + __slots__ = ( + "_event_publisher", + "_timeout_seconds", + "_clock", + "_state", + "_last_activity_at", + ) + + def __init__( + self, + event_publisher: AcquisitionRuntimeEventPublisherProtocol, + *, + timeout_seconds: float, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if not isinstance(timeout_seconds, (int, float)) or isinstance( + timeout_seconds, + bool, + ): + raise TypeError( + "timeout_seconds must be an integer or float" + ) + + if timeout_seconds <= 0: + raise ValueError( + "timeout_seconds must be positive" + ) + + if not callable(clock): + raise TypeError("clock must be callable") + + self._event_publisher = event_publisher + self._timeout_seconds = float(timeout_seconds) + self._clock = clock + + self._state = HeartbeatState.IDLE + self._last_activity_at: float | None = None + + @property + def state(self) -> HeartbeatState: + """Вернуть текущее состояние Heartbeat Monitor.""" + return self._state + + @property + def timeout_seconds(self) -> float: + """Вернуть настроенный интервал timeout.""" + return self._timeout_seconds + + @property + def last_activity_at(self) -> float | None: + """Вернуть monotonic-время последней активности.""" + return self._last_activity_at + + def start(self) -> None: + """ + Начать мониторинг с текущего monotonic-времени. + """ + self._last_activity_at = self._clock() + self._state = HeartbeatState.MONITORING + + def stop(self) -> None: + """ + Остановить мониторинг и удалить runtime-состояние Heartbeat. + """ + self._last_activity_at = None + self._state = HeartbeatState.IDLE + + def record_activity(self) -> None: + """ + Зафиксировать текущий момент как последнюю активность. + + Получение новой активности также переводит Monitor из состояния + TIMED_OUT обратно в MONITORING. + """ + self._last_activity_at = self._clock() + self._state = HeartbeatState.MONITORING + + async def check_timeout(self) -> bool: + """ + Проверить, истёк ли настроенный интервал активности. + + HeartbeatTimeoutEvent публикуется только один раз для каждого + периода отсутствия активности. + + После новой активности Monitor снова может зафиксировать timeout. + """ + if self._state is not HeartbeatState.MONITORING: + return False + + last_activity_at = self._last_activity_at + + if last_activity_at is None: + return False + + elapsed_seconds = self._clock() - last_activity_at + + if elapsed_seconds < self._timeout_seconds: + return False + + self._state = HeartbeatState.TIMED_OUT + + await self._event_publisher.publish( + HeartbeatTimeoutEvent( + timeout_seconds=self._timeout_seconds, + ) + ) + + return True diff --git a/app/src/market_data/acquisition/runtime/reconnect.py b/app/src/market_data/acquisition/runtime/reconnect.py index fccc002..dc8dfb0 100644 --- a/app/src/market_data/acquisition/runtime/reconnect.py +++ b/app/src/market_data/acquisition/runtime/reconnect.py @@ -1 +1,186 @@ -# app/src/market_data/acquisition/runtime/reconnect.py \ No newline at end of file +# app/src/market_data/acquisition/runtime/reconnect.py + +from __future__ import annotations + +from enum import Enum +from typing import Protocol, runtime_checkable + +from src.market_data.acquisition.runtime.runtime_commands import ( + ConnectCommand, +) +from src.market_data.acquisition.runtime.runtime_events import ( + ReconnectCompletedEvent, + ReconnectFailedEvent, + ReconnectStartedEvent, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionRuntimeCommandDispatcherProtocol, + AcquisitionRuntimeEventPublisherProtocol, + WebSocketSubscriptionManagerProtocol, +) + + +class ReconnectState(str, Enum): + """ + Текущее состояние ReconnectCoordinator. + + Состояния описывают только lifecycle одной инфраструктурной + операции reconnect. + + Состояния Heartbeat, Supervisor и Trade Recovery принадлежат + соответствующим специализированным компонентам. + """ + + DISCONNECTED = "disconnected" + CONNECTING = "connecting" + RESTORING_SUBSCRIPTIONS = "restoring_subscriptions" + CONNECTED = "connected" + FAILED = "failed" + + +@runtime_checkable +class ReconnectCoordinatorProtocol(Protocol): + """ + Контракт координатора повторного подключения Acquisition Runtime. + """ + + @property + def state(self) -> ReconnectState: + """ + Вернуть текущее состояние reconnect lifecycle. + """ + ... + + @property + def attempt(self) -> int: + """ + Вернуть номер последней начатой попытки reconnect. + """ + ... + + async def reconnect(self) -> None: + """ + Выполнить одну попытку повторного подключения. + """ + ... + + +class ReconnectCoordinator: + """ + Координатор одной попытки повторного подключения. + + Coordinator: + + - увеличивает номер попытки; + - публикует ReconnectStartedEvent; + - передаёт ConnectCommand в Runtime Dispatcher; + - восстанавливает зарегистрированные подписки; + - публикует ReconnectCompletedEvent; + - при ошибке публикует ReconnectFailedEvent; + - сохраняет текущее состояние lifecycle. + + Coordinator не выполняет: + + - retry loop; + - backoff; + - scheduling; + - heartbeat monitoring; + - Trade Recovery; + - обработку рыночных сообщений. + """ + + __slots__ = ( + "_command_dispatcher", + "_subscription_manager", + "_event_publisher", + "_state", + "_attempt", + ) + + def __init__( + self, + command_dispatcher: AcquisitionRuntimeCommandDispatcherProtocol, + subscription_manager: WebSocketSubscriptionManagerProtocol, + event_publisher: AcquisitionRuntimeEventPublisherProtocol, + ) -> None: + self._command_dispatcher = command_dispatcher + self._subscription_manager = subscription_manager + self._event_publisher = event_publisher + + self._state = ReconnectState.DISCONNECTED + self._attempt = 0 + + @property + def state(self) -> ReconnectState: + """ + Вернуть текущее состояние reconnect lifecycle. + """ + return self._state + + @property + def attempt(self) -> int: + """ + Вернуть номер последней начатой попытки reconnect. + """ + return self._attempt + + async def reconnect(self) -> None: + """ + Выполнить одну попытку повторного подключения. + + Последовательность: + + ReconnectStartedEvent + ↓ + ConnectCommand + ↓ + restore_subscriptions() + ↓ + ReconnectCompletedEvent + + Raises: + Exception: + Любая ошибка Runtime Dispatcher или Subscription Manager + распространяется вызывающему компоненту без обёртки. + """ + self._attempt += 1 + current_attempt = self._attempt + + self._state = ReconnectState.CONNECTING + + await self._event_publisher.publish( + ReconnectStartedEvent( + attempt=current_attempt, + ) + ) + + try: + await self._command_dispatcher.dispatch( + ConnectCommand() + ) + + self._state = ( + ReconnectState.RESTORING_SUBSCRIPTIONS + ) + + await self._subscription_manager.restore_subscriptions() + + except Exception as error: + self._state = ReconnectState.FAILED + + await self._event_publisher.publish( + ReconnectFailedEvent( + attempt=current_attempt, + reason=str(error), + ) + ) + + raise + + self._state = ReconnectState.CONNECTED + + await self._event_publisher.publish( + ReconnectCompletedEvent( + attempt=current_attempt, + ) + ) diff --git a/app/src/market_data/acquisition/runtime/runtime_recovery_coordinator.py b/app/src/market_data/acquisition/runtime/runtime_recovery_coordinator.py new file mode 100644 index 0000000..9f82632 --- /dev/null +++ b/app/src/market_data/acquisition/runtime/runtime_recovery_coordinator.py @@ -0,0 +1,295 @@ +# app/src/market_data/acquisition/runtime/ +# runtime_recovery_coordinator.py + +from __future__ import annotations + +from datetime import datetime, timezone + +from src.market_data.acquisition.consistency.trade_stream_state_store_exceptions import ( + TradeStreamStateNotFoundError, +) +from src.market_data.acquisition.consistency.trade_stream_state_store_protocol import ( + TradeStreamStateStoreProtocol, +) +from src.market_data.acquisition.models.trade import Trade +from src.market_data.acquisition.recovery.trade_recovery_protocol import ( + TradeRecoveryProtocol, +) +from src.market_data.acquisition.recovery.trade_recovery_request import ( + TradeRecoveryRequest, +) +from src.market_data.acquisition.recovery.trade_recovery_result import ( + TradeRecoveryResult, +) +from src.market_data.acquisition.recovery.trade_recovery_window_planner import ( + TradeRecoveryWindowPlanner, +) + + +class RuntimeRecoveryCoordinator: + """ + Runtime-координатор восстановления Trade Stream. + + Coordinator связывает существующие компоненты: + + - TradeStreamStateStoreProtocol; + - checkpoint в TradeStreamState.last_trade; + - TradeRecoveryWindowPlanner; + - TradeRecoveryProtocol. + + Последовательность восстановления: + + state_store.get(symbol) + ↓ + TradeStreamState.last_trade + ↓ + executed_at → Unix milliseconds + ↓ + TradeRecoveryWindowPlanner.build_windows() + ↓ + TradeRecoveryRequest для каждого окна + ↓ + TradeRecoveryProtocol.recover() + ↓ + объединённый TradeRecoveryResult + + Coordinator не выполняет: + + - WebSocket connect или disconnect; + - reconnect; + - восстановление подписок; + - heartbeat monitoring; + - scheduling; + - retry или backoff; + - публикацию Runtime Events; + - создание нового TradeStreamState; + - обработку и проверку согласованности Trade. + + Отсутствие состояния или последней принятой сделки является + штатной ситуацией: восстановление пропускается и возвращается + пустой TradeRecoveryResult. + + Все остальные ошибки State Store, Window Planner и Trade Recovery + распространяются вызывающему коду без обёртки. + """ + + __slots__ = ( + "_state_store", + "_window_planner", + "_recovery_controller", + ) + + def __init__( + self, + *, + state_store: TradeStreamStateStoreProtocol, + window_planner: TradeRecoveryWindowPlanner, + recovery_controller: TradeRecoveryProtocol, + ) -> None: + """ + Создать Runtime Recovery Coordinator. + + Args: + state_store: + Общее хранилище состояния Trade Stream Consistency. + + Должен передаваться тот же экземпляр хранилища, + который используется основным WebSocket-потоком. + + window_planner: + Планировщик допустимых временных окон восстановления. + + recovery_controller: + Существующий stateless-контроллер Trade Recovery. + """ + self._state_store = state_store + self._window_planner = window_planner + self._recovery_controller = recovery_controller + + def recover( + self, + *, + symbol: str, + recovery_end_time: int, + ) -> TradeRecoveryResult: + """ + Восстановить пропущенные сделки одного инструмента. + + Начальная граница определяется по времени последней сделки, + принятой общим Trade Stream Consistency Layer. + + Конечная граница передаётся вызывающим Runtime-компонентом. + Coordinator не получает системное время самостоятельно. + + Возможный повтор последней принятой сделки на левой границе + безопасно устраняется существующим Consistency Layer. + + Args: + symbol: + Символ торгового инструмента. + + recovery_end_time: + Конечная граница восстановления в миллисекундах + Unix time. + + Returns: + Объединённый TradeRecoveryResult для полного диапазона. + + Если состояние или checkpoint отсутствуют, возвращается + пустой результат с границами, равными recovery_end_time. + + Raises: + TypeError: + Если symbol или recovery_end_time имеют неверный тип. + + ValueError: + Если symbol пустой, recovery_end_time отрицательный, + checkpoint не содержит timezone-aware datetime либо + конечная граница расположена раньше checkpoint. + + Exception: + Ошибки Window Planner и Trade Recovery Controller + распространяются без изменения. + """ + self._validate_input( + symbol=symbol, + recovery_end_time=recovery_end_time, + ) + + try: + state = self._state_store.get(symbol) + except TradeStreamStateNotFoundError: + return self._build_empty_result( + symbol=symbol, + boundary_time=recovery_end_time, + ) + + checkpoint_trade = state.last_trade + + if checkpoint_trade is None: + return self._build_empty_result( + symbol=symbol, + boundary_time=recovery_end_time, + ) + + recovery_start_time = self._datetime_to_unix_ms( + checkpoint_trade.executed_at, + ) + + windows = self._window_planner.build_windows( + symbol=symbol, + start_time=recovery_start_time, + end_time=recovery_end_time, + ) + + recovered_trades: list[Trade] = [] + + for window in windows: + window_result = self._recovery_controller.recover( + TradeRecoveryRequest( + symbol=window.symbol, + start_time=window.start_time, + end_time=window.end_time, + ) + ) + + recovered_trades.extend( + window_result.recovered_trades, + ) + + return TradeRecoveryResult( + symbol=symbol, + requested_start_time=recovery_start_time, + requested_end_time=recovery_end_time, + recovered_trades=tuple(recovered_trades), + ) + + @staticmethod + def _validate_input( + *, + symbol: str, + recovery_end_time: int, + ) -> None: + """ + Проверить входные параметры до обращения к State Store. + """ + if not isinstance(symbol, str): + raise TypeError( + "symbol должен иметь тип str." + ) + + if not symbol.strip(): + raise ValueError( + "symbol не должен быть пустым." + ) + + if isinstance(recovery_end_time, bool) or not isinstance( + recovery_end_time, + int, + ): + raise TypeError( + "recovery_end_time должен иметь тип int." + ) + + if recovery_end_time < 0: + raise ValueError( + "recovery_end_time не должен быть отрицательным." + ) + + @staticmethod + def _datetime_to_unix_ms( + value: datetime, + ) -> int: + """ + Преобразовать timezone-aware datetime в Unix milliseconds. + + Расчёт выполняется относительно UTC epoch без использования + float, чтобы не вносить погрешность преобразования timestamp. + """ + if not isinstance(value, datetime): + raise TypeError( + "checkpoint executed_at должен иметь тип datetime." + ) + + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError( + "checkpoint executed_at должен содержать timezone." + ) + + utc_value = value.astimezone( + timezone.utc, + ) + + epoch = datetime( + 1970, + 1, + 1, + tzinfo=timezone.utc, + ) + + delta = utc_value - epoch + + return ( + delta.days * 86_400_000 + + delta.seconds * 1_000 + + delta.microseconds // 1_000 + ) + + @staticmethod + def _build_empty_result( + *, + symbol: str, + boundary_time: int, + ) -> TradeRecoveryResult: + """ + Создать пустой результат при отсутствии Recovery Checkpoint. + + Равные границы явно обозначают отсутствие диапазона, + который можно восстановить на основе существующего состояния. + """ + return TradeRecoveryResult( + symbol=symbol, + requested_start_time=boundary_time, + requested_end_time=boundary_time, + recovered_trades=(), + ) diff --git a/app/src/market_data/acquisition/runtime/runtime_recovery_protocol.py b/app/src/market_data/acquisition/runtime/runtime_recovery_protocol.py new file mode 100644 index 0000000..cd367df --- /dev/null +++ b/app/src/market_data/acquisition/runtime/runtime_recovery_protocol.py @@ -0,0 +1,72 @@ +# app/src/market_data/acquisition/runtime/runtime_recovery_protocol.py + +""" +Публичный контракт Runtime Recovery Coordinator. + +Runtime Recovery Coordinator связывает Runtime Layer с существующим +Trade Recovery Layer, но не выполняет reconnect, WebSocket-операции, +heartbeat, scheduling или восстановление подписок. +""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from src.market_data.acquisition.recovery.trade_recovery_result import ( + TradeRecoveryResult, +) + + +@runtime_checkable +class RuntimeRecoveryProtocol(Protocol): + """ + Контракт Runtime-координатора восстановления Trade Stream. + + Реализация должна: + + - получить существующий checkpoint торгового инструмента; + - определить полный временной диапазон восстановления; + - разбить диапазон на допустимые Recovery Window; + - последовательно выполнить Trade Recovery для каждого окна; + - вернуть объединённый TradeRecoveryResult. + + Реализация не должна: + + - создавать новое состояние Trade Stream; + - выполнять WebSocket connect или disconnect; + - восстанавливать WebSocket-подписки; + - управлять Heartbeat Monitor; + - управлять Runtime Supervisor; + - запускать Scheduler; + - выполнять retry или backoff; + - скрывать ошибки нижележащих компонентов. + """ + + def recover( + self, + *, + symbol: str, + recovery_end_time: int, + ) -> TradeRecoveryResult: + """ + Восстановить пропущенные сделки одного торгового инструмента. + + Args: + symbol: + Символ торгового инструмента. + + recovery_end_time: + Правая граница восстанавливаемого диапазона + в миллисекундах Unix time. + + Значение определяется вызывающим Runtime-компонентом. + Coordinator самостоятельно текущее время не вычисляет. + + Returns: + Объединённый результат восстановления всего диапазона. + + Если существующее состояние или checkpoint отсутствуют, + реализация возвращает пустой TradeRecoveryResult и не создаёт + новое состояние Trade Stream. + """ + ... diff --git a/app/src/market_data/acquisition/runtime/scheduler.py b/app/src/market_data/acquisition/runtime/scheduler.py index e69de29..fce13ca 100644 --- a/app/src/market_data/acquisition/runtime/scheduler.py +++ b/app/src/market_data/acquisition/runtime/scheduler.py @@ -0,0 +1,199 @@ +# app/src/market_data/acquisition/runtime/scheduler.py + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable +from typing import Protocol, runtime_checkable + +from src.market_data.acquisition.runtime.heartbeat import ( + HeartbeatMonitorProtocol, +) +from src.market_data.acquisition.runtime.supervisor import ( + RuntimeSupervisorProtocol, +) + + +RuntimeSleep = Callable[[float], Awaitable[None]] + + +@runtime_checkable +class RuntimeSchedulerProtocol(Protocol): + """ + Контракт периодического планировщика Acquisition Runtime. + """ + + @property + def running(self) -> bool: + """ + Вернуть признак активного scheduler loop. + """ + ... + + @property + def interval_seconds(self) -> float: + """ + Вернуть интервал между периодическими проверками. + """ + ... + + async def start(self) -> None: + """ + Запустить scheduler loop. + + Метод выполняется до вызова stop() либо возникновения ошибки. + """ + ... + + def stop(self) -> None: + """ + Запросить остановку scheduler loop. + """ + ... + + async def run_once(self) -> bool: + """ + Выполнить одну проверку Heartbeat. + + Вернуть True, если был обнаружен timeout. + """ + ... + + +class RuntimeScheduler: + """ + Периодический планировщик Acquisition Runtime. + + Scheduler отвечает только за время выполнения проверок: + + HeartbeatMonitor.check_timeout() + │ + ├── False + │ └── ожидание следующего интервала + │ + └── True + └── RuntimeSupervisor + .handle_heartbeat_timeout() + + Scheduler не выполняет: + + - reconnect; + - Heartbeat calculations; + - управление WebSocket; + - Trade Recovery; + - backoff; + - retry policy; + - публикацию Runtime Events. + + Ошибки Heartbeat Monitor, Runtime Supervisor и sleep-функции + распространяются вызывающему коду без обёртки. + """ + + __slots__ = ( + "_heartbeat_monitor", + "_runtime_supervisor", + "_interval_seconds", + "_sleep", + "_running", + ) + + def __init__( + self, + heartbeat_monitor: HeartbeatMonitorProtocol, + runtime_supervisor: RuntimeSupervisorProtocol, + *, + interval_seconds: float, + sleep: RuntimeSleep = asyncio.sleep, + ) -> None: + if not isinstance(interval_seconds, (int, float)) or isinstance( + interval_seconds, + bool, + ): + raise TypeError( + "interval_seconds must be an integer or float" + ) + + if interval_seconds <= 0: + raise ValueError( + "interval_seconds must be positive" + ) + + if not callable(sleep): + raise TypeError("sleep must be callable") + + self._heartbeat_monitor = heartbeat_monitor + self._runtime_supervisor = runtime_supervisor + self._interval_seconds = float(interval_seconds) + self._sleep = sleep + self._running = False + + @property + def running(self) -> bool: + """ + Вернуть признак активного scheduler loop. + """ + return self._running + + @property + def interval_seconds(self) -> float: + """ + Вернуть интервал между проверками. + """ + return self._interval_seconds + + async def start(self) -> None: + """ + Запустить периодический scheduler loop. + + Если Scheduler уже запущен, повторный вызов немедленно + завершается и не создаёт второй параллельный цикл. + + Scheduler всегда сбрасывает running в False: + + - после stop(); + - после ошибки; + - после отмены внешней asyncio-задачи. + """ + if self._running: + return + + self._running = True + + try: + while self._running: + await self.run_once() + + if not self._running: + break + + await self._sleep( + self._interval_seconds, + ) + finally: + self._running = False + + def stop(self) -> None: + """ + Запросить завершение scheduler loop. + + Метод идемпотентен. Он не отменяет внешнюю asyncio-задачу, + а останавливает цикл на ближайшей управляемой границе. + """ + self._running = False + + async def run_once(self) -> bool: + """ + Выполнить одну проверку Heartbeat. + + Если Heartbeat Monitor подтверждает timeout, Scheduler + передаёт его Runtime Supervisor. + + Возвращаемое значение отражает только результат проверки + Heartbeat Monitor. + """ + timed_out = await self._heartbeat_monitor.check_timeout() + + if timed_out: + await self._runtime_supervisor.handle_heartbeat_timeout() + + return timed_out diff --git a/app/src/market_data/acquisition/runtime/supervisor.py b/app/src/market_data/acquisition/runtime/supervisor.py index 7fe2905..49d71b1 100644 --- a/app/src/market_data/acquisition/runtime/supervisor.py +++ b/app/src/market_data/acquisition/runtime/supervisor.py @@ -1 +1,203 @@ -# app/src/market_data/acquisition/runtime/supervisor.py \ No newline at end of file +# app/src/market_data/acquisition/runtime/supervisor.py + +from __future__ import annotations + +from enum import Enum +from typing import Protocol, runtime_checkable + +from src.market_data.acquisition.runtime.heartbeat import ( + HeartbeatMonitorProtocol, +) +from src.market_data.acquisition.runtime.reconnect import ( + ReconnectCoordinatorProtocol, +) + + +class RuntimeSupervisorState(str, Enum): + """ + Текущее состояние Runtime Supervisor. + """ + + STOPPED = "stopped" + RUNNING = "running" + RECONNECTING = "reconnecting" + FAILED = "failed" + + +@runtime_checkable +class RuntimeSupervisorProtocol(Protocol): + """ + Контракт координатора lifecycle Acquisition Runtime. + """ + + @property + def state(self) -> RuntimeSupervisorState: + """ + Вернуть текущее состояние Runtime Supervisor. + """ + ... + + def start(self) -> None: + """ + Запустить supervision и Heartbeat Monitor. + """ + ... + + def stop(self) -> None: + """ + Остановить supervision и Heartbeat Monitor. + """ + ... + + def notify_activity(self) -> None: + """ + Передать Heartbeat Monitor информацию о Runtime activity. + """ + ... + + async def handle_heartbeat_timeout(self) -> bool: + """ + Обработать подтверждённый Heartbeat timeout. + + Вернуть True, если была выполнена попытка reconnect. + """ + ... + + +class RuntimeSupervisor: + """ + Координатор lifecycle Acquisition Runtime. + + Supervisor объединяет: + + - HeartbeatMonitor; + - ReconnectCoordinator; + - общее состояние Runtime Session. + + Supervisor: + + - запускает и останавливает Heartbeat Monitor; + - принимает уведомления о Runtime activity; + - запускает одну попытку reconnect после Heartbeat timeout; + - после успешного reconnect начинает новый период Heartbeat; + - фиксирует состояния STOPPED, RUNNING, RECONNECTING и FAILED; + - предотвращает параллельный reconnect. + + Supervisor не выполняет: + + - периодические проверки Heartbeat; + - retry loop; + - backoff; + - scheduling; + - Trade Recovery; + - управление WebSocket Transport; + - публикацию Runtime Events. + + Периодический вызов Heartbeat и передача timeout в Supervisor + будут ответственностью Runtime Scheduler. + """ + + __slots__ = ( + "_heartbeat_monitor", + "_reconnect_coordinator", + "_state", + ) + + def __init__( + self, + heartbeat_monitor: HeartbeatMonitorProtocol, + reconnect_coordinator: ReconnectCoordinatorProtocol, + ) -> None: + self._heartbeat_monitor = heartbeat_monitor + self._reconnect_coordinator = reconnect_coordinator + self._state = RuntimeSupervisorState.STOPPED + + @property + def state(self) -> RuntimeSupervisorState: + """ + Вернуть текущее состояние Runtime Supervisor. + """ + return self._state + + def start(self) -> None: + """ + Запустить supervision. + + Повторный вызов start() начинает новый Heartbeat monitoring + period и сохраняет Supervisor в состоянии RUNNING. + """ + self._heartbeat_monitor.start() + self._state = RuntimeSupervisorState.RUNNING + + def stop(self) -> None: + """ + Остановить supervision. + + Повторный вызов stop() безопасен и сохраняет состояние STOPPED. + """ + self._heartbeat_monitor.stop() + self._state = RuntimeSupervisorState.STOPPED + + def notify_activity(self) -> None: + """ + Передать информацию о Runtime activity в Heartbeat Monitor. + + Активность учитывается только во время RUNNING lifecycle. + События, полученные после stop() либо во время reconnect, + не должны неявно запускать Heartbeat Monitor. + """ + if self._state is not RuntimeSupervisorState.RUNNING: + return + + self._heartbeat_monitor.record_activity() + + async def handle_heartbeat_timeout(self) -> bool: + """ + Обработать подтверждённый Heartbeat timeout. + + Последовательность: + + RUNNING + ↓ + stop Heartbeat + ↓ + RECONNECTING + ↓ + ReconnectCoordinator.reconnect() + ├── success: + │ start Heartbeat + │ RUNNING + │ + └── failure: + FAILED + exception propagates + + Если Supervisor не находится в состоянии RUNNING, + новая попытка reconnect не запускается. + + Returns: + bool: + True, если reconnect был выполнен успешно. + False, если Supervisor не находился в RUNNING. + + Raises: + Exception: + Исходная ошибка ReconnectCoordinator распространяется + вызывающему компоненту без обёртки. + """ + if self._state is not RuntimeSupervisorState.RUNNING: + return False + + self._heartbeat_monitor.stop() + self._state = RuntimeSupervisorState.RECONNECTING + + try: + await self._reconnect_coordinator.reconnect() + except Exception: + self._state = RuntimeSupervisorState.FAILED + raise + + self._heartbeat_monitor.start() + self._state = RuntimeSupervisorState.RUNNING + + return True diff --git a/app/src/market_data/acquisition/trade_stream_runtime_composition.py b/app/src/market_data/acquisition/trade_stream_runtime_composition.py new file mode 100644 index 0000000..8f903dc --- /dev/null +++ b/app/src/market_data/acquisition/trade_stream_runtime_composition.py @@ -0,0 +1,184 @@ +# app/src/market_data/acquisition/ +# trade_stream_runtime_composition.py + +from __future__ import annotations + +import asyncio +import time +from collections.abc import Callable +from dataclasses import dataclass + +from src.market_data.acquisition.adapters.dzengi.rest import ( + DzengiTradesDocumentSource, +) +from src.market_data.acquisition.consistency.trade_stream_consistency_controller import ( + TradeStreamConsistencyController, +) +from src.market_data.acquisition.consistency.trade_stream_state_store import ( + TradeStreamStateStore, +) +from src.market_data.acquisition.recovery.trade_recovery_controller import ( + TradeRecoveryController, +) +from src.market_data.acquisition.recovery.trade_recovery_window_planner import ( + DEFAULT_TRADE_RECOVERY_WINDOW_MS, + TradeRecoveryWindowPlanner, +) +from src.market_data.acquisition.runtime.acquisition_runtime_service import ( + AcquisitionRuntimeService, +) +from src.market_data.acquisition.runtime.heartbeat import ( + HeartbeatMonitor, +) +from src.market_data.acquisition.runtime.reconnect import ( + ReconnectCoordinator, +) +from src.market_data.acquisition.runtime.runtime_recovery_coordinator import ( + RuntimeRecoveryCoordinator, +) +from src.market_data.acquisition.runtime.scheduler import ( + RuntimeScheduler, + RuntimeSleep, +) +from src.market_data.acquisition.runtime.supervisor import ( + RuntimeSupervisor, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionRuntimeEventPublisherProtocol, + WebSocketSessionProtocol, + WebSocketSubscriptionManagerProtocol, + WebSocketTransportProtocol, +) +from src.market_data.acquisition.trade_stream_acquisition_service import ( + TradeStreamAcquisitionService, +) +from src.market_data.acquisition.trade_stream_message_adapter_protocol import ( + TradeStreamMessageAdapterProtocol, +) + + +@dataclass(frozen=True, slots=True) +class TradeStreamRuntimeComposition: + """ + Неизменяемый результат композиции Trade Stream Runtime. + + Объект предоставляет явно построенный граф зависимостей и не является + Registry либо Service Locator. Все компоненты создаются factory-функцией + ровно один раз и повторно используют общие stateful-зависимости. + + Создание Composition не запускает WebSocket, Heartbeat, Supervisor + или Scheduler и не выполняет Recovery. + """ + + state_store: TradeStreamStateStore + consistency_controller: TradeStreamConsistencyController + recovery_controller: TradeRecoveryController + recovery_window_planner: TradeRecoveryWindowPlanner + runtime_recovery_coordinator: RuntimeRecoveryCoordinator + + acquisition_runtime_service: AcquisitionRuntimeService + trade_stream_acquisition_service: TradeStreamAcquisitionService + + reconnect_coordinator: ReconnectCoordinator + heartbeat_monitor: HeartbeatMonitor + runtime_supervisor: RuntimeSupervisor + runtime_scheduler: RuntimeScheduler + + +def build_trade_stream_runtime_composition( + *, + session: WebSocketSessionProtocol, + transport: WebSocketTransportProtocol, + subscription_manager: WebSocketSubscriptionManagerProtocol, + event_publisher: AcquisitionRuntimeEventPublisherProtocol, + message_adapter: TradeStreamMessageAdapterProtocol, + recovery_document_source: DzengiTradesDocumentSource, + heartbeat_timeout_seconds: float, + scheduler_interval_seconds: float, + max_recovery_window_ms: int = DEFAULT_TRADE_RECOVERY_WINDOW_MS, + heartbeat_clock: Callable[[], float] = time.monotonic, + scheduler_sleep: RuntimeSleep = asyncio.sleep, +) -> TradeStreamRuntimeComposition: + """ + Построить изолированный граф зависимостей Trade Stream Runtime. + + Factory создаёт единые экземпляры TradeStreamStateStore и + TradeStreamConsistencyController. Благодаря этому Live Stream + и Recovery используют один checkpoint и одинаковые правила + согласованности сделок. + + Внешние WebSocket-зависимости передаются через Protocol-контракты. + Factory не создаёт production transport, не читает Settings, + не запускает lifecycle и не создаёт фоновые asyncio-задачи. + """ + state_store = TradeStreamStateStore() + + consistency_controller = TradeStreamConsistencyController( + state_store, + ) + + recovery_controller = TradeRecoveryController( + document_source=recovery_document_source, + consistency_controller=consistency_controller, + ) + + recovery_window_planner = TradeRecoveryWindowPlanner( + max_window_ms=max_recovery_window_ms, + ) + + runtime_recovery_coordinator = RuntimeRecoveryCoordinator( + state_store=state_store, + window_planner=recovery_window_planner, + recovery_controller=recovery_controller, + ) + + acquisition_runtime_service = AcquisitionRuntimeService( + session=session, + transport=transport, + subscription_manager=subscription_manager, + event_publisher=event_publisher, + ) + + trade_stream_acquisition_service = TradeStreamAcquisitionService( + runtime_service=acquisition_runtime_service, + adapter=message_adapter, + consistency_controller=consistency_controller, + ) + + reconnect_coordinator = ReconnectCoordinator( + command_dispatcher=acquisition_runtime_service, + subscription_manager=subscription_manager, + event_publisher=event_publisher, + ) + + heartbeat_monitor = HeartbeatMonitor( + event_publisher=event_publisher, + timeout_seconds=heartbeat_timeout_seconds, + clock=heartbeat_clock, + ) + + runtime_supervisor = RuntimeSupervisor( + heartbeat_monitor=heartbeat_monitor, + reconnect_coordinator=reconnect_coordinator, + ) + + runtime_scheduler = RuntimeScheduler( + heartbeat_monitor=heartbeat_monitor, + runtime_supervisor=runtime_supervisor, + interval_seconds=scheduler_interval_seconds, + sleep=scheduler_sleep, + ) + + return TradeStreamRuntimeComposition( + state_store=state_store, + consistency_controller=consistency_controller, + recovery_controller=recovery_controller, + recovery_window_planner=recovery_window_planner, + runtime_recovery_coordinator=runtime_recovery_coordinator, + acquisition_runtime_service=acquisition_runtime_service, + trade_stream_acquisition_service=trade_stream_acquisition_service, + reconnect_coordinator=reconnect_coordinator, + heartbeat_monitor=heartbeat_monitor, + runtime_supervisor=runtime_supervisor, + runtime_scheduler=runtime_scheduler, + ) diff --git a/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state.py b/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state.py index 7a8e35a..4f52b67 100644 --- a/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state.py +++ b/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state.py @@ -1,3 +1,5 @@ +# app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state.py + from __future__ import annotations from datetime import datetime, timezone @@ -53,6 +55,13 @@ def test_trade_stream_state_uses_slots() -> None: assert not hasattr(state, "__dict__") +def test_new_state_has_no_checkpoint() -> None: + state = TradeStreamState(symbol="BTCUSD") + + assert state.last_trade_id is None + assert state.last_trade is None + + def test_accepts_first_trade() -> None: state = TradeStreamState(symbol="BTCUSD") trade = _trade() @@ -61,6 +70,17 @@ def test_accepts_first_trade() -> None: assert result == trade assert state.last_trade_id == trade.trade_id + assert state.last_trade is trade + + +def test_first_accepted_trade_becomes_checkpoint() -> None: + state = TradeStreamState(symbol="BTCUSD") + trade = _trade() + + state.accept(trade) + + assert state.last_trade is trade + assert state.last_trade_id == trade.trade_id def test_accepts_trade_with_greater_trade_id() -> None: @@ -73,6 +93,19 @@ def test_accepts_trade_with_greater_trade_id() -> None: assert result == second_trade assert state.last_trade_id == second_trade.trade_id + assert state.last_trade is second_trade + + +def test_next_accepted_trade_replaces_checkpoint() -> None: + state = TradeStreamState(symbol="BTCUSD") + first_trade = _trade(trade_id=100) + second_trade = _trade(trade_id=101) + + state.accept(first_trade) + state.accept(second_trade) + + assert state.last_trade is second_trade + assert state.last_trade_id == second_trade.trade_id def test_accepts_trade_with_gap() -> None: @@ -85,6 +118,17 @@ def test_accepts_trade_with_gap() -> None: assert result == trade_after_gap assert state.last_trade_id == trade_after_gap.trade_id + assert state.last_trade is trade_after_gap + + +def test_checkpoint_preserves_trade_identity() -> None: + state = TradeStreamState(symbol="BTCUSD") + trade = _trade() + + result = state.accept(trade) + + assert result is trade + assert state.last_trade is trade def test_returns_none_for_identical_duplicate() -> None: @@ -96,6 +140,22 @@ def test_returns_none_for_identical_duplicate() -> None: assert result is None assert state.last_trade_id == trade.trade_id + assert state.last_trade is trade + + +def test_identical_duplicate_does_not_change_checkpoint() -> None: + state = TradeStreamState(symbol="BTCUSD") + original_trade = _trade() + + state.accept(original_trade) + + duplicate_trade = _trade() + + result = state.accept(duplicate_trade) + + assert result is None + assert state.last_trade is original_trade + assert state.last_trade_id == original_trade.trade_id def test_raises_consistency_error_for_conflicting_duplicate() -> None: @@ -115,6 +175,26 @@ def test_raises_consistency_error_for_conflicting_duplicate() -> None: state.accept(conflicting_trade) +def test_conflicting_duplicate_does_not_change_checkpoint() -> None: + state = TradeStreamState(symbol="BTCUSD") + original_trade = _trade( + trade_id=100, + price=Decimal("50000.00"), + ) + conflicting_trade = _trade( + trade_id=100, + price=Decimal("50001.00"), + ) + + state.accept(original_trade) + + with pytest.raises(TradeConsistencyError): + state.accept(conflicting_trade) + + assert state.last_trade is original_trade + assert state.last_trade_id == original_trade.trade_id + + def test_raises_ordering_error_for_older_trade() -> None: state = TradeStreamState(symbol="BTCUSD") current_trade = _trade(trade_id=100) @@ -126,6 +206,20 @@ def test_raises_ordering_error_for_older_trade() -> None: state.accept(older_trade) +def test_older_trade_does_not_change_checkpoint() -> None: + state = TradeStreamState(symbol="BTCUSD") + current_trade = _trade(trade_id=100) + older_trade = _trade(trade_id=99) + + state.accept(current_trade) + + with pytest.raises(TradeOrderingError): + state.accept(older_trade) + + assert state.last_trade is current_trade + assert state.last_trade_id == current_trade.trade_id + + def test_returns_none_for_duplicate_still_inside_window() -> None: state = TradeStreamState( symbol="BTCUSD", @@ -143,6 +237,29 @@ def test_returns_none_for_duplicate_still_inside_window() -> None: assert result is None assert state.last_trade_id == third_trade.trade_id + assert state.last_trade is third_trade + + +def test_duplicate_inside_window_does_not_change_checkpoint() -> None: + state = TradeStreamState( + symbol="BTCUSD", + deduplication_window_size=3, + ) + first_trade = _trade(trade_id=100) + second_trade = _trade(trade_id=101) + third_trade = _trade(trade_id=102) + + state.accept(first_trade) + state.accept(second_trade) + state.accept(third_trade) + + duplicate_trade = _trade(trade_id=100) + + result = state.accept(duplicate_trade) + + assert result is None + assert state.last_trade is third_trade + assert state.last_trade_id == third_trade.trade_id def test_raises_ordering_error_after_trade_leaves_window() -> None: @@ -162,6 +279,26 @@ def test_raises_ordering_error_after_trade_leaves_window() -> None: state.accept(first_trade) +def test_ordering_error_after_window_does_not_change_checkpoint() -> None: + state = TradeStreamState( + symbol="BTCUSD", + deduplication_window_size=2, + ) + first_trade = _trade(trade_id=100) + second_trade = _trade(trade_id=101) + third_trade = _trade(trade_id=102) + + state.accept(first_trade) + state.accept(second_trade) + state.accept(third_trade) + + with pytest.raises(TradeOrderingError): + state.accept(first_trade) + + assert state.last_trade is third_trade + assert state.last_trade_id == third_trade.trade_id + + def test_rejects_unexpected_symbol() -> None: state = TradeStreamState(symbol="BTCUSD") trade = _trade(symbol="ETHUSD") @@ -170,6 +307,36 @@ def test_rejects_unexpected_symbol() -> None: state.accept(trade) +def test_unexpected_symbol_does_not_change_checkpoint() -> None: + state = TradeStreamState(symbol="BTCUSD") + accepted_trade = _trade(trade_id=100) + + state.accept(accepted_trade) + + unexpected_trade = _trade( + symbol="ETHUSD", + trade_id=101, + ) + + with pytest.raises(ValueError): + state.accept(unexpected_trade) + + assert state.last_trade is accepted_trade + assert state.last_trade_id == accepted_trade.trade_id + + +def test_checkpoint_trade_id_matches_last_trade_id() -> None: + state = TradeStreamState(symbol="BTCUSD") + first_trade = _trade(trade_id=100) + second_trade = _trade(trade_id=105) + + state.accept(first_trade) + state.accept(second_trade) + + assert state.last_trade is not None + assert state.last_trade.trade_id == state.last_trade_id + + def test_rejects_empty_symbol() -> None: with pytest.raises(ValueError): TradeStreamState(symbol="") @@ -189,4 +356,4 @@ def test_rejects_non_positive_window_size( TradeStreamState( symbol="BTCUSD", deduplication_window_size=window_size, - ) \ No newline at end of file + ) diff --git a/app/tests/unit/market_data/acquisition/recovery/test_trade_recovery_window.py b/app/tests/unit/market_data/acquisition/recovery/test_trade_recovery_window.py new file mode 100644 index 0000000..91c9ee8 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/recovery/test_trade_recovery_window.py @@ -0,0 +1,158 @@ +# app/tests/unit/market_data/acquisition/recovery/test_trade_recovery_window.py + +from __future__ import annotations + +from dataclasses import FrozenInstanceError + +import pytest + +from src.market_data.acquisition.recovery.trade_recovery_window import ( + TradeRecoveryWindow, +) + + +def test_creates_valid_window() -> None: + window = TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=1_700_000_000_000, + end_time=1_700_000_100_000, + ) + + assert window.symbol == "BTCUSDT" + assert window.start_time == 1_700_000_000_000 + assert window.end_time == 1_700_000_100_000 + + +def test_allows_equal_start_and_end_time() -> None: + window = TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=1_700_000_000_000, + end_time=1_700_000_000_000, + ) + + assert window.start_time == window.end_time + + +@pytest.mark.parametrize( + "symbol", + [ + "", + " ", + ], +) +def test_rejects_empty_symbol( + symbol: str, +) -> None: + with pytest.raises(ValueError): + TradeRecoveryWindow( + symbol=symbol, + start_time=1, + end_time=2, + ) + + +@pytest.mark.parametrize( + "symbol", + [ + None, + 123, + True, + ], +) +def test_rejects_non_string_symbol( + symbol: object, +) -> None: + with pytest.raises(TypeError): + TradeRecoveryWindow( + symbol=symbol, # type: ignore[arg-type] + start_time=1, + end_time=2, + ) + + +@pytest.mark.parametrize( + "start_time", + [ + 1.5, + "1000", + None, + True, + ], +) +def test_rejects_invalid_start_time_type( + start_time: object, +) -> None: + with pytest.raises(TypeError): + TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=start_time, # type: ignore[arg-type] + end_time=2, + ) + + +@pytest.mark.parametrize( + "end_time", + [ + 1.5, + "1000", + None, + True, + ], +) +def test_rejects_invalid_end_time_type( + end_time: object, +) -> None: + with pytest.raises(TypeError): + TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=1, + end_time=end_time, # type: ignore[arg-type] + ) + + +def test_rejects_negative_start_time() -> None: + with pytest.raises(ValueError): + TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=-1, + end_time=2, + ) + + +def test_rejects_negative_end_time() -> None: + with pytest.raises(ValueError): + TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=1, + end_time=-1, + ) + + +def test_rejects_start_time_greater_than_end_time() -> None: + with pytest.raises(ValueError): + TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=2, + end_time=1, + ) + + +def test_window_is_immutable() -> None: + window = TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=1, + end_time=2, + ) + + with pytest.raises(FrozenInstanceError): + window.end_time = 3 # type: ignore[misc] + + +def test_window_uses_slots() -> None: + window = TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=1, + end_time=2, + ) + + assert not hasattr(window, "__dict__") diff --git a/app/tests/unit/market_data/acquisition/recovery/test_trade_recovery_window_planner.py b/app/tests/unit/market_data/acquisition/recovery/test_trade_recovery_window_planner.py new file mode 100644 index 0000000..e423cac --- /dev/null +++ b/app/tests/unit/market_data/acquisition/recovery/test_trade_recovery_window_planner.py @@ -0,0 +1,380 @@ +# app/tests/unit/market_data/acquisition/recovery/test_trade_recovery_window_planner.py + +from __future__ import annotations + +import pytest + +from src.market_data.acquisition.recovery.trade_recovery_window import ( + TradeRecoveryWindow, +) +from src.market_data.acquisition.recovery.trade_recovery_window_planner import ( + DEFAULT_TRADE_RECOVERY_WINDOW_MS, + MAX_TRADE_RECOVERY_REQUEST_WINDOW_MS, + TradeRecoveryWindowPlanner, +) + + +def test_planner_uses_slots() -> None: + planner = TradeRecoveryWindowPlanner() + + assert not hasattr(planner, "__dict__") + + +def test_uses_default_max_window() -> None: + planner = TradeRecoveryWindowPlanner() + + assert ( + planner.max_window_ms + == DEFAULT_TRADE_RECOVERY_WINDOW_MS + ) + + +def test_accepts_custom_max_window() -> None: + planner = TradeRecoveryWindowPlanner( + max_window_ms=1_000, + ) + + assert planner.max_window_ms == 1_000 + + +@pytest.mark.parametrize( + "max_window_ms", + [ + 1.5, + "1000", + None, + True, + ], +) +def test_rejects_invalid_max_window_type( + max_window_ms: object, +) -> None: + with pytest.raises(TypeError): + TradeRecoveryWindowPlanner( + max_window_ms=max_window_ms, # type: ignore[arg-type] + ) + + +@pytest.mark.parametrize( + "max_window_ms", + [ + 0, + -1, + ], +) +def test_rejects_non_positive_max_window( + max_window_ms: int, +) -> None: + with pytest.raises(ValueError): + TradeRecoveryWindowPlanner( + max_window_ms=max_window_ms, + ) + + +def test_rejects_window_larger_than_request_limit() -> None: + with pytest.raises(ValueError): + TradeRecoveryWindowPlanner( + max_window_ms=( + MAX_TRADE_RECOVERY_REQUEST_WINDOW_MS + 1 + ), + ) + + +def test_returns_empty_tuple_for_equal_boundaries() -> None: + planner = TradeRecoveryWindowPlanner() + + result = planner.build_windows( + symbol="BTCUSDT", + start_time=1_000, + end_time=1_000, + ) + + assert result == () + assert isinstance(result, tuple) + + +def test_builds_single_window_for_short_range() -> None: + planner = TradeRecoveryWindowPlanner( + max_window_ms=1_000, + ) + + result = planner.build_windows( + symbol="BTCUSDT", + start_time=1_000, + end_time=1_500, + ) + + assert result == ( + TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=1_000, + end_time=1_500, + ), + ) + + +def test_builds_single_window_at_exact_configured_maximum() -> None: + planner = TradeRecoveryWindowPlanner( + max_window_ms=1_000, + ) + + result = planner.build_windows( + symbol="BTCUSDT", + start_time=1_000, + end_time=2_000, + ) + + assert result == ( + TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=1_000, + end_time=2_000, + ), + ) + + +def test_splits_range_into_multiple_windows() -> None: + planner = TradeRecoveryWindowPlanner( + max_window_ms=1_000, + ) + + result = planner.build_windows( + symbol="BTCUSDT", + start_time=1_000, + end_time=3_500, + ) + + assert result == ( + TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=1_000, + end_time=2_000, + ), + TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=2_000, + end_time=3_000, + ), + TradeRecoveryWindow( + symbol="BTCUSDT", + start_time=3_000, + end_time=3_500, + ), + ) + + +def test_returns_tuple_for_multiple_windows() -> None: + planner = TradeRecoveryWindowPlanner( + max_window_ms=1_000, + ) + + result = planner.build_windows( + symbol="BTCUSDT", + start_time=0, + end_time=2_500, + ) + + assert isinstance(result, tuple) + + +def test_preserves_symbol_in_every_window() -> None: + planner = TradeRecoveryWindowPlanner( + max_window_ms=1_000, + ) + + result = planner.build_windows( + symbol="ETHUSDT", + start_time=0, + end_time=2_500, + ) + + assert result + assert all( + window.symbol == "ETHUSDT" + for window in result + ) + + +def test_windows_are_contiguous() -> None: + planner = TradeRecoveryWindowPlanner( + max_window_ms=1_000, + ) + + result = planner.build_windows( + symbol="BTCUSDT", + start_time=0, + end_time=3_500, + ) + + for previous, current in zip( + result, + result[1:], + strict=False, + ): + assert previous.end_time == current.start_time + + +def test_windows_cover_complete_range() -> None: + planner = TradeRecoveryWindowPlanner( + max_window_ms=1_000, + ) + + result = planner.build_windows( + symbol="BTCUSDT", + start_time=500, + end_time=3_750, + ) + + assert result[0].start_time == 500 + assert result[-1].end_time == 3_750 + + +def test_no_window_exceeds_configured_size() -> None: + planner = TradeRecoveryWindowPlanner( + max_window_ms=1_000, + ) + + result = planner.build_windows( + symbol="BTCUSDT", + start_time=0, + end_time=4_500, + ) + + assert all( + window.end_time - window.start_time <= 1_000 + for window in result + ) + + +def test_default_windows_satisfy_recovery_request_limit() -> None: + planner = TradeRecoveryWindowPlanner() + + result = planner.build_windows( + symbol="BTCUSDT", + start_time=0, + end_time=7_500_000, + ) + + assert result + assert all( + window.end_time - window.start_time < 3_600_000 + for window in result + ) + + +@pytest.mark.parametrize( + "symbol", + [ + "", + " ", + ], +) +def test_rejects_empty_symbol( + symbol: str, +) -> None: + planner = TradeRecoveryWindowPlanner() + + with pytest.raises(ValueError): + planner.build_windows( + symbol=symbol, + start_time=1, + end_time=2, + ) + + +@pytest.mark.parametrize( + "symbol", + [ + None, + 123, + True, + ], +) +def test_rejects_non_string_symbol( + symbol: object, +) -> None: + planner = TradeRecoveryWindowPlanner() + + with pytest.raises(TypeError): + planner.build_windows( + symbol=symbol, # type: ignore[arg-type] + start_time=1, + end_time=2, + ) + + +@pytest.mark.parametrize( + "start_time", + [ + 1.5, + "1000", + None, + True, + ], +) +def test_rejects_invalid_start_time_type( + start_time: object, +) -> None: + planner = TradeRecoveryWindowPlanner() + + with pytest.raises(TypeError): + planner.build_windows( + symbol="BTCUSDT", + start_time=start_time, # type: ignore[arg-type] + end_time=2, + ) + + +@pytest.mark.parametrize( + "end_time", + [ + 1.5, + "1000", + None, + True, + ], +) +def test_rejects_invalid_end_time_type( + end_time: object, +) -> None: + planner = TradeRecoveryWindowPlanner() + + with pytest.raises(TypeError): + planner.build_windows( + symbol="BTCUSDT", + start_time=1, + end_time=end_time, # type: ignore[arg-type] + ) + + +def test_rejects_negative_start_time() -> None: + planner = TradeRecoveryWindowPlanner() + + with pytest.raises(ValueError): + planner.build_windows( + symbol="BTCUSDT", + start_time=-1, + end_time=2, + ) + + +def test_rejects_negative_end_time() -> None: + planner = TradeRecoveryWindowPlanner() + + with pytest.raises(ValueError): + planner.build_windows( + symbol="BTCUSDT", + start_time=1, + end_time=-1, + ) + + +def test_rejects_start_time_greater_than_end_time() -> None: + planner = TradeRecoveryWindowPlanner() + + with pytest.raises(ValueError): + planner.build_windows( + symbol="BTCUSDT", + start_time=2, + end_time=1, + ) diff --git a/app/tests/unit/market_data/acquisition/runtime/test_heartbeat_monitor.py b/app/tests/unit/market_data/acquisition/runtime/test_heartbeat_monitor.py new file mode 100644 index 0000000..ab8882d --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_heartbeat_monitor.py @@ -0,0 +1,466 @@ +# app/tests/unit/market_data/acquisition/runtime/test_heartbeat_monitor.py + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from src.market_data.acquisition.runtime.heartbeat import ( + HeartbeatMonitor, + HeartbeatMonitorProtocol, + HeartbeatState, +) +from src.market_data.acquisition.runtime.runtime_events import ( + HeartbeatTimeoutEvent, +) + + +class FakeClock: + def __init__( + self, + initial_value: float = 0.0, + ) -> None: + self.value = initial_value + + def __call__(self) -> float: + return self.value + + def advance( + self, + seconds: float, + ) -> None: + self.value += seconds + + +class FakeEventPublisher: + def __init__(self) -> None: + self.events: list[Any] = [] + + async def publish( + self, + event: Any, + ) -> None: + self.events.append(event) + + +def create_monitor( + *, + timeout_seconds: float = 10.0, + initial_clock_value: float = 0.0, +) -> tuple[ + HeartbeatMonitor, + FakeClock, + FakeEventPublisher, +]: + clock = FakeClock( + initial_value=initial_clock_value, + ) + publisher = FakeEventPublisher() + + monitor = HeartbeatMonitor( + event_publisher=publisher, + timeout_seconds=timeout_seconds, + clock=clock, + ) + + return ( + monitor, + clock, + publisher, + ) + + +def test_monitor_implements_protocol() -> None: + monitor, *_ = create_monitor() + + assert isinstance( + monitor, + HeartbeatMonitorProtocol, + ) + + +def test_monitor_uses_slots() -> None: + monitor, *_ = create_monitor() + + assert not hasattr(monitor, "__dict__") + + +def test_initial_state_is_idle() -> None: + monitor, *_ = create_monitor() + + assert monitor.state is HeartbeatState.IDLE + assert monitor.last_activity_at is None + + +def test_exposes_timeout_seconds() -> None: + monitor, *_ = create_monitor( + timeout_seconds=15.5, + ) + + assert monitor.timeout_seconds == 15.5 + + +@pytest.mark.parametrize( + "timeout_seconds", + [ + 1.5, + 10, + ], +) +def test_accepts_positive_timeout( + timeout_seconds: float, +) -> None: + monitor, *_ = create_monitor( + timeout_seconds=timeout_seconds, + ) + + assert monitor.timeout_seconds == float(timeout_seconds) + + +@pytest.mark.parametrize( + "timeout_seconds", + [ + "10", + None, + True, + ], +) +def test_rejects_invalid_timeout_type( + timeout_seconds: object, +) -> None: + with pytest.raises(TypeError): + HeartbeatMonitor( + event_publisher=FakeEventPublisher(), + timeout_seconds=timeout_seconds, # type: ignore[arg-type] + clock=FakeClock(), + ) + + +@pytest.mark.parametrize( + "timeout_seconds", + [ + 0, + -1, + -0.5, + ], +) +def test_rejects_non_positive_timeout( + timeout_seconds: float, +) -> None: + with pytest.raises(ValueError): + HeartbeatMonitor( + event_publisher=FakeEventPublisher(), + timeout_seconds=timeout_seconds, + clock=FakeClock(), + ) + + +def test_rejects_non_callable_clock() -> None: + with pytest.raises(TypeError): + HeartbeatMonitor( + event_publisher=FakeEventPublisher(), + timeout_seconds=10.0, + clock=object(), # type: ignore[arg-type] + ) + + +def test_start_begins_monitoring() -> None: + monitor, clock, _ = create_monitor( + initial_clock_value=100.0, + ) + + monitor.start() + + assert monitor.state is HeartbeatState.MONITORING + assert monitor.last_activity_at == 100.0 + + +def test_stop_returns_monitor_to_idle() -> None: + monitor, *_ = create_monitor() + + monitor.start() + monitor.stop() + + assert monitor.state is HeartbeatState.IDLE + assert monitor.last_activity_at is None + + +def test_record_activity_starts_monitoring() -> None: + monitor, clock, _ = create_monitor( + initial_clock_value=50.0, + ) + + monitor.record_activity() + + assert monitor.state is HeartbeatState.MONITORING + assert monitor.last_activity_at == 50.0 + + +def test_record_activity_updates_last_activity_time() -> None: + monitor, clock, _ = create_monitor() + + monitor.record_activity() + + clock.advance(3.5) + monitor.record_activity() + + assert monitor.last_activity_at == 3.5 + + +def test_check_timeout_returns_false_while_idle() -> None: + monitor, _, publisher = create_monitor() + + result = asyncio.run( + monitor.check_timeout() + ) + + assert result is False + assert publisher.events == [] + assert monitor.state is HeartbeatState.IDLE + + +def test_check_timeout_returns_false_before_threshold() -> None: + monitor, clock, publisher = create_monitor( + timeout_seconds=10.0, + ) + + monitor.start() + clock.advance(9.999) + + result = asyncio.run( + monitor.check_timeout() + ) + + assert result is False + assert publisher.events == [] + assert monitor.state is HeartbeatState.MONITORING + + +def test_check_timeout_triggers_at_exact_threshold() -> None: + monitor, clock, publisher = create_monitor( + timeout_seconds=10.0, + ) + + monitor.start() + clock.advance(10.0) + + result = asyncio.run( + monitor.check_timeout() + ) + + assert result is True + assert monitor.state is HeartbeatState.TIMED_OUT + assert publisher.events == [ + HeartbeatTimeoutEvent( + timeout_seconds=10.0, + ), + ] + + +def test_check_timeout_triggers_after_threshold() -> None: + monitor, clock, publisher = create_monitor( + timeout_seconds=10.0, + ) + + monitor.start() + clock.advance(15.0) + + result = asyncio.run( + monitor.check_timeout() + ) + + assert result is True + assert publisher.events == [ + HeartbeatTimeoutEvent( + timeout_seconds=10.0, + ), + ] + + +def test_timeout_event_is_published_only_once_per_inactivity_period() -> None: + monitor, clock, publisher = create_monitor( + timeout_seconds=10.0, + ) + + monitor.start() + clock.advance(10.0) + + first_result = asyncio.run( + monitor.check_timeout() + ) + + clock.advance(5.0) + + second_result = asyncio.run( + monitor.check_timeout() + ) + + assert first_result is True + assert second_result is False + assert publisher.events == [ + HeartbeatTimeoutEvent( + timeout_seconds=10.0, + ), + ] + + +def test_record_activity_resets_timed_out_state() -> None: + monitor, clock, _ = create_monitor( + timeout_seconds=10.0, + ) + + monitor.start() + clock.advance(10.0) + + asyncio.run( + monitor.check_timeout() + ) + + clock.advance(1.0) + monitor.record_activity() + + assert monitor.state is HeartbeatState.MONITORING + assert monitor.last_activity_at == 11.0 + + +def test_new_activity_allows_future_timeout_event() -> None: + monitor, clock, publisher = create_monitor( + timeout_seconds=10.0, + ) + + monitor.start() + clock.advance(10.0) + + first_result = asyncio.run( + monitor.check_timeout() + ) + + clock.advance(1.0) + monitor.record_activity() + + clock.advance(10.0) + + second_result = asyncio.run( + monitor.check_timeout() + ) + + assert first_result is True + assert second_result is True + assert publisher.events == [ + HeartbeatTimeoutEvent( + timeout_seconds=10.0, + ), + HeartbeatTimeoutEvent( + timeout_seconds=10.0, + ), + ] + + +def test_stop_after_timeout_clears_runtime_state() -> None: + monitor, clock, _ = create_monitor( + timeout_seconds=10.0, + ) + + monitor.start() + clock.advance(10.0) + + asyncio.run( + monitor.check_timeout() + ) + + monitor.stop() + + assert monitor.state is HeartbeatState.IDLE + assert monitor.last_activity_at is None + + +def test_start_after_timeout_begins_new_monitoring_period() -> None: + monitor, clock, publisher = create_monitor( + timeout_seconds=10.0, + ) + + monitor.start() + clock.advance(10.0) + + asyncio.run( + monitor.check_timeout() + ) + + clock.advance(5.0) + monitor.start() + + assert monitor.state is HeartbeatState.MONITORING + assert monitor.last_activity_at == 15.0 + + clock.advance(10.0) + + result = asyncio.run( + monitor.check_timeout() + ) + + assert result is True + assert publisher.events == [ + HeartbeatTimeoutEvent( + timeout_seconds=10.0, + ), + HeartbeatTimeoutEvent( + timeout_seconds=10.0, + ), + ] + + +def test_event_publisher_error_is_propagated() -> None: + class BrokenEventPublisher(FakeEventPublisher): + async def publish( + self, + event: Any, + ) -> None: + raise RuntimeError("publish failed") + + clock = FakeClock() + + monitor = HeartbeatMonitor( + event_publisher=BrokenEventPublisher(), + timeout_seconds=10.0, + clock=clock, + ) + + monitor.start() + clock.advance(10.0) + + with pytest.raises( + RuntimeError, + match="publish failed", + ): + asyncio.run( + monitor.check_timeout() + ) + + +def test_publisher_error_leaves_monitor_timed_out() -> None: + class BrokenEventPublisher(FakeEventPublisher): + async def publish( + self, + event: Any, + ) -> None: + raise RuntimeError("publish failed") + + clock = FakeClock() + + monitor = HeartbeatMonitor( + event_publisher=BrokenEventPublisher(), + timeout_seconds=10.0, + clock=clock, + ) + + monitor.start() + clock.advance(10.0) + + with pytest.raises(RuntimeError): + asyncio.run( + monitor.check_timeout() + ) + + assert monitor.state is HeartbeatState.TIMED_OUT diff --git a/app/tests/unit/market_data/acquisition/runtime/test_reconnect_coordinator.py b/app/tests/unit/market_data/acquisition/runtime/test_reconnect_coordinator.py new file mode 100644 index 0000000..dd237e7 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_reconnect_coordinator.py @@ -0,0 +1,349 @@ +# app/tests/unit/market_data/acquisition/runtime/test_reconnect_coordinator.py + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from src.market_data.acquisition.runtime.reconnect import ( + ReconnectCoordinator, + ReconnectCoordinatorProtocol, + ReconnectState, +) +from src.market_data.acquisition.runtime.runtime_commands import ( + ConnectCommand, +) +from src.market_data.acquisition.runtime.runtime_events import ( + ReconnectCompletedEvent, + ReconnectFailedEvent, + ReconnectStartedEvent, +) + + +class FakeCommandDispatcher: + def __init__(self) -> None: + self.commands: list[Any] = [] + + async def dispatch( + self, + command: Any, + ) -> None: + self.commands.append(command) + + +class FakeSubscriptionManager: + def __init__(self) -> None: + self.restore_calls = 0 + + async def subscribe( + self, + subscription_key: str, + message: Any, + ) -> None: + return None + + async def unsubscribe( + self, + subscription_key: str, + message: Any, + ) -> None: + return None + + async def restore_subscriptions(self) -> None: + self.restore_calls += 1 + + async def clear_subscriptions(self) -> None: + return None + + +class FakeEventPublisher: + def __init__(self) -> None: + self.events: list[Any] = [] + + async def publish( + self, + event: Any, + ) -> None: + self.events.append(event) + + +def create_coordinator() -> tuple[ + ReconnectCoordinator, + FakeCommandDispatcher, + FakeSubscriptionManager, + FakeEventPublisher, +]: + dispatcher = FakeCommandDispatcher() + subscriptions = FakeSubscriptionManager() + publisher = FakeEventPublisher() + + coordinator = ReconnectCoordinator( + command_dispatcher=dispatcher, + subscription_manager=subscriptions, + event_publisher=publisher, + ) + + return ( + coordinator, + dispatcher, + subscriptions, + publisher, + ) + + +def test_coordinator_implements_protocol() -> None: + coordinator, *_ = create_coordinator() + + assert isinstance( + coordinator, + ReconnectCoordinatorProtocol, + ) + + +def test_coordinator_uses_slots() -> None: + coordinator, *_ = create_coordinator() + + assert not hasattr(coordinator, "__dict__") + + +def test_initial_state_is_disconnected() -> None: + coordinator, *_ = create_coordinator() + + assert coordinator.state is ReconnectState.DISCONNECTED + assert coordinator.attempt == 0 + + +def test_reconnect_dispatches_connect_command() -> None: + coordinator, dispatcher, *_ = create_coordinator() + + asyncio.run(coordinator.reconnect()) + + assert len(dispatcher.commands) == 1 + assert isinstance( + dispatcher.commands[0], + ConnectCommand, + ) + + +def test_reconnect_restores_subscriptions() -> None: + coordinator, _, subscriptions, _ = create_coordinator() + + asyncio.run(coordinator.reconnect()) + + assert subscriptions.restore_calls == 1 + + +def test_successful_reconnect_publishes_lifecycle_events() -> None: + coordinator, _, _, publisher = create_coordinator() + + asyncio.run(coordinator.reconnect()) + + assert publisher.events == [ + ReconnectStartedEvent(attempt=1), + ReconnectCompletedEvent(attempt=1), + ] + + +def test_successful_reconnect_sets_connected_state() -> None: + coordinator, *_ = create_coordinator() + + asyncio.run(coordinator.reconnect()) + + assert coordinator.state is ReconnectState.CONNECTED + assert coordinator.attempt == 1 + + +def test_attempt_increments_for_each_reconnect() -> None: + coordinator, *_ = create_coordinator() + + asyncio.run(coordinator.reconnect()) + asyncio.run(coordinator.reconnect()) + + assert coordinator.attempt == 2 + + +def test_second_reconnect_uses_next_attempt_number() -> None: + coordinator, _, _, publisher = create_coordinator() + + asyncio.run(coordinator.reconnect()) + asyncio.run(coordinator.reconnect()) + + assert publisher.events == [ + ReconnectStartedEvent(attempt=1), + ReconnectCompletedEvent(attempt=1), + ReconnectStartedEvent(attempt=2), + ReconnectCompletedEvent(attempt=2), + ] + + +def test_connect_error_publishes_failed_event() -> None: + class BrokenCommandDispatcher(FakeCommandDispatcher): + async def dispatch( + self, + command: Any, + ) -> None: + self.commands.append(command) + raise RuntimeError("connection failed") + + dispatcher = BrokenCommandDispatcher() + subscriptions = FakeSubscriptionManager() + publisher = FakeEventPublisher() + + coordinator = ReconnectCoordinator( + command_dispatcher=dispatcher, + subscription_manager=subscriptions, + event_publisher=publisher, + ) + + with pytest.raises( + RuntimeError, + match="connection failed", + ): + asyncio.run(coordinator.reconnect()) + + assert publisher.events == [ + ReconnectStartedEvent(attempt=1), + ReconnectFailedEvent( + attempt=1, + reason="connection failed", + ), + ] + + +def test_connect_error_sets_failed_state() -> None: + class BrokenCommandDispatcher(FakeCommandDispatcher): + async def dispatch( + self, + command: Any, + ) -> None: + raise RuntimeError("connection failed") + + coordinator = ReconnectCoordinator( + command_dispatcher=BrokenCommandDispatcher(), + subscription_manager=FakeSubscriptionManager(), + event_publisher=FakeEventPublisher(), + ) + + with pytest.raises(RuntimeError): + asyncio.run(coordinator.reconnect()) + + assert coordinator.state is ReconnectState.FAILED + assert coordinator.attempt == 1 + + +def test_connect_error_does_not_restore_subscriptions() -> None: + class BrokenCommandDispatcher(FakeCommandDispatcher): + async def dispatch( + self, + command: Any, + ) -> None: + raise RuntimeError("connection failed") + + subscriptions = FakeSubscriptionManager() + + coordinator = ReconnectCoordinator( + command_dispatcher=BrokenCommandDispatcher(), + subscription_manager=subscriptions, + event_publisher=FakeEventPublisher(), + ) + + with pytest.raises(RuntimeError): + asyncio.run(coordinator.reconnect()) + + assert subscriptions.restore_calls == 0 + + +def test_subscription_restore_error_publishes_failed_event() -> None: + class BrokenSubscriptionManager(FakeSubscriptionManager): + async def restore_subscriptions(self) -> None: + self.restore_calls += 1 + raise RuntimeError("restore failed") + + dispatcher = FakeCommandDispatcher() + subscriptions = BrokenSubscriptionManager() + publisher = FakeEventPublisher() + + coordinator = ReconnectCoordinator( + command_dispatcher=dispatcher, + subscription_manager=subscriptions, + event_publisher=publisher, + ) + + with pytest.raises( + RuntimeError, + match="restore failed", + ): + asyncio.run(coordinator.reconnect()) + + assert publisher.events == [ + ReconnectStartedEvent(attempt=1), + ReconnectFailedEvent( + attempt=1, + reason="restore failed", + ), + ] + + +def test_subscription_restore_error_sets_failed_state() -> None: + class BrokenSubscriptionManager(FakeSubscriptionManager): + async def restore_subscriptions(self) -> None: + raise RuntimeError("restore failed") + + coordinator = ReconnectCoordinator( + command_dispatcher=FakeCommandDispatcher(), + subscription_manager=BrokenSubscriptionManager(), + event_publisher=FakeEventPublisher(), + ) + + with pytest.raises(RuntimeError): + asyncio.run(coordinator.reconnect()) + + assert coordinator.state is ReconnectState.FAILED + + +def test_failed_attempt_can_be_retried() -> None: + class FailOnceCommandDispatcher(FakeCommandDispatcher): + def __init__(self) -> None: + super().__init__() + self.calls = 0 + + async def dispatch( + self, + command: Any, + ) -> None: + self.calls += 1 + self.commands.append(command) + + if self.calls == 1: + raise RuntimeError("temporary failure") + + dispatcher = FailOnceCommandDispatcher() + subscriptions = FakeSubscriptionManager() + publisher = FakeEventPublisher() + + coordinator = ReconnectCoordinator( + command_dispatcher=dispatcher, + subscription_manager=subscriptions, + event_publisher=publisher, + ) + + with pytest.raises(RuntimeError): + asyncio.run(coordinator.reconnect()) + + asyncio.run(coordinator.reconnect()) + + assert coordinator.attempt == 2 + assert coordinator.state is ReconnectState.CONNECTED + assert subscriptions.restore_calls == 1 + + assert publisher.events == [ + ReconnectStartedEvent(attempt=1), + ReconnectFailedEvent( + attempt=1, + reason="temporary failure", + ), + ReconnectStartedEvent(attempt=2), + ReconnectCompletedEvent(attempt=2), + ] diff --git a/app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py b/app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py new file mode 100644 index 0000000..a349019 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py @@ -0,0 +1,989 @@ +# app/tests/unit/market_data/acquisition/runtime/ +# test_runtime_recovery_coordinator.py + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any + +import pytest + +from src.market_data.acquisition.consistency.trade_stream_state import ( + TradeStreamState, +) +from src.market_data.acquisition.consistency.trade_stream_state_store_exceptions import ( + TradeStreamStateNotFoundError, +) +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.acquisition.recovery.trade_recovery_request import ( + TradeRecoveryRequest, +) +from src.market_data.acquisition.recovery.trade_recovery_result import ( + TradeRecoveryResult, +) +from src.market_data.acquisition.recovery.trade_recovery_window import ( + TradeRecoveryWindow, +) +from src.market_data.acquisition.recovery.trade_recovery_window_planner import ( + TradeRecoveryWindowPlanner, +) +from src.market_data.acquisition.runtime.runtime_recovery_coordinator import ( + RuntimeRecoveryCoordinator, +) +from src.market_data.acquisition.runtime.runtime_recovery_protocol import ( + RuntimeRecoveryProtocol, +) + + +SYMBOL = "BTC/USD_LEVERAGE" + +CHECKPOINT_TIME = datetime( + 2026, + 1, + 1, + 0, + 0, + 0, + 123000, + tzinfo=timezone.utc, +) + +CHECKPOINT_TIME_MS = 1_767_225_600_123 +RECOVERY_END_TIME_MS = CHECKPOINT_TIME_MS + 5_000 + + +def make_trade( + *, + trade_id: int = 1, + symbol: str = SYMBOL, + executed_at: datetime = CHECKPOINT_TIME, +) -> Trade: + """ + Создать каноническую Trade для Runtime Recovery tests. + """ + return Trade( + symbol=symbol, + trade_id=trade_id, + price=Decimal("64555.55"), + quantity=Decimal("0.002"), + executed_at=executed_at, + aggressor_side=TradeAggressorSide.BUY, + source="test", + ) + + +def make_state_with_checkpoint( + *, + checkpoint: Trade | None = None, +) -> TradeStreamState: + """ + Создать TradeStreamState с последней принятой сделкой. + """ + checkpoint_trade = checkpoint or make_trade() + + state = TradeStreamState( + symbol=checkpoint_trade.symbol, + ) + + accepted_trade = state.accept( + checkpoint_trade, + ) + + assert accepted_trade is checkpoint_trade + assert state.last_trade is checkpoint_trade + + return state + + +class FakeStateStore: + """ + Управляемая реализация TradeStreamStateStoreProtocol. + """ + + def __init__( + self, + *, + states: dict[str, TradeStreamState] | None = None, + ) -> None: + self._states = dict( + states or {}, + ) + + self.get_calls: list[str] = [] + self.get_or_create_calls: list[str] = [] + self.contains_calls: list[str] = [] + self.remove_calls: list[str] = [] + self.clear_calls = 0 + + def get_or_create( + self, + symbol: str, + ) -> TradeStreamState: + self.get_or_create_calls.append( + symbol, + ) + + state = self._states.get( + symbol, + ) + + if state is None: + state = TradeStreamState( + symbol=symbol, + ) + self._states[symbol] = state + + return state + + def get( + self, + symbol: str, + ) -> TradeStreamState: + self.get_calls.append( + symbol, + ) + + try: + return self._states[symbol] + except KeyError as error: + raise TradeStreamStateNotFoundError( + f"Trade Stream state для {symbol!r} не найден." + ) from error + + def contains( + self, + symbol: str, + ) -> bool: + self.contains_calls.append( + symbol, + ) + + return symbol in self._states + + def remove( + self, + symbol: str, + ) -> None: + self.remove_calls.append( + symbol, + ) + + try: + del self._states[symbol] + except KeyError as error: + raise TradeStreamStateNotFoundError( + f"Trade Stream state для {symbol!r} не найден." + ) from error + + def clear(self) -> None: + self.clear_calls += 1 + self._states.clear() + + +class RecordingWindowPlanner: + """ + Planner с заранее заданным результатом. + """ + + def __init__( + self, + *, + windows: tuple[TradeRecoveryWindow, ...] = (), + ) -> None: + self._windows = windows + + self.calls: list[ + dict[str, Any] + ] = [] + + @property + def max_window_ms(self) -> int: + return 3_599_999 + + def build_windows( + self, + *, + symbol: str, + start_time: int, + end_time: int, + ) -> tuple[TradeRecoveryWindow, ...]: + self.calls.append( + { + "symbol": symbol, + "start_time": start_time, + "end_time": end_time, + } + ) + + return self._windows + + +class RecordingRecoveryController: + """ + Recovery Controller с записью полученных запросов. + """ + + def __init__( + self, + *, + results: tuple[TradeRecoveryResult, ...] = (), + ) -> None: + self._results = list( + results, + ) + + self.requests: list[ + TradeRecoveryRequest + ] = [] + + def recover( + self, + request: TradeRecoveryRequest, + ) -> TradeRecoveryResult: + self.requests.append( + request, + ) + + if self._results: + return self._results.pop(0) + + return TradeRecoveryResult( + symbol=request.symbol, + requested_start_time=request.start_time, + requested_end_time=request.end_time, + recovered_trades=(), + ) + + +def create_coordinator( + *, + state_store: FakeStateStore | None = None, + window_planner: RecordingWindowPlanner | None = None, + recovery_controller: RecordingRecoveryController | None = None, +) -> tuple[ + RuntimeRecoveryCoordinator, + FakeStateStore, + RecordingWindowPlanner, + RecordingRecoveryController, +]: + """ + Создать Coordinator и управляемые зависимости. + """ + resolved_state_store = ( + state_store + or FakeStateStore() + ) + + resolved_window_planner = ( + window_planner + or RecordingWindowPlanner() + ) + + resolved_recovery_controller = ( + recovery_controller + or RecordingRecoveryController() + ) + + coordinator = RuntimeRecoveryCoordinator( + state_store=resolved_state_store, + window_planner=resolved_window_planner, # type: ignore[arg-type] + recovery_controller=resolved_recovery_controller, + ) + + return ( + coordinator, + resolved_state_store, + resolved_window_planner, + resolved_recovery_controller, + ) + + +def test_coordinator_implements_protocol() -> None: + coordinator, *_ = create_coordinator() + + assert isinstance( + coordinator, + RuntimeRecoveryProtocol, + ) + + +def test_coordinator_uses_slots() -> None: + coordinator, *_ = create_coordinator() + + assert not hasattr( + coordinator, + "__dict__", + ) + + +@pytest.mark.parametrize( + "symbol", + [ + None, + 123, + True, + ], +) +def test_rejects_non_string_symbol( + symbol: object, +) -> None: + coordinator, state_store, planner, recovery = ( + create_coordinator() + ) + + with pytest.raises( + TypeError, + match="symbol", + ): + coordinator.recover( + symbol=symbol, # type: ignore[arg-type] + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert state_store.get_calls == [] + assert planner.calls == [] + assert recovery.requests == [] + + +@pytest.mark.parametrize( + "symbol", + [ + "", + " ", + ], +) +def test_rejects_empty_symbol( + symbol: str, +) -> None: + coordinator, state_store, planner, recovery = ( + create_coordinator() + ) + + with pytest.raises( + ValueError, + match="symbol", + ): + coordinator.recover( + symbol=symbol, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert state_store.get_calls == [] + assert planner.calls == [] + assert recovery.requests == [] + + +@pytest.mark.parametrize( + "recovery_end_time", + [ + 1.5, + "1000", + None, + True, + ], +) +def test_rejects_invalid_recovery_end_time_type( + recovery_end_time: object, +) -> None: + coordinator, state_store, planner, recovery = ( + create_coordinator() + ) + + with pytest.raises( + TypeError, + match="recovery_end_time", + ): + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=recovery_end_time, # type: ignore[arg-type] + ) + + assert state_store.get_calls == [] + assert planner.calls == [] + assert recovery.requests == [] + + +def test_rejects_negative_recovery_end_time() -> None: + coordinator, state_store, planner, recovery = ( + create_coordinator() + ) + + with pytest.raises( + ValueError, + match="recovery_end_time", + ): + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=-1, + ) + + assert state_store.get_calls == [] + assert planner.calls == [] + assert recovery.requests == [] + + +def test_reads_state_by_symbol() -> None: + state_store = FakeStateStore( + states={ + SYMBOL: make_state_with_checkpoint(), + } + ) + + coordinator, _, _, _ = create_coordinator( + state_store=state_store, + ) + + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert state_store.get_calls == [ + SYMBOL, + ] + + +def test_does_not_create_state_during_recovery() -> None: + coordinator, state_store, _, _ = ( + create_coordinator() + ) + + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert state_store.get_or_create_calls == [] + + +def test_missing_state_returns_empty_result() -> None: + coordinator, _, planner, recovery = ( + create_coordinator() + ) + + result = coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert result == TradeRecoveryResult( + symbol=SYMBOL, + requested_start_time=RECOVERY_END_TIME_MS, + requested_end_time=RECOVERY_END_TIME_MS, + recovered_trades=(), + ) + + assert planner.calls == [] + assert recovery.requests == [] + + +def test_state_without_checkpoint_returns_empty_result() -> None: + state_store = FakeStateStore( + states={ + SYMBOL: TradeStreamState( + symbol=SYMBOL, + ), + } + ) + + coordinator, _, planner, recovery = ( + create_coordinator( + state_store=state_store, + ) + ) + + result = coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert result.is_empty is True + + assert ( + result.requested_start_time + == RECOVERY_END_TIME_MS + ) + assert ( + result.requested_end_time + == RECOVERY_END_TIME_MS + ) + + assert planner.calls == [] + assert recovery.requests == [] + + +def test_checkpoint_time_is_passed_to_planner_as_unix_ms() -> None: + state_store = FakeStateStore( + states={ + SYMBOL: make_state_with_checkpoint(), + } + ) + + planner = RecordingWindowPlanner() + + coordinator, _, _, _ = create_coordinator( + state_store=state_store, + window_planner=planner, + ) + + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert planner.calls == [ + { + "symbol": SYMBOL, + "start_time": CHECKPOINT_TIME_MS, + "end_time": RECOVERY_END_TIME_MS, + } + ] + + +def test_checkpoint_timezone_is_normalized_to_utc() -> None: + offset_timezone = timezone( + timedelta(hours=3), + ) + + checkpoint = make_trade( + executed_at=datetime( + 2026, + 1, + 1, + 3, + 0, + 0, + 123000, + tzinfo=offset_timezone, + ), + ) + + state_store = FakeStateStore( + states={ + SYMBOL: make_state_with_checkpoint( + checkpoint=checkpoint, + ), + } + ) + + planner = RecordingWindowPlanner() + + coordinator, _, _, _ = create_coordinator( + state_store=state_store, + window_planner=planner, + ) + + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert planner.calls[0]["start_time"] == ( + CHECKPOINT_TIME_MS + ) + + +def test_rejects_naive_checkpoint_datetime() -> None: + checkpoint = make_trade( + executed_at=datetime( + 2026, + 1, + 1, + 0, + 0, + 0, + 123000, + ), + ) + + state_store = FakeStateStore( + states={ + SYMBOL: make_state_with_checkpoint( + checkpoint=checkpoint, + ), + } + ) + + coordinator, _, planner, recovery = ( + create_coordinator( + state_store=state_store, + ) + ) + + with pytest.raises( + ValueError, + match="timezone", + ): + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert planner.calls == [] + assert recovery.requests == [] + + +def test_empty_planner_result_returns_empty_recovery() -> None: + state_store = FakeStateStore( + states={ + SYMBOL: make_state_with_checkpoint(), + } + ) + + planner = RecordingWindowPlanner( + windows=(), + ) + + coordinator, _, _, recovery = create_coordinator( + state_store=state_store, + window_planner=planner, + ) + + result = coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert result.is_empty is True + assert recovery.requests == [] + + +def test_single_window_is_forwarded_to_controller() -> None: + window = TradeRecoveryWindow( + symbol=SYMBOL, + start_time=CHECKPOINT_TIME_MS, + end_time=RECOVERY_END_TIME_MS, + ) + + state_store = FakeStateStore( + states={ + SYMBOL: make_state_with_checkpoint(), + } + ) + + planner = RecordingWindowPlanner( + windows=(window,), + ) + + recovery = RecordingRecoveryController() + + coordinator, *_ = create_coordinator( + state_store=state_store, + window_planner=planner, + recovery_controller=recovery, + ) + + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert len(recovery.requests) == 1 + + request = recovery.requests[0] + + assert request.symbol == SYMBOL + assert request.start_time == window.start_time + assert request.end_time == window.end_time + + +def test_multiple_windows_are_processed_sequentially() -> None: + windows = ( + TradeRecoveryWindow( + symbol=SYMBOL, + start_time=CHECKPOINT_TIME_MS, + end_time=CHECKPOINT_TIME_MS + 1_000, + ), + TradeRecoveryWindow( + symbol=SYMBOL, + start_time=CHECKPOINT_TIME_MS + 1_000, + end_time=CHECKPOINT_TIME_MS + 3_000, + ), + TradeRecoveryWindow( + symbol=SYMBOL, + start_time=CHECKPOINT_TIME_MS + 3_000, + end_time=RECOVERY_END_TIME_MS, + ), + ) + + state_store = FakeStateStore( + states={ + SYMBOL: make_state_with_checkpoint(), + } + ) + + planner = RecordingWindowPlanner( + windows=windows, + ) + + recovery = RecordingRecoveryController() + + coordinator, *_ = create_coordinator( + state_store=state_store, + window_planner=planner, + recovery_controller=recovery, + ) + + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert [ + (r.start_time, r.end_time) + for r in recovery.requests + ] == [ + ( + CHECKPOINT_TIME_MS, + CHECKPOINT_TIME_MS + 1_000, + ), + ( + CHECKPOINT_TIME_MS + 1_000, + CHECKPOINT_TIME_MS + 3_000, + ), + ( + CHECKPOINT_TIME_MS + 3_000, + RECOVERY_END_TIME_MS, + ), + ] + + +def test_multiple_window_results_are_aggregated() -> None: + middle_time = CHECKPOINT_TIME_MS + 2_500 + + window_one = TradeRecoveryWindow( + symbol=SYMBOL, + start_time=CHECKPOINT_TIME_MS, + end_time=middle_time, + ) + + window_two = TradeRecoveryWindow( + symbol=SYMBOL, + start_time=middle_time, + end_time=RECOVERY_END_TIME_MS, + ) + + first_trade = make_trade( + trade_id=100, + ) + + second_trade = make_trade( + trade_id=200, + ) + + result_one = TradeRecoveryResult( + symbol=SYMBOL, + requested_start_time=CHECKPOINT_TIME_MS, + requested_end_time=middle_time, + recovered_trades=( + first_trade, + ), + ) + + result_two = TradeRecoveryResult( + symbol=SYMBOL, + requested_start_time=middle_time, + requested_end_time=RECOVERY_END_TIME_MS, + recovered_trades=( + second_trade, + ), + ) + + coordinator, *_ = create_coordinator( + state_store=FakeStateStore( + states={ + SYMBOL: make_state_with_checkpoint(), + } + ), + window_planner=RecordingWindowPlanner( + windows=( + window_one, + window_two, + ), + ), + recovery_controller=RecordingRecoveryController( + results=( + result_one, + result_two, + ), + ), + ) + + result = coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert result.recovered_trades == ( + first_trade, + second_trade, + ) + assert ( + result.requested_start_time + == CHECKPOINT_TIME_MS + ) + assert ( + result.requested_end_time + == RECOVERY_END_TIME_MS + ) + assert result.first_trade is first_trade + assert result.last_trade is second_trade + + +def test_single_window_result_is_aggregated_into_full_result() -> None: + recovered_trade = make_trade( + trade_id=777, + ) + + expected = TradeRecoveryResult( + symbol=SYMBOL, + requested_start_time=CHECKPOINT_TIME_MS, + requested_end_time=RECOVERY_END_TIME_MS, + recovered_trades=( + recovered_trade, + ), + ) + + coordinator, *_ = create_coordinator( + state_store=FakeStateStore( + states={ + SYMBOL: make_state_with_checkpoint(), + } + ), + window_planner=RecordingWindowPlanner( + windows=( + TradeRecoveryWindow( + symbol=SYMBOL, + start_time=CHECKPOINT_TIME_MS, + end_time=RECOVERY_END_TIME_MS, + ), + ), + ), + recovery_controller=RecordingRecoveryController( + results=(expected,), + ), + ) + + result = coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert result is not expected + assert result == expected + assert result.recovered_trades == ( + recovered_trade, + ) + assert result.first_trade is recovered_trade + assert result.last_trade is recovered_trade + + +def test_controller_error_is_not_swallowed() -> None: + class BrokenRecoveryController( + RecordingRecoveryController, + ): + def recover( + self, + request: TradeRecoveryRequest, + ) -> TradeRecoveryResult: + raise RuntimeError( + "controller failed", + ) + + coordinator, *_ = create_coordinator( + state_store=FakeStateStore( + states={ + SYMBOL: make_state_with_checkpoint(), + } + ), + window_planner=RecordingWindowPlanner( + windows=( + TradeRecoveryWindow( + symbol=SYMBOL, + start_time=CHECKPOINT_TIME_MS, + end_time=RECOVERY_END_TIME_MS, + ), + ), + ), + recovery_controller=BrokenRecoveryController(), + ) + + with pytest.raises( + RuntimeError, + match="controller failed", + ): + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + +def test_planner_error_is_not_swallowed() -> None: + class BrokenPlanner( + TradeRecoveryWindowPlanner, + ): + def __init__(self) -> None: + pass + + @property + def max_window_ms(self) -> int: + return 1 + + def build_windows( + self, + *, + symbol: str, + start_time: int, + end_time: int, + ) -> tuple[TradeRecoveryWindow, ...]: + raise RuntimeError( + "planner failed", + ) + + coordinator, *_ = create_coordinator( + state_store=FakeStateStore( + states={ + SYMBOL: make_state_with_checkpoint(), + } + ), + window_planner=BrokenPlanner(), + ) + + with pytest.raises( + RuntimeError, + match="planner failed", + ): + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + +def test_store_error_is_not_swallowed() -> None: + class BrokenStore( + FakeStateStore, + ): + def get( + self, + symbol: str, + ) -> TradeStreamState: + raise RuntimeError( + "store failed", + ) + + coordinator, *_ = create_coordinator( + state_store=BrokenStore(), + ) + + with pytest.raises( + RuntimeError, + match="store failed", + ): + coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) diff --git a/app/tests/unit/market_data/acquisition/runtime/test_runtime_scheduler.py b/app/tests/unit/market_data/acquisition/runtime/test_runtime_scheduler.py new file mode 100644 index 0000000..3a44853 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_runtime_scheduler.py @@ -0,0 +1,590 @@ +# app/tests/unit/market_data/acquisition/runtime/test_runtime_scheduler.py + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable +from typing import Callable + +import pytest + +from src.market_data.acquisition.runtime.heartbeat import ( + HeartbeatState, +) +from src.market_data.acquisition.runtime.scheduler import ( + RuntimeScheduler, + RuntimeSchedulerProtocol, +) +from src.market_data.acquisition.runtime.supervisor import ( + RuntimeSupervisorState, +) + + +class FakeHeartbeatMonitor: + def __init__( + self, + results: list[bool] | None = None, + ) -> None: + self._results = list(results or []) + self.check_timeout_calls = 0 + + @property + def state(self) -> HeartbeatState: + return HeartbeatState.MONITORING + + @property + def timeout_seconds(self) -> float: + return 10.0 + + @property + def last_activity_at(self) -> float | None: + return 0.0 + + def start(self) -> None: + return None + + def stop(self) -> None: + return None + + def record_activity(self) -> None: + return None + + async def check_timeout(self) -> bool: + self.check_timeout_calls += 1 + + if not self._results: + return False + + return self._results.pop(0) + + +class FakeRuntimeSupervisor: + def __init__(self) -> None: + self.handle_timeout_calls = 0 + + @property + def state(self) -> RuntimeSupervisorState: + return RuntimeSupervisorState.RUNNING + + def start(self) -> None: + return None + + def stop(self) -> None: + return None + + def notify_activity(self) -> None: + return None + + async def handle_heartbeat_timeout(self) -> bool: + self.handle_timeout_calls += 1 + return True + + +class RecordingSleep: + def __init__( + self, + *, + on_call: Callable[[], None] | None = None, + ) -> None: + self.calls: list[float] = [] + self._on_call = on_call + + async def __call__( + self, + seconds: float, + ) -> None: + self.calls.append(seconds) + + if self._on_call is not None: + self._on_call() + + +def create_scheduler( + *, + heartbeat_results: list[bool] | None = None, + interval_seconds: float = 1.0, + sleep: Callable[[float], Awaitable[None]] | None = None, +) -> tuple[ + RuntimeScheduler, + FakeHeartbeatMonitor, + FakeRuntimeSupervisor, +]: + heartbeat = FakeHeartbeatMonitor( + results=heartbeat_results, + ) + supervisor = FakeRuntimeSupervisor() + + scheduler = RuntimeScheduler( + heartbeat_monitor=heartbeat, + runtime_supervisor=supervisor, + interval_seconds=interval_seconds, + sleep=sleep or asyncio.sleep, + ) + + return ( + scheduler, + heartbeat, + supervisor, + ) + + +def test_scheduler_implements_protocol() -> None: + scheduler, *_ = create_scheduler() + + assert isinstance( + scheduler, + RuntimeSchedulerProtocol, + ) + + +def test_scheduler_uses_slots() -> None: + scheduler, *_ = create_scheduler() + + assert not hasattr(scheduler, "__dict__") + + +def test_initial_state_is_not_running() -> None: + scheduler, *_ = create_scheduler() + + assert scheduler.running is False + + +def test_exposes_interval_seconds() -> None: + scheduler, *_ = create_scheduler( + interval_seconds=2.5, + ) + + assert scheduler.interval_seconds == 2.5 + + +@pytest.mark.parametrize( + "interval_seconds", + [ + 1, + 1.5, + ], +) +def test_accepts_positive_interval( + interval_seconds: float, +) -> None: + scheduler, *_ = create_scheduler( + interval_seconds=interval_seconds, + ) + + assert scheduler.interval_seconds == float(interval_seconds) + + +@pytest.mark.parametrize( + "interval_seconds", + [ + "1", + None, + True, + ], +) +def test_rejects_invalid_interval_type( + interval_seconds: object, +) -> None: + with pytest.raises(TypeError): + RuntimeScheduler( + heartbeat_monitor=FakeHeartbeatMonitor(), + runtime_supervisor=FakeRuntimeSupervisor(), + interval_seconds=interval_seconds, # type: ignore[arg-type] + ) + + +@pytest.mark.parametrize( + "interval_seconds", + [ + 0, + -1, + -0.5, + ], +) +def test_rejects_non_positive_interval( + interval_seconds: float, +) -> None: + with pytest.raises(ValueError): + RuntimeScheduler( + heartbeat_monitor=FakeHeartbeatMonitor(), + runtime_supervisor=FakeRuntimeSupervisor(), + interval_seconds=interval_seconds, + ) + + +def test_rejects_non_callable_sleep() -> None: + with pytest.raises(TypeError): + RuntimeScheduler( + heartbeat_monitor=FakeHeartbeatMonitor(), + runtime_supervisor=FakeRuntimeSupervisor(), + interval_seconds=1.0, + sleep=object(), # type: ignore[arg-type] + ) + + +def test_run_once_checks_heartbeat() -> None: + scheduler, heartbeat, supervisor = create_scheduler( + heartbeat_results=[False], + ) + + result = asyncio.run( + scheduler.run_once() + ) + + assert result is False + assert heartbeat.check_timeout_calls == 1 + assert supervisor.handle_timeout_calls == 0 + + +def test_run_once_calls_supervisor_on_timeout() -> None: + scheduler, heartbeat, supervisor = create_scheduler( + heartbeat_results=[True], + ) + + result = asyncio.run( + scheduler.run_once() + ) + + assert result is True + assert heartbeat.check_timeout_calls == 1 + assert supervisor.handle_timeout_calls == 1 + + +def test_run_once_does_not_call_supervisor_without_timeout() -> None: + scheduler, _, supervisor = create_scheduler( + heartbeat_results=[False], + ) + + asyncio.run( + scheduler.run_once() + ) + + assert supervisor.handle_timeout_calls == 0 + + +def test_stop_is_safe_before_start() -> None: + scheduler, *_ = create_scheduler() + + scheduler.stop() + + assert scheduler.running is False + + +def test_repeated_stop_is_safe() -> None: + scheduler, *_ = create_scheduler() + + scheduler.stop() + scheduler.stop() + + assert scheduler.running is False + + +def test_start_runs_periodic_check() -> None: + scheduler: RuntimeScheduler + + def stop_scheduler() -> None: + scheduler.stop() + + sleep = RecordingSleep( + on_call=stop_scheduler, + ) + + scheduler, heartbeat, supervisor = create_scheduler( + heartbeat_results=[False], + sleep=sleep, + ) + + asyncio.run( + scheduler.start() + ) + + assert heartbeat.check_timeout_calls == 1 + assert supervisor.handle_timeout_calls == 0 + assert sleep.calls == [1.0] + assert scheduler.running is False + + +def test_start_uses_configured_interval() -> None: + scheduler: RuntimeScheduler + + def stop_scheduler() -> None: + scheduler.stop() + + sleep = RecordingSleep( + on_call=stop_scheduler, + ) + + scheduler, _, _ = create_scheduler( + heartbeat_results=[False], + interval_seconds=2.5, + sleep=sleep, + ) + + asyncio.run( + scheduler.start() + ) + + assert sleep.calls == [2.5] + + +def test_scheduler_handles_timeout_during_loop() -> None: + scheduler: RuntimeScheduler + + def stop_scheduler() -> None: + scheduler.stop() + + sleep = RecordingSleep( + on_call=stop_scheduler, + ) + + scheduler, heartbeat, supervisor = create_scheduler( + heartbeat_results=[True], + sleep=sleep, + ) + + asyncio.run( + scheduler.start() + ) + + assert heartbeat.check_timeout_calls == 1 + assert supervisor.handle_timeout_calls == 1 + assert scheduler.running is False + + +def test_scheduler_can_run_multiple_iterations() -> None: + scheduler: RuntimeScheduler + sleep_calls = 0 + + async def sleep( + seconds: float, + ) -> None: + nonlocal sleep_calls + + assert seconds == 1.0 + + sleep_calls += 1 + + if sleep_calls == 2: + scheduler.stop() + + scheduler, heartbeat, supervisor = create_scheduler( + heartbeat_results=[ + False, + True, + ], + sleep=sleep, + ) + + asyncio.run( + scheduler.start() + ) + + assert heartbeat.check_timeout_calls == 2 + assert supervisor.handle_timeout_calls == 1 + assert sleep_calls == 2 + + +def test_stop_during_run_once_prevents_sleep() -> None: + heartbeat = FakeHeartbeatMonitor( + results=[False], + ) + supervisor = FakeRuntimeSupervisor() + sleep = RecordingSleep() + + scheduler: RuntimeScheduler + + class StoppingHeartbeatMonitor(FakeHeartbeatMonitor): + async def check_timeout(self) -> bool: + self.check_timeout_calls += 1 + scheduler.stop() + return False + + heartbeat = StoppingHeartbeatMonitor() + + scheduler = RuntimeScheduler( + heartbeat_monitor=heartbeat, + runtime_supervisor=supervisor, + interval_seconds=1.0, + sleep=sleep, + ) + + asyncio.run( + scheduler.start() + ) + + assert heartbeat.check_timeout_calls == 1 + assert sleep.calls == [] + assert scheduler.running is False + + +def test_repeated_start_while_running_does_not_create_second_loop() -> None: + heartbeat = FakeHeartbeatMonitor( + results=[False], + ) + supervisor = FakeRuntimeSupervisor() + + scheduler: RuntimeScheduler + nested_start_completed = False + + async def sleep( + seconds: float, + ) -> None: + nonlocal nested_start_completed + + assert seconds == 1.0 + + await scheduler.start() + nested_start_completed = True + scheduler.stop() + + scheduler = RuntimeScheduler( + heartbeat_monitor=heartbeat, + runtime_supervisor=supervisor, + interval_seconds=1.0, + sleep=sleep, + ) + + asyncio.run( + scheduler.start() + ) + + assert nested_start_completed is True + assert heartbeat.check_timeout_calls == 1 + assert scheduler.running is False + + +def test_scheduler_can_restart_after_stop() -> None: + scheduler: RuntimeScheduler + sleep_calls = 0 + + async def sleep( + seconds: float, + ) -> None: + nonlocal sleep_calls + + sleep_calls += 1 + scheduler.stop() + + scheduler, heartbeat, _ = create_scheduler( + heartbeat_results=[ + False, + False, + ], + sleep=sleep, + ) + + asyncio.run( + scheduler.start() + ) + asyncio.run( + scheduler.start() + ) + + assert heartbeat.check_timeout_calls == 2 + assert sleep_calls == 2 + assert scheduler.running is False + + +def test_heartbeat_error_is_propagated() -> None: + class BrokenHeartbeatMonitor(FakeHeartbeatMonitor): + async def check_timeout(self) -> bool: + raise RuntimeError("heartbeat failed") + + scheduler = RuntimeScheduler( + heartbeat_monitor=BrokenHeartbeatMonitor(), + runtime_supervisor=FakeRuntimeSupervisor(), + interval_seconds=1.0, + ) + + with pytest.raises( + RuntimeError, + match="heartbeat failed", + ): + asyncio.run( + scheduler.run_once() + ) + + +def test_supervisor_error_is_propagated() -> None: + class BrokenRuntimeSupervisor(FakeRuntimeSupervisor): + async def handle_heartbeat_timeout(self) -> bool: + raise RuntimeError("supervisor failed") + + scheduler = RuntimeScheduler( + heartbeat_monitor=FakeHeartbeatMonitor( + results=[True], + ), + runtime_supervisor=BrokenRuntimeSupervisor(), + interval_seconds=1.0, + ) + + with pytest.raises( + RuntimeError, + match="supervisor failed", + ): + asyncio.run( + scheduler.run_once() + ) + + +def test_start_resets_running_after_heartbeat_error() -> None: + class BrokenHeartbeatMonitor(FakeHeartbeatMonitor): + async def check_timeout(self) -> bool: + raise RuntimeError("heartbeat failed") + + scheduler = RuntimeScheduler( + heartbeat_monitor=BrokenHeartbeatMonitor(), + runtime_supervisor=FakeRuntimeSupervisor(), + interval_seconds=1.0, + ) + + with pytest.raises(RuntimeError): + asyncio.run( + scheduler.start() + ) + + assert scheduler.running is False + + +def test_start_resets_running_after_supervisor_error() -> None: + class BrokenRuntimeSupervisor(FakeRuntimeSupervisor): + async def handle_heartbeat_timeout(self) -> bool: + raise RuntimeError("supervisor failed") + + scheduler = RuntimeScheduler( + heartbeat_monitor=FakeHeartbeatMonitor( + results=[True], + ), + runtime_supervisor=BrokenRuntimeSupervisor(), + interval_seconds=1.0, + ) + + with pytest.raises(RuntimeError): + asyncio.run( + scheduler.start() + ) + + assert scheduler.running is False + + +def test_sleep_error_is_propagated_and_resets_running() -> None: + async def broken_sleep( + seconds: float, + ) -> None: + raise RuntimeError("sleep failed") + + scheduler, heartbeat, _ = create_scheduler( + heartbeat_results=[False], + sleep=broken_sleep, + ) + + with pytest.raises( + RuntimeError, + match="sleep failed", + ): + asyncio.run( + scheduler.start() + ) + + assert heartbeat.check_timeout_calls == 1 + assert scheduler.running is False diff --git a/app/tests/unit/market_data/acquisition/runtime/test_runtime_supervisor.py b/app/tests/unit/market_data/acquisition/runtime/test_runtime_supervisor.py new file mode 100644 index 0000000..7f4143d --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_runtime_supervisor.py @@ -0,0 +1,528 @@ +# app/tests/unit/market_data/acquisition/runtime/test_runtime_supervisor.py + +from __future__ import annotations + +import asyncio + +import pytest + +from src.market_data.acquisition.runtime.heartbeat import ( + HeartbeatState, +) +from src.market_data.acquisition.runtime.reconnect import ( + ReconnectState, +) +from src.market_data.acquisition.runtime.supervisor import ( + RuntimeSupervisor, + RuntimeSupervisorProtocol, + RuntimeSupervisorState, +) + + +class FakeHeartbeatMonitor: + def __init__(self) -> None: + self.start_calls = 0 + self.stop_calls = 0 + self.record_activity_calls = 0 + + self._state = HeartbeatState.IDLE + self._last_activity_at: float | None = None + + @property + def state(self) -> HeartbeatState: + return self._state + + @property + def timeout_seconds(self) -> float: + return 10.0 + + @property + def last_activity_at(self) -> float | None: + return self._last_activity_at + + def start(self) -> None: + self.start_calls += 1 + self._last_activity_at = float(self.start_calls) + self._state = HeartbeatState.MONITORING + + def stop(self) -> None: + self.stop_calls += 1 + self._last_activity_at = None + self._state = HeartbeatState.IDLE + + def record_activity(self) -> None: + self.record_activity_calls += 1 + self._last_activity_at = float( + self.record_activity_calls + ) + self._state = HeartbeatState.MONITORING + + async def check_timeout(self) -> bool: + return False + + +class FakeReconnectCoordinator: + def __init__(self) -> None: + self.reconnect_calls = 0 + self._attempt = 0 + self._state = ReconnectState.DISCONNECTED + + @property + def state(self) -> ReconnectState: + return self._state + + @property + def attempt(self) -> int: + return self._attempt + + async def reconnect(self) -> None: + self.reconnect_calls += 1 + self._attempt += 1 + self._state = ReconnectState.CONNECTED + + +def create_supervisor() -> tuple[ + RuntimeSupervisor, + FakeHeartbeatMonitor, + FakeReconnectCoordinator, +]: + heartbeat = FakeHeartbeatMonitor() + reconnect = FakeReconnectCoordinator() + + supervisor = RuntimeSupervisor( + heartbeat_monitor=heartbeat, + reconnect_coordinator=reconnect, + ) + + return ( + supervisor, + heartbeat, + reconnect, + ) + + +def test_supervisor_implements_protocol() -> None: + supervisor, *_ = create_supervisor() + + assert isinstance( + supervisor, + RuntimeSupervisorProtocol, + ) + + +def test_supervisor_uses_slots() -> None: + supervisor, *_ = create_supervisor() + + assert not hasattr(supervisor, "__dict__") + + +def test_initial_state_is_stopped() -> None: + supervisor, heartbeat, reconnect = create_supervisor() + + assert supervisor.state is RuntimeSupervisorState.STOPPED + assert heartbeat.start_calls == 0 + assert heartbeat.stop_calls == 0 + assert reconnect.reconnect_calls == 0 + + +def test_start_starts_heartbeat() -> None: + supervisor, heartbeat, _ = create_supervisor() + + supervisor.start() + + assert heartbeat.start_calls == 1 + assert heartbeat.state is HeartbeatState.MONITORING + + +def test_start_sets_running_state() -> None: + supervisor, *_ = create_supervisor() + + supervisor.start() + + assert supervisor.state is RuntimeSupervisorState.RUNNING + + +def test_repeated_start_begins_new_monitoring_period() -> None: + supervisor, heartbeat, _ = create_supervisor() + + supervisor.start() + supervisor.start() + + assert heartbeat.start_calls == 2 + assert supervisor.state is RuntimeSupervisorState.RUNNING + + +def test_stop_stops_heartbeat() -> None: + supervisor, heartbeat, _ = create_supervisor() + + supervisor.start() + supervisor.stop() + + assert heartbeat.stop_calls == 1 + assert heartbeat.state is HeartbeatState.IDLE + + +def test_stop_sets_stopped_state() -> None: + supervisor, *_ = create_supervisor() + + supervisor.start() + supervisor.stop() + + assert supervisor.state is RuntimeSupervisorState.STOPPED + + +def test_stop_is_safe_before_start() -> None: + supervisor, heartbeat, reconnect = create_supervisor() + + supervisor.stop() + + assert supervisor.state is RuntimeSupervisorState.STOPPED + assert heartbeat.stop_calls == 1 + assert reconnect.reconnect_calls == 0 + + +def test_repeated_stop_remains_stopped() -> None: + supervisor, heartbeat, _ = create_supervisor() + + supervisor.stop() + supervisor.stop() + + assert heartbeat.stop_calls == 2 + assert supervisor.state is RuntimeSupervisorState.STOPPED + + +def test_notify_activity_delegates_while_running() -> None: + supervisor, heartbeat, _ = create_supervisor() + + supervisor.start() + supervisor.notify_activity() + + assert heartbeat.record_activity_calls == 1 + + +def test_notify_activity_does_nothing_while_stopped() -> None: + supervisor, heartbeat, _ = create_supervisor() + + supervisor.notify_activity() + + assert heartbeat.record_activity_calls == 0 + assert supervisor.state is RuntimeSupervisorState.STOPPED + + +def test_timeout_does_nothing_while_stopped() -> None: + supervisor, heartbeat, reconnect = create_supervisor() + + result = asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + assert result is False + assert reconnect.reconnect_calls == 0 + assert heartbeat.start_calls == 0 + assert heartbeat.stop_calls == 0 + assert supervisor.state is RuntimeSupervisorState.STOPPED + + +def test_timeout_stops_heartbeat_before_reconnect() -> None: + call_order: list[str] = [] + + class OrderedHeartbeatMonitor(FakeHeartbeatMonitor): + def stop(self) -> None: + call_order.append("heartbeat.stop") + super().stop() + + class OrderedReconnectCoordinator( + FakeReconnectCoordinator + ): + async def reconnect(self) -> None: + call_order.append("reconnect") + await super().reconnect() + + heartbeat = OrderedHeartbeatMonitor() + reconnect = OrderedReconnectCoordinator() + + supervisor = RuntimeSupervisor( + heartbeat_monitor=heartbeat, + reconnect_coordinator=reconnect, + ) + + supervisor.start() + + asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + assert call_order == [ + "heartbeat.stop", + "reconnect", + ] + + +def test_timeout_runs_single_reconnect_attempt() -> None: + supervisor, _, reconnect = create_supervisor() + + supervisor.start() + + result = asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + assert result is True + assert reconnect.reconnect_calls == 1 + + +def test_successful_reconnect_restarts_heartbeat() -> None: + supervisor, heartbeat, _ = create_supervisor() + + supervisor.start() + + asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + assert heartbeat.stop_calls == 1 + assert heartbeat.start_calls == 2 + assert heartbeat.state is HeartbeatState.MONITORING + + +def test_successful_reconnect_returns_to_running() -> None: + supervisor, _, reconnect = create_supervisor() + + supervisor.start() + + result = asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + assert result is True + assert supervisor.state is RuntimeSupervisorState.RUNNING + assert reconnect.state is ReconnectState.CONNECTED + + +def test_notify_activity_works_after_successful_reconnect() -> None: + supervisor, heartbeat, _ = create_supervisor() + + supervisor.start() + + asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + supervisor.notify_activity() + + assert heartbeat.record_activity_calls == 1 + + +def test_reconnect_error_is_propagated() -> None: + class BrokenReconnectCoordinator( + FakeReconnectCoordinator + ): + async def reconnect(self) -> None: + self.reconnect_calls += 1 + self._attempt += 1 + self._state = ReconnectState.FAILED + + raise RuntimeError("reconnect failed") + + heartbeat = FakeHeartbeatMonitor() + reconnect = BrokenReconnectCoordinator() + + supervisor = RuntimeSupervisor( + heartbeat_monitor=heartbeat, + reconnect_coordinator=reconnect, + ) + + supervisor.start() + + with pytest.raises( + RuntimeError, + match="reconnect failed", + ): + asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + +def test_reconnect_error_sets_failed_state() -> None: + class BrokenReconnectCoordinator( + FakeReconnectCoordinator + ): + async def reconnect(self) -> None: + self.reconnect_calls += 1 + self._state = ReconnectState.FAILED + raise RuntimeError("reconnect failed") + + heartbeat = FakeHeartbeatMonitor() + + supervisor = RuntimeSupervisor( + heartbeat_monitor=heartbeat, + reconnect_coordinator=( + BrokenReconnectCoordinator() + ), + ) + + supervisor.start() + + with pytest.raises(RuntimeError): + asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + assert supervisor.state is RuntimeSupervisorState.FAILED + + +def test_reconnect_error_leaves_heartbeat_stopped() -> None: + class BrokenReconnectCoordinator( + FakeReconnectCoordinator + ): + async def reconnect(self) -> None: + raise RuntimeError("reconnect failed") + + heartbeat = FakeHeartbeatMonitor() + + supervisor = RuntimeSupervisor( + heartbeat_monitor=heartbeat, + reconnect_coordinator=( + BrokenReconnectCoordinator() + ), + ) + + supervisor.start() + + with pytest.raises(RuntimeError): + asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + assert heartbeat.start_calls == 1 + assert heartbeat.stop_calls == 1 + assert heartbeat.state is HeartbeatState.IDLE + + +def test_timeout_does_not_start_second_reconnect_after_failure() -> None: + class BrokenReconnectCoordinator( + FakeReconnectCoordinator + ): + async def reconnect(self) -> None: + self.reconnect_calls += 1 + raise RuntimeError("reconnect failed") + + heartbeat = FakeHeartbeatMonitor() + reconnect = BrokenReconnectCoordinator() + + supervisor = RuntimeSupervisor( + heartbeat_monitor=heartbeat, + reconnect_coordinator=reconnect, + ) + + supervisor.start() + + with pytest.raises(RuntimeError): + asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + second_result = asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + assert second_result is False + assert reconnect.reconnect_calls == 1 + assert supervisor.state is RuntimeSupervisorState.FAILED + + +def test_start_can_restart_supervision_after_failure() -> None: + class FailOnceReconnectCoordinator( + FakeReconnectCoordinator + ): + async def reconnect(self) -> None: + self.reconnect_calls += 1 + self._attempt += 1 + + if self.reconnect_calls == 1: + self._state = ReconnectState.FAILED + raise RuntimeError("temporary failure") + + self._state = ReconnectState.CONNECTED + + heartbeat = FakeHeartbeatMonitor() + reconnect = FailOnceReconnectCoordinator() + + supervisor = RuntimeSupervisor( + heartbeat_monitor=heartbeat, + reconnect_coordinator=reconnect, + ) + + supervisor.start() + + with pytest.raises(RuntimeError): + asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + supervisor.start() + + result = asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + assert result is True + assert reconnect.reconnect_calls == 2 + assert supervisor.state is RuntimeSupervisorState.RUNNING + + +def test_notify_activity_does_nothing_after_failure() -> None: + class BrokenReconnectCoordinator( + FakeReconnectCoordinator + ): + async def reconnect(self) -> None: + raise RuntimeError("reconnect failed") + + heartbeat = FakeHeartbeatMonitor() + + supervisor = RuntimeSupervisor( + heartbeat_monitor=heartbeat, + reconnect_coordinator=( + BrokenReconnectCoordinator() + ), + ) + + supervisor.start() + + with pytest.raises(RuntimeError): + asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + supervisor.notify_activity() + + assert heartbeat.record_activity_calls == 0 + + +def test_stop_can_reset_failed_supervisor() -> None: + class BrokenReconnectCoordinator( + FakeReconnectCoordinator + ): + async def reconnect(self) -> None: + raise RuntimeError("reconnect failed") + + heartbeat = FakeHeartbeatMonitor() + + supervisor = RuntimeSupervisor( + heartbeat_monitor=heartbeat, + reconnect_coordinator=( + BrokenReconnectCoordinator() + ), + ) + + supervisor.start() + + with pytest.raises(RuntimeError): + asyncio.run( + supervisor.handle_heartbeat_timeout() + ) + + supervisor.stop() + + assert supervisor.state is RuntimeSupervisorState.STOPPED + assert heartbeat.state is HeartbeatState.IDLE diff --git a/app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py b/app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py new file mode 100644 index 0000000..240f6ad --- /dev/null +++ b/app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py @@ -0,0 +1,752 @@ +# app/tests/unit/market_data/acquisition/ +# test_trade_stream_runtime_composition.py + +from __future__ import annotations + +import asyncio +from dataclasses import FrozenInstanceError, dataclass +from datetime import datetime, timezone +from decimal import Decimal + +import pytest + +from src.market_data.acquisition.adapters.dzengi.rest import ( + DzengiTradesDocumentSource, +) +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.acquisition.runtime.acquisition_runtime_service_protocol import ( + AcquisitionRuntimeServiceProtocol, +) +from src.market_data.acquisition.runtime.heartbeat import ( + HeartbeatMonitorProtocol, + HeartbeatState, +) +from src.market_data.acquisition.runtime.reconnect import ( + ReconnectCoordinatorProtocol, + ReconnectState, +) +from src.market_data.acquisition.runtime.runtime_events import ( + ReconnectCompletedEvent, + ReconnectStartedEvent, +) +from src.market_data.acquisition.runtime.runtime_recovery_protocol import ( + RuntimeRecoveryProtocol, +) +from src.market_data.acquisition.runtime.scheduler import ( + RuntimeSchedulerProtocol, +) +from src.market_data.acquisition.runtime.supervisor import ( + RuntimeSupervisorProtocol, + RuntimeSupervisorState, +) +from src.market_data.acquisition.runtime.websocket_protocol import ( + AcquisitionRuntimeCommandDispatcherProtocol, + AcquisitionRuntimeEvent, + AcquisitionSubscriptionMessage, +) +from src.market_data.acquisition.trade_stream_acquisition_protocol import ( + TradeStreamAcquisitionServiceProtocol, +) +from src.market_data.acquisition.trade_stream_message_adapter_protocol import ( + TradeStreamMappedMessage, +) +from src.market_data.acquisition.trade_stream_runtime_composition import ( + TradeStreamRuntimeComposition, + build_trade_stream_runtime_composition, +) + + +SYMBOL = "BTC/USD_LEVERAGE" + +CHECKPOINT_TIME = datetime( + 2026, + 7, + 29, + 12, + 0, + 0, + 123000, + tzinfo=timezone.utc, +) + +CHECKPOINT_TIME_MS = 1_785_326_400_123 +RECOVERY_END_TIME_MS = CHECKPOINT_TIME_MS + 5_000 + + +def make_trade( + *, + trade_id: int = 100, + executed_at: datetime = CHECKPOINT_TIME, +) -> Trade: + return Trade( + symbol=SYMBOL, + trade_id=trade_id, + price=Decimal("64555.55"), + quantity=Decimal("0.002"), + executed_at=executed_at, + aggressor_side=TradeAggressorSide.BUY, + source="test", + ) + + +def make_raw_trade( + *, + trade_id: int, + timestamp: int, +) -> dict[str, object]: + return { + "a": trade_id, + "p": "64556.00", + "q": "0.003", + "T": timestamp, + "m": False, + } + + +class FakeSession: + def __init__(self) -> None: + self.start_calls = 0 + self.stop_calls = 0 + + @property + def is_connected(self) -> bool: + return self.start_calls > self.stop_calls + + async def start(self) -> None: + self.start_calls += 1 + + async def stop(self) -> None: + self.stop_calls += 1 + + +class FakeTransport: + def __init__(self) -> None: + self.connect_calls = 0 + self.disconnect_calls = 0 + self.sent_messages: list[str | bytes] = [] + self.receive_calls = 0 + + async def connect(self) -> None: + self.connect_calls += 1 + + async def disconnect(self) -> None: + self.disconnect_calls += 1 + + async def send( + self, + message: str | bytes, + ) -> None: + self.sent_messages.append(message) + + async def receive(self) -> str | bytes: + self.receive_calls += 1 + return "" + + +class FakeSubscriptionManager: + def __init__(self) -> None: + self.subscriptions: list[ + tuple[str, AcquisitionSubscriptionMessage] + ] = [] + self.unsubscriptions: list[ + tuple[str, AcquisitionSubscriptionMessage] + ] = [] + self.restore_calls = 0 + self.clear_calls = 0 + + async def subscribe( + self, + subscription_key: str, + message: AcquisitionSubscriptionMessage, + ) -> None: + self.subscriptions.append( + ( + subscription_key, + message, + ) + ) + + async def unsubscribe( + self, + subscription_key: str, + message: AcquisitionSubscriptionMessage, + ) -> None: + self.unsubscriptions.append( + ( + subscription_key, + message, + ) + ) + + async def restore_subscriptions(self) -> None: + self.restore_calls += 1 + + async def clear_subscriptions(self) -> None: + self.clear_calls += 1 + + +class FakeEventPublisher: + def __init__(self) -> None: + self.events: list[AcquisitionRuntimeEvent] = [] + + async def publish( + self, + event: AcquisitionRuntimeEvent, + ) -> None: + self.events.append(event) + + +class FakeMessageAdapter: + def __init__( + self, + result: TradeStreamMappedMessage, + ) -> None: + self._result = result + self.documents: list[object] = [] + + def map_message( + self, + document: object, + ) -> TradeStreamMappedMessage: + self.documents.append(document) + return self._result + + +class StubTradesDocumentSource( + DzengiTradesDocumentSource, +): + def __init__( + self, + document: object, + ) -> None: + super().__init__() + + self.document = document + self.calls: list[ + tuple[ + str, + int | None, + int | None, + int | None, + ] + ] = [] + + def fetch_trades_document( + self, + symbol: str, + *, + start_time: int | None = None, + end_time: int | None = None, + limit: int | None = None, + ) -> object: + self.calls.append( + ( + symbol, + start_time, + end_time, + limit, + ) + ) + + return self.document + + +class FakeClock: + def __init__( + self, + value: float = 100.0, + ) -> None: + self.value = value + + def __call__(self) -> float: + return self.value + + +class RecordingSleep: + def __init__(self) -> None: + self.calls: list[float] = [] + + async def __call__( + self, + seconds: float, + ) -> None: + self.calls.append(seconds) + + +@dataclass(slots=True) +class CompositionDependencies: + session: FakeSession + transport: FakeTransport + subscription_manager: FakeSubscriptionManager + event_publisher: FakeEventPublisher + message_adapter: FakeMessageAdapter + recovery_document_source: StubTradesDocumentSource + heartbeat_clock: FakeClock + scheduler_sleep: RecordingSleep + + +def create_composition( + *, + trade: Trade | None = None, + recovery_document: object = (), + heartbeat_timeout_seconds: float = 10.0, + scheduler_interval_seconds: float = 1.0, + max_recovery_window_ms: int = 3_599_999, +) -> tuple[ + TradeStreamRuntimeComposition, + CompositionDependencies, +]: + dependencies = CompositionDependencies( + session=FakeSession(), + transport=FakeTransport(), + subscription_manager=FakeSubscriptionManager(), + event_publisher=FakeEventPublisher(), + message_adapter=FakeMessageAdapter( + trade or make_trade(), + ), + recovery_document_source=StubTradesDocumentSource( + recovery_document, + ), + heartbeat_clock=FakeClock(), + scheduler_sleep=RecordingSleep(), + ) + + composition = build_trade_stream_runtime_composition( + session=dependencies.session, + transport=dependencies.transport, + subscription_manager=dependencies.subscription_manager, + event_publisher=dependencies.event_publisher, + message_adapter=dependencies.message_adapter, + recovery_document_source=( + dependencies.recovery_document_source + ), + heartbeat_timeout_seconds=heartbeat_timeout_seconds, + scheduler_interval_seconds=scheduler_interval_seconds, + max_recovery_window_ms=max_recovery_window_ms, + heartbeat_clock=dependencies.heartbeat_clock, + scheduler_sleep=dependencies.scheduler_sleep, + ) + + return ( + composition, + dependencies, + ) + + +def test_composition_uses_slots_and_is_frozen() -> None: + composition, *_ = create_composition() + + assert not hasattr( + composition, + "__dict__", + ) + + with pytest.raises(FrozenInstanceError): + composition.runtime_scheduler = ( # type: ignore[misc] + composition.runtime_scheduler + ) + + +def test_components_implement_public_protocols() -> None: + composition, *_ = create_composition() + + assert isinstance( + composition.acquisition_runtime_service, + AcquisitionRuntimeServiceProtocol, + ) + assert isinstance( + composition.acquisition_runtime_service, + AcquisitionRuntimeCommandDispatcherProtocol, + ) + assert isinstance( + composition.trade_stream_acquisition_service, + TradeStreamAcquisitionServiceProtocol, + ) + assert isinstance( + composition.runtime_recovery_coordinator, + RuntimeRecoveryProtocol, + ) + assert isinstance( + composition.reconnect_coordinator, + ReconnectCoordinatorProtocol, + ) + assert isinstance( + composition.heartbeat_monitor, + HeartbeatMonitorProtocol, + ) + assert isinstance( + composition.runtime_supervisor, + RuntimeSupervisorProtocol, + ) + assert isinstance( + composition.runtime_scheduler, + RuntimeSchedulerProtocol, + ) + + +def test_external_dependencies_are_reused() -> None: + composition, dependencies = create_composition() + + runtime_service = composition.acquisition_runtime_service + + assert runtime_service._session is dependencies.session + assert runtime_service._transport is dependencies.transport + assert ( + runtime_service._subscription_manager + is dependencies.subscription_manager + ) + assert ( + runtime_service._event_publisher + is dependencies.event_publisher + ) + + assert ( + composition.trade_stream_acquisition_service._adapter + is dependencies.message_adapter + ) + assert ( + composition.recovery_controller._document_source + is dependencies.recovery_document_source + ) + + +def test_live_stream_and_recovery_share_consistency_state() -> None: + composition, *_ = create_composition() + + assert ( + composition.consistency_controller._state_store + is composition.state_store + ) + assert ( + composition.trade_stream_acquisition_service + ._consistency_controller + is composition.consistency_controller + ) + assert ( + composition.recovery_controller._consistency_controller + is composition.consistency_controller + ) + assert ( + composition.runtime_recovery_coordinator._state_store + is composition.state_store + ) + + +def test_runtime_components_share_lifecycle_dependencies() -> None: + composition, dependencies = create_composition() + + assert ( + composition.trade_stream_acquisition_service._runtime_service + is composition.acquisition_runtime_service + ) + assert ( + composition.reconnect_coordinator._command_dispatcher + is composition.acquisition_runtime_service + ) + assert ( + composition.reconnect_coordinator._subscription_manager + is dependencies.subscription_manager + ) + assert ( + composition.reconnect_coordinator._event_publisher + is dependencies.event_publisher + ) + assert ( + composition.runtime_supervisor._heartbeat_monitor + is composition.heartbeat_monitor + ) + assert ( + composition.runtime_supervisor._reconnect_coordinator + is composition.reconnect_coordinator + ) + assert ( + composition.runtime_scheduler._heartbeat_monitor + is composition.heartbeat_monitor + ) + assert ( + composition.runtime_scheduler._runtime_supervisor + is composition.runtime_supervisor + ) + + +def test_configuration_is_forwarded() -> None: + composition, *_ = create_composition( + heartbeat_timeout_seconds=15.5, + scheduler_interval_seconds=2.5, + max_recovery_window_ms=2_000, + ) + + assert composition.heartbeat_monitor.timeout_seconds == 15.5 + assert composition.runtime_scheduler.interval_seconds == 2.5 + assert composition.recovery_window_planner.max_window_ms == 2_000 + + +def test_clock_and_sleep_are_forwarded() -> None: + composition, dependencies = create_composition() + + assert ( + composition.heartbeat_monitor._clock + is dependencies.heartbeat_clock + ) + assert ( + composition.runtime_scheduler._sleep + is dependencies.scheduler_sleep + ) + + +def test_creation_has_no_runtime_side_effects() -> None: + composition, dependencies = create_composition() + + assert dependencies.session.start_calls == 0 + assert dependencies.session.stop_calls == 0 + + assert dependencies.transport.connect_calls == 0 + assert dependencies.transport.disconnect_calls == 0 + assert dependencies.transport.sent_messages == [] + assert dependencies.transport.receive_calls == 0 + + assert dependencies.subscription_manager.subscriptions == [] + assert dependencies.subscription_manager.unsubscriptions == [] + assert dependencies.subscription_manager.restore_calls == 0 + assert dependencies.subscription_manager.clear_calls == 0 + + assert dependencies.event_publisher.events == [] + assert dependencies.recovery_document_source.calls == [] + + assert composition.heartbeat_monitor.state is HeartbeatState.IDLE + assert ( + composition.runtime_supervisor.state + is RuntimeSupervisorState.STOPPED + ) + assert ( + composition.reconnect_coordinator.state + is ReconnectState.DISCONNECTED + ) + assert composition.runtime_scheduler.running is False + + +def test_live_checkpoint_is_visible_to_runtime_recovery() -> None: + checkpoint_trade = make_trade() + + composition, dependencies = create_composition( + trade=checkpoint_trade, + recovery_document=[], + ) + + accepted_trade = ( + composition.trade_stream_acquisition_service.handle_message( + { + "destination": "internal.trade", + } + ) + ) + + result = composition.runtime_recovery_coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + assert accepted_trade is checkpoint_trade + assert result.requested_start_time == CHECKPOINT_TIME_MS + assert result.requested_end_time == RECOVERY_END_TIME_MS + assert result.is_empty is True + + assert dependencies.recovery_document_source.calls == [ + ( + SYMBOL, + CHECKPOINT_TIME_MS, + RECOVERY_END_TIME_MS, + None, + ) + ] + + +def test_recovery_advances_shared_live_checkpoint() -> None: + recovered_trade_id = 101 + + composition, *_ = create_composition( + recovery_document=[ + make_raw_trade( + trade_id=recovered_trade_id, + timestamp=CHECKPOINT_TIME_MS + 1_000, + ), + ], + ) + + composition.trade_stream_acquisition_service.handle_message( + { + "destination": "internal.trade", + } + ) + + result = composition.runtime_recovery_coordinator.recover( + symbol=SYMBOL, + recovery_end_time=RECOVERY_END_TIME_MS, + ) + + state = composition.state_store.get( + SYMBOL, + ) + + assert result.recovered_count == 1 + assert result.last_trade is state.last_trade + assert state.last_trade_id == recovered_trade_id + assert state.last_trade is not None + assert state.last_trade.trade_id == recovered_trade_id + + +def test_subscription_uses_composed_runtime_service() -> None: + composition, dependencies = create_composition() + + asyncio.run( + composition.trade_stream_acquisition_service.subscribe( + ( + SYMBOL, + ), + correlation_id="composition-test", + ) + ) + + assert len( + dependencies.subscription_manager.subscriptions + ) == 1 + + subscription_key, _ = ( + dependencies.subscription_manager.subscriptions[0] + ) + + assert SYMBOL in subscription_key + assert dependencies.session.start_calls == 0 + + +def test_reconnect_uses_composed_runtime_dependencies() -> None: + composition, dependencies = create_composition() + + asyncio.run( + composition.reconnect_coordinator.reconnect() + ) + + assert dependencies.session.start_calls == 1 + assert dependencies.subscription_manager.restore_calls == 1 + assert len(dependencies.event_publisher.events) == 2 + assert isinstance( + dependencies.event_publisher.events[0], + ReconnectStartedEvent, + ) + assert isinstance( + dependencies.event_publisher.events[1], + ReconnectCompletedEvent, + ) + + +def test_separate_compositions_have_independent_state() -> None: + first, _ = create_composition() + second, _ = create_composition() + + assert first is not second + assert first.state_store is not second.state_store + assert ( + first.consistency_controller + is not second.consistency_controller + ) + assert ( + first.runtime_recovery_coordinator + is not second.runtime_recovery_coordinator + ) + assert ( + first.runtime_supervisor + is not second.runtime_supervisor + ) + assert ( + first.runtime_scheduler + is not second.runtime_scheduler + ) + + +@pytest.mark.parametrize( + "heartbeat_timeout_seconds", + [ + 0, + -1, + True, + "10", + ], +) +def test_invalid_heartbeat_configuration_is_not_hidden( + heartbeat_timeout_seconds: object, +) -> None: + dependencies = CompositionDependencies( + session=FakeSession(), + transport=FakeTransport(), + subscription_manager=FakeSubscriptionManager(), + event_publisher=FakeEventPublisher(), + message_adapter=FakeMessageAdapter( + make_trade(), + ), + recovery_document_source=StubTradesDocumentSource( + [], + ), + heartbeat_clock=FakeClock(), + scheduler_sleep=RecordingSleep(), + ) + + with pytest.raises( + (TypeError, ValueError), + ): + build_trade_stream_runtime_composition( + session=dependencies.session, + transport=dependencies.transport, + subscription_manager=dependencies.subscription_manager, + event_publisher=dependencies.event_publisher, + message_adapter=dependencies.message_adapter, + recovery_document_source=( + dependencies.recovery_document_source + ), + heartbeat_timeout_seconds=heartbeat_timeout_seconds, # type: ignore[arg-type] + scheduler_interval_seconds=1.0, + heartbeat_clock=dependencies.heartbeat_clock, + scheduler_sleep=dependencies.scheduler_sleep, + ) + + +@pytest.mark.parametrize( + "scheduler_interval_seconds", + [ + 0, + -1, + True, + "1", + ], +) +def test_invalid_scheduler_configuration_is_not_hidden( + scheduler_interval_seconds: object, +) -> None: + with pytest.raises( + (TypeError, ValueError), + ): + create_composition( + scheduler_interval_seconds=scheduler_interval_seconds, # type: ignore[arg-type] + ) + + +@pytest.mark.parametrize( + "max_recovery_window_ms", + [ + 0, + -1, + True, + 3_600_000, + ], +) +def test_invalid_recovery_window_configuration_is_not_hidden( + max_recovery_window_ms: object, +) -> None: + with pytest.raises( + (TypeError, ValueError), + ): + create_composition( + max_recovery_window_ms=max_recovery_window_ms, # type: ignore[arg-type] + ) diff --git a/docs/migrations/build_060_24.md b/docs/migrations/build_060_24.md new file mode 100644 index 0000000..8ae71f5 --- /dev/null +++ b/docs/migrations/build_060_24.md @@ -0,0 +1,552 @@ +# Build 060.24 — Runtime Recovery Architecture + +**Engineering Migration Report** + +--- + +## Контроль документа + +| Свойство | Значение | +|---|---| +| Build | 060.24 | +| Название | Runtime Recovery Architecture | +| Статус | Completed | +| Проект | Dzentra | +| Подсистема | Market Data Acquisition | +| Компонент | Trade Stream Runtime | +| Версия документа | 2.0 | + +## Связанные документы + +- `build_060_24_architecture.md` — итоговая архитектурная спецификация Build. +- `build_060_23.md` — Trade Stream Acquisition Integration. +- `build_060_23_architecture.md` — архитектура интеграции Trade Stream Acquisition. +- `build_060_22.md` — Runtime Service Integration. +- `build_060_20_1.md` — Trade Stream State Ownership Alignment. +- `build_060_19.md` — Trade Recovery. +- `build_060_18.md` — Trade Stream Consistency. + +--- + +## 1. Исходное состояние + +После Build 060.23 проект имел рабочий конвейер обработки одной live-сделки: + +```text +WebSocket message + │ + ▼ +TradeStreamMessageAdapterProtocol + │ + ▼ +Canonical Trade + │ + ▼ +TradeStreamConsistencyController + │ + ▼ +Trade | None +``` + +Также уже существовал REST Recovery Layer: + +```text +TradeRecoveryRequest + │ + ▼ +TradeRecoveryController + │ + ▼ +TradeRecoveryNormalizer + │ + ▼ +TradeStreamConsistencyController +``` + +Эти два конвейера использовали Consistency Layer, но Runtime ещё не +обладал архитектурой обнаружения потери активности, повторного подключения +и координации восстановления пропущенного диапазона. + +## 2. Цель Build + +Цель Build 060.24 — реализовать внутреннюю Runtime Recovery Architecture: + +- определить единственный checkpoint Trade Stream; +- представить и рассчитать допустимые окна восстановления; +- реализовать изолированные runtime-компоненты reconnect, heartbeat, + supervisor и scheduler; +- связать checkpoint, planner и существующий Recovery Controller; +- построить единый граф зависимостей Live Stream и Recovery; +- не запускать production lifecycle и не создавать сетевые подключения + во время композиции. + +Build создаёт завершённую архитектурную основу восстановления в памяти. +Её подключение к реальному WebSocket lifecycle, bootstrap приложения +и production-конфигурации относится к Build 060.25. + +--- + +## 3. Выполненный roadmap + +| Подэтап | Название | Фактическая реализация | +|---|---|---| +| 060.24.1 | RuntimeCheckpoint | `TradeStreamState.last_trade` | +| 060.24.2 | RuntimeSessionState | `RuntimeSupervisorState` и локальные состояния runtime-компонентов | +| 060.24.3 | RecoveryPlanner | `TradeRecoveryWindowPlanner` | +| 060.24.4 | RecoveryWindowCalculator | алгоритм `TradeRecoveryWindowPlanner.build_windows()` | +| 060.24.5 | ReconnectCoordinator | `ReconnectCoordinator` | +| 060.24.6 | Heartbeat | `HeartbeatMonitor` | +| 060.24.7 | Supervisor | `RuntimeSupervisor` | +| 060.24.8 | Scheduler | `RuntimeScheduler` | +| 060.24.9.1 | RuntimeRecoveryCoordinator | `RuntimeRecoveryProtocol` и `RuntimeRecoveryCoordinator` | +| 060.24.9.2 | Runtime Composition | `TradeStreamRuntimeComposition` и factory-функция сборки | + +`RuntimeSessionState` не введён как отдельная дублирующая модель. +Состояние runtime-сессии выражено через `RuntimeSupervisorState`, а +локальные детали принадлежат `ReconnectState` и `HeartbeatState`. + +`RecoveryWindowCalculator` не введён как отдельный класс. Расчёт является +единственной ответственностью `TradeRecoveryWindowPlanner`, поэтому его +выделение создало бы искусственный слой без самостоятельного состояния +или политики. + +--- + +## 4. Реализованные компоненты + +### 4.1. RuntimeCheckpoint + +`TradeStreamState` хранит: + +```text +last_trade_id +last_trade +recent trade window +``` + +Checkpoint — это `last_trade`, то есть последняя каноническая сделка, +принятая Consistency Layer. + +Он обновляется только после успешного `accept()`. Полный дубликат, +конфликтующий дубликат, нарушение порядка или неожиданный symbol +не переводят checkpoint вперёд. + +Основной инвариант: + +```text +last_trade is None +⇔ +last_trade_id is None +``` + +Если checkpoint существует: + +```text +last_trade.trade_id == last_trade_id +last_trade.symbol == TradeStreamState.symbol +``` + +Checkpoint находится только в оперативной памяти. Его долговременное +хранение относится к Build 060.28. + +### 4.2. TradeRecoveryWindow + +`TradeRecoveryWindow` — immutable value object с `slots`: + +```text +symbol +start_time +end_time +``` + +Модель проверяет типы и запрещает: + +- пустой symbol; +- отрицательные временные границы; +- `start_time > end_time`; +- использование `bool` вместо целочисленного timestamp. + +Window не выполняет Recovery и не содержит transport-, retry- или +exchange-состояние. + +### 4.3. TradeRecoveryWindowPlanner + +Planner принимает полный диапазон и возвращает: + +```text +tuple[TradeRecoveryWindow, ...] +``` + +Каждое окно удовлетворяет ограничению существующего +`TradeRecoveryRequest`: + +```text +end_time - start_time < 3_600_000 ms +``` + +Значение по умолчанию: + +```text +3_599_999 ms +``` + +Соседние окна имеют общую границу: + +```text +previous.end_time == next.start_time +``` + +Возможный повтор сделки на границе устраняется общим Consistency Layer. +При равных `start_time` и `end_time` Planner возвращает пустой tuple. + +Planner: + +- не выполняет REST-запросы; +- не создаёт `TradeRecoveryRequest`; +- не читает системное время; +- не зависит от WebSocket и Runtime lifecycle. + +### 4.4. ReconnectCoordinator + +`ReconnectCoordinator` выполняет одну попытку reconnect: + +```text +ReconnectStartedEvent + │ + ▼ +ConnectCommand + │ + ▼ +restore_subscriptions() + │ + ▼ +ReconnectCompletedEvent +``` + +При ошибке публикуется `ReconnectFailedEvent`, состояние переходит +в `FAILED`, а исходное исключение распространяется вызывающему коду. + +Coordinator хранит номер попытки и состояние: + +```text +DISCONNECTED +CONNECTING +RESTORING_SUBSCRIPTIONS +CONNECTED +FAILED +``` + +Retry loop и backoff в этот компонент не входят. + +### 4.5. HeartbeatMonitor + +Heartbeat является пассивным монитором: + +- использует monotonic clock; +- хранит время последней активности; +- определяет timeout детерминированным вызовом `check_timeout()`; +- публикует один `HeartbeatTimeoutEvent` на один период отсутствия + активности; +- возвращается в monitoring после новой активности. + +Heartbeat не создаёт фоновых задач, не выполняет reconnect и не управляет +WebSocket. + +### 4.6. RuntimeSupervisor + +Supervisor владеет состоянием runtime-сессии: + +```text +STOPPED +RUNNING +RECONNECTING +FAILED +``` + +Он: + +- запускает и останавливает Heartbeat; +- принимает уведомления об активности; +- запускает одну reconnect-попытку после подтверждённого timeout; +- предотвращает параллельный reconnect; +- после успешного reconnect начинает новый heartbeat-период; +- сохраняет `FAILED` и распространяет ошибку reconnect. + +Supervisor не выполняет периодический цикл, recovery, retry или backoff. + +### 4.7. RuntimeScheduler + +Scheduler отвечает только за периодичность: + +```text +HeartbeatMonitor.check_timeout() + │ + ├── False → следующий интервал + │ + └── True → RuntimeSupervisor.handle_heartbeat_timeout() +``` + +Цикл запускается явным `await RuntimeScheduler.start()`. Scheduler +не создаёт `asyncio.Task` самостоятельно. `stop()` запрашивает безопасное +завершение на ближайшей управляемой границе. + +### 4.8. RuntimeRecoveryCoordinator + +Coordinator соединяет существующие слои: + +```text +TradeStreamStateStore + │ + ▼ +TradeStreamState.last_trade + │ + ▼ +executed_at → Unix milliseconds + │ + ▼ +TradeRecoveryWindowPlanner + │ + ▼ +TradeRecoveryRequest[] + │ + ▼ +TradeRecoveryController + │ + ▼ +aggregated TradeRecoveryResult +``` + +Свойства реализации: + +- использует существующее состояние и не создаёт новый + `TradeStreamState`; +- принимает `recovery_end_time` от вызывающего runtime-компонента; +- преобразует только timezone-aware `datetime`; +- не использует float при переводе времени в Unix milliseconds; +- выполняет окна последовательно; +- объединяет восстановленные сделки в один результат; +- не скрывает ошибки Store, Planner или Recovery Controller. + +При отсутствии состояния или checkpoint возвращается пустой +`TradeRecoveryResult`; обе его границы равны `recovery_end_time`. + +### 4.9. Runtime Composition + +Factory: + +```text +build_trade_stream_runtime_composition(...) +``` + +создаёт один граф зависимостей и возвращает immutable +`TradeStreamRuntimeComposition`. + +Ключевой identity-инвариант: + +```text +Live Stream ───┐ + ├── one TradeStreamConsistencyController +Recovery ──────┘ + │ + ▼ + one TradeStreamStateStore +``` + +Благодаря этому live-сделки и восстановленные сделки проходят одинаковые +правила согласованности и используют один checkpoint. + +Внешние WebSocket-зависимости передаются через Protocol-контракты. +Создание Composition: + +- не читает Settings; +- не создаёт production transport; +- не подключается к сети; +- не запускает Scheduler или Supervisor; +- не создаёт фоновые задачи; +- не выполняет Recovery. + +--- + +## 5. Архитектурные границы + +После Build действуют следующие границы. + +| Компонент | Владеет | Не владеет | +|---|---|---| +| `TradeStreamState` | checkpoint и deduplication window | recovery orchestration | +| `TradeRecoveryWindowPlanner` | расчёт окон | REST и runtime lifecycle | +| `ReconnectCoordinator` | одна reconnect-попытка | retry, backoff и recovery | +| `HeartbeatMonitor` | контроль активности | scheduling и reconnect | +| `RuntimeSupervisor` | состояние сессии и реакция на timeout | периодический цикл и recovery | +| `RuntimeScheduler` | периодический вызов проверок | reconnect-логика и heartbeat-вычисления | +| `RuntimeRecoveryCoordinator` | recovery orchestration одного symbol | WebSocket lifecycle | +| Composition root | создание и связывание объектов | запуск production lifecycle | + +Ни один из компонентов Build не импортирует bootstrap приложения, +торговую логику или Telegram-инфраструктуру. + +--- + +## 6. Состав изменений + +### Production + +```text +app/src/market_data/acquisition/ +├── consistency/ +│ └── trade_stream_state.py +├── recovery/ +│ ├── trade_recovery_window.py +│ └── trade_recovery_window_planner.py +├── runtime/ +│ ├── heartbeat.py +│ ├── reconnect.py +│ ├── runtime_recovery_coordinator.py +│ ├── runtime_recovery_protocol.py +│ ├── scheduler.py +│ └── supervisor.py +└── trade_stream_runtime_composition.py +``` + +### Tests + +```text +app/tests/unit/market_data/acquisition/ +├── consistency/ +│ └── test_trade_stream_state.py +├── recovery/ +│ ├── test_trade_recovery_window.py +│ └── test_trade_recovery_window_planner.py +├── runtime/ +│ ├── test_heartbeat_monitor.py +│ ├── test_reconnect_coordinator.py +│ ├── test_runtime_recovery_coordinator.py +│ ├── test_runtime_scheduler.py +│ └── test_runtime_supervisor.py +└── test_trade_stream_runtime_composition.py +``` + +### Documentation + +```text +docs/migrations/ +├── build_060_24.md +└── build_060_24_architecture.md +``` + +Изменение `.gitignore`, присутствующее в рабочем дереве, не является +частью Build 060.24. + +--- + +## 7. Тестирование + +Непосредственно связанные с Build девять тестовых модулей содержат +230 тестов. + +| Область | Тестов | +|---|---:| +| Trade Stream State | 24 | +| Recovery Window | 20 | +| Recovery Window Planner | 36 | +| Reconnect Coordinator | 15 | +| Heartbeat Monitor | 28 | +| Runtime Supervisor | 25 | +| Runtime Scheduler | 30 | +| Runtime Recovery Coordinator | 27 | +| Runtime Composition | 25 | +| **Итого** | **230** | + +Контрольный результат: + +```text +230 passed +``` + +Полная регрессия проекта: + +```text +1620 passed +0 failed +``` + +Тесты являются unit- и composition-тестами. Реальные сценарии +WebSocket disconnect, повторной подписки, reconnect → recovery +и стресс-тестирование относятся к Build 060.25–060.26. + +--- + +## 8. Обратная совместимость + +Build сохраняет существующие публичные контракты: + +- каноническая модель `Trade` не изменена; +- Recovery Controller и Recovery Request не изменены; +- live acquisition продолжает использовать Consistency Protocol; +- transport protocols остаются внешними зависимостями Runtime; +- `main.py` и bootstrap не изменены. + +Расширение `TradeStreamState` добавляет checkpoint, не изменяя контракт +`accept()`. + +--- + +## 9. Что не входит в Build + +Build 060.24 не включает: + +- создание production WebSocket transport/session; +- подключение Composition к `main.py` или bootstrap; +- запуск scheduler-задачи; +- передачу live-активности в Supervisor; +- выбор symbol и `recovery_end_time` для production recovery; +- автоматическую цепочку reconnect → recovery; +- переключение между Recovery и Live Stream; +- retry/backoff policy; +- постоянное хранение checkpoint; +- интеграционные reconnect/recovery и stress tests. + +Эти ограничения означают, что формулировка +«Trade Stream самостоятельно переживает обрыв WebSocket» на данном этапе +является архитектурной целью, а не активированным production-поведением. + +Production Runtime Integration выполняется в Build 060.25. + +--- + +## 10. Definition of Done + +- [x] Checkpoint принадлежит `TradeStreamState`. +- [x] Checkpoint обновляется только принятой сделкой. +- [x] Реализована immutable-модель Recovery Window. +- [x] Реализован расчёт одного или нескольких допустимых окон. +- [x] Реализована одна reconnect-попытка и восстановление подписок. +- [x] Реализован пассивный Heartbeat. +- [x] Реализовано состояние и поведение Runtime Supervisor. +- [x] Реализован периодический Runtime Scheduler. +- [x] Реализован Runtime Recovery Coordinator. +- [x] Реализована Runtime Composition с общим Consistency Layer. +- [x] Создание Composition не имеет lifecycle- и network-side effects. +- [x] Целевые тесты проходят. +- [x] Полная регрессия проходит. +- [ ] Production lifecycle подключён к приложению — Build 060.25. +- [ ] End-to-end reconnect/recovery проверен — Build 060.25–060.26. + +--- + +## 11. Итог + +Build 060.24 завершает внутреннюю Runtime Recovery Architecture. + +Проект располагает: + +- надёжным checkpoint в Consistency Layer; +- детерминированным планированием Recovery Window; +- изолированными компонентами reconnect, heartbeat, supervisor + и scheduler; +- координатором последовательного восстановления; +- явным composition root с единым состоянием для live и recovery. + +Следующий этап не должен заново проектировать эти компоненты. +Задача Build 060.25 — подключить готовый граф к production WebSocket +lifecycle и определить end-to-end порядок запуска, reconnect и recovery. diff --git a/docs/migrations/build_060_24_architecture.md b/docs/migrations/build_060_24_architecture.md new file mode 100644 index 0000000..fdb82f2 --- /dev/null +++ b/docs/migrations/build_060_24_architecture.md @@ -0,0 +1,1010 @@ +# Build 060.24 — Runtime Recovery Architecture Specification + +**Статус:** Accepted + +**Build:** 060.24 + +**Подсистема:** Market Data Acquisition / Trade Stream Runtime + +--- + +## 1. Назначение + +Документ фиксирует фактическую архитектуру Build 060.24 после завершения +подэтапов 060.24.1–060.24.9.2. + +Build создаёт компоненты, необходимые для восстановления Trade Stream +после потери активности, и связывает их в единый граф зависимостей. + +Документ отделяет: + +- завершённую внутреннюю архитектуру Build 060.24; +- production lifecycle и end-to-end интеграцию Build 060.25; +- интеграционные и стресс-сценарии Build 060.26; +- постоянное хранение Build 060.27–060.28. + +--- + +## 2. Архитектурный контекст + +### 2.1. Live pipeline + +```text +WebSocket Transport + │ + ▼ +AcquisitionRuntimeService + │ + ▼ +TradeStreamMessageAdapterProtocol + │ + ▼ +TradeStreamAcquisitionService + │ + ▼ +TradeStreamConsistencyController + │ + ▼ +Canonical Trade | None +``` + +### 2.2. Recovery pipeline + +```text +TradeRecoveryRequest + │ + ▼ +TradeRecoveryController + │ + ▼ +DzengiTradesDocumentSource + │ + ▼ +TradeRecoveryNormalizer + │ + ▼ +TradeStreamConsistencyController + │ + ▼ +Canonical Trade[] +``` + +### 2.3. Проблема до Build + +До Build компоненты Live и Recovery существовали отдельно. +Отсутствовали: + +- надёжная временная точка начала восстановления; +- разбиение диапазона на допустимые REST-окна; +- контроль потери runtime-активности; +- одна формализованная reconnect-попытка; +- состояние runtime-сессии; +- периодический вызов heartbeat; +- координатор checkpoint → recovery; +- composition root с общими stateful-зависимостями. + +--- + +## 3. Итоговая модель + +```text + PRODUCTION INPUTS + + WebSocket Session / Transport / Subscriptions / Event Publisher + │ + ▼ + TradeStreamRuntimeComposition + │ + ┌────────────────────┼────────────────────┐ + │ │ │ + ▼ ▼ ▼ + LIVE ACQUISITION RUNTIME LIFECYCLE RECOVERY + │ │ │ + │ HeartbeatMonitor │ + │ │ │ + │ RuntimeScheduler │ + │ │ │ + │ RuntimeSupervisor │ + │ │ │ + │ ReconnectCoordinator │ + │ │ + ▼ ▼ + TradeStreamConsistencyController ◄──── TradeRecoveryController + │ ▲ + ▼ │ + TradeStreamStateStore ──► RuntimeRecoveryCoordinator + │ │ + ▼ ▼ + TradeStreamState TradeRecoveryWindowPlanner + │ │ + └──── checkpoint ─────────┘ +``` + +Composition создаёт граф, но не запускает его. Владельцем production +задач и lifecycle будет внешний bootstrap Build 060.25. + +--- + +## 4. Слои и направления зависимостей + +Допустимое направление: + +```text +Composition + │ + ├── Runtime + ├── Recovery + ├── Consistency + └── Acquisition + +Runtime Recovery Coordinator + │ + ├── State Store Protocol + ├── Window Planner + └── Recovery Protocol + +Recovery Controller + │ + └── Consistency Protocol + +Acquisition Service + │ + └── Consistency Protocol +``` + +Запрещённые обратные зависимости: + +```text +Consistency ─X─► Runtime +Recovery ─X─► WebSocket lifecycle +Heartbeat ─X─► Reconnect implementation +Scheduler ─X─► Transport +Supervisor ─X─► Recovery Controller +``` + +Composition размещена в корне `market_data.acquisition`, а не внутри +`runtime`, потому что она знает сразу о нескольких соседних слоях. + +--- + +## 5. Ownership состояния + +### 5.1. Trade Stream state + +Владелец: + +```text +TradeStreamStateStore + │ + ▼ +TradeStreamState(symbol) +``` + +Содержит: + +- последнюю принятую сделку; +- идентификатор последней принятой сделки; +- ограниченное окно deduplication. + +### 5.2. Runtime session state + +Отдельный `RuntimeSessionState` не создаётся. + +Общее состояние lifecycle принадлежит Supervisor: + +```text +RuntimeSupervisorState +├── STOPPED +├── RUNNING +├── RECONNECTING +└── FAILED +``` + +Локальное состояние операции reconnect принадлежит +`ReconnectCoordinator`: + +```text +ReconnectState +├── DISCONNECTED +├── CONNECTING +├── RESTORING_SUBSCRIPTIONS +├── CONNECTED +└── FAILED +``` + +Состояние контроля активности принадлежит Heartbeat: + +```text +HeartbeatState +├── IDLE +├── MONITORING +└── TIMED_OUT +``` + +Это разделение предотвращает появление общего mutable-объекта, +дублирующего состояние специализированных компонентов. + +### 5.3. Scheduler state + +Scheduler хранит только: + +```text +running: bool +``` + +Флаг описывает состояние его цикла и не является состоянием +WebSocket-сессии. + +--- + +## 6. RuntimeCheckpoint + +### 6.1. Владелец + +Checkpoint принадлежит только: + +```text +TradeStreamState.last_trade +``` + +Runtime, Recovery Coordinator и Supervisor могут читать checkpoint, +но не владеют им и не изменяют его. + +### 6.2. Причина хранения полного Trade + +Полный канонический `Trade` уже содержит согласованные: + +```text +symbol +trade_id +executed_at +price +quantity +side +source +``` + +Отдельные поля checkpoint создавали бы риск рассинхронизации. + +### 6.3. Инварианты + +```text +last_trade is None +⇔ +last_trade_id is None +``` + +```text +last_trade.trade_id == last_trade_id +``` + +```text +last_trade.symbol == state.symbol +``` + +Checkpoint изменяется только после принятия сделки. Ошибка или duplicate +не должны частично изменить состояние. + +### 6.4. Ограничение + +Checkpoint является in-memory. Перезапуск процесса его уничтожает. +Persistent Checkpoint относится к Build 060.28. + +--- + +## 7. Recovery Window и расчёт диапазона + +### 7.1. TradeRecoveryWindow + +Value object: + +```text +TradeRecoveryWindow( + symbol: str, + start_time: int, + end_time: int, +) +``` + +Свойства: + +- immutable; +- slots; +- валиден сразу после создания; +- независим от REST, Runtime и конкретной биржи. + +### 7.2. TradeRecoveryWindowPlanner + +Planner объединяет роли RecoveryPlanner и RecoveryWindowCalculator: + +```text +build_windows( + symbol, + start_time, + end_time, +) -> tuple[TradeRecoveryWindow, ...] +``` + +Отдельный Calculator не вводится, потому что расчёт окон: + +- не имеет собственного состояния; +- не имеет политики за пределами Planner; +- не используется независимо от построения recovery plan. + +### 7.3. Временные границы + +`start_time` и `end_time` представлены Unix milliseconds. + +Допустимый диапазон: + +```text +0 <= start_time <= end_time +``` + +Равные границы означают отсутствие необходимого восстановления: + +```text +start_time == end_time + │ + ▼ +() +``` + +### 7.4. Размер окна + +Ограничение Recovery Request: + +```text +end_time - start_time < 3_600_000 +``` + +Максимальное значение Planner: + +```text +3_599_999 ms +``` + +### 7.5. Непрерывность + +Planner строит последовательность: + +```text +[start, boundary_1] +[boundary_1, boundary_2] +[boundary_2, end] +``` + +Соседние окна разделяют одну границу. Это исключает временной разрыв. +Повтор сделки на общей границе безопасен, потому что Recovery использует +тот же Consistency Layer. + +--- + +## 8. Reconnect + +### 8.1. Ответственность + +`ReconnectCoordinator.reconnect()` выполняет ровно одну попытку: + +```text +attempt += 1 + │ + ▼ +publish ReconnectStartedEvent + │ + ▼ +dispatch ConnectCommand + │ + ▼ +restore_subscriptions() + │ + ▼ +publish ReconnectCompletedEvent +``` + +### 8.2. Ошибка + +```text +connect or restore error + │ + ▼ +state = FAILED + │ + ▼ +publish ReconnectFailedEvent + │ + ▼ +raise original exception +``` + +### 8.3. Граница + +Reconnect не решает: + +- когда запускаться; +- сколько раз повторяться; +- какой backoff использовать; +- требуется ли Trade Recovery; +- когда возобновлять live processing. + +--- + +## 9. Heartbeat + +Heartbeat — пассивный детектор отсутствия активности. + +```text +start() + │ + ▼ +MONITORING + │ + ├── record_activity() → reset monotonic boundary + │ + └── check_timeout() + │ + ├── elapsed < timeout → False + └── elapsed >= timeout + │ + ▼ + TIMED_OUT + │ + ▼ + HeartbeatTimeoutEvent +``` + +Событие публикуется один раз до новой активности или нового `start()`. + +Используется `time.monotonic`, а не wall clock. Инъекция clock делает +проверки детерминированными. + +Heartbeat: + +- не содержит собственного периодического цикла; +- не знает о Supervisor; +- не выполняет reconnect; +- не создаёт задачи. + +--- + +## 10. Runtime Supervisor + +Supervisor координирует Heartbeat и Reconnect. + +### 10.1. Запуск + +```text +STOPPED + │ start() + ▼ +heartbeat.start() + │ + ▼ +RUNNING +``` + +### 10.2. Активность + +`notify_activity()` передаётся Heartbeat только в состоянии `RUNNING`. +Активность после остановки или во время reconnect не запускает monitor +неявно. + +### 10.3. Timeout + +```text +RUNNING + │ confirmed timeout + ▼ +heartbeat.stop() + │ + ▼ +RECONNECTING + │ + ▼ +ReconnectCoordinator.reconnect() + │ + ├── success → heartbeat.start() → RUNNING + └── failure → FAILED → raise +``` + +Проверка исходного состояния предотвращает параллельный reconnect. + +### 10.4. Граница + +Supervisor не: + +- вызывает `HeartbeatMonitor.check_timeout()`; +- выполняет периодический sleep; +- планирует retry; +- выполняет recovery; +- управляет Transport напрямую. + +--- + +## 11. Runtime Scheduler + +Scheduler владеет только временем вызовов: + +```text +while running: + timed_out = await heartbeat.check_timeout() + + if timed_out: + await supervisor.handle_heartbeat_timeout() + + await sleep(interval_seconds) +``` + +Реальная реализация дополнительно: + +- не запускает второй цикл при повторном `start()`; +- позволяет выполнить один шаг через `run_once()`; +- прекращает цикл через идемпотентный `stop()`; +- сбрасывает `running` в `finally`; +- распространяет ошибки зависимостей без обёртки; +- позволяет инъецировать sleep-функцию. + +Scheduler не создаёт свою `asyncio.Task`. Внешний lifecycle определяет, +где и как await/cancel выполняются. + +--- + +## 12. Runtime Recovery Coordinator + +### 12.1. Контракт + +```text +recover( + symbol: str, + recovery_end_time: int, +) -> TradeRecoveryResult +``` + +`RuntimeRecoveryProtocol` является structural protocol. Реализация +не наследует Protocol напрямую, что сохраняет рабочий `__slots__` +и позволяет проверять совместимость через `runtime_checkable`. + +### 12.2. Последовательность + +```text +state_store.get(symbol) + │ + ├── state absent ───────────┐ + │ │ + ▼ │ +state.last_trade │ + │ │ + ├── checkpoint absent ──────┤ + │ │ + ▼ ▼ +executed_at → Unix ms empty TradeRecoveryResult + │ + ▼ +window_planner.build_windows() + │ + ▼ +for each window: + TradeRecoveryRequest + │ + ▼ + recovery_controller.recover() + │ + ▼ +aggregate recovered_trades + │ + ▼ +TradeRecoveryResult +``` + +### 12.3. Time policy + +Начальная граница берётся только из checkpoint. + +Конечная граница передаётся извне: + +```text +recovery_end_time +``` + +Coordinator не читает часы самостоятельно. Это оставляет production +policy владельцу lifecycle и обеспечивает детерминированные тесты. + +`executed_at` обязан быть timezone-aware. Преобразование в Unix +milliseconds выполняется через UTC epoch и целочисленную арифметику. + +### 12.4. Empty recovery + +Если state или checkpoint отсутствуют: + +```text +requested_start_time = recovery_end_time +requested_end_time = recovery_end_time +recovered_trades = () +``` + +Coordinator не создаёт state автоматически, потому что владельцем +жизненного цикла state остаётся Consistency Layer. + +### 12.5. Error policy + +Ошибки Store, Planner и Recovery Controller распространяются без +преобразования. Runtime lifecycle должен решить, как публиковать ошибку +и какую retry policy применять. + +--- + +## 13. Runtime Composition + +### 13.1. Входные зависимости + +Factory принимает: + +- WebSocket session; +- transport; +- subscription manager; +- runtime event publisher; +- message adapter; +- recovery document source; +- heartbeat timeout; +- scheduler interval; +- необязательные window size, clock и sleep. + +### 13.2. Создаваемые объекты + +```text +TradeStreamStateStore +TradeStreamConsistencyController +TradeRecoveryController +TradeRecoveryWindowPlanner +RuntimeRecoveryCoordinator +AcquisitionRuntimeService +TradeStreamAcquisitionService +ReconnectCoordinator +HeartbeatMonitor +RuntimeSupervisor +RuntimeScheduler +``` + +### 13.3. Identity graph + +```text +TradeStreamAcquisitionService + │ + └──────────────┐ + ▼ + TradeStreamConsistencyController + ▲ + │ +TradeRecoveryController + │ + ▼ +RuntimeRecoveryCoordinator + +TradeStreamConsistencyController + │ + ▼ +TradeStreamStateStore + ▲ + │ +RuntimeRecoveryCoordinator +``` + +Один store и один consistency controller являются обязательным +инвариантом композиции. + +### 13.4. Side-effect policy + +Factory только создаёт объекты. + +Она не: + +- выполняет connect; +- создаёт asyncio-задачу; +- запускает Scheduler; +- запускает Supervisor; +- читает Settings; +- выполняет Recovery; +- публикует Runtime Events. + +Это позволяет безопасно строить граф в тестах и в будущем bootstrap. + +### 13.5. Не Service Locator + +`TradeStreamRuntimeComposition` — typed assembly result, а не глобальный +registry. Он immutable и явно передаётся владельцу lifecycle. + +--- + +## 14. Протоколы + +На границах используются Protocol-контракты: + +```text +WebSocketSessionProtocol +WebSocketTransportProtocol +WebSocketSubscriptionManagerProtocol +AcquisitionRuntimeCommandDispatcherProtocol +AcquisitionRuntimeEventPublisherProtocol +HeartbeatMonitorProtocol +ReconnectCoordinatorProtocol +RuntimeSupervisorProtocol +RuntimeSchedulerProtocol +RuntimeRecoveryProtocol +TradeStreamStateStoreProtocol +TradeRecoveryProtocol +``` + +Протоколы позволяют: + +- независимо тестировать компоненты; +- не привязывать runtime-логику к конкретному transport; +- инъецировать clock и sleep; +- сохранить однонаправленные зависимости. + +--- + +## 15. Ошибки и восстановление состояния + +| Компонент | Поведение при ошибке | +|---|---| +| Checkpoint | не изменяется при отклонённой сделке | +| Planner | отклоняет некорректный диапазон до построения окон | +| Reconnect | публикует failure event, устанавливает `FAILED`, распространяет ошибку | +| Heartbeat | распространяет ошибку publisher | +| Supervisor | устанавливает `FAILED`, распространяет reconnect error | +| Scheduler | завершает цикл, сбрасывает `running`, распространяет ошибку | +| Recovery Coordinator | не скрывает ошибки Store, Planner и Recovery | +| Composition | ошибка конструктора прерывает сборку без запуска lifecycle | + +Build не вводит общий error envelope и retry policy. + +--- + +## 16. Concurrency и lifecycle + +Компоненты Build не создают неявных фоновых задач. + +Единственный длительный цикл находится в: + +```text +await RuntimeScheduler.start() +``` + +Владелец внешнего lifecycle обязан: + +- создать и хранить task Scheduler; +- вызвать `RuntimeSupervisor.start()`; +- передавать live activity через `notify_activity()`; +- остановить Scheduler и Supervisor; +- обработать cancellation; +- определить reconnect/recovery order. + +Эти обязанности относятся к Production Runtime Integration. + +--- + +## 17. Архитектурные решения + +### ADR-060.24-001 — Checkpoint принадлежит Consistency Layer + +**Статус:** Accepted + +`TradeStreamState.last_trade` является единственным источником точки +восстановления. + +### ADR-060.24-002 — Checkpoint хранит полный Canonical Trade + +**Статус:** Accepted + +Отдельные timestamp, symbol и trade id не дублируются в Runtime. + +### ADR-060.24-003 — RuntimeSessionState распределён по владельцам + +**Статус:** Accepted + +Общий lifecycle хранится Supervisor, локальные состояния — Heartbeat +и Reconnect. Общий mutable session object не создаётся. + +### ADR-060.24-004 — Window Calculator является частью Planner + +**Статус:** Accepted + +Расчёт окна не имеет независимой ответственности, поэтому отдельный +Calculator не вводится. + +### ADR-060.24-005 — Shared boundary устраняется Consistency Layer + +**Статус:** Accepted + +Окна имеют общую временную границу, а возможные повторы удаляются +централизованной deduplication-логикой. + +### ADR-060.24-006 — Heartbeat пассивен + +**Статус:** Accepted + +Heartbeat не владеет task или scheduler. Он только хранит активность +и проверяет timeout. + +### ADR-060.24-007 — Reconnect выполняет одну попытку + +**Статус:** Accepted + +Retry и backoff остаются внешней policy. + +### ADR-060.24-008 — Recovery End Time передаётся извне + +**Статус:** Accepted + +Coordinator не читает системное время и не принимает production policy. + +### ADR-060.24-009 — Runtime Recovery не владеет WebSocket + +**Статус:** Accepted + +Recovery Coordinator связывает state, planning и REST Recovery, +не выполняя connect, disconnect или subscription restore. + +### ADR-060.24-010 — Composition не запускает lifecycle + +**Статус:** Accepted + +Factory создаёт граф без network-, task- и configuration-side effects. + +--- + +## 18. Проверяемые инварианты + +### State + +- checkpoint равен последней принятой сделке; +- checkpoint и `last_trade_id` согласованы; +- duplicate и error не изменяют checkpoint. + +### Planning + +- каждое окно валидно; +- окна не превышают configured maximum; +- окна покрывают полный диапазон без разрывов; +- равные границы дают пустой plan. + +### Runtime + +- Heartbeat публикует timeout однократно; +- Supervisor не запускает параллельный reconnect; +- Reconnect восстанавливает подписки только после connect; +- Scheduler не создаёт второй loop; +- ошибки не скрываются. + +### Recovery + +- отсутствие state/checkpoint не создаёт state; +- окна выполняются последовательно; +- результаты агрегируются в исходных границах; +- recovered Trade сохраняют identity; +- timezone и Unix millisecond conversion детерминированы. + +### Composition + +- live и recovery используют один consistency controller; +- coordinator и consistency controller используют один state store; +- runtime-компоненты получают ровно те зависимости, которые созданы + factory; +- создание графа не вызывает runtime-методы. + +--- + +## 19. Тестовая стратегия + +Build проверяется на трёх уровнях. + +### Value objects и чистые вычисления + +- checkpoint invariants; +- Recovery Window validation; +- Window Planner boundaries. + +### Изолированные runtime-компоненты + +- protocol compatibility; +- state transitions; +- event order; +- async interaction; +- error propagation; +- injected clock и sleep. + +### Composition + +- типы компонентов; +- identity общего Store и Consistency Controller; +- отсутствие side effects; +- передача конфигурации и внешних зависимостей. + +Итоговый целевой набор: + +```text +230 passed +``` + +Полная регрессия: + +```text +1620 passed +``` + +End-to-end WebSocket и stress tests не подменяются unit-тестами +и остаются обязательной частью Build 060.25–060.26. + +--- + +## 20. Граница Build 060.25 + +Build 060.25 должен добавить production-владельца lifecycle: + +```text +bootstrap + │ + ▼ +build composition + │ + ▼ +connect and subscribe + │ + ▼ +start supervisor and scheduler task + │ + ▼ +receive messages and notify activity + │ + ▼ +disconnect detected + │ + ▼ +reconnect and restore subscriptions + │ + ▼ +choose recovery_end_time + │ + ▼ +RuntimeRecoveryCoordinator.recover() + │ + ▼ +resume steady live lifecycle +``` + +До появления этой интеграции нельзя утверждать, что production +Trade Stream автоматически переживает WebSocket disconnect. + +--- + +## 21. Что не входит в архитектуру Build 060.24 + +- production bootstrap; +- конкретный WebSocket lifecycle owner; +- автоматический вызов Runtime Recovery после reconnect; +- policy выбора `recovery_end_time`; +- retry/backoff; +- parallel recovery нескольких symbols; +- persistent checkpoint; +- долговременное Market Data Storage; +- historical query и replay services; +- integration, reconnect, recovery и stress tests полного процесса. + +--- + +## 22. Итог + +Build 060.24 формирует завершённый внутренний набор компонентов Runtime +Recovery и фиксирует их границы. + +Архитектура: + +- сохраняет checkpoint в единственном владельце; +- отделяет вычисление окон от выполнения Recovery; +- разделяет heartbeat, scheduling, supervision и reconnect; +- не смешивает Recovery с Transport; +- собирает live и recovery поверх общего Consistency Layer; +- не скрывает production lifecycle внутри factory-функции. + +Следующий Build должен интегрировать эту архитектуру, а не менять её +ответственности.