From 8c485e32b191c317e0b6ccc8b9d85d4b83c73177 Mon Sep 17 00:00:00 2001 From: Sergey Date: Sat, 1 Aug 2026 20:55:32 +0300 Subject: [PATCH] Build 060.28: implement Persistent Checkpoint and Startup Recovery --- app/.env.example | 2 + app/src/bootstrap/app_factory.py | 5 + app/src/bootstrap/trade_stream_runtime.py | 40 + app/src/core/config.py | 24 +- .../acquisition/checkpoint/__init__.py | 11 + .../checkpoint/trade_stream_state_hydrator.py | 327 +++++ .../trade_observation_sink_protocol.py | 13 +- .../trade_stream_consistency_controller.py | 7 +- .../consistency/trade_stream_state.py | 79 +- .../consistency/trade_stream_state_store.py | 41 +- .../trade_stream_state_store_exceptions.py | 8 +- .../trade_stream_state_store_protocol.py | 13 +- app/src/market_data/acquisition/exceptions.py | 14 + .../runtime_reconnect_recovery_coordinator.py | 3 +- .../runtime_startup_recovery_coordinator.py | 295 +++++ .../trade_stream_production_runtime.py | 206 +++- .../trade_stream_runtime_composition.py | 48 + app/src/market_data/storage/__init__.py | 8 + app/src/market_data/storage/contracts.py | 141 +++ app/src/market_data/storage/exceptions.py | 12 + .../storage/postgres_partitions.py | 51 + .../storage/postgres_trade_repository.py | 899 +++++++++++++- .../storage/trade_storage_observation_sink.py | 37 +- app/src/storage/migrations.py | 38 + .../runtime/loopback_trade_exchange.py | 20 + .../test_trade_stream_loopback_integration.py | 431 +++++++ ...ostgres_partition_retention_integration.py | 475 ++++++++ .../test_postgres_schema_integration.py | 176 +++ ...trade_checkpoint_repository_integration.py | 707 +++++++++++ ...rade_checkpoint_consistency_integration.py | 209 ++++ ...t_trade_runtime_persistence_integration.py | 1 + ...eam_persistent_cancellation_integration.py | 185 +++ ...e_stream_persistent_failure_integration.py | 357 ++++++ ...e_stream_persistent_restart_integration.py | 1082 +++++++++++++++++ ...sistent_transaction_failure_integration.py | 480 ++++++++ ...rade_stream_state_hydration_integration.py | 283 +++++ app/tests/support/postgres_market_data.py | 159 ++- app/tests/support/trade_stream_runtime.py | 10 +- app/tests/unit/bootstrap/test_app_factory.py | 20 +- app/tests/unit/bootstrap/test_application.py | 11 + .../test_market_data_storage_bootstrap.py | 16 + .../bootstrap/test_trade_stream_runtime.py | 148 ++- app/tests/unit/core/test_config.py | 42 +- .../test_trade_stream_state_hydrator.py | 557 +++++++++ ...est_trade_stream_consistency_controller.py | 72 +- .../consistency/test_trade_stream_state.py | 263 ++++ .../test_trade_stream_state_store.py | 107 +- ..._runtime_reconnect_recovery_coordinator.py | 4 +- .../test_runtime_recovery_coordinator.py | 12 + ...st_runtime_startup_recovery_coordinator.py | 462 +++++++ .../test_trade_stream_production_runtime.py | 753 +++++++++++- ...websocket_runtime_reconnect_integration.py | 7 + .../test_trade_stream_runtime_composition.py | 357 +++++- .../storage/test_checkpoint_contracts.py | 260 ++++ .../storage/test_postgres_partitions.py | 83 ++ ...st_postgres_trade_checkpoint_repository.py | 817 +++++++++++++ .../test_trade_storage_observation_sink.py | 169 ++- app/tests/unit/storage/test_migrations.py | 25 +- .../unit/test_postgres_market_data_support.py | 276 +++++ .../unit/test_trade_stream_runtime_support.py | 28 + docs/migrations/build_060_28.md | 244 ++++ docs/migrations/build_060_28_architecture.md | 851 +++++++++++++ docs/roadmap/master-roadmap.md | 41 +- 63 files changed, 12430 insertions(+), 92 deletions(-) create mode 100644 app/src/market_data/acquisition/checkpoint/__init__.py create mode 100644 app/src/market_data/acquisition/checkpoint/trade_stream_state_hydrator.py create mode 100644 app/src/market_data/acquisition/runtime/runtime_startup_recovery_coordinator.py create mode 100644 app/tests/integration/market_data/storage/test_postgres_trade_checkpoint_repository_integration.py create mode 100644 app/tests/integration/market_data/storage/test_trade_checkpoint_consistency_integration.py create mode 100644 app/tests/integration/market_data/storage/test_trade_stream_persistent_cancellation_integration.py create mode 100644 app/tests/integration/market_data/storage/test_trade_stream_persistent_failure_integration.py create mode 100644 app/tests/integration/market_data/storage/test_trade_stream_persistent_restart_integration.py create mode 100644 app/tests/integration/market_data/storage/test_trade_stream_persistent_transaction_failure_integration.py create mode 100644 app/tests/integration/market_data/storage/test_trade_stream_state_hydration_integration.py create mode 100644 app/tests/unit/market_data/acquisition/checkpoint/test_trade_stream_state_hydrator.py create mode 100644 app/tests/unit/market_data/acquisition/runtime/test_runtime_startup_recovery_coordinator.py create mode 100644 app/tests/unit/market_data/storage/test_checkpoint_contracts.py create mode 100644 app/tests/unit/market_data/storage/test_postgres_trade_checkpoint_repository.py create mode 100644 docs/migrations/build_060_28.md create mode 100644 docs/migrations/build_060_28_architecture.md diff --git a/app/.env.example b/app/.env.example index fb4306d..d2c3ebe 100644 --- a/app/.env.example +++ b/app/.env.example @@ -33,3 +33,5 @@ TRADE_STREAM_CLOSE_TIMEOUT_SECONDS=10 TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS=30 TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS=5 TRADE_STREAM_RECOVERY_WINDOW_MS=3599999 +TRADE_STREAM_SUBSCRIPTION_ACK_TIMEOUT_SECONDS=10 +TRADE_STREAM_STARTUP_MARKET_BUFFER_CAPACITY=10000 diff --git a/app/src/bootstrap/app_factory.py b/app/src/bootstrap/app_factory.py index ea81548..f654917 100644 --- a/app/src/bootstrap/app_factory.py +++ b/app/src/bootstrap/app_factory.py @@ -52,6 +52,11 @@ def create_app() -> ApplicationComposition: if market_data_storage is not None else None ), + checkpoint_storage=( + market_data_storage.trade_repository + if market_data_storage is not None + else None + ), ) bot = Bot( diff --git a/app/src/bootstrap/trade_stream_runtime.py b/app/src/bootstrap/trade_stream_runtime.py index 2a10c01..35a53b2 100644 --- a/app/src/bootstrap/trade_stream_runtime.py +++ b/app/src/bootstrap/trade_stream_runtime.py @@ -38,12 +38,16 @@ from src.market_data.acquisition.runtime.websocket_subscription_manager import ( from src.market_data.acquisition.trade_stream_runtime_composition import ( build_trade_stream_runtime_composition, ) +from src.market_data.storage.contracts import ( + TradeCheckpointStorageProtocol, +) def build_trade_stream_production_runtime( settings: Settings, *, trade_observation_sink: TradeObservationSinkProtocol | None = None, + checkpoint_storage: TradeCheckpointStorageProtocol | None = None, ) -> TradeStreamProductionRuntime | None: """ Собрать Production Trade Stream Runtime без запуска lifecycle. @@ -53,10 +57,31 @@ def build_trade_stream_production_runtime( Runtime создаются один раз и затем передаются по identity. """ trade_stream = settings.trade_stream + storage_enabled = settings.market_data_storage.enabled if not trade_stream.enabled: + if storage_enabled: + raise RuntimeError( + "Trade Stream must be enabled when Market Data " + "Storage is enabled." + ) + return None + if storage_enabled and ( + trade_observation_sink is None + or checkpoint_storage is None + ): + raise RuntimeError( + "Enabled Market Data Storage requires both Trade " + "observation sink and checkpoint storage." + ) + + if not storage_enabled and checkpoint_storage is not None: + raise RuntimeError( + "Checkpoint storage requires enabled Market Data Storage." + ) + headers = { "Origin": settings.exchange_base_url.rstrip("/"), "Content-Type": "application/json", @@ -105,6 +130,12 @@ def build_trade_stream_production_runtime( ), trade_observation_sink=trade_observation_sink, max_recovery_window_ms=trade_stream.recovery_window_ms, + checkpoint_storage=checkpoint_storage, + checkpoint_venue=( + settings.exchange_name + if checkpoint_storage is not None + else None + ), ) return TradeStreamProductionRuntime( @@ -128,4 +159,13 @@ def build_trade_stream_production_runtime( runtime_supervisor=composition.runtime_supervisor, runtime_scheduler=composition.runtime_scheduler, symbols=trade_stream.symbols, + startup_recovery_coordinator=( + composition.runtime_startup_recovery_coordinator + ), + subscription_ack_timeout_seconds=( + trade_stream.subscription_ack_timeout_seconds + ), + startup_market_buffer_capacity=( + trade_stream.startup_market_buffer_capacity + ), ) diff --git a/app/src/core/config.py b/app/src/core/config.py index 7aab9c6..7d9f76a 100644 --- a/app/src/core/config.py +++ b/app/src/core/config.py @@ -9,6 +9,8 @@ from pathlib import Path from dotenv import load_dotenv +from src.market_data.acquisition.symbols import normalize_symbol + # корень проекта BASE_DIR = Path(__file__).resolve().parents[2] @@ -33,6 +35,8 @@ class TradeStreamSettings: heartbeat_timeout_seconds: float scheduler_interval_seconds: float recovery_window_ms: int + subscription_ack_timeout_seconds: float = 10.0 + startup_market_buffer_capacity: int = 10_000 @dataclass(frozen=True, slots=True) @@ -200,7 +204,7 @@ def _parse_trade_stream_symbols( return tuple( sorted( { - symbol.strip() + normalize_symbol(symbol) for symbol in raw_symbols } ) @@ -228,6 +232,8 @@ def _load_trade_stream_settings( heartbeat_timeout_seconds=30.0, scheduler_interval_seconds=5.0, recovery_window_ms=3_599_999, + subscription_ack_timeout_seconds=10.0, + startup_market_buffer_capacity=10_000, ) websocket_url = os.getenv( @@ -300,6 +306,22 @@ def _load_trade_stream_settings( name="TRADE_STREAM_RECOVERY_WINDOW_MS", default=3_599_999, ), + subscription_ack_timeout_seconds=_parse_positive_float( + os.getenv( + "TRADE_STREAM_SUBSCRIPTION_ACK_TIMEOUT_SECONDS", + "10", + ), + name="TRADE_STREAM_SUBSCRIPTION_ACK_TIMEOUT_SECONDS", + default=10.0, + ), + startup_market_buffer_capacity=_parse_positive_int( + os.getenv( + "TRADE_STREAM_STARTUP_MARKET_BUFFER_CAPACITY", + "10000", + ), + name="TRADE_STREAM_STARTUP_MARKET_BUFFER_CAPACITY", + default=10_000, + ), ) diff --git a/app/src/market_data/acquisition/checkpoint/__init__.py b/app/src/market_data/acquisition/checkpoint/__init__.py new file mode 100644 index 0000000..a1d225a --- /dev/null +++ b/app/src/market_data/acquisition/checkpoint/__init__.py @@ -0,0 +1,11 @@ +"""Восстановление persistent Trade Stream checkpoint.""" + +from src.market_data.acquisition.checkpoint.trade_stream_state_hydrator import ( + TradeStreamStateHydrator, + TradeStreamStateHydratorProtocol, +) + +__all__ = ( + "TradeStreamStateHydrator", + "TradeStreamStateHydratorProtocol", +) diff --git a/app/src/market_data/acquisition/checkpoint/trade_stream_state_hydrator.py b/app/src/market_data/acquisition/checkpoint/trade_stream_state_hydrator.py new file mode 100644 index 0000000..ee141f6 --- /dev/null +++ b/app/src/market_data/acquisition/checkpoint/trade_stream_state_hydrator.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from src.market_data.acquisition.consistency.trade_stream_exceptions import ( + TradeConsistencyError, + TradeOrderingError, +) +from src.market_data.acquisition.consistency.trade_stream_state import ( + DEFAULT_DEDUPLICATION_WINDOW_SIZE, + TradeStreamState, +) +from src.market_data.acquisition.consistency.trade_stream_state_store_exceptions import ( + TradeStreamStateStoreInitializationError, +) +from src.market_data.acquisition.consistency.trade_stream_state_store_protocol import ( + TradeStreamStateStoreProtocol, +) +from src.market_data.acquisition.models.trade import Trade +from src.market_data.acquisition.symbols import normalize_symbol +from src.market_data.storage.contracts import ( + PersistentTradeCheckpoint, + TradeCheckpointStorageProtocol, +) +from src.market_data.storage.exceptions import ( + MarketDataCheckpointIntegrityError, +) + + +@runtime_checkable +class TradeStreamStateHydratorProtocol(Protocol): + """Контракт восстановления Consistency state до запуска Runtime.""" + + def hydrate( + self, + *, + symbols: tuple[str, ...], + ) -> tuple[TradeStreamState, ...]: + """Восстановить и атомарно опубликовать состояния symbols.""" + ... + + +class TradeStreamStateHydrator: + """Восстанавливает TradeStreamState из persistent checkpoint.""" + + __slots__ = ( + "_checkpoint_storage", + "_state_store", + "_venue", + "_deduplication_window_size", + ) + + def __init__( + self, + *, + checkpoint_storage: TradeCheckpointStorageProtocol, + state_store: TradeStreamStateStoreProtocol, + venue: str, + deduplication_window_size: int = ( + DEFAULT_DEDUPLICATION_WINDOW_SIZE + ), + ) -> None: + if not isinstance( + checkpoint_storage, + TradeCheckpointStorageProtocol, + ): + raise TypeError( + "checkpoint_storage must implement " + "TradeCheckpointStorageProtocol" + ) + + if not isinstance(state_store, TradeStreamStateStoreProtocol): + raise TypeError( + "state_store must implement TradeStreamStateStoreProtocol" + ) + + if not isinstance(venue, str): + raise TypeError("venue must be a string") + + normalized_venue = venue.strip().lower() + + if not normalized_venue: + raise ValueError("venue must not be empty") + + if ( + isinstance(deduplication_window_size, bool) + or not isinstance(deduplication_window_size, int) + ): + raise TypeError( + "deduplication_window_size must be an integer" + ) + + if deduplication_window_size <= 0: + raise ValueError( + "deduplication_window_size must be positive" + ) + + self._checkpoint_storage = checkpoint_storage + self._state_store = state_store + self._venue = normalized_venue + self._deduplication_window_size = deduplication_window_size + + def hydrate( + self, + *, + symbols: tuple[str, ...], + ) -> tuple[TradeStreamState, ...]: + normalized_symbols = self._normalize_symbols(symbols) + + if not self._state_store.is_empty(): + raise TradeStreamStateStoreInitializationError( + "Trade Stream State Store уже содержит состояния." + ) + + states = tuple( + self._prepare_state(symbol) + for symbol in normalized_symbols + ) + self._state_store.initialize(states) + return states + + def _prepare_state( + self, + symbol: str, + ) -> TradeStreamState: + checkpoint = self._checkpoint_storage.load_checkpoint( + venue=self._venue, + symbol=symbol, + ) + + if checkpoint is not None: + return self._state_from_checkpoint( + symbol=symbol, + checkpoint=checkpoint, + ) + + latest_tail = self._checkpoint_storage.load_latest_trade_tail( + venue=self._venue, + symbol=symbol, + limit=self._deduplication_window_size, + ) + + if not isinstance(latest_tail, tuple): + raise MarketDataCheckpointIntegrityError( + "Checkpoint storage returned non-tuple latest Trade tail." + ) + + if not latest_tail: + concurrent_checkpoint = ( + self._checkpoint_storage.load_checkpoint( + venue=self._venue, + symbol=symbol, + ) + ) + + if concurrent_checkpoint is not None: + return self._state_from_checkpoint( + symbol=symbol, + checkpoint=concurrent_checkpoint, + ) + + return TradeStreamState.from_history( + symbol=symbol, + trades=(), + deduplication_window_size=( + self._deduplication_window_size + ), + ) + + candidate_state = self._state_from_tail( + symbol=symbol, + trades=latest_tail, + ) + candidate = candidate_state.last_trade + + if candidate is None: + raise MarketDataCheckpointIntegrityError( + "Latest Trade tail unexpectedly produced empty state." + ) + + adopted = ( + self._checkpoint_storage.adopt_existing_trade_as_checkpoint( + venue=self._venue, + trade=candidate, + ) + ) + self._validate_checkpoint( + checkpoint=adopted, + symbol=symbol, + expected_trade=candidate, + ) + return self._state_from_checkpoint( + symbol=symbol, + checkpoint=adopted, + ) + + def _state_from_checkpoint( + self, + *, + symbol: str, + checkpoint: PersistentTradeCheckpoint, + ) -> TradeStreamState: + self._validate_checkpoint( + checkpoint=checkpoint, + symbol=symbol, + expected_trade=None, + ) + tail = self._checkpoint_storage.load_checkpoint_tail( + venue=self._venue, + checkpoint=checkpoint, + limit=self._deduplication_window_size, + ) + state = self._state_from_tail( + symbol=symbol, + trades=tail, + ) + + if state.last_trade is None or not self._same_market_trade( + state.last_trade, + checkpoint.trade, + ): + raise MarketDataCheckpointIntegrityError( + "Hydrated tail does not end at persistent checkpoint." + ) + + return state + + def _state_from_tail( + self, + *, + symbol: str, + trades: object, + ) -> TradeStreamState: + if not isinstance(trades, tuple): + raise MarketDataCheckpointIntegrityError( + "Checkpoint storage returned non-tuple Trade tail." + ) + + try: + return TradeStreamState.from_history( + symbol=symbol, + trades=trades, + deduplication_window_size=( + self._deduplication_window_size + ), + ) + except ( + TypeError, + ValueError, + TradeConsistencyError, + TradeOrderingError, + ) as error: + raise MarketDataCheckpointIntegrityError( + "Persistent Trade tail cannot hydrate Consistency state." + ) from error + + def _validate_checkpoint( + self, + *, + checkpoint: object, + symbol: str, + expected_trade: Trade | None, + ) -> None: + if not isinstance(checkpoint, PersistentTradeCheckpoint): + raise MarketDataCheckpointIntegrityError( + "Checkpoint storage returned invalid checkpoint." + ) + + if checkpoint.venue.strip().lower() != self._venue: + raise MarketDataCheckpointIntegrityError( + "Persistent checkpoint venue does not match hydrator." + ) + + if checkpoint.trade.symbol.strip().upper() != symbol: + raise MarketDataCheckpointIntegrityError( + "Persistent checkpoint symbol does not match request." + ) + + if expected_trade is not None and not self._same_market_trade( + checkpoint.trade, + expected_trade, + ): + raise MarketDataCheckpointIntegrityError( + "Adopted checkpoint does not match candidate Trade." + ) + + @staticmethod + def _same_market_trade( + first: Trade, + second: Trade, + ) -> bool: + return ( + first.symbol == second.symbol + and first.trade_id == second.trade_id + and first.executed_at == second.executed_at + and first.price == second.price + and first.quantity == second.quantity + and first.aggressor_side is second.aggressor_side + ) + + @staticmethod + def _normalize_symbols( + symbols: tuple[str, ...], + ) -> tuple[str, ...]: + if not isinstance(symbols, tuple): + raise TypeError("symbols must be a tuple") + + if not symbols: + raise ValueError("symbols must not be empty") + + normalized: list[str] = [] + + for symbol in symbols: + if not isinstance(symbol, str): + raise TypeError("symbols must contain only strings") + + value = normalize_symbol(symbol) + + if not value: + raise ValueError("symbols must not contain empty values") + + if value in normalized: + raise ValueError("symbols must not contain duplicates") + + normalized.append(value) + + return tuple(normalized) diff --git a/app/src/market_data/acquisition/consistency/trade_observation_sink_protocol.py b/app/src/market_data/acquisition/consistency/trade_observation_sink_protocol.py index 5aedc9f..88c482e 100644 --- a/app/src/market_data/acquisition/consistency/trade_observation_sink_protocol.py +++ b/app/src/market_data/acquisition/consistency/trade_observation_sink_protocol.py @@ -9,9 +9,18 @@ from src.market_data.acquisition.models.trade import Trade class TradeObservationSinkProtocol(Protocol): """Надёжный приёмник одного наблюдения канонической сделки.""" - def persist( + def persist_accepted( + self, + trade: Trade, + *, + expected_trade: Trade | None, + ) -> None: + """Сохранить новую Trade и атомарно продвинуть checkpoint.""" + ... + + def persist_duplicate( self, trade: Trade, ) -> None: - """Надёжно сохранить наблюдение до продвижения контрольной точки.""" + """Сохранить provenance дубликата без продвижения checkpoint.""" ... diff --git a/app/src/market_data/acquisition/consistency/trade_stream_consistency_controller.py b/app/src/market_data/acquisition/consistency/trade_stream_consistency_controller.py index fb75c58..8793cc7 100644 --- a/app/src/market_data/acquisition/consistency/trade_stream_consistency_controller.py +++ b/app/src/market_data/acquisition/consistency/trade_stream_consistency_controller.py @@ -60,7 +60,12 @@ class TradeStreamConsistencyController( return state.accept( trade, before_checkpoint=( - sink.persist + sink.persist_accepted + if sink is not None + else None + ), + on_duplicate=( + sink.persist_duplicate if sink is not None else None ), diff --git a/app/src/market_data/acquisition/consistency/trade_stream_state.py b/app/src/market_data/acquisition/consistency/trade_stream_state.py index 55c013f..272006b 100644 --- a/app/src/market_data/acquisition/consistency/trade_stream_state.py +++ b/app/src/market_data/acquisition/consistency/trade_stream_state.py @@ -5,6 +5,7 @@ from __future__ import annotations from collections import deque from collections.abc import Callable from dataclasses import dataclass, field +from typing import Protocol, Self from src.market_data.acquisition.consistency.trade_stream_exceptions import ( TradeConsistencyError, @@ -17,6 +18,19 @@ from src.market_data.acquisition.trade_id_sequence import ( ) DEFAULT_DEDUPLICATION_WINDOW_SIZE = 10_000 +TradeDuplicateCallback = Callable[[Trade], None] + + +class TradeCheckpointCallback(Protocol): + """Callback атомарной записи перед in-memory checkpoint.""" + + def __call__( + self, + trade: Trade, + *, + expected_trade: Trade | None, + ) -> None: + ... @dataclass(slots=True) @@ -49,11 +63,47 @@ class TradeStreamState: ) self._trades = {} + @classmethod + def from_history( + cls, + *, + symbol: str, + trades: tuple[Trade, ...], + deduplication_window_size: int = ( + DEFAULT_DEDUPLICATION_WINDOW_SIZE + ), + ) -> Self: + """Строго восстановить временное состояние из durable tail.""" + if not isinstance(trades, tuple): + raise TypeError("trades must be a tuple") + + if len(trades) > deduplication_window_size: + raise ValueError( + "trades must fit deduplication_window_size" + ) + + state = cls( + symbol=symbol, + deduplication_window_size=deduplication_window_size, + ) + + for trade in trades: + if not isinstance(trade, Trade): + raise TypeError("trades must contain only Trade") + + if state.accept(trade) is None: + raise TradeConsistencyError( + "Hydration history contains a duplicate Trade." + ) + + return state + def accept( self, trade: Trade, *, - before_checkpoint: Callable[[Trade], None] | None = None, + before_checkpoint: TradeCheckpointCallback | None = None, + on_duplicate: TradeDuplicateCallback | None = None, ) -> Trade | None: """ Проверить сделку на согласованность. @@ -71,6 +121,9 @@ class TradeStreamState: ): raise TypeError("before_checkpoint must be callable") + if on_duplicate is not None and not callable(on_duplicate): + raise TypeError("on_duplicate must be callable") + if trade.symbol != self.symbol: raise ValueError( f"Unexpected symbol: {trade.symbol!r}" @@ -93,9 +146,9 @@ class TradeStreamState: previous, trade, ): - self._run_before_checkpoint( + self._run_duplicate_observation( trade, - callback=before_checkpoint, + callback=on_duplicate, ) return None @@ -108,9 +161,9 @@ class TradeStreamState: previous, trade, ): - self._run_before_checkpoint( + self._run_duplicate_observation( trade, - callback=before_checkpoint, + callback=on_duplicate, ) return None @@ -118,6 +171,7 @@ class TradeStreamState: self._run_before_checkpoint( trade, + expected_trade=self.last_trade, callback=before_checkpoint, ) @@ -132,7 +186,20 @@ class TradeStreamState: def _run_before_checkpoint( trade: Trade, *, - callback: Callable[[Trade], None] | None, + expected_trade: Trade | None, + callback: TradeCheckpointCallback | None, + ) -> None: + if callback is not None: + callback( + trade, + expected_trade=expected_trade, + ) + + @staticmethod + def _run_duplicate_observation( + trade: Trade, + *, + callback: TradeDuplicateCallback | None, ) -> None: if callback is not None: callback(trade) diff --git a/app/src/market_data/acquisition/consistency/trade_stream_state_store.py b/app/src/market_data/acquisition/consistency/trade_stream_state_store.py index 2a9a91e..fba6623 100644 --- a/app/src/market_data/acquisition/consistency/trade_stream_state_store.py +++ b/app/src/market_data/acquisition/consistency/trade_stream_state_store.py @@ -19,6 +19,7 @@ from src.market_data.acquisition.consistency.trade_stream_state import ( ) from src.market_data.acquisition.consistency.trade_stream_state_store_exceptions import ( TradeStreamStateNotFoundError, + TradeStreamStateStoreInitializationError, ) from src.market_data.acquisition.consistency.trade_stream_state_store_protocol import ( TradeStreamStateStoreProtocol, @@ -38,6 +39,43 @@ class TradeStreamStateStore(TradeStreamStateStoreProtocol): Создаёт пустое хранилище состояний. """ self._states: dict[str, TradeStreamState] = {} + self._initialized = False + + def initialize( + self, + states: tuple[TradeStreamState, ...], + ) -> None: + """Однократно опубликовать полностью подготовленный набор.""" + if not isinstance(states, tuple): + raise TypeError("states must be a tuple") + + if self._initialized or self._states: + raise TradeStreamStateStoreInitializationError( + "Trade Stream State Store уже содержит состояния." + ) + + prepared: dict[str, TradeStreamState] = {} + + for state in states: + if not isinstance(state, TradeStreamState): + raise TypeError( + "states must contain only TradeStreamState" + ) + + if state.symbol in prepared: + raise TradeStreamStateStoreInitializationError( + f"Состояние для символа {state.symbol!r} " + "передано повторно." + ) + + prepared[state.symbol] = state + + self._states.update(prepared) + self._initialized = True + + def is_empty(self) -> bool: + """Проверить, что хранилище не содержит состояний.""" + return not self._states and not self._initialized def get_or_create( self, @@ -126,4 +164,5 @@ class TradeStreamStateStore(TradeStreamStateStoreProtocol): """ Полностью очищает хранилище состояний. """ - self._states.clear() \ No newline at end of file + self._states.clear() + self._initialized = False diff --git a/app/src/market_data/acquisition/consistency/trade_stream_state_store_exceptions.py b/app/src/market_data/acquisition/consistency/trade_stream_state_store_exceptions.py index 1d38917..9c7f9f9 100644 --- a/app/src/market_data/acquisition/consistency/trade_stream_state_store_exceptions.py +++ b/app/src/market_data/acquisition/consistency/trade_stream_state_store_exceptions.py @@ -25,4 +25,10 @@ class TradeStreamStateNotFoundError( ): """ Состояние торгового инструмента отсутствует. - """ \ No newline at end of file + """ + + +class TradeStreamStateStoreInitializationError( + TradeStreamStateStoreError, +): + """Хранилище состояний нельзя безопасно инициализировать.""" diff --git a/app/src/market_data/acquisition/consistency/trade_stream_state_store_protocol.py b/app/src/market_data/acquisition/consistency/trade_stream_state_store_protocol.py index 4d15cf0..e5a66ce 100644 --- a/app/src/market_data/acquisition/consistency/trade_stream_state_store_protocol.py +++ b/app/src/market_data/acquisition/consistency/trade_stream_state_store_protocol.py @@ -29,6 +29,17 @@ class TradeStreamStateStoreProtocol(Protocol): Протокол хранилища состояний Trade Stream Consistency. """ + def initialize( + self, + states: tuple[TradeStreamState, ...], + ) -> None: + """Однократно и атомарно опубликовать готовые состояния.""" + ... + + def is_empty(self) -> bool: + """Проверить, что ни одно состояние ещё не опубликовано.""" + ... + def get_or_create( self, symbol: str, @@ -104,4 +115,4 @@ class TradeStreamStateStoreProtocol(Protocol): """ Полностью очищает хранилище состояний. """ - ... \ No newline at end of file + ... diff --git a/app/src/market_data/acquisition/exceptions.py b/app/src/market_data/acquisition/exceptions.py index b0ce85f..83f4323 100644 --- a/app/src/market_data/acquisition/exceptions.py +++ b/app/src/market_data/acquisition/exceptions.py @@ -166,6 +166,20 @@ class WebSocketControlMessageError(MarketDataAcquisitionError): pass +# Истекло время ожидания подтверждения стартовой WebSocket-подписки. +class WebSocketSubscriptionAckTimeoutError( + MarketDataAcquisitionError, +): + pass + + +# Переполнен буфер market-сообщений до подтверждения подписки. +class WebSocketStartupMarketBufferOverflowError( + MarketDataAcquisitionError, +): + pass + + # Ошибка низкоуровневой WebSocket-инфраструктуры Acquisition Runtime. class WebSocketTransportError(MarketDataAcquisitionError): pass diff --git a/app/src/market_data/acquisition/runtime/runtime_reconnect_recovery_coordinator.py b/app/src/market_data/acquisition/runtime/runtime_reconnect_recovery_coordinator.py index 505599e..3277e52 100644 --- a/app/src/market_data/acquisition/runtime/runtime_reconnect_recovery_coordinator.py +++ b/app/src/market_data/acquisition/runtime/runtime_reconnect_recovery_coordinator.py @@ -18,6 +18,7 @@ from src.market_data.acquisition.runtime.reconnect import ( from src.market_data.acquisition.runtime.runtime_recovery_protocol import ( RuntimeRecoveryProtocol, ) +from src.market_data.acquisition.symbols import normalize_symbol RuntimeUnixTimeMillisecondsClock = Callable[[], int] @@ -354,7 +355,7 @@ class RuntimeReconnectRecoveryCoordinator: "symbols must contain only strings" ) - normalized_symbol = symbol.strip() + normalized_symbol = normalize_symbol(symbol) if normalized_symbol: normalized_symbols.add(normalized_symbol) diff --git a/app/src/market_data/acquisition/runtime/runtime_startup_recovery_coordinator.py b/app/src/market_data/acquisition/runtime/runtime_startup_recovery_coordinator.py new file mode 100644 index 0000000..06d153c --- /dev/null +++ b/app/src/market_data/acquisition/runtime/runtime_startup_recovery_coordinator.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +import asyncio +from typing import Protocol, TypeVar, runtime_checkable + +from src.market_data.acquisition.checkpoint.trade_stream_state_hydrator import ( + TradeStreamStateHydratorProtocol, +) +from src.market_data.acquisition.consistency.trade_stream_state import ( + TradeStreamState, +) +from src.market_data.acquisition.recovery.trade_recovery_result import ( + TradeRecoveryResult, +) +from src.market_data.acquisition.runtime.live_processing_gate import ( + RuntimeLiveProcessingGateProtocol, +) +from src.market_data.acquisition.runtime.runtime_reconnect_recovery_coordinator import ( + RuntimeUnixTimeMillisecondsClock, + system_unix_time_ms, +) +from src.market_data.acquisition.runtime.runtime_recovery_protocol import ( + RuntimeRecoveryProtocol, +) +from src.market_data.acquisition.symbols import normalize_symbol + + +_WorkerResult = TypeVar("_WorkerResult") + + +@runtime_checkable +class RuntimeStartupRecoveryProtocol(Protocol): + """Контракт восстановления состояния при запуске Trade Stream.""" + + @property + def live_processing_gate( + self, + ) -> RuntimeLiveProcessingGateProtocol: + """Вернуть общий gate Startup Recovery и Live processing.""" + ... + + @property + def symbols(self) -> tuple[str, ...]: + """Вернуть нормализованные символы Startup Recovery.""" + ... + + async def hydrate_once( + self, + ) -> tuple[TradeStreamState, ...]: + """Один раз восстановить Consistency state из хранилища.""" + ... + + async def recover_after_ack( + self, + ) -> tuple[TradeRecoveryResult, ...]: + """Выполнить REST Recovery после подтверждения подписки.""" + ... + + +class RuntimeStartupRecoveryCoordinator: + """ + Координирует blocking-операции Startup Hydration и Recovery. + + Владельцем общей границы subscribe → ACK → Recovery → buffered Live + остаётся Production Runtime. Координатор хранит тот же gate для + проверки единства dependency graph, но самостоятельно его не + захватывает. + """ + + __slots__ = ( + "_state_hydrator", + "_recovery_coordinator", + "_live_processing_gate", + "_symbols", + "_clock", + "_hydration_lock", + "_hydration_completed", + "_hydrated_states", + "_hydration_task", + "_recovery_task", + ) + + def __init__( + self, + *, + state_hydrator: TradeStreamStateHydratorProtocol, + recovery_coordinator: RuntimeRecoveryProtocol, + live_processing_gate: RuntimeLiveProcessingGateProtocol, + symbols: tuple[str, ...], + clock: RuntimeUnixTimeMillisecondsClock = system_unix_time_ms, + ) -> None: + if not callable(clock): + raise TypeError("clock must be callable") + + self._state_hydrator = state_hydrator + self._recovery_coordinator = recovery_coordinator + self._live_processing_gate = live_processing_gate + self._symbols = self._normalize_symbols(symbols) + self._clock = clock + self._hydration_lock = asyncio.Lock() + self._hydration_completed = False + self._hydrated_states: tuple[TradeStreamState, ...] = () + self._hydration_task: ( + asyncio.Task[tuple[TradeStreamState, ...]] | None + ) = None + self._recovery_task: ( + asyncio.Task[tuple[TradeRecoveryResult, ...]] | None + ) = None + + @property + def live_processing_gate( + self, + ) -> RuntimeLiveProcessingGateProtocol: + """Вернуть общий gate Startup Recovery и Live processing.""" + return self._live_processing_gate + + @property + def symbols(self) -> tuple[str, ...]: + """Вернуть нормализованные символы Startup Recovery.""" + return self._symbols + + async def hydrate_once( + self, + ) -> tuple[TradeStreamState, ...]: + """ + Восстановить Consistency state не более одного успешного раза. + + При отмене вызывающего coroutine уже запущенный blocking worker + завершается до возврата управления. Если worker успел успешно + восстановить state, результат сохраняется для следующего вызова. + """ + if self._hydration_completed: + return self._hydrated_states + + async with self._hydration_lock: + if self._hydration_completed: + return self._hydrated_states + + hydration_task = asyncio.create_task( + asyncio.to_thread( + self._hydrate_symbols, + ), + name="trade-stream-state-hydration", + ) + self._hydration_task = hydration_task + + try: + states = await asyncio.shield( + hydration_task, + ) + except asyncio.CancelledError as cancellation: + try: + states = await self._wait_for_worker_completion( + hydration_task, + ) + except BaseException as hydration_error: + cancellation.add_note( + "Trade Stream state hydration also failed: " + f"{type(hydration_error).__name__}." + ) + else: + self._store_hydration_result(states) + + raise + else: + self._store_hydration_result(states) + return states + finally: + if self._hydration_task is hydration_task: + self._hydration_task = None + + async def recover_after_ack( + self, + ) -> tuple[TradeRecoveryResult, ...]: + """ + Последовательно восстановить symbols по одной временной границе. + + Вызывающий Production Runtime обязан удерживать общий live gate + от отправки подписки до завершения buffered Live processing. + """ + recovery_end_time = self._clock() + self._validate_recovery_end_time(recovery_end_time) + + recovery_task = asyncio.create_task( + asyncio.to_thread( + self._recover_symbols, + recovery_end_time, + ), + name="trade-stream-startup-recovery", + ) + self._recovery_task = recovery_task + + try: + return await asyncio.shield( + recovery_task, + ) + except asyncio.CancelledError as cancellation: + try: + await self._wait_for_worker_completion( + recovery_task, + ) + except BaseException as recovery_error: + cancellation.add_note( + "Trade Stream Startup Recovery also failed: " + f"{type(recovery_error).__name__}." + ) + + raise + finally: + if self._recovery_task is recovery_task: + self._recovery_task = None + + def _hydrate_symbols( + self, + ) -> tuple[TradeStreamState, ...]: + return self._state_hydrator.hydrate( + symbols=self._symbols, + ) + + def _recover_symbols( + self, + recovery_end_time: int, + ) -> tuple[TradeRecoveryResult, ...]: + return tuple( + self._recovery_coordinator.recover( + symbol=symbol, + recovery_end_time=recovery_end_time, + ) + for symbol in self._symbols + ) + + def _store_hydration_result( + self, + states: tuple[TradeStreamState, ...], + ) -> None: + self._hydrated_states = states + self._hydration_completed = True + + @staticmethod + async def _wait_for_worker_completion( + worker_task: asyncio.Task[_WorkerResult], + ) -> _WorkerResult: + while not worker_task.done(): + try: + await asyncio.shield( + worker_task, + ) + except asyncio.CancelledError: + continue + except BaseException: + break + + return worker_task.result() + + @staticmethod + def _normalize_symbols( + symbols: tuple[str, ...], + ) -> tuple[str, ...]: + if not isinstance(symbols, tuple): + raise TypeError("symbols must be a tuple") + + normalized_symbols: set[str] = set() + + for symbol in symbols: + if not isinstance(symbol, str): + raise TypeError( + "symbols must contain only strings" + ) + + normalized_symbol = normalize_symbol(symbol) + + if normalized_symbol: + normalized_symbols.add(normalized_symbol) + + if not normalized_symbols: + raise ValueError( + "symbols must contain at least one non-empty symbol" + ) + + return tuple(sorted(normalized_symbols)) + + @staticmethod + def _validate_recovery_end_time( + recovery_end_time: int, + ) -> None: + if isinstance(recovery_end_time, bool) or not isinstance( + recovery_end_time, + int, + ): + raise TypeError("clock must return an integer") + + if recovery_end_time < 0: + raise ValueError( + "clock must not return a negative value" + ) diff --git a/app/src/market_data/acquisition/runtime/trade_stream_production_runtime.py b/app/src/market_data/acquisition/runtime/trade_stream_production_runtime.py index cd2cfd6..35c6db4 100644 --- a/app/src/market_data/acquisition/runtime/trade_stream_production_runtime.py +++ b/app/src/market_data/acquisition/runtime/trade_stream_production_runtime.py @@ -2,12 +2,16 @@ from __future__ import annotations import asyncio import json +import math +from collections import deque from enum import Enum from typing import Protocol, runtime_checkable from uuid import uuid4 from src.market_data.acquisition.exceptions import ( WebSocketMessageDecodeError, + WebSocketStartupMarketBufferOverflowError, + WebSocketSubscriptionAckTimeoutError, WebSocketTransportError, ) from src.market_data.acquisition.models.trade import Trade @@ -17,6 +21,9 @@ from src.market_data.acquisition.runtime.live_processing_gate import ( from src.market_data.acquisition.runtime.runtime_reconnect_recovery_coordinator import ( RuntimeReconnectRecoveryProtocol, ) +from src.market_data.acquisition.runtime.runtime_startup_recovery_coordinator import ( + RuntimeStartupRecoveryProtocol, +) from src.market_data.acquisition.runtime.scheduler import ( RuntimeSchedulerProtocol, ) @@ -47,6 +54,7 @@ from src.market_data.acquisition.runtime.websocket_protocol import ( from src.market_data.acquisition.trade_stream_acquisition_protocol import ( TradeStreamAcquisitionServiceProtocol, ) +from src.market_data.acquisition.symbols import normalize_symbol class TradeStreamProductionRuntimeState(str, Enum): @@ -116,6 +124,9 @@ class TradeStreamProductionRuntime: "_control_message_handler", "_live_processing_gate", "_reconnect_recovery_coordinator", + "_startup_recovery_coordinator", + "_subscription_ack_timeout_seconds", + "_startup_market_buffer_capacity", "_runtime_supervisor", "_runtime_scheduler", "_symbols", @@ -148,6 +159,11 @@ class TradeStreamProductionRuntime: runtime_supervisor: RuntimeSupervisorProtocol, runtime_scheduler: RuntimeSchedulerProtocol, symbols: tuple[str, ...], + startup_recovery_coordinator: ( + RuntimeStartupRecoveryProtocol | None + ) = None, + subscription_ack_timeout_seconds: float = 10.0, + startup_market_buffer_capacity: int = 10_000, ) -> None: self._session = session self._transport = transport @@ -174,6 +190,63 @@ class TradeStreamProductionRuntime: "must use the same symbols." ) + if startup_recovery_coordinator is not None: + if ( + startup_recovery_coordinator.live_processing_gate + is not live_processing_gate + ): + raise ValueError( + "Production Runtime and startup recovery " + "coordinator must share one live processing gate." + ) + + if ( + startup_recovery_coordinator.symbols + != normalized_symbols + ): + raise ValueError( + "Production Runtime and startup recovery " + "coordinator must use the same symbols." + ) + + if ( + isinstance(subscription_ack_timeout_seconds, bool) + or not isinstance( + subscription_ack_timeout_seconds, + (int, float), + ) + ): + raise TypeError( + "subscription_ack_timeout_seconds must be a number" + ) + + normalized_ack_timeout = float( + subscription_ack_timeout_seconds, + ) + + if ( + not math.isfinite(normalized_ack_timeout) + or normalized_ack_timeout <= 0.0 + ): + raise ValueError( + "subscription_ack_timeout_seconds must be finite " + "and greater than zero" + ) + + if ( + isinstance(startup_market_buffer_capacity, bool) + or not isinstance(startup_market_buffer_capacity, int) + ): + raise TypeError( + "startup_market_buffer_capacity must be an integer" + ) + + if startup_market_buffer_capacity <= 0: + raise ValueError( + "startup_market_buffer_capacity must be greater " + "than zero" + ) + if runtime_scheduler.runtime_supervisor is not runtime_supervisor: raise ValueError( "Production Runtime and Scheduler must share one " @@ -190,6 +263,15 @@ class TradeStreamProductionRuntime: self._reconnect_recovery_coordinator = ( reconnect_recovery_coordinator ) + self._startup_recovery_coordinator = ( + startup_recovery_coordinator + ) + self._subscription_ack_timeout_seconds = ( + normalized_ack_timeout + ) + self._startup_market_buffer_capacity = ( + startup_market_buffer_capacity + ) self._runtime_supervisor = runtime_supervisor self._runtime_scheduler = runtime_scheduler self._symbols = normalized_symbols @@ -386,6 +468,19 @@ class TradeStreamProductionRuntime: raise async def _start_sequence(self) -> None: + startup_recovery_coordinator = ( + self._startup_recovery_coordinator + ) + + if startup_recovery_coordinator is not None: + try: + await startup_recovery_coordinator.hydrate_once() + except asyncio.CancelledError: + raise + except Exception as error: + self._live_processing_gate.fail(error) + raise + try: await self._session.start() except asyncio.CancelledError: @@ -396,6 +491,10 @@ class TradeStreamProductionRuntime: reason=str(error), ) ) + + if startup_recovery_coordinator is not None: + self._live_processing_gate.fail(error) + raise self._session_started = True @@ -410,11 +509,110 @@ class TradeStreamProductionRuntime: correlation_id = str(uuid4()) self._subscription_correlation_id = correlation_id - await self._trade_stream_service.subscribe( - self._symbols, - correlation_id=correlation_id, + if startup_recovery_coordinator is None: + await self._trade_stream_service.subscribe( + self._symbols, + correlation_id=correlation_id, + ) + return + + try: + async with self._live_processing_gate: + await self._trade_stream_service.subscribe( + self._symbols, + correlation_id=correlation_id, + ) + + buffered_documents = await self._wait_for_startup_ack( + correlation_id=correlation_id, + ) + + await startup_recovery_coordinator.recover_after_ack() + + for document in buffered_documents: + await self._process_market_document(document) + except asyncio.CancelledError: + raise + except Exception as error: + self._live_processing_gate.fail(error) + raise + + async def _wait_for_startup_ack( + self, + *, + correlation_id: str, + ) -> deque[object]: + """ + Дождаться ACK одним WebSocket consumer и сохранить ранний market FIFO. + + Timeout запускается только после успешной отправки подписки. + После принятого ACK этот метод больше не читает транспорт. + """ + buffered_documents: deque[object] = deque() + ack_timeout = asyncio.timeout( + self._subscription_ack_timeout_seconds, ) + try: + async with ack_timeout: + while True: + raw_message = await self._transport.receive() + + transport_message = self._wrap_transport_message( + raw_message, + ) + + await self._event_publisher.publish( + MessageReceivedEvent( + message=transport_message, + ) + ) + + document = self._decode_message(raw_message) + message_kind = self._message_classifier.classify( + document, + ) + + if ( + message_kind + is WebSocketInboundMessageKind.CONTROL + ): + self._control_message_handler.handle( + document, + expected_correlation_id=correlation_id, + ) + return buffered_documents + + if ( + message_kind + is not WebSocketInboundMessageKind.MARKET + ): + raise TypeError( + "message classifier must return " + "WebSocketInboundMessageKind" + ) + + if ( + len(buffered_documents) + >= self._startup_market_buffer_capacity + ): + raise ( + WebSocketStartupMarketBufferOverflowError( + "Переполнен буфер market-сообщений " + "до подтверждения WebSocket-подписки." + ) + ) + + buffered_documents.append(document) + except TimeoutError as error: + if not ack_timeout.expired(): + raise + + raise WebSocketSubscriptionAckTimeoutError( + "Истекло время ожидания подтверждения стартовой " + "WebSocket-подписки." + ) from error + async def _receive_loop(self) -> None: while not self._stop_requested.is_set(): observed_generation = ( @@ -810,7 +1008,7 @@ class TradeStreamProductionRuntime: "symbols must contain only strings" ) - normalized_symbol = symbol.strip() + normalized_symbol = normalize_symbol(symbol) if normalized_symbol: normalized_symbols.add(normalized_symbol) diff --git a/app/src/market_data/acquisition/trade_stream_runtime_composition.py b/app/src/market_data/acquisition/trade_stream_runtime_composition.py index 802e9ac..9c3614b 100644 --- a/app/src/market_data/acquisition/trade_stream_runtime_composition.py +++ b/app/src/market_data/acquisition/trade_stream_runtime_composition.py @@ -12,6 +12,9 @@ from typing import Protocol from src.market_data.acquisition.adapters.dzengi.rest import ( DzengiTradesDocumentSource, ) +from src.market_data.acquisition.checkpoint.trade_stream_state_hydrator import ( + TradeStreamStateHydrator, +) from src.market_data.acquisition.consistency.trade_stream_consistency_controller import ( TradeStreamConsistencyController, ) @@ -51,6 +54,9 @@ from src.market_data.acquisition.runtime.runtime_liveness_probe import ( from src.market_data.acquisition.runtime.runtime_recovery_coordinator import ( RuntimeRecoveryCoordinator, ) +from src.market_data.acquisition.runtime.runtime_startup_recovery_coordinator import ( + RuntimeStartupRecoveryCoordinator, +) from src.market_data.acquisition.runtime.scheduler import ( RuntimeScheduler, RuntimeSleep, @@ -70,6 +76,9 @@ from src.market_data.acquisition.trade_stream_acquisition_service import ( from src.market_data.acquisition.trade_stream_message_adapter_protocol import ( TradeStreamMessageAdapterProtocol, ) +from src.market_data.storage.contracts import ( + TradeCheckpointStorageProtocol, +) class _TradeStreamRuntimeTransportProtocol( @@ -99,6 +108,10 @@ class TradeStreamRuntimeComposition: recovery_controller: TradeRecoveryController recovery_window_planner: TradeRecoveryWindowPlanner runtime_recovery_coordinator: RuntimeRecoveryCoordinator + state_hydrator: TradeStreamStateHydrator | None + runtime_startup_recovery_coordinator: ( + RuntimeStartupRecoveryCoordinator | None + ) acquisition_runtime_service: AcquisitionRuntimeService trade_stream_acquisition_service: TradeStreamAcquisitionService @@ -132,6 +145,8 @@ def build_trade_stream_runtime_composition( system_unix_time_ms ), scheduler_sleep: RuntimeSleep = asyncio.sleep, + checkpoint_storage: TradeCheckpointStorageProtocol | None = None, + checkpoint_venue: str | None = None, ) -> TradeStreamRuntimeComposition: """ Построить изолированный граф зависимостей Trade Stream Runtime. @@ -145,6 +160,12 @@ def build_trade_stream_runtime_composition( Factory не создаёт production transport, не читает Settings, не запускает lifecycle и не создаёт фоновые asyncio-задачи. """ + if (checkpoint_storage is None) != (checkpoint_venue is None): + raise ValueError( + "checkpoint_storage и checkpoint_venue должны быть " + "переданы вместе" + ) + state_store = TradeStreamStateStore() consistency_controller = TradeStreamConsistencyController( @@ -198,6 +219,29 @@ def build_trade_stream_runtime_composition( ) ) + state_hydrator: TradeStreamStateHydrator | None = None + runtime_startup_recovery_coordinator: ( + RuntimeStartupRecoveryCoordinator | None + ) = None + + if checkpoint_storage is not None and checkpoint_venue is not None: + state_hydrator = TradeStreamStateHydrator( + checkpoint_storage=checkpoint_storage, + state_store=state_store, + venue=checkpoint_venue, + ) + runtime_startup_recovery_coordinator = ( + RuntimeStartupRecoveryCoordinator( + state_hydrator=state_hydrator, + recovery_coordinator=runtime_recovery_coordinator, + live_processing_gate=live_processing_gate, + symbols=( + runtime_reconnect_recovery_coordinator.symbols + ), + clock=recovery_end_time_clock, + ) + ) + heartbeat_monitor = HeartbeatMonitor( event_publisher=event_publisher, timeout_seconds=heartbeat_timeout_seconds, @@ -226,6 +270,10 @@ def build_trade_stream_runtime_composition( recovery_controller=recovery_controller, recovery_window_planner=recovery_window_planner, runtime_recovery_coordinator=runtime_recovery_coordinator, + state_hydrator=state_hydrator, + runtime_startup_recovery_coordinator=( + runtime_startup_recovery_coordinator + ), acquisition_runtime_service=acquisition_runtime_service, trade_stream_acquisition_service=trade_stream_acquisition_service, reconnect_coordinator=reconnect_coordinator, diff --git a/app/src/market_data/storage/__init__.py b/app/src/market_data/storage/__init__.py index d56c385..14c6734 100644 --- a/app/src/market_data/storage/__init__.py +++ b/app/src/market_data/storage/__init__.py @@ -5,10 +5,14 @@ from src.market_data.storage.contracts import ( MarketDataBatchWriteResult, MarketDataWriteResult, MarketDataWriteStatus, + PersistentTradeCheckpoint, QuoteStorageProtocol, + TradeCheckpointStorageProtocol, TradeStorageProtocol, ) from src.market_data.storage.exceptions import ( + MarketDataCheckpointConflictError, + MarketDataCheckpointIntegrityError, MarketDataStorageConfigurationError, MarketDataStorageConflictError, MarketDataStorageError, @@ -46,6 +50,8 @@ from src.market_data.storage.postgres_partitions import ( __all__ = ( "CandleStorageProtocol", "MarketDataBatchWriteResult", + "MarketDataCheckpointConflictError", + "MarketDataCheckpointIntegrityError", "MarketDataPartition", "MarketDataPartitionResult", "MarketDataPartitionType", @@ -60,12 +66,14 @@ __all__ = ( "MarketDataStorageValidationError", "MarketDataWriteResult", "MarketDataWriteStatus", + "PersistentTradeCheckpoint", "PostgresCandleRepository", "PostgresMarketDataPartitionManager", "PostgresMarketDataRetentionService", "PostgresQuoteRepository", "PostgresTradeRepository", "QuoteStorageProtocol", + "TradeCheckpointStorageProtocol", "TradeStorageProtocol", "TradeObservationClock", "TradeStorageObservationSink", diff --git a/app/src/market_data/storage/contracts.py b/app/src/market_data/storage/contracts.py index 977b384..6de887a 100644 --- a/app/src/market_data/storage/contracts.py +++ b/app/src/market_data/storage/contracts.py @@ -8,6 +8,9 @@ from typing import Protocol, runtime_checkable from src.market_data.acquisition.models.candle import Candle from src.market_data.acquisition.models.quote import Quote from src.market_data.acquisition.models.trade import Trade +from src.market_data.acquisition.trade_id_sequence import ( + validate_signed_trade_id, +) class MarketDataWriteStatus(Enum): @@ -61,6 +64,78 @@ class MarketDataBatchWriteResult: ) +@dataclass(frozen=True, slots=True) +class PersistentTradeCheckpoint: + """ + Подтверждённая persistent-точка состояния Trade Stream. + """ + + venue: str + trade: Trade + revision: int + updated_at: datetime + checkpoint_schema_version: int = 1 + + def __post_init__(self) -> None: + if not isinstance(self.venue, str): + raise TypeError("venue must be a string") + + if not self.venue.strip(): + raise ValueError("venue must not be empty") + + if not isinstance(self.trade, Trade): + raise TypeError("trade must be a Canonical Trade") + + if not isinstance(self.trade.symbol, str): + raise TypeError("trade.symbol must be a string") + + if not self.trade.symbol.strip(): + raise ValueError("trade.symbol must not be empty") + + validate_signed_trade_id(self.trade.trade_id) + + if ( + not isinstance(self.trade.executed_at, datetime) + or self.trade.executed_at.tzinfo is None + or self.trade.executed_at.utcoffset() is None + ): + raise ValueError( + "trade.executed_at must contain timezone" + ) + + for field_name, value in ( + ("revision", self.revision), + ( + "checkpoint_schema_version", + self.checkpoint_schema_version, + ), + ): + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{field_name} must be an integer") + + if value <= 0: + raise ValueError(f"{field_name} must be positive") + + if not isinstance(self.updated_at, datetime): + raise TypeError("updated_at must be a datetime") + + if ( + self.updated_at.tzinfo is None + or self.updated_at.utcoffset() is None + ): + raise ValueError("updated_at must contain timezone") + + @property + def identity(self) -> tuple[str, str, int, datetime]: + """Вернуть durable identity checkpoint Trade.""" + return ( + self.venue, + self.trade.symbol, + self.trade.trade_id, + self.trade.executed_at, + ) + + @runtime_checkable class TradeStorageProtocol(Protocol): """Граница постоянного хранения канонических сделок только для записи.""" @@ -84,6 +159,72 @@ class TradeStorageProtocol(Protocol): ... +@runtime_checkable +class TradeCheckpointStorageProtocol(Protocol): + """ + Узкая граница хранения и восстановления Trade checkpoint. + """ + + def load_checkpoint( + self, + *, + venue: str, + symbol: str, + ) -> PersistentTradeCheckpoint | None: + """ + Загрузить checkpoint, подтверждённый Canonical Trade history. + """ + ... + + def load_checkpoint_tail( + self, + *, + venue: str, + checkpoint: PersistentTradeCheckpoint, + limit: int, + ) -> tuple[Trade, ...]: + """Загрузить bounded history до указанного checkpoint.""" + ... + + def load_latest_trade_tail( + self, + *, + venue: str, + symbol: str, + limit: int, + ) -> tuple[Trade, ...]: + """ + Загрузить bounded tail для первой инициализации checkpoint. + """ + ... + + def adopt_existing_trade_as_checkpoint( + self, + *, + venue: str, + trade: Trade, + ) -> PersistentTradeCheckpoint: + """ + Создать первый checkpoint на существующей durable Trade. + + Операция не должна повторно записывать наблюдение Trade либо + изменять её provenance и observation timestamps. + """ + ... + + def store_trade_and_advance_checkpoint( + self, + *, + venue: str, + expected_trade: Trade | None, + trade: Trade, + observed_at: datetime, + ) -> PersistentTradeCheckpoint: + """ + Атомарно сохранить принятую Trade и продвинуть checkpoint. + """ + ... + @runtime_checkable class QuoteStorageProtocol(Protocol): """Граница постоянного хранения канонических котировок для записи.""" diff --git a/app/src/market_data/storage/exceptions.py b/app/src/market_data/storage/exceptions.py index 4a08b6f..e8624b3 100644 --- a/app/src/market_data/storage/exceptions.py +++ b/app/src/market_data/storage/exceptions.py @@ -19,3 +19,15 @@ class MarketDataStorageConflictError(MarketDataStorageError): class MarketDataStorageOperationError(MarketDataStorageError): """Ошибка операции постоянного хранилища с базой данных.""" + + +class MarketDataCheckpointConflictError( + MarketDataStorageConflictError, +): + """ + Persistent checkpoint не совпал с ожидаемым состоянием writer. + """ + + +class MarketDataCheckpointIntegrityError(MarketDataStorageError): + """Persistent checkpoint не подтверждается Canonical Trade history.""" diff --git a/app/src/market_data/storage/postgres_partitions.py b/app/src/market_data/storage/postgres_partitions.py index 9d84ec7..b2c7620 100644 --- a/app/src/market_data/storage/postgres_partitions.py +++ b/app/src/market_data/storage/postgres_partitions.py @@ -21,6 +21,32 @@ from src.market_data.storage.postgres_repository_support import ( MARKET_DATA_PARTITION_ADVISORY_LOCK_ID = 0x445A504152544E _SCHEMA_NAME = "market_data" +_TRADE_CHECKPOINT_FK_NAME = "trade_stream_checkpoints_trade_fk" + + +def _drop_trade_checkpoint_foreign_key(*, cursor: Any) -> None: + cursor.execute( + sql.SQL("ALTER TABLE {} DROP CONSTRAINT {}").format( + sql.Identifier(_SCHEMA_NAME, "trade_stream_checkpoints"), + sql.Identifier(_TRADE_CHECKPOINT_FK_NAME), + ) + ) + + +def _restore_trade_checkpoint_foreign_key(*, cursor: Any) -> None: + cursor.execute( + sql.SQL( + "ALTER TABLE {} ADD CONSTRAINT {} " + "FOREIGN KEY (venue, symbol, trade_id, executed_at) " + "REFERENCES {} (venue, symbol, trade_id, executed_at) " + "ON UPDATE NO ACTION ON DELETE NO ACTION " + "DEFERRABLE INITIALLY DEFERRED" + ).format( + sql.Identifier(_SCHEMA_NAME, "trade_stream_checkpoints"), + sql.Identifier(_TRADE_CHECKPOINT_FK_NAME), + sql.Identifier(_SCHEMA_NAME, "trades"), + ) + ) class MarketDataPartitionType(Enum): @@ -456,6 +482,9 @@ class PostgresMarketDataPartitionManager: partition=partition, spec=spec, ) + + if partition.data_type is MarketDataPartitionType.TRADES: + _restore_trade_checkpoint_foreign_key(cursor=cursor) except MarketDataStorageError: raise except Exception as error: @@ -486,9 +515,20 @@ class PostgresMarketDataPartitionManager: range_start = sql.Literal(partition.range_start) range_end = sql.Literal(partition.range_end) + if partition.data_type is MarketDataPartitionType.TRADES: + cursor.execute( + sql.SQL( + "LOCK TABLE {} IN SHARE ROW EXCLUSIVE MODE" + ).format(parent) + ) + cursor.execute( sql.SQL("LOCK TABLE {} IN ACCESS EXCLUSIVE MODE").format(default) ) + + if partition.data_type is MarketDataPartitionType.TRADES: + _drop_trade_checkpoint_foreign_key(cursor=cursor) + cursor.execute( sql.SQL("CREATE TABLE {} (LIKE {} INCLUDING ALL)").format( child, @@ -625,6 +665,14 @@ class PostgresMarketDataRetentionService: ) ) + maintains_trade_checkpoints = any( + data_type is MarketDataPartitionType.TRADES + for data_type, _ in configured + ) + + if maintains_trade_checkpoints: + _drop_trade_checkpoint_foreign_key(cursor=cursor) + entries = tuple( self._apply_one( cursor=cursor, @@ -633,6 +681,9 @@ class PostgresMarketDataRetentionService: ) for data_type, days in configured ) + + if maintains_trade_checkpoints: + _restore_trade_checkpoint_foreign_key(cursor=cursor) except MarketDataStorageError: raise except Exception as error: diff --git a/app/src/market_data/storage/postgres_trade_repository.py b/app/src/market_data/storage/postgres_trade_repository.py index 83e211f..a3fd9c0 100644 --- a/app/src/market_data/storage/postgres_trade_repository.py +++ b/app/src/market_data/storage/postgres_trade_repository.py @@ -10,14 +10,18 @@ from src.market_data.acquisition.models.trade import ( TradeAggressorSide, ) from src.market_data.acquisition.trade_id_sequence import ( + is_trade_id_newer, validate_signed_trade_id, ) from src.market_data.storage.contracts import ( MarketDataBatchWriteResult, MarketDataWriteResult, MarketDataWriteStatus, + PersistentTradeCheckpoint, ) from src.market_data.storage.exceptions import ( + MarketDataCheckpointConflictError, + MarketDataCheckpointIntegrityError, MarketDataStorageConflictError, MarketDataStorageError, MarketDataStorageOperationError, @@ -33,6 +37,7 @@ from src.market_data.storage.postgres_repository_support import ( CANONICAL_TRADE_SCHEMA_VERSION = 1 +CHECKPOINT_SCHEMA_VERSION = 1 _INSERT_TRADE_SQL = """ INSERT INTO market_data.trades ( @@ -84,6 +89,165 @@ WHERE venue = %s AND executed_at = %s """ +_SELECT_CHECKPOINT_SQL = """ +SELECT + checkpoint.venue, + checkpoint.symbol, + checkpoint.trade_id, + checkpoint.executed_at, + checkpoint.revision, + checkpoint.updated_at, + checkpoint.checkpoint_schema_version, + trade_row.venue, + trade_row.symbol, + trade_row.trade_id, + trade_row.executed_at, + trade_row.price, + trade_row.quantity, + trade_row.aggressor_side, + trade_row.source, + trade_row.canonical_schema_version +FROM market_data.trade_stream_checkpoints AS checkpoint +LEFT JOIN market_data.trades AS trade_row + ON trade_row.venue = checkpoint.venue + AND trade_row.symbol = checkpoint.symbol + AND trade_row.trade_id = checkpoint.trade_id + AND trade_row.executed_at = checkpoint.executed_at +WHERE checkpoint.venue = %s + AND checkpoint.symbol = %s +""" + +_SELECT_CHECKPOINT_FOR_UPDATE_SQL = ( + _SELECT_CHECKPOINT_SQL + + "FOR UPDATE OF checkpoint\n" +) + +_SELECT_TRADE_FOR_ADOPTION_SQL = """ +SELECT + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + canonical_schema_version +FROM market_data.trades +WHERE venue = %s + AND symbol = %s + AND trade_id = %s + AND executed_at = %s +FOR SHARE +""" + +_INSERT_CHECKPOINT_SQL = """ +INSERT INTO market_data.trade_stream_checkpoints ( + venue, + symbol, + trade_id, + executed_at, + revision, + checkpoint_schema_version +) +VALUES (%s, %s, %s, %s, 1, %s) +ON CONFLICT (venue, symbol) DO NOTHING +RETURNING 1 +""" + +_UPDATE_CHECKPOINT_SQL = """ +UPDATE market_data.trade_stream_checkpoints +SET trade_id = %s, + executed_at = %s, + revision = revision + 1, + updated_at = NOW(), + checkpoint_schema_version = %s +WHERE venue = %s + AND symbol = %s + AND revision = %s +RETURNING 1 +""" + +_SELECT_LATEST_TRADE_SQL = """ +SELECT + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + canonical_schema_version +FROM market_data.trades +WHERE venue = %s + AND symbol = %s +ORDER BY first_observed_at DESC, + executed_at DESC, + trade_id DESC +LIMIT 1 +""" + +_SELECT_TRADE_TAIL_SQL = """ +WITH reference_trade AS ( + SELECT first_observed_at + FROM market_data.trades + WHERE venue = %s + AND symbol = %s + AND trade_id = %s + AND executed_at = %s +), +ranked_trade_cycles AS ( + SELECT + trade_row.symbol, + trade_row.trade_id, + trade_row.executed_at, + trade_row.price, + trade_row.quantity, + trade_row.aggressor_side, + trade_row.source, + trade_row.canonical_schema_version, + trade_row.first_observed_at, + ROW_NUMBER() OVER ( + PARTITION BY trade_row.trade_id + ORDER BY trade_row.first_observed_at DESC, + trade_row.executed_at DESC + ) AS cycle_rank + FROM market_data.trades AS trade_row + CROSS JOIN reference_trade + WHERE trade_row.venue = %s + AND trade_row.symbol = %s + AND trade_row.first_observed_at + <= reference_trade.first_observed_at +), +current_trade_cycle AS ( + SELECT + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + canonical_schema_version, + first_observed_at + FROM ranked_trade_cycles + WHERE cycle_rank = 1 +) +SELECT + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + canonical_schema_version +FROM current_trade_cycle +ORDER BY first_observed_at DESC, + executed_at DESC, + trade_id DESC +LIMIT %s +""" + @dataclass(frozen=True, slots=True) class _PreparedTrade: @@ -106,7 +270,7 @@ class _PreparedTrade: class PostgresTradeRepository: - """Транзакционный модуль записи канонических сделок в PostgreSQL.""" + """Единый транзакционный модуль Trades и persistent checkpoint.""" __slots__ = ("_connection_provider",) @@ -120,6 +284,284 @@ class PostgresTradeRepository: self._connection_provider = connection_provider + def load_checkpoint( + self, + *, + venue: str, + symbol: str, + ) -> PersistentTradeCheckpoint | None: + normalized_venue = normalize_venue(venue) + normalized_symbol = normalize_non_empty_text( + symbol, + field_name="symbol", + ).upper() + + try: + with self._connection_provider() as connection: + with connection.cursor() as cursor: + return self._load_checkpoint( + cursor=cursor, + venue=normalized_venue, + symbol=normalized_symbol, + for_update=False, + ) + except MarketDataStorageError: + raise + except Exception as error: + raise MarketDataStorageOperationError( + "Failed to load persistent Trade checkpoint." + ) from error + + def load_checkpoint_tail( + self, + *, + venue: str, + checkpoint: PersistentTradeCheckpoint, + limit: int, + ) -> tuple[Trade, ...]: + normalized_venue = normalize_venue(venue) + normalized_limit = self._validate_tail_limit(limit) + + if not isinstance(checkpoint, PersistentTradeCheckpoint): + raise MarketDataStorageValidationError( + "checkpoint must be PersistentTradeCheckpoint" + ) + + checkpoint_venue = normalize_venue(checkpoint.venue) + checkpoint_symbol = normalize_non_empty_text( + checkpoint.trade.symbol, + field_name="checkpoint.trade.symbol", + ).upper() + + if checkpoint_venue != normalized_venue: + raise MarketDataStorageValidationError( + "checkpoint venue does not match venue" + ) + + try: + with self._connection_provider() as connection: + with connection.cursor() as cursor: + trades = self._load_trade_tail( + cursor=cursor, + venue=normalized_venue, + symbol=checkpoint_symbol, + reference_trade_id=checkpoint.trade.trade_id, + reference_executed_at=normalize_aware_datetime( + checkpoint.trade.executed_at, + field_name="checkpoint.trade.executed_at", + ), + limit=normalized_limit, + ) + except MarketDataStorageError: + raise + except Exception as error: + raise MarketDataStorageOperationError( + "Failed to load persistent Trade checkpoint tail." + ) from error + + return self._validate_trade_tail( + trades=trades, + expected_last_identity=( + checkpoint_venue, + checkpoint_symbol, + checkpoint.trade.trade_id, + normalize_aware_datetime( + checkpoint.trade.executed_at, + field_name="checkpoint.trade.executed_at", + ), + ), + ) + + def load_latest_trade_tail( + self, + *, + venue: str, + symbol: str, + limit: int, + ) -> tuple[Trade, ...]: + normalized_venue = normalize_venue(venue) + normalized_symbol = normalize_non_empty_text( + symbol, + field_name="symbol", + ).upper() + normalized_limit = self._validate_tail_limit(limit) + + try: + with self._connection_provider() as connection: + with connection.cursor() as cursor: + cursor.execute( + _SELECT_LATEST_TRADE_SQL, + ( + normalized_venue, + normalized_symbol, + ), + ) + latest_row = cursor.fetchone() + + if latest_row is None: + return () + + latest_trade = self._trade_from_row(latest_row) + trades = self._load_trade_tail( + cursor=cursor, + venue=normalized_venue, + symbol=normalized_symbol, + reference_trade_id=latest_trade.trade_id, + reference_executed_at=latest_trade.executed_at, + limit=normalized_limit, + ) + except MarketDataStorageError: + raise + except Exception as error: + raise MarketDataStorageOperationError( + "Failed to load latest persistent Trade tail." + ) from error + + return self._validate_trade_tail( + trades=trades, + expected_last_identity=self._trade_identity( + venue=normalized_venue, + trade=latest_trade, + ), + ) + + def adopt_existing_trade_as_checkpoint( + self, + *, + venue: str, + trade: Trade, + ) -> PersistentTradeCheckpoint: + normalized_venue = normalize_venue(venue) + + if not isinstance(trade, Trade): + raise MarketDataStorageValidationError( + "trade must be a Canonical Trade" + ) + + prepared = self._prepare_trade( + trade=trade, + observed_at=trade.executed_at, + ) + candidate_identity = self._prepared_identity( + venue=normalized_venue, + trade=prepared, + ) + + try: + with self._connection_provider() as connection: + with connection.cursor() as cursor: + durable_trade = self._load_trade_for_adoption( + cursor=cursor, + venue=normalized_venue, + trade=prepared, + ) + + if durable_trade is None: + raise MarketDataCheckpointIntegrityError( + "Checkpoint candidate is absent from durable " + "Canonical Trade history." + ) + + if not self._prepared_trade_matches( + prepared=prepared, + stored=durable_trade, + ): + raise MarketDataCheckpointConflictError( + "Checkpoint candidate conflicts with durable " + "Canonical Trade facts." + ) + + cursor.execute( + _INSERT_CHECKPOINT_SQL, + ( + *candidate_identity, + CHECKPOINT_SCHEMA_VERSION, + ), + ) + inserted = cursor.fetchone() is not None + checkpoint = self._load_checkpoint( + cursor=cursor, + venue=normalized_venue, + symbol=prepared.symbol, + for_update=True, + ) + + if checkpoint is None: + raise MarketDataCheckpointIntegrityError( + "Adopted checkpoint disappeared after insert." + ) + + checkpoint_matches = ( + self._normalized_checkpoint_identity(checkpoint) + == candidate_identity + and self._prepared_trade_matches( + prepared=prepared, + stored=checkpoint.trade, + ) + and checkpoint.revision == 1 + ) + + if not checkpoint_matches: + if inserted: + raise MarketDataCheckpointIntegrityError( + "Adopted checkpoint does not match durable " + "candidate Trade." + ) + + raise MarketDataCheckpointConflictError( + "Persistent checkpoint already points to another " + "Trade or revision." + ) + except MarketDataStorageError: + raise + except Exception as error: + raise MarketDataStorageOperationError( + "Failed to adopt existing Trade as persistent checkpoint." + ) from error + + return checkpoint + + def store_trade_and_advance_checkpoint( + self, + *, + venue: str, + expected_trade: Trade | None, + trade: Trade, + observed_at: datetime, + ) -> PersistentTradeCheckpoint: + normalized_venue = normalize_venue(venue) + prepared = self._prepare_trade( + trade=trade, + observed_at=observed_at, + ) + expected_identity = self._prepare_expected_identity( + venue=normalized_venue, + expected_trade=expected_trade, + ) + + try: + with self._connection_provider() as connection: + with connection.cursor() as cursor: + self._store_prepared_trade( + cursor=cursor, + venue=normalized_venue, + trade=prepared, + ) + checkpoint = self._advance_checkpoint( + cursor=cursor, + venue=normalized_venue, + expected_identity=expected_identity, + trade=prepared, + ) + except MarketDataStorageError: + raise + except Exception as error: + raise MarketDataStorageOperationError( + "Failed to atomically store Trade and advance checkpoint." + ) from error + + return checkpoint + def store_trade( self, *, @@ -219,6 +661,461 @@ class PostgresTradeRepository: provenance_updated_count=provenance_updated_count, ) + def _advance_checkpoint( + self, + *, + cursor: Any, + venue: str, + expected_identity: tuple[str, str, int, datetime] | None, + trade: _PreparedTrade, + ) -> PersistentTradeCheckpoint: + candidate_identity = self._prepared_identity( + venue=venue, + trade=trade, + ) + + if expected_identity is None: + cursor.execute( + _INSERT_CHECKPOINT_SQL, + ( + *candidate_identity, + CHECKPOINT_SCHEMA_VERSION, + ), + ) + inserted = cursor.fetchone() is not None + current = self._load_checkpoint( + cursor=cursor, + venue=venue, + symbol=trade.symbol, + for_update=True, + ) + + if current is None: + raise MarketDataCheckpointIntegrityError( + "Checkpoint disappeared after first insert." + ) + + if inserted: + if ( + self._normalized_checkpoint_identity(current) + != candidate_identity + ): + raise MarketDataCheckpointIntegrityError( + "First checkpoint does not match candidate Trade." + ) + + return current + else: + current = self._load_checkpoint( + cursor=cursor, + venue=venue, + symbol=trade.symbol, + for_update=True, + ) + + if current is None: + raise MarketDataCheckpointConflictError( + "Expected persistent checkpoint does not exist." + ) + + current_identity = self._normalized_checkpoint_identity(current) + + if current_identity == candidate_identity: + return current + + if expected_identity is None or current_identity != expected_identity: + raise MarketDataCheckpointConflictError( + "Persistent checkpoint differs from expected Trade." + ) + + try: + candidate_is_newer = is_trade_id_newer( + trade.trade_id, + current.trade.trade_id, + ) + except (TypeError, ValueError) as error: + raise MarketDataCheckpointConflictError( + "Candidate Trade has ambiguous checkpoint ordering." + ) from error + + if not candidate_is_newer: + raise MarketDataCheckpointConflictError( + "Candidate Trade does not advance persistent checkpoint." + ) + + cursor.execute( + _UPDATE_CHECKPOINT_SQL, + ( + trade.trade_id, + trade.executed_at, + CHECKPOINT_SCHEMA_VERSION, + venue, + trade.symbol, + current.revision, + ), + ) + + if cursor.fetchone() is None: + raise MarketDataCheckpointConflictError( + "Persistent checkpoint changed during update." + ) + + updated = self._load_checkpoint( + cursor=cursor, + venue=venue, + symbol=trade.symbol, + for_update=True, + ) + + if ( + updated is None + or self._normalized_checkpoint_identity(updated) + != candidate_identity + or updated.revision != current.revision + 1 + ): + raise MarketDataCheckpointIntegrityError( + "Updated checkpoint does not match committed Trade." + ) + + return updated + + def _load_checkpoint( + self, + *, + cursor: Any, + venue: str, + symbol: str, + for_update: bool, + ) -> PersistentTradeCheckpoint | None: + cursor.execute( + ( + _SELECT_CHECKPOINT_FOR_UPDATE_SQL + if for_update + else _SELECT_CHECKPOINT_SQL + ), + ( + venue, + symbol, + ), + ) + row = cursor.fetchone() + + if row is None: + return None + + return self._checkpoint_from_row( + row=row, + expected_venue=venue, + expected_symbol=symbol, + ) + + def _load_trade_for_adoption( + self, + *, + cursor: Any, + venue: str, + trade: _PreparedTrade, + ) -> Trade | None: + cursor.execute( + _SELECT_TRADE_FOR_ADOPTION_SQL, + self._identity_parameters( + venue=venue, + trade=trade, + ), + ) + row = cursor.fetchone() + + if row is None: + return None + + return self._trade_from_row(row) + + def _load_trade_tail( + self, + *, + cursor: Any, + venue: str, + symbol: str, + reference_trade_id: int, + reference_executed_at: datetime, + limit: int, + ) -> tuple[Trade, ...]: + cursor.execute( + _SELECT_TRADE_TAIL_SQL, + ( + venue, + symbol, + reference_trade_id, + reference_executed_at, + venue, + symbol, + limit, + ), + ) + descending = tuple( + self._trade_from_row(row) + for row in cursor.fetchall() + ) + return tuple(reversed(descending)) + + def _checkpoint_from_row( + self, + *, + row: object, + expected_venue: str, + expected_symbol: str, + ) -> PersistentTradeCheckpoint: + if not isinstance(row, tuple) or len(row) != 16: + raise MarketDataCheckpointIntegrityError( + "PostgreSQL returned invalid checkpoint row." + ) + + ( + checkpoint_venue, + checkpoint_symbol, + checkpoint_trade_id, + checkpoint_executed_at, + revision, + updated_at, + checkpoint_schema_version, + trade_venue, + trade_symbol, + trade_id, + trade_executed_at, + price, + quantity, + aggressor_side, + source, + canonical_schema_version, + ) = row + + if ( + checkpoint_venue != expected_venue + or checkpoint_symbol != expected_symbol + or trade_venue != checkpoint_venue + or trade_symbol != checkpoint_symbol + or trade_id != checkpoint_trade_id + or trade_executed_at != checkpoint_executed_at + or checkpoint_schema_version != CHECKPOINT_SCHEMA_VERSION + ): + raise MarketDataCheckpointIntegrityError( + "Checkpoint is not confirmed by exact Canonical Trade." + ) + + trade = self._trade_from_row( + ( + trade_symbol, + trade_id, + trade_executed_at, + price, + quantity, + aggressor_side, + source, + canonical_schema_version, + ) + ) + + try: + return PersistentTradeCheckpoint( + venue=checkpoint_venue, + trade=trade, + revision=revision, + updated_at=updated_at, + checkpoint_schema_version=checkpoint_schema_version, + ) + except (TypeError, ValueError) as error: + raise MarketDataCheckpointIntegrityError( + "PostgreSQL returned invalid checkpoint values." + ) from error + + @staticmethod + def _trade_from_row(row: object) -> Trade: + if not isinstance(row, tuple) or len(row) != 8: + raise MarketDataCheckpointIntegrityError( + "PostgreSQL returned invalid Canonical Trade row." + ) + + ( + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + canonical_schema_version, + ) = row + + try: + normalized_symbol = normalize_non_empty_text( + symbol, + field_name="stored trade.symbol", + ).upper() + validate_signed_trade_id(trade_id) + normalized_executed_at = normalize_aware_datetime( + executed_at, + field_name="stored trade.executed_at", + ) + normalized_price = validate_decimal( + price, + field_name="stored trade.price", + ) + normalized_quantity = validate_decimal( + quantity, + field_name="stored trade.quantity", + ) + normalized_source = normalize_non_empty_text( + source, + field_name="stored trade.source", + ) + normalized_side = TradeAggressorSide(aggressor_side) + except (TypeError, ValueError, MarketDataStorageError) as error: + raise MarketDataCheckpointIntegrityError( + "PostgreSQL returned invalid Canonical Trade values." + ) from error + + if canonical_schema_version != CANONICAL_TRADE_SCHEMA_VERSION: + raise MarketDataCheckpointIntegrityError( + "Unsupported Canonical Trade schema version." + ) + + return Trade( + symbol=normalized_symbol, + trade_id=trade_id, + price=normalized_price, + quantity=normalized_quantity, + executed_at=normalized_executed_at, + aggressor_side=normalized_side, + source=normalized_source, + ) + + def _prepare_expected_identity( + self, + *, + venue: str, + expected_trade: Trade | None, + ) -> tuple[str, str, int, datetime] | None: + if expected_trade is None: + return None + + if not isinstance(expected_trade, Trade): + raise MarketDataStorageValidationError( + "expected_trade must be a Canonical Trade or None" + ) + + prepared = self._prepare_trade( + trade=expected_trade, + observed_at=expected_trade.executed_at, + ) + return self._prepared_identity( + venue=venue, + trade=prepared, + ) + + @staticmethod + def _prepared_identity( + *, + venue: str, + trade: _PreparedTrade, + ) -> tuple[str, str, int, datetime]: + return ( + venue, + trade.symbol, + trade.trade_id, + trade.executed_at, + ) + + @staticmethod + def _normalized_checkpoint_identity( + checkpoint: PersistentTradeCheckpoint, + ) -> tuple[str, str, int, datetime]: + return ( + checkpoint.venue.strip(), + checkpoint.trade.symbol.strip().upper(), + checkpoint.trade.trade_id, + checkpoint.trade.executed_at, + ) + + @staticmethod + def _trade_identity( + *, + venue: str, + trade: Trade, + ) -> tuple[str, str, int, datetime]: + return ( + venue, + trade.symbol, + trade.trade_id, + trade.executed_at, + ) + + @staticmethod + def _prepared_trade_matches( + *, + prepared: _PreparedTrade, + stored: Trade, + ) -> bool: + return ( + stored.symbol == prepared.symbol + and stored.trade_id == prepared.trade_id + and stored.executed_at == prepared.executed_at + and stored.price == prepared.price + and stored.quantity == prepared.quantity + and stored.aggressor_side.value == prepared.aggressor_side + and stored.source == prepared.source + ) + + @classmethod + def _validate_trade_tail( + cls, + *, + trades: tuple[Trade, ...], + expected_last_identity: tuple[str, str, int, datetime], + ) -> tuple[Trade, ...]: + if ( + not trades + or cls._trade_identity( + venue=expected_last_identity[0], + trade=trades[-1], + ) + != expected_last_identity + ): + raise MarketDataCheckpointIntegrityError( + "Trade tail does not end at expected durable Trade." + ) + + for previous, current in zip(trades, trades[1:], strict=False): + try: + current_is_newer = is_trade_id_newer( + current.trade_id, + previous.trade_id, + ) + except (TypeError, ValueError) as error: + raise MarketDataCheckpointIntegrityError( + "Trade tail contains ambiguous rollover ordering." + ) from error + + if not current_is_newer: + raise MarketDataCheckpointIntegrityError( + "Trade tail is not strictly rollover-ordered." + ) + + return trades + + @staticmethod + def _validate_tail_limit(limit: int) -> int: + if isinstance(limit, bool) or not isinstance(limit, int): + raise MarketDataStorageValidationError( + "limit must be an integer" + ) + + if limit <= 0: + raise MarketDataStorageValidationError( + "limit must be positive" + ) + + return limit + def _store_prepared_trade( self, *, diff --git a/app/src/market_data/storage/trade_storage_observation_sink.py b/app/src/market_data/storage/trade_storage_observation_sink.py index 1cd52a6..c64d3eb 100644 --- a/app/src/market_data/storage/trade_storage_observation_sink.py +++ b/app/src/market_data/storage/trade_storage_observation_sink.py @@ -2,10 +2,13 @@ from __future__ import annotations from collections.abc import Callable from datetime import datetime, timezone +from typing import cast from src.market_data.acquisition.models.trade import Trade from src.market_data.storage.contracts import ( MarketDataWriteResult, + PersistentTradeCheckpoint, + TradeCheckpointStorageProtocol, TradeStorageProtocol, ) from src.market_data.storage.postgres_repository_support import ( @@ -25,6 +28,7 @@ class TradeStorageObservationSink: """Адаптер наблюдений канонических сделок к постоянному хранилищу.""" __slots__ = ( + "_checkpoint_storage", "_trade_storage", "_venue", "_clock", @@ -42,14 +46,45 @@ class TradeStorageObservationSink: "trade_storage must implement TradeStorageProtocol" ) + if not isinstance(trade_storage, TradeCheckpointStorageProtocol): + raise TypeError( + "trade_storage must implement " + "TradeCheckpointStorageProtocol" + ) + if not callable(clock): raise TypeError("clock must be callable") self._trade_storage = trade_storage + self._checkpoint_storage = cast( + TradeCheckpointStorageProtocol, + trade_storage, + ) self._venue = normalize_venue(venue) self._clock = clock - def persist( + def persist_accepted( + self, + trade: Trade, + *, + expected_trade: Trade | None, + ) -> None: + checkpoint = ( + self._checkpoint_storage.store_trade_and_advance_checkpoint( + venue=self._venue, + expected_trade=expected_trade, + trade=trade, + observed_at=self._clock(), + ) + ) + + if not isinstance(checkpoint, PersistentTradeCheckpoint): + raise TypeError( + "trade_storage.store_trade_and_advance_checkpoint() " + "must return PersistentTradeCheckpoint" + ) + + def persist_duplicate( self, trade: Trade, ) -> None: diff --git a/app/src/storage/migrations.py b/app/src/storage/migrations.py index 51b51e3..94e4fac 100644 --- a/app/src/storage/migrations.py +++ b/app/src/storage/migrations.py @@ -292,6 +292,44 @@ STORAGE_MIGRATIONS = ( """, ), ), + StorageMigration( + version=8, + name="create_trade_stream_checkpoints", + statements=( + """ + CREATE TABLE market_data.trade_stream_checkpoints ( + venue TEXT NOT NULL, + symbol TEXT NOT NULL, + trade_id INTEGER NOT NULL, + executed_at TIMESTAMPTZ NOT NULL, + revision BIGINT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + checkpoint_schema_version INTEGER NOT NULL DEFAULT 1, + PRIMARY KEY (venue, symbol), + CONSTRAINT trade_stream_checkpoints_trade_fk + FOREIGN KEY ( + venue, + symbol, + trade_id, + executed_at + ) REFERENCES market_data.trades ( + venue, + symbol, + trade_id, + executed_at + ) + ON UPDATE NO ACTION + ON DELETE NO ACTION + DEFERRABLE INITIALLY DEFERRED, + CHECK (trade_id BETWEEN -2147483648 AND 2147483647), + CHECK (BTRIM(venue) <> ''), + CHECK (BTRIM(symbol) <> ''), + CHECK (revision > 0), + CHECK (checkpoint_schema_version > 0) + ) + """, + ), + ), ) diff --git a/app/tests/integration/market_data/acquisition/runtime/loopback_trade_exchange.py b/app/tests/integration/market_data/acquisition/runtime/loopback_trade_exchange.py index 8fa0a1d..b948a03 100644 --- a/app/tests/integration/market_data/acquisition/runtime/loopback_trade_exchange.py +++ b/app/tests/integration/market_data/acquisition/runtime/loopback_trade_exchange.py @@ -214,6 +214,26 @@ class LoopbackTradeWebSocketServer: ), ) + async def send_ack( + self, + connection_index: int, + *, + correlation_id: str, + status: str = "OK", + destination: str = "trades.subscribe", + ) -> None: + """Отправить управляемый ответ на Trade subscription.""" + await self.send_raw( + connection_index, + json.dumps( + { + "correlationId": correlation_id, + "destination": destination, + "status": status, + } + ), + ) + async def abort_connection( self, connection_index: int, diff --git a/app/tests/integration/market_data/acquisition/runtime/test_trade_stream_loopback_integration.py b/app/tests/integration/market_data/acquisition/runtime/test_trade_stream_loopback_integration.py index 432bc37..6d90c29 100644 --- a/app/tests/integration/market_data/acquisition/runtime/test_trade_stream_loopback_integration.py +++ b/app/tests/integration/market_data/acquisition/runtime/test_trade_stream_loopback_integration.py @@ -3,10 +3,29 @@ from __future__ import annotations import asyncio import threading import time +from datetime import datetime, timezone +from decimal import Decimal from typing import Any import pytest +from src.core.config import Settings +from src.integrations.exchange.rest_client import ExchangeRestClient +from src.market_data.acquisition.adapters.dzengi.rest import ( + DzengiTradesDocumentSource, +) +from src.market_data.acquisition.adapters.dzengi.websocket import ( + DzengiUnifiedWebSocketAdapter, +) +from src.market_data.acquisition.adapters.dzengi.websocket_control_message_handler import ( + DzengiWebSocketControlMessageHandler, +) +from src.market_data.acquisition.adapters.dzengi.websocket_inbound_message_classifier import ( + DzengiWebSocketInboundMessageClassifier, +) +from src.market_data.acquisition.adapters.dzengi.websocket_transport import ( + DzengiWebSocketTransport, +) from tests.integration.market_data.acquisition.runtime import ( loopback_trade_exchange, ) @@ -22,6 +41,13 @@ from src.market_data.acquisition.exceptions import ( TradeTransportError, WebSocketMessageDecodeError, ) +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.acquisition.runtime.acquisition_runtime_event_publisher import ( + AcquisitionRuntimeEventPublisher, +) from src.market_data.acquisition.runtime.runtime_reconnect_recovery_coordinator import ( RuntimeReconnectRecoveryCoordinator, ) @@ -29,10 +55,22 @@ from src.market_data.acquisition.runtime.trade_stream_production_runtime import TradeStreamProductionRuntime, TradeStreamProductionRuntimeState, ) +from src.market_data.acquisition.runtime.websocket_session import ( + WebSocketSession, +) +from src.market_data.acquisition.runtime.websocket_subscription_manager import ( + WebSocketSubscriptionManager, +) +from src.market_data.acquisition.trade_stream_runtime_composition import ( + TradeStreamRuntimeComposition, + build_trade_stream_runtime_composition, +) +from src.market_data.storage.contracts import PersistentTradeCheckpoint from tests.support.trade_stream_runtime import ( SYMBOL, assert_no_owned_tasks, build_runtime, + make_settings, reconnect_coordinator_from, run_scenario, start_runtime, @@ -44,6 +82,187 @@ from tests.support.trade_stream_runtime import ( pytestmark = pytest.mark.integration +class StaticCheckpointStorage: + """Локальное checkpoint-хранилище для startup boundary.""" + + def __init__( + self, + *, + checkpoint: PersistentTradeCheckpoint, + tail: tuple[Trade, ...], + events: list[str], + ) -> None: + self._checkpoint = checkpoint + self._tail = tail + self._events = events + + def load_checkpoint( + self, + *, + venue: str, + symbol: str, + ) -> PersistentTradeCheckpoint | None: + assert venue == "dzengi" + assert symbol == SYMBOL + self._events.append("checkpoint.load") + return self._checkpoint + + def load_checkpoint_tail( + self, + *, + venue: str, + checkpoint: PersistentTradeCheckpoint, + limit: int, + ) -> tuple[Trade, ...]: + assert venue == "dzengi" + assert checkpoint is self._checkpoint + assert limit > 0 + self._events.append("checkpoint.tail") + return self._tail + + def load_latest_trade_tail( + self, + *, + venue: str, + symbol: str, + limit: int, + ) -> tuple[Trade, ...]: + raise AssertionError("Checkpoint уже существует.") + + def adopt_existing_trade_as_checkpoint( + self, + *, + venue: str, + trade: Trade, + ) -> PersistentTradeCheckpoint: + raise AssertionError("Checkpoint уже существует.") + + def store_trade_and_advance_checkpoint( + self, + *, + venue: str, + expected_trade: Trade | None, + trade: Trade, + observed_at: datetime, + ) -> PersistentTradeCheckpoint: + raise AssertionError("Startup Hydrator не выполняет запись.") + + +class RecordingTradeObservationSink: + """Фиксирует точный порядок durable acceptance в тесте.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._accepted_trade_ids: list[int] = [] + + @property + def accepted_trade_ids(self) -> tuple[int, ...]: + with self._lock: + return tuple(self._accepted_trade_ids) + + def persist_accepted( + self, + trade: Trade, + *, + expected_trade: Trade | None, + ) -> None: + del expected_trade + + with self._lock: + self._accepted_trade_ids.append(trade.trade_id) + + def persist_duplicate(self, trade: Trade) -> None: + del trade + + +def make_checkpoint_trade(*, timestamp_ms: int) -> Trade: + return Trade( + symbol=SYMBOL, + trade_id=100, + price=Decimal("64555.55"), + quantity=Decimal("0.002"), + executed_at=datetime.fromtimestamp( + timestamp_ms / 1_000, + tz=timezone.utc, + ), + aggressor_side=TradeAggressorSide.BUY, + source="postgres_trade_history", + ) + + +def build_startup_recovery_runtime( + *, + settings: Settings, + checkpoint_storage: StaticCheckpointStorage, + observation_sink: RecordingTradeObservationSink, + recovery_end_time_ms: int, +) -> tuple[TradeStreamProductionRuntime, TradeStreamRuntimeComposition]: + """Собрать persistent startup-граф без использования Bootstrap.""" + transport = DzengiWebSocketTransport( + url=settings.trade_stream.websocket_url, + headers={ + "Origin": settings.exchange_base_url.rstrip("/"), + "Content-Type": "application/json", + }, + open_timeout=settings.trade_stream.open_timeout_seconds, + ping_interval=None, + ping_timeout=None, + probe_timeout=settings.trade_stream.probe_timeout_seconds, + close_timeout=settings.trade_stream.close_timeout_seconds, + ) + session = WebSocketSession(transport) + subscription_manager = WebSocketSubscriptionManager( + transport, + supports_unsubscribe=False, + ) + event_publisher = AcquisitionRuntimeEventPublisher() + composition = build_trade_stream_runtime_composition( + session=session, + transport=transport, + subscription_manager=subscription_manager, + event_publisher=event_publisher, + message_adapter=DzengiUnifiedWebSocketAdapter(), + recovery_document_source=DzengiTradesDocumentSource( + ExchangeRestClient(settings=settings), + ), + symbols=(SYMBOL,), + heartbeat_timeout_seconds=60.0, + scheduler_interval_seconds=60.0, + trade_observation_sink=observation_sink, + max_recovery_window_ms=3_599_999, + recovery_end_time_clock=lambda: recovery_end_time_ms, + checkpoint_storage=checkpoint_storage, + checkpoint_venue="dzengi", + ) + startup_recovery = ( + composition.runtime_startup_recovery_coordinator + ) + assert startup_recovery is not None + + runtime = TradeStreamProductionRuntime( + session=session, + transport=transport, + subscription_manager=subscription_manager, + event_publisher=event_publisher, + trade_stream_service=( + composition.trade_stream_acquisition_service + ), + message_classifier=DzengiWebSocketInboundMessageClassifier(), + control_message_handler=DzengiWebSocketControlMessageHandler(), + live_processing_gate=composition.live_processing_gate, + reconnect_recovery_coordinator=( + composition.runtime_reconnect_recovery_coordinator + ), + runtime_supervisor=composition.runtime_supervisor, + runtime_scheduler=composition.runtime_scheduler, + symbols=(SYMBOL,), + startup_recovery_coordinator=startup_recovery, + subscription_ack_timeout_seconds=1.0, + startup_market_buffer_capacity=10, + ) + return runtime, composition + + def make_recovered_trade( *, trade_id: int, @@ -277,6 +496,218 @@ def test_real_loopback_websocket_updates_shared_checkpoint() -> None: run_scenario(scenario()) +def test_startup_recovery_precedes_buffered_live_market() -> None: + async def scenario() -> None: + events: list[str] = [] + recovery_release = threading.Event() + base_time_ms = time.time_ns() // 1_000_000 - 10_000 + checkpoint_trade = make_checkpoint_trade( + timestamp_ms=base_time_ms, + ) + checkpoint = PersistentTradeCheckpoint( + venue="dzengi", + trade=checkpoint_trade, + revision=7, + updated_at=datetime.now(timezone.utc), + ) + checkpoint_storage = StaticCheckpointStorage( + checkpoint=checkpoint, + tail=(checkpoint_trade,), + events=events, + ) + observation_sink = RecordingTradeObservationSink() + websocket = LoopbackTradeWebSocketServer( + events=events, + auto_ack=False, + ) + rest = LoopbackTradeRestServer( + responses=( + LoopbackHttpResponse( + body=[ + make_recovered_trade( + trade_id=101, + timestamp_ms=base_time_ms + 100, + ), + ], + release=recovery_release, + ), + ), + events=events, + ) + + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + settings = make_settings( + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + ) + runtime, composition = build_startup_recovery_runtime( + settings=settings, + checkpoint_storage=checkpoint_storage, + observation_sink=observation_sink, + recovery_end_time_ms=base_time_ms + 150, + ) + runtime_task = asyncio.create_task( + runtime.run(), + name="trade-stream-runtime", + ) + + try: + await websocket.wait_for_subscriptions(1) + + assert runtime.state is ( + TradeStreamProductionRuntimeState.STARTING + ) + assert composition.state_store.get( + SYMBOL, + ).last_trade_id == 100 + assert events.index("checkpoint.tail") < events.index( + "ws.connect:0" + ) + + await websocket.send_trade( + 0, + symbol=SYMBOL, + trade_id=102, + timestamp_ms=base_time_ms + 200, + ) + subscription = websocket.subscriptions[0] + await websocket.send_ack( + 0, + correlation_id=subscription.correlation_id, + ) + await rest.wait_for_requests(1) + + assert runtime.state is ( + TradeStreamProductionRuntimeState.STARTING + ) + assert composition.state_store.get( + SYMBOL, + ).last_trade_id == 100 + assert observation_sink.accepted_trade_ids == () + + recovery_release.set() + await wait_until( + lambda: runtime.state + is TradeStreamProductionRuntimeState.RUNNING, + ) + await wait_until( + lambda: composition.state_store.get( + SYMBOL, + ).last_trade_id + == 102, + ) + + assert observation_sink.accepted_trade_ids == ( + 101, + 102, + ) + assert events.index("ws.subscribe:0") < events.index( + "rest.request" + ) + assert runtime_task.done() is False + finally: + recovery_release.set() + await stop_runtime(runtime, runtime_task) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + await assert_no_owned_tasks() + + run_scenario(scenario()) + + +def test_startup_stop_waits_for_blocked_recovery_worker() -> None: + async def scenario() -> None: + events: list[str] = [] + recovery_release = threading.Event() + base_time_ms = time.time_ns() // 1_000_000 - 10_000 + checkpoint_trade = make_checkpoint_trade( + timestamp_ms=base_time_ms, + ) + checkpoint = PersistentTradeCheckpoint( + venue="dzengi", + trade=checkpoint_trade, + revision=3, + updated_at=datetime.now(timezone.utc), + ) + checkpoint_storage = StaticCheckpointStorage( + checkpoint=checkpoint, + tail=(checkpoint_trade,), + events=events, + ) + observation_sink = RecordingTradeObservationSink() + websocket = LoopbackTradeWebSocketServer(events=events) + rest = LoopbackTradeRestServer( + responses=( + LoopbackHttpResponse( + body=[], + release=recovery_release, + ), + ), + events=events, + ) + + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + settings = make_settings( + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + ) + runtime, composition = build_startup_recovery_runtime( + settings=settings, + checkpoint_storage=checkpoint_storage, + observation_sink=observation_sink, + recovery_end_time_ms=base_time_ms + 150, + ) + runtime_task = asyncio.create_task( + runtime.run(), + name="trade-stream-runtime", + ) + stop_task: asyncio.Task[None] | None = None + + try: + await websocket.wait_for_subscriptions(1) + await rest.wait_for_requests(1) + + stop_task = asyncio.create_task(runtime.stop()) + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert stop_task.done() is False + assert runtime_task.done() is False + assert composition.live_processing_gate.locked is True + + recovery_release.set() + await asyncio.wait_for(stop_task, timeout=3.0) + await asyncio.wait_for(runtime_task, timeout=3.0) + + assert runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert composition.live_processing_gate.locked is False + assert composition.live_processing_gate.failed is False + assert observation_sink.accepted_trade_ids == () + finally: + recovery_release.set() + + if stop_task is not None and not stop_task.done(): + await stop_task + + if not runtime_task.done(): + await stop_runtime(runtime, runtime_task) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + await assert_no_owned_tasks() + + run_scenario(scenario()) + + def test_reconnect_restores_then_recovers_before_buffered_live() -> None: async def scenario() -> None: events: list[str] = [] diff --git a/app/tests/integration/market_data/storage/test_postgres_partition_retention_integration.py b/app/tests/integration/market_data/storage/test_postgres_partition_retention_integration.py index a4bf907..a6f2149 100644 --- a/app/tests/integration/market_data/storage/test_postgres_partition_retention_integration.py +++ b/app/tests/integration/market_data/storage/test_postgres_partition_retention_integration.py @@ -14,9 +14,11 @@ from src.market_data.acquisition.models.trade import ( ) from src.market_data.storage import ( MARKET_DATA_PARTITION_ADVISORY_LOCK_ID, + MarketDataPartitionResult, MarketDataPartitionType, MarketDataRetentionPolicy, MarketDataStorageOperationError, + PersistentTradeCheckpoint, PostgresMarketDataPartitionManager, PostgresMarketDataRetentionService, PostgresQuoteRepository, @@ -27,6 +29,7 @@ from tests.support.postgres_market_data import ( PostgresTestSettings, connect_postgres_test_database, wait_for_postgres_advisory_lock_waiters, + wait_for_postgres_relation_lock_waiters, ) @@ -34,6 +37,7 @@ pytestmark = pytest.mark.integration VENUE = "dzengi" SYMBOL = "BTC/USD_LEVERAGE" +PARTITION_WRITER_BARRIER_LOCK_ID = 0x445A5041525457 def _trade(*, trade_id: int, executed_at: datetime) -> Trade: @@ -60,6 +64,91 @@ def _quote(*, received_at: datetime) -> Quote: ) +def _insert_checkpoint( + pool: PostgresConnectionPool, + *, + trade: Trade, + revision: int = 1, +) -> None: + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + INSERT INTO market_data.trade_stream_checkpoints ( + venue, + symbol, + trade_id, + executed_at, + revision + ) + VALUES (%s, %s, %s, %s, %s) + """, + ( + VENUE, + trade.symbol, + trade.trade_id, + trade.executed_at, + revision, + ), + ) + + +def _advance_checkpoint( + pool: PostgresConnectionPool, + *, + trade: Trade, + revision: int, +) -> None: + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + UPDATE market_data.trade_stream_checkpoints + SET trade_id = %s, + executed_at = %s, + revision = %s, + updated_at = NOW() + WHERE venue = %s + AND symbol = %s + """, + ( + trade.trade_id, + trade.executed_at, + revision, + VENUE, + trade.symbol, + ), + ) + assert cursor.rowcount == 1 + + +def _assert_checkpoint_foreign_key_is_deferred_no_action( + pool: PostgresConnectionPool, +) -> None: + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT constraint_row.confupdtype::text, + constraint_row.confdeltype::text, + constraint_row.condeferrable, + constraint_row.condeferred + FROM pg_catalog.pg_constraint AS constraint_row + JOIN pg_catalog.pg_class AS table_row + ON table_row.oid = constraint_row.conrelid + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = table_row.relnamespace + WHERE namespace.nspname = 'market_data' + AND table_row.relname = 'trade_stream_checkpoints' + AND constraint_row.conname = + 'trade_stream_checkpoints_trade_fk' + """ + ) + row = cursor.fetchone() + + assert row == ("a", "a", True, True) + + def test_real_partition_creation_moves_default_row_and_is_idempotent( migrated_postgres_pool: PostgresConnectionPool, ) -> None: @@ -99,6 +188,56 @@ def test_real_partition_creation_moves_default_row_and_is_idempotent( assert relation == ("market_data.trades_2026_07",) +def test_checkpoint_trade_moves_from_default_to_month_partition( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + event_time = datetime(2026, 7, 15, tzinfo=timezone.utc) + trade = _trade(trade_id=1, executed_at=event_time) + repository = PostgresTradeRepository( + connection_provider=migrated_postgres_pool.connection, + ) + manager = PostgresMarketDataPartitionManager( + connection_provider=migrated_postgres_pool.connection, + ) + repository.store_trade( + venue=VENUE, + trade=trade, + observed_at=event_time, + ) + _insert_checkpoint( + migrated_postgres_pool, + trade=trade, + ) + + result = manager.ensure_month_partition( + data_type=MarketDataPartitionType.TRADES, + month=event_time, + ) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT trade_row.tableoid::regclass::text, + checkpoint.revision + FROM market_data.trade_stream_checkpoints AS checkpoint + JOIN market_data.trades AS trade_row + ON trade_row.venue = checkpoint.venue + AND trade_row.symbol = checkpoint.symbol + AND trade_row.trade_id = checkpoint.trade_id + AND trade_row.executed_at = checkpoint.executed_at + """ + ) + restored = cursor.fetchone() + + assert result.created is True + assert result.moved_row_count == 1 + assert restored == ("market_data.trades_2026_07", 1) + _assert_checkpoint_foreign_key_is_deferred_no_action( + migrated_postgres_pool + ) + + def test_two_real_partition_callers_create_one_partition( migrated_postgres_pool: PostgresConnectionPool, postgres_test_settings: PostgresTestSettings, @@ -150,6 +289,164 @@ def test_two_real_partition_callers_create_one_partition( assert sorted(results) == [(False, 0), (True, 0)] +def test_partition_creation_and_checkpoint_writer_use_one_lock_order( + migrated_postgres_pool: PostgresConnectionPool, + postgres_test_settings: PostgresTestSettings, +) -> None: + existing_month = datetime(2026, 7, 1, tzinfo=timezone.utc) + new_month = datetime(2026, 8, 1, tzinfo=timezone.utc) + previous_trade = _trade( + trade_id=1, + executed_at=datetime(2026, 7, 14, tzinfo=timezone.utc), + ) + candidate_trade = _trade( + trade_id=2, + executed_at=datetime(2026, 7, 15, tzinfo=timezone.utc), + ) + repository = PostgresTradeRepository( + connection_provider=migrated_postgres_pool.connection, + ) + manager = PostgresMarketDataPartitionManager( + connection_provider=migrated_postgres_pool.connection, + ) + manager.ensure_month_partition( + data_type=MarketDataPartitionType.TRADES, + month=existing_month, + ) + repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=previous_trade, + observed_at=previous_trade.executed_at, + ) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + f""" + CREATE FUNCTION market_data.block_partition_writer() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + PERFORM pg_advisory_xact_lock( + {PARTITION_WRITER_BARRIER_LOCK_ID} + ); + RETURN NEW; + END; + $$ + """ + ) + cursor.execute( + """ + CREATE TRIGGER block_partition_writer + BEFORE INSERT ON market_data.trades_2026_07 + FOR EACH ROW + EXECUTE FUNCTION market_data.block_partition_writer() + """ + ) + + caller_ids: set[int] = set() + caller_ids_lock = threading.Lock() + + def store_and_advance_checkpoint() -> PersistentTradeCheckpoint: + with caller_ids_lock: + caller_ids.add(threading.get_ident()) + return repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=previous_trade, + trade=candidate_trade, + observed_at=candidate_trade.executed_at, + ) + + def create_partition() -> MarketDataPartitionResult: + with caller_ids_lock: + caller_ids.add(threading.get_ident()) + return manager.ensure_month_partition( + data_type=MarketDataPartitionType.TRADES, + month=new_month, + ) + + with connect_postgres_test_database( + postgres_test_settings, + ) as control: + with control.cursor() as cursor: + cursor.execute( + "SELECT pg_advisory_lock(%s)", + (PARTITION_WRITER_BARRIER_LOCK_ID,), + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + try: + writer_future = executor.submit( + store_and_advance_checkpoint + ) + wait_for_postgres_advisory_lock_waiters( + control, + lock_id=PARTITION_WRITER_BARRIER_LOCK_ID, + expected_count=1, + ) + manager_future = executor.submit(create_partition) + wait_for_postgres_relation_lock_waiters( + control, + relation_name="market_data.trades", + expected_count=1, + ) + + finally: + with control.cursor() as cursor: + cursor.execute( + "SELECT pg_advisory_unlock(%s)", + (PARTITION_WRITER_BARRIER_LOCK_ID,), + ) + barrier_released = cursor.fetchone() + + if barrier_released != (True,): + raise AssertionError("Writer barrier was not released.") + + try: + checkpoint = writer_future.result(timeout=10.0) + partition_result = manager_future.result(timeout=10.0) + finally: + writer_future.cancel() + manager_future.cancel() + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT trade_row.tableoid::regclass::text, + checkpoint.trade_id, + checkpoint.revision + FROM market_data.trades AS trade_row + JOIN market_data.trade_stream_checkpoints AS checkpoint + ON checkpoint.venue = trade_row.venue + AND checkpoint.symbol = trade_row.symbol + AND checkpoint.trade_id = trade_row.trade_id + AND checkpoint.executed_at = trade_row.executed_at + WHERE trade_row.venue = %s + AND trade_row.symbol = %s + AND trade_row.trade_id = %s + """, + ( + VENUE, + SYMBOL, + candidate_trade.trade_id, + ), + ) + persisted = cursor.fetchone() + + assert len(caller_ids) == 2 + assert checkpoint.trade == candidate_trade + assert checkpoint.revision == 2 + assert partition_result.created is True + assert partition_result.moved_row_count == 0 + assert persisted == ("market_data.trades_2026_07", 2, 2) + _assert_checkpoint_foreign_key_is_deferred_no_action( + migrated_postgres_pool + ) + + def test_real_retention_uses_exact_cutoff( migrated_postgres_pool: PostgresConnectionPool, ) -> None: @@ -189,6 +486,184 @@ def test_real_retention_uses_exact_cutoff( assert remaining == (2, 3) +def test_retention_rolls_back_for_active_checkpoint_in_default( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + now = datetime(2026, 8, 15, 12, 0, tzinfo=timezone.utc) + old_time = now - timedelta(days=30) + trade = _trade(trade_id=1, executed_at=old_time) + repository = PostgresTradeRepository( + connection_provider=migrated_postgres_pool.connection, + ) + service = PostgresMarketDataRetentionService( + connection_provider=migrated_postgres_pool.connection, + ) + repository.store_trade( + venue=VENUE, + trade=trade, + observed_at=now, + ) + _insert_checkpoint( + migrated_postgres_pool, + trade=trade, + ) + + with pytest.raises(MarketDataStorageOperationError): + service.apply( + policy=MarketDataRetentionPolicy( + enabled=True, + trade_days=10, + ), + now=now, + ) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT COUNT(*) FROM market_data.trades") + trade_count = cursor.fetchone() + cursor.execute( + "SELECT COUNT(*) " + "FROM market_data.trade_stream_checkpoints" + ) + checkpoint_count = cursor.fetchone() + + assert trade_count == (1,) + assert checkpoint_count == (1,) + _assert_checkpoint_foreign_key_is_deferred_no_action( + migrated_postgres_pool + ) + + +def test_retention_rolls_back_for_active_checkpoint_in_partition( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + now = datetime(2026, 8, 15, 12, 0, tzinfo=timezone.utc) + old_time = datetime(2026, 6, 15, tzinfo=timezone.utc) + trade = _trade(trade_id=1, executed_at=old_time) + repository = PostgresTradeRepository( + connection_provider=migrated_postgres_pool.connection, + ) + manager = PostgresMarketDataPartitionManager( + connection_provider=migrated_postgres_pool.connection, + ) + service = PostgresMarketDataRetentionService( + connection_provider=migrated_postgres_pool.connection, + ) + manager.ensure_month_partition( + data_type=MarketDataPartitionType.TRADES, + month=old_time, + ) + repository.store_trade( + venue=VENUE, + trade=trade, + observed_at=now, + ) + _insert_checkpoint( + migrated_postgres_pool, + trade=trade, + ) + + with pytest.raises(MarketDataStorageOperationError): + service.apply( + policy=MarketDataRetentionPolicy( + enabled=True, + trade_days=10, + ), + now=now, + ) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT COUNT(*) FROM market_data.trades") + trade_count = cursor.fetchone() + cursor.execute( + """ + SELECT COUNT(*) + FROM market_data.partition_registry + WHERE partition_name = 'trades_2026_06' + """ + ) + registry_count = cursor.fetchone() + + assert trade_count == (1,) + assert registry_count == (1,) + + +def test_retention_drops_old_partition_after_checkpoint_advances( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + now = datetime(2026, 8, 15, 12, 0, tzinfo=timezone.utc) + old_time = datetime(2026, 1, 15, tzinfo=timezone.utc) + current_time = datetime(2026, 7, 15, tzinfo=timezone.utc) + old_trade = _trade(trade_id=1, executed_at=old_time) + current_trade = _trade(trade_id=2, executed_at=current_time) + repository = PostgresTradeRepository( + connection_provider=migrated_postgres_pool.connection, + ) + manager = PostgresMarketDataPartitionManager( + connection_provider=migrated_postgres_pool.connection, + ) + service = PostgresMarketDataRetentionService( + connection_provider=migrated_postgres_pool.connection, + ) + manager.ensure_month_partition( + data_type=MarketDataPartitionType.TRADES, + month=old_time, + ) + repository.store_trade( + venue=VENUE, + trade=old_trade, + observed_at=now, + ) + repository.store_trade( + venue=VENUE, + trade=current_trade, + observed_at=now, + ) + _insert_checkpoint( + migrated_postgres_pool, + trade=old_trade, + ) + _advance_checkpoint( + migrated_postgres_pool, + trade=current_trade, + revision=2, + ) + + result = service.apply( + policy=MarketDataRetentionPolicy( + enabled=True, + trade_days=90, + ), + now=now, + ) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + "SELECT trade_id FROM market_data.trades ORDER BY trade_id" + ) + remaining_trade_ids = tuple( + row[0] for row in cursor.fetchall() + ) + cursor.execute( + """ + SELECT trade_id, revision + FROM market_data.trade_stream_checkpoints + """ + ) + checkpoint_row = cursor.fetchone() + + trade_result = result.entries[0] + assert trade_result.dropped_partitions == ("trades_2026_01",) + assert trade_result.dropped_row_count == 1 + assert remaining_trade_ids == (2,) + assert checkpoint_row == (2, 2) + _assert_checkpoint_foreign_key_is_deferred_no_action( + migrated_postgres_pool + ) + + def test_real_retention_failure_rolls_back_all_data_types( migrated_postgres_pool: PostgresConnectionPool, ) -> None: diff --git a/app/tests/integration/market_data/storage/test_postgres_schema_integration.py b/app/tests/integration/market_data/storage/test_postgres_schema_integration.py index 1232b2f..37779fe 100644 --- a/app/tests/integration/market_data/storage/test_postgres_schema_integration.py +++ b/app/tests/integration/market_data/storage/test_postgres_schema_integration.py @@ -1,9 +1,11 @@ from __future__ import annotations from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone import threading import pytest +from psycopg.errors import ForeignKeyViolation from src.storage.exceptions import StorageMigrationError from src.storage.migrations import ( @@ -104,6 +106,51 @@ def test_real_migrations_create_expected_schema_and_are_idempotent( partitioned_tables = tuple( row[0] for row in cursor.fetchall() ) + cursor.execute( + """ + SELECT column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_schema = 'market_data' + AND table_name = 'trade_stream_checkpoints' + ORDER BY ordinal_position + """ + ) + checkpoint_columns = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT pg_get_constraintdef(constraint_row.oid) + FROM pg_catalog.pg_constraint AS constraint_row + JOIN pg_catalog.pg_class AS table_row + ON table_row.oid = constraint_row.conrelid + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = table_row.relnamespace + WHERE namespace.nspname = 'market_data' + AND table_row.relname = 'trade_stream_checkpoints' + ORDER BY constraint_row.contype, + constraint_row.conname + """ + ) + checkpoint_constraints = tuple( + row[0] for row in cursor.fetchall() + ) + cursor.execute( + """ + SELECT constraint_row.confupdtype::text, + constraint_row.confdeltype::text, + constraint_row.condeferrable, + constraint_row.condeferred + FROM pg_catalog.pg_constraint AS constraint_row + JOIN pg_catalog.pg_class AS table_row + ON table_row.oid = constraint_row.conrelid + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = table_row.relnamespace + WHERE namespace.nspname = 'market_data' + AND table_row.relname = 'trade_stream_checkpoints' + AND constraint_row.conname = + 'trade_stream_checkpoints_trade_fk' + """ + ) + checkpoint_foreign_key = cursor.fetchone() finally: pool.close() @@ -121,11 +168,140 @@ def test_real_migrations_create_expected_schema_and_are_idempotent( "quotes", "trades", ) + assert checkpoint_columns == ( + ("venue", "text", "NO"), + ("symbol", "text", "NO"), + ("trade_id", "integer", "NO"), + ("executed_at", "timestamp with time zone", "NO"), + ("revision", "bigint", "NO"), + ("updated_at", "timestamp with time zone", "NO"), + ("checkpoint_schema_version", "integer", "NO"), + ) + assert any( + definition + == "PRIMARY KEY (venue, symbol)" + for definition in checkpoint_constraints + ) + assert any( + definition.startswith( + "FOREIGN KEY (venue, symbol, trade_id, executed_at) " + "REFERENCES market_data.trades" + ) + and "DEFERRABLE INITIALLY DEFERRED" in definition + for definition in checkpoint_constraints + ) + assert checkpoint_foreign_key == ("a", "a", True, True) with connect_postgres_test_database(postgres_test_settings) as control: assert count_other_test_connections(control) == 0 +def test_checkpoint_foreign_key_rejects_orphan_and_trade_deletion( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + executed_at = datetime(2026, 8, 1, 10, 0, tzinfo=timezone.utc) + + with pytest.raises(ForeignKeyViolation): + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + INSERT INTO market_data.trade_stream_checkpoints ( + venue, + symbol, + trade_id, + executed_at, + revision + ) + VALUES (%s, %s, %s, %s, %s) + """, + ( + "DZENGI", + "BTC/USD_LEVERAGE", + 123, + executed_at, + 1, + ), + ) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + INSERT INTO market_data.trades ( + venue, + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + observation_sources, + canonical_schema_version + ) + VALUES ( + %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s + ) + """, + ( + "DZENGI", + "BTC/USD_LEVERAGE", + 123, + executed_at, + "65000.25", + "0.001", + "buy", + "dzengi_websocket_trade", + executed_at, + executed_at, + ["dzengi_websocket_trade"], + 1, + ), + ) + cursor.execute( + """ + INSERT INTO market_data.trade_stream_checkpoints ( + venue, + symbol, + trade_id, + executed_at, + revision + ) + VALUES (%s, %s, %s, %s, %s) + """, + ( + "DZENGI", + "BTC/USD_LEVERAGE", + 123, + executed_at, + 1, + ), + ) + + with pytest.raises(ForeignKeyViolation): + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + DELETE FROM market_data.trades + WHERE venue = %s + AND symbol = %s + AND trade_id = %s + AND executed_at = %s + """, + ( + "DZENGI", + "BTC/USD_LEVERAGE", + 123, + executed_at, + ), + ) + + def test_two_real_migration_runners_apply_each_version_once( postgres_test_settings: PostgresTestSettings, ) -> None: diff --git a/app/tests/integration/market_data/storage/test_postgres_trade_checkpoint_repository_integration.py b/app/tests/integration/market_data/storage/test_postgres_trade_checkpoint_repository_integration.py new file mode 100644 index 0000000..34f04c0 --- /dev/null +++ b/app/tests/integration/market_data/storage/test_postgres_trade_checkpoint_repository_integration.py @@ -0,0 +1,707 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from decimal import Decimal +import threading + +import pytest + +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.acquisition.trade_id_sequence import ( + SIGNED_TRADE_ID_MAX, + SIGNED_TRADE_ID_MIN, +) +from src.market_data.storage import ( + MarketDataCheckpointConflictError, + MarketDataCheckpointIntegrityError, + MarketDataStorageOperationError, + PostgresTradeRepository, +) +from src.storage.postgres_pool import PostgresConnectionPool + + +pytestmark = pytest.mark.integration + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +EVENT_TIME = datetime(2026, 8, 1, 10, 0, tzinfo=timezone.utc) +OBSERVED_AT = EVENT_TIME + timedelta(seconds=1) + + +def _trade( + *, + trade_id: int, + offset_ms: int = 0, + source: str = "dzengi_websocket_trade", +) -> Trade: + return Trade( + symbol=SYMBOL, + trade_id=trade_id, + price=Decimal("65000.25"), + quantity=Decimal("0.001"), + executed_at=EVENT_TIME + timedelta(milliseconds=offset_ms), + aggressor_side=TradeAggressorSide.BUY, + source=source, + ) + + +def _repository( + pool: PostgresConnectionPool, +) -> PostgresTradeRepository: + return PostgresTradeRepository( + connection_provider=pool.connection, + ) + + +def _stored_trade_metadata( + pool: PostgresConnectionPool, + *, + trade: Trade, +) -> tuple[object, ...] | None: + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT + source, + first_observed_at, + last_observed_at, + observation_sources + FROM market_data.trades + WHERE venue = %s + AND symbol = %s + AND trade_id = %s + AND executed_at = %s + """, + ( + VENUE, + trade.symbol, + trade.trade_id, + trade.executed_at, + ), + ) + return cursor.fetchone() + + +def test_real_checkpoint_commit_load_and_rollover_tail( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + first = _trade(trade_id=SIGNED_TRADE_ID_MAX) + second = _trade( + trade_id=SIGNED_TRADE_ID_MIN, + offset_ms=1, + ) + third = _trade( + trade_id=SIGNED_TRADE_ID_MIN + 1, + offset_ms=2, + ) + + first_checkpoint = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=first, + observed_at=OBSERVED_AT, + ) + second_checkpoint = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=first, + trade=second, + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + third_checkpoint = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=second, + trade=third, + observed_at=OBSERVED_AT + timedelta(seconds=2), + ) + + loaded = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + tail = repository.load_checkpoint_tail( + venue=VENUE, + checkpoint=third_checkpoint, + limit=3, + ) + + assert first_checkpoint.revision == 1 + assert second_checkpoint.revision == 2 + assert third_checkpoint.revision == 3 + assert loaded == third_checkpoint + assert tail == (first, second, third) + + +def test_real_latest_tail_orders_minus_one_to_zero_without_checkpoint( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + first = _trade(trade_id=-1) + second = _trade(trade_id=0, offset_ms=1) + repository.store_trade( + venue=VENUE, + trade=first, + observed_at=OBSERVED_AT, + ) + repository.store_trade( + venue=VENUE, + trade=second, + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + + tail = repository.load_latest_trade_tail( + venue=VENUE, + symbol=SYMBOL, + limit=2, + ) + + assert tail == (first, second) + + +def test_real_latest_tail_uses_acceptance_order_not_exchange_time( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + first = _trade( + trade_id=SIGNED_TRADE_ID_MAX, + offset_ms=2, + ) + second = _trade( + trade_id=SIGNED_TRADE_ID_MIN, + offset_ms=1, + ) + repository.store_trade( + venue=VENUE, + trade=first, + observed_at=OBSERVED_AT, + ) + repository.store_trade( + venue=VENUE, + trade=second, + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + + tail = repository.load_latest_trade_tail( + venue=VENUE, + symbol=SYMBOL, + limit=2, + ) + + assert first.executed_at > second.executed_at + assert tail == (first, second) + + +def test_real_checkpoint_tail_uses_only_latest_raw_id_cycle( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + previous_cycle = _trade(trade_id=100) + current_predecessor = _trade(trade_id=99, offset_ms=1) + current = _trade(trade_id=100, offset_ms=2) + repository.store_trade( + venue=VENUE, + trade=previous_cycle, + observed_at=OBSERVED_AT, + ) + repository.store_trade( + venue=VENUE, + trade=current_predecessor, + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + checkpoint = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=current, + observed_at=OBSERVED_AT + timedelta(seconds=2), + ) + + tail = repository.load_checkpoint_tail( + venue=VENUE, + checkpoint=checkpoint, + limit=2, + ) + + assert previous_cycle.trade_id == current.trade_id + assert previous_cycle.executed_at != current.executed_at + assert tail == (current_predecessor, current) + + +def test_real_adoption_preserves_trade_metadata_and_is_idempotent( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + trade = _trade(trade_id=100) + repository.store_trade( + venue=VENUE, + trade=trade, + observed_at=OBSERVED_AT, + ) + before = _stored_trade_metadata( + migrated_postgres_pool, + trade=trade, + ) + + adopted = repository.adopt_existing_trade_as_checkpoint( + venue=VENUE, + trade=trade, + ) + repeated = repository.adopt_existing_trade_as_checkpoint( + venue=VENUE, + trade=trade, + ) + after = _stored_trade_metadata( + migrated_postgres_pool, + trade=trade, + ) + + assert adopted.revision == 1 + assert repeated == adopted + assert before == after + + +def test_real_adoption_rejects_missing_and_conflicting_durable_trade( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + missing = _trade(trade_id=100) + + with pytest.raises(MarketDataCheckpointIntegrityError): + repository.adopt_existing_trade_as_checkpoint( + venue=VENUE, + trade=missing, + ) + + durable = _trade(trade_id=101, offset_ms=1) + repository.store_trade( + venue=VENUE, + trade=durable, + observed_at=OBSERVED_AT, + ) + + with pytest.raises(MarketDataCheckpointConflictError): + repository.adopt_existing_trade_as_checkpoint( + venue=VENUE, + trade=replace(durable, price=Decimal("65000.26")), + ) + + assert repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) is None + + +def test_two_real_identical_adoption_callers_converge( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + trade = _trade(trade_id=100) + repository.store_trade( + venue=VENUE, + trade=trade, + observed_at=OBSERVED_AT, + ) + before = _stored_trade_metadata( + migrated_postgres_pool, + trade=trade, + ) + start_barrier = threading.Barrier(2) + caller_ids: set[int] = set() + caller_ids_lock = threading.Lock() + + def adopt() -> int: + with caller_ids_lock: + caller_ids.add(threading.get_ident()) + start_barrier.wait(timeout=5.0) + checkpoint = repository.adopt_existing_trade_as_checkpoint( + venue=VENUE, + trade=trade, + ) + return checkpoint.revision + + with ThreadPoolExecutor(max_workers=2) as executor: + revisions = tuple(executor.map(lambda _: adopt(), range(2))) + + after = _stored_trade_metadata( + migrated_postgres_pool, + trade=trade, + ) + + assert len(caller_ids) == 2 + assert revisions == (1, 1) + assert before == after + + +def test_two_real_different_adoption_callers_allow_one_winner( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + candidates = ( + _trade(trade_id=100), + _trade(trade_id=101, offset_ms=1), + ) + + for index, candidate in enumerate(candidates): + repository.store_trade( + venue=VENUE, + trade=candidate, + observed_at=OBSERVED_AT + timedelta(seconds=index), + ) + + before = tuple( + _stored_trade_metadata( + migrated_postgres_pool, + trade=candidate, + ) + for candidate in candidates + ) + start_barrier = threading.Barrier(2) + caller_ids: set[int] = set() + caller_ids_lock = threading.Lock() + + def adopt(candidate: Trade) -> tuple[str, int]: + with caller_ids_lock: + caller_ids.add(threading.get_ident()) + start_barrier.wait(timeout=5.0) + + try: + checkpoint = repository.adopt_existing_trade_as_checkpoint( + venue=VENUE, + trade=candidate, + ) + except MarketDataCheckpointConflictError: + return ("conflict", candidate.trade_id) + + return ("committed", checkpoint.trade.trade_id) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = tuple(executor.map(adopt, candidates)) + + loaded = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + after = tuple( + _stored_trade_metadata( + migrated_postgres_pool, + trade=candidate, + ) + for candidate in candidates + ) + committed_ids = tuple( + trade_id + for status, trade_id in results + if status == "committed" + ) + + assert len(caller_ids) == 2 + assert tuple(status for status, _ in results).count("committed") == 1 + assert tuple(status for status, _ in results).count("conflict") == 1 + assert loaded is not None + assert loaded.trade.trade_id == committed_ids[0] + assert loaded.revision == 1 + assert before == after + + +def test_real_stale_writer_rolls_back_candidate_trade( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + first = _trade(trade_id=100) + winner = _trade(trade_id=101, offset_ms=1) + stale_candidate = _trade(trade_id=102, offset_ms=2) + repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=first, + observed_at=OBSERVED_AT, + ) + winner_checkpoint = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=first, + trade=winner, + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + + with pytest.raises(MarketDataCheckpointConflictError): + repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=first, + trade=stale_candidate, + observed_at=OBSERVED_AT + timedelta(seconds=2), + ) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + "SELECT trade_id FROM market_data.trades ORDER BY trade_id" + ) + trade_ids = tuple(row[0] for row in cursor.fetchall()) + + loaded = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + assert trade_ids == (100, 101) + assert loaded == winner_checkpoint + + +def test_real_retry_after_commit_preserves_revision_and_updates_provenance( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + first = _trade(trade_id=100) + current = _trade(trade_id=101, offset_ms=1) + repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=first, + observed_at=OBSERVED_AT, + ) + committed = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=first, + trade=current, + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + + repeated = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=first, + trade=replace(current, source="dzengi"), + observed_at=OBSERVED_AT + timedelta(seconds=2), + ) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT observation_sources + FROM market_data.trades + WHERE trade_id = 101 + """ + ) + sources = cursor.fetchone() + + assert repeated == committed + assert repeated.revision == 2 + assert sources is not None + assert set(sources[0]) == {"dzengi_websocket_trade", "dzengi"} + + +def test_two_real_first_writers_leave_only_winner_trade( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + candidates = ( + _trade(trade_id=100), + _trade(trade_id=101, offset_ms=1), + ) + start_barrier = threading.Barrier(2) + caller_ids: set[int] = set() + caller_ids_lock = threading.Lock() + + def commit(trade: Trade) -> tuple[str, int]: + with caller_ids_lock: + caller_ids.add(threading.get_ident()) + start_barrier.wait(timeout=5.0) + + try: + checkpoint = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=trade, + observed_at=OBSERVED_AT, + ) + except MarketDataCheckpointConflictError: + return ("conflict", trade.trade_id) + + return ("committed", checkpoint.trade.trade_id) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = tuple(executor.map(commit, candidates)) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT trade_id FROM market_data.trades") + stored_trade_ids = tuple(row[0] for row in cursor.fetchall()) + + loaded = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + committed_ids = tuple( + trade_id + for status, trade_id in results + if status == "committed" + ) + assert len(caller_ids) == 2 + assert tuple(status for status, _ in results).count("committed") == 1 + assert tuple(status for status, _ in results).count("conflict") == 1 + assert stored_trade_ids == committed_ids + assert loaded is not None + assert loaded.trade.trade_id == committed_ids[0] + + +def test_two_real_identical_first_writers_converge_idempotently( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + candidates = ( + _trade(trade_id=100), + _trade( + trade_id=100, + source="dzengi", + ), + ) + start_barrier = threading.Barrier(2) + caller_ids: set[int] = set() + caller_ids_lock = threading.Lock() + + def commit(trade: Trade) -> int: + with caller_ids_lock: + caller_ids.add(threading.get_ident()) + start_barrier.wait(timeout=5.0) + checkpoint = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=trade, + observed_at=OBSERVED_AT, + ) + return checkpoint.revision + + with ThreadPoolExecutor(max_workers=2) as executor: + revisions = tuple(executor.map(commit, candidates)) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + "SELECT observation_sources FROM market_data.trades" + ) + rows = tuple(cursor.fetchall()) + + assert len(caller_ids) == 2 + assert revisions == (1, 1) + assert len(rows) == 1 + assert set(rows[0][0]) == {"dzengi_websocket_trade", "dzengi"} + + +def test_two_real_checkpoint_updates_allow_only_one_winner( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + previous = _trade(trade_id=100) + candidates = ( + _trade(trade_id=101, offset_ms=1), + _trade(trade_id=102, offset_ms=2), + ) + repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=previous, + observed_at=OBSERVED_AT, + ) + start_barrier = threading.Barrier(2) + caller_ids: set[int] = set() + caller_ids_lock = threading.Lock() + + def commit(trade: Trade) -> tuple[str, int]: + with caller_ids_lock: + caller_ids.add(threading.get_ident()) + start_barrier.wait(timeout=5.0) + + try: + checkpoint = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=previous, + trade=trade, + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + except MarketDataCheckpointConflictError: + return ("conflict", trade.trade_id) + + return ("committed", checkpoint.trade.trade_id) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = tuple(executor.map(commit, candidates)) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + "SELECT trade_id FROM market_data.trades ORDER BY trade_id" + ) + stored_trade_ids = tuple(row[0] for row in cursor.fetchall()) + + loaded = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + committed_ids = tuple( + trade_id + for status, trade_id in results + if status == "committed" + ) + assert len(caller_ids) == 2 + assert tuple(status for status, _ in results).count("committed") == 1 + assert tuple(status for status, _ in results).count("conflict") == 1 + assert stored_trade_ids == (100, committed_ids[0]) + assert loaded is not None + assert loaded.trade.trade_id == committed_ids[0] + assert loaded.revision == 2 + + +def test_real_checkpoint_failure_rolls_back_inserted_trade( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + trade = _trade(trade_id=100) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + CREATE FUNCTION market_data.reject_checkpoint_write() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + RAISE EXCEPTION 'checkpoint write rejected'; + END; + $$ + """ + ) + cursor.execute( + """ + CREATE TRIGGER reject_checkpoint_write + BEFORE INSERT OR UPDATE + ON market_data.trade_stream_checkpoints + FOR EACH ROW + EXECUTE FUNCTION market_data.reject_checkpoint_write() + """ + ) + + with pytest.raises(MarketDataStorageOperationError): + repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=trade, + observed_at=OBSERVED_AT, + ) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT COUNT(*) FROM market_data.trades") + trade_count = cursor.fetchone() + cursor.execute( + "SELECT COUNT(*) " + "FROM market_data.trade_stream_checkpoints" + ) + checkpoint_count = cursor.fetchone() + + assert trade_count == (0,) + assert checkpoint_count == (0,) diff --git a/app/tests/integration/market_data/storage/test_trade_checkpoint_consistency_integration.py b/app/tests/integration/market_data/storage/test_trade_checkpoint_consistency_integration.py new file mode 100644 index 0000000..c72183f --- /dev/null +++ b/app/tests/integration/market_data/storage/test_trade_checkpoint_consistency_integration.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +import pytest + +from src.market_data.acquisition.consistency.trade_stream_consistency_controller import ( + TradeStreamConsistencyController, +) +from src.market_data.acquisition.consistency.trade_stream_state_store import ( + TradeStreamStateStore, +) +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.storage import ( + MarketDataCheckpointConflictError, + PostgresTradeRepository, + TradeStorageObservationSink, +) +from src.storage.postgres_pool import PostgresConnectionPool + + +pytestmark = pytest.mark.integration + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +EXECUTED_AT = datetime(2026, 8, 1, 12, 0, tzinfo=timezone.utc) +OBSERVED_AT = EXECUTED_AT + timedelta(seconds=1) + + +def _trade( + *, + trade_id: int, + offset_ms: int = 0, + source: str = "dzengi_websocket_trade", +) -> Trade: + return Trade( + symbol=SYMBOL, + trade_id=trade_id, + price=Decimal("65000.25"), + quantity=Decimal("0.001"), + executed_at=EXECUTED_AT + timedelta(milliseconds=offset_ms), + aggressor_side=TradeAggressorSide.BUY, + source=source, + ) + + +def _repository( + pool: PostgresConnectionPool, +) -> PostgresTradeRepository: + return PostgresTradeRepository( + connection_provider=pool.connection, + ) + + +def _controller( + pool: PostgresConnectionPool, +) -> tuple[ + TradeStreamConsistencyController, + TradeStreamStateStore, +]: + repository = _repository(pool) + state_store = TradeStreamStateStore() + sink = TradeStorageObservationSink( + trade_storage=repository, + venue=VENUE, + clock=lambda: OBSERVED_AT, + ) + return ( + TradeStreamConsistencyController( + state_store=state_store, + trade_observation_sink=sink, + ), + state_store, + ) + + +def _stored_trade_ids( + pool: PostgresConnectionPool, +) -> tuple[int, ...]: + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT trade_id + FROM market_data.trades + WHERE venue = %s AND symbol = %s + ORDER BY trade_id + """, + (VENUE, SYMBOL), + ) + return tuple(row[0] for row in cursor.fetchall()) + + +def test_real_consistency_atomically_advances_durable_checkpoint( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + controller, state_store = _controller(migrated_postgres_pool) + repository = _repository(migrated_postgres_pool) + first = _trade(trade_id=100) + second = _trade(trade_id=101, offset_ms=1) + + controller.accept(first) + result = controller.accept(second) + + checkpoint = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + state = state_store.get(SYMBOL) + + assert result is second + assert checkpoint is not None + assert checkpoint.trade == second + assert checkpoint.revision == 2 + assert state.last_trade is second + assert state.last_trade_id == second.trade_id + assert _stored_trade_ids(migrated_postgres_pool) == (100, 101) + + +def test_real_duplicate_updates_provenance_without_checkpoint_advance( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + controller, state_store = _controller(migrated_postgres_pool) + repository = _repository(migrated_postgres_pool) + original = _trade(trade_id=200) + duplicate = _trade( + trade_id=200, + source="dzengi", + ) + controller.accept(original) + before_duplicate = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + + result = controller.accept(duplicate) + + after_duplicate = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + state = state_store.get(SYMBOL) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT observation_sources + FROM market_data.trades + WHERE venue = %s + AND symbol = %s + AND trade_id = %s + AND executed_at = %s + """, + ( + VENUE, + SYMBOL, + original.trade_id, + original.executed_at, + ), + ) + provenance = cursor.fetchone() + + assert result is None + assert before_duplicate is not None + assert after_duplicate is not None + assert after_duplicate == before_duplicate + assert after_duplicate.revision == 1 + assert provenance == ( + ["dzengi_websocket_trade", "dzengi"], + ) + assert state.last_trade is original + assert state.last_trade_id == original.trade_id + + +def test_real_checkpoint_conflict_rolls_back_trade_and_in_memory_state( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + controller, state_store = _controller(migrated_postgres_pool) + repository = _repository(migrated_postgres_pool) + first = _trade(trade_id=300) + stale_candidate = _trade(trade_id=301, offset_ms=1) + winner = _trade(trade_id=302, offset_ms=2) + controller.accept(first) + winner_checkpoint = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=first, + trade=winner, + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + + with pytest.raises(MarketDataCheckpointConflictError): + controller.accept(stale_candidate) + + checkpoint = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + state = state_store.get(SYMBOL) + + assert checkpoint == winner_checkpoint + assert _stored_trade_ids(migrated_postgres_pool) == (300, 302) + assert state.last_trade is first + assert state.last_trade_id == first.trade_id + assert stale_candidate.trade_id not in state._trades diff --git a/app/tests/integration/market_data/storage/test_trade_runtime_persistence_integration.py b/app/tests/integration/market_data/storage/test_trade_runtime_persistence_integration.py index 794eb20..ab971e7 100644 --- a/app/tests/integration/market_data/storage/test_trade_runtime_persistence_integration.py +++ b/app/tests/integration/market_data/storage/test_trade_runtime_persistence_integration.py @@ -391,6 +391,7 @@ def test_application_owns_real_storage_before_and_after_runtime( runtime = build_trade_stream_production_runtime( settings, trade_observation_sink=storage.trade_observation_sink, + checkpoint_storage=storage.trade_repository, ) assert runtime is not None dispatcher = ControlledDispatcher() diff --git a/app/tests/integration/market_data/storage/test_trade_stream_persistent_cancellation_integration.py b/app/tests/integration/market_data/storage/test_trade_stream_persistent_cancellation_integration.py new file mode 100644 index 0000000..48fc12d --- /dev/null +++ b/app/tests/integration/market_data/storage/test_trade_stream_persistent_cancellation_integration.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import asyncio +import time +from datetime import datetime, timezone +from typing import Any + +import pytest + +from src.bootstrap.market_data_storage import build_market_data_storage +from src.market_data.acquisition.runtime.trade_stream_production_runtime import ( + TradeStreamProductionRuntimeState, +) +from tests.integration.market_data.acquisition.runtime.loopback_trade_exchange import ( + LoopbackTradeEnvironment, + LoopbackTradeRestServer, + LoopbackTradeWebSocketServer, +) +from tests.integration.market_data.storage.test_trade_stream_persistent_restart_integration import ( + APPLICATION_TIMEOUT_SECONDS, + VENUE, + PersistentApplication, + _assert_no_pool_connections, + _build_application, + _canonical_trade, + _cleanup_application, + _settings_for_database, +) +from tests.support.postgres_market_data import ( + PostgresTestSettings, + connect_postgres_test_database, + wait_for_postgres_relation_lock_waiters, +) +from tests.support.trade_stream_runtime import ( + assert_no_owned_tasks, + run_scenario, +) + + +pytestmark = pytest.mark.integration + +CHECKPOINT_RELATION_NAME = "market_data.trade_stream_checkpoints" + + +def test_cancellation_waits_for_blocked_hydration_before_pool_close( + postgres_test_settings: PostgresTestSettings, +) -> None: + async def scenario() -> None: + base_time_ms = time.time_ns() // 1_000_000 - 10_000 + websocket = LoopbackTradeWebSocketServer(auto_ack=False) + rest = LoopbackTradeRestServer() + application: PersistentApplication | None = None + seed_storage = None + blocker = None + + try: + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + settings = _settings_for_database( + postgres=postgres_test_settings, + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + ) + seed_storage = build_market_data_storage(settings) + + assert seed_storage is not None + + await asyncio.to_thread(seed_storage.lifecycle.start) + await asyncio.to_thread( + seed_storage.trade_repository + .store_trade_and_advance_checkpoint, + venue=VENUE, + expected_trade=None, + trade=_canonical_trade( + trade_id=500, + timestamp_ms=base_time_ms, + ), + observed_at=datetime.now(timezone.utc), + ) + await asyncio.to_thread(seed_storage.lifecycle.stop) + + assert seed_storage.connection_pool.is_open is False + await asyncio.to_thread( + _assert_no_pool_connections, + postgres_test_settings, + ) + + blocker = connect_postgres_test_database( + postgres_test_settings, + autocommit=False, + ) + + with blocker.cursor() as cursor: + cursor.execute( + "LOCK TABLE " + "market_data.trade_stream_checkpoints " + "IN ACCESS EXCLUSIVE MODE" + ) + + application = _build_application( + settings, + task_name="persistent-application-cancellation", + ) + await application.dispatcher.started.wait() + await asyncio.to_thread( + wait_for_postgres_relation_lock_waiters, + blocker, + relation_name=CHECKPOINT_RELATION_NAME, + expected_count=1, + ) + + assert application.runtime.state is ( + TradeStreamProductionRuntimeState.STARTING + ) + assert application.storage.connection_pool.is_open is True + assert websocket.connection_count == 0 + assert rest.request_count == 0 + + application.task.cancel() + await asyncio.wait_for( + application.dispatcher.cancelled.wait(), + timeout=APPLICATION_TIMEOUT_SECONDS, + ) + + assert application.task.done() is False + assert application.task.cancelling() == 1 + assert application.storage.connection_pool.is_open is True + + application.task.cancel() + await asyncio.sleep(0) + + assert application.task.done() is False + assert application.task.cancelling() == 2 + assert application.storage.connection_pool.is_open is True + + await asyncio.to_thread(blocker.rollback) + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for( + asyncio.shield(application.task), + timeout=APPLICATION_TIMEOUT_SECONDS, + ) + + runtime_graph: Any = application.runtime + + assert application.task.cancelled() is True + assert application.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert runtime_graph._live_processing_gate.failed is False + assert runtime_graph._live_processing_gate.locked is False + assert application.dispatcher.cancelled.is_set() + assert application.storage.connection_pool.is_open is False + assert application.storage.lifecycle.started is False + assert application.bot_session.close_calls == 1 + finally: + if blocker is not None: + try: + if not blocker.closed: + await asyncio.to_thread(blocker.rollback) + finally: + await asyncio.to_thread(blocker.close) + + await _cleanup_application(application) + + if ( + seed_storage is not None + and seed_storage.connection_pool.is_open + ): + await asyncio.to_thread(seed_storage.lifecycle.stop) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + await asyncio.to_thread( + _assert_no_pool_connections, + postgres_test_settings, + ) + await assert_no_owned_tasks() + + run_scenario( + scenario(), + timeout_seconds=60.0, + ) diff --git a/app/tests/integration/market_data/storage/test_trade_stream_persistent_failure_integration.py b/app/tests/integration/market_data/storage/test_trade_stream_persistent_failure_integration.py new file mode 100644 index 0000000..7d0a77f --- /dev/null +++ b/app/tests/integration/market_data/storage/test_trade_stream_persistent_failure_integration.py @@ -0,0 +1,357 @@ +from __future__ import annotations + +import asyncio +import threading +import time +from datetime import datetime, timezone +from typing import Any + +import pytest + +from src.bootstrap.market_data_storage import ( + MarketDataStorageBootstrapComposition, + build_market_data_storage, +) +from src.core.config import Settings +from src.integrations.exchange.exceptions import ExchangeResponseError +from src.market_data.acquisition.exceptions import TradeTransportError +from src.market_data.acquisition.runtime.trade_stream_production_runtime import ( + TradeStreamProductionRuntimeState, +) +from src.market_data.storage import MarketDataCheckpointIntegrityError +from tests.integration.market_data.acquisition.runtime.loopback_trade_exchange import ( + LoopbackHttpResponse, + LoopbackTradeEnvironment, + LoopbackTradeRestServer, + LoopbackTradeWebSocketServer, +) +from tests.integration.market_data.storage import ( + test_trade_stream_persistent_restart_integration as restart_harness, +) +from tests.support.postgres_market_data import ( + PostgresTestSettings, + connect_postgres_test_database, +) +from tests.support.trade_stream_runtime import ( + SYMBOL, + assert_no_owned_tasks, + run_scenario, + state_store_from, +) + + +pytestmark = pytest.mark.integration + +VENUE = "dzengi" +CHECKPOINT_FOREIGN_KEY = "trade_stream_checkpoints_trade_fk" + + +def _prepare_storage( + settings: Settings, +) -> MarketDataStorageBootstrapComposition: + storage = build_market_data_storage(settings) + + assert storage is not None + storage.lifecycle.start() + return storage + + +def _stop_storage( + storage: MarketDataStorageBootstrapComposition | None, +) -> None: + if storage is not None and storage.connection_pool.is_open: + storage.lifecycle.stop() + + +def _insert_orphan_checkpoint( + postgres: PostgresTestSettings, + *, + trade_id: int, + executed_at: datetime, +) -> None: + """Создать повреждённую точку только в одноразовой тестовой БД.""" + with connect_postgres_test_database(postgres) as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT COUNT(*) + FROM pg_catalog.pg_constraint + WHERE conname = %s + AND conrelid = ( + 'market_data.trade_stream_checkpoints'::regclass + ) + """, + (CHECKPOINT_FOREIGN_KEY,), + ) + assert cursor.fetchone() == (1,) + cursor.execute( + """ + ALTER TABLE market_data.trade_stream_checkpoints + DROP CONSTRAINT trade_stream_checkpoints_trade_fk + """ + ) + cursor.execute( + """ + INSERT INTO market_data.trade_stream_checkpoints ( + venue, + symbol, + trade_id, + executed_at, + revision, + checkpoint_schema_version + ) + VALUES (%s, %s, %s, %s, 1, 1) + """, + ( + VENUE, + SYMBOL, + trade_id, + executed_at, + ), + ) + + +def _seed_checkpoint( + settings: Settings, + *, + trade_id: int, + timestamp_ms: int, +) -> None: + storage: MarketDataStorageBootstrapComposition | None = None + + try: + storage = _prepare_storage(settings) + trade = restart_harness._canonical_trade( + trade_id=trade_id, + timestamp_ms=timestamp_ms, + ) + storage.trade_repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=trade, + observed_at=datetime.now(timezone.utc), + ) + finally: + _stop_storage(storage) + + +def test_orphan_checkpoint_fails_before_network_io( + postgres_test_settings: PostgresTestSettings, +) -> None: + async def scenario() -> None: + base_time_ms = time.time_ns() // 1_000_000 - 10_000 + websocket = LoopbackTradeWebSocketServer(auto_ack=False) + rest = LoopbackTradeRestServer() + application: restart_harness.PersistentApplication | None = None + setup_storage: MarketDataStorageBootstrapComposition | None = None + + try: + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + settings = restart_harness._settings_for_database( + postgres=postgres_test_settings, + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + ) + setup_storage = await asyncio.to_thread( + _prepare_storage, + settings, + ) + await asyncio.to_thread( + setup_storage.lifecycle.stop, + ) + await asyncio.to_thread( + _insert_orphan_checkpoint, + postgres_test_settings, + trade_id=300, + executed_at=datetime.fromtimestamp( + base_time_ms / 1_000, + tz=timezone.utc, + ), + ) + application = restart_harness._build_application( + settings, + task_name="persistent-application-orphan-checkpoint", + ) + + with pytest.raises( + MarketDataCheckpointIntegrityError, + ) as captured: + await asyncio.wait_for( + asyncio.shield(application.task), + timeout=restart_harness.APPLICATION_TIMEOUT_SECONDS, + ) + + assert type(captured.value) is ( + MarketDataCheckpointIntegrityError + ) + runtime_graph: Any = application.runtime + gate = runtime_graph._live_processing_gate + + assert application.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + assert gate.failed is True + assert gate.locked is False + assert gate._failure is captured.value + assert runtime_graph._session.is_connected is False + assert ( + runtime_graph._subscription_manager.subscription_keys + == () + ) + assert application.dispatcher.cancelled.is_set() + assert state_store_from(application.runtime).is_empty() is True + assert websocket.connection_count == 0 + assert websocket.subscriptions == () + assert rest.request_count == 0 + assert application.storage.connection_pool.is_open is False + assert application.storage.lifecycle.started is False + assert application.bot_session.close_calls == 1 + await asyncio.to_thread( + restart_harness._assert_no_pool_connections, + postgres_test_settings, + ) + finally: + await restart_harness._cleanup_application(application) + await asyncio.to_thread(_stop_storage, setup_storage) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + await assert_no_owned_tasks() + + run_scenario( + scenario(), + timeout_seconds=60.0, + ) + + +def test_rest_failure_rejects_buffered_live_and_preserves_checkpoint( + postgres_test_settings: PostgresTestSettings, +) -> None: + async def scenario() -> None: + recovery_release = threading.Event() + base_time_ms = time.time_ns() // 1_000_000 - 10_000 + websocket = LoopbackTradeWebSocketServer(auto_ack=False) + rest = LoopbackTradeRestServer( + responses=( + LoopbackHttpResponse( + body={"error": "recovery unavailable"}, + status=500, + release=recovery_release, + ), + ) + ) + application: restart_harness.PersistentApplication | None = None + + try: + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + settings = restart_harness._settings_for_database( + postgres=postgres_test_settings, + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + ) + await asyncio.to_thread( + _seed_checkpoint, + settings, + trade_id=400, + timestamp_ms=base_time_ms, + ) + seeded_snapshot = await asyncio.to_thread( + restart_harness._snapshot_database, + postgres_test_settings, + ) + + assert tuple(row[0] for row in seeded_snapshot.trades) == ( + 400, + ) + assert seeded_snapshot.checkpoint == (400, 1, 1) + + application = restart_harness._build_application( + settings, + task_name="persistent-application-rest-failure", + ) + await application.dispatcher.started.wait() + await websocket.wait_for_subscriptions(1) + state_store = state_store_from(application.runtime) + + assert state_store.get(SYMBOL).last_trade_id == 400 + + await websocket.send_trade( + 0, + symbol=SYMBOL, + trade_id=402, + timestamp_ms=base_time_ms + 200, + ) + subscription = websocket.subscriptions[0] + await websocket.send_ack( + 0, + correlation_id=subscription.correlation_id, + ) + await rest.wait_for_requests(1) + runtime_graph: Any = application.runtime + gate = runtime_graph._live_processing_gate + + assert gate.locked is True + assert gate.failed is False + assert application.runtime.state is ( + TradeStreamProductionRuntimeState.STARTING + ) + assert state_store.get(SYMBOL).last_trade_id == 400 + assert await asyncio.to_thread( + restart_harness._snapshot_database, + postgres_test_settings, + ) == seeded_snapshot + + recovery_release.set() + + with pytest.raises(TradeTransportError) as captured: + await asyncio.wait_for( + asyncio.shield(application.task), + timeout=restart_harness.APPLICATION_TIMEOUT_SECONDS, + ) + + assert type(captured.value) is TradeTransportError + assert type(captured.value.__cause__) is ExchangeResponseError + failed_snapshot = await asyncio.to_thread( + restart_harness._snapshot_database, + postgres_test_settings, + ) + + assert failed_snapshot == seeded_snapshot + assert state_store.get(SYMBOL).last_trade_id == 400 + assert application.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + assert gate.failed is True + assert gate.locked is False + assert gate._failure is captured.value + assert runtime_graph._session.is_connected is False + assert ( + runtime_graph._subscription_manager.subscription_keys + == () + ) + assert application.dispatcher.cancelled.is_set() + assert application.storage.connection_pool.is_open is False + assert application.storage.lifecycle.started is False + assert application.bot_session.close_calls == 1 + await asyncio.to_thread( + restart_harness._assert_no_pool_connections, + postgres_test_settings, + ) + finally: + recovery_release.set() + await restart_harness._cleanup_application(application) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + await assert_no_owned_tasks() + + run_scenario( + scenario(), + timeout_seconds=60.0, + ) diff --git a/app/tests/integration/market_data/storage/test_trade_stream_persistent_restart_integration.py b/app/tests/integration/market_data/storage/test_trade_stream_persistent_restart_integration.py new file mode 100644 index 0000000..57be1e8 --- /dev/null +++ b/app/tests/integration/market_data/storage/test_trade_stream_persistent_restart_integration.py @@ -0,0 +1,1082 @@ +from __future__ import annotations + +import asyncio +import threading +import time +from collections.abc import Callable +from dataclasses import dataclass, replace +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from types import SimpleNamespace +from typing import Any, cast + +import pytest +from aiogram import Bot, Dispatcher +from psycopg.conninfo import conninfo_to_dict + +from src.bootstrap.application import ( + ApplicationComposition, + run_application, +) +from src.bootstrap.market_data_storage import ( + MarketDataStorageBootstrapComposition, + build_market_data_storage, +) +from src.bootstrap.trade_stream_runtime import ( + build_trade_stream_production_runtime, +) +from src.core.config import MarketDataStorageSettings, Settings +from src.market_data.acquisition.consistency.trade_observation_sink_protocol import ( + TradeObservationSinkProtocol, +) +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.acquisition.runtime.trade_stream_production_runtime import ( + TradeStreamProductionRuntime, + TradeStreamProductionRuntimeState, +) +from src.market_data.storage.contracts import ( + PersistentTradeCheckpoint, + TradeCheckpointStorageProtocol, +) +from tests.integration.market_data.acquisition.runtime.loopback_trade_exchange import ( + LoopbackHttpResponse, + LoopbackTradeEnvironment, + LoopbackTradeRestServer, + LoopbackTradeWebSocketServer, + wait_until, +) +from tests.support.postgres_market_data import ( + PostgresTestSettings, + connect_postgres_test_database, + count_other_test_connections, +) +from tests.support.trade_stream_runtime import ( + SYMBOL, + assert_no_owned_tasks, + make_settings, + run_scenario, + state_store_from, +) + + +pytestmark = pytest.mark.integration + +VENUE = "dzengi" +APPLICATION_TIMEOUT_SECONDS = 10.0 + + +class FakeBotSession: + def __init__(self) -> None: + self.close_calls = 0 + + async def close(self) -> None: + self.close_calls += 1 + + +class ControlledDispatcher: + def __init__(self) -> None: + self.started = asyncio.Event() + self.release = asyncio.Event() + self.cancelled = asyncio.Event() + + async def start_polling( + self, + bot: object, + *, + close_bot_session: bool, + ) -> None: + del bot + assert close_bot_session is False + self.started.set() + + try: + await self.release.wait() + except asyncio.CancelledError: + self.cancelled.set() + raise + + +@dataclass(slots=True) +class PersistentApplication: + storage: MarketDataStorageBootstrapComposition + runtime: TradeStreamProductionRuntime + dispatcher: ControlledDispatcher + bot_session: FakeBotSession + task: asyncio.Task[None] + + +@dataclass(frozen=True, slots=True) +class DatabaseSnapshot: + trades: tuple[tuple[Any, ...], ...] + checkpoint: tuple[Any, ...] | None + + +class BlockingCheckpointStorage: + """Один раз остановить выбранную checkpoint-операцию в тесте.""" + + __slots__ = ( + "_block_count", + "_blocked_operation", + "_delegate", + "_lock", + "entered", + "release", + ) + + def __init__( + self, + *, + delegate: TradeCheckpointStorageProtocol, + blocked_operation: str, + ) -> None: + if not isinstance(delegate, TradeCheckpointStorageProtocol): + raise TypeError( + "delegate must implement TradeCheckpointStorageProtocol" + ) + + if blocked_operation not in { + "load_checkpoint", + "adopt_existing_trade_as_checkpoint", + }: + raise ValueError("unsupported checkpoint operation barrier") + + self._delegate = delegate + self._blocked_operation = blocked_operation + self._block_count = 0 + self._lock = threading.Lock() + self.entered = threading.Event() + self.release = threading.Event() + + @property + def block_count(self) -> int: + with self._lock: + return self._block_count + + def load_checkpoint( + self, + *, + venue: str, + symbol: str, + ) -> PersistentTradeCheckpoint | None: + self._block_once("load_checkpoint") + return self._delegate.load_checkpoint( + venue=venue, + symbol=symbol, + ) + + def load_checkpoint_tail( + self, + *, + venue: str, + checkpoint: PersistentTradeCheckpoint, + limit: int, + ) -> tuple[Trade, ...]: + return self._delegate.load_checkpoint_tail( + venue=venue, + checkpoint=checkpoint, + limit=limit, + ) + + def load_latest_trade_tail( + self, + *, + venue: str, + symbol: str, + limit: int, + ) -> tuple[Trade, ...]: + return self._delegate.load_latest_trade_tail( + venue=venue, + symbol=symbol, + limit=limit, + ) + + def adopt_existing_trade_as_checkpoint( + self, + *, + venue: str, + trade: Trade, + ) -> PersistentTradeCheckpoint: + self._block_once("adopt_existing_trade_as_checkpoint") + return self._delegate.adopt_existing_trade_as_checkpoint( + venue=venue, + trade=trade, + ) + + def store_trade_and_advance_checkpoint( + self, + *, + venue: str, + expected_trade: Trade | None, + trade: Trade, + observed_at: datetime, + ) -> PersistentTradeCheckpoint: + return self._delegate.store_trade_and_advance_checkpoint( + venue=venue, + expected_trade=expected_trade, + trade=trade, + observed_at=observed_at, + ) + + def _block_once( + self, + operation: str, + ) -> None: + if operation != self._blocked_operation: + return + + with self._lock: + if self._block_count > 0: + return + + self._block_count = 1 + + self.entered.set() + + if not self.release.wait(timeout=APPLICATION_TIMEOUT_SECONDS): + raise TimeoutError( + "Истекло время ожидания освобождения checkpoint barrier." + ) + + +def _settings_for_database( + *, + postgres: PostgresTestSettings, + websocket_url: str, + rest_base_url: str, +) -> Settings: + parameters = conninfo_to_dict(postgres.dsn) + host_value = parameters.get("host") or parameters.get("hostaddr") + + if host_value is None: + raise AssertionError("PostgreSQL integration DSN has no host") + + port_value = parameters.get("port") + base_settings = make_settings( + websocket_url=websocket_url, + rest_base_url=rest_base_url, + ) + + return replace( + base_settings, + trade_stream=replace( + base_settings.trade_stream, + subscription_ack_timeout_seconds=2.0, + startup_market_buffer_capacity=10, + ), + db_host=str(host_value), + db_port=int(str(port_value if port_value is not None else 5432)), + db_name=str(parameters["dbname"]), + db_user=str(parameters.get("user", "")), + db_password=str(parameters.get("password", "")), + market_data_storage=MarketDataStorageSettings( + enabled=True, + pool_min_size=1, + pool_max_size=4, + pool_timeout_seconds=5.0, + ), + ) + + +def _build_application( + settings: Settings, + *, + task_name: str, + trade_observation_sink_factory: Callable[ + [MarketDataStorageBootstrapComposition], + TradeObservationSinkProtocol, + ] + | None = None, + checkpoint_storage_factory: Callable[ + [MarketDataStorageBootstrapComposition], + TradeCheckpointStorageProtocol, + ] + | None = None, +) -> PersistentApplication: + storage = build_market_data_storage(settings) + + assert storage is not None + + trade_observation_sink = ( + storage.trade_observation_sink + if trade_observation_sink_factory is None + else trade_observation_sink_factory(storage) + ) + checkpoint_storage = ( + storage.trade_repository + if checkpoint_storage_factory is None + else checkpoint_storage_factory(storage) + ) + + runtime = build_trade_stream_production_runtime( + settings, + trade_observation_sink=trade_observation_sink, + checkpoint_storage=checkpoint_storage, + ) + + assert isinstance(runtime, TradeStreamProductionRuntime) + + dispatcher = ControlledDispatcher() + bot_session = FakeBotSession() + bot = SimpleNamespace(session=bot_session) + task = asyncio.create_task( + run_application( + ApplicationComposition( + bot=cast(Bot, cast(object, bot)), + dispatcher=cast( + Dispatcher, + cast(object, dispatcher), + ), + trade_stream_runtime=runtime, + market_data_storage_lifecycle=storage.lifecycle, + ) + ), + name=task_name, + ) + return PersistentApplication( + storage=storage, + runtime=runtime, + dispatcher=dispatcher, + bot_session=bot_session, + task=task, + ) + + +async def _stop_application( + application: PersistentApplication, +) -> None: + if not application.task.done(): + application.dispatcher.release.set() + + await asyncio.wait_for( + asyncio.shield(application.task), + timeout=APPLICATION_TIMEOUT_SECONDS, + ) + assert application.storage.connection_pool.is_open is False + assert application.storage.lifecycle.started is False + assert application.bot_session.close_calls == 1 + runtime_graph: Any = application.runtime + + assert application.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + assert runtime_graph._session.is_connected is False + assert runtime_graph._subscription_manager.subscription_keys == () + assert runtime_graph._live_processing_gate.locked is False + + +async def _cleanup_application( + application: PersistentApplication | None, +) -> None: + if application is None: + return + + if not application.task.done(): + application.dispatcher.release.set() + + try: + await asyncio.wait_for( + asyncio.shield(application.task), + timeout=APPLICATION_TIMEOUT_SECONDS, + ) + except BaseException: + if not application.task.done(): + application.task.cancel() + + try: + await application.task + except BaseException: + pass + + +def _snapshot_database( + postgres: PostgresTestSettings, +) -> DatabaseSnapshot: + with connect_postgres_test_database(postgres) as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT + trade_id, + source, + observation_sources, + first_observed_at, + last_observed_at + FROM market_data.trades + WHERE venue = %s AND symbol = %s + ORDER BY first_observed_at, executed_at, trade_id + """, + (VENUE, SYMBOL), + ) + trades = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT trade_id, revision, checkpoint_schema_version + FROM market_data.trade_stream_checkpoints + WHERE venue = %s AND symbol = %s + """, + (VENUE, SYMBOL), + ) + checkpoint = cursor.fetchone() + + return DatabaseSnapshot( + trades=trades, + checkpoint=checkpoint, + ) + + +def _assert_no_pool_connections( + postgres: PostgresTestSettings, +) -> None: + with connect_postgres_test_database(postgres) as connection: + assert count_other_test_connections(connection) == 0 + + +def _rest_trade( + *, + trade_id: int, + timestamp_ms: int, +) -> dict[str, object]: + return { + "a": trade_id, + "p": "64555.55", + "q": "0.002", + "T": timestamp_ms, + "m": False, + } + + +def _canonical_trade( + *, + trade_id: int, + timestamp_ms: int, +) -> Trade: + return Trade( + symbol=SYMBOL, + trade_id=trade_id, + price=Decimal("64555.55"), + quantity=Decimal("0.002"), + executed_at=datetime.fromtimestamp( + timestamp_ms / 1_000, + tz=timezone.utc, + ), + aggressor_side=TradeAggressorSide.BUY, + source="dzengi_websocket_trade", + ) + + +def _seed_trade_history( + settings: Settings, + *, + trade: Trade, + with_checkpoint: bool, +) -> None: + storage = build_market_data_storage(settings) + + assert storage is not None + + try: + storage.lifecycle.start() + + if with_checkpoint: + storage.trade_repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=trade, + observed_at=trade.executed_at, + ) + else: + storage.trade_repository.store_trade( + venue=VENUE, + trade=trade, + observed_at=trade.executed_at, + ) + finally: + if storage.connection_pool.is_open: + storage.lifecycle.stop() + + assert storage.connection_pool.is_open is False + assert storage.lifecycle.started is False + + +def test_checkpoint_hydration_finishes_before_first_network_io( + postgres_test_settings: PostgresTestSettings, +) -> None: + async def scenario() -> None: + base_time_ms = time.time_ns() // 1_000_000 - 10_000 + websocket = LoopbackTradeWebSocketServer(auto_ack=False) + rest = LoopbackTradeRestServer() + application: PersistentApplication | None = None + barrier: BlockingCheckpointStorage | None = None + + try: + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + settings = _settings_for_database( + postgres=postgres_test_settings, + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + ) + await asyncio.to_thread( + _seed_trade_history, + settings, + trade=_canonical_trade( + trade_id=50, + timestamp_ms=base_time_ms, + ), + with_checkpoint=True, + ) + + def checkpoint_storage_factory( + storage: MarketDataStorageBootstrapComposition, + ) -> TradeCheckpointStorageProtocol: + nonlocal barrier + barrier = BlockingCheckpointStorage( + delegate=storage.trade_repository, + blocked_operation="load_checkpoint", + ) + return barrier + + application = _build_application( + settings, + task_name=( + "persistent-application-hydration-network-boundary" + ), + checkpoint_storage_factory=( + checkpoint_storage_factory + ), + ) + + assert barrier is not None + + await application.dispatcher.started.wait() + await wait_until(barrier.entered.is_set) + + assert barrier.block_count == 1 + assert application.runtime.state is ( + TradeStreamProductionRuntimeState.STARTING + ) + assert state_store_from(application.runtime).is_empty() + assert websocket.connection_count == 0 + assert websocket.active_connection_count == 0 + assert websocket.subscriptions == () + assert rest.request_count == 0 + + barrier.release.set() + await websocket.wait_for_subscriptions(1) + state_store = state_store_from(application.runtime) + + assert state_store.get(SYMBOL).last_trade_id == 50 + + subscription = websocket.subscriptions[0] + await websocket.send_ack( + 0, + correlation_id=subscription.correlation_id, + ) + await wait_until( + lambda: application.runtime.state + is TradeStreamProductionRuntimeState.RUNNING, + ) + await _stop_application(application) + await asyncio.to_thread( + _assert_no_pool_connections, + postgres_test_settings, + ) + finally: + if barrier is not None: + barrier.release.set() + + await _cleanup_application(application) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + await assert_no_owned_tasks() + + run_scenario( + scenario(), + timeout_seconds=60.0, + ) + + +def test_first_adoption_finishes_before_first_network_io( + postgres_test_settings: PostgresTestSettings, +) -> None: + async def scenario() -> None: + base_time_ms = time.time_ns() // 1_000_000 - 10_000 + websocket = LoopbackTradeWebSocketServer(auto_ack=False) + rest = LoopbackTradeRestServer() + application: PersistentApplication | None = None + barrier: BlockingCheckpointStorage | None = None + + try: + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + settings = _settings_for_database( + postgres=postgres_test_settings, + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + ) + await asyncio.to_thread( + _seed_trade_history, + settings, + trade=_canonical_trade( + trade_id=60, + timestamp_ms=base_time_ms, + ), + with_checkpoint=False, + ) + + def checkpoint_storage_factory( + storage: MarketDataStorageBootstrapComposition, + ) -> TradeCheckpointStorageProtocol: + nonlocal barrier + barrier = BlockingCheckpointStorage( + delegate=storage.trade_repository, + blocked_operation=( + "adopt_existing_trade_as_checkpoint" + ), + ) + return barrier + + application = _build_application( + settings, + task_name=( + "persistent-application-adoption-network-boundary" + ), + checkpoint_storage_factory=( + checkpoint_storage_factory + ), + ) + + assert barrier is not None + + await application.dispatcher.started.wait() + await wait_until(barrier.entered.is_set) + blocked_snapshot = await asyncio.to_thread( + _snapshot_database, + postgres_test_settings, + ) + + assert barrier.block_count == 1 + assert application.runtime.state is ( + TradeStreamProductionRuntimeState.STARTING + ) + assert state_store_from(application.runtime).is_empty() + assert blocked_snapshot.checkpoint is None + assert websocket.connection_count == 0 + assert websocket.active_connection_count == 0 + assert websocket.subscriptions == () + assert rest.request_count == 0 + + barrier.release.set() + await websocket.wait_for_subscriptions(1) + state_store = state_store_from(application.runtime) + adopted_snapshot = await asyncio.to_thread( + _snapshot_database, + postgres_test_settings, + ) + + assert state_store.get(SYMBOL).last_trade_id == 60 + assert adopted_snapshot.checkpoint == (60, 1, 1) + + subscription = websocket.subscriptions[0] + await websocket.send_ack( + 0, + correlation_id=subscription.correlation_id, + ) + await wait_until( + lambda: application.runtime.state + is TradeStreamProductionRuntimeState.RUNNING, + ) + await _stop_application(application) + await asyncio.to_thread( + _assert_no_pool_connections, + postgres_test_settings, + ) + finally: + if barrier is not None: + barrier.release.set() + + await _cleanup_application(application) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + await assert_no_owned_tasks() + + run_scenario( + scenario(), + timeout_seconds=60.0, + ) + + +def test_real_restart_hydrates_recovers_gap_and_drains_live_fifo( + postgres_test_settings: PostgresTestSettings, +) -> None: + async def scenario() -> None: + recovery_release = threading.Event() + base_time_ms = time.time_ns() // 1_000_000 - 10_000 + websocket = LoopbackTradeWebSocketServer(auto_ack=False) + rest = LoopbackTradeRestServer( + responses=( + LoopbackHttpResponse( + body=[ + _rest_trade( + trade_id=101, + timestamp_ms=base_time_ms + 100, + ), + _rest_trade( + trade_id=102, + timestamp_ms=base_time_ms + 200, + ), + _rest_trade( + trade_id=103, + timestamp_ms=base_time_ms + 300, + ), + ], + release=recovery_release, + ), + ) + ) + first: PersistentApplication | None = None + second: PersistentApplication | None = None + + try: + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + settings = _settings_for_database( + postgres=postgres_test_settings, + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + ) + first = _build_application( + settings, + task_name="persistent-application-first", + ) + await first.dispatcher.started.wait() + await websocket.wait_for_subscriptions(1) + first_subscription = websocket.subscriptions[0] + await websocket.send_ack( + 0, + correlation_id=first_subscription.correlation_id, + ) + await wait_until( + lambda: first.runtime.state + is TradeStreamProductionRuntimeState.RUNNING, + ) + + await websocket.send_trade( + 0, + symbol=SYMBOL, + trade_id=100, + timestamp_ms=base_time_ms, + ) + await websocket.send_trade( + 0, + symbol=SYMBOL, + trade_id=101, + timestamp_ms=base_time_ms + 100, + ) + first_store = state_store_from(first.runtime) + await wait_until( + lambda: first_store.get(SYMBOL).last_trade_id == 101, + ) + first_state = first_store.get(SYMBOL) + + await _stop_application(first) + await wait_until( + lambda: websocket.active_handler_count == 0, + ) + await assert_no_owned_tasks() + first_snapshot = await asyncio.to_thread( + _snapshot_database, + postgres_test_settings, + ) + + assert tuple(row[0] for row in first_snapshot.trades) == ( + 100, + 101, + ) + assert first_snapshot.checkpoint == (101, 2, 1) + await asyncio.to_thread( + _assert_no_pool_connections, + postgres_test_settings, + ) + + second = _build_application( + settings, + task_name="persistent-application-second", + ) + + assert second.storage is not first.storage + assert second.storage.trade_repository is not ( + first.storage.trade_repository + ) + assert second.runtime is not first.runtime + + await second.dispatcher.started.wait() + await websocket.wait_for_subscriptions(2) + second_store = state_store_from(second.runtime) + hydrated_state = second_store.get(SYMBOL) + + assert second_store is not first_store + assert hydrated_state is not first_state + assert hydrated_state.last_trade_id == 101 + assert tuple(hydrated_state._trade_window) == (100, 101) + assert second.runtime.state is ( + TradeStreamProductionRuntimeState.STARTING + ) + + await websocket.send_trade( + 1, + symbol=SYMBOL, + trade_id=104, + timestamp_ms=base_time_ms + 400, + ) + second_subscription = websocket.subscriptions[1] + await websocket.send_ack( + 1, + correlation_id=second_subscription.correlation_id, + ) + await rest.wait_for_requests(1) + + startup: Any = second.runtime + gate = startup._live_processing_gate + blocked_snapshot = await asyncio.to_thread( + _snapshot_database, + postgres_test_settings, + ) + + assert gate.locked is True + assert gate.failed is False + assert second.runtime.state is ( + TradeStreamProductionRuntimeState.STARTING + ) + assert second_store.get(SYMBOL).last_trade_id == 101 + assert blocked_snapshot.checkpoint == (101, 2, 1) + assert tuple(row[0] for row in blocked_snapshot.trades) == ( + 100, + 101, + ) + + recovery_release.set() + await wait_until( + lambda: second.runtime.state + is TradeStreamProductionRuntimeState.RUNNING, + ) + await wait_until( + lambda: second_store.get(SYMBOL).last_trade_id == 104, + ) + + final_snapshot = await asyncio.to_thread( + _snapshot_database, + postgres_test_settings, + ) + + assert tuple(row[0] for row in final_snapshot.trades) == ( + 100, + 101, + 102, + 103, + 104, + ) + assert final_snapshot.checkpoint == (104, 5, 1) + assert final_snapshot.trades[1][1] == ( + "dzengi_websocket_trade" + ) + assert final_snapshot.trades[1][2] == [ + "dzengi_websocket_trade", + "dzengi", + ] + assert gate.locked is False + assert gate.failed is False + + await _stop_application(second) + await asyncio.to_thread( + _assert_no_pool_connections, + postgres_test_settings, + ) + finally: + recovery_release.set() + await _cleanup_application(second) + await _cleanup_application(first) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + await assert_no_owned_tasks() + + run_scenario( + scenario(), + timeout_seconds=60.0, + ) + + +def test_real_restart_adopts_legacy_history_before_network( + postgres_test_settings: PostgresTestSettings, +) -> None: + async def scenario() -> None: + base_time_ms = time.time_ns() // 1_000_000 - 10_000 + websocket = LoopbackTradeWebSocketServer(auto_ack=False) + rest = LoopbackTradeRestServer( + responses=( + LoopbackHttpResponse( + body=[ + _rest_trade( + trade_id=201, + timestamp_ms=base_time_ms + 100, + ), + _rest_trade( + trade_id=202, + timestamp_ms=base_time_ms + 200, + ), + ] + ), + ) + ) + application: PersistentApplication | None = None + legacy_storage: MarketDataStorageBootstrapComposition | None = None + + try: + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + settings = _settings_for_database( + postgres=postgres_test_settings, + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + ) + legacy_storage = build_market_data_storage(settings) + + assert legacy_storage is not None + + await asyncio.to_thread(legacy_storage.lifecycle.start) + first = _canonical_trade( + trade_id=200, + timestamp_ms=base_time_ms, + ) + second = _canonical_trade( + trade_id=201, + timestamp_ms=base_time_ms + 100, + ) + observed_at = datetime.now(timezone.utc) + await asyncio.to_thread( + legacy_storage.trade_repository.store_trade, + venue=VENUE, + trade=first, + observed_at=observed_at, + ) + await asyncio.to_thread( + legacy_storage.trade_repository.store_trade, + venue=VENUE, + trade=second, + observed_at=observed_at + timedelta(seconds=1), + ) + before_adoption = await asyncio.to_thread( + _snapshot_database, + postgres_test_settings, + ) + await asyncio.to_thread(legacy_storage.lifecycle.stop) + + assert before_adoption.checkpoint is None + assert legacy_storage.connection_pool.is_open is False + await asyncio.to_thread( + _assert_no_pool_connections, + postgres_test_settings, + ) + + application = _build_application( + settings, + task_name="persistent-application-adoption", + ) + + assert application.storage is not legacy_storage + assert application.storage.trade_repository is not ( + legacy_storage.trade_repository + ) + + await application.dispatcher.started.wait() + await websocket.wait_for_subscriptions(1) + state_store = state_store_from(application.runtime) + adopted_snapshot = await asyncio.to_thread( + _snapshot_database, + postgres_test_settings, + ) + + assert state_store.get(SYMBOL).last_trade_id == 201 + assert tuple(state_store.get(SYMBOL)._trade_window) == ( + 200, + 201, + ) + assert adopted_snapshot.checkpoint == (201, 1, 1) + assert adopted_snapshot.trades == before_adoption.trades + assert application.runtime.state is ( + TradeStreamProductionRuntimeState.STARTING + ) + + await websocket.send_trade( + 0, + symbol=SYMBOL, + trade_id=203, + timestamp_ms=base_time_ms + 300, + ) + subscription = websocket.subscriptions[0] + await websocket.send_ack( + 0, + correlation_id=subscription.correlation_id, + ) + await rest.wait_for_requests(1) + await wait_until( + lambda: application.runtime.state + is TradeStreamProductionRuntimeState.RUNNING, + ) + await wait_until( + lambda: state_store.get(SYMBOL).last_trade_id == 203, + ) + + final_snapshot = await asyncio.to_thread( + _snapshot_database, + postgres_test_settings, + ) + + assert tuple(row[0] for row in final_snapshot.trades) == ( + 200, + 201, + 202, + 203, + ) + assert final_snapshot.checkpoint == (203, 3, 1) + assert final_snapshot.trades[1][2] == [ + "dzengi_websocket_trade", + "dzengi", + ] + + await _stop_application(application) + await asyncio.to_thread( + _assert_no_pool_connections, + postgres_test_settings, + ) + finally: + await _cleanup_application(application) + + if ( + legacy_storage is not None + and legacy_storage.connection_pool.is_open + ): + await asyncio.to_thread(legacy_storage.lifecycle.stop) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + await assert_no_owned_tasks() + + run_scenario( + scenario(), + timeout_seconds=60.0, + ) diff --git a/app/tests/integration/market_data/storage/test_trade_stream_persistent_transaction_failure_integration.py b/app/tests/integration/market_data/storage/test_trade_stream_persistent_transaction_failure_integration.py new file mode 100644 index 0000000..3db0412 --- /dev/null +++ b/app/tests/integration/market_data/storage/test_trade_stream_persistent_transaction_failure_integration.py @@ -0,0 +1,480 @@ +from __future__ import annotations + +import asyncio +import time +from datetime import datetime, timezone +from typing import Any + +import psycopg +import pytest + +from src.bootstrap.market_data_storage import ( + MarketDataStorageBootstrapComposition, + build_market_data_storage, +) +from src.core.config import Settings +from src.market_data.acquisition.consistency.trade_observation_sink_protocol import ( + TradeObservationSinkProtocol, +) +from src.market_data.acquisition.models.trade import Trade +from src.market_data.acquisition.runtime.trade_stream_production_runtime import ( + TradeStreamProductionRuntimeState, +) +from src.market_data.storage.exceptions import ( + MarketDataStorageOperationError, +) +from tests.integration.market_data.acquisition.runtime.loopback_trade_exchange import ( + LoopbackHttpResponse, + LoopbackTradeEnvironment, + LoopbackTradeRestServer, + LoopbackTradeWebSocketServer, + wait_until, +) +from tests.integration.market_data.storage import ( + test_trade_stream_persistent_restart_integration as restart_harness, +) +from tests.support.postgres_market_data import ( + PostgresTestSettings, + connect_postgres_test_database, +) +from tests.support.trade_stream_runtime import ( + SYMBOL, + assert_no_owned_tasks, + run_scenario, + state_store_from, +) + + +pytestmark = pytest.mark.integration + +VENUE = "dzengi" +REJECT_TRIGGER = "reject_runtime_checkpoint_write" +REJECT_FUNCTION = "market_data.reject_runtime_checkpoint_write" + + +class PostCommitTestError(RuntimeError): + """Искусственная ошибка после подтверждённой фиксации в PostgreSQL.""" + + +class CommitThenFailObservationSink: + """Сохранить принятую сделку и имитировать потерю ответа.""" + + __slots__ = ( + "_accepted_trades", + "_delegate", + "_expected_error", + ) + + def __init__( + self, + *, + delegate: TradeObservationSinkProtocol, + expected_error: PostCommitTestError, + ) -> None: + self._delegate = delegate + self._expected_error = expected_error + self._accepted_trades: list[Trade] = [] + + @property + def accepted_trades(self) -> tuple[Trade, ...]: + return tuple(self._accepted_trades) + + def persist_accepted( + self, + trade: Trade, + *, + expected_trade: Trade | None, + ) -> None: + self._delegate.persist_accepted( + trade, + expected_trade=expected_trade, + ) + self._accepted_trades.append(trade) + raise self._expected_error + + def persist_duplicate( + self, + trade: Trade, + ) -> None: + self._delegate.persist_duplicate(trade) + + +def _seed_checkpoint( + settings: Settings, + *, + trade_id: int, + timestamp_ms: int, +) -> None: + storage = build_market_data_storage(settings) + + assert storage is not None + + try: + storage.lifecycle.start() + storage.trade_repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=restart_harness._canonical_trade( + trade_id=trade_id, + timestamp_ms=timestamp_ms, + ), + observed_at=datetime.now(timezone.utc), + ) + finally: + if storage.connection_pool.is_open: + storage.lifecycle.stop() + + +def _install_checkpoint_rejection_trigger( + postgres: PostgresTestSettings, +) -> None: + with connect_postgres_test_database(postgres) as connection: + with connection.cursor() as cursor: + cursor.execute( + f""" + CREATE FUNCTION {REJECT_FUNCTION}() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + RAISE EXCEPTION 'runtime checkpoint write rejected'; + END; + $$ + """ + ) + cursor.execute( + f""" + CREATE TRIGGER {REJECT_TRIGGER} + BEFORE INSERT OR UPDATE + ON market_data.trade_stream_checkpoints + FOR EACH ROW + EXECUTE FUNCTION {REJECT_FUNCTION}() + """ + ) + + +def _remove_checkpoint_rejection_trigger( + postgres: PostgresTestSettings, +) -> None: + with connect_postgres_test_database(postgres) as connection: + with connection.cursor() as cursor: + cursor.execute( + f""" + DROP TRIGGER IF EXISTS {REJECT_TRIGGER} + ON market_data.trade_stream_checkpoints + """ + ) + cursor.execute( + f"DROP FUNCTION IF EXISTS {REJECT_FUNCTION}()" + ) + + +def test_checkpoint_write_failure_rolls_back_trade_and_runtime_state( + postgres_test_settings: PostgresTestSettings, +) -> None: + async def scenario() -> None: + base_time_ms = time.time_ns() // 1_000_000 - 10_000 + websocket = LoopbackTradeWebSocketServer(auto_ack=False) + rest = LoopbackTradeRestServer() + application: restart_harness.PersistentApplication | None = None + trigger_installed = False + + try: + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + settings = restart_harness._settings_for_database( + postgres=postgres_test_settings, + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + ) + await asyncio.to_thread( + _seed_checkpoint, + settings, + trade_id=300, + timestamp_ms=base_time_ms, + ) + seeded_snapshot = await asyncio.to_thread( + restart_harness._snapshot_database, + postgres_test_settings, + ) + await asyncio.to_thread( + _install_checkpoint_rejection_trigger, + postgres_test_settings, + ) + trigger_installed = True + application = restart_harness._build_application( + settings, + task_name="persistent-application-before-commit-failure", + ) + await application.dispatcher.started.wait() + await websocket.wait_for_subscriptions(1) + subscription = websocket.subscriptions[0] + await websocket.send_ack( + 0, + correlation_id=subscription.correlation_id, + ) + await wait_until( + lambda: application.runtime.state + is TradeStreamProductionRuntimeState.RUNNING, + ) + state_store = state_store_from(application.runtime) + state_before_failure = state_store.get(SYMBOL) + + assert state_before_failure.last_trade_id == 300 + + await websocket.send_trade( + 0, + symbol=SYMBOL, + trade_id=301, + timestamp_ms=base_time_ms + 100, + ) + + with pytest.raises( + MarketDataStorageOperationError, + ) as captured: + await asyncio.wait_for( + asyncio.shield(application.task), + timeout=restart_harness.APPLICATION_TIMEOUT_SECONDS, + ) + + assert type(captured.value) is ( + MarketDataStorageOperationError + ) + database_error = captured.value.__cause__ + + assert type(database_error) is psycopg.errors.RaiseException + assert database_error.sqlstate == "P0001" + failed_snapshot = await asyncio.to_thread( + restart_harness._snapshot_database, + postgres_test_settings, + ) + runtime_graph: Any = application.runtime + gate = runtime_graph._live_processing_gate + + assert failed_snapshot == seeded_snapshot + assert state_store.get(SYMBOL) is state_before_failure + assert state_before_failure.last_trade_id == 300 + assert application.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + assert gate.failed is True + assert gate.locked is False + assert gate._failure is captured.value + assert runtime_graph._session.is_connected is False + assert ( + runtime_graph._subscription_manager.subscription_keys + == () + ) + assert application.dispatcher.cancelled.is_set() + assert application.storage.connection_pool.is_open is False + assert application.storage.lifecycle.started is False + assert application.bot_session.close_calls == 1 + await asyncio.to_thread( + restart_harness._assert_no_pool_connections, + postgres_test_settings, + ) + finally: + await restart_harness._cleanup_application(application) + + if trigger_installed: + await asyncio.to_thread( + _remove_checkpoint_rejection_trigger, + postgres_test_settings, + ) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + await assert_no_owned_tasks() + + run_scenario( + scenario(), + timeout_seconds=60.0, + ) + + +def test_post_commit_error_is_recovered_by_fresh_runtime_graph( + postgres_test_settings: PostgresTestSettings, +) -> None: + async def scenario() -> None: + base_time_ms = time.time_ns() // 1_000_000 - 10_000 + expected_error = PostCommitTestError( + "caller did not observe committed checkpoint" + ) + websocket = LoopbackTradeWebSocketServer(auto_ack=False) + rest = LoopbackTradeRestServer( + responses=( + LoopbackHttpResponse( + body=[ + restart_harness._rest_trade( + trade_id=600, + timestamp_ms=base_time_ms, + ), + ] + ), + ) + ) + first: restart_harness.PersistentApplication | None = None + second: restart_harness.PersistentApplication | None = None + failing_sink: CommitThenFailObservationSink | None = None + + try: + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + settings = restart_harness._settings_for_database( + postgres=postgres_test_settings, + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + ) + + def failing_sink_factory( + storage: MarketDataStorageBootstrapComposition, + ) -> TradeObservationSinkProtocol: + nonlocal failing_sink + failing_sink = CommitThenFailObservationSink( + delegate=storage.trade_observation_sink, + expected_error=expected_error, + ) + return failing_sink + + first = restart_harness._build_application( + settings, + task_name="persistent-application-post-commit-failure", + trade_observation_sink_factory=failing_sink_factory, + ) + await first.dispatcher.started.wait() + await websocket.wait_for_subscriptions(1) + first_subscription = websocket.subscriptions[0] + await websocket.send_ack( + 0, + correlation_id=first_subscription.correlation_id, + ) + await wait_until( + lambda: first.runtime.state + is TradeStreamProductionRuntimeState.RUNNING, + ) + first_store = state_store_from(first.runtime) + first_state = first_store.get(SYMBOL) + + assert first_state.last_trade_id is None + + await websocket.send_trade( + 0, + symbol=SYMBOL, + trade_id=600, + timestamp_ms=base_time_ms, + ) + + with pytest.raises(PostCommitTestError) as captured: + await asyncio.wait_for( + asyncio.shield(first.task), + timeout=restart_harness.APPLICATION_TIMEOUT_SECONDS, + ) + + assert captured.value is expected_error + assert failing_sink is not None + assert len(failing_sink.accepted_trades) == 1 + committed_object = failing_sink.accepted_trades[0] + committed_snapshot = await asyncio.to_thread( + restart_harness._snapshot_database, + postgres_test_settings, + ) + + assert tuple(row[0] for row in committed_snapshot.trades) == ( + 600, + ) + assert committed_snapshot.checkpoint == (600, 1, 1) + assert first_state.last_trade_id is None + assert first.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + first_graph: Any = first.runtime + + assert first_graph._live_processing_gate._failure is ( + expected_error + ) + assert first_graph._session.is_connected is False + assert ( + first_graph._subscription_manager.subscription_keys + == () + ) + assert first.dispatcher.cancelled.is_set() + assert first.storage.connection_pool.is_open is False + await wait_until( + lambda: websocket.active_handler_count == 0, + ) + await assert_no_owned_tasks() + await asyncio.to_thread( + restart_harness._assert_no_pool_connections, + postgres_test_settings, + ) + + second = restart_harness._build_application( + settings, + task_name="persistent-application-post-commit-restart", + ) + + assert second.storage is not first.storage + assert second.storage.trade_repository is not ( + first.storage.trade_repository + ) + assert second.runtime is not first.runtime + + await second.dispatcher.started.wait() + await websocket.wait_for_subscriptions(2) + second_store = state_store_from(second.runtime) + second_state = second_store.get(SYMBOL) + hydrated_object = second_state.last_trade + + assert second_state is not first_state + assert hydrated_object == committed_object + assert hydrated_object is not committed_object + assert second_state.last_trade_id == 600 + assert tuple(second_state._trade_window) == (600,) + + second_subscription = websocket.subscriptions[1] + await websocket.send_ack( + 1, + correlation_id=second_subscription.correlation_id, + ) + await rest.wait_for_requests(1) + await wait_until( + lambda: second.runtime.state + is TradeStreamProductionRuntimeState.RUNNING, + ) + recovered_snapshot = await asyncio.to_thread( + restart_harness._snapshot_database, + postgres_test_settings, + ) + + assert tuple(row[0] for row in recovered_snapshot.trades) == ( + 600, + ) + assert recovered_snapshot.checkpoint == (600, 1, 1) + assert recovered_snapshot.trades[0][2] == [ + "dzengi_websocket_trade", + "dzengi", + ] + assert second_state.last_trade_id == 600 + assert second_state.last_trade is hydrated_object + + await restart_harness._stop_application(second) + await asyncio.to_thread( + restart_harness._assert_no_pool_connections, + postgres_test_settings, + ) + finally: + await restart_harness._cleanup_application(second) + await restart_harness._cleanup_application(first) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + await assert_no_owned_tasks() + + run_scenario( + scenario(), + timeout_seconds=60.0, + ) diff --git a/app/tests/integration/market_data/storage/test_trade_stream_state_hydration_integration.py b/app/tests/integration/market_data/storage/test_trade_stream_state_hydration_integration.py new file mode 100644 index 0000000..e193b99 --- /dev/null +++ b/app/tests/integration/market_data/storage/test_trade_stream_state_hydration_integration.py @@ -0,0 +1,283 @@ +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +import pytest + +from src.market_data.acquisition.checkpoint import ( + TradeStreamStateHydrator, +) +from src.market_data.acquisition.consistency.trade_stream_consistency_controller import ( + TradeStreamConsistencyController, +) +from src.market_data.acquisition.consistency.trade_stream_state_store import ( + TradeStreamStateStore, +) +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.acquisition.trade_id_sequence import ( + SIGNED_TRADE_ID_MAX, + SIGNED_TRADE_ID_MIN, +) +from src.market_data.storage import ( + MarketDataCheckpointIntegrityError, + PostgresTradeRepository, + TradeStorageObservationSink, +) +from src.storage.postgres_pool import PostgresConnectionPool + + +pytestmark = pytest.mark.integration + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +SECOND_SYMBOL = "ETH/USD_LEVERAGE" +EVENT_TIME = datetime(2026, 8, 1, 14, 0, tzinfo=timezone.utc) +OBSERVED_AT = EVENT_TIME + timedelta(seconds=1) + + +def _trade( + *, + trade_id: int, + offset_ms: int = 0, + symbol: str = SYMBOL, + source: str = "dzengi_websocket_trade", +) -> Trade: + return Trade( + symbol=symbol, + trade_id=trade_id, + price=Decimal("65000.25"), + quantity=Decimal("0.001"), + executed_at=EVENT_TIME + timedelta(milliseconds=offset_ms), + aggressor_side=TradeAggressorSide.BUY, + source=source, + ) + + +def _repository( + pool: PostgresConnectionPool, +) -> PostgresTradeRepository: + return PostgresTradeRepository( + connection_provider=pool.connection, + ) + + +def _trade_metadata( + pool: PostgresConnectionPool, + *, + symbol: str, +) -> tuple[tuple[object, ...], ...]: + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT + trade_id, + executed_at, + source, + first_observed_at, + last_observed_at, + observation_sources + FROM market_data.trades + WHERE venue = %s AND symbol = %s + ORDER BY first_observed_at, executed_at, trade_id + """, + (VENUE, symbol), + ) + return tuple(cursor.fetchall()) + + +def test_real_hydration_adopts_history_without_rewriting_trades( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + first = _trade(trade_id=100) + second = _trade(trade_id=101, offset_ms=1) + repository.store_trade( + venue=VENUE, + trade=first, + observed_at=OBSERVED_AT, + ) + repository.store_trade( + venue=VENUE, + trade=second, + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + metadata_before = _trade_metadata( + migrated_postgres_pool, + symbol=SYMBOL, + ) + + first_store = TradeStreamStateStore() + first_states = TradeStreamStateHydrator( + checkpoint_storage=repository, + state_store=first_store, + venue=VENUE, + deduplication_window_size=2, + ).hydrate(symbols=(SYMBOL,)) + first_checkpoint = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + + assert len(first_states) == 1 + assert tuple(first_states[0]._trade_window) == (100, 101) + assert first_states[0].last_trade == second + assert first_checkpoint is not None + assert first_checkpoint.trade == second + assert first_checkpoint.revision == 1 + assert _trade_metadata( + migrated_postgres_pool, + symbol=SYMBOL, + ) == metadata_before + + restart_store = TradeStreamStateStore() + restart_state = TradeStreamStateHydrator( + checkpoint_storage=repository, + state_store=restart_store, + venue=VENUE, + deduplication_window_size=2, + ).hydrate(symbols=(SYMBOL,))[0] + repeated_checkpoint = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + + assert tuple(restart_state._trade_window) == (100, 101) + assert restart_state.accept( + replace(first, source="dzengi_recovery"), + ) is None + assert repeated_checkpoint is not None + assert repeated_checkpoint == first_checkpoint + assert repeated_checkpoint.revision == 1 + assert _trade_metadata( + migrated_postgres_pool, + symbol=SYMBOL, + ) == metadata_before + + controller = TradeStreamConsistencyController( + state_store=restart_store, + trade_observation_sink=TradeStorageObservationSink( + trade_storage=repository, + venue=VENUE, + clock=lambda: OBSERVED_AT + timedelta(seconds=2), + ), + ) + third = _trade(trade_id=102, offset_ms=2) + + assert controller.accept(third) is third + + advanced_checkpoint = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + assert advanced_checkpoint is not None + assert advanced_checkpoint.trade == third + assert advanced_checkpoint.revision == 2 + assert restart_state.last_trade is third + + +@pytest.mark.parametrize( + ("first_trade_id", "second_trade_id", "next_trade_id"), + ( + (SIGNED_TRADE_ID_MAX, SIGNED_TRADE_ID_MIN, SIGNED_TRADE_ID_MIN + 1), + (-1, 0, 1), + ), +) +def test_real_hydration_restores_rollover_aware_deduplication_tail( + migrated_postgres_pool: PostgresConnectionPool, + first_trade_id: int, + second_trade_id: int, + next_trade_id: int, +) -> None: + repository = _repository(migrated_postgres_pool) + first = _trade(trade_id=first_trade_id) + second = _trade(trade_id=second_trade_id, offset_ms=1) + repository.store_trade( + venue=VENUE, + trade=first, + observed_at=OBSERVED_AT, + ) + repository.store_trade( + venue=VENUE, + trade=second, + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + state_store = TradeStreamStateStore() + + state = TradeStreamStateHydrator( + checkpoint_storage=repository, + state_store=state_store, + venue=VENUE, + deduplication_window_size=2, + ).hydrate(symbols=(SYMBOL,))[0] + + assert tuple(state._trade_window) == ( + first_trade_id, + second_trade_id, + ) + assert state.last_trade == second + assert state.accept( + replace(first, source="dzengi_recovery"), + ) is None + + next_trade = _trade(trade_id=next_trade_id, offset_ms=2) + assert state.accept(next_trade) is next_trade + assert state.last_trade is next_trade + + +def test_real_multi_symbol_failure_does_not_publish_partial_state( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + repository = _repository(migrated_postgres_pool) + repository.store_trade( + venue=VENUE, + trade=_trade(trade_id=10), + observed_at=OBSERVED_AT, + ) + repository.store_trade( + venue=VENUE, + trade=_trade( + symbol=SECOND_SYMBOL, + trade_id=20, + offset_ms=1, + ), + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + repository.store_trade( + venue=VENUE, + trade=_trade( + symbol=SECOND_SYMBOL, + trade_id=19, + offset_ms=2, + ), + observed_at=OBSERVED_AT + timedelta(seconds=2), + ) + state_store = TradeStreamStateStore() + + with pytest.raises(MarketDataCheckpointIntegrityError): + TradeStreamStateHydrator( + checkpoint_storage=repository, + state_store=state_store, + venue=VENUE, + deduplication_window_size=2, + ).hydrate(symbols=(SYMBOL, SECOND_SYMBOL)) + + assert state_store.is_empty() is True + assert state_store.contains(SYMBOL) is False + assert state_store.contains(SECOND_SYMBOL) is False + + adopted_first_symbol = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + assert adopted_first_symbol is not None + assert adopted_first_symbol.trade.trade_id == 10 + assert repository.load_checkpoint( + venue=VENUE, + symbol=SECOND_SYMBOL, + ) is None diff --git a/app/tests/support/postgres_market_data.py b/app/tests/support/postgres_market_data.py index 1fed0fc..493e195 100644 --- a/app/tests/support/postgres_market_data.py +++ b/app/tests/support/postgres_market_data.py @@ -1,6 +1,7 @@ from __future__ import annotations import ipaddress +import math import re import time from collections.abc import Mapping @@ -177,28 +178,26 @@ def release_postgres_test_lock( def count_other_test_connections( connection: psycopg.Connection[Any], ) -> int: - """Посчитать оставшиеся соединения стенда и пула с тестовой базой.""" + """Посчитать клиентские соединения с проверенной тестовой БД.""" + database_name = _validated_postgres_test_control_database_name(connection) + with connection.cursor() as cursor: cursor.execute( """ SELECT COUNT(*) - FROM pg_stat_activity - WHERE datname = current_database() - AND application_name = %s + FROM pg_catalog.pg_stat_activity + WHERE datname = %s + AND backend_type = 'client backend' + AND application_name IS DISTINCT FROM %s """, - (POSTGRES_TEST_APPLICATION_NAME,), + (database_name, POSTGRES_TEST_CONTROL_APPLICATION_NAME), ) row = cursor.fetchone() - if ( - not isinstance(row, tuple) - or len(row) != 1 - or isinstance(row[0], bool) - or not isinstance(row[0], int) - ): - raise RuntimeError("PostgreSQL returned an invalid connection count") - - return row[0] + return _validated_postgres_count( + row, + error_message="PostgreSQL returned an invalid connection count", + ) def wait_for_postgres_advisory_lock_waiters( @@ -260,6 +259,138 @@ def wait_for_postgres_advisory_lock_waiters( time.sleep(0.01) +def wait_for_postgres_relation_lock_waiters( + connection: psycopg.Connection[Any], + *, + relation_name: str, + expected_count: int, + timeout_seconds: float = 5.0, +) -> None: + """Дождаться точного числа ожидающих блокировку заданной таблицы.""" + if not isinstance(relation_name, str) or not relation_name.strip(): + raise ValueError("relation_name must be a non-empty string") + + if ( + isinstance(expected_count, bool) + or not isinstance(expected_count, int) + or expected_count <= 0 + ): + raise ValueError("expected_count must be a positive integer") + + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, (int, float)) + or not math.isfinite(float(timeout_seconds)) + or timeout_seconds <= 0 + ): + raise ValueError("timeout_seconds must be a positive finite number") + + database_name = _validated_postgres_test_control_database_name(connection) + + with connection.cursor() as cursor: + cursor.execute( + "SELECT pg_catalog.to_regclass(%s)::oid", + (relation_name,), + ) + relation_row = cursor.fetchone() + + if ( + not isinstance(relation_row, tuple) + or len(relation_row) != 1 + or isinstance(relation_row[0], bool) + or not isinstance(relation_row[0], int) + or relation_row[0] <= 0 + ): + raise RuntimeError( + "PostgreSQL did not resolve the requested test relation" + ) + + relation_oid = relation_row[0] + deadline = time.monotonic() + timeout_seconds + + while True: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT COUNT(*) + FROM pg_catalog.pg_locks + WHERE locktype = 'relation' + AND database = ( + SELECT oid + FROM pg_catalog.pg_database + WHERE datname = %s + ) + AND relation = %s + AND NOT granted + """, + (database_name, relation_oid), + ) + row = cursor.fetchone() + + observed_count = _validated_postgres_count( + row, + error_message=( + "PostgreSQL returned an invalid relation-lock waiter count" + ), + ) + + if observed_count == expected_count: + return + + if time.monotonic() >= deadline: + raise TimeoutError( + "PostgreSQL did not observe all relation-lock callers; " + f"expected {expected_count}, observed {observed_count}." + ) + + time.sleep(0.01) + + +def _validated_postgres_test_control_database_name( + connection: psycopg.Connection[Any], +) -> str: + """Повторно подтвердить безопасную БД и управляющее соединение.""" + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT current_database(), current_setting('application_name') + """ + ) + identity = cursor.fetchone() + + if ( + not isinstance(identity, tuple) + or len(identity) != 2 + or not isinstance(identity[0], str) + or not _SAFE_DATABASE_NAME.fullmatch(identity[0]) + or identity[1] != POSTGRES_TEST_CONTROL_APPLICATION_NAME + ): + raise RuntimeError( + "PostgreSQL connection is not the validated test control " + "connection." + ) + + return identity[0] + + +def _validated_postgres_count( + row: object, + *, + error_message: str, +) -> int: + """Проверить форму и тип результата PostgreSQL COUNT(*).""" + if ( + not isinstance(row, tuple) + or len(row) != 1 + or isinstance(row[0], bool) + or not isinstance(row[0], int) + or row[0] < 0 + ): + raise RuntimeError(error_message) + + return row[0] + + def _validate_local_endpoint(parameters: Mapping[str, object]) -> None: service = str(parameters.get("service", "")).strip() host = str(parameters.get("host", "")).strip() diff --git a/app/tests/support/trade_stream_runtime.py b/app/tests/support/trade_stream_runtime.py index f6b9097..df9cd65 100644 --- a/app/tests/support/trade_stream_runtime.py +++ b/app/tests/support/trade_stream_runtime.py @@ -30,12 +30,17 @@ RUNTIME_CLEANUP_TIMEOUT_SECONDS = 5.0 OWNED_TASK_NAMES = frozenset( { + "application-shutdown", "market-data-storage-shutdown", "market-data-storage-startup", + "telegram-polling", + "trade-stream-market-processing", "trade-stream-receive", "trade-stream-runtime", "trade-stream-runtime-recovery", "trade-stream-scheduler", + "trade-stream-startup-recovery", + "trade-stream-state-hydration", "trade-stream-startup", } ) @@ -173,7 +178,10 @@ def active_owned_task_names() -> tuple[str, ...]: for task in asyncio.all_tasks() if task is not current_task and not task.done() - and task.get_name() in OWNED_TASK_NAMES + and ( + task.get_name() in OWNED_TASK_NAMES + or task.get_name().startswith("persistent-application-") + ) ) ) diff --git a/app/tests/unit/bootstrap/test_app_factory.py b/app/tests/unit/bootstrap/test_app_factory.py index e607d3a..ae04755 100644 --- a/app/tests/unit/bootstrap/test_app_factory.py +++ b/app/tests/unit/bootstrap/test_app_factory.py @@ -37,7 +37,10 @@ class RecordingJournal: del event, message, context -def make_settings() -> SimpleNamespace: +def make_settings( + *, + storage_enabled: bool = True, +) -> SimpleNamespace: return SimpleNamespace( bot_token="test-token", bot_parse_mode="HTML", @@ -46,7 +49,9 @@ def make_settings() -> SimpleNamespace: exchange_name="dzengi", default_symbol="BTC/USD_LEVERAGE", trade_stream=SimpleNamespace(enabled=True), - market_data_storage=SimpleNamespace(enabled=True), + market_data_storage=SimpleNamespace( + enabled=storage_enabled, + ), ) @@ -59,9 +64,11 @@ def test_create_app_builds_one_application_composition( runtime = object() storage_lifecycle = object() storage_sink = object() + storage_repository = object() storage = SimpleNamespace( lifecycle=storage_lifecycle, trade_observation_sink=storage_sink, + trade_repository=storage_repository, ) journal = RecordingJournal() observed_runtime_settings: list[object] = [] @@ -103,9 +110,11 @@ def test_create_app_builds_one_application_composition( received_settings: object, *, trade_observation_sink: object, + checkpoint_storage: object, ) -> object: observed_runtime_settings.append(received_settings) assert trade_observation_sink is storage_sink + assert checkpoint_storage is storage_repository return runtime monkeypatch.setattr( @@ -162,7 +171,7 @@ def test_runtime_build_error_is_fatal( monkeypatch.setattr( app_factory, "load_settings", - make_settings, + lambda: make_settings(storage_enabled=False), ) monkeypatch.setattr( app_factory, @@ -189,8 +198,11 @@ def test_runtime_build_error_is_fatal( settings: object, *, trade_observation_sink: object, + checkpoint_storage: object, ) -> None: - del settings, trade_observation_sink + del settings + assert trade_observation_sink is None + assert checkpoint_storage is None raise expected monkeypatch.setattr( diff --git a/app/tests/unit/bootstrap/test_application.py b/app/tests/unit/bootstrap/test_application.py index 3498e6b..eae8cd7 100644 --- a/app/tests/unit/bootstrap/test_application.py +++ b/app/tests/unit/bootstrap/test_application.py @@ -557,6 +557,8 @@ def test_application_cancellation_performs_full_cleanup() -> None: "telegram-polling", "trade-stream-runtime", "application-shutdown", + "market-data-storage-startup", + "market-data-storage-shutdown", } } @@ -637,6 +639,8 @@ def test_simultaneous_root_failures_are_awaited_deterministically() -> None: "telegram-polling", "trade-stream-runtime", "application-shutdown", + "market-data-storage-startup", + "market-data-storage-shutdown", } } @@ -647,6 +651,7 @@ def test_repeated_cancellation_does_not_interrupt_cleanup() -> None: async def scenario() -> None: dispatcher = FakeDispatcher() runtime = BlockingStopRuntime() + storage = FakeStorageLifecycle() bot = FakeBot() task = asyncio.create_task( run_application( @@ -654,6 +659,7 @@ def test_repeated_cancellation_does_not_interrupt_cleanup() -> None: dispatcher=dispatcher, runtime=runtime, bot=bot, + storage_lifecycle=storage, ) ) ) @@ -664,6 +670,8 @@ def test_repeated_cancellation_does_not_interrupt_cleanup() -> None: await runtime.stop_entered.wait() task.cancel() + await asyncio.sleep(0) + assert storage.stop_calls == 0 runtime.stop_release.set() with pytest.raises(asyncio.CancelledError): @@ -673,6 +681,7 @@ def test_repeated_cancellation_does_not_interrupt_cleanup() -> None: assert dispatcher.cancelled.is_set() assert runtime.stop_calls == 1 assert runtime.stopped.is_set() + assert storage.stop_calls == 1 assert bot.session.close_calls == 1 await asyncio.sleep(0) assert not { @@ -685,6 +694,8 @@ def test_repeated_cancellation_does_not_interrupt_cleanup() -> None: "telegram-polling", "trade-stream-runtime", "application-shutdown", + "market-data-storage-startup", + "market-data-storage-shutdown", } } diff --git a/app/tests/unit/bootstrap/test_market_data_storage_bootstrap.py b/app/tests/unit/bootstrap/test_market_data_storage_bootstrap.py index 708004b..7381e2d 100644 --- a/app/tests/unit/bootstrap/test_market_data_storage_bootstrap.py +++ b/app/tests/unit/bootstrap/test_market_data_storage_bootstrap.py @@ -15,6 +15,10 @@ from src.core.config import ( Settings, TradeStreamSettings, ) +from src.market_data.storage.contracts import ( + TradeCheckpointStorageProtocol, + TradeStorageProtocol, +) def make_settings( @@ -159,6 +163,18 @@ def test_builds_shared_graph_without_opening_pool() -> None: composition.trade_observation_sink._trade_storage is composition.trade_repository ) + assert ( + composition.trade_observation_sink._checkpoint_storage + is composition.trade_repository + ) + assert isinstance( + composition.trade_repository, + TradeStorageProtocol, + ) + assert isinstance( + composition.trade_repository, + TradeCheckpointStorageProtocol, + ) assert composition.trade_observation_sink._venue == "dzengi" conninfo = conninfo_to_dict(composition.connection_pool._conninfo) diff --git a/app/tests/unit/bootstrap/test_trade_stream_runtime.py b/app/tests/unit/bootstrap/test_trade_stream_runtime.py index e26322e..a97677b 100644 --- a/app/tests/unit/bootstrap/test_trade_stream_runtime.py +++ b/app/tests/unit/bootstrap/test_trade_stream_runtime.py @@ -2,6 +2,10 @@ from __future__ import annotations from typing import Any +import pytest + +import src.bootstrap.trade_stream_runtime as runtime_factory +from src.bootstrap.market_data_storage import build_market_data_storage from src.bootstrap.trade_stream_runtime import ( build_trade_stream_production_runtime, ) @@ -21,7 +25,16 @@ class RecordingTradeObservationSink: def __init__(self) -> None: self.observations: list[Trade] = [] - def persist(self, trade: Trade) -> None: + def persist_accepted( + self, + trade: Trade, + *, + expected_trade: Trade | None, + ) -> None: + del expected_trade + self.observations.append(trade) + + def persist_duplicate(self, trade: Trade) -> None: self.observations.append(trade) @@ -29,6 +42,9 @@ def make_settings( *, enabled: bool = True, api_key: str = "api-key", + storage_enabled: bool = False, + subscription_ack_timeout_seconds: float = 12.5, + startup_market_buffer_capacity: int = 1_234, ) -> Settings: return Settings( bot_token="test-token", @@ -58,6 +74,12 @@ def make_settings( heartbeat_timeout_seconds=31.0, scheduler_interval_seconds=6.0, recovery_window_ms=123_456, + subscription_ack_timeout_seconds=( + subscription_ack_timeout_seconds + ), + startup_market_buffer_capacity=( + startup_market_buffer_capacity + ), ), db_host="localhost", db_port=5432, @@ -65,7 +87,7 @@ def make_settings( db_user="test", db_password="test", market_data_storage=MarketDataStorageSettings( - enabled=False, + enabled=storage_enabled, pool_min_size=1, pool_max_size=4, pool_timeout_seconds=10.0, @@ -81,6 +103,16 @@ def test_disabled_feature_does_not_build_runtime() -> None: assert build_trade_stream_production_runtime(settings) is None +def test_enabled_storage_requires_enabled_trade_stream() -> None: + settings = make_settings( + enabled=False, + storage_enabled=True, + ) + + with pytest.raises(RuntimeError, match="Trade Stream"): + build_trade_stream_production_runtime(settings) + + def test_builds_runtime_without_starting_lifecycle() -> None: runtime = build_trade_stream_production_runtime( make_settings(), @@ -120,6 +152,116 @@ def test_uses_one_shared_stateful_dependency_graph() -> None: assert runtime_graph._runtime_scheduler.runtime_supervisor is ( runtime_graph._runtime_supervisor ) + assert runtime_graph._startup_recovery_coordinator is None + + +def test_persistent_runtime_reuses_one_storage_graph_without_io() -> None: + settings = make_settings(storage_enabled=True) + storage = build_market_data_storage(settings) + + assert storage is not None + + runtime = build_trade_stream_production_runtime( + settings, + trade_observation_sink=storage.trade_observation_sink, + checkpoint_storage=storage.trade_repository, + ) + + assert isinstance(runtime, TradeStreamProductionRuntime) + runtime_graph: Any = runtime + startup_recovery = runtime_graph._startup_recovery_coordinator + live_controller = ( + runtime_graph._trade_stream_service._consistency_controller + ) + + assert startup_recovery is not None + assert ( + startup_recovery._state_hydrator._checkpoint_storage + is storage.trade_repository + ) + assert ( + storage.trade_observation_sink._trade_storage + is storage.trade_repository + ) + assert ( + storage.trade_observation_sink._checkpoint_storage + is storage.trade_repository + ) + assert live_controller._trade_observation_sink is ( + storage.trade_observation_sink + ) + assert startup_recovery._state_hydrator._state_store is ( + live_controller._state_store + ) + assert startup_recovery._state_hydrator._venue == "dzengi" + assert storage.trade_observation_sink._venue == "dzengi" + assert storage.connection_pool.is_open is False + assert storage.lifecycle.started is False + assert startup_recovery._hydration_task is None + assert startup_recovery._recovery_task is None + assert runtime_graph._startup_task is None + assert runtime_graph._receive_task is None + assert runtime_graph._scheduler_task is None + + +@pytest.mark.parametrize( + ("include_sink", "include_checkpoint"), + ( + (False, False), + (True, False), + (False, True), + ), +) +def test_enabled_storage_rejects_incomplete_runtime_graph_before_transport( + monkeypatch: pytest.MonkeyPatch, + include_sink: bool, + include_checkpoint: bool, +) -> None: + settings = make_settings(storage_enabled=True) + storage = build_market_data_storage(settings) + + assert storage is not None + + def unexpected_transport(**kwargs: object) -> None: + del kwargs + raise AssertionError("Transport graph must not be created.") + + monkeypatch.setattr( + runtime_factory, + "DzengiWebSocketTransport", + unexpected_transport, + ) + + with pytest.raises(RuntimeError, match="requires both"): + build_trade_stream_production_runtime( + settings, + trade_observation_sink=( + storage.trade_observation_sink + if include_sink + else None + ), + checkpoint_storage=( + storage.trade_repository + if include_checkpoint + else None + ), + ) + + assert storage.connection_pool.is_open is False + + +def test_disabled_storage_rejects_checkpoint_dependency() -> None: + settings = make_settings() + persistent_settings = make_settings(storage_enabled=True) + storage = build_market_data_storage(persistent_settings) + + assert storage is not None + + with pytest.raises(RuntimeError, match="requires enabled"): + build_trade_stream_production_runtime( + settings, + checkpoint_storage=storage.trade_repository, + ) def test_optional_storage_sink_is_shared_by_live_and_recovery() -> None: @@ -172,6 +314,8 @@ def test_applies_explicit_transport_and_runtime_settings() -> None: "ETH/USD_LEVERAGE", ) assert runtime_graph._runtime_scheduler.interval_seconds == 6.0 + assert runtime_graph._subscription_ack_timeout_seconds == 12.5 + assert runtime_graph._startup_market_buffer_capacity == 1_234 assert ( runtime_graph._runtime_supervisor._heartbeat_monitor.timeout_seconds == 31.0 diff --git a/app/tests/unit/core/test_config.py b/app/tests/unit/core/test_config.py index 6726d06..cad3bfc 100644 --- a/app/tests/unit/core/test_config.py +++ b/app/tests/unit/core/test_config.py @@ -15,6 +15,8 @@ _TRADE_STREAM_VARIABLES = ( "TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS", "TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS", "TRADE_STREAM_RECOVERY_WINDOW_MS", + "TRADE_STREAM_SUBSCRIPTION_ACK_TIMEOUT_SECONDS", + "TRADE_STREAM_STARTUP_MARKET_BUFFER_CAPACITY", ) _MARKET_DATA_STORAGE_VARIABLES = ( @@ -69,6 +71,8 @@ def test_trade_stream_is_disabled_by_default( assert settings.trade_stream.enabled is False assert settings.trade_stream.websocket_url == "" assert settings.trade_stream.symbols == () + assert settings.trade_stream.subscription_ack_timeout_seconds == 10.0 + assert settings.trade_stream.startup_market_buffer_capacity == 10_000 def test_disabled_trade_stream_ignores_dependent_values( @@ -76,6 +80,14 @@ def test_disabled_trade_stream_ignores_dependent_values( ) -> None: prepare_environment(monkeypatch) monkeypatch.setenv("TRADE_STREAM_OPEN_TIMEOUT_SECONDS", "invalid") + monkeypatch.setenv( + "TRADE_STREAM_SUBSCRIPTION_ACK_TIMEOUT_SECONDS", + "invalid", + ) + monkeypatch.setenv( + "TRADE_STREAM_STARTUP_MARKET_BUFFER_CAPACITY", + "invalid", + ) monkeypatch.setenv("TRADE_STREAM_SYMBOLS", ",") settings = load_settings() @@ -144,7 +156,7 @@ def test_enabled_trade_stream_parses_independent_settings( enable_trade_stream(monkeypatch) monkeypatch.setenv( "TRADE_STREAM_SYMBOLS", - " ETH/USD_LEVERAGE, BTC/USD_LEVERAGE,ETH/USD_LEVERAGE ", + " eth/usd_leverage, BTC/USD_LEVERAGE,ETH/USD_LEVERAGE ", ) monkeypatch.setenv("TRADE_STREAM_OPEN_TIMEOUT_SECONDS", "11.5") monkeypatch.setenv("TRADE_STREAM_PROBE_TIMEOUT_SECONDS", "21") @@ -152,6 +164,14 @@ def test_enabled_trade_stream_parses_independent_settings( monkeypatch.setenv("TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS", "31") monkeypatch.setenv("TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS", "6") monkeypatch.setenv("TRADE_STREAM_RECOVERY_WINDOW_MS", "123456") + monkeypatch.setenv( + "TRADE_STREAM_SUBSCRIPTION_ACK_TIMEOUT_SECONDS", + "12.5", + ) + monkeypatch.setenv( + "TRADE_STREAM_STARTUP_MARKET_BUFFER_CAPACITY", + "1234", + ) settings = load_settings() trade_stream = settings.trade_stream @@ -168,6 +188,20 @@ def test_enabled_trade_stream_parses_independent_settings( assert trade_stream.heartbeat_timeout_seconds == 31.0 assert trade_stream.scheduler_interval_seconds == 6.0 assert trade_stream.recovery_window_ms == 123_456 + assert trade_stream.subscription_ack_timeout_seconds == 12.5 + assert trade_stream.startup_market_buffer_capacity == 1_234 + + +def test_enabled_trade_stream_uses_startup_boundary_defaults( + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepare_environment(monkeypatch) + enable_trade_stream(monkeypatch) + + trade_stream = load_settings().trade_stream + + assert trade_stream.subscription_ack_timeout_seconds == 10.0 + assert trade_stream.startup_market_buffer_capacity == 10_000 @pytest.mark.parametrize( @@ -180,6 +214,12 @@ def test_enabled_trade_stream_parses_independent_settings( ("TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS", "invalid"), ("TRADE_STREAM_RECOVERY_WINDOW_MS", "1.5"), ("TRADE_STREAM_RECOVERY_WINDOW_MS", "0"), + ("TRADE_STREAM_SUBSCRIPTION_ACK_TIMEOUT_SECONDS", "0"), + ("TRADE_STREAM_SUBSCRIPTION_ACK_TIMEOUT_SECONDS", "nan"), + ("TRADE_STREAM_SUBSCRIPTION_ACK_TIMEOUT_SECONDS", "invalid"), + ("TRADE_STREAM_STARTUP_MARKET_BUFFER_CAPACITY", "0"), + ("TRADE_STREAM_STARTUP_MARKET_BUFFER_CAPACITY", "-1"), + ("TRADE_STREAM_STARTUP_MARKET_BUFFER_CAPACITY", "1.5"), ), ) def test_enabled_trade_stream_rejects_invalid_numeric_settings( diff --git a/app/tests/unit/market_data/acquisition/checkpoint/test_trade_stream_state_hydrator.py b/app/tests/unit/market_data/acquisition/checkpoint/test_trade_stream_state_hydrator.py new file mode 100644 index 0000000..fef91a8 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/checkpoint/test_trade_stream_state_hydrator.py @@ -0,0 +1,557 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +import pytest + +from src.market_data.acquisition.checkpoint.trade_stream_state_hydrator import ( + TradeStreamStateHydrator, + TradeStreamStateHydratorProtocol, +) +from src.market_data.acquisition.consistency.trade_stream_state_store import ( + TradeStreamStateStore, +) +from src.market_data.acquisition.consistency.trade_stream_state_store_exceptions import ( + TradeStreamStateStoreInitializationError, +) +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.storage.contracts import ( + PersistentTradeCheckpoint, + TradeCheckpointStorageProtocol, +) +from src.market_data.storage.exceptions import ( + MarketDataCheckpointIntegrityError, +) + + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +SECOND_SYMBOL = "ETH/USD_LEVERAGE" +BASE_TIME = datetime(2026, 8, 1, 10, 0, tzinfo=timezone.utc) + + +def make_trade( + *, + symbol: str = SYMBOL, + trade_id: int = 100, + second: int = 0, + source: str = "dzengi_websocket_trade", +) -> Trade: + return Trade( + symbol=symbol, + trade_id=trade_id, + price=Decimal("65000.25"), + quantity=Decimal("0.001"), + executed_at=BASE_TIME + timedelta(seconds=second), + aggressor_side=TradeAggressorSide.BUY, + source=source, + ) + + +def make_checkpoint( + trade: Trade, + *, + venue: str = VENUE, + revision: int = 1, +) -> PersistentTradeCheckpoint: + return PersistentTradeCheckpoint( + venue=venue, + trade=trade, + revision=revision, + updated_at=BASE_TIME + timedelta(minutes=1), + ) + + +CheckpointResult = PersistentTradeCheckpoint | None | BaseException +TailResult = tuple[Trade, ...] | object | BaseException +AdoptionResult = PersistentTradeCheckpoint | BaseException + + +class RecordingCheckpointStorage: + """Настраиваемый fake полного checkpoint-контракта.""" + + def __init__( + self, + *, + checkpoint_results: dict[ + str, + tuple[CheckpointResult, ...], + ] + | None = None, + checkpoint_tails: dict[str, TailResult] | None = None, + latest_tails: dict[str, TailResult] | None = None, + adoption_results: dict[str, AdoptionResult] | None = None, + ) -> None: + self.checkpoint_results = checkpoint_results or {} + self.checkpoint_tails = checkpoint_tails or {} + self.latest_tails = latest_tails or {} + self.adoption_results = adoption_results or {} + self.calls: list[tuple[object, ...]] = [] + self._checkpoint_offsets: dict[str, int] = {} + + @property + def operation_names(self) -> list[str]: + return [str(call[0]) for call in self.calls] + + def load_checkpoint( + self, + *, + venue: str, + symbol: str, + ) -> PersistentTradeCheckpoint | None: + self.calls.append(("load_checkpoint", venue, symbol)) + results = self.checkpoint_results.get(symbol, (None,)) + offset = self._checkpoint_offsets.get(symbol, 0) + self._checkpoint_offsets[symbol] = offset + 1 + result = results[min(offset, len(results) - 1)] + + if isinstance(result, BaseException): + raise result + + return result + + def load_checkpoint_tail( + self, + *, + venue: str, + checkpoint: PersistentTradeCheckpoint, + limit: int, + ) -> tuple[Trade, ...]: + self.calls.append( + ( + "load_checkpoint_tail", + venue, + checkpoint, + limit, + ) + ) + result = self.checkpoint_tails.get( + checkpoint.trade.symbol, + (checkpoint.trade,), + ) + + if isinstance(result, BaseException): + raise result + + return result # type: ignore[return-value] + + def load_latest_trade_tail( + self, + *, + venue: str, + symbol: str, + limit: int, + ) -> tuple[Trade, ...]: + self.calls.append( + ("load_latest_trade_tail", venue, symbol, limit) + ) + result = self.latest_tails.get(symbol, ()) + + if isinstance(result, BaseException): + raise result + + return result # type: ignore[return-value] + + def adopt_existing_trade_as_checkpoint( + self, + *, + venue: str, + trade: Trade, + ) -> PersistentTradeCheckpoint: + self.calls.append( + ("adopt_existing_trade_as_checkpoint", venue, trade) + ) + result = self.adoption_results.get( + trade.symbol, + make_checkpoint(trade, venue=venue), + ) + + if isinstance(result, BaseException): + raise result + + return result + + def store_trade_and_advance_checkpoint( + self, + *, + venue: str, + expected_trade: Trade | None, + trade: Trade, + observed_at: datetime, + ) -> PersistentTradeCheckpoint: + raise AssertionError( + "Hydrator не должен использовать writer checkpoint." + ) + + +def make_hydrator( + storage: RecordingCheckpointStorage, + *, + state_store: TradeStreamStateStore | None = None, + window_size: int = 3, +) -> tuple[TradeStreamStateHydrator, TradeStreamStateStore]: + store = state_store or TradeStreamStateStore() + return ( + TradeStreamStateHydrator( + checkpoint_storage=storage, + state_store=store, + venue=VENUE, + deduplication_window_size=window_size, + ), + store, + ) + + +def test_implements_protocol_uses_slots_and_constructor_has_no_io() -> None: + storage = RecordingCheckpointStorage() + + hydrator, _ = make_hydrator(storage) + + assert isinstance(storage, TradeCheckpointStorageProtocol) + assert isinstance(hydrator, TradeStreamStateHydratorProtocol) + assert not hasattr(hydrator, "__dict__") + assert storage.calls == [] + + +def test_existing_checkpoint_restores_bounded_tail_and_publishes_state() -> None: + trades = ( + make_trade(trade_id=100, second=0), + make_trade(trade_id=101, second=1), + make_trade(trade_id=102, second=2), + ) + checkpoint = make_checkpoint(trades[-1], revision=7) + storage = RecordingCheckpointStorage( + checkpoint_results={SYMBOL: (checkpoint,)}, + checkpoint_tails={SYMBOL: trades}, + ) + hydrator, store = make_hydrator(storage) + + states = hydrator.hydrate(symbols=(SYMBOL,)) + + assert len(states) == 1 + assert states[0].last_trade is trades[-1] + assert states[0].last_trade_id == 102 + assert store.get(SYMBOL) is states[0] + assert storage.calls == [ + ("load_checkpoint", VENUE, SYMBOL), + ("load_checkpoint_tail", VENUE, checkpoint, 3), + ] + + +def test_first_adoption_reloads_tail_before_publishing_state() -> None: + candidate_tail = ( + make_trade(trade_id=100, second=0), + make_trade(trade_id=101, second=1), + ) + adopted = make_checkpoint(candidate_tail[-1]) + reloaded_tail = ( + make_trade( + trade_id=99, + second=-1, + source="postgres_trade_history", + ), + make_trade( + trade_id=100, + second=0, + source="postgres_trade_history", + ), + make_trade( + trade_id=101, + second=1, + source="postgres_trade_history", + ), + ) + storage = RecordingCheckpointStorage( + checkpoint_results={SYMBOL: (None,)}, + latest_tails={SYMBOL: candidate_tail}, + adoption_results={SYMBOL: adopted}, + checkpoint_tails={SYMBOL: reloaded_tail}, + ) + hydrator, store = make_hydrator(storage) + + (state,) = hydrator.hydrate(symbols=(SYMBOL,)) + + assert state.last_trade is reloaded_tail[-1] + assert store.get(SYMBOL) is state + assert storage.calls == [ + ("load_checkpoint", VENUE, SYMBOL), + ("load_latest_trade_tail", VENUE, SYMBOL, 3), + ( + "adopt_existing_trade_as_checkpoint", + VENUE, + candidate_tail[-1], + ), + ("load_checkpoint_tail", VENUE, adopted, 3), + ] + + +def test_empty_history_rechecks_checkpoint_and_uses_concurrent_value() -> None: + trade = make_trade(trade_id=100) + concurrent_checkpoint = make_checkpoint(trade, revision=2) + storage = RecordingCheckpointStorage( + checkpoint_results={ + SYMBOL: (None, concurrent_checkpoint), + }, + latest_tails={SYMBOL: ()}, + checkpoint_tails={SYMBOL: (trade,)}, + ) + hydrator, store = make_hydrator(storage) + + (state,) = hydrator.hydrate(symbols=(SYMBOL,)) + + assert state.last_trade is trade + assert store.get(SYMBOL) is state + assert storage.calls == [ + ("load_checkpoint", VENUE, SYMBOL), + ("load_latest_trade_tail", VENUE, SYMBOL, 3), + ("load_checkpoint", VENUE, SYMBOL), + ( + "load_checkpoint_tail", + VENUE, + concurrent_checkpoint, + 3, + ), + ] + + +def test_empty_history_without_checkpoint_publishes_empty_state() -> None: + storage = RecordingCheckpointStorage( + checkpoint_results={SYMBOL: (None, None)}, + latest_tails={SYMBOL: ()}, + ) + hydrator, store = make_hydrator(storage) + + (state,) = hydrator.hydrate(symbols=(SYMBOL,)) + + assert state.last_trade is None + assert state.last_trade_id is None + assert store.get(SYMBOL) is state + assert storage.operation_names == [ + "load_checkpoint", + "load_latest_trade_tail", + "load_checkpoint", + ] + + +@pytest.mark.parametrize( + "malformed_tail", + ( + None, + [], + ), +) +def test_rejects_falsey_non_tuple_latest_tail( + malformed_tail: object, +) -> None: + storage = RecordingCheckpointStorage( + checkpoint_results={SYMBOL: (None, None)}, + latest_tails={SYMBOL: malformed_tail}, + ) + hydrator, store = make_hydrator(storage) + + with pytest.raises( + MarketDataCheckpointIntegrityError, + match="non-tuple latest Trade tail", + ): + hydrator.hydrate(symbols=(SYMBOL,)) + + assert store.is_empty() is True + assert storage.operation_names == [ + "load_checkpoint", + "load_latest_trade_tail", + ] + + +@pytest.mark.parametrize( + ("previous_trade_id", "checkpoint_trade_id"), + ( + (2**31 - 1, -(2**31)), + (-1, 0), + ), +) +def test_restores_tail_across_signed_rollover( + previous_trade_id: int, + checkpoint_trade_id: int, +) -> None: + trades = ( + make_trade(trade_id=previous_trade_id, second=0), + make_trade(trade_id=checkpoint_trade_id, second=1), + ) + checkpoint = make_checkpoint(trades[-1]) + storage = RecordingCheckpointStorage( + checkpoint_results={SYMBOL: (checkpoint,)}, + checkpoint_tails={SYMBOL: trades}, + ) + hydrator, _ = make_hydrator(storage) + + (state,) = hydrator.hydrate(symbols=(SYMBOL,)) + + assert state.last_trade is trades[-1] + assert state.last_trade_id == checkpoint_trade_id + + +def test_rejects_tail_that_does_not_end_at_checkpoint() -> None: + checkpoint_trade = make_trade(trade_id=102, second=2) + checkpoint = make_checkpoint(checkpoint_trade) + storage = RecordingCheckpointStorage( + checkpoint_results={SYMBOL: (checkpoint,)}, + checkpoint_tails={ + SYMBOL: ( + make_trade(trade_id=100, second=0), + make_trade(trade_id=101, second=1), + ) + }, + ) + hydrator, store = make_hydrator(storage) + + with pytest.raises( + MarketDataCheckpointIntegrityError, + match="does not end", + ): + hydrator.hydrate(symbols=(SYMBOL,)) + + assert store.contains(SYMBOL) is False + + +def test_rejects_tail_larger_than_deduplication_window() -> None: + trades = tuple( + make_trade(trade_id=trade_id, second=trade_id - 100) + for trade_id in range(100, 103) + ) + checkpoint = make_checkpoint(trades[-1]) + storage = RecordingCheckpointStorage( + checkpoint_results={SYMBOL: (checkpoint,)}, + checkpoint_tails={SYMBOL: trades}, + ) + hydrator, store = make_hydrator(storage, window_size=2) + + with pytest.raises( + MarketDataCheckpointIntegrityError, + match="cannot hydrate", + ): + hydrator.hydrate(symbols=(SYMBOL,)) + + assert store.contains(SYMBOL) is False + + +@pytest.mark.parametrize( + "malformed_tail", + ( + [make_trade()], + (object(),), + ), +) +def test_rejects_malformed_checkpoint_tail( + malformed_tail: object, +) -> None: + checkpoint = make_checkpoint(make_trade()) + storage = RecordingCheckpointStorage( + checkpoint_results={SYMBOL: (checkpoint,)}, + checkpoint_tails={SYMBOL: malformed_tail}, + ) + hydrator, store = make_hydrator(storage) + + with pytest.raises(MarketDataCheckpointIntegrityError): + hydrator.hydrate(symbols=(SYMBOL,)) + + assert store.contains(SYMBOL) is False + + +def test_storage_error_is_not_swallowed_and_state_is_not_published() -> None: + failure = RuntimeError("storage failed") + storage = RecordingCheckpointStorage( + checkpoint_results={SYMBOL: (failure,)}, + ) + hydrator, store = make_hydrator(storage) + + with pytest.raises(RuntimeError, match="storage failed"): + hydrator.hydrate(symbols=(SYMBOL,)) + + assert store.contains(SYMBOL) is False + assert storage.operation_names == ["load_checkpoint"] + + +def test_adoption_error_is_not_swallowed_and_state_is_not_published() -> None: + trade = make_trade() + failure = RuntimeError("adoption failed") + storage = RecordingCheckpointStorage( + checkpoint_results={SYMBOL: (None,)}, + latest_tails={SYMBOL: (trade,)}, + adoption_results={SYMBOL: failure}, + ) + hydrator, store = make_hydrator(storage) + + with pytest.raises(RuntimeError, match="adoption failed"): + hydrator.hydrate(symbols=(SYMBOL,)) + + assert store.contains(SYMBOL) is False + assert storage.operation_names == [ + "load_checkpoint", + "load_latest_trade_tail", + "adopt_existing_trade_as_checkpoint", + ] + + +def test_rejects_adopted_checkpoint_for_different_candidate() -> None: + candidate = make_trade(trade_id=100) + different_trade = make_trade(trade_id=101, second=1) + storage = RecordingCheckpointStorage( + checkpoint_results={SYMBOL: (None,)}, + latest_tails={SYMBOL: (candidate,)}, + adoption_results={ + SYMBOL: make_checkpoint(different_trade), + }, + ) + hydrator, store = make_hydrator(storage) + + with pytest.raises( + MarketDataCheckpointIntegrityError, + match="does not match candidate", + ): + hydrator.hydrate(symbols=(SYMBOL,)) + + assert store.contains(SYMBOL) is False + assert "load_checkpoint_tail" not in storage.operation_names + + +def test_second_symbol_failure_does_not_publish_first_state() -> None: + first_trade = make_trade(symbol=SYMBOL) + first_checkpoint = make_checkpoint(first_trade) + second_failure = RuntimeError("second symbol failed") + storage = RecordingCheckpointStorage( + checkpoint_results={ + SYMBOL: (first_checkpoint,), + SECOND_SYMBOL: (second_failure,), + }, + checkpoint_tails={SYMBOL: (first_trade,)}, + ) + hydrator, store = make_hydrator(storage) + + with pytest.raises(RuntimeError, match="second symbol failed"): + hydrator.hydrate(symbols=(SYMBOL, SECOND_SYMBOL)) + + assert store.contains(SYMBOL) is False + assert store.contains(SECOND_SYMBOL) is False + assert storage.operation_names == [ + "load_checkpoint", + "load_checkpoint_tail", + "load_checkpoint", + ] + + +def test_nonempty_store_is_rejected_before_storage_io() -> None: + storage = RecordingCheckpointStorage() + store = TradeStreamStateStore() + existing_state = store.get_or_create(SYMBOL) + hydrator, _ = make_hydrator(storage, state_store=store) + + with pytest.raises(TradeStreamStateStoreInitializationError): + hydrator.hydrate(symbols=(SYMBOL, SECOND_SYMBOL)) + + assert store.get(SYMBOL) is existing_state + assert store.contains(SECOND_SYMBOL) is False + assert storage.calls == [] diff --git a/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_consistency_controller.py b/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_consistency_controller.py index a99d921..9d0a45c 100644 --- a/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_consistency_controller.py +++ b/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_consistency_controller.py @@ -78,12 +78,27 @@ class RecordingTradeObservationSink: ) -> None: self.error = error self.observations: list[Trade] = [] + self.accepted: list[tuple[Trade, Trade | None]] = [] + self.duplicates: list[Trade] = [] - def persist( + def persist_accepted( + self, + trade: Trade, + *, + expected_trade: Trade | None, + ) -> None: + self.observations.append(trade) + self.accepted.append((trade, expected_trade)) + + if self.error is not None: + raise self.error + + def persist_duplicate( self, trade: Trade, ) -> None: self.observations.append(trade) + self.duplicates.append(trade) if self.error is not None: raise self.error @@ -221,6 +236,8 @@ def test_persists_trade_before_advancing_checkpoint( assert result is trade assert sink.observations == [trade] + assert sink.accepted == [(trade, None)] + assert sink.duplicates == [] assert state.last_trade is trade @@ -247,6 +264,10 @@ def test_persistence_failure_leaves_checkpoint_unchanged( state = state_store.get(first_trade.symbol) assert error_info.value is storage_error + assert sink.accepted == [ + (first_trade, None), + (failed_trade, first_trade), + ] assert state.last_trade is first_trade assert state.last_trade_id == first_trade.trade_id assert failed_trade.trade_id not in state._trades @@ -280,10 +301,59 @@ def test_valid_duplicate_is_persisted_without_checkpoint_advance( websocket_trade, rest_duplicate, ] + assert sink.accepted == [(websocket_trade, None)] + assert sink.duplicates == [rest_duplicate] assert state.last_trade is websocket_trade assert state.last_trade_id == websocket_trade.trade_id +def test_duplicate_persistence_failure_keeps_checkpoint( + state_store: TradeStreamStateStore, +) -> None: + storage_error = RuntimeError("storage failed") + sink = RecordingTradeObservationSink() + controller = TradeStreamConsistencyController( + state_store=state_store, + trade_observation_sink=sink, + ) + original = _trade(source="dzengi_websocket_trade") + duplicate = _trade(source="dzengi") + controller.accept(original) + sink.error = storage_error + + with pytest.raises(RuntimeError, match="storage failed") as error_info: + controller.accept(duplicate) + + state = state_store.get(original.symbol) + + assert error_info.value is storage_error + assert sink.accepted == [(original, None)] + assert sink.duplicates == [duplicate] + assert state.last_trade is original + assert state.last_trade_id == original.trade_id + + +def test_rollover_advance_uses_previous_trade_as_expected_checkpoint( + state_store: TradeStreamStateStore, +) -> None: + sink = RecordingTradeObservationSink() + controller = TradeStreamConsistencyController( + state_store=state_store, + trade_observation_sink=sink, + ) + previous = _trade(trade_id=2**31 - 1) + current = _trade(trade_id=-(2**31)) + + controller.accept(previous) + controller.accept(current) + + assert sink.accepted == [ + (previous, None), + (current, previous), + ] + assert sink.duplicates == [] + + def test_invalid_trades_do_not_reach_persistence_sink( state_store: TradeStreamStateStore, ) -> None: diff --git a/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state.py b/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state.py index 66dd6df..a0c07b7 100644 --- a/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state.py +++ b/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state.py @@ -420,6 +420,88 @@ def test_checkpoint_trade_id_matches_last_trade_id() -> None: assert state.last_trade.trade_id == state.last_trade_id +def test_checkpoint_callback_receives_expected_previous_trade() -> None: + state = TradeStreamState(symbol="BTCUSD") + first_trade = _trade(trade_id=100) + second_trade = _trade(trade_id=101) + calls: list[tuple[Trade, Trade | None]] = [] + + def before_checkpoint( + trade: Trade, + *, + expected_trade: Trade | None, + ) -> None: + calls.append((trade, expected_trade)) + assert state.last_trade is expected_trade + + state.accept( + first_trade, + before_checkpoint=before_checkpoint, + ) + state.accept( + second_trade, + before_checkpoint=before_checkpoint, + ) + + assert calls == [ + (first_trade, None), + (second_trade, first_trade), + ] + assert state.last_trade is second_trade + + +def test_duplicate_uses_only_duplicate_callback() -> None: + state = TradeStreamState( + symbol="BTCUSD", + deduplication_window_size=3, + ) + first_trade = _trade(trade_id=100) + latest_trade = _trade(trade_id=101) + duplicate = _trade( + trade_id=100, + source="dzengi_websocket_trade", + ) + checkpoint_calls: list[tuple[Trade, Trade | None]] = [] + duplicate_calls: list[Trade] = [] + + state.accept(first_trade) + state.accept(latest_trade) + result = state.accept( + duplicate, + before_checkpoint=lambda trade, expected_trade: ( + checkpoint_calls.append((trade, expected_trade)) + ), + on_duplicate=duplicate_calls.append, + ) + + assert result is None + assert checkpoint_calls == [] + assert duplicate_calls == [duplicate] + assert state.last_trade is latest_trade + + +def test_duplicate_callback_failure_keeps_checkpoint() -> None: + state = TradeStreamState(symbol="BTCUSD") + original = _trade(source="dzengi_websocket_trade") + duplicate = _trade(source="dzengi") + storage_error = RuntimeError("storage failed") + state.accept(original) + + def fail_duplicate(trade: Trade) -> None: + assert trade is duplicate + raise storage_error + + with pytest.raises(RuntimeError, match="storage failed") as error_info: + state.accept( + duplicate, + on_duplicate=fail_duplicate, + ) + + assert error_info.value is storage_error + assert state.last_trade is original + assert state.last_trade_id == original.trade_id + + def test_rejects_empty_symbol() -> None: with pytest.raises(ValueError): TradeStreamState(symbol="") @@ -440,3 +522,184 @@ def test_rejects_non_positive_window_size( symbol="BTCUSD", deduplication_window_size=window_size, ) + + +def test_from_history_builds_empty_state() -> None: + state = TradeStreamState.from_history( + symbol="BTCUSD", + trades=(), + deduplication_window_size=3, + ) + + assert state.symbol == "BTCUSD" + assert state.deduplication_window_size == 3 + assert state.last_trade_id is None + assert state.last_trade is None + + +def test_from_history_restores_valid_deduplication_window() -> None: + first_trade = _trade(trade_id=100) + second_trade = _trade(trade_id=101) + last_trade = _trade(trade_id=102) + + state = TradeStreamState.from_history( + symbol="BTCUSD", + trades=( + first_trade, + second_trade, + last_trade, + ), + deduplication_window_size=3, + ) + + assert state.last_trade is last_trade + assert state.last_trade_id == last_trade.trade_id + assert state.accept(_trade(trade_id=100)) is None + assert state.last_trade is last_trade + + +def test_from_history_keeps_exact_window_boundary() -> None: + first_trade = _trade(trade_id=100) + second_trade = _trade(trade_id=101) + third_trade = _trade(trade_id=102) + + state = TradeStreamState.from_history( + symbol="BTCUSD", + trades=( + first_trade, + second_trade, + third_trade, + ), + deduplication_window_size=3, + ) + + next_trade = _trade(trade_id=103) + + assert state.accept(next_trade) is next_trade + assert state.accept(_trade(trade_id=101)) is None + + with pytest.raises(TradeOrderingError): + state.accept(_trade(trade_id=100)) + + +@pytest.mark.parametrize( + ("first_trade_id", "next_trade_id"), + ( + (2**31 - 1, -(2**31)), + (-1, 0), + ), +) +def test_from_history_restores_signed_rollover_sequence( + first_trade_id: int, + next_trade_id: int, +) -> None: + first_trade = _trade(trade_id=first_trade_id) + next_trade = _trade(trade_id=next_trade_id) + + state = TradeStreamState.from_history( + symbol="BTCUSD", + trades=(first_trade, next_trade), + deduplication_window_size=2, + ) + + assert state.last_trade is next_trade + assert state.last_trade_id == next_trade_id + assert state.accept(_trade(trade_id=first_trade_id)) is None + + +def test_from_history_accepts_next_trade_after_restoration() -> None: + last_history_trade = _trade(trade_id=101) + state = TradeStreamState.from_history( + symbol="BTCUSD", + trades=( + _trade(trade_id=100), + last_history_trade, + ), + deduplication_window_size=3, + ) + next_trade = _trade(trade_id=102) + + result = state.accept(next_trade) + + assert result is next_trade + assert state.last_trade is next_trade + assert state.last_trade_id == next_trade.trade_id + + +def test_from_history_rejects_identical_duplicate_strictly() -> None: + original = _trade(trade_id=100) + duplicate = _trade( + trade_id=100, + source="dzengi_websocket_trade", + ) + + with pytest.raises( + TradeConsistencyError, + match="duplicate Trade", + ): + TradeStreamState.from_history( + symbol="BTCUSD", + trades=(original, duplicate), + deduplication_window_size=2, + ) + + +def test_from_history_rejects_conflicting_trade_id() -> None: + original = _trade( + trade_id=100, + price=Decimal("50000.00"), + ) + conflict = _trade( + trade_id=100, + price=Decimal("50001.00"), + ) + + with pytest.raises(TradeConsistencyError): + TradeStreamState.from_history( + symbol="BTCUSD", + trades=(original, conflict), + deduplication_window_size=2, + ) + + +def test_from_history_rejects_reverse_sequence() -> None: + with pytest.raises(TradeOrderingError): + TradeStreamState.from_history( + symbol="BTCUSD", + trades=( + _trade(trade_id=101), + _trade(trade_id=100), + ), + deduplication_window_size=2, + ) + + +def test_from_history_rejects_half_cycle_sequence() -> None: + with pytest.raises( + ValueError, + match="exactly half", + ): + TradeStreamState.from_history( + symbol="BTCUSD", + trades=( + _trade(trade_id=0), + _trade(trade_id=-(2**31)), + ), + deduplication_window_size=2, + ) + + +def test_from_history_rejects_history_larger_than_window() -> None: + with pytest.raises( + ValueError, + match="fit deduplication_window_size", + ): + TradeStreamState.from_history( + symbol="BTCUSD", + trades=( + _trade(trade_id=100), + _trade(trade_id=101), + _trade(trade_id=102), + ), + deduplication_window_size=2, + ) diff --git a/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state_store.py b/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state_store.py index 5a6cb59..4a0aec4 100644 --- a/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state_store.py +++ b/app/tests/unit/market_data/acquisition/consistency/test_trade_stream_state_store.py @@ -2,6 +2,8 @@ from __future__ import annotations +from typing import cast + import pytest from src.market_data.acquisition.consistency.trade_stream_state import ( @@ -12,6 +14,7 @@ from src.market_data.acquisition.consistency.trade_stream_state_store import ( ) from src.market_data.acquisition.consistency.trade_stream_state_store_exceptions import ( TradeStreamStateNotFoundError, + TradeStreamStateStoreInitializationError, ) from src.market_data.acquisition.consistency.trade_stream_state_store_protocol import ( TradeStreamStateStoreProtocol, @@ -140,4 +143,106 @@ def test_clear_is_idempotent_for_empty_store() -> None: store.clear() store.clear() - assert store.contains("BTCUSD") is False \ No newline at end of file + assert store.contains("BTCUSD") is False + + +def test_initialize_publishes_all_states_preserving_identity() -> None: + store = TradeStreamStateStore() + btc_state = TradeStreamState(symbol="BTCUSD") + eth_state = TradeStreamState(symbol="ETHUSD") + + store.initialize((btc_state, eth_state)) + + assert store.get("BTCUSD") is btc_state + assert store.get("ETHUSD") is eth_state + + +def test_initialize_rejects_duplicate_symbols_atomically() -> None: + store = TradeStreamStateStore() + + with pytest.raises( + TradeStreamStateStoreInitializationError, + match="повторно", + ): + store.initialize( + ( + TradeStreamState(symbol="BTCUSD"), + TradeStreamState(symbol="BTCUSD"), + ) + ) + + assert store.contains("BTCUSD") is False + + +def test_initialize_rejects_invalid_item_atomically() -> None: + store = TradeStreamStateStore() + + with pytest.raises( + TypeError, + match="TradeStreamState", + ): + store.initialize( + cast( + tuple[TradeStreamState, ...], + ( + TradeStreamState(symbol="BTCUSD"), + object(), + ), + ), + ) + + assert store.contains("BTCUSD") is False + + +def test_initialize_rejects_non_empty_store() -> None: + store = TradeStreamStateStore() + existing_state = store.get_or_create("BTCUSD") + + with pytest.raises( + TradeStreamStateStoreInitializationError, + match="уже содержит", + ): + store.initialize( + (TradeStreamState(symbol="ETHUSD"),) + ) + + assert store.get("BTCUSD") is existing_state + assert store.contains("ETHUSD") is False + + +def test_initialize_can_be_called_only_once() -> None: + store = TradeStreamStateStore() + original_state = TradeStreamState(symbol="BTCUSD") + store.initialize((original_state,)) + + with pytest.raises( + TradeStreamStateStoreInitializationError, + match="уже содержит", + ): + store.initialize( + (TradeStreamState(symbol="ETHUSD"),) + ) + + assert store.get("BTCUSD") is original_state + assert store.contains("ETHUSD") is False + + +def test_empty_initialize_is_still_one_time_initialization() -> None: + store = TradeStreamStateStore() + store.initialize(()) + + with pytest.raises(TradeStreamStateStoreInitializationError): + store.initialize(()) + + +def test_clear_allows_store_to_be_initialized_again() -> None: + store = TradeStreamStateStore() + store.initialize((TradeStreamState(symbol="BTCUSD"),)) + + store.clear() + + eth_state = TradeStreamState(symbol="ETHUSD") + store.initialize((eth_state,)) + + assert store.contains("BTCUSD") is False + assert store.get("ETHUSD") is eth_state diff --git a/app/tests/unit/market_data/acquisition/runtime/test_runtime_reconnect_recovery_coordinator.py b/app/tests/unit/market_data/acquisition/runtime/test_runtime_reconnect_recovery_coordinator.py index a923cab..cbe5e29 100644 --- a/app/tests/unit/market_data/acquisition/runtime/test_runtime_reconnect_recovery_coordinator.py +++ b/app/tests/unit/market_data/acquisition/runtime/test_runtime_reconnect_recovery_coordinator.py @@ -266,8 +266,8 @@ def test_reconnect_restore_boundary_and_recovery_order() -> None: order, ) = create_coordinator( symbols=( - f" {ETH} ", - BTC, + f" {ETH.lower()} ", + BTC.lower(), ETH, ), ) diff --git a/app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py b/app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py index 60763d4..327b477 100644 --- a/app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py +++ b/app/tests/unit/market_data/acquisition/runtime/test_runtime_recovery_coordinator.py @@ -119,6 +119,18 @@ class FakeStateStore: self.remove_calls: list[str] = [] self.clear_calls = 0 + def initialize( + self, + states: tuple[TradeStreamState, ...], + ) -> None: + self._states = { + state.symbol: state + for state in states + } + + def is_empty(self) -> bool: + return not self._states + def get_or_create( self, symbol: str, diff --git a/app/tests/unit/market_data/acquisition/runtime/test_runtime_startup_recovery_coordinator.py b/app/tests/unit/market_data/acquisition/runtime/test_runtime_startup_recovery_coordinator.py new file mode 100644 index 0000000..f01e107 --- /dev/null +++ b/app/tests/unit/market_data/acquisition/runtime/test_runtime_startup_recovery_coordinator.py @@ -0,0 +1,462 @@ +from __future__ import annotations + +import asyncio +import threading + +import pytest + +from src.market_data.acquisition.consistency.trade_stream_state import ( + TradeStreamState, +) +from src.market_data.acquisition.recovery.trade_recovery_result import ( + TradeRecoveryResult, +) +from src.market_data.acquisition.runtime.live_processing_gate import ( + RuntimeLiveProcessingGate, +) +from src.market_data.acquisition.runtime.runtime_startup_recovery_coordinator import ( + RuntimeStartupRecoveryCoordinator, + RuntimeStartupRecoveryProtocol, +) + + +BTC = "BTC/USD_LEVERAGE" +ETH = "ETH/USD_LEVERAGE" +RECOVERY_END_TIME_MS = 1_785_326_405_123 +HYDRATION_TASK_NAME = "trade-stream-state-hydration" +RECOVERY_TASK_NAME = "trade-stream-startup-recovery" + + +class FakeStateHydrator: + def __init__( + self, + *, + states: tuple[TradeStreamState, ...], + error: Exception | None = None, + started: threading.Event | None = None, + release: threading.Event | None = None, + ) -> None: + self._states = states + self._error = error + self._started = started + self._release = release + self.calls: list[tuple[str, ...]] = [] + self.thread_ids: list[int] = [] + + def hydrate( + self, + *, + symbols: tuple[str, ...], + ) -> tuple[TradeStreamState, ...]: + self.calls.append(symbols) + self.thread_ids.append(threading.get_ident()) + + if self._started is not None: + self._started.set() + + if self._release is not None and not self._release.wait( + timeout=2.0, + ): + raise AssertionError("hydration release was not signalled") + + if self._error is not None: + raise self._error + + return self._states + + +class FakeRecoveryCoordinator: + def __init__( + self, + *, + error: Exception | None = None, + started: threading.Event | None = None, + release: threading.Event | None = None, + ) -> None: + self._error = error + self._started = started + self._release = release + self.calls: list[tuple[str, int]] = [] + self.thread_ids: list[int] = [] + + def recover( + self, + *, + symbol: str, + recovery_end_time: int, + ) -> TradeRecoveryResult: + self.calls.append( + ( + symbol, + recovery_end_time, + ) + ) + self.thread_ids.append(threading.get_ident()) + + if self._started is not None: + self._started.set() + + if self._release is not None and not self._release.wait( + timeout=2.0, + ): + raise AssertionError("recovery release was not signalled") + + if self._error is not None: + raise self._error + + return TradeRecoveryResult( + symbol=symbol, + requested_start_time=recovery_end_time, + requested_end_time=recovery_end_time, + recovered_trades=(), + ) + + +class RecordingClock: + def __init__( + self, + value: object = RECOVERY_END_TIME_MS, + ) -> None: + self._value = value + self.calls = 0 + + def __call__(self) -> int: + self.calls += 1 + return self._value # type: ignore[return-value] + + +def create_coordinator( + *, + symbols: tuple[str, ...] = (BTC,), + states: tuple[TradeStreamState, ...] | None = None, + hydration_error: Exception | None = None, + hydration_started: threading.Event | None = None, + hydration_release: threading.Event | None = None, + recovery_error: Exception | None = None, + recovery_started: threading.Event | None = None, + recovery_release: threading.Event | None = None, + clock_value: object = RECOVERY_END_TIME_MS, +) -> tuple[ + RuntimeStartupRecoveryCoordinator, + FakeStateHydrator, + FakeRecoveryCoordinator, + RuntimeLiveProcessingGate, + RecordingClock, +]: + hydrated_states = states or ( + TradeStreamState(symbol=BTC), + ) + hydrator = FakeStateHydrator( + states=hydrated_states, + error=hydration_error, + started=hydration_started, + release=hydration_release, + ) + recovery = FakeRecoveryCoordinator( + error=recovery_error, + started=recovery_started, + release=recovery_release, + ) + gate = RuntimeLiveProcessingGate() + clock = RecordingClock(clock_value) + coordinator = RuntimeStartupRecoveryCoordinator( + state_hydrator=hydrator, + recovery_coordinator=recovery, + live_processing_gate=gate, + symbols=symbols, + clock=clock, + ) + return coordinator, hydrator, recovery, gate, clock + + +async def wait_until( + predicate: object, +) -> None: + for _ in range(100): + if callable(predicate) and predicate(): + return + + await asyncio.sleep(0) + + raise AssertionError("condition was not reached") + + +def test_implements_protocol_uses_slots_and_constructor_has_no_io() -> None: + coordinator, hydrator, recovery, gate, clock = create_coordinator( + symbols=( + f" {ETH.lower()} ", + BTC.lower(), + ETH, + ), + ) + + assert isinstance(coordinator, RuntimeStartupRecoveryProtocol) + assert not hasattr(coordinator, "__dict__") + assert coordinator.live_processing_gate is gate + assert coordinator.symbols == (BTC, ETH) + assert hydrator.calls == [] + assert recovery.calls == [] + assert clock.calls == 0 + assert gate.locked is False + + +@pytest.mark.parametrize( + ("symbols", "error_type"), + [ + ([], TypeError), + ((), ValueError), + (("", " "), ValueError), + ((BTC, 1), TypeError), + ], +) +def test_rejects_invalid_symbols( + symbols: object, + error_type: type[Exception], +) -> None: + with pytest.raises(error_type): + create_coordinator( + symbols=symbols, # type: ignore[arg-type] + ) + + +def test_hydration_is_idempotent_and_runs_off_event_loop() -> None: + async def scenario() -> tuple[ + tuple[TradeStreamState, ...], + tuple[TradeStreamState, ...], + int, + ]: + states = ( + TradeStreamState(symbol=BTC), + ) + coordinator, hydrator, *_ = create_coordinator( + states=states, + ) + event_loop_thread_id = threading.get_ident() + + first = await coordinator.hydrate_once() + second = await coordinator.hydrate_once() + + assert hydrator.calls == [(BTC,)] + assert len(hydrator.thread_ids) == 1 + return first, second, event_loop_thread_id + + first, second, event_loop_thread_id = asyncio.run(scenario()) + + assert first is second + assert first[0].symbol == BTC + coordinator, hydrator, *_ = create_coordinator() + asyncio.run(coordinator.hydrate_once()) + assert hydrator.thread_ids[0] != event_loop_thread_id + + +def test_recovery_uses_one_clock_boundary_and_symbol_order() -> None: + async def scenario() -> tuple[ + tuple[TradeRecoveryResult, ...], + FakeRecoveryCoordinator, + RecordingClock, + int, + ]: + coordinator, _, recovery, gate, clock = create_coordinator( + symbols=( + f" {ETH.lower()} ", + BTC.lower(), + ETH, + ), + ) + event_loop_thread_id = threading.get_ident() + + async with gate: + results = await coordinator.recover_after_ack() + + return results, recovery, clock, event_loop_thread_id + + results, recovery, clock, event_loop_thread_id = asyncio.run( + scenario() + ) + + assert tuple(result.symbol for result in results) == (BTC, ETH) + assert recovery.calls == [ + (BTC, RECOVERY_END_TIME_MS), + (ETH, RECOVERY_END_TIME_MS), + ] + assert clock.calls == 1 + assert len(set(recovery.thread_ids)) == 1 + assert recovery.thread_ids[0] != event_loop_thread_id + + +def test_cancelled_hydration_waits_and_caches_success() -> None: + async def scenario() -> tuple[ + RuntimeStartupRecoveryCoordinator, + FakeStateHydrator, + tuple[TradeStreamState, ...], + ]: + started = threading.Event() + release = threading.Event() + states = ( + TradeStreamState(symbol=BTC), + ) + coordinator, hydrator, *_ = create_coordinator( + states=states, + hydration_started=started, + hydration_release=release, + ) + task = asyncio.create_task( + coordinator.hydrate_once(), + ) + await wait_until(started.is_set) + joined_task = asyncio.create_task( + coordinator.hydrate_once(), + ) + await asyncio.sleep(0) + + task.cancel() + await asyncio.sleep(0) + assert task.done() is False + assert joined_task.done() is False + + release.set() + with pytest.raises(asyncio.CancelledError): + await task + + cached = await joined_task + assert await coordinator.hydrate_once() is cached + return coordinator, hydrator, cached + + coordinator, hydrator, cached = asyncio.run(scenario()) + + assert cached[0].symbol == BTC + assert hydrator.calls == [(BTC,)] + assert coordinator._hydration_task is None + + +def test_cancelled_recovery_waits_for_worker_completion() -> None: + async def scenario() -> tuple[ + RuntimeStartupRecoveryCoordinator, + FakeRecoveryCoordinator, + ]: + started = threading.Event() + release = threading.Event() + coordinator, _, recovery, gate, _ = create_coordinator( + recovery_started=started, + recovery_release=release, + ) + + async def run_recovery() -> None: + async with gate: + await coordinator.recover_after_ack() + + task = asyncio.create_task(run_recovery()) + await wait_until(started.is_set) + + task.cancel() + await asyncio.sleep(0) + assert task.done() is False + assert gate.locked is True + + release.set() + with pytest.raises(asyncio.CancelledError): + await task + + assert gate.locked is False + return coordinator, recovery + + coordinator, recovery = asyncio.run(scenario()) + + assert recovery.calls == [(BTC, RECOVERY_END_TIME_MS)] + assert coordinator._recovery_task is None + + +@pytest.mark.parametrize( + "operation", + [ + "hydration", + "recovery", + ], +) +def test_worker_error_identity_is_preserved( + operation: str, +) -> None: + error = RuntimeError(f"{operation} failed") + + async def scenario() -> RuntimeStartupRecoveryCoordinator: + coordinator, *_ = create_coordinator( + hydration_error=( + error if operation == "hydration" else None + ), + recovery_error=( + error if operation == "recovery" else None + ), + ) + + with pytest.raises(RuntimeError) as raised: + if operation == "hydration": + await coordinator.hydrate_once() + else: + await coordinator.recover_after_ack() + + assert raised.value is error + return coordinator + + coordinator = asyncio.run(scenario()) + + assert coordinator._hydration_task is None + assert coordinator._recovery_task is None + + +@pytest.mark.parametrize( + ("clock_value", "error_type"), + [ + (True, TypeError), + (1.5, TypeError), + ("1", TypeError), + (-1, ValueError), + ], +) +def test_invalid_clock_result_fails_before_recovery( + clock_value: object, + error_type: type[Exception], +) -> None: + async def scenario() -> tuple[ + FakeRecoveryCoordinator, + RecordingClock, + RuntimeStartupRecoveryCoordinator, + ]: + coordinator, _, recovery, _, clock = create_coordinator( + clock_value=clock_value, + ) + + with pytest.raises(error_type): + await coordinator.recover_after_ack() + + return recovery, clock, coordinator + + recovery, clock, coordinator = asyncio.run(scenario()) + + assert recovery.calls == [] + assert clock.calls == 1 + assert coordinator._recovery_task is None + + +def test_completed_operations_leave_no_owned_tasks() -> None: + async def scenario() -> None: + coordinator, *_ = create_coordinator() + + await coordinator.hydrate_once() + await coordinator.recover_after_ack() + + pending_owned_tasks = { + task.get_name() + for task in asyncio.all_tasks() + if task is not asyncio.current_task() + and not task.done() + and task.get_name() + in { + HYDRATION_TASK_NAME, + RECOVERY_TASK_NAME, + } + } + assert pending_owned_tasks == set() + assert coordinator._hydration_task is None + assert coordinator._recovery_task is None + + asyncio.run(scenario()) diff --git a/app/tests/unit/market_data/acquisition/runtime/test_trade_stream_production_runtime.py b/app/tests/unit/market_data/acquisition/runtime/test_trade_stream_production_runtime.py index d69fde1..f513971 100644 --- a/app/tests/unit/market_data/acquisition/runtime/test_trade_stream_production_runtime.py +++ b/app/tests/unit/market_data/acquisition/runtime/test_trade_stream_production_runtime.py @@ -24,10 +24,15 @@ from src.market_data.acquisition.consistency.trade_stream_consistency_controller from src.market_data.acquisition.consistency.trade_stream_state_store import ( TradeStreamStateStore, ) +from src.market_data.acquisition.consistency.trade_stream_state import ( + TradeStreamState, +) from src.market_data.acquisition.exceptions import ( WebSocketControlMessageError, WebSocketMessageDecodeError, WebSocketMessageRoutingError, + WebSocketStartupMarketBufferOverflowError, + WebSocketSubscriptionAckTimeoutError, WebSocketTransportError, ) from src.market_data.acquisition.models.trade import Trade @@ -37,6 +42,9 @@ from src.market_data.acquisition.runtime.runtime_events import ( DisconnectedEvent, MessageReceivedEvent, ) +from src.market_data.acquisition.recovery.trade_recovery_result import ( + TradeRecoveryResult, +) from src.market_data.acquisition.runtime.live_processing_gate import ( RuntimeLiveProcessingGate, ) @@ -66,6 +74,7 @@ from src.market_data.acquisition.runtime.websocket_protocol import ( from src.market_data.acquisition.trade_stream_acquisition_service import ( TradeStreamAcquisitionService, ) +from src.market_data.acquisition.symbols import normalize_symbol SYMBOL = "BTC/USD_LEVERAGE" @@ -153,6 +162,8 @@ class FakeTransport: self._incoming = deque(incoming) self._message_available = asyncio.Event() self.receive_calls = 0 + self.active_receivers = 0 + self.max_active_receivers = 0 self.probe_calls = 0 self.waiting = asyncio.Event() @@ -171,24 +182,32 @@ class FakeTransport: async def receive(self) -> str | bytes: self.receive_calls += 1 self._calls.append("transport.receive") + self.active_receivers += 1 + self.max_active_receivers = max( + self.max_active_receivers, + self.active_receivers, + ) try: - while not self._incoming: - self.waiting.set() - await self._message_available.wait() - self._message_available.clear() - except asyncio.CancelledError: - self._calls.append( - "transport.receive.cancelled", - ) - raise + try: + while not self._incoming: + self.waiting.set() + await self._message_available.wait() + self._message_available.clear() + except asyncio.CancelledError: + self._calls.append( + "transport.receive.cancelled", + ) + raise - result = self._incoming.popleft() + result = self._incoming.popleft() - if isinstance(result, BaseException): - raise result + if isinstance(result, BaseException): + raise result - return result + return result + finally: + self.active_receivers -= 1 def feed( self, @@ -302,9 +321,9 @@ class FakeReconnectRecoveryCoordinator: self._symbols = tuple( sorted( { - symbol.strip() + normalize_symbol(symbol) for symbol in valid_symbols - if symbol.strip() + if normalize_symbol(symbol) } ) ) @@ -369,6 +388,80 @@ class FakeReconnectRecoveryCoordinator: raise self._error +class FakeStartupRecoveryCoordinator: + def __init__( + self, + *, + calls: list[str], + live_processing_gate: RuntimeLiveProcessingGate, + symbols: tuple[str, ...], + hydration_error: Exception | None = None, + recovery_error: Exception | None = None, + hydration_release: asyncio.Event | None = None, + recovery_release: asyncio.Event | None = None, + ) -> None: + self._calls = calls + self._live_processing_gate = live_processing_gate + self._symbols = tuple( + sorted( + { + normalize_symbol(symbol) + for symbol in symbols + if normalize_symbol(symbol) + } + ) + ) + self._hydration_error = hydration_error + self._recovery_error = recovery_error + self._hydration_release = hydration_release + self._recovery_release = recovery_release + self.hydration_entered = asyncio.Event() + self.recovery_entered = asyncio.Event() + self.hydrate_calls = 0 + self.recover_calls = 0 + + @property + def live_processing_gate( + self, + ) -> RuntimeLiveProcessingGate: + return self._live_processing_gate + + @property + def symbols(self) -> tuple[str, ...]: + return self._symbols + + async def hydrate_once(self) -> tuple[TradeStreamState, ...]: + self.hydrate_calls += 1 + self._calls.append("startup_recovery.hydrate") + self.hydration_entered.set() + + if self._hydration_release is not None: + await self._hydration_release.wait() + + if self._hydration_error is not None: + raise self._hydration_error + + return () + + async def recover_after_ack(self) -> tuple[TradeRecoveryResult, ...]: + self.recover_calls += 1 + self._calls.append("startup_recovery.recover") + self.recovery_entered.set() + + if not self._live_processing_gate.locked: + raise AssertionError( + "Production Runtime должен удерживать startup gate." + ) + + if self._recovery_release is not None: + await self._recovery_release.wait() + + if self._recovery_error is not None: + raise self._recovery_error + + return () + + class FakeTradeStreamService: def __init__( self, @@ -623,6 +716,15 @@ class RuntimeDependencies: scheduler_uses_different_transport: bool = False, symbols: tuple[str, ...] = (SYMBOL,), recovery_symbols: tuple[str, ...] | None = None, + startup_recovery_enabled: bool = False, + startup_hydration_error: Exception | None = None, + startup_recovery_error: Exception | None = None, + startup_hydration_release: asyncio.Event | None = None, + startup_recovery_release: asyncio.Event | None = None, + startup_recovery_symbols: tuple[str, ...] | None = None, + startup_recovery_uses_different_gate: bool = False, + subscription_ack_timeout_seconds: float = 10.0, + startup_market_buffer_capacity: int = 10_000, ) -> None: self.calls: list[str] = [] self.session = FakeSession( @@ -663,6 +765,27 @@ class RuntimeDependencies: else recovery_symbols ), ) + self.startup_recovery = ( + FakeStartupRecoveryCoordinator( + calls=self.calls, + live_processing_gate=( + RuntimeLiveProcessingGate() + if startup_recovery_uses_different_gate + else self.live_processing_gate + ), + symbols=( + symbols + if startup_recovery_symbols is None + else startup_recovery_symbols + ), + hydration_error=startup_hydration_error, + recovery_error=startup_recovery_error, + hydration_release=startup_hydration_release, + recovery_release=startup_recovery_release, + ) + if startup_recovery_enabled + else None + ) self.supervisor = FakeRuntimeSupervisor( calls=self.calls, start_error=supervisor_start_error, @@ -711,6 +834,13 @@ class RuntimeDependencies: runtime_supervisor=self.supervisor, runtime_scheduler=self.scheduler, symbols=symbols, + startup_recovery_coordinator=self.startup_recovery, + subscription_ack_timeout_seconds=( + subscription_ack_timeout_seconds + ), + startup_market_buffer_capacity=( + startup_market_buffer_capacity + ), ) @@ -773,6 +903,21 @@ def test_rejects_invalid_symbols( ) +def test_canonicalizes_case_variants_across_runtime_graph() -> None: + dependencies = RuntimeDependencies( + symbols=( + f" {SYMBOL.lower()} ", + SYMBOL, + ), + startup_recovery_enabled=True, + ) + + assert dependencies.runtime._symbols == (SYMBOL,) + assert dependencies.reconnect_recovery.symbols == (SYMBOL,) + assert dependencies.startup_recovery is not None + assert dependencies.startup_recovery.symbols == (SYMBOL,) + + def test_rejects_symbols_different_from_recovery_coordinator() -> None: with pytest.raises( ValueError, @@ -804,6 +949,577 @@ def test_rejects_scheduler_with_different_transport() -> None: ) +def test_rejects_startup_recovery_with_different_gate() -> None: + with pytest.raises( + ValueError, + match="startup recovery coordinator must share one", + ): + RuntimeDependencies( + startup_recovery_enabled=True, + startup_recovery_uses_different_gate=True, + ) + + +def test_rejects_startup_recovery_with_different_symbols() -> None: + with pytest.raises( + ValueError, + match="startup recovery coordinator must use the same symbols", + ): + RuntimeDependencies( + startup_recovery_enabled=True, + startup_recovery_symbols=(ETH_SYMBOL,), + ) + + +@pytest.mark.parametrize( + ("timeout", "error_type"), + [ + (True, TypeError), + ("10", TypeError), + (0.0, ValueError), + (-1.0, ValueError), + (float("inf"), ValueError), + (float("nan"), ValueError), + ], +) +def test_rejects_invalid_subscription_ack_timeout( + timeout: object, + error_type: type[Exception], +) -> None: + with pytest.raises(error_type): + RuntimeDependencies( + subscription_ack_timeout_seconds=timeout, # type: ignore[arg-type] + ) + + +@pytest.mark.parametrize( + ("capacity", "error_type"), + [ + (True, TypeError), + (1.0, TypeError), + (0, ValueError), + (-1, ValueError), + ], +) +def test_rejects_invalid_startup_market_buffer_capacity( + capacity: object, + error_type: type[Exception], +) -> None: + with pytest.raises(error_type): + RuntimeDependencies( + startup_market_buffer_capacity=capacity, # type: ignore[arg-type] + ) + + +def test_startup_recovery_preserves_boundary_order_and_one_receiver() -> None: + first_document = { + "destination": "internal.trade", + "payload": { + "symbol": SYMBOL, + "sequence": 1, + }, + } + second_document = { + "destination": "internal.trade", + "payload": { + "symbol": SYMBOL, + "sequence": 2, + }, + } + post_ack_document = { + "destination": "internal.trade", + "payload": { + "symbol": SYMBOL, + "sequence": 3, + }, + } + + async def scenario() -> RuntimeDependencies: + recovery_release = asyncio.Event() + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + startup_recovery_release=recovery_release, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.subscribe_entered.wait() + correlation_id = ( + dependencies.service.subscribe_correlation_ids[0] + ) + assert isinstance(correlation_id, str) + + dependencies.transport.feed(json.dumps(first_document)) + dependencies.transport.feed(json.dumps(second_document)) + dependencies.transport.feed( + make_control_message( + correlation_id=correlation_id, + ) + ) + + startup_recovery = dependencies.startup_recovery + assert startup_recovery is not None + await startup_recovery.recovery_entered.wait() + + assert dependencies.live_processing_gate.locked is True + assert dependencies.service.documents == [] + assert dependencies.transport.receive_calls == 3 + assert dependencies.supervisor.start_calls == 0 + assert dependencies.scheduler.start_calls == 0 + + dependencies.transport.feed( + json.dumps(post_ack_document), + ) + await asyncio.sleep(0) + assert dependencies.transport.receive_calls == 3 + + recovery_release.set() + await dependencies.scheduler.started.wait() + await wait_until( + lambda: dependencies.service.documents + == [ + first_document, + second_document, + post_ack_document, + ], + ) + await wait_until( + lambda: dependencies.transport.receive_calls == 5, + ) + + assert dependencies.service.documents == [ + first_document, + second_document, + post_ack_document, + ] + assert dependencies.transport.max_active_receivers == 1 + + await dependencies.runtime.stop() + await runtime_task + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.calls.index( + "startup_recovery.hydrate" + ) < dependencies.calls.index("session.start") + assert dependencies.calls.index( + "service.subscribe" + ) < dependencies.calls.index("startup_recovery.recover") + assert dependencies.calls.index( + "startup_recovery.recover" + ) < dependencies.calls.index("service.handle_message") + assert dependencies.calls.index( + "service.handle_message" + ) < dependencies.calls.index("supervisor.start") + + +def test_startup_ack_timeout_is_terminal() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + subscription_ack_timeout_seconds=0.01, + ) + + with pytest.raises( + WebSocketSubscriptionAckTimeoutError, + match="Истекло время", + ): + await dependencies.runtime.run() + + return dependencies + + dependencies = asyncio.run(scenario()) + startup_recovery = dependencies.startup_recovery + + assert startup_recovery is not None + assert startup_recovery.recover_calls == 0 + assert dependencies.live_processing_gate.failed is True + assert dependencies.supervisor.start_calls == 0 + assert dependencies.scheduler.start_calls == 0 + assert dependencies.transport.max_active_receivers == 1 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_startup_hydration_failure_prevents_network_io() -> None: + hydration_error = RuntimeError("startup hydration failed") + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + startup_hydration_error=hydration_error, + ) + + with pytest.raises( + RuntimeError, + match="startup hydration failed", + ) as error_info: + await dependencies.runtime.run() + + assert error_info.value is hydration_error + return dependencies + + dependencies = asyncio.run(scenario()) + startup_recovery = dependencies.startup_recovery + + assert startup_recovery is not None + assert startup_recovery.hydrate_calls == 1 + assert startup_recovery.recover_calls == 0 + assert dependencies.session.start_calls == 0 + assert dependencies.service.subscribe_calls == [] + assert dependencies.live_processing_gate.failed is True + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_persistent_startup_connect_failure_marks_gate_failed() -> None: + connect_error = RuntimeError("persistent connect failed") + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + start_error=connect_error, + ) + + with pytest.raises( + RuntimeError, + match="persistent connect failed", + ) as error_info: + await dependencies.runtime.run() + + assert error_info.value is connect_error + return dependencies + + dependencies = asyncio.run(scenario()) + startup_recovery = dependencies.startup_recovery + + assert startup_recovery is not None + assert startup_recovery.hydrate_calls == 1 + assert startup_recovery.recover_calls == 0 + assert dependencies.service.subscribe_calls == [] + assert dependencies.live_processing_gate.failed is True + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.FAILED + ) + + +def test_negative_startup_ack_is_terminal_before_recovery() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.subscribe_entered.wait() + correlation_id = ( + dependencies.service.subscribe_correlation_ids[0] + ) + assert isinstance(correlation_id, str) + dependencies.transport.feed( + make_control_message( + correlation_id=correlation_id, + status="ERROR", + ) + ) + + with pytest.raises( + WebSocketControlMessageError, + match="отклонил", + ): + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + startup_recovery = dependencies.startup_recovery + + assert startup_recovery is not None + assert startup_recovery.recover_calls == 0 + assert dependencies.live_processing_gate.failed is True + assert dependencies.supervisor.start_calls == 0 + + +@pytest.mark.parametrize( + ("control_document_factory", "error_match"), + [ + ( + lambda correlation_id: { + "correlationId": "unknown-request", + "destination": "trades.subscribe", + "status": "OK", + }, + "неизвестным correlationId", + ), + ( + lambda correlation_id: { + "correlationId": correlation_id, + "destination": "trades.subscribe", + }, + "непустой строковый status", + ), + ( + lambda correlation_id: { + "correlationId": correlation_id, + "destination": "unknown.destination", + "status": "OK", + }, + "неизвестным destination", + ), + ], +) +def test_invalid_startup_control_message_is_terminal( + control_document_factory: Callable[[str], object], + error_match: str, +) -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.subscribe_entered.wait() + correlation_id = ( + dependencies.service.subscribe_correlation_ids[0] + ) + assert isinstance(correlation_id, str) + dependencies.transport.feed( + json.dumps( + control_document_factory(correlation_id), + ) + ) + + with pytest.raises( + WebSocketMessageRoutingError, + match=error_match, + ): + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + startup_recovery = dependencies.startup_recovery + + assert startup_recovery is not None + assert startup_recovery.recover_calls == 0 + assert dependencies.service.documents == [] + assert dependencies.live_processing_gate.failed is True + assert dependencies.reconnect_recovery.calls == [] + + +def test_startup_transport_error_is_terminal_without_reconnect() -> None: + transport_error = WebSocketTransportError( + "startup transport failed", + ) + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + incoming=(transport_error,), + ) + + with pytest.raises( + WebSocketTransportError, + match="startup transport failed", + ) as error_info: + await dependencies.runtime.run() + + assert error_info.value is transport_error + return dependencies + + dependencies = asyncio.run(scenario()) + startup_recovery = dependencies.startup_recovery + + assert startup_recovery is not None + assert startup_recovery.recover_calls == 0 + assert dependencies.reconnect_recovery.calls == [] + assert dependencies.live_processing_gate.failed is True + assert dependencies.supervisor.start_calls == 0 + + +def test_startup_market_buffer_overflow_is_terminal() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + startup_market_buffer_capacity=1, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.subscribe_entered.wait() + dependencies.transport.feed(MARKET_MESSAGE) + dependencies.transport.feed(MARKET_MESSAGE) + + with pytest.raises( + WebSocketStartupMarketBufferOverflowError, + match="Переполнен буфер", + ): + await runtime_task + + return dependencies + + dependencies = asyncio.run(scenario()) + startup_recovery = dependencies.startup_recovery + + assert startup_recovery is not None + assert startup_recovery.recover_calls == 0 + assert dependencies.service.documents == [] + assert dependencies.live_processing_gate.failed is True + + +def test_startup_recovery_failure_prevents_fifo_drain() -> None: + recovery_error = RuntimeError("startup recovery failed") + + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + startup_recovery_error=recovery_error, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.subscribe_entered.wait() + correlation_id = ( + dependencies.service.subscribe_correlation_ids[0] + ) + assert isinstance(correlation_id, str) + dependencies.transport.feed(MARKET_MESSAGE) + dependencies.transport.feed( + make_control_message( + correlation_id=correlation_id, + ) + ) + + with pytest.raises( + RuntimeError, + match="startup recovery failed", + ) as error_info: + await runtime_task + + assert error_info.value is recovery_error + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.documents == [] + assert dependencies.live_processing_gate.failed is True + assert dependencies.supervisor.start_calls == 0 + assert dependencies.scheduler.start_calls == 0 + + +def test_stop_during_startup_ack_wait_releases_boundary() -> None: + async def scenario() -> RuntimeDependencies: + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.subscribe_entered.wait() + await dependencies.transport.waiting.wait() + await asyncio.wait_for( + dependencies.runtime.stop(), + timeout=1.0, + ) + await runtime_task + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.live_processing_gate.locked is False + assert dependencies.live_processing_gate.failed is False + assert dependencies.supervisor.start_calls == 0 + assert dependencies.transport.active_receivers == 0 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + + +def test_stop_during_startup_hydration_prevents_connection() -> None: + async def scenario() -> RuntimeDependencies: + hydration_release = asyncio.Event() + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + startup_hydration_release=hydration_release, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + startup_recovery = dependencies.startup_recovery + assert startup_recovery is not None + await startup_recovery.hydration_entered.wait() + + await asyncio.wait_for( + dependencies.runtime.stop(), + timeout=1.0, + ) + await runtime_task + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.session.start_calls == 0 + assert dependencies.live_processing_gate.failed is False + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + + +def test_stop_during_startup_recovery_prevents_fifo_drain() -> None: + async def scenario() -> RuntimeDependencies: + recovery_release = asyncio.Event() + dependencies = RuntimeDependencies( + startup_recovery_enabled=True, + startup_recovery_release=recovery_release, + ) + runtime_task = asyncio.create_task( + dependencies.runtime.run(), + ) + + await dependencies.service.subscribe_entered.wait() + correlation_id = ( + dependencies.service.subscribe_correlation_ids[0] + ) + assert isinstance(correlation_id, str) + dependencies.transport.feed(MARKET_MESSAGE) + dependencies.transport.feed( + make_control_message( + correlation_id=correlation_id, + ) + ) + + startup_recovery = dependencies.startup_recovery + assert startup_recovery is not None + await startup_recovery.recovery_entered.wait() + + await asyncio.wait_for( + dependencies.runtime.stop(), + timeout=1.0, + ) + await runtime_task + return dependencies + + dependencies = asyncio.run(scenario()) + + assert dependencies.service.documents == [] + assert dependencies.live_processing_gate.locked is False + assert dependencies.live_processing_gate.failed is False + assert dependencies.supervisor.start_calls == 0 + assert dependencies.runtime.state is ( + TradeStreamProductionRuntimeState.STOPPED + ) + + def test_locked_gate_does_not_leave_runtime_partially_started() -> None: async def scenario() -> RuntimeDependencies: dependencies = RuntimeDependencies() @@ -893,7 +1609,12 @@ def test_scheduler_claim_blocks_external_start_during_runtime_startup() -> None: def test_startup_receive_and_shutdown_order() -> None: async def scenario() -> RuntimeDependencies: - dependencies = RuntimeDependencies() + dependencies = RuntimeDependencies( + symbols=( + f" {SYMBOL.lower()} ", + SYMBOL, + ), + ) runtime_task = asyncio.create_task( dependencies.runtime.run(), ) diff --git a/app/tests/unit/market_data/acquisition/runtime/test_websocket_runtime_reconnect_integration.py b/app/tests/unit/market_data/acquisition/runtime/test_websocket_runtime_reconnect_integration.py index 623b349..f433d12 100644 --- a/app/tests/unit/market_data/acquisition/runtime/test_websocket_runtime_reconnect_integration.py +++ b/app/tests/unit/market_data/acquisition/runtime/test_websocket_runtime_reconnect_integration.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +from collections.abc import Awaitable from typing import Any import pytest @@ -73,6 +74,12 @@ class FakeConnection: async def recv(self) -> str | bytes: return "" + async def ping(self) -> Awaitable[float]: + async def wait_for_pong() -> float: + return 0.001 + + return wait_for_pong() + class RecordingConnector: def __init__( diff --git a/app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py b/app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py index 9c8776b..43d4df0 100644 --- a/app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py +++ b/app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py @@ -13,9 +13,15 @@ import pytest from src.market_data.acquisition.adapters.dzengi.rest import ( DzengiTradesDocumentSource, ) +from src.market_data.acquisition.checkpoint.trade_stream_state_hydrator import ( + TradeStreamStateHydratorProtocol, +) from src.market_data.acquisition.consistency.trade_observation_sink_protocol import ( TradeObservationSinkProtocol, ) +from src.market_data.acquisition.consistency.trade_stream_state import ( + TradeStreamState, +) from src.market_data.acquisition.models.trade import ( Trade, TradeAggressorSide, @@ -44,6 +50,12 @@ from src.market_data.acquisition.runtime.runtime_events import ( from src.market_data.acquisition.runtime.runtime_recovery_protocol import ( RuntimeRecoveryProtocol, ) +from src.market_data.acquisition.recovery.trade_recovery_result import ( + TradeRecoveryResult, +) +from src.market_data.acquisition.runtime.runtime_startup_recovery_coordinator import ( + RuntimeStartupRecoveryProtocol, +) from src.market_data.acquisition.runtime.scheduler import ( RuntimeSchedulerProtocol, ) @@ -66,6 +78,10 @@ from src.market_data.acquisition.trade_stream_runtime_composition import ( TradeStreamRuntimeComposition, build_trade_stream_runtime_composition, ) +from src.market_data.storage.contracts import ( + PersistentTradeCheckpoint, + TradeCheckpointStorageProtocol, +) SYMBOL = "BTC/USD_LEVERAGE" @@ -326,17 +342,152 @@ class RecordingTradeObservationSink: ) -> None: self._fail_on_trade_id = fail_on_trade_id self.observations: list[Trade] = [] + self.accepted: list[tuple[Trade, Trade | None]] = [] + self.duplicates: list[Trade] = [] - def persist( + def persist_accepted( + self, + trade: Trade, + *, + expected_trade: Trade | None, + ) -> None: + self.observations.append(trade) + self.accepted.append((trade, expected_trade)) + + if trade.trade_id == self._fail_on_trade_id: + raise RuntimeError("storage failed") + + def persist_duplicate( self, trade: Trade, ) -> None: self.observations.append(trade) + self.duplicates.append(trade) if trade.trade_id == self._fail_on_trade_id: raise RuntimeError("storage failed") +class RecordingCheckpointStorage: + """Checkpoint storage, фиксирующий нежелательный I/O.""" + + def __init__(self) -> None: + self.calls: list[str] = [] + + def load_checkpoint( + self, + *, + venue: str, + symbol: str, + ) -> PersistentTradeCheckpoint | None: + self.calls.append("load_checkpoint") + raise AssertionError("Composition не должна читать checkpoint.") + + def load_checkpoint_tail( + self, + *, + venue: str, + checkpoint: PersistentTradeCheckpoint, + limit: int, + ) -> tuple[Trade, ...]: + self.calls.append("load_checkpoint_tail") + raise AssertionError("Composition не должна читать Trade tail.") + + def load_latest_trade_tail( + self, + *, + venue: str, + symbol: str, + limit: int, + ) -> tuple[Trade, ...]: + self.calls.append("load_latest_trade_tail") + raise AssertionError("Composition не должна читать Trade tail.") + + def adopt_existing_trade_as_checkpoint( + self, + *, + venue: str, + trade: Trade, + ) -> PersistentTradeCheckpoint: + self.calls.append("adopt_existing_trade_as_checkpoint") + raise AssertionError("Composition не должна создавать checkpoint.") + + def store_trade_and_advance_checkpoint( + self, + *, + venue: str, + expected_trade: Trade | None, + trade: Trade, + observed_at: datetime, + ) -> PersistentTradeCheckpoint: + self.calls.append("store_trade_and_advance_checkpoint") + raise AssertionError("Composition не должна записывать checkpoint.") + + +class CheckpointBackedStorage: + """Checkpoint storage для проверки общего канонического ключа.""" + + def __init__( + self, + checkpoint: PersistentTradeCheckpoint, + ) -> None: + self._checkpoint = checkpoint + self.calls: list[tuple[object, ...]] = [] + + def load_checkpoint( + self, + *, + venue: str, + symbol: str, + ) -> PersistentTradeCheckpoint | None: + self.calls.append(("load_checkpoint", venue, symbol)) + return self._checkpoint + + def load_checkpoint_tail( + self, + *, + venue: str, + checkpoint: PersistentTradeCheckpoint, + limit: int, + ) -> tuple[Trade, ...]: + self.calls.append( + ( + "load_checkpoint_tail", + venue, + checkpoint, + limit, + ) + ) + return (checkpoint.trade,) + + def load_latest_trade_tail( + self, + *, + venue: str, + symbol: str, + limit: int, + ) -> tuple[Trade, ...]: + raise AssertionError("Latest Trade tail не должен запрашиваться.") + + def adopt_existing_trade_as_checkpoint( + self, + *, + venue: str, + trade: Trade, + ) -> PersistentTradeCheckpoint: + raise AssertionError("Checkpoint не должен создаваться.") + + def store_trade_and_advance_checkpoint( + self, + *, + venue: str, + expected_trade: Trade | None, + trade: Trade, + observed_at: datetime, + ) -> PersistentTradeCheckpoint: + raise AssertionError("Checkpoint не должен записываться.") + + @dataclass(slots=True) class CompositionDependencies: session: FakeSession @@ -352,6 +503,7 @@ class CompositionDependencies: def create_composition( *, + symbols: tuple[str, ...] = (SYMBOL,), trade: Trade | None = None, recovery_document: object = (), heartbeat_timeout_seconds: float = 10.0, @@ -359,6 +511,8 @@ def create_composition( max_recovery_window_ms: int = 3_599_999, probe_results: tuple[bool, ...] = (True,), trade_observation_sink: TradeObservationSinkProtocol | None = None, + checkpoint_storage: TradeCheckpointStorageProtocol | None = None, + checkpoint_venue: str | None = None, ) -> tuple[ TradeStreamRuntimeComposition, CompositionDependencies, @@ -390,7 +544,7 @@ def create_composition( recovery_document_source=( dependencies.recovery_document_source ), - symbols=(SYMBOL,), + symbols=symbols, heartbeat_timeout_seconds=heartbeat_timeout_seconds, scheduler_interval_seconds=scheduler_interval_seconds, trade_observation_sink=trade_observation_sink, @@ -400,6 +554,8 @@ def create_composition( dependencies.recovery_end_time_clock ), scheduler_sleep=dependencies.scheduler_sleep, + checkpoint_storage=checkpoint_storage, + checkpoint_venue=checkpoint_venue, ) return ( @@ -467,6 +623,196 @@ def test_components_implement_public_protocols() -> None: ) +def test_checkpoint_components_are_absent_without_configuration() -> None: + composition, *_ = create_composition() + + assert composition.state_hydrator is None + assert composition.runtime_startup_recovery_coordinator is None + + +def test_checkpoint_components_implement_public_protocols() -> None: + storage = RecordingCheckpointStorage() + composition, *_ = create_composition( + checkpoint_storage=storage, + checkpoint_venue="Dzengi", + ) + + assert isinstance(storage, TradeCheckpointStorageProtocol) + assert isinstance( + composition.state_hydrator, + TradeStreamStateHydratorProtocol, + ) + assert isinstance( + composition.runtime_startup_recovery_coordinator, + RuntimeStartupRecoveryProtocol, + ) + + +def test_checkpoint_runtime_reuses_shared_dependency_graph() -> None: + storage = RecordingCheckpointStorage() + composition, dependencies = create_composition( + checkpoint_storage=storage, + checkpoint_venue="Dzengi", + ) + state_hydrator = composition.state_hydrator + startup_recovery = ( + composition.runtime_startup_recovery_coordinator + ) + + assert state_hydrator is not None + assert startup_recovery is not None + assert state_hydrator._state_store is composition.state_store + assert startup_recovery._state_hydrator is state_hydrator + assert ( + startup_recovery._recovery_coordinator + is composition.runtime_recovery_coordinator + ) + assert ( + startup_recovery.live_processing_gate + is composition.live_processing_gate + ) + assert ( + startup_recovery.symbols + == composition.runtime_reconnect_recovery_coordinator.symbols + == (SYMBOL,) + ) + assert ( + startup_recovery._clock + is dependencies.recovery_end_time_clock + ) + assert ( + composition.runtime_reconnect_recovery_coordinator._clock + is dependencies.recovery_end_time_clock + ) + + +def test_lowercase_symbol_uses_one_key_for_hydration_and_recovery() -> None: + checkpoint = PersistentTradeCheckpoint( + venue="dzengi", + trade=make_trade(), + revision=1, + updated_at=CHECKPOINT_TIME, + ) + storage = CheckpointBackedStorage(checkpoint) + composition, dependencies = create_composition( + symbols=(f" {SYMBOL.lower()} ",), + recovery_document=[], + checkpoint_storage=storage, + checkpoint_venue="Dzengi", + ) + startup_recovery = ( + composition.runtime_startup_recovery_coordinator + ) + + assert startup_recovery is not None + + async def scenario() -> tuple[ + tuple[TradeStreamState, ...], + tuple[TradeRecoveryResult, ...], + ]: + states = await startup_recovery.hydrate_once() + + async with composition.live_processing_gate: + results = await startup_recovery.recover_after_ack() + + return states, results + + states, results = asyncio.run(scenario()) + + assert startup_recovery.symbols == (SYMBOL,) + assert ( + composition.runtime_reconnect_recovery_coordinator.symbols + == (SYMBOL,) + ) + assert states[0].symbol == SYMBOL + assert composition.state_store.get(SYMBOL) is states[0] + assert composition.state_store.contains(SYMBOL.lower()) is False + assert results[0].symbol == SYMBOL + assert storage.calls[0] == ( + "load_checkpoint", + "dzengi", + SYMBOL, + ) + assert dependencies.recovery_document_source.calls + assert dependencies.recovery_document_source.calls[0][0] == SYMBOL + + +def test_case_variant_duplicates_collapse_before_hydration_worker() -> None: + checkpoint = PersistentTradeCheckpoint( + venue="dzengi", + trade=make_trade(), + revision=1, + updated_at=CHECKPOINT_TIME, + ) + storage = CheckpointBackedStorage(checkpoint) + composition, _ = create_composition( + symbols=( + SYMBOL.lower(), + SYMBOL, + f" {SYMBOL.lower()} ", + ), + checkpoint_storage=storage, + checkpoint_venue="dzengi", + ) + startup_recovery = ( + composition.runtime_startup_recovery_coordinator + ) + + assert startup_recovery is not None + + states = asyncio.run(startup_recovery.hydrate_once()) + + assert startup_recovery.symbols == (SYMBOL,) + assert len(states) == 1 + assert states[0].symbol == SYMBOL + assert [call[0] for call in storage.calls] == [ + "load_checkpoint", + "load_checkpoint_tail", + ] + + +@pytest.mark.parametrize( + ("checkpoint_storage", "checkpoint_venue"), + [ + (RecordingCheckpointStorage(), None), + (None, "dzengi"), + ], +) +def test_partial_checkpoint_configuration_is_rejected_without_io( + checkpoint_storage: TradeCheckpointStorageProtocol | None, + checkpoint_venue: str | None, +) -> None: + with pytest.raises( + ValueError, + match="должны быть переданы вместе", + ): + create_composition( + checkpoint_storage=checkpoint_storage, + checkpoint_venue=checkpoint_venue, + ) + + if isinstance(checkpoint_storage, RecordingCheckpointStorage): + assert checkpoint_storage.calls == [] + + +def test_checkpoint_composition_has_no_io_or_background_tasks() -> None: + storage = RecordingCheckpointStorage() + composition, dependencies = create_composition( + checkpoint_storage=storage, + checkpoint_venue="dzengi", + ) + startup_recovery = ( + composition.runtime_startup_recovery_coordinator + ) + + assert startup_recovery is not None + assert storage.calls == [] + assert dependencies.recovery_document_source.calls == [] + assert dependencies.recovery_end_time_clock.calls == 0 + assert startup_recovery._hydration_task is None + assert startup_recovery._recovery_task is None + + def test_external_dependencies_are_reused() -> None: composition, dependencies = create_composition() @@ -548,6 +894,11 @@ def test_live_and_recovery_share_optional_persistence_sink() -> None: trade.trade_id for trade in sink.observations ] == [100, recovered_trade_id] + assert sink.accepted == [ + (live_trade, None), + (recovery_result.last_trade, live_trade), + ] + assert sink.duplicates == [] def test_recovery_duplicate_updates_persistence_without_checkpoint_change( @@ -589,6 +940,8 @@ def test_recovery_duplicate_updates_persistence_without_checkpoint_change( assert len(sink.observations) == 2 assert sink.observations[0] is live_trade assert sink.observations[1].source == "dzengi" + assert sink.accepted == [(live_trade, None)] + assert sink.duplicates == [sink.observations[1]] assert state.last_trade is live_trade diff --git a/app/tests/unit/market_data/storage/test_checkpoint_contracts.py b/app/tests/unit/market_data/storage/test_checkpoint_contracts.py new file mode 100644 index 0000000..57ab0ab --- /dev/null +++ b/app/tests/unit/market_data/storage/test_checkpoint_contracts.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timezone +from decimal import Decimal +from typing import Any + +import pytest + +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.storage import ( + MarketDataCheckpointConflictError, + MarketDataCheckpointIntegrityError, + MarketDataStorageConflictError, + MarketDataStorageError, + PersistentTradeCheckpoint, + TradeCheckpointStorageProtocol, +) + + +VENUE = "DZENGI" +SYMBOL = "BTC/USD_LEVERAGE" +EXECUTED_AT = datetime(2026, 8, 1, 10, 0, tzinfo=timezone.utc) +UPDATED_AT = datetime(2026, 8, 1, 10, 1, tzinfo=timezone.utc) + + +def make_trade( + *, + symbol: str = SYMBOL, + trade_id: int = 123, + executed_at: datetime = EXECUTED_AT, +) -> Trade: + return Trade( + symbol=symbol, + trade_id=trade_id, + price=Decimal("65000.25"), + quantity=Decimal("0.001"), + executed_at=executed_at, + aggressor_side=TradeAggressorSide.BUY, + source="dzengi_websocket_trade", + ) + + +def make_checkpoint() -> PersistentTradeCheckpoint: + return PersistentTradeCheckpoint( + venue=VENUE, + trade=make_trade(), + revision=7, + updated_at=UPDATED_AT, + ) + + +class RecordingCheckpointStorage: + def __init__(self) -> None: + self.checkpoint = make_checkpoint() + + def load_checkpoint( + self, + *, + venue: str, + symbol: str, + ) -> PersistentTradeCheckpoint | None: + return self.checkpoint + + def load_checkpoint_tail( + self, + *, + venue: str, + checkpoint: PersistentTradeCheckpoint, + limit: int, + ) -> tuple[Trade, ...]: + return (checkpoint.trade,) + + def load_latest_trade_tail( + self, + *, + venue: str, + symbol: str, + limit: int, + ) -> tuple[Trade, ...]: + return (self.checkpoint.trade,) + + def adopt_existing_trade_as_checkpoint( + self, + *, + venue: str, + trade: Trade, + ) -> PersistentTradeCheckpoint: + return replace( + self.checkpoint, + venue=venue, + trade=trade, + revision=1, + ) + + def store_trade_and_advance_checkpoint( + self, + *, + venue: str, + expected_trade: Trade | None, + trade: Trade, + observed_at: datetime, + ) -> PersistentTradeCheckpoint: + return replace( + self.checkpoint, + trade=trade, + revision=self.checkpoint.revision + 1, + updated_at=observed_at, + ) + + +def test_checkpoint_preserves_full_trade_and_durable_identity() -> None: + checkpoint = make_checkpoint() + + assert checkpoint.venue == VENUE + assert checkpoint.trade is not None + assert checkpoint.revision == 7 + assert checkpoint.updated_at == UPDATED_AT + assert checkpoint.checkpoint_schema_version == 1 + assert checkpoint.identity == ( + VENUE, + SYMBOL, + 123, + EXECUTED_AT, + ) + + +def test_checkpoint_storage_protocol_is_runtime_checkable() -> None: + assert isinstance( + RecordingCheckpointStorage(), + TradeCheckpointStorageProtocol, + ) + + +@pytest.mark.parametrize("venue", ("", " ", "\t")) +def test_checkpoint_rejects_empty_venue(venue: str) -> None: + with pytest.raises(ValueError, match="venue"): + PersistentTradeCheckpoint( + venue=venue, + trade=make_trade(), + revision=1, + updated_at=UPDATED_AT, + ) + + +def test_checkpoint_rejects_non_string_venue() -> None: + with pytest.raises(TypeError, match="venue"): + PersistentTradeCheckpoint( + venue=123, # type: ignore[arg-type] + trade=make_trade(), + revision=1, + updated_at=UPDATED_AT, + ) + + +def test_checkpoint_rejects_non_trade_payload() -> None: + with pytest.raises(TypeError, match="Canonical Trade"): + PersistentTradeCheckpoint( + venue=VENUE, + trade=object(), # type: ignore[arg-type] + revision=1, + updated_at=UPDATED_AT, + ) + + +def test_checkpoint_rejects_empty_trade_symbol() -> None: + with pytest.raises(ValueError, match="trade.symbol"): + PersistentTradeCheckpoint( + venue=VENUE, + trade=make_trade(symbol=" "), + revision=1, + updated_at=UPDATED_AT, + ) + + +def test_checkpoint_rejects_naive_trade_time() -> None: + with pytest.raises(ValueError, match="trade.executed_at"): + PersistentTradeCheckpoint( + venue=VENUE, + trade=make_trade( + executed_at=datetime(2026, 8, 1, 10, 0), + ), + revision=1, + updated_at=UPDATED_AT, + ) + + +@pytest.mark.parametrize("trade_id", (-2_147_483_649, 2_147_483_648)) +def test_checkpoint_rejects_trade_id_outside_signed_range( + trade_id: int, +) -> None: + with pytest.raises(ValueError, match="signed 32-bit"): + PersistentTradeCheckpoint( + venue=VENUE, + trade=make_trade(trade_id=trade_id), + revision=1, + updated_at=UPDATED_AT, + ) + + +@pytest.mark.parametrize("revision", (0, -1)) +def test_checkpoint_rejects_non_positive_revision(revision: int) -> None: + with pytest.raises(ValueError, match="revision"): + PersistentTradeCheckpoint( + venue=VENUE, + trade=make_trade(), + revision=revision, + updated_at=UPDATED_AT, + ) + + +@pytest.mark.parametrize("revision", (True, 1.5, "1", None)) +def test_checkpoint_rejects_non_integer_revision( + revision: Any, +) -> None: + with pytest.raises(TypeError, match="revision"): + PersistentTradeCheckpoint( + venue=VENUE, + trade=make_trade(), + revision=revision, + updated_at=UPDATED_AT, + ) + + +def test_checkpoint_rejects_naive_updated_at() -> None: + with pytest.raises(ValueError, match="updated_at"): + PersistentTradeCheckpoint( + venue=VENUE, + trade=make_trade(), + revision=1, + updated_at=datetime(2026, 8, 1, 10, 1), + ) + + +@pytest.mark.parametrize("version", (0, -1)) +def test_checkpoint_rejects_non_positive_schema_version( + version: int, +) -> None: + with pytest.raises(ValueError, match="checkpoint_schema_version"): + PersistentTradeCheckpoint( + venue=VENUE, + trade=make_trade(), + revision=1, + updated_at=UPDATED_AT, + checkpoint_schema_version=version, + ) + + +def test_checkpoint_errors_preserve_storage_hierarchy() -> None: + assert issubclass( + MarketDataCheckpointConflictError, + MarketDataStorageConflictError, + ) + assert issubclass( + MarketDataCheckpointIntegrityError, + MarketDataStorageError, + ) diff --git a/app/tests/unit/market_data/storage/test_postgres_partitions.py b/app/tests/unit/market_data/storage/test_postgres_partitions.py index 1e60617..53a84eb 100644 --- a/app/tests/unit/market_data/storage/test_postgres_partitions.py +++ b/app/tests/unit/market_data/storage/test_postgres_partitions.py @@ -129,6 +129,9 @@ class PartitionCursor: if normalized.startswith("ALTER TABLE") and "ADD CONSTRAINT" in normalized: return + if normalized.startswith("ALTER TABLE") and "DROP CONSTRAINT" in normalized: + return + if normalized.startswith("WITH moved_rows AS"): self.rowcount = self._connection.moved_row_count return @@ -481,6 +484,70 @@ def test_manager_creates_partition_moves_default_rows_and_registers_it() -> None assert all("::timestamptz" in statement for statement, _ in ddl_calls) +def test_trade_partition_rebuilds_checkpoint_foreign_key_around_move() -> None: + _, connection, provider = _dependencies() + manager = PostgresMarketDataPartitionManager( + connection_provider=provider + ) + + manager.ensure_month_partition( + data_type=MarketDataPartitionType.TRADES, + month=JULY, + ) + + statements = tuple(statement for statement, _ in connection.calls) + parent_lock_index = statements.index( + 'LOCK TABLE "market_data"."trades" ' + "IN SHARE ROW EXCLUSIVE MODE" + ) + default_lock_index = statements.index( + 'LOCK TABLE "market_data"."trades_default" ' + "IN ACCESS EXCLUSIVE MODE" + ) + drop_index = statements.index( + 'ALTER TABLE "market_data"."trade_stream_checkpoints" ' + 'DROP CONSTRAINT "trade_stream_checkpoints_trade_fk"' + ) + move_index = next( + index + for index, statement in enumerate(statements) + if statement.startswith("WITH moved_rows AS") + ) + restore_index = next( + index + for index, statement in enumerate(statements) + if "ADD CONSTRAINT \"trade_stream_checkpoints_trade_fk\"" in statement + ) + + assert ( + parent_lock_index + < default_lock_index + < drop_index + < move_index + < restore_index + ) + restored_sql = statements[restore_index] + assert "ON UPDATE NO ACTION ON DELETE NO ACTION" in restored_sql + assert "DEFERRABLE INITIALLY DEFERRED" in restored_sql + + +def test_non_trade_partition_does_not_touch_checkpoint_foreign_key() -> None: + _, connection, provider = _dependencies() + manager = PostgresMarketDataPartitionManager( + connection_provider=provider + ) + + manager.ensure_month_partition( + data_type=MarketDataPartitionType.QUOTES, + month=JULY, + ) + + assert not any( + "trade_stream_checkpoints_trade_fk" in statement + for statement, _ in connection.calls + ) + + def test_manager_is_idempotent_after_registered_partition_exists() -> None: _, connection, provider = _dependencies() manager = PostgresMarketDataPartitionManager( @@ -718,6 +785,22 @@ def test_retention_drops_complete_month_and_deletes_partial_history() -> None: for statement, _ in connection.calls ) + statements = tuple(statement for statement, _ in connection.calls) + drop_index = statements.index( + 'ALTER TABLE "market_data"."trade_stream_checkpoints" ' + 'DROP CONSTRAINT "trade_stream_checkpoints_trade_fk"' + ) + delete_index = statements.index( + 'DELETE FROM "market_data"."trades" ' + 'WHERE "executed_at" < %s' + ) + restore_index = next( + index + for index, statement in enumerate(statements) + if "ADD CONSTRAINT \"trade_stream_checkpoints_trade_fk\"" in statement + ) + assert drop_index < delete_index < restore_index + def test_unconfigured_data_type_is_unlimited_and_not_touched() -> None: database, connection, provider = _dependencies() diff --git a/app/tests/unit/market_data/storage/test_postgres_trade_checkpoint_repository.py b/app/tests/unit/market_data/storage/test_postgres_trade_checkpoint_repository.py new file mode 100644 index 0000000..4ac03e4 --- /dev/null +++ b/app/tests/unit/market_data/storage/test_postgres_trade_checkpoint_repository.py @@ -0,0 +1,817 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any + +import pytest + +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.acquisition.trade_id_sequence import ( + SIGNED_TRADE_ID_MAX, + SIGNED_TRADE_ID_MIN, +) +from src.market_data.storage import ( + MarketDataCheckpointConflictError, + MarketDataCheckpointIntegrityError, + MarketDataStorageOperationError, + MarketDataStorageValidationError, + PersistentTradeCheckpoint, + PostgresTradeRepository, + TradeCheckpointStorageProtocol, +) + + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +EXECUTED_AT = datetime(2026, 8, 1, 10, 0, tzinfo=timezone.utc) +OBSERVED_AT = EXECUTED_AT + timedelta(seconds=1) + + +def _trade( + *, + trade_id: int = 100, + executed_at: datetime = EXECUTED_AT, + price: Decimal = Decimal("65000.25"), + source: str = "dzengi_websocket_trade", +) -> Trade: + return Trade( + symbol=SYMBOL, + trade_id=trade_id, + price=price, + quantity=Decimal("0.001"), + executed_at=executed_at, + aggressor_side=TradeAggressorSide.BUY, + source=source, + ) + + +def _trade_row(trade: Trade) -> tuple[object, ...]: + return ( + trade.symbol, + trade.trade_id, + trade.executed_at, + trade.price, + trade.quantity, + trade.aggressor_side.value, + trade.source, + 1, + ) + + +def _checkpoint_row( + trade: Trade, + *, + revision: int, + updated_at: datetime = OBSERVED_AT, +) -> tuple[object, ...]: + return ( + VENUE, + trade.symbol, + trade.trade_id, + trade.executed_at, + revision, + updated_at, + 1, + VENUE, + trade.symbol, + trade.trade_id, + trade.executed_at, + trade.price, + trade.quantity, + trade.aggressor_side.value, + trade.source, + 1, + ) + + +def _existing_trade_row(trade: Trade) -> tuple[object, ...]: + return ( + trade.price, + trade.quantity, + trade.aggressor_side.value, + OBSERVED_AT, + OBSERVED_AT, + [trade.source], + 1, + ) + + +@dataclass(frozen=True) +class SqlStep: + starts_with: str + fetchone: object = None + fetchall: tuple[tuple[object, ...], ...] = () + error: BaseException | None = None + + +class ScriptedCursor: + def __init__(self, connection: ScriptedConnection) -> None: + self._connection = connection + self._fetchone: object = None + self._fetchall: tuple[tuple[object, ...], ...] = () + + def __enter__(self) -> ScriptedCursor: + return self + + def __exit__(self, *args: object) -> None: + return None + + def execute( + self, + statement: str, + parameters: tuple[Any, ...], + ) -> None: + normalized = " ".join(statement.split()) + self._connection.calls.append((normalized, parameters)) + + if not self._connection.steps: + raise AssertionError(f"Unexpected SQL: {normalized}") + + step = self._connection.steps.pop(0) + + if not normalized.startswith(step.starts_with): + raise AssertionError( + f"Expected SQL starting with {step.starts_with!r}, " + f"received {normalized!r}." + ) + + if step.error is not None: + raise step.error + + self._fetchone = step.fetchone + self._fetchall = step.fetchall + + def fetchone(self) -> object: + return self._fetchone + + def fetchall(self) -> tuple[tuple[object, ...], ...]: + return self._fetchall + + +class ScriptedConnection: + def __init__(self, steps: tuple[SqlStep, ...]) -> None: + self.steps = list(steps) + self.calls: list[tuple[str, tuple[Any, ...]]] = [] + self.exit_exception_types: list[type[BaseException] | None] = [] + + def __enter__(self) -> ScriptedConnection: + return self + + def __exit__( + self, + exception_type: type[BaseException] | None, + exception: BaseException | None, + traceback: object, + ) -> None: + self.exit_exception_types.append(exception_type) + return None + + def cursor(self) -> ScriptedCursor: + return ScriptedCursor(self) + + +class RecordingProvider: + def __init__(self, connection: ScriptedConnection) -> None: + self.connection = connection + self.calls = 0 + + def __call__(self) -> ScriptedConnection: + self.calls += 1 + return self.connection + + +def _repository( + *steps: SqlStep, +) -> tuple[PostgresTradeRepository, ScriptedConnection, RecordingProvider]: + connection = ScriptedConnection(steps) + provider = RecordingProvider(connection) + repository = PostgresTradeRepository( + connection_provider=provider, + ) + return repository, connection, provider + + +def test_repository_matches_checkpoint_storage_protocol() -> None: + repository, _, _ = _repository() + + assert isinstance(repository, TradeCheckpointStorageProtocol) + + +def test_load_checkpoint_returns_none_without_persistent_state() -> None: + repository, connection, provider = _repository( + SqlStep(starts_with="SELECT checkpoint.venue"), + ) + + result = repository.load_checkpoint( + venue=" dzengi ", + symbol=" btc/usd_leverage ", + ) + + assert result is None + assert provider.calls == 1 + assert connection.calls[0][1] == (VENUE, SYMBOL) + assert connection.steps == [] + + +def test_load_checkpoint_restores_exact_canonical_trade() -> None: + trade = _trade() + repository, connection, _ = _repository( + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(trade, revision=7), + ), + ) + + result = repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + + assert result == PersistentTradeCheckpoint( + venue=VENUE, + trade=trade, + revision=7, + updated_at=OBSERVED_AT, + ) + assert "FOR UPDATE" not in connection.calls[0][0] + + +def test_load_checkpoint_rejects_missing_durable_trade() -> None: + trade = _trade() + orphan_row = list(_checkpoint_row(trade, revision=1)) + orphan_row[7:] = [None] * 9 + repository, connection, _ = _repository( + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=tuple(orphan_row), + ), + ) + + with pytest.raises( + MarketDataCheckpointIntegrityError, + match="exact Canonical Trade", + ): + repository.load_checkpoint( + venue=VENUE, + symbol=SYMBOL, + ) + + assert connection.exit_exception_types == [ + MarketDataCheckpointIntegrityError + ] + + +def test_checkpoint_tail_is_returned_oldest_to_checkpoint_across_rollover( +) -> None: + previous = _trade( + trade_id=SIGNED_TRADE_ID_MAX, + executed_at=EXECUTED_AT, + ) + current = _trade( + trade_id=SIGNED_TRADE_ID_MIN, + executed_at=EXECUTED_AT + timedelta(milliseconds=1), + ) + checkpoint = PersistentTradeCheckpoint( + venue=VENUE, + trade=current, + revision=2, + updated_at=OBSERVED_AT, + ) + repository, connection, _ = _repository( + SqlStep( + starts_with="WITH reference_trade AS", + fetchall=( + _trade_row(current), + _trade_row(previous), + ), + ), + ) + + result = repository.load_checkpoint_tail( + venue=VENUE, + checkpoint=checkpoint, + limit=2, + ) + + assert result == (previous, current) + parameters = connection.calls[0][1] + assert parameters[:4] == ( + VENUE, + SYMBOL, + SIGNED_TRADE_ID_MIN, + current.executed_at, + ) + assert parameters[-1] == 2 + + +def test_checkpoint_tail_rejects_non_strict_trade_id_order() -> None: + previous = _trade(trade_id=100) + conflicting = _trade( + trade_id=100, + executed_at=EXECUTED_AT + timedelta(milliseconds=1), + ) + checkpoint = PersistentTradeCheckpoint( + venue=VENUE, + trade=conflicting, + revision=2, + updated_at=OBSERVED_AT, + ) + repository, connection, _ = _repository( + SqlStep( + starts_with="WITH reference_trade AS", + fetchall=( + _trade_row(conflicting), + _trade_row(previous), + ), + ), + ) + + with pytest.raises( + MarketDataCheckpointIntegrityError, + match="strictly rollover-ordered", + ): + repository.load_checkpoint_tail( + venue=VENUE, + checkpoint=checkpoint, + limit=2, + ) + + assert connection.exit_exception_types == [None] + + +@pytest.mark.parametrize("limit", (0, -1, True, 1.5, "1")) +def test_tail_rejects_invalid_limit_without_io(limit: object) -> None: + repository, _, provider = _repository() + + with pytest.raises(MarketDataStorageValidationError, match="limit"): + repository.load_latest_trade_tail( + venue=VENUE, + symbol=SYMBOL, + limit=limit, # type: ignore[arg-type] + ) + + assert provider.calls == 0 + + +def test_latest_tail_returns_empty_without_trade_history() -> None: + repository, connection, _ = _repository( + SqlStep(starts_with="SELECT symbol, trade_id"), + ) + + result = repository.load_latest_trade_tail( + venue=VENUE, + symbol=SYMBOL, + limit=10, + ) + + assert result == () + assert len(connection.calls) == 1 + + +def test_adopts_existing_trade_without_mutating_trade_history() -> None: + trade = _trade() + repository, connection, provider = _repository( + SqlStep( + starts_with="SELECT symbol, trade_id", + fetchone=_trade_row(trade), + ), + SqlStep( + starts_with="INSERT INTO market_data.trade_stream_checkpoints", + fetchone=(1,), + ), + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(trade, revision=1), + ), + ) + + result = repository.adopt_existing_trade_as_checkpoint( + venue=" dzengi ", + trade=trade, + ) + + assert result.trade == trade + assert result.revision == 1 + assert provider.calls == 1 + assert connection.calls[0][1] == ( + VENUE, + SYMBOL, + trade.trade_id, + trade.executed_at, + ) + assert "FOR SHARE" in connection.calls[0][0] + assert not any( + statement.startswith("INSERT INTO market_data.trades") + or statement.startswith("UPDATE market_data.trades") + for statement, _ in connection.calls + ) + assert connection.exit_exception_types == [None] + assert connection.steps == [] + + +def test_repeated_adoption_of_same_trade_is_idempotent() -> None: + trade = _trade() + repository, connection, _ = _repository( + SqlStep( + starts_with="SELECT symbol, trade_id", + fetchone=_trade_row(trade), + ), + SqlStep( + starts_with="INSERT INTO market_data.trade_stream_checkpoints", + ), + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(trade, revision=1), + ), + ) + + result = repository.adopt_existing_trade_as_checkpoint( + venue=VENUE, + trade=trade, + ) + + assert result.trade == trade + assert result.revision == 1 + assert not any( + statement.startswith( + "UPDATE market_data.trade_stream_checkpoints" + ) + for statement, _ in connection.calls + ) + assert connection.exit_exception_types == [None] + + +def test_adoption_rejects_missing_durable_trade() -> None: + repository, connection, _ = _repository( + SqlStep(starts_with="SELECT symbol, trade_id"), + ) + + with pytest.raises( + MarketDataCheckpointIntegrityError, + match="absent from durable", + ): + repository.adopt_existing_trade_as_checkpoint( + venue=VENUE, + trade=_trade(), + ) + + assert len(connection.calls) == 1 + assert connection.exit_exception_types == [ + MarketDataCheckpointIntegrityError + ] + + +def test_adoption_rejects_conflicting_durable_payload() -> None: + candidate = _trade() + durable = _trade(price=Decimal("65000.26")) + repository, connection, _ = _repository( + SqlStep( + starts_with="SELECT symbol, trade_id", + fetchone=_trade_row(durable), + ), + ) + + with pytest.raises( + MarketDataCheckpointConflictError, + match="conflicts with durable", + ): + repository.adopt_existing_trade_as_checkpoint( + venue=VENUE, + trade=candidate, + ) + + assert len(connection.calls) == 1 + assert connection.exit_exception_types == [ + MarketDataCheckpointConflictError + ] + + +def test_adoption_rejects_checkpoint_of_another_trade() -> None: + candidate = _trade(trade_id=100) + existing = _trade( + trade_id=101, + executed_at=EXECUTED_AT + timedelta(milliseconds=1), + ) + repository, connection, _ = _repository( + SqlStep( + starts_with="SELECT symbol, trade_id", + fetchone=_trade_row(candidate), + ), + SqlStep( + starts_with="INSERT INTO market_data.trade_stream_checkpoints", + ), + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(existing, revision=1), + ), + ) + + with pytest.raises( + MarketDataCheckpointConflictError, + match="already points", + ): + repository.adopt_existing_trade_as_checkpoint( + venue=VENUE, + trade=candidate, + ) + + assert connection.exit_exception_types == [ + MarketDataCheckpointConflictError + ] + + +def test_adoption_database_error_rolls_back_transaction() -> None: + trade = _trade() + repository, connection, _ = _repository( + SqlStep( + starts_with="SELECT symbol, trade_id", + fetchone=_trade_row(trade), + ), + SqlStep( + starts_with="INSERT INTO market_data.trade_stream_checkpoints", + error=RuntimeError("database failed"), + ), + ) + + with pytest.raises( + MarketDataStorageOperationError, + match="adopt existing Trade", + ) as error_info: + repository.adopt_existing_trade_as_checkpoint( + venue=VENUE, + trade=trade, + ) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert connection.exit_exception_types == [RuntimeError] + + +def test_first_checkpoint_is_inserted_with_trade_in_one_transaction() -> None: + trade = _trade() + repository, connection, provider = _repository( + SqlStep( + starts_with="INSERT INTO market_data.trades", + fetchone=(1,), + ), + SqlStep( + starts_with="INSERT INTO market_data.trade_stream_checkpoints", + fetchone=(1,), + ), + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(trade, revision=1), + ), + ) + + result = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=trade, + observed_at=OBSERVED_AT, + ) + + assert result.trade == trade + assert result.revision == 1 + assert provider.calls == 1 + assert connection.exit_exception_types == [None] + assert "FOR UPDATE OF checkpoint" in connection.calls[-1][0] + assert connection.steps == [] + + +def test_first_checkpoint_mismatch_rolls_back_transaction() -> None: + candidate = _trade(trade_id=100) + unexpected = _trade( + trade_id=101, + executed_at=EXECUTED_AT + timedelta(milliseconds=1), + ) + repository, connection, _ = _repository( + SqlStep( + starts_with="INSERT INTO market_data.trades", + fetchone=(1,), + ), + SqlStep( + starts_with="INSERT INTO market_data.trade_stream_checkpoints", + fetchone=(1,), + ), + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(unexpected, revision=1), + ), + ) + + with pytest.raises( + MarketDataCheckpointIntegrityError, + match="First checkpoint", + ): + repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=None, + trade=candidate, + observed_at=OBSERVED_AT, + ) + + assert connection.exit_exception_types == [ + MarketDataCheckpointIntegrityError + ] + + +def test_existing_checkpoint_advances_with_revision_cas() -> None: + previous = _trade(trade_id=100) + current = _trade( + trade_id=101, + executed_at=EXECUTED_AT + timedelta(milliseconds=1), + ) + repository, connection, _ = _repository( + SqlStep( + starts_with="INSERT INTO market_data.trades", + fetchone=(1,), + ), + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(previous, revision=7), + ), + SqlStep( + starts_with="UPDATE market_data.trade_stream_checkpoints", + fetchone=(1,), + ), + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(current, revision=8), + ), + ) + + result = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=previous, + trade=current, + observed_at=OBSERVED_AT, + ) + + assert result.trade == current + assert result.revision == 8 + update_parameters = connection.calls[2][1] + assert update_parameters[-1] == 7 + assert connection.exit_exception_types == [None] + + +def test_retry_after_committed_candidate_keeps_original_revision() -> None: + previous = _trade(trade_id=100) + current = _trade( + trade_id=101, + executed_at=EXECUTED_AT + timedelta(milliseconds=1), + ) + repository, connection, _ = _repository( + SqlStep(starts_with="INSERT INTO market_data.trades"), + SqlStep( + starts_with="SELECT price, quantity", + fetchone=_existing_trade_row(current), + ), + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(current, revision=8), + ), + ) + + result = repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=previous, + trade=current, + observed_at=OBSERVED_AT, + ) + + assert result.revision == 8 + assert not any( + statement.startswith( + "UPDATE market_data.trade_stream_checkpoints" + ) + for statement, _ in connection.calls + ) + assert connection.exit_exception_types == [None] + + +def test_stale_expected_checkpoint_rolls_back_candidate_trade() -> None: + stale = _trade(trade_id=100) + database_current = _trade( + trade_id=101, + executed_at=EXECUTED_AT + timedelta(milliseconds=1), + ) + candidate = _trade( + trade_id=102, + executed_at=EXECUTED_AT + timedelta(milliseconds=2), + ) + repository, connection, _ = _repository( + SqlStep( + starts_with="INSERT INTO market_data.trades", + fetchone=(1,), + ), + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(database_current, revision=2), + ), + ) + + with pytest.raises( + MarketDataCheckpointConflictError, + match="differs from expected", + ): + repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=stale, + trade=candidate, + observed_at=OBSERVED_AT, + ) + + assert connection.exit_exception_types == [ + MarketDataCheckpointConflictError + ] + + +def test_checkpoint_database_error_rolls_back_whole_transaction() -> None: + previous = _trade(trade_id=100) + current = _trade( + trade_id=101, + executed_at=EXECUTED_AT + timedelta(milliseconds=1), + ) + repository, connection, _ = _repository( + SqlStep( + starts_with="INSERT INTO market_data.trades", + fetchone=(1,), + ), + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(previous, revision=1), + ), + SqlStep( + starts_with="UPDATE market_data.trade_stream_checkpoints", + error=RuntimeError("database failed"), + ), + ) + + with pytest.raises( + MarketDataStorageOperationError, + match="atomically", + ) as error_info: + repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=previous, + trade=current, + observed_at=OBSERVED_AT, + ) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert connection.exit_exception_types == [RuntimeError] + + +def test_half_cycle_candidate_is_rejected_as_ambiguous() -> None: + previous = _trade(trade_id=0) + ambiguous = _trade( + trade_id=SIGNED_TRADE_ID_MIN, + executed_at=EXECUTED_AT + timedelta(milliseconds=1), + ) + repository, connection, _ = _repository( + SqlStep( + starts_with="INSERT INTO market_data.trades", + fetchone=(1,), + ), + SqlStep( + starts_with="SELECT checkpoint.venue", + fetchone=_checkpoint_row(previous, revision=1), + ), + ) + + with pytest.raises( + MarketDataCheckpointConflictError, + match="ambiguous", + ): + repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=previous, + trade=ambiguous, + observed_at=OBSERVED_AT, + ) + + assert connection.exit_exception_types == [ + MarketDataCheckpointConflictError + ] + + +def test_invalid_expected_trade_is_rejected_without_io() -> None: + repository, _, provider = _repository() + + with pytest.raises( + MarketDataStorageValidationError, + match="expected_trade", + ): + repository.store_trade_and_advance_checkpoint( + venue=VENUE, + expected_trade=object(), # type: ignore[arg-type] + trade=_trade(), + observed_at=OBSERVED_AT, + ) + + assert provider.calls == 0 diff --git a/app/tests/unit/market_data/storage/test_trade_storage_observation_sink.py b/app/tests/unit/market_data/storage/test_trade_storage_observation_sink.py index 2e55feb..ad4b77f 100644 --- a/app/tests/unit/market_data/storage/test_trade_storage_observation_sink.py +++ b/app/tests/unit/market_data/storage/test_trade_storage_observation_sink.py @@ -17,6 +17,7 @@ from src.market_data.storage import ( MarketDataStorageValidationError, MarketDataWriteResult, MarketDataWriteStatus, + PersistentTradeCheckpoint, TradeStorageObservationSink, ) @@ -42,13 +43,18 @@ class RecordingTradeStorage: self, *, result: object | None = None, + checkpoint_result: object | None = None, error: Exception | None = None, ) -> None: self.result = result or MarketDataWriteResult( status=MarketDataWriteStatus.INSERTED, ) + self.checkpoint_result = checkpoint_result self.error = error self.calls: list[tuple[str, Trade, datetime]] = [] + self.checkpoint_calls: list[ + tuple[str, Trade | None, Trade, datetime] + ] = [] def store_trade( self, @@ -73,12 +79,91 @@ class RecordingTradeStorage: ) -> MarketDataBatchWriteResult: raise AssertionError("Runtime persists observations one by one") + def load_checkpoint( + self, + *, + venue: str, + symbol: str, + ) -> PersistentTradeCheckpoint | None: + raise AssertionError((venue, symbol)) + + def load_checkpoint_tail( + self, + *, + venue: str, + checkpoint: PersistentTradeCheckpoint, + limit: int, + ) -> tuple[Trade, ...]: + raise AssertionError((venue, checkpoint, limit)) + + def load_latest_trade_tail( + self, + *, + venue: str, + symbol: str, + limit: int, + ) -> tuple[Trade, ...]: + raise AssertionError((venue, symbol, limit)) + + def adopt_existing_trade_as_checkpoint( + self, + *, + venue: str, + trade: Trade, + ) -> PersistentTradeCheckpoint: + raise AssertionError((venue, trade)) + + def store_trade_and_advance_checkpoint( + self, + *, + venue: str, + expected_trade: Trade | None, + trade: Trade, + observed_at: datetime, + ) -> PersistentTradeCheckpoint: + self.checkpoint_calls.append( + (venue, expected_trade, trade, observed_at) + ) + + if self.error is not None: + raise self.error + + if self.checkpoint_result is not None: + return self.checkpoint_result # type: ignore[return-value] + + return PersistentTradeCheckpoint( + venue=venue, + trade=trade, + revision=1, + updated_at=observed_at, + ) + + +class RecordingWriteOnlyTradeStorage: + def store_trade( + self, + *, + venue: str, + trade: Trade, + observed_at: datetime, + ) -> MarketDataWriteResult: + raise AssertionError((venue, trade, observed_at)) + + def store_trades( + self, + *, + venue: str, + trades: tuple[Trade, ...], + observed_at: datetime, + ) -> MarketDataBatchWriteResult: + raise AssertionError((venue, trades, observed_at)) + @pytest.mark.parametrize( "status", tuple(MarketDataWriteStatus), ) -def test_forwards_observation_and_accepts_all_success_statuses( +def test_duplicate_forwards_observation_and_accepts_all_success_statuses( status: MarketDataWriteStatus, ) -> None: storage = RecordingTradeStorage( @@ -91,7 +176,7 @@ def test_forwards_observation_and_accepts_all_success_statuses( ) trade = make_trade() - result = sink.persist(trade) + result = sink.persist_duplicate(trade) assert result is None assert storage.calls == [ @@ -101,6 +186,42 @@ def test_forwards_observation_and_accepts_all_success_statuses( OBSERVED_AT, ) ] + assert storage.checkpoint_calls == [] + + +def test_accepted_trade_atomically_advances_checkpoint() -> None: + storage = RecordingTradeStorage() + sink = TradeStorageObservationSink( + trade_storage=storage, + venue=" dzengi ", + clock=lambda: OBSERVED_AT, + ) + previous_trade = make_trade() + trade = Trade( + symbol=previous_trade.symbol, + trade_id=previous_trade.trade_id + 1, + price=previous_trade.price, + quantity=previous_trade.quantity, + executed_at=previous_trade.executed_at, + aggressor_side=previous_trade.aggressor_side, + source=previous_trade.source, + ) + + result = sink.persist_accepted( + trade, + expected_trade=previous_trade, + ) + + assert result is None + assert storage.checkpoint_calls == [ + ( + VENUE, + previous_trade, + trade, + OBSERVED_AT, + ) + ] + assert storage.calls == [] def test_implements_acquisition_side_sink_protocol() -> None: @@ -113,7 +234,11 @@ def test_implements_acquisition_side_sink_protocol() -> None: assert not hasattr(sink, "__dict__") -def test_storage_error_is_not_wrapped() -> None: +@pytest.mark.parametrize( + "operation", + ("accepted", "duplicate"), +) +def test_storage_error_is_not_wrapped(operation: str) -> None: storage_error = RuntimeError("storage failed") sink = TradeStorageObservationSink( trade_storage=RecordingTradeStorage(error=storage_error), @@ -125,7 +250,13 @@ def test_storage_error_is_not_wrapped() -> None: RuntimeError, match="storage failed", ) as error_info: - sink.persist(make_trade()) + if operation == "accepted": + sink.persist_accepted( + make_trade(), + expected_trade=None, + ) + else: + sink.persist_duplicate(make_trade()) assert error_info.value is storage_error @@ -141,7 +272,26 @@ def test_rejects_invalid_storage_result() -> None: TypeError, match="MarketDataWriteResult", ): - sink.persist(make_trade()) + sink.persist_duplicate(make_trade()) + + +def test_rejects_invalid_checkpoint_result() -> None: + sink = TradeStorageObservationSink( + trade_storage=RecordingTradeStorage( + checkpoint_result=object(), + ), + venue=VENUE, + clock=lambda: OBSERVED_AT, + ) + + with pytest.raises( + TypeError, + match="PersistentTradeCheckpoint", + ): + sink.persist_accepted( + make_trade(), + expected_trade=None, + ) def test_rejects_invalid_dependencies() -> None: @@ -151,6 +301,15 @@ def test_rejects_invalid_dependencies() -> None: venue=VENUE, ) + with pytest.raises( + TypeError, + match="TradeCheckpointStorageProtocol", + ): + TradeStorageObservationSink( + trade_storage=RecordingWriteOnlyTradeStorage(), + venue=VENUE, + ) + with pytest.raises(TypeError, match="clock must be callable"): TradeStorageObservationSink( trade_storage=RecordingTradeStorage(), diff --git a/app/tests/unit/storage/test_migrations.py b/app/tests/unit/storage/test_migrations.py index 45bdf26..b3f788e 100644 --- a/app/tests/unit/storage/test_migrations.py +++ b/app/tests/unit/storage/test_migrations.py @@ -105,6 +105,7 @@ def test_default_migrations_have_stable_order_and_names() -> None: (5, "add_trade_observation_sources"), (6, "add_quote_and_candle_observation_sources"), (7, "create_market_data_partition_registry"), + (8, "create_trade_stream_checkpoints"), ) @@ -119,8 +120,8 @@ def test_default_schema_defines_partitions_identities_and_constraints() -> None: assert "CREATE TABLE market_data.trades" in sql assert "PRIMARY KEY (venue, symbol, trade_id, executed_at)" in sql assert "trade_id BETWEEN -2147483648 AND 2147483647" in sql - assert sql.count("CHECK (BTRIM(venue) <> '')") == 3 - assert sql.count("CHECK (BTRIM(symbol) <> '')") == 3 + assert sql.count("CHECK (BTRIM(venue) <> '')") == 4 + assert sql.count("CHECK (BTRIM(symbol) <> '')") == 4 assert sql.count("CHECK (BTRIM(source) <> '')") == 3 assert "PARTITION BY RANGE (executed_at)" in sql assert "CREATE TABLE market_data.quotes" in sql @@ -143,6 +144,18 @@ def test_default_schema_defines_partitions_identities_and_constraints() -> None: assert "partition_bound TEXT NOT NULL" in sql assert "BTRIM(partition_bound) <> ''" in sql assert "range_end > range_start" in sql + assert "CREATE TABLE market_data.trade_stream_checkpoints" in sql + assert "PRIMARY KEY (venue, symbol)" in sql + assert "revision BIGINT NOT NULL" in sql + assert "checkpoint_schema_version INTEGER NOT NULL DEFAULT 1" in sql + assert "CONSTRAINT trade_stream_checkpoints_trade_fk" in sql + assert "FOREIGN KEY (" in sql + assert ") REFERENCES market_data.trades (" in sql + assert "ON UPDATE NO ACTION" in sql + assert "ON DELETE NO ACTION" in sql + assert "DEFERRABLE INITIALLY DEFERRED" in sql + assert "CHECK (revision > 0)" in sql + assert "CHECK (checkpoint_schema_version > 0)" in sql def test_run_locks_and_applies_every_pending_migration_in_order() -> None: @@ -150,7 +163,7 @@ def test_run_locks_and_applies_every_pending_migration_in_order() -> None: result = runner.run() - assert result == (1, 2, 3, 4, 5, 6, 7) + assert result == (1, 2, 3, 4, 5, 6, 7, 8) assert provider.calls == 1 assert connection.entered == 1 assert connection.exited == 1 @@ -167,7 +180,7 @@ def test_run_locks_and_applies_every_pending_migration_in_order() -> None: ) and isinstance(parameters, tuple) ) - assert inserted_versions == (1, 2, 3, 4, 5, 6, 7) + assert inserted_versions == (1, 2, 3, 4, 5, 6, 7, 8) def test_run_skips_already_applied_migrations() -> None: @@ -196,7 +209,7 @@ def test_run_applies_only_migrations_after_existing_prefix() -> None: result = runner.run() - assert result == (3, 4, 5, 6, 7) + assert result == (3, 4, 5, 6, 7, 8) inserted_versions = tuple( parameters[0] for statement, parameters in cursor.calls @@ -205,7 +218,7 @@ def test_run_applies_only_migrations_after_existing_prefix() -> None: ) and isinstance(parameters, tuple) ) - assert inserted_versions == (3, 4, 5, 6, 7) + assert inserted_versions == (3, 4, 5, 6, 7, 8) def test_run_rejects_unknown_applied_version() -> None: diff --git a/app/tests/unit/test_postgres_market_data_support.py b/app/tests/unit/test_postgres_market_data_support.py index 91b6162..0c16500 100644 --- a/app/tests/unit/test_postgres_market_data_support.py +++ b/app/tests/unit/test_postgres_market_data_support.py @@ -5,11 +5,14 @@ from typing import Any import pytest from psycopg.conninfo import conninfo_to_dict +from tests.support import postgres_market_data from tests.support.postgres_market_data import ( POSTGRES_TEST_APPLICATION_NAME, POSTGRES_TEST_CONTROL_APPLICATION_NAME, + count_other_test_connections, load_postgres_test_settings, reset_postgres_test_database, + wait_for_postgres_relation_lock_waiters, ) @@ -59,6 +62,52 @@ class RecordingControlConnection: ) +class QueuedCursor: + def __init__(self, connection: QueuedControlConnection) -> None: + self._connection = connection + + def __enter__(self) -> QueuedCursor: + return self + + def __exit__(self, *_: object) -> None: + return None + + def execute( + self, + statement: object, + parameters: object = None, + ) -> None: + self._connection.statements.append( + (" ".join(str(statement).split()), parameters) + ) + + def fetchone(self) -> object: + if not self._connection.rows: + raise AssertionError("Не подготовлен ответ PostgreSQL") + + return self._connection.rows.pop(0) + + +class QueuedControlConnection: + def __init__(self, *, rows: list[object]) -> None: + self.rows = list(rows) + self.statements: list[tuple[str, object]] = [] + + def cursor(self) -> QueuedCursor: + return QueuedCursor(self) + + +class FakeClock: + def __init__(self) -> None: + self.current = 0.0 + + def monotonic(self) -> float: + return self.current + + def sleep(self, seconds: float) -> None: + self.current += seconds + + def test_postgres_harness_is_disabled_without_explicit_flag() -> None: assert load_postgres_test_settings({}) is None @@ -188,3 +237,230 @@ def test_postgres_reset_refuses_unvalidated_connection_before_drop( assert connection.statements == [ "SELECT current_database(), current_setting('application_name')", ] + + +def test_connection_count_includes_every_non_control_database_session() -> None: + database_name = "dzentra_test_all_connections" + connection: Any = QueuedControlConnection( + rows=[ + (database_name, POSTGRES_TEST_CONTROL_APPLICATION_NAME), + (3,), + ] + ) + + result = count_other_test_connections(connection) + + assert result == 3 + assert connection.statements == [ + ( + "SELECT current_database(), " + "current_setting('application_name')", + None, + ), + ( + "SELECT COUNT(*) FROM pg_catalog.pg_stat_activity " + "WHERE datname = %s " + "AND backend_type = 'client backend' " + "AND application_name IS DISTINCT FROM %s", + (database_name, POSTGRES_TEST_CONTROL_APPLICATION_NAME), + ), + ] + + +@pytest.mark.parametrize( + "identity", + ( + ("production", POSTGRES_TEST_CONTROL_APPLICATION_NAME), + ("dzentra_test_all_connections", "another-application"), + None, + ), +) +def test_connection_count_revalidates_control_connection( + identity: object, +) -> None: + connection: Any = QueuedControlConnection(rows=[identity, (0,)]) + + with pytest.raises(RuntimeError, match="validated test control"): + count_other_test_connections(connection) + + assert len(connection.statements) == 1 + + +@pytest.mark.parametrize( + "count_row", + ( + (True,), + (-1,), + ("1",), + [1], + None, + ), +) +def test_connection_count_rejects_invalid_postgres_result( + count_row: object, +) -> None: + connection: Any = QueuedControlConnection( + rows=[ + ( + "dzentra_test_all_connections", + POSTGRES_TEST_CONTROL_APPLICATION_NAME, + ), + count_row, + ] + ) + + with pytest.raises(RuntimeError, match="invalid connection count"): + count_other_test_connections(connection) + + +def test_relation_waiter_uses_validated_database_and_exact_count( + monkeypatch: pytest.MonkeyPatch, +) -> None: + database_name = "dzentra_test_relation_waiter" + connection: Any = QueuedControlConnection( + rows=[ + (database_name, POSTGRES_TEST_CONTROL_APPLICATION_NAME), + (16_384,), + (0,), + (2,), + ] + ) + clock = FakeClock() + monkeypatch.setattr(postgres_market_data.time, "monotonic", clock.monotonic) + monkeypatch.setattr(postgres_market_data.time, "sleep", clock.sleep) + + wait_for_postgres_relation_lock_waiters( + connection, + relation_name="market_data.trade_stream_checkpoints", + expected_count=2, + timeout_seconds=1.0, + ) + + assert connection.rows == [] + assert connection.statements[1] == ( + "SELECT pg_catalog.to_regclass(%s)::oid", + ("market_data.trade_stream_checkpoints",), + ) + lock_statement, lock_parameters = connection.statements[2] + assert "locktype = 'relation'" in lock_statement + assert "AND NOT granted" in lock_statement + assert lock_parameters == (database_name, 16_384) + assert connection.statements[3] == ( + lock_statement, + lock_parameters, + ) + + +def test_relation_waiter_times_out_with_observed_count( + monkeypatch: pytest.MonkeyPatch, +) -> None: + connection: Any = QueuedControlConnection( + rows=[ + ( + "dzentra_test_relation_timeout", + POSTGRES_TEST_CONTROL_APPLICATION_NAME, + ), + (42,), + (0,), + (0,), + ] + ) + clock = FakeClock() + monkeypatch.setattr(postgres_market_data.time, "monotonic", clock.monotonic) + monkeypatch.setattr(postgres_market_data.time, "sleep", clock.sleep) + + with pytest.raises( + TimeoutError, + match=r"expected 1, observed 0\.", + ): + wait_for_postgres_relation_lock_waiters( + connection, + relation_name="market_data.trade_stream_checkpoints", + expected_count=1, + timeout_seconds=0.01, + ) + + +@pytest.mark.parametrize( + "relation_row", + ( + None, + (), + (None,), + (True,), + ("42",), + (0,), + ), +) +def test_relation_waiter_rejects_unresolved_or_invalid_relation( + relation_row: object, +) -> None: + connection: Any = QueuedControlConnection( + rows=[ + ( + "dzentra_test_relation_shape", + POSTGRES_TEST_CONTROL_APPLICATION_NAME, + ), + relation_row, + ] + ) + + with pytest.raises(RuntimeError, match="requested test relation"): + wait_for_postgres_relation_lock_waiters( + connection, + relation_name="market_data.trade_stream_checkpoints", + expected_count=1, + ) + + +@pytest.mark.parametrize( + "waiter_row", + ( + None, + (), + (True,), + (-1,), + ("1",), + [1], + ), +) +def test_relation_waiter_rejects_invalid_count_result( + waiter_row: object, +) -> None: + connection: Any = QueuedControlConnection( + rows=[ + ( + "dzentra_test_waiter_shape", + POSTGRES_TEST_CONTROL_APPLICATION_NAME, + ), + (42,), + waiter_row, + ] + ) + + with pytest.raises(RuntimeError, match="invalid relation-lock waiter"): + wait_for_postgres_relation_lock_waiters( + connection, + relation_name="market_data.trade_stream_checkpoints", + expected_count=1, + ) + + +@pytest.mark.parametrize( + "timeout_seconds", + (0.0, -1.0, float("inf"), float("nan"), True), +) +def test_relation_waiter_requires_positive_finite_timeout( + timeout_seconds: float, +) -> None: + connection: Any = QueuedControlConnection(rows=[]) + + with pytest.raises(ValueError, match="positive finite"): + wait_for_postgres_relation_lock_waiters( + connection, + relation_name="market_data.trade_stream_checkpoints", + expected_count=1, + timeout_seconds=timeout_seconds, + ) + + assert connection.statements == [] diff --git a/app/tests/unit/test_trade_stream_runtime_support.py b/app/tests/unit/test_trade_stream_runtime_support.py index 1a43650..2f6c7c0 100644 --- a/app/tests/unit/test_trade_stream_runtime_support.py +++ b/app/tests/unit/test_trade_stream_runtime_support.py @@ -5,6 +5,7 @@ import asyncio import pytest from tests.support.trade_stream_runtime import ( + active_owned_task_names, run_scenario, wait_until_or_runtime_exit, ) @@ -90,3 +91,30 @@ def test_wait_until_or_runtime_exit_rejects_clean_early_runtime_exit() -> None: ) run_scenario(scenario()) + + +def test_owned_task_detection_covers_application_and_processing_tasks() -> None: + async def scenario() -> None: + release = asyncio.Event() + names = ( + "application-shutdown", + "telegram-polling", + "trade-stream-market-processing", + "persistent-application-verification", + ) + tasks = tuple( + asyncio.create_task(release.wait(), name=name) + for name in names + ) + + try: + await asyncio.sleep(0) + + assert active_owned_task_names() == tuple(sorted(names)) + finally: + release.set() + await asyncio.gather(*tasks) + + assert active_owned_task_names() == () + + run_scenario(scenario()) diff --git a/docs/migrations/build_060_28.md b/docs/migrations/build_060_28.md new file mode 100644 index 0000000..86aa9f9 --- /dev/null +++ b/docs/migrations/build_060_28.md @@ -0,0 +1,244 @@ +# Build 060.28 — Persistent Checkpoint and Startup Recovery + +**Engineering Migration Report** + +--- + +## Контроль документа + +| Свойство | Значение | +|---|---| +| Build | 060.28 | +| Статус | Completed | +| Подсистема | Market Data Acquisition / Persistent Checkpoint | +| Компонент | Trade Stream Startup Recovery | +| Дата завершения | 2026-08-01 | +| Версия | 1.0 | + +--- + +## Связанные документы + +- `build_060_28_architecture.md` — архитектура, решения и подробные + результаты подэтапов 060.28.0–060.28.8; +- `build_060_27.md` — Persistent Market Data Storage; +- `dzentra_target_architecture.md` — место Market Data Acquisition в + целевой архитектуре Dzentra; +- `master-roadmap.md` — дальнейшая последовательность Build. + +--- + +## 1. Назначение Build + +Build 060.28 сделал operational checkpoint Trade Stream постоянным и +добавил восстановление Runtime после перезапуска процесса. + +PostgreSQL Canonical Trades остаются источником рыночных фактов. +Persistent checkpoint хранит только подтверждённую точку в этой истории +и позволяет восстановить bounded deduplication window до подключения к +бирже. + +Итоговая последовательность запуска: + +```text +PostgreSQL pool + migrations + ↓ +checkpoint hydration или first-adoption + ↓ +WebSocket connect + subscription ACK + ↓ +REST Recovery при закрытом Live gate + ↓ +buffered Live в исходном порядке + ↓ +обычный Production Runtime +``` + +--- + +## 2. Завершённые подэтапы + +| Подэтап | Название | Статус | +|---|---|---| +| 060.28.0 | Architecture, Contracts and Failure Policy | Accepted | +| 060.28.1 | PostgreSQL Checkpoint Schema and Migration | Accepted | +| 060.28.2 | Checkpoint Repository and Atomic Trade Commit | Accepted | +| 060.28.3 | Consistency Persistence Integration | Accepted | +| 060.28.4 | State Hydration and Deduplication Restoration | Accepted | +| 060.28.5 | Startup Recovery and ACK/Live Boundary | Accepted | +| 060.28.6 | Bootstrap and Lifecycle Integration | Accepted | +| 060.28.7 | PostgreSQL Restart and Failure Verification | Accepted | +| 060.28.8 | Final Regression and Acceptance | Accepted | + +Каждый подэтап проходил отдельный read-only review. Findings +исправлялись и закрывались regression-тестами до принятия. + +--- + +## 3. Реализованная архитектура + +### 3.1. Schema и единый repository + +Migration 8 создала непартиционированную таблицу +`market_data.trade_stream_checkpoints` с ключом `(venue, symbol)`. +Checkpoint ссылается на точную Canonical Trade identity через deferred +`ON UPDATE/DELETE NO ACTION` foreign key. + +`PostgresTradeRepository` остаётся единственным владельцем записи +Trades и реализует узкий `TradeCheckpointStorageProtocol`. Новая Trade и +продвижение checkpoint выполняются одним соединением и одной +транзакцией. Expected identity и revision обеспечивают CAS-защиту от +stale writer; повтор уже зафиксированной candidate Trade идемпотентен. + +Duplicate Trade обновляет provenance без продвижения persistent или +in-memory checkpoint. + +### 3.2. Consistency и Hydration + +`TradeStreamConsistencyController` продвигает in-memory state только +после успешного durable commit. Ошибка PostgreSQL является фатальной и +не скрывается. + +`TradeStreamStateHydrator` до запуска сети: + +1. загружает и проверяет persistent checkpoint; +2. восстанавливает bounded rollover-aware Trade tail; +3. создаёт временные `TradeStreamState`; +4. атомарно публикует весь набор в общий State Store. + +Если после Build 060.27 история уже существует, а checkpoint ещё нет, +последняя проверенная Trade один раз принимается как revision `1` без +повторного наблюдения и изменения provenance. + +### 3.3. Startup Recovery + +`RuntimeStartupRecoveryCoordinator` выполняет blocking Hydration и REST +Recovery через принадлежащие Runtime задачи. Cancellation ожидает уже +начатый worker, поэтому PostgreSQL pool не закрывается под выполняющейся +операцией. + +Production Runtime использует одного WebSocket consumer до завершения +startup. Он ожидает положительный subscription ACK с конечным timeout, +складывает ранние market-сообщения в ограниченный FIFO, выполняет REST +Recovery при закрытом общем gate и затем разбирает FIFO в исходном +порядке. + +Negative ACK, timeout, переполнение FIFO, повреждённый checkpoint или +ошибка Recovery завершают startup без открытия Live processing. + +### 3.4. Bootstrap и lifecycle + +Отдельный checkpoint feature flag не добавлен. Persistent checkpoint +включается только вместе с `MARKET_DATA_STORAGE_ENABLED=true` и +использует тот же экземпляр `PostgresTradeRepository`, что и запись +Canonical Trades. + +Composition не выполняет I/O и не создаёт фоновых задач. Application +сохраняет порядок: + +```text +pool open → migrations → Runtime start +Runtime stop → ожидание workers → pool close +``` + +Неполная пара Trade sink/checkpoint storage отклоняется до создания +сетевых компонентов. + +### 3.5. Partitions и Retention + +Foreign key временно снимается только внутри транзакции обслуживания +Trade partitions и восстанавливается до commit с тем же строгим +контрактом. Попытка Retention удалить активную checkpoint Trade +откатывает всю операцию. + +Partition Manager и atomic writer используют единый порядок блокировок: + +```text +market_data.trades parent + ↓ +trades_default + ↓ +trade_stream_checkpoints +``` + +Это исключает deadlock при одновременной записи в существующую месячную +partition и создании новой. + +--- + +## 4. Real PostgreSQL verification + +Безопасный opt-in harness требует отдельную локальную базу с именем +`dzentra_test_*` и два явных параметра: + +- `DZENTRA_RUN_POSTGRES_TESTS=1`; +- `DZENTRA_TEST_POSTGRES_DSN`. + +Реальный PostgreSQL 16 подтвердил: + +- restart с восстановлением checkpoint и deduplication tail; +- first-adoption существующей истории 060.27; +- отсутствие network I/O до завершения Hydration/adoption; +- rollback ошибки до commit и восстановление после ошибки после commit; +- фатальный orphan checkpoint и REST failure при закрытом Live gate; +- ожидание blocking Hydration при cancellation; +- rollover-границы `INT32_MAX → INT32_MIN` и `-1 → 0`; +- защиту активной checkpoint Trade при Partition/Retention; +- отсутствие deadlock между Partition Manager и atomic writer; +- отсутствие оставшихся tasks, threads и PostgreSQL connections. + +--- + +## 5. Финальные результаты + +Итоговая приёмка выполнена 2026-08-01: + +```text +Partition lock-order unit target: 21 passed +Restart/failure repeated ten times: 90 passed +Partition/checkpoint race ten times: 10 passed +Full PostgreSQL Storage integration: 51 passed +Full integration with PostgreSQL: 64 passed +PostgreSQL suite without opt-in: 51 skipped +Fixed stress target: 3 passed, 1 deselected +Full offline regression: 2305 passed, 69 deselected +git diff --check: clean +Untracked files whitespace check: clean +``` + +В integration и stress наборах `ResourceWarning` считался ошибкой. +Одноразовый PostgreSQL-контейнер после приёмки остановлен и удалён. +Итоговый read-only review не обнаружил открытых findings. + +--- + +## 6. Границы Build + +Build 060.28 намеренно не реализует: + +- общий Data Access Layer и Historical Queries; +- Replay API и детерминированные replay-часы; +- подключение persistent Quote/Candle consumers; +- автоматический Retention Scheduler; +- distributed lease, leader election и HA failover; +- бесконечный Recovery retry/backoff. + +Эти обязанности относятся к следующим Build. Полная итоговая ревизия +документации Market Data Acquisition остаётся задачей Build 060.30. + +Постороннее пользовательское изменение `.gitignore` не относится к +Build 060.28 и не должно включаться в его staging. + +--- + +## 7. Итог + +Build 060.28 завершён и принят. + +Dzentra восстанавливает подтверждённое состояние Trade Stream из +PostgreSQL после перезапуска, заполняет downtime gap через REST и только +после этого продолжает Live processing. Durable history, persistent +checkpoint и in-memory state теперь продвигаются в одном проверяемом +порядке без скрытой потери последовательности. + +Следующий этап — Build 060.29, Market Data Access and Replay. diff --git a/docs/migrations/build_060_28_architecture.md b/docs/migrations/build_060_28_architecture.md new file mode 100644 index 0000000..3c7607e --- /dev/null +++ b/docs/migrations/build_060_28_architecture.md @@ -0,0 +1,851 @@ +# Build 060.28 — Persistent Checkpoint and Startup Recovery Architecture + +**Статус:** Completed + +**Build:** 060.28 + +**Подсистема:** Market Data Acquisition / Persistent Checkpoint + +**Дата начала:** 2026-08-01 + +**Дата завершения:** 2026-08-01 + +**Версия документа:** 1.14 + +--- + +## 1. Назначение + +Build 060.28 сохраняет operational checkpoint Trade Stream в +PostgreSQL и восстанавливает Runtime после перезапуска процесса. + +Build использует Canonical Trade history, созданную в 060.27, как более +достоверный источник рыночных фактов. Persistent checkpoint является +операционным указателем на подтверждённую строку истории и не заменяет +Market Data Storage. + +После завершения Build новый процесс должен: + +1. проверить persistent checkpoint по durable Trade history; +2. восстановить checkpoint и deduplication window в Consistency Layer; +3. подключить WebSocket и подтвердить подписку; +4. восстановить Trades за время остановки через REST; +5. только затем продолжить обработку накопленных Live сообщений. + +--- + +## 2. Статус подэтапов + +| Подэтап | Название | Статус | +|---|---|---| +| 060.28.0 | Architecture, Contracts and Failure Policy | Accepted | +| 060.28.1 | PostgreSQL Checkpoint Schema and Migration | Accepted | +| 060.28.2 | Checkpoint Repository and Atomic Trade Commit | Accepted | +| 060.28.3 | Consistency Persistence Integration | Accepted | +| 060.28.4 | State Hydration and Deduplication Restoration | Accepted | +| 060.28.5 | Startup Recovery and ACK/Live Boundary | Accepted | +| 060.28.6 | Bootstrap and Lifecycle Integration | Accepted | +| 060.28.7 | PostgreSQL Restart and Failure Verification | Accepted | +| 060.28.8 | Final Regression and Acceptance | Accepted | + +060.28.0 зафиксировал immutable `PersistentTradeCheckpoint`, узкий +runtime-checkable `TradeCheckpointStorageProtocol` и специализированные +checkpoint conflict/integrity errors. Контракты не импортируют Runtime, +Recovery, Bootstrap или PostgreSQL implementation. + +060.28.1 добавил versioned migration 8 с непартиционированной таблицей +`market_data.trade_stream_checkpoints`. Реальный PostgreSQL подтвердил +структуру колонок, idempotent migration history, точный foreign key и +запрет orphan/delete для активной checkpoint Trade. + +Оба подэтапа приняты после отдельного read-only review и реальной +PostgreSQL-проверки. Они реализованы без checkpoint repository, +Consistency integration, Runtime wiring, Bootstrap I/O и изменения +feature flags. + +060.28.2 расширил существующий `PostgresTradeRepository`: он остаётся +единственным владельцем записи Canonical Trades и одновременно +реализует `TradeCheckpointStorageProtocol`. Trade write, checkpoint +lock, expected-identity CAS и revision update выполняются через одно +соединение и одну transaction. Repository распознаёт уже +зафиксированную candidate Trade как безопасный повтор, восстанавливает +bounded rollover-aware tails и явно отклоняет stale/ambiguous writer. + +Подэтап принят после отдельного read-only review и исправления двух +rollover-tail findings. Durable acceptance anchor использует +`first_observed_at`, а повторившийся raw Trade ID выбирается только из +последнего цикла не позже exact checkpoint Trade. Consistency, Runtime, +Bootstrap и feature flags не изменялись. + +060.28.3 разделил persistence-вызовы по уже принятому решению +Consistency Layer. Новая Trade передаётся в атомарный +`store_trade_and_advance_checkpoint` вместе с точной предыдущей +`last_trade`, а полный дубликат вызывает только `store_trade` для +обновления provenance. Storage не повторяет rollover-классификацию. +Ошибка любой ветви распространяется вызывающему коду; новая Trade не +попадает в in-memory state до успешного PostgreSQL commit. + +Подэтап реализован без hydration, Startup Recovery, новых feature flags, +изменения Bootstrap/settings или lifecycle. Отдельный приёмочный +read-only review подтвердил границы ответственности, порядок durable +commit → in-memory checkpoint и полный rollback при CAS-конфликте; +findings не выявлено, подэтап принят. + +060.28.4 добавил строгую фабрику `TradeStreamState.from_history`, +одноразовую атомарную публикацию через `TradeStreamStateStore.initialize` +и отдельный `TradeStreamStateHydrator`. Hydrator сначала полностью +готовит состояния всех символов и только затем публикует их в общий +Store; при ошибке ни один частично собранный state не становится +доступен Runtime. + +Для перехода с Build 060.27 storage-контракт получил отдельную операцию +`adopt_existing_trade_as_checkpoint`. Она создаёт revision `1` на уже +существующей durable Trade, не выполняет повторную запись наблюдения и +не меняет `source`, provenance, `first_observed_at` или +`last_observed_at`. После adoption Hydrator повторно загружает tail уже +от окончательной persistent-точки и заново проверяет его. Подэтап +реализован без подключения к Bootstrap, Runtime lifecycle и Startup +Recovery. Приёмочный review выявил один случай, когда ложный non-tuple +ответ Storage мог быть принят за пустую историю. После исправления +только `tuple()` означает отсутствие истории, а `None` и `[]` приводят +к фатальной integrity-ошибке без публикации state. Повторный review +findings не выявил; подэтап принят. + +060.28.5 добавил отдельный `RuntimeStartupRecoveryCoordinator` и +встроил persistent startup-путь в `TradeStreamProductionRuntime`. +Hydration выполняется до сетевого подключения. После подписки Runtime +одним consumer ожидает положительный ACK, сохраняет ранние Live +документы в ограниченный FIFO, выполняет REST Recovery при закрытом +общем gate и только затем обрабатывает накопленный Live-поток. + +Первый приёмочный review выявил расхождение регистра между Hydrator и +Startup Recovery, способное бесшумно пропустить восстановление для +lowercase-конфигурации. Все Runtime-границы и Hydrator переведены на +единый `normalize_symbol`; варианты одного символа в разном регистре +схлопываются до запуска blocking worker. Regression-тест проходит +полную цепочку Hydration → exact-key State Store → REST Recovery. +Повторный независимый review findings не выявил; подэтап принят без +изменения Bootstrap и settings. + +060.28.6 подключил persistent checkpoint к Production Bootstrap без +нового feature flag. При включённом Market Data Storage один экземпляр +`PostgresTradeRepository` одновременно используется как владелец +durable Trades и checkpoint storage, а Runtime получает созданный +Composition `RuntimeStartupRecoveryCoordinator`. Неполный persistent- +граф отклоняется до создания WebSocket/REST-компонентов. + +Настройки ACK timeout и вместимости стартового FIFO получили строгую +валидацию и явные environment-параметры. Сборка приложения по-прежнему +не открывает PostgreSQL, не выполняет migrations, Hydration или Recovery +и не создаёт фоновых задач. Application сохраняет порядок pool open + +migrations → Runtime и Runtime stop + ожидание workers → pool close. +После исправления устаревшего PostgreSQL integration harness повторный +read-only review findings не выявил; подэтап принят. + +060.28.7 добавил opt-in проверку полного Application lifecycle на +одноразовом PostgreSQL 16: перезапуск на той же durable history, +first-adoption истории без checkpoint, откат незавершённой transaction, +повтор после ошибки persistence и cancellation во время blocking I/O. +Все сценарии используют только явно заданную локальную тестовую базу и +проверяют отсутствие оставшихся соединений, задач и обработчиков. + +Первый приёмочный review выявил недостаточно строгую проверку порядка: +исходные restart-сценарии наблюдали восстановленное состояние только к +моменту подписки. Два детерминированных regression-сценария теперь +останавливают соответственно `load_checkpoint` и +`adopt_existing_trade_as_checkpoint` внутри storage-вызова и до снятия +barrier подтверждают отсутствие WebSocket/REST I/O. После исправления +два независимых read-only review findings не выявили; production-код не +изменялся, подэтап принят. + +060.28.8 выполнил verification-only матрицу всего Build: десятикратный +прогон restart/failure-сценариев, полный PostgreSQL Storage и общий +integration-наборы, opt-out isolation, fixed stress и полную стандартную +регрессию. Итоговый review выявил один потенциальный PostgreSQL deadlock +между Partition Manager и атомарным Trade/checkpoint writer. + +Partition Manager переведён на единый порядок блокировок parent Trades → +default partition → checkpoint table. Детерминированный PostgreSQL-тест +подтвердил реальное участие обоих конкурентных callers и отсутствие +deadlock; сценарий стабильно прошёл десять повторов. После повторения +всей приёмочной матрицы итоговый review findings не выявил. Подэтап и +Build 060.28 приняты. + +--- + +## 3. Исходные инварианты + +Build продолжает принятые решения 060.24–060.27: + +1. Operational checkpoint принадлежит только + `TradeStreamState.last_trade`. +2. Live и Recovery используют один `TradeStreamStateStore` и один + `TradeStreamConsistencyController`. +3. Canonical Trade сохраняется до продвижения in-memory checkpoint. +4. Ошибка включённого persistent storage является фатальной. +5. Идентичность durable Trade равна + `(venue, symbol, trade_id, executed_at)`. +6. Порядок Trade ID использует единый signed 32-bit rollover-aware + контракт. +7. PostgreSQL pool открывается до Runtime и закрывается только после + полной остановки Runtime. +8. Composition не выполняет I/O и не создаёт фоновых задач. + +--- + +## 4. Иерархия достоверности + +```text +PostgreSQL Canonical Trades + │ подтверждённые рыночные факты + ▼ +Persistent Trade Stream Checkpoint + │ операционный указатель + ▼ +TradeStreamState + │ оперативное состояние процесса + ▼ +Live / Recovery processing +``` + +Persistent checkpoint нельзя использовать, если соответствующая +Canonical Trade отсутствует или отличается от durable history. + +Checkpoint не хранит независимую копию цены, количества, стороны и +source. Он хранит точную identity Trade. Checkpoint Service получает +полный Canonical `Trade` соединением checkpoint с `market_data.trades`. + +--- + +## 5. Архитектурная граница + +```text +Consistency Layer + │ acquisition-side Protocol + ▼ +Checkpoint Service + │ storage contracts + ▼ +PostgreSQL checkpoint repository + │ общий managed pool + ▼ +market_data.trade_stream_checkpoints + │ exact durable identity + ▼ +market_data.trades +``` + +Consistency, Recovery и Runtime не импортируют `psycopg`, PostgreSQL +repository или pool. PostgreSQL implementation зависит от Canonical +Models и storage contracts, но не от Runtime lifecycle. + +Общий `MarketDataStorage` остаётся write-only фасадом Trades, Quotes и +Candles. Узкие checkpoint reads не превращаются в Historical Query API; +общий Data Access Layer относится к Build 060.29. + +--- + +## 6. Storage contracts + +### 6.1. PersistentTradeCheckpoint + +Неизменяемый результат чтения checkpoint содержит: + +```text +PersistentTradeCheckpoint +├── venue: str +├── trade: Trade +├── revision: int +├── updated_at: timezone-aware datetime +└── checkpoint_schema_version: int +``` + +Полный `Trade` уже проверен по durable history и является checkpoint, +который можно установить в Consistency state. + +### 6.2. TradeCheckpointStorageProtocol + +Узкий storage-контракт предусматривает: + +```text +load_checkpoint(venue, symbol) +load_checkpoint_tail(venue, checkpoint, limit) +load_latest_trade_tail(venue, symbol, limit) +adopt_existing_trade_as_checkpoint(venue, trade) +store_trade_and_advance_checkpoint( + venue, + expected_trade, + trade, + observed_at, +) +``` + +Назначение операций: + +- `load_checkpoint` возвращает только подтверждённый полный checkpoint; +- `load_checkpoint_tail` загружает bounded history для восстановления + deduplication window; +- `load_latest_trade_tail` используется только при первой инициализации + после Build 060.27, если история существует, а checkpoint ещё нет; +- `adopt_existing_trade_as_checkpoint` создаёт первый checkpoint на + точной уже существующей durable Trade без повторной записи Trade; +- `store_trade_and_advance_checkpoint` атомарно сохраняет принятую Trade + и продвигает persistent checkpoint. + +Основные read/commit-операции реализованы в 060.28.2, а безопасная +first-adoption — в 060.28.4 существующим `PostgresTradeRepository`. +Отдельный второй writer не создаётся; общий `MarketDataStorage` +сохраняет write-only границу, а checkpoint reads доступны только через +узкий `TradeCheckpointStorageProtocol`. + +--- + +## 7. PostgreSQL schema + +Migration 8 создаёт: + +```text +market_data.trade_stream_checkpoints +├── venue TEXT +├── symbol TEXT +├── trade_id INTEGER +├── executed_at TIMESTAMPTZ +├── revision BIGINT +├── updated_at TIMESTAMPTZ +└── checkpoint_schema_version INTEGER +``` + +Ограничения: + +```text +PRIMARY KEY (venue, symbol) + +FOREIGN KEY (venue, symbol, trade_id, executed_at) + REFERENCES market_data.trades ( + venue, + symbol, + trade_id, + executed_at + ) + ON UPDATE NO ACTION + ON DELETE NO ACTION + DEFERRABLE INITIALLY DEFERRED +``` + +Дополнительно проверяются: + +- непустые `venue` и `symbol`; +- signed 32-bit диапазон `trade_id`; +- положительные `revision` и `checkpoint_schema_version`; +- timezone semantics через PostgreSQL `TIMESTAMPTZ`. + +Таблица не партиционируется: для `(venue, symbol)` существует ровно одна +маленькая operational row. + +--- + +## 8. Атомарное продвижение + +Для новой принятой Trade `PostgresTradeRepository` выполняет: + +```text +BEGIN + │ + ├── store Canonical Trade + ├── lock current checkpoint row + ├── verify expected previous Trade identity + ├── advance checkpoint and revision + └── COMMIT + ↓ +advance in-memory TradeStreamState +``` + +Если Trade write или checkpoint update завершается ошибкой, вся +PostgreSQL transaction откатывается, а in-memory state не изменяется. + +Точный дубликат продолжает обновлять provenance в `market_data.trades`, +но не продвигает persistent или in-memory checkpoint. + +`expected_trade` защищает от stale writer. Если database checkpoint не +равен ожидаемому предыдущему checkpoint, операция завершается явным +конфликтом. Это защита от повреждения, а не distributed leader election. + +--- + +## 9. Восстановление Consistency state + +Одного `last_trade` недостаточно. После перезапуска REST Recovery может +вернуть более старые Trades у общей временной границы. Поэтому Startup +должен восстановить bounded deduplication tail до checkpoint. + +Правила: + +1. Загружается не более текущего `deduplication_window_size`. +2. Порядок нормализуется общим rollover-aware контрактом. +3. Проверяются symbol, Canonical payload и конечная checkpoint identity. +4. Сначала собирается новый временный `TradeStreamState`. +5. Store получает состояние только после полной успешной проверки. +6. Hydration не вызывает persistence sink и не записывает provenance. +7. Первый checkpoint на истории 060.27 создаётся только после строгой + проверки временного state. +8. После first-adoption tail повторно загружается относительно уже + созданного checkpoint и проверяется заново. +9. Все символы публикуются одним `initialize`; ошибка одного символа не + оставляет частично заполненный State Store. + +Обязательные границы порядка: + +```text +INT32_MAX → INT32_MIN +-1 → 0 +``` + +--- + +## 10. Первый запуск после Build 060.27 + +### 10.1. Нет checkpoint и нет Trades + +Состояние остаётся пустым. REST backfill без известной начальной границы +не выполняется. Первый принятый Live Trade создаёт первый checkpoint. + +### 10.2. Нет checkpoint, но есть durable Trades + +Checkpoint Service загружает bounded latest tail, проверяет его порядок +и один раз создаёт checkpoint на последней подтверждённой Trade. Этот +путь предназначен для перехода с завершённого Build 060.27. + +### 10.3. Checkpoint существует + +Checkpoint обязан ссылаться на существующую точную Canonical Trade. +Отсутствие строки, несовпадение payload или неоднозначный rollover-order +являются фатальной ошибкой startup. Тихий переход к пустому состоянию +запрещён. + +--- + +## 11. Startup Recovery sequence + +```text +open PostgreSQL pool + ↓ +apply migrations + ↓ +load checkpoint and deduplication tail + ↓ +hydrate TradeStreamState + ↓ +connect WebSocket + ↓ +send Trade subscription + ↓ +receive successful ACK + ↓ +capture recovery_end_time + ↓ +REST Recovery under shared live gate + ↓ +process buffered WebSocket market messages + ↓ +start normal receive / Supervisor / Scheduler lifecycle +``` + +До завершения Recovery существует только один consumer WebSocket. +Market messages, необходимые для получения последующего ACK, временно +буферизуются в исходном порядке и не передаются в Consistency Layer. + +ACK имеет конечный timeout. Negative ACK, timeout, checkpoint error или +Recovery error являются terminal startup failure. + +REST Recovery использует существующие `TradeRecoveryWindowPlanner`, +`TradeRecoveryController` и общий Consistency state. При crash посреди +Recovery следующий процесс начинает с последней атомарно сохранённой +checkpoint Trade. + +--- + +## 12. Retention policy + +Retention не может удалить Trade, на которую указывает активный +checkpoint. + +Защита состоит из трёх уровней: + +1. `FOREIGN KEY ... ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED` + сохраняет ссылочную целостность и позволяет фиксировать Trade и + checkpoint одной transaction; +2. PostgreSQL создаёт внутренние FK-зависимости от каждой Trade + partition, поэтому Partition Manager и Retention Service под общим + advisory lock транзакционно снимают FK только на время переноса или + удаления секции и восстанавливают его с тем же строгим контрактом до + commit; +3. повторное создание FK проверяет всю checkpoint table. Если Retention + затронул активную checkpoint Trade, проверка завершается ошибкой и + PostgreSQL откатывает удаление данных, registry и изменение FK. + +Partition Manager сначала блокирует parent `market_data.trades` в +`SHARE ROW EXCLUSIVE`, затем default Trade partition и checkpoint table. +Этот порядок согласован с реализованной атомарной записью Trade → +checkpoint и исключает взаимную блокировку с writer в существующей +месячной partition. + +Попытка удалить активную checkpoint Trade откатывает всю retention- +операцию. Автоматическое удаление или откат checkpoint запрещены. + +--- + +## 13. Feature flag и lifecycle + +Отдельный checkpoint feature flag не вводится. + +```text +MARKET_DATA_STORAGE_ENABLED=false + → прежний полностью in-memory Runtime + +MARKET_DATA_STORAGE_ENABLED=true + → durable Trades + Persistent Checkpoint + Startup Recovery +``` + +Таким образом persistent checkpoint нельзя включить без durable Market +Data. Существующий Storage flag остаётся выключенным по умолчанию. + +Composition только строит зависимости. Checkpoint I/O начинается после +открытия pool и migrations. Shutdown сохраняет существующий порядок: + +```text +Trade Runtime stop + ↓ +await all persistence/recovery workers + ↓ +PostgreSQL pool close +``` + +Checkpoint не создаёт Scheduler, periodic flush или fire-and-forget +задачи. Каждая принятая Trade фиксируется inline до in-memory checkpoint. + +--- + +## 14. Ошибки и конкурентность + +Фатальными являются: + +- checkpoint с отсутствующей durable Trade; +- несовпадающая checkpoint identity или Canonical payload; +- stale expected checkpoint; +- ошибка атомарной Trade/checkpoint transaction; +- неоднозначный signed rollover-order; +- неуспешная hydration; +- ошибка Startup Recovery; +- ACK timeout или negative ACK. + +Повторная загрузка checkpoint является read-only и идемпотентна. +Повтор атомарного commit после неоднозначного сетевого результата должен +распознавать уже зафиксированную ту же candidate identity. + +Build предполагает одного активного Trade Runtime для `(venue, symbol)`. +Optimistic checkpoint conflict не даёт второму экземпляру незаметно +повредить состояние. Distributed lease и автоматический failover не +входят в scope. + +--- + +## 15. Критерии приёмки + +1. Storage contracts не импортируют Runtime, Recovery или Bootstrap. +2. Checkpoint table имеет точный ключ `(venue, symbol)`. +3. Checkpoint ссылается на durable Trade identity. +4. Schema запрещает пустые identity fields и неверные версии. +5. Migration остаётся versioned, атомарной и идемпотентной. +6. Accepted Trade и checkpoint фиксируются одной transaction. +7. Ошибка checkpoint откатывает новую Trade. +8. Duplicate provenance не продвигает checkpoint. +9. Stale concurrent writer получает явный conflict. +10. Hydration восстанавливает checkpoint и deduplication tail. +11. Rollover boundaries восстанавливаются в правильном порядке. +12. Повреждённый persistent checkpoint не скрывается. +13. Startup Recovery заканчивается до buffered Live processing. +14. Cancellation ожидает уже начатые blocking workers. +15. Retention не удаляет активную checkpoint Trade. +16. Composition не выполняет PostgreSQL или network I/O. +17. Runtime полностью останавливается до pool close. +18. После тестов не остаются asyncio tasks, threads или connections. +19. Целевые, integration и полная regression проходят успешно. + +--- + +## 16. Test strategy + +### Unit + +- value object и runtime-checkable Protocol; +- schema SQL и migration order; +- repository load/commit/CAS и полный rollback; +- hydration и bounded deduplication tail; +- no-checkpoint и first-adoption policy; +- ACK timeout, buffering, ordering и cancellation. + +### PostgreSQL integration + +- реальный foreign key и transactional commit; +- concurrent first insert и stale update; +- crash boundaries до и после commit; +- restart с той же disposable database; +- retention active-checkpoint protection; +- отсутствие оставшихся pool connections. + +### Runtime integration + +- первый процесс сохраняет Live Trades и завершается; +- второй процесс восстанавливает checkpoint и deduplication tail; +- REST заполняет downtime gap; +- buffered Live Trades обрабатываются после Recovery; +- duplicate boundary обновляет provenance один раз; +- failure не открывает live gate и не оставляет owned tasks. + +--- + +## 17. Вне scope + +- общие Historical Queries и Data Access Layer; +- Replay API и replay clocks; +- persistent Quotes/Candles consumers; +- автоматический Retention Scheduler; +- distributed runtime lease, leader election и HA failover; +- бесконечный REST retry/backoff; +- восстановление истории без известной durable начальной границы. + +Эти обязанности относятся к следующим Build либо отдельным production +инициативам. + +--- + +## 18. Архитектурные решения + +### ADR-060.28-001 — Durable Trades являются источником рыночных фактов + +**Статус:** Accepted + +Persistent checkpoint хранит identity и проверяется по Canonical Trade +history. Он не заменяет Market Data Storage. + +### ADR-060.28-002 — Trade и persistent checkpoint фиксируются атомарно + +**Статус:** Accepted + +Accepted Trade и её checkpoint принадлежат одной PostgreSQL transaction. +In-memory checkpoint продвигается только после commit. + +### ADR-060.28-003 — Consistency остаётся владельцем checkpoint + +**Статус:** Accepted + +Storage хранит operational pointer, но решение о принятии Trade и +продвижении состояния остаётся в Consistency Layer. + +### ADR-060.28-004 — Startup восстанавливает deduplication tail + +**Статус:** Accepted + +Восстановление только `last_trade` недостаточно для безопасной общей +границы REST и Live. + +### ADR-060.28-005 — Startup Recovery завершается до buffered Live + +**Статус:** Accepted + +Подписка подтверждается ACK, затем REST Recovery обрабатывается при +закрытом общем gate и только после этого разрешаются Live Trades. + +### ADR-060.28-006 — Retention не удаляет активную точку восстановления + +**Статус:** Accepted + +Удаление checkpoint Trade запрещено ссылочной целостностью. Во время +partition maintenance Retention Service восстанавливает и повторно +проверяет тот же FK до commit; ошибка откатывает всю transaction. + +### ADR-060.28-007 — Отдельный checkpoint feature flag не вводится + +**Статус:** Accepted + +Persistent Checkpoint является частью явно включённого Market Data +Storage и не может работать без durable Trades. + +### ADR-060.28-008 — Optimistic conflict защищает от второго writer + +**Статус:** Accepted + +Checkpoint update проверяет ожидаемую previous identity. Полноценный +distributed lease не входит в Build. + +--- + +## 19. Verification evidence + +Результаты реализации 060.28.0–060.28.1 на 2026-08-01: + +```text +Checkpoint contracts and migration unit target: 51 passed +Expanded Storage unit target: 255 passed +Targeted PostgreSQL schema/partition/retention: 12 passed +Full PostgreSQL Storage integration: 20 passed +git diff --check: clean +``` + +PostgreSQL 16 запускался в одноразовом локальном контейнере. После +проверки foreign key, idempotent migrations, переноса checkpoint Trade +между default/monthly partitions и Retention rollback контейнер +остановлен и удалён. Повторный read-only review findings не выявил. +Checkpoint repository, Runtime и Bootstrap в этих подэтапах не +подключались. + +Дополнительные результаты реализации 060.28.2: + +```text +Checkpoint repository unit target: 69 passed +Expanded Storage unit target: 275 passed +Real PostgreSQL checkpoint repository target: 10 passed +Full PostgreSQL Storage integration: 30 passed +git diff --check: clean +``` + +Реальный PostgreSQL подтвердил атомарный rollback Trade при checkpoint +failure/conflict, идемпотентный повтор после commit, два конкурирующих +первых writer, два конкурирующих CAS update и границы +`INT32_MAX → INT32_MIN` / `-1 → 0`. PostgreSQL-контейнер после проверки +остановлен и удалён. Consistency, Runtime и Bootstrap не изменялись. + +Финальный review дополнительно подтвердил порядок при убывающем +`executed_at` и растущем Trade ID, а также исключение старого полного +32-битного цикла при повторе того же raw ID. Повторный review findings +не выявил; 060.28.2 принят. + +Результаты реализации 060.28.3: + +```text +Targeted Consistency/Storage unit: 104 passed +Full unit regression: 2169 passed +Targeted PostgreSQL Consistency integration: 3 passed +Existing PostgreSQL Runtime persistence: 4 passed +Full PostgreSQL Storage integration: 33 passed +git diff --check: clean +``` + +Реальный PostgreSQL подтвердил синхронное продвижение persistent и +in-memory checkpoint, сохранение provenance дубликата без изменения +revision и полный rollback candidate Trade при CAS-конфликте. Отдельный +приёмочный read-only review findings не выявил; 060.28.3 принят. + +Результаты реализации 060.28.4: + +```text +Targeted hydration/repository unit: 140 passed +Full unit regression: 2209 passed +Targeted PostgreSQL adoption/hydration: 18 passed +Full PostgreSQL Storage integration: 41 passed +git diff --check: clean +``` + +Unit-тесты подтвердили strict hydration, восстановление bounded dedup +window, атомарную публикацию нескольких символов, ошибки каждого шага и +границы `INT32_MAX → INT32_MIN` / `-1 → 0`. Реальный PostgreSQL +подтвердил неизменность provenance и observation timestamps при +first-adoption, идемпотентный restart без роста revision, восстановление +дубликатов после restart и последующее атомарное продвижение revision +`1 → 2`. Дополнительные regression-тесты подтвердили запрет falsey +non-tuple ответа Storage без fallback к пустому state. Повторный +read-only review findings не выявил; 060.28.4 принят. Runtime, +Bootstrap, settings и Startup Recovery не изменялись. + +Результаты реализации 060.28.5: + +```text +Targeted Runtime/Checkpoint unit: 257 passed +Loopback Runtime integration: 13 passed +Critical startup ordering repeated: 10/10 +Full standard regression: 2263 passed +Deselected integration/stress/live tests: 59 +git diff --check: clean +``` + +Проверены Hydration до первого network I/O, положительный и ошибочный +ACK, ограниченный FIFO ранних Live-сообщений, строгий порядок +Recovery → buffered Live, частичный startup, cancellation blocking +workers, cleanup owned tasks и legacy-путь без persistent checkpoint. +Отдельные regression-сценарии подтвердили единый uppercase-ключ для +lowercase symbols и безопасное схлопывание case-variant duplicates до +Hydration. Два независимых повторных read-only review findings не +выявили; 060.28.5 принят. + +Результаты реализации 060.28.6: + +```text +Targeted PostgreSQL Application lifecycle: 1 passed +Full PostgreSQL Storage integration: 41 passed +Full integration with PostgreSQL: 54 passed +Full standard regression: 2276 passed +Deselected integration/stress/live tests: 59 +git diff --check: clean +``` + +Подтверждены единый persistent dependency graph, отсутствие I/O и +фоновых задач при Bootstrap-сборке, строгий fail-fast для неполной +пары sink/checkpoint storage, Hydration и Startup Recovery до Live +processing, а также закрытие PostgreSQL pool только после полной +остановки Runtime. Повторная cancellation не прерывает cleanup. +Исправленный реальный Application-сценарий дополнительно подтвердил +отсутствие оставшихся owned tasks и соединений PostgreSQL. Два +повторных read-only review findings не выявили; 060.28.6 принят. + +Результаты реализации 060.28.7: + +```text +PostgreSQL test-support unit: 52 passed +Persistent restart/failure scenarios: 9 passed +Network-order barriers repeated: 10 x 2 passed +Full PostgreSQL Storage integration: 50 passed +Full integration with PostgreSQL: 63 passed +PostgreSQL opt-out isolation: 50 skipped +Full standard regression: 2305 passed +Deselected integration/stress/live tests: 68 +git diff --check: clean +``` + +Реальный PostgreSQL подтвердил восстановление checkpoint и bounded +deduplication tail после перезапуска, безопасную first-adoption, +атомарный rollback, повтор после persistence failure и завершение уже +начатого blocking worker при cancellation. Два новых barrier-сценария +отдельно доказали, что Hydration и first-adoption завершаются до первого +WebSocket/REST I/O; каждый сценарий стабильно прошёл десять повторов. +Opt-out запуск пропустил все 50 PostgreSQL-тестов без подключения к базе. +Одноразовый контейнер после проверки остановлен и удалён. Финальный +приёмочный review findings не выявил; 060.28.7 принят. + +Финальные результаты 060.28.8 после исправления lock order: + +```text +Partition lock-order unit target: 21 passed +Restart/failure scenarios repeated: 10 x 9 passed +Partition/checkpoint race repeated: 10 x 1 passed +Full PostgreSQL Storage integration: 51 passed +Full integration with PostgreSQL: 64 passed +PostgreSQL opt-out isolation: 51 skipped +Fixed stress target: 3 passed, 1 deselected +Full standard regression: 2305 passed +Deselected integration/stress/live tests: 69 +git diff --check: clean +Untracked files whitespace check: clean +``` + +В integration и stress наборах `ResourceWarning` считался ошибкой. +Одноразовый PostgreSQL 16 после проверки остановлен и удалён. Итоговый +review полного Build подтвердил архитектурные границы, атомарность, +CAS/rollover, Startup Recovery, lifecycle/cancellation, Retention и +единый порядок блокировок; открытых findings не осталось. + +--- + +## 20. Следующий Build + +Build 060.28 завершён и принят. Следующий этап — Build 060.29, Market +Data Access and Replay. diff --git a/docs/roadmap/master-roadmap.md b/docs/roadmap/master-roadmap.md index a1896b9..5ce35b6 100644 --- a/docs/roadmap/master-roadmap.md +++ b/docs/roadmap/master-roadmap.md @@ -6,10 +6,10 @@ |---|---| | Тип | Master Delivery Roadmap | | Статус | Active | -| Версия | 2.1 | +| Версия | 2.2 | | Дата актуализации | 2026-08-01 | -| Текущий завершённый Build | 060.27 | -| Текущий Build | 060.28 — Planned | +| Текущий завершённый Build | 060.28 | +| Текущий Build | 060.29 — Planned | --- @@ -42,26 +42,26 @@ test evidence находятся в документах конкретных Bu ```text Market Data Acquisition ↓ -Persistent Market Data Storage завершён +Persistent Checkpoint and Startup Recovery завершён ↓ -Build 060.27 +Build 060.28 ↓ -Completed → следующий Build 060.28 +Completed → следующий Build 060.29 ``` -Build 060.27 завершён и принят: +Build 060.28 завершён и принят: -- реализованы versioned PostgreSQL migrations и управляемый pool; -- созданы Canonical repositories для Trades, Quotes и Candles; -- реализованы monthly partitions, provenance и opt-in retention; -- Trade Runtime сохраняет Live и Recovery Trades до checkpoint; -- Bootstrap управляет порядком Storage startup и shutdown; -- real PostgreSQL integration и финальная регрессия приняты. +- persistent checkpoint подтверждается Canonical Trade history; +- Trade и checkpoint продвигаются одной PostgreSQL transaction; +- при запуске восстанавливаются checkpoint и deduplication tail; +- Startup Recovery завершается до buffered Live processing; +- Bootstrap и cancellation сохраняют строгий lifecycle pool/Runtime; +- restart/failure, concurrency и финальная регрессия приняты. Подробности: ```text -docs/migrations/build_060_27_architecture.md +docs/migrations/build_060_28.md ``` --- @@ -82,6 +82,7 @@ docs/migrations/build_060_27_architecture.md | 060.25 | Production Runtime Integration | Completed | | 060.26 | Integration and Regression | Completed | | 060.27 | Persistent Market Data Storage | Completed | +| 060.28 | Persistent Checkpoint and Startup Recovery | Completed | ## Build 060.26 — Integration and Regression @@ -122,7 +123,7 @@ Build не должен превращать сетевые сценарии в ### Build 060.28 — Persistent Checkpoint and Startup Recovery -**Статус:** Planned +**Статус:** Completed Назначение: @@ -136,6 +137,12 @@ Build не должен превращать сетевые сценарии в Persistent checkpoint не заменяет Market Data Storage и не должен считаться более достоверным, чем подтверждённая сохранённая история. +Результат: после перезапуска Trade Stream восстанавливает подтверждённый +checkpoint и bounded deduplication tail, заполняет downtime gap через +REST и только затем продолжает buffered Live processing. + +Подробный итог: `docs/migrations/build_060_28.md`. + ### Build 060.29 — Market Data Access and Replay **Статус:** Planned @@ -1732,6 +1739,6 @@ read-only архитектурного анализа. Старый ориент Актуальная контрольная точка: ```text -Завершён: Build 060.27 — Persistent Market Data Storage -Следующий: Build 060.28 — Persistent Checkpoint and Startup Recovery +Завершён: Build 060.28 — Persistent Checkpoint and Startup Recovery +Следующий: Build 060.29 — Market Data Access and Replay ```