diff --git a/.vscode/settings.json b/.vscode/settings.json index 38007ad..88fef83 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,6 @@ { "python.defaultInterpreterPath": "app/.venv/bin/python", - "python-envs.defaultEnvManager": "ms-python.python:system" + "python-envs.defaultEnvManager": "ms-python.python:system", + "python.analysis.typeCheckingMode": "standard", + "python.analysis.diagnosticMode": "workspace" } diff --git a/app/requirements-dev.txt b/app/requirements-dev.txt new file mode 100644 index 0000000..3a717f0 --- /dev/null +++ b/app/requirements-dev.txt @@ -0,0 +1,4 @@ +-r requirements.txt + +pytest==9.1.1 +pyright[nodejs]==1.1.411 diff --git a/app/src/market_data/access/__init__.py b/app/src/market_data/access/__init__.py new file mode 100644 index 0000000..b166b39 --- /dev/null +++ b/app/src/market_data/access/__init__.py @@ -0,0 +1,86 @@ +from src.market_data.access.contracts import ( + CandleRevisionHistoryReaderProtocol, + MarketDataHistoricalAccessProtocol, + QuoteHistoryReaderProtocol, + TradeHistoryReaderProtocol, +) +from src.market_data.access.exceptions import ( + MarketDataAccessError, + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, + MarketDataAccessValidationError, + MarketDataCursorError, +) +from src.market_data.access.market_data_historical_access import ( + MarketDataHistoricalAccess, +) +from src.market_data.access.models import ( + HISTORY_CURSOR_VERSION, + HISTORY_PAGE_LIMIT_DEFAULT, + HISTORY_PAGE_LIMIT_MAX, + CandleRevisionHistoryCursor, + CandleRevisionHistoryPage, + CandleRevisionHistoryQuery, + CandleRevisionHistoryRecord, + HistoricalTimeRange, + HistoryPage, + HistoryQuery, + HistoryRecord, + QuoteHistoryCursor, + QuoteHistoryPage, + QuoteHistoryQuery, + QuoteHistoryRecord, + TradeHistoryCursor, + TradeHistoryPage, + TradeHistoryQuery, + TradeHistoryRecord, +) +from src.market_data.access.postgres_candle_revision_history_repository import ( + PostgresCandleRevisionHistoryRepository, +) +from src.market_data.access.postgres_history_support import ( + PostgresHistoryConnectionProvider, +) +from src.market_data.access.postgres_quote_history_repository import ( + PostgresQuoteHistoryRepository, +) +from src.market_data.access.postgres_trade_history_repository import ( + PostgresTradeHistoryRepository, +) + + +__all__ = ( + "HISTORY_CURSOR_VERSION", + "HISTORY_PAGE_LIMIT_DEFAULT", + "HISTORY_PAGE_LIMIT_MAX", + "CandleRevisionHistoryCursor", + "CandleRevisionHistoryPage", + "CandleRevisionHistoryQuery", + "CandleRevisionHistoryReaderProtocol", + "CandleRevisionHistoryRecord", + "HistoricalTimeRange", + "HistoryPage", + "HistoryQuery", + "HistoryRecord", + "MarketDataAccessError", + "MarketDataAccessIntegrityError", + "MarketDataAccessOperationError", + "MarketDataAccessValidationError", + "MarketDataCursorError", + "MarketDataHistoricalAccess", + "MarketDataHistoricalAccessProtocol", + "PostgresCandleRevisionHistoryRepository", + "PostgresHistoryConnectionProvider", + "PostgresQuoteHistoryRepository", + "PostgresTradeHistoryRepository", + "QuoteHistoryCursor", + "QuoteHistoryPage", + "QuoteHistoryQuery", + "QuoteHistoryReaderProtocol", + "QuoteHistoryRecord", + "TradeHistoryCursor", + "TradeHistoryPage", + "TradeHistoryQuery", + "TradeHistoryReaderProtocol", + "TradeHistoryRecord", +) diff --git a/app/src/market_data/access/contracts.py b/app/src/market_data/access/contracts.py new file mode 100644 index 0000000..a912e1e --- /dev/null +++ b/app/src/market_data/access/contracts.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from src.market_data.access.models import ( + CandleRevisionHistoryPage, + CandleRevisionHistoryQuery, + QuoteHistoryPage, + QuoteHistoryQuery, + TradeHistoryPage, + TradeHistoryQuery, +) + + +@runtime_checkable +class TradeHistoryReaderProtocol(Protocol): + """Граница чтения одной страницы Canonical Trade History.""" + + def query_trades( + self, + query: TradeHistoryQuery, + ) -> TradeHistoryPage: + ... + + +@runtime_checkable +class QuoteHistoryReaderProtocol(Protocol): + """Граница чтения одной страницы Canonical Quote History.""" + + def query_quotes( + self, + query: QuoteHistoryQuery, + ) -> QuoteHistoryPage: + ... + + +@runtime_checkable +class CandleRevisionHistoryReaderProtocol(Protocol): + """Граница чтения одной страницы Canonical Candle revisions.""" + + def query_candle_revisions( + self, + query: CandleRevisionHistoryQuery, + ) -> CandleRevisionHistoryPage: + ... + + +@runtime_checkable +class MarketDataHistoricalAccessProtocol( + TradeHistoryReaderProtocol, + QuoteHistoryReaderProtocol, + CandleRevisionHistoryReaderProtocol, + Protocol, +): + """Объединённая DB-neutral граница Historical Access.""" diff --git a/app/src/market_data/access/exceptions.py b/app/src/market_data/access/exceptions.py new file mode 100644 index 0000000..8b0e35d --- /dev/null +++ b/app/src/market_data/access/exceptions.py @@ -0,0 +1,21 @@ +from __future__ import annotations + + +class MarketDataAccessError(Exception): + """Базовая ошибка доступа к историческим рыночным данным.""" + + +class MarketDataAccessValidationError(MarketDataAccessError): + """Недопустимый запрос или значение на публичной границе чтения.""" + + +class MarketDataCursorError(MarketDataAccessValidationError): + """Курсор повреждён либо принадлежит другому запросу.""" + + +class MarketDataAccessIntegrityError(MarketDataAccessError): + """Сохранённая строка не соответствует Canonical-контракту.""" + + +class MarketDataAccessOperationError(MarketDataAccessError): + """Ошибка backend-операции Historical Access.""" diff --git a/app/src/market_data/access/market_data_historical_access.py b/app/src/market_data/access/market_data_historical_access.py new file mode 100644 index 0000000..31b2d92 --- /dev/null +++ b/app/src/market_data/access/market_data_historical_access.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from src.market_data.access.contracts import ( + CandleRevisionHistoryReaderProtocol, + QuoteHistoryReaderProtocol, + TradeHistoryReaderProtocol, +) +from src.market_data.access.models import ( + CandleRevisionHistoryPage, + CandleRevisionHistoryQuery, + QuoteHistoryPage, + QuoteHistoryQuery, + TradeHistoryPage, + TradeHistoryQuery, +) + + +class MarketDataHistoricalAccess: + """Независимый от базы данных фасад трёх модулей чтения истории.""" + + __slots__ = ( + "_trade_reader", + "_quote_reader", + "_candle_revision_reader", + ) + + def __init__( + self, + *, + trade_reader: TradeHistoryReaderProtocol, + quote_reader: QuoteHistoryReaderProtocol, + candle_revision_reader: CandleRevisionHistoryReaderProtocol, + ) -> None: + if not isinstance(trade_reader, TradeHistoryReaderProtocol): + raise TypeError( + "trade_reader must implement TradeHistoryReaderProtocol" + ) + + if not isinstance(quote_reader, QuoteHistoryReaderProtocol): + raise TypeError( + "quote_reader must implement QuoteHistoryReaderProtocol" + ) + + if not isinstance( + candle_revision_reader, + CandleRevisionHistoryReaderProtocol, + ): + raise TypeError( + "candle_revision_reader must implement " + "CandleRevisionHistoryReaderProtocol" + ) + + self._trade_reader = trade_reader + self._quote_reader = quote_reader + self._candle_revision_reader = candle_revision_reader + + def query_trades( + self, + query: TradeHistoryQuery, + ) -> TradeHistoryPage: + """Передать запрос истории сделок соответствующему модулю.""" + return self._trade_reader.query_trades(query) + + def query_quotes( + self, + query: QuoteHistoryQuery, + ) -> QuoteHistoryPage: + """Передать запрос истории котировок соответствующему модулю.""" + return self._quote_reader.query_quotes(query) + + def query_candle_revisions( + self, + query: CandleRevisionHistoryQuery, + ) -> CandleRevisionHistoryPage: + """Передать запрос истории свечей соответствующему модулю.""" + return self._candle_revision_reader.query_candle_revisions(query) diff --git a/app/src/market_data/access/models.py b/app/src/market_data/access/models.py new file mode 100644 index 0000000..d8157af --- /dev/null +++ b/app/src/market_data/access/models.py @@ -0,0 +1,901 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Callable + +from src.market_data.acquisition.models.candle import Candle +from src.market_data.acquisition.models.quote import Quote +from src.market_data.acquisition.models.trade import Trade +from src.market_data.acquisition.symbols import normalize_symbol + + +HISTORY_CURSOR_VERSION = 1 +HISTORY_PAGE_LIMIT_DEFAULT = 500 +HISTORY_PAGE_LIMIT_MAX = 1_000 + + +def _normalize_non_empty_text( + value: object, + *, + field_name: str, + uppercase: bool = False, +) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be a string") + + normalized = value.strip() + + if not normalized: + raise ValueError(f"{field_name} must not be empty") + + return normalized.upper() if uppercase else normalized + + +def _normalize_symbol(value: object, *, field_name: str) -> str: + raw_symbol = _normalize_non_empty_text( + value, + field_name=field_name, + ) + normalized = normalize_symbol(raw_symbol) + + if not normalized: + raise ValueError(f"{field_name} must not be empty") + + return normalized + + +def _normalize_utc_datetime( + value: object, + *, + field_name: str, +) -> datetime: + if not isinstance(value, datetime): + raise TypeError(f"{field_name} must be a datetime") + + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{field_name} must contain timezone") + + return value.astimezone(timezone.utc) + + +def _normalize_positive_integer( + value: object, + *, + field_name: str, + maximum: int | None = None, +) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{field_name} must be an integer") + + if value <= 0: + raise ValueError(f"{field_name} must be positive") + + if maximum is not None and value > maximum: + raise ValueError( + f"{field_name} must not exceed {maximum}" + ) + + return value + + +def _normalize_observation_sources( + value: object, + *, + primary_source: object, +) -> tuple[str, ...]: + if not isinstance(value, tuple): + raise TypeError("observation_sources must be a tuple") + + if not value: + raise ValueError("observation_sources must not be empty") + + normalized = tuple( + _normalize_non_empty_text( + source, + field_name="observation_sources item", + ) + for source in value + ) + + if len(set(normalized)) != len(normalized): + raise ValueError("observation_sources must be unique") + + normalized_primary = _normalize_non_empty_text( + primary_source, + field_name="payload.source", + ) + + if normalized[0] != normalized_primary: + raise ValueError( + "first observation source must match payload.source" + ) + + return normalized + + +def _validate_canonical_symbol_and_source( + *, + symbol: object, + source: object, +) -> None: + normalized_symbol = _normalize_symbol( + symbol, + field_name="payload.symbol", + ) + + if normalized_symbol != symbol: + raise ValueError("payload.symbol must be canonical") + + normalized_source = _normalize_non_empty_text( + source, + field_name="payload.source", + ) + + if normalized_source != source: + raise ValueError("payload.source must be canonical") + + +@dataclass(frozen=True, slots=True) +class HistoricalTimeRange: + """Полуоткрытый UTC-диапазон исторического запроса ``[start, end)``.""" + + start_time: datetime + end_time: datetime + + def __post_init__(self) -> None: + normalized_start = _normalize_utc_datetime( + self.start_time, + field_name="start_time", + ) + normalized_end = _normalize_utc_datetime( + self.end_time, + field_name="end_time", + ) + + if normalized_start >= normalized_end: + raise ValueError("start_time must be earlier than end_time") + + object.__setattr__(self, "start_time", normalized_start) + object.__setattr__(self, "end_time", normalized_end) + + def contains(self, instant: datetime) -> bool: + """Проверить принадлежность времени полуоткрытому диапазону.""" + normalized = _normalize_utc_datetime( + instant, + field_name="instant", + ) + return self.start_time <= normalized < self.end_time + + +@dataclass(frozen=True, slots=True) +class TradeHistoryRecord: + """Canonical Trade и неизменяемые технические данные его хранения.""" + + venue: str + trade: Trade + first_observed_at: datetime + last_observed_at: datetime + observation_sources: tuple[str, ...] + replay_sequence: int + canonical_schema_version: int = 1 + + def __post_init__(self) -> None: + normalized_venue = _normalize_non_empty_text( + self.venue, + field_name="venue", + ) + + if type(self.trade) is not Trade: + raise TypeError("trade must be a Canonical Trade") + + _validate_canonical_symbol_and_source( + symbol=self.trade.symbol, + source=self.trade.source, + ) + _normalize_utc_datetime( + self.trade.executed_at, + field_name="trade.executed_at", + ) + normalized_first = _normalize_utc_datetime( + self.first_observed_at, + field_name="first_observed_at", + ) + normalized_last = _normalize_utc_datetime( + self.last_observed_at, + field_name="last_observed_at", + ) + + if normalized_last < normalized_first: + raise ValueError( + "last_observed_at must not precede first_observed_at" + ) + + normalized_sources = _normalize_observation_sources( + self.observation_sources, + primary_source=self.trade.source, + ) + normalized_sequence = _normalize_positive_integer( + self.replay_sequence, + field_name="replay_sequence", + ) + normalized_version = _normalize_positive_integer( + self.canonical_schema_version, + field_name="canonical_schema_version", + ) + + object.__setattr__(self, "venue", normalized_venue) + object.__setattr__( + self, + "first_observed_at", + normalized_first, + ) + object.__setattr__( + self, + "last_observed_at", + normalized_last, + ) + object.__setattr__( + self, + "observation_sources", + normalized_sources, + ) + object.__setattr__( + self, + "replay_sequence", + normalized_sequence, + ) + object.__setattr__( + self, + "canonical_schema_version", + normalized_version, + ) + + @property + def symbol(self) -> str: + return self.trade.symbol + + @property + def event_time(self) -> datetime: + return self.trade.executed_at.astimezone(timezone.utc) + + @property + def order_key(self) -> tuple[datetime, int]: + return (self.event_time, self.replay_sequence) + + +@dataclass(frozen=True, slots=True) +class QuoteHistoryRecord: + """Canonical Quote и неизменяемые технические данные его хранения.""" + + venue: str + quote: Quote + observation_sources: tuple[str, ...] + replay_sequence: int + canonical_schema_version: int = 1 + + def __post_init__(self) -> None: + normalized_venue = _normalize_non_empty_text( + self.venue, + field_name="venue", + ) + + if type(self.quote) is not Quote: + raise TypeError("quote must be a Canonical Quote") + + _validate_canonical_symbol_and_source( + symbol=self.quote.symbol, + source=self.quote.source, + ) + _normalize_utc_datetime( + self.quote.received_at, + field_name="quote.received_at", + ) + + if self.quote.exchange_timestamp is not None: + _normalize_utc_datetime( + self.quote.exchange_timestamp, + field_name="quote.exchange_timestamp", + ) + + normalized_sources = _normalize_observation_sources( + self.observation_sources, + primary_source=self.quote.source, + ) + normalized_sequence = _normalize_positive_integer( + self.replay_sequence, + field_name="replay_sequence", + ) + normalized_version = _normalize_positive_integer( + self.canonical_schema_version, + field_name="canonical_schema_version", + ) + + object.__setattr__(self, "venue", normalized_venue) + object.__setattr__( + self, + "observation_sources", + normalized_sources, + ) + object.__setattr__( + self, + "replay_sequence", + normalized_sequence, + ) + object.__setattr__( + self, + "canonical_schema_version", + normalized_version, + ) + + @property + def symbol(self) -> str: + return self.quote.symbol + + @property + def event_time(self) -> datetime: + return self.quote.received_at.astimezone(timezone.utc) + + @property + def order_key(self) -> tuple[datetime, int]: + return (self.event_time, self.replay_sequence) + + +@dataclass(frozen=True, slots=True) +class CandleRevisionHistoryRecord: + """Canonical Candle и метаданные одной наблюдавшейся ревизии.""" + + venue: str + candle: Candle + observed_at: datetime + is_final: bool + observation_sources: tuple[str, ...] + replay_sequence: int + canonical_schema_version: int = 1 + + def __post_init__(self) -> None: + normalized_venue = _normalize_non_empty_text( + self.venue, + field_name="venue", + ) + + if type(self.candle) is not Candle: + raise TypeError("candle must be a Canonical Candle") + + _validate_canonical_symbol_and_source( + symbol=self.candle.symbol, + source=self.candle.source, + ) + normalized_interval = _normalize_non_empty_text( + self.candle.interval, + field_name="candle.interval", + ) + + if normalized_interval != self.candle.interval: + raise ValueError("candle.interval must be canonical") + normalized_open_time = _normalize_utc_datetime( + self.candle.open_time, + field_name="candle.open_time", + ) + normalized_observed_at = _normalize_utc_datetime( + self.observed_at, + field_name="observed_at", + ) + + if normalized_observed_at < normalized_open_time: + raise ValueError("observed_at must not precede candle.open_time") + + if not isinstance(self.is_final, bool): + raise TypeError("is_final must be a boolean") + + normalized_sources = _normalize_observation_sources( + self.observation_sources, + primary_source=self.candle.source, + ) + normalized_sequence = _normalize_positive_integer( + self.replay_sequence, + field_name="replay_sequence", + ) + normalized_version = _normalize_positive_integer( + self.canonical_schema_version, + field_name="canonical_schema_version", + ) + + object.__setattr__(self, "venue", normalized_venue) + object.__setattr__( + self, + "observed_at", + normalized_observed_at, + ) + object.__setattr__( + self, + "observation_sources", + normalized_sources, + ) + object.__setattr__( + self, + "replay_sequence", + normalized_sequence, + ) + object.__setattr__( + self, + "canonical_schema_version", + normalized_version, + ) + + @property + def symbol(self) -> str: + return self.candle.symbol + + @property + def interval(self) -> str: + return self.candle.interval + + @property + def event_time(self) -> datetime: + """Вернуть ось Historical Query — время открытия свечи.""" + return self.candle.open_time.astimezone(timezone.utc) + + @property + def replay_at(self) -> datetime: + """Вернуть ось Replay — время наблюдения конкретной ревизии.""" + return self.observed_at + + @property + def order_key(self) -> tuple[datetime, int]: + return (self.event_time, self.replay_sequence) + + +@dataclass(frozen=True, slots=True) +class TradeHistoryCursor: + """Структурированный keyset-курсор Trade History.""" + + venue: str + symbol: str + time_range: HistoricalTimeRange + executed_at: datetime + replay_sequence: int + version: int = HISTORY_CURSOR_VERSION + + def __post_init__(self) -> None: + normalized_venue, normalized_symbol = _normalize_cursor_scope( + venue=self.venue, + symbol=self.symbol, + time_range=self.time_range, + ) + normalized_time = _normalize_cursor_position( + cursor_time=self.executed_at, + replay_sequence=self.replay_sequence, + time_range=self.time_range, + ) + _validate_cursor_version(self.version) + + object.__setattr__(self, "venue", normalized_venue) + object.__setattr__(self, "symbol", normalized_symbol) + object.__setattr__(self, "executed_at", normalized_time) + + +@dataclass(frozen=True, slots=True) +class QuoteHistoryCursor: + """Структурированный keyset-курсор Quote History.""" + + venue: str + symbol: str + time_range: HistoricalTimeRange + received_at: datetime + replay_sequence: int + version: int = HISTORY_CURSOR_VERSION + + def __post_init__(self) -> None: + normalized_venue, normalized_symbol = _normalize_cursor_scope( + venue=self.venue, + symbol=self.symbol, + time_range=self.time_range, + ) + normalized_time = _normalize_cursor_position( + cursor_time=self.received_at, + replay_sequence=self.replay_sequence, + time_range=self.time_range, + ) + _validate_cursor_version(self.version) + + object.__setattr__(self, "venue", normalized_venue) + object.__setattr__(self, "symbol", normalized_symbol) + object.__setattr__(self, "received_at", normalized_time) + + +@dataclass(frozen=True, slots=True) +class CandleRevisionHistoryCursor: + """Структурированный keyset-курсор Candle Revision History.""" + + venue: str + symbol: str + interval: str + time_range: HistoricalTimeRange + open_time: datetime + replay_sequence: int + version: int = HISTORY_CURSOR_VERSION + + def __post_init__(self) -> None: + normalized_venue, normalized_symbol = _normalize_cursor_scope( + venue=self.venue, + symbol=self.symbol, + time_range=self.time_range, + ) + normalized_interval = _normalize_non_empty_text( + self.interval, + field_name="interval", + ) + normalized_time = _normalize_cursor_position( + cursor_time=self.open_time, + replay_sequence=self.replay_sequence, + time_range=self.time_range, + ) + _validate_cursor_version(self.version) + + object.__setattr__(self, "venue", normalized_venue) + object.__setattr__(self, "symbol", normalized_symbol) + object.__setattr__(self, "interval", normalized_interval) + object.__setattr__(self, "open_time", normalized_time) + + +def _normalize_cursor_scope( + *, + venue: object, + symbol: object, + time_range: object, +) -> tuple[str, str]: + if type(time_range) is not HistoricalTimeRange: + raise TypeError("time_range must be HistoricalTimeRange") + + return ( + _normalize_non_empty_text(venue, field_name="venue"), + _normalize_symbol(symbol, field_name="symbol"), + ) + + +def _normalize_cursor_position( + *, + cursor_time: object, + replay_sequence: object, + time_range: HistoricalTimeRange, +) -> datetime: + normalized_time = _normalize_utc_datetime( + cursor_time, + field_name="cursor time", + ) + _normalize_positive_integer( + replay_sequence, + field_name="replay_sequence", + ) + + if not time_range.contains(normalized_time): + raise ValueError("cursor time must belong to query range") + + return normalized_time + + +def _validate_cursor_version(version: object) -> None: + if isinstance(version, bool) or not isinstance(version, int): + raise TypeError("version must be an integer") + + if version != HISTORY_CURSOR_VERSION: + raise ValueError("unsupported history cursor version") + + +@dataclass(frozen=True, slots=True) +class TradeHistoryQuery: + """Один forward-only запрос истории Trades.""" + + venue: str + symbol: str + time_range: HistoricalTimeRange + limit: int = HISTORY_PAGE_LIMIT_DEFAULT + cursor: TradeHistoryCursor | None = None + + def __post_init__(self) -> None: + venue, symbol = _normalize_query( + venue=self.venue, + symbol=self.symbol, + time_range=self.time_range, + limit=self.limit, + ) + _validate_query_cursor( + cursor=self.cursor, + cursor_type=TradeHistoryCursor, + venue=venue, + symbol=symbol, + time_range=self.time_range, + ) + object.__setattr__(self, "venue", venue) + object.__setattr__(self, "symbol", symbol) + + +@dataclass(frozen=True, slots=True) +class QuoteHistoryQuery: + """Один forward-only запрос истории Quotes.""" + + venue: str + symbol: str + time_range: HistoricalTimeRange + limit: int = HISTORY_PAGE_LIMIT_DEFAULT + cursor: QuoteHistoryCursor | None = None + + def __post_init__(self) -> None: + venue, symbol = _normalize_query( + venue=self.venue, + symbol=self.symbol, + time_range=self.time_range, + limit=self.limit, + ) + _validate_query_cursor( + cursor=self.cursor, + cursor_type=QuoteHistoryCursor, + venue=venue, + symbol=symbol, + time_range=self.time_range, + ) + object.__setattr__(self, "venue", venue) + object.__setattr__(self, "symbol", symbol) + + +@dataclass(frozen=True, slots=True) +class CandleRevisionHistoryQuery: + """Один forward-only запрос истории ревизий Candles.""" + + venue: str + symbol: str + interval: str + time_range: HistoricalTimeRange + limit: int = HISTORY_PAGE_LIMIT_DEFAULT + cursor: CandleRevisionHistoryCursor | None = None + + def __post_init__(self) -> None: + venue, symbol = _normalize_query( + venue=self.venue, + symbol=self.symbol, + time_range=self.time_range, + limit=self.limit, + ) + interval = _normalize_non_empty_text( + self.interval, + field_name="interval", + ) + _validate_query_cursor( + cursor=self.cursor, + cursor_type=CandleRevisionHistoryCursor, + venue=venue, + symbol=symbol, + time_range=self.time_range, + interval=interval, + ) + object.__setattr__(self, "venue", venue) + object.__setattr__(self, "symbol", symbol) + object.__setattr__(self, "interval", interval) + + +def _normalize_query( + *, + venue: object, + symbol: object, + time_range: object, + limit: object, +) -> tuple[str, str]: + if type(time_range) is not HistoricalTimeRange: + raise TypeError("time_range must be HistoricalTimeRange") + + _normalize_positive_integer( + limit, + field_name="limit", + maximum=HISTORY_PAGE_LIMIT_MAX, + ) + return ( + _normalize_non_empty_text(venue, field_name="venue"), + _normalize_symbol(symbol, field_name="symbol"), + ) + + +def _validate_query_cursor( + *, + cursor: object, + cursor_type: type[object], + venue: str, + symbol: str, + time_range: HistoricalTimeRange, + interval: str | None = None, +) -> None: + if cursor is None: + return + + if type(cursor) is not cursor_type: + raise TypeError(f"cursor must be {cursor_type.__name__} or None") + + if ( + getattr(cursor, "venue") != venue + or getattr(cursor, "symbol") != symbol + or getattr(cursor, "time_range") != time_range + ): + raise ValueError("cursor does not belong to query scope") + + if interval is not None and getattr(cursor, "interval") != interval: + raise ValueError("cursor does not belong to query interval") + + +@dataclass(frozen=True, slots=True) +class TradeHistoryPage: + """Одна типизированная страница Trade History.""" + + query: TradeHistoryQuery + items: tuple[TradeHistoryRecord, ...] + next_cursor: TradeHistoryCursor | None = None + + def __post_init__(self) -> None: + _validate_page( + query=self.query, + query_type=TradeHistoryQuery, + items=self.items, + item_type=TradeHistoryRecord, + next_cursor=self.next_cursor, + cursor_type=TradeHistoryCursor, + cursor_time_name="executed_at", + ) + + @property + def has_more(self) -> bool: + return self.next_cursor is not None + + +@dataclass(frozen=True, slots=True) +class QuoteHistoryPage: + """Одна типизированная страница Quote History.""" + + query: QuoteHistoryQuery + items: tuple[QuoteHistoryRecord, ...] + next_cursor: QuoteHistoryCursor | None = None + + def __post_init__(self) -> None: + _validate_page( + query=self.query, + query_type=QuoteHistoryQuery, + items=self.items, + item_type=QuoteHistoryRecord, + next_cursor=self.next_cursor, + cursor_type=QuoteHistoryCursor, + cursor_time_name="received_at", + ) + + @property + def has_more(self) -> bool: + return self.next_cursor is not None + + +@dataclass(frozen=True, slots=True) +class CandleRevisionHistoryPage: + """Одна типизированная страница Candle Revision History.""" + + query: CandleRevisionHistoryQuery + items: tuple[CandleRevisionHistoryRecord, ...] + next_cursor: CandleRevisionHistoryCursor | None = None + + def __post_init__(self) -> None: + _validate_page( + query=self.query, + query_type=CandleRevisionHistoryQuery, + items=self.items, + item_type=CandleRevisionHistoryRecord, + next_cursor=self.next_cursor, + cursor_type=CandleRevisionHistoryCursor, + cursor_time_name="open_time", + ) + + @property + def has_more(self) -> bool: + return self.next_cursor is not None + + +def _validate_page( + *, + query: object, + query_type: type[object], + items: object, + item_type: type[object], + next_cursor: object, + cursor_type: type[object], + cursor_time_name: str, +) -> None: + if type(query) is not query_type: + raise TypeError(f"query must be {query_type.__name__}") + + if type(items) is not tuple: + raise TypeError("items must be a tuple") + + if len(items) > getattr(query, "limit"): + raise ValueError("items must not exceed query limit") + + if any(type(item) is not item_type for item in items): + raise TypeError(f"items must contain only {item_type.__name__}") + + if next_cursor is not None and type(next_cursor) is not cursor_type: + raise TypeError( + f"next_cursor must be {cursor_type.__name__} or None" + ) + + if not items: + if next_cursor is not None: + raise ValueError("empty page must not have next_cursor") + return + + for previous, current in zip(items, items[1:], strict=False): + if getattr(previous, "order_key") >= getattr(current, "order_key"): + raise ValueError("page items must be strictly ordered") + + for item in items: + if ( + getattr(item, "venue") != getattr(query, "venue") + or getattr(item, "symbol") != getattr(query, "symbol") + ): + raise ValueError("page items must belong to one query scope") + + if getattr(item, "interval", None) != getattr( + query, + "interval", + None, + ): + raise ValueError("page items must belong to one interval") + + if not getattr(query, "time_range").contains( + getattr(item, "event_time") + ): + raise ValueError("page item time must belong to query range") + + query_cursor = getattr(query, "cursor") + + if query_cursor is not None: + incoming_key = ( + getattr(query_cursor, cursor_time_name), + getattr(query_cursor, "replay_sequence"), + ) + + if getattr(items[0], "order_key") <= incoming_key: + raise ValueError("page items must follow query cursor") + + if next_cursor is None: + return + + last = items[-1] + + if ( + getattr(next_cursor, "venue") != getattr(query, "venue") + or getattr(next_cursor, "symbol") != getattr(query, "symbol") + or getattr(next_cursor, "time_range") + != getattr(query, "time_range") + or getattr(next_cursor, "replay_sequence") + != getattr(last, "replay_sequence") + or getattr(next_cursor, cursor_time_name) + != getattr(last, "event_time") + ): + raise ValueError("next_cursor must point to the last page item") + + cursor_interval = getattr(next_cursor, "interval", None) + + if cursor_interval != getattr(query, "interval", None): + raise ValueError("next_cursor must belong to page interval") + + +HistoryRecord = ( + TradeHistoryRecord + | QuoteHistoryRecord + | CandleRevisionHistoryRecord +) +HistoryQuery = ( + TradeHistoryQuery + | QuoteHistoryQuery + | CandleRevisionHistoryQuery +) +HistoryPage = ( + TradeHistoryPage + | QuoteHistoryPage + | CandleRevisionHistoryPage +) +HistoryOrderKey = tuple[datetime, int] +HistoryOrderKeyFactory = Callable[[Any], HistoryOrderKey] diff --git a/app/src/market_data/access/postgres_candle_revision_history_repository.py b/app/src/market_data/access/postgres_candle_revision_history_repository.py new file mode 100644 index 0000000..526be3d --- /dev/null +++ b/app/src/market_data/access/postgres_candle_revision_history_repository.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +from collections.abc import Sequence +from datetime import datetime + +from src.market_data.access.exceptions import ( + MarketDataAccessError, + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, + MarketDataAccessValidationError, +) +from src.market_data.access.models import ( + CandleRevisionHistoryCursor, + CandleRevisionHistoryPage, + CandleRevisionHistoryQuery, + CandleRevisionHistoryRecord, +) +from src.market_data.access.postgres_history_support import ( + PostgresHistoryConnectionProvider, + candle_history_record_from_row, +) + + +_CANDLE_REVISION_HISTORY_COLUMNS_SQL = """ + venue, + symbol, + interval, + open_time, + observed_at, + open_price, + high_price, + low_price, + close_price, + volume, + is_final, + source, + observation_sources, + replay_sequence, + canonical_schema_version +""" + +_SELECT_CANDLE_REVISION_HISTORY_SQL = f""" +SELECT +{_CANDLE_REVISION_HISTORY_COLUMNS_SQL} +FROM market_data.candle_revisions +WHERE venue = %s + AND symbol = %s + AND interval = %s + AND open_time >= %s + AND open_time < %s +ORDER BY open_time ASC, + replay_sequence ASC +LIMIT %s +""" + +_SELECT_CANDLE_REVISION_HISTORY_AFTER_CURSOR_SQL = f""" +SELECT +{_CANDLE_REVISION_HISTORY_COLUMNS_SQL} +FROM market_data.candle_revisions +WHERE venue = %s + AND symbol = %s + AND interval = %s + AND open_time >= %s + AND open_time < %s + AND (open_time, replay_sequence) > (%s, %s) +ORDER BY open_time ASC, + replay_sequence ASC +LIMIT %s +""" + + +class PostgresCandleRevisionHistoryRepository: + """PostgreSQL-адаптер одной страницы истории ревизий свечей.""" + + __slots__ = ("_connection_provider",) + + def __init__( + self, + *, + connection_provider: PostgresHistoryConnectionProvider, + ) -> None: + if not callable(connection_provider): + raise TypeError("connection_provider must be callable") + + self._connection_provider = connection_provider + + def query_candle_revisions( + self, + query: CandleRevisionHistoryQuery, + ) -> CandleRevisionHistoryPage: + """Прочитать страницу в устойчивом порядке времени открытия.""" + if type(query) is not CandleRevisionHistoryQuery: + raise MarketDataAccessValidationError( + "query must be CandleRevisionHistoryQuery" + ) + + sql, parameters = self._build_query(query) + + try: + with self._connection_provider() as connection: + with connection.cursor() as cursor: + cursor.execute(sql, parameters) + rows = cursor.fetchall() + return self._page_from_rows(query=query, rows=rows) + except MarketDataAccessError: + raise + except Exception as error: + raise MarketDataAccessOperationError( + "Failed to query PostgreSQL Candle Revision History." + ) from error + + @staticmethod + def _build_query( + query: CandleRevisionHistoryQuery, + ) -> tuple[str, tuple[object, ...]]: + fetch_limit = query.limit + 1 + cursor = query.cursor + + if cursor is None: + return ( + _SELECT_CANDLE_REVISION_HISTORY_SQL, + ( + query.venue, + query.symbol, + query.interval, + query.time_range.start_time, + query.time_range.end_time, + fetch_limit, + ), + ) + + return ( + _SELECT_CANDLE_REVISION_HISTORY_AFTER_CURSOR_SQL, + ( + query.venue, + query.symbol, + query.interval, + query.time_range.start_time, + query.time_range.end_time, + cursor.open_time, + cursor.replay_sequence, + fetch_limit, + ), + ) + + @classmethod + def _page_from_rows( + cls, + *, + query: CandleRevisionHistoryQuery, + rows: object, + ) -> CandleRevisionHistoryPage: + if not isinstance(rows, Sequence) or isinstance( + rows, + (str, bytes, bytearray), + ): + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Candle Revision History rows." + ) + + if len(rows) > query.limit + 1: + raise MarketDataAccessIntegrityError( + "PostgreSQL exceeded the requested Candle History limit." + ) + + records = tuple( + cls._record_from_row(row=row, query=query) + for row in rows + ) + cls._validate_records(records=records, query=query) + + has_more = len(records) > query.limit + items = records[: query.limit] + next_cursor = ( + cls._cursor_from_record(query=query, record=items[-1]) + if has_more + else None + ) + + try: + return CandleRevisionHistoryPage( + query=query, + items=items, + next_cursor=next_cursor, + ) + except (TypeError, ValueError) as error: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned inconsistent Candle History page." + ) from error + + @staticmethod + def _record_from_row( + *, + row: object, + query: CandleRevisionHistoryQuery, + ) -> CandleRevisionHistoryRecord: + record = candle_history_record_from_row(row) + + if ( + record.venue != query.venue + or record.symbol != query.symbol + or record.interval != query.interval + ): + raise MarketDataAccessIntegrityError( + "PostgreSQL returned Candle revision outside query scope." + ) + + if not query.time_range.contains(record.event_time): + raise MarketDataAccessIntegrityError( + "PostgreSQL returned Candle revision outside query range." + ) + + return record + + @staticmethod + def _validate_records( + *, + records: tuple[CandleRevisionHistoryRecord, ...], + query: CandleRevisionHistoryQuery, + ) -> None: + seen_sequences: set[int] = set() + previous_key: tuple[datetime, int] | None = None + incoming_key = ( + ( + query.cursor.open_time, + query.cursor.replay_sequence, + ) + if query.cursor is not None + else None + ) + + for record in records: + if record.replay_sequence in seen_sequences: + raise MarketDataAccessIntegrityError( + "Candle History contains duplicate replay_sequence." + ) + + if incoming_key is not None and record.order_key <= incoming_key: + raise MarketDataAccessIntegrityError( + "Candle History row does not follow query cursor." + ) + + if previous_key is not None and record.order_key <= previous_key: + raise MarketDataAccessIntegrityError( + "Candle History rows are not strictly ordered." + ) + + seen_sequences.add(record.replay_sequence) + previous_key = record.order_key + + @staticmethod + def _cursor_from_record( + *, + query: CandleRevisionHistoryQuery, + record: CandleRevisionHistoryRecord, + ) -> CandleRevisionHistoryCursor: + try: + return CandleRevisionHistoryCursor( + venue=query.venue, + symbol=query.symbol, + interval=query.interval, + time_range=query.time_range, + open_time=record.event_time, + replay_sequence=record.replay_sequence, + ) + except (TypeError, ValueError) as error: + raise MarketDataAccessIntegrityError( + "Cannot create Candle History cursor from stored row." + ) from error diff --git a/app/src/market_data/access/postgres_history_support.py b/app/src/market_data/access/postgres_history_support.py new file mode 100644 index 0000000..df3d04c --- /dev/null +++ b/app/src/market_data/access/postgres_history_support.py @@ -0,0 +1,486 @@ +from __future__ import annotations + +from collections.abc import Callable +from contextlib import AbstractContextManager +from datetime import datetime, timezone +from decimal import Decimal +from typing import Any + +from src.market_data.access.exceptions import MarketDataAccessIntegrityError +from src.market_data.access.models import ( + CandleRevisionHistoryRecord, + QuoteHistoryRecord, + TradeHistoryRecord, +) +from src.market_data.acquisition.models.candle import Candle +from src.market_data.acquisition.models.quote import Quote +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.acquisition.symbols import normalize_symbol +from src.market_data.acquisition.trade_id_sequence import ( + validate_signed_trade_id, +) + + +CANONICAL_TRADE_SCHEMA_VERSION = 1 +CANONICAL_QUOTE_SCHEMA_VERSION = 1 +CANONICAL_CANDLE_SCHEMA_VERSION = 1 + +PostgresHistoryConnectionProvider = Callable[ + [], + AbstractContextManager[Any], +] + + +def canonical_text(value: object, *, field_name: str) -> str: + """Проверить каноническую непустую строку без скрытой нормализации.""" + if type(value) is not str: + raise TypeError(f"{field_name} must be a string") + + normalized = value.strip() + + if not normalized or normalized != value: + raise ValueError(f"{field_name} must be canonical") + + return value + + +def canonical_symbol(value: object, *, field_name: str) -> str: + """Проверить уже канонизированный символ сохранённой строки.""" + raw_symbol = canonical_text(value, field_name=field_name) + normalized = normalize_symbol(raw_symbol) + + if normalized != raw_symbol: + raise ValueError(f"{field_name} must be canonical") + + return normalized + + +def aware_utc_datetime(value: object, *, field_name: str) -> datetime: + """Проверить datetime с часовым поясом и привести его к UTC.""" + if ( + type(value) is not datetime + or value.tzinfo is None + or value.utcoffset() is None + ): + raise TypeError(f"{field_name} must be timezone-aware datetime") + + return value.astimezone(timezone.utc) + + +def positive_integer(value: object, *, field_name: str) -> int: + """Проверить точный положительный целочисленный тип.""" + if type(value) is not int: + raise TypeError(f"{field_name} must be an integer") + + if value <= 0: + raise ValueError(f"{field_name} must be positive") + + return value + + +def signed_trade_id(value: object) -> int: + """Проверить знаковый 32-битный идентификатор без изменения.""" + if type(value) is not int: + raise TypeError("stored trade.trade_id must be an integer") + + validate_signed_trade_id(value) + return value + + +def decimal_value( + value: object, + *, + field_name: str, + allow_zero: bool = False, +) -> Decimal: + """Проверить точный конечный Decimal и допустимый знак.""" + if type(value) is not Decimal or not value.is_finite(): + raise TypeError(f"{field_name} must be a finite Decimal") + + if allow_zero: + if value < 0: + raise ValueError(f"{field_name} must not be negative") + elif value <= 0: + raise ValueError(f"{field_name} must be positive") + + return value + + +def observation_sources( + value: object, + *, + primary_source: str, + field_name: str, +) -> tuple[str, ...]: + """Проверить порядок и уникальность источников наблюдения.""" + if type(value) is not list: + raise TypeError(f"{field_name} must be a list") + + if not value: + raise ValueError(f"{field_name} must not be empty") + + normalized = tuple( + canonical_text( + source, + field_name=f"{field_name} item", + ) + for source in value + ) + + if len(set(normalized)) != len(normalized): + raise ValueError(f"{field_name} must be unique") + + if normalized[0] != primary_source: + raise ValueError( + "first observation source must match stored payload.source" + ) + + return normalized + + +def trade_history_record_from_row(row: object) -> TradeHistoryRecord: + """Материализовать строго проверенную строку канонической сделки.""" + if type(row) is not tuple or len(row) != 13: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Trade History row." + ) + + ( + venue, + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + stored_sources, + replay_sequence, + canonical_schema_version, + ) = row + + try: + normalized_venue = canonical_text( + venue, + field_name="stored trade.venue", + ) + normalized_symbol = canonical_symbol( + symbol, + field_name="stored trade.symbol", + ) + normalized_trade_id = signed_trade_id(trade_id) + normalized_executed_at = aware_utc_datetime( + executed_at, + field_name="stored trade.executed_at", + ) + normalized_price = decimal_value( + price, + field_name="stored trade.price", + ) + normalized_quantity = decimal_value( + quantity, + field_name="stored trade.quantity", + ) + + if type(aggressor_side) is not str: + raise TypeError("stored trade.aggressor_side must be a string") + + normalized_side = TradeAggressorSide(aggressor_side) + normalized_source = canonical_text( + source, + field_name="stored trade.source", + ) + normalized_first_observed_at = aware_utc_datetime( + first_observed_at, + field_name="stored trade.first_observed_at", + ) + normalized_last_observed_at = aware_utc_datetime( + last_observed_at, + field_name="stored trade.last_observed_at", + ) + normalized_sources = observation_sources( + stored_sources, + primary_source=normalized_source, + field_name="stored trade.observation_sources", + ) + normalized_sequence = positive_integer( + replay_sequence, + field_name="stored trade.replay_sequence", + ) + normalized_version = positive_integer( + canonical_schema_version, + field_name="stored trade.canonical_schema_version", + ) + + if normalized_version != CANONICAL_TRADE_SCHEMA_VERSION: + raise ValueError("unsupported Canonical Trade schema version") + + trade = Trade( + symbol=normalized_symbol, + trade_id=normalized_trade_id, + price=normalized_price, + quantity=normalized_quantity, + executed_at=normalized_executed_at, + aggressor_side=normalized_side, + source=normalized_source, + ) + return TradeHistoryRecord( + venue=normalized_venue, + trade=trade, + first_observed_at=normalized_first_observed_at, + last_observed_at=normalized_last_observed_at, + observation_sources=normalized_sources, + replay_sequence=normalized_sequence, + canonical_schema_version=normalized_version, + ) + except (TypeError, ValueError, ArithmeticError) as error: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Canonical Trade values." + ) from error + + +def quote_history_record_from_row(row: object) -> QuoteHistoryRecord: + """Материализовать строго проверенную строку канонической котировки.""" + if type(row) is not tuple or len(row) != 11: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Quote History row." + ) + + ( + venue, + symbol, + received_at, + exchange_timestamp, + last_price, + bid_price, + ask_price, + source, + stored_sources, + replay_sequence, + canonical_schema_version, + ) = row + + try: + normalized_venue = canonical_text( + venue, + field_name="stored quote.venue", + ) + normalized_symbol = canonical_symbol( + symbol, + field_name="stored quote.symbol", + ) + normalized_received_at = aware_utc_datetime( + received_at, + field_name="stored quote.received_at", + ) + normalized_exchange_timestamp = ( + aware_utc_datetime( + exchange_timestamp, + field_name="stored quote.exchange_timestamp", + ) + if exchange_timestamp is not None + else None + ) + normalized_last_price = decimal_value( + last_price, + field_name="stored quote.last_price", + ) + normalized_bid_price = decimal_value( + bid_price, + field_name="stored quote.bid_price", + ) + normalized_ask_price = decimal_value( + ask_price, + field_name="stored quote.ask_price", + ) + + if normalized_bid_price > normalized_ask_price: + raise ValueError("stored quote.bid_price exceeds ask_price") + + normalized_source = canonical_text( + source, + field_name="stored quote.source", + ) + normalized_sources = observation_sources( + stored_sources, + primary_source=normalized_source, + field_name="stored quote.observation_sources", + ) + normalized_sequence = positive_integer( + replay_sequence, + field_name="stored quote.replay_sequence", + ) + normalized_version = positive_integer( + canonical_schema_version, + field_name="stored quote.canonical_schema_version", + ) + + if normalized_version != CANONICAL_QUOTE_SCHEMA_VERSION: + raise ValueError("unsupported Canonical Quote schema version") + + quote = Quote( + symbol=normalized_symbol, + last_price=normalized_last_price, + bid_price=normalized_bid_price, + ask_price=normalized_ask_price, + exchange_timestamp=normalized_exchange_timestamp, + received_at=normalized_received_at, + source=normalized_source, + ) + return QuoteHistoryRecord( + venue=normalized_venue, + quote=quote, + observation_sources=normalized_sources, + replay_sequence=normalized_sequence, + canonical_schema_version=normalized_version, + ) + except (TypeError, ValueError, ArithmeticError) as error: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Canonical Quote values." + ) from error + + +def candle_history_record_from_row( + row: object, +) -> CandleRevisionHistoryRecord: + """Материализовать строго проверенную строку ревизии свечи.""" + if type(row) is not tuple or len(row) != 15: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Candle History row." + ) + + ( + venue, + symbol, + interval, + open_time, + observed_at, + open_price, + high_price, + low_price, + close_price, + volume, + is_final, + source, + stored_sources, + replay_sequence, + canonical_schema_version, + ) = row + + try: + normalized_venue = canonical_text( + venue, + field_name="stored candle.venue", + ) + normalized_symbol = canonical_symbol( + symbol, + field_name="stored candle.symbol", + ) + normalized_interval = canonical_text( + interval, + field_name="stored candle.interval", + ) + normalized_open_time = aware_utc_datetime( + open_time, + field_name="stored candle.open_time", + ) + normalized_observed_at = aware_utc_datetime( + observed_at, + field_name="stored candle.observed_at", + ) + + if normalized_observed_at < normalized_open_time: + raise ValueError("stored candle.observed_at precedes open_time") + + normalized_open_price = decimal_value( + open_price, + field_name="stored candle.open_price", + ) + normalized_high_price = decimal_value( + high_price, + field_name="stored candle.high_price", + ) + normalized_low_price = decimal_value( + low_price, + field_name="stored candle.low_price", + ) + normalized_close_price = decimal_value( + close_price, + field_name="stored candle.close_price", + ) + normalized_volume = decimal_value( + volume, + field_name="stored candle.volume", + allow_zero=True, + ) + + if normalized_low_price > normalized_high_price: + raise ValueError("stored candle.low_price exceeds high_price") + + if not ( + normalized_low_price + <= normalized_open_price + <= normalized_high_price + ): + raise ValueError("stored candle.open_price is outside range") + + if not ( + normalized_low_price + <= normalized_close_price + <= normalized_high_price + ): + raise ValueError("stored candle.close_price is outside range") + + if type(is_final) is not bool: + raise TypeError("stored candle.is_final must be a boolean") + + normalized_source = canonical_text( + source, + field_name="stored candle.source", + ) + normalized_sources = observation_sources( + stored_sources, + primary_source=normalized_source, + field_name="stored candle.observation_sources", + ) + normalized_sequence = positive_integer( + replay_sequence, + field_name="stored candle.replay_sequence", + ) + normalized_version = positive_integer( + canonical_schema_version, + field_name="stored candle.canonical_schema_version", + ) + + if normalized_version != CANONICAL_CANDLE_SCHEMA_VERSION: + raise ValueError("unsupported Canonical Candle schema version") + + candle = Candle( + symbol=normalized_symbol, + interval=normalized_interval, + open_time=normalized_open_time, + open_price=normalized_open_price, + high_price=normalized_high_price, + low_price=normalized_low_price, + close_price=normalized_close_price, + volume=normalized_volume, + source=normalized_source, + ) + return CandleRevisionHistoryRecord( + venue=normalized_venue, + candle=candle, + observed_at=normalized_observed_at, + is_final=is_final, + observation_sources=normalized_sources, + replay_sequence=normalized_sequence, + canonical_schema_version=normalized_version, + ) + except (TypeError, ValueError, ArithmeticError) as error: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Canonical Candle values." + ) from error diff --git a/app/src/market_data/access/postgres_quote_history_repository.py b/app/src/market_data/access/postgres_quote_history_repository.py new file mode 100644 index 0000000..61172af --- /dev/null +++ b/app/src/market_data/access/postgres_quote_history_repository.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +from collections.abc import Sequence +from datetime import datetime + +from src.market_data.access.exceptions import ( + MarketDataAccessError, + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, + MarketDataAccessValidationError, +) +from src.market_data.access.models import ( + QuoteHistoryCursor, + QuoteHistoryPage, + QuoteHistoryQuery, + QuoteHistoryRecord, +) +from src.market_data.access.postgres_history_support import ( + PostgresHistoryConnectionProvider, + quote_history_record_from_row, +) + + +_QUOTE_HISTORY_COLUMNS_SQL = """ + venue, + symbol, + received_at, + exchange_timestamp, + last_price, + bid_price, + ask_price, + source, + observation_sources, + replay_sequence, + canonical_schema_version +""" + +_SELECT_QUOTE_HISTORY_SQL = f""" +SELECT +{_QUOTE_HISTORY_COLUMNS_SQL} +FROM market_data.quotes +WHERE venue = %s + AND symbol = %s + AND received_at >= %s + AND received_at < %s +ORDER BY received_at ASC, + replay_sequence ASC +LIMIT %s +""" + +_SELECT_QUOTE_HISTORY_AFTER_CURSOR_SQL = f""" +SELECT +{_QUOTE_HISTORY_COLUMNS_SQL} +FROM market_data.quotes +WHERE venue = %s + AND symbol = %s + AND received_at >= %s + AND received_at < %s + AND (received_at, replay_sequence) > (%s, %s) +ORDER BY received_at ASC, + replay_sequence ASC +LIMIT %s +""" + + +class PostgresQuoteHistoryRepository: + """PostgreSQL-адаптер одной страницы канонической истории котировок.""" + + __slots__ = ("_connection_provider",) + + def __init__( + self, + *, + connection_provider: PostgresHistoryConnectionProvider, + ) -> None: + if not callable(connection_provider): + raise TypeError("connection_provider must be callable") + + self._connection_provider = connection_provider + + def query_quotes(self, query: QuoteHistoryQuery) -> QuoteHistoryPage: + """Прочитать ограниченную страницу с устойчивой сортировкой.""" + if type(query) is not QuoteHistoryQuery: + raise MarketDataAccessValidationError( + "query must be QuoteHistoryQuery" + ) + + sql, parameters = self._build_query(query) + + try: + with self._connection_provider() as connection: + with connection.cursor() as cursor: + cursor.execute(sql, parameters) + rows = cursor.fetchall() + return self._page_from_rows(query=query, rows=rows) + except MarketDataAccessError: + raise + except Exception as error: + raise MarketDataAccessOperationError( + "Failed to query PostgreSQL Quote History." + ) from error + + @staticmethod + def _build_query( + query: QuoteHistoryQuery, + ) -> tuple[str, tuple[object, ...]]: + fetch_limit = query.limit + 1 + cursor = query.cursor + + if cursor is None: + return ( + _SELECT_QUOTE_HISTORY_SQL, + ( + query.venue, + query.symbol, + query.time_range.start_time, + query.time_range.end_time, + fetch_limit, + ), + ) + + return ( + _SELECT_QUOTE_HISTORY_AFTER_CURSOR_SQL, + ( + query.venue, + query.symbol, + query.time_range.start_time, + query.time_range.end_time, + cursor.received_at, + cursor.replay_sequence, + fetch_limit, + ), + ) + + @classmethod + def _page_from_rows( + cls, + *, + query: QuoteHistoryQuery, + rows: object, + ) -> QuoteHistoryPage: + if not isinstance(rows, Sequence) or isinstance( + rows, + (str, bytes, bytearray), + ): + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Quote History rows." + ) + + if len(rows) > query.limit + 1: + raise MarketDataAccessIntegrityError( + "PostgreSQL exceeded the requested Quote History limit." + ) + + records = tuple( + cls._record_from_row(row=row, query=query) + for row in rows + ) + cls._validate_records(records=records, query=query) + + has_more = len(records) > query.limit + items = records[: query.limit] + next_cursor = ( + cls._cursor_from_record(query=query, record=items[-1]) + if has_more + else None + ) + + try: + return QuoteHistoryPage( + query=query, + items=items, + next_cursor=next_cursor, + ) + except (TypeError, ValueError) as error: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned inconsistent Quote History page." + ) from error + + @staticmethod + def _record_from_row( + *, + row: object, + query: QuoteHistoryQuery, + ) -> QuoteHistoryRecord: + record = quote_history_record_from_row(row) + + if record.venue != query.venue or record.symbol != query.symbol: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned Quote outside query scope." + ) + + if not query.time_range.contains(record.event_time): + raise MarketDataAccessIntegrityError( + "PostgreSQL returned Quote outside query range." + ) + + return record + + @staticmethod + def _validate_records( + *, + records: tuple[QuoteHistoryRecord, ...], + query: QuoteHistoryQuery, + ) -> None: + seen_sequences: set[int] = set() + previous_key: tuple[datetime, int] | None = None + incoming_key = ( + ( + query.cursor.received_at, + query.cursor.replay_sequence, + ) + if query.cursor is not None + else None + ) + + for record in records: + if record.replay_sequence in seen_sequences: + raise MarketDataAccessIntegrityError( + "Quote History contains duplicate replay_sequence." + ) + + if incoming_key is not None and record.order_key <= incoming_key: + raise MarketDataAccessIntegrityError( + "Quote History row does not follow query cursor." + ) + + if previous_key is not None and record.order_key <= previous_key: + raise MarketDataAccessIntegrityError( + "Quote History rows are not strictly ordered." + ) + + seen_sequences.add(record.replay_sequence) + previous_key = record.order_key + + @staticmethod + def _cursor_from_record( + *, + query: QuoteHistoryQuery, + record: QuoteHistoryRecord, + ) -> QuoteHistoryCursor: + try: + return QuoteHistoryCursor( + venue=query.venue, + symbol=query.symbol, + time_range=query.time_range, + received_at=record.event_time, + replay_sequence=record.replay_sequence, + ) + except (TypeError, ValueError) as error: + raise MarketDataAccessIntegrityError( + "Cannot create Quote History cursor from stored row." + ) from error diff --git a/app/src/market_data/access/postgres_trade_history_repository.py b/app/src/market_data/access/postgres_trade_history_repository.py new file mode 100644 index 0000000..c2b636a --- /dev/null +++ b/app/src/market_data/access/postgres_trade_history_repository.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +from collections.abc import Sequence +from datetime import datetime +from typing import Any + +from src.market_data.access.exceptions import ( + MarketDataAccessError, + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, + MarketDataAccessValidationError, +) +from src.market_data.access.models import ( + TradeHistoryCursor, + TradeHistoryPage, + TradeHistoryQuery, + TradeHistoryRecord, +) +from src.market_data.access.postgres_history_support import ( + PostgresHistoryConnectionProvider, + trade_history_record_from_row, +) + +_TRADE_HISTORY_COLUMNS_SQL = """ + venue, + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + observation_sources, + replay_sequence, + canonical_schema_version +""" + +_SELECT_TRADE_HISTORY_SQL = f""" +SELECT +{_TRADE_HISTORY_COLUMNS_SQL} +FROM market_data.trades +WHERE venue = %s + AND symbol = %s + AND executed_at >= %s + AND executed_at < %s +ORDER BY executed_at ASC, + replay_sequence ASC +LIMIT %s +""" + +_SELECT_TRADE_HISTORY_AFTER_CURSOR_SQL = f""" +SELECT +{_TRADE_HISTORY_COLUMNS_SQL} +FROM market_data.trades +WHERE venue = %s + AND symbol = %s + AND executed_at >= %s + AND executed_at < %s + AND (executed_at, replay_sequence) > (%s, %s) +ORDER BY executed_at ASC, + replay_sequence ASC +LIMIT %s +""" + + +class PostgresTradeHistoryRepository: + """PostgreSQL-адаптер одной страницы канонической истории сделок.""" + + __slots__ = ("_connection_provider",) + + def __init__( + self, + *, + connection_provider: PostgresHistoryConnectionProvider, + ) -> None: + if not callable(connection_provider): + raise TypeError("connection_provider must be callable") + + self._connection_provider = connection_provider + + def query_trades(self, query: TradeHistoryQuery) -> TradeHistoryPage: + """Прочитать одну bounded страницу в устойчивом keyset-порядке.""" + if type(query) is not TradeHistoryQuery: + raise MarketDataAccessValidationError( + "query must be TradeHistoryQuery" + ) + + sql, parameters = self._build_query(query) + + try: + with self._connection_provider() as connection: + with connection.cursor() as cursor: + cursor.execute(sql, parameters) + rows = cursor.fetchall() + return self._page_from_rows(query=query, rows=rows) + except MarketDataAccessError: + raise + except Exception as error: + raise MarketDataAccessOperationError( + "Failed to query PostgreSQL Trade History." + ) from error + + @staticmethod + def _build_query( + query: TradeHistoryQuery, + ) -> tuple[str, tuple[object, ...]]: + fetch_limit = query.limit + 1 + cursor = query.cursor + + if cursor is None: + return ( + _SELECT_TRADE_HISTORY_SQL, + ( + query.venue, + query.symbol, + query.time_range.start_time, + query.time_range.end_time, + fetch_limit, + ), + ) + + return ( + _SELECT_TRADE_HISTORY_AFTER_CURSOR_SQL, + ( + query.venue, + query.symbol, + query.time_range.start_time, + query.time_range.end_time, + cursor.executed_at, + cursor.replay_sequence, + fetch_limit, + ), + ) + + @classmethod + def _page_from_rows( + cls, + *, + query: TradeHistoryQuery, + rows: object, + ) -> TradeHistoryPage: + if not isinstance(rows, Sequence) or isinstance( + rows, + (str, bytes, bytearray), + ): + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Trade History rows." + ) + + if len(rows) > query.limit + 1: + raise MarketDataAccessIntegrityError( + "PostgreSQL exceeded the requested Trade History limit." + ) + + records = tuple( + cls._record_from_row(row=row, query=query) + for row in rows + ) + cls._validate_records(records=records, query=query) + + has_more = len(records) > query.limit + items = records[: query.limit] + next_cursor = ( + cls._cursor_from_record(query=query, record=items[-1]) + if has_more + else None + ) + + try: + return TradeHistoryPage( + query=query, + items=items, + next_cursor=next_cursor, + ) + except (TypeError, ValueError) as error: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned inconsistent Trade History page." + ) from error + + @classmethod + def _record_from_row( + cls, + *, + row: object, + query: TradeHistoryQuery, + ) -> TradeHistoryRecord: + record = trade_history_record_from_row(row) + + try: + if record.venue != query.venue or record.symbol != query.symbol: + raise ValueError("stored Trade is outside query scope") + + if not query.time_range.contains(record.event_time): + raise ValueError("stored Trade is outside query range") + except (TypeError, ValueError) as error: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Canonical Trade values." + ) from error + + return record + + @staticmethod + def _validate_records( + *, + records: tuple[TradeHistoryRecord, ...], + query: TradeHistoryQuery, + ) -> None: + seen_sequences: set[int] = set() + previous_key: tuple[datetime, int] | None = None + incoming_key = ( + ( + query.cursor.executed_at, + query.cursor.replay_sequence, + ) + if query.cursor is not None + else None + ) + + for record in records: + if record.replay_sequence in seen_sequences: + raise MarketDataAccessIntegrityError( + "Trade History contains duplicate replay_sequence." + ) + + if incoming_key is not None and record.order_key <= incoming_key: + raise MarketDataAccessIntegrityError( + "Trade History row does not follow query cursor." + ) + + if previous_key is not None and record.order_key <= previous_key: + raise MarketDataAccessIntegrityError( + "Trade History rows are not strictly ordered." + ) + + seen_sequences.add(record.replay_sequence) + previous_key = record.order_key + + @staticmethod + def _cursor_from_record( + *, + query: TradeHistoryQuery, + record: TradeHistoryRecord, + ) -> TradeHistoryCursor: + try: + return TradeHistoryCursor( + venue=query.venue, + symbol=query.symbol, + time_range=query.time_range, + executed_at=record.event_time, + replay_sequence=record.replay_sequence, + ) + except (TypeError, ValueError) as error: + raise MarketDataAccessIntegrityError( + "Cannot create Trade History cursor from stored row." + ) from error diff --git a/app/src/market_data/replay/__init__.py b/app/src/market_data/replay/__init__.py new file mode 100644 index 0000000..0e95541 --- /dev/null +++ b/app/src/market_data/replay/__init__.py @@ -0,0 +1,62 @@ +from src.market_data.replay.contracts import ( + MarketDataClockProtocol, + ReplayClockProtocol, + ReplayConsumerFactoryProtocol, + ReplayConsumerProtocol, + ReplayPlanBuilderProtocol, + ReplaySessionProtocol, +) +from src.market_data.replay.deterministic_replay_clock import ( + DeterministicReplayClock, +) +from src.market_data.replay.exceptions import ( + MarketDataReplayError, + MarketDataReplayValidationError, + ReplayClockError, + ReplayPlanLimitExceededError, + ReplaySessionStateError, +) +from src.market_data.replay.models import ( + REPLAY_PLAN_MAX_RECORDS_DEFAULT, + REPLAY_PLAN_MAX_RECORDS_LIMIT, + CanonicalMarketData, + ReplayDataType, + ReplayEvent, + ReplayPlan, + ReplayPlanRequest, + ReplaySessionState, +) +from src.market_data.replay.postgres_replay_plan_builder import ( + PostgresReplayPlanBuilder, +) +from src.market_data.replay.replay_session import ReplaySession +from src.market_data.replay.replay_session_factory import ( + ReplaySessionFactory, +) + + +__all__ = ( + "REPLAY_PLAN_MAX_RECORDS_DEFAULT", + "REPLAY_PLAN_MAX_RECORDS_LIMIT", + "CanonicalMarketData", + "DeterministicReplayClock", + "MarketDataClockProtocol", + "MarketDataReplayError", + "MarketDataReplayValidationError", + "PostgresReplayPlanBuilder", + "ReplayClockError", + "ReplayClockProtocol", + "ReplayConsumerFactoryProtocol", + "ReplayConsumerProtocol", + "ReplayDataType", + "ReplayEvent", + "ReplayPlan", + "ReplayPlanBuilderProtocol", + "ReplayPlanLimitExceededError", + "ReplayPlanRequest", + "ReplaySession", + "ReplaySessionFactory", + "ReplaySessionProtocol", + "ReplaySessionState", + "ReplaySessionStateError", +) diff --git a/app/src/market_data/replay/contracts.py b/app/src/market_data/replay/contracts.py new file mode 100644 index 0000000..3ec5832 --- /dev/null +++ b/app/src/market_data/replay/contracts.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Protocol, runtime_checkable + +from src.market_data.replay.models import ( + ReplayEvent, + ReplayPlan, + ReplayPlanRequest, + ReplaySessionState, +) + + +@runtime_checkable +class MarketDataClockProtocol(Protocol): + """Доступ consumer к текущему времени без права его изменять.""" + + @property + def now(self) -> datetime: + ... + + +@runtime_checkable +class ReplayClockProtocol(MarketDataClockProtocol, Protocol): + """Управляющая граница Virtual Clock одной Replay Session.""" + + def advance_to(self, instant: datetime) -> None: + ... + + +@runtime_checkable +class ReplayConsumerProtocol(Protocol): + """Один последовательный async consumer Canonical Replay events.""" + + async def consume(self, event: ReplayEvent) -> None: + ... + + +@runtime_checkable +class ReplayConsumerFactoryProtocol(Protocol): + """Граница создания отдельного consumer для одной Replay Session.""" + + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + ... + + +@runtime_checkable +class ReplayPlanBuilderProtocol(Protocol): + """Граница создания bounded snapshot до запуска Replay Session.""" + + def create_plan(self, request: ReplayPlanRequest) -> ReplayPlan: + ... + + +@runtime_checkable +class ReplaySessionProtocol(Protocol): + """Одноразовая исполняемая Replay Session без hidden root task.""" + + @property + def state(self) -> ReplaySessionState: + ... + + @property + def plan(self) -> ReplayPlan: + ... + + @property + def clock(self) -> MarketDataClockProtocol: + ... + + async def run(self) -> None: + ... diff --git a/app/src/market_data/replay/deterministic_replay_clock.py b/app/src/market_data/replay/deterministic_replay_clock.py new file mode 100644 index 0000000..2930d0f --- /dev/null +++ b/app/src/market_data/replay/deterministic_replay_clock.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from src.market_data.replay.exceptions import ReplayClockError + + +def _normalize_utc_datetime( + value: object, + *, + field_name: str, +) -> datetime: + """Проверить время с часовым поясом и канонизировать его в UTC.""" + if not isinstance(value, datetime): + raise TypeError(f"{field_name} must be a datetime") + + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{field_name} must contain timezone") + + normalized = value.astimezone(timezone.utc) + + # Сохраняем совместимость с datetime-подклассами на входе, но не + # переносим их изменённое поведение во внутреннее состояние Clock. + return datetime( + normalized.year, + normalized.month, + normalized.day, + normalized.hour, + normalized.minute, + normalized.second, + normalized.microsecond, + tzinfo=timezone.utc, + ) + + +class DeterministicReplayClock: + """Синхронные логические часы одного детерминированного Replay.""" + + __slots__ = ("_now",) + + def __init__(self, initial_time: datetime) -> None: + self._now = _normalize_utc_datetime( + initial_time, + field_name="initial_time", + ) + + @property + def now(self) -> datetime: + """Вернуть текущий канонический UTC-момент Replay.""" + return self._now + + def advance_to(self, instant: datetime) -> None: + """Перевести логическое время вперёд или оставить его прежним.""" + normalized = _normalize_utc_datetime( + instant, + field_name="instant", + ) + + if normalized < self._now: + raise ReplayClockError( + "Replay Clock cannot move backwards." + ) + + if normalized == self._now: + return + + self._now = normalized diff --git a/app/src/market_data/replay/exceptions.py b/app/src/market_data/replay/exceptions.py new file mode 100644 index 0000000..4ebc5a9 --- /dev/null +++ b/app/src/market_data/replay/exceptions.py @@ -0,0 +1,21 @@ +from __future__ import annotations + + +class MarketDataReplayError(Exception): + """Базовая ошибка детерминированного Market Data Replay.""" + + +class MarketDataReplayValidationError(MarketDataReplayError): + """Недопустимый Replay request, plan или dependency.""" + + +class ReplayPlanLimitExceededError(MarketDataReplayError): + """Snapshot содержит больше событий, чем разрешает Replay request.""" + + +class ReplayClockError(MarketDataReplayError): + """Virtual Clock получил недопустимый переход времени.""" + + +class ReplaySessionStateError(MarketDataReplayError): + """Операция недопустима в текущем состоянии Replay Session.""" diff --git a/app/src/market_data/replay/models.py b/app/src/market_data/replay/models.py new file mode 100644 index 0000000..ef3266f --- /dev/null +++ b/app/src/market_data/replay/models.py @@ -0,0 +1,371 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum + +from src.market_data.access.models import HistoricalTimeRange +from src.market_data.acquisition.models.candle import Candle +from src.market_data.acquisition.models.quote import Quote +from src.market_data.acquisition.models.trade import Trade +from src.market_data.acquisition.symbols import normalize_symbol + + +REPLAY_PLAN_MAX_RECORDS_DEFAULT = 100_000 +REPLAY_PLAN_MAX_RECORDS_LIMIT = 100_000 + + +class ReplayDataType(Enum): + """Тип Canonical payload в одном Replay Plan.""" + + TRADE = "trade" + QUOTE = "quote" + CANDLE_REVISION = "candle_revision" + + +class ReplaySessionState(Enum): + """Одноразовый lifecycle одной Replay Session.""" + + CREATED = "created" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +CanonicalMarketData = Trade | Quote | Candle + + +def _normalize_non_empty_text( + value: object, + *, + field_name: str, +) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be a string") + + normalized = value.strip() + + if not normalized: + raise ValueError(f"{field_name} must not be empty") + + return normalized + + +def _normalize_utc_datetime( + value: object, + *, + field_name: str, +) -> datetime: + if not isinstance(value, datetime): + raise TypeError(f"{field_name} must be a datetime") + + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{field_name} must contain timezone") + + return value.astimezone(timezone.utc) + + +def _normalize_positive_integer( + value: object, + *, + field_name: str, + maximum: int | None = None, +) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{field_name} must be an integer") + + if value <= 0: + raise ValueError(f"{field_name} must be positive") + + if maximum is not None and value > maximum: + raise ValueError( + f"{field_name} must not exceed {maximum}" + ) + + return value + + +def _canonical_payload_time(payload: CanonicalMarketData) -> datetime: + if isinstance(payload, Trade): + return _normalize_utc_datetime( + payload.executed_at, + field_name="payload.executed_at", + ) + + if isinstance(payload, Quote): + return _normalize_utc_datetime( + payload.received_at, + field_name="payload.received_at", + ) + + return _normalize_utc_datetime( + payload.open_time, + field_name="payload.open_time", + ) + + +@dataclass(frozen=True, slots=True) +class ReplayEvent: + """Canonical payload с устойчивой позицией во времени Replay.""" + + venue: str + replay_at: datetime + replay_sequence: int + payload: CanonicalMarketData + candle_is_final: bool | None = None + + def __post_init__(self) -> None: + normalized_venue = _normalize_non_empty_text( + self.venue, + field_name="venue", + ) + normalized_replay_at = _normalize_utc_datetime( + self.replay_at, + field_name="replay_at", + ) + normalized_sequence = _normalize_positive_integer( + self.replay_sequence, + field_name="replay_sequence", + ) + + if type(self.payload) not in (Trade, Quote, Candle): + raise TypeError( + "payload must be a Canonical Trade, Quote or Candle" + ) + + normalized_symbol = normalize_symbol( + _normalize_non_empty_text( + self.payload.symbol, + field_name="payload.symbol", + ) + ) + + if normalized_symbol != self.payload.symbol: + raise ValueError("payload.symbol must be canonical") + + normalized_source = _normalize_non_empty_text( + self.payload.source, + field_name="payload.source", + ) + + if normalized_source != self.payload.source: + raise ValueError("payload.source must be canonical") + + payload_time = _canonical_payload_time(self.payload) + + if isinstance(self.payload, Candle): + normalized_interval = _normalize_non_empty_text( + self.payload.interval, + field_name="payload.interval", + ) + + if normalized_interval != self.payload.interval: + raise ValueError("payload.interval must be canonical") + + if not isinstance(self.candle_is_final, bool): + raise TypeError( + "candle_is_final must be a boolean for Candle" + ) + + if normalized_replay_at < payload_time: + raise ValueError( + "Candle replay_at must not precede open_time" + ) + else: + if self.candle_is_final is not None: + raise ValueError( + "candle_is_final must be None for Trade and Quote" + ) + + if normalized_replay_at != payload_time: + raise ValueError( + "replay_at must match Canonical payload event time" + ) + + object.__setattr__(self, "venue", normalized_venue) + object.__setattr__(self, "replay_at", normalized_replay_at) + object.__setattr__( + self, + "replay_sequence", + normalized_sequence, + ) + + @property + def symbol(self) -> str: + return self.payload.symbol + + @property + def data_type(self) -> ReplayDataType: + if isinstance(self.payload, Trade): + return ReplayDataType.TRADE + + if isinstance(self.payload, Quote): + return ReplayDataType.QUOTE + + return ReplayDataType.CANDLE_REVISION + + @property + def order_key(self) -> tuple[datetime, int]: + return (self.replay_at, self.replay_sequence) + + +@dataclass(frozen=True, slots=True) +class ReplayPlanRequest: + """Неизменяемый scope одного bounded Replay snapshot.""" + + venue: str + symbols: tuple[str, ...] + data_types: tuple[ReplayDataType, ...] + time_range: HistoricalTimeRange + candle_intervals: tuple[str, ...] = () + max_records: int = REPLAY_PLAN_MAX_RECORDS_DEFAULT + + def __post_init__(self) -> None: + venue = _normalize_non_empty_text( + self.venue, + field_name="venue", + ) + + if type(self.symbols) is not tuple: + raise TypeError("symbols must be a tuple") + + if not self.symbols: + raise ValueError("symbols must not be empty") + + symbols = tuple( + normalize_symbol( + _normalize_non_empty_text( + symbol, + field_name="symbols item", + ) + ) + for symbol in self.symbols + ) + + if len(set(symbols)) != len(symbols): + raise ValueError("symbols must be unique after normalization") + + if type(self.data_types) is not tuple: + raise TypeError("data_types must be a tuple") + + if not self.data_types: + raise ValueError("data_types must not be empty") + + if any( + not isinstance(data_type, ReplayDataType) + for data_type in self.data_types + ): + raise TypeError("data_types must contain only ReplayDataType") + + if len(set(self.data_types)) != len(self.data_types): + raise ValueError("data_types must be unique") + + if type(self.time_range) is not HistoricalTimeRange: + raise TypeError("time_range must be HistoricalTimeRange") + + if type(self.candle_intervals) is not tuple: + raise TypeError("candle_intervals must be a tuple") + + candle_intervals = tuple( + _normalize_non_empty_text( + interval, + field_name="candle_intervals item", + ) + for interval in self.candle_intervals + ) + + if len(set(candle_intervals)) != len(candle_intervals): + raise ValueError("candle_intervals must be unique") + + includes_candles = ( + ReplayDataType.CANDLE_REVISION in self.data_types + ) + + if includes_candles and not candle_intervals: + raise ValueError( + "candle_intervals are required for Candle replay" + ) + + if not includes_candles and candle_intervals: + raise ValueError( + "candle_intervals require Candle revision data type" + ) + + max_records = _normalize_positive_integer( + self.max_records, + field_name="max_records", + maximum=REPLAY_PLAN_MAX_RECORDS_LIMIT, + ) + + object.__setattr__(self, "venue", venue) + object.__setattr__(self, "symbols", symbols) + object.__setattr__( + self, + "candle_intervals", + candle_intervals, + ) + object.__setattr__(self, "max_records", max_records) + + +@dataclass(frozen=True, slots=True) +class ReplayPlan: + """Полностью материализованный и DB-независимый Replay snapshot.""" + + request: ReplayPlanRequest + events: tuple[ReplayEvent, ...] + + def __post_init__(self) -> None: + if type(self.request) is not ReplayPlanRequest: + raise TypeError("request must be ReplayPlanRequest") + + if type(self.events) is not tuple: + raise TypeError("events must be a tuple") + + if len(self.events) > self.request.max_records: + raise ValueError("events exceed request.max_records") + + if any(type(event) is not ReplayEvent for event in self.events): + raise TypeError("events must contain only ReplayEvent") + + seen_sequences: set[int] = set() + + for index, event in enumerate(self.events): + if event.venue != self.request.venue: + raise ValueError("event venue is outside Replay request") + + if event.symbol not in self.request.symbols: + raise ValueError("event symbol is outside Replay request") + + if event.data_type not in self.request.data_types: + raise ValueError("event data type is outside Replay request") + + if not self.request.time_range.contains(event.replay_at): + raise ValueError("event time is outside Replay request") + + if isinstance(event.payload, Candle) and ( + event.payload.interval + not in self.request.candle_intervals + ): + raise ValueError( + "Candle interval is outside Replay request" + ) + + if event.replay_sequence in seen_sequences: + raise ValueError( + "replay_sequence must be globally unique in plan" + ) + + seen_sequences.add(event.replay_sequence) + + if index > 0 and ( + self.events[index - 1].order_key >= event.order_key + ): + raise ValueError("events must be strictly ordered") + + @property + def is_empty(self) -> bool: + return not self.events + + def __len__(self) -> int: + return len(self.events) diff --git a/app/src/market_data/replay/postgres_replay_plan_builder.py b/app/src/market_data/replay/postgres_replay_plan_builder.py new file mode 100644 index 0000000..10c412e --- /dev/null +++ b/app/src/market_data/replay/postgres_replay_plan_builder.py @@ -0,0 +1,436 @@ +from __future__ import annotations + +from collections.abc import Sequence + +from src.market_data.access.exceptions import ( + MarketDataAccessError, + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, +) +from src.market_data.access.postgres_history_support import ( + PostgresHistoryConnectionProvider, + candle_history_record_from_row, + quote_history_record_from_row, + trade_history_record_from_row, +) +from src.market_data.replay.exceptions import ( + MarketDataReplayValidationError, + ReplayPlanLimitExceededError, +) +from src.market_data.replay.models import ( + ReplayDataType, + ReplayEvent, + ReplayPlan, + ReplayPlanRequest, +) + + +_SET_SNAPSHOT_TRANSACTION_SQL = ( + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY" +) + +_REPLAY_SNAPSHOT_COLUMNS_SQL = """ + data_type, + replay_at, + replay_sequence, + venue, + symbol, + trade_id, + executed_at, + trade_price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + received_at, + exchange_timestamp, + last_price, + bid_price, + ask_price, + interval, + open_time, + observed_at, + open_price, + high_price, + low_price, + close_price, + volume, + is_final, + observation_sources, + canonical_schema_version +""" + +_TRADE_SNAPSHOT_BRANCH_SQL = """ +SELECT + 'trade'::TEXT AS data_type, + executed_at AS replay_at, + replay_sequence, + venue, + symbol, + trade_id, + executed_at, + price AS trade_price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + NULL::TIMESTAMPTZ AS received_at, + NULL::TIMESTAMPTZ AS exchange_timestamp, + NULL::NUMERIC AS last_price, + NULL::NUMERIC AS bid_price, + NULL::NUMERIC AS ask_price, + NULL::TEXT AS interval, + NULL::TIMESTAMPTZ AS open_time, + NULL::TIMESTAMPTZ AS observed_at, + NULL::NUMERIC AS open_price, + NULL::NUMERIC AS high_price, + NULL::NUMERIC AS low_price, + NULL::NUMERIC AS close_price, + NULL::NUMERIC AS volume, + NULL::BOOLEAN AS is_final, + observation_sources, + canonical_schema_version +FROM market_data.trades +WHERE venue = %s + AND symbol = ANY(%s) + AND executed_at >= %s + AND executed_at < %s +""" + +_QUOTE_SNAPSHOT_BRANCH_SQL = """ +SELECT + 'quote'::TEXT AS data_type, + received_at AS replay_at, + replay_sequence, + venue, + symbol, + NULL::INTEGER AS trade_id, + NULL::TIMESTAMPTZ AS executed_at, + NULL::NUMERIC AS trade_price, + NULL::NUMERIC AS quantity, + NULL::TEXT AS aggressor_side, + source, + NULL::TIMESTAMPTZ AS first_observed_at, + NULL::TIMESTAMPTZ AS last_observed_at, + received_at, + exchange_timestamp, + last_price, + bid_price, + ask_price, + NULL::TEXT AS interval, + NULL::TIMESTAMPTZ AS open_time, + NULL::TIMESTAMPTZ AS observed_at, + NULL::NUMERIC AS open_price, + NULL::NUMERIC AS high_price, + NULL::NUMERIC AS low_price, + NULL::NUMERIC AS close_price, + NULL::NUMERIC AS volume, + NULL::BOOLEAN AS is_final, + observation_sources, + canonical_schema_version +FROM market_data.quotes +WHERE venue = %s + AND symbol = ANY(%s) + AND received_at >= %s + AND received_at < %s +""" + +_CANDLE_SNAPSHOT_BRANCH_SQL = """ +SELECT + 'candle_revision'::TEXT AS data_type, + observed_at AS replay_at, + replay_sequence, + venue, + symbol, + NULL::INTEGER AS trade_id, + NULL::TIMESTAMPTZ AS executed_at, + NULL::NUMERIC AS trade_price, + NULL::NUMERIC AS quantity, + NULL::TEXT AS aggressor_side, + source, + NULL::TIMESTAMPTZ AS first_observed_at, + NULL::TIMESTAMPTZ AS last_observed_at, + NULL::TIMESTAMPTZ AS received_at, + NULL::TIMESTAMPTZ AS exchange_timestamp, + NULL::NUMERIC AS last_price, + NULL::NUMERIC AS bid_price, + NULL::NUMERIC AS ask_price, + interval, + open_time, + observed_at, + open_price, + high_price, + low_price, + close_price, + volume, + is_final, + observation_sources, + canonical_schema_version +FROM market_data.candle_revisions +WHERE venue = %s + AND symbol = ANY(%s) + AND observed_at >= %s + AND observed_at < %s + AND interval = ANY(%s) +""" + +_SNAPSHOT_BRANCHES = { + ReplayDataType.TRADE: _TRADE_SNAPSHOT_BRANCH_SQL, + ReplayDataType.QUOTE: _QUOTE_SNAPSHOT_BRANCH_SQL, + ReplayDataType.CANDLE_REVISION: _CANDLE_SNAPSHOT_BRANCH_SQL, +} + +_SNAPSHOT_DATA_TYPE_ORDER = ( + ReplayDataType.TRADE, + ReplayDataType.QUOTE, + ReplayDataType.CANDLE_REVISION, +) + +_SNAPSHOT_ROW_LENGTH = 29 + + +class PostgresReplayPlanBuilder: + """Создать один ограниченный снимок воспроизведения из PostgreSQL.""" + + __slots__ = ("_connection_provider",) + + def __init__( + self, + *, + connection_provider: PostgresHistoryConnectionProvider, + ) -> None: + if not callable(connection_provider): + raise TypeError("connection_provider must be callable") + + self._connection_provider = connection_provider + + def create_plan(self, request: ReplayPlanRequest) -> ReplayPlan: + """Материализовать план в короткой транзакции только для чтения.""" + if type(request) is not ReplayPlanRequest: + raise MarketDataReplayValidationError( + "request must be ReplayPlanRequest" + ) + + sql, parameters = self._build_snapshot_query(request) + + try: + with self._connection_provider() as connection: + with connection.transaction(): + with connection.cursor() as cursor: + cursor.execute(_SET_SNAPSHOT_TRANSACTION_SQL) + cursor.execute(sql, parameters) + rows = cursor.fetchall() + + return self._plan_from_rows(request=request, rows=rows) + except (MarketDataAccessError, ReplayPlanLimitExceededError): + raise + except Exception as error: + raise MarketDataAccessOperationError( + "Failed to create PostgreSQL Replay snapshot." + ) from error + + @staticmethod + def _build_snapshot_query( + request: ReplayPlanRequest, + ) -> tuple[str, tuple[object, ...]]: + requested_types = set(request.data_types) + branches: list[str] = [] + parameters: list[object] = [] + + for data_type in _SNAPSHOT_DATA_TYPE_ORDER: + if data_type not in requested_types: + continue + + branches.append(_SNAPSHOT_BRANCHES[data_type]) + parameters.extend( + ( + request.venue, + list(request.symbols), + request.time_range.start_time, + request.time_range.end_time, + ) + ) + + if data_type is ReplayDataType.CANDLE_REVISION: + parameters.append(list(request.candle_intervals)) + + union_sql = "\nUNION ALL\n".join(branches) + sql = f""" +SELECT +{_REPLAY_SNAPSHOT_COLUMNS_SQL} +FROM ( +{union_sql} +) AS replay_snapshot +ORDER BY replay_at ASC, + replay_sequence ASC +LIMIT %s +""" + parameters.append(request.max_records + 1) + return sql, tuple(parameters) + + @classmethod + def _plan_from_rows( + cls, + *, + request: ReplayPlanRequest, + rows: object, + ) -> ReplayPlan: + if not isinstance(rows, Sequence) or isinstance( + rows, + (str, bytes, bytearray), + ): + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Replay snapshot rows." + ) + + if len(rows) > request.max_records + 1: + raise MarketDataAccessIntegrityError( + "PostgreSQL exceeded the requested Replay snapshot limit." + ) + + if len(rows) == request.max_records + 1: + raise ReplayPlanLimitExceededError( + "Replay snapshot exceeds request.max_records." + ) + + events = tuple(cls._event_from_row(row) for row in rows) + + try: + return ReplayPlan(request=request, events=events) + except (TypeError, ValueError) as error: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned inconsistent Replay snapshot." + ) from error + + @staticmethod + def _event_from_row(row: object) -> ReplayEvent: + if type(row) is not tuple or len(row) != _SNAPSHOT_ROW_LENGTH: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Replay snapshot row." + ) + + ( + data_type, + replay_at, + replay_sequence, + venue, + symbol, + trade_id, + executed_at, + trade_price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + received_at, + exchange_timestamp, + last_price, + bid_price, + ask_price, + interval, + open_time, + observed_at, + open_price, + high_price, + low_price, + close_price, + volume, + is_final, + observation_sources, + canonical_schema_version, + ) = row + + if type(data_type) is not str: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned invalid Replay data type." + ) + + if data_type == ReplayDataType.TRADE.value: + record = trade_history_record_from_row( + ( + venue, + symbol, + trade_id, + executed_at, + trade_price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + observation_sources, + replay_sequence, + canonical_schema_version, + ) + ) + payload = record.trade + expected_replay_at = record.event_time + candle_is_final = None + elif data_type == ReplayDataType.QUOTE.value: + record = quote_history_record_from_row( + ( + venue, + symbol, + received_at, + exchange_timestamp, + last_price, + bid_price, + ask_price, + source, + observation_sources, + replay_sequence, + canonical_schema_version, + ) + ) + payload = record.quote + expected_replay_at = record.event_time + candle_is_final = None + elif data_type == ReplayDataType.CANDLE_REVISION.value: + record = candle_history_record_from_row( + ( + venue, + symbol, + interval, + open_time, + observed_at, + open_price, + high_price, + low_price, + close_price, + volume, + is_final, + source, + observation_sources, + replay_sequence, + canonical_schema_version, + ) + ) + payload = record.candle + expected_replay_at = record.observed_at + candle_is_final = record.is_final + else: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned unknown Replay data type." + ) + + try: + if replay_at != expected_replay_at: + raise ValueError("Replay event time differs from payload") + + if replay_sequence != record.replay_sequence: + raise ValueError("Replay sequence differs from payload") + + return ReplayEvent( + venue=record.venue, + replay_at=expected_replay_at, + replay_sequence=record.replay_sequence, + payload=payload, + candle_is_final=candle_is_final, + ) + except (TypeError, ValueError) as error: + raise MarketDataAccessIntegrityError( + "PostgreSQL returned inconsistent Replay event." + ) from error diff --git a/app/src/market_data/replay/replay_session.py b/app/src/market_data/replay/replay_session.py new file mode 100644 index 0000000..546e459 --- /dev/null +++ b/app/src/market_data/replay/replay_session.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import asyncio +import inspect +from datetime import datetime, timezone + +from src.market_data.replay.contracts import ( + MarketDataClockProtocol, + ReplayClockProtocol, + ReplayConsumerProtocol, +) +from src.market_data.replay.exceptions import ( + MarketDataReplayValidationError, + ReplaySessionStateError, +) +from src.market_data.replay.models import ( + ReplayPlan, + ReplaySessionState, +) + + +class ReplaySession: + """ + Одноразовая последовательная Session детерминированного Replay. + """ + + __slots__ = ( + "_clock", + "_consumer", + "_plan", + "_state", + ) + + def __init__( + self, + *, + plan: ReplayPlan, + clock: ReplayClockProtocol, + consumer: ReplayConsumerProtocol, + ) -> None: + if type(plan) is not ReplayPlan: + raise TypeError("plan must be a ReplayPlan") + + if isinstance(clock, type) or not isinstance( + clock, + ReplayClockProtocol, + ): + raise TypeError("clock must implement ReplayClockProtocol") + + if ( + not callable(clock.advance_to) + or inspect.iscoroutinefunction(clock.advance_to) + ): + raise TypeError("clock.advance_to must be synchronous") + + if isinstance(consumer, type) or not isinstance( + consumer, + ReplayConsumerProtocol, + ): + raise TypeError( + "consumer must implement ReplayConsumerProtocol" + ) + + if not inspect.iscoroutinefunction(consumer.consume): + raise TypeError("consumer.consume must be asynchronous") + + clock_now = clock.now + + if ( + type(clock_now) is not datetime + or clock_now.tzinfo is not timezone.utc + or clock_now.fold != 0 + ): + raise MarketDataReplayValidationError( + "Replay Session Clock must contain canonical UTC time." + ) + + if clock_now != plan.request.time_range.start_time: + raise MarketDataReplayValidationError( + "Replay Session Clock must start at the Replay request " + "start time." + ) + + self._clock = clock + self._consumer = consumer + self._plan = plan + self._state = ReplaySessionState.CREATED + + @property + def state(self) -> ReplaySessionState: + """Вернуть текущее необратимое состояние Session.""" + return self._state + + @property + def plan(self) -> ReplayPlan: + """Вернуть неизменяемый Replay Plan этой Session.""" + return self._plan + + @property + def clock(self) -> MarketDataClockProtocol: + """Предоставить Clock через границу только для чтения.""" + return self._clock + + async def run(self) -> None: + """Последовательно воспроизвести Plan ровно один раз.""" + if self._state is not ReplaySessionState.CREATED: + raise ReplaySessionStateError( + "Replay Session can only run from CREATED state." + ) + + self._state = ReplaySessionState.RUNNING + + try: + for event in self._plan.events: + self._clock.advance_to(event.replay_at) + await self._consumer.consume(event) + except asyncio.CancelledError: + self._state = ReplaySessionState.CANCELLED + raise + except BaseException: + self._state = ReplaySessionState.FAILED + raise + + self._state = ReplaySessionState.COMPLETED diff --git a/app/src/market_data/replay/replay_session_factory.py b/app/src/market_data/replay/replay_session_factory.py new file mode 100644 index 0000000..a0bd08a --- /dev/null +++ b/app/src/market_data/replay/replay_session_factory.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import inspect + +from src.market_data.replay.contracts import ( + ReplayConsumerFactoryProtocol, + ReplayPlanBuilderProtocol, +) +from src.market_data.replay.deterministic_replay_clock import ( + DeterministicReplayClock, +) +from src.market_data.replay.exceptions import ( + MarketDataReplayValidationError, +) +from src.market_data.replay.models import ( + ReplayPlan, + ReplayPlanRequest, +) +from src.market_data.replay.replay_session import ReplaySession + + +class ReplaySessionFactory: + """Явно подготовить независимый сеанс Replay без его запуска.""" + + __slots__ = ( + "_consumer_factory", + "_plan_builder", + ) + + def __init__( + self, + *, + plan_builder: ReplayPlanBuilderProtocol, + consumer_factory: ReplayConsumerFactoryProtocol, + ) -> None: + if isinstance(plan_builder, type) or not isinstance( + plan_builder, + ReplayPlanBuilderProtocol, + ): + raise TypeError( + "plan_builder must implement ReplayPlanBuilderProtocol" + ) + + if ( + not callable(plan_builder.create_plan) + or inspect.iscoroutinefunction(plan_builder.create_plan) + ): + raise TypeError("plan_builder.create_plan must be synchronous") + + if isinstance(consumer_factory, type) or not isinstance( + consumer_factory, + ReplayConsumerFactoryProtocol, + ): + raise TypeError( + "consumer_factory must implement " + "ReplayConsumerFactoryProtocol" + ) + + if ( + not callable(consumer_factory.create_consumer) + or inspect.iscoroutinefunction( + consumer_factory.create_consumer + ) + ): + raise TypeError( + "consumer_factory.create_consumer must be synchronous" + ) + + self._plan_builder = plan_builder + self._consumer_factory = consumer_factory + + def prepare_session( + self, + request: ReplayPlanRequest, + ) -> ReplaySession: + """Синхронно собрать новый граф зависимостей одного сеанса.""" + if type(request) is not ReplayPlanRequest: + raise MarketDataReplayValidationError( + "request must be ReplayPlanRequest" + ) + + plan = self._plan_builder.create_plan(request) + + if type(plan) is not ReplayPlan: + raise MarketDataReplayValidationError( + "plan_builder must return an exact ReplayPlan" + ) + + if plan.request is not request: + raise MarketDataReplayValidationError( + "Replay Plan must preserve Replay request identity" + ) + + clock = DeterministicReplayClock( + plan.request.time_range.start_time + ) + consumer = self._consumer_factory.create_consumer( + plan=plan, + clock=clock, + ) + + return ReplaySession( + plan=plan, + clock=clock, + consumer=consumer, + ) diff --git a/app/src/market_data/storage/postgres_partitions.py b/app/src/market_data/storage/postgres_partitions.py index b2c7620..b922d5d 100644 --- a/app/src/market_data/storage/postgres_partitions.py +++ b/app/src/market_data/storage/postgres_partitions.py @@ -17,9 +17,9 @@ from src.market_data.storage.postgres_repository_support import ( PostgresRepositoryConnectionProvider, normalize_aware_datetime, ) +from src.storage.migrations import MARKET_DATA_PARTITION_ADVISORY_LOCK_ID -MARKET_DATA_PARTITION_ADVISORY_LOCK_ID = 0x445A504152544E _SCHEMA_NAME = "market_data" _TRADE_CHECKPOINT_FK_NAME = "trade_stream_checkpoints_trade_fk" diff --git a/app/src/market_data/storage/postgres_trade_repository.py b/app/src/market_data/storage/postgres_trade_repository.py index a3fd9c0..c0fd5aa 100644 --- a/app/src/market_data/storage/postgres_trade_repository.py +++ b/app/src/market_data/storage/postgres_trade_repository.py @@ -61,6 +61,35 @@ ON CONFLICT (venue, symbol, trade_id, executed_at) DO NOTHING RETURNING 1 """ +_INSERT_TRADE_WITH_REPLAY_SEQUENCE_SQL = """ +INSERT INTO market_data.trades ( + venue, + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + observation_sources, + replay_sequence, + canonical_schema_version +) +VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s +) +ON CONFLICT (venue, symbol, trade_id, executed_at) DO NOTHING +RETURNING 1 +""" + +_ALLOCATE_REPLAY_SEQUENCES_SQL = """ +SELECT nextval('market_data.replay_sequence'::regclass) +FROM generate_series(1, %s) +ORDER BY 1 +""" + _SELECT_TRADE_FOR_UPDATE_SQL = """ SELECT price, @@ -632,14 +661,27 @@ class PostgresTradeRepository: try: with self._connection_provider() as connection: with connection.cursor() as cursor: - for prepared in sorted( - prepared_trades, - key=lambda item: item.identity_order_key, + replay_sequences = self._allocate_replay_sequences( + cursor=cursor, + count=len(prepared_trades), + ) + prepared_with_sequences = tuple( + zip( + prepared_trades, + replay_sequences, + strict=True, + ) + ) + + for prepared, replay_sequence in sorted( + prepared_with_sequences, + key=lambda item: item[0].identity_order_key, ): status = self._store_prepared_trade( cursor=cursor, venue=normalized_venue, trade=prepared, + replay_sequence=replay_sequence, ) if status is MarketDataWriteStatus.INSERTED: @@ -1122,25 +1164,34 @@ class PostgresTradeRepository: cursor: Any, venue: str, trade: _PreparedTrade, + replay_sequence: int | None = None, ) -> MarketDataWriteStatus: - cursor.execute( - _INSERT_TRADE_SQL, - ( - venue, - trade.symbol, - trade.trade_id, - trade.executed_at, - trade.price, - trade.quantity, - trade.aggressor_side, - trade.source, - trade.observed_at, - trade.observed_at, - [trade.source], - CANONICAL_TRADE_SCHEMA_VERSION, - ), + parameters = ( + venue, + trade.symbol, + trade.trade_id, + trade.executed_at, + trade.price, + trade.quantity, + trade.aggressor_side, + trade.source, + trade.observed_at, + trade.observed_at, + [trade.source], ) + if replay_sequence is None: + statement = _INSERT_TRADE_SQL + parameters += (CANONICAL_TRADE_SCHEMA_VERSION,) + else: + statement = _INSERT_TRADE_WITH_REPLAY_SEQUENCE_SQL + parameters += ( + replay_sequence, + CANONICAL_TRADE_SCHEMA_VERSION, + ) + + cursor.execute(statement, parameters) + if cursor.fetchone() is not None: return MarketDataWriteStatus.INSERTED @@ -1216,6 +1267,53 @@ class PostgresTradeRepository: return MarketDataWriteStatus.PROVENANCE_UPDATED + @staticmethod + def _allocate_replay_sequences( + *, + cursor: Any, + count: int, + ) -> tuple[int, ...]: + cursor.execute( + _ALLOCATE_REPLAY_SEQUENCES_SQL, + (count,), + ) + rows = cursor.fetchall() + + if not isinstance(rows, list | tuple) or len(rows) != count: + raise MarketDataStorageOperationError( + "PostgreSQL returned invalid Replay sequence allocation." + ) + + sequences: list[int] = [] + + for row in rows: + if ( + not isinstance(row, tuple) + or len(row) != 1 + or isinstance(row[0], bool) + or not isinstance(row[0], int) + or row[0] <= 0 + ): + raise MarketDataStorageOperationError( + "PostgreSQL returned invalid Replay sequence value." + ) + + sequences.append(row[0]) + + if any( + current <= previous + for previous, current in zip( + sequences, + sequences[1:], + strict=False, + ) + ): + raise MarketDataStorageOperationError( + "PostgreSQL returned unordered Replay sequence values." + ) + + return tuple(sequences) + @staticmethod def _identity_parameters( *, diff --git a/app/src/storage/migrations.py b/app/src/storage/migrations.py index 94e4fac..d693f7b 100644 --- a/app/src/storage/migrations.py +++ b/app/src/storage/migrations.py @@ -9,6 +9,7 @@ from src.storage.exceptions import StorageMigrationError STORAGE_MIGRATION_ADVISORY_LOCK_ID = 0x445A454E545241 +MARKET_DATA_PARTITION_ADVISORY_LOCK_ID = 0x445A504152544E _CREATE_HISTORY_TABLE_SQL = """ CREATE TABLE IF NOT EXISTS public.storage_schema_migrations ( @@ -330,6 +331,292 @@ STORAGE_MIGRATIONS = ( """, ), ), + StorageMigration( + version=9, + name="add_global_market_data_replay_sequence", + statements=( + ( + "SELECT pg_advisory_xact_lock(" + f"{MARKET_DATA_PARTITION_ADVISORY_LOCK_ID}" + ")" + ), + "LOCK TABLE market_data.trades IN ACCESS EXCLUSIVE MODE", + "LOCK TABLE market_data.quotes IN ACCESS EXCLUSIVE MODE", + ( + "LOCK TABLE market_data.candle_revisions " + "IN ACCESS EXCLUSIVE MODE" + ), + """ + CREATE SEQUENCE market_data.replay_sequence + AS BIGINT + INCREMENT BY 1 + MINVALUE 1 + START WITH 1 + CACHE 1 + NO CYCLE + OWNED BY NONE + """, + """ + ALTER TABLE market_data.trades + ADD COLUMN replay_sequence BIGINT + """, + """ + ALTER TABLE market_data.quotes + ADD COLUMN replay_sequence BIGINT + """, + """ + ALTER TABLE market_data.candle_revisions + ADD COLUMN replay_sequence BIGINT + """, + """ + CREATE TEMPORARY TABLE market_data_replay_sequence_backfill + ON COMMIT DROP + AS + SELECT + data_type_rank, + venue, + symbol, + trade_id, + executed_at, + received_at, + interval, + open_time, + observed_at, + ROW_NUMBER() OVER ( + ORDER BY + event_time, + data_type_rank, + venue COLLATE "C", + symbol COLLATE "C", + trade_executed_at, + trade_id, + quote_received_at, + candle_interval COLLATE "C", + candle_open_time, + candle_observed_at + ) AS replay_sequence + FROM ( + SELECT + 1::SMALLINT AS data_type_rank, + executed_at AS event_time, + venue, + symbol, + trade_id, + executed_at, + NULL::TIMESTAMPTZ AS received_at, + NULL::TEXT AS interval, + NULL::TIMESTAMPTZ AS open_time, + NULL::TIMESTAMPTZ AS observed_at, + executed_at AS trade_executed_at, + NULL::TIMESTAMPTZ AS quote_received_at, + NULL::TEXT AS candle_interval, + NULL::TIMESTAMPTZ AS candle_open_time, + NULL::TIMESTAMPTZ AS candle_observed_at + FROM market_data.trades + + UNION ALL + + SELECT + 2::SMALLINT AS data_type_rank, + received_at AS event_time, + venue, + symbol, + NULL::INTEGER AS trade_id, + NULL::TIMESTAMPTZ AS executed_at, + received_at, + NULL::TEXT AS interval, + NULL::TIMESTAMPTZ AS open_time, + NULL::TIMESTAMPTZ AS observed_at, + NULL::TIMESTAMPTZ AS trade_executed_at, + received_at AS quote_received_at, + NULL::TEXT AS candle_interval, + NULL::TIMESTAMPTZ AS candle_open_time, + NULL::TIMESTAMPTZ AS candle_observed_at + FROM market_data.quotes + + UNION ALL + + SELECT + 3::SMALLINT AS data_type_rank, + observed_at AS event_time, + venue, + symbol, + NULL::INTEGER AS trade_id, + NULL::TIMESTAMPTZ AS executed_at, + NULL::TIMESTAMPTZ AS received_at, + interval, + open_time, + observed_at, + NULL::TIMESTAMPTZ AS trade_executed_at, + NULL::TIMESTAMPTZ AS quote_received_at, + interval AS candle_interval, + open_time AS candle_open_time, + observed_at AS candle_observed_at + FROM market_data.candle_revisions + ) AS durable_rows + """, + """ + UPDATE market_data.trades AS target + SET replay_sequence = backfill.replay_sequence + FROM market_data_replay_sequence_backfill AS backfill + WHERE backfill.data_type_rank = 1 + AND target.venue = backfill.venue + AND target.symbol = backfill.symbol + AND target.trade_id = backfill.trade_id + AND target.executed_at = backfill.executed_at + """, + """ + UPDATE market_data.quotes AS target + SET replay_sequence = backfill.replay_sequence + FROM market_data_replay_sequence_backfill AS backfill + WHERE backfill.data_type_rank = 2 + AND target.venue = backfill.venue + AND target.symbol = backfill.symbol + AND target.received_at = backfill.received_at + """, + """ + UPDATE market_data.candle_revisions AS target + SET replay_sequence = backfill.replay_sequence + FROM market_data_replay_sequence_backfill AS backfill + WHERE backfill.data_type_rank = 3 + AND target.venue = backfill.venue + AND target.symbol = backfill.symbol + AND target.interval = backfill.interval + AND target.open_time = backfill.open_time + AND target.observed_at = backfill.observed_at + """, + """ + SELECT pg_catalog.setval( + 'market_data.replay_sequence'::REGCLASS, + COALESCE( + ( + SELECT MAX(replay_sequence) + FROM market_data_replay_sequence_backfill + ), + 1 + ), + EXISTS ( + SELECT 1 + FROM market_data_replay_sequence_backfill + ) + ) + """, + """ + ALTER TABLE market_data.trades + ALTER COLUMN replay_sequence + SET DEFAULT nextval( + 'market_data.replay_sequence'::REGCLASS + ) + """, + """ + ALTER TABLE market_data.quotes + ALTER COLUMN replay_sequence + SET DEFAULT nextval( + 'market_data.replay_sequence'::REGCLASS + ) + """, + """ + ALTER TABLE market_data.candle_revisions + ALTER COLUMN replay_sequence + SET DEFAULT nextval( + 'market_data.replay_sequence'::REGCLASS + ) + """, + """ + ALTER TABLE market_data.trades + ALTER COLUMN replay_sequence SET NOT NULL, + ADD CONSTRAINT trades_replay_sequence_positive + CHECK (replay_sequence > 0) + """, + """ + ALTER TABLE market_data.quotes + ALTER COLUMN replay_sequence SET NOT NULL, + ADD CONSTRAINT quotes_replay_sequence_positive + CHECK (replay_sequence > 0) + """, + """ + ALTER TABLE market_data.candle_revisions + ALTER COLUMN replay_sequence SET NOT NULL, + ADD CONSTRAINT candle_revisions_replay_sequence_positive + CHECK (replay_sequence > 0) + """, + """ + CREATE FUNCTION market_data.reject_replay_sequence_change() + RETURNS TRIGGER + LANGUAGE plpgsql + AS $function$ + BEGIN + IF NEW.replay_sequence IS DISTINCT FROM OLD.replay_sequence THEN + RAISE EXCEPTION USING + ERRCODE = '23514', + MESSAGE = 'market_data replay_sequence is immutable'; + END IF; + + RETURN NEW; + END; + $function$ + """, + """ + CREATE TRIGGER trades_replay_sequence_immutable + BEFORE UPDATE OF replay_sequence + ON market_data.trades + FOR EACH ROW + EXECUTE FUNCTION market_data.reject_replay_sequence_change() + """, + """ + CREATE TRIGGER quotes_replay_sequence_immutable + BEFORE UPDATE OF replay_sequence + ON market_data.quotes + FOR EACH ROW + EXECUTE FUNCTION market_data.reject_replay_sequence_change() + """, + """ + CREATE TRIGGER candle_revisions_replay_sequence_immutable + BEFORE UPDATE OF replay_sequence + ON market_data.candle_revisions + FOR EACH ROW + EXECUTE FUNCTION market_data.reject_replay_sequence_change() + """, + """ + CREATE INDEX trades_history_keyset_idx + ON market_data.trades ( + venue, + symbol, + executed_at, + replay_sequence + ) + """, + """ + CREATE INDEX quotes_history_keyset_idx + ON market_data.quotes ( + venue, + symbol, + received_at, + replay_sequence + ) + """, + """ + CREATE INDEX candle_revisions_history_keyset_idx + ON market_data.candle_revisions ( + venue, + symbol, + interval, + open_time, + replay_sequence + ) + """, + """ + CREATE INDEX candle_revisions_replay_keyset_idx + ON market_data.candle_revisions ( + venue, + symbol, + interval, + observed_at, + replay_sequence + ) + """, + ), + ), ) diff --git a/app/tests/integration/market_data/storage/test_postgres_historical_replay_integration.py b/app/tests/integration/market_data/storage/test_postgres_historical_replay_integration.py new file mode 100644 index 0000000..dce5344 --- /dev/null +++ b/app/tests/integration/market_data/storage/test_postgres_historical_replay_integration.py @@ -0,0 +1,312 @@ +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +import pytest + +from src.market_data.access.models import ( + CandleRevisionHistoryQuery, + HistoricalTimeRange, + QuoteHistoryQuery, +) +from src.market_data.access.postgres_candle_revision_history_repository import ( + PostgresCandleRevisionHistoryRepository, +) +from src.market_data.access.postgres_quote_history_repository import ( + PostgresQuoteHistoryRepository, +) +from src.market_data.acquisition.models.candle import Candle +from src.market_data.acquisition.models.quote import Quote +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.replay.exceptions import ReplayPlanLimitExceededError +from src.market_data.replay.models import ( + ReplayDataType, + ReplayPlanRequest, +) +from src.market_data.replay.postgres_replay_plan_builder import ( + PostgresReplayPlanBuilder, +) +from src.market_data.storage.postgres_candle_repository import ( + PostgresCandleRepository, +) +from src.market_data.storage.postgres_quote_repository import ( + PostgresQuoteRepository, +) +from src.market_data.storage.postgres_trade_repository import ( + PostgresTradeRepository, +) +from src.storage.postgres_pool import PostgresConnectionPool + + +pytestmark = pytest.mark.integration + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +START = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) +END = START + timedelta(minutes=10) + + +def _quote( + *, + received_at: datetime, + source: str = "dzengi_websocket_quote", +) -> Quote: + return Quote( + symbol=SYMBOL, + last_price=Decimal("65000.25"), + bid_price=Decimal("65000.00"), + ask_price=Decimal("65000.50"), + exchange_timestamp=received_at - timedelta(milliseconds=1), + received_at=received_at, + source=source, + ) + + +def _candle( + *, + open_time: datetime, + close_price: Decimal = Decimal("105"), + interval: str = "1m", +) -> Candle: + return Candle( + symbol=SYMBOL, + interval=interval, + open_time=open_time, + open_price=Decimal("100"), + high_price=Decimal("110"), + low_price=Decimal("90"), + close_price=close_price, + volume=Decimal("10"), + source="rest_klines:bid", + ) + + +def _trade(*, executed_at: datetime) -> Trade: + return Trade( + symbol=SYMBOL, + trade_id=100, + price=Decimal("65000.25"), + quantity=Decimal("0.001"), + executed_at=executed_at, + aggressor_side=TradeAggressorSide.BUY, + source="dzengi_websocket_trade", + ) + + +def test_real_quote_and_candle_history_use_public_axes_and_keysets( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + quote_writer = PostgresQuoteRepository( + connection_provider=migrated_postgres_pool.connection, + ) + candle_writer = PostgresCandleRepository( + connection_provider=migrated_postgres_pool.connection, + ) + quote_reader = PostgresQuoteHistoryRepository( + connection_provider=migrated_postgres_pool.connection, + ) + candle_reader = PostgresCandleRevisionHistoryRepository( + connection_provider=migrated_postgres_pool.connection, + ) + + quote_writer.store_quote( + venue=VENUE, + quote=_quote(received_at=START - timedelta(microseconds=1)), + ) + first_quote = _quote(received_at=START) + second_quote = _quote(received_at=START + timedelta(minutes=1)) + quote_writer.store_quote(venue=VENUE, quote=first_quote) + quote_writer.store_quote( + venue=VENUE, + quote=replace(first_quote, source="dzengi"), + ) + quote_writer.store_quote(venue=VENUE, quote=second_quote) + quote_writer.store_quote( + venue=VENUE, + quote=_quote(received_at=END), + ) + + first_quote_page = quote_reader.query_quotes( + QuoteHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=HistoricalTimeRange(START, END), + limit=1, + ) + ) + assert first_quote_page.next_cursor is not None + second_quote_page = quote_reader.query_quotes( + QuoteHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=HistoricalTimeRange(START, END), + limit=2, + cursor=first_quote_page.next_cursor, + ) + ) + quote_records = first_quote_page.items + second_quote_page.items + + assert [record.quote for record in quote_records] == [ + first_quote, + second_quote, + ] + assert quote_records[0].observation_sources == ( + "dzengi_websocket_quote", + "dzengi", + ) + assert second_quote_page.next_cursor is None + + candle_writer.store_candle_revision( + venue=VENUE, + candle=_candle(open_time=START), + observed_at=START + timedelta(seconds=10), + is_final=False, + ) + candle_writer.store_candle_revision( + venue=VENUE, + candle=_candle( + open_time=START, + close_price=Decimal("106"), + ), + observed_at=START + timedelta(seconds=20), + is_final=True, + ) + candle_writer.store_candle_revision( + venue=VENUE, + candle=_candle(open_time=END), + observed_at=END + timedelta(seconds=1), + is_final=True, + ) + + first_candle_page = candle_reader.query_candle_revisions( + CandleRevisionHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + interval="1m", + time_range=HistoricalTimeRange(START, END), + limit=1, + ) + ) + assert first_candle_page.next_cursor is not None + second_candle_page = candle_reader.query_candle_revisions( + CandleRevisionHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + interval="1m", + time_range=HistoricalTimeRange(START, END), + limit=2, + cursor=first_candle_page.next_cursor, + ) + ) + candle_records = first_candle_page.items + second_candle_page.items + + assert [record.observed_at for record in candle_records] == [ + START + timedelta(seconds=10), + START + timedelta(seconds=20), + ] + assert [record.is_final for record in candle_records] == [False, True] + assert second_candle_page.next_cursor is None + + +def test_real_replay_snapshot_uses_observed_candle_axis_and_global_order( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + trade_writer = PostgresTradeRepository( + connection_provider=migrated_postgres_pool.connection, + ) + quote_writer = PostgresQuoteRepository( + connection_provider=migrated_postgres_pool.connection, + ) + candle_writer = PostgresCandleRepository( + connection_provider=migrated_postgres_pool.connection, + ) + candle_reader = PostgresCandleRevisionHistoryRepository( + connection_provider=migrated_postgres_pool.connection, + ) + builder = PostgresReplayPlanBuilder( + connection_provider=migrated_postgres_pool.connection, + ) + shared_time = START + timedelta(minutes=1) + included_candle = _candle(open_time=START - timedelta(minutes=1)) + excluded_candle = _candle( + open_time=START + timedelta(minutes=3), + close_price=Decimal("107"), + ) + + trade_writer.store_trade( + venue=VENUE, + trade=_trade(executed_at=shared_time), + observed_at=shared_time, + ) + quote_writer.store_quote( + venue=VENUE, + quote=_quote(received_at=shared_time), + ) + candle_writer.store_candle_revision( + venue=VENUE, + candle=included_candle, + observed_at=START + timedelta(minutes=2), + is_final=True, + ) + candle_writer.store_candle_revision( + venue=VENUE, + candle=excluded_candle, + observed_at=END + timedelta(minutes=1), + is_final=True, + ) + + request = ReplayPlanRequest( + venue=VENUE, + symbols=(SYMBOL,), + data_types=( + ReplayDataType.TRADE, + ReplayDataType.QUOTE, + ReplayDataType.CANDLE_REVISION, + ), + time_range=HistoricalTimeRange(START, END), + candle_intervals=("1m",), + max_records=10, + ) + plan = builder.create_plan(request) + + assert [event.data_type for event in plan.events] == [ + ReplayDataType.TRADE, + ReplayDataType.QUOTE, + ReplayDataType.CANDLE_REVISION, + ] + assert [event.replay_at for event in plan.events] == [ + shared_time, + shared_time, + START + timedelta(minutes=2), + ] + assert plan.events[2].payload == included_candle + assert plan.events[2].candle_is_final is True + assert [event.order_key for event in plan.events] == sorted( + event.order_key for event in plan.events + ) + assert ( + plan.events[0].replay_sequence + < plan.events[1].replay_sequence + ) + assert len({event.replay_sequence for event in plan.events}) == 3 + + public_candles = candle_reader.query_candle_revisions( + CandleRevisionHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + interval="1m", + time_range=HistoricalTimeRange(START, END), + limit=10, + ) + ) + assert tuple( + record.candle for record in public_candles.items + ) == (excluded_candle,) + + with pytest.raises(ReplayPlanLimitExceededError): + builder.create_plan(replace(request, max_records=2)) diff --git a/app/tests/integration/market_data/storage/test_postgres_replay_runtime_integration.py b/app/tests/integration/market_data/storage/test_postgres_replay_runtime_integration.py new file mode 100644 index 0000000..043c633 --- /dev/null +++ b/app/tests/integration/market_data/storage/test_postgres_replay_runtime_integration.py @@ -0,0 +1,1035 @@ +from __future__ import annotations + +import asyncio +import threading +from concurrent.futures import ThreadPoolExecutor +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from types import TracebackType +from typing import Any, Iterator + +import psycopg +import pytest +from psycopg.conninfo import make_conninfo + +from src.market_data.access import ( + HistoricalTimeRange, + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, +) +from src.market_data.acquisition.models.candle import Candle +from src.market_data.acquisition.models.quote import Quote +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.replay import ( + MarketDataClockProtocol, + PostgresReplayPlanBuilder, + ReplayConsumerProtocol, + ReplayDataType, + ReplayEvent, + ReplayPlan, + ReplayPlanLimitExceededError, + ReplayPlanRequest, + ReplaySessionFactory, + ReplaySessionState, +) +from src.market_data.storage import ( + PostgresCandleRepository, + PostgresQuoteRepository, + PostgresTradeRepository, +) +from src.storage.exceptions import PostgresConnectionPoolError +from src.storage.postgres_pool import PostgresConnectionPool +from tests.support.postgres_market_data import ( + PostgresTestSettings, + connect_postgres_test_database, +) + + +pytestmark = pytest.mark.integration + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +START = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) +END = START + timedelta(minutes=10) + + +@dataclass(frozen=True, slots=True) +class DurableSnapshot: + trades: tuple[tuple[Any, ...], ...] + quotes: tuple[tuple[Any, ...], ...] + candles: tuple[tuple[Any, ...], ...] + checkpoints: tuple[tuple[Any, ...], ...] + replay_sequence: tuple[Any, ...] + + +class RecordingConsumer: + def __init__(self, clock: MarketDataClockProtocol) -> None: + self.clock = clock + self.events: list[ReplayEvent] = [] + self.observed_times: list[datetime] = [] + + async def consume(self, event: ReplayEvent) -> None: + self.events.append(event) + self.observed_times.append(self.clock.now) + + +class RecordingConsumerFactory: + def __init__(self) -> None: + self._lock = threading.Lock() + self.callers: list[int] = [] + self.plans: list[ReplayPlan] = [] + self.clocks: list[MarketDataClockProtocol] = [] + self.consumers: list[RecordingConsumer] = [] + + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + consumer = RecordingConsumer(clock) + + with self._lock: + self.callers.append(threading.get_ident()) + self.plans.append(plan) + self.clocks.append(clock) + self.consumers.append(consumer) + + return consumer + + +class PoolProbeConsumerFactory(RecordingConsumerFactory): + def __init__(self, pool: PostgresConnectionPool) -> None: + super().__init__() + self._pool = pool + + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + _assert_pool_reusable(self._pool) + return super().create_consumer(plan=plan, clock=clock) + + +class FailingConsumer: + def __init__( + self, + *, + clock: MarketDataClockProtocol, + error: BaseException, + fail_at: int, + ) -> None: + self.clock = clock + self.error = error + self.fail_at = fail_at + self.events: list[ReplayEvent] = [] + + async def consume(self, event: ReplayEvent) -> None: + self.events.append(event) + + if len(self.events) == self.fail_at: + raise self.error + + +class FailingConsumerFactory: + def __init__(self, *, error: BaseException, fail_at: int) -> None: + self.error = error + self.fail_at = fail_at + self.calls = 0 + self.consumer: FailingConsumer | None = None + + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + self.calls += 1 + self.consumer = FailingConsumer( + clock=clock, + error=self.error, + fail_at=self.fail_at, + ) + return self.consumer + + +class BlockingConsumer: + def __init__(self, clock: MarketDataClockProtocol) -> None: + self.clock = clock + self.entered = asyncio.Event() + self.events: list[ReplayEvent] = [] + + async def consume(self, event: ReplayEvent) -> None: + self.events.append(event) + self.entered.set() + await asyncio.Future() + + +class BlockingConsumerFactory: + def __init__(self) -> None: + self.calls = 0 + self.consumer: BlockingConsumer | None = None + + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + self.calls += 1 + self.consumer = BlockingConsumer(clock) + return self.consumer + + +class SnapshotBarrierCursor: + def __init__( + self, + cursor_context: AbstractContextManager[Any], + *, + snapshot_ready: threading.Event, + writer_committed: threading.Event, + ) -> None: + self._cursor_context = cursor_context + self._cursor: Any | None = None + self._snapshot_ready = snapshot_ready + self._writer_committed = writer_committed + + def __enter__(self) -> SnapshotBarrierCursor: + self._cursor = self._cursor_context.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool | None: + return self._cursor_context.__exit__( + exc_type, + exc_value, + traceback, + ) + + def execute( + self, + query: object, + parameters: object = None, + ) -> None: + cursor = self._required_cursor() + cursor.execute(query, parameters) + + if not str(query).strip().startswith("SET TRANSACTION"): + return + + cursor.execute("SHOW transaction_isolation") + assert cursor.fetchone() == ("repeatable read",) + cursor.execute("SHOW transaction_read_only") + assert cursor.fetchone() == ("on",) + cursor.execute("SELECT pg_current_snapshot()") + snapshot = cursor.fetchone() + assert isinstance(snapshot, tuple) and snapshot[0] is not None + self._snapshot_ready.set() + + if not self._writer_committed.wait(timeout=5.0): + raise TimeoutError("Concurrent PostgreSQL writer did not commit.") + + def fetchall(self) -> object: + return self._required_cursor().fetchall() + + def _required_cursor(self) -> Any: + if self._cursor is None: + raise RuntimeError("PostgreSQL cursor context is not entered.") + + return self._cursor + + +class SnapshotBarrierConnection: + def __init__( + self, + connection: Any, + *, + snapshot_ready: threading.Event, + writer_committed: threading.Event, + ) -> None: + self._connection = connection + self._snapshot_ready = snapshot_ready + self._writer_committed = writer_committed + + def transaction(self) -> AbstractContextManager[Any]: + return self._connection.transaction() + + def cursor(self) -> SnapshotBarrierCursor: + return SnapshotBarrierCursor( + self._connection.cursor(), + snapshot_ready=self._snapshot_ready, + writer_committed=self._writer_committed, + ) + + +class SnapshotBarrierConnectionProvider: + def __init__( + self, + pool: PostgresConnectionPool, + *, + snapshot_ready: threading.Event, + writer_committed: threading.Event, + ) -> None: + self._pool = pool + self._snapshot_ready = snapshot_ready + self._writer_committed = writer_committed + + @contextmanager + def __call__(self) -> Iterator[SnapshotBarrierConnection]: + with self._pool.connection() as connection: + yield SnapshotBarrierConnection( + connection, + snapshot_ready=self._snapshot_ready, + writer_committed=self._writer_committed, + ) + + +class ConcurrentConnectionProvider: + def __init__(self, pool: PostgresConnectionPool) -> None: + self._pool = pool + self._barrier = threading.Barrier(2) + self._lock = threading.Lock() + self.callers: list[int] = [] + + @contextmanager + def __call__(self) -> Iterator[Any]: + with self._pool.connection() as connection: + with self._lock: + self.callers.append(threading.get_ident()) + + self._barrier.wait(timeout=5.0) + yield connection + + +class TerminatingConnectionProvider: + def __init__( + self, + pool: PostgresConnectionPool, + *, + settings: PostgresTestSettings, + application_name: str, + ) -> None: + self._pool = pool + self._settings = settings + self._application_name = application_name + self._terminate_once = True + + @contextmanager + def __call__(self) -> Iterator[Any]: + with self._pool.connection() as connection: + if self._terminate_once: + self._terminate_once = False + self._terminate_backend(connection) + + yield connection + + def _terminate_backend(self, connection: Any) -> None: + backend_pid = connection.info.backend_pid + database_name = connection.info.dbname + application_name = connection.info.parameter_status( + "application_name" + ) + + if database_name != self._settings.database_name: + raise RuntimeError( + "Refusing to terminate a backend outside the test database." + ) + + if application_name != self._application_name: + raise RuntimeError( + "Refusing to terminate a backend with another application." + ) + + with connect_postgres_test_database(self._settings) as control: + with control.cursor() as cursor: + cursor.execute( + """ + SELECT pg_terminate_backend(activity.pid, %s) + FROM pg_catalog.pg_stat_activity AS activity + WHERE activity.pid = %s + AND activity.datname = %s + AND activity.application_name = %s + AND activity.backend_type = 'client backend' + AND activity.pid <> pg_backend_pid() + """, + ( + 5_000, + backend_pid, + self._settings.database_name, + self._application_name, + ), + ) + result = tuple(cursor.fetchall()) + + if result != ((True,),): + raise RuntimeError( + "The exact validated PostgreSQL test backend was not " + "terminated." + ) + + +def _trade( + *, + trade_id: int, + executed_at: datetime, +) -> Trade: + return Trade( + symbol=SYMBOL, + trade_id=trade_id, + price=Decimal("65000.25") + Decimal(trade_id) / 100, + quantity=Decimal("0.001"), + executed_at=executed_at, + aggressor_side=TradeAggressorSide.BUY, + source="dzengi_websocket_trade", + ) + + +def _quote(*, received_at: datetime) -> Quote: + return Quote( + symbol=SYMBOL, + last_price=Decimal("65000.25"), + bid_price=Decimal("65000.00"), + ask_price=Decimal("65000.50"), + exchange_timestamp=received_at - timedelta(milliseconds=1), + received_at=received_at, + source="dzengi_websocket_quote", + ) + + +def _candle(*, open_time: datetime) -> Candle: + return Candle( + symbol=SYMBOL, + interval="1m", + open_time=open_time, + open_price=Decimal("64900"), + high_price=Decimal("65100"), + low_price=Decimal("64800"), + close_price=Decimal("65000"), + volume=Decimal("10"), + source="rest_klines:bid", + ) + + +def _request( + *, + data_types: tuple[ReplayDataType, ...] = (ReplayDataType.TRADE,), + max_records: int = 100, +) -> ReplayPlanRequest: + return ReplayPlanRequest( + venue=VENUE, + symbols=(SYMBOL,), + data_types=data_types, + time_range=HistoricalTimeRange(START, END), + candle_intervals=("1m",) + if ReplayDataType.CANDLE_REVISION in data_types + else (), + max_records=max_records, + ) + + +def _store_trade( + pool: PostgresConnectionPool, + *, + trade_id: int, + executed_at: datetime, +) -> Trade: + trade = _trade(trade_id=trade_id, executed_at=executed_at) + PostgresTradeRepository( + connection_provider=pool.connection, + ).store_trade( + venue=VENUE, + trade=trade, + observed_at=executed_at + timedelta(milliseconds=1), + ) + return trade + + +def _store_mixed_snapshot( + pool: PostgresConnectionPool, +) -> tuple[Trade, Quote, Candle, datetime]: + shared_time = START + timedelta(minutes=1) + trade = _store_trade( + pool, + trade_id=101, + executed_at=shared_time, + ) + quote = _quote(received_at=shared_time) + PostgresQuoteRepository( + connection_provider=pool.connection, + ).store_quote(venue=VENUE, quote=quote) + candle = _candle(open_time=START) + PostgresCandleRepository( + connection_provider=pool.connection, + ).store_candle_revision( + venue=VENUE, + candle=candle, + observed_at=shared_time, + is_final=True, + ) + return trade, quote, candle, shared_time + + +def _factory( + pool: PostgresConnectionPool, + consumer_factory: Any, +) -> ReplaySessionFactory: + return ReplaySessionFactory( + plan_builder=PostgresReplayPlanBuilder( + connection_provider=pool.connection, + ), + consumer_factory=consumer_factory, + ) + + +def _new_single_connection_pool( + settings: PostgresTestSettings, + *, + name: str, +) -> PostgresConnectionPool: + return PostgresConnectionPool( + conninfo=settings.dsn, + min_size=1, + max_size=1, + timeout_seconds=2.0, + name=name, + ) + + +def _assert_pool_reusable(pool: PostgresConnectionPool) -> None: + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT 1") + assert cursor.fetchone() == (1,) + + +def _durable_snapshot(pool: PostgresConnectionPool) -> DurableSnapshot: + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT * + FROM market_data.trades + ORDER BY replay_sequence + """ + ) + trades = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT * + FROM market_data.quotes + ORDER BY replay_sequence + """ + ) + quotes = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT * + FROM market_data.candle_revisions + ORDER BY replay_sequence + """ + ) + candles = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT * + FROM market_data.trade_stream_checkpoints + ORDER BY venue, symbol + """ + ) + checkpoints = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT last_value, is_called + FROM market_data.replay_sequence + """ + ) + replay_sequence = cursor.fetchone() + + assert replay_sequence is not None + return DurableSnapshot( + trades=trades, + quotes=quotes, + candles=candles, + checkpoints=checkpoints, + replay_sequence=replay_sequence, + ) + + +def test_real_mixed_snapshot_composition_and_session_preserve_global_order( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + trade, quote, candle, shared_time = _store_mixed_snapshot( + migrated_postgres_pool + ) + request = _request( + data_types=( + ReplayDataType.TRADE, + ReplayDataType.QUOTE, + ReplayDataType.CANDLE_REVISION, + ) + ) + consumer_factory = RecordingConsumerFactory() + session = _factory( + migrated_postgres_pool, + consumer_factory, + ).prepare_session(request) + before_replay = _durable_snapshot(migrated_postgres_pool) + + assert session.state is ReplaySessionState.CREATED + assert [event.data_type for event in session.plan.events] == [ + ReplayDataType.TRADE, + ReplayDataType.QUOTE, + ReplayDataType.CANDLE_REVISION, + ] + assert [event.payload for event in session.plan.events] == [ + trade, + quote, + candle, + ] + assert [event.replay_at for event in session.plan.events] == [ + shared_time, + shared_time, + shared_time, + ] + assert [event.replay_sequence for event in session.plan.events] == [ + 1, + 2, + 3, + ] + assert session.plan.events[2].candle_is_final is True + + asyncio.run(session.run()) + + consumer = consumer_factory.consumers[0] + assert session.state is ReplaySessionState.COMPLETED + assert consumer.clock is session.clock + assert tuple(consumer.events) == session.plan.events + assert consumer.observed_times == [shared_time, shared_time, shared_time] + assert _durable_snapshot(migrated_postgres_pool) == before_replay + + +def test_empty_real_snapshot_completes_without_consumer_calls( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + consumer_factory = RecordingConsumerFactory() + session = _factory( + migrated_postgres_pool, + consumer_factory, + ).prepare_session(_request()) + + assert session.state is ReplaySessionState.CREATED + assert session.plan.events == () + assert len(consumer_factory.consumers) == 1 + + asyncio.run(session.run()) + + assert session.state is ReplaySessionState.COMPLETED + assert consumer_factory.consumers[0].events == [] + assert session.clock.now == START + + +def test_repeatable_read_snapshot_excludes_concurrent_commit_until_next_plan( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + first = _store_trade( + migrated_postgres_pool, + trade_id=201, + executed_at=START + timedelta(minutes=1), + ) + request = _request() + snapshot_ready = threading.Event() + writer_committed = threading.Event() + provider = SnapshotBarrierConnectionProvider( + migrated_postgres_pool, + snapshot_ready=snapshot_ready, + writer_committed=writer_committed, + ) + first_factory = ReplaySessionFactory( + plan_builder=PostgresReplayPlanBuilder( + connection_provider=provider, + ), + consumer_factory=RecordingConsumerFactory(), + ) + + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(first_factory.prepare_session, request) + + try: + assert snapshot_ready.wait(timeout=5.0) + second = _store_trade( + migrated_postgres_pool, + trade_id=202, + executed_at=START + timedelta(minutes=2), + ) + finally: + writer_committed.set() + + first_session = future.result(timeout=10.0) + + second_session = _factory( + migrated_postgres_pool, + RecordingConsumerFactory(), + ).prepare_session(request) + + assert [event.payload for event in first_session.plan.events] == [first] + assert [event.payload for event in second_session.plan.events] == [ + first, + second, + ] + + +def test_single_connection_pool_is_released_before_consumer_and_playback( + migrated_postgres_pool: PostgresConnectionPool, + postgres_test_settings: PostgresTestSettings, +) -> None: + _store_trade( + migrated_postgres_pool, + trade_id=301, + executed_at=START + timedelta(minutes=1), + ) + pool = _new_single_connection_pool( + postgres_test_settings, + name="replay-resource-release", + ) + pool.open() + consumer_factory = PoolProbeConsumerFactory(pool) + factory = _factory(pool, consumer_factory) + + try: + session = factory.prepare_session(_request()) + pool.close() + asyncio.run(session.run()) + + assert session.state is ReplaySessionState.COMPLETED + assert len(consumer_factory.consumers[0].events) == 1 + + pool.open() + repeated = factory.prepare_session(_request()) + assert repeated is not session + assert repeated.plan is not session.plan + assert repeated.clock is not session.clock + _assert_pool_reusable(pool) + finally: + pool.close() + + +def test_limit_failure_creates_no_consumer_and_releases_connection( + migrated_postgres_pool: PostgresConnectionPool, + postgres_test_settings: PostgresTestSettings, +) -> None: + _store_trade( + migrated_postgres_pool, + trade_id=401, + executed_at=START + timedelta(minutes=1), + ) + _store_trade( + migrated_postgres_pool, + trade_id=402, + executed_at=START + timedelta(minutes=2), + ) + pool = _new_single_connection_pool( + postgres_test_settings, + name="replay-limit-failure", + ) + pool.open() + consumer_factory = RecordingConsumerFactory() + factory = _factory(pool, consumer_factory) + + try: + with pytest.raises(ReplayPlanLimitExceededError): + factory.prepare_session(_request(max_records=1)) + + assert consumer_factory.consumers == [] + _assert_pool_reusable(pool) + finally: + pool.close() + + +def test_integrity_failure_creates_no_consumer_and_clean_retry_succeeds( + migrated_postgres_pool: PostgresConnectionPool, + postgres_test_settings: PostgresTestSettings, +) -> None: + trade = _store_trade( + migrated_postgres_pool, + trade_id=501, + executed_at=START + timedelta(minutes=1), + ) + pool = _new_single_connection_pool( + postgres_test_settings, + name="replay-integrity-failure", + ) + pool.open() + consumer_factory = RecordingConsumerFactory() + factory = _factory(pool, consumer_factory) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + UPDATE market_data.trades + SET canonical_schema_version = 2 + WHERE venue = %s + AND symbol = %s + AND trade_id = %s + AND executed_at = %s + """, + (VENUE, SYMBOL, trade.trade_id, trade.executed_at), + ) + + try: + with pytest.raises(MarketDataAccessIntegrityError): + factory.prepare_session(_request()) + + assert consumer_factory.consumers == [] + _assert_pool_reusable(pool) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + UPDATE market_data.trades + SET canonical_schema_version = 1 + WHERE venue = %s + AND symbol = %s + AND trade_id = %s + AND executed_at = %s + """, + (VENUE, SYMBOL, trade.trade_id, trade.executed_at), + ) + + session = factory.prepare_session(_request()) + assert [event.payload for event in session.plan.events] == [trade] + assert len(consumer_factory.consumers) == 1 + finally: + pool.close() + + +def test_closed_pool_failure_is_wrapped_and_same_factory_recovers( + migrated_postgres_pool: PostgresConnectionPool, + postgres_test_settings: PostgresTestSettings, +) -> None: + _store_trade( + migrated_postgres_pool, + trade_id=601, + executed_at=START + timedelta(minutes=1), + ) + pool = _new_single_connection_pool( + postgres_test_settings, + name="replay-closed-provider", + ) + pool.open() + pool.close() + consumer_factory = RecordingConsumerFactory() + factory = _factory(pool, consumer_factory) + + with pytest.raises(MarketDataAccessOperationError) as error_info: + factory.prepare_session(_request()) + + assert isinstance(error_info.value.__cause__, PostgresConnectionPoolError) + assert consumer_factory.consumers == [] + + try: + pool.open() + session = factory.prepare_session(_request()) + assert len(session.plan.events) == 1 + assert len(consumer_factory.consumers) == 1 + finally: + pool.close() + + +def test_consumer_failure_after_real_snapshot_keeps_database_detached( + migrated_postgres_pool: PostgresConnectionPool, + postgres_test_settings: PostgresTestSettings, +) -> None: + for offset in range(3): + _store_trade( + migrated_postgres_pool, + trade_id=701 + offset, + executed_at=START + timedelta(minutes=offset + 1), + ) + + before = _durable_snapshot(migrated_postgres_pool) + error = RuntimeError("consumer failed") + consumer_factory = FailingConsumerFactory(error=error, fail_at=2) + pool = _new_single_connection_pool( + postgres_test_settings, + name="replay-consumer-failure", + ) + pool.open() + + try: + session = _factory(pool, consumer_factory).prepare_session( + _request() + ) + pool.close() + + with pytest.raises(RuntimeError) as error_info: + asyncio.run(session.run()) + + assert error_info.value is error + assert session.state is ReplaySessionState.FAILED + assert consumer_factory.consumer is not None + assert len(consumer_factory.consumer.events) == 2 + assert session.clock.now == session.plan.events[1].replay_at + finally: + pool.close() + + assert _durable_snapshot(migrated_postgres_pool) == before + + +def test_consumer_cancellation_after_real_snapshot_leaves_no_owned_task( + migrated_postgres_pool: PostgresConnectionPool, + postgres_test_settings: PostgresTestSettings, +) -> None: + for offset in range(2): + _store_trade( + migrated_postgres_pool, + trade_id=801 + offset, + executed_at=START + timedelta(minutes=offset + 1), + ) + + before = _durable_snapshot(migrated_postgres_pool) + consumer_factory = BlockingConsumerFactory() + pool = _new_single_connection_pool( + postgres_test_settings, + name="replay-consumer-cancellation", + ) + pool.open() + + try: + session = _factory(pool, consumer_factory).prepare_session( + _request() + ) + pool.close() + + async def scenario() -> None: + consumer = consumer_factory.consumer + assert consumer is not None + task = asyncio.create_task( + session.run(), + name="test-replay-cancellation", + ) + await asyncio.wait_for(consumer.entered.wait(), timeout=2.0) + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + + assert task.done() + + asyncio.run(scenario()) + + assert session.state is ReplaySessionState.CANCELLED + assert consumer_factory.consumer is not None + assert len(consumer_factory.consumer.events) == 1 + assert session.clock.now == session.plan.events[0].replay_at + finally: + pool.close() + + assert _durable_snapshot(migrated_postgres_pool) == before + + +def test_two_real_concurrent_preparations_use_independent_graphs( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + _store_mixed_snapshot(migrated_postgres_pool) + request = _request( + data_types=( + ReplayDataType.TRADE, + ReplayDataType.QUOTE, + ReplayDataType.CANDLE_REVISION, + ) + ) + provider = ConcurrentConnectionProvider(migrated_postgres_pool) + consumer_factory = RecordingConsumerFactory() + factory = ReplaySessionFactory( + plan_builder=PostgresReplayPlanBuilder( + connection_provider=provider, + ), + consumer_factory=consumer_factory, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [ + executor.submit(factory.prepare_session, request) + for _ in range(2) + ] + sessions = [future.result(timeout=10.0) for future in futures] + + assert len(set(provider.callers)) == 2 + assert len(set(consumer_factory.callers)) == 2 + assert sessions[0] is not sessions[1] + assert sessions[0].plan is not sessions[1].plan + assert sessions[0].plan == sessions[1].plan + assert sessions[0].clock is not sessions[1].clock + assert consumer_factory.consumers[0] is not ( + consumer_factory.consumers[1] + ) + assert {id(clock) for clock in consumer_factory.clocks} == { + id(session.clock) for session in sessions + } + assert all( + session.state is ReplaySessionState.CREATED + for session in sessions + ) + _assert_pool_reusable(migrated_postgres_pool) + + +def test_real_backend_failure_is_wrapped_and_pool_replaces_connection( + migrated_postgres_pool: PostgresConnectionPool, + postgres_test_settings: PostgresTestSettings, +) -> None: + _store_trade( + migrated_postgres_pool, + trade_id=901, + executed_at=START + timedelta(minutes=1), + ) + application_name = "dzentra-replay-backend-failure" + pool = PostgresConnectionPool( + conninfo=make_conninfo( + postgres_test_settings.dsn, + application_name=application_name, + ), + min_size=1, + max_size=1, + timeout_seconds=5.0, + name="replay-terminated-backend", + ) + pool.open() + provider = TerminatingConnectionProvider( + pool, + settings=postgres_test_settings, + application_name=application_name, + ) + consumer_factory = RecordingConsumerFactory() + factory = ReplaySessionFactory( + plan_builder=PostgresReplayPlanBuilder( + connection_provider=provider, + ), + consumer_factory=consumer_factory, + ) + + try: + with pytest.raises(MarketDataAccessOperationError) as error_info: + factory.prepare_session(_request()) + + assert isinstance(error_info.value.__cause__, psycopg.Error) + assert consumer_factory.consumers == [] + + session = factory.prepare_session(_request()) + assert len(session.plan.events) == 1 + assert len(consumer_factory.consumers) == 1 + _assert_pool_reusable(pool) + finally: + pool.close() diff --git a/app/tests/integration/market_data/storage/test_postgres_replay_sequence_migration_integration.py b/app/tests/integration/market_data/storage/test_postgres_replay_sequence_migration_integration.py new file mode 100644 index 0000000..009b79c --- /dev/null +++ b/app/tests/integration/market_data/storage/test_postgres_replay_sequence_migration_integration.py @@ -0,0 +1,1097 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import threading +from typing import Any + +import pytest +from psycopg import sql +from psycopg.errors import CheckViolation + +from src.market_data.storage import ( + MarketDataPartitionType, + PostgresMarketDataPartitionManager, +) +from src.storage.exceptions import StorageMigrationError +from src.storage.migrations import ( + MARKET_DATA_PARTITION_ADVISORY_LOCK_ID, + STORAGE_MIGRATIONS, + StorageMigration, + StorageMigrationRunner, +) +from src.storage.postgres_pool import PostgresConnectionPool +from tests.support.postgres_market_data import ( + PostgresTestSettings, + connect_postgres_test_database, + wait_for_postgres_advisory_lock_waiters, + wait_for_postgres_relation_lock_waiters, +) + + +pytestmark = pytest.mark.integration + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" + + +def _open_pool( + settings: PostgresTestSettings, + *, + name: str, +) -> PostgresConnectionPool: + pool = PostgresConnectionPool( + conninfo=settings.dsn, + min_size=1, + max_size=4, + timeout_seconds=5.0, + name=name, + ) + pool.open() + return pool + + +def _apply_base_migrations(pool: PostgresConnectionPool) -> None: + result = StorageMigrationRunner( + connection_provider=pool.connection, + migrations=STORAGE_MIGRATIONS[:8], + ).run() + + assert result == tuple(range(1, 9)) + + +def _insert_trade( + cursor: Any, + *, + trade_id: int, + executed_at: datetime, + venue: str = VENUE, + symbol: str = SYMBOL, + return_sequence: bool = False, +) -> int | None: + returning = " RETURNING replay_sequence" if return_sequence else "" + cursor.execute( + """ + INSERT INTO market_data.trades ( + venue, + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + observation_sources + ) + VALUES ( + %s, %s, %s, %s, 100, 1, 'buy', + 'dzengi_websocket_trade', %s, %s, + ARRAY['dzengi_websocket_trade'] + ) + """ + + returning, + ( + venue, + symbol, + trade_id, + executed_at, + executed_at, + executed_at, + ), + ) + + if not return_sequence: + return None + + row = cursor.fetchone() + assert isinstance(row, tuple) + assert len(row) == 1 + assert isinstance(row[0], int) + return row[0] + + +def _insert_quote( + cursor: Any, + *, + received_at: datetime, + venue: str = VENUE, + symbol: str = SYMBOL, + return_sequence: bool = False, +) -> int | None: + returning = " RETURNING replay_sequence" if return_sequence else "" + cursor.execute( + """ + INSERT INTO market_data.quotes ( + venue, + symbol, + received_at, + exchange_timestamp, + last_price, + bid_price, + ask_price, + source, + observation_sources + ) + VALUES ( + %s, %s, %s, %s, 100, 99, 101, + 'dzengi', ARRAY['dzengi'] + ) + """ + + returning, + ( + venue, + symbol, + received_at, + received_at, + ), + ) + + if not return_sequence: + return None + + row = cursor.fetchone() + assert isinstance(row, tuple) + assert len(row) == 1 + assert isinstance(row[0], int) + return row[0] + + +def _insert_candle( + cursor: Any, + *, + interval: str, + open_time: datetime, + observed_at: datetime, + venue: str = VENUE, + symbol: str = SYMBOL, + return_sequence: bool = False, +) -> int | None: + returning = " RETURNING replay_sequence" if return_sequence else "" + cursor.execute( + """ + INSERT INTO market_data.candle_revisions ( + venue, + symbol, + interval, + open_time, + observed_at, + open_price, + high_price, + low_price, + close_price, + volume, + is_final, + source, + observation_sources + ) + VALUES ( + %s, %s, %s, %s, %s, + 100, 101, 99, 100, 1, TRUE, + 'dzengi', ARRAY['dzengi'] + ) + """ + + returning, + ( + venue, + symbol, + interval, + open_time, + observed_at, + ), + ) + + if not return_sequence: + return None + + row = cursor.fetchone() + assert isinstance(row, tuple) + assert len(row) == 1 + assert isinstance(row[0], int) + return row[0] + + +def test_real_empty_migration_uses_one_unowned_global_allocator( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT + pg_catalog.format_type(sequence.seqtypid, NULL), + sequence.seqstart, + sequence.seqincrement, + sequence.seqmin, + sequence.seqcycle, + sequence.seqcache + FROM pg_catalog.pg_sequence AS sequence + WHERE sequence.seqrelid = + 'market_data.replay_sequence'::REGCLASS + """ + ) + sequence_definition = cursor.fetchone() + cursor.execute( + """ + SELECT COUNT(*) + FROM pg_catalog.pg_depend AS dependency + WHERE dependency.classid = 'pg_catalog.pg_class'::REGCLASS + AND dependency.objid = + 'market_data.replay_sequence'::REGCLASS + AND dependency.deptype = 'a' + """ + ) + sequence_owner_count = cursor.fetchone() + cursor.execute( + """ + SELECT table_name, data_type, is_nullable, column_default + FROM information_schema.columns + WHERE table_schema = 'market_data' + AND column_name = 'replay_sequence' + AND table_name IN ( + 'trades', + 'trades_default', + 'quotes', + 'quotes_default', + 'candle_revisions', + 'candle_revisions_default' + ) + ORDER BY table_name + """ + ) + replay_columns = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT table_row.relname, constraint_row.conname + FROM pg_catalog.pg_constraint AS constraint_row + JOIN pg_catalog.pg_class AS table_row + ON table_row.oid = constraint_row.conrelid + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = table_row.relnamespace + WHERE namespace.nspname = 'market_data' + AND constraint_row.conname IN ( + 'trades_replay_sequence_positive', + 'quotes_replay_sequence_positive', + 'candle_revisions_replay_sequence_positive' + ) + ORDER BY table_row.relname + """ + ) + replay_constraints = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT table_row.relname, trigger_row.tgname + FROM pg_catalog.pg_trigger AS trigger_row + JOIN pg_catalog.pg_class AS table_row + ON table_row.oid = trigger_row.tgrelid + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = table_row.relnamespace + WHERE namespace.nspname = 'market_data' + AND NOT trigger_row.tgisinternal + ORDER BY table_row.relname + """ + ) + replay_triggers = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT table_row.relname + FROM pg_catalog.pg_class AS table_row + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = table_row.relnamespace + WHERE namespace.nspname = 'market_data' + AND table_row.relkind IN ('r', 'p') + AND table_row.relname ILIKE '%replay%' + ORDER BY table_row.relname + """ + ) + replay_tables = tuple(cursor.fetchall()) + + first = _insert_trade( + cursor, + trade_id=1, + executed_at=datetime(2026, 8, 2, 1, tzinfo=timezone.utc), + return_sequence=True, + ) + second = _insert_quote( + cursor, + received_at=datetime(2026, 8, 2, 2, tzinfo=timezone.utc), + return_sequence=True, + ) + third = _insert_candle( + cursor, + interval="1m", + open_time=datetime(2026, 8, 2, 3, tzinfo=timezone.utc), + observed_at=datetime(2026, 8, 2, 3, 1, tzinfo=timezone.utc), + return_sequence=True, + ) + + assert sequence_definition == ("bigint", 1, 1, 1, False, 1) + assert sequence_owner_count == (0,) + assert len(replay_columns) == 6 + assert all( + data_type == "bigint" + and is_nullable == "NO" + and column_default + == "nextval('market_data.replay_sequence'::regclass)" + for _, data_type, is_nullable, column_default in replay_columns + ) + assert replay_constraints == ( + ( + "candle_revisions", + "candle_revisions_replay_sequence_positive", + ), + ( + "candle_revisions_default", + "candle_revisions_replay_sequence_positive", + ), + ("quotes", "quotes_replay_sequence_positive"), + ("quotes_default", "quotes_replay_sequence_positive"), + ("trades", "trades_replay_sequence_positive"), + ("trades_default", "trades_replay_sequence_positive"), + ) + assert replay_triggers == ( + ( + "candle_revisions", + "candle_revisions_replay_sequence_immutable", + ), + ( + "candle_revisions_default", + "candle_revisions_replay_sequence_immutable", + ), + ("quotes", "quotes_replay_sequence_immutable"), + ("quotes_default", "quotes_replay_sequence_immutable"), + ("trades", "trades_replay_sequence_immutable"), + ("trades_default", "trades_replay_sequence_immutable"), + ) + assert replay_tables == () + assert (first, second, third) == (1, 2, 3) + + +def test_real_backfill_uses_exact_deterministic_cross_type_order( + postgres_test_settings: PostgresTestSettings, +) -> None: + pool = _open_pool( + postgres_test_settings, + name="replay-backfill", + ) + event_time = datetime(2026, 8, 2, 10, tzinfo=timezone.utc) + later_time = datetime(2026, 8, 2, 11, tzinfo=timezone.utc) + candle_open_time = datetime(2026, 8, 2, 9, tzinfo=timezone.utc) + + try: + _apply_base_migrations(pool) + + with pool.connection() as connection: + with connection.cursor() as cursor: + _insert_trade( + cursor, + trade_id=10, + executed_at=later_time, + ) + _insert_candle( + cursor, + interval="1m", + open_time=candle_open_time, + observed_at=event_time, + ) + _insert_quote( + cursor, + venue="a", + received_at=event_time, + ) + _insert_trade( + cursor, + trade_id=2_147_483_647, + executed_at=event_time, + ) + _insert_candle( + cursor, + interval="1D", + open_time=candle_open_time, + observed_at=event_time, + ) + _insert_quote( + cursor, + venue="Z", + received_at=event_time, + ) + _insert_trade( + cursor, + trade_id=-2_147_483_648, + executed_at=event_time, + ) + + result = StorageMigrationRunner( + connection_provider=pool.connection, + ).run() + + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT data_type, venue, durable_identity, replay_sequence + FROM ( + SELECT + 'trade'::TEXT AS data_type, + venue, + trade_id::TEXT AS durable_identity, + replay_sequence + FROM market_data.trades + + UNION ALL + + SELECT + 'quote'::TEXT AS data_type, + venue, + received_at::TEXT AS durable_identity, + replay_sequence + FROM market_data.quotes + + UNION ALL + + SELECT + 'candle'::TEXT AS data_type, + venue, + interval AS durable_identity, + replay_sequence + FROM market_data.candle_revisions + ) AS replay_rows + ORDER BY replay_sequence + """ + ) + replay_rows = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT last_value, is_called + FROM market_data.replay_sequence + """ + ) + sequence_state = cursor.fetchone() + cursor.execute( + """ + SELECT replay_sequence + FROM market_data.trades + WHERE trade_id = -2147483648 + """ + ) + original_sequence = cursor.fetchone() + cursor.execute( + """ + UPDATE market_data.trades + SET observation_sources = ARRAY[ + 'dzengi_websocket_trade', + 'dzengi' + ], + last_observed_at = %s + WHERE trade_id = -2147483648 + """, + (later_time,), + ) + cursor.execute( + """ + SELECT replay_sequence + FROM market_data.trades + WHERE trade_id = -2147483648 + """ + ) + after_provenance_update = cursor.fetchone() + next_values = ( + _insert_quote( + cursor, + received_at=datetime( + 2026, + 8, + 2, + 12, + tzinfo=timezone.utc, + ), + return_sequence=True, + ), + _insert_trade( + cursor, + trade_id=11, + executed_at=datetime( + 2026, + 8, + 2, + 13, + tzinfo=timezone.utc, + ), + return_sequence=True, + ), + _insert_candle( + cursor, + interval="5m", + open_time=datetime( + 2026, + 8, + 2, + 14, + tzinfo=timezone.utc, + ), + observed_at=datetime( + 2026, + 8, + 2, + 14, + 1, + tzinfo=timezone.utc, + ), + return_sequence=True, + ), + ) + finally: + pool.close() + + assert result == (9,) + assert replay_rows == ( + ("trade", VENUE, "-2147483648", 1), + ("trade", VENUE, "2147483647", 2), + ("quote", "Z", "2026-08-02 10:00:00+00", 3), + ("quote", "a", "2026-08-02 10:00:00+00", 4), + ("candle", VENUE, "1D", 5), + ("candle", VENUE, "1m", 6), + ("trade", VENUE, "10", 7), + ) + assert sequence_state == (7, True) + assert original_sequence == (1,) + assert after_provenance_update == original_sequence + assert next_values == (8, 9, 10) + + +def _assert_replay_sequence_update_is_rejected( + pool: PostgresConnectionPool, + *, + table_name: str, +) -> None: + with pytest.raises(CheckViolation): + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + sql.SQL( + "UPDATE {} " + "SET replay_sequence = replay_sequence + 1000" + ).format( + sql.Identifier("market_data", table_name), + ) + ) + + +def test_real_existing_and_future_partitions_inherit_replay_contract( + postgres_test_settings: PostgresTestSettings, +) -> None: + pool = _open_pool( + postgres_test_settings, + name="replay-partitions", + ) + existing_time = datetime(2026, 1, 15, 10, tzinfo=timezone.utc) + future_time = datetime(2027, 2, 15, 10, tzinfo=timezone.utc) + default_time = datetime(2029, 3, 15, 10, tzinfo=timezone.utc) + data_types = tuple(MarketDataPartitionType) + + try: + _apply_base_migrations(pool) + manager = PostgresMarketDataPartitionManager( + connection_provider=pool.connection, + ) + existing_partitions = tuple( + manager.ensure_month_partition( + data_type=data_type, + month=existing_time, + ).partition.partition_name + for data_type in data_types + ) + + assert StorageMigrationRunner( + connection_provider=pool.connection, + ).run() == (9,) + + future_partitions = tuple( + manager.ensure_month_partition( + data_type=data_type, + month=future_time, + ).partition.partition_name + for data_type in data_types + ) + + with pool.connection() as connection: + with connection.cursor() as cursor: + generated_sequences = ( + _insert_trade( + cursor, + trade_id=101, + executed_at=existing_time, + return_sequence=True, + ), + _insert_trade( + cursor, + trade_id=102, + executed_at=future_time, + return_sequence=True, + ), + _insert_trade( + cursor, + trade_id=103, + executed_at=default_time, + return_sequence=True, + ), + _insert_quote( + cursor, + received_at=existing_time, + return_sequence=True, + ), + _insert_quote( + cursor, + received_at=future_time, + return_sequence=True, + ), + _insert_quote( + cursor, + received_at=default_time, + return_sequence=True, + ), + _insert_candle( + cursor, + interval="1m", + open_time=existing_time, + observed_at=existing_time, + return_sequence=True, + ), + _insert_candle( + cursor, + interval="1m", + open_time=future_time, + observed_at=future_time, + return_sequence=True, + ), + _insert_candle( + cursor, + interval="1m", + open_time=default_time, + observed_at=default_time, + return_sequence=True, + ), + ) + cursor.execute( + """ + SELECT table_name, data_type, is_nullable, column_default + FROM information_schema.columns + WHERE table_schema = 'market_data' + AND column_name = 'replay_sequence' + ORDER BY table_name + """ + ) + replay_columns = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT table_row.relname, constraint_row.conname + FROM pg_catalog.pg_constraint AS constraint_row + JOIN pg_catalog.pg_class AS table_row + ON table_row.oid = constraint_row.conrelid + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = table_row.relnamespace + WHERE namespace.nspname = 'market_data' + AND constraint_row.conname IN ( + 'trades_replay_sequence_positive', + 'quotes_replay_sequence_positive', + 'candle_revisions_replay_sequence_positive' + ) + ORDER BY table_row.relname + """ + ) + replay_constraints = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT table_row.relname, trigger_row.tgname + FROM pg_catalog.pg_trigger AS trigger_row + JOIN pg_catalog.pg_class AS table_row + ON table_row.oid = trigger_row.tgrelid + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = table_row.relnamespace + WHERE namespace.nspname = 'market_data' + AND NOT trigger_row.tgisinternal + ORDER BY table_row.relname + """ + ) + replay_triggers = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT parent_index.relname, COUNT(*) + FROM pg_catalog.pg_inherits AS inheritance + JOIN pg_catalog.pg_class AS parent_index + ON parent_index.oid = inheritance.inhparent + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = parent_index.relnamespace + WHERE namespace.nspname = 'market_data' + AND parent_index.relname IN ( + 'trades_history_keyset_idx', + 'quotes_history_keyset_idx', + 'candle_revisions_history_keyset_idx', + 'candle_revisions_replay_keyset_idx' + ) + GROUP BY parent_index.relname + ORDER BY parent_index.relname + """ + ) + attached_index_counts = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT tableoid::REGCLASS::TEXT + FROM market_data.trades + ORDER BY trade_id + """ + ) + trade_relations = tuple(row[0] for row in cursor.fetchall()) + cursor.execute( + """ + SELECT tableoid::REGCLASS::TEXT + FROM market_data.quotes + ORDER BY received_at + """ + ) + quote_relations = tuple(row[0] for row in cursor.fetchall()) + cursor.execute( + """ + SELECT tableoid::REGCLASS::TEXT + FROM market_data.candle_revisions + ORDER BY open_time + """ + ) + candle_relations = tuple( + row[0] for row in cursor.fetchall() + ) + + parent_tables = ( + "trades", + "quotes", + "candle_revisions", + ) + default_partitions = ( + "trades_default", + "quotes_default", + "candle_revisions_default", + ) + expected_relations = set( + parent_tables + + default_partitions + + existing_partitions + + future_partitions + ) + + assert generated_sequences == tuple(range(1, 10)) + assert {row[0] for row in replay_columns} == expected_relations + assert all( + data_type == "bigint" + and is_nullable == "NO" + and column_default + == "nextval('market_data.replay_sequence'::regclass)" + for _, data_type, is_nullable, column_default in replay_columns + ) + assert {row[0] for row in replay_constraints} == expected_relations + assert {row[0] for row in replay_triggers} == expected_relations + assert attached_index_counts == ( + ("candle_revisions_history_keyset_idx", 3), + ("candle_revisions_replay_keyset_idx", 3), + ("quotes_history_keyset_idx", 3), + ("trades_history_keyset_idx", 3), + ) + assert trade_relations == ( + f"market_data.{existing_partitions[0]}", + f"market_data.{future_partitions[0]}", + "market_data.trades_default", + ) + assert quote_relations == ( + f"market_data.{existing_partitions[1]}", + f"market_data.{future_partitions[1]}", + "market_data.quotes_default", + ) + assert candle_relations == ( + f"market_data.{existing_partitions[2]}", + f"market_data.{future_partitions[2]}", + "market_data.candle_revisions_default", + ) + + for table_name in tuple(expected_relations): + _assert_replay_sequence_update_is_rejected( + pool, + table_name=table_name, + ) + finally: + pool.close() + + +def test_real_migration_waits_for_writer_and_backfills_committed_row( + postgres_test_settings: PostgresTestSettings, +) -> None: + pool = _open_pool( + postgres_test_settings, + name="replay-writer-blocking", + ) + writer = connect_postgres_test_database( + postgres_test_settings, + autocommit=False, + ) + migration_thread_ids: set[int] = set() + + try: + _apply_base_migrations(pool) + event_time = datetime(2026, 8, 2, 15, tzinfo=timezone.utc) + + with writer.cursor() as cursor: + _insert_trade( + cursor, + trade_id=500, + executed_at=event_time, + ) + + runner = StorageMigrationRunner( + connection_provider=pool.connection, + ) + + def run_migration() -> tuple[int, ...]: + migration_thread_ids.add(threading.get_ident()) + return runner.run() + + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(run_migration) + + try: + wait_for_postgres_relation_lock_waiters( + writer, + relation_name="market_data.trades", + expected_count=1, + ) + finally: + writer.commit() + + migration_result = future.result(timeout=10.0) + + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT replay_sequence + FROM market_data.trades + WHERE venue = %s + AND symbol = %s + AND trade_id = 500 + AND executed_at = %s + """, + (VENUE, SYMBOL, event_time), + ) + stored_sequence = cursor.fetchone() + finally: + writer.close() + pool.close() + + assert len(migration_thread_ids) == 1 + assert migration_result == (9,) + assert stored_sequence == (1,) + + +def test_real_migration_and_partition_manager_share_one_lock( + postgres_test_settings: PostgresTestSettings, +) -> None: + pool = _open_pool( + postgres_test_settings, + name="replay-partition-race", + ) + control = connect_postgres_test_database( + postgres_test_settings, + autocommit=False, + ) + caller_thread_ids: set[int] = set() + caller_thread_ids_lock = threading.Lock() + start_barrier = threading.Barrier(3) + + try: + _apply_base_migrations(pool) + manager = PostgresMarketDataPartitionManager( + connection_provider=pool.connection, + ) + runner = StorageMigrationRunner( + connection_provider=pool.connection, + ) + + with control.cursor() as cursor: + cursor.execute( + "SELECT pg_advisory_xact_lock(%s)", + (MARKET_DATA_PARTITION_ADVISORY_LOCK_ID,), + ) + + def record_caller() -> None: + with caller_thread_ids_lock: + caller_thread_ids.add(threading.get_ident()) + start_barrier.wait(timeout=5.0) + + def run_migration() -> tuple[int, ...]: + record_caller() + return runner.run() + + def create_partition() -> str: + record_caller() + result = manager.ensure_month_partition( + data_type=MarketDataPartitionType.QUOTES, + month=datetime(2028, 4, 1, tzinfo=timezone.utc), + ) + assert result.created is True + return result.partition.partition_name + + with ThreadPoolExecutor(max_workers=2) as executor: + migration_future = executor.submit(run_migration) + partition_future = executor.submit(create_partition) + start_barrier.wait(timeout=5.0) + + try: + wait_for_postgres_advisory_lock_waiters( + control, + lock_id=MARKET_DATA_PARTITION_ADVISORY_LOCK_ID, + expected_count=2, + ) + finally: + control.commit() + + migration_result = migration_future.result(timeout=10.0) + partition_name = partition_future.result(timeout=10.0) + + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT data_type, is_nullable, column_default + FROM information_schema.columns + WHERE table_schema = 'market_data' + AND table_name = %s + AND column_name = 'replay_sequence' + """, + (partition_name,), + ) + replay_column = cursor.fetchone() + cursor.execute( + """ + SELECT trigger_row.tgname + FROM pg_catalog.pg_trigger AS trigger_row + JOIN pg_catalog.pg_class AS table_row + ON table_row.oid = trigger_row.tgrelid + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = table_row.relnamespace + WHERE namespace.nspname = 'market_data' + AND table_row.relname = %s + AND NOT trigger_row.tgisinternal + """, + (partition_name,), + ) + replay_trigger = cursor.fetchone() + cursor.execute( + """ + SELECT parent_index.relname + FROM pg_catalog.pg_inherits AS inheritance + JOIN pg_catalog.pg_class AS child_index + ON child_index.oid = inheritance.inhrelid + JOIN pg_catalog.pg_class AS parent_index + ON parent_index.oid = inheritance.inhparent + JOIN pg_catalog.pg_index AS child_index_state + ON child_index_state.indexrelid = child_index.oid + JOIN pg_catalog.pg_class AS child_table + ON child_table.oid = child_index_state.indrelid + WHERE child_table.relname = %s + AND parent_index.relname = + 'quotes_history_keyset_idx' + """, + (partition_name,), + ) + replay_index = cursor.fetchone() + finally: + control.close() + pool.close() + + assert len(caller_thread_ids) == 2 + assert migration_result == (9,) + assert replay_column == ( + "bigint", + "NO", + "nextval('market_data.replay_sequence'::regclass)", + ) + assert replay_trigger == ("quotes_replay_sequence_immutable",) + assert replay_index == ("quotes_history_keyset_idx",) + + +def test_real_failed_migration_rolls_back_and_clean_retry_succeeds( + postgres_test_settings: PostgresTestSettings, +) -> None: + pool = _open_pool( + postgres_test_settings, + name="replay-migration-rollback", + ) + event_time = datetime(2026, 8, 2, 16, tzinfo=timezone.utc) + + try: + _apply_base_migrations(pool) + + with pool.connection() as connection: + with connection.cursor() as cursor: + _insert_trade( + cursor, + trade_id=600, + executed_at=event_time, + ) + + migration = STORAGE_MIGRATIONS[8] + broken_migration = StorageMigration( + version=migration.version, + name=migration.name, + statements=(*migration.statements, "SELECT 1 / 0"), + ) + broken_runner = StorageMigrationRunner( + connection_provider=pool.connection, + migrations=(*STORAGE_MIGRATIONS[:8], broken_migration), + ) + + with pytest.raises(StorageMigrationError): + broken_runner.run() + + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT + pg_catalog.to_regclass( + 'market_data.replay_sequence' + ), + EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'market_data' + AND table_name = 'trades' + AND column_name = 'replay_sequence' + ), + EXISTS ( + SELECT 1 + FROM public.storage_schema_migrations + WHERE version = 9 + ) + """ + ) + rolled_back_state = cursor.fetchone() + + clean_result = StorageMigrationRunner( + connection_provider=pool.connection, + ).run() + + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT replay_sequence + FROM market_data.trades + WHERE trade_id = 600 + """ + ) + stored_sequence = cursor.fetchone() + finally: + pool.close() + + assert rolled_back_state == (None, False, False) + assert clean_result == (9,) + assert stored_sequence == (1,) diff --git a/app/tests/integration/market_data/storage/test_postgres_trade_history_integration.py b/app/tests/integration/market_data/storage/test_postgres_trade_history_integration.py new file mode 100644 index 0000000..6f4ce28 --- /dev/null +++ b/app/tests/integration/market_data/storage/test_postgres_trade_history_integration.py @@ -0,0 +1,340 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from decimal import Decimal +import threading + +import pytest + +from src.market_data.access import ( + HistoricalTimeRange, + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, + PostgresTradeHistoryRepository, + TradeHistoryCursor, + TradeHistoryQuery, +) +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.acquisition.trade_id_sequence import ( + SIGNED_TRADE_ID_MAX, + SIGNED_TRADE_ID_MIN, +) +from src.market_data.storage import ( + MarketDataStorageConflictError, + MarketDataWriteStatus, + PostgresTradeRepository, +) +from src.storage.exceptions import PostgresConnectionPoolError +from src.storage.postgres_pool import PostgresConnectionPool +from tests.support.postgres_market_data import PostgresTestSettings + + +pytestmark = pytest.mark.integration + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +START = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) +END = START + timedelta(hours=1) +SOURCE = "dzengi_websocket_trade" + + +def _trade( + *, + trade_id: int, + executed_at: datetime = START, + source: str = SOURCE, +) -> Trade: + return Trade( + symbol=SYMBOL, + trade_id=trade_id, + price=Decimal("65000.25"), + quantity=Decimal("0.001"), + executed_at=executed_at, + aggressor_side=TradeAggressorSide.BUY, + source=source, + ) + + +def _query( + *, + limit: int, + cursor: TradeHistoryCursor | None = None, +) -> TradeHistoryQuery: + return TradeHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=HistoricalTimeRange(START, END), + limit=limit, + cursor=cursor, + ) + + +def test_real_trade_history_uses_half_open_rollover_keyset_order( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + writer = PostgresTradeRepository( + connection_provider=migrated_postgres_pool.connection, + ) + reader = PostgresTradeHistoryRepository( + connection_provider=migrated_postgres_pool.connection, + ) + rollover_ids = ( + SIGNED_TRADE_ID_MAX, + SIGNED_TRADE_ID_MIN, + -1, + 0, + ) + + writer.store_trade( + venue=VENUE, + trade=_trade( + trade_id=10, + executed_at=START - timedelta(microseconds=1), + ), + observed_at=START, + ) + writer.store_trades( + venue=VENUE, + trades=tuple( + _trade(trade_id=trade_id) + for trade_id in rollover_ids + ), + observed_at=START + timedelta(seconds=1), + ) + writer.store_trade( + venue=VENUE, + trade=_trade(trade_id=11, executed_at=END), + observed_at=END + timedelta(seconds=1), + ) + + first_page = reader.query_trades(_query(limit=2)) + + assert [record.trade.trade_id for record in first_page.items] == [ + SIGNED_TRADE_ID_MAX, + SIGNED_TRADE_ID_MIN, + ] + assert first_page.next_cursor is not None + assert ( + first_page.next_cursor.executed_at, + first_page.next_cursor.replay_sequence, + ) == first_page.items[-1].order_key + + second_page = reader.query_trades( + _query( + limit=3, + cursor=first_page.next_cursor, + ) + ) + records = first_page.items + second_page.items + + assert [record.trade.trade_id for record in records] == list( + rollover_ids + ) + assert [record.event_time for record in records] == [START] * 4 + assert [record.replay_sequence for record in records] == sorted( + record.replay_sequence for record in records + ) + assert len({record.replay_sequence for record in records}) == 4 + assert second_page.next_cursor is None + + +def test_real_trade_history_preserves_ordinal_and_returns_provenance( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + writer = PostgresTradeRepository( + connection_provider=migrated_postgres_pool.connection, + ) + reader = PostgresTradeHistoryRepository( + connection_provider=migrated_postgres_pool.connection, + ) + original = _trade(trade_id=100) + first_observed_at = START + timedelta(seconds=1) + last_observed_at = START + timedelta(seconds=5) + + inserted = writer.store_trade( + venue=VENUE, + trade=original, + observed_at=first_observed_at, + ) + original_record = reader.query_trades(_query(limit=10)).items[0] + duplicate = writer.store_trade( + venue=VENUE, + trade=original, + observed_at=first_observed_at, + ) + provenance = writer.store_trade( + venue=VENUE, + trade=replace(original, source="dzengi"), + observed_at=last_observed_at, + ) + updated_record = reader.query_trades(_query(limit=10)).items[0] + + assert inserted.status is MarketDataWriteStatus.INSERTED + assert duplicate.status is MarketDataWriteStatus.DUPLICATE + assert provenance.status is MarketDataWriteStatus.PROVENANCE_UPDATED + assert updated_record.replay_sequence == original_record.replay_sequence + assert updated_record.trade == original + assert updated_record.first_observed_at == first_observed_at + assert updated_record.last_observed_at == last_observed_at + assert updated_record.observation_sources == ( + SOURCE, + "dzengi", + ) + + +def test_real_trade_history_omits_rolled_back_batch_and_keeps_gap( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + writer = PostgresTradeRepository( + connection_provider=migrated_postgres_pool.connection, + ) + reader = PostgresTradeHistoryRepository( + connection_provider=migrated_postgres_pool.connection, + ) + existing = _trade(trade_id=2) + writer.store_trade( + venue=VENUE, + trade=existing, + observed_at=START + timedelta(seconds=1), + ) + + with pytest.raises(MarketDataStorageConflictError): + writer.store_trades( + venue=VENUE, + trades=( + _trade(trade_id=1), + replace(existing, price=Decimal("999")), + ), + observed_at=START + timedelta(seconds=2), + ) + + writer.store_trade( + venue=VENUE, + trade=_trade(trade_id=3), + observed_at=START + timedelta(seconds=3), + ) + records = reader.query_trades(_query(limit=10)).items + + assert [record.trade.trade_id for record in records] == [2, 3] + assert [record.replay_sequence for record in records] == [1, 4] + + +def test_two_real_batches_keep_local_order_and_global_unique_sequences( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + writer = PostgresTradeRepository( + connection_provider=migrated_postgres_pool.connection, + ) + reader = PostgresTradeHistoryRepository( + connection_provider=migrated_postgres_pool.connection, + ) + batches = ( + ( + _trade(trade_id=SIGNED_TRADE_ID_MAX), + _trade(trade_id=SIGNED_TRADE_ID_MIN), + ), + ( + _trade(trade_id=-1), + _trade(trade_id=0), + ), + ) + start_barrier = threading.Barrier(2) + caller_ids: set[int] = set() + caller_ids_lock = threading.Lock() + + def store_batch(trades: tuple[Trade, ...]) -> int: + with caller_ids_lock: + caller_ids.add(threading.get_ident()) + start_barrier.wait(timeout=5.0) + return writer.store_trades( + venue=VENUE, + trades=trades, + observed_at=START + timedelta(seconds=1), + ).inserted_count + + with ThreadPoolExecutor(max_workers=2) as executor: + inserted_counts = tuple(executor.map(store_batch, batches)) + + records = reader.query_trades(_query(limit=10)).items + sequence_by_trade_id = { + record.trade.trade_id: record.replay_sequence + for record in records + } + + assert inserted_counts == (2, 2) + assert len(caller_ids) == 2 + assert len(records) == 4 + assert len(set(sequence_by_trade_id.values())) == 4 + assert ( + sequence_by_trade_id[SIGNED_TRADE_ID_MAX] + < sequence_by_trade_id[SIGNED_TRADE_ID_MIN] + ) + assert sequence_by_trade_id[-1] < sequence_by_trade_id[0] + + +def test_real_trade_history_rejects_corrupted_schema_version( + migrated_postgres_pool: PostgresConnectionPool, +) -> None: + writer = PostgresTradeRepository( + connection_provider=migrated_postgres_pool.connection, + ) + reader = PostgresTradeHistoryRepository( + connection_provider=migrated_postgres_pool.connection, + ) + trade = _trade(trade_id=200) + writer.store_trade( + venue=VENUE, + trade=trade, + observed_at=START + timedelta(seconds=1), + ) + + with migrated_postgres_pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + UPDATE market_data.trades + SET canonical_schema_version = 2 + WHERE venue = %s + AND symbol = %s + AND trade_id = %s + AND executed_at = %s + """, + ( + VENUE, + SYMBOL, + trade.trade_id, + trade.executed_at, + ), + ) + + with pytest.raises(MarketDataAccessIntegrityError): + reader.query_trades(_query(limit=10)) + + +def test_real_trade_history_wraps_closed_pool_provider_failure( + migrated_postgres_pool: PostgresConnectionPool, + postgres_test_settings: PostgresTestSettings, +) -> None: + assert migrated_postgres_pool.is_open + closed_pool = PostgresConnectionPool( + conninfo=postgres_test_settings.dsn, + min_size=1, + max_size=1, + timeout_seconds=5.0, + name="trade-history-closed-provider", + ) + closed_pool.open() + closed_pool.close() + reader = PostgresTradeHistoryRepository( + connection_provider=closed_pool.connection, + ) + + with pytest.raises(MarketDataAccessOperationError) as error_info: + reader.query_trades(_query(limit=10)) + + assert isinstance(error_info.value.__cause__, PostgresConnectionPoolError) diff --git a/app/tests/integration/market_data/storage/test_trade_runtime_replay_integration.py b/app/tests/integration/market_data/storage/test_trade_runtime_replay_integration.py new file mode 100644 index 0000000..50388ee --- /dev/null +++ b/app/tests/integration/market_data/storage/test_trade_runtime_replay_integration.py @@ -0,0 +1,413 @@ +from __future__ import annotations + +import asyncio +import time +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any + +import pytest + +from src.market_data.access import ( + HistoricalTimeRange, + PostgresTradeHistoryRepository, + TradeHistoryQuery, + TradeHistoryRecord, +) +from src.market_data.acquisition.models.trade import Trade +from src.market_data.acquisition.trade_id_sequence import ( + SIGNED_TRADE_ID_MAX, + SIGNED_TRADE_ID_MIN, +) +from src.market_data.replay import ( + MarketDataClockProtocol, + PostgresReplayPlanBuilder, + ReplayConsumerProtocol, + ReplayDataType, + ReplayEvent, + ReplayPlan, + ReplayPlanRequest, + ReplaySession, + ReplaySessionFactory, + ReplaySessionState, +) +from src.market_data.storage import ( + PostgresTradeRepository, + TradeStorageObservationSink, +) +from src.storage.postgres_pool import PostgresConnectionPool +from tests.integration.market_data.acquisition.runtime.loopback_trade_exchange import ( + LoopbackTradeEnvironment, + LoopbackTradeRestServer, + LoopbackTradeWebSocketServer, + wait_until, +) +from tests.support.postgres_market_data import ( + PostgresTestSettings, + connect_postgres_test_database, + count_other_test_connections, +) +from tests.support.trade_stream_runtime import ( + SYMBOL, + assert_no_owned_tasks, + build_runtime, + run_scenario, + start_runtime, + state_store_from, + stop_runtime, +) + + +pytestmark = pytest.mark.integration + +VENUE = "dzengi" + + +@dataclass(frozen=True, slots=True) +class DatabaseSnapshot: + trades: tuple[tuple[Any, ...], ...] + checkpoints: tuple[tuple[Any, ...], ...] + replay_sequence: tuple[Any, ...] + + +class RecordingReplayConsumer: + def __init__(self, clock: MarketDataClockProtocol) -> None: + self.clock = clock + self.events: list[ReplayEvent] = [] + self.observed_times: list[datetime] = [] + + async def consume(self, event: ReplayEvent) -> None: + self.events.append(event) + self.observed_times.append(self.clock.now) + + +class RecordingReplayConsumerFactory: + def __init__(self) -> None: + self.plans: list[ReplayPlan] = [] + self.clocks: list[MarketDataClockProtocol] = [] + self.consumers: list[RecordingReplayConsumer] = [] + + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + consumer = RecordingReplayConsumer(clock) + self.plans.append(plan) + self.clocks.append(clock) + self.consumers.append(consumer) + return consumer + + +def _trade_sink( + pool: PostgresConnectionPool, +) -> TradeStorageObservationSink: + return TradeStorageObservationSink( + trade_storage=PostgresTradeRepository( + connection_provider=pool.connection, + ), + venue=VENUE, + ) + + +def _database_snapshot( + pool: PostgresConnectionPool, +) -> DatabaseSnapshot: + with pool.connection() as connection: + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT + venue, + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + observation_sources, + replay_sequence, + canonical_schema_version + FROM market_data.trades + ORDER BY replay_sequence + """ + ) + trades = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT + venue, + symbol, + trade_id, + executed_at, + revision, + checkpoint_schema_version + FROM market_data.trade_stream_checkpoints + ORDER BY venue, symbol + """ + ) + checkpoints = tuple(cursor.fetchall()) + cursor.execute( + """ + SELECT last_value, is_called + FROM market_data.replay_sequence + """ + ) + replay_sequence = cursor.fetchone() + + assert replay_sequence is not None + + return DatabaseSnapshot( + trades=trades, + checkpoints=checkpoints, + replay_sequence=replay_sequence, + ) + + +def _wait_for_no_other_postgres_connections( + settings: PostgresTestSettings, + *, + timeout_seconds: float = 2.0, +) -> None: + deadline = time.monotonic() + timeout_seconds + + with connect_postgres_test_database(settings) as control: + while True: + observed_count = count_other_test_connections(control) + + if observed_count == 0: + return + + if time.monotonic() >= deadline: + raise AssertionError( + "PostgreSQL test connections were not released; " + f"observed {observed_count}." + ) + + time.sleep(0.01) + + +def _prepare_history_and_replay( + *, + pool: PostgresConnectionPool, + request: ReplayPlanRequest, + consumer_factory: RecordingReplayConsumerFactory, +) -> tuple[ + tuple[TradeHistoryRecord, ...], + ReplaySession, + DatabaseSnapshot, +]: + history = PostgresTradeHistoryRepository( + connection_provider=pool.connection, + ).query_trades( + TradeHistoryQuery( + venue=request.venue, + symbol=request.symbols[0], + time_range=request.time_range, + limit=request.max_records, + ) + ) + session = ReplaySessionFactory( + plan_builder=PostgresReplayPlanBuilder( + connection_provider=pool.connection, + ), + consumer_factory=consumer_factory, + ).prepare_session(request) + return history.items, session, _database_snapshot(pool) + + +def test_loopback_runtime_storage_history_and_replay_are_one_read_only_path( + migrated_postgres_pool: PostgresConnectionPool, + postgres_test_settings: PostgresTestSettings, +) -> None: + async def scenario() -> None: + websocket = LoopbackTradeWebSocketServer() + rest = LoopbackTradeRestServer() + start_timestamp_ms = time.time_ns() // 1_000_000 + end_timestamp_ms = start_timestamp_ms + 1 + + async with LoopbackTradeEnvironment( + websocket=websocket, + rest=rest, + ) as environment: + runtime = build_runtime( + websocket_url=environment.websocket_url, + rest_base_url=rest.base_url, + trade_observation_sink=_trade_sink( + migrated_postgres_pool + ), + ) + runtime_task: asyncio.Task[None] | None = None + + try: + runtime_task = await start_runtime(runtime) + await websocket.wait_for_subscriptions(1) + await websocket.send_trade( + 0, + symbol=SYMBOL, + trade_id=SIGNED_TRADE_ID_MAX - 1, + timestamp_ms=start_timestamp_ms - 1, + price="64555.54", + quantity="0.001", + ) + state_store = state_store_from(runtime) + await wait_until( + lambda: ( + state_store.contains(SYMBOL) + and state_store.get(SYMBOL).last_trade_id + == SIGNED_TRADE_ID_MAX - 1 + ) + ) + await websocket.send_trade( + 0, + symbol=SYMBOL, + trade_id=SIGNED_TRADE_ID_MAX, + timestamp_ms=start_timestamp_ms, + price="64555.55", + quantity="0.002", + ) + await wait_until( + lambda: state_store.get(SYMBOL).last_trade_id + == SIGNED_TRADE_ID_MAX + ) + await websocket.send_trade( + 0, + symbol=SYMBOL, + trade_id=SIGNED_TRADE_ID_MIN, + timestamp_ms=start_timestamp_ms, + price="64555.56", + quantity="0.003", + ) + await wait_until( + lambda: state_store.get(SYMBOL).last_trade_id + == SIGNED_TRADE_ID_MIN + ) + await websocket.send_trade( + 0, + symbol=SYMBOL, + trade_id=SIGNED_TRADE_ID_MIN + 1, + timestamp_ms=end_timestamp_ms, + price="64555.57", + quantity="0.004", + ) + await wait_until( + lambda: state_store.get(SYMBOL).last_trade_id + == SIGNED_TRADE_ID_MIN + 1 + ) + finally: + if runtime_task is not None: + await stop_runtime(runtime, runtime_task) + + assert websocket.active_handler_count == 0 + assert rest.thread_is_alive is False + + request = ReplayPlanRequest( + venue=VENUE, + symbols=(SYMBOL,), + data_types=(ReplayDataType.TRADE,), + time_range=HistoricalTimeRange( + start_time=datetime.fromtimestamp( + start_timestamp_ms / 1_000, + tz=timezone.utc, + ), + end_time=datetime.fromtimestamp( + end_timestamp_ms / 1_000, + tz=timezone.utc, + ), + ), + max_records=10, + ) + consumer_factory = RecordingReplayConsumerFactory() + history, session, before_replay = await asyncio.to_thread( + _prepare_history_and_replay, + pool=migrated_postgres_pool, + request=request, + consumer_factory=consumer_factory, + ) + + assert session.state is ReplaySessionState.CREATED + assert [record.trade.trade_id for record in history] == [ + SIGNED_TRADE_ID_MAX, + SIGNED_TRADE_ID_MIN, + ] + assert [record.trade.price for record in history] == [ + Decimal("64555.55"), + Decimal("64555.56"), + ] + assert [record.trade.quantity for record in history] == [ + Decimal("0.002"), + Decimal("0.003"), + ] + replayed_trades: list[Trade] = [] + for event in session.plan.events: + payload = event.payload + + if not isinstance(payload, Trade): + raise AssertionError( + "Replay event must contain Trade payload." + ) + + replayed_trades.append(payload) + assert replayed_trades == [record.trade for record in history] + assert [ + (event.payload, event.replay_sequence) + for event in session.plan.events + ] == [ + (record.trade, record.replay_sequence) + for record in history + ] + assert consumer_factory.plans == [session.plan] + assert consumer_factory.plans[0] is session.plan + assert consumer_factory.clocks == [session.clock] + assert before_replay.checkpoints == ( + ( + VENUE, + SYMBOL, + SIGNED_TRADE_ID_MIN + 1, + request.time_range.end_time, + 4, + 1, + ), + ) + + migrated_postgres_pool.close() + + try: + _wait_for_no_other_postgres_connections( + postgres_test_settings + ) + + await session.run() + finally: + migrated_postgres_pool.open() + + consumer = consumer_factory.consumers[0] + assert session.state is ReplaySessionState.COMPLETED + assert consumer.clock is session.clock + assert tuple(consumer.events) == session.plan.events + assert all( + actual is expected + for actual, expected in zip( + consumer.events, + session.plan.events, + strict=True, + ) + ) + assert consumer.observed_times == [ + event.replay_at for event in session.plan.events + ] + assert session.clock.now == session.plan.events[-1].replay_at + + after_replay = await asyncio.to_thread( + _database_snapshot, + migrated_postgres_pool, + ) + assert after_replay == before_replay + await assert_no_owned_tasks() + + run_scenario(scenario()) diff --git a/app/tests/static/test_python_type_gate.py b/app/tests/static/test_python_type_gate.py new file mode 100644 index 0000000..0cacdf8 --- /dev/null +++ b/app/tests/static/test_python_type_gate.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[3] + + +def test_python_type_gate_is_clean() -> None: + result = subprocess.run( + ( + sys.executable, + "-m", + "pyright", + "--project", + str(PROJECT_ROOT / "pyrightconfig.json"), + "--warnings", + ), + cwd=PROJECT_ROOT, + capture_output=True, + text=True, + check=False, + timeout=30.0, + ) + + output = "\n".join( + part.strip() + for part in (result.stdout, result.stderr) + if part.strip() + ) + + assert result.returncode == 0, output diff --git a/app/tests/unit/market_data/access/test_market_data_access_contracts.py b/app/tests/unit/market_data/access/test_market_data_access_contracts.py new file mode 100644 index 0000000..2b99c23 --- /dev/null +++ b/app/tests/unit/market_data/access/test_market_data_access_contracts.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from src.market_data.access import ( + CandleRevisionHistoryPage, + CandleRevisionHistoryQuery, + CandleRevisionHistoryReaderProtocol, + MarketDataAccessError, + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, + MarketDataAccessValidationError, + MarketDataCursorError, + MarketDataHistoricalAccessProtocol, + QuoteHistoryPage, + QuoteHistoryQuery, + QuoteHistoryReaderProtocol, + TradeHistoryPage, + TradeHistoryQuery, + TradeHistoryReaderProtocol, +) + + +class RecordingHistoricalAccess: + def query_trades( + self, + query: TradeHistoryQuery, + ) -> TradeHistoryPage: + return TradeHistoryPage(query=query, items=()) + + def query_quotes( + self, + query: QuoteHistoryQuery, + ) -> QuoteHistoryPage: + return QuoteHistoryPage(query=query, items=()) + + def query_candle_revisions( + self, + query: CandleRevisionHistoryQuery, + ) -> CandleRevisionHistoryPage: + return CandleRevisionHistoryPage(query=query, items=()) + + +def test_history_reader_protocols_are_runtime_checkable() -> None: + access = RecordingHistoricalAccess() + + assert isinstance(access, TradeHistoryReaderProtocol) + assert isinstance(access, QuoteHistoryReaderProtocol) + assert isinstance(access, CandleRevisionHistoryReaderProtocol) + assert isinstance(access, MarketDataHistoricalAccessProtocol) + + +def test_access_error_hierarchy_is_specialized() -> None: + assert issubclass(MarketDataAccessValidationError, MarketDataAccessError) + assert issubclass(MarketDataCursorError, MarketDataAccessValidationError) + assert issubclass(MarketDataAccessIntegrityError, MarketDataAccessError) + assert issubclass(MarketDataAccessOperationError, MarketDataAccessError) diff --git a/app/tests/unit/market_data/access/test_market_data_access_models.py b/app/tests/unit/market_data/access/test_market_data_access_models.py new file mode 100644 index 0000000..2f72f2e --- /dev/null +++ b/app/tests/unit/market_data/access/test_market_data_access_models.py @@ -0,0 +1,882 @@ +from __future__ import annotations + +from dataclasses import FrozenInstanceError, replace +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any + +import pytest + +from src.market_data.access import ( + HISTORY_PAGE_LIMIT_MAX, + CandleRevisionHistoryCursor, + CandleRevisionHistoryPage, + CandleRevisionHistoryQuery, + CandleRevisionHistoryRecord, + HistoricalTimeRange, + QuoteHistoryCursor, + QuoteHistoryPage, + QuoteHistoryQuery, + QuoteHistoryRecord, + TradeHistoryCursor, + TradeHistoryPage, + TradeHistoryQuery, + TradeHistoryRecord, +) +from src.market_data.acquisition.models.candle import Candle +from src.market_data.acquisition.models.quote import Quote +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) + + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +START = datetime(2026, 8, 2, 10, 0, tzinfo=timezone.utc) +END = START + timedelta(hours=1) +SOURCE = "dzengi_websocket_trade" + + +def make_trade( + *, + trade_id: int = 100, + executed_at: datetime = START + timedelta(minutes=1), + symbol: str = SYMBOL, + source: str = SOURCE, +) -> Trade: + return Trade( + symbol=symbol, + trade_id=trade_id, + price=Decimal("65000.25"), + quantity=Decimal("0.001"), + executed_at=executed_at, + aggressor_side=TradeAggressorSide.BUY, + source=source, + ) + + +def make_quote( + *, + received_at: datetime = START + timedelta(minutes=2), +) -> Quote: + return Quote( + symbol=SYMBOL, + last_price=Decimal("65000"), + bid_price=Decimal("64999"), + ask_price=Decimal("65001"), + exchange_timestamp=received_at - timedelta(milliseconds=1), + received_at=received_at, + source="dzengi_rest_quote", + ) + + +def make_candle( + *, + open_time: datetime = START, + interval: str = "1m", +) -> Candle: + return Candle( + symbol=SYMBOL, + interval=interval, + open_time=open_time, + open_price=Decimal("64900"), + high_price=Decimal("65100"), + low_price=Decimal("64800"), + close_price=Decimal("65000"), + volume=Decimal("12.5"), + source="dzengi_rest_candle", + ) + + +def make_range() -> HistoricalTimeRange: + return HistoricalTimeRange(start_time=START, end_time=END) + + +def make_trade_query( + *, + cursor: TradeHistoryCursor | None = None, + limit: int = 500, +) -> TradeHistoryQuery: + return TradeHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + limit=limit, + cursor=cursor, + ) + + +def make_trade_record( + *, + trade_id: int = 100, + executed_at: datetime = START + timedelta(minutes=1), + replay_sequence: int = 10, +) -> TradeHistoryRecord: + return TradeHistoryRecord( + venue=VENUE, + trade=make_trade( + trade_id=trade_id, + executed_at=executed_at, + ), + first_observed_at=executed_at + timedelta(seconds=1), + last_observed_at=executed_at + timedelta(seconds=2), + observation_sources=(SOURCE,), + replay_sequence=replay_sequence, + ) + + +def make_quote_record( + *, + received_at: datetime = START + timedelta(minutes=2), + replay_sequence: int = 20, +) -> QuoteHistoryRecord: + quote = make_quote(received_at=received_at) + return QuoteHistoryRecord( + venue=VENUE, + quote=quote, + observation_sources=(quote.source,), + replay_sequence=replay_sequence, + ) + + +def make_candle_record( + *, + open_time: datetime = START, + observed_at: datetime = START + timedelta(seconds=30), + replay_sequence: int = 30, + interval: str = "1m", +) -> CandleRevisionHistoryRecord: + candle = make_candle(open_time=open_time, interval=interval) + return CandleRevisionHistoryRecord( + venue=VENUE, + candle=candle, + observed_at=observed_at, + is_final=False, + observation_sources=(candle.source,), + replay_sequence=replay_sequence, + ) + + +def test_time_range_is_half_open_and_normalized_to_utc() -> None: + offset = timezone(timedelta(hours=3)) + time_range = HistoricalTimeRange( + start_time=START.astimezone(offset), + end_time=END.astimezone(offset), + ) + + assert time_range.start_time == START + assert time_range.start_time.tzinfo is timezone.utc + assert time_range.end_time == END + assert time_range.contains(START) + assert time_range.contains(END - timedelta(microseconds=1)) + assert not time_range.contains(END) + + +@pytest.mark.parametrize( + ("start_time", "end_time", "error_type"), + ( + (datetime(2026, 8, 2, 10, 0), END, ValueError), + (START, datetime(2026, 8, 2, 11, 0), ValueError), + (START, START, ValueError), + (END, START, ValueError), + ("2026-08-02", END, TypeError), + ), +) +def test_time_range_rejects_invalid_boundaries( + start_time: Any, + end_time: Any, + error_type: type[Exception], +) -> None: + with pytest.raises(error_type): + HistoricalTimeRange( + start_time=start_time, + end_time=end_time, + ) + + +def test_history_records_preserve_exact_canonical_payloads() -> None: + trade = make_trade() + quote = make_quote() + candle = make_candle() + + trade_record = TradeHistoryRecord( + venue=" dzengi ", + trade=trade, + first_observed_at=trade.executed_at, + last_observed_at=trade.executed_at, + observation_sources=(trade.source,), + replay_sequence=1, + ) + quote_record = QuoteHistoryRecord( + venue=VENUE, + quote=quote, + observation_sources=(quote.source,), + replay_sequence=2, + ) + candle_record = CandleRevisionHistoryRecord( + venue=VENUE, + candle=candle, + observed_at=candle.open_time, + is_final=True, + observation_sources=(candle.source,), + replay_sequence=3, + ) + + assert trade_record.venue == VENUE + assert trade_record.trade is trade + assert quote_record.quote is quote + assert candle_record.candle is candle + assert trade_record.event_time == trade.executed_at + assert quote_record.event_time == quote.received_at + assert candle_record.event_time == candle.open_time + assert candle_record.replay_at == candle.open_time + + +@pytest.mark.parametrize("invalid_sequence", (True, 0, -1, 1.5, "1")) +def test_records_reject_invalid_replay_sequence( + invalid_sequence: Any, +) -> None: + with pytest.raises((TypeError, ValueError), match="replay_sequence"): + make_trade_record(replay_sequence=invalid_sequence) + + +@pytest.mark.parametrize( + "observation_sources", + ( + [], + (), + ("",), + (SOURCE, SOURCE), + ("another_source",), + ), +) +def test_trade_record_rejects_invalid_provenance( + observation_sources: Any, +) -> None: + trade = make_trade() + + with pytest.raises((TypeError, ValueError)): + TradeHistoryRecord( + venue=VENUE, + trade=trade, + first_observed_at=trade.executed_at, + last_observed_at=trade.executed_at, + observation_sources=observation_sources, + replay_sequence=1, + ) + + +def test_trade_record_rejects_reversed_observation_times() -> None: + trade = make_trade() + + with pytest.raises(ValueError, match="last_observed_at"): + TradeHistoryRecord( + venue=VENUE, + trade=trade, + first_observed_at=trade.executed_at + timedelta(seconds=1), + last_observed_at=trade.executed_at, + observation_sources=(trade.source,), + replay_sequence=1, + ) + + +def test_candle_record_uses_open_time_for_history_and_observed_for_replay() -> None: + record = make_candle_record() + + assert record.event_time == START + assert record.replay_at == START + timedelta(seconds=30) + assert record.interval == "1m" + + +def test_candle_record_rejects_observation_before_open_time() -> None: + with pytest.raises(ValueError, match="observed_at"): + make_candle_record( + observed_at=START - timedelta(microseconds=1), + ) + + +@pytest.mark.parametrize("is_final", (0, 1, None, "true")) +def test_candle_record_requires_exact_boolean(is_final: Any) -> None: + candle = make_candle() + + with pytest.raises(TypeError, match="is_final"): + CandleRevisionHistoryRecord( + venue=VENUE, + candle=candle, + observed_at=candle.open_time, + is_final=is_final, + observation_sources=(candle.source,), + replay_sequence=1, + ) + + +def test_trade_cursor_normalizes_scope_and_time() -> None: + offset = timezone(timedelta(hours=3)) + cursor = TradeHistoryCursor( + venue=" dzengi ", + symbol=" btc/usd_leverage ", + time_range=make_range(), + executed_at=(START + timedelta(minutes=1)).astimezone(offset), + replay_sequence=5, + ) + + assert cursor.venue == VENUE + assert cursor.symbol == SYMBOL + assert cursor.executed_at.tzinfo is timezone.utc + + +@pytest.mark.parametrize( + "cursor_time", + ( + START - timedelta(microseconds=1), + END, + END + timedelta(microseconds=1), + ), +) +def test_cursor_position_must_belong_to_query_range( + cursor_time: datetime, +) -> None: + with pytest.raises(ValueError, match="query range"): + TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + executed_at=cursor_time, + replay_sequence=1, + ) + + +def test_cursor_rejects_unknown_version() -> None: + with pytest.raises(ValueError, match="version"): + TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + executed_at=START, + replay_sequence=1, + version=2, + ) + + +@pytest.mark.parametrize("limit", (1, HISTORY_PAGE_LIMIT_MAX)) +def test_trade_query_accepts_limit_boundaries(limit: int) -> None: + query = TradeHistoryQuery( + venue=" dzengi ", + symbol="btc/usd_leverage", + time_range=make_range(), + limit=limit, + ) + + assert query.venue == VENUE + assert query.symbol == SYMBOL + assert query.limit == limit + + +@pytest.mark.parametrize("limit", (True, 0, -1, 1.5, 1001)) +def test_query_rejects_invalid_limit(limit: Any) -> None: + with pytest.raises((TypeError, ValueError), match="limit"): + TradeHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + limit=limit, + ) + + +def test_query_accepts_cursor_when_only_page_limit_changes() -> None: + time_range = make_range() + cursor = TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=time_range, + executed_at=START, + replay_sequence=1, + ) + + query = TradeHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=time_range, + limit=17, + cursor=cursor, + ) + + assert query.cursor is cursor + + +def test_query_rejects_cursor_from_another_scope() -> None: + cursor = TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + executed_at=START, + replay_sequence=1, + ) + + with pytest.raises(ValueError, match="scope"): + TradeHistoryQuery( + venue=VENUE, + symbol="ETH/USD_LEVERAGE", + time_range=make_range(), + cursor=cursor, + ) + + +def test_query_rejects_cursor_of_another_data_type() -> None: + quote_cursor = QuoteHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + received_at=START, + replay_sequence=1, + ) + + with pytest.raises(TypeError, match="TradeHistoryCursor"): + TradeHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + cursor=quote_cursor, # type: ignore[arg-type] + ) + + +def test_candle_query_preserves_interval_case() -> None: + query = CandleRevisionHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + interval=" 1M ", + time_range=make_range(), + ) + + assert query.interval == "1M" + + +def test_empty_page_is_valid_without_cursor() -> None: + page = TradeHistoryPage(query=make_trade_query(), items=()) + + assert page.items == () + assert page.has_more is False + + +def test_empty_page_rejects_cursor() -> None: + cursor = TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + executed_at=START, + replay_sequence=1, + ) + + with pytest.raises(ValueError, match="empty page"): + TradeHistoryPage( + query=make_trade_query(), + items=(), + next_cursor=cursor, + ) + + +def test_trade_page_accepts_signed_rollover_at_equal_timestamp() -> None: + rollover_time = START + timedelta(minutes=1) + first = make_trade_record( + trade_id=2_147_483_647, + executed_at=rollover_time, + replay_sequence=10, + ) + second = make_trade_record( + trade_id=-2_147_483_648, + executed_at=rollover_time, + replay_sequence=11, + ) + cursor = TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + executed_at=rollover_time, + replay_sequence=11, + ) + + page = TradeHistoryPage( + query=make_trade_query(), + items=(first, second), + next_cursor=cursor, + ) + + assert page.items == (first, second) + assert page.has_more is True + + +def test_trade_page_accepts_negative_one_to_zero_boundary() -> None: + event_time = START + timedelta(minutes=1) + page = TradeHistoryPage( + query=make_trade_query(), + items=( + make_trade_record( + trade_id=-1, + executed_at=event_time, + replay_sequence=20, + ), + make_trade_record( + trade_id=0, + executed_at=event_time, + replay_sequence=21, + ), + ), + ) + + assert [item.trade.trade_id for item in page.items] == [-1, 0] + + +def test_page_rejects_reverse_or_duplicate_order_key() -> None: + first = make_trade_record(replay_sequence=2) + second = make_trade_record(replay_sequence=1) + + with pytest.raises(ValueError, match="strictly ordered"): + TradeHistoryPage( + query=make_trade_query(), + items=(first, second), + ) + + +def test_page_rejects_cursor_not_pointing_to_last_item() -> None: + item = make_trade_record(replay_sequence=10) + cursor = TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + executed_at=item.event_time, + replay_sequence=9, + ) + + with pytest.raises(ValueError, match="last page item"): + TradeHistoryPage( + query=make_trade_query(), + items=(item,), + next_cursor=cursor, + ) + + +@pytest.mark.parametrize( + "page", + ( + QuoteHistoryPage( + query=QuoteHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + ), + items=(make_quote_record(),), + ), + CandleRevisionHistoryPage( + query=CandleRevisionHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + interval="1m", + time_range=make_range(), + ), + items=(make_candle_record(),), + ), + ), +) +def test_quote_and_candle_pages_are_typed_and_immutable(page: Any) -> None: + assert not hasattr(page, "__dict__") + + with pytest.raises(FrozenInstanceError): + setattr(page, "items", ()) + + +def test_cursor_and_query_classes_use_slots_and_are_frozen() -> None: + query = QuoteHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + ) + + assert not hasattr(query, "__dict__") + + with pytest.raises(FrozenInstanceError): + setattr(query, "venue", "other") + + +def test_cursor_window_mismatch_is_rejected() -> None: + cursor = TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + executed_at=START, + replay_sequence=1, + ) + shifted = HistoricalTimeRange( + start_time=START - timedelta(minutes=1), + end_time=END, + ) + + with pytest.raises(ValueError, match="scope"): + TradeHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=shifted, + cursor=cursor, + ) + + +def test_candle_cursor_interval_mismatch_is_rejected() -> None: + cursor = CandleRevisionHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + interval="1m", + time_range=make_range(), + open_time=START, + replay_sequence=1, + ) + + with pytest.raises(ValueError, match="interval"): + CandleRevisionHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + interval="5m", + time_range=make_range(), + cursor=cursor, + ) + + +def test_page_rejects_mixed_query_scope() -> None: + first = make_trade_record(replay_sequence=1) + second = replace( + make_trade_record( + executed_at=START + timedelta(minutes=2), + replay_sequence=2, + ), + venue="other", + ) + + with pytest.raises(ValueError, match="one query scope"): + TradeHistoryPage( + query=make_trade_query(), + items=(first, second), + ) + + +def test_terminal_page_rejects_item_outside_query_range() -> None: + item = make_trade_record( + executed_at=START - timedelta(microseconds=1), + ) + + with pytest.raises(ValueError, match="query range"): + TradeHistoryPage( + query=make_trade_query(), + items=(item,), + ) + + +def test_page_rejects_more_items_than_query_limit() -> None: + first = make_trade_record(replay_sequence=1) + second = make_trade_record( + executed_at=START + timedelta(minutes=2), + replay_sequence=2, + ) + + with pytest.raises(ValueError, match="query limit"): + TradeHistoryPage( + query=make_trade_query(limit=1), + items=(first, second), + ) + + +def test_page_items_must_follow_incoming_cursor() -> None: + cursor = TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + executed_at=START + timedelta(minutes=1), + replay_sequence=10, + ) + + with pytest.raises(ValueError, match="follow query cursor"): + TradeHistoryPage( + query=make_trade_query(cursor=cursor), + items=(make_trade_record(replay_sequence=10),), + ) + + +def test_records_reject_canonical_payload_subclasses() -> None: + class TradeSubclass(Trade): + pass + + class QuoteSubclass(Quote): + pass + + class CandleSubclass(Candle): + pass + + trade = make_trade() + quote = make_quote() + candle = make_candle() + + trade_subclass = TradeSubclass( + symbol=trade.symbol, + trade_id=trade.trade_id, + price=trade.price, + quantity=trade.quantity, + executed_at=trade.executed_at, + aggressor_side=trade.aggressor_side, + source=trade.source, + ) + quote_subclass = QuoteSubclass( + symbol=quote.symbol, + last_price=quote.last_price, + bid_price=quote.bid_price, + ask_price=quote.ask_price, + exchange_timestamp=quote.exchange_timestamp, + received_at=quote.received_at, + source=quote.source, + ) + candle_subclass = CandleSubclass( + symbol=candle.symbol, + interval=candle.interval, + open_time=candle.open_time, + open_price=candle.open_price, + high_price=candle.high_price, + low_price=candle.low_price, + close_price=candle.close_price, + volume=candle.volume, + source=candle.source, + ) + + with pytest.raises(TypeError, match="Canonical Trade"): + TradeHistoryRecord( + venue=VENUE, + trade=trade_subclass, + first_observed_at=trade.executed_at, + last_observed_at=trade.executed_at, + observation_sources=(trade.source,), + replay_sequence=1, + ) + + with pytest.raises(TypeError, match="Canonical Quote"): + QuoteHistoryRecord( + venue=VENUE, + quote=quote_subclass, + observation_sources=(quote.source,), + replay_sequence=1, + ) + + with pytest.raises(TypeError, match="Canonical Candle"): + CandleRevisionHistoryRecord( + venue=VENUE, + candle=candle_subclass, + observed_at=candle.open_time, + is_final=False, + observation_sources=(candle.source,), + replay_sequence=1, + ) + + +def test_cursor_and_query_reject_time_range_subclass() -> None: + class HistoricalTimeRangeSubclass(HistoricalTimeRange): + pass + + time_range = HistoricalTimeRangeSubclass(START, END) + + with pytest.raises(TypeError, match="HistoricalTimeRange"): + TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=time_range, + executed_at=START, + replay_sequence=1, + ) + + with pytest.raises(TypeError, match="HistoricalTimeRange"): + TradeHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=time_range, + ) + + +def test_query_rejects_cursor_subclass() -> None: + class TradeHistoryCursorSubclass(TradeHistoryCursor): + pass + + cursor = TradeHistoryCursorSubclass( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + executed_at=START, + replay_sequence=1, + ) + + with pytest.raises(TypeError, match="TradeHistoryCursor"): + TradeHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + cursor=cursor, + ) + + +def test_page_rejects_query_record_and_cursor_subclasses() -> None: + class TradeHistoryQuerySubclass(TradeHistoryQuery): + pass + + class TradeHistoryRecordSubclass(TradeHistoryRecord): + @property + def order_key(self) -> tuple[datetime, int]: + return (END, 1) + + class TradeHistoryCursorSubclass(TradeHistoryCursor): + pass + + item = make_trade_record(replay_sequence=10) + + with pytest.raises(TypeError, match="TradeHistoryQuery"): + TradeHistoryPage( + query=TradeHistoryQuerySubclass( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + ), + items=(item,), + ) + + item_subclass = TradeHistoryRecordSubclass( + venue=item.venue, + trade=item.trade, + first_observed_at=item.first_observed_at, + last_observed_at=item.last_observed_at, + observation_sources=item.observation_sources, + replay_sequence=item.replay_sequence, + ) + + with pytest.raises(TypeError, match="TradeHistoryRecord"): + TradeHistoryPage( + query=make_trade_query(), + items=(item_subclass,), + ) + + cursor_subclass = TradeHistoryCursorSubclass( + venue=VENUE, + symbol=SYMBOL, + time_range=make_range(), + executed_at=item.event_time, + replay_sequence=item.replay_sequence, + ) + + with pytest.raises(TypeError, match="TradeHistoryCursor"): + TradeHistoryPage( + query=make_trade_query(), + items=(item,), + next_cursor=cursor_subclass, + ) + + +def test_page_rejects_items_tuple_subclass() -> None: + class ItemsTupleSubclass(tuple): + pass + + with pytest.raises(TypeError, match="items must be a tuple"): + TradeHistoryPage( + query=make_trade_query(), + items=ItemsTupleSubclass((make_trade_record(),)), + ) diff --git a/app/tests/unit/market_data/access/test_market_data_historical_access.py b/app/tests/unit/market_data/access/test_market_data_historical_access.py new file mode 100644 index 0000000..20b6530 --- /dev/null +++ b/app/tests/unit/market_data/access/test_market_data_historical_access.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Any + +import pytest + +from src.market_data.access.contracts import ( + MarketDataHistoricalAccessProtocol, +) +from src.market_data.access.market_data_historical_access import ( + MarketDataHistoricalAccess, +) +from src.market_data.access.models import ( + CandleRevisionHistoryPage, + CandleRevisionHistoryQuery, + HistoricalTimeRange, + QuoteHistoryPage, + QuoteHistoryQuery, + TradeHistoryPage, + TradeHistoryQuery, +) + + +NOW = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) +TIME_RANGE = HistoricalTimeRange( + start_time=NOW, + end_time=NOW + timedelta(hours=1), +) + + +class RecordingTradeReader: + def __init__(self, page: TradeHistoryPage) -> None: + self.page = page + self.queries: list[TradeHistoryQuery] = [] + + def query_trades(self, query: TradeHistoryQuery) -> TradeHistoryPage: + self.queries.append(query) + return self.page + + +class RecordingQuoteReader: + def __init__(self, page: QuoteHistoryPage) -> None: + self.page = page + self.queries: list[QuoteHistoryQuery] = [] + + def query_quotes(self, query: QuoteHistoryQuery) -> QuoteHistoryPage: + self.queries.append(query) + return self.page + + +class RecordingCandleReader: + def __init__(self, page: CandleRevisionHistoryPage) -> None: + self.page = page + self.queries: list[CandleRevisionHistoryQuery] = [] + + def query_candle_revisions( + self, + query: CandleRevisionHistoryQuery, + ) -> CandleRevisionHistoryPage: + self.queries.append(query) + return self.page + + +class BrokenTradeReader: + def __init__(self, error: RuntimeError) -> None: + self.error = error + + def query_trades(self, query: TradeHistoryQuery) -> TradeHistoryPage: + del query + raise self.error + + +def make_dependencies() -> tuple[ + MarketDataHistoricalAccess, + RecordingTradeReader, + RecordingQuoteReader, + RecordingCandleReader, +]: + trade_query = TradeHistoryQuery( + venue="dzengi", + symbol="BTC/USD_LEVERAGE", + time_range=TIME_RANGE, + ) + quote_query = QuoteHistoryQuery( + venue="dzengi", + symbol="BTC/USD_LEVERAGE", + time_range=TIME_RANGE, + ) + candle_query = CandleRevisionHistoryQuery( + venue="dzengi", + symbol="BTC/USD_LEVERAGE", + interval="1m", + time_range=TIME_RANGE, + ) + trade_reader = RecordingTradeReader( + TradeHistoryPage(query=trade_query, items=()), + ) + quote_reader = RecordingQuoteReader( + QuoteHistoryPage(query=quote_query, items=()), + ) + candle_reader = RecordingCandleReader( + CandleRevisionHistoryPage(query=candle_query, items=()), + ) + access = MarketDataHistoricalAccess( + trade_reader=trade_reader, + quote_reader=quote_reader, + candle_revision_reader=candle_reader, + ) + return access, trade_reader, quote_reader, candle_reader + + +def test_implements_combined_protocol_and_uses_slots() -> None: + access, *_ = make_dependencies() + + assert isinstance(access, MarketDataHistoricalAccessProtocol) + assert not hasattr(access, "__dict__") + + +@pytest.mark.parametrize( + "dependency_name", + ( + "trade_reader", + "quote_reader", + "candle_revision_reader", + ), +) +def test_rejects_dependency_without_required_protocol( + dependency_name: str, +) -> None: + _, trade_reader, quote_reader, candle_reader = make_dependencies() + dependencies: dict[str, Any] = { + "trade_reader": trade_reader, + "quote_reader": quote_reader, + "candle_revision_reader": candle_reader, + } + dependencies[dependency_name] = object() + + with pytest.raises(TypeError, match=dependency_name): + MarketDataHistoricalAccess(**dependencies) + + +def test_delegates_each_query_without_rebuilding_page() -> None: + access, trade_reader, quote_reader, candle_reader = make_dependencies() + trade_query = TradeHistoryQuery( + venue="dzengi", + symbol="BTC/USD_LEVERAGE", + time_range=TIME_RANGE, + limit=10, + ) + quote_query = QuoteHistoryQuery( + venue="dzengi", + symbol="BTC/USD_LEVERAGE", + time_range=TIME_RANGE, + limit=20, + ) + candle_query = CandleRevisionHistoryQuery( + venue="dzengi", + symbol="BTC/USD_LEVERAGE", + interval="1m", + time_range=TIME_RANGE, + limit=30, + ) + + trade_page = access.query_trades(trade_query) + quote_page = access.query_quotes(quote_query) + candle_page = access.query_candle_revisions(candle_query) + + assert trade_page is trade_reader.page + assert quote_page is quote_reader.page + assert candle_page is candle_reader.page + assert trade_reader.queries == [trade_query] + assert quote_reader.queries == [quote_query] + assert candle_reader.queries == [candle_query] + + +def test_does_not_swallow_reader_error() -> None: + _, _, quote_reader, candle_reader = make_dependencies() + expected = RuntimeError("reader failed") + broken_reader = BrokenTradeReader(expected) + access = MarketDataHistoricalAccess( + trade_reader=broken_reader, + quote_reader=quote_reader, + candle_revision_reader=candle_reader, + ) + query = TradeHistoryQuery( + venue="dzengi", + symbol="BTC/USD_LEVERAGE", + time_range=TIME_RANGE, + ) + + with pytest.raises(RuntimeError) as captured: + access.query_trades(query) + + assert captured.value is expected diff --git a/app/tests/unit/market_data/access/test_postgres_candle_revision_history_repository.py b/app/tests/unit/market_data/access/test_postgres_candle_revision_history_repository.py new file mode 100644 index 0000000..d74757c --- /dev/null +++ b/app/tests/unit/market_data/access/test_postgres_candle_revision_history_repository.py @@ -0,0 +1,620 @@ +from __future__ import annotations + +from contextlib import nullcontext +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any + +import pytest + +from src.market_data.access.contracts import ( + CandleRevisionHistoryReaderProtocol, +) +from src.market_data.access.exceptions import ( + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, + MarketDataAccessValidationError, +) +from src.market_data.access.models import ( + CandleRevisionHistoryCursor, + CandleRevisionHistoryQuery, + HistoricalTimeRange, +) +from src.market_data.access.postgres_candle_revision_history_repository import ( + PostgresCandleRevisionHistoryRepository, +) +from src.market_data.acquisition.models.candle import Candle + + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +INTERVAL = "1m" +START = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) +END = START + timedelta(hours=1) +SOURCE = "dzengi_websocket_candle" +_DEFAULT = object() + + +class RecordingCursor: + def __init__(self, rows: object = ()) -> None: + self.rows = rows + self.calls: list[tuple[str, tuple[object, ...]]] = [] + self.enter_calls = 0 + self.exit_exception_types: list[type[BaseException] | None] = [] + self.execute_error: BaseException | None = None + self.fetchall_error: BaseException | None = None + self.exit_error: BaseException | None = None + + def __enter__(self) -> RecordingCursor: + self.enter_calls += 1 + return self + + def __exit__( + self, + exception_type: type[BaseException] | None, + exception: BaseException | None, + traceback: object, + ) -> None: + self.exit_exception_types.append(exception_type) + + if self.exit_error is not None: + raise self.exit_error + + return None + + def execute(self, sql: str, parameters: tuple[object, ...]) -> None: + self.calls.append((sql, parameters)) + + if self.execute_error is not None: + raise self.execute_error + + def fetchall(self) -> object: + if self.fetchall_error is not None: + raise self.fetchall_error + + return self.rows + + +class RecordingConnection: + def __init__(self, cursor: RecordingCursor) -> None: + self._cursor = cursor + self.enter_calls = 0 + self.cursor_calls = 0 + self.exit_exception_types: list[type[BaseException] | None] = [] + self.cursor_error: BaseException | None = None + self.exit_error: BaseException | None = None + + def __enter__(self) -> RecordingConnection: + self.enter_calls += 1 + return self + + def __exit__( + self, + exception_type: type[BaseException] | None, + exception: BaseException | None, + traceback: object, + ) -> None: + self.exit_exception_types.append(exception_type) + + if self.exit_error is not None: + raise self.exit_error + + return None + + def cursor(self) -> RecordingCursor: + self.cursor_calls += 1 + + if self.cursor_error is not None: + raise self.cursor_error + + return self._cursor + + +@dataclass +class RecordingProvider: + connection: RecordingConnection + calls: int = 0 + error: BaseException | None = None + + def __call__(self) -> RecordingConnection: + self.calls += 1 + + if self.error is not None: + raise self.error + + return self.connection + + +def make_query( + *, + interval: str = INTERVAL, + limit: int = 3, + cursor: CandleRevisionHistoryCursor | None = None, +) -> CandleRevisionHistoryQuery: + return CandleRevisionHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + interval=interval, + time_range=HistoricalTimeRange(START, END), + limit=limit, + cursor=cursor, + ) + + +def make_row( + *, + venue: object = VENUE, + symbol: object = SYMBOL, + interval: object = INTERVAL, + open_time: object = START + timedelta(minutes=1), + observed_at: object = _DEFAULT, + open_price: object = Decimal("64000"), + high_price: object = Decimal("64200"), + low_price: object = Decimal("63900"), + close_price: object = Decimal("64150"), + volume: object = Decimal("1.25"), + is_final: object = False, + source: object = SOURCE, + observation_sources: object = _DEFAULT, + replay_sequence: object = 10, + canonical_schema_version: object = 1, +) -> tuple[object, ...]: + resolved_observed_at = ( + START + timedelta(minutes=1, seconds=10) + if observed_at is _DEFAULT + else observed_at + ) + resolved_sources = ( + [SOURCE] + if observation_sources is _DEFAULT + else observation_sources + ) + return ( + venue, + symbol, + interval, + open_time, + resolved_observed_at, + open_price, + high_price, + low_price, + close_price, + volume, + is_final, + source, + resolved_sources, + replay_sequence, + canonical_schema_version, + ) + + +def dependencies( + rows: object = (), +) -> tuple[ + PostgresCandleRevisionHistoryRepository, + RecordingCursor, + RecordingConnection, + RecordingProvider, +]: + cursor = RecordingCursor(rows) + connection = RecordingConnection(cursor) + provider = RecordingProvider(connection) + repository = PostgresCandleRevisionHistoryRepository( + connection_provider=provider, + ) + return repository, cursor, connection, provider + + +def normalized_sql(sql: str) -> str: + return " ".join(sql.split()) + + +def test_constructor_is_no_io_slotted_and_matches_protocol() -> None: + repository, _, _, provider = dependencies() + + assert provider.calls == 0 + assert not hasattr(repository, "__dict__") + assert isinstance(repository, CandleRevisionHistoryReaderProtocol) + + +def test_constructor_rejects_non_callable_provider() -> None: + with pytest.raises(TypeError, match="connection_provider"): + PostgresCandleRevisionHistoryRepository( + connection_provider=None, # type: ignore[arg-type] + ) + + +def test_exact_query_is_validated_before_connection_borrow() -> None: + class QuerySubclass(CandleRevisionHistoryQuery): + pass + + repository, _, _, provider = dependencies() + query = QuerySubclass( + venue=VENUE, + symbol=SYMBOL, + interval=INTERVAL, + time_range=HistoricalTimeRange(START, END), + ) + + with pytest.raises(MarketDataAccessValidationError, match="query"): + repository.query_candle_revisions(query) + + assert provider.calls == 0 + + +def test_first_page_uses_open_time_half_open_order_and_limit_plus_one() -> None: + first_open = START + timedelta(minutes=1) + second_open = START + timedelta(minutes=2) + repository, cursor, connection, provider = dependencies( + [ + make_row(open_time=first_open, replay_sequence=10), + make_row( + open_time=second_open, + observed_at=second_open + timedelta(seconds=10), + is_final=True, + replay_sequence=11, + ), + ] + ) + query = make_query(limit=3) + + page = repository.query_candle_revisions(query) + + sql, parameters = cursor.calls[0] + compact_sql = normalized_sql(sql) + assert "open_time >= %s" in compact_sql + assert "open_time < %s" in compact_sql + assert "observed_at >= %s" not in compact_sql + assert "(open_time, replay_sequence) >" not in compact_sql + assert "ORDER BY open_time ASC, replay_sequence ASC" in compact_sql + assert compact_sql.endswith("LIMIT %s") + assert parameters == (VENUE, SYMBOL, INTERVAL, START, END, 4) + assert [item.replay_sequence for item in page.items] == [10, 11] + assert type(page.items[0].candle) is Candle + assert page.items[0].event_time is first_open + assert page.items[1].is_final is True + assert page.next_cursor is None + assert provider.calls == 1 + assert connection.enter_calls == 1 + assert connection.cursor_calls == 1 + assert cursor.enter_calls == 1 + assert cursor.exit_exception_types == [None] + assert connection.exit_exception_types == [None] + + +def test_interval_is_case_sensitive_and_not_normalized() -> None: + repository, cursor, _, _ = dependencies([]) + query = make_query(interval="1M") + + repository.query_candle_revisions(query) + + assert cursor.calls[0][1][2] == "1M" + + +def test_keyset_query_uses_exact_open_time_cursor_tuple() -> None: + cursor_time = START + timedelta(minutes=5) + incoming = CandleRevisionHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + interval=INTERVAL, + time_range=HistoricalTimeRange(START, END), + open_time=cursor_time, + replay_sequence=25, + ) + repository, cursor, _, _ = dependencies( + [ + make_row( + open_time=cursor_time, + observed_at=cursor_time + timedelta(seconds=10), + replay_sequence=26, + ) + ] + ) + + repository.query_candle_revisions(make_query(limit=2, cursor=incoming)) + + sql, parameters = cursor.calls[0] + assert ( + "(open_time, replay_sequence) > (%s, %s)" + in normalized_sql(sql) + ) + assert parameters == ( + VENUE, + SYMBOL, + INTERVAL, + START, + END, + cursor_time, + 25, + 3, + ) + + +def test_limit_plus_one_creates_cursor_from_last_returned_item() -> None: + open_times = tuple( + START + timedelta(minutes=index) for index in (1, 2, 3) + ) + repository, _, _, _ = dependencies( + [ + make_row( + open_time=open_time, + observed_at=open_time + timedelta(seconds=10), + replay_sequence=10 + index, + ) + for index, open_time in enumerate(open_times) + ] + ) + query = make_query(limit=2) + + page = repository.query_candle_revisions(query) + + assert [item.replay_sequence for item in page.items] == [10, 11] + assert page.next_cursor is not None + assert page.next_cursor.venue == VENUE + assert page.next_cursor.symbol == SYMBOL + assert page.next_cursor.interval == INTERVAL + assert page.next_cursor.time_range is query.time_range + assert page.next_cursor.open_time == open_times[1] + assert page.next_cursor.replay_sequence == 11 + + +def test_empty_result_is_valid_without_cursor() -> None: + repository, _, _, _ = dependencies([]) + + page = repository.query_candle_revisions(make_query()) + + assert page.items == () + assert page.next_cursor is None + assert page.has_more is False + + +def test_history_range_uses_open_time_not_observed_at() -> None: + open_time = START + timedelta(minutes=1) + observed_at = END + timedelta(minutes=10) + repository, _, _, _ = dependencies( + [make_row(open_time=open_time, observed_at=observed_at)] + ) + + page = repository.query_candle_revisions(make_query()) + + assert page.items[0].event_time == open_time + assert page.items[0].replay_at == observed_at + + +@pytest.mark.parametrize( + ("overrides", "error_match"), + ( + ({"venue": "other"}, "scope"), + ({"symbol": "ETH/USD_LEVERAGE"}, "scope"), + ({"interval": "1M"}, "scope"), + ({"open_time": END, "observed_at": END}, "range"), + ), +) +def test_rows_outside_exact_scope_or_range_are_rejected( + overrides: dict[str, object], + error_match: str, +) -> None: + repository, _, _, _ = dependencies([make_row(**overrides)]) + + with pytest.raises(MarketDataAccessIntegrityError, match=error_match): + repository.query_candle_revisions(make_query()) + + +@pytest.mark.parametrize( + "overrides", + ( + {"venue": " dzengi"}, + {"symbol": "btc/usd_leverage"}, + {"interval": " 1m"}, + {"open_time": START.replace(tzinfo=None)}, + {"observed_at": START.replace(tzinfo=None)}, + {"observed_at": START}, + {"open_price": Decimal("0")}, + {"high_price": Decimal("NaN")}, + {"low_price": Decimal("65000")}, + {"close_price": Decimal("65000")}, + {"volume": Decimal("-0.01")}, + {"is_final": 1}, + {"source": " source"}, + {"observation_sources": ()}, + {"observation_sources": []}, + {"observation_sources": [SOURCE, SOURCE]}, + {"observation_sources": ["recovery", SOURCE]}, + {"replay_sequence": True}, + {"replay_sequence": 0}, + {"canonical_schema_version": 2}, + ), +) +def test_corrupt_stored_values_raise_integrity_error( + overrides: dict[str, object], +) -> None: + repository, cursor, connection, _ = dependencies( + [make_row(**overrides)] + ) + + with pytest.raises( + MarketDataAccessIntegrityError, + match="invalid Canonical Candle", + ): + repository.query_candle_revisions(make_query()) + + assert cursor.exit_exception_types == [MarketDataAccessIntegrityError] + assert connection.exit_exception_types == [ + MarketDataAccessIntegrityError + ] + + +@pytest.mark.parametrize("row", ((), [object()] * 15, (object(),) * 14)) +def test_invalid_row_shape_raises_integrity_error(row: object) -> None: + repository, _, _, _ = dependencies([row]) + + with pytest.raises(MarketDataAccessIntegrityError, match="row"): + repository.query_candle_revisions(make_query()) + + +def test_invalid_rows_collection_and_excess_rows_are_rejected() -> None: + repository, _, _, _ = dependencies(iter(())) + + with pytest.raises(MarketDataAccessIntegrityError, match="rows"): + repository.query_candle_revisions(make_query()) + + repository, _, _, _ = dependencies( + [ + make_row( + open_time=START + timedelta(minutes=index + 1), + observed_at=START + timedelta(minutes=index + 1, seconds=1), + replay_sequence=index + 1, + ) + for index in range(3) + ] + ) + + with pytest.raises(MarketDataAccessIntegrityError, match="limit"): + repository.query_candle_revisions(make_query(limit=1)) + + +def test_unordered_rows_and_duplicate_global_sequence_are_rejected() -> None: + earlier = START + timedelta(minutes=1) + later = START + timedelta(minutes=2) + unordered, _, _, _ = dependencies( + [ + make_row( + open_time=later, + observed_at=later, + replay_sequence=10, + ), + make_row( + open_time=earlier, + observed_at=earlier, + replay_sequence=11, + ), + ] + ) + + with pytest.raises(MarketDataAccessIntegrityError, match="ordered"): + unordered.query_candle_revisions(make_query()) + + duplicate, _, _, _ = dependencies( + [ + make_row( + open_time=earlier, + observed_at=earlier, + replay_sequence=10, + ), + make_row( + open_time=later, + observed_at=later, + replay_sequence=10, + ), + ] + ) + + with pytest.raises( + MarketDataAccessIntegrityError, + match="duplicate replay_sequence", + ): + duplicate.query_candle_revisions(make_query()) + + +def test_rows_must_strictly_follow_incoming_cursor() -> None: + cursor_time = START + timedelta(minutes=1) + incoming = CandleRevisionHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + interval=INTERVAL, + time_range=HistoricalTimeRange(START, END), + open_time=cursor_time, + replay_sequence=10, + ) + repository, _, _, _ = dependencies( + [ + make_row( + open_time=cursor_time, + observed_at=cursor_time, + replay_sequence=10, + ) + ] + ) + + with pytest.raises(MarketDataAccessIntegrityError, match="cursor"): + repository.query_candle_revisions(make_query(cursor=incoming)) + + +def test_database_error_is_wrapped_after_context_cleanup() -> None: + repository, cursor, connection, _ = dependencies() + cursor.execute_error = RuntimeError("database failed") + + with pytest.raises(MarketDataAccessOperationError) as error_info: + repository.query_candle_revisions(make_query()) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert cursor.exit_exception_types == [RuntimeError] + assert connection.exit_exception_types == [RuntimeError] + + +def test_provider_error_is_wrapped_without_entering_connection() -> None: + repository, _, connection, provider = dependencies() + provider.error = RuntimeError("provider failed") + + with pytest.raises(MarketDataAccessOperationError) as error_info: + repository.query_candle_revisions(make_query()) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert provider.calls == 1 + assert connection.enter_calls == 0 + + +def test_base_exception_is_not_wrapped_and_reaches_cleanup() -> None: + repository, cursor, connection, _ = dependencies() + cursor.execute_error = KeyboardInterrupt() + + with pytest.raises(KeyboardInterrupt): + repository.query_candle_revisions(make_query()) + + assert cursor.exit_exception_types == [KeyboardInterrupt] + assert connection.exit_exception_types == [KeyboardInterrupt] + + +def test_cleanup_error_obeys_exception_and_base_exception_contract() -> None: + repository, cursor, connection, _ = dependencies([]) + cursor.exit_error = RuntimeError("cursor cleanup failed") + + with pytest.raises(MarketDataAccessOperationError) as error_info: + repository.query_candle_revisions(make_query()) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert cursor.exit_exception_types == [None] + assert connection.exit_exception_types == [RuntimeError] + + repository, cursor, connection, _ = dependencies([]) + cursor.exit_error = KeyboardInterrupt() + + with pytest.raises(KeyboardInterrupt): + repository.query_candle_revisions(make_query()) + + assert cursor.exit_exception_types == [None] + assert connection.exit_exception_types == [KeyboardInterrupt] + + +def test_bound_provider_can_reuse_caller_owned_connection() -> None: + cursor = RecordingCursor([make_row()]) + connection = RecordingConnection(cursor) + provider_calls = 0 + + def bound_provider() -> Any: + nonlocal provider_calls + provider_calls += 1 + return nullcontext(connection) + + repository = PostgresCandleRevisionHistoryRepository( + connection_provider=bound_provider, + ) + + page = repository.query_candle_revisions(make_query()) + + assert len(page.items) == 1 + assert provider_calls == 1 + assert connection.enter_calls == 0 + assert connection.exit_exception_types == [] + assert cursor.exit_exception_types == [None] diff --git a/app/tests/unit/market_data/access/test_postgres_quote_history_repository.py b/app/tests/unit/market_data/access/test_postgres_quote_history_repository.py new file mode 100644 index 0000000..4af647a --- /dev/null +++ b/app/tests/unit/market_data/access/test_postgres_quote_history_repository.py @@ -0,0 +1,529 @@ +from __future__ import annotations + +from contextlib import nullcontext +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any + +import pytest + +from src.market_data.access.contracts import QuoteHistoryReaderProtocol +from src.market_data.access.exceptions import ( + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, + MarketDataAccessValidationError, +) +from src.market_data.access.models import ( + HistoricalTimeRange, + QuoteHistoryCursor, + QuoteHistoryQuery, +) +from src.market_data.access.postgres_quote_history_repository import ( + PostgresQuoteHistoryRepository, +) +from src.market_data.acquisition.models.quote import Quote + + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +START = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) +END = START + timedelta(hours=1) +SOURCE = "dzengi_websocket_quote" + + +class RecordingCursor: + def __init__(self, rows: object = ()) -> None: + self.rows = rows + self.calls: list[tuple[str, tuple[object, ...]]] = [] + self.enter_calls = 0 + self.exit_exception_types: list[type[BaseException] | None] = [] + self.execute_error: BaseException | None = None + self.fetchall_error: BaseException | None = None + self.exit_error: BaseException | None = None + + def __enter__(self) -> RecordingCursor: + self.enter_calls += 1 + return self + + def __exit__( + self, + exception_type: type[BaseException] | None, + exception: BaseException | None, + traceback: object, + ) -> None: + self.exit_exception_types.append(exception_type) + + if self.exit_error is not None: + raise self.exit_error + + return None + + def execute(self, sql: str, parameters: tuple[object, ...]) -> None: + self.calls.append((sql, parameters)) + + if self.execute_error is not None: + raise self.execute_error + + def fetchall(self) -> object: + if self.fetchall_error is not None: + raise self.fetchall_error + + return self.rows + + +class RecordingConnection: + def __init__(self, cursor: RecordingCursor) -> None: + self._cursor = cursor + self.enter_calls = 0 + self.cursor_calls = 0 + self.exit_exception_types: list[type[BaseException] | None] = [] + self.cursor_error: BaseException | None = None + self.exit_error: BaseException | None = None + + def __enter__(self) -> RecordingConnection: + self.enter_calls += 1 + return self + + def __exit__( + self, + exception_type: type[BaseException] | None, + exception: BaseException | None, + traceback: object, + ) -> None: + self.exit_exception_types.append(exception_type) + + if self.exit_error is not None: + raise self.exit_error + + return None + + def cursor(self) -> RecordingCursor: + self.cursor_calls += 1 + + if self.cursor_error is not None: + raise self.cursor_error + + return self._cursor + + +@dataclass +class RecordingProvider: + connection: RecordingConnection + calls: int = 0 + error: BaseException | None = None + + def __call__(self) -> RecordingConnection: + self.calls += 1 + + if self.error is not None: + raise self.error + + return self.connection + + +def make_query( + *, + limit: int = 3, + cursor: QuoteHistoryCursor | None = None, +) -> QuoteHistoryQuery: + return QuoteHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=HistoricalTimeRange(START, END), + limit=limit, + cursor=cursor, + ) + + +def make_row( + *, + venue: object = VENUE, + symbol: object = SYMBOL, + received_at: object = START + timedelta(minutes=1), + exchange_timestamp: object = START + timedelta(seconds=59), + last_price: object = Decimal("64159.45"), + bid_price: object = Decimal("64159.40"), + ask_price: object = Decimal("64159.50"), + source: object = SOURCE, + observation_sources: object = None, + replay_sequence: object = 10, + canonical_schema_version: object = 1, +) -> tuple[object, ...]: + sources = [SOURCE] if observation_sources is None else observation_sources + return ( + venue, + symbol, + received_at, + exchange_timestamp, + last_price, + bid_price, + ask_price, + source, + sources, + replay_sequence, + canonical_schema_version, + ) + + +def dependencies( + rows: object = (), +) -> tuple[ + PostgresQuoteHistoryRepository, + RecordingCursor, + RecordingConnection, + RecordingProvider, +]: + cursor = RecordingCursor(rows) + connection = RecordingConnection(cursor) + provider = RecordingProvider(connection) + repository = PostgresQuoteHistoryRepository( + connection_provider=provider, + ) + return repository, cursor, connection, provider + + +def normalized_sql(sql: str) -> str: + return " ".join(sql.split()) + + +def test_constructor_is_no_io_slotted_and_matches_protocol() -> None: + repository, _, _, provider = dependencies() + + assert provider.calls == 0 + assert not hasattr(repository, "__dict__") + assert isinstance(repository, QuoteHistoryReaderProtocol) + + +def test_constructor_rejects_non_callable_provider() -> None: + with pytest.raises(TypeError, match="connection_provider"): + PostgresQuoteHistoryRepository( + connection_provider=None, # type: ignore[arg-type] + ) + + +def test_exact_query_is_validated_before_connection_borrow() -> None: + class QuoteHistoryQuerySubclass(QuoteHistoryQuery): + pass + + repository, _, _, provider = dependencies() + query = QuoteHistoryQuerySubclass( + venue=VENUE, + symbol=SYMBOL, + time_range=HistoricalTimeRange(START, END), + ) + + with pytest.raises(MarketDataAccessValidationError, match="query"): + repository.query_quotes(query) + + assert provider.calls == 0 + + +def test_first_page_uses_received_time_half_open_order_and_limit() -> None: + first_time = START + timedelta(minutes=1) + second_time = START + timedelta(minutes=2) + repository, cursor, connection, provider = dependencies( + [ + make_row(received_at=first_time, replay_sequence=10), + make_row(received_at=second_time, replay_sequence=11), + ] + ) + query = make_query(limit=3) + + page = repository.query_quotes(query) + + sql, parameters = cursor.calls[0] + compact_sql = normalized_sql(sql) + assert "received_at >= %s" in compact_sql + assert "received_at < %s" in compact_sql + assert "(received_at, replay_sequence) >" not in compact_sql + assert "ORDER BY received_at ASC, replay_sequence ASC" in compact_sql + assert compact_sql.endswith("LIMIT %s") + assert parameters == (VENUE, SYMBOL, START, END, 4) + assert [item.replay_sequence for item in page.items] == [10, 11] + assert type(page.items[0].quote) is Quote + assert page.items[0].quote.received_at is first_time + assert page.items[0].observation_sources == (SOURCE,) + assert page.next_cursor is None + assert provider.calls == 1 + assert connection.enter_calls == 1 + assert connection.cursor_calls == 1 + assert cursor.enter_calls == 1 + assert cursor.exit_exception_types == [None] + assert connection.exit_exception_types == [None] + + +def test_keyset_query_uses_exact_received_at_and_sequence_cursor() -> None: + cursor_position = START + timedelta(minutes=5) + incoming_cursor = QuoteHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=HistoricalTimeRange(START, END), + received_at=cursor_position, + replay_sequence=25, + ) + repository, recording_cursor, _, _ = dependencies( + [make_row(received_at=cursor_position, replay_sequence=26)] + ) + + repository.query_quotes(make_query(limit=2, cursor=incoming_cursor)) + + sql, parameters = recording_cursor.calls[0] + assert ( + "(received_at, replay_sequence) > (%s, %s)" + in normalized_sql(sql) + ) + assert parameters == ( + VENUE, + SYMBOL, + START, + END, + cursor_position, + 25, + 3, + ) + + +def test_limit_plus_one_creates_cursor_from_last_returned_quote() -> None: + times = tuple(START + timedelta(minutes=index) for index in (1, 2, 3)) + repository, _, _, _ = dependencies( + [ + make_row(received_at=event_time, replay_sequence=10 + index) + for index, event_time in enumerate(times) + ] + ) + query = make_query(limit=2) + + page = repository.query_quotes(query) + + assert len(page.items) == 2 + assert [item.replay_sequence for item in page.items] == [10, 11] + assert page.next_cursor is not None + assert page.next_cursor.venue == query.venue + assert page.next_cursor.symbol == query.symbol + assert page.next_cursor.time_range is query.time_range + assert page.next_cursor.received_at == times[1] + assert page.next_cursor.replay_sequence == 11 + + +def test_empty_result_and_exact_start_are_valid() -> None: + empty_repository, _, _, _ = dependencies([]) + + empty_page = empty_repository.query_quotes(make_query()) + + assert empty_page.items == () + assert empty_page.next_cursor is None + assert empty_page.has_more is False + + start_repository, _, _, _ = dependencies( + [make_row(received_at=START)] + ) + + start_page = start_repository.query_quotes(make_query()) + + assert start_page.items[0].event_time == START + + +def test_none_exchange_timestamp_is_preserved() -> None: + repository, _, _, _ = dependencies( + [make_row(exchange_timestamp=None)] + ) + + page = repository.query_quotes(make_query()) + + assert page.items[0].quote.exchange_timestamp is None + + +def test_backend_cannot_return_more_than_limit_plus_one() -> None: + repository, _, _, _ = dependencies( + [ + make_row( + received_at=START + timedelta(minutes=index + 1), + replay_sequence=10 + index, + ) + for index in range(3) + ] + ) + + with pytest.raises(MarketDataAccessIntegrityError, match="limit"): + repository.query_quotes(make_query(limit=1)) + + +@pytest.mark.parametrize( + "overrides", + ( + {"venue": "other"}, + {"venue": " dzengi "}, + {"symbol": "btc/usd_leverage"}, + {"received_at": END}, + {"received_at": START.replace(tzinfo=None)}, + {"exchange_timestamp": START.replace(tzinfo=None)}, + {"last_price": Decimal("0")}, + {"last_price": Decimal("NaN")}, + {"last_price": 1}, + {"bid_price": Decimal("64160")}, + {"ask_price": Decimal("0")}, + {"source": " source "}, + {"observation_sources": ()}, + {"observation_sources": []}, + {"observation_sources": [SOURCE, SOURCE]}, + {"observation_sources": ["recovery", SOURCE]}, + {"replay_sequence": True}, + {"replay_sequence": 0}, + {"canonical_schema_version": 2}, + ), +) +def test_corrupt_or_out_of_scope_values_raise_integrity_error( + overrides: dict[str, object], +) -> None: + repository, cursor, connection, _ = dependencies( + [make_row(**overrides)] + ) + + with pytest.raises(MarketDataAccessIntegrityError): + repository.query_quotes(make_query()) + + assert cursor.exit_exception_types == [MarketDataAccessIntegrityError] + assert connection.exit_exception_types == [MarketDataAccessIntegrityError] + + +@pytest.mark.parametrize( + "row", + ((), [object()] * 11, (object(),) * 10), +) +def test_invalid_row_shape_raises_integrity_error(row: object) -> None: + repository, _, _, _ = dependencies([row]) + + with pytest.raises(MarketDataAccessIntegrityError, match="row"): + repository.query_quotes(make_query()) + + +@pytest.mark.parametrize("rows", (None, "rows", object())) +def test_invalid_rows_collection_raises_integrity_error(rows: object) -> None: + repository, _, _, _ = dependencies(rows) + + with pytest.raises(MarketDataAccessIntegrityError, match="rows"): + repository.query_quotes(make_query()) + + +def test_unordered_rows_and_duplicate_global_sequence_are_rejected() -> None: + later = START + timedelta(minutes=2) + earlier = START + timedelta(minutes=1) + unordered_repository, _, _, _ = dependencies( + [ + make_row(received_at=later, replay_sequence=10), + make_row(received_at=earlier, replay_sequence=11), + ] + ) + + with pytest.raises(MarketDataAccessIntegrityError, match="ordered"): + unordered_repository.query_quotes(make_query()) + + duplicate_repository, _, _, _ = dependencies( + [ + make_row(received_at=earlier, replay_sequence=10), + make_row(received_at=later, replay_sequence=10), + ] + ) + + with pytest.raises( + MarketDataAccessIntegrityError, + match="duplicate replay_sequence", + ): + duplicate_repository.query_quotes(make_query()) + + +def test_rows_must_strictly_follow_incoming_cursor() -> None: + cursor_time = START + timedelta(minutes=1) + incoming = QuoteHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=HistoricalTimeRange(START, END), + received_at=cursor_time, + replay_sequence=10, + ) + repository, _, _, _ = dependencies( + [make_row(received_at=cursor_time, replay_sequence=10)] + ) + + with pytest.raises(MarketDataAccessIntegrityError, match="cursor"): + repository.query_quotes(make_query(cursor=incoming)) + + +def test_database_error_is_wrapped_after_context_cleanup() -> None: + repository, cursor, connection, _ = dependencies() + cursor.execute_error = RuntimeError("database failed") + + with pytest.raises(MarketDataAccessOperationError) as error_info: + repository.query_quotes(make_query()) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert cursor.exit_exception_types == [RuntimeError] + assert connection.exit_exception_types == [RuntimeError] + + +def test_provider_error_is_wrapped_without_entering_connection() -> None: + repository, _, connection, provider = dependencies() + provider.error = RuntimeError("provider failed") + + with pytest.raises(MarketDataAccessOperationError) as error_info: + repository.query_quotes(make_query()) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert provider.calls == 1 + assert connection.enter_calls == 0 + + +def test_keyboard_interrupt_is_not_wrapped_and_reaches_cleanup() -> None: + repository, cursor, connection, _ = dependencies() + cursor.execute_error = KeyboardInterrupt() + + with pytest.raises(KeyboardInterrupt): + repository.query_quotes(make_query()) + + assert cursor.exit_exception_types == [KeyboardInterrupt] + assert connection.exit_exception_types == [KeyboardInterrupt] + + +def test_cleanup_error_obeys_exception_and_base_exception_contract() -> None: + repository, cursor, connection, _ = dependencies([]) + cursor.exit_error = RuntimeError("cursor cleanup failed") + + with pytest.raises(MarketDataAccessOperationError) as error_info: + repository.query_quotes(make_query()) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert cursor.exit_exception_types == [None] + assert connection.exit_exception_types == [RuntimeError] + + repository, cursor, connection, _ = dependencies([]) + cursor.exit_error = KeyboardInterrupt() + + with pytest.raises(KeyboardInterrupt): + repository.query_quotes(make_query()) + + assert cursor.exit_exception_types == [None] + assert connection.exit_exception_types == [KeyboardInterrupt] + + +def test_bound_provider_can_reuse_caller_owned_connection() -> None: + cursor = RecordingCursor([make_row()]) + connection = RecordingConnection(cursor) + provider_calls = 0 + + def bound_provider() -> Any: + nonlocal provider_calls + provider_calls += 1 + return nullcontext(connection) + + repository = PostgresQuoteHistoryRepository( + connection_provider=bound_provider, + ) + + page = repository.query_quotes(make_query()) + + assert len(page.items) == 1 + assert provider_calls == 1 + assert connection.enter_calls == 0 + assert connection.exit_exception_types == [] + assert cursor.exit_exception_types == [None] diff --git a/app/tests/unit/market_data/access/test_postgres_trade_history_repository.py b/app/tests/unit/market_data/access/test_postgres_trade_history_repository.py new file mode 100644 index 0000000..4fc554e --- /dev/null +++ b/app/tests/unit/market_data/access/test_postgres_trade_history_repository.py @@ -0,0 +1,622 @@ +from __future__ import annotations + +from contextlib import nullcontext +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any + +import pytest + +from src.market_data.access import ( + HistoricalTimeRange, + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, + MarketDataAccessValidationError, + PostgresTradeHistoryRepository, + TradeHistoryCursor, + TradeHistoryQuery, + TradeHistoryReaderProtocol, +) +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.acquisition.trade_id_sequence import ( + SIGNED_TRADE_ID_MAX, + SIGNED_TRADE_ID_MIN, +) + + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +START = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) +END = START + timedelta(hours=1) +SOURCE = "dzengi_websocket_trade" + + +class RecordingCursor: + def __init__(self, rows: object = ()) -> None: + self.rows = rows + self.calls: list[tuple[str, tuple[object, ...]]] = [] + self.enter_calls = 0 + self.exit_exception_types: list[type[BaseException] | None] = [] + self.execute_error: BaseException | None = None + self.fetchall_error: BaseException | None = None + self.exit_error: BaseException | None = None + + def __enter__(self) -> RecordingCursor: + self.enter_calls += 1 + return self + + def __exit__( + self, + exception_type: type[BaseException] | None, + exception: BaseException | None, + traceback: object, + ) -> None: + self.exit_exception_types.append(exception_type) + + if self.exit_error is not None: + raise self.exit_error + + return None + + def execute(self, sql: str, parameters: tuple[object, ...]) -> None: + self.calls.append((sql, parameters)) + + if self.execute_error is not None: + raise self.execute_error + + def fetchall(self) -> object: + if self.fetchall_error is not None: + raise self.fetchall_error + + return self.rows + + +class RecordingConnection: + def __init__(self, cursor: RecordingCursor) -> None: + self._cursor = cursor + self.enter_calls = 0 + self.cursor_calls = 0 + self.exit_exception_types: list[type[BaseException] | None] = [] + self.cursor_error: BaseException | None = None + self.exit_error: BaseException | None = None + + def __enter__(self) -> RecordingConnection: + self.enter_calls += 1 + return self + + def __exit__( + self, + exception_type: type[BaseException] | None, + exception: BaseException | None, + traceback: object, + ) -> None: + self.exit_exception_types.append(exception_type) + + if self.exit_error is not None: + raise self.exit_error + + return None + + def cursor(self) -> RecordingCursor: + self.cursor_calls += 1 + + if self.cursor_error is not None: + raise self.cursor_error + + return self._cursor + + +@dataclass +class RecordingProvider: + connection: RecordingConnection + calls: int = 0 + error: BaseException | None = None + + def __call__(self) -> RecordingConnection: + self.calls += 1 + + if self.error is not None: + raise self.error + + return self.connection + + +def make_query( + *, + limit: int = 3, + cursor: TradeHistoryCursor | None = None, +) -> TradeHistoryQuery: + return TradeHistoryQuery( + venue=VENUE, + symbol=SYMBOL, + time_range=HistoricalTimeRange(START, END), + limit=limit, + cursor=cursor, + ) + + +def make_row( + *, + venue: object = VENUE, + symbol: object = SYMBOL, + trade_id: object = 100, + executed_at: object = START + timedelta(minutes=1), + price: object = Decimal("64159.45"), + quantity: object = Decimal("0.125"), + aggressor_side: object = "buy", + source: object = SOURCE, + first_observed_at: object = START + timedelta(minutes=1, seconds=1), + last_observed_at: object = START + timedelta(minutes=1, seconds=2), + observation_sources: object = None, + replay_sequence: object = 10, + canonical_schema_version: object = 1, +) -> tuple[object, ...]: + sources = [SOURCE] if observation_sources is None else observation_sources + return ( + venue, + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + sources, + replay_sequence, + canonical_schema_version, + ) + + +def dependencies( + rows: object = (), +) -> tuple[ + PostgresTradeHistoryRepository, + RecordingCursor, + RecordingConnection, + RecordingProvider, +]: + cursor = RecordingCursor(rows) + connection = RecordingConnection(cursor) + provider = RecordingProvider(connection) + repository = PostgresTradeHistoryRepository( + connection_provider=provider, + ) + return repository, cursor, connection, provider + + +def normalized_sql(sql: str) -> str: + return " ".join(sql.split()) + + +def test_constructor_is_no_io_slotted_and_matches_protocol() -> None: + repository, _, _, provider = dependencies() + + assert provider.calls == 0 + assert not hasattr(repository, "__dict__") + assert isinstance(repository, TradeHistoryReaderProtocol) + + +def test_constructor_rejects_non_callable_provider() -> None: + with pytest.raises(TypeError, match="connection_provider"): + PostgresTradeHistoryRepository( + connection_provider=None, # type: ignore[arg-type] + ) + + +def test_exact_query_is_validated_before_connection_borrow() -> None: + class TradeHistoryQuerySubclass(TradeHistoryQuery): + pass + + repository, _, _, provider = dependencies() + query = TradeHistoryQuerySubclass( + venue=VENUE, + symbol=SYMBOL, + time_range=HistoricalTimeRange(START, END), + ) + + with pytest.raises(MarketDataAccessValidationError, match="query"): + repository.query_trades(query) + + assert provider.calls == 0 + + +def test_first_page_uses_half_open_ordered_limit_plus_one_query() -> None: + first_time = START + timedelta(minutes=1) + second_time = START + timedelta(minutes=2) + repository, cursor, connection, provider = dependencies( + [ + make_row(executed_at=first_time, replay_sequence=10), + make_row( + trade_id=101, + executed_at=second_time, + first_observed_at=second_time + timedelta(seconds=1), + last_observed_at=second_time + timedelta(seconds=2), + replay_sequence=11, + ), + ] + ) + query = make_query(limit=3) + + page = repository.query_trades(query) + + sql, parameters = cursor.calls[0] + compact_sql = normalized_sql(sql) + assert "executed_at >= %s" in compact_sql + assert "executed_at < %s" in compact_sql + assert "(executed_at, replay_sequence) >" not in compact_sql + assert "ORDER BY executed_at ASC, replay_sequence ASC" in compact_sql + assert compact_sql.endswith("LIMIT %s") + assert parameters == (VENUE, SYMBOL, START, END, 4) + assert [item.replay_sequence for item in page.items] == [10, 11] + assert type(page.items[0].trade) is Trade + assert page.items[0].trade.executed_at is first_time + assert page.items[0].trade.aggressor_side is TradeAggressorSide.BUY + assert page.items[0].observation_sources == (SOURCE,) + assert page.next_cursor is None + assert provider.calls == 1 + assert connection.enter_calls == 1 + assert connection.cursor_calls == 1 + assert cursor.enter_calls == 1 + assert cursor.exit_exception_types == [None] + assert connection.exit_exception_types == [None] + + +def test_keyset_query_uses_exact_cursor_tuple() -> None: + cursor_position = START + timedelta(minutes=5) + incoming_cursor = TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=HistoricalTimeRange(START, END), + executed_at=cursor_position, + replay_sequence=25, + ) + row_time = cursor_position + repository, recording_cursor, _, _ = dependencies( + [ + make_row( + executed_at=row_time, + first_observed_at=row_time + timedelta(seconds=1), + last_observed_at=row_time + timedelta(seconds=2), + replay_sequence=26, + ) + ] + ) + query = make_query(limit=2, cursor=incoming_cursor) + + repository.query_trades(query) + + sql, parameters = recording_cursor.calls[0] + assert ( + "(executed_at, replay_sequence) > (%s, %s)" + in normalized_sql(sql) + ) + assert parameters == ( + VENUE, + SYMBOL, + START, + END, + cursor_position, + 25, + 3, + ) + + +def test_trade_id_rollover_does_not_participate_in_history_order() -> None: + event_time = START + timedelta(minutes=1) + repository, _, _, _ = dependencies( + [ + make_row( + trade_id=SIGNED_TRADE_ID_MAX, + executed_at=event_time, + replay_sequence=10, + ), + make_row( + trade_id=SIGNED_TRADE_ID_MIN, + executed_at=event_time, + replay_sequence=11, + ), + make_row( + trade_id=-1, + executed_at=event_time, + replay_sequence=12, + ), + make_row( + trade_id=0, + executed_at=event_time, + replay_sequence=13, + ), + ] + ) + + page = repository.query_trades(make_query(limit=4)) + + assert [item.trade.trade_id for item in page.items] == [ + SIGNED_TRADE_ID_MAX, + SIGNED_TRADE_ID_MIN, + -1, + 0, + ] + + +def test_limit_plus_one_creates_cursor_from_last_returned_item() -> None: + times = tuple(START + timedelta(minutes=index) for index in (1, 2, 3)) + repository, _, _, _ = dependencies( + [ + make_row( + trade_id=100 + index, + executed_at=event_time, + first_observed_at=event_time + timedelta(seconds=1), + last_observed_at=event_time + timedelta(seconds=2), + replay_sequence=10 + index, + ) + for index, event_time in enumerate(times) + ] + ) + query = make_query(limit=2) + + page = repository.query_trades(query) + + assert len(page.items) == 2 + assert [item.replay_sequence for item in page.items] == [10, 11] + assert page.next_cursor is not None + assert page.next_cursor.venue == query.venue + assert page.next_cursor.symbol == query.symbol + assert page.next_cursor.time_range is query.time_range + assert page.next_cursor.executed_at == times[1] + assert page.next_cursor.replay_sequence == 11 + + +def test_empty_result_is_valid_without_cursor() -> None: + repository, _, _, _ = dependencies([]) + + page = repository.query_trades(make_query()) + + assert page.items == () + assert page.next_cursor is None + assert page.has_more is False + + +def test_half_open_range_includes_exact_start() -> None: + repository, _, _, _ = dependencies( + [ + make_row( + executed_at=START, + first_observed_at=START, + last_observed_at=START, + ) + ] + ) + + page = repository.query_trades(make_query()) + + assert page.items[0].event_time == START + + +def test_backend_cannot_return_more_than_limit_plus_one() -> None: + query = make_query(limit=1) + rows = [ + make_row( + trade_id=100 + index, + executed_at=START + timedelta(minutes=index + 1), + first_observed_at=START + timedelta(minutes=index + 1, seconds=1), + last_observed_at=START + timedelta(minutes=index + 1, seconds=2), + replay_sequence=10 + index, + ) + for index in range(3) + ] + repository, _, _, _ = dependencies(rows) + + with pytest.raises(MarketDataAccessIntegrityError, match="limit"): + repository.query_trades(query) + + +@pytest.mark.parametrize( + ("overrides", "error_match"), + ( + ({"venue": "other"}, "invalid Canonical Trade"), + ({"symbol": "btc/usd_leverage"}, "invalid Canonical Trade"), + ({"trade_id": True}, "invalid Canonical Trade"), + ({"trade_id": SIGNED_TRADE_ID_MAX + 1}, "invalid Canonical Trade"), + ({"executed_at": END}, "invalid Canonical Trade"), + ({"price": Decimal("0")}, "invalid Canonical Trade"), + ({"price": Decimal("NaN")}, "invalid Canonical Trade"), + ({"quantity": Decimal("0")}, "invalid Canonical Trade"), + ({"aggressor_side": "hold"}, "invalid Canonical Trade"), + ({"source": " source "}, "invalid Canonical Trade"), + ( + {"first_observed_at": START.replace(tzinfo=None)}, + "invalid Canonical Trade", + ), + ( + { + "first_observed_at": START + timedelta(minutes=3), + "last_observed_at": START + timedelta(minutes=2), + }, + "invalid Canonical Trade", + ), + ({"observation_sources": ()}, "invalid Canonical Trade"), + ({"observation_sources": []}, "invalid Canonical Trade"), + ( + {"observation_sources": [SOURCE, SOURCE]}, + "invalid Canonical Trade", + ), + ( + {"observation_sources": ["recovery", SOURCE]}, + "invalid Canonical Trade", + ), + ({"replay_sequence": True}, "invalid Canonical Trade"), + ({"replay_sequence": 0}, "invalid Canonical Trade"), + ({"canonical_schema_version": 2}, "invalid Canonical Trade"), + ), +) +def test_corrupt_stored_values_raise_integrity_error( + overrides: dict[str, object], + error_match: str, +) -> None: + repository, cursor, connection, _ = dependencies( + [make_row(**overrides)] + ) + + with pytest.raises(MarketDataAccessIntegrityError, match=error_match): + repository.query_trades(make_query()) + + assert cursor.exit_exception_types == [MarketDataAccessIntegrityError] + assert connection.exit_exception_types == [MarketDataAccessIntegrityError] + + +@pytest.mark.parametrize("row", ((), [object()] * 13, (object(),) * 12)) +def test_invalid_row_shape_raises_integrity_error(row: object) -> None: + repository, _, _, _ = dependencies([row]) + + with pytest.raises(MarketDataAccessIntegrityError, match="row"): + repository.query_trades(make_query()) + + +def test_unordered_rows_and_duplicate_global_sequence_are_rejected() -> None: + later = START + timedelta(minutes=2) + earlier = START + timedelta(minutes=1) + unordered_repository, _, _, _ = dependencies( + [ + make_row( + executed_at=later, + first_observed_at=later + timedelta(seconds=1), + last_observed_at=later + timedelta(seconds=2), + replay_sequence=10, + ), + make_row( + executed_at=earlier, + first_observed_at=earlier + timedelta(seconds=1), + last_observed_at=earlier + timedelta(seconds=2), + replay_sequence=11, + ), + ] + ) + + with pytest.raises(MarketDataAccessIntegrityError, match="ordered"): + unordered_repository.query_trades(make_query()) + + duplicate_repository, _, _, _ = dependencies( + [ + make_row( + executed_at=earlier, + first_observed_at=earlier + timedelta(seconds=1), + last_observed_at=earlier + timedelta(seconds=2), + replay_sequence=10, + ), + make_row( + trade_id=101, + executed_at=later, + first_observed_at=later + timedelta(seconds=1), + last_observed_at=later + timedelta(seconds=2), + replay_sequence=10, + ), + ] + ) + + with pytest.raises( + MarketDataAccessIntegrityError, + match="duplicate replay_sequence", + ): + duplicate_repository.query_trades(make_query()) + + +def test_rows_must_strictly_follow_incoming_cursor() -> None: + cursor_time = START + timedelta(minutes=1) + incoming = TradeHistoryCursor( + venue=VENUE, + symbol=SYMBOL, + time_range=HistoricalTimeRange(START, END), + executed_at=cursor_time, + replay_sequence=10, + ) + repository, _, _, _ = dependencies( + [make_row(executed_at=cursor_time, replay_sequence=10)] + ) + + with pytest.raises(MarketDataAccessIntegrityError, match="cursor"): + repository.query_trades(make_query(cursor=incoming)) + + +def test_database_error_is_wrapped_after_context_cleanup() -> None: + repository, cursor, connection, _ = dependencies() + cursor.execute_error = RuntimeError("database failed") + + with pytest.raises(MarketDataAccessOperationError) as error_info: + repository.query_trades(make_query()) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert cursor.exit_exception_types == [RuntimeError] + assert connection.exit_exception_types == [RuntimeError] + + +def test_provider_error_is_wrapped_without_entering_connection() -> None: + repository, _, connection, provider = dependencies() + provider.error = RuntimeError("provider failed") + + with pytest.raises(MarketDataAccessOperationError) as error_info: + repository.query_trades(make_query()) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert provider.calls == 1 + assert connection.enter_calls == 0 + + +def test_keyboard_interrupt_is_not_wrapped_and_reaches_cleanup() -> None: + repository, cursor, connection, _ = dependencies() + cursor.execute_error = KeyboardInterrupt() + + with pytest.raises(KeyboardInterrupt): + repository.query_trades(make_query()) + + assert cursor.exit_exception_types == [KeyboardInterrupt] + assert connection.exit_exception_types == [KeyboardInterrupt] + + +def test_cleanup_error_obeys_exception_and_base_exception_contract() -> None: + repository, cursor, connection, _ = dependencies([]) + cursor.exit_error = RuntimeError("cursor cleanup failed") + + with pytest.raises(MarketDataAccessOperationError) as error_info: + repository.query_trades(make_query()) + + assert isinstance(error_info.value.__cause__, RuntimeError) + assert cursor.exit_exception_types == [None] + assert connection.exit_exception_types == [RuntimeError] + + repository, cursor, connection, _ = dependencies([]) + cursor.exit_error = KeyboardInterrupt() + + with pytest.raises(KeyboardInterrupt): + repository.query_trades(make_query()) + + assert cursor.exit_exception_types == [None] + assert connection.exit_exception_types == [KeyboardInterrupt] + + +def test_bound_provider_can_reuse_caller_owned_connection() -> None: + cursor = RecordingCursor([make_row()]) + connection = RecordingConnection(cursor) + provider_calls = 0 + + def bound_provider() -> Any: + nonlocal provider_calls + provider_calls += 1 + return nullcontext(connection) + + repository = PostgresTradeHistoryRepository( + connection_provider=bound_provider, + ) + + page = repository.query_trades(make_query()) + + assert len(page.items) == 1 + assert provider_calls == 1 + assert connection.enter_calls == 0 + assert connection.exit_exception_types == [] + assert cursor.exit_exception_types == [None] diff --git a/app/tests/unit/market_data/replay/conftest.py b/app/tests/unit/market_data/replay/conftest.py new file mode 100644 index 0000000..b8eb820 --- /dev/null +++ b/app/tests/unit/market_data/replay/conftest.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest + +from src.market_data.access import HistoricalTimeRange +from src.market_data.replay import ( + ReplayDataType, + ReplayPlan, + ReplayPlanRequest, +) + + +@pytest.fixture +def empty_replay_plan() -> ReplayPlan: + start = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) + request = ReplayPlanRequest( + venue="dzengi", + symbols=("BTC/USD_LEVERAGE",), + data_types=(ReplayDataType.TRADE,), + time_range=HistoricalTimeRange( + start_time=start, + end_time=start + timedelta(hours=1), + ), + max_records=100, + ) + return ReplayPlan(request=request, events=()) diff --git a/app/tests/unit/market_data/replay/test_deterministic_replay_clock.py b/app/tests/unit/market_data/replay/test_deterministic_replay_clock.py new file mode 100644 index 0000000..790caf3 --- /dev/null +++ b/app/tests/unit/market_data/replay/test_deterministic_replay_clock.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +import inspect + +import pytest + +from src.market_data.replay.contracts import ( + MarketDataClockProtocol, + ReplayClockProtocol, +) +from src.market_data.replay.deterministic_replay_clock import ( + DeterministicReplayClock, +) +from src.market_data.replay.exceptions import ReplayClockError + + +START = datetime( + 2026, + 8, + 2, + 12, + 0, + 0, + 123456, + tzinfo=timezone.utc, +) + + +def test_matches_protocols_uses_slots_and_is_synchronous() -> None: + clock = DeterministicReplayClock(START) + + assert isinstance(clock, MarketDataClockProtocol) + assert isinstance(clock, ReplayClockProtocol) + assert not hasattr(clock, "__dict__") + assert inspect.iscoroutinefunction(clock.advance_to) is False + + +def test_starts_at_exact_canonical_utc_time() -> None: + clock = DeterministicReplayClock(START) + + assert clock.now == START + assert type(clock.now) is datetime + assert clock.now.tzinfo is timezone.utc + + +def test_normalizes_non_utc_initial_time_without_losing_precision() -> None: + offset = timezone(timedelta(hours=3, minutes=30)) + initial_time = datetime( + 2026, + 8, + 2, + 15, + 30, + 0, + 654321, + tzinfo=offset, + ) + + clock = DeterministicReplayClock(initial_time) + + assert clock.now == datetime( + 2026, + 8, + 2, + 12, + 0, + 0, + 654321, + tzinfo=timezone.utc, + ) + assert type(clock.now) is datetime + + +def test_accepts_datetime_subclass_but_stores_base_datetime() -> None: + class CompatibleDatetime(datetime): + pass + + initial_time = CompatibleDatetime( + 2026, + 8, + 2, + 12, + 0, + tzinfo=timezone.utc, + ) + + clock = DeterministicReplayClock(initial_time) + + assert clock.now == initial_time + assert type(clock.now) is datetime + + +@pytest.mark.parametrize("invalid", (None, "2026-08-02", 1, object())) +def test_rejects_non_datetime_initial_value(invalid: object) -> None: + with pytest.raises(TypeError, match="initial_time"): + DeterministicReplayClock(invalid) # type: ignore[arg-type] + + +def test_rejects_naive_initial_time() -> None: + with pytest.raises(ValueError, match="timezone"): + DeterministicReplayClock(START.replace(tzinfo=None)) + + +def test_advances_forward_with_microsecond_precision() -> None: + clock = DeterministicReplayClock(START) + expected = START + timedelta(microseconds=1) + + result = clock.advance_to(expected) + + assert result is None + assert clock.now == expected + + +def test_allows_repeated_equal_time_for_distinct_replay_events() -> None: + clock = DeterministicReplayClock(START) + + clock.advance_to(START) + clock.advance_to(START) + + assert clock.now == START + + +def test_equal_instant_with_another_offset_is_idempotent() -> None: + clock = DeterministicReplayClock(START) + equal_instant = START.astimezone(timezone(timedelta(hours=-4))) + + clock.advance_to(equal_instant) + + assert clock.now == START + assert clock.now.tzinfo is timezone.utc + + +def test_equal_utc_instant_with_another_fold_is_complete_no_op() -> None: + clock = DeterministicReplayClock(START) + previous = clock.now + + clock.advance_to(START.replace(fold=1)) + + assert clock.now is previous + assert clock.now.fold == 0 + + +def test_accepts_datetime_subclass_during_advance() -> None: + class CompatibleDatetime(datetime): + pass + + clock = DeterministicReplayClock(START) + later = CompatibleDatetime( + 2026, + 8, + 2, + 12, + 1, + tzinfo=timezone.utc, + ) + + clock.advance_to(later) + + assert clock.now == later + assert type(clock.now) is datetime + + +def test_rejects_backward_transition_without_changing_state() -> None: + clock = DeterministicReplayClock(START) + later = START + timedelta(minutes=1) + clock.advance_to(later) + + with pytest.raises(ReplayClockError, match="backwards"): + clock.advance_to(START) + + assert clock.now == later + + +@pytest.mark.parametrize("invalid", (None, "later", 1, object())) +def test_invalid_advance_type_does_not_change_state(invalid: object) -> None: + clock = DeterministicReplayClock(START) + + with pytest.raises(TypeError, match="instant"): + clock.advance_to(invalid) # type: ignore[arg-type] + + assert clock.now == START + + +def test_naive_advance_does_not_change_state() -> None: + clock = DeterministicReplayClock(START) + + with pytest.raises(ValueError, match="timezone"): + clock.advance_to(START.replace(tzinfo=None)) + + assert clock.now == START + + +def test_two_clocks_have_independent_state() -> None: + first = DeterministicReplayClock(START) + second = DeterministicReplayClock(START) + + first.advance_to(START + timedelta(hours=1)) + + assert first.now == START + timedelta(hours=1) + assert second.now == START + + +def test_now_is_read_only_and_lifecycle_extensions_are_absent() -> None: + clock = DeterministicReplayClock(START) + + with pytest.raises(AttributeError): + setattr(clock, "now", START + timedelta(hours=1)) + + assert not hasattr(clock, "reset") + assert not hasattr(clock, "advance_by") + assert not hasattr(clock, "start") + assert not hasattr(clock, "stop") + assert clock.now == START diff --git a/app/tests/unit/market_data/replay/test_market_data_replay_contracts.py b/app/tests/unit/market_data/replay/test_market_data_replay_contracts.py new file mode 100644 index 0000000..bdb7490 --- /dev/null +++ b/app/tests/unit/market_data/replay/test_market_data_replay_contracts.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from datetime import datetime + +from src.market_data.replay import ( + MarketDataClockProtocol, + MarketDataReplayError, + MarketDataReplayValidationError, + ReplayClockError, + ReplayClockProtocol, + ReplayConsumerProtocol, + ReplayEvent, + ReplayPlan, + ReplayPlanBuilderProtocol, + ReplayPlanLimitExceededError, + ReplayPlanRequest, + ReplaySessionProtocol, + ReplaySessionState, + ReplaySessionStateError, +) + + +class FakeClock: + def __init__(self, now: datetime) -> None: + self._now = now + + @property + def now(self) -> datetime: + return self._now + + def advance_to(self, instant: datetime) -> None: + self._now = instant + + +class RecordingConsumer: + def __init__(self) -> None: + self.events: list[ReplayEvent] = [] + + async def consume(self, event: ReplayEvent) -> None: + self.events.append(event) + + +class RecordingPlanBuilder: + def __init__(self, plan: ReplayPlan) -> None: + self.plan = plan + + def create_plan(self, request: ReplayPlanRequest) -> ReplayPlan: + return self.plan + + +class RecordingSession: + def __init__(self, plan: ReplayPlan, clock: FakeClock) -> None: + self._plan = plan + self._clock = clock + + @property + def state(self) -> ReplaySessionState: + return ReplaySessionState.CREATED + + @property + def plan(self) -> ReplayPlan: + return self._plan + + @property + def clock(self) -> MarketDataClockProtocol: + return self._clock + + async def run(self) -> None: + return None + + +def test_replay_protocols_are_runtime_checkable( + empty_replay_plan: ReplayPlan, +) -> None: + clock = FakeClock(empty_replay_plan.request.time_range.start_time) + + assert isinstance(clock, MarketDataClockProtocol) + assert isinstance(clock, ReplayClockProtocol) + assert isinstance(RecordingConsumer(), ReplayConsumerProtocol) + assert isinstance( + RecordingPlanBuilder(empty_replay_plan), + ReplayPlanBuilderProtocol, + ) + assert isinstance( + RecordingSession(empty_replay_plan, clock), + ReplaySessionProtocol, + ) + + +def test_replay_error_hierarchy_is_specialized() -> None: + assert issubclass(MarketDataReplayValidationError, MarketDataReplayError) + assert issubclass(ReplayPlanLimitExceededError, MarketDataReplayError) + assert issubclass(ReplayClockError, MarketDataReplayError) + assert issubclass(ReplaySessionStateError, MarketDataReplayError) diff --git a/app/tests/unit/market_data/replay/test_market_data_replay_models.py b/app/tests/unit/market_data/replay/test_market_data_replay_models.py new file mode 100644 index 0000000..814e344 --- /dev/null +++ b/app/tests/unit/market_data/replay/test_market_data_replay_models.py @@ -0,0 +1,659 @@ +from __future__ import annotations + +from dataclasses import FrozenInstanceError +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any + +import pytest + +from src.market_data.access import HistoricalTimeRange +from src.market_data.acquisition.models.candle import Candle +from src.market_data.acquisition.models.quote import Quote +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.replay import ( + REPLAY_PLAN_MAX_RECORDS_LIMIT, + ReplayDataType, + ReplayEvent, + ReplayPlan, + ReplayPlanRequest, + ReplaySessionState, +) + + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +START = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) +END = START + timedelta(hours=1) + + +def make_trade( + *, + trade_id: int = 100, + executed_at: datetime = START + timedelta(minutes=1), + symbol: str = SYMBOL, +) -> Trade: + return Trade( + symbol=symbol, + trade_id=trade_id, + price=Decimal("65000"), + quantity=Decimal("0.001"), + executed_at=executed_at, + aggressor_side=TradeAggressorSide.BUY, + source="dzengi_websocket_trade", + ) + + +def make_quote( + *, + received_at: datetime = START + timedelta(minutes=2), +) -> Quote: + return Quote( + symbol=SYMBOL, + last_price=Decimal("65000"), + bid_price=Decimal("64999"), + ask_price=Decimal("65001"), + exchange_timestamp=received_at - timedelta(milliseconds=1), + received_at=received_at, + source="dzengi_rest_quote", + ) + + +def make_candle(*, interval: str = "1m") -> Candle: + return Candle( + symbol=SYMBOL, + interval=interval, + open_time=START, + open_price=Decimal("64900"), + high_price=Decimal("65100"), + low_price=Decimal("64800"), + close_price=Decimal("65000"), + volume=Decimal("10"), + source="dzengi_rest_candle", + ) + + +def make_request( + *, + data_types: tuple[ReplayDataType, ...] = (ReplayDataType.TRADE,), + candle_intervals: tuple[str, ...] = (), + max_records: int = 100, +) -> ReplayPlanRequest: + return ReplayPlanRequest( + venue=VENUE, + symbols=(SYMBOL,), + data_types=data_types, + time_range=HistoricalTimeRange( + start_time=START, + end_time=END, + ), + candle_intervals=candle_intervals, + max_records=max_records, + ) + + +def make_trade_event( + *, + trade_id: int = 100, + replay_at: datetime = START + timedelta(minutes=1), + replay_sequence: int = 1, + venue: str = VENUE, + symbol: str = SYMBOL, +) -> ReplayEvent: + trade = make_trade( + trade_id=trade_id, + executed_at=replay_at, + symbol=symbol, + ) + return ReplayEvent( + venue=venue, + replay_at=replay_at, + replay_sequence=replay_sequence, + payload=trade, + ) + + +def trade_id_from_event(event: ReplayEvent) -> int: + payload = event.payload + + if not isinstance(payload, Trade): + raise AssertionError("Replay event must contain Trade payload.") + + return payload.trade_id + + +def test_trade_event_preserves_payload_identity_and_exact_event_time() -> None: + trade = make_trade() + event = ReplayEvent( + venue=" dzengi ", + replay_at=trade.executed_at, + replay_sequence=10, + payload=trade, + ) + + assert event.venue == VENUE + assert event.payload is trade + assert event.symbol == SYMBOL + assert event.data_type is ReplayDataType.TRADE + assert event.order_key == (trade.executed_at, 10) + assert event.candle_is_final is None + + +def test_quote_event_uses_received_time_not_exchange_time() -> None: + quote = make_quote() + event = ReplayEvent( + venue=VENUE, + replay_at=quote.received_at, + replay_sequence=11, + payload=quote, + ) + + assert event.data_type is ReplayDataType.QUOTE + assert event.replay_at == quote.received_at + assert event.replay_at != quote.exchange_timestamp + + +def test_candle_event_uses_observation_time_and_final_metadata() -> None: + candle = make_candle() + observed_at = candle.open_time + timedelta(seconds=30) + event = ReplayEvent( + venue=VENUE, + replay_at=observed_at, + replay_sequence=12, + payload=candle, + candle_is_final=False, + ) + + assert event.data_type is ReplayDataType.CANDLE_REVISION + assert event.replay_at == observed_at + assert event.candle_is_final is False + + +def test_event_normalizes_aware_time_to_utc() -> None: + offset = timezone(timedelta(hours=3)) + trade = make_trade() + event = ReplayEvent( + venue=VENUE, + replay_at=trade.executed_at.astimezone(offset), + replay_sequence=1, + payload=trade, + ) + + assert event.replay_at.tzinfo is timezone.utc + assert event.replay_at == trade.executed_at + + +@pytest.mark.parametrize("invalid_sequence", (True, 0, -1, 1.5, "1")) +def test_event_rejects_invalid_sequence(invalid_sequence: Any) -> None: + trade = make_trade() + + with pytest.raises((TypeError, ValueError), match="replay_sequence"): + ReplayEvent( + venue=VENUE, + replay_at=trade.executed_at, + replay_sequence=invalid_sequence, + payload=trade, + ) + + +def test_trade_event_rejects_mismatched_replay_time() -> None: + trade = make_trade() + + with pytest.raises(ValueError, match="event time"): + ReplayEvent( + venue=VENUE, + replay_at=trade.executed_at + timedelta(microseconds=1), + replay_sequence=1, + payload=trade, + ) + + +def test_candle_event_rejects_time_before_open() -> None: + candle = make_candle() + + with pytest.raises(ValueError, match="open_time"): + ReplayEvent( + venue=VENUE, + replay_at=candle.open_time - timedelta(microseconds=1), + replay_sequence=1, + payload=candle, + candle_is_final=False, + ) + + +@pytest.mark.parametrize("candle_is_final", (None, 0, 1, "true")) +def test_candle_event_requires_exact_final_boolean( + candle_is_final: Any, +) -> None: + candle = make_candle() + + with pytest.raises(TypeError, match="candle_is_final"): + ReplayEvent( + venue=VENUE, + replay_at=candle.open_time, + replay_sequence=1, + payload=candle, + candle_is_final=candle_is_final, + ) + + +def test_trade_event_rejects_candle_metadata() -> None: + trade = make_trade() + + with pytest.raises(ValueError, match="must be None"): + ReplayEvent( + venue=VENUE, + replay_at=trade.executed_at, + replay_sequence=1, + payload=trade, + candle_is_final=False, + ) + + +def test_request_normalizes_symbols_and_preserves_interval_case() -> None: + request = ReplayPlanRequest( + venue=" dzengi ", + symbols=(" btc/usd_leverage ",), + data_types=(ReplayDataType.CANDLE_REVISION,), + time_range=HistoricalTimeRange(START, END), + candle_intervals=(" 1M ",), + ) + + assert request.venue == VENUE + assert request.symbols == (SYMBOL,) + assert request.candle_intervals == ("1M",) + + +@pytest.mark.parametrize( + "symbols", + ( + [], + (), + ("",), + ("BTC/USD_LEVERAGE", "btc/usd_leverage"), + ), +) +def test_request_rejects_invalid_symbols(symbols: Any) -> None: + with pytest.raises((TypeError, ValueError)): + ReplayPlanRequest( + venue=VENUE, + symbols=symbols, + data_types=(ReplayDataType.TRADE,), + time_range=HistoricalTimeRange(START, END), + ) + + +def test_request_requires_intervals_only_for_candles() -> None: + with pytest.raises(ValueError, match="required"): + make_request(data_types=(ReplayDataType.CANDLE_REVISION,)) + + with pytest.raises(ValueError, match="require Candle"): + make_request(candle_intervals=("1m",)) + + +@pytest.mark.parametrize( + "max_records", + (True, 0, -1, 1.5, REPLAY_PLAN_MAX_RECORDS_LIMIT + 1), +) +def test_request_rejects_invalid_max_records(max_records: Any) -> None: + with pytest.raises((TypeError, ValueError), match="max_records"): + make_request(max_records=max_records) + + +def test_empty_plan_is_valid_and_preserves_request_scope() -> None: + request = make_request() + plan = ReplayPlan(request=request, events=()) + + assert plan.request is request + assert plan.events == () + assert plan.is_empty is True + assert len(plan) == 0 + + +def test_plan_preserves_event_and_payload_identity() -> None: + request = make_request() + event = make_trade_event() + plan = ReplayPlan(request=request, events=(event,)) + + assert plan.events[0] is event + assert plan.events[0].payload is event.payload + + +def test_plan_accepts_rollover_order_by_time_and_sequence() -> None: + same_time = START + timedelta(minutes=1) + first = make_trade_event( + trade_id=2_147_483_647, + replay_at=same_time, + replay_sequence=10, + ) + second = make_trade_event( + trade_id=-2_147_483_648, + replay_at=same_time, + replay_sequence=11, + ) + + plan = ReplayPlan( + request=make_request(), + events=(first, second), + ) + + assert [trade_id_from_event(event) for event in plan.events] == [ + 2_147_483_647, + -2_147_483_648, + ] + + +def test_plan_accepts_negative_one_to_zero_at_equal_time() -> None: + same_time = START + timedelta(minutes=1) + plan = ReplayPlan( + request=make_request(), + events=( + make_trade_event( + trade_id=-1, + replay_at=same_time, + replay_sequence=20, + ), + make_trade_event( + trade_id=0, + replay_at=same_time, + replay_sequence=21, + ), + ), + ) + + assert [trade_id_from_event(event) for event in plan.events] == [-1, 0] + + +def test_plan_rejects_reverse_order() -> None: + with pytest.raises(ValueError, match="strictly ordered"): + ReplayPlan( + request=make_request(), + events=( + make_trade_event(replay_sequence=2), + make_trade_event(replay_sequence=1), + ), + ) + + +def test_plan_rejects_reverse_event_time_with_increasing_sequence() -> None: + with pytest.raises(ValueError, match="strictly ordered"): + ReplayPlan( + request=make_request(), + events=( + make_trade_event( + replay_at=START + timedelta(minutes=2), + replay_sequence=1, + ), + make_trade_event( + replay_at=START + timedelta(minutes=1), + replay_sequence=2, + ), + ), + ) + + +def test_plan_rejects_duplicate_global_sequence() -> None: + with pytest.raises(ValueError, match="globally unique"): + ReplayPlan( + request=make_request(), + events=( + make_trade_event( + replay_at=START + timedelta(minutes=1), + replay_sequence=1, + ), + make_trade_event( + replay_at=START + timedelta(minutes=2), + replay_sequence=1, + ), + ), + ) + + +def test_plan_rejects_event_outside_time_range() -> None: + event = make_trade_event(replay_at=END) + + with pytest.raises(ValueError, match="outside Replay request"): + ReplayPlan(request=make_request(), events=(event,)) + + +def test_plan_rejects_data_type_outside_request() -> None: + quote = make_quote() + event = ReplayEvent( + venue=VENUE, + replay_at=quote.received_at, + replay_sequence=1, + payload=quote, + ) + + with pytest.raises(ValueError, match="data type"): + ReplayPlan(request=make_request(), events=(event,)) + + +def test_plan_rejects_event_from_another_venue() -> None: + event = make_trade_event(venue="other") + + with pytest.raises(ValueError, match="event venue"): + ReplayPlan(request=make_request(), events=(event,)) + + +def test_plan_rejects_event_from_another_symbol() -> None: + event = make_trade_event(symbol="ETH/USD_LEVERAGE") + + with pytest.raises(ValueError, match="event symbol"): + ReplayPlan(request=make_request(), events=(event,)) + + +def test_plan_rejects_candle_interval_outside_request() -> None: + candle = make_candle(interval="5m") + event = ReplayEvent( + venue=VENUE, + replay_at=candle.open_time, + replay_sequence=1, + payload=candle, + candle_is_final=False, + ) + request = make_request( + data_types=(ReplayDataType.CANDLE_REVISION,), + candle_intervals=("1m",), + ) + + with pytest.raises(ValueError, match="Candle interval"): + ReplayPlan(request=request, events=(event,)) + + +def test_plan_rejects_more_events_than_request_limit() -> None: + event = make_trade_event() + + with pytest.raises(ValueError, match="max_records"): + ReplayPlan( + request=make_request(max_records=1), + events=(event, make_trade_event(replay_sequence=2)), + ) + + +def test_replay_models_are_frozen_and_slotted() -> None: + event = make_trade_event() + request = make_request() + plan = ReplayPlan(request=request, events=(event,)) + + for model in (event, request, plan): + assert not hasattr(model, "__dict__") + + with pytest.raises(FrozenInstanceError): + setattr(event, "venue", "other") + + +def test_session_states_are_explicit_and_complete() -> None: + assert tuple(ReplaySessionState) == ( + ReplaySessionState.CREATED, + ReplaySessionState.RUNNING, + ReplaySessionState.COMPLETED, + ReplaySessionState.FAILED, + ReplaySessionState.CANCELLED, + ) + + +def test_event_rejects_canonical_payload_subclasses() -> None: + class TradeSubclass(Trade): + pass + + class QuoteSubclass(Quote): + pass + + class CandleSubclass(Candle): + pass + + trade = make_trade() + quote = make_quote() + candle = make_candle() + + payloads = ( + ( + TradeSubclass( + symbol=trade.symbol, + trade_id=trade.trade_id, + price=trade.price, + quantity=trade.quantity, + executed_at=trade.executed_at, + aggressor_side=trade.aggressor_side, + source=trade.source, + ), + trade.executed_at, + None, + ), + ( + QuoteSubclass( + symbol=quote.symbol, + last_price=quote.last_price, + bid_price=quote.bid_price, + ask_price=quote.ask_price, + exchange_timestamp=quote.exchange_timestamp, + received_at=quote.received_at, + source=quote.source, + ), + quote.received_at, + None, + ), + ( + CandleSubclass( + symbol=candle.symbol, + interval=candle.interval, + open_time=candle.open_time, + open_price=candle.open_price, + high_price=candle.high_price, + low_price=candle.low_price, + close_price=candle.close_price, + volume=candle.volume, + source=candle.source, + ), + candle.open_time, + False, + ), + ) + + for payload, replay_at, candle_is_final in payloads: + with pytest.raises(TypeError, match="Canonical Trade, Quote or Candle"): + ReplayEvent( + venue=VENUE, + replay_at=replay_at, + replay_sequence=1, + payload=payload, + candle_is_final=candle_is_final, + ) + + +def test_request_rejects_time_range_subclass() -> None: + class HistoricalTimeRangeSubclass(HistoricalTimeRange): + def contains(self, instant: datetime) -> bool: + return True + + with pytest.raises(TypeError, match="HistoricalTimeRange"): + ReplayPlanRequest( + venue=VENUE, + symbols=(SYMBOL,), + data_types=(ReplayDataType.TRADE,), + time_range=HistoricalTimeRangeSubclass(START, END), + ) + + +def test_request_rejects_tuple_subclasses() -> None: + class TupleSubclass(tuple): + pass + + with pytest.raises(TypeError, match="symbols must be a tuple"): + ReplayPlanRequest( + venue=VENUE, + symbols=TupleSubclass((SYMBOL,)), + data_types=(ReplayDataType.TRADE,), + time_range=HistoricalTimeRange(START, END), + ) + + with pytest.raises(TypeError, match="data_types must be a tuple"): + ReplayPlanRequest( + venue=VENUE, + symbols=(SYMBOL,), + data_types=TupleSubclass((ReplayDataType.TRADE,)), + time_range=HistoricalTimeRange(START, END), + ) + + with pytest.raises(TypeError, match="candle_intervals must be a tuple"): + ReplayPlanRequest( + venue=VENUE, + symbols=(SYMBOL,), + data_types=(ReplayDataType.CANDLE_REVISION,), + time_range=HistoricalTimeRange(START, END), + candle_intervals=TupleSubclass(("1m",)), + ) + + +def test_plan_rejects_request_and_event_subclasses() -> None: + class ReplayPlanRequestSubclass(ReplayPlanRequest): + pass + + class ReplayEventSubclass(ReplayEvent): + @property + def symbol(self) -> str: + return SYMBOL + + @property + def data_type(self) -> ReplayDataType: + return ReplayDataType.TRADE + + @property + def order_key(self) -> tuple[datetime, int]: + return (START, 1) + + request_subclass = ReplayPlanRequestSubclass( + venue=VENUE, + symbols=(SYMBOL,), + data_types=(ReplayDataType.TRADE,), + time_range=HistoricalTimeRange(START, END), + ) + + with pytest.raises(TypeError, match="ReplayPlanRequest"): + ReplayPlan(request=request_subclass, events=()) + + event = make_trade_event() + event_subclass = ReplayEventSubclass( + venue=event.venue, + replay_at=event.replay_at, + replay_sequence=event.replay_sequence, + payload=event.payload, + ) + + with pytest.raises(TypeError, match="ReplayEvent"): + ReplayPlan(request=make_request(), events=(event_subclass,)) + + +def test_plan_rejects_events_tuple_subclass() -> None: + class EventsTupleSubclass(tuple): + pass + + with pytest.raises(TypeError, match="events must be a tuple"): + ReplayPlan( + request=make_request(), + events=EventsTupleSubclass((make_trade_event(),)), + ) diff --git a/app/tests/unit/market_data/replay/test_postgres_replay_plan_builder.py b/app/tests/unit/market_data/replay/test_postgres_replay_plan_builder.py new file mode 100644 index 0000000..983524f --- /dev/null +++ b/app/tests/unit/market_data/replay/test_postgres_replay_plan_builder.py @@ -0,0 +1,699 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any + +import pytest + +from src.market_data.access.exceptions import ( + MarketDataAccessIntegrityError, + MarketDataAccessOperationError, +) +from src.market_data.access.models import HistoricalTimeRange +from src.market_data.acquisition.models.candle import Candle +from src.market_data.acquisition.models.quote import Quote +from src.market_data.acquisition.models.trade import Trade +from src.market_data.replay.contracts import ReplayPlanBuilderProtocol +from src.market_data.replay.exceptions import ( + MarketDataReplayValidationError, + ReplayPlanLimitExceededError, +) +from src.market_data.replay.models import ( + ReplayDataType, + ReplayPlanRequest, +) +from src.market_data.replay.postgres_replay_plan_builder import ( + PostgresReplayPlanBuilder, +) + + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +START = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) +END = START + timedelta(hours=1) +TRADE_SOURCE = "dzengi_websocket_trade" +QUOTE_SOURCE = "dzengi_rest_quote" +CANDLE_SOURCE = "dzengi_rest_candle" + + +class RecordingCursor: + def __init__( + self, + *, + events: list[tuple[object, ...]], + rows: object = (), + ) -> None: + self._events = events + self.rows = rows + self.calls: list[tuple[str, object]] = [] + self.execute_errors: list[BaseException | None] = [] + self.fetchall_error: BaseException | None = None + + def __enter__(self) -> RecordingCursor: + self._events.append(("cursor_enter",)) + return self + + def __exit__( + self, + exception_type: type[BaseException] | None, + exception: BaseException | None, + traceback: object, + ) -> None: + self._events.append(("cursor_exit", exception_type)) + return None + + def execute( + self, + sql: str, + parameters: object = None, + ) -> None: + self.calls.append((sql, parameters)) + self._events.append(("execute", normalized_sql(sql), parameters)) + + error = ( + self.execute_errors.pop(0) + if self.execute_errors + else None + ) + + if error is not None: + raise error + + def fetchall(self) -> object: + self._events.append(("fetchall",)) + + if self.fetchall_error is not None: + raise self.fetchall_error + + return self.rows + + +class RecordingTransaction: + def __init__(self, events: list[tuple[object, ...]]) -> None: + self._events = events + + def __enter__(self) -> RecordingTransaction: + self._events.append(("transaction_enter",)) + return self + + def __exit__( + self, + exception_type: type[BaseException] | None, + exception: BaseException | None, + traceback: object, + ) -> None: + self._events.append(("transaction_exit", exception_type)) + return None + + +class RecordingConnection: + def __init__( + self, + *, + events: list[tuple[object, ...]], + cursor: RecordingCursor, + ) -> None: + self._events = events + self._cursor = cursor + self.transaction_calls = 0 + self.cursor_calls = 0 + + def __enter__(self) -> RecordingConnection: + self._events.append(("connection_enter",)) + return self + + def __exit__( + self, + exception_type: type[BaseException] | None, + exception: BaseException | None, + traceback: object, + ) -> None: + self._events.append(("connection_exit", exception_type)) + return None + + def transaction(self) -> RecordingTransaction: + self.transaction_calls += 1 + self._events.append(("transaction",)) + return RecordingTransaction(self._events) + + def cursor(self) -> RecordingCursor: + self.cursor_calls += 1 + self._events.append(("cursor",)) + return self._cursor + + +@dataclass +class RecordingProvider: + connection: RecordingConnection + events: list[tuple[object, ...]] + calls: int = 0 + error: BaseException | None = None + + def __call__(self) -> RecordingConnection: + self.calls += 1 + self.events.append(("provider",)) + + if self.error is not None: + raise self.error + + return self.connection + + +def normalized_sql(sql: str) -> str: + return " ".join(sql.split()) + + +def make_request( + *, + data_types: tuple[ReplayDataType, ...] = (ReplayDataType.TRADE,), + candle_intervals: tuple[str, ...] = (), + max_records: int = 10, + venue: str = VENUE, +) -> ReplayPlanRequest: + return ReplayPlanRequest( + venue=venue, + symbols=(SYMBOL,), + data_types=data_types, + time_range=HistoricalTimeRange(START, END), + candle_intervals=candle_intervals, + max_records=max_records, + ) + + +def make_trade_row( + *, + replay_at: object = START + timedelta(minutes=1), + replay_sequence: object = 10, + venue: object = VENUE, + trade_id: object = 100, + price: object = Decimal("65000"), +) -> tuple[object, ...]: + return ( + "trade", + replay_at, + replay_sequence, + venue, + SYMBOL, + trade_id, + replay_at, + price, + Decimal("0.01"), + "buy", + TRADE_SOURCE, + START + timedelta(seconds=1), + START + timedelta(seconds=2), + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + [TRADE_SOURCE], + 1, + ) + + +def make_quote_row( + *, + replay_at: object = START + timedelta(minutes=2), + replay_sequence: object = 11, +) -> tuple[object, ...]: + return ( + "quote", + replay_at, + replay_sequence, + VENUE, + SYMBOL, + None, + None, + None, + None, + None, + QUOTE_SOURCE, + None, + None, + replay_at, + START + timedelta(minutes=2) - timedelta(milliseconds=1), + Decimal("65001"), + Decimal("65000"), + Decimal("65002"), + None, + None, + None, + None, + None, + None, + None, + None, + None, + [QUOTE_SOURCE], + 1, + ) + + +def make_candle_row( + *, + replay_at: object = START + timedelta(minutes=3), + replay_sequence: object = 12, + open_time: object = START - timedelta(hours=1), + interval: object = "1m", + is_final: object = True, +) -> tuple[object, ...]: + return ( + "candle_revision", + replay_at, + replay_sequence, + VENUE, + SYMBOL, + None, + None, + None, + None, + None, + CANDLE_SOURCE, + None, + None, + None, + None, + None, + None, + None, + interval, + open_time, + replay_at, + Decimal("64900"), + Decimal("65100"), + Decimal("64800"), + Decimal("65000"), + Decimal("10"), + is_final, + [CANDLE_SOURCE], + 1, + ) + + +def dependencies( + rows: object = (), +) -> tuple[ + PostgresReplayPlanBuilder, + RecordingCursor, + RecordingConnection, + RecordingProvider, + list[tuple[object, ...]], +]: + events: list[tuple[object, ...]] = [] + cursor = RecordingCursor(events=events, rows=rows) + connection = RecordingConnection(events=events, cursor=cursor) + provider = RecordingProvider(connection=connection, events=events) + builder = PostgresReplayPlanBuilder(connection_provider=provider) + return builder, cursor, connection, provider, events + + +def test_constructor_is_no_io_slotted_and_matches_protocol() -> None: + builder, _, _, provider, _ = dependencies() + + assert provider.calls == 0 + assert not hasattr(builder, "__dict__") + assert isinstance(builder, ReplayPlanBuilderProtocol) + + +def test_constructor_rejects_non_callable_provider() -> None: + with pytest.raises(TypeError, match="connection_provider"): + PostgresReplayPlanBuilder( + connection_provider=None, # type: ignore[arg-type] + ) + + +def test_exact_request_is_validated_before_connection_borrow() -> None: + class RequestSubclass(ReplayPlanRequest): + pass + + builder, _, _, provider, _ = dependencies() + request = RequestSubclass( + venue=VENUE, + symbols=(SYMBOL,), + data_types=(ReplayDataType.TRADE,), + time_range=HistoricalTimeRange(START, END), + ) + + with pytest.raises(MarketDataReplayValidationError, match="request"): + builder.create_plan(request) + + assert provider.calls == 0 + + +def test_trade_snapshot_uses_one_half_open_parameterized_query() -> None: + builder, cursor, connection, provider, events = dependencies() + request = make_request(max_records=7, venue="tenant'value") + + plan = builder.create_plan(request) + + assert plan.is_empty is True + assert provider.calls == 1 + assert connection.transaction_calls == 1 + assert connection.cursor_calls == 1 + assert len(cursor.calls) == 2 + setup_sql, setup_parameters = cursor.calls[0] + sql, parameters = cursor.calls[1] + compact_sql = normalized_sql(sql) + assert normalized_sql(setup_sql) == ( + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY" + ) + assert setup_parameters is None + assert "FROM market_data.trades" in compact_sql + assert "FROM market_data.quotes" not in compact_sql + assert "FROM market_data.candle_revisions" not in compact_sql + assert "executed_at >= %s" in compact_sql + assert "executed_at < %s" in compact_sql + assert "ORDER BY replay_at ASC, replay_sequence ASC" in compact_sql + assert compact_sql.endswith("LIMIT %s") + assert "tenant'value" not in sql + assert parameters == ( + "tenant'value", + [SYMBOL], + START, + END, + 8, + ) + assert events == [ + ("provider",), + ("connection_enter",), + ("transaction",), + ("transaction_enter",), + ("cursor",), + ("cursor_enter",), + ( + "execute", + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY", + None, + ), + ("execute", compact_sql, parameters), + ("fetchall",), + ("cursor_exit", None), + ("transaction_exit", None), + ("connection_exit", None), + ] + + +def test_all_requested_types_use_static_union_and_own_time_axes() -> None: + builder, cursor, _, _, _ = dependencies() + request = make_request( + data_types=( + ReplayDataType.CANDLE_REVISION, + ReplayDataType.TRADE, + ReplayDataType.QUOTE, + ), + candle_intervals=("1m", "5m"), + max_records=20, + ) + + builder.create_plan(request) + + sql, parameters = cursor.calls[1] + compact_sql = normalized_sql(sql) + assert compact_sql.count("UNION ALL") == 2 + assert "executed_at >= %s" in compact_sql + assert "received_at >= %s" in compact_sql + assert "observed_at >= %s" in compact_sql + assert "open_time >= %s" not in compact_sql + assert "interval = ANY(%s)" in compact_sql + assert parameters == ( + VENUE, + [SYMBOL], + START, + END, + VENUE, + [SYMBOL], + START, + END, + VENUE, + [SYMBOL], + START, + END, + ["1m", "5m"], + 21, + ) + + +def test_materializes_globally_ordered_canonical_events() -> None: + event_time = START + timedelta(minutes=2) + builder, _, _, _, _ = dependencies( + [ + make_trade_row( + replay_at=START + timedelta(minutes=1), + replay_sequence=10, + ), + make_quote_row( + replay_at=event_time, + replay_sequence=11, + ), + make_candle_row( + replay_at=event_time, + replay_sequence=12, + ), + ] + ) + request = make_request( + data_types=( + ReplayDataType.TRADE, + ReplayDataType.QUOTE, + ReplayDataType.CANDLE_REVISION, + ), + candle_intervals=("1m",), + ) + + plan = builder.create_plan(request) + + assert [event.replay_sequence for event in plan.events] == [10, 11, 12] + assert type(plan.events[0].payload) is Trade + assert type(plan.events[1].payload) is Quote + candle_payload = plan.events[2].payload + assert type(candle_payload) is Candle + assert isinstance(candle_payload, Candle) + assert plan.events[2].replay_at == event_time + assert candle_payload.open_time < START + assert plan.events[2].candle_is_final is True + + +def test_candle_snapshot_filters_by_observed_time_not_open_time() -> None: + builder, cursor, _, _, _ = dependencies( + [make_candle_row(open_time=START - timedelta(days=1))] + ) + request = make_request( + data_types=(ReplayDataType.CANDLE_REVISION,), + candle_intervals=("1m",), + ) + + plan = builder.create_plan(request) + + sql, _ = cursor.calls[1] + compact_sql = normalized_sql(sql) + assert "observed_at >= %s" in compact_sql + assert "observed_at < %s" in compact_sql + assert "open_time >= %s" not in compact_sql + assert len(plan.events) == 1 + + +def test_limit_plus_one_raises_without_partial_plan() -> None: + builder, _, _, _, events = dependencies( + [ + make_trade_row(replay_sequence=10), + make_trade_row( + replay_at=START + timedelta(minutes=2), + replay_sequence=11, + trade_id=101, + ), + ] + ) + + with pytest.raises(ReplayPlanLimitExceededError, match="max_records"): + builder.create_plan(make_request(max_records=1)) + + assert events[-3:] == [ + ("cursor_exit", None), + ("transaction_exit", None), + ("connection_exit", None), + ] + + +def test_more_than_limit_plus_one_is_backend_integrity_error() -> None: + builder, _, _, _, _ = dependencies( + [ + make_trade_row(replay_sequence=10), + make_trade_row(replay_sequence=11), + make_trade_row(replay_sequence=12), + ] + ) + + with pytest.raises(MarketDataAccessIntegrityError, match="limit"): + builder.create_plan(make_request(max_records=1)) + + +@pytest.mark.parametrize("rows", (None, 1, "rows", b"rows")) +def test_rejects_invalid_rows_container(rows: object) -> None: + builder, _, _, _, _ = dependencies(rows) + + with pytest.raises(MarketDataAccessIntegrityError, match="rows"): + builder.create_plan(make_request()) + + +@pytest.mark.parametrize( + "row", + ( + (), + ("trade",), + ("unknown",) + make_trade_row()[1:], + (1,) + make_trade_row()[1:], + ), +) +def test_rejects_invalid_snapshot_row_shape_or_type( + row: tuple[object, ...], +) -> None: + builder, _, _, _, _ = dependencies([row]) + + with pytest.raises(MarketDataAccessIntegrityError): + builder.create_plan(make_request()) + + +@pytest.mark.parametrize( + "row, replay_request", + ( + ( + make_trade_row(price=Decimal("0")), + make_request(), + ), + ( + make_quote_row()[:-1] + (2,), + make_request(data_types=(ReplayDataType.QUOTE,)), + ), + ( + make_candle_row(is_final=1), + make_request( + data_types=(ReplayDataType.CANDLE_REVISION,), + candle_intervals=("1m",), + ), + ), + ), +) +def test_shared_mappers_reject_corrupt_canonical_values( + row: tuple[object, ...], + replay_request: ReplayPlanRequest, +) -> None: + builder, _, _, _, _ = dependencies([row]) + + with pytest.raises(MarketDataAccessIntegrityError): + builder.create_plan(replay_request) + + +def test_rejects_replay_time_that_differs_from_payload_time() -> None: + row = list(make_trade_row()) + row[1] = START + timedelta(minutes=2) + builder, _, _, _, _ = dependencies([tuple(row)]) + + with pytest.raises(MarketDataAccessIntegrityError, match="event"): + builder.create_plan(make_request()) + + +@pytest.mark.parametrize( + "rows", + ( + ( + make_trade_row(replay_sequence=10), + make_trade_row( + replay_at=START + timedelta(minutes=2), + replay_sequence=10, + trade_id=101, + ), + ), + ( + make_trade_row( + replay_at=START + timedelta(minutes=2), + replay_sequence=11, + ), + make_trade_row( + replay_at=START + timedelta(minutes=1), + replay_sequence=10, + trade_id=101, + ), + ), + ), +) +def test_rejects_duplicate_sequence_or_unordered_rows( + rows: tuple[tuple[object, ...], ...], +) -> None: + builder, _, _, _, _ = dependencies(rows) + + with pytest.raises(MarketDataAccessIntegrityError, match="snapshot"): + builder.create_plan(make_request()) + + +def test_rejects_event_outside_request_scope() -> None: + builder, _, _, _, _ = dependencies( + [make_trade_row(venue="another")] + ) + + with pytest.raises(MarketDataAccessIntegrityError, match="snapshot"): + builder.create_plan(make_request()) + + +@pytest.mark.parametrize("failure_point", ("provider", "execute", "fetchall")) +def test_backend_error_is_wrapped_with_original_cause( + failure_point: str, +) -> None: + builder, cursor, _, provider, events = dependencies() + backend_error = RuntimeError("backend failed") + + if failure_point == "provider": + provider.error = backend_error + elif failure_point == "execute": + cursor.execute_errors = [None, backend_error] + else: + cursor.fetchall_error = backend_error + + with pytest.raises(MarketDataAccessOperationError) as raised: + builder.create_plan(make_request()) + + assert raised.value.__cause__ is backend_error + + if failure_point != "provider": + assert events[-3:] == [ + ("cursor_exit", RuntimeError), + ("transaction_exit", RuntimeError), + ("connection_exit", RuntimeError), + ] + + +def test_transaction_setup_error_is_wrapped_and_query_is_not_executed() -> None: + builder, cursor, _, _, _ = dependencies() + setup_error = RuntimeError("cannot configure transaction") + cursor.execute_errors = [setup_error] + + with pytest.raises(MarketDataAccessOperationError) as raised: + builder.create_plan(make_request()) + + assert raised.value.__cause__ is setup_error + assert len(cursor.calls) == 1 + + +def test_base_exception_is_not_swallowed_and_contexts_are_closed() -> None: + builder, cursor, _, _, events = dependencies() + cursor.fetchall_error = KeyboardInterrupt() + + with pytest.raises(KeyboardInterrupt): + builder.create_plan(make_request()) + + assert events[-3:] == [ + ("cursor_exit", KeyboardInterrupt), + ("transaction_exit", KeyboardInterrupt), + ("connection_exit", KeyboardInterrupt), + ] diff --git a/app/tests/unit/market_data/replay/test_replay_session.py b/app/tests/unit/market_data/replay/test_replay_session.py new file mode 100644 index 0000000..c580c73 --- /dev/null +++ b/app/tests/unit/market_data/replay/test_replay_session.py @@ -0,0 +1,807 @@ +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import Any + +import pytest + +import src.market_data.replay as replay_package +from src.market_data.access import HistoricalTimeRange +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.replay import ( + MarketDataReplayValidationError, + ReplayDataType, + ReplayEvent, + ReplayPlan, + ReplayPlanRequest, + ReplaySession, + ReplaySessionProtocol, + ReplaySessionState, + ReplaySessionStateError, +) + + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +START = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) +FIRST_TIME = START + timedelta(minutes=1) +SECOND_TIME = START + timedelta(minutes=2) +THIRD_TIME = START + timedelta(minutes=3) +END = START + timedelta(hours=1) + + +def make_trade_event( + *, + trade_id: int, + replay_at: datetime, + replay_sequence: int, +) -> ReplayEvent: + trade = Trade( + symbol=SYMBOL, + trade_id=trade_id, + price=Decimal("65000"), + quantity=Decimal("0.001"), + executed_at=replay_at, + aggressor_side=TradeAggressorSide.BUY, + source="dzengi_websocket_trade", + ) + return ReplayEvent( + venue=VENUE, + replay_at=replay_at, + replay_sequence=replay_sequence, + payload=trade, + ) + + +def make_plan(*, empty: bool = False) -> ReplayPlan: + request = ReplayPlanRequest( + venue=VENUE, + symbols=(SYMBOL,), + data_types=(ReplayDataType.TRADE,), + time_range=HistoricalTimeRange( + start_time=START, + end_time=END, + ), + max_records=100, + ) + events = () if empty else ( + make_trade_event( + trade_id=1, + replay_at=FIRST_TIME, + replay_sequence=1, + ), + make_trade_event( + trade_id=2, + replay_at=FIRST_TIME, + replay_sequence=2, + ), + make_trade_event( + trade_id=3, + replay_at=SECOND_TIME, + replay_sequence=3, + ), + make_trade_event( + trade_id=4, + replay_at=THIRD_TIME, + replay_sequence=4, + ), + ) + return ReplayPlan(request=request, events=events) + + +class RecordingClock: + def __init__(self, now: datetime = START) -> None: + self._now = now + self.advance_calls: list[datetime] = [] + + @property + def now(self) -> datetime: + return self._now + + def advance_to(self, instant: datetime) -> None: + self.advance_calls.append(instant) + self._now = instant + + +class RecordingConsumer: + def __init__(self, clock: RecordingClock) -> None: + self.clock = clock + self.events: list[ReplayEvent] = [] + self.observed_times: list[datetime] = [] + self.tasks: list[asyncio.Task[Any] | None] = [] + + async def consume(self, event: ReplayEvent) -> None: + self.events.append(event) + self.observed_times.append(self.clock.now) + self.tasks.append(asyncio.current_task()) + + +def make_session( + *, + plan: ReplayPlan | None = None, + clock: RecordingClock | None = None, + consumer: RecordingConsumer | None = None, +) -> tuple[ReplaySession, ReplayPlan, RecordingClock, RecordingConsumer]: + resolved_plan = make_plan() if plan is None else plan + resolved_clock = ( + RecordingClock(resolved_plan.request.time_range.start_time) + if clock is None + else clock + ) + resolved_consumer = ( + RecordingConsumer(resolved_clock) + if consumer is None + else consumer + ) + session = ReplaySession( + plan=resolved_plan, + clock=resolved_clock, + consumer=resolved_consumer, + ) + return ( + session, + resolved_plan, + resolved_clock, + resolved_consumer, + ) + + +def test_matches_protocol_uses_slots_and_preserves_dependencies() -> None: + session, plan, clock, _ = make_session() + + assert isinstance(session, ReplaySessionProtocol) + assert not hasattr(session, "__dict__") + assert session.state is ReplaySessionState.CREATED + assert session.plan is plan + assert session.clock is clock + assert replay_package.ReplaySession is ReplaySession + assert not hasattr(replay_package, "ReplayEngine") + + +def test_construction_does_not_advance_clock_or_call_consumer() -> None: + session, _, clock, consumer = make_session() + + assert session.state is ReplaySessionState.CREATED + assert clock.advance_calls == [] + assert consumer.events == [] + + +@pytest.mark.parametrize("invalid", (None, object(), "plan")) +def test_rejects_non_plan(invalid: object) -> None: + clock = RecordingClock() + + with pytest.raises(TypeError, match="plan"): + ReplaySession( + plan=invalid, # type: ignore[arg-type] + clock=clock, + consumer=RecordingConsumer(clock), + ) + + +def test_rejects_plan_subclass() -> None: + class ReplayPlanSubclass(ReplayPlan): + pass + + plan = make_plan() + subclass = ReplayPlanSubclass( + request=plan.request, + events=plan.events, + ) + clock = RecordingClock() + + with pytest.raises(TypeError, match="plan"): + ReplaySession( + plan=subclass, + clock=clock, + consumer=RecordingConsumer(clock), + ) + + +def test_rejects_clock_without_protocol() -> None: + class InvalidClock: + @property + def now(self) -> datetime: + return START + + with pytest.raises(TypeError, match="clock"): + ReplaySession( + plan=make_plan(), + clock=InvalidClock(), # type: ignore[arg-type] + consumer=RecordingConsumer(RecordingClock()), + ) + + +def test_rejects_clock_class_before_any_runtime_action() -> None: + class ClockClass: + advance_calls: list[datetime] = [] + + @property + def now(self) -> datetime: + return START + + def advance_to(self, instant: datetime) -> None: + self.advance_calls.append(instant) + + consumer_clock = RecordingClock() + consumer = RecordingConsumer(consumer_clock) + + with pytest.raises(TypeError, match="clock"): + ReplaySession( + plan=make_plan(), + clock=ClockClass, # type: ignore[arg-type] + consumer=consumer, + ) + + assert ClockClass.advance_calls == [] + assert consumer.events == [] + + +def test_rejects_asynchronous_clock_advance() -> None: + class AsyncClock: + @property + def now(self) -> datetime: + return START + + async def advance_to(self, instant: datetime) -> None: + return None + + clock = AsyncClock() + + with pytest.raises(TypeError, match="synchronous"): + ReplaySession( + plan=make_plan(), + clock=clock, # type: ignore[arg-type] + consumer=RecordingConsumer(RecordingClock()), + ) + + +def test_rejects_consumer_without_protocol() -> None: + class InvalidConsumer: + pass + + with pytest.raises(TypeError, match="consumer"): + ReplaySession( + plan=make_plan(), + clock=RecordingClock(), + consumer=InvalidConsumer(), # type: ignore[arg-type] + ) + + +def test_rejects_consumer_class_before_advancing_clock() -> None: + clock = RecordingClock() + + with pytest.raises(TypeError, match="consumer"): + ReplaySession( + plan=make_plan(), + clock=clock, + consumer=RecordingConsumer, # type: ignore[arg-type] + ) + + assert clock.now == START + assert clock.advance_calls == [] + + +def test_rejects_synchronous_consumer() -> None: + class SyncConsumer: + def consume(self, event: ReplayEvent) -> None: + return None + + with pytest.raises(TypeError, match="asynchronous"): + ReplaySession( + plan=make_plan(), + clock=RecordingClock(), + consumer=SyncConsumer(), # type: ignore[arg-type] + ) + + +@pytest.mark.parametrize( + "now", + ( + START.replace(tzinfo=None), + START.astimezone(timezone(timedelta(hours=3))), + START.replace(fold=1), + ), +) +def test_rejects_non_canonical_clock_time(now: datetime) -> None: + clock = RecordingClock(now) + + with pytest.raises(MarketDataReplayValidationError, match="canonical"): + ReplaySession( + plan=make_plan(), + clock=clock, + consumer=RecordingConsumer(clock), + ) + + assert clock.advance_calls == [] + + +def test_rejects_datetime_subclass_from_clock() -> None: + class CompatibleDatetime(datetime): + pass + + now = CompatibleDatetime( + 2026, + 8, + 2, + 12, + 0, + tzinfo=timezone.utc, + ) + clock = RecordingClock(now) + + with pytest.raises(MarketDataReplayValidationError, match="canonical"): + ReplaySession( + plan=make_plan(), + clock=clock, + consumer=RecordingConsumer(clock), + ) + + +def test_rejects_non_datetime_clock_time() -> None: + class InvalidNowClock: + @property + def now(self) -> datetime: + return object() # type: ignore[return-value] + + def advance_to(self, instant: datetime) -> None: + return None + + clock = InvalidNowClock() + + with pytest.raises(MarketDataReplayValidationError, match="canonical"): + ReplaySession( + plan=make_plan(), + clock=clock, + consumer=RecordingConsumer(RecordingClock()), + ) + + +@pytest.mark.parametrize( + "now", + (START - timedelta(microseconds=1), START + timedelta(microseconds=1)), +) +def test_rejects_clock_outside_exact_start_without_reset( + now: datetime, +) -> None: + clock = RecordingClock(now) + + with pytest.raises(MarketDataReplayValidationError, match="start"): + ReplaySession( + plan=make_plan(), + clock=clock, + consumer=RecordingConsumer(clock), + ) + + assert clock.now is now + assert clock.advance_calls == [] + + +def test_clock_now_error_is_not_swallowed() -> None: + expected = RuntimeError("clock now failed") + + class BrokenClock(RecordingClock): + @property + def now(self) -> datetime: + raise expected + + clock = BrokenClock() + + with pytest.raises(RuntimeError) as captured: + ReplaySession( + plan=make_plan(), + clock=clock, + consumer=RecordingConsumer(clock), + ) + + assert captured.value is expected + + +def test_run_preserves_order_identity_and_advances_before_consumer() -> None: + async def scenario() -> None: + session, plan, clock, consumer = make_session() + caller_task = asyncio.current_task() + + result = await session.run() + + assert result is None + assert session.state is ReplaySessionState.COMPLETED + assert tuple(consumer.events) == plan.events + assert all( + actual is expected + for actual, expected in zip( + consumer.events, + plan.events, + strict=True, + ) + ) + assert clock.advance_calls == [ + event.replay_at for event in plan.events + ] + assert consumer.observed_times == clock.advance_calls + assert all(task is caller_task for task in consumer.tasks) + assert clock.now == plan.events[-1].replay_at + + asyncio.run(scenario()) + + +def test_empty_plan_completes_without_dependency_calls() -> None: + async def scenario() -> None: + plan = make_plan(empty=True) + session, _, clock, consumer = make_session(plan=plan) + initial_time = clock.now + + result = await session.run() + + assert result is None + assert session.state is ReplaySessionState.COMPLETED + assert clock.now is initial_time + assert clock.advance_calls == [] + assert consumer.events == [] + + asyncio.run(scenario()) + + +def test_delivery_is_strictly_sequential() -> None: + class YieldingConsumer(RecordingConsumer): + def __init__(self, clock: RecordingClock) -> None: + super().__init__(clock) + self.active = 0 + self.maximum_active = 0 + + async def consume(self, event: ReplayEvent) -> None: + self.active += 1 + self.maximum_active = max(self.maximum_active, self.active) + try: + await asyncio.sleep(0) + await super().consume(event) + finally: + self.active -= 1 + + async def scenario() -> None: + plan = make_plan() + clock = RecordingClock() + consumer = YieldingConsumer(clock) + session, *_ = make_session( + plan=plan, + clock=clock, + consumer=consumer, + ) + + await session.run() + + assert consumer.maximum_active == 1 + assert tuple(consumer.events) == plan.events + + asyncio.run(scenario()) + + +def test_clock_error_fails_without_delivering_current_or_suffix() -> None: + expected = RuntimeError("clock failed") + + class BrokenClock(RecordingClock): + def advance_to(self, instant: datetime) -> None: + self.advance_calls.append(instant) + if len(self.advance_calls) == 3: + raise expected + self._now = instant + + async def scenario() -> None: + plan = make_plan() + clock = BrokenClock() + consumer = RecordingConsumer(clock) + session, *_ = make_session( + plan=plan, + clock=clock, + consumer=consumer, + ) + + with pytest.raises(RuntimeError) as captured: + await session.run() + + assert captured.value is expected + assert session.state is ReplaySessionState.FAILED + assert tuple(consumer.events) == plan.events[:2] + assert clock.advance_calls == [ + event.replay_at for event in plan.events[:3] + ] + assert clock.now == plan.events[1].replay_at + + asyncio.run(scenario()) + + +def test_consumer_error_fails_without_retry_rollback_or_suffix() -> None: + expected = RuntimeError("consumer failed") + + class BrokenConsumer(RecordingConsumer): + async def consume(self, event: ReplayEvent) -> None: + await super().consume(event) + if len(self.events) == 3: + raise expected + + async def scenario() -> None: + plan = make_plan() + clock = RecordingClock() + consumer = BrokenConsumer(clock) + session, *_ = make_session( + plan=plan, + clock=clock, + consumer=consumer, + ) + + with pytest.raises(RuntimeError) as captured: + await session.run() + + assert captured.value is expected + assert session.state is ReplaySessionState.FAILED + assert tuple(consumer.events) == plan.events[:3] + assert clock.now == plan.events[2].replay_at + + with pytest.raises(ReplaySessionStateError): + await session.run() + + assert tuple(consumer.events) == plan.events[:3] + assert session.state is ReplaySessionState.FAILED + + asyncio.run(scenario()) + + +def test_non_exception_base_error_preserves_identity_and_failed_state() -> None: + class ReplaySignal(BaseException): + pass + + expected = ReplaySignal("stop") + + class BrokenConsumer(RecordingConsumer): + async def consume(self, event: ReplayEvent) -> None: + raise expected + + async def scenario() -> None: + plan = make_plan() + clock = RecordingClock() + session, *_ = make_session( + plan=plan, + clock=clock, + consumer=BrokenConsumer(clock), + ) + + with pytest.raises(ReplaySignal) as captured: + await session.run() + + assert captured.value is expected + assert session.state is ReplaySessionState.FAILED + assert clock.now == plan.events[0].replay_at + + asyncio.run(scenario()) + + +def test_consumer_cancellation_preserves_identity_and_cancelled_state() -> None: + expected = asyncio.CancelledError("consumer cancelled") + + class CancellingConsumer(RecordingConsumer): + async def consume(self, event: ReplayEvent) -> None: + await super().consume(event) + raise expected + + async def scenario() -> None: + plan = make_plan() + clock = RecordingClock() + consumer = CancellingConsumer(clock) + session, *_ = make_session( + plan=plan, + clock=clock, + consumer=consumer, + ) + + with pytest.raises(asyncio.CancelledError) as captured: + await session.run() + + assert captured.value is expected + assert session.state is ReplaySessionState.CANCELLED + assert consumer.events == [plan.events[0]] + assert clock.now == plan.events[0].replay_at + + with pytest.raises(ReplaySessionStateError): + await session.run() + + assert session.state is ReplaySessionState.CANCELLED + + asyncio.run(scenario()) + + +def test_external_cancellation_during_consumer_stays_cancelled() -> None: + class BlockingConsumer(RecordingConsumer): + def __init__(self, clock: RecordingClock) -> None: + super().__init__(clock) + self.entered = asyncio.Event() + self.release = asyncio.Event() + + async def consume(self, event: ReplayEvent) -> None: + await super().consume(event) + self.entered.set() + await self.release.wait() + + async def scenario() -> None: + plan = make_plan() + clock = RecordingClock() + consumer = BlockingConsumer(clock) + session, *_ = make_session( + plan=plan, + clock=clock, + consumer=consumer, + ) + task = asyncio.create_task(session.run()) + await consumer.entered.wait() + + assert session.state is ReplaySessionState.RUNNING + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + + assert session.state is ReplaySessionState.CANCELLED + assert consumer.events == [plan.events[0]] + assert clock.now == plan.events[0].replay_at + + asyncio.run(scenario()) + + +def test_concurrent_caller_is_rejected_without_damaging_first() -> None: + class BlockingConsumer(RecordingConsumer): + def __init__(self, clock: RecordingClock) -> None: + super().__init__(clock) + self.entered = asyncio.Event() + self.release = asyncio.Event() + + async def consume(self, event: ReplayEvent) -> None: + await super().consume(event) + if len(self.events) == 1: + self.entered.set() + await self.release.wait() + + async def scenario() -> None: + plan = make_plan() + clock = RecordingClock() + consumer = BlockingConsumer(clock) + session, *_ = make_session( + plan=plan, + clock=clock, + consumer=consumer, + ) + first = asyncio.create_task(session.run()) + await consumer.entered.wait() + + assert first.done() is False + assert session.state is ReplaySessionState.RUNNING + + with pytest.raises(ReplaySessionStateError): + await session.run() + + assert first.done() is False + assert session.state is ReplaySessionState.RUNNING + consumer.release.set() + await first + + assert session.state is ReplaySessionState.COMPLETED + assert tuple(consumer.events) == plan.events + + asyncio.run(scenario()) + + +def test_caught_reentrant_run_does_not_damage_outer_run() -> None: + class ReentrantConsumer(RecordingConsumer): + def __init__(self, clock: RecordingClock) -> None: + super().__init__(clock) + self.session: ReplaySession | None = None + self.reentrant_errors: list[ReplaySessionStateError] = [] + + async def consume(self, event: ReplayEvent) -> None: + await super().consume(event) + assert self.session is not None + try: + await self.session.run() + except ReplaySessionStateError as error: + self.reentrant_errors.append(error) + + async def scenario() -> None: + plan = make_plan() + clock = RecordingClock() + consumer = ReentrantConsumer(clock) + session, *_ = make_session( + plan=plan, + clock=clock, + consumer=consumer, + ) + consumer.session = session + + await session.run() + + assert session.state is ReplaySessionState.COMPLETED + assert tuple(consumer.events) == plan.events + assert len(consumer.reentrant_errors) == len(plan.events) + + asyncio.run(scenario()) + + +def test_uncaught_reentrant_run_fails_outer_run() -> None: + class ReentrantConsumer(RecordingConsumer): + def __init__(self, clock: RecordingClock) -> None: + super().__init__(clock) + self.session: ReplaySession | None = None + + async def consume(self, event: ReplayEvent) -> None: + await super().consume(event) + assert self.session is not None + await self.session.run() + + async def scenario() -> None: + plan = make_plan() + clock = RecordingClock() + consumer = ReentrantConsumer(clock) + session, *_ = make_session( + plan=plan, + clock=clock, + consumer=consumer, + ) + consumer.session = session + + with pytest.raises(ReplaySessionStateError): + await session.run() + + assert session.state is ReplaySessionState.FAILED + assert consumer.events == [plan.events[0]] + + asyncio.run(scenario()) + + +def test_repeated_run_after_completion_is_rejected() -> None: + async def scenario() -> None: + session, plan, _, consumer = make_session() + await session.run() + + with pytest.raises(ReplaySessionStateError): + await session.run() + + assert session.state is ReplaySessionState.COMPLETED + assert tuple(consumer.events) == plan.events + + asyncio.run(scenario()) + + +def test_two_sessions_share_immutable_plan_but_not_state_or_clock() -> None: + async def scenario() -> None: + plan = make_plan() + first, _, first_clock, first_consumer = make_session(plan=plan) + second, _, second_clock, second_consumer = make_session(plan=plan) + + await first.run() + + assert first.state is ReplaySessionState.COMPLETED + assert second.state is ReplaySessionState.CREATED + assert first.plan is second.plan is plan + assert first_clock is not second_clock + assert second_clock.now == START + assert second_consumer.events == [] + + await second.run() + + assert tuple(first_consumer.events) == plan.events + assert tuple(second_consumer.events) == plan.events + assert second.state is ReplaySessionState.COMPLETED + + asyncio.run(scenario()) + + +def test_lifecycle_and_hidden_task_extensions_are_absent() -> None: + session, *_ = make_session() + + assert not hasattr(session, "start") + assert not hasattr(session, "stop") + assert not hasattr(session, "close") + assert not hasattr(session, "reset") + assert not hasattr(session, "pause") + assert not hasattr(session, "resume") diff --git a/app/tests/unit/market_data/replay/test_replay_session_factory.py b/app/tests/unit/market_data/replay/test_replay_session_factory.py new file mode 100644 index 0000000..4b0eb46 --- /dev/null +++ b/app/tests/unit/market_data/replay/test_replay_session_factory.py @@ -0,0 +1,883 @@ +from __future__ import annotations + +import asyncio +import inspect +import threading +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import cast + +import pytest + +import src.market_data.replay as replay_package +import src.market_data.replay.replay_session_factory as factory_module +from src.market_data.access import HistoricalTimeRange +from src.market_data.acquisition.models.trade import ( + Trade, + TradeAggressorSide, +) +from src.market_data.replay import ( + DeterministicReplayClock, + MarketDataClockProtocol, + MarketDataReplayValidationError, + ReplayConsumerFactoryProtocol, + ReplayConsumerProtocol, + ReplayClockProtocol, + ReplayDataType, + ReplayEvent, + ReplayPlan, + ReplayPlanBuilderProtocol, + ReplayPlanRequest, + ReplaySession, + ReplaySessionFactory, + ReplaySessionState, +) + + +VENUE = "dzengi" +SYMBOL = "BTC/USD_LEVERAGE" +START = datetime(2026, 8, 2, 12, 0, tzinfo=timezone.utc) +FIRST_TIME = START + timedelta(minutes=1) +SECOND_TIME = START + timedelta(minutes=2) +END = START + timedelta(hours=1) + + +def make_request() -> ReplayPlanRequest: + return ReplayPlanRequest( + venue=VENUE, + symbols=(SYMBOL,), + data_types=(ReplayDataType.TRADE,), + time_range=HistoricalTimeRange( + start_time=START, + end_time=END, + ), + max_records=100, + ) + + +def make_trade_event( + *, + trade_id: int, + replay_at: datetime, + replay_sequence: int, +) -> ReplayEvent: + return ReplayEvent( + venue=VENUE, + replay_at=replay_at, + replay_sequence=replay_sequence, + payload=Trade( + symbol=SYMBOL, + trade_id=trade_id, + price=Decimal("65000"), + quantity=Decimal("0.001"), + executed_at=replay_at, + aggressor_side=TradeAggressorSide.BUY, + source="dzengi_websocket_trade", + ), + ) + + +def make_plan( + *, + request: ReplayPlanRequest | None = None, + empty: bool = False, +) -> ReplayPlan: + resolved_request = make_request() if request is None else request + events = () if empty else ( + make_trade_event( + trade_id=1, + replay_at=FIRST_TIME, + replay_sequence=1, + ), + make_trade_event( + trade_id=2, + replay_at=SECOND_TIME, + replay_sequence=2, + ), + ) + return ReplayPlan( + request=resolved_request, + events=events, + ) + + +class RecordingConsumer: + def __init__(self, clock: MarketDataClockProtocol) -> None: + self.clock = clock + self.events: list[ReplayEvent] = [] + self.observed_times: list[datetime] = [] + self.start_calls = 0 + self.stop_calls = 0 + self.close_calls = 0 + + async def consume(self, event: ReplayEvent) -> None: + self.events.append(event) + self.observed_times.append(self.clock.now) + + def start(self) -> None: + self.start_calls += 1 + + def stop(self) -> None: + self.stop_calls += 1 + + def close(self) -> None: + self.close_calls += 1 + + +class RecordingPlanBuilder: + def __init__( + self, + plan: ReplayPlan, + *, + actions: list[str] | None = None, + ) -> None: + self.plan = plan + self.actions = actions + self.requests: list[ReplayPlanRequest] = [] + self.thread_ids: list[int] = [] + + def create_plan(self, request: ReplayPlanRequest) -> ReplayPlan: + if self.actions is not None: + self.actions.append("builder") + self.requests.append(request) + self.thread_ids.append(threading.get_ident()) + return self.plan + + +class RecordingConsumerFactory: + def __init__(self, *, actions: list[str] | None = None) -> None: + self.actions = actions + self.plans: list[ReplayPlan] = [] + self.clocks: list[MarketDataClockProtocol] = [] + self.consumers: list[RecordingConsumer] = [] + + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + if self.actions is not None: + self.actions.append("consumer") + consumer = RecordingConsumer(clock) + self.plans.append(plan) + self.clocks.append(clock) + self.consumers.append(consumer) + return consumer + + +def create_factory( + *, + plan: ReplayPlan | None = None, +) -> tuple[ + ReplaySessionFactory, + ReplayPlan, + RecordingPlanBuilder, + RecordingConsumerFactory, +]: + resolved_plan = make_plan() if plan is None else plan + plan_builder = RecordingPlanBuilder(resolved_plan) + consumer_factory = RecordingConsumerFactory() + return ( + ReplaySessionFactory( + plan_builder=plan_builder, + consumer_factory=consumer_factory, + ), + resolved_plan, + plan_builder, + consumer_factory, + ) + + +def test_matches_protocols_uses_slots_and_is_exported() -> None: + factory, _, plan_builder, consumer_factory = create_factory() + + assert isinstance(plan_builder, ReplayPlanBuilderProtocol) + assert isinstance(consumer_factory, ReplayConsumerFactoryProtocol) + assert not hasattr(factory, "__dict__") + assert replay_package.ReplaySessionFactory is ReplaySessionFactory + assert ( + replay_package.ReplayConsumerFactoryProtocol + is ReplayConsumerFactoryProtocol + ) + + +def test_constructor_only_preserves_dependencies() -> None: + plan = make_plan() + plan_builder = RecordingPlanBuilder(plan) + consumer_factory = RecordingConsumerFactory() + + factory = ReplaySessionFactory( + plan_builder=plan_builder, + consumer_factory=consumer_factory, + ) + + assert isinstance(factory, ReplaySessionFactory) + assert plan_builder.requests == [] + assert consumer_factory.plans == [] + assert consumer_factory.clocks == [] + assert consumer_factory.consumers == [] + + +def test_constructor_does_not_create_clock_session_or_task( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class ForbiddenClock: + def __init__(self, initial_time: datetime) -> None: + raise AssertionError("Clock must not be created") + + def forbidden_session(**kwargs: object) -> ReplaySession: + raise AssertionError("Session must not be created") + + monkeypatch.setattr( + factory_module, + "DeterministicReplayClock", + ForbiddenClock, + ) + monkeypatch.setattr( + factory_module, + "ReplaySession", + forbidden_session, + ) + + async def scenario() -> None: + tasks_before = set(asyncio.all_tasks()) + + factory, _, _, _ = create_factory() + + assert isinstance(factory, ReplaySessionFactory) + assert set(asyncio.all_tasks()) == tasks_before + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("invalid", (None, object(), "builder")) +def test_rejects_invalid_plan_builder(invalid: object) -> None: + with pytest.raises(TypeError, match="plan_builder"): + ReplaySessionFactory( + plan_builder=cast(ReplayPlanBuilderProtocol, invalid), + consumer_factory=RecordingConsumerFactory(), + ) + + +def test_rejects_plan_builder_class() -> None: + class PlanBuilderClass: + def create_plan( + self, + request: ReplayPlanRequest, + ) -> ReplayPlan: + return make_plan(request=request) + + with pytest.raises(TypeError, match="plan_builder"): + ReplaySessionFactory( + plan_builder=cast( + ReplayPlanBuilderProtocol, + PlanBuilderClass, + ), + consumer_factory=RecordingConsumerFactory(), + ) + + +def test_rejects_asynchronous_plan_builder() -> None: + class AsyncPlanBuilder: + async def create_plan( + self, + request: ReplayPlanRequest, + ) -> ReplayPlan: + return make_plan(request=request) + + with pytest.raises(TypeError, match="synchronous"): + ReplaySessionFactory( + plan_builder=cast( + ReplayPlanBuilderProtocol, + AsyncPlanBuilder(), + ), + consumer_factory=RecordingConsumerFactory(), + ) + + +@pytest.mark.parametrize("invalid", (None, object(), "factory")) +def test_rejects_invalid_consumer_factory(invalid: object) -> None: + with pytest.raises(TypeError, match="consumer_factory"): + ReplaySessionFactory( + plan_builder=RecordingPlanBuilder(make_plan()), + consumer_factory=cast( + ReplayConsumerFactoryProtocol, + invalid, + ), + ) + + +def test_rejects_consumer_factory_class() -> None: + class ConsumerFactoryClass: + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + return RecordingConsumer(clock) + + with pytest.raises(TypeError, match="consumer_factory"): + ReplaySessionFactory( + plan_builder=RecordingPlanBuilder(make_plan()), + consumer_factory=cast( + ReplayConsumerFactoryProtocol, + ConsumerFactoryClass, + ), + ) + + +def test_rejects_asynchronous_consumer_factory() -> None: + class AsyncConsumerFactory: + async def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + return RecordingConsumer(clock) + + with pytest.raises(TypeError, match="synchronous"): + ReplaySessionFactory( + plan_builder=RecordingPlanBuilder(make_plan()), + consumer_factory=cast( + ReplayConsumerFactoryProtocol, + AsyncConsumerFactory(), + ), + ) + + +def test_prepare_is_synchronous_and_run_remains_asynchronous() -> None: + assert not inspect.iscoroutinefunction( + ReplaySessionFactory.prepare_session + ) + assert inspect.iscoroutinefunction(ReplaySession.run) + + +@pytest.mark.parametrize("invalid", (None, object(), "request")) +def test_rejects_invalid_request_before_dependencies( + invalid: object, +) -> None: + factory, _, plan_builder, consumer_factory = create_factory() + + with pytest.raises(MarketDataReplayValidationError, match="request"): + factory.prepare_session( + cast(ReplayPlanRequest, invalid) + ) + + assert plan_builder.requests == [] + assert consumer_factory.plans == [] + + +def test_rejects_request_subclass_before_dependencies() -> None: + class ReplayPlanRequestSubclass(ReplayPlanRequest): + pass + + request = make_request() + subclass = ReplayPlanRequestSubclass( + venue=request.venue, + symbols=request.symbols, + data_types=request.data_types, + time_range=request.time_range, + candle_intervals=request.candle_intervals, + max_records=request.max_records, + ) + factory, _, plan_builder, consumer_factory = create_factory() + + with pytest.raises(MarketDataReplayValidationError, match="request"): + factory.prepare_session(subclass) + + assert plan_builder.requests == [] + assert consumer_factory.plans == [] + + +def test_builds_in_order_and_preserves_all_identities( + monkeypatch: pytest.MonkeyPatch, +) -> None: + actions: list[str] = [] + request = make_request() + plan = make_plan(request=request) + plan_builder = RecordingPlanBuilder(plan, actions=actions) + consumer_factory = RecordingConsumerFactory(actions=actions) + real_session = ReplaySession + + class OrderedClock(DeterministicReplayClock): + def __init__(self, initial_time: datetime) -> None: + actions.append("clock") + super().__init__(initial_time) + + def ordered_session( + *, + plan: ReplayPlan, + clock: ReplayClockProtocol, + consumer: ReplayConsumerProtocol, + ) -> ReplaySession: + actions.append("session") + return real_session( + plan=plan, + clock=clock, + consumer=consumer, + ) + + monkeypatch.setattr( + factory_module, + "DeterministicReplayClock", + OrderedClock, + ) + monkeypatch.setattr( + factory_module, + "ReplaySession", + ordered_session, + ) + factory = ReplaySessionFactory( + plan_builder=plan_builder, + consumer_factory=consumer_factory, + ) + + session = factory.prepare_session(request) + clock = consumer_factory.clocks[0] + + assert actions == ["builder", "clock", "consumer", "session"] + assert plan_builder.requests == [request] + assert plan_builder.requests[0] is request + assert consumer_factory.plans == [plan] + assert consumer_factory.plans[0] is plan + assert session.plan is plan + assert session.clock is clock + assert isinstance(clock, OrderedClock) + assert clock.now == START + assert session.state is ReplaySessionState.CREATED + + +def test_prepare_runs_builder_in_caller_thread() -> None: + factory, plan, plan_builder, consumer_factory = create_factory() + caller_thread_id = threading.get_ident() + + session = factory.prepare_session(plan.request) + + assert isinstance(session, ReplaySession) + assert plan_builder.thread_ids == [caller_thread_id] + assert consumer_factory.consumers[0].events == [] + + +def test_prepare_does_not_start_consumer_or_session_lifecycle() -> None: + factory, plan, _, consumer_factory = create_factory() + + async def scenario() -> None: + tasks_before = set(asyncio.all_tasks()) + + session = factory.prepare_session(plan.request) + consumer = consumer_factory.consumers[0] + + assert session.state is ReplaySessionState.CREATED + assert consumer.events == [] + assert consumer.start_calls == 0 + assert consumer.stop_calls == 0 + assert consumer.close_calls == 0 + assert set(asyncio.all_tasks()) == tasks_before + + asyncio.run(scenario()) + + +def test_consumer_observes_start_before_run_and_event_time_during_run( +) -> None: + factory, plan, _, consumer_factory = create_factory() + session = factory.prepare_session(plan.request) + consumer = consumer_factory.consumers[0] + + assert consumer.clock is session.clock + assert consumer.clock.now == START + assert consumer.events == [] + + asyncio.run(session.run()) + + assert tuple(consumer.events) == plan.events + assert consumer.observed_times == [FIRST_TIME, SECOND_TIME] + assert session.clock.now == SECOND_TIME + assert session.state is ReplaySessionState.COMPLETED + + +def test_empty_plan_builds_created_no_op_session() -> None: + request = make_request() + plan = make_plan(request=request, empty=True) + factory, _, _, consumer_factory = create_factory(plan=plan) + + session = factory.prepare_session(request) + consumer = consumer_factory.consumers[0] + + assert session.state is ReplaySessionState.CREATED + assert session.clock.now == START + + asyncio.run(session.run()) + + assert session.state is ReplaySessionState.COMPLETED + assert session.clock.now == START + assert consumer.events == [] + + +def test_repeated_prepare_creates_fresh_dependency_graph() -> None: + factory, plan, plan_builder, consumer_factory = create_factory() + + first = factory.prepare_session(plan.request) + second = factory.prepare_session(plan.request) + + assert first is not second + assert first.plan is plan + assert second.plan is plan + assert first.clock is not second.clock + assert consumer_factory.consumers[0] is not ( + consumer_factory.consumers[1] + ) + assert consumer_factory.clocks == [first.clock, second.clock] + assert plan_builder.requests == [plan.request, plan.request] + + asyncio.run(first.run()) + + assert first.state is ReplaySessionState.COMPLETED + assert second.state is ReplaySessionState.CREATED + assert second.clock.now == START + assert consumer_factory.consumers[1].events == [] + + +class FatalPreparationError(BaseException): + pass + + +class RaisingPlanBuilder: + def __init__(self, error: BaseException) -> None: + self.error = error + self.requests: list[ReplayPlanRequest] = [] + + def create_plan(self, request: ReplayPlanRequest) -> ReplayPlan: + self.requests.append(request) + raise self.error + + +@pytest.mark.parametrize( + "expected", + ( + RuntimeError("builder failed"), + FatalPreparationError("builder fatal"), + asyncio.CancelledError("builder cancelled"), + ), +) +def test_builder_error_is_not_wrapped_and_stops_preparation( + expected: BaseException, +) -> None: + consumer_factory = RecordingConsumerFactory() + plan_builder = RaisingPlanBuilder(expected) + factory = ReplaySessionFactory( + plan_builder=plan_builder, + consumer_factory=consumer_factory, + ) + request = make_request() + + with pytest.raises(type(expected)) as captured: + factory.prepare_session(request) + + assert captured.value is expected + assert plan_builder.requests == [request] + assert consumer_factory.plans == [] + assert consumer_factory.clocks == [] + + +class InvalidResultPlanBuilder: + def __init__(self, result: object) -> None: + self.result = result + self.requests: list[ReplayPlanRequest] = [] + + def create_plan(self, request: ReplayPlanRequest) -> ReplayPlan: + self.requests.append(request) + return cast(ReplayPlan, self.result) + + +@pytest.mark.parametrize("invalid", (None, object(), "plan")) +def test_rejects_invalid_builder_result_before_consumer( + invalid: object, +) -> None: + plan_builder = InvalidResultPlanBuilder(invalid) + consumer_factory = RecordingConsumerFactory() + factory = ReplaySessionFactory( + plan_builder=plan_builder, + consumer_factory=consumer_factory, + ) + request = make_request() + + with pytest.raises(MarketDataReplayValidationError, match="ReplayPlan"): + factory.prepare_session(request) + + assert plan_builder.requests == [request] + assert consumer_factory.plans == [] + + +def test_rejects_plan_subclass_before_consumer() -> None: + class ReplayPlanSubclass(ReplayPlan): + pass + + request = make_request() + valid = make_plan(request=request) + subclass = ReplayPlanSubclass( + request=valid.request, + events=valid.events, + ) + plan_builder = InvalidResultPlanBuilder(subclass) + consumer_factory = RecordingConsumerFactory() + factory = ReplaySessionFactory( + plan_builder=plan_builder, + consumer_factory=consumer_factory, + ) + + with pytest.raises(MarketDataReplayValidationError, match="exact"): + factory.prepare_session(request) + + assert consumer_factory.plans == [] + + +def test_rejects_equal_plan_with_different_request_identity() -> None: + request = make_request() + copied_request = make_request() + assert copied_request == request + assert copied_request is not request + plan = make_plan(request=copied_request) + plan_builder = RecordingPlanBuilder(plan) + consumer_factory = RecordingConsumerFactory() + factory = ReplaySessionFactory( + plan_builder=plan_builder, + consumer_factory=consumer_factory, + ) + + with pytest.raises(MarketDataReplayValidationError, match="identity"): + factory.prepare_session(request) + + assert plan_builder.requests == [request] + assert consumer_factory.plans == [] + + +class RaisingConsumerFactory: + def __init__(self, error: BaseException) -> None: + self.error = error + self.plans: list[ReplayPlan] = [] + self.clocks: list[MarketDataClockProtocol] = [] + + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + self.plans.append(plan) + self.clocks.append(clock) + raise self.error + + +@pytest.mark.parametrize( + "expected", + ( + RuntimeError("consumer factory failed"), + FatalPreparationError("consumer factory fatal"), + asyncio.CancelledError("consumer factory cancelled"), + ), +) +def test_consumer_factory_error_is_not_wrapped_or_retried( + expected: BaseException, +) -> None: + request = make_request() + plan = make_plan(request=request) + consumer_factory = RaisingConsumerFactory(expected) + factory = ReplaySessionFactory( + plan_builder=RecordingPlanBuilder(plan), + consumer_factory=consumer_factory, + ) + + with pytest.raises(type(expected)) as captured: + factory.prepare_session(request) + + assert captured.value is expected + assert consumer_factory.plans == [plan] + assert len(consumer_factory.clocks) == 1 + assert consumer_factory.clocks[0].now == START + + +class InvalidConsumerFactory: + def __init__(self, result: object) -> None: + self.result = result + self.calls = 0 + + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + self.calls += 1 + return cast(ReplayConsumerProtocol, self.result) + + +@pytest.mark.parametrize("invalid", (None, object(), "consumer")) +def test_rejects_invalid_consumer_result(invalid: object) -> None: + request = make_request() + consumer_factory = InvalidConsumerFactory(invalid) + factory = ReplaySessionFactory( + plan_builder=RecordingPlanBuilder(make_plan(request=request)), + consumer_factory=consumer_factory, + ) + + with pytest.raises(TypeError, match="consumer"): + factory.prepare_session(request) + + assert consumer_factory.calls == 1 + + +def test_rejects_consumer_class_result() -> None: + request = make_request() + consumer_factory = InvalidConsumerFactory(RecordingConsumer) + factory = ReplaySessionFactory( + plan_builder=RecordingPlanBuilder(make_plan(request=request)), + consumer_factory=consumer_factory, + ) + + with pytest.raises(TypeError, match="consumer"): + factory.prepare_session(request) + + +def test_rejects_synchronous_consumer_result() -> None: + class SynchronousConsumer: + def consume(self, event: ReplayEvent) -> None: + return None + + request = make_request() + consumer_factory = InvalidConsumerFactory(SynchronousConsumer()) + factory = ReplaySessionFactory( + plan_builder=RecordingPlanBuilder(make_plan(request=request)), + consumer_factory=consumer_factory, + ) + + with pytest.raises(TypeError, match="asynchronous"): + factory.prepare_session(request) + + +def test_keyword_incompatible_consumer_factory_error_is_not_wrapped() -> None: + class PositionalOnlyConsumerFactory: + def create_consumer( + self, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + /, + ) -> ReplayConsumerProtocol: + return RecordingConsumer(clock) + + request = make_request() + factory = ReplaySessionFactory( + plan_builder=RecordingPlanBuilder(make_plan(request=request)), + consumer_factory=cast( + ReplayConsumerFactoryProtocol, + PositionalOnlyConsumerFactory(), + ), + ) + + with pytest.raises(TypeError, match="keyword"): + factory.prepare_session(request) + + +def test_failed_preparation_does_not_poison_next_call() -> None: + class RecoveringConsumerFactory(RecordingConsumerFactory): + def __init__(self) -> None: + super().__init__() + self.attempts = 0 + + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + self.attempts += 1 + if self.attempts == 1: + raise RuntimeError("first attempt failed") + return super().create_consumer(plan=plan, clock=clock) + + request = make_request() + plan = make_plan(request=request) + consumer_factory = RecoveringConsumerFactory() + factory = ReplaySessionFactory( + plan_builder=RecordingPlanBuilder(plan), + consumer_factory=consumer_factory, + ) + + with pytest.raises(RuntimeError, match="first attempt failed"): + factory.prepare_session(request) + + session = factory.prepare_session(request) + + assert session.state is ReplaySessionState.CREATED + assert consumer_factory.attempts == 2 + assert len(consumer_factory.consumers) == 1 + + +def test_two_concurrent_callers_receive_independent_graphs() -> None: + request = make_request() + plan = make_plan(request=request) + builder_barrier = threading.Barrier(2) + consumer_barrier = threading.Barrier(2) + lock = threading.Lock() + + class ConcurrentPlanBuilder: + def __init__(self) -> None: + self.callers: list[int] = [] + + def create_plan(self, request: ReplayPlanRequest) -> ReplayPlan: + with lock: + self.callers.append(threading.get_ident()) + builder_barrier.wait(timeout=5) + return plan + + class ConcurrentConsumerFactory: + def __init__(self) -> None: + self.calls: list[ + tuple[int, MarketDataClockProtocol, RecordingConsumer] + ] = [] + + def create_consumer( + self, + *, + plan: ReplayPlan, + clock: MarketDataClockProtocol, + ) -> ReplayConsumerProtocol: + consumer = RecordingConsumer(clock) + with lock: + self.calls.append( + (threading.get_ident(), clock, consumer) + ) + consumer_barrier.wait(timeout=5) + return consumer + + plan_builder = ConcurrentPlanBuilder() + consumer_factory = ConcurrentConsumerFactory() + factory = ReplaySessionFactory( + plan_builder=plan_builder, + consumer_factory=consumer_factory, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [ + executor.submit(factory.prepare_session, request) + for _ in range(2) + ] + sessions = [future.result(timeout=10) for future in futures] + + assert len(set(plan_builder.callers)) == 2 + assert len({call[0] for call in consumer_factory.calls}) == 2 + assert sessions[0] is not sessions[1] + assert sessions[0].clock is not sessions[1].clock + assert consumer_factory.calls[0][2] is not ( + consumer_factory.calls[1][2] + ) + assert {id(call[1]) for call in consumer_factory.calls} == { + id(session.clock) for session in sessions + } + assert all( + session.state is ReplaySessionState.CREATED + for session in sessions + ) diff --git a/app/tests/unit/market_data/storage/test_postgres_trade_repository.py b/app/tests/unit/market_data/storage/test_postgres_trade_repository.py index d8149d3..9bddef1 100644 --- a/app/tests/unit/market_data/storage/test_postgres_trade_repository.py +++ b/app/tests/unit/market_data/storage/test_postgres_trade_repository.py @@ -38,6 +38,7 @@ TradeRow = dict[str, Any] @dataclass class TransactionalTradeDatabase: rows: dict[TradeKey, TradeRow] = field(default_factory=dict) + next_replay_sequence: int = 1 class TransactionalCursor: @@ -47,6 +48,7 @@ class TransactionalCursor: ) -> None: self._connection = connection self._fetchone_result: object = None + self._fetchall_result: list[tuple[int]] = [] def __enter__(self) -> TransactionalCursor: return self @@ -78,6 +80,12 @@ class TransactionalCursor: self._insert(parameters) return + if normalized.startswith( + "SELECT nextval('market_data.replay_sequence'::regclass)" + ): + self._allocate_replay_sequences(parameters) + return + if normalized.startswith("SELECT price, quantity"): self._select(parameters) return @@ -91,21 +99,45 @@ class TransactionalCursor: def fetchone(self) -> object: return self._fetchone_result + def fetchall(self) -> list[tuple[int]]: + return list(self._fetchall_result) + def _insert(self, parameters: tuple[Any, ...]) -> None: - ( - venue, - symbol, - trade_id, - executed_at, - price, - quantity, - aggressor_side, - source, - first_observed_at, - last_observed_at, - observation_sources, - canonical_schema_version, - ) = parameters + if len(parameters) == 12: + ( + venue, + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + observation_sources, + canonical_schema_version, + ) = parameters + replay_sequence = self._next_replay_sequence() + elif len(parameters) == 13: + ( + venue, + symbol, + trade_id, + executed_at, + price, + quantity, + aggressor_side, + source, + first_observed_at, + last_observed_at, + observation_sources, + replay_sequence, + canonical_schema_version, + ) = parameters + else: + raise AssertionError("Unexpected Trade INSERT parameters") + key = (venue, symbol, trade_id, executed_at) working_rows = self._connection.working_rows @@ -121,10 +153,26 @@ class TransactionalCursor: "first_observed_at": first_observed_at, "last_observed_at": last_observed_at, "observation_sources": list(observation_sources), + "replay_sequence": replay_sequence, "canonical_schema_version": canonical_schema_version, } self._fetchone_result = (1,) + def _allocate_replay_sequences( + self, + parameters: tuple[Any, ...], + ) -> None: + (count,) = parameters + self._fetchall_result = [ + (self._next_replay_sequence(),) + for _ in range(count) + ] + + def _next_replay_sequence(self) -> int: + value = self._connection.database.next_replay_sequence + self._connection.database.next_replay_sequence += 1 + return value + def _select(self, parameters: tuple[Any, ...]) -> None: key = parameters row = self._connection.working_rows.get(key) @@ -181,6 +229,10 @@ class TransactionalConnection: def cursor(self) -> TransactionalCursor: return TransactionalCursor(self) + @property + def database(self) -> TransactionalTradeDatabase: + return self._database + @dataclass class RecordingConnectionProvider: @@ -261,6 +313,7 @@ def test_store_trade_inserts_canonical_payload_and_provenance() -> None: "first_observed_at": OBSERVED_AT, "last_observed_at": OBSERVED_AT, "observation_sources": ["dzengi_websocket_trade"], + "replay_sequence": 1, "canonical_schema_version": 1, } @@ -459,6 +512,122 @@ def test_batch_uses_stable_identity_order() -> None: assert inserted_symbols == ("A", "B", "C") +def test_batch_allocates_replay_sequence_in_input_order() -> None: + repository, database, connection, _ = _repository() + + repository.store_trades( + venue=VENUE, + trades=( + _trade(symbol="C", trade_id=3), + _trade(symbol="A", trade_id=1), + _trade(symbol="B", trade_id=2), + ), + observed_at=OBSERVED_AT, + ) + + inserted = tuple( + (parameters[1], parameters[11]) + for statement, parameters in connection.calls + if statement.startswith("INSERT INTO market_data.trades") + ) + + assert inserted == ( + ("A", 2), + ("B", 3), + ("C", 1), + ) + assert { + key[1]: row["replay_sequence"] + for key, row in database.rows.items() + } == { + "A": 2, + "B": 3, + "C": 1, + } + + +@pytest.mark.parametrize( + ("first_trade_id", "second_trade_id"), + ( + (SIGNED_TRADE_ID_MAX, SIGNED_TRADE_ID_MIN), + (-1, 0), + ), +) +def test_batch_replay_sequence_preserves_signed_rollover_input_order( + first_trade_id: int, + second_trade_id: int, +) -> None: + repository, database, _, _ = _repository() + + repository.store_trades( + venue=VENUE, + trades=( + _trade(trade_id=first_trade_id), + _trade(trade_id=second_trade_id), + ), + observed_at=OBSERVED_AT, + ) + + assert database.rows[ + (VENUE, SYMBOL, first_trade_id, EXECUTED_AT) + ]["replay_sequence"] == 1 + assert database.rows[ + (VENUE, SYMBOL, second_trade_id, EXECUTED_AT) + ]["replay_sequence"] == 2 + + +def test_batch_duplicate_and_provenance_keep_original_replay_sequence() -> None: + repository, database, _, _ = _repository() + original = _trade() + repository.store_trade( + venue=VENUE, + trade=original, + observed_at=OBSERVED_AT, + ) + + repository.store_trades( + venue=VENUE, + trades=( + replace(original, source="dzengi"), + original, + ), + observed_at=OBSERVED_AT + timedelta(seconds=1), + ) + + assert _only_row(database)["replay_sequence"] == 1 + assert database.next_replay_sequence == 4 + + +def test_failed_batch_keeps_consumed_replay_sequence_gap() -> None: + repository, database, _, _ = _repository() + existing = _trade(symbol="B", trade_id=2) + repository.store_trade( + venue=VENUE, + trade=existing, + observed_at=OBSERVED_AT, + ) + + with pytest.raises(MarketDataStorageConflictError): + repository.store_trades( + venue=VENUE, + trades=( + _trade(symbol="A", trade_id=1), + replace(existing, price=Decimal("999")), + ), + observed_at=OBSERVED_AT, + ) + + repository.store_trade( + venue=VENUE, + trade=_trade(symbol="C", trade_id=3), + observed_at=OBSERVED_AT, + ) + + assert database.rows[ + (VENUE, "C", 3, EXECUTED_AT) + ]["replay_sequence"] == 4 + + def test_batch_conflict_rolls_back_preceding_insert() -> None: repository, database, connection, _ = _repository() existing = _trade(symbol="B", trade_id=2) diff --git a/app/tests/unit/storage/test_migrations.py b/app/tests/unit/storage/test_migrations.py index b3f788e..269a9a4 100644 --- a/app/tests/unit/storage/test_migrations.py +++ b/app/tests/unit/storage/test_migrations.py @@ -7,6 +7,7 @@ import pytest from src.storage.exceptions import StorageMigrationError from src.storage.migrations import ( + MARKET_DATA_PARTITION_ADVISORY_LOCK_ID, STORAGE_MIGRATION_ADVISORY_LOCK_ID, STORAGE_MIGRATIONS, StorageMigration, @@ -106,6 +107,7 @@ def test_default_migrations_have_stable_order_and_names() -> None: (6, "add_quote_and_candle_observation_sources"), (7, "create_market_data_partition_registry"), (8, "create_trade_stream_checkpoints"), + (9, "add_global_market_data_replay_sequence"), ) @@ -158,12 +160,73 @@ def test_default_schema_defines_partitions_identities_and_constraints() -> None: assert "CHECK (checkpoint_schema_version > 0)" in sql +def test_replay_sequence_migration_has_atomic_global_order_contract() -> None: + migration = STORAGE_MIGRATIONS[-1] + statements = tuple( + " ".join(statement.split()) + for statement in migration.statements + ) + sql = "\n".join(statements) + + assert migration.version == 9 + assert migration.name == "add_global_market_data_replay_sequence" + assert statements[:4] == ( + ( + "SELECT pg_advisory_xact_lock(" + f"{MARKET_DATA_PARTITION_ADVISORY_LOCK_ID}" + ")" + ), + "LOCK TABLE market_data.trades IN ACCESS EXCLUSIVE MODE", + "LOCK TABLE market_data.quotes IN ACCESS EXCLUSIVE MODE", + ( + "LOCK TABLE market_data.candle_revisions " + "IN ACCESS EXCLUSIVE MODE" + ), + ) + assert "CREATE SEQUENCE market_data.replay_sequence AS BIGINT" in sql + assert "MINVALUE 1" in sql + assert "CACHE 1" in sql + assert "NO CYCLE" in sql + assert "OWNED BY NONE" in sql + assert sql.count("ADD COLUMN replay_sequence BIGINT") == 3 + assert "CREATE TABLE market_data.replay_sequence" not in sql + assert "CREATE TEMPORARY TABLE market_data_replay_sequence_backfill" in sql + assert "ON COMMIT DROP" in sql + assert "ROW_NUMBER() OVER" in sql + assert "UNION ALL" in sql + assert "event_time, data_type_rank, venue COLLATE \"C\"" in sql + assert "symbol COLLATE \"C\"" in sql + assert "candle_interval COLLATE \"C\"" in sql + assert "ctid" not in sql.lower() + assert sql.count("SET replay_sequence = backfill.replay_sequence") == 3 + assert "target.trade_id = backfill.trade_id" in sql + assert "target.received_at = backfill.received_at" in sql + assert "target.interval = backfill.interval" in sql + assert "target.open_time = backfill.open_time" in sql + assert "target.observed_at = backfill.observed_at" in sql + assert "SELECT pg_catalog.setval(" in sql + assert "EXISTS ( SELECT 1 FROM market_data_replay_sequence_backfill )" in sql + assert sql.count("SET DEFAULT nextval(") == 3 + assert sql.count("ALTER COLUMN replay_sequence SET NOT NULL") == 3 + assert sql.count("CHECK (replay_sequence > 0)") == 3 + assert "CREATE FUNCTION market_data.reject_replay_sequence_change()" in sql + assert sql.count("BEFORE UPDATE OF replay_sequence") == 3 + assert "CREATE INDEX trades_history_keyset_idx" in sql + assert "CREATE INDEX quotes_history_keyset_idx" in sql + assert "CREATE INDEX candle_revisions_history_keyset_idx" in sql + assert "CREATE INDEX candle_revisions_replay_keyset_idx" in sql + assert "executed_at, replay_sequence" in sql + assert "received_at, replay_sequence" in sql + assert "interval, open_time, replay_sequence" in sql + assert "interval, observed_at, replay_sequence" in sql + + def test_run_locks_and_applies_every_pending_migration_in_order() -> None: runner, cursor, connection, provider = _runner() result = runner.run() - assert result == (1, 2, 3, 4, 5, 6, 7, 8) + assert result == (1, 2, 3, 4, 5, 6, 7, 8, 9) assert provider.calls == 1 assert connection.entered == 1 assert connection.exited == 1 @@ -180,7 +243,7 @@ def test_run_locks_and_applies_every_pending_migration_in_order() -> None: ) and isinstance(parameters, tuple) ) - assert inserted_versions == (1, 2, 3, 4, 5, 6, 7, 8) + assert inserted_versions == (1, 2, 3, 4, 5, 6, 7, 8, 9) def test_run_skips_already_applied_migrations() -> None: @@ -209,7 +272,7 @@ def test_run_applies_only_migrations_after_existing_prefix() -> None: result = runner.run() - assert result == (3, 4, 5, 6, 7, 8) + assert result == (3, 4, 5, 6, 7, 8, 9) inserted_versions = tuple( parameters[0] for statement, parameters in cursor.calls @@ -218,7 +281,7 @@ def test_run_applies_only_migrations_after_existing_prefix() -> None: ) and isinstance(parameters, tuple) ) - assert inserted_versions == (3, 4, 5, 6, 7, 8) + assert inserted_versions == (3, 4, 5, 6, 7, 8, 9) def test_run_rejects_unknown_applied_version() -> None: diff --git a/docs/migrations/build_060_29.md b/docs/migrations/build_060_29.md new file mode 100644 index 0000000..8d994e7 --- /dev/null +++ b/docs/migrations/build_060_29.md @@ -0,0 +1,236 @@ +# Build 060.29 — Market Data Access and Replay + +**Engineering Migration Report** + +--- + +## Контроль документа + +| Свойство | Значение | +|---|---| +| Build | 060.29 | +| Статус | Completed | +| Подсистема | Market Data / Historical Access and Replay | +| Компонент | Canonical History and Deterministic Replay | +| Дата завершения | 2026-08-02 | +| Версия | 1.0 | + +--- + +## Связанные документы + +- `build_060_29_architecture.md` — архитектура, решения и подробные + результаты подэтапов 060.29.0–060.29.8; +- `build_060_28.md` — Persistent Checkpoint and Startup Recovery; +- `dzentra_target_architecture.md` — целевая архитектура Dzentra; +- `master-roadmap.md` — дальнейшая последовательность Build. + +--- + +## 1. Назначение Build + +Build 060.29 добавил безопасное чтение сохранённых Canonical Market +Data и их детерминированное воспроизведение. + +Итоговая цепочка: + +```text +PostgreSQL Canonical Market Data + ↓ +Historical Access + ↓ +bounded immutable ReplayPlan + ↓ +ReplaySession + Deterministic Clock + ↓ +явно переданный Canonical consumer +``` + +Historical Access и Replay являются read-side подсистемами. Они не +заменяют Acquisition Runtime, не продвигают operational checkpoint и не +записывают воспроизводимые события обратно в Market Data Storage. + +--- + +## 2. Завершённые подэтапы + +| Подэтап | Название | Статус | +|---|---|---| +| 060.29.0 | Architecture, Boundaries and Ordering Policy | Accepted | +| 060.29.1 | Historical Access and Replay Contracts | Accepted | +| 060.29.2 | Global Replay Sequence Migration and PostgreSQL Trade History | Accepted | +| 060.29.3 | Quote/Candle History and Snapshot Plan Builder | Accepted | +| 060.29.4 | Deterministic Replay Clock | Accepted | +| 060.29.5 | Replay Session and Engine | Accepted | +| 060.29.6 | Consumer and Composition Integration | Accepted | +| 060.29.7 | PostgreSQL Replay and Failure Verification | Accepted | +| 060.29.8 | Final Regression and Acceptance | Accepted | + +Каждый подэтап проходил отдельный read-only review. Findings +исправлялись и закрывались regression-тестами до принятия. + +--- + +## 3. Реализованная архитектура + +### 3.1. Глобальная последовательность Replay + +Migration 9 добавила один shared PostgreSQL sequence для Trades, Quotes +и Candle revisions. Неизменяемый `replay_sequence` задаёт общий +детерминированный порядок для событий с одинаковым временем и +сохраняется при duplicate или provenance update. + +Существующие writers совместимы с sequence для одиночных, пакетных и +конкурентных записей. Default, существующие месячные и будущие partitions +получают одинаковые constraints, trigger и keyset indexes. + +Migration выполняется атомарно и использует единый порядок блокировок с +Partition Manager и writers. + +### 3.2. Historical Access + +DB-neutral read-контракты отделены от write-only Storage API. Отдельные +PostgreSQL readers возвращают типизированные неизменяемые страницы: + +- Trades — по `executed_at`; +- Quotes — по `received_at`; +- Candle revisions — по `open_time`. + +Запросы используют полуоткрытый временной диапазон и forward keyset +pagination. Cursor связан с точным query scope. Строки PostgreSQL строго +проверяются перед созданием Canonical `Trade`, `Quote` или `Candle`. + +### 3.3. Bounded Replay snapshot + +Replay Plan Builder выполняет один статический snapshot-запрос в +короткой транзакции `READ ONLY REPEATABLE READ`. Все выбранные события +материализуются до возврата `ReplayPlan`; cursor, transaction и +connection освобождаются до первого вызова consumer. + +Порядок Replay равен `(replay_at, replay_sequence)`. Для Candle временем +Replay является `observed_at`, поэтому ревизия не появляется раньше +момента, когда она стала известна системе. Превышение `max_records` +завершается явной ошибкой без частичного plan. + +### 3.4. Clock, Session и Composition + +`DeterministicReplayClock` работает только с aware UTC-временем, +разрешает равное время и запрещает движение назад. Он не читает wall +clock и не имеет reset. + +`ReplaySession` является необратимой one-shot сущностью. Она +последовательно продвигает Clock и ожидает один consumer-вызов для +каждого события. Ошибки и cancellation сохраняют явное terminal state и +не скрываются. + +`ReplaySessionFactory` синхронно подготавливает новый граф +`Plan → Clock → consumer → Session` для каждого вызова. Default consumer, +автоматический startup, скрытая задача и отдельный `ReplayEngine` не +добавлены. Caller отдельно владеет `session.run()`. + +### 3.5. Статическая проверка типов + +Pyright закреплён как обязательный gate в режиме `standard`, совпадающем +с Pylance проекта. Канонический запуск — `scripts/check_python_types.sh`; +допустимый результат — только `0 errors, 0 warnings`. Тот же gate входит +в обычную offline pytest-регрессию. + +--- + +## 4. Real PostgreSQL verification + +Безопасный opt-in harness требует отдельную локальную базу с именем +`dzentra_test_*`, явный флаг и отдельный DSN. + +Настоящий PostgreSQL 16 подтвердил: + +- атомарный детерминированный backfill migration 9; +- общий sequence и совместимость всех writers/partitions; +- keyset pagination Trades, Quotes и Candle revisions; +- единый mixed-type Replay order; +- устойчивый `REPEATABLE READ` snapshot при concurrent commit; +- освобождение PostgreSQL resources до playback; +- limit, integrity, backend, consumer и cancellation paths; +- независимые графы двух одновременно готовящихся Replay Session; +- полный путь `Loopback Runtime → Storage → History → Replay`; +- отсутствие изменения durable данных и operational checkpoint. + +--- + +## 5. Финальные результаты + +Итоговая приёмка выполнена 2026-08-02: + +```text +Pyright mandatory gate: 0 errors, 0 warnings +Pyright pytest gate: 1 passed +Expanded Access/Replay/Storage unit: 545 passed +PostgreSQL Replay target repeated: 10 × 12 passed +Full PostgreSQL Storage integration: 77 passed +Full integration with PostgreSQL: 90 passed +PostgreSQL suite without opt-in: 77 skipped +Fixed stress target: 3 passed, 1 deselected +Full offline regression: 2681 passed, 95 deselected +pip check: clean +Compileall через временный pycache: clean +Tracked/untracked whitespace checks: clean +Три независимых read-only review: clean +``` + +В integration и stress наборах `ResourceWarning` считался ошибкой. +Production-код на финальном подэтапе 060.29.8 не изменялся: матрица не +выявила реального дефекта. + +--- + +## 6. Эксплуатационное предупреждение migration 9 + +Migration 9 является блокирующей. Её длительность пропорциональна уже +накопленному объёму Market Data, потому что существующие строки получают +глобальный `replay_sequence`, после чего создаются constraints и +индексы. + +Для текущей небольшой базы выбранный атомарный вариант разумен. Перед +применением к большой production-базе обязательны резервная копия, +замер длительности на сопоставимом объёме и отдельное maintenance +window. Online staged migration в Build 060.29 не реализована и требует +отдельного архитектурного решения. + +--- + +## 7. Границы Build + +Build 060.29 намеренно не реализует: + +- initial historical backfill с биржи; +- гарантию полной биржевой истории без completeness metadata; +- точное воспроизведение порядка исходных сетевых пакетов; +- production consumers для persistent Quotes и Candles; +- автоматический запуск Replay из Bootstrap; +- default/no-op Replay consumer; +- Backtesting, Analytics API или торговую симуляцию; +- online migration 9 для большой production-базы. + +Каждая Historical page отражает committed-состояние на время своего +запроса. Последовательная pagination охватывает все фактически +сохранённые строки диапазона только при неизменном dataset между +страницами; единого межстраничного snapshot нет. Период доступной истории +определяется моментом включения persistent storage и Retention Policy, а +не самим Replay API. + +Постороннее пользовательское изменение `.gitignore` не относится к +Build 060.29 и не должно включаться в его staging. + +--- + +## 8. Итог + +Build 060.29 завершён и принят. + +Dzentra получила отдельный Historical Access, общий детерминированный +порядок сохранённых Canonical Market Data и caller-owned Replay без +скрытого lifecycle. Сохранённую рыночную историю теперь можно безопасно +читать и воспроизводить одинаковыми Canonical объектами. + +Следующий этап — Build 060.30, итоговый аудит и документальное закрытие +ветки Trades Feed. diff --git a/docs/migrations/build_060_29_architecture.md b/docs/migrations/build_060_29_architecture.md new file mode 100644 index 0000000..a9188d0 --- /dev/null +++ b/docs/migrations/build_060_29_architecture.md @@ -0,0 +1,1461 @@ +# Build 060.29 — Market Data Access and Replay Architecture + +**Статус:** Completed + +**Build:** 060.29 + +**Подсистема:** Market Data / Historical Access and Replay + +**Дата начала:** 2026-08-02 + +**Дата завершения:** 2026-08-02 + +**Версия документа:** 1.9 + +--- + +## 1. Назначение + +Build 060.29 добавляет безопасное чтение сохранённых Canonical Market +Data и их детерминированное воспроизведение. + +Build опирается на: + +- PostgreSQL Canonical Storage из 060.27; +- Persistent Checkpoint и Startup Recovery из 060.28; +- существующие неизменяемые Canonical `Trade`, `Quote` и `Candle`. + +Итоговая цепочка должна выглядеть так: + +```text +PostgreSQL Canonical Market Data + ↓ +Historical Access + ↓ +bounded immutable ReplayPlan + ↓ +ReplaySession + Virtual Clock + ↓ +Canonical consumer +``` + +Historical Access и Replay являются read-side подсистемами. Они не +заменяют Acquisition Runtime, не продвигают operational checkpoint и не +записывают повторно воспроизводимые события в Market Data Storage. + +--- + +## 2. Статус подэтапов + +| Подэтап | Название | Статус | +|---|---|---| +| 060.29.0 | Architecture, Boundaries and Ordering Policy | Accepted | +| 060.29.1 | Historical Access and Replay Contracts | Accepted | +| 060.29.2 | Global Replay Sequence Migration and PostgreSQL Trade History | Accepted | +| 060.29.3 | Quote/Candle History and Snapshot Plan Builder | Accepted | +| 060.29.4 | Deterministic Replay Clock | Accepted | +| 060.29.5 | Replay Session and Engine | Accepted | +| 060.29.6 | Consumer and Composition Integration | Accepted | +| 060.29.7 | PostgreSQL Replay and Failure Verification | Accepted | +| 060.29.8 | Final Regression and Acceptance | Accepted | + +Подэтапы 060.29.0–060.29.6 приняты после отдельных приёмочных +read-only review. Реализация 060.29.2 ограничена versioned migration 9, +совместимостью существующих writers и отдельным PostgreSQL Trade +History reader. 060.29.3 добавляет Quote/Candle History и отдельный +bounded Replay snapshot. 060.29.4 добавляет детерминированные UTC-часы +без wall clock, reset и скрытого lifecycle. На этом этапе Build оставался +`In Progress`. Контракт 060.29.5 утверждён; отдельный `ReplayEngine` не +создаётся, а принятый playback engine реализован внутри +`ReplaySession.run()`. Контракт 060.29.6 утверждён: composition использует +обязательную фабрику consumer, один синхронный blocking +`ReplaySessionFactory`, новый граф зависимостей на каждый вызов и не +выполняет автоматический startup. Реализация 060.29.6 принята после +чистого формального read-only review. + +Контракт 060.29.7 утверждён как verification-only: новые тесты используют +существующий безопасный opt-in PostgreSQL harness, реальные Storage, +Historical Access, ReplayPlan Builder, Composition и Session, но не +добавляют production consumer, Bootstrap wiring или automatic startup. +Production-код заранее не изменяется. + +060.29.7 принят после исправления найденных Pylance/Pyright diagnostics, +введения обязательного static gate, полной повторной регрессии и +чистого формального read-only review. Единственная production-правка +подэтапа уточняет статическое сужение уже проверенного точного union-типа +без изменения runtime-контракта или поведения Replay. + +060.29.8 выполнил финальную verification-only матрицу без изменения +production-кода. Повторены статическая проверка, unit, PostgreSQL, +integration, opt-out, fixed stress и полная offline-регрессия. Три +независимых read-only review не выявили открытых findings; подэтап и +Build 060.29 приняты. + +--- + +## 3. Исходные инварианты 060.27–060.28 + +1. Canonical Market Data уже прошли transport validation и mapping. +2. `MarketDataStorage` остаётся фасадом только для записи. +3. Checkpoint reads остаются узкой границей Startup Hydration и не + превращаются в Historical API. +4. Production Runtime сохраняет Canonical Trade до продвижения + operational checkpoint. +5. Live и Recovery используют одного владельца записи Trades. +6. PostgreSQL pool имеет явный Application lifecycle. +7. Consistency, Recovery и Runtime не зависят от `psycopg`. +8. Trade ID использует signed 32-bit rollover-aware контракт. +9. Retention по умолчанию выключен и запускается только явно. + +--- + +## 4. Принятые архитектурные решения + +### 4.1. Отдельная read-side граница + +Historical Access размещается отдельно от существующего write-only +Storage API: + +```text +market_data.access contracts + ↑ +PostgreSQL History adapters +``` + +Существующие `TradeStorageProtocol`, `QuoteStorageProtocol`, +`CandleStorageProtocol`, `TradeCheckpointStorageProtocol` и +`MarketDataStorage` не расширяются историческими запросами. + +PostgreSQL implementation зависит от DB-neutral read-контрактов. +Контракты не импортируют `psycopg`, pool, Bootstrap, Runtime, Recovery, +Telegram или Trading. + +### 4.2. Scope типов данных + +Historical Access предусматривает чтение трёх уже существующих +Canonical таблиц: + +- Trades; +- Quotes; +- Candle revisions. + +Это не подключает persistent Quote/Candle consumers и не объявляет их +Production Runtime завершённым. Полная production-проверка +`Live → Storage → Historical → Replay` в этом Build обязательна для +Trades. Quote/Candle read-path проверяется на repository и PostgreSQL +уровне. + +### 4.3. Совместимость Live и Replay + +Live и Replay используют одинаковые Canonical classes и значения: + +```text +Trade | Quote | Candle +``` + +PostgreSQL reader создаёт новый Canonical объект, поэтому identity с +первоначальным Live-объектом не обещается. После материализации +Historical record и созданный из него Replay event используют один и +тот же payload без повторного копирования. + +Replay не создаёт `ReplayTrade`, не меняет `source` на `replay` и не +восстанавливает transport document. Технический Replay envelope не +объявляется существующим Live Runtime event-контрактом. Совместимость +гарантируется на уровне точных Canonical payload classes и значений. + +### 4.4. Гарантируемый порядок + +Build гарантирует детерминированную event-time хронологию Canonical +данных. Он не обещает точное повторение порядка исходных сетевых +пакетов, потому что raw transport arrival log не сохраняется. + +Оси времени: + +| Тип | Historical Query | Replay | +|---|---|---| +| Trade | `executed_at` | `executed_at` | +| Quote | `received_at` | `received_at` | +| Candle revision | `open_time` | `observed_at` | + +`exchange_timestamp` Quote не используется как Replay time, чтобы не +создавать look-ahead. Candle revision воспроизводится в момент её +наблюдения, а не задним числом в `open_time`. + +--- + +## 5. Архитектурная граница и направление импортов + +```text +Canonical Models + ↑ ↑ +Historical contracts Replay contracts + ↑ ↑ +PostgreSQL readers → ReplayPlan builder + ↓ + ReplaySession + ↓ + async consumer +``` + +Запрещённые зависимости: + +```text +access/replay → acquisition.runtime +access/replay → storage write facade +access/replay → bootstrap/application +access/replay → telegram/trading +access/replay → runtime_events/EventBus +``` + +Replay не вызывает `TradeStreamConsistencyController`. Иначе +воспроизведение изменило бы durable history и operational checkpoint. + +--- + +## 6. Historical contracts + +### 6.1. Time range + +Все публичные запросы используют timezone-aware полуоткрытый диапазон: + +```text +[start_time, end_time) +``` + +Время нормализуется в UTC. Требуется строгое +`start_time < end_time`. + +### 6.2. Records + +Historical record хранит точный Canonical payload и технические поля, +которые не должны добавляться в Canonical model: + +```text +TradeHistoryRecord +├── venue +├── trade: Trade +├── first_observed_at +├── last_observed_at +├── observation_sources +├── replay_sequence +└── canonical_schema_version + +QuoteHistoryRecord +├── venue +├── quote: Quote +├── observation_sources +├── replay_sequence +└── canonical_schema_version + +CandleRevisionHistoryRecord +├── venue +├── candle: Candle +├── observed_at +├── is_final +├── observation_sources +├── replay_sequence +└── canonical_schema_version +``` + +Records являются immutable, используют `slots` и не копируют Canonical +payload. + +### 6.3. Queries и pages + +Каждый запрос содержит один `venue`, один canonical `symbol`, один +UTC-диапазон и bounded page limit. Candle query дополнительно содержит +case-sensitive `interval`. + +Пустая страница является корректным результатом. Повреждённая строка, +неизвестная версия Canonical schema или неверный enum являются +integrity error; строка не пропускается молча. + +--- + +## 7. Global replay sequence, ordering и keyset + +### 7.1. Причина нового durable tie-breaker + +Текущие таблицы не содержат подходящего immutable порядка Replay: + +- signed `trade_id` нельзя сортировать обычным числовым сравнением на + границах rollover; +- `first_observed_at` может измениться при обновлении provenance; +- между разными типами данных нет общего устойчивого tie-breaker. + +060.29.2 добавляет одну глобальную положительную +`replay_sequence` для Trades, Quotes и Candle revisions. + +Для новых строк sequence назначается только первой durable-вставке и не +изменяется при duplicate/provenance update. Пропуски sequence после +rollback допустимы: важен порядок, а не непрерывность значений. + +### 7.2. Детерминированный backfill существующих строк + +До migration 060.29.2 база не сохраняла global ordinal. Поэтому точный +первоначальный transport, insertion или commit order существующих строк +восстановить невозможно. + +Backfill использует один явно фиксированный порядок: + +```text +Replay event_time +→ data_type rank: Trade=1, Quote=2, Candle revision=3 +→ venue COLLATE "C" +→ symbol COLLATE "C" +→ полная durable identity типа +``` + +Полная identity: + +```text +Trade: executed_at, trade_id +Quote: received_at +Candle revision: interval COLLATE "C", open_time, observed_at +``` + +Этот порядок воспроизводим для одинакового committed dataset, но не +выдаётся за исторический порядок сетевых пакетов. В частности, для уже +существующих Trades с одинаковым `executed_at` backfill не может +доказать первоначальный rollover order. + +Для новых строк immutable sequence сохраняет порядок первой записи при +одинаковом event time. Batch writer обязан назначать sequence в порядке +входного Canonical batch, даже если блокировки строк берутся в другом +безопасном порядке. Между конкурентными транзакциями global sequence +остаётся устойчивым tie-breaker, но не объявляется commit chronology. + +### 7.3. Stable order + +Historical и Replay ordering используют: + +```text +(event_time, replay_sequence) +``` + +`trade_id` не входит в cursor и order key. Для новых строк одинаковый +timestamp сохраняет durable insertion order на границах: + +```text +INT32_MAX → INT32_MIN +-1 → 0 +``` + +Для backfilled строк порядок остаётся детерминированным по policy +раздела 7.2, но не заявляется точным первоначальным rollover order. + +### 7.4. Pagination + +Historical Queries используют только forward keyset pagination. +`OFFSET` запрещён. + +Структурированный cursor содержит: + +- тип данных; +- query scope; +- исходный временной диапазон; +- последнее `event_time`; +- последнее `replay_sequence`; +- version. + +Cursor другого типа, symbol, venue, interval или окна отклоняется. +Page limit можно менять между запросами с тем же cursor. + +### 7.5. Shared sequence и durable schema + +Migration 9 `add_global_market_data_replay_sequence` создаёт один общий +PostgreSQL sequence: + +```text +market_data.replay_sequence +BIGINT +MINVALUE 1 +CACHE 1 +NO CYCLE +OWNED BY NONE +``` + +Отдельная registry/counter table не создаётся. Единственным +распределителем ordinal для новых Trades, Quotes и Candle revisions +является shared sequence. Его нельзя привязать через `OWNED BY` к одной +колонке, потому что он обслуживает сразу три таблицы. + +Каждая из таблиц получает колонку `replay_sequence BIGINT`, общий +`DEFAULT nextval(...)`, `NOT NULL` и именованный `CHECK (> 0)`. После +backfill изменение уже назначенного значения запрещает database trigger +на partitioned parent. Тот же контракт обязан действовать в default, +ранее созданных monthly и будущих partitions. + +Один cross-table `UNIQUE` constraint для трёх partitioned tables в +PostgreSQL недоступен. Глобальная уникальность обеспечивается shared +sequence и контролируемыми repositories. Ручная вставка явно заданного +`replay_sequence` вне этих adapters не входит в поддерживаемый контракт. + +### 7.6. Блокирующая migration и атомарный backfill + +Migration 9 выполняется существующим `StorageMigrationRunner` в одной +transaction. Сначала runner владеет общим +`STORAGE_MIGRATION_ADVISORY_LOCK_ID`. Затем первым statement migration 9, +до любых table locks, берётся единый +`MARKET_DATA_PARTITION_ADVISORY_LOCK_ID`. Этот же partition advisory lock +используют `PostgresMarketDataPartitionManager` и +`PostgresMarketDataRetentionService`. + +Полный обязательный порядок блокировок: + +```text +STORAGE_MIGRATION_ADVISORY_LOCK_ID +→ MARKET_DATA_PARTITION_ADVISORY_LOCK_ID +→ ACCESS EXCLUSIVE trades +→ ACCESS EXCLUSIVE quotes +→ ACCESS EXCLUSIVE candle_revisions +``` + +Так migration, Partition Manager и Retention сериализуют partition DDL +до захвата table locks. Ни один участник не может одновременно держать +lock default partition и ждать parent table, пока migration держит +parent и ждёт default. Это исключает deadlock `parent ↔ default`. + +После двух advisory locks для parent tables и их partitions рекурсивно +берётся `ACCESS EXCLUSIVE` lock в одном порядке: + +```text +trades → quotes → candle_revisions +``` + +Блокировка запрещает конкурентные writes и partition DDL до завершения +migration. Так новая строка не может появиться между построением +глобального порядка, установкой defaults и включением ограничений. + +Backfill строит transaction-local mapping через один `UNION ALL` всех +трёх типов и назначает `row_number()` по policy раздела 7.2. Обновление +строк выполняется по полной durable primary identity. Использование +`ctid` и зависимость от физического порядка строк или порядка вычисления +`nextval()` запрещены. + +После backfill sequence переводится за максимальное назначенное +значение. Для пустого dataset следующий вызов возвращает `1`. Только +после этого устанавливаются `DEFAULT`, `NOT NULL`, positive checks, +immutable triggers и partitioned keyset indexes: + +```text +Trades: (venue, symbol, executed_at, replay_sequence) +Quotes: (venue, symbol, received_at, replay_sequence) +Candles: (venue, symbol, interval, open_time, replay_sequence) +Candles replay preparation: + (venue, symbol, interval, observed_at, replay_sequence) +``` + +Существующий Trade index с `trade_id` в migration 9 не удаляется. +Ошибка любого statement откатывает schema, backfill и запись версии 9; +повторный запуск начинается с согласованного состояния. + +> **Внимание: migration 9 является блокирующей.** Её длительность +> пропорциональна числу сохранённых строк, стоимости backfill и созданию +> indexes. Перед применением к большой production database обязательны +> замер времени и дополнительного дискового места на её актуальной +> копии, проверенный backup, rollback plan и согласованное maintenance +> window. Online staged migration с chunked backfill, concurrent indexes +> и поэтапным включением ограничений в 060.29.2 не реализуется. Если она +> понадобится, это будет отдельное архитектурное и deployment-решение. + +### 7.7. Совместимость writers + +Single Trade, атомарная запись Trade с checkpoint, Quote и Candle +используют database default. Duplicate либо provenance update сохраняет +первоначальный `replay_sequence` и не включает эту колонку в `UPDATE`. + +Trade batch сначала полностью валидируется. Затем writer назначает по +одному sequence каждому элементу строго в исходном Canonical tuple. +Только связанные пары `(prepared_trade, replay_sequence)` сортируются по +существующему identity lock order и записываются. Поэтому безопасный +порядок захвата row locks не меняет порядок событий внутри исходного +batch, включая `INT32_MAX → INT32_MIN` и `-1 → 0`. + +Sequence не является transaction counter: duplicate, conflict или +rollback могут оставить пропуск. Между конкурентными transactions он +показывает порядок выдачи ordinal, но не обещает commit chronology. + +### 7.8. PostgreSQL Trade History reader + +060.29.2 добавляет отдельный synchronous provider-based adapter +`PostgresTradeHistoryRepository`. Он реализует только +`TradeHistoryReaderProtocol`, не расширяет write facade, не читает +checkpoint и не владеет lifecycle PostgreSQL pool. + +Query использует один parameterized `SELECT`, полуоткрытый фильтр +`[start_time, end_time)`, forward keyset predicate +`(executed_at, replay_sequence) > (...)`, тот же ascending `ORDER BY` и +`LIMIT query.limit + 1`. `OFFSET`, `SELECT *`, `FOR UPDATE` и любые +writes запрещены. `next_cursor` указывает на последний возвращённый +record только тогда, когда получена лишняя строка. + +Reader явно проецирует все Canonical и provenance columns и строго +проверяет signed Trade ID, Decimal values, aggressor side, UTC times, +Canonical schema version, observation sources, query scope и positive +sequence. Повреждённая строка не пропускается. + +Обычная страница является текущим committed view одного statement. +Repository не выполняет `commit`, `rollback` и не меняет isolation +level: transaction принадлежит connection provider. Это позволяет +обычным Historical callers явно владеть границей чтения. 060.29.3 +повторно использует тот же строгий row mapper, но строит Replay через +отдельный snapshot query внутри короткой `READ ONLY REPEATABLE READ` +transaction. + +Неверный тип query отклоняется до получения connection как +`MarketDataAccessValidationError`. Повреждённая строка или page +invariant дают `MarketDataAccessIntegrityError`; backend/provider error +даёт `MarketDataAccessOperationError` с исходной причиной. Существующие +Access errors не переоборачиваются, а `KeyboardInterrupt` и +`SystemExit` не перехватываются. + +### 7.9. PostgreSQL Quote/Candle History readers + +060.29.3 добавляет независимые `PostgresQuoteHistoryRepository` и +`PostgresCandleRevisionHistoryRepository`. Они используют тот же +provider-based lifecycle и ту же строгую политику ошибок, что и Trade +History reader, но сохраняют собственные публичные оси истории: + +```text +Quote: (received_at, replay_sequence) +Candle revision: (open_time, replay_sequence) +``` + +Quote query использует полуоткрытый диапазон `received_at` и явно +проецирует Canonical payload, provenance, global sequence и schema +version. Candle query дополнительно требует один case-sensitive +`interval` и использует полуоткрытый диапазон `open_time`. Оба reader +используют forward keyset pagination и `LIMIT query.limit + 1`. + +Строгая материализация PostgreSQL-строк вынесена в общий внутренний +модуль. Trade, Quote, Candle History и Replay Plan Builder поэтому +одинаково проверяют Canonical values, UTC, provenance, schema version и +positive global sequence. Извлечение Trade-маппера не меняет публичное +поведение принятого 060.29.2. + +`MarketDataHistoricalAccess` является DB-neutral фасадом трёх reader +protocols. Его создание не выполняет I/O; фасад не владеет pool и не +переоборачивает ошибки конкретного reader. + +--- + +## 8. Snapshot и concurrent writes + +Historical pages являются текущим committed view. Между страницами +Live/Recovery могут добавить строки, а Retention может удалить данные. +Поэтому обычная пагинация не обещает межстраничный snapshot. + +Replay не читает медленно меняющиеся страницы во время playback. +Сначала в одной короткой транзакции: + +```text +READ ONLY +REPEATABLE READ +``` + +материализуется bounded `ReplayPlan`. После этого cursor, transaction и +connection закрываются, а Replay работает только с immutable tuple в +памяти. + +Долговременная PostgreSQL transaction на период виртуального времени +запрещена. Превышение `max_records` является явной ошибкой, а не +частичным либо молчаливо обрезанным Replay. + +### 8.1. Отдельный PostgreSQL Replay snapshot query + +Публичный Candle History query нельзя использовать как источник Replay: +он выбирает свечи по `open_time`, тогда как конкретная ревизия должна +воспроизводиться по `observed_at`. + +`PostgresReplayPlanBuilder` выполняет один отдельный parameterized +`UNION ALL` query только для запрошенных типов: + +```text +Trade → replay_at = executed_at +Quote → replay_at = received_at +Candle revision → replay_at = observed_at +``` + +Static SQL branches выбираются только по `ReplayDataType`; venue, +symbols, временной диапазон и intervals всегда передаются параметрами. +Итоговый query сортирует все типы по +`(replay_at, replay_sequence)` и запрашивает `max_records + 1` строк. +Лишняя строка вызывает `ReplayPlanLimitExceededError`; частичный plan не +создаётся. + +Builder владеет только короткими cursor и transaction внутри одного +вызова `create_plan()`. Первой SQL-командой transaction устанавливается: + +```sql +SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY; +``` + +Pool и connection provider принадлежат приложению. После materialization +transaction и connection закрыты, а возвращённый `ReplayPlan` не зависит +от PostgreSQL. Повреждённые строки дают +`MarketDataAccessIntegrityError`, backend/provider failures — +`MarketDataAccessOperationError` с исходной причиной, превышение лимита +остаётся отдельной Replay-ошибкой. `BaseException` не перехватывается. + +--- + +## 9. Replay contracts + +### 9.1. ReplayEvent и ReplayPlan + +`ReplayEvent` содержит: + +- `venue`; +- UTC `replay_at`; +- immutable `replay_sequence`; +- точный Canonical payload; +- `is_final` только для Candle revision. + +`ReplayPlan` хранит исходный immutable request и полностью +материализованный tuple событий. Пустой plan является допустимым no-op. +Непустой plan строго возрастает по `(replay_at, replay_sequence)` и не +содержит повторяющихся global sequence. + +### 9.2. Clock + +Контракты разделяются: + +- `MarketDataClockProtocol` — только чтение текущего времени; +- `ReplayClockProtocol` — управляемое `advance_to()`. + +Одна Replay Session владеет одними Virtual Clock. Стартовое время равно +`request.start_time`, включая пустой plan. Время не может уменьшаться. + +060.29.4 добавляет concrete `DeterministicReplayClock` с единственным +изменяемым состоянием — текущим UTC-временем. Clock не читает системное +время, не выполняет sleep, I/O и не создаёт задач. + +Начальное значение и каждый `advance_to()` принимают timezone-aware +`datetime`, совместимый с уже принятыми Replay time contracts, и +канонизируют его в обычный UTC `datetime`. Naive datetime запрещён. + +Переход вперёд изменяет `now`. Переход на тот же абсолютный момент +разрешён как идемпотентный no-op: несколько событий могут иметь один +`replay_at` и различаться только `replay_sequence`. Только переход назад +вызывает `ReplayClockError`; после любой ошибки состояние не меняется. + +Clock не знает о `ReplayPlan`, sequence и верхней границе request. У него +нет `reset()`, `advance_by()`, `start()` и `stop()`. Один экземпляр +принадлежит одной Session и не разделяется между Session или OS threads. +Внутренние lock и owner registry не добавляются; управление +последовательно и синхронно выполняет будущая Replay Session. + +### 9.3. Consumer и Session + +Одна Session имеет одного async consumer. Доставка выполняется строго +последовательно: + +```text +clock.advance_to(event.replay_at) + ↓ +await consumer.consume(event) +``` + +Fan-out, fire-and-forget и error isolation в Build не добавляются. + +060.29.5 добавляет concrete `ReplaySession`. Отдельный класс +`ReplayEngine` не создаётся: engine является последовательным циклом +внутри `ReplaySession.run()`. + +Конструктор получает уже материализованный точный `ReplayPlan`, один +`ReplayClockProtocol` и один `ReplayConsumerProtocol`. Clock передаётся +Session в исключительное владение и до запуска обязан находиться в +каноническом UTC-времени `plan.request.time_range.start_time`. +Несовпадение считается ошибкой dependency и даёт +`MarketDataReplayValidationError`; Session не выполняет скрытый reset. + +Plan сохраняется по identity и может использоваться для создания других +независимых Session. Public `clock` предоставляет consumer-facing +границу `MarketDataClockProtocol`, а lifecycle consumer остаётся у +caller/composition. + +Session является one-shot и имеет состояния: + +```text +CREATED → RUNNING → COMPLETED + ↘ FAILED + ↘ CANCELLED +``` + +Повторный, конкурентный или рекурсивный `run()` запрещён. Session +переводится в `RUNNING` до первого `await`, поэтому внутри одного event +loop второй caller немедленно получает `ReplaySessionStateError` без +lock и без влияния на первый запуск. Для повторения создаются новые +Session и Clock. + +Empty plan штатно проходит `CREATED → RUNNING → COMPLETED`, не вызывает +Clock или consumer и оставляет время в `request.start_time`. `run()` не +возвращает Result DTO или progress: успешный результат равен `None`. + +### 9.4. Consumer Factory и Composition + +060.29.6 добавляет только две новые публичные границы: + +```text +ReplayConsumerFactoryProtocol +ReplaySessionFactory.prepare_session(request) +``` + +`ReplayConsumerFactoryProtocol` синхронно создаёт отдельный consumer для +одной Session и получает: + +- точный immutable `ReplayPlan`; +- Clock этой же Session через read-only `MarketDataClockProtocol`. + +Factory обязана возвращать новый stateful consumer при каждом вызове, +не выполнять I/O, не создавать task и не запускать lifecycle consumer. +Default/no-op consumer не предоставляется: конкретный consumer выбирает +вызывающая подсистема с реальным use case. + +`ReplaySessionFactory` получает только `ReplayPlanBuilderProtocol` и +`ReplayConsumerFactoryProtocol`. Конструктор сохраняет зависимости и не +выполняет I/O. Явный `prepare_session()` выполняет строго одну +последовательность: + +```text +exact ReplayPlanRequest validation + ↓ +plan_builder.create_plan(request) + ↓ +exact ReplayPlan + plan.request identity validation + ↓ +fresh DeterministicReplayClock(request.start_time) + ↓ +consumer_factory.create_consumer(plan, тот же Clock) + ↓ +fresh ReplaySession(plan, тот же Clock, consumer) +``` + +Метод `prepare_session()` намеренно является синхронным и потенциально +блокирующим: concrete PostgreSQL builder выполняет чтение snapshot в +потоке вызывающего кода. Composition не скрывает SQL через +`asyncio.to_thread()`, потому что cancellation asyncio task не остановит +уже выполняющийся SQL в worker thread. Выбор отдельного worker остаётся +у внешнего caller. + +Возвращается обычная `ReplaySession` в состоянии `CREATED`. +Composition не вводит Result DTO, отдельный `ReplayEngine`, service, +registry, cache, lock или single-flight. Caller отдельно и явно +выполняет `await session.run()` и владеет этой coroutine/task. + +Каждый вызов `prepare_session()` создаёт новый Clock, consumer и +Session. Один immutable plan может быть повторно возвращён injected +builder, но изменяемые части графа между Session не разделяются. Один и +тот же concrete Clock по identity передаётся consumer factory через +read-only Protocol и Session через управляющий Protocol. + +Composition не сериализует параллельные вызовы. Thread-safety общего +injected builder и consumer factory является их внешним контрактом; +два уже созданных графа не разделяют изменяемые Session/Clock/consumer. + +Ошибки builder и consumer factory распространяются без обёртки, retry и +частичного результата. Неверный тип request, неверный exact plan или +подмена `plan.request` дают `MarketDataReplayValidationError` до создания +следующей зависимости. Class objects и async implementations вместо +синхронных dependency instances отклоняются при создании composition. + +060.29.6 не меняет Bootstrap, settings, SQL, PostgreSQL pool, Runtime, +Telegram или startup приложения. Подготовка выполняется только после +явного вызова `prepare_session()`, а playback — только после явного +вызова `run()`. + +--- + +## 10. Lifecycle, cancellation и ошибки + +1. Создание contracts, access facade и `ReplaySessionFactory` не + выполняет I/O; явный `prepare_session()` является blocking-границей. +2. Historical readers используют существующий managed pool, но не + открывают и не закрывают его. +3. ReplayPlan полностью отделяется от PostgreSQL до playback. +4. Caller владеет coroutine task `ReplaySession.run()`. +5. Session не создаёт скрытую root/background task. +6. Consumer error распространяется вызывающему коду и останавливает + доставку следующих событий. +7. `CancelledError` не проглатывается. +8. Clock error является terminal для Session. +9. Частично доставленный consumer prefix нельзя транзакционно отменить; + повтор выполняется новой Session по явному решению caller. +10. Clock переводится до consumer; после consumer failure или + cancellation остаётся во времени начатого события. +11. Исходные error и cancellation распространяются без обёртки; Session + только фиксирует `FAILED` или `CANCELLED`. +12. Session не создаёт task, lock, connection, retry или cleanup и не + управляет lifecycle consumer. + +--- + +## 11. Scope 060.29.0–060.29.1 + +### 060.29.0 + +- этот architecture document; +- read/write boundary; +- event-time и ordering policy; +- global replay sequence policy; +- snapshot, resource и failure policy; +- разбиение Build и acceptance criteria. + +### 060.29.1 + +- immutable Historical records; +- time range, queries, typed cursors и pages; +- runtime-checkable Historical reader protocols; +- отдельная error hierarchy; +- Replay request, event, materialized plan и session state; +- clock, consumer, plan builder и session Protocol-контракты; +- DB-neutral unit-тесты. + +060.29.0–060.29.1 не выполняют SQL, network I/O, filesystem I/O и не +создают asyncio tasks. + +--- + +## 12. Следующие подэтапы + +### 060.29.2 + +- migration 9 с shared global `replay_sequence` без registry table; +- блокирующий атомарный backfill существующих строк; +- immutable sequence и partitioned keyset indexes; +- совместимость single/batch writers без изменения checkpoint semantics; +- отдельный PostgreSQL Trade Historical reader; +- unit- и opt-in PostgreSQL verification. + +Детальный утверждённый контракт закреплён в разделах 7.5–7.8. + +### 060.29.3 + +- Quote History по `(received_at, replay_sequence)`; +- Candle History по `(open_time, replay_sequence)` и одному interval; +- общий DB-neutral Historical Access facade; +- единые строгие PostgreSQL row mappers; +- отдельный snapshot query с Candle replay axis `observed_at`; +- короткая `READ ONLY REPEATABLE READ` transaction; +- bounded immutable ReplayPlan и fail-fast `max_records + 1`. + +### 060.29.4 + +- concrete `DeterministicReplayClock`; +- UTC normalization без чтения wall clock; +- разрешённый равный переход и запрет движения назад; +- single-owner policy без lock, reset и lifecycle; +- public export и DB-neutral unit-тесты. + +### 060.29.5 + +- concrete `ReplaySession` без отдельного `ReplayEngine`; +- injected Clock/consumer и fail-fast проверка начального времени; +- one-shot lifecycle и последовательный playback loop; +- error/cancellation propagation без retry и rollback; +- DB-neutral async unit-тесты и public export. + +### 060.29.6 + +- `ReplayConsumerFactoryProtocol` для обязательного concrete consumer; +- единый `ReplaySessionFactory.prepare_session()`; +- явная blocking composition + `request → plan → fresh Clock → fresh consumer → fresh Session`; +- один Clock по identity для Session и consumer через разные + Protocol-границы; +- отсутствие default consumer, скрытого `to_thread`, cache и + автоматического startup; +- DB-neutral unit-тесты composition, identity, ошибок и конкурентности. + +### 060.29.7 + +- verification-only integration без предварительного production diff; +- существующий opt-in harness только для локальной `dzentra_test_*` БД; +- локальный путь + `Loopback Trade Runtime → Storage → Historical Access → Replay`; +- mixed-type PostgreSQL snapshot и caller-owned playback; +- реальная граница `READ ONLY REPEATABLE READ` при concurrent commit; +- возврат PostgreSQL resources до первого consumer call; +- limit, integrity, provider/backend и consumer failure paths; +- независимые графы двух concurrent blocking preparations; +- десятикратный повтор нового PostgreSQL target. + +### 060.29.8 + +- обязательный Pyright gate, compileall и проверка зависимостей; +- расширенный Access/Replay/Storage/migration unit-набор; +- десятикратный PostgreSQL Replay target; +- полный PostgreSQL Storage и общий integration-наборы; +- отдельная opt-out проверка без подключения к базе; +- fixed stress и полная offline-регрессия; +- итоговый read-only review архитектуры, SQL, ordering, lifecycle, + обработки ошибок, освобождения ресурсов и документации. + +Подэтап остаётся verification-only. Dzengi live и acquisition soak не +являются обязательными: Build не добавляет live endpoint wiring или +долгоживущие Replay-задачи, а детерминированный путь +`Loopback Runtime → Storage → Historical Access → Replay` проверен на +локальной PostgreSQL. + +--- + +## 13. Test strategy + +### Обязательная статическая типизация + +- Pyright `1.1.411` запускается в режиме `standard`, совпадающем с + используемым Pylance; +- каноническая команда — `scripts/check_python_types.sh`; +- допустимый результат — только `0 errors, 0 warnings`; +- проверяются Market Data, Storage, Bootstrap, Runtime Events и все их + unit/integration/stress/live/support-тесты; +- gate входит в обычную offline pytest-регрессию через отдельный + static-тест и поэтому не может быть пропущен при приёмке Build; +- `type: ignore`, отключение диагностик или ослабление режима не + используются для исправления новых ошибок; +- область проверки в следующих Build может только расширяться. + +### Contracts + +- frozen/slots; +- строгие типы, включая запрет `bool` вместо `int`; +- UTC normalization и запрет naive datetime; +- полуоткрытые временные границы; +- canonical symbol, venue и case-sensitive Candle interval; +- provenance и Canonical schema version; +- cursor scope/version; +- typed pages и строгий order key; +- runtime-checkable Protocols. + +### Ordering + +- одинаковое event time и разные sequence; +- `INT32_MAX → INT32_MIN`; +- `-1 → 0`; +- provenance update не меняет sequence; +- размер страницы не меняет итоговый порядок. + +### Migration и writers + +- детерминированный mixed-type backfill независимо от physical order; +- shared sequence во всех parent/default/existing/future partitions; +- direct update sequence запрещён trigger; +- duplicate и provenance update сохраняют первоначальный ordinal; +- batch input order сохраняется независимо от identity lock order; +- concurrent migration callers, writer wait и partition-manager race; +- partial failure откатывает migration 9, clean retry проходит; +- gaps после duplicate и transaction rollback считаются корректными. + +### PostgreSQL Trade History + +- start inclusive и end exclusive; +- empty, exact-limit и extra-row pages; +- смена page limit не создаёт skips или duplicates; +- equal timestamp и signed rollover упорядочиваются по sequence; +- default и monthly partitions дают один стабильный результат; +- повреждённая строка и неизвестная schema version не пропускаются; +- DB failure сохраняет cause и не оставляет connection/cursor; +- чтение не меняет Trade, provenance, checkpoint или sequence. + +### PostgreSQL Quote/Candle History и Replay snapshot + +- Quote start inclusive и end exclusive по `received_at`; +- Candle History start inclusive и end exclusive по `open_time`; +- case-sensitive Candle interval и устойчивые keyset pages; +- строгая проверка Quote prices и Candle OHLCV/revision metadata; +- Candle с `open_time` вне Replay range и `observed_at` внутри включается; +- Candle с `open_time` внутри и `observed_at` вне исключается; +- static branches включают только запрошенные data types; +- один global order для одинакового времени разных типов; +- `max_records + 1` даёт ошибку без частичного plan; +- transaction mode, cleanup, backend cause и `BaseException` проверены. + +### Replay lifecycle + +- Protocol, slots, initial `CREATED` и read-only properties; +- fail-fast dependency validation и точное начальное время Clock; +- empty plan без вызовов Clock/consumer; +- точный порядок tuple, Clock-before-consumer и event identity; +- одинаковое `replay_at` с разными global sequence; +- последовательная backpressure без параллельной доставки; +- consumer/Clock failure и исходная error identity; +- cancellation без доставки suffix и без проглатывания; +- concurrent/repeated/reentrant `run()` rejection; +- независимые Session и Clock при общем immutable plan; +- отсутствие hidden tasks, retries, cleanup и PostgreSQL connections. + +### Deterministic Replay Clock + +- начальное UTC-время и normalization разных UTC offsets; +- запрет non-datetime и naive datetime; +- прямой переход с сохранением микросекунд; +- повторный переход на тот же абсолютный момент; +- несколько событий с одинаковым `replay_at`; +- обратный переход через `ReplayClockError` без изменения `now`; +- независимость Clock разных Session; +- read-only `now`, `__slots__` и отсутствие reset/lifecycle/tasks/I/O. + +### Consumer и Composition + +- runtime-checkable `ReplayConsumerFactoryProtocol` и public exports; +- fail-fast constructor validation, включая class objects и async + implementations; +- exact request до builder и identity request в созданном plan; +- порядок builder → Clock → consumer → Session; +- один Clock по identity для consumer factory и Session; +- новый Clock, consumer и Session при каждом вызове; +- пустой plan без автоматического запуска consumer; +- blocking preparation в caller thread без скрытого `to_thread`; +- ошибки builder/factory по identity, без retry и wrapping; +- отсутствие hidden tasks, lifecycle calls, cache и automatic startup; +- независимые графы для двух concurrent callers. + +### PostgreSQL Replay and Failure Verification + +- opt-in отключён без точного флага и явного локального DSN; +- destructive reset разрешён только для проверенной `dzentra_test_*` БД; +- local Loopback Trade проходит Runtime, Storage, History и Replay; +- Trade, Quote и Candle revisions воспроизводятся в global order; +- empty snapshot остаётся штатным no-op; +- test-only barrier фиксирует snapshot до concurrent commit без `sleep`; +- первая Session не видит post-snapshot commit, следующая видит его; +- pool с `max_size=1` освобождается до playback и может быть переоткрыт; +- limit/integrity/provider/backend errors не создают consumer или Session; +- consumer failure/cancellation происходят после освобождения DB; +- Replay не меняет Market Data или persistent checkpoint; +- два blocking caller реально участвуют и получают отдельные графы; +- threads, tasks, cursors, transactions и connections закрываются; +- новый target проходит однократно и десять раз с + `ResourceWarning` как error. + +--- + +## 14. Вне scope + +- raw exchange documents и точный transport arrival log; +- загрузка рыночной истории до первого запуска persistent storage; +- автоматический historical backfill; +- пользовательские ордера, fills, позиции и private account operations; +- аналитические вычисления и Feature Engineering; +- стратегии, Risk, Portfolio и Order Management; +- полноценный Backtesting; +- wall-clock pacing, speed, pause, seek и resume; +- автоматический Retention Scheduler; +- persistent Quote/Candle production consumers; +- distributed lease, leader election и HA failover; +- Telegram/HTTP public API; +- автоматический запуск Replay вместе с приложением; +- online staged migration global sequence без maintenance window. + +--- + +## 15. Граница доступного периода + +После 060.29 база предоставляет запросы и Replay только для данных, +которые действительно находятся в PostgreSQL. + +Начальная граница production Trade history: + +```text +первый успешный запуск +MARKET_DATA_STORAGE_ENABLED=true +и включённого Production Trade Stream +``` + +060.28 заполняет bounded downtime gap после уже известного persistent +checkpoint, если нужные Trades ещё доступны в REST API Dzengi. Он не +загружает всю историю до первого checkpoint. + +Конечная граница Trade history постоянно продвигается работающим +Runtime. Retention по умолчанию выключен, поэтому программного срока +удаления истории нет. Фактический период всё равно ограничен: + +- временем первого production-запуска storage; +- периодами, когда Runtime или база были недоступны; +- глубиной истории, доступной Recovery API; +- ручной очисткой, будущей Retention policy и объёмом диска. + +Для Quotes и Candle revisions production consumers пока не подключены. +Historical Access возвращает только строки, фактически записанные через +их repositories; полнота production Quote/Candle history не заявляется. + +Каждая Historical page отражает committed-состояние на время +собственного запроса. Последовательная pagination даёт доступ ко всем +фактически сохранённым строкам диапазона только при неизменном dataset +между страницами. Live/Recovery-запись и Retention могут изменить этот +набор; единый межстраничный snapshot и полная биржевая история без +отдельного completeness metadata не гарантируются. Initial backfill +потребует нового согласованного Build. + +--- + +## 16. Acceptance criteria + +060.29.0–060.29.1 могут быть приняты, если: + +1. Read contracts не меняют write-only Storage API. +2. Нет SQL, `psycopg`, repository, Runtime или Bootstrap implementation. +3. Все public value objects immutable и slotted. +4. Query windows aware, UTC и half-open. +5. Cursor строго связан с query scope. +6. `trade_id` не входит в Historical order key. +7. Records и Replay events сохраняют identity Canonical payload. +8. ReplayPlan строго упорядочен и bounded. +9. Clock/Consumer/Session представлены только Protocol-контрактами. +10. Комментарии и docstrings новых файлов написаны по-русски. +11. Целевые unit-тесты проходят. +12. Отдельный read-only review не выявляет findings. + +060.29.2 может быть принят, если: + +1. Migration 9 атомарно и детерминированно заполняет global sequence. +2. Shared sequence является единственным allocator; registry table нет. +3. Все partitions получают positive `NOT NULL` sequence и keyset indexes. +4. Provenance update и duplicate не меняют уже записанный ordinal. +5. Trade batch сохраняет input order до identity lock sorting. +6. Migration concurrency, blocking и rollback проверены на PostgreSQL. +7. Trade History использует half-open range и forward keyset pagination. +8. Reader строго проверяет строки и сохраняет error cause. +9. Reader не владеет pool lifecycle и не меняет Runtime/Bootstrap. +10. Unit, PostgreSQL target и полная регрессия проходят. +11. Отдельный acceptance read-only review не выявляет findings. + +060.29.3 может быть принят, если: + +1. Quote History использует `received_at` и forward keyset pagination. +2. Candle History использует `open_time` и case-sensitive interval. +3. Все три reader используют единые строгие PostgreSQL row mappers. +4. Historical Access facade не выполняет I/O и не владеет pool. +5. Replay Builder использует отдельный snapshot query, а не Candle + History pagination. +6. Candle Replay фильтруется по `observed_at`, а не по `open_time`. +7. Snapshot materializes в одной короткой `READ ONLY REPEATABLE READ` + transaction и закрывает все ресурсы до возврата plan. +8. SQL branches статичны, а пользовательские значения параметризованы. +9. Итоговый порядок строго возрастает по + `(replay_at, replay_sequence)` для всех типов. +10. `max_records + 1` даёт `ReplayPlanLimitExceededError` без частичного + результата. +11. Unit, PostgreSQL target и полная регрессия проходят. +12. Отдельный acceptance read-only review не выявляет findings. + +060.29.4 может быть принят, если: + +1. Clock начинается с переданного aware времени и хранит его в UTC. +2. Clock не читает wall clock и не выполняет I/O или sleep. +3. `advance_to()` разрешает прямой и равный переход. +4. Обратный переход даёт `ReplayClockError` и не меняет `now`. +5. Ошибка типа или naive datetime не меняет состояние. +6. Один Clock не содержит shared Session state или внутренних lock. +7. Нет reset, relative advance, lifecycle и скрытых asyncio tasks. +8. Clock соответствует `MarketDataClockProtocol` и + `ReplayClockProtocol`. +9. В подэтапе нет Replay Session, consumer delivery или composition. +10. Unit и полная регрессия проходят. +11. Отдельный acceptance read-only review не выявляет findings. + +060.29.5 может быть принят, если: + +1. Concrete `ReplaySession` соответствует `ReplaySessionProtocol` и не + вводит отдельный `ReplayEngine`. +2. Exact `ReplayPlan`, `ReplayClockProtocol` и `ReplayConsumerProtocol` + проверяются до первого запуска. +3. Clock начинается точно в каноническом UTC + `plan.request.time_range.start_time`; reset не выполняется. +4. `run()` переводит `CREATED → RUNNING` до первого `await` и допускает + только один запуск. +5. Для каждого события Clock продвигается до одного последовательного + `await consumer.consume(event)`. +6. Empty plan завершается без вызовов dependency и не меняет Clock. +7. Ошибка переводит Session в `FAILED`, cancellation — в `CANCELLED`, а + исходный объект исключения распространяется без обёртки. +8. Повторный, конкурентный и рекурсивный запуск дают + `ReplaySessionStateError`, не повреждая активный или terminal state. +9. Session не создаёт tasks, locks, I/O, retries, rollback или cleanup. +10. В подэтапе нет Plan Builder composition, выбора consumer, Bootstrap + или автоматического startup. +11. Unit и полная регрессия проходят. +12. Отдельный acceptance read-only review не выявляет findings. + +060.29.6 может быть принят, если: + +1. `ReplayConsumerFactoryProtocol` является единственной новой границей + выбора concrete consumer. +2. `ReplaySessionFactory` проверяет синхронные dependency instances и не + выполняет I/O или создание объектов графа в конструкторе. +3. `prepare_session()` является синхронной blocking-границей и не + использует скрытый `to_thread` или background task. +4. Builder получает точный исходный request ровно один раз, а результат + обязан быть точным `ReplayPlan` с тем же request по identity. +5. Порядок сборки равен plan → Clock → consumer → Session. +6. Consumer factory и Session получают один Clock по identity через + разные Protocol-границы. +7. Каждый вызов создаёт новые Clock, consumer и Session без cache, + registry, lock и single-flight. +8. Подготовленная Session остаётся в `CREATED`; consumer, lifecycle и + playback не запускаются автоматически. +9. Default/no-op consumer, отдельный ReplayEngine, Result DTO, Bootstrap + и settings не добавлены. +10. Ошибки dependencies и cancellation распространяются без обёртки, + retry и частично возвращённой Session. +11. Unit и полная offline-регрессия проходят. +12. Отдельный acceptance read-only review не выявляет findings. + +060.29.7 может быть принят, если: + +1. Подэтап остаётся verification-only, если тесты не выявили реальный + production defect. +2. Используется существующий opt-in PostgreSQL harness с проверкой + локального endpoint и имени `dzentra_test_*`. +3. Local Trade проходит Production Runtime, PostgreSQL Storage, + Historical Access, ReplayPlan, Composition и Session. +4. Mixed Trade/Quote/Candle plan воспроизводится в точном global order, + а consumer наблюдает уже продвинутое время Clock. +5. Реальный `REPEATABLE READ` snapshot не видит commit после + зафиксированной точки, а следующая Session видит его полностью. +6. PostgreSQL cursor, transaction и connection освобождены до первого + consumer call; подготовленная Session работает после закрытия pool. +7. Limit, integrity, provider/backend errors не создают consumer или + частичную Session, а pool после контролируемой ошибки остаётся либо + снова становится пригодным к работе. +8. Consumer failure и cancellation сохраняют принятый lifecycle Session + без удержания PostgreSQL resources и изменения durable данных. +9. Два concurrent caller действительно выполняют real preparation и + получают разные Plan, Clock, consumer и Session. +10. Нет незавершённых PostgreSQL resources, threads или asyncio tasks. +11. Обязательный Pyright gate проходит с нулём errors и warnings. +12. Новый target проходит один раз и десять раз, полный PostgreSQL + Storage integration, Replay unit и offline regression проходят. +13. Отдельный acceptance read-only review не выявляет findings. + +060.29.8 может быть принят, если: + +1. Pyright завершается с `0 errors, 0 warnings`, static pytest gate, + compileall и `pip check` проходят. +2. Расширенный Access/Replay/Storage/migration unit-набор проходит. +3. Двенадцать PostgreSQL Replay-сценариев стабильно проходят десять + последовательных запусков с `ResourceWarning` как ошибкой. +4. Полный PostgreSQL Storage и общий integration-наборы проходят с + `ResourceWarning` как ошибкой. +5. Без opt-in flag и DSN весь PostgreSQL Storage-набор пропускается и не + устанавливает соединение с базой. +6. Fixed stress и полная offline-регрессия проходят. +7. Нет необъяснённых skips, незавершённых tasks, threads, cursors, + transactions или PostgreSQL connections. +8. Replay не изменяет Canonical Market Data, operational checkpoint или + production lifecycle. +9. Итоговые read-only review архитектуры, SQL, тестов и документации не + выявляют findings P0–P3. +10. Подэтап не меняет production-код, если матрица не воспроизвела + реальный дефект. + +--- + +## 17. ADR + +### ADR-060.29-001 — Read API не расширяет write-only Storage facade + +Historical Access получает отдельные DB-neutral contracts и adapters. + +### ADR-060.29-002 — Replay использует event-time chronology + +Порядок исходных transport packets не заявляется без raw arrival log. + +### ADR-060.29-003 — Global immutable replay sequence + +Один PostgreSQL sequence без registry table применяется ко всем +Canonical Market Data и не изменяется при provenance update. + +### ADR-060.29-004 — Replay materializes bounded snapshot + +PostgreSQL transaction завершается до начала виртуального playback. + +### ADR-060.29-005 — Canonical payload не заменяется Replay DTO + +Replay envelope сохраняет точный `Trade`, `Quote` или `Candle`. + +### ADR-060.29-006 — Session является one-shot и caller-owned + +Session не создаёт скрытую задачу и не скрывает consumer/cancellation +errors. + +### ADR-060.29-007 — Migration 9 использует maintenance window + +060.29.2 выбирает атомарную блокирующую migration. Online staged rollout +для большой production database требует отдельного решения. + +### ADR-060.29-008 — Candle History и Replay используют разные оси + +Публичная Candle History выбирает ревизии по `open_time`. Replay Plan +Builder использует отдельный snapshot query и выбирает те же ревизии по +`observed_at`, то есть в момент, когда они фактически стали известны. + +### ADR-060.29-009 — Replay engine находится внутри one-shot Session + +060.29.5 не вводит отдельный `ReplayEngine`. Последовательный playback +выполняет `ReplaySession.run()`, а создание plan, fresh Clock и consumer +остаётся явной задачей Composition 060.29.6. + +### ADR-060.29-010 — Composition разделяет preparation и playback + +060.29.6 использует один синхронный blocking +`ReplaySessionFactory.prepare_session()` без скрытого worker и +автоматического startup. Каждый вызов создаёт новый Clock, обязательный +consumer и `ReplaySession`; caller отдельно владеет `session.run()`. + +### ADR-060.29-011 — PostgreSQL acceptance остаётся verification-only + +060.29.7 не добавляет production consumer или новые runtime-границы. +Он проверяет уже принятые Storage, Historical, snapshot, Composition и +Session на локальной PostgreSQL. Production-код может измениться только +при воспроизводимом нарушении принятого контракта и только вместе с +минимальным regression-тестом; test hooks в production запрещены. + +--- + +## 18. Test evidence + +Итоговая приёмочная проверка 060.29.0–060.29.1: + +```text +Historical Access + Replay unit target: 111 passed +Full offline regression: 2416 passed, 69 deselected +Historical Access exact-type re-review: clean +Replay exact-type re-review: clean +Combined acceptance read-only review: clean +``` + +Подэтапы 060.29.0–060.29.1 приняты. На этом этапе Build 060.29 оставался +`In Progress`. + +Итоговая приёмочная проверка 060.29.2: + +```text +Target unit: 227 passed +Full Market Data Storage unit: 290 passed +New 060.29.2 PostgreSQL integration: 12 passed +Full PostgreSQL Storage integration: 63 passed +Full integration with PostgreSQL: 76 passed +Full offline regression: 2461 passed, 81 deselected +Compileall через временный pycache: clean +Migration acceptance read-only review: clean +Reader/writer acceptance read-only review: clean +Повторный целевой набор review: 90 passed +``` + +Подэтапы 060.29.2–060.29.3 приняты. На этом этапе Build 060.29 оставался +`In Progress`. + +Итоговая приёмочная проверка 060.29.3: + +```text +Historical Access + Replay unit target: 270 passed +Expanded Access/Replay/Storage unit: 446 passed +New 060.29.3 PostgreSQL integration: 2 passed +Full PostgreSQL Storage integration: 65 passed +Full integration with PostgreSQL: 78 passed +Full offline regression: 2581 passed, 83 deselected +Compileall через временный pycache: clean +Повторный целевой unit-набор: 270 passed +Formal acceptance read-only review: clean +``` + +Подэтапы 060.29.3–060.29.4 приняты после отдельных формальных +приёмочных read-only review. + +Проверка реализации Clock 060.29.4: + +```text +Deterministic Replay Clock unit target: 22 passed +Full Replay unit target: 100 passed +Historical Access + Replay unit target: 292 passed +Full offline regression: 2603 passed, 83 deselected +Compileall через временный pycache: clean +git diff --check: clean +Короткий post-fix read-only review: clean +Formal acceptance read-only review: clean +``` + +Контракт и реализация 060.29.5 приняты после исправления fail-fast +проверки class objects и чистого повторного read-only review. + +Проверка реализации Replay Session 060.29.5: + +```text +Replay Session unit target: 34 passed +Full Replay unit target: 134 passed +Historical Access + Replay unit target: 326 passed +Full offline regression: 2637 passed, 83 deselected +Compileall через временный pycache: clean +git diff --check: clean +Короткий implementation read-only review: clean +Post-fix acceptance read-only review: clean +``` + +Проверка реализации Consumer and Composition Integration 060.29.6: + +```text +Replay Session Factory unit target: 43 passed +Full Replay unit target: 177 passed +Historical Access + Replay unit target: 369 passed +Full offline regression: 2680 passed, 83 deselected +Compileall через временный pycache: clean +Tracked и untracked whitespace checks: clean +Implementation read-only review: clean +Formal acceptance read-only review: clean +``` + +Подэтап 060.29.6 принят. На этом этапе Build 060.29 оставался +`In Progress`. + +Проверка реализации PostgreSQL Replay and Failure Verification 060.29.7: + +```text +Новый PostgreSQL Replay target: 12 passed +Десятикратный прогон target: 10 × 12 passed +Full PostgreSQL Storage integration: 77 passed +Full integration with PostgreSQL: 90 passed +Historical Access + Replay unit target: 369 passed +PostgreSQL opt-out: 77 skipped +Pyright mandatory gate: 0 errors, 0 warnings +Pyright pytest gate: 1 passed +Full offline regression: 2681 passed, 95 deselected +Compileall через временный pycache: clean +Два повторных read-only review: clean +Production-код в 060.29.7: только static narrowing fix; + контракт и поведение неизменны +``` + +Подэтап 060.29.7 принят после чистого формального приёмочного read-only +review. На этом этапе Build 060.29 оставался `In Progress` до выполнения +060.29.8. + +Финальная приёмочная проверка 060.29.8: + +```text +Pyright mandatory gate: 0 errors, 0 warnings +Pyright pytest gate: 1 passed +Expanded Access/Replay/Storage unit: 545 passed +PostgreSQL Replay target repeated: 10 × 12 passed +Full PostgreSQL Storage integration: 77 passed +Full integration with PostgreSQL: 90 passed +PostgreSQL opt-out isolation: 77 skipped +Fixed stress target: 3 passed, 1 deselected +Full offline regression: 2681 passed, 95 deselected +pip check: clean +Compileall через временный pycache: clean +Tracked/untracked whitespace checks: clean +Три независимых read-only review: clean +Production-код в 060.29.8: без изменений +``` + +В integration и stress наборах `ResourceWarning` считался ошибкой. +Подэтап 060.29.8 и весь Build 060.29 приняты. + +--- + +## 19. Следующий Build + +Build 060.29 завершён и принят. Следующий этап — Build 060.30, итоговый +аудит и документальное закрытие ветки Trades Feed. diff --git a/docs/roadmap/master-roadmap.md b/docs/roadmap/master-roadmap.md index 5ce35b6..33b44bf 100644 --- a/docs/roadmap/master-roadmap.md +++ b/docs/roadmap/master-roadmap.md @@ -6,10 +6,10 @@ |---|---| | Тип | Master Delivery Roadmap | | Статус | Active | -| Версия | 2.2 | -| Дата актуализации | 2026-08-01 | -| Текущий завершённый Build | 060.28 | -| Текущий Build | 060.29 — Planned | +| Версия | 2.3 | +| Дата актуализации | 2026-08-02 | +| Текущий завершённый Build | 060.29 | +| Текущий Build | 060.30 — Planned | --- @@ -42,33 +42,33 @@ test evidence находятся в документах конкретных Bu ```text Market Data Acquisition ↓ -Persistent Checkpoint and Startup Recovery завершён +Market Data Access and Replay завершён ↓ -Build 060.28 +Build 060.29 ↓ -Completed → следующий Build 060.29 +Completed → следующий Build 060.30 ``` -Build 060.28 завершён и принят: +Build 060.29 завершён и принят: -- persistent checkpoint подтверждается Canonical Trade history; -- Trade и checkpoint продвигаются одной PostgreSQL transaction; -- при запуске восстанавливаются checkpoint и deduplication tail; -- Startup Recovery завершается до buffered Live processing; -- Bootstrap и cancellation сохраняют строгий lifecycle pool/Runtime; -- restart/failure, concurrency и финальная регрессия приняты. +- Historical Access отделён от write-only Storage API; +- migration 9 добавила общий immutable Replay sequence; +- Trades, Quotes и Candle revisions читаются устойчивыми keyset pages; +- Replay строится как bounded `REPEATABLE READ` snapshot; +- Clock, Session и Composition остаются детерминированными и caller-owned; +- PostgreSQL failure paths и финальная регрессия приняты. Подробности: ```text -docs/migrations/build_060_28.md +docs/migrations/build_060_29.md ``` --- # Активная программа — Market Data Acquisition -## Завершённая ветка Trades Feed +## Завершённые Build ветки Trades Feed | Build | Результат | Статус | |---|---|---| @@ -83,6 +83,7 @@ docs/migrations/build_060_28.md | 060.26 | Integration and Regression | Completed | | 060.27 | Persistent Market Data Storage | Completed | | 060.28 | Persistent Checkpoint and Startup Recovery | Completed | +| 060.29 | Market Data Access and Replay | Completed | ## Build 060.26 — Integration and Regression @@ -145,7 +146,7 @@ REST и только затем продолжает buffered Live processing. ### Build 060.29 — Market Data Access and Replay -**Статус:** Planned +**Статус:** Completed Назначение: @@ -159,6 +160,12 @@ REST и только затем продолжает buffered Live processing. Build. Они принадлежат Market Data Processing и Feature Engineering и получат отдельный scope после появления устойчивого Storage/Replay. +Результат: сохранённые Canonical Trades, Quotes и Candle revisions +доступны через отдельный Historical Access и могут воспроизводиться в +детерминированном global order через caller-owned Replay Session. + +Подробный итог: `docs/migrations/build_060_29.md`. + ### Build 060.30 — Market Data Acquisition Final Documentation **Статус:** Planned @@ -1739,6 +1746,6 @@ read-only архитектурного анализа. Старый ориент Актуальная контрольная точка: ```text -Завершён: Build 060.28 — Persistent Checkpoint and Startup Recovery -Следующий: Build 060.29 — Market Data Access and Replay +Завершён: Build 060.29 — Market Data Access and Replay +Следующий: Build 060.30 — Market Data Acquisition Final Documentation ``` diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..8d20805 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,35 @@ +{ + "include": [ + "app/src/bootstrap", + "app/src/core", + "app/src/integrations", + "app/src/market_data", + "app/src/runtime_events", + "app/src/storage", + "app/tests/integration/market_data", + "app/tests/live/market_data", + "app/tests/static", + "app/tests/stress/market_data", + "app/tests/support", + "app/tests/unit/bootstrap", + "app/tests/unit/core", + "app/tests/unit/integrations", + "app/tests/unit/market_data", + "app/tests/unit/storage", + "app/tests/unit/test_live_trade_stream_support.py", + "app/tests/unit/test_postgres_market_data_support.py", + "app/tests/unit/test_trade_stream_runtime_support.py" + ], + "venvPath": "app", + "venv": ".venv", + "pythonVersion": "3.12", + "typeCheckingMode": "standard", + "executionEnvironments": [ + { + "root": "app", + "extraPaths": [ + "app" + ] + } + ] +} diff --git a/scripts/check_python_types.sh b/scripts/check_python_types.sh new file mode 100755 index 0000000..3c7d3e2 --- /dev/null +++ b/scripts/check_python_types.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +cd "${project_root}" +app/.venv/bin/python -m pyright --project pyrightconfig.json --warnings