Build 060.25: implement Production Runtime Integration
This commit is contained in:
@@ -17,4 +17,14 @@ EXCHANGE_API_KEY=
|
||||
EXCHANGE_API_SECRET=
|
||||
EXCHANGE_TIMEOUT_SEC=10
|
||||
EXCHANGE_TESTNET=true
|
||||
DEFAULT_SYMBOL=BTC/USD_LEVERAGE
|
||||
DEFAULT_SYMBOL=BTC/USD_LEVERAGE
|
||||
|
||||
TRADE_STREAM_ENABLED=false
|
||||
TRADE_STREAM_WS_URL=
|
||||
TRADE_STREAM_SYMBOLS=
|
||||
TRADE_STREAM_OPEN_TIMEOUT_SECONDS=10
|
||||
TRADE_STREAM_PROBE_TIMEOUT_SECONDS=20
|
||||
TRADE_STREAM_CLOSE_TIMEOUT_SECONDS=10
|
||||
TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS=30
|
||||
TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS=5
|
||||
TRADE_STREAM_RECOVERY_WINDOW_MS=3599999
|
||||
|
||||
@@ -5,7 +5,11 @@ from __future__ import annotations
|
||||
from aiogram import Bot, Dispatcher
|
||||
from aiogram.client.default import DefaultBotProperties
|
||||
|
||||
from src.bootstrap.application import ApplicationComposition
|
||||
from src.bootstrap.logging import setup_logging
|
||||
from src.bootstrap.trade_stream_runtime import (
|
||||
build_trade_stream_production_runtime,
|
||||
)
|
||||
from src.core.config import load_settings
|
||||
from src.notifications.targets import NotificationTargetRegistry
|
||||
from src.storage.schema import init_schema
|
||||
@@ -13,7 +17,7 @@ from src.telegram.routers import setup_routers
|
||||
from src.trading.journal.service import JournalService
|
||||
|
||||
|
||||
def create_app() -> tuple[Bot, Dispatcher]:
|
||||
def create_app() -> ApplicationComposition:
|
||||
settings = load_settings()
|
||||
|
||||
setup_logging(settings.log_level)
|
||||
@@ -37,18 +41,9 @@ def create_app() -> tuple[Bot, Dispatcher]:
|
||||
pass
|
||||
raise
|
||||
|
||||
try:
|
||||
journal.log_info(
|
||||
"app_started",
|
||||
"Приложение запущено",
|
||||
{
|
||||
"env": settings.app_env,
|
||||
"exchange_name": settings.exchange_name,
|
||||
"default_symbol": settings.default_symbol,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
trade_stream_runtime = (
|
||||
build_trade_stream_production_runtime(settings)
|
||||
)
|
||||
|
||||
bot = Bot(
|
||||
token=settings.bot_token,
|
||||
@@ -61,4 +56,24 @@ def create_app() -> tuple[Bot, Dispatcher]:
|
||||
|
||||
setup_routers(dispatcher)
|
||||
|
||||
return bot, dispatcher
|
||||
try:
|
||||
journal.log_info(
|
||||
"app_started",
|
||||
"Приложение запущено",
|
||||
{
|
||||
"env": settings.app_env,
|
||||
"exchange_name": settings.exchange_name,
|
||||
"default_symbol": settings.default_symbol,
|
||||
"trade_stream_enabled": (
|
||||
settings.trade_stream.enabled
|
||||
),
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return ApplicationComposition(
|
||||
bot=bot,
|
||||
dispatcher=dispatcher,
|
||||
trade_stream_runtime=trade_stream_runtime,
|
||||
)
|
||||
|
||||
254
app/src/bootstrap/application.py
Normal file
254
app/src/bootstrap/application.py
Normal file
@@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
|
||||
from aiogram import Bot, Dispatcher
|
||||
|
||||
from src.market_data.acquisition.runtime.trade_stream_production_runtime import (
|
||||
TradeStreamProductionRuntimeProtocol,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ApplicationComposition:
|
||||
"""Корневые компоненты одного запущенного экземпляра приложения."""
|
||||
|
||||
bot: Bot
|
||||
dispatcher: Dispatcher
|
||||
trade_stream_runtime: (
|
||||
TradeStreamProductionRuntimeProtocol | None
|
||||
)
|
||||
|
||||
|
||||
async def run_application(
|
||||
application: ApplicationComposition,
|
||||
) -> None:
|
||||
"""
|
||||
Выполнять Telegram polling и опциональный Trade Stream как одно целое.
|
||||
|
||||
Ошибка включённого Trade Stream считается фатальной для всего
|
||||
приложения. Остановка Telegram, Trade Stream и bot session
|
||||
выполняется одним владельцем и не оставляет фоновых root tasks.
|
||||
"""
|
||||
polling_task = asyncio.create_task(
|
||||
application.dispatcher.start_polling(
|
||||
application.bot,
|
||||
close_bot_session=False,
|
||||
),
|
||||
name="telegram-polling",
|
||||
)
|
||||
runtime_task = (
|
||||
asyncio.create_task(
|
||||
application.trade_stream_runtime.run(),
|
||||
name="trade-stream-runtime",
|
||||
)
|
||||
if application.trade_stream_runtime is not None
|
||||
else None
|
||||
)
|
||||
|
||||
primary_error: BaseException | None = None
|
||||
|
||||
try:
|
||||
await _wait_for_root_tasks(
|
||||
polling_task=polling_task,
|
||||
runtime_task=runtime_task,
|
||||
)
|
||||
except BaseException as error:
|
||||
primary_error = error
|
||||
|
||||
cleanup_task = asyncio.create_task(
|
||||
_shutdown_application(
|
||||
application=application,
|
||||
polling_task=polling_task,
|
||||
runtime_task=runtime_task,
|
||||
primary_error=primary_error,
|
||||
),
|
||||
name="application-shutdown",
|
||||
)
|
||||
cleanup_error = await _await_cleanup(
|
||||
cleanup_task,
|
||||
primary_error=primary_error,
|
||||
)
|
||||
|
||||
if primary_error is not None:
|
||||
if cleanup_error is not None:
|
||||
primary_error.add_note(
|
||||
"Application cleanup also failed: "
|
||||
f"{type(cleanup_error).__name__}."
|
||||
)
|
||||
|
||||
raise primary_error.with_traceback(
|
||||
primary_error.__traceback__,
|
||||
)
|
||||
|
||||
if cleanup_error is not None:
|
||||
raise cleanup_error.with_traceback(
|
||||
cleanup_error.__traceback__,
|
||||
)
|
||||
|
||||
|
||||
async def _wait_for_root_tasks(
|
||||
*,
|
||||
polling_task: asyncio.Task[None],
|
||||
runtime_task: asyncio.Task[None] | None,
|
||||
) -> None:
|
||||
if runtime_task is None:
|
||||
await polling_task
|
||||
return
|
||||
|
||||
done_tasks, _ = await asyncio.wait(
|
||||
(
|
||||
polling_task,
|
||||
runtime_task,
|
||||
),
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
|
||||
if runtime_task in done_tasks:
|
||||
if runtime_task.cancelled():
|
||||
raise RuntimeError(
|
||||
"Trade Stream Runtime was cancelled unexpectedly."
|
||||
)
|
||||
|
||||
runtime_error = runtime_task.exception()
|
||||
|
||||
if runtime_error is not None:
|
||||
raise runtime_error.with_traceback(
|
||||
runtime_error.__traceback__,
|
||||
)
|
||||
|
||||
raise RuntimeError(
|
||||
"Trade Stream Runtime terminated unexpectedly."
|
||||
)
|
||||
|
||||
await polling_task
|
||||
|
||||
|
||||
async def _shutdown_application(
|
||||
*,
|
||||
application: ApplicationComposition,
|
||||
polling_task: asyncio.Task[None],
|
||||
runtime_task: asyncio.Task[None] | None,
|
||||
primary_error: BaseException | None,
|
||||
) -> BaseException | None:
|
||||
cleanup_error: BaseException | None = None
|
||||
polling_cancelled = False
|
||||
runtime_cancelled = False
|
||||
|
||||
if not polling_task.done():
|
||||
polling_cancelled = True
|
||||
polling_task.cancel()
|
||||
|
||||
runtime = application.trade_stream_runtime
|
||||
|
||||
if runtime is not None:
|
||||
try:
|
||||
await runtime.stop()
|
||||
except BaseException as error:
|
||||
cleanup_error = _merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
|
||||
if runtime_task is not None and not runtime_task.done():
|
||||
runtime_cancelled = True
|
||||
runtime_task.cancel()
|
||||
|
||||
cleanup_error = await _observe_root_task(
|
||||
polling_task,
|
||||
expected_cancellation=polling_cancelled,
|
||||
primary_error=primary_error,
|
||||
cleanup_error=cleanup_error,
|
||||
)
|
||||
|
||||
if runtime_task is not None:
|
||||
cleanup_error = await _observe_root_task(
|
||||
runtime_task,
|
||||
expected_cancellation=runtime_cancelled,
|
||||
primary_error=primary_error,
|
||||
cleanup_error=cleanup_error,
|
||||
)
|
||||
|
||||
try:
|
||||
await application.bot.session.close()
|
||||
except BaseException as error:
|
||||
cleanup_error = _merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
|
||||
return cleanup_error
|
||||
|
||||
|
||||
async def _observe_root_task(
|
||||
task: asyncio.Task[None],
|
||||
*,
|
||||
expected_cancellation: bool,
|
||||
primary_error: BaseException | None,
|
||||
cleanup_error: BaseException | None,
|
||||
) -> BaseException | None:
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError as error:
|
||||
if (
|
||||
not expected_cancellation
|
||||
and error is not primary_error
|
||||
and not isinstance(primary_error, asyncio.CancelledError)
|
||||
):
|
||||
return _merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
except BaseException as error:
|
||||
if error is not primary_error:
|
||||
return _merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
|
||||
return cleanup_error
|
||||
|
||||
|
||||
async def _await_cleanup(
|
||||
cleanup_task: asyncio.Task[BaseException | None],
|
||||
*,
|
||||
primary_error: BaseException | None,
|
||||
) -> BaseException | None:
|
||||
interrupted_error: asyncio.CancelledError | None = None
|
||||
|
||||
while not cleanup_task.done():
|
||||
try:
|
||||
await asyncio.shield(cleanup_task)
|
||||
except asyncio.CancelledError as error:
|
||||
if primary_error is None:
|
||||
primary_error = error
|
||||
interrupted_error = error
|
||||
continue
|
||||
|
||||
cleanup_error = cleanup_task.result()
|
||||
|
||||
if interrupted_error is not None:
|
||||
if cleanup_error is not None:
|
||||
interrupted_error.add_note(
|
||||
"Application cleanup also failed: "
|
||||
f"{type(cleanup_error).__name__}."
|
||||
)
|
||||
|
||||
raise interrupted_error
|
||||
|
||||
return cleanup_error
|
||||
|
||||
|
||||
def _merge_cleanup_error(
|
||||
current_error: BaseException | None,
|
||||
new_error: BaseException,
|
||||
) -> BaseException:
|
||||
if current_error is None:
|
||||
return new_error
|
||||
|
||||
current_error.add_note(
|
||||
"Additional application cleanup failure: "
|
||||
f"{type(new_error).__name__}."
|
||||
)
|
||||
return current_error
|
||||
125
app/src/bootstrap/trade_stream_runtime.py
Normal file
125
app/src/bootstrap/trade_stream_runtime.py
Normal file
@@ -0,0 +1,125 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from src.core.config import Settings
|
||||
from src.integrations.exchange.rest_client import ExchangeRestClient
|
||||
from src.market_data.acquisition.adapters.dzengi.rest import (
|
||||
DzengiTradesDocumentSource,
|
||||
)
|
||||
from src.market_data.acquisition.adapters.dzengi.websocket import (
|
||||
DzengiUnifiedWebSocketAdapter,
|
||||
)
|
||||
from src.market_data.acquisition.adapters.dzengi.websocket_control_message_handler import (
|
||||
DzengiWebSocketControlMessageHandler,
|
||||
)
|
||||
from src.market_data.acquisition.adapters.dzengi.websocket_inbound_message_classifier import (
|
||||
DzengiWebSocketInboundMessageClassifier,
|
||||
)
|
||||
from src.market_data.acquisition.adapters.dzengi.websocket_transport import (
|
||||
DzengiWebSocketTransport,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.acquisition_runtime_event_logging_consumer import (
|
||||
AcquisitionRuntimeEventLoggingConsumer,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.acquisition_runtime_event_publisher import (
|
||||
AcquisitionRuntimeEventPublisher,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.trade_stream_production_runtime import (
|
||||
TradeStreamProductionRuntime,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_session import (
|
||||
WebSocketSession,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_subscription_manager import (
|
||||
WebSocketSubscriptionManager,
|
||||
)
|
||||
from src.market_data.acquisition.trade_stream_runtime_composition import (
|
||||
build_trade_stream_runtime_composition,
|
||||
)
|
||||
|
||||
|
||||
def build_trade_stream_production_runtime(
|
||||
settings: Settings,
|
||||
) -> TradeStreamProductionRuntime | None:
|
||||
"""
|
||||
Собрать Production Trade Stream Runtime без запуска lifecycle.
|
||||
|
||||
Выключенный feature flag возвращает ``None`` до создания
|
||||
WebSocket/REST-зависимостей. Все stateful-компоненты включённого
|
||||
Runtime создаются один раз и затем передаются по identity.
|
||||
"""
|
||||
trade_stream = settings.trade_stream
|
||||
|
||||
if not trade_stream.enabled:
|
||||
return None
|
||||
|
||||
headers = {
|
||||
"Origin": settings.exchange_base_url.rstrip("/"),
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
if settings.exchange_api_key:
|
||||
headers["X-MBX-APIKEY"] = settings.exchange_api_key
|
||||
|
||||
transport = DzengiWebSocketTransport(
|
||||
url=trade_stream.websocket_url,
|
||||
headers=headers,
|
||||
open_timeout=trade_stream.open_timeout_seconds,
|
||||
ping_interval=None,
|
||||
ping_timeout=None,
|
||||
probe_timeout=trade_stream.probe_timeout_seconds,
|
||||
close_timeout=trade_stream.close_timeout_seconds,
|
||||
)
|
||||
session = WebSocketSession(transport)
|
||||
subscription_manager = WebSocketSubscriptionManager(
|
||||
transport,
|
||||
supports_unsubscribe=False,
|
||||
)
|
||||
event_publisher = AcquisitionRuntimeEventPublisher(
|
||||
(
|
||||
AcquisitionRuntimeEventLoggingConsumer(),
|
||||
)
|
||||
)
|
||||
recovery_document_source = DzengiTradesDocumentSource(
|
||||
ExchangeRestClient(settings=settings),
|
||||
)
|
||||
message_adapter = DzengiUnifiedWebSocketAdapter()
|
||||
|
||||
composition = build_trade_stream_runtime_composition(
|
||||
session=session,
|
||||
transport=transport,
|
||||
subscription_manager=subscription_manager,
|
||||
event_publisher=event_publisher,
|
||||
message_adapter=message_adapter,
|
||||
recovery_document_source=recovery_document_source,
|
||||
symbols=trade_stream.symbols,
|
||||
heartbeat_timeout_seconds=(
|
||||
trade_stream.heartbeat_timeout_seconds
|
||||
),
|
||||
scheduler_interval_seconds=(
|
||||
trade_stream.scheduler_interval_seconds
|
||||
),
|
||||
max_recovery_window_ms=trade_stream.recovery_window_ms,
|
||||
)
|
||||
|
||||
return TradeStreamProductionRuntime(
|
||||
session=session,
|
||||
transport=transport,
|
||||
subscription_manager=subscription_manager,
|
||||
event_publisher=event_publisher,
|
||||
trade_stream_service=(
|
||||
composition.trade_stream_acquisition_service
|
||||
),
|
||||
message_classifier=(
|
||||
DzengiWebSocketInboundMessageClassifier()
|
||||
),
|
||||
control_message_handler=(
|
||||
DzengiWebSocketControlMessageHandler()
|
||||
),
|
||||
live_processing_gate=composition.live_processing_gate,
|
||||
reconnect_recovery_coordinator=(
|
||||
composition.runtime_reconnect_recovery_coordinator
|
||||
),
|
||||
runtime_supervisor=composition.runtime_supervisor,
|
||||
runtime_scheduler=composition.runtime_scheduler,
|
||||
symbols=trade_stream.symbols,
|
||||
)
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
@@ -19,6 +20,21 @@ ENV_FILE = BASE_DIR / ".env"
|
||||
load_dotenv(ENV_FILE)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TradeStreamSettings:
|
||||
"""Настройки Production Trade Stream Runtime."""
|
||||
|
||||
enabled: bool
|
||||
websocket_url: str
|
||||
symbols: tuple[str, ...]
|
||||
open_timeout_seconds: float
|
||||
probe_timeout_seconds: float
|
||||
close_timeout_seconds: float
|
||||
heartbeat_timeout_seconds: float
|
||||
scheduler_interval_seconds: float
|
||||
recovery_window_ms: int
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Settings:
|
||||
# Telegram
|
||||
@@ -40,6 +56,7 @@ class Settings:
|
||||
exchange_timeout_sec: int
|
||||
exchange_testnet: bool
|
||||
default_symbol: str
|
||||
trade_stream: TradeStreamSettings
|
||||
|
||||
# Database
|
||||
db_host: str
|
||||
@@ -75,6 +92,206 @@ def _parse_int(raw_value: str, default: int) -> int:
|
||||
return int(value)
|
||||
|
||||
|
||||
def _parse_strict_bool(
|
||||
raw_value: str,
|
||||
*,
|
||||
name: str,
|
||||
default: bool,
|
||||
) -> bool:
|
||||
value = (raw_value or "").strip().lower()
|
||||
|
||||
if not value:
|
||||
return default
|
||||
|
||||
if value in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
|
||||
if value in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
|
||||
raise ValueError(
|
||||
f"{name} must be a boolean value"
|
||||
)
|
||||
|
||||
|
||||
def _parse_positive_float(
|
||||
raw_value: str,
|
||||
*,
|
||||
name: str,
|
||||
default: float,
|
||||
) -> float:
|
||||
value = (raw_value or "").strip()
|
||||
|
||||
try:
|
||||
result = (
|
||||
float(value)
|
||||
if value
|
||||
else float(default)
|
||||
)
|
||||
except ValueError as error:
|
||||
raise ValueError(
|
||||
f"{name} must be a number"
|
||||
) from error
|
||||
|
||||
if not math.isfinite(result) or result <= 0:
|
||||
raise ValueError(
|
||||
f"{name} must be positive and finite"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _parse_positive_int(
|
||||
raw_value: str,
|
||||
*,
|
||||
name: str,
|
||||
default: int,
|
||||
) -> int:
|
||||
value = (raw_value or "").strip()
|
||||
|
||||
try:
|
||||
result = (
|
||||
int(value)
|
||||
if value
|
||||
else int(default)
|
||||
)
|
||||
except ValueError as error:
|
||||
raise ValueError(
|
||||
f"{name} must be an integer"
|
||||
) from error
|
||||
|
||||
if result <= 0:
|
||||
raise ValueError(
|
||||
f"{name} must be positive"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _parse_trade_stream_symbols(
|
||||
raw_value: str,
|
||||
) -> tuple[str, ...]:
|
||||
value = (raw_value or "").strip()
|
||||
|
||||
if not value:
|
||||
raise RuntimeError(
|
||||
"TRADE_STREAM_SYMBOLS is required when "
|
||||
"TRADE_STREAM_ENABLED is true"
|
||||
)
|
||||
|
||||
raw_symbols = value.split(",")
|
||||
|
||||
if any(not symbol.strip() for symbol in raw_symbols):
|
||||
raise ValueError(
|
||||
"TRADE_STREAM_SYMBOLS must not contain empty symbols"
|
||||
)
|
||||
|
||||
return tuple(
|
||||
sorted(
|
||||
{
|
||||
symbol.strip()
|
||||
for symbol in raw_symbols
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _load_trade_stream_settings(
|
||||
*,
|
||||
exchange_base_url: str,
|
||||
) -> TradeStreamSettings:
|
||||
enabled = _parse_strict_bool(
|
||||
os.getenv("TRADE_STREAM_ENABLED", "false"),
|
||||
name="TRADE_STREAM_ENABLED",
|
||||
default=False,
|
||||
)
|
||||
|
||||
if not enabled:
|
||||
return TradeStreamSettings(
|
||||
enabled=False,
|
||||
websocket_url="",
|
||||
symbols=(),
|
||||
open_timeout_seconds=10.0,
|
||||
probe_timeout_seconds=20.0,
|
||||
close_timeout_seconds=10.0,
|
||||
heartbeat_timeout_seconds=30.0,
|
||||
scheduler_interval_seconds=5.0,
|
||||
recovery_window_ms=3_599_999,
|
||||
)
|
||||
|
||||
websocket_url = os.getenv(
|
||||
"TRADE_STREAM_WS_URL",
|
||||
"",
|
||||
).strip()
|
||||
|
||||
if not websocket_url:
|
||||
raise RuntimeError(
|
||||
"TRADE_STREAM_WS_URL is required when "
|
||||
"TRADE_STREAM_ENABLED is true"
|
||||
)
|
||||
|
||||
if not exchange_base_url:
|
||||
raise RuntimeError(
|
||||
"EXCHANGE_BASE_URL is required for Trade Stream Recovery"
|
||||
)
|
||||
|
||||
return TradeStreamSettings(
|
||||
enabled=True,
|
||||
websocket_url=websocket_url,
|
||||
symbols=_parse_trade_stream_symbols(
|
||||
os.getenv("TRADE_STREAM_SYMBOLS", ""),
|
||||
),
|
||||
open_timeout_seconds=_parse_positive_float(
|
||||
os.getenv(
|
||||
"TRADE_STREAM_OPEN_TIMEOUT_SECONDS",
|
||||
"10",
|
||||
),
|
||||
name="TRADE_STREAM_OPEN_TIMEOUT_SECONDS",
|
||||
default=10.0,
|
||||
),
|
||||
probe_timeout_seconds=_parse_positive_float(
|
||||
os.getenv(
|
||||
"TRADE_STREAM_PROBE_TIMEOUT_SECONDS",
|
||||
"20",
|
||||
),
|
||||
name="TRADE_STREAM_PROBE_TIMEOUT_SECONDS",
|
||||
default=20.0,
|
||||
),
|
||||
close_timeout_seconds=_parse_positive_float(
|
||||
os.getenv(
|
||||
"TRADE_STREAM_CLOSE_TIMEOUT_SECONDS",
|
||||
"10",
|
||||
),
|
||||
name="TRADE_STREAM_CLOSE_TIMEOUT_SECONDS",
|
||||
default=10.0,
|
||||
),
|
||||
heartbeat_timeout_seconds=_parse_positive_float(
|
||||
os.getenv(
|
||||
"TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS",
|
||||
"30",
|
||||
),
|
||||
name="TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS",
|
||||
default=30.0,
|
||||
),
|
||||
scheduler_interval_seconds=_parse_positive_float(
|
||||
os.getenv(
|
||||
"TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS",
|
||||
"5",
|
||||
),
|
||||
name="TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS",
|
||||
default=5.0,
|
||||
),
|
||||
recovery_window_ms=_parse_positive_int(
|
||||
os.getenv(
|
||||
"TRADE_STREAM_RECOVERY_WINDOW_MS",
|
||||
"3599999",
|
||||
),
|
||||
name="TRADE_STREAM_RECOVERY_WINDOW_MS",
|
||||
default=3_599_999,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# load all settings
|
||||
def load_settings() -> Settings:
|
||||
bot_token = os.getenv("BOT_TOKEN", "").strip()
|
||||
@@ -82,6 +299,11 @@ def load_settings() -> Settings:
|
||||
if not bot_token:
|
||||
raise RuntimeError("BOT_TOKEN is not set in app/.env")
|
||||
|
||||
exchange_base_url = os.getenv(
|
||||
"EXCHANGE_BASE_URL",
|
||||
"",
|
||||
).strip()
|
||||
|
||||
return Settings(
|
||||
# Telegram
|
||||
bot_token=bot_token,
|
||||
@@ -99,7 +321,7 @@ def load_settings() -> Settings:
|
||||
# Exchange
|
||||
exchange_enabled=_parse_bool(os.getenv("EXCHANGE_ENABLED", "false")),
|
||||
exchange_name=os.getenv("EXCHANGE_NAME", "dzengi").strip() or "dzengi",
|
||||
exchange_base_url=os.getenv("EXCHANGE_BASE_URL", "").strip(),
|
||||
exchange_base_url=exchange_base_url,
|
||||
exchange_ws_url=os.getenv("EXCHANGE_WS_URL", "").strip(),
|
||||
exchange_api_key=os.getenv("EXCHANGE_API_KEY", "").strip(),
|
||||
exchange_api_secret=os.getenv("EXCHANGE_API_SECRET", "").strip(),
|
||||
@@ -107,6 +329,9 @@ def load_settings() -> Settings:
|
||||
exchange_testnet=_parse_bool(os.getenv("EXCHANGE_TESTNET", "false")),
|
||||
default_symbol=os.getenv("DEFAULT_SYMBOL", "ETH/USD_LEVERAGE").strip()
|
||||
or "ETH/USD_LEVERAGE",
|
||||
trade_stream=_load_trade_stream_settings(
|
||||
exchange_base_url=exchange_base_url,
|
||||
),
|
||||
|
||||
# Database
|
||||
db_host=os.getenv("DB_HOST", "localhost").strip() or "localhost",
|
||||
@@ -114,4 +339,4 @@ def load_settings() -> Settings:
|
||||
db_name=os.getenv("DB_NAME", "dzentra_bot").strip() or "dzentra_bot",
|
||||
db_user=os.getenv("DB_USER", "dzentra_bot").strip() or "dzentra_bot",
|
||||
db_password=os.getenv("DB_PASSWORD", "").strip(),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from src.core.config import load_settings
|
||||
from src.core.config import Settings, load_settings
|
||||
from src.integrations.exchange.exceptions import (
|
||||
ExchangeConnectionError,
|
||||
ExchangeResponseError,
|
||||
@@ -15,8 +15,15 @@ from src.integrations.exchange.exceptions import (
|
||||
|
||||
|
||||
class ExchangeRestClient:
|
||||
def __init__(self) -> None:
|
||||
self.settings = load_settings()
|
||||
def __init__(
|
||||
self,
|
||||
settings: Settings | None = None,
|
||||
) -> None:
|
||||
self.settings = (
|
||||
settings
|
||||
if settings is not None
|
||||
else load_settings()
|
||||
)
|
||||
if not self.settings.exchange_base_url:
|
||||
raise ExchangeConnectionError("EXCHANGE_BASE_URL is empty.")
|
||||
self.base_url = self.settings.exchange_base_url.rstrip("/")
|
||||
@@ -133,4 +140,4 @@ class ExchangeRestClient:
|
||||
if not isinstance(payload, dict):
|
||||
raise ExchangeResponseError("Exchange response is not a JSON object.")
|
||||
|
||||
return payload
|
||||
return payload
|
||||
|
||||
@@ -2,26 +2,14 @@
|
||||
|
||||
import asyncio
|
||||
|
||||
from src.bootstrap.application import run_application
|
||||
from src.bootstrap.app_factory import create_app
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# создаём bot + dispatcher
|
||||
bot, dispatcher = create_app()
|
||||
|
||||
# WebSocket stream временно отключён.
|
||||
# Причина: Dzengi Swagger содержит wss:/api/v1/* endpoints,
|
||||
# но runtime probe не нашёл endpoint с WebSocket Upgrade 101.
|
||||
#
|
||||
# Когда Dzengi подтвердит рабочий WS endpoint,
|
||||
# можно будет вернуть запуск:
|
||||
#
|
||||
# from src.integrations.exchange.market_stream import start_market_stream
|
||||
# market_stream_task = asyncio.create_task(start_market_stream())
|
||||
|
||||
# запускаем Telegram polling
|
||||
await dispatcher.start_polling(bot)
|
||||
application = create_app()
|
||||
await run_application(application)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
asyncio.run(main())
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from src.market_data.acquisition.exceptions import (
|
||||
WebSocketControlMessageError,
|
||||
WebSocketMessageRoutingError,
|
||||
)
|
||||
|
||||
|
||||
_TRADE_SUBSCRIPTION_DESTINATION = "trades.subscribe"
|
||||
_SUCCESS_STATUS = "OK"
|
||||
|
||||
|
||||
class DzengiWebSocketControlMessageHandler:
|
||||
"""
|
||||
Provider-specific проверка Dzengi control/ACK сообщений.
|
||||
|
||||
Handler не управляет lifecycle и не хранит pending requests.
|
||||
Ожидаемый correlation ID передаётся владельцем Runtime.
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def handle(
|
||||
self,
|
||||
document: object,
|
||||
*,
|
||||
expected_correlation_id: str,
|
||||
) -> None:
|
||||
"""Проверить ACK текущей Trade Stream подписки."""
|
||||
if not isinstance(document, dict):
|
||||
raise WebSocketMessageRoutingError(
|
||||
"Control message Dzengi WebSocket должен быть объектом."
|
||||
)
|
||||
|
||||
correlation_id = document.get("correlationId")
|
||||
|
||||
if correlation_id != expected_correlation_id:
|
||||
raise WebSocketMessageRoutingError(
|
||||
"Получен control message с неизвестным correlationId."
|
||||
)
|
||||
|
||||
status = document.get("status")
|
||||
|
||||
if not isinstance(status, str) or not status.strip():
|
||||
raise WebSocketMessageRoutingError(
|
||||
"Control message Dzengi WebSocket должен содержать "
|
||||
"непустой строковый status."
|
||||
)
|
||||
|
||||
if status != _SUCCESS_STATUS:
|
||||
raise WebSocketControlMessageError(
|
||||
"Dzengi отклонил Trade Stream subscription ACK."
|
||||
)
|
||||
|
||||
if (
|
||||
document.get("destination")
|
||||
!= _TRADE_SUBSCRIPTION_DESTINATION
|
||||
):
|
||||
raise WebSocketMessageRoutingError(
|
||||
"Получен успешный control message с неизвестным "
|
||||
"destination."
|
||||
)
|
||||
@@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from src.market_data.acquisition.exceptions import (
|
||||
WebSocketMessageRoutingError,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_inbound_message import (
|
||||
WebSocketInboundMessageKind,
|
||||
)
|
||||
|
||||
|
||||
_DZENGI_MARKET_DESTINATIONS = frozenset(
|
||||
{
|
||||
"internal.trade",
|
||||
"ohlc.event",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class DzengiWebSocketInboundMessageClassifier:
|
||||
"""
|
||||
Классификатор декодированных входящих сообщений Dzengi.
|
||||
|
||||
Market markers имеют приоритет над correlationId, чтобы market
|
||||
document с дополнительными transport metadata не был ошибочно
|
||||
принят за control response.
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def classify(
|
||||
self,
|
||||
document: object,
|
||||
) -> WebSocketInboundMessageKind:
|
||||
"""Разделить market documents и ответы на Runtime-команды."""
|
||||
if not isinstance(document, dict):
|
||||
raise WebSocketMessageRoutingError(
|
||||
"Сообщение Dzengi WebSocket должно быть объектом."
|
||||
)
|
||||
|
||||
destination = document.get("destination")
|
||||
|
||||
if (
|
||||
(
|
||||
isinstance(destination, str)
|
||||
and destination in _DZENGI_MARKET_DESTINATIONS
|
||||
)
|
||||
or "Payload" in document
|
||||
):
|
||||
return WebSocketInboundMessageKind.MARKET
|
||||
|
||||
if "correlationId" in document:
|
||||
self._validate_correlation_id(
|
||||
document["correlationId"],
|
||||
)
|
||||
return WebSocketInboundMessageKind.CONTROL
|
||||
|
||||
raise WebSocketMessageRoutingError(
|
||||
"Не удалось определить категорию сообщения "
|
||||
"Dzengi WebSocket."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_correlation_id(
|
||||
correlation_id: object,
|
||||
) -> None:
|
||||
if (
|
||||
isinstance(correlation_id, bool)
|
||||
or not isinstance(correlation_id, (str, int))
|
||||
):
|
||||
raise WebSocketMessageRoutingError(
|
||||
"correlationId сообщения Dzengi WebSocket должен "
|
||||
"быть строкой или целым числом."
|
||||
)
|
||||
|
||||
if (
|
||||
isinstance(correlation_id, str)
|
||||
and not correlation_id.strip()
|
||||
):
|
||||
raise WebSocketMessageRoutingError(
|
||||
"correlationId сообщения Dzengi WebSocket "
|
||||
"не должен быть пустым."
|
||||
)
|
||||
@@ -0,0 +1,445 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from typing import Protocol, cast
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from websockets.asyncio.client import connect as websocket_connect
|
||||
from websockets.protocol import State
|
||||
from websockets.typing import Subprotocol
|
||||
|
||||
from src.market_data.acquisition.exceptions import (
|
||||
WebSocketTransportError,
|
||||
WebSocketTransportNotConnectedError,
|
||||
)
|
||||
|
||||
|
||||
class _WebSocketConnectionProtocol(Protocol):
|
||||
"""
|
||||
Минимальный контракт соединения.
|
||||
"""
|
||||
|
||||
@property
|
||||
def state(self) -> State:
|
||||
"""Вернуть состояние WebSocket-соединения."""
|
||||
...
|
||||
|
||||
async def close(
|
||||
self,
|
||||
code: int = 1000,
|
||||
reason: str = "",
|
||||
) -> None:
|
||||
"""Закрыть WebSocket-соединение."""
|
||||
...
|
||||
|
||||
async def send(
|
||||
self,
|
||||
message: str | bytes,
|
||||
) -> None:
|
||||
"""Отправить одно сообщение."""
|
||||
...
|
||||
|
||||
async def recv(self) -> str | bytes:
|
||||
"""Получить одно сообщение."""
|
||||
...
|
||||
|
||||
async def ping(self) -> Awaitable[float]:
|
||||
"""Отправить Ping и вернуть ожидание соответствующего Pong."""
|
||||
...
|
||||
|
||||
|
||||
WebSocketConnector = Callable[
|
||||
...,
|
||||
Awaitable[_WebSocketConnectionProtocol],
|
||||
]
|
||||
|
||||
|
||||
def build_dzengi_websocket_url(
|
||||
raw_url: str,
|
||||
) -> str:
|
||||
"""
|
||||
Нормализовать Dzengi HTTP/WebSocket URL до endpoint ``/connect``.
|
||||
"""
|
||||
if not isinstance(raw_url, str):
|
||||
raise TypeError("raw_url must be a string")
|
||||
|
||||
normalized_url = raw_url.strip()
|
||||
|
||||
if not normalized_url:
|
||||
raise ValueError("raw_url must not be empty")
|
||||
|
||||
parsed = urlsplit(normalized_url)
|
||||
|
||||
scheme = {
|
||||
"http": "ws",
|
||||
"https": "wss",
|
||||
"ws": "ws",
|
||||
"wss": "wss",
|
||||
}.get(parsed.scheme.lower())
|
||||
|
||||
if scheme is None or not parsed.netloc:
|
||||
raise ValueError(
|
||||
"raw_url must be an absolute HTTP or WebSocket URL"
|
||||
)
|
||||
|
||||
if parsed.fragment:
|
||||
raise ValueError("raw_url must not contain a fragment")
|
||||
|
||||
path = parsed.path.rstrip("/")
|
||||
|
||||
if not path.endswith("/connect"):
|
||||
path = f"{path}/connect"
|
||||
|
||||
return urlunsplit(
|
||||
(
|
||||
scheme,
|
||||
parsed.netloc,
|
||||
path,
|
||||
parsed.query,
|
||||
"",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class DzengiWebSocketTransport:
|
||||
"""
|
||||
Низкоуровневый WebSocket-транспорт Dzengi.
|
||||
|
||||
Transport управляет только соединением и непрозрачными
|
||||
``str``/``bytes`` сообщениями. JSON parsing, subscription routing,
|
||||
recovery и lifecycle находятся за пределами этого компонента.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_url",
|
||||
"_headers",
|
||||
"_open_timeout",
|
||||
"_ping_interval",
|
||||
"_ping_timeout",
|
||||
"_probe_timeout",
|
||||
"_close_timeout",
|
||||
"_connector",
|
||||
"_connection",
|
||||
"_lifecycle_lock",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
url: str,
|
||||
headers: Mapping[str, str] | None = None,
|
||||
open_timeout: float | None = 10.0,
|
||||
ping_interval: float | None = 20.0,
|
||||
ping_timeout: float | None = 20.0,
|
||||
probe_timeout: float | None = None,
|
||||
close_timeout: float | None = 10.0,
|
||||
connector: WebSocketConnector | None = None,
|
||||
) -> None:
|
||||
self._url = build_dzengi_websocket_url(url)
|
||||
self._headers = self._validate_headers(headers)
|
||||
self._open_timeout = self._validate_timeout(
|
||||
"open_timeout",
|
||||
open_timeout,
|
||||
)
|
||||
self._ping_interval = self._validate_timeout(
|
||||
"ping_interval",
|
||||
ping_interval,
|
||||
)
|
||||
self._ping_timeout = self._validate_timeout(
|
||||
"ping_timeout",
|
||||
ping_timeout,
|
||||
)
|
||||
self._probe_timeout = self._validate_probe_timeout(
|
||||
(
|
||||
probe_timeout
|
||||
if probe_timeout is not None
|
||||
else (
|
||||
ping_timeout
|
||||
if ping_timeout is not None
|
||||
else 20.0
|
||||
)
|
||||
),
|
||||
)
|
||||
self._close_timeout = self._validate_timeout(
|
||||
"close_timeout",
|
||||
close_timeout,
|
||||
)
|
||||
self._connector = (
|
||||
connector
|
||||
if connector is not None
|
||||
else cast(
|
||||
WebSocketConnector,
|
||||
websocket_connect,
|
||||
)
|
||||
)
|
||||
self._connection: _WebSocketConnectionProtocol | None = None
|
||||
self._lifecycle_lock = asyncio.Lock()
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
"""Вернуть нормализованный WebSocket URL."""
|
||||
return self._url
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Показывает, открыто ли соединение."""
|
||||
connection = self._connection
|
||||
|
||||
return (
|
||||
connection is not None
|
||||
and connection.state is State.OPEN
|
||||
)
|
||||
|
||||
async def connect(self) -> None:
|
||||
"""
|
||||
Идемпотентно открыть WebSocket-соединение Dzengi.
|
||||
"""
|
||||
async with self._lifecycle_lock:
|
||||
if self.is_connected:
|
||||
return
|
||||
|
||||
self._connection = None
|
||||
|
||||
try:
|
||||
connection = await self._connector(
|
||||
self._url,
|
||||
additional_headers=(
|
||||
self._headers
|
||||
if self._headers
|
||||
else None
|
||||
),
|
||||
subprotocols=(
|
||||
Subprotocol("json"),
|
||||
),
|
||||
open_timeout=self._open_timeout,
|
||||
ping_interval=self._ping_interval,
|
||||
ping_timeout=self._ping_timeout,
|
||||
close_timeout=self._close_timeout,
|
||||
)
|
||||
except Exception as error:
|
||||
raise WebSocketTransportError(
|
||||
"Не удалось открыть Dzengi WebSocket connection: "
|
||||
f"{error}"
|
||||
) from error
|
||||
|
||||
self._connection = connection
|
||||
|
||||
if self.is_connected:
|
||||
return
|
||||
|
||||
self._connection = None
|
||||
|
||||
try:
|
||||
await connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
raise WebSocketTransportError(
|
||||
"Dzengi WebSocket connection не перешло "
|
||||
"в состояние OPEN."
|
||||
)
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
"""
|
||||
Идемпотентно закрыть WebSocket-соединение.
|
||||
"""
|
||||
async with self._lifecycle_lock:
|
||||
connection = self._connection
|
||||
self._connection = None
|
||||
|
||||
if connection is None:
|
||||
return
|
||||
|
||||
try:
|
||||
await connection.close()
|
||||
except Exception as error:
|
||||
raise WebSocketTransportError(
|
||||
"Не удалось закрыть Dzengi WebSocket connection: "
|
||||
f"{error}"
|
||||
) from error
|
||||
|
||||
async def send(
|
||||
self,
|
||||
message: str | bytes,
|
||||
) -> None:
|
||||
"""Отправить непрозрачное транспортное сообщение."""
|
||||
if not isinstance(message, (str, bytes)):
|
||||
raise TypeError("message must be str or bytes")
|
||||
|
||||
connection = self._require_connection()
|
||||
|
||||
try:
|
||||
await connection.send(message)
|
||||
except Exception as error:
|
||||
self._discard_closed_connection(connection)
|
||||
|
||||
raise WebSocketTransportError(
|
||||
"Не удалось отправить Dzengi WebSocket message: "
|
||||
f"{error}"
|
||||
) from error
|
||||
|
||||
async def receive(self) -> str | bytes:
|
||||
"""Получить непрозрачное транспортное сообщение."""
|
||||
connection = self._require_connection()
|
||||
|
||||
try:
|
||||
return await connection.recv()
|
||||
except Exception as error:
|
||||
self._discard_closed_connection(connection)
|
||||
|
||||
raise WebSocketTransportError(
|
||||
"Не удалось получить Dzengi WebSocket message: "
|
||||
f"{error}"
|
||||
) from error
|
||||
|
||||
async def probe(self) -> bool:
|
||||
"""
|
||||
Подтвердить liveness текущего connection через Ping/Pong.
|
||||
|
||||
Закрытое соединение и отсутствие Pong являются штатным
|
||||
отрицательным результатом. Cancellation не перехватывается.
|
||||
"""
|
||||
connection = self._connection
|
||||
|
||||
if (
|
||||
connection is None
|
||||
or connection.state is not State.OPEN
|
||||
):
|
||||
self._connection = None
|
||||
return False
|
||||
|
||||
try:
|
||||
pong_waiter = await connection.ping()
|
||||
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(
|
||||
pong_waiter,
|
||||
),
|
||||
timeout=self._probe_timeout,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
return False
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as error:
|
||||
self._discard_closed_connection(connection)
|
||||
|
||||
if connection.state is not State.OPEN:
|
||||
return False
|
||||
|
||||
raise WebSocketTransportError(
|
||||
"Не удалось проверить Dzengi WebSocket liveness: "
|
||||
f"{error}"
|
||||
) from error
|
||||
|
||||
return (
|
||||
self._connection is connection
|
||||
and connection.state is State.OPEN
|
||||
)
|
||||
|
||||
def _require_connection(
|
||||
self,
|
||||
) -> _WebSocketConnectionProtocol:
|
||||
"""
|
||||
Вернуть открытое соединение.
|
||||
"""
|
||||
connection = self._connection
|
||||
|
||||
if (
|
||||
connection is None
|
||||
or connection.state is not State.OPEN
|
||||
):
|
||||
self._connection = None
|
||||
|
||||
raise WebSocketTransportNotConnectedError(
|
||||
"Dzengi WebSocket transport is not connected."
|
||||
)
|
||||
|
||||
return connection
|
||||
|
||||
def _discard_closed_connection(
|
||||
self,
|
||||
connection: _WebSocketConnectionProtocol,
|
||||
) -> None:
|
||||
"""
|
||||
Удалить ссылку на закрытое соединение.
|
||||
"""
|
||||
if (
|
||||
self._connection is connection
|
||||
and connection.state is not State.OPEN
|
||||
):
|
||||
self._connection = None
|
||||
|
||||
@staticmethod
|
||||
def _validate_headers(
|
||||
headers: Mapping[str, str] | None,
|
||||
) -> dict[str, str]:
|
||||
if headers is None:
|
||||
return {}
|
||||
|
||||
if not isinstance(headers, Mapping):
|
||||
raise TypeError("headers must be a mapping")
|
||||
|
||||
normalized_headers: dict[str, str] = {}
|
||||
|
||||
for name, value in headers.items():
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise ValueError(
|
||||
"header names must be non-empty strings"
|
||||
)
|
||||
|
||||
if not isinstance(value, str):
|
||||
raise TypeError("header values must be strings")
|
||||
|
||||
normalized_headers[name] = value
|
||||
|
||||
return normalized_headers
|
||||
|
||||
@staticmethod
|
||||
def _validate_timeout(
|
||||
name: str,
|
||||
value: float | None,
|
||||
) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
if (
|
||||
isinstance(value, bool)
|
||||
or not isinstance(value, (int, float))
|
||||
):
|
||||
raise TypeError(
|
||||
f"{name} must be an integer, float, or None"
|
||||
)
|
||||
|
||||
if value <= 0:
|
||||
raise ValueError(
|
||||
f"{name} must be positive"
|
||||
)
|
||||
|
||||
return float(value)
|
||||
|
||||
@staticmethod
|
||||
def _validate_probe_timeout(
|
||||
value: float,
|
||||
) -> float:
|
||||
if (
|
||||
isinstance(value, bool)
|
||||
or not isinstance(value, (int, float))
|
||||
):
|
||||
raise TypeError(
|
||||
"probe_timeout must be an integer or float"
|
||||
)
|
||||
|
||||
normalized_value = float(value)
|
||||
|
||||
if (
|
||||
not math.isfinite(normalized_value)
|
||||
or normalized_value <= 0
|
||||
):
|
||||
raise ValueError(
|
||||
"probe_timeout must be positive and finite"
|
||||
)
|
||||
|
||||
return normalized_value
|
||||
@@ -153,4 +153,31 @@ class TradeFeedRegistryError(MarketDataAcquisitionError):
|
||||
# Ошибка определения типа входящего WebSocket-сообщения
|
||||
# и выбора специализированного адаптера.
|
||||
class WebSocketMessageRoutingError(MarketDataAcquisitionError):
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
# Ошибка декодирования сырого WebSocket-сообщения.
|
||||
class WebSocketMessageDecodeError(MarketDataAcquisitionError):
|
||||
pass
|
||||
|
||||
|
||||
# Ошибка обработки control/ACK сообщения WebSocket-провайдера.
|
||||
class WebSocketControlMessageError(MarketDataAcquisitionError):
|
||||
pass
|
||||
|
||||
|
||||
# Ошибка низкоуровневой WebSocket-инфраструктуры Acquisition Runtime.
|
||||
class WebSocketTransportError(MarketDataAcquisitionError):
|
||||
pass
|
||||
|
||||
|
||||
# Попытка использовать WebSocket-транспорт без открытого соединения.
|
||||
class WebSocketTransportNotConnectedError(WebSocketTransportError):
|
||||
pass
|
||||
|
||||
|
||||
# WebSocket-провайдер не поддерживает транспортную операцию unsubscribe.
|
||||
class WebSocketUnsubscribeNotSupportedError(
|
||||
MarketDataAcquisitionError,
|
||||
):
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from src.market_data.acquisition.runtime.runtime_events import (
|
||||
ConnectedEvent,
|
||||
ConnectFailedEvent,
|
||||
DisconnectedEvent,
|
||||
HeartbeatTimeoutEvent,
|
||||
MessageReceivedEvent,
|
||||
MessageSentEvent,
|
||||
ReconnectCompletedEvent,
|
||||
ReconnectFailedEvent,
|
||||
ReconnectStartedEvent,
|
||||
TransportMessage,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_protocol import (
|
||||
AcquisitionRuntimeEvent,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AcquisitionRuntimeEventLoggingConsumer:
|
||||
"""
|
||||
Безопасно записывает Acquisition Runtime Events в standard logging.
|
||||
|
||||
Содержимое transport payload намеренно не журналируется. Для
|
||||
MessageReceivedEvent и MessageSentEvent фиксируются только тип
|
||||
сообщения и размер payload.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_logger",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
event_logger: logging.Logger = logger,
|
||||
) -> None:
|
||||
if not isinstance(event_logger, logging.Logger):
|
||||
raise TypeError(
|
||||
"event_logger must be a logging.Logger"
|
||||
)
|
||||
|
||||
self._logger = event_logger
|
||||
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
"""Записать одно Acquisition Runtime Event."""
|
||||
if isinstance(event, ConnectedEvent):
|
||||
self._logger.info(
|
||||
"Acquisition WebSocket connected."
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(event, DisconnectedEvent):
|
||||
self._logger.info(
|
||||
"Acquisition WebSocket disconnected."
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(event, ConnectFailedEvent):
|
||||
self._logger.error(
|
||||
"Acquisition WebSocket connection failed: %s",
|
||||
event.reason,
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(event, MessageReceivedEvent):
|
||||
self._log_transport_message(
|
||||
direction="received",
|
||||
message=event.message,
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(event, MessageSentEvent):
|
||||
self._log_transport_message(
|
||||
direction="sent",
|
||||
message=event.message,
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(event, ReconnectStartedEvent):
|
||||
self._logger.info(
|
||||
"Acquisition WebSocket reconnect started: attempt=%d",
|
||||
event.attempt,
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(event, ReconnectCompletedEvent):
|
||||
self._logger.info(
|
||||
"Acquisition WebSocket reconnect completed: attempt=%d",
|
||||
event.attempt,
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(event, ReconnectFailedEvent):
|
||||
self._logger.error(
|
||||
"Acquisition WebSocket reconnect failed: "
|
||||
"attempt=%d reason=%s",
|
||||
event.attempt,
|
||||
event.reason,
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(event, HeartbeatTimeoutEvent):
|
||||
self._logger.warning(
|
||||
"Acquisition WebSocket heartbeat timeout: "
|
||||
"timeout_seconds=%s",
|
||||
event.timeout_seconds,
|
||||
)
|
||||
return
|
||||
|
||||
raise TypeError(
|
||||
"event must be an AcquisitionRuntimeEvent"
|
||||
)
|
||||
|
||||
def _log_transport_message(
|
||||
self,
|
||||
*,
|
||||
direction: str,
|
||||
message: TransportMessage,
|
||||
) -> None:
|
||||
self._logger.debug(
|
||||
"Acquisition WebSocket message %s: "
|
||||
"message_type=%s payload_size=%d",
|
||||
direction,
|
||||
type(message).__name__,
|
||||
len(message.payload),
|
||||
)
|
||||
@@ -0,0 +1,148 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Iterable
|
||||
from contextvars import ContextVar
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from src.market_data.acquisition.runtime.websocket_protocol import (
|
||||
AcquisitionRuntimeEvent,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_active_publisher_ids: ContextVar[tuple[int, ...]] = ContextVar(
|
||||
"active_acquisition_runtime_event_publisher_ids",
|
||||
default=(),
|
||||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class AcquisitionRuntimeEventConsumerProtocol(Protocol):
|
||||
"""
|
||||
Контракт одного потребителя Acquisition Runtime Events.
|
||||
|
||||
Consumer обрабатывает уже произошедший инфраструктурный факт
|
||||
и не управляет lifecycle Publisher.
|
||||
"""
|
||||
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
"""Обработать одно Acquisition Runtime Event."""
|
||||
...
|
||||
|
||||
|
||||
class AcquisitionRuntimeEventPublisher:
|
||||
"""
|
||||
Последовательный in-process Publisher Acquisition Runtime Events.
|
||||
|
||||
Один вызов publish() полностью доставляет событие всем Consumer
|
||||
в порядке их регистрации. Параллельные вызовы сериализуются одним
|
||||
asyncio.Lock.
|
||||
|
||||
Publisher не создаёт фоновых задач, не хранит очередь и не управляет
|
||||
lifecycle Consumer.
|
||||
|
||||
Обычная ошибка одного Consumer журналируется и не препятствует
|
||||
доставке остальным Consumer. Сигналы отмены и другие BaseException
|
||||
не перехватываются.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_consumers",
|
||||
"_publish_lock",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
consumers: Iterable[
|
||||
AcquisitionRuntimeEventConsumerProtocol
|
||||
] = (),
|
||||
) -> None:
|
||||
registered_consumers = tuple(consumers)
|
||||
|
||||
for consumer in registered_consumers:
|
||||
if not isinstance(
|
||||
consumer,
|
||||
AcquisitionRuntimeEventConsumerProtocol,
|
||||
):
|
||||
raise TypeError(
|
||||
"consumer must satisfy "
|
||||
"AcquisitionRuntimeEventConsumerProtocol"
|
||||
)
|
||||
|
||||
self._consumers = registered_consumers
|
||||
self._publish_lock = asyncio.Lock()
|
||||
|
||||
async def publish(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
"""
|
||||
Последовательно доставить событие всем Consumer.
|
||||
|
||||
Рекурсивная публикация через тот же Publisher запрещена.
|
||||
Такое ограничение предотвращает deadlock на publish lock.
|
||||
"""
|
||||
if not isinstance(event, AcquisitionRuntimeEvent):
|
||||
raise TypeError(
|
||||
"event must be an AcquisitionRuntimeEvent"
|
||||
)
|
||||
|
||||
publisher_id = id(self)
|
||||
active_publisher_ids = _active_publisher_ids.get()
|
||||
|
||||
if publisher_id in active_publisher_ids:
|
||||
raise RuntimeError(
|
||||
"Recursive publication through the same "
|
||||
"AcquisitionRuntimeEventPublisher is not supported."
|
||||
)
|
||||
|
||||
active_token = _active_publisher_ids.set(
|
||||
(*active_publisher_ids, publisher_id)
|
||||
)
|
||||
|
||||
try:
|
||||
async with self._publish_lock:
|
||||
for consumer in self._consumers:
|
||||
try:
|
||||
await consumer.consume(event)
|
||||
except Exception as error:
|
||||
self._report_consumer_error(
|
||||
event=event,
|
||||
consumer=consumer,
|
||||
error=error,
|
||||
)
|
||||
finally:
|
||||
_active_publisher_ids.reset(active_token)
|
||||
|
||||
@staticmethod
|
||||
def _report_consumer_error(
|
||||
*,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
consumer: AcquisitionRuntimeEventConsumerProtocol,
|
||||
error: Exception,
|
||||
) -> None:
|
||||
"""
|
||||
Безопасно зафиксировать ошибку Consumer.
|
||||
|
||||
Текст исключения и traceback намеренно не журналируются:
|
||||
они могут содержать repr(event) с transport payload.
|
||||
|
||||
Ошибка logging handler не должна прерывать Runtime lifecycle
|
||||
или доставку события остальным Consumer.
|
||||
"""
|
||||
try:
|
||||
logger.error(
|
||||
"Acquisition Runtime Event consumer failed: "
|
||||
"event=%s consumer=%s error_type=%s",
|
||||
type(event).__name__,
|
||||
type(consumer).__name__,
|
||||
type(error).__name__,
|
||||
)
|
||||
except Exception:
|
||||
return
|
||||
118
app/src/market_data/acquisition/runtime/live_processing_gate.py
Normal file
118
app/src/market_data/acquisition/runtime/live_processing_gate.py
Normal file
@@ -0,0 +1,118 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import TracebackType
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
import asyncio
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class RuntimeLiveProcessingGateProtocol(Protocol):
|
||||
"""
|
||||
Контракт общей границы Live processing и Runtime Recovery.
|
||||
"""
|
||||
|
||||
@property
|
||||
def locked(self) -> bool:
|
||||
"""Показывает, закрыт ли gate текущей операцией."""
|
||||
...
|
||||
|
||||
@property
|
||||
def failed(self) -> bool:
|
||||
"""Показывает, запрещено ли продолжать Live processing."""
|
||||
...
|
||||
|
||||
def fail(
|
||||
self,
|
||||
error: Exception,
|
||||
) -> None:
|
||||
"""Запретить Live processing после terminal Runtime error."""
|
||||
...
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Разрешить Live processing для нового Runtime lifecycle."""
|
||||
...
|
||||
|
||||
async def __aenter__(
|
||||
self,
|
||||
) -> RuntimeLiveProcessingGateProtocol:
|
||||
"""Закрыть gate и дождаться эксклюзивного доступа."""
|
||||
...
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_value: BaseException | None,
|
||||
traceback: TracebackType | None,
|
||||
) -> None:
|
||||
"""Открыть gate после завершения защищённой операции."""
|
||||
...
|
||||
|
||||
|
||||
class RuntimeLiveProcessingGate:
|
||||
"""
|
||||
Единый asyncio gate для Live processing и Recovery.
|
||||
|
||||
Gate не содержит market state и не создаёт фоновые задачи.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_lock",
|
||||
"_failure",
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = asyncio.Lock()
|
||||
self._failure: Exception | None = None
|
||||
|
||||
@property
|
||||
def locked(self) -> bool:
|
||||
"""Показывает, закрыт ли gate текущей операцией."""
|
||||
return self._lock.locked()
|
||||
|
||||
@property
|
||||
def failed(self) -> bool:
|
||||
"""Показывает, запрещено ли продолжать Live processing."""
|
||||
return self._failure is not None
|
||||
|
||||
def fail(
|
||||
self,
|
||||
error: Exception,
|
||||
) -> None:
|
||||
"""Запретить Live processing после terminal Runtime error."""
|
||||
if not isinstance(error, Exception):
|
||||
raise TypeError("error must be an Exception")
|
||||
|
||||
self._failure = error
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Разрешить Live processing для нового Runtime lifecycle."""
|
||||
if self._lock.locked():
|
||||
raise RuntimeError(
|
||||
"Live processing gate cannot be reset while locked."
|
||||
)
|
||||
|
||||
self._failure = None
|
||||
|
||||
async def __aenter__(
|
||||
self,
|
||||
) -> RuntimeLiveProcessingGate:
|
||||
"""Закрыть gate и дождаться эксклюзивного доступа."""
|
||||
await self._lock.acquire()
|
||||
|
||||
failure = self._failure
|
||||
|
||||
if failure is not None:
|
||||
self._lock.release()
|
||||
raise failure
|
||||
|
||||
return self
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_value: BaseException | None,
|
||||
traceback: TracebackType | None,
|
||||
) -> None:
|
||||
"""Открыть gate после завершения защищённой операции."""
|
||||
self._lock.release()
|
||||
@@ -7,6 +7,7 @@ from typing import Protocol, runtime_checkable
|
||||
|
||||
from src.market_data.acquisition.runtime.runtime_commands import (
|
||||
ConnectCommand,
|
||||
DisconnectCommand,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_events import (
|
||||
ReconnectCompletedEvent,
|
||||
@@ -73,7 +74,7 @@ class ReconnectCoordinator:
|
||||
|
||||
- увеличивает номер попытки;
|
||||
- публикует ReconnectStartedEvent;
|
||||
- передаёт ConnectCommand в Runtime Dispatcher;
|
||||
- передаёт DisconnectCommand и ConnectCommand в Runtime Dispatcher;
|
||||
- восстанавливает зарегистрированные подписки;
|
||||
- публикует ReconnectCompletedEvent;
|
||||
- при ошибке публикует ReconnectFailedEvent;
|
||||
@@ -132,6 +133,8 @@ class ReconnectCoordinator:
|
||||
|
||||
ReconnectStartedEvent
|
||||
↓
|
||||
DisconnectCommand
|
||||
↓
|
||||
ConnectCommand
|
||||
↓
|
||||
restore_subscriptions()
|
||||
@@ -155,6 +158,10 @@ class ReconnectCoordinator:
|
||||
)
|
||||
|
||||
try:
|
||||
await self._command_dispatcher.dispatch(
|
||||
DisconnectCommand()
|
||||
)
|
||||
|
||||
await self._command_dispatcher.dispatch(
|
||||
ConnectCommand()
|
||||
)
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class RuntimeLivenessProbeProtocol(Protocol):
|
||||
"""
|
||||
Контракт активной проверки transport liveness.
|
||||
|
||||
Успешная проверка подтверждается transport-level ответом, а не
|
||||
наличием market events.
|
||||
"""
|
||||
|
||||
async def probe(self) -> bool:
|
||||
"""
|
||||
Проверить активность текущего transport connection.
|
||||
|
||||
Вернуть True только после подтверждённого transport response.
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,401 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from src.market_data.acquisition.recovery.trade_recovery_result import (
|
||||
TradeRecoveryResult,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.live_processing_gate import (
|
||||
RuntimeLiveProcessingGateProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.reconnect import (
|
||||
ReconnectCoordinatorProtocol,
|
||||
ReconnectState,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_recovery_protocol import (
|
||||
RuntimeRecoveryProtocol,
|
||||
)
|
||||
|
||||
|
||||
RuntimeUnixTimeMillisecondsClock = Callable[[], int]
|
||||
|
||||
|
||||
def system_unix_time_ms() -> int:
|
||||
"""Вернуть текущее Unix time без промежуточного float."""
|
||||
return time.time_ns() // 1_000_000
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class RuntimeReconnectRecoveryProtocol(
|
||||
ReconnectCoordinatorProtocol,
|
||||
Protocol,
|
||||
):
|
||||
"""
|
||||
Контракт единой reconnect → restore → recovery операции.
|
||||
"""
|
||||
|
||||
@property
|
||||
def generation(self) -> int:
|
||||
"""Вернуть поколение последней начатой операции."""
|
||||
...
|
||||
|
||||
@property
|
||||
def live_processing_gate(
|
||||
self,
|
||||
) -> RuntimeLiveProcessingGateProtocol:
|
||||
"""Вернуть общий gate Live processing и Recovery."""
|
||||
...
|
||||
|
||||
@property
|
||||
def symbols(self) -> tuple[str, ...]:
|
||||
"""Вернуть нормализованные Recovery symbols."""
|
||||
...
|
||||
|
||||
async def reconnect_after_transport_failure(
|
||||
self,
|
||||
*,
|
||||
observed_generation: int,
|
||||
) -> None:
|
||||
"""
|
||||
Восстановить connection, если ошибка относится к текущему
|
||||
поколению.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class _ReconnectRecoveryOperation:
|
||||
__slots__ = (
|
||||
"source_generation",
|
||||
"completion",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
source_generation: int,
|
||||
completion: asyncio.Future[None],
|
||||
) -> None:
|
||||
self.source_generation = source_generation
|
||||
self.completion = completion
|
||||
|
||||
|
||||
class RuntimeReconnectRecoveryCoordinator:
|
||||
"""
|
||||
Single-flight координатор reconnect → restore → recovery.
|
||||
|
||||
Существующий ReconnectCoordinator сохраняет ответственность только
|
||||
за Disconnect → Connect → restore subscriptions. Этот компонент
|
||||
закрывает общий Live processing gate и после reconnect выполняет
|
||||
синхронный Runtime Recovery вне event loop.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_reconnect_coordinator",
|
||||
"_recovery_coordinator",
|
||||
"_live_processing_gate",
|
||||
"_symbols",
|
||||
"_clock",
|
||||
"_coordination_lock",
|
||||
"_generation",
|
||||
"_active_operation",
|
||||
"_last_operation",
|
||||
"_recovery_task",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
reconnect_coordinator: ReconnectCoordinatorProtocol,
|
||||
recovery_coordinator: RuntimeRecoveryProtocol,
|
||||
live_processing_gate: RuntimeLiveProcessingGateProtocol,
|
||||
symbols: tuple[str, ...],
|
||||
clock: RuntimeUnixTimeMillisecondsClock = system_unix_time_ms,
|
||||
) -> None:
|
||||
if not callable(clock):
|
||||
raise TypeError("clock must be callable")
|
||||
|
||||
self._reconnect_coordinator = reconnect_coordinator
|
||||
self._recovery_coordinator = recovery_coordinator
|
||||
self._live_processing_gate = live_processing_gate
|
||||
self._symbols = self._normalize_symbols(symbols)
|
||||
self._clock = clock
|
||||
|
||||
self._coordination_lock = asyncio.Lock()
|
||||
self._generation = 0
|
||||
self._active_operation: _ReconnectRecoveryOperation | None = None
|
||||
self._last_operation: _ReconnectRecoveryOperation | None = None
|
||||
self._recovery_task: (
|
||||
asyncio.Task[tuple[TradeRecoveryResult, ...]] | None
|
||||
) = None
|
||||
|
||||
@property
|
||||
def state(self) -> ReconnectState:
|
||||
"""Вернуть состояние базовой reconnect-операции."""
|
||||
return self._reconnect_coordinator.state
|
||||
|
||||
@property
|
||||
def attempt(self) -> int:
|
||||
"""Вернуть номер последней базовой reconnect-попытки."""
|
||||
return self._reconnect_coordinator.attempt
|
||||
|
||||
@property
|
||||
def generation(self) -> int:
|
||||
"""Вернуть поколение последней начатой операции."""
|
||||
return self._generation
|
||||
|
||||
@property
|
||||
def live_processing_gate(
|
||||
self,
|
||||
) -> RuntimeLiveProcessingGateProtocol:
|
||||
"""Вернуть общий gate Live processing и Recovery."""
|
||||
return self._live_processing_gate
|
||||
|
||||
@property
|
||||
def symbols(self) -> tuple[str, ...]:
|
||||
"""Вернуть нормализованные Recovery symbols."""
|
||||
return self._symbols
|
||||
|
||||
async def reconnect(self) -> None:
|
||||
"""
|
||||
Выполнить либо присоединиться к текущей single-flight операции.
|
||||
"""
|
||||
await self._run_or_join(
|
||||
observed_generation=None,
|
||||
)
|
||||
|
||||
async def reconnect_after_transport_failure(
|
||||
self,
|
||||
*,
|
||||
observed_generation: int,
|
||||
) -> None:
|
||||
"""
|
||||
Обработать transport error конкретного connection generation.
|
||||
|
||||
Ошибка старого connection присоединяется к уже выполняемой
|
||||
операции либо использует её завершённый результат и не запускает
|
||||
второй reconnect.
|
||||
"""
|
||||
self._validate_generation(observed_generation)
|
||||
|
||||
await self._run_or_join(
|
||||
observed_generation=observed_generation,
|
||||
)
|
||||
|
||||
async def _run_or_join(
|
||||
self,
|
||||
*,
|
||||
observed_generation: int | None,
|
||||
) -> None:
|
||||
operation, is_leader = await self._select_operation(
|
||||
observed_generation=observed_generation,
|
||||
)
|
||||
|
||||
if operation is None:
|
||||
return
|
||||
|
||||
if not is_leader:
|
||||
await asyncio.shield(
|
||||
operation.completion,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
await self._execute_operation()
|
||||
except asyncio.CancelledError:
|
||||
operation.completion.cancel()
|
||||
raise
|
||||
except BaseException as error:
|
||||
operation.completion.set_exception(error)
|
||||
operation.completion.exception()
|
||||
raise
|
||||
else:
|
||||
operation.completion.set_result(None)
|
||||
finally:
|
||||
if self._active_operation is operation:
|
||||
self._last_operation = operation
|
||||
self._active_operation = None
|
||||
|
||||
async def _select_operation(
|
||||
self,
|
||||
*,
|
||||
observed_generation: int | None,
|
||||
) -> tuple[_ReconnectRecoveryOperation | None, bool]:
|
||||
async with self._coordination_lock:
|
||||
active_operation = self._active_operation
|
||||
|
||||
if active_operation is not None:
|
||||
return (
|
||||
active_operation,
|
||||
False,
|
||||
)
|
||||
|
||||
if (
|
||||
observed_generation is not None
|
||||
and observed_generation != self._generation
|
||||
):
|
||||
last_operation = self._last_operation
|
||||
|
||||
if (
|
||||
last_operation is not None
|
||||
and last_operation.source_generation
|
||||
== observed_generation
|
||||
):
|
||||
return (
|
||||
last_operation,
|
||||
False,
|
||||
)
|
||||
|
||||
return (
|
||||
None,
|
||||
False,
|
||||
)
|
||||
|
||||
source_generation = self._generation
|
||||
self._generation += 1
|
||||
|
||||
operation = _ReconnectRecoveryOperation(
|
||||
source_generation=source_generation,
|
||||
completion=asyncio.get_running_loop().create_future(),
|
||||
)
|
||||
self._active_operation = operation
|
||||
|
||||
return (
|
||||
operation,
|
||||
True,
|
||||
)
|
||||
|
||||
async def _execute_operation(self) -> None:
|
||||
async with self._live_processing_gate:
|
||||
try:
|
||||
await self._reconnect_coordinator.reconnect()
|
||||
|
||||
recovery_end_time = self._clock()
|
||||
self._validate_recovery_end_time(
|
||||
recovery_end_time,
|
||||
)
|
||||
|
||||
recovery_task = asyncio.create_task(
|
||||
asyncio.to_thread(
|
||||
self._recover_symbols,
|
||||
recovery_end_time,
|
||||
),
|
||||
name="trade-stream-runtime-recovery",
|
||||
)
|
||||
self._recovery_task = recovery_task
|
||||
|
||||
try:
|
||||
await asyncio.shield(
|
||||
recovery_task,
|
||||
)
|
||||
except asyncio.CancelledError as cancellation:
|
||||
try:
|
||||
await self._wait_for_recovery_completion(
|
||||
recovery_task,
|
||||
)
|
||||
except BaseException as recovery_error:
|
||||
cancellation.add_note(
|
||||
"Trade Stream Runtime Recovery also failed: "
|
||||
f"{type(recovery_error).__name__}."
|
||||
)
|
||||
|
||||
raise
|
||||
finally:
|
||||
if self._recovery_task is recovery_task:
|
||||
self._recovery_task = None
|
||||
except Exception as error:
|
||||
self._live_processing_gate.fail(error)
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
async def _wait_for_recovery_completion(
|
||||
recovery_task: asyncio.Task[
|
||||
tuple[TradeRecoveryResult, ...]
|
||||
],
|
||||
) -> None:
|
||||
while not recovery_task.done():
|
||||
try:
|
||||
await asyncio.shield(
|
||||
recovery_task,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
continue
|
||||
except BaseException:
|
||||
break
|
||||
|
||||
recovery_task.result()
|
||||
|
||||
def _recover_symbols(
|
||||
self,
|
||||
recovery_end_time: int,
|
||||
) -> tuple[TradeRecoveryResult, ...]:
|
||||
return tuple(
|
||||
self._recovery_coordinator.recover(
|
||||
symbol=symbol,
|
||||
recovery_end_time=recovery_end_time,
|
||||
)
|
||||
for symbol in self._symbols
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_symbols(
|
||||
symbols: tuple[str, ...],
|
||||
) -> tuple[str, ...]:
|
||||
if not isinstance(symbols, tuple):
|
||||
raise TypeError("symbols must be a tuple")
|
||||
|
||||
normalized_symbols: set[str] = set()
|
||||
|
||||
for symbol in symbols:
|
||||
if not isinstance(symbol, str):
|
||||
raise TypeError(
|
||||
"symbols must contain only strings"
|
||||
)
|
||||
|
||||
normalized_symbol = symbol.strip()
|
||||
|
||||
if normalized_symbol:
|
||||
normalized_symbols.add(normalized_symbol)
|
||||
|
||||
if not normalized_symbols:
|
||||
raise ValueError(
|
||||
"symbols must contain at least one non-empty symbol"
|
||||
)
|
||||
|
||||
return tuple(sorted(normalized_symbols))
|
||||
|
||||
@staticmethod
|
||||
def _validate_generation(
|
||||
generation: int,
|
||||
) -> None:
|
||||
if isinstance(generation, bool) or not isinstance(
|
||||
generation,
|
||||
int,
|
||||
):
|
||||
raise TypeError(
|
||||
"observed_generation must be an integer"
|
||||
)
|
||||
|
||||
if generation < 0:
|
||||
raise ValueError(
|
||||
"observed_generation must not be negative"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_recovery_end_time(
|
||||
recovery_end_time: int,
|
||||
) -> None:
|
||||
if isinstance(recovery_end_time, bool) or not isinstance(
|
||||
recovery_end_time,
|
||||
int,
|
||||
):
|
||||
raise TypeError(
|
||||
"clock must return an integer"
|
||||
)
|
||||
|
||||
if recovery_end_time < 0:
|
||||
raise ValueError(
|
||||
"clock must not return a negative value"
|
||||
)
|
||||
@@ -9,6 +9,9 @@ from typing import Protocol, runtime_checkable
|
||||
from src.market_data.acquisition.runtime.heartbeat import (
|
||||
HeartbeatMonitorProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_liveness_probe import (
|
||||
RuntimeLivenessProbeProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.supervisor import (
|
||||
RuntimeSupervisorProtocol,
|
||||
)
|
||||
@@ -37,7 +40,29 @@ class RuntimeSchedulerProtocol(Protocol):
|
||||
"""
|
||||
...
|
||||
|
||||
async def start(self) -> None:
|
||||
@property
|
||||
def liveness_probe(self) -> RuntimeLivenessProbeProtocol:
|
||||
"""Вернуть используемый transport liveness probe."""
|
||||
...
|
||||
|
||||
@property
|
||||
def runtime_supervisor(self) -> RuntimeSupervisorProtocol:
|
||||
"""Вернуть Supervisor, получающий timeout."""
|
||||
...
|
||||
|
||||
def claim(self, owner: object) -> None:
|
||||
"""Закрепить Scheduler за единственным lifecycle owner."""
|
||||
...
|
||||
|
||||
def release(self, owner: object) -> None:
|
||||
"""Освободить Scheduler после завершения owned task."""
|
||||
...
|
||||
|
||||
async def start(
|
||||
self,
|
||||
*,
|
||||
owner: object | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Запустить scheduler loop.
|
||||
|
||||
@@ -66,6 +91,12 @@ class RuntimeScheduler:
|
||||
|
||||
Scheduler отвечает только за время выполнения проверок:
|
||||
|
||||
RuntimeLivenessProbe.probe()
|
||||
│
|
||||
├── True
|
||||
│ └── RuntimeSupervisor.notify_activity()
|
||||
│
|
||||
▼
|
||||
HeartbeatMonitor.check_timeout()
|
||||
│
|
||||
├── False
|
||||
@@ -85,20 +116,23 @@ class RuntimeScheduler:
|
||||
- retry policy;
|
||||
- публикацию Runtime Events.
|
||||
|
||||
Ошибки Heartbeat Monitor, Runtime Supervisor и sleep-функции
|
||||
распространяются вызывающему коду без обёртки.
|
||||
Ошибки liveness probe, Heartbeat Monitor, Runtime Supervisor и
|
||||
sleep-функции распространяются вызывающему коду без обёртки.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_liveness_probe",
|
||||
"_heartbeat_monitor",
|
||||
"_runtime_supervisor",
|
||||
"_interval_seconds",
|
||||
"_sleep",
|
||||
"_running",
|
||||
"_owner",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
liveness_probe: RuntimeLivenessProbeProtocol,
|
||||
heartbeat_monitor: HeartbeatMonitorProtocol,
|
||||
runtime_supervisor: RuntimeSupervisorProtocol,
|
||||
*,
|
||||
@@ -121,11 +155,13 @@ class RuntimeScheduler:
|
||||
if not callable(sleep):
|
||||
raise TypeError("sleep must be callable")
|
||||
|
||||
self._liveness_probe = liveness_probe
|
||||
self._heartbeat_monitor = heartbeat_monitor
|
||||
self._runtime_supervisor = runtime_supervisor
|
||||
self._interval_seconds = float(interval_seconds)
|
||||
self._sleep = sleep
|
||||
self._running = False
|
||||
self._owner: object | None = None
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
@@ -141,7 +177,57 @@ class RuntimeScheduler:
|
||||
"""
|
||||
return self._interval_seconds
|
||||
|
||||
async def start(self) -> None:
|
||||
@property
|
||||
def liveness_probe(self) -> RuntimeLivenessProbeProtocol:
|
||||
"""Вернуть используемый transport liveness probe."""
|
||||
return self._liveness_probe
|
||||
|
||||
@property
|
||||
def runtime_supervisor(self) -> RuntimeSupervisorProtocol:
|
||||
"""Вернуть Supervisor, получающий timeout."""
|
||||
return self._runtime_supervisor
|
||||
|
||||
def claim(self, owner: object) -> None:
|
||||
"""
|
||||
Атомарно закрепить Scheduler за lifecycle owner.
|
||||
|
||||
После claim только этот owner может запустить loop. Claim не
|
||||
создаёт asyncio-задачу и безопасен до длительного startup.
|
||||
"""
|
||||
if owner is None:
|
||||
raise TypeError("owner must not be None")
|
||||
|
||||
if self._owner is owner:
|
||||
return
|
||||
|
||||
if self._owner is not None or self._running:
|
||||
raise RuntimeError(
|
||||
"Runtime Scheduler is already owned or active."
|
||||
)
|
||||
|
||||
self._owner = owner
|
||||
|
||||
def release(self, owner: object) -> None:
|
||||
"""
|
||||
Освободить Scheduler после остановки owned loop.
|
||||
"""
|
||||
if self._owner is not owner:
|
||||
raise RuntimeError(
|
||||
"Runtime Scheduler ownership does not match."
|
||||
)
|
||||
|
||||
if self._running:
|
||||
raise RuntimeError(
|
||||
"Cannot release a running Runtime Scheduler."
|
||||
)
|
||||
|
||||
self._owner = None
|
||||
|
||||
async def start(
|
||||
self,
|
||||
*,
|
||||
owner: object | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Запустить периодический scheduler loop.
|
||||
|
||||
@@ -154,6 +240,16 @@ class RuntimeScheduler:
|
||||
- после ошибки;
|
||||
- после отмены внешней asyncio-задачи.
|
||||
"""
|
||||
if self._owner is not None and self._owner is not owner:
|
||||
raise RuntimeError(
|
||||
"Runtime Scheduler is owned by another lifecycle."
|
||||
)
|
||||
|
||||
if self._owner is None and owner is not None:
|
||||
raise RuntimeError(
|
||||
"Runtime Scheduler owner must claim it before start."
|
||||
)
|
||||
|
||||
if self._running:
|
||||
return
|
||||
|
||||
@@ -183,14 +279,25 @@ class RuntimeScheduler:
|
||||
|
||||
async def run_once(self) -> bool:
|
||||
"""
|
||||
Выполнить одну проверку Heartbeat.
|
||||
Выполнить одну liveness- и Heartbeat-проверку.
|
||||
|
||||
Если Heartbeat Monitor подтверждает timeout, Scheduler
|
||||
передаёт его Runtime Supervisor.
|
||||
Успешный transport probe обновляет время активности через
|
||||
Supervisor. Если Heartbeat Monitor подтверждает timeout,
|
||||
Scheduler передаёт его Runtime Supervisor.
|
||||
|
||||
Возвращаемое значение отражает только результат проверки
|
||||
Heartbeat Monitor.
|
||||
"""
|
||||
connection_is_live = await self._liveness_probe.probe()
|
||||
|
||||
if not isinstance(connection_is_live, bool):
|
||||
raise TypeError(
|
||||
"liveness probe must return a boolean"
|
||||
)
|
||||
|
||||
if connection_is_live:
|
||||
self._runtime_supervisor.notify_activity()
|
||||
|
||||
timed_out = await self._heartbeat_monitor.check_timeout()
|
||||
|
||||
if timed_out:
|
||||
|
||||
@@ -24,6 +24,28 @@ class RuntimeSupervisorState(str, Enum):
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class RuntimeSupervisorReconnectProtocol(
|
||||
ReconnectCoordinatorProtocol,
|
||||
Protocol,
|
||||
):
|
||||
"""
|
||||
Generation-aware reconnect-контракт Runtime Supervisor.
|
||||
"""
|
||||
|
||||
@property
|
||||
def generation(self) -> int:
|
||||
"""Вернуть текущее connection generation."""
|
||||
...
|
||||
|
||||
async def reconnect_after_transport_failure(
|
||||
self,
|
||||
*,
|
||||
observed_generation: int,
|
||||
) -> None:
|
||||
"""Восстановить только наблюдавшееся поколение connection."""
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class RuntimeSupervisorProtocol(Protocol):
|
||||
"""
|
||||
@@ -100,16 +122,18 @@ class RuntimeSupervisor:
|
||||
__slots__ = (
|
||||
"_heartbeat_monitor",
|
||||
"_reconnect_coordinator",
|
||||
"_observed_generation",
|
||||
"_state",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
heartbeat_monitor: HeartbeatMonitorProtocol,
|
||||
reconnect_coordinator: ReconnectCoordinatorProtocol,
|
||||
reconnect_coordinator: RuntimeSupervisorReconnectProtocol,
|
||||
) -> None:
|
||||
self._heartbeat_monitor = heartbeat_monitor
|
||||
self._reconnect_coordinator = reconnect_coordinator
|
||||
self._observed_generation = reconnect_coordinator.generation
|
||||
self._state = RuntimeSupervisorState.STOPPED
|
||||
|
||||
@property
|
||||
@@ -127,6 +151,9 @@ class RuntimeSupervisor:
|
||||
period и сохраняет Supervisor в состоянии RUNNING.
|
||||
"""
|
||||
self._heartbeat_monitor.start()
|
||||
self._observed_generation = (
|
||||
self._reconnect_coordinator.generation
|
||||
)
|
||||
self._state = RuntimeSupervisorState.RUNNING
|
||||
|
||||
def stop(self) -> None:
|
||||
@@ -150,6 +177,9 @@ class RuntimeSupervisor:
|
||||
return
|
||||
|
||||
self._heartbeat_monitor.record_activity()
|
||||
self._observed_generation = (
|
||||
self._reconnect_coordinator.generation
|
||||
)
|
||||
|
||||
async def handle_heartbeat_timeout(self) -> bool:
|
||||
"""
|
||||
@@ -163,7 +193,7 @@ class RuntimeSupervisor:
|
||||
↓
|
||||
RECONNECTING
|
||||
↓
|
||||
ReconnectCoordinator.reconnect()
|
||||
reconnect текущего connection generation
|
||||
├── success:
|
||||
│ start Heartbeat
|
||||
│ RUNNING
|
||||
@@ -190,14 +220,23 @@ class RuntimeSupervisor:
|
||||
|
||||
self._heartbeat_monitor.stop()
|
||||
self._state = RuntimeSupervisorState.RECONNECTING
|
||||
observed_generation = self._observed_generation
|
||||
|
||||
try:
|
||||
await self._reconnect_coordinator.reconnect()
|
||||
await (
|
||||
self._reconnect_coordinator
|
||||
.reconnect_after_transport_failure(
|
||||
observed_generation=observed_generation,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
self._state = RuntimeSupervisorState.FAILED
|
||||
raise
|
||||
|
||||
self._heartbeat_monitor.start()
|
||||
self._observed_generation = (
|
||||
self._reconnect_coordinator.generation
|
||||
)
|
||||
self._state = RuntimeSupervisorState.RUNNING
|
||||
|
||||
return True
|
||||
|
||||
@@ -0,0 +1,763 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from enum import Enum
|
||||
from typing import Protocol, runtime_checkable
|
||||
from uuid import uuid4
|
||||
|
||||
from src.market_data.acquisition.exceptions import (
|
||||
WebSocketMessageDecodeError,
|
||||
WebSocketTransportError,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.live_processing_gate import (
|
||||
RuntimeLiveProcessingGateProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_reconnect_recovery_coordinator import (
|
||||
RuntimeReconnectRecoveryProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.scheduler import (
|
||||
RuntimeSchedulerProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.supervisor import (
|
||||
RuntimeSupervisorProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_events import (
|
||||
ConnectedEvent,
|
||||
ConnectFailedEvent,
|
||||
DisconnectedEvent,
|
||||
MessageReceivedEvent,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.transport_messages import (
|
||||
TransportBinaryMessage,
|
||||
TransportTextMessage,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_inbound_message import (
|
||||
WebSocketControlMessageHandlerProtocol,
|
||||
WebSocketInboundMessageClassifierProtocol,
|
||||
WebSocketInboundMessageKind,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_protocol import (
|
||||
AcquisitionRuntimeEventPublisherProtocol,
|
||||
WebSocketSessionProtocol,
|
||||
WebSocketSubscriptionManagerProtocol,
|
||||
WebSocketTransportProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.trade_stream_acquisition_protocol import (
|
||||
TradeStreamAcquisitionServiceProtocol,
|
||||
)
|
||||
|
||||
|
||||
class TradeStreamProductionRuntimeState(str, Enum):
|
||||
"""
|
||||
Состояние высокоуровневого lifecycle Trade Stream Runtime.
|
||||
"""
|
||||
|
||||
STOPPED = "stopped"
|
||||
STARTING = "starting"
|
||||
RUNNING = "running"
|
||||
STOPPING = "stopping"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class TradeStreamProductionRuntimeProtocol(Protocol):
|
||||
"""
|
||||
Контракт единственного владельца Trade Stream runtime execution.
|
||||
"""
|
||||
|
||||
@property
|
||||
def state(self) -> TradeStreamProductionRuntimeState:
|
||||
"""Вернуть текущее lifecycle-состояние."""
|
||||
...
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
"""Показывает, выполняется ли receive lifecycle."""
|
||||
...
|
||||
|
||||
async def run(self) -> None:
|
||||
"""Запустить Runtime и выполнять его до stop или ошибки."""
|
||||
...
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Идемпотентно остановить Runtime и дождаться cleanup."""
|
||||
...
|
||||
|
||||
|
||||
class TradeStreamProductionRuntime:
|
||||
"""
|
||||
Высокоуровневый владелец lifecycle Trade Stream.
|
||||
|
||||
Runtime:
|
||||
|
||||
- запускает WebSocket Session;
|
||||
- регистрирует Trade subscription;
|
||||
- создаёт ровно одну receive task;
|
||||
- декодирует JSON и отделяет control messages от market documents;
|
||||
- передаёт market documents в TradeStreamAcquisitionService;
|
||||
- после transport failure выполняет reconnect и Runtime Recovery;
|
||||
- запускает Runtime Supervisor и одну Scheduler task;
|
||||
- отменяет и await-ит созданные им startup, Scheduler и receive tasks;
|
||||
- выполняет полный cleanup при stop, cancellation и ошибке.
|
||||
|
||||
Scheduler останавливается до Supervisor и receive loop. Благодаря
|
||||
этому shutdown не может запустить новую heartbeat recovery-операцию.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_session",
|
||||
"_transport",
|
||||
"_subscription_manager",
|
||||
"_event_publisher",
|
||||
"_trade_stream_service",
|
||||
"_message_classifier",
|
||||
"_control_message_handler",
|
||||
"_live_processing_gate",
|
||||
"_reconnect_recovery_coordinator",
|
||||
"_runtime_supervisor",
|
||||
"_runtime_scheduler",
|
||||
"_symbols",
|
||||
"_state",
|
||||
"_startup_task",
|
||||
"_receive_task",
|
||||
"_scheduler_task",
|
||||
"_scheduler_claimed",
|
||||
"_supervisor_started",
|
||||
"_session_started",
|
||||
"_subscription_correlation_id",
|
||||
"_lifecycle_lock",
|
||||
"_stop_requested",
|
||||
"_stopped",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
session: WebSocketSessionProtocol,
|
||||
transport: WebSocketTransportProtocol,
|
||||
subscription_manager: WebSocketSubscriptionManagerProtocol,
|
||||
event_publisher: AcquisitionRuntimeEventPublisherProtocol,
|
||||
trade_stream_service: TradeStreamAcquisitionServiceProtocol,
|
||||
message_classifier: WebSocketInboundMessageClassifierProtocol,
|
||||
control_message_handler: WebSocketControlMessageHandlerProtocol,
|
||||
live_processing_gate: RuntimeLiveProcessingGateProtocol,
|
||||
reconnect_recovery_coordinator: RuntimeReconnectRecoveryProtocol,
|
||||
runtime_supervisor: RuntimeSupervisorProtocol,
|
||||
runtime_scheduler: RuntimeSchedulerProtocol,
|
||||
symbols: tuple[str, ...],
|
||||
) -> None:
|
||||
self._session = session
|
||||
self._transport = transport
|
||||
self._subscription_manager = subscription_manager
|
||||
self._event_publisher = event_publisher
|
||||
self._trade_stream_service = trade_stream_service
|
||||
self._message_classifier = message_classifier
|
||||
self._control_message_handler = control_message_handler
|
||||
|
||||
normalized_symbols = self._normalize_symbols(symbols)
|
||||
|
||||
if (
|
||||
reconnect_recovery_coordinator.live_processing_gate
|
||||
is not live_processing_gate
|
||||
):
|
||||
raise ValueError(
|
||||
"Production Runtime and reconnect/recovery coordinator "
|
||||
"must share one live processing gate."
|
||||
)
|
||||
|
||||
if reconnect_recovery_coordinator.symbols != normalized_symbols:
|
||||
raise ValueError(
|
||||
"Production Runtime and reconnect/recovery coordinator "
|
||||
"must use the same symbols."
|
||||
)
|
||||
|
||||
if runtime_scheduler.runtime_supervisor is not runtime_supervisor:
|
||||
raise ValueError(
|
||||
"Production Runtime and Scheduler must share one "
|
||||
"Runtime Supervisor."
|
||||
)
|
||||
|
||||
if runtime_scheduler.liveness_probe is not transport:
|
||||
raise ValueError(
|
||||
"Production Runtime and Scheduler must share one "
|
||||
"WebSocket Transport."
|
||||
)
|
||||
|
||||
self._live_processing_gate = live_processing_gate
|
||||
self._reconnect_recovery_coordinator = (
|
||||
reconnect_recovery_coordinator
|
||||
)
|
||||
self._runtime_supervisor = runtime_supervisor
|
||||
self._runtime_scheduler = runtime_scheduler
|
||||
self._symbols = normalized_symbols
|
||||
|
||||
self._state = TradeStreamProductionRuntimeState.STOPPED
|
||||
self._startup_task: asyncio.Task[None] | None = None
|
||||
self._receive_task: asyncio.Task[None] | None = None
|
||||
self._scheduler_task: asyncio.Task[None] | None = None
|
||||
self._scheduler_claimed = False
|
||||
self._supervisor_started = False
|
||||
self._session_started = False
|
||||
self._subscription_correlation_id: str | None = None
|
||||
|
||||
self._lifecycle_lock = asyncio.Lock()
|
||||
self._stop_requested = asyncio.Event()
|
||||
self._stopped = asyncio.Event()
|
||||
self._stopped.set()
|
||||
|
||||
@property
|
||||
def state(self) -> TradeStreamProductionRuntimeState:
|
||||
"""Вернуть текущее lifecycle-состояние."""
|
||||
return self._state
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
"""Показывает, выполняется ли receive lifecycle."""
|
||||
return self._state is TradeStreamProductionRuntimeState.RUNNING
|
||||
|
||||
async def run(self) -> None:
|
||||
"""
|
||||
Запустить Runtime и выполнять его до stop или terminal error.
|
||||
|
||||
Метод не создаёт скрытую корневую задачу. Внешний bootstrap
|
||||
владеет coroutine task ``run()``, а Runtime владеет созданными
|
||||
внутри startup, Scheduler и receive tasks.
|
||||
"""
|
||||
await self._begin_run()
|
||||
|
||||
primary_error: BaseException | None = None
|
||||
|
||||
try:
|
||||
await self._run_lifecycle()
|
||||
except BaseException as error:
|
||||
primary_error = error
|
||||
|
||||
cleanup_error = await self._shutdown(
|
||||
primary_error=primary_error,
|
||||
)
|
||||
|
||||
failed = (
|
||||
isinstance(primary_error, Exception)
|
||||
or isinstance(cleanup_error, Exception)
|
||||
)
|
||||
self._finish_run(failed=failed)
|
||||
|
||||
if primary_error is not None:
|
||||
if cleanup_error is not None:
|
||||
primary_error.add_note(
|
||||
"Trade Stream Runtime cleanup also failed: "
|
||||
f"{type(cleanup_error).__name__}."
|
||||
)
|
||||
|
||||
raise primary_error.with_traceback(
|
||||
primary_error.__traceback__,
|
||||
)
|
||||
|
||||
if cleanup_error is not None:
|
||||
raise cleanup_error.with_traceback(
|
||||
cleanup_error.__traceback__,
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""
|
||||
Запросить остановку и дождаться полного cleanup.
|
||||
|
||||
stop() не отменяет внешнюю task ``run()``. Он останавливает
|
||||
только startup, Scheduler и receive tasks, принадлежащие
|
||||
Production Runtime.
|
||||
"""
|
||||
async with self._lifecycle_lock:
|
||||
if self._state in {
|
||||
TradeStreamProductionRuntimeState.STOPPED,
|
||||
TradeStreamProductionRuntimeState.FAILED,
|
||||
}:
|
||||
return
|
||||
|
||||
if (
|
||||
self._state
|
||||
is not TradeStreamProductionRuntimeState.STOPPING
|
||||
):
|
||||
self._state = TradeStreamProductionRuntimeState.STOPPING
|
||||
self._stop_requested.set()
|
||||
|
||||
startup_task = self._startup_task
|
||||
scheduler_task = self._scheduler_task
|
||||
|
||||
if (
|
||||
startup_task is not None
|
||||
and not startup_task.done()
|
||||
):
|
||||
startup_task.cancel()
|
||||
|
||||
if (
|
||||
scheduler_task is not None
|
||||
and not scheduler_task.done()
|
||||
):
|
||||
self._runtime_scheduler.stop()
|
||||
scheduler_task.cancel()
|
||||
|
||||
await self._stopped.wait()
|
||||
|
||||
async def _begin_run(self) -> None:
|
||||
async with self._lifecycle_lock:
|
||||
if self._state not in {
|
||||
TradeStreamProductionRuntimeState.STOPPED,
|
||||
TradeStreamProductionRuntimeState.FAILED,
|
||||
}:
|
||||
raise RuntimeError(
|
||||
"Trade Stream Production Runtime is already active."
|
||||
)
|
||||
|
||||
self._runtime_scheduler.claim(self)
|
||||
self._scheduler_claimed = True
|
||||
|
||||
try:
|
||||
self._live_processing_gate.reset()
|
||||
except BaseException:
|
||||
self._runtime_scheduler.release(self)
|
||||
self._scheduler_claimed = False
|
||||
raise
|
||||
|
||||
self._state = TradeStreamProductionRuntimeState.STARTING
|
||||
self._startup_task = None
|
||||
self._receive_task = None
|
||||
self._scheduler_task = None
|
||||
self._supervisor_started = False
|
||||
self._session_started = False
|
||||
self._subscription_correlation_id = None
|
||||
self._stop_requested.clear()
|
||||
self._stopped.clear()
|
||||
|
||||
async def _run_lifecycle(self) -> None:
|
||||
startup_task = asyncio.create_task(
|
||||
self._start_sequence(),
|
||||
name="trade-stream-startup",
|
||||
)
|
||||
|
||||
async with self._lifecycle_lock:
|
||||
self._startup_task = startup_task
|
||||
|
||||
if self._stop_requested.is_set():
|
||||
startup_task.cancel()
|
||||
|
||||
try:
|
||||
await startup_task
|
||||
except asyncio.CancelledError:
|
||||
if not self._stop_requested.is_set():
|
||||
raise
|
||||
|
||||
return
|
||||
|
||||
async with self._lifecycle_lock:
|
||||
if self._stop_requested.is_set():
|
||||
return
|
||||
|
||||
self._supervisor_started = True
|
||||
self._runtime_supervisor.start()
|
||||
|
||||
receive_task = asyncio.create_task(
|
||||
self._receive_loop(),
|
||||
name="trade-stream-receive",
|
||||
)
|
||||
scheduler_task = asyncio.create_task(
|
||||
self._runtime_scheduler.start(
|
||||
owner=self,
|
||||
),
|
||||
name="trade-stream-scheduler",
|
||||
)
|
||||
self._receive_task = receive_task
|
||||
self._scheduler_task = scheduler_task
|
||||
self._state = TradeStreamProductionRuntimeState.RUNNING
|
||||
|
||||
try:
|
||||
await self._wait_for_runtime_task(
|
||||
receive_task=receive_task,
|
||||
scheduler_task=scheduler_task,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
if not self._stop_requested.is_set():
|
||||
raise
|
||||
|
||||
async def _start_sequence(self) -> None:
|
||||
try:
|
||||
await self._session.start()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as error:
|
||||
await self._event_publisher.publish(
|
||||
ConnectFailedEvent(
|
||||
reason=str(error),
|
||||
)
|
||||
)
|
||||
raise
|
||||
|
||||
self._session_started = True
|
||||
|
||||
await self._event_publisher.publish(
|
||||
ConnectedEvent(),
|
||||
)
|
||||
|
||||
if self._stop_requested.is_set():
|
||||
return
|
||||
|
||||
correlation_id = str(uuid4())
|
||||
self._subscription_correlation_id = correlation_id
|
||||
|
||||
await self._trade_stream_service.subscribe(
|
||||
self._symbols,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
|
||||
async def _receive_loop(self) -> None:
|
||||
while not self._stop_requested.is_set():
|
||||
observed_generation = (
|
||||
self._reconnect_recovery_coordinator.generation
|
||||
)
|
||||
|
||||
try:
|
||||
raw_message = await self._transport.receive()
|
||||
except WebSocketTransportError:
|
||||
if self._stop_requested.is_set():
|
||||
return
|
||||
|
||||
await (
|
||||
self._reconnect_recovery_coordinator
|
||||
.reconnect_after_transport_failure(
|
||||
observed_generation=observed_generation,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
if self._stop_requested.is_set():
|
||||
return
|
||||
|
||||
self._runtime_supervisor.notify_activity()
|
||||
|
||||
transport_message = self._wrap_transport_message(
|
||||
raw_message,
|
||||
)
|
||||
|
||||
await self._event_publisher.publish(
|
||||
MessageReceivedEvent(
|
||||
message=transport_message,
|
||||
)
|
||||
)
|
||||
|
||||
document = self._decode_message(
|
||||
raw_message,
|
||||
)
|
||||
|
||||
async with self._live_processing_gate:
|
||||
if self._stop_requested.is_set():
|
||||
return
|
||||
|
||||
message_kind = self._message_classifier.classify(
|
||||
document,
|
||||
)
|
||||
|
||||
if (
|
||||
message_kind
|
||||
is WebSocketInboundMessageKind.CONTROL
|
||||
):
|
||||
correlation_id = (
|
||||
self._subscription_correlation_id
|
||||
)
|
||||
|
||||
if correlation_id is None:
|
||||
raise RuntimeError(
|
||||
"Control message received without an "
|
||||
"active subscription correlation ID."
|
||||
)
|
||||
|
||||
self._control_message_handler.handle(
|
||||
document,
|
||||
expected_correlation_id=correlation_id,
|
||||
)
|
||||
continue
|
||||
|
||||
if (
|
||||
message_kind
|
||||
is not WebSocketInboundMessageKind.MARKET
|
||||
):
|
||||
raise TypeError(
|
||||
"message classifier must return "
|
||||
"WebSocketInboundMessageKind"
|
||||
)
|
||||
|
||||
self._trade_stream_service.handle_message(
|
||||
document,
|
||||
)
|
||||
|
||||
async def _shutdown(
|
||||
self,
|
||||
*,
|
||||
primary_error: BaseException | None,
|
||||
) -> BaseException | None:
|
||||
cleanup_error: BaseException | None = None
|
||||
startup_task = self._startup_task
|
||||
scheduler_task = self._scheduler_task
|
||||
receive_task = self._receive_task
|
||||
|
||||
self._stop_requested.set()
|
||||
|
||||
if (
|
||||
startup_task is not None
|
||||
and not startup_task.done()
|
||||
):
|
||||
startup_task.cancel()
|
||||
try:
|
||||
await startup_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except BaseException as error:
|
||||
cleanup_error = self._merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
|
||||
if scheduler_task is not None:
|
||||
if not scheduler_task.done():
|
||||
try:
|
||||
self._runtime_scheduler.stop()
|
||||
except BaseException as error:
|
||||
cleanup_error = self._merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
|
||||
scheduler_task.cancel()
|
||||
|
||||
try:
|
||||
await scheduler_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except BaseException as error:
|
||||
if error is not primary_error:
|
||||
cleanup_error = self._merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
|
||||
if self._scheduler_claimed:
|
||||
try:
|
||||
self._runtime_scheduler.release(self)
|
||||
except BaseException as error:
|
||||
cleanup_error = self._merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
else:
|
||||
self._scheduler_claimed = False
|
||||
|
||||
if self._supervisor_started:
|
||||
try:
|
||||
self._runtime_supervisor.stop()
|
||||
except BaseException as error:
|
||||
cleanup_error = self._merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
finally:
|
||||
self._supervisor_started = False
|
||||
|
||||
if receive_task is not None:
|
||||
if not receive_task.done():
|
||||
receive_task.cancel()
|
||||
|
||||
try:
|
||||
await receive_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except BaseException as error:
|
||||
if error is not primary_error:
|
||||
cleanup_error = self._merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
|
||||
session_stopped = False
|
||||
|
||||
try:
|
||||
await self._session.stop()
|
||||
session_stopped = True
|
||||
except BaseException as error:
|
||||
cleanup_error = self._merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
|
||||
try:
|
||||
await self._subscription_manager.clear_subscriptions()
|
||||
except BaseException as error:
|
||||
cleanup_error = self._merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
|
||||
if self._session_started and session_stopped:
|
||||
try:
|
||||
await self._event_publisher.publish(
|
||||
DisconnectedEvent(),
|
||||
)
|
||||
except BaseException as error:
|
||||
cleanup_error = self._merge_cleanup_error(
|
||||
cleanup_error,
|
||||
error,
|
||||
)
|
||||
|
||||
self._startup_task = None
|
||||
self._receive_task = None
|
||||
self._scheduler_task = None
|
||||
self._session_started = False
|
||||
self._subscription_correlation_id = None
|
||||
|
||||
return cleanup_error
|
||||
|
||||
async def _wait_for_runtime_task(
|
||||
self,
|
||||
*,
|
||||
receive_task: asyncio.Task[None],
|
||||
scheduler_task: asyncio.Task[None],
|
||||
) -> None:
|
||||
done_tasks, _ = await asyncio.wait(
|
||||
(
|
||||
receive_task,
|
||||
scheduler_task,
|
||||
),
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
|
||||
task_errors: list[BaseException] = []
|
||||
cancelled_task_names: list[str] = []
|
||||
|
||||
for task in (
|
||||
receive_task,
|
||||
scheduler_task,
|
||||
):
|
||||
if task not in done_tasks:
|
||||
continue
|
||||
|
||||
if task.cancelled():
|
||||
cancelled_task_names.append(
|
||||
task.get_name(),
|
||||
)
|
||||
continue
|
||||
|
||||
error = task.exception()
|
||||
|
||||
if error is not None:
|
||||
task_errors.append(error)
|
||||
|
||||
if task_errors:
|
||||
primary_error = task_errors[0]
|
||||
|
||||
for additional_error in task_errors[1:]:
|
||||
primary_error.add_note(
|
||||
"Additional Trade Stream Runtime task failure: "
|
||||
f"{type(additional_error).__name__}."
|
||||
)
|
||||
|
||||
raise primary_error.with_traceback(
|
||||
primary_error.__traceback__,
|
||||
)
|
||||
|
||||
if self._stop_requested.is_set():
|
||||
return
|
||||
|
||||
if cancelled_task_names:
|
||||
raise RuntimeError(
|
||||
"Trade Stream Runtime task was cancelled unexpectedly: "
|
||||
f"{', '.join(cancelled_task_names)}."
|
||||
)
|
||||
|
||||
completed_task_names = ", ".join(
|
||||
task.get_name()
|
||||
for task in done_tasks
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Trade Stream Runtime task terminated unexpectedly: "
|
||||
f"{completed_task_names}."
|
||||
)
|
||||
|
||||
def _finish_run(
|
||||
self,
|
||||
*,
|
||||
failed: bool,
|
||||
) -> None:
|
||||
self._state = (
|
||||
TradeStreamProductionRuntimeState.FAILED
|
||||
if failed
|
||||
else TradeStreamProductionRuntimeState.STOPPED
|
||||
)
|
||||
self._stopped.set()
|
||||
|
||||
@staticmethod
|
||||
def _merge_cleanup_error(
|
||||
current_error: BaseException | None,
|
||||
new_error: BaseException,
|
||||
) -> BaseException:
|
||||
if current_error is None:
|
||||
return new_error
|
||||
|
||||
current_error.add_note(
|
||||
"Additional Trade Stream Runtime cleanup failure: "
|
||||
f"{type(new_error).__name__}."
|
||||
)
|
||||
return current_error
|
||||
|
||||
@staticmethod
|
||||
def _wrap_transport_message(
|
||||
raw_message: str | bytes,
|
||||
) -> TransportTextMessage | TransportBinaryMessage:
|
||||
if isinstance(raw_message, str):
|
||||
return TransportTextMessage(
|
||||
payload=raw_message,
|
||||
)
|
||||
|
||||
if isinstance(raw_message, bytes):
|
||||
return TransportBinaryMessage(
|
||||
payload=raw_message,
|
||||
)
|
||||
|
||||
raise TypeError(
|
||||
"WebSocket transport receive() must return str or bytes"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _decode_message(
|
||||
raw_message: str | bytes,
|
||||
) -> object:
|
||||
try:
|
||||
return json.loads(raw_message)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as error:
|
||||
raise WebSocketMessageDecodeError(
|
||||
"Не удалось декодировать JSON из WebSocket message."
|
||||
) from error
|
||||
|
||||
@staticmethod
|
||||
def _normalize_symbols(
|
||||
symbols: tuple[str, ...],
|
||||
) -> tuple[str, ...]:
|
||||
if not isinstance(symbols, tuple):
|
||||
raise TypeError("symbols must be a tuple")
|
||||
|
||||
normalized_symbols: set[str] = set()
|
||||
|
||||
for symbol in symbols:
|
||||
if not isinstance(symbol, str):
|
||||
raise TypeError(
|
||||
"symbols must contain only strings"
|
||||
)
|
||||
|
||||
normalized_symbol = symbol.strip()
|
||||
|
||||
if normalized_symbol:
|
||||
normalized_symbols.add(normalized_symbol)
|
||||
|
||||
if not normalized_symbols:
|
||||
raise ValueError(
|
||||
"symbols must contain at least one non-empty symbol"
|
||||
)
|
||||
|
||||
return tuple(sorted(normalized_symbols))
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
class WebSocketInboundMessageKind(str, Enum):
|
||||
"""
|
||||
Категория декодированного входящего WebSocket-сообщения.
|
||||
"""
|
||||
|
||||
MARKET = "market"
|
||||
CONTROL = "control"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class WebSocketInboundMessageClassifierProtocol(Protocol):
|
||||
"""
|
||||
Контракт классификации декодированного WebSocket-документа.
|
||||
|
||||
Конкретная реализация содержит provider-specific знания о форме
|
||||
market и control messages. Production Runtime использует только
|
||||
итоговую категорию и не интерпретирует exchange-specific поля.
|
||||
"""
|
||||
|
||||
def classify(
|
||||
self,
|
||||
document: object,
|
||||
) -> WebSocketInboundMessageKind:
|
||||
"""Классифицировать один декодированный документ."""
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class WebSocketControlMessageHandlerProtocol(Protocol):
|
||||
"""
|
||||
Контракт provider-specific проверки входящего control/ACK.
|
||||
|
||||
Production Runtime передаёт ожидаемый correlation ID, но не
|
||||
интерпретирует provider-specific поля ответа самостоятельно.
|
||||
"""
|
||||
|
||||
def handle(
|
||||
self,
|
||||
document: object,
|
||||
*,
|
||||
expected_correlation_id: str,
|
||||
) -> None:
|
||||
"""
|
||||
Проверить принадлежность и успешность control/ACK сообщения.
|
||||
"""
|
||||
...
|
||||
81
app/src/market_data/acquisition/runtime/websocket_session.py
Normal file
81
app/src/market_data/acquisition/runtime/websocket_session.py
Normal file
@@ -0,0 +1,81 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Protocol
|
||||
|
||||
from src.market_data.acquisition.runtime.websocket_protocol import (
|
||||
WebSocketTransportProtocol,
|
||||
)
|
||||
|
||||
|
||||
class _SessionTransportProtocol(
|
||||
WebSocketTransportProtocol,
|
||||
Protocol,
|
||||
):
|
||||
"""
|
||||
Transport-контракт для Session lifecycle.
|
||||
"""
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Показывает состояние Transport."""
|
||||
...
|
||||
|
||||
|
||||
class WebSocketSession:
|
||||
"""
|
||||
Идемпотентный lifecycle WebSocket-сессии.
|
||||
|
||||
Session сериализует start/stop, делегирует сетевые операции
|
||||
Transport и не содержит parsing, subscriptions или recovery.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_transport",
|
||||
"_started",
|
||||
"_lifecycle_lock",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
transport: _SessionTransportProtocol,
|
||||
) -> None:
|
||||
self._transport = transport
|
||||
self._started = False
|
||||
self._lifecycle_lock = asyncio.Lock()
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Показывает состояние Session."""
|
||||
return (
|
||||
self._started
|
||||
and self._transport.is_connected
|
||||
)
|
||||
|
||||
async def start(self) -> None:
|
||||
"""
|
||||
Открыть Session без повторного соединения.
|
||||
"""
|
||||
async with self._lifecycle_lock:
|
||||
if self.is_connected:
|
||||
return
|
||||
|
||||
self._started = False
|
||||
await self._transport.connect()
|
||||
self._started = True
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""
|
||||
Закрыть Session и сбросить локальное состояние.
|
||||
"""
|
||||
async with self._lifecycle_lock:
|
||||
if (
|
||||
not self._started
|
||||
and not self._transport.is_connected
|
||||
):
|
||||
return
|
||||
|
||||
try:
|
||||
await self._transport.disconnect()
|
||||
finally:
|
||||
self._started = False
|
||||
@@ -0,0 +1,176 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from src.market_data.acquisition.exceptions import (
|
||||
WebSocketUnsubscribeNotSupportedError,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.transport_messages import (
|
||||
TransportBinaryMessage,
|
||||
TransportTextMessage,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_protocol import (
|
||||
AcquisitionSubscriptionMessage,
|
||||
WebSocketTransportProtocol,
|
||||
)
|
||||
|
||||
|
||||
class WebSocketSubscriptionManager:
|
||||
"""
|
||||
Реестр WebSocket-подписок и механизм их восстановления.
|
||||
|
||||
Manager хранит только универсальные transport messages.
|
||||
Он не разбирает payload и не содержит exchange-specific знаний.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_transport",
|
||||
"_supports_unsubscribe",
|
||||
"_subscriptions",
|
||||
"_active_subscription_keys",
|
||||
"_lock",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
transport: WebSocketTransportProtocol,
|
||||
*,
|
||||
supports_unsubscribe: bool = False,
|
||||
) -> None:
|
||||
if not isinstance(supports_unsubscribe, bool):
|
||||
raise TypeError(
|
||||
"supports_unsubscribe must be a boolean"
|
||||
)
|
||||
|
||||
self._transport = transport
|
||||
self._supports_unsubscribe = supports_unsubscribe
|
||||
self._subscriptions: dict[
|
||||
str,
|
||||
AcquisitionSubscriptionMessage,
|
||||
] = {}
|
||||
self._active_subscription_keys: set[str] = set()
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
@property
|
||||
def subscription_keys(self) -> tuple[str, ...]:
|
||||
"""Вернуть ключи в порядке регистрации."""
|
||||
return tuple(self._subscriptions)
|
||||
|
||||
async def subscribe(
|
||||
self,
|
||||
subscription_key: str,
|
||||
message: AcquisitionSubscriptionMessage,
|
||||
) -> None:
|
||||
"""
|
||||
Идемпотентно отправить и зарегистрировать подписку.
|
||||
|
||||
Desired subscription сохраняется до сетевой отправки.
|
||||
При ошибке она остаётся pending для последующего retry/reconnect.
|
||||
"""
|
||||
self._validate_subscription_key(subscription_key)
|
||||
self._validate_message(message)
|
||||
|
||||
async with self._lock:
|
||||
if subscription_key in self._active_subscription_keys:
|
||||
return
|
||||
|
||||
self._subscriptions[subscription_key] = message
|
||||
|
||||
await self._send(message)
|
||||
self._active_subscription_keys.add(
|
||||
subscription_key,
|
||||
)
|
||||
|
||||
async def unsubscribe(
|
||||
self,
|
||||
subscription_key: str,
|
||||
message: AcquisitionSubscriptionMessage,
|
||||
) -> None:
|
||||
"""
|
||||
Отправить unsubscribe только при явной поддержке провайдером.
|
||||
|
||||
Dzengi ``trades.unsubscribe`` не поддерживает. Поэтому
|
||||
``supports_unsubscribe`` по умолчанию равно False.
|
||||
"""
|
||||
self._validate_subscription_key(subscription_key)
|
||||
self._validate_message(message)
|
||||
|
||||
if not self._supports_unsubscribe:
|
||||
raise WebSocketUnsubscribeNotSupportedError(
|
||||
"WebSocket provider does not support unsubscribe."
|
||||
)
|
||||
|
||||
async with self._lock:
|
||||
if subscription_key not in self._subscriptions:
|
||||
return
|
||||
|
||||
if subscription_key in self._active_subscription_keys:
|
||||
await self._send(message)
|
||||
|
||||
del self._subscriptions[subscription_key]
|
||||
self._active_subscription_keys.discard(
|
||||
subscription_key,
|
||||
)
|
||||
|
||||
async def restore_subscriptions(self) -> None:
|
||||
"""
|
||||
Последовательно восстановить подписки.
|
||||
"""
|
||||
async with self._lock:
|
||||
self._active_subscription_keys.clear()
|
||||
|
||||
for subscription_key, message in (
|
||||
self._subscriptions.items()
|
||||
):
|
||||
await self._send(message)
|
||||
self._active_subscription_keys.add(
|
||||
subscription_key,
|
||||
)
|
||||
|
||||
async def clear_subscriptions(self) -> None:
|
||||
"""
|
||||
Очистить реестр без отправки сообщений.
|
||||
"""
|
||||
async with self._lock:
|
||||
self._subscriptions.clear()
|
||||
self._active_subscription_keys.clear()
|
||||
|
||||
async def _send(
|
||||
self,
|
||||
message: AcquisitionSubscriptionMessage,
|
||||
) -> None:
|
||||
if isinstance(message, TransportTextMessage):
|
||||
await self._transport.send(message.payload)
|
||||
return
|
||||
|
||||
await self._transport.send(message.payload)
|
||||
|
||||
@staticmethod
|
||||
def _validate_subscription_key(
|
||||
subscription_key: str,
|
||||
) -> None:
|
||||
if not isinstance(subscription_key, str):
|
||||
raise TypeError(
|
||||
"subscription_key must be a string"
|
||||
)
|
||||
|
||||
if not subscription_key.strip():
|
||||
raise ValueError(
|
||||
"subscription_key must not be empty"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_message(
|
||||
message: AcquisitionSubscriptionMessage,
|
||||
) -> None:
|
||||
if not isinstance(
|
||||
message,
|
||||
(
|
||||
TransportTextMessage,
|
||||
TransportBinaryMessage,
|
||||
),
|
||||
):
|
||||
raise TypeError(
|
||||
"message must be TransportTextMessage "
|
||||
"or TransportBinaryMessage"
|
||||
)
|
||||
@@ -7,6 +7,7 @@ import asyncio
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
from src.market_data.acquisition.adapters.dzengi.rest import (
|
||||
DzengiTradesDocumentSource,
|
||||
@@ -30,9 +31,20 @@ from src.market_data.acquisition.runtime.acquisition_runtime_service import (
|
||||
from src.market_data.acquisition.runtime.heartbeat import (
|
||||
HeartbeatMonitor,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.live_processing_gate import (
|
||||
RuntimeLiveProcessingGate,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.reconnect import (
|
||||
ReconnectCoordinator,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_reconnect_recovery_coordinator import (
|
||||
RuntimeReconnectRecoveryCoordinator,
|
||||
RuntimeUnixTimeMillisecondsClock,
|
||||
system_unix_time_ms,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_liveness_probe import (
|
||||
RuntimeLivenessProbeProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_recovery_coordinator import (
|
||||
RuntimeRecoveryCoordinator,
|
||||
)
|
||||
@@ -57,6 +69,14 @@ from src.market_data.acquisition.trade_stream_message_adapter_protocol import (
|
||||
)
|
||||
|
||||
|
||||
class _TradeStreamRuntimeTransportProtocol(
|
||||
WebSocketTransportProtocol,
|
||||
RuntimeLivenessProbeProtocol,
|
||||
Protocol,
|
||||
):
|
||||
"""Общий Transport и liveness-контракт Composition."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TradeStreamRuntimeComposition:
|
||||
"""
|
||||
@@ -80,6 +100,11 @@ class TradeStreamRuntimeComposition:
|
||||
trade_stream_acquisition_service: TradeStreamAcquisitionService
|
||||
|
||||
reconnect_coordinator: ReconnectCoordinator
|
||||
live_processing_gate: RuntimeLiveProcessingGate
|
||||
liveness_probe: RuntimeLivenessProbeProtocol
|
||||
runtime_reconnect_recovery_coordinator: (
|
||||
RuntimeReconnectRecoveryCoordinator
|
||||
)
|
||||
heartbeat_monitor: HeartbeatMonitor
|
||||
runtime_supervisor: RuntimeSupervisor
|
||||
runtime_scheduler: RuntimeScheduler
|
||||
@@ -88,15 +113,19 @@ class TradeStreamRuntimeComposition:
|
||||
def build_trade_stream_runtime_composition(
|
||||
*,
|
||||
session: WebSocketSessionProtocol,
|
||||
transport: WebSocketTransportProtocol,
|
||||
transport: _TradeStreamRuntimeTransportProtocol,
|
||||
subscription_manager: WebSocketSubscriptionManagerProtocol,
|
||||
event_publisher: AcquisitionRuntimeEventPublisherProtocol,
|
||||
message_adapter: TradeStreamMessageAdapterProtocol,
|
||||
recovery_document_source: DzengiTradesDocumentSource,
|
||||
symbols: tuple[str, ...],
|
||||
heartbeat_timeout_seconds: float,
|
||||
scheduler_interval_seconds: float,
|
||||
max_recovery_window_ms: int = DEFAULT_TRADE_RECOVERY_WINDOW_MS,
|
||||
heartbeat_clock: Callable[[], float] = time.monotonic,
|
||||
recovery_end_time_clock: RuntimeUnixTimeMillisecondsClock = (
|
||||
system_unix_time_ms
|
||||
),
|
||||
scheduler_sleep: RuntimeSleep = asyncio.sleep,
|
||||
) -> TradeStreamRuntimeComposition:
|
||||
"""
|
||||
@@ -151,6 +180,18 @@ def build_trade_stream_runtime_composition(
|
||||
event_publisher=event_publisher,
|
||||
)
|
||||
|
||||
live_processing_gate = RuntimeLiveProcessingGate()
|
||||
|
||||
runtime_reconnect_recovery_coordinator = (
|
||||
RuntimeReconnectRecoveryCoordinator(
|
||||
reconnect_coordinator=reconnect_coordinator,
|
||||
recovery_coordinator=runtime_recovery_coordinator,
|
||||
live_processing_gate=live_processing_gate,
|
||||
symbols=symbols,
|
||||
clock=recovery_end_time_clock,
|
||||
)
|
||||
)
|
||||
|
||||
heartbeat_monitor = HeartbeatMonitor(
|
||||
event_publisher=event_publisher,
|
||||
timeout_seconds=heartbeat_timeout_seconds,
|
||||
@@ -159,10 +200,13 @@ def build_trade_stream_runtime_composition(
|
||||
|
||||
runtime_supervisor = RuntimeSupervisor(
|
||||
heartbeat_monitor=heartbeat_monitor,
|
||||
reconnect_coordinator=reconnect_coordinator,
|
||||
reconnect_coordinator=(
|
||||
runtime_reconnect_recovery_coordinator
|
||||
),
|
||||
)
|
||||
|
||||
runtime_scheduler = RuntimeScheduler(
|
||||
liveness_probe=transport,
|
||||
heartbeat_monitor=heartbeat_monitor,
|
||||
runtime_supervisor=runtime_supervisor,
|
||||
interval_seconds=scheduler_interval_seconds,
|
||||
@@ -178,6 +222,11 @@ def build_trade_stream_runtime_composition(
|
||||
acquisition_runtime_service=acquisition_runtime_service,
|
||||
trade_stream_acquisition_service=trade_stream_acquisition_service,
|
||||
reconnect_coordinator=reconnect_coordinator,
|
||||
live_processing_gate=live_processing_gate,
|
||||
liveness_probe=transport,
|
||||
runtime_reconnect_recovery_coordinator=(
|
||||
runtime_reconnect_recovery_coordinator
|
||||
),
|
||||
heartbeat_monitor=heartbeat_monitor,
|
||||
runtime_supervisor=runtime_supervisor,
|
||||
runtime_scheduler=runtime_scheduler,
|
||||
|
||||
166
app/tests/unit/bootstrap/test_app_factory.py
Normal file
166
app/tests/unit/bootstrap/test_app_factory.py
Normal file
@@ -0,0 +1,166 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
import src.bootstrap.app_factory as app_factory
|
||||
from src.bootstrap.application import ApplicationComposition
|
||||
|
||||
|
||||
class RecordingJournal:
|
||||
def __init__(self) -> None:
|
||||
self.info_calls: list[
|
||||
tuple[str, str, dict[str, object]]
|
||||
] = []
|
||||
|
||||
def log_info(
|
||||
self,
|
||||
event: str,
|
||||
message: str,
|
||||
context: dict[str, object],
|
||||
) -> None:
|
||||
self.info_calls.append(
|
||||
(
|
||||
event,
|
||||
message,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
def log_critical(
|
||||
self,
|
||||
event: str,
|
||||
message: str,
|
||||
context: dict[str, object],
|
||||
) -> None:
|
||||
del event, message, context
|
||||
|
||||
|
||||
def make_settings() -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
bot_token="test-token",
|
||||
bot_parse_mode="HTML",
|
||||
log_level="INFO",
|
||||
app_env="test",
|
||||
exchange_name="dzengi",
|
||||
default_symbol="BTC/USD_LEVERAGE",
|
||||
trade_stream=SimpleNamespace(enabled=True),
|
||||
)
|
||||
|
||||
|
||||
def test_create_app_builds_one_application_composition(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
settings = make_settings()
|
||||
bot = object()
|
||||
dispatcher = object()
|
||||
runtime = object()
|
||||
journal = RecordingJournal()
|
||||
observed_runtime_settings: list[object] = []
|
||||
registered_bots: list[object] = []
|
||||
routed_dispatchers: list[object] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"load_settings",
|
||||
lambda: settings,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"setup_logging",
|
||||
lambda level: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"init_schema",
|
||||
lambda: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"JournalService",
|
||||
lambda: journal,
|
||||
)
|
||||
|
||||
def build_runtime(received_settings: object) -> object:
|
||||
observed_runtime_settings.append(received_settings)
|
||||
return runtime
|
||||
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"build_trade_stream_production_runtime",
|
||||
build_runtime,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"Bot",
|
||||
lambda **kwargs: bot,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"Dispatcher",
|
||||
lambda: dispatcher,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
app_factory.NotificationTargetRegistry,
|
||||
"set_bot",
|
||||
registered_bots.append,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"setup_routers",
|
||||
routed_dispatchers.append,
|
||||
)
|
||||
|
||||
application = app_factory.create_app()
|
||||
|
||||
assert isinstance(application, ApplicationComposition)
|
||||
assert application.bot is bot
|
||||
assert application.dispatcher is dispatcher
|
||||
assert application.trade_stream_runtime is runtime
|
||||
assert observed_runtime_settings == [settings]
|
||||
assert registered_bots == [bot]
|
||||
assert routed_dispatchers == [dispatcher]
|
||||
assert journal.info_calls[0][2]["trade_stream_enabled"] is True
|
||||
|
||||
|
||||
def test_runtime_build_error_is_fatal(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
expected = RuntimeError("invalid Trade Stream settings")
|
||||
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"load_settings",
|
||||
make_settings,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"setup_logging",
|
||||
lambda level: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"init_schema",
|
||||
lambda: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"JournalService",
|
||||
RecordingJournal,
|
||||
)
|
||||
|
||||
def fail_runtime_build(settings: object) -> None:
|
||||
del settings
|
||||
raise expected
|
||||
|
||||
monkeypatch.setattr(
|
||||
app_factory,
|
||||
"build_trade_stream_production_runtime",
|
||||
fail_runtime_build,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
app_factory.create_app()
|
||||
|
||||
assert exc_info.value is expected
|
||||
442
app/tests/unit/bootstrap/test_application.py
Normal file
442
app/tests/unit/bootstrap/test_application.py
Normal file
@@ -0,0 +1,442 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from src.bootstrap.application import (
|
||||
ApplicationComposition,
|
||||
run_application,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.trade_stream_production_runtime import (
|
||||
TradeStreamProductionRuntimeState,
|
||||
)
|
||||
|
||||
|
||||
class FakeBotSession:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
close_error: BaseException | None = None,
|
||||
) -> None:
|
||||
self.close_calls = 0
|
||||
self.close_error = close_error
|
||||
|
||||
async def close(self) -> None:
|
||||
self.close_calls += 1
|
||||
|
||||
if self.close_error is not None:
|
||||
raise self.close_error
|
||||
|
||||
|
||||
class FakeBot:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
close_error: BaseException | None = None,
|
||||
) -> None:
|
||||
self.session = FakeBotSession(
|
||||
close_error=close_error,
|
||||
)
|
||||
|
||||
|
||||
class FakeDispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
return_immediately: bool = False,
|
||||
error: BaseException | None = None,
|
||||
) -> None:
|
||||
self.started = asyncio.Event()
|
||||
self.release = asyncio.Event()
|
||||
self.cancelled = asyncio.Event()
|
||||
self.return_immediately = return_immediately
|
||||
self.error = error
|
||||
self.close_bot_session_values: list[bool] = []
|
||||
|
||||
async def start_polling(
|
||||
self,
|
||||
bot: FakeBot,
|
||||
*,
|
||||
close_bot_session: bool,
|
||||
) -> None:
|
||||
del bot
|
||||
self.close_bot_session_values.append(close_bot_session)
|
||||
self.started.set()
|
||||
|
||||
try:
|
||||
if not self.return_immediately:
|
||||
await self.release.wait()
|
||||
except asyncio.CancelledError:
|
||||
self.cancelled.set()
|
||||
raise
|
||||
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
|
||||
|
||||
class FakeRuntime:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
return_immediately: bool = False,
|
||||
error: BaseException | None = None,
|
||||
stop_error: BaseException | None = None,
|
||||
) -> None:
|
||||
self.started = asyncio.Event()
|
||||
self.release = asyncio.Event()
|
||||
self.stopped = asyncio.Event()
|
||||
self.return_immediately = return_immediately
|
||||
self.error = error
|
||||
self.stop_error = stop_error
|
||||
self.stop_calls = 0
|
||||
|
||||
@property
|
||||
def state(self) -> TradeStreamProductionRuntimeState:
|
||||
return (
|
||||
TradeStreamProductionRuntimeState.RUNNING
|
||||
if self.started.is_set() and not self.stopped.is_set()
|
||||
else TradeStreamProductionRuntimeState.STOPPED
|
||||
)
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.state is TradeStreamProductionRuntimeState.RUNNING
|
||||
|
||||
async def run(self) -> None:
|
||||
self.started.set()
|
||||
|
||||
if not self.return_immediately:
|
||||
await self.release.wait()
|
||||
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
|
||||
async def stop(self) -> None:
|
||||
self.stop_calls += 1
|
||||
self.release.set()
|
||||
self.stopped.set()
|
||||
|
||||
if self.stop_error is not None:
|
||||
raise self.stop_error
|
||||
|
||||
|
||||
class BlockingStopRuntime(FakeRuntime):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.stop_entered = asyncio.Event()
|
||||
self.stop_release = asyncio.Event()
|
||||
|
||||
async def stop(self) -> None:
|
||||
self.stop_calls += 1
|
||||
self.stop_entered.set()
|
||||
await self.stop_release.wait()
|
||||
self.release.set()
|
||||
self.stopped.set()
|
||||
|
||||
|
||||
def make_application(
|
||||
*,
|
||||
dispatcher: FakeDispatcher,
|
||||
runtime: FakeRuntime | None,
|
||||
bot: FakeBot | None = None,
|
||||
) -> ApplicationComposition:
|
||||
return ApplicationComposition(
|
||||
bot=bot or FakeBot(), # type: ignore[arg-type]
|
||||
dispatcher=dispatcher, # type: ignore[arg-type]
|
||||
trade_stream_runtime=runtime,
|
||||
)
|
||||
|
||||
|
||||
def test_disabled_runtime_runs_only_polling_and_closes_bot() -> None:
|
||||
async def scenario() -> None:
|
||||
dispatcher = FakeDispatcher(return_immediately=True)
|
||||
bot = FakeBot()
|
||||
|
||||
await run_application(
|
||||
make_application(
|
||||
dispatcher=dispatcher,
|
||||
runtime=None,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
|
||||
assert dispatcher.close_bot_session_values == [False]
|
||||
assert bot.session.close_calls == 1
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_polling_completion_stops_and_awaits_runtime() -> None:
|
||||
async def scenario() -> None:
|
||||
dispatcher = FakeDispatcher(return_immediately=True)
|
||||
runtime = FakeRuntime()
|
||||
bot = FakeBot()
|
||||
|
||||
await run_application(
|
||||
make_application(
|
||||
dispatcher=dispatcher,
|
||||
runtime=runtime,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
|
||||
assert runtime.started.is_set()
|
||||
assert runtime.stop_calls == 1
|
||||
assert runtime.stopped.is_set()
|
||||
assert bot.session.close_calls == 1
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_runtime_error_is_fatal_and_stops_polling() -> None:
|
||||
async def scenario() -> None:
|
||||
expected = RuntimeError("trade stream failed")
|
||||
dispatcher = FakeDispatcher()
|
||||
runtime = FakeRuntime(
|
||||
return_immediately=True,
|
||||
error=expected,
|
||||
)
|
||||
bot = FakeBot()
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
await run_application(
|
||||
make_application(
|
||||
dispatcher=dispatcher,
|
||||
runtime=runtime,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
|
||||
assert exc_info.value is expected
|
||||
assert dispatcher.cancelled.is_set()
|
||||
assert runtime.stop_calls == 1
|
||||
assert bot.session.close_calls == 1
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_normal_runtime_completion_is_fatal() -> None:
|
||||
async def scenario() -> None:
|
||||
dispatcher = FakeDispatcher()
|
||||
runtime = FakeRuntime(return_immediately=True)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="terminated unexpectedly",
|
||||
):
|
||||
await run_application(
|
||||
make_application(
|
||||
dispatcher=dispatcher,
|
||||
runtime=runtime,
|
||||
)
|
||||
)
|
||||
|
||||
assert dispatcher.cancelled.is_set()
|
||||
assert runtime.stop_calls == 1
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_application_cancellation_performs_full_cleanup() -> None:
|
||||
async def scenario() -> None:
|
||||
dispatcher = FakeDispatcher()
|
||||
runtime = FakeRuntime()
|
||||
bot = FakeBot()
|
||||
task = asyncio.create_task(
|
||||
run_application(
|
||||
make_application(
|
||||
dispatcher=dispatcher,
|
||||
runtime=runtime,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
await dispatcher.started.wait()
|
||||
await runtime.started.wait()
|
||||
task.cancel()
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
assert dispatcher.cancelled.is_set()
|
||||
assert runtime.stop_calls == 1
|
||||
assert runtime.stopped.is_set()
|
||||
assert bot.session.close_calls == 1
|
||||
await asyncio.sleep(0)
|
||||
assert not {
|
||||
child.get_name()
|
||||
for child in asyncio.all_tasks()
|
||||
if child is not asyncio.current_task()
|
||||
and not child.done()
|
||||
and child.get_name()
|
||||
in {
|
||||
"telegram-polling",
|
||||
"trade-stream-runtime",
|
||||
"application-shutdown",
|
||||
}
|
||||
}
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_polling_error_stops_runtime_and_preserves_error() -> None:
|
||||
async def scenario() -> None:
|
||||
expected = RuntimeError("polling failed")
|
||||
dispatcher = FakeDispatcher(
|
||||
return_immediately=True,
|
||||
error=expected,
|
||||
)
|
||||
runtime = FakeRuntime()
|
||||
bot = FakeBot()
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
await run_application(
|
||||
make_application(
|
||||
dispatcher=dispatcher,
|
||||
runtime=runtime,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
|
||||
assert exc_info.value is expected
|
||||
assert runtime.stop_calls == 1
|
||||
assert runtime.stopped.is_set()
|
||||
assert bot.session.close_calls == 1
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_simultaneous_root_failures_are_awaited_deterministically() -> None:
|
||||
async def scenario() -> None:
|
||||
polling_error = RuntimeError("polling failed")
|
||||
runtime_error = ValueError("trade stream failed")
|
||||
dispatcher = FakeDispatcher(
|
||||
error=polling_error,
|
||||
)
|
||||
runtime = FakeRuntime(
|
||||
error=runtime_error,
|
||||
)
|
||||
bot = FakeBot()
|
||||
task = asyncio.create_task(
|
||||
run_application(
|
||||
make_application(
|
||||
dispatcher=dispatcher,
|
||||
runtime=runtime,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
await dispatcher.started.wait()
|
||||
await runtime.started.wait()
|
||||
dispatcher.release.set()
|
||||
runtime.release.set()
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
await task
|
||||
|
||||
assert exc_info.value is runtime_error
|
||||
assert runtime.stop_calls == 1
|
||||
assert bot.session.close_calls == 1
|
||||
assert any(
|
||||
"cleanup also failed" in note
|
||||
for note in getattr(runtime_error, "__notes__", ())
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
assert not {
|
||||
child.get_name()
|
||||
for child in asyncio.all_tasks()
|
||||
if child is not asyncio.current_task()
|
||||
and not child.done()
|
||||
and child.get_name()
|
||||
in {
|
||||
"telegram-polling",
|
||||
"trade-stream-runtime",
|
||||
"application-shutdown",
|
||||
}
|
||||
}
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_repeated_cancellation_does_not_interrupt_cleanup() -> None:
|
||||
async def scenario() -> None:
|
||||
dispatcher = FakeDispatcher()
|
||||
runtime = BlockingStopRuntime()
|
||||
bot = FakeBot()
|
||||
task = asyncio.create_task(
|
||||
run_application(
|
||||
make_application(
|
||||
dispatcher=dispatcher,
|
||||
runtime=runtime,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
await dispatcher.started.wait()
|
||||
await runtime.started.wait()
|
||||
task.cancel()
|
||||
await runtime.stop_entered.wait()
|
||||
|
||||
task.cancel()
|
||||
runtime.stop_release.set()
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
assert task.cancelled() is True
|
||||
assert dispatcher.cancelled.is_set()
|
||||
assert runtime.stop_calls == 1
|
||||
assert runtime.stopped.is_set()
|
||||
assert bot.session.close_calls == 1
|
||||
await asyncio.sleep(0)
|
||||
assert not {
|
||||
child.get_name()
|
||||
for child in asyncio.all_tasks()
|
||||
if child is not asyncio.current_task()
|
||||
and not child.done()
|
||||
and child.get_name()
|
||||
in {
|
||||
"telegram-polling",
|
||||
"trade-stream-runtime",
|
||||
"application-shutdown",
|
||||
}
|
||||
}
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_cleanup_error_does_not_replace_runtime_error() -> None:
|
||||
async def scenario() -> None:
|
||||
expected = RuntimeError("trade stream failed")
|
||||
dispatcher = FakeDispatcher()
|
||||
runtime = FakeRuntime(
|
||||
return_immediately=True,
|
||||
error=expected,
|
||||
stop_error=RuntimeError("stop failed"),
|
||||
)
|
||||
bot = FakeBot(
|
||||
close_error=RuntimeError("close failed"),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
await run_application(
|
||||
make_application(
|
||||
dispatcher=dispatcher,
|
||||
runtime=runtime,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
|
||||
assert exc_info.value is expected
|
||||
assert bot.session.close_calls == 1
|
||||
assert any(
|
||||
"cleanup also failed" in note
|
||||
for note in getattr(expected, "__notes__", ())
|
||||
)
|
||||
|
||||
asyncio.run(scenario())
|
||||
@@ -0,0 +1,890 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from websockets.protocol import State
|
||||
|
||||
import src.bootstrap.trade_stream_runtime as production_factory
|
||||
from src.bootstrap.application import (
|
||||
ApplicationComposition,
|
||||
run_application,
|
||||
)
|
||||
from src.core.config import Settings, TradeStreamSettings
|
||||
from src.market_data.acquisition.adapters.dzengi.websocket_transport import (
|
||||
DzengiWebSocketTransport,
|
||||
)
|
||||
from src.market_data.acquisition.consistency.trade_stream_state_store import (
|
||||
TradeStreamStateStore,
|
||||
)
|
||||
from src.market_data.acquisition.exceptions import (
|
||||
TradeTransportError,
|
||||
WebSocketTransportError,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.trade_stream_production_runtime import (
|
||||
TradeStreamProductionRuntime,
|
||||
TradeStreamProductionRuntimeState,
|
||||
)
|
||||
|
||||
|
||||
SYMBOL = "BTC/USD_LEVERAGE"
|
||||
|
||||
_OWNED_TASK_NAMES = frozenset(
|
||||
{
|
||||
"application-shutdown",
|
||||
"telegram-polling",
|
||||
"trade-stream-receive",
|
||||
"trade-stream-runtime",
|
||||
"trade-stream-runtime-recovery",
|
||||
"trade-stream-scheduler",
|
||||
"trade-stream-startup",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class FakeBotSession:
|
||||
def __init__(self) -> None:
|
||||
self.close_calls = 0
|
||||
|
||||
async def close(self) -> None:
|
||||
self.close_calls += 1
|
||||
|
||||
|
||||
class FakeBot:
|
||||
def __init__(self) -> None:
|
||||
self.session = FakeBotSession()
|
||||
|
||||
|
||||
class ControlledDispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
return_immediately: bool = False,
|
||||
) -> None:
|
||||
self.return_immediately = return_immediately
|
||||
self.started = asyncio.Event()
|
||||
self.release = asyncio.Event()
|
||||
self.cancelled = asyncio.Event()
|
||||
self.close_bot_session_values: list[bool] = []
|
||||
|
||||
async def start_polling(
|
||||
self,
|
||||
bot: FakeBot,
|
||||
*,
|
||||
close_bot_session: bool,
|
||||
) -> None:
|
||||
del bot
|
||||
self.close_bot_session_values.append(close_bot_session)
|
||||
self.started.set()
|
||||
|
||||
if self.return_immediately:
|
||||
return
|
||||
|
||||
try:
|
||||
await self.release.wait()
|
||||
except asyncio.CancelledError:
|
||||
self.cancelled.set()
|
||||
raise
|
||||
|
||||
|
||||
class ScriptedConnection:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
name: str,
|
||||
calls: list[str],
|
||||
send_error: Exception | None = None,
|
||||
close_on_receive_error: bool = True,
|
||||
) -> None:
|
||||
self.name = name
|
||||
self.calls = calls
|
||||
self.send_error = send_error
|
||||
self.close_on_receive_error = close_on_receive_error
|
||||
|
||||
self.state = State.OPEN
|
||||
self.incoming: asyncio.Queue[
|
||||
str | bytes | BaseException
|
||||
] = asyncio.Queue()
|
||||
self.sent_messages: list[str | bytes] = []
|
||||
self.subscription_sent = asyncio.Event()
|
||||
self.close_calls = 0
|
||||
self.ping_calls = 0
|
||||
|
||||
async def close(
|
||||
self,
|
||||
code: int = 1000,
|
||||
reason: str = "",
|
||||
) -> None:
|
||||
del code, reason
|
||||
self.calls.append(f"{self.name}.close")
|
||||
self.close_calls += 1
|
||||
self.state = State.CLOSED
|
||||
|
||||
async def send(
|
||||
self,
|
||||
message: str | bytes,
|
||||
) -> None:
|
||||
self.calls.append(f"{self.name}.send")
|
||||
|
||||
if self.send_error is not None:
|
||||
self.state = State.CLOSED
|
||||
raise self.send_error
|
||||
|
||||
self.sent_messages.append(message)
|
||||
self.subscription_sent.set()
|
||||
|
||||
if not isinstance(message, str):
|
||||
return
|
||||
|
||||
document = json.loads(message)
|
||||
|
||||
if document.get("destination") != "trades.subscribe":
|
||||
return
|
||||
|
||||
self.feed(
|
||||
json.dumps(
|
||||
{
|
||||
"correlationId": document["correlationId"],
|
||||
"destination": "trades.subscribe",
|
||||
"status": "OK",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def recv(self) -> str | bytes:
|
||||
item = await self.incoming.get()
|
||||
|
||||
if isinstance(item, BaseException):
|
||||
self.calls.append(f"{self.name}.receive_error")
|
||||
|
||||
if self.close_on_receive_error:
|
||||
self.state = State.CLOSED
|
||||
|
||||
raise item
|
||||
|
||||
self.calls.append(f"{self.name}.receive")
|
||||
return item
|
||||
|
||||
async def ping(self) -> Awaitable[float]:
|
||||
self.calls.append(f"{self.name}.ping")
|
||||
self.ping_calls += 1
|
||||
|
||||
async def wait_for_pong() -> float:
|
||||
return 0.001
|
||||
|
||||
return wait_for_pong()
|
||||
|
||||
def feed(
|
||||
self,
|
||||
item: str | bytes | BaseException,
|
||||
) -> None:
|
||||
self.incoming.put_nowait(item)
|
||||
|
||||
|
||||
class ScriptedConnector:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
calls: list[str],
|
||||
connections: tuple[ScriptedConnection, ...] = (),
|
||||
error: Exception | None = None,
|
||||
) -> None:
|
||||
self.calls = calls
|
||||
self.connections = list(connections)
|
||||
self.error = error
|
||||
self.options: list[dict[str, Any]] = []
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
url: str,
|
||||
**kwargs: Any,
|
||||
) -> ScriptedConnection:
|
||||
self.calls.append("connector.connect")
|
||||
self.options.append(
|
||||
{
|
||||
"url": url,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
|
||||
return self.connections.pop(0)
|
||||
|
||||
|
||||
class BlockingRestClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
calls: list[str],
|
||||
document: object,
|
||||
error: Exception | None = None,
|
||||
) -> None:
|
||||
self.calls = calls
|
||||
self.document = document
|
||||
self.error = error
|
||||
self.entered = threading.Event()
|
||||
self.release = threading.Event()
|
||||
self.requests: list[
|
||||
tuple[str, dict[str, str] | None]
|
||||
] = []
|
||||
|
||||
def get_payload(
|
||||
self,
|
||||
path: str,
|
||||
params: dict[str, str] | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> object:
|
||||
del headers
|
||||
self.calls.append("recovery.fetch")
|
||||
self.requests.append(
|
||||
(
|
||||
path,
|
||||
params,
|
||||
)
|
||||
)
|
||||
self.entered.set()
|
||||
|
||||
if not self.release.wait(timeout=5):
|
||||
raise TimeoutError("test did not release REST recovery")
|
||||
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
|
||||
return self.document
|
||||
|
||||
|
||||
def make_settings(
|
||||
*,
|
||||
enabled: bool = True,
|
||||
) -> Settings:
|
||||
return Settings(
|
||||
bot_token="test-token",
|
||||
bot_parse_mode="HTML",
|
||||
app_env="test",
|
||||
log_level="INFO",
|
||||
tz="UTC",
|
||||
exchange_enabled=True,
|
||||
exchange_name="dzengi",
|
||||
exchange_base_url="https://rest.example.test",
|
||||
exchange_ws_url="wss://legacy.example.test",
|
||||
exchange_api_key="",
|
||||
exchange_api_secret="",
|
||||
exchange_timeout_sec=10,
|
||||
exchange_testnet=True,
|
||||
default_symbol="LEGACY",
|
||||
trade_stream=TradeStreamSettings(
|
||||
enabled=enabled,
|
||||
websocket_url="wss://stream.example.test",
|
||||
symbols=(SYMBOL,),
|
||||
open_timeout_seconds=1.0,
|
||||
probe_timeout_seconds=1.0,
|
||||
close_timeout_seconds=1.0,
|
||||
heartbeat_timeout_seconds=120.0,
|
||||
scheduler_interval_seconds=60.0,
|
||||
recovery_window_ms=3_599_999,
|
||||
),
|
||||
db_host="localhost",
|
||||
db_port=5432,
|
||||
db_name="test",
|
||||
db_user="test",
|
||||
db_password="test",
|
||||
debug_enabled=False,
|
||||
journal_debug_enabled=False,
|
||||
)
|
||||
|
||||
|
||||
def make_trade_document(
|
||||
*,
|
||||
trade_id: int,
|
||||
timestamp: int,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"status": "OK",
|
||||
"destination": "internal.trade",
|
||||
"payload": {
|
||||
"id": trade_id,
|
||||
"price": "64555.55",
|
||||
"size": "0.002",
|
||||
"ts": timestamp,
|
||||
"symbol": SYMBOL,
|
||||
"buyer": True,
|
||||
"orderId": f"order-{trade_id}",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def make_recovered_trade(
|
||||
*,
|
||||
trade_id: int,
|
||||
timestamp: int,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"a": trade_id,
|
||||
"p": "64555.56",
|
||||
"q": "0.003",
|
||||
"T": timestamp,
|
||||
"m": False,
|
||||
}
|
||||
|
||||
|
||||
def install_connector(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
connector: ScriptedConnector,
|
||||
) -> list[DzengiWebSocketTransport]:
|
||||
transport_type = production_factory.DzengiWebSocketTransport
|
||||
transports: list[DzengiWebSocketTransport] = []
|
||||
|
||||
def build_transport(
|
||||
**kwargs: Any,
|
||||
) -> DzengiWebSocketTransport:
|
||||
transport = transport_type(
|
||||
connector=connector,
|
||||
**kwargs,
|
||||
)
|
||||
transports.append(transport)
|
||||
return transport
|
||||
|
||||
monkeypatch.setattr(
|
||||
production_factory,
|
||||
"DzengiWebSocketTransport",
|
||||
build_transport,
|
||||
)
|
||||
|
||||
return transports
|
||||
|
||||
|
||||
def make_application(
|
||||
*,
|
||||
runtime: TradeStreamProductionRuntime | None,
|
||||
dispatcher: ControlledDispatcher,
|
||||
bot: FakeBot,
|
||||
) -> ApplicationComposition:
|
||||
return ApplicationComposition(
|
||||
bot=bot, # type: ignore[arg-type]
|
||||
dispatcher=dispatcher, # type: ignore[arg-type]
|
||||
trade_stream_runtime=runtime,
|
||||
)
|
||||
|
||||
|
||||
async def wait_until(
|
||||
predicate: Callable[[], bool],
|
||||
) -> None:
|
||||
for _ in range(1_000):
|
||||
if predicate():
|
||||
return
|
||||
|
||||
await asyncio.sleep(0)
|
||||
|
||||
raise AssertionError("condition was not reached")
|
||||
|
||||
|
||||
async def assert_no_owned_tasks() -> None:
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert not {
|
||||
task.get_name()
|
||||
for task in asyncio.all_tasks()
|
||||
if task is not asyncio.current_task()
|
||||
and not task.done()
|
||||
and task.get_name() in _OWNED_TASK_NAMES
|
||||
}
|
||||
|
||||
|
||||
def state_store_from(
|
||||
runtime: TradeStreamProductionRuntime,
|
||||
) -> TradeStreamStateStore:
|
||||
return (
|
||||
runtime
|
||||
._reconnect_recovery_coordinator
|
||||
._recovery_coordinator
|
||||
._state_store
|
||||
)
|
||||
|
||||
|
||||
def test_disabled_feature_runs_only_telegram_without_runtime_graph(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
def forbidden_dependency(
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
del args, kwargs
|
||||
raise AssertionError("disabled feature built a dependency")
|
||||
|
||||
monkeypatch.setattr(
|
||||
production_factory,
|
||||
"DzengiWebSocketTransport",
|
||||
forbidden_dependency,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
production_factory,
|
||||
"ExchangeRestClient",
|
||||
forbidden_dependency,
|
||||
)
|
||||
|
||||
runtime = (
|
||||
production_factory.build_trade_stream_production_runtime(
|
||||
make_settings(enabled=False),
|
||||
)
|
||||
)
|
||||
|
||||
async def scenario() -> None:
|
||||
bot = FakeBot()
|
||||
dispatcher = ControlledDispatcher(
|
||||
return_immediately=True,
|
||||
)
|
||||
|
||||
await run_application(
|
||||
make_application(
|
||||
runtime=runtime,
|
||||
dispatcher=dispatcher,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
|
||||
assert dispatcher.close_bot_session_values == [False]
|
||||
assert bot.session.close_calls == 1
|
||||
await assert_no_owned_tasks()
|
||||
|
||||
assert runtime is None
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_production_factory_processes_ack_trade_and_shutdown(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
async def scenario() -> None:
|
||||
calls: list[str] = []
|
||||
connection = ScriptedConnection(
|
||||
name="connection",
|
||||
calls=calls,
|
||||
)
|
||||
connector = ScriptedConnector(
|
||||
calls=calls,
|
||||
connections=(connection,),
|
||||
)
|
||||
transports = install_connector(
|
||||
monkeypatch,
|
||||
connector,
|
||||
)
|
||||
runtime = (
|
||||
production_factory.build_trade_stream_production_runtime(
|
||||
make_settings(),
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(runtime, TradeStreamProductionRuntime)
|
||||
|
||||
bot = FakeBot()
|
||||
dispatcher = ControlledDispatcher()
|
||||
application_task = asyncio.create_task(
|
||||
run_application(
|
||||
make_application(
|
||||
runtime=runtime,
|
||||
dispatcher=dispatcher,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
await connection.subscription_sent.wait()
|
||||
|
||||
timestamp = time.time_ns() // 1_000_000
|
||||
connection.feed(
|
||||
json.dumps(
|
||||
make_trade_document(
|
||||
trade_id=100,
|
||||
timestamp=timestamp,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
state_store = state_store_from(runtime)
|
||||
await wait_until(
|
||||
lambda: (
|
||||
state_store.contains(SYMBOL)
|
||||
and state_store.get(SYMBOL).last_trade_id == 100
|
||||
)
|
||||
)
|
||||
|
||||
dispatcher.release.set()
|
||||
await application_task
|
||||
|
||||
subscription = json.loads(
|
||||
connection.sent_messages[0],
|
||||
)
|
||||
|
||||
assert subscription["destination"] == "trades.subscribe"
|
||||
assert subscription["payload"]["symbols"] == [SYMBOL]
|
||||
assert connector.options[0]["ping_interval"] is None
|
||||
assert connector.options[0]["ping_timeout"] is None
|
||||
assert connection.close_calls == 1
|
||||
assert runtime.state is (
|
||||
TradeStreamProductionRuntimeState.STOPPED
|
||||
)
|
||||
assert runtime._subscription_manager.subscription_keys == ()
|
||||
assert bot.session.close_calls == 1
|
||||
assert len(transports) == 1
|
||||
await assert_no_owned_tasks()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_connection_startup_failure_is_fatal_and_leaves_no_tasks(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
async def scenario() -> None:
|
||||
calls: list[str] = []
|
||||
connector = ScriptedConnector(
|
||||
calls=calls,
|
||||
error=OSError("connect failed"),
|
||||
)
|
||||
install_connector(
|
||||
monkeypatch,
|
||||
connector,
|
||||
)
|
||||
runtime = (
|
||||
production_factory.build_trade_stream_production_runtime(
|
||||
make_settings(),
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(runtime, TradeStreamProductionRuntime)
|
||||
|
||||
bot = FakeBot()
|
||||
dispatcher = ControlledDispatcher()
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketTransportError,
|
||||
match="connect failed",
|
||||
) as exc_info:
|
||||
await run_application(
|
||||
make_application(
|
||||
runtime=runtime,
|
||||
dispatcher=dispatcher,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(exc_info.value.__cause__, OSError)
|
||||
assert dispatcher.cancelled.is_set()
|
||||
assert runtime.state is (
|
||||
TradeStreamProductionRuntimeState.FAILED
|
||||
)
|
||||
assert runtime._session.is_connected is False
|
||||
assert runtime._subscription_manager.subscription_keys == ()
|
||||
assert bot.session.close_calls == 1
|
||||
await assert_no_owned_tasks()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_subscription_startup_failure_rolls_back_concrete_graph(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
async def scenario() -> None:
|
||||
calls: list[str] = []
|
||||
connection = ScriptedConnection(
|
||||
name="connection",
|
||||
calls=calls,
|
||||
send_error=OSError("subscription failed"),
|
||||
)
|
||||
connector = ScriptedConnector(
|
||||
calls=calls,
|
||||
connections=(connection,),
|
||||
)
|
||||
install_connector(
|
||||
monkeypatch,
|
||||
connector,
|
||||
)
|
||||
runtime = (
|
||||
production_factory.build_trade_stream_production_runtime(
|
||||
make_settings(),
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(runtime, TradeStreamProductionRuntime)
|
||||
|
||||
bot = FakeBot()
|
||||
dispatcher = ControlledDispatcher()
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketTransportError,
|
||||
match="subscription failed",
|
||||
):
|
||||
await run_application(
|
||||
make_application(
|
||||
runtime=runtime,
|
||||
dispatcher=dispatcher,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
|
||||
assert dispatcher.cancelled.is_set()
|
||||
assert connection.state is State.CLOSED
|
||||
assert runtime.state is (
|
||||
TradeStreamProductionRuntimeState.FAILED
|
||||
)
|
||||
assert runtime._session.is_connected is False
|
||||
assert runtime._subscription_manager.subscription_keys == ()
|
||||
assert bot.session.close_calls == 1
|
||||
await assert_no_owned_tasks()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_reconnect_restores_recovers_then_processes_buffered_trade(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
async def scenario() -> None:
|
||||
calls: list[str] = []
|
||||
first_connection = ScriptedConnection(
|
||||
name="first",
|
||||
calls=calls,
|
||||
close_on_receive_error=False,
|
||||
)
|
||||
second_connection = ScriptedConnection(
|
||||
name="second",
|
||||
calls=calls,
|
||||
)
|
||||
connector = ScriptedConnector(
|
||||
calls=calls,
|
||||
connections=(
|
||||
first_connection,
|
||||
second_connection,
|
||||
),
|
||||
)
|
||||
install_connector(
|
||||
monkeypatch,
|
||||
connector,
|
||||
)
|
||||
|
||||
live_timestamp = (
|
||||
time.time_ns() // 1_000_000
|
||||
) - 10_000
|
||||
rest_client = BlockingRestClient(
|
||||
calls=calls,
|
||||
document=[
|
||||
make_recovered_trade(
|
||||
trade_id=101,
|
||||
timestamp=live_timestamp + 1,
|
||||
)
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
production_factory,
|
||||
"ExchangeRestClient",
|
||||
lambda settings: rest_client,
|
||||
)
|
||||
|
||||
runtime = (
|
||||
production_factory.build_trade_stream_production_runtime(
|
||||
make_settings(),
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(runtime, TradeStreamProductionRuntime)
|
||||
|
||||
bot = FakeBot()
|
||||
dispatcher = ControlledDispatcher()
|
||||
application_task = asyncio.create_task(
|
||||
run_application(
|
||||
make_application(
|
||||
runtime=runtime,
|
||||
dispatcher=dispatcher,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
await first_connection.subscription_sent.wait()
|
||||
first_connection.feed(
|
||||
json.dumps(
|
||||
make_trade_document(
|
||||
trade_id=100,
|
||||
timestamp=live_timestamp,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
state_store = state_store_from(runtime)
|
||||
await wait_until(
|
||||
lambda: (
|
||||
state_store.contains(SYMBOL)
|
||||
and state_store.get(SYMBOL).last_trade_id == 100
|
||||
)
|
||||
)
|
||||
|
||||
first_connection.feed(
|
||||
OSError("connection dropped"),
|
||||
)
|
||||
|
||||
await wait_until(rest_client.entered.is_set)
|
||||
|
||||
assert second_connection.subscription_sent.is_set()
|
||||
assert state_store.get(SYMBOL).last_trade_id == 100
|
||||
|
||||
second_connection.feed(
|
||||
json.dumps(
|
||||
make_trade_document(
|
||||
trade_id=102,
|
||||
timestamp=live_timestamp + 2,
|
||||
)
|
||||
)
|
||||
)
|
||||
rest_client.release.set()
|
||||
|
||||
await wait_until(
|
||||
lambda: state_store.get(SYMBOL).last_trade_id == 102
|
||||
)
|
||||
|
||||
dispatcher.release.set()
|
||||
await application_task
|
||||
|
||||
state = state_store.get(SYMBOL)
|
||||
|
||||
assert state.last_trade_id == 102
|
||||
assert state.last_trade is not None
|
||||
assert state.last_trade.executed_at.timestamp() == pytest.approx(
|
||||
(live_timestamp + 2) / 1_000,
|
||||
)
|
||||
assert len(connector.options) == 2
|
||||
assert first_connection.close_calls == 1
|
||||
assert second_connection.close_calls == 1
|
||||
assert len(first_connection.sent_messages) == 1
|
||||
assert len(second_connection.sent_messages) == 1
|
||||
assert calls.index("second.send") < calls.index(
|
||||
"recovery.fetch"
|
||||
)
|
||||
assert rest_client.requests[0][0] == "/api/v1/aggTrades"
|
||||
request_params = rest_client.requests[0][1]
|
||||
|
||||
assert request_params is not None
|
||||
assert request_params["symbol"] == SYMBOL
|
||||
assert runtime._reconnect_recovery_coordinator.generation == 1
|
||||
assert runtime.state is (
|
||||
TradeStreamProductionRuntimeState.STOPPED
|
||||
)
|
||||
assert bot.session.close_calls == 1
|
||||
await assert_no_owned_tasks()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_recovery_failure_rejects_buffered_trade_and_cleans_up(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
async def scenario() -> None:
|
||||
calls: list[str] = []
|
||||
first_connection = ScriptedConnection(
|
||||
name="first",
|
||||
calls=calls,
|
||||
close_on_receive_error=False,
|
||||
)
|
||||
second_connection = ScriptedConnection(
|
||||
name="second",
|
||||
calls=calls,
|
||||
)
|
||||
connector = ScriptedConnector(
|
||||
calls=calls,
|
||||
connections=(
|
||||
first_connection,
|
||||
second_connection,
|
||||
),
|
||||
)
|
||||
install_connector(
|
||||
monkeypatch,
|
||||
connector,
|
||||
)
|
||||
|
||||
live_timestamp = (
|
||||
time.time_ns() // 1_000_000
|
||||
) - 10_000
|
||||
rest_client = BlockingRestClient(
|
||||
calls=calls,
|
||||
document=[],
|
||||
error=OSError("recovery failed"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
production_factory,
|
||||
"ExchangeRestClient",
|
||||
lambda settings: rest_client,
|
||||
)
|
||||
|
||||
runtime = (
|
||||
production_factory.build_trade_stream_production_runtime(
|
||||
make_settings(),
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(runtime, TradeStreamProductionRuntime)
|
||||
|
||||
bot = FakeBot()
|
||||
dispatcher = ControlledDispatcher()
|
||||
application_task = asyncio.create_task(
|
||||
run_application(
|
||||
make_application(
|
||||
runtime=runtime,
|
||||
dispatcher=dispatcher,
|
||||
bot=bot,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
await first_connection.subscription_sent.wait()
|
||||
first_connection.feed(
|
||||
json.dumps(
|
||||
make_trade_document(
|
||||
trade_id=100,
|
||||
timestamp=live_timestamp,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
state_store = state_store_from(runtime)
|
||||
await wait_until(
|
||||
lambda: (
|
||||
state_store.contains(SYMBOL)
|
||||
and state_store.get(SYMBOL).last_trade_id == 100
|
||||
)
|
||||
)
|
||||
|
||||
first_connection.feed(
|
||||
OSError("connection dropped"),
|
||||
)
|
||||
await wait_until(rest_client.entered.is_set)
|
||||
|
||||
second_connection.feed(
|
||||
json.dumps(
|
||||
make_trade_document(
|
||||
trade_id=102,
|
||||
timestamp=live_timestamp + 2,
|
||||
)
|
||||
)
|
||||
)
|
||||
rest_client.release.set()
|
||||
|
||||
with pytest.raises(
|
||||
TradeTransportError,
|
||||
match="recovery failed",
|
||||
):
|
||||
await application_task
|
||||
|
||||
assert state_store.get(SYMBOL).last_trade_id == 100
|
||||
assert runtime._live_processing_gate.failed is True
|
||||
assert dispatcher.cancelled.is_set()
|
||||
assert runtime.state is (
|
||||
TradeStreamProductionRuntimeState.FAILED
|
||||
)
|
||||
assert second_connection.close_calls == 1
|
||||
assert bot.session.close_calls == 1
|
||||
await assert_no_owned_tasks()
|
||||
|
||||
asyncio.run(scenario())
|
||||
157
app/tests/unit/bootstrap/test_trade_stream_runtime.py
Normal file
157
app/tests/unit/bootstrap/test_trade_stream_runtime.py
Normal file
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from src.bootstrap.trade_stream_runtime import (
|
||||
build_trade_stream_production_runtime,
|
||||
)
|
||||
from src.core.config import Settings, TradeStreamSettings
|
||||
from src.market_data.acquisition.runtime.trade_stream_production_runtime import (
|
||||
TradeStreamProductionRuntime,
|
||||
TradeStreamProductionRuntimeState,
|
||||
)
|
||||
|
||||
|
||||
def make_settings(
|
||||
*,
|
||||
enabled: bool = True,
|
||||
api_key: str = "api-key",
|
||||
) -> Settings:
|
||||
return Settings(
|
||||
bot_token="test-token",
|
||||
bot_parse_mode="HTML",
|
||||
app_env="test",
|
||||
log_level="INFO",
|
||||
tz="UTC",
|
||||
exchange_enabled=False,
|
||||
exchange_name="dzengi",
|
||||
exchange_base_url="https://rest.example.test/",
|
||||
exchange_ws_url="wss://legacy.example.test",
|
||||
exchange_api_key=api_key,
|
||||
exchange_api_secret="secret",
|
||||
exchange_timeout_sec=17,
|
||||
exchange_testnet=True,
|
||||
default_symbol="LEGACY",
|
||||
trade_stream=TradeStreamSettings(
|
||||
enabled=enabled,
|
||||
websocket_url="wss://stream.example.test/root",
|
||||
symbols=(
|
||||
"ETH/USD_LEVERAGE",
|
||||
"BTC/USD_LEVERAGE",
|
||||
),
|
||||
open_timeout_seconds=11.0,
|
||||
probe_timeout_seconds=21.0,
|
||||
close_timeout_seconds=9.0,
|
||||
heartbeat_timeout_seconds=31.0,
|
||||
scheduler_interval_seconds=6.0,
|
||||
recovery_window_ms=123_456,
|
||||
),
|
||||
db_host="localhost",
|
||||
db_port=5432,
|
||||
db_name="test",
|
||||
db_user="test",
|
||||
db_password="test",
|
||||
debug_enabled=False,
|
||||
journal_debug_enabled=False,
|
||||
)
|
||||
|
||||
|
||||
def test_disabled_feature_does_not_build_runtime() -> None:
|
||||
settings = make_settings(enabled=False)
|
||||
|
||||
assert build_trade_stream_production_runtime(settings) is None
|
||||
|
||||
|
||||
def test_builds_runtime_without_starting_lifecycle() -> None:
|
||||
runtime = build_trade_stream_production_runtime(
|
||||
make_settings(),
|
||||
)
|
||||
|
||||
assert isinstance(runtime, TradeStreamProductionRuntime)
|
||||
assert runtime.state is TradeStreamProductionRuntimeState.STOPPED
|
||||
assert runtime.running is False
|
||||
assert runtime._startup_task is None
|
||||
assert runtime._receive_task is None
|
||||
assert runtime._scheduler_task is None
|
||||
|
||||
|
||||
def test_uses_one_shared_stateful_dependency_graph() -> None:
|
||||
runtime = build_trade_stream_production_runtime(
|
||||
make_settings(),
|
||||
)
|
||||
|
||||
assert isinstance(runtime, TradeStreamProductionRuntime)
|
||||
|
||||
transport = runtime._transport
|
||||
service = runtime._trade_stream_service
|
||||
reconnect_recovery = runtime._reconnect_recovery_coordinator
|
||||
recovery = reconnect_recovery._recovery_coordinator
|
||||
|
||||
assert runtime._session._transport is transport
|
||||
assert runtime._subscription_manager._transport is transport
|
||||
assert runtime._runtime_scheduler.liveness_probe is transport
|
||||
assert (
|
||||
service._consistency_controller
|
||||
is recovery._recovery_controller._consistency_controller
|
||||
)
|
||||
assert runtime._live_processing_gate is (
|
||||
reconnect_recovery.live_processing_gate
|
||||
)
|
||||
assert runtime._runtime_scheduler.runtime_supervisor is (
|
||||
runtime._runtime_supervisor
|
||||
)
|
||||
|
||||
|
||||
def test_applies_explicit_transport_and_runtime_settings() -> None:
|
||||
settings = make_settings()
|
||||
|
||||
runtime = build_trade_stream_production_runtime(settings)
|
||||
|
||||
assert isinstance(runtime, TradeStreamProductionRuntime)
|
||||
transport = runtime._transport
|
||||
|
||||
assert transport._url == "wss://stream.example.test/root/connect"
|
||||
assert transport._headers == {
|
||||
"Origin": "https://rest.example.test",
|
||||
"Content-Type": "application/json",
|
||||
"X-MBX-APIKEY": "api-key",
|
||||
}
|
||||
assert transport._open_timeout == 11.0
|
||||
assert transport._probe_timeout == 21.0
|
||||
assert transport._close_timeout == 9.0
|
||||
assert transport._ping_interval is None
|
||||
assert transport._ping_timeout is None
|
||||
assert runtime._symbols == (
|
||||
"BTC/USD_LEVERAGE",
|
||||
"ETH/USD_LEVERAGE",
|
||||
)
|
||||
assert runtime._runtime_scheduler.interval_seconds == 6.0
|
||||
assert (
|
||||
runtime._runtime_supervisor._heartbeat_monitor.timeout_seconds
|
||||
== 31.0
|
||||
)
|
||||
assert (
|
||||
runtime._reconnect_recovery_coordinator
|
||||
._recovery_coordinator
|
||||
._window_planner
|
||||
.max_window_ms
|
||||
== 123_456
|
||||
)
|
||||
|
||||
|
||||
def test_recovery_rest_client_reuses_settings_snapshot() -> None:
|
||||
settings = make_settings(api_key="")
|
||||
|
||||
runtime = build_trade_stream_production_runtime(settings)
|
||||
|
||||
assert isinstance(runtime, TradeStreamProductionRuntime)
|
||||
document_source = (
|
||||
runtime._reconnect_recovery_coordinator
|
||||
._recovery_coordinator
|
||||
._recovery_controller
|
||||
._document_source
|
||||
)
|
||||
rest_client = document_source._client
|
||||
|
||||
assert rest_client.settings is settings
|
||||
assert rest_client.base_url == "https://rest.example.test"
|
||||
assert rest_client.timeout == 17
|
||||
assert "X-MBX-APIKEY" not in runtime._transport._headers
|
||||
202
app/tests/unit/core/test_config.py
Normal file
202
app/tests/unit/core/test_config.py
Normal file
@@ -0,0 +1,202 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.config import load_settings
|
||||
|
||||
|
||||
_TRADE_STREAM_VARIABLES = (
|
||||
"TRADE_STREAM_ENABLED",
|
||||
"TRADE_STREAM_WS_URL",
|
||||
"TRADE_STREAM_SYMBOLS",
|
||||
"TRADE_STREAM_OPEN_TIMEOUT_SECONDS",
|
||||
"TRADE_STREAM_PROBE_TIMEOUT_SECONDS",
|
||||
"TRADE_STREAM_CLOSE_TIMEOUT_SECONDS",
|
||||
"TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS",
|
||||
"TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS",
|
||||
"TRADE_STREAM_RECOVERY_WINDOW_MS",
|
||||
)
|
||||
|
||||
|
||||
def prepare_environment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("BOT_TOKEN", "test-token")
|
||||
monkeypatch.delenv("EXCHANGE_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("EXCHANGE_ENABLED", raising=False)
|
||||
|
||||
for variable in _TRADE_STREAM_VARIABLES:
|
||||
monkeypatch.delenv(variable, raising=False)
|
||||
|
||||
|
||||
def enable_trade_stream(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("TRADE_STREAM_ENABLED", "true")
|
||||
monkeypatch.setenv(
|
||||
"TRADE_STREAM_WS_URL",
|
||||
"wss://stream.example.test",
|
||||
)
|
||||
monkeypatch.setenv(
|
||||
"TRADE_STREAM_SYMBOLS",
|
||||
"ETH/USD_LEVERAGE,BTC/USD_LEVERAGE",
|
||||
)
|
||||
monkeypatch.setenv(
|
||||
"EXCHANGE_BASE_URL",
|
||||
"https://rest.example.test",
|
||||
)
|
||||
|
||||
|
||||
def test_trade_stream_is_disabled_by_default(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
prepare_environment(monkeypatch)
|
||||
monkeypatch.setenv("EXCHANGE_ENABLED", "true")
|
||||
|
||||
settings = load_settings()
|
||||
|
||||
assert settings.exchange_enabled is True
|
||||
assert settings.trade_stream.enabled is False
|
||||
assert settings.trade_stream.websocket_url == ""
|
||||
assert settings.trade_stream.symbols == ()
|
||||
|
||||
|
||||
def test_disabled_trade_stream_ignores_dependent_values(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
prepare_environment(monkeypatch)
|
||||
monkeypatch.setenv("TRADE_STREAM_OPEN_TIMEOUT_SECONDS", "invalid")
|
||||
monkeypatch.setenv("TRADE_STREAM_SYMBOLS", ",")
|
||||
|
||||
settings = load_settings()
|
||||
|
||||
assert settings.trade_stream.enabled is False
|
||||
|
||||
|
||||
def test_trade_stream_flag_is_strict(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
prepare_environment(monkeypatch)
|
||||
monkeypatch.setenv("TRADE_STREAM_ENABLED", "sometimes")
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="TRADE_STREAM_ENABLED",
|
||||
):
|
||||
load_settings()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("missing_variable", "message"),
|
||||
(
|
||||
(
|
||||
"TRADE_STREAM_WS_URL",
|
||||
"TRADE_STREAM_WS_URL",
|
||||
),
|
||||
(
|
||||
"TRADE_STREAM_SYMBOLS",
|
||||
"TRADE_STREAM_SYMBOLS",
|
||||
),
|
||||
(
|
||||
"EXCHANGE_BASE_URL",
|
||||
"EXCHANGE_BASE_URL",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_enabled_trade_stream_requires_explicit_endpoints_and_symbols(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
missing_variable: str,
|
||||
message: str,
|
||||
) -> None:
|
||||
prepare_environment(monkeypatch)
|
||||
enable_trade_stream(monkeypatch)
|
||||
monkeypatch.setenv(
|
||||
"EXCHANGE_WS_URL",
|
||||
"wss://legacy-fallback.example.test",
|
||||
)
|
||||
monkeypatch.setenv(
|
||||
"DEFAULT_SYMBOL",
|
||||
"LEGACY_FALLBACK",
|
||||
)
|
||||
monkeypatch.delenv(missing_variable)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match=message,
|
||||
):
|
||||
load_settings()
|
||||
|
||||
|
||||
def test_enabled_trade_stream_parses_independent_settings(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
prepare_environment(monkeypatch)
|
||||
enable_trade_stream(monkeypatch)
|
||||
monkeypatch.setenv(
|
||||
"TRADE_STREAM_SYMBOLS",
|
||||
" ETH/USD_LEVERAGE, BTC/USD_LEVERAGE,ETH/USD_LEVERAGE ",
|
||||
)
|
||||
monkeypatch.setenv("TRADE_STREAM_OPEN_TIMEOUT_SECONDS", "11.5")
|
||||
monkeypatch.setenv("TRADE_STREAM_PROBE_TIMEOUT_SECONDS", "21")
|
||||
monkeypatch.setenv("TRADE_STREAM_CLOSE_TIMEOUT_SECONDS", "9")
|
||||
monkeypatch.setenv("TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS", "31")
|
||||
monkeypatch.setenv("TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS", "6")
|
||||
monkeypatch.setenv("TRADE_STREAM_RECOVERY_WINDOW_MS", "123456")
|
||||
|
||||
settings = load_settings()
|
||||
trade_stream = settings.trade_stream
|
||||
|
||||
assert trade_stream.enabled is True
|
||||
assert trade_stream.websocket_url == "wss://stream.example.test"
|
||||
assert trade_stream.symbols == (
|
||||
"BTC/USD_LEVERAGE",
|
||||
"ETH/USD_LEVERAGE",
|
||||
)
|
||||
assert trade_stream.open_timeout_seconds == 11.5
|
||||
assert trade_stream.probe_timeout_seconds == 21.0
|
||||
assert trade_stream.close_timeout_seconds == 9.0
|
||||
assert trade_stream.heartbeat_timeout_seconds == 31.0
|
||||
assert trade_stream.scheduler_interval_seconds == 6.0
|
||||
assert trade_stream.recovery_window_ms == 123_456
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("variable", "value"),
|
||||
(
|
||||
("TRADE_STREAM_OPEN_TIMEOUT_SECONDS", "0"),
|
||||
("TRADE_STREAM_PROBE_TIMEOUT_SECONDS", "-1"),
|
||||
("TRADE_STREAM_CLOSE_TIMEOUT_SECONDS", "nan"),
|
||||
("TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS", "inf"),
|
||||
("TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS", "invalid"),
|
||||
("TRADE_STREAM_RECOVERY_WINDOW_MS", "1.5"),
|
||||
("TRADE_STREAM_RECOVERY_WINDOW_MS", "0"),
|
||||
),
|
||||
)
|
||||
def test_enabled_trade_stream_rejects_invalid_numeric_settings(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
variable: str,
|
||||
value: str,
|
||||
) -> None:
|
||||
prepare_environment(monkeypatch)
|
||||
enable_trade_stream(monkeypatch)
|
||||
monkeypatch.setenv(variable, value)
|
||||
|
||||
with pytest.raises(ValueError, match=variable):
|
||||
load_settings()
|
||||
|
||||
|
||||
def test_symbols_must_not_contain_empty_items(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
prepare_environment(monkeypatch)
|
||||
enable_trade_stream(monkeypatch)
|
||||
monkeypatch.setenv(
|
||||
"TRADE_STREAM_SYMBOLS",
|
||||
"BTC/USD_LEVERAGE,,ETH/USD_LEVERAGE",
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="empty symbols",
|
||||
):
|
||||
load_settings()
|
||||
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from src.market_data.acquisition.adapters.dzengi.websocket_control_message_handler import (
|
||||
DzengiWebSocketControlMessageHandler,
|
||||
)
|
||||
from src.market_data.acquisition.exceptions import (
|
||||
WebSocketControlMessageError,
|
||||
WebSocketMessageRoutingError,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_inbound_message import (
|
||||
WebSocketControlMessageHandlerProtocol,
|
||||
)
|
||||
|
||||
|
||||
CORRELATION_ID = "trade-subscription-1"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def handler() -> DzengiWebSocketControlMessageHandler:
|
||||
return DzengiWebSocketControlMessageHandler()
|
||||
|
||||
|
||||
def test_implements_public_protocol_and_uses_slots(
|
||||
handler: DzengiWebSocketControlMessageHandler,
|
||||
) -> None:
|
||||
assert isinstance(
|
||||
handler,
|
||||
WebSocketControlMessageHandlerProtocol,
|
||||
)
|
||||
assert not hasattr(handler, "__dict__")
|
||||
|
||||
|
||||
def test_accepts_matching_successful_subscription_ack(
|
||||
handler: DzengiWebSocketControlMessageHandler,
|
||||
) -> None:
|
||||
handler.handle(
|
||||
{
|
||||
"correlationId": CORRELATION_ID,
|
||||
"destination": "trades.subscribe",
|
||||
"status": "OK",
|
||||
},
|
||||
expected_correlation_id=CORRELATION_ID,
|
||||
)
|
||||
|
||||
|
||||
def test_rejects_negative_subscription_ack(
|
||||
handler: DzengiWebSocketControlMessageHandler,
|
||||
) -> None:
|
||||
with pytest.raises(
|
||||
WebSocketControlMessageError,
|
||||
match="отклонил",
|
||||
):
|
||||
handler.handle(
|
||||
{
|
||||
"correlationId": CORRELATION_ID,
|
||||
"status": "ERROR",
|
||||
"payload": {
|
||||
"errorCode": "BAD_REQUEST",
|
||||
},
|
||||
},
|
||||
expected_correlation_id=CORRELATION_ID,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"document",
|
||||
[
|
||||
None,
|
||||
{},
|
||||
{
|
||||
"correlationId": "unknown",
|
||||
"destination": "trades.subscribe",
|
||||
"status": "OK",
|
||||
},
|
||||
{
|
||||
"correlationId": CORRELATION_ID,
|
||||
"destination": "unknown",
|
||||
"status": "OK",
|
||||
},
|
||||
{
|
||||
"correlationId": CORRELATION_ID,
|
||||
"destination": "trades.subscribe",
|
||||
},
|
||||
{
|
||||
"correlationId": CORRELATION_ID,
|
||||
"destination": "trades.subscribe",
|
||||
"status": "",
|
||||
},
|
||||
{
|
||||
"correlationId": CORRELATION_ID,
|
||||
"destination": "trades.subscribe",
|
||||
"status": None,
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_rejects_unknown_or_malformed_control_messages(
|
||||
handler: DzengiWebSocketControlMessageHandler,
|
||||
document: object,
|
||||
) -> None:
|
||||
with pytest.raises(WebSocketMessageRoutingError):
|
||||
handler.handle(
|
||||
document,
|
||||
expected_correlation_id=CORRELATION_ID,
|
||||
)
|
||||
@@ -0,0 +1,137 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from src.market_data.acquisition.adapters.dzengi.websocket_inbound_message_classifier import (
|
||||
DzengiWebSocketInboundMessageClassifier,
|
||||
)
|
||||
from src.market_data.acquisition.exceptions import (
|
||||
WebSocketMessageRoutingError,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_inbound_message import (
|
||||
WebSocketInboundMessageClassifierProtocol,
|
||||
WebSocketInboundMessageKind,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def classifier() -> DzengiWebSocketInboundMessageClassifier:
|
||||
return DzengiWebSocketInboundMessageClassifier()
|
||||
|
||||
|
||||
def test_implements_public_protocol(
|
||||
classifier: DzengiWebSocketInboundMessageClassifier,
|
||||
) -> None:
|
||||
assert isinstance(
|
||||
classifier,
|
||||
WebSocketInboundMessageClassifierProtocol,
|
||||
)
|
||||
|
||||
|
||||
def test_uses_slots(
|
||||
classifier: DzengiWebSocketInboundMessageClassifier,
|
||||
) -> None:
|
||||
assert not hasattr(
|
||||
classifier,
|
||||
"__dict__",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"document",
|
||||
[
|
||||
{
|
||||
"destination": "internal.trade",
|
||||
},
|
||||
{
|
||||
"destination": "ohlc.event",
|
||||
},
|
||||
{
|
||||
"Payload": {},
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_recognizes_market_documents(
|
||||
classifier: DzengiWebSocketInboundMessageClassifier,
|
||||
document: object,
|
||||
) -> None:
|
||||
assert (
|
||||
classifier.classify(document)
|
||||
is WebSocketInboundMessageKind.MARKET
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"correlation_id",
|
||||
[
|
||||
"request-1",
|
||||
1,
|
||||
],
|
||||
)
|
||||
def test_recognizes_control_responses(
|
||||
classifier: DzengiWebSocketInboundMessageClassifier,
|
||||
correlation_id: str | int,
|
||||
) -> None:
|
||||
assert (
|
||||
classifier.classify(
|
||||
{
|
||||
"correlationId": correlation_id,
|
||||
"destination": "trades.subscribe",
|
||||
}
|
||||
)
|
||||
is WebSocketInboundMessageKind.CONTROL
|
||||
)
|
||||
|
||||
|
||||
def test_market_markers_take_priority_over_correlation_id(
|
||||
classifier: DzengiWebSocketInboundMessageClassifier,
|
||||
) -> None:
|
||||
assert (
|
||||
classifier.classify(
|
||||
{
|
||||
"destination": "internal.trade",
|
||||
"correlationId": "request-1",
|
||||
}
|
||||
)
|
||||
is WebSocketInboundMessageKind.MARKET
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"document",
|
||||
[
|
||||
None,
|
||||
[],
|
||||
"message",
|
||||
{},
|
||||
{
|
||||
"destination": "unknown",
|
||||
},
|
||||
{
|
||||
"destination": [],
|
||||
},
|
||||
{
|
||||
"correlationId": "",
|
||||
},
|
||||
{
|
||||
"correlationId": " ",
|
||||
},
|
||||
{
|
||||
"correlationId": None,
|
||||
},
|
||||
{
|
||||
"correlationId": True,
|
||||
},
|
||||
{
|
||||
"correlationId": 1.5,
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_or_unknown_documents(
|
||||
classifier: DzengiWebSocketInboundMessageClassifier,
|
||||
document: object,
|
||||
) -> None:
|
||||
with pytest.raises(
|
||||
WebSocketMessageRoutingError,
|
||||
):
|
||||
classifier.classify(document)
|
||||
@@ -0,0 +1,573 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from websockets.protocol import State
|
||||
|
||||
from src.market_data.acquisition.adapters.dzengi.websocket_transport import (
|
||||
DzengiWebSocketTransport,
|
||||
build_dzengi_websocket_url,
|
||||
)
|
||||
from src.market_data.acquisition.exceptions import (
|
||||
WebSocketTransportError,
|
||||
WebSocketTransportNotConnectedError,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_protocol import (
|
||||
WebSocketTransportProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_liveness_probe import (
|
||||
RuntimeLivenessProbeProtocol,
|
||||
)
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
incoming: tuple[str | bytes, ...] = (),
|
||||
state: State = State.OPEN,
|
||||
) -> None:
|
||||
self.state = state
|
||||
self.incoming = list(incoming)
|
||||
self.sent_messages: list[str | bytes] = []
|
||||
self.close_calls = 0
|
||||
self.send_error: Exception | None = None
|
||||
self.receive_error: Exception | None = None
|
||||
self.ping_error: Exception | None = None
|
||||
self.pong_error: Exception | None = None
|
||||
self.pong_gate: asyncio.Event | None = None
|
||||
self.ping_calls = 0
|
||||
self.close_error: Exception | None = None
|
||||
|
||||
async def close(
|
||||
self,
|
||||
code: int = 1000,
|
||||
reason: str = "",
|
||||
) -> None:
|
||||
self.close_calls += 1
|
||||
self.state = State.CLOSED
|
||||
|
||||
if self.close_error is not None:
|
||||
raise self.close_error
|
||||
|
||||
async def send(
|
||||
self,
|
||||
message: str | bytes,
|
||||
) -> None:
|
||||
if self.send_error is not None:
|
||||
self.state = State.CLOSED
|
||||
raise self.send_error
|
||||
|
||||
self.sent_messages.append(message)
|
||||
|
||||
async def recv(self) -> str | bytes:
|
||||
if self.receive_error is not None:
|
||||
self.state = State.CLOSED
|
||||
raise self.receive_error
|
||||
|
||||
return self.incoming.pop(0)
|
||||
|
||||
async def ping(self) -> Awaitable[float]:
|
||||
self.ping_calls += 1
|
||||
|
||||
if self.ping_error is not None:
|
||||
raise self.ping_error
|
||||
|
||||
async def wait_for_pong() -> float:
|
||||
if self.pong_gate is not None:
|
||||
await self.pong_gate.wait()
|
||||
|
||||
if self.pong_error is not None:
|
||||
raise self.pong_error
|
||||
|
||||
return 0.01
|
||||
|
||||
return wait_for_pong()
|
||||
|
||||
|
||||
class RecordingConnector:
|
||||
def __init__(
|
||||
self,
|
||||
*connections: FakeConnection,
|
||||
) -> None:
|
||||
self.connections = list(connections)
|
||||
self.calls: list[
|
||||
tuple[str, dict[str, Any]]
|
||||
] = []
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
url: str,
|
||||
**kwargs: Any,
|
||||
) -> FakeConnection:
|
||||
self.calls.append(
|
||||
(
|
||||
url,
|
||||
kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
return self.connections.pop(0)
|
||||
|
||||
|
||||
def create_transport(
|
||||
connection: FakeConnection | None = None,
|
||||
) -> tuple[
|
||||
DzengiWebSocketTransport,
|
||||
FakeConnection,
|
||||
RecordingConnector,
|
||||
]:
|
||||
resolved_connection = connection or FakeConnection()
|
||||
connector = RecordingConnector(
|
||||
resolved_connection,
|
||||
)
|
||||
transport = DzengiWebSocketTransport(
|
||||
url="https://api-adapter.dzengi.com",
|
||||
headers={
|
||||
"Origin": "https://api-adapter.dzengi.com",
|
||||
"X-MBX-APIKEY": "test-key",
|
||||
},
|
||||
open_timeout=11,
|
||||
ping_interval=12,
|
||||
ping_timeout=13,
|
||||
close_timeout=14,
|
||||
connector=connector,
|
||||
)
|
||||
|
||||
return (
|
||||
transport,
|
||||
resolved_connection,
|
||||
connector,
|
||||
)
|
||||
|
||||
|
||||
def test_transport_implements_protocol() -> None:
|
||||
transport, *_ = create_transport()
|
||||
|
||||
assert isinstance(
|
||||
transport,
|
||||
WebSocketTransportProtocol,
|
||||
)
|
||||
assert isinstance(
|
||||
transport,
|
||||
RuntimeLivenessProbeProtocol,
|
||||
)
|
||||
|
||||
|
||||
def test_transport_uses_slots() -> None:
|
||||
transport, *_ = create_transport()
|
||||
|
||||
assert not hasattr(transport, "__dict__")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw_url", "expected"),
|
||||
[
|
||||
(
|
||||
"https://api-adapter.dzengi.com",
|
||||
"wss://api-adapter.dzengi.com/connect",
|
||||
),
|
||||
(
|
||||
"http://localhost:8080/",
|
||||
"ws://localhost:8080/connect",
|
||||
),
|
||||
(
|
||||
"wss://api-adapter.dzengi.com/connect/",
|
||||
"wss://api-adapter.dzengi.com/connect",
|
||||
),
|
||||
(
|
||||
"ws://localhost:8080/custom?token=abc",
|
||||
"ws://localhost:8080/custom/connect?token=abc",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_build_url_normalizes_supported_urls(
|
||||
raw_url: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
assert build_dzengi_websocket_url(raw_url) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_url",
|
||||
[
|
||||
"",
|
||||
" ",
|
||||
"ftp://api-adapter.dzengi.com",
|
||||
"api-adapter.dzengi.com",
|
||||
"wss:///connect",
|
||||
"wss://api-adapter.dzengi.com/#fragment",
|
||||
],
|
||||
)
|
||||
def test_build_url_rejects_invalid_values(
|
||||
raw_url: str,
|
||||
) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
build_dzengi_websocket_url(raw_url)
|
||||
|
||||
|
||||
def test_build_url_rejects_non_string() -> None:
|
||||
with pytest.raises(TypeError):
|
||||
build_dzengi_websocket_url(123) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_connect_forwards_connection_options() -> None:
|
||||
transport, _, connector = create_transport()
|
||||
|
||||
asyncio.run(transport.connect())
|
||||
|
||||
assert transport.is_connected is True
|
||||
assert len(connector.calls) == 1
|
||||
|
||||
url, options = connector.calls[0]
|
||||
|
||||
assert url == "wss://api-adapter.dzengi.com/connect"
|
||||
assert options["additional_headers"] == {
|
||||
"Origin": "https://api-adapter.dzengi.com",
|
||||
"X-MBX-APIKEY": "test-key",
|
||||
}
|
||||
assert tuple(options["subprotocols"]) == ("json",)
|
||||
assert options["open_timeout"] == 11.0
|
||||
assert options["ping_interval"] == 12.0
|
||||
assert options["ping_timeout"] == 13.0
|
||||
assert options["close_timeout"] == 14.0
|
||||
|
||||
|
||||
def test_connect_is_idempotent() -> None:
|
||||
transport, _, connector = create_transport()
|
||||
|
||||
async def scenario() -> None:
|
||||
await transport.connect()
|
||||
await transport.connect()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert len(connector.calls) == 1
|
||||
|
||||
|
||||
def test_connect_wraps_connector_error() -> None:
|
||||
async def broken_connector(
|
||||
url: str,
|
||||
**kwargs: Any,
|
||||
) -> FakeConnection:
|
||||
raise OSError("connection failed")
|
||||
|
||||
transport = DzengiWebSocketTransport(
|
||||
url="wss://api-adapter.dzengi.com/connect",
|
||||
connector=broken_connector,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketTransportError,
|
||||
match="connection failed",
|
||||
) as exc_info:
|
||||
asyncio.run(transport.connect())
|
||||
|
||||
assert isinstance(exc_info.value.__cause__, OSError)
|
||||
assert transport.is_connected is False
|
||||
|
||||
|
||||
def test_connect_rejects_non_open_connection() -> None:
|
||||
connection = FakeConnection(
|
||||
state=State.CLOSED,
|
||||
)
|
||||
transport, _, _ = create_transport(connection)
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketTransportError,
|
||||
match="состояние OPEN",
|
||||
):
|
||||
asyncio.run(transport.connect())
|
||||
|
||||
assert connection.close_calls == 1
|
||||
assert transport.is_connected is False
|
||||
|
||||
|
||||
def test_disconnect_closes_connection_and_is_idempotent() -> None:
|
||||
transport, connection, _ = create_transport()
|
||||
|
||||
async def scenario() -> None:
|
||||
await transport.connect()
|
||||
await transport.disconnect()
|
||||
await transport.disconnect()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert connection.close_calls == 1
|
||||
assert transport.is_connected is False
|
||||
|
||||
|
||||
def test_disconnect_clears_connection_after_close_error() -> None:
|
||||
connection = FakeConnection()
|
||||
connection.close_error = RuntimeError("close failed")
|
||||
transport, _, _ = create_transport(connection)
|
||||
|
||||
async def scenario() -> None:
|
||||
await transport.connect()
|
||||
await transport.disconnect()
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketTransportError,
|
||||
match="close failed",
|
||||
):
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert transport.is_connected is False
|
||||
|
||||
|
||||
def test_send_forwards_text_and_binary_messages() -> None:
|
||||
transport, connection, _ = create_transport()
|
||||
|
||||
async def scenario() -> None:
|
||||
await transport.connect()
|
||||
await transport.send("text")
|
||||
await transport.send(b"binary")
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert connection.sent_messages == [
|
||||
"text",
|
||||
b"binary",
|
||||
]
|
||||
|
||||
|
||||
def test_send_requires_open_connection() -> None:
|
||||
transport, _, _ = create_transport()
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketTransportNotConnectedError,
|
||||
):
|
||||
asyncio.run(transport.send("message"))
|
||||
|
||||
|
||||
def test_send_error_is_wrapped_and_discards_closed_connection() -> None:
|
||||
connection = FakeConnection()
|
||||
connection.send_error = RuntimeError("send failed")
|
||||
transport, _, _ = create_transport(connection)
|
||||
|
||||
async def scenario() -> None:
|
||||
await transport.connect()
|
||||
await transport.send("message")
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketTransportError,
|
||||
match="send failed",
|
||||
):
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert transport.is_connected is False
|
||||
|
||||
|
||||
def test_receive_returns_text_and_binary_messages() -> None:
|
||||
connection = FakeConnection(
|
||||
incoming=(
|
||||
"text",
|
||||
b"binary",
|
||||
),
|
||||
)
|
||||
transport, _, _ = create_transport(connection)
|
||||
|
||||
async def scenario() -> tuple[str | bytes, str | bytes]:
|
||||
await transport.connect()
|
||||
|
||||
return (
|
||||
await transport.receive(),
|
||||
await transport.receive(),
|
||||
)
|
||||
|
||||
assert asyncio.run(scenario()) == (
|
||||
"text",
|
||||
b"binary",
|
||||
)
|
||||
|
||||
|
||||
def test_receive_requires_open_connection() -> None:
|
||||
transport, _, _ = create_transport()
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketTransportNotConnectedError,
|
||||
):
|
||||
asyncio.run(transport.receive())
|
||||
|
||||
|
||||
def test_receive_error_is_wrapped_and_discards_closed_connection() -> None:
|
||||
connection = FakeConnection()
|
||||
connection.receive_error = RuntimeError("receive failed")
|
||||
transport, _, _ = create_transport(connection)
|
||||
|
||||
async def scenario() -> None:
|
||||
await transport.connect()
|
||||
await transport.receive()
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketTransportError,
|
||||
match="receive failed",
|
||||
):
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert transport.is_connected is False
|
||||
|
||||
|
||||
def test_probe_returns_true_after_pong() -> None:
|
||||
transport, connection, _ = create_transport()
|
||||
|
||||
async def scenario() -> bool:
|
||||
await transport.connect()
|
||||
return await transport.probe()
|
||||
|
||||
assert asyncio.run(scenario()) is True
|
||||
assert connection.ping_calls == 1
|
||||
|
||||
|
||||
def test_probe_returns_false_without_open_connection() -> None:
|
||||
transport, connection, _ = create_transport()
|
||||
|
||||
assert asyncio.run(transport.probe()) is False
|
||||
assert connection.ping_calls == 0
|
||||
|
||||
|
||||
def test_probe_returns_false_after_pong_timeout() -> None:
|
||||
connection = FakeConnection()
|
||||
connection.pong_gate = asyncio.Event()
|
||||
connector = RecordingConnector(connection)
|
||||
transport = DzengiWebSocketTransport(
|
||||
url="wss://api-adapter.dzengi.com",
|
||||
ping_timeout=0.001,
|
||||
connector=connector,
|
||||
)
|
||||
|
||||
async def scenario() -> bool:
|
||||
await transport.connect()
|
||||
return await transport.probe()
|
||||
|
||||
assert asyncio.run(scenario()) is False
|
||||
assert connection.ping_calls == 1
|
||||
assert transport.is_connected is True
|
||||
|
||||
|
||||
def test_probe_timeout_remains_finite_when_ping_timeout_is_disabled() -> None:
|
||||
connection = FakeConnection()
|
||||
connection.pong_gate = asyncio.Event()
|
||||
connector = RecordingConnector(connection)
|
||||
transport = DzengiWebSocketTransport(
|
||||
url="wss://api-adapter.dzengi.com",
|
||||
ping_timeout=None,
|
||||
probe_timeout=0.001,
|
||||
connector=connector,
|
||||
)
|
||||
|
||||
async def scenario() -> bool:
|
||||
await transport.connect()
|
||||
return await transport.probe()
|
||||
|
||||
assert asyncio.run(scenario()) is False
|
||||
assert connection.ping_calls == 1
|
||||
assert connector.calls[0][1]["ping_timeout"] is None
|
||||
|
||||
|
||||
def test_probe_returns_false_when_connection_closes() -> None:
|
||||
connection = FakeConnection()
|
||||
connection.pong_error = RuntimeError("connection closed")
|
||||
transport, _, _ = create_transport(connection)
|
||||
|
||||
async def scenario() -> bool:
|
||||
await transport.connect()
|
||||
connection.state = State.CLOSED
|
||||
return await transport.probe()
|
||||
|
||||
assert asyncio.run(scenario()) is False
|
||||
assert transport.is_connected is False
|
||||
|
||||
|
||||
def test_probe_wraps_unexpected_ping_error_on_open_connection() -> None:
|
||||
connection = FakeConnection()
|
||||
connection.ping_error = RuntimeError("ping failed")
|
||||
transport, _, _ = create_transport(connection)
|
||||
|
||||
async def scenario() -> None:
|
||||
await transport.connect()
|
||||
await transport.probe()
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketTransportError,
|
||||
match="ping failed",
|
||||
) as error_info:
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert isinstance(error_info.value.__cause__, RuntimeError)
|
||||
assert transport.is_connected is True
|
||||
|
||||
|
||||
def test_probe_cancellation_is_not_swallowed() -> None:
|
||||
async def scenario() -> DzengiWebSocketTransport:
|
||||
connection = FakeConnection()
|
||||
connection.pong_gate = asyncio.Event()
|
||||
transport, _, _ = create_transport(connection)
|
||||
await transport.connect()
|
||||
|
||||
probe_task = asyncio.create_task(
|
||||
transport.probe(),
|
||||
)
|
||||
|
||||
while connection.ping_calls == 0:
|
||||
await asyncio.sleep(0)
|
||||
|
||||
probe_task.cancel()
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await probe_task
|
||||
|
||||
return transport
|
||||
|
||||
assert asyncio.run(scenario()).is_connected is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "value"),
|
||||
[
|
||||
("open_timeout", 0),
|
||||
("ping_interval", -1),
|
||||
("ping_timeout", True),
|
||||
("probe_timeout", float("inf")),
|
||||
("close_timeout", "10"),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_timeout(
|
||||
name: str,
|
||||
value: object,
|
||||
) -> None:
|
||||
options = {
|
||||
name: value,
|
||||
}
|
||||
|
||||
with pytest.raises(
|
||||
(TypeError, ValueError),
|
||||
):
|
||||
DzengiWebSocketTransport(
|
||||
url="wss://api-adapter.dzengi.com",
|
||||
**options, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
|
||||
def test_copies_headers_from_caller() -> None:
|
||||
headers = {
|
||||
"Origin": "https://api-adapter.dzengi.com",
|
||||
}
|
||||
connection = FakeConnection()
|
||||
connector = RecordingConnector(connection)
|
||||
transport = DzengiWebSocketTransport(
|
||||
url="wss://api-adapter.dzengi.com",
|
||||
headers=headers,
|
||||
connector=connector,
|
||||
)
|
||||
|
||||
headers["Origin"] = "changed"
|
||||
|
||||
asyncio.run(transport.connect())
|
||||
|
||||
assert connector.calls[0][1]["additional_headers"] == {
|
||||
"Origin": "https://api-adapter.dzengi.com",
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from src.market_data.acquisition.runtime.acquisition_runtime_event_logging_consumer import (
|
||||
AcquisitionRuntimeEventLoggingConsumer,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.acquisition_runtime_event_publisher import (
|
||||
AcquisitionRuntimeEventConsumerProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_events import (
|
||||
ConnectedEvent,
|
||||
ConnectFailedEvent,
|
||||
DisconnectedEvent,
|
||||
HeartbeatTimeoutEvent,
|
||||
MessageReceivedEvent,
|
||||
MessageSentEvent,
|
||||
ReconnectCompletedEvent,
|
||||
ReconnectFailedEvent,
|
||||
ReconnectStartedEvent,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.transport_messages import (
|
||||
TransportBinaryMessage,
|
||||
TransportTextMessage,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_protocol import (
|
||||
AcquisitionRuntimeEvent,
|
||||
)
|
||||
|
||||
|
||||
TEST_LOGGER_NAME = "tests.acquisition_runtime_events"
|
||||
|
||||
|
||||
def create_consumer() -> AcquisitionRuntimeEventLoggingConsumer:
|
||||
return AcquisitionRuntimeEventLoggingConsumer(
|
||||
event_logger=logging.getLogger(
|
||||
TEST_LOGGER_NAME,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_logging_consumer_satisfies_protocol() -> None:
|
||||
assert isinstance(
|
||||
create_consumer(),
|
||||
AcquisitionRuntimeEventConsumerProtocol,
|
||||
)
|
||||
|
||||
|
||||
def test_logging_consumer_uses_slots() -> None:
|
||||
consumer = create_consumer()
|
||||
|
||||
assert not hasattr(consumer, "__dict__")
|
||||
|
||||
|
||||
def test_invalid_logger_is_rejected() -> None:
|
||||
with pytest.raises(
|
||||
TypeError,
|
||||
match="logging.Logger",
|
||||
):
|
||||
AcquisitionRuntimeEventLoggingConsumer(
|
||||
event_logger=object(), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"event",
|
||||
"expected_level",
|
||||
"expected_message",
|
||||
),
|
||||
(
|
||||
(
|
||||
ConnectedEvent(),
|
||||
logging.INFO,
|
||||
"connected",
|
||||
),
|
||||
(
|
||||
DisconnectedEvent(),
|
||||
logging.INFO,
|
||||
"disconnected",
|
||||
),
|
||||
(
|
||||
ConnectFailedEvent(
|
||||
reason="connection refused",
|
||||
),
|
||||
logging.ERROR,
|
||||
"connection refused",
|
||||
),
|
||||
(
|
||||
ReconnectStartedEvent(
|
||||
attempt=1,
|
||||
),
|
||||
logging.INFO,
|
||||
"attempt=1",
|
||||
),
|
||||
(
|
||||
ReconnectCompletedEvent(
|
||||
attempt=2,
|
||||
),
|
||||
logging.INFO,
|
||||
"attempt=2",
|
||||
),
|
||||
(
|
||||
ReconnectFailedEvent(
|
||||
attempt=3,
|
||||
reason="timeout",
|
||||
),
|
||||
logging.ERROR,
|
||||
"attempt=3 reason=timeout",
|
||||
),
|
||||
(
|
||||
HeartbeatTimeoutEvent(
|
||||
timeout_seconds=30.0,
|
||||
),
|
||||
logging.WARNING,
|
||||
"timeout_seconds=30.0",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_lifecycle_event_uses_expected_log_level(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
expected_level: int,
|
||||
expected_message: str,
|
||||
) -> None:
|
||||
consumer = create_consumer()
|
||||
|
||||
caplog.set_level(
|
||||
logging.DEBUG,
|
||||
logger=TEST_LOGGER_NAME,
|
||||
)
|
||||
|
||||
asyncio.run(
|
||||
consumer.consume(event)
|
||||
)
|
||||
|
||||
assert len(caplog.records) == 1
|
||||
assert caplog.records[0].levelno == expected_level
|
||||
assert expected_message in caplog.records[0].getMessage()
|
||||
|
||||
|
||||
def test_message_events_log_metadata_without_payload(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
consumer = create_consumer()
|
||||
text_payload = "secret-subscription-payload"
|
||||
binary_payload = b"secret-binary-payload"
|
||||
|
||||
caplog.set_level(
|
||||
logging.DEBUG,
|
||||
logger=TEST_LOGGER_NAME,
|
||||
)
|
||||
|
||||
async def consume_messages() -> None:
|
||||
await consumer.consume(
|
||||
MessageReceivedEvent(
|
||||
message=TransportTextMessage(
|
||||
payload=text_payload,
|
||||
),
|
||||
)
|
||||
)
|
||||
await consumer.consume(
|
||||
MessageSentEvent(
|
||||
message=TransportBinaryMessage(
|
||||
payload=binary_payload,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
asyncio.run(consume_messages())
|
||||
|
||||
assert len(caplog.records) == 2
|
||||
assert (
|
||||
"message_type=TransportTextMessage "
|
||||
f"payload_size={len(text_payload)}"
|
||||
in caplog.records[0].getMessage()
|
||||
)
|
||||
assert (
|
||||
"message_type=TransportBinaryMessage "
|
||||
f"payload_size={len(binary_payload)}"
|
||||
in caplog.records[1].getMessage()
|
||||
)
|
||||
assert text_payload not in caplog.text
|
||||
assert binary_payload.decode() not in caplog.text
|
||||
|
||||
|
||||
def test_invalid_event_is_rejected() -> None:
|
||||
consumer = create_consumer()
|
||||
|
||||
with pytest.raises(
|
||||
TypeError,
|
||||
match="AcquisitionRuntimeEvent",
|
||||
):
|
||||
asyncio.run(
|
||||
consumer.consume(
|
||||
object(), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,679 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from src.market_data.acquisition.runtime.acquisition_runtime_event_publisher import (
|
||||
AcquisitionRuntimeEventConsumerProtocol,
|
||||
AcquisitionRuntimeEventPublisher,
|
||||
logger as publisher_logger,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.heartbeat import (
|
||||
HeartbeatMonitor,
|
||||
HeartbeatState,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.reconnect import (
|
||||
ReconnectCoordinator,
|
||||
ReconnectState,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_commands import (
|
||||
ConnectCommand,
|
||||
DisconnectCommand,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_events import (
|
||||
ConnectedEvent,
|
||||
DisconnectedEvent,
|
||||
MessageReceivedEvent,
|
||||
ReconnectStartedEvent,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.transport_messages import (
|
||||
TransportTextMessage,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_protocol import (
|
||||
AcquisitionRuntimeCommand,
|
||||
AcquisitionRuntimeEvent,
|
||||
AcquisitionRuntimeEventPublisherProtocol,
|
||||
AcquisitionSubscriptionMessage,
|
||||
)
|
||||
|
||||
|
||||
PUBLISHER_LOGGER_NAME = (
|
||||
"src.market_data.acquisition.runtime."
|
||||
"acquisition_runtime_event_publisher"
|
||||
)
|
||||
|
||||
|
||||
class RecordingConsumer:
|
||||
def __init__(self) -> None:
|
||||
self.events: list[AcquisitionRuntimeEvent] = []
|
||||
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
self.events.append(event)
|
||||
|
||||
|
||||
def test_publisher_satisfies_protocol() -> None:
|
||||
publisher = AcquisitionRuntimeEventPublisher()
|
||||
|
||||
assert isinstance(
|
||||
publisher,
|
||||
AcquisitionRuntimeEventPublisherProtocol,
|
||||
)
|
||||
|
||||
|
||||
def test_consumer_satisfies_protocol() -> None:
|
||||
assert isinstance(
|
||||
RecordingConsumer(),
|
||||
AcquisitionRuntimeEventConsumerProtocol,
|
||||
)
|
||||
|
||||
|
||||
def test_publisher_uses_slots() -> None:
|
||||
publisher = AcquisitionRuntimeEventPublisher()
|
||||
|
||||
assert not hasattr(publisher, "__dict__")
|
||||
|
||||
|
||||
def test_empty_publisher_accepts_event() -> None:
|
||||
publisher = AcquisitionRuntimeEventPublisher()
|
||||
|
||||
asyncio.run(
|
||||
publisher.publish(
|
||||
ConnectedEvent(),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_constructor_copies_consumer_collection() -> None:
|
||||
first_consumer = RecordingConsumer()
|
||||
consumers = [first_consumer]
|
||||
publisher = AcquisitionRuntimeEventPublisher(consumers)
|
||||
|
||||
second_consumer = RecordingConsumer()
|
||||
consumers.append(second_consumer)
|
||||
|
||||
event = ConnectedEvent()
|
||||
asyncio.run(publisher.publish(event))
|
||||
|
||||
assert first_consumer.events == [event]
|
||||
assert second_consumer.events == []
|
||||
|
||||
|
||||
def test_invalid_consumer_is_rejected() -> None:
|
||||
with pytest.raises(
|
||||
TypeError,
|
||||
match="AcquisitionRuntimeEventConsumerProtocol",
|
||||
):
|
||||
AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
object(), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_event_is_rejected() -> None:
|
||||
publisher = AcquisitionRuntimeEventPublisher()
|
||||
|
||||
with pytest.raises(
|
||||
TypeError,
|
||||
match="AcquisitionRuntimeEvent",
|
||||
):
|
||||
asyncio.run(
|
||||
publisher.publish(
|
||||
object(), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_event_is_delivered_to_consumers_in_registration_order() -> None:
|
||||
calls: list[tuple[str, AcquisitionRuntimeEvent]] = []
|
||||
|
||||
class NamedConsumer:
|
||||
def __init__(self, name: str) -> None:
|
||||
self._name = name
|
||||
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
calls.append(
|
||||
(
|
||||
self._name,
|
||||
event,
|
||||
)
|
||||
)
|
||||
|
||||
event = ConnectedEvent()
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
NamedConsumer("first"),
|
||||
NamedConsumer("second"),
|
||||
)
|
||||
)
|
||||
|
||||
asyncio.run(publisher.publish(event))
|
||||
|
||||
assert calls == [
|
||||
(
|
||||
"first",
|
||||
event,
|
||||
),
|
||||
(
|
||||
"second",
|
||||
event,
|
||||
),
|
||||
]
|
||||
assert calls[0][1] is event
|
||||
assert calls[1][1] is event
|
||||
|
||||
|
||||
def test_publish_waits_for_consumer_completion() -> None:
|
||||
completed = False
|
||||
|
||||
class YieldingConsumer:
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
nonlocal completed
|
||||
|
||||
await asyncio.sleep(0)
|
||||
completed = True
|
||||
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
YieldingConsumer(),
|
||||
)
|
||||
)
|
||||
|
||||
asyncio.run(
|
||||
publisher.publish(
|
||||
ConnectedEvent(),
|
||||
)
|
||||
)
|
||||
|
||||
assert completed is True
|
||||
|
||||
|
||||
def test_concurrent_publications_are_serialized() -> None:
|
||||
calls: list[tuple[str, int]] = []
|
||||
|
||||
async def scenario() -> None:
|
||||
first_started = asyncio.Event()
|
||||
release_first = asyncio.Event()
|
||||
|
||||
class BlockingConsumer:
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
assert isinstance(
|
||||
event,
|
||||
ReconnectStartedEvent,
|
||||
)
|
||||
|
||||
calls.append(
|
||||
(
|
||||
"start",
|
||||
event.attempt,
|
||||
)
|
||||
)
|
||||
|
||||
if event.attempt == 1:
|
||||
first_started.set()
|
||||
await release_first.wait()
|
||||
|
||||
calls.append(
|
||||
(
|
||||
"end",
|
||||
event.attempt,
|
||||
)
|
||||
)
|
||||
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
BlockingConsumer(),
|
||||
)
|
||||
)
|
||||
|
||||
first_task = asyncio.create_task(
|
||||
publisher.publish(
|
||||
ReconnectStartedEvent(
|
||||
attempt=1,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
await first_started.wait()
|
||||
|
||||
second_task = asyncio.create_task(
|
||||
publisher.publish(
|
||||
ReconnectStartedEvent(
|
||||
attempt=2,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert calls == [
|
||||
(
|
||||
"start",
|
||||
1,
|
||||
),
|
||||
]
|
||||
|
||||
release_first.set()
|
||||
|
||||
await asyncio.gather(
|
||||
first_task,
|
||||
second_task,
|
||||
)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert calls == [
|
||||
(
|
||||
"start",
|
||||
1,
|
||||
),
|
||||
(
|
||||
"end",
|
||||
1,
|
||||
),
|
||||
(
|
||||
"start",
|
||||
2,
|
||||
),
|
||||
(
|
||||
"end",
|
||||
2,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def test_consumer_error_is_logged_and_delivery_continues(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
class BrokenConsumer:
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
raise RuntimeError(
|
||||
"sensitive consumer failure detail",
|
||||
)
|
||||
|
||||
recording_consumer = RecordingConsumer()
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
BrokenConsumer(),
|
||||
recording_consumer,
|
||||
)
|
||||
)
|
||||
event = ConnectedEvent()
|
||||
|
||||
caplog.set_level(
|
||||
logging.ERROR,
|
||||
logger=PUBLISHER_LOGGER_NAME,
|
||||
)
|
||||
|
||||
asyncio.run(publisher.publish(event))
|
||||
|
||||
assert recording_consumer.events == [event]
|
||||
assert len(caplog.records) == 1
|
||||
assert (
|
||||
"event=ConnectedEvent consumer=BrokenConsumer "
|
||||
"error_type=RuntimeError"
|
||||
in caplog.records[0].getMessage()
|
||||
)
|
||||
assert caplog.records[0].exc_info is None
|
||||
assert (
|
||||
"sensitive consumer failure detail"
|
||||
not in caplog.text
|
||||
)
|
||||
|
||||
|
||||
def test_consumer_error_log_does_not_include_transport_payload(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
class PayloadEchoingBrokenConsumer:
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
raise RuntimeError(
|
||||
f"consumer rejected {event!r}"
|
||||
)
|
||||
|
||||
payload = "secret-runtime-payload"
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
PayloadEchoingBrokenConsumer(),
|
||||
)
|
||||
)
|
||||
|
||||
caplog.set_level(
|
||||
logging.ERROR,
|
||||
logger=PUBLISHER_LOGGER_NAME,
|
||||
)
|
||||
|
||||
asyncio.run(
|
||||
publisher.publish(
|
||||
MessageReceivedEvent(
|
||||
message=TransportTextMessage(
|
||||
payload=payload,
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert payload not in caplog.text
|
||||
assert "consumer rejected" not in caplog.text
|
||||
assert "error_type=RuntimeError" in caplog.text
|
||||
|
||||
|
||||
def test_logging_handler_error_does_not_escape_or_stop_delivery() -> None:
|
||||
class BrokenConsumer:
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
raise RuntimeError(
|
||||
"consumer failed",
|
||||
)
|
||||
|
||||
class RaisingHandler(logging.Handler):
|
||||
def emit(
|
||||
self,
|
||||
record: logging.LogRecord,
|
||||
) -> None:
|
||||
raise RuntimeError(
|
||||
"logging failed",
|
||||
)
|
||||
|
||||
handler = RaisingHandler()
|
||||
recording_consumer = RecordingConsumer()
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
BrokenConsumer(),
|
||||
recording_consumer,
|
||||
)
|
||||
)
|
||||
event = ConnectedEvent()
|
||||
|
||||
publisher_logger.addHandler(handler)
|
||||
|
||||
try:
|
||||
asyncio.run(publisher.publish(event))
|
||||
finally:
|
||||
publisher_logger.removeHandler(handler)
|
||||
|
||||
assert recording_consumer.events == [event]
|
||||
|
||||
|
||||
def test_consumer_cancellation_is_propagated() -> None:
|
||||
class CancelledConsumer:
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
CancelledConsumer(),
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
asyncio.run(
|
||||
publisher.publish(
|
||||
ConnectedEvent(),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_publisher_can_be_reused_after_consumer_cancellation() -> None:
|
||||
class CancelOnceConsumer:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
self.calls += 1
|
||||
|
||||
if self.calls == 1:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
cancel_once_consumer = CancelOnceConsumer()
|
||||
recording_consumer = RecordingConsumer()
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
cancel_once_consumer,
|
||||
recording_consumer,
|
||||
)
|
||||
)
|
||||
second_event = DisconnectedEvent()
|
||||
|
||||
async def scenario() -> None:
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await publisher.publish(
|
||||
ConnectedEvent(),
|
||||
)
|
||||
|
||||
await publisher.publish(second_event)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert cancel_once_consumer.calls == 2
|
||||
assert recording_consumer.events == [
|
||||
second_event,
|
||||
]
|
||||
|
||||
|
||||
def test_recursive_publication_is_rejected_without_deadlock(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
class RecursiveConsumer:
|
||||
def __init__(self) -> None:
|
||||
self.publisher: (
|
||||
AcquisitionRuntimeEventPublisher | None
|
||||
) = None
|
||||
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
assert self.publisher is not None
|
||||
|
||||
await self.publisher.publish(
|
||||
DisconnectedEvent(),
|
||||
)
|
||||
|
||||
recursive_consumer = RecursiveConsumer()
|
||||
recording_consumer = RecordingConsumer()
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
recursive_consumer,
|
||||
recording_consumer,
|
||||
)
|
||||
)
|
||||
recursive_consumer.publisher = publisher
|
||||
event = ConnectedEvent()
|
||||
|
||||
caplog.set_level(
|
||||
logging.ERROR,
|
||||
logger=PUBLISHER_LOGGER_NAME,
|
||||
)
|
||||
|
||||
asyncio.run(publisher.publish(event))
|
||||
|
||||
assert recording_consumer.events == [event]
|
||||
assert (
|
||||
"event=ConnectedEvent consumer=RecursiveConsumer "
|
||||
"error_type=RuntimeError"
|
||||
in caplog.text
|
||||
)
|
||||
|
||||
|
||||
def test_child_task_recursive_publication_is_rejected() -> None:
|
||||
class ChildTaskRecursiveConsumer:
|
||||
def __init__(self) -> None:
|
||||
self.publisher: (
|
||||
AcquisitionRuntimeEventPublisher | None
|
||||
) = None
|
||||
self.rejected = False
|
||||
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
assert self.publisher is not None
|
||||
|
||||
if not isinstance(event, ConnectedEvent):
|
||||
return
|
||||
|
||||
try:
|
||||
await asyncio.create_task(
|
||||
self.publisher.publish(
|
||||
DisconnectedEvent(),
|
||||
)
|
||||
)
|
||||
except RuntimeError:
|
||||
self.rejected = True
|
||||
|
||||
recursive_consumer = ChildTaskRecursiveConsumer()
|
||||
recording_consumer = RecordingConsumer()
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
recursive_consumer,
|
||||
recording_consumer,
|
||||
)
|
||||
)
|
||||
recursive_consumer.publisher = publisher
|
||||
event = ConnectedEvent()
|
||||
|
||||
asyncio.run(
|
||||
asyncio.wait_for(
|
||||
publisher.publish(event),
|
||||
timeout=1.0,
|
||||
)
|
||||
)
|
||||
|
||||
assert recursive_consumer.rejected is True
|
||||
assert recording_consumer.events == [event]
|
||||
|
||||
|
||||
def test_consumer_error_does_not_interrupt_heartbeat_timeout() -> None:
|
||||
class BrokenConsumer:
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
raise RuntimeError(
|
||||
"consumer failed",
|
||||
)
|
||||
|
||||
current_time = [0.0]
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
BrokenConsumer(),
|
||||
)
|
||||
)
|
||||
monitor = HeartbeatMonitor(
|
||||
event_publisher=publisher,
|
||||
timeout_seconds=10.0,
|
||||
clock=lambda: current_time[0],
|
||||
)
|
||||
|
||||
monitor.start()
|
||||
current_time[0] = 10.0
|
||||
|
||||
result = asyncio.run(
|
||||
monitor.check_timeout()
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert monitor.state is HeartbeatState.TIMED_OUT
|
||||
|
||||
|
||||
def test_consumer_error_does_not_interrupt_reconnect() -> None:
|
||||
class BrokenConsumer:
|
||||
async def consume(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
raise RuntimeError(
|
||||
"consumer failed",
|
||||
)
|
||||
|
||||
class RecordingCommandDispatcher:
|
||||
def __init__(self) -> None:
|
||||
self.commands: list[
|
||||
AcquisitionRuntimeCommand
|
||||
] = []
|
||||
|
||||
async def dispatch(
|
||||
self,
|
||||
command: AcquisitionRuntimeCommand,
|
||||
) -> None:
|
||||
self.commands.append(command)
|
||||
|
||||
class RecordingSubscriptionManager:
|
||||
def __init__(self) -> None:
|
||||
self.restore_calls = 0
|
||||
|
||||
async def subscribe(
|
||||
self,
|
||||
subscription_key: str,
|
||||
message: AcquisitionSubscriptionMessage,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
async def unsubscribe(
|
||||
self,
|
||||
subscription_key: str,
|
||||
message: AcquisitionSubscriptionMessage,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
async def restore_subscriptions(self) -> None:
|
||||
self.restore_calls += 1
|
||||
|
||||
async def clear_subscriptions(self) -> None:
|
||||
return None
|
||||
|
||||
dispatcher = RecordingCommandDispatcher()
|
||||
subscriptions = RecordingSubscriptionManager()
|
||||
publisher = AcquisitionRuntimeEventPublisher(
|
||||
consumers=(
|
||||
BrokenConsumer(),
|
||||
)
|
||||
)
|
||||
coordinator = ReconnectCoordinator(
|
||||
command_dispatcher=dispatcher,
|
||||
subscription_manager=subscriptions,
|
||||
event_publisher=publisher,
|
||||
)
|
||||
|
||||
asyncio.run(
|
||||
coordinator.reconnect()
|
||||
)
|
||||
|
||||
assert coordinator.state is ReconnectState.CONNECTED
|
||||
assert coordinator.attempt == 1
|
||||
assert [
|
||||
type(command)
|
||||
for command in dispatcher.commands
|
||||
] == [
|
||||
DisconnectCommand,
|
||||
ConnectCommand,
|
||||
]
|
||||
assert subscriptions.restore_calls == 1
|
||||
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from src.market_data.acquisition.runtime.live_processing_gate import (
|
||||
RuntimeLiveProcessingGate,
|
||||
RuntimeLiveProcessingGateProtocol,
|
||||
)
|
||||
|
||||
|
||||
def test_implements_protocol_uses_slots_and_starts_open() -> None:
|
||||
gate = RuntimeLiveProcessingGate()
|
||||
|
||||
assert isinstance(
|
||||
gate,
|
||||
RuntimeLiveProcessingGateProtocol,
|
||||
)
|
||||
assert not hasattr(gate, "__dict__")
|
||||
assert gate.locked is False
|
||||
assert gate.failed is False
|
||||
|
||||
|
||||
def test_serializes_protected_operations() -> None:
|
||||
async def scenario() -> list[str]:
|
||||
gate = RuntimeLiveProcessingGate()
|
||||
order: list[str] = []
|
||||
|
||||
async def contender() -> None:
|
||||
async with gate:
|
||||
order.append("contender")
|
||||
|
||||
async with gate:
|
||||
order.append("owner")
|
||||
contender_task = asyncio.create_task(
|
||||
contender(),
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert contender_task.done() is False
|
||||
assert gate.locked is True
|
||||
|
||||
await contender_task
|
||||
return order
|
||||
|
||||
assert asyncio.run(scenario()) == [
|
||||
"owner",
|
||||
"contender",
|
||||
]
|
||||
|
||||
|
||||
def test_releases_gate_after_error() -> None:
|
||||
async def scenario() -> RuntimeLiveProcessingGate:
|
||||
gate = RuntimeLiveProcessingGate()
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="protected failure",
|
||||
):
|
||||
async with gate:
|
||||
raise RuntimeError(
|
||||
"protected failure",
|
||||
)
|
||||
|
||||
return gate
|
||||
|
||||
assert asyncio.run(scenario()).locked is False
|
||||
|
||||
|
||||
def test_failed_gate_rejects_waiters_until_reset() -> None:
|
||||
async def scenario() -> RuntimeLiveProcessingGate:
|
||||
gate = RuntimeLiveProcessingGate()
|
||||
failure = RuntimeError("recovery failed")
|
||||
|
||||
gate.fail(failure)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="recovery failed",
|
||||
) as error_info:
|
||||
async with gate:
|
||||
raise AssertionError(
|
||||
"failed gate must not admit live processing"
|
||||
)
|
||||
|
||||
assert error_info.value is failure
|
||||
assert gate.locked is False
|
||||
assert gate.failed is True
|
||||
|
||||
gate.reset()
|
||||
|
||||
async with gate:
|
||||
assert gate.locked is True
|
||||
|
||||
return gate
|
||||
|
||||
gate = asyncio.run(scenario())
|
||||
|
||||
assert gate.locked is False
|
||||
assert gate.failed is False
|
||||
|
||||
|
||||
def test_rejects_reset_while_gate_is_locked() -> None:
|
||||
async def scenario() -> None:
|
||||
gate = RuntimeLiveProcessingGate()
|
||||
|
||||
async with gate:
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="cannot be reset while locked",
|
||||
):
|
||||
gate.reset()
|
||||
|
||||
asyncio.run(scenario())
|
||||
@@ -14,6 +14,7 @@ from src.market_data.acquisition.runtime.reconnect import (
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_commands import (
|
||||
ConnectCommand,
|
||||
DisconnectCommand,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_events import (
|
||||
ReconnectCompletedEvent,
|
||||
@@ -115,16 +116,14 @@ def test_initial_state_is_disconnected() -> None:
|
||||
assert coordinator.attempt == 0
|
||||
|
||||
|
||||
def test_reconnect_dispatches_connect_command() -> None:
|
||||
def test_reconnect_dispatches_disconnect_then_connect() -> None:
|
||||
coordinator, dispatcher, *_ = create_coordinator()
|
||||
|
||||
asyncio.run(coordinator.reconnect())
|
||||
|
||||
assert len(dispatcher.commands) == 1
|
||||
assert isinstance(
|
||||
dispatcher.commands[0],
|
||||
ConnectCommand,
|
||||
)
|
||||
assert len(dispatcher.commands) == 2
|
||||
assert isinstance(dispatcher.commands[0], DisconnectCommand)
|
||||
assert isinstance(dispatcher.commands[1], ConnectCommand)
|
||||
|
||||
|
||||
def test_reconnect_restores_subscriptions() -> None:
|
||||
@@ -185,7 +184,9 @@ def test_connect_error_publishes_failed_event() -> None:
|
||||
command: Any,
|
||||
) -> None:
|
||||
self.commands.append(command)
|
||||
raise RuntimeError("connection failed")
|
||||
|
||||
if isinstance(command, ConnectCommand):
|
||||
raise RuntimeError("connection failed")
|
||||
|
||||
dispatcher = BrokenCommandDispatcher()
|
||||
subscriptions = FakeSubscriptionManager()
|
||||
@@ -218,7 +219,8 @@ def test_connect_error_sets_failed_state() -> None:
|
||||
self,
|
||||
command: Any,
|
||||
) -> None:
|
||||
raise RuntimeError("connection failed")
|
||||
if isinstance(command, ConnectCommand):
|
||||
raise RuntimeError("connection failed")
|
||||
|
||||
coordinator = ReconnectCoordinator(
|
||||
command_dispatcher=BrokenCommandDispatcher(),
|
||||
@@ -239,7 +241,8 @@ def test_connect_error_does_not_restore_subscriptions() -> None:
|
||||
self,
|
||||
command: Any,
|
||||
) -> None:
|
||||
raise RuntimeError("connection failed")
|
||||
if isinstance(command, ConnectCommand):
|
||||
raise RuntimeError("connection failed")
|
||||
|
||||
subscriptions = FakeSubscriptionManager()
|
||||
|
||||
|
||||
@@ -0,0 +1,711 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from src.market_data.acquisition.recovery.trade_recovery_result import (
|
||||
TradeRecoveryResult,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.live_processing_gate import (
|
||||
RuntimeLiveProcessingGate,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.reconnect import (
|
||||
ReconnectState,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_reconnect_recovery_coordinator import (
|
||||
RuntimeReconnectRecoveryCoordinator,
|
||||
RuntimeReconnectRecoveryProtocol,
|
||||
)
|
||||
|
||||
|
||||
BTC = "BTC/USD_LEVERAGE"
|
||||
ETH = "ETH/USD_LEVERAGE"
|
||||
RECOVERY_END_TIME_MS = 1_785_326_405_123
|
||||
|
||||
|
||||
class FakeReconnectCoordinator:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
order: list[str],
|
||||
gate: RuntimeLiveProcessingGate,
|
||||
error: Exception | None = None,
|
||||
entered: asyncio.Event | None = None,
|
||||
release: asyncio.Event | None = None,
|
||||
) -> None:
|
||||
self._order = order
|
||||
self._gate = gate
|
||||
self._error = error
|
||||
self._entered = entered
|
||||
self._release = release
|
||||
self._state = ReconnectState.DISCONNECTED
|
||||
self._attempt = 0
|
||||
|
||||
@property
|
||||
def state(self) -> ReconnectState:
|
||||
return self._state
|
||||
|
||||
@property
|
||||
def attempt(self) -> int:
|
||||
return self._attempt
|
||||
|
||||
async def reconnect(self) -> None:
|
||||
assert self._gate.locked is True
|
||||
|
||||
self._attempt += 1
|
||||
self._state = ReconnectState.CONNECTING
|
||||
self._order.append("reconnect")
|
||||
|
||||
if self._entered is not None:
|
||||
self._entered.set()
|
||||
|
||||
if self._release is not None:
|
||||
await self._release.wait()
|
||||
|
||||
if self._error is not None:
|
||||
self._state = ReconnectState.FAILED
|
||||
raise self._error
|
||||
|
||||
self._state = ReconnectState.CONNECTED
|
||||
|
||||
|
||||
class FakeRecoveryCoordinator:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
order: list[str],
|
||||
gate: RuntimeLiveProcessingGate,
|
||||
error: Exception | None = None,
|
||||
started: threading.Event | None = None,
|
||||
release: threading.Event | None = None,
|
||||
) -> None:
|
||||
self._order = order
|
||||
self._gate = gate
|
||||
self._error = error
|
||||
self._started = started
|
||||
self._release = release
|
||||
self.calls: list[tuple[str, int]] = []
|
||||
self.thread_ids: list[int] = []
|
||||
|
||||
def recover(
|
||||
self,
|
||||
*,
|
||||
symbol: str,
|
||||
recovery_end_time: int,
|
||||
) -> TradeRecoveryResult:
|
||||
assert self._gate.locked is True
|
||||
|
||||
self.calls.append(
|
||||
(
|
||||
symbol,
|
||||
recovery_end_time,
|
||||
)
|
||||
)
|
||||
self.thread_ids.append(
|
||||
threading.get_ident(),
|
||||
)
|
||||
self._order.append(
|
||||
f"recover:{symbol}",
|
||||
)
|
||||
|
||||
if self._started is not None:
|
||||
self._started.set()
|
||||
|
||||
if self._release is not None:
|
||||
if not self._release.wait(timeout=2.0):
|
||||
raise AssertionError(
|
||||
"recovery release was not signalled"
|
||||
)
|
||||
|
||||
if self._error is not None:
|
||||
raise self._error
|
||||
|
||||
return TradeRecoveryResult(
|
||||
symbol=symbol,
|
||||
requested_start_time=recovery_end_time,
|
||||
requested_end_time=recovery_end_time,
|
||||
recovered_trades=(),
|
||||
)
|
||||
|
||||
|
||||
class RecordingClock:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
order: list[str],
|
||||
value: object = RECOVERY_END_TIME_MS,
|
||||
) -> None:
|
||||
self._order = order
|
||||
self._value = value
|
||||
self.calls = 0
|
||||
|
||||
def __call__(self) -> int:
|
||||
self.calls += 1
|
||||
self._order.append("clock")
|
||||
return self._value # type: ignore[return-value]
|
||||
|
||||
|
||||
def create_coordinator(
|
||||
*,
|
||||
symbols: tuple[str, ...] = (BTC,),
|
||||
reconnect_error: Exception | None = None,
|
||||
recovery_error: Exception | None = None,
|
||||
reconnect_entered: asyncio.Event | None = None,
|
||||
reconnect_release: asyncio.Event | None = None,
|
||||
recovery_started: threading.Event | None = None,
|
||||
recovery_release: threading.Event | None = None,
|
||||
clock_value: object = RECOVERY_END_TIME_MS,
|
||||
) -> tuple[
|
||||
RuntimeReconnectRecoveryCoordinator,
|
||||
RuntimeLiveProcessingGate,
|
||||
FakeReconnectCoordinator,
|
||||
FakeRecoveryCoordinator,
|
||||
RecordingClock,
|
||||
list[str],
|
||||
]:
|
||||
order: list[str] = []
|
||||
gate = RuntimeLiveProcessingGate()
|
||||
reconnect = FakeReconnectCoordinator(
|
||||
order=order,
|
||||
gate=gate,
|
||||
error=reconnect_error,
|
||||
entered=reconnect_entered,
|
||||
release=reconnect_release,
|
||||
)
|
||||
recovery = FakeRecoveryCoordinator(
|
||||
order=order,
|
||||
gate=gate,
|
||||
error=recovery_error,
|
||||
started=recovery_started,
|
||||
release=recovery_release,
|
||||
)
|
||||
clock = RecordingClock(
|
||||
order=order,
|
||||
value=clock_value,
|
||||
)
|
||||
coordinator = RuntimeReconnectRecoveryCoordinator(
|
||||
reconnect_coordinator=reconnect,
|
||||
recovery_coordinator=recovery,
|
||||
live_processing_gate=gate,
|
||||
symbols=symbols,
|
||||
clock=clock,
|
||||
)
|
||||
|
||||
return (
|
||||
coordinator,
|
||||
gate,
|
||||
reconnect,
|
||||
recovery,
|
||||
clock,
|
||||
order,
|
||||
)
|
||||
|
||||
|
||||
async def wait_until(
|
||||
predicate: object,
|
||||
) -> None:
|
||||
for _ in range(100):
|
||||
if callable(predicate) and predicate():
|
||||
return
|
||||
|
||||
await asyncio.sleep(0)
|
||||
|
||||
raise AssertionError("condition was not reached")
|
||||
|
||||
|
||||
def test_implements_protocol_uses_slots_and_delegates_state() -> None:
|
||||
coordinator, _, reconnect, *_ = create_coordinator()
|
||||
|
||||
assert isinstance(
|
||||
coordinator,
|
||||
RuntimeReconnectRecoveryProtocol,
|
||||
)
|
||||
assert not hasattr(coordinator, "__dict__")
|
||||
assert coordinator.state is ReconnectState.DISCONNECTED
|
||||
assert coordinator.attempt == 0
|
||||
assert coordinator.generation == 0
|
||||
assert coordinator.live_processing_gate is coordinator._live_processing_gate
|
||||
assert coordinator.symbols == (BTC,)
|
||||
|
||||
asyncio.run(coordinator.reconnect())
|
||||
|
||||
assert coordinator.state is reconnect.state
|
||||
assert coordinator.state is ReconnectState.CONNECTED
|
||||
assert coordinator.attempt == 1
|
||||
assert coordinator.generation == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("symbols", "error_type"),
|
||||
[
|
||||
([], TypeError),
|
||||
((), ValueError),
|
||||
(("", " "), ValueError),
|
||||
((BTC, 1), TypeError),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_symbols(
|
||||
symbols: object,
|
||||
error_type: type[Exception],
|
||||
) -> None:
|
||||
with pytest.raises(error_type):
|
||||
create_coordinator(
|
||||
symbols=symbols, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
|
||||
def test_reconnect_restore_boundary_and_recovery_order() -> None:
|
||||
(
|
||||
coordinator,
|
||||
gate,
|
||||
_,
|
||||
recovery,
|
||||
clock,
|
||||
order,
|
||||
) = create_coordinator(
|
||||
symbols=(
|
||||
f" {ETH} ",
|
||||
BTC,
|
||||
ETH,
|
||||
),
|
||||
)
|
||||
main_thread_id = threading.get_ident()
|
||||
|
||||
asyncio.run(coordinator.reconnect())
|
||||
|
||||
assert order == [
|
||||
"reconnect",
|
||||
"clock",
|
||||
f"recover:{BTC}",
|
||||
f"recover:{ETH}",
|
||||
]
|
||||
assert recovery.calls == [
|
||||
(
|
||||
BTC,
|
||||
RECOVERY_END_TIME_MS,
|
||||
),
|
||||
(
|
||||
ETH,
|
||||
RECOVERY_END_TIME_MS,
|
||||
),
|
||||
]
|
||||
assert clock.calls == 1
|
||||
assert all(
|
||||
thread_id != main_thread_id
|
||||
for thread_id in recovery.thread_ids
|
||||
)
|
||||
assert gate.locked is False
|
||||
|
||||
|
||||
def test_live_processing_waits_until_recovery_finishes() -> None:
|
||||
async def scenario() -> list[str]:
|
||||
recovery_started = threading.Event()
|
||||
recovery_release = threading.Event()
|
||||
(
|
||||
coordinator,
|
||||
gate,
|
||||
*_,
|
||||
) = create_coordinator(
|
||||
recovery_started=recovery_started,
|
||||
recovery_release=recovery_release,
|
||||
)
|
||||
order: list[str] = []
|
||||
|
||||
coordinator_task = asyncio.create_task(
|
||||
coordinator.reconnect(),
|
||||
)
|
||||
await wait_until(
|
||||
recovery_started.is_set,
|
||||
)
|
||||
|
||||
async def process_live() -> None:
|
||||
async with gate:
|
||||
order.append("live")
|
||||
|
||||
live_task = asyncio.create_task(
|
||||
process_live(),
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert live_task.done() is False
|
||||
assert gate.locked is True
|
||||
|
||||
recovery_release.set()
|
||||
await coordinator_task
|
||||
await live_task
|
||||
|
||||
return order
|
||||
|
||||
assert asyncio.run(scenario()) == [
|
||||
"live",
|
||||
]
|
||||
|
||||
|
||||
def test_concurrent_callers_share_one_operation() -> None:
|
||||
async def scenario() -> tuple[
|
||||
RuntimeReconnectRecoveryCoordinator,
|
||||
FakeReconnectCoordinator,
|
||||
FakeRecoveryCoordinator,
|
||||
]:
|
||||
reconnect_entered = asyncio.Event()
|
||||
reconnect_release = asyncio.Event()
|
||||
(
|
||||
coordinator,
|
||||
_,
|
||||
reconnect,
|
||||
recovery,
|
||||
*_,
|
||||
) = create_coordinator(
|
||||
reconnect_entered=reconnect_entered,
|
||||
reconnect_release=reconnect_release,
|
||||
)
|
||||
|
||||
first = asyncio.create_task(
|
||||
coordinator.reconnect(),
|
||||
)
|
||||
await reconnect_entered.wait()
|
||||
|
||||
second = asyncio.create_task(
|
||||
coordinator.reconnect(),
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert reconnect.attempt == 1
|
||||
|
||||
reconnect_release.set()
|
||||
await asyncio.gather(
|
||||
first,
|
||||
second,
|
||||
)
|
||||
|
||||
return (
|
||||
coordinator,
|
||||
reconnect,
|
||||
recovery,
|
||||
)
|
||||
|
||||
coordinator, reconnect, recovery = asyncio.run(
|
||||
scenario()
|
||||
)
|
||||
|
||||
assert reconnect.attempt == 1
|
||||
assert len(recovery.calls) == 1
|
||||
assert coordinator.generation == 1
|
||||
|
||||
|
||||
def test_timeout_and_transport_error_share_one_operation() -> None:
|
||||
async def scenario() -> tuple[
|
||||
RuntimeReconnectRecoveryCoordinator,
|
||||
FakeReconnectCoordinator,
|
||||
FakeRecoveryCoordinator,
|
||||
]:
|
||||
reconnect_entered = asyncio.Event()
|
||||
reconnect_release = asyncio.Event()
|
||||
(
|
||||
coordinator,
|
||||
_,
|
||||
reconnect,
|
||||
recovery,
|
||||
*_,
|
||||
) = create_coordinator(
|
||||
reconnect_entered=reconnect_entered,
|
||||
reconnect_release=reconnect_release,
|
||||
)
|
||||
observed_generation = coordinator.generation
|
||||
|
||||
timeout_task = asyncio.create_task(
|
||||
coordinator.reconnect(),
|
||||
)
|
||||
await reconnect_entered.wait()
|
||||
|
||||
transport_task = asyncio.create_task(
|
||||
coordinator.reconnect_after_transport_failure(
|
||||
observed_generation=observed_generation,
|
||||
),
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert reconnect.attempt == 1
|
||||
reconnect_release.set()
|
||||
|
||||
await asyncio.gather(
|
||||
timeout_task,
|
||||
transport_task,
|
||||
)
|
||||
|
||||
return (
|
||||
coordinator,
|
||||
reconnect,
|
||||
recovery,
|
||||
)
|
||||
|
||||
coordinator, reconnect, recovery = asyncio.run(
|
||||
scenario()
|
||||
)
|
||||
|
||||
assert reconnect.attempt == 1
|
||||
assert len(recovery.calls) == 1
|
||||
assert coordinator.generation == 1
|
||||
|
||||
|
||||
def test_stale_transport_error_reuses_completed_operation() -> None:
|
||||
(
|
||||
coordinator,
|
||||
_,
|
||||
reconnect,
|
||||
recovery,
|
||||
*_,
|
||||
) = create_coordinator()
|
||||
|
||||
observed_generation = coordinator.generation
|
||||
|
||||
asyncio.run(coordinator.reconnect())
|
||||
asyncio.run(
|
||||
coordinator.reconnect_after_transport_failure(
|
||||
observed_generation=observed_generation,
|
||||
)
|
||||
)
|
||||
|
||||
assert reconnect.attempt == 1
|
||||
assert len(recovery.calls) == 1
|
||||
|
||||
|
||||
def test_reconnect_error_skips_clock_and_recovery() -> None:
|
||||
reconnect_error = RuntimeError("reconnect failed")
|
||||
(
|
||||
coordinator,
|
||||
gate,
|
||||
_,
|
||||
recovery,
|
||||
clock,
|
||||
_,
|
||||
) = create_coordinator(
|
||||
reconnect_error=reconnect_error,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="reconnect failed",
|
||||
) as error_info:
|
||||
asyncio.run(coordinator.reconnect())
|
||||
|
||||
assert error_info.value is reconnect_error
|
||||
assert recovery.calls == []
|
||||
assert clock.calls == 0
|
||||
assert gate.locked is False
|
||||
assert gate.failed is True
|
||||
|
||||
|
||||
def test_recovery_error_is_not_wrapped() -> None:
|
||||
recovery_error = RuntimeError("recovery failed")
|
||||
(
|
||||
coordinator,
|
||||
gate,
|
||||
*_,
|
||||
) = create_coordinator(
|
||||
recovery_error=recovery_error,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="recovery failed",
|
||||
) as error_info:
|
||||
asyncio.run(coordinator.reconnect())
|
||||
|
||||
assert error_info.value is recovery_error
|
||||
assert gate.locked is False
|
||||
assert gate.failed is True
|
||||
assert coordinator._recovery_task is None
|
||||
|
||||
|
||||
def test_recovery_error_rejects_buffered_live_processing() -> None:
|
||||
async def scenario() -> tuple[
|
||||
RuntimeLiveProcessingGate,
|
||||
list[str],
|
||||
]:
|
||||
recovery_error = RuntimeError("recovery failed")
|
||||
recovery_started = threading.Event()
|
||||
recovery_release = threading.Event()
|
||||
(
|
||||
coordinator,
|
||||
gate,
|
||||
*_,
|
||||
) = create_coordinator(
|
||||
recovery_error=recovery_error,
|
||||
recovery_started=recovery_started,
|
||||
recovery_release=recovery_release,
|
||||
)
|
||||
processed: list[str] = []
|
||||
coordinator_task = asyncio.create_task(
|
||||
coordinator.reconnect(),
|
||||
)
|
||||
await wait_until(
|
||||
recovery_started.is_set,
|
||||
)
|
||||
|
||||
async def process_live() -> None:
|
||||
async with gate:
|
||||
processed.append("live")
|
||||
|
||||
live_task = asyncio.create_task(
|
||||
process_live(),
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert live_task.done() is False
|
||||
recovery_release.set()
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="recovery failed",
|
||||
):
|
||||
await coordinator_task
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="recovery failed",
|
||||
):
|
||||
await live_task
|
||||
|
||||
return (
|
||||
gate,
|
||||
processed,
|
||||
)
|
||||
|
||||
gate, processed = asyncio.run(scenario())
|
||||
|
||||
assert processed == []
|
||||
assert gate.locked is False
|
||||
assert gate.failed is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("clock_value", "error_type"),
|
||||
[
|
||||
(True, TypeError),
|
||||
(1.5, TypeError),
|
||||
(-1, ValueError),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_clock_result(
|
||||
clock_value: object,
|
||||
error_type: type[Exception],
|
||||
) -> None:
|
||||
coordinator, gate, *_ = create_coordinator(
|
||||
clock_value=clock_value,
|
||||
)
|
||||
|
||||
with pytest.raises(error_type):
|
||||
asyncio.run(coordinator.reconnect())
|
||||
|
||||
assert gate.locked is False
|
||||
|
||||
|
||||
def test_cancellation_waits_for_worker_before_opening_gate() -> None:
|
||||
async def scenario() -> tuple[
|
||||
RuntimeReconnectRecoveryCoordinator,
|
||||
RuntimeLiveProcessingGate,
|
||||
]:
|
||||
recovery_started = threading.Event()
|
||||
recovery_release = threading.Event()
|
||||
(
|
||||
coordinator,
|
||||
gate,
|
||||
*_,
|
||||
) = create_coordinator(
|
||||
recovery_started=recovery_started,
|
||||
recovery_release=recovery_release,
|
||||
)
|
||||
coordinator_task = asyncio.create_task(
|
||||
coordinator.reconnect(),
|
||||
)
|
||||
await wait_until(
|
||||
recovery_started.is_set,
|
||||
)
|
||||
|
||||
coordinator_task.cancel()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert coordinator_task.done() is False
|
||||
assert gate.locked is True
|
||||
assert coordinator._recovery_task is not None
|
||||
|
||||
recovery_release.set()
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await coordinator_task
|
||||
|
||||
return (
|
||||
coordinator,
|
||||
gate,
|
||||
)
|
||||
|
||||
coordinator, gate = asyncio.run(scenario())
|
||||
|
||||
assert gate.locked is False
|
||||
assert coordinator._recovery_task is None
|
||||
|
||||
|
||||
def test_repeated_cancellation_waits_for_worker_before_opening_gate() -> None:
|
||||
async def scenario() -> tuple[
|
||||
RuntimeReconnectRecoveryCoordinator,
|
||||
RuntimeLiveProcessingGate,
|
||||
list[str],
|
||||
]:
|
||||
recovery_started = threading.Event()
|
||||
recovery_release = threading.Event()
|
||||
(
|
||||
coordinator,
|
||||
gate,
|
||||
*_,
|
||||
) = create_coordinator(
|
||||
recovery_started=recovery_started,
|
||||
recovery_release=recovery_release,
|
||||
)
|
||||
processed: list[str] = []
|
||||
coordinator_task = asyncio.create_task(
|
||||
coordinator.reconnect(),
|
||||
)
|
||||
await wait_until(
|
||||
recovery_started.is_set,
|
||||
)
|
||||
|
||||
async def process_live() -> None:
|
||||
async with gate:
|
||||
processed.append("live")
|
||||
|
||||
live_task = asyncio.create_task(
|
||||
process_live(),
|
||||
)
|
||||
|
||||
coordinator_task.cancel()
|
||||
await asyncio.sleep(0)
|
||||
coordinator_task.cancel()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert coordinator_task.done() is False
|
||||
assert live_task.done() is False
|
||||
assert gate.locked is True
|
||||
assert coordinator._recovery_task is not None
|
||||
|
||||
recovery_release.set()
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await coordinator_task
|
||||
|
||||
await live_task
|
||||
|
||||
return (
|
||||
coordinator,
|
||||
gate,
|
||||
processed,
|
||||
)
|
||||
|
||||
coordinator, gate, processed = asyncio.run(scenario())
|
||||
|
||||
assert processed == ["live"]
|
||||
assert gate.locked is False
|
||||
assert gate.failed is False
|
||||
assert coordinator._recovery_task is None
|
||||
@@ -184,7 +184,9 @@ class FakeStateStore:
|
||||
self._states.clear()
|
||||
|
||||
|
||||
class RecordingWindowPlanner:
|
||||
class RecordingWindowPlanner(
|
||||
TradeRecoveryWindowPlanner,
|
||||
):
|
||||
"""
|
||||
Planner с заранее заданным результатом.
|
||||
"""
|
||||
@@ -290,7 +292,7 @@ def create_coordinator(
|
||||
|
||||
coordinator = RuntimeRecoveryCoordinator(
|
||||
state_store=resolved_state_store,
|
||||
window_planner=resolved_window_planner, # type: ignore[arg-type]
|
||||
window_planner=resolved_window_planner,
|
||||
recovery_controller=resolved_recovery_controller,
|
||||
)
|
||||
|
||||
@@ -924,15 +926,8 @@ def test_controller_error_is_not_swallowed() -> None:
|
||||
|
||||
def test_planner_error_is_not_swallowed() -> None:
|
||||
class BrokenPlanner(
|
||||
TradeRecoveryWindowPlanner,
|
||||
RecordingWindowPlanner,
|
||||
):
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
@property
|
||||
def max_window_ms(self) -> int:
|
||||
return 1
|
||||
|
||||
def build_windows(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -11,6 +11,9 @@ import pytest
|
||||
from src.market_data.acquisition.runtime.heartbeat import (
|
||||
HeartbeatState,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_liveness_probe import (
|
||||
RuntimeLivenessProbeProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.scheduler import (
|
||||
RuntimeScheduler,
|
||||
RuntimeSchedulerProtocol,
|
||||
@@ -58,9 +61,37 @@ class FakeHeartbeatMonitor:
|
||||
return self._results.pop(0)
|
||||
|
||||
|
||||
class FakeLivenessProbe:
|
||||
def __init__(
|
||||
self,
|
||||
results: list[object] | None = None,
|
||||
*,
|
||||
error: BaseException | None = None,
|
||||
) -> None:
|
||||
self._results = list(
|
||||
results
|
||||
if results is not None
|
||||
else [True]
|
||||
)
|
||||
self._error = error
|
||||
self.calls = 0
|
||||
|
||||
async def probe(self) -> bool:
|
||||
self.calls += 1
|
||||
|
||||
if self._error is not None:
|
||||
raise self._error
|
||||
|
||||
if not self._results:
|
||||
return True
|
||||
|
||||
return self._results.pop(0) # type: ignore[return-value]
|
||||
|
||||
|
||||
class FakeRuntimeSupervisor:
|
||||
def __init__(self) -> None:
|
||||
self.handle_timeout_calls = 0
|
||||
self.notify_activity_calls = 0
|
||||
|
||||
@property
|
||||
def state(self) -> RuntimeSupervisorState:
|
||||
@@ -73,7 +104,7 @@ class FakeRuntimeSupervisor:
|
||||
return None
|
||||
|
||||
def notify_activity(self) -> None:
|
||||
return None
|
||||
self.notify_activity_calls += 1
|
||||
|
||||
async def handle_heartbeat_timeout(self) -> bool:
|
||||
self.handle_timeout_calls += 1
|
||||
@@ -101,6 +132,7 @@ class RecordingSleep:
|
||||
|
||||
def create_scheduler(
|
||||
*,
|
||||
liveness_results: list[object] | None = None,
|
||||
heartbeat_results: list[bool] | None = None,
|
||||
interval_seconds: float = 1.0,
|
||||
sleep: Callable[[float], Awaitable[None]] | None = None,
|
||||
@@ -115,6 +147,9 @@ def create_scheduler(
|
||||
supervisor = FakeRuntimeSupervisor()
|
||||
|
||||
scheduler = RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(
|
||||
results=liveness_results,
|
||||
),
|
||||
heartbeat_monitor=heartbeat,
|
||||
runtime_supervisor=supervisor,
|
||||
interval_seconds=interval_seconds,
|
||||
@@ -131,6 +166,10 @@ def create_scheduler(
|
||||
def test_scheduler_implements_protocol() -> None:
|
||||
scheduler, *_ = create_scheduler()
|
||||
|
||||
assert isinstance(
|
||||
scheduler._liveness_probe,
|
||||
RuntimeLivenessProbeProtocol,
|
||||
)
|
||||
assert isinstance(
|
||||
scheduler,
|
||||
RuntimeSchedulerProtocol,
|
||||
@@ -149,6 +188,71 @@ def test_initial_state_is_not_running() -> None:
|
||||
assert scheduler.running is False
|
||||
|
||||
|
||||
def test_claim_blocks_start_without_matching_owner() -> None:
|
||||
scheduler, *_ = create_scheduler()
|
||||
owner = object()
|
||||
scheduler.claim(owner)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="owned by another lifecycle",
|
||||
):
|
||||
asyncio.run(
|
||||
scheduler.start()
|
||||
)
|
||||
|
||||
scheduler.release(owner)
|
||||
|
||||
|
||||
def test_claim_rejects_active_scheduler() -> None:
|
||||
scheduler, *_ = create_scheduler()
|
||||
scheduler._running = True
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="already owned or active",
|
||||
):
|
||||
scheduler.claim(object())
|
||||
|
||||
|
||||
def test_release_rejects_running_owned_scheduler() -> None:
|
||||
scheduler, *_ = create_scheduler()
|
||||
owner = object()
|
||||
scheduler.claim(owner)
|
||||
scheduler._running = True
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="Cannot release",
|
||||
):
|
||||
scheduler.release(owner)
|
||||
|
||||
|
||||
def test_matching_owner_can_run_and_release_scheduler() -> None:
|
||||
scheduler_holder: list[RuntimeScheduler] = []
|
||||
|
||||
async def stop_after_first_iteration(
|
||||
seconds: float,
|
||||
) -> None:
|
||||
scheduler_holder[0].stop()
|
||||
|
||||
scheduler, *_ = create_scheduler(
|
||||
sleep=stop_after_first_iteration,
|
||||
)
|
||||
scheduler_holder.append(scheduler)
|
||||
owner = object()
|
||||
scheduler.claim(owner)
|
||||
|
||||
asyncio.run(
|
||||
scheduler.start(
|
||||
owner=owner,
|
||||
)
|
||||
)
|
||||
scheduler.release(owner)
|
||||
|
||||
assert scheduler.running is False
|
||||
|
||||
|
||||
def test_exposes_interval_seconds() -> None:
|
||||
scheduler, *_ = create_scheduler(
|
||||
interval_seconds=2.5,
|
||||
@@ -187,6 +291,7 @@ def test_rejects_invalid_interval_type(
|
||||
) -> None:
|
||||
with pytest.raises(TypeError):
|
||||
RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(),
|
||||
heartbeat_monitor=FakeHeartbeatMonitor(),
|
||||
runtime_supervisor=FakeRuntimeSupervisor(),
|
||||
interval_seconds=interval_seconds, # type: ignore[arg-type]
|
||||
@@ -206,6 +311,7 @@ def test_rejects_non_positive_interval(
|
||||
) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(),
|
||||
heartbeat_monitor=FakeHeartbeatMonitor(),
|
||||
runtime_supervisor=FakeRuntimeSupervisor(),
|
||||
interval_seconds=interval_seconds,
|
||||
@@ -215,6 +321,7 @@ def test_rejects_non_positive_interval(
|
||||
def test_rejects_non_callable_sleep() -> None:
|
||||
with pytest.raises(TypeError):
|
||||
RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(),
|
||||
heartbeat_monitor=FakeHeartbeatMonitor(),
|
||||
runtime_supervisor=FakeRuntimeSupervisor(),
|
||||
interval_seconds=1.0,
|
||||
@@ -233,9 +340,43 @@ def test_run_once_checks_heartbeat() -> None:
|
||||
|
||||
assert result is False
|
||||
assert heartbeat.check_timeout_calls == 1
|
||||
assert supervisor.notify_activity_calls == 1
|
||||
assert supervisor.handle_timeout_calls == 0
|
||||
|
||||
|
||||
def test_failed_probe_does_not_record_activity() -> None:
|
||||
scheduler, heartbeat, supervisor = create_scheduler(
|
||||
liveness_results=[False],
|
||||
heartbeat_results=[False],
|
||||
)
|
||||
|
||||
result = asyncio.run(
|
||||
scheduler.run_once()
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert heartbeat.check_timeout_calls == 1
|
||||
assert supervisor.notify_activity_calls == 0
|
||||
assert supervisor.handle_timeout_calls == 0
|
||||
|
||||
|
||||
def test_rejects_non_boolean_probe_result() -> None:
|
||||
scheduler, heartbeat, supervisor = create_scheduler(
|
||||
liveness_results=["alive"],
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
TypeError,
|
||||
match="must return a boolean",
|
||||
):
|
||||
asyncio.run(
|
||||
scheduler.run_once()
|
||||
)
|
||||
|
||||
assert heartbeat.check_timeout_calls == 0
|
||||
assert supervisor.notify_activity_calls == 0
|
||||
|
||||
|
||||
def test_run_once_calls_supervisor_on_timeout() -> None:
|
||||
scheduler, heartbeat, supervisor = create_scheduler(
|
||||
heartbeat_results=[True],
|
||||
@@ -402,6 +543,7 @@ def test_stop_during_run_once_prevents_sleep() -> None:
|
||||
heartbeat = StoppingHeartbeatMonitor()
|
||||
|
||||
scheduler = RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(),
|
||||
heartbeat_monitor=heartbeat,
|
||||
runtime_supervisor=supervisor,
|
||||
interval_seconds=1.0,
|
||||
@@ -438,6 +580,7 @@ def test_repeated_start_while_running_does_not_create_second_loop() -> None:
|
||||
scheduler.stop()
|
||||
|
||||
scheduler = RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(),
|
||||
heartbeat_monitor=heartbeat,
|
||||
runtime_supervisor=supervisor,
|
||||
interval_seconds=1.0,
|
||||
@@ -491,6 +634,7 @@ def test_heartbeat_error_is_propagated() -> None:
|
||||
raise RuntimeError("heartbeat failed")
|
||||
|
||||
scheduler = RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(),
|
||||
heartbeat_monitor=BrokenHeartbeatMonitor(),
|
||||
runtime_supervisor=FakeRuntimeSupervisor(),
|
||||
interval_seconds=1.0,
|
||||
@@ -511,6 +655,7 @@ def test_supervisor_error_is_propagated() -> None:
|
||||
raise RuntimeError("supervisor failed")
|
||||
|
||||
scheduler = RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(),
|
||||
heartbeat_monitor=FakeHeartbeatMonitor(
|
||||
results=[True],
|
||||
),
|
||||
@@ -533,6 +678,7 @@ def test_start_resets_running_after_heartbeat_error() -> None:
|
||||
raise RuntimeError("heartbeat failed")
|
||||
|
||||
scheduler = RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(),
|
||||
heartbeat_monitor=BrokenHeartbeatMonitor(),
|
||||
runtime_supervisor=FakeRuntimeSupervisor(),
|
||||
interval_seconds=1.0,
|
||||
@@ -552,6 +698,7 @@ def test_start_resets_running_after_supervisor_error() -> None:
|
||||
raise RuntimeError("supervisor failed")
|
||||
|
||||
scheduler = RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(),
|
||||
heartbeat_monitor=FakeHeartbeatMonitor(
|
||||
results=[True],
|
||||
),
|
||||
@@ -588,3 +735,45 @@ def test_sleep_error_is_propagated_and_resets_running() -> None:
|
||||
|
||||
assert heartbeat.check_timeout_calls == 1
|
||||
assert scheduler.running is False
|
||||
|
||||
|
||||
def test_liveness_error_is_propagated_and_resets_running() -> None:
|
||||
liveness_error = RuntimeError("liveness failed")
|
||||
scheduler = RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(
|
||||
error=liveness_error,
|
||||
),
|
||||
heartbeat_monitor=FakeHeartbeatMonitor(),
|
||||
runtime_supervisor=FakeRuntimeSupervisor(),
|
||||
interval_seconds=1.0,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="liveness failed",
|
||||
) as error_info:
|
||||
asyncio.run(
|
||||
scheduler.start()
|
||||
)
|
||||
|
||||
assert error_info.value is liveness_error
|
||||
assert scheduler.running is False
|
||||
|
||||
|
||||
def test_liveness_cancellation_is_propagated() -> None:
|
||||
cancellation = asyncio.CancelledError()
|
||||
scheduler = RuntimeScheduler(
|
||||
liveness_probe=FakeLivenessProbe(
|
||||
error=cancellation,
|
||||
),
|
||||
heartbeat_monitor=FakeHeartbeatMonitor(),
|
||||
runtime_supervisor=FakeRuntimeSupervisor(),
|
||||
interval_seconds=1.0,
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
asyncio.run(
|
||||
scheduler.start()
|
||||
)
|
||||
|
||||
assert scheduler.running is False
|
||||
|
||||
@@ -65,6 +65,7 @@ class FakeReconnectCoordinator:
|
||||
def __init__(self) -> None:
|
||||
self.reconnect_calls = 0
|
||||
self._attempt = 0
|
||||
self._generation = 0
|
||||
self._state = ReconnectState.DISCONNECTED
|
||||
|
||||
@property
|
||||
@@ -75,11 +76,26 @@ class FakeReconnectCoordinator:
|
||||
def attempt(self) -> int:
|
||||
return self._attempt
|
||||
|
||||
@property
|
||||
def generation(self) -> int:
|
||||
return self._generation
|
||||
|
||||
async def reconnect(self) -> None:
|
||||
self.reconnect_calls += 1
|
||||
self._attempt += 1
|
||||
self._generation += 1
|
||||
self._state = ReconnectState.CONNECTED
|
||||
|
||||
async def reconnect_after_transport_failure(
|
||||
self,
|
||||
*,
|
||||
observed_generation: int,
|
||||
) -> None:
|
||||
if observed_generation != self._generation:
|
||||
return
|
||||
|
||||
await self.reconnect()
|
||||
|
||||
|
||||
def create_supervisor() -> tuple[
|
||||
RuntimeSupervisor,
|
||||
@@ -271,6 +287,41 @@ def test_timeout_runs_single_reconnect_attempt() -> None:
|
||||
assert reconnect.reconnect_calls == 1
|
||||
|
||||
|
||||
def test_delayed_timeout_reuses_completed_transport_reconnect() -> None:
|
||||
async def scenario() -> tuple[
|
||||
RuntimeSupervisor,
|
||||
FakeHeartbeatMonitor,
|
||||
FakeReconnectCoordinator,
|
||||
bool,
|
||||
]:
|
||||
supervisor, heartbeat, reconnect = create_supervisor()
|
||||
supervisor.start()
|
||||
observed_generation = reconnect.generation
|
||||
|
||||
await reconnect.reconnect_after_transport_failure(
|
||||
observed_generation=observed_generation,
|
||||
)
|
||||
result = await supervisor.handle_heartbeat_timeout()
|
||||
|
||||
return (
|
||||
supervisor,
|
||||
heartbeat,
|
||||
reconnect,
|
||||
result,
|
||||
)
|
||||
|
||||
supervisor, heartbeat, reconnect, result = asyncio.run(
|
||||
scenario()
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert reconnect.reconnect_calls == 1
|
||||
assert reconnect.generation == 1
|
||||
assert heartbeat.stop_calls == 1
|
||||
assert heartbeat.start_calls == 2
|
||||
assert supervisor.state is RuntimeSupervisorState.RUNNING
|
||||
|
||||
|
||||
def test_successful_reconnect_restarts_heartbeat() -> None:
|
||||
supervisor, heartbeat, _ = create_supervisor()
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,252 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from websockets.protocol import State
|
||||
|
||||
from src.market_data.acquisition.adapters.dzengi.websocket_transport import (
|
||||
DzengiWebSocketTransport,
|
||||
)
|
||||
from src.market_data.acquisition.exceptions import (
|
||||
WebSocketTransportError,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.acquisition_runtime_service import (
|
||||
AcquisitionRuntimeService,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.reconnect import (
|
||||
ReconnectCoordinator,
|
||||
ReconnectState,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_events import (
|
||||
ReconnectCompletedEvent,
|
||||
ReconnectStartedEvent,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.transport_messages import (
|
||||
TransportTextMessage,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_session import (
|
||||
WebSocketSession,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_protocol import (
|
||||
AcquisitionRuntimeEvent,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_subscription_manager import (
|
||||
WebSocketSubscriptionManager,
|
||||
)
|
||||
|
||||
|
||||
SUBSCRIPTION_KEY = "trades:BTC/USD_LEVERAGE"
|
||||
SUBSCRIPTION_PAYLOAD = '{"destination":"trades.subscribe"}'
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
send_error: Exception | None = None,
|
||||
) -> None:
|
||||
self.state = State.OPEN
|
||||
self.send_error = send_error
|
||||
self.sent_messages: list[str | bytes] = []
|
||||
self.close_calls = 0
|
||||
|
||||
async def close(
|
||||
self,
|
||||
code: int = 1000,
|
||||
reason: str = "",
|
||||
) -> None:
|
||||
self.close_calls += 1
|
||||
self.state = State.CLOSED
|
||||
|
||||
async def send(
|
||||
self,
|
||||
message: str | bytes,
|
||||
) -> None:
|
||||
if self.send_error is not None:
|
||||
self.state = State.CLOSED
|
||||
raise self.send_error
|
||||
|
||||
self.sent_messages.append(message)
|
||||
|
||||
async def recv(self) -> str | bytes:
|
||||
return ""
|
||||
|
||||
|
||||
class RecordingConnector:
|
||||
def __init__(
|
||||
self,
|
||||
*connections: FakeConnection,
|
||||
) -> None:
|
||||
self._connections = list(connections)
|
||||
self.calls: list[
|
||||
tuple[str, dict[str, Any]]
|
||||
] = []
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
url: str,
|
||||
**kwargs: Any,
|
||||
) -> FakeConnection:
|
||||
self.calls.append(
|
||||
(
|
||||
url,
|
||||
kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
return self._connections.pop(0)
|
||||
|
||||
|
||||
class RecordingEventPublisher:
|
||||
def __init__(self) -> None:
|
||||
self.events: list[AcquisitionRuntimeEvent] = []
|
||||
|
||||
async def publish(
|
||||
self,
|
||||
event: AcquisitionRuntimeEvent,
|
||||
) -> None:
|
||||
self.events.append(event)
|
||||
|
||||
|
||||
def create_runtime(
|
||||
*connections: FakeConnection,
|
||||
) -> tuple[
|
||||
WebSocketSession,
|
||||
WebSocketSubscriptionManager,
|
||||
ReconnectCoordinator,
|
||||
RecordingConnector,
|
||||
RecordingEventPublisher,
|
||||
]:
|
||||
connector = RecordingConnector(
|
||||
*connections,
|
||||
)
|
||||
transport = DzengiWebSocketTransport(
|
||||
url="wss://api-adapter.dzengi.com",
|
||||
connector=connector,
|
||||
)
|
||||
session = WebSocketSession(
|
||||
transport,
|
||||
)
|
||||
subscriptions = WebSocketSubscriptionManager(
|
||||
transport,
|
||||
)
|
||||
publisher = RecordingEventPublisher()
|
||||
runtime_service = AcquisitionRuntimeService(
|
||||
session=session,
|
||||
transport=transport,
|
||||
subscription_manager=subscriptions,
|
||||
event_publisher=publisher,
|
||||
)
|
||||
reconnect = ReconnectCoordinator(
|
||||
command_dispatcher=runtime_service,
|
||||
subscription_manager=subscriptions,
|
||||
event_publisher=publisher,
|
||||
)
|
||||
|
||||
return (
|
||||
session,
|
||||
subscriptions,
|
||||
reconnect,
|
||||
connector,
|
||||
publisher,
|
||||
)
|
||||
|
||||
|
||||
def test_reconnect_replaces_open_connection_before_restore() -> None:
|
||||
first_connection = FakeConnection()
|
||||
second_connection = FakeConnection()
|
||||
(
|
||||
session,
|
||||
subscriptions,
|
||||
reconnect,
|
||||
connector,
|
||||
publisher,
|
||||
) = create_runtime(
|
||||
first_connection,
|
||||
second_connection,
|
||||
)
|
||||
|
||||
async def scenario() -> None:
|
||||
await session.start()
|
||||
await subscriptions.subscribe(
|
||||
SUBSCRIPTION_KEY,
|
||||
TransportTextMessage(
|
||||
payload=SUBSCRIPTION_PAYLOAD,
|
||||
),
|
||||
)
|
||||
await reconnect.reconnect()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert len(connector.calls) == 2
|
||||
assert first_connection.close_calls == 1
|
||||
assert first_connection.state is State.CLOSED
|
||||
assert first_connection.sent_messages == [
|
||||
SUBSCRIPTION_PAYLOAD,
|
||||
]
|
||||
assert second_connection.sent_messages == [
|
||||
SUBSCRIPTION_PAYLOAD,
|
||||
]
|
||||
assert session.is_connected is True
|
||||
assert reconnect.state is ReconnectState.CONNECTED
|
||||
assert publisher.events == [
|
||||
ReconnectStartedEvent(attempt=1),
|
||||
ReconnectCompletedEvent(attempt=1),
|
||||
]
|
||||
|
||||
|
||||
def test_reconnect_restores_subscription_after_initial_send_failure() -> None:
|
||||
first_connection = FakeConnection(
|
||||
send_error=RuntimeError(
|
||||
"socket dropped while subscribing",
|
||||
),
|
||||
)
|
||||
second_connection = FakeConnection()
|
||||
(
|
||||
session,
|
||||
subscriptions,
|
||||
reconnect,
|
||||
connector,
|
||||
publisher,
|
||||
) = create_runtime(
|
||||
first_connection,
|
||||
second_connection,
|
||||
)
|
||||
|
||||
async def scenario() -> None:
|
||||
await session.start()
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketTransportError,
|
||||
match="socket dropped while subscribing",
|
||||
):
|
||||
await subscriptions.subscribe(
|
||||
SUBSCRIPTION_KEY,
|
||||
TransportTextMessage(
|
||||
payload=SUBSCRIPTION_PAYLOAD,
|
||||
),
|
||||
)
|
||||
|
||||
assert subscriptions.subscription_keys == (
|
||||
SUBSCRIPTION_KEY,
|
||||
)
|
||||
|
||||
await reconnect.reconnect()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert len(connector.calls) == 2
|
||||
assert second_connection.sent_messages == [
|
||||
SUBSCRIPTION_PAYLOAD,
|
||||
]
|
||||
assert subscriptions.subscription_keys == (
|
||||
SUBSCRIPTION_KEY,
|
||||
)
|
||||
assert session.is_connected is True
|
||||
assert reconnect.state is ReconnectState.CONNECTED
|
||||
assert publisher.events == [
|
||||
ReconnectStartedEvent(attempt=1),
|
||||
ReconnectCompletedEvent(attempt=1),
|
||||
]
|
||||
@@ -0,0 +1,196 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from src.market_data.acquisition.runtime.websocket_protocol import (
|
||||
WebSocketSessionProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_session import (
|
||||
WebSocketSession,
|
||||
)
|
||||
|
||||
|
||||
class FakeTransport:
|
||||
def __init__(self) -> None:
|
||||
self.connected = False
|
||||
self.connect_calls = 0
|
||||
self.disconnect_calls = 0
|
||||
self.connect_error: Exception | None = None
|
||||
self.disconnect_error: Exception | None = None
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
return self.connected
|
||||
|
||||
async def connect(self) -> None:
|
||||
self.connect_calls += 1
|
||||
|
||||
if self.connect_error is not None:
|
||||
raise self.connect_error
|
||||
|
||||
self.connected = True
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
self.disconnect_calls += 1
|
||||
self.connected = False
|
||||
|
||||
if self.disconnect_error is not None:
|
||||
raise self.disconnect_error
|
||||
|
||||
async def send(
|
||||
self,
|
||||
message: str | bytes,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
async def receive(self) -> str | bytes:
|
||||
return ""
|
||||
|
||||
|
||||
def create_session() -> tuple[
|
||||
WebSocketSession,
|
||||
FakeTransport,
|
||||
]:
|
||||
transport = FakeTransport()
|
||||
session = WebSocketSession(
|
||||
transport,
|
||||
)
|
||||
|
||||
return (
|
||||
session,
|
||||
transport,
|
||||
)
|
||||
|
||||
|
||||
def test_session_implements_protocol() -> None:
|
||||
session, _ = create_session()
|
||||
|
||||
assert isinstance(
|
||||
session,
|
||||
WebSocketSessionProtocol,
|
||||
)
|
||||
|
||||
|
||||
def test_session_uses_slots() -> None:
|
||||
session, _ = create_session()
|
||||
|
||||
assert not hasattr(session, "__dict__")
|
||||
|
||||
|
||||
def test_session_is_initially_disconnected() -> None:
|
||||
session, _ = create_session()
|
||||
|
||||
assert session.is_connected is False
|
||||
|
||||
|
||||
def test_start_connects_transport() -> None:
|
||||
session, transport = create_session()
|
||||
|
||||
asyncio.run(session.start())
|
||||
|
||||
assert session.is_connected is True
|
||||
assert transport.connect_calls == 1
|
||||
|
||||
|
||||
def test_start_is_idempotent() -> None:
|
||||
session, transport = create_session()
|
||||
|
||||
async def scenario() -> None:
|
||||
await session.start()
|
||||
await session.start()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert session.is_connected is True
|
||||
assert transport.connect_calls == 1
|
||||
|
||||
|
||||
def test_concurrent_start_creates_one_connection() -> None:
|
||||
session, transport = create_session()
|
||||
|
||||
async def scenario() -> None:
|
||||
await asyncio.gather(
|
||||
session.start(),
|
||||
session.start(),
|
||||
)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert session.is_connected is True
|
||||
assert transport.connect_calls == 1
|
||||
|
||||
|
||||
def test_start_error_leaves_session_disconnected() -> None:
|
||||
session, transport = create_session()
|
||||
transport.connect_error = RuntimeError("start failed")
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="start failed",
|
||||
):
|
||||
asyncio.run(session.start())
|
||||
|
||||
assert session.is_connected is False
|
||||
|
||||
|
||||
def test_start_reconnects_after_remote_disconnect() -> None:
|
||||
session, transport = create_session()
|
||||
|
||||
async def scenario() -> None:
|
||||
await session.start()
|
||||
transport.connected = False
|
||||
|
||||
assert session.is_connected is False
|
||||
|
||||
await session.start()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert session.is_connected is True
|
||||
assert transport.connect_calls == 2
|
||||
|
||||
|
||||
def test_stop_disconnects_transport() -> None:
|
||||
session, transport = create_session()
|
||||
|
||||
async def scenario() -> None:
|
||||
await session.start()
|
||||
await session.stop()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert session.is_connected is False
|
||||
assert transport.disconnect_calls == 1
|
||||
|
||||
|
||||
def test_stop_is_idempotent() -> None:
|
||||
session, transport = create_session()
|
||||
|
||||
async def scenario() -> None:
|
||||
await session.start()
|
||||
await session.stop()
|
||||
await session.stop()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert transport.disconnect_calls == 1
|
||||
assert session.is_connected is False
|
||||
|
||||
|
||||
def test_stop_error_still_clears_session_state() -> None:
|
||||
session, transport = create_session()
|
||||
transport.disconnect_error = RuntimeError("stop failed")
|
||||
|
||||
async def scenario() -> None:
|
||||
await session.start()
|
||||
await session.stop()
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="stop failed",
|
||||
):
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert session.is_connected is False
|
||||
@@ -0,0 +1,419 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from src.market_data.acquisition.exceptions import (
|
||||
WebSocketUnsubscribeNotSupportedError,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.transport_messages import (
|
||||
TransportBinaryMessage,
|
||||
TransportTextMessage,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_protocol import (
|
||||
WebSocketSubscriptionManagerProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.websocket_subscription_manager import (
|
||||
WebSocketSubscriptionManager,
|
||||
)
|
||||
|
||||
|
||||
class RecordingTransport:
|
||||
def __init__(self) -> None:
|
||||
self.messages: list[str | bytes] = []
|
||||
self.send_error: Exception | None = None
|
||||
|
||||
async def connect(self) -> None:
|
||||
return None
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
return None
|
||||
|
||||
async def send(
|
||||
self,
|
||||
message: str | bytes,
|
||||
) -> None:
|
||||
if self.send_error is not None:
|
||||
raise self.send_error
|
||||
|
||||
self.messages.append(message)
|
||||
|
||||
async def receive(self) -> str | bytes:
|
||||
return ""
|
||||
|
||||
|
||||
def create_manager(
|
||||
*,
|
||||
supports_unsubscribe: bool = False,
|
||||
) -> tuple[
|
||||
WebSocketSubscriptionManager,
|
||||
RecordingTransport,
|
||||
]:
|
||||
transport = RecordingTransport()
|
||||
manager = WebSocketSubscriptionManager(
|
||||
transport,
|
||||
supports_unsubscribe=supports_unsubscribe,
|
||||
)
|
||||
|
||||
return (
|
||||
manager,
|
||||
transport,
|
||||
)
|
||||
|
||||
|
||||
def test_manager_implements_protocol() -> None:
|
||||
manager, _ = create_manager()
|
||||
|
||||
assert isinstance(
|
||||
manager,
|
||||
WebSocketSubscriptionManagerProtocol,
|
||||
)
|
||||
|
||||
|
||||
def test_manager_uses_slots() -> None:
|
||||
manager, _ = create_manager()
|
||||
|
||||
assert not hasattr(manager, "__dict__")
|
||||
|
||||
|
||||
def test_manager_starts_with_empty_registry() -> None:
|
||||
manager, _ = create_manager()
|
||||
|
||||
assert manager.subscription_keys == ()
|
||||
|
||||
|
||||
def test_subscribe_sends_and_registers_text_message() -> None:
|
||||
manager, transport = create_manager()
|
||||
message = TransportTextMessage(
|
||||
payload='{"destination":"trades.subscribe"}',
|
||||
)
|
||||
|
||||
asyncio.run(
|
||||
manager.subscribe(
|
||||
"trades:BTC",
|
||||
message,
|
||||
)
|
||||
)
|
||||
|
||||
assert transport.messages == [
|
||||
message.payload,
|
||||
]
|
||||
assert manager.subscription_keys == (
|
||||
"trades:BTC",
|
||||
)
|
||||
|
||||
|
||||
def test_subscribe_sends_binary_message() -> None:
|
||||
manager, transport = create_manager()
|
||||
message = TransportBinaryMessage(
|
||||
payload=b"\x01\x02",
|
||||
)
|
||||
|
||||
asyncio.run(
|
||||
manager.subscribe(
|
||||
"binary",
|
||||
message,
|
||||
)
|
||||
)
|
||||
|
||||
assert transport.messages == [
|
||||
b"\x01\x02",
|
||||
]
|
||||
assert manager.subscription_keys == (
|
||||
"binary",
|
||||
)
|
||||
|
||||
|
||||
def test_duplicate_subscription_key_is_idempotent() -> None:
|
||||
manager, transport = create_manager()
|
||||
first_message = TransportTextMessage(
|
||||
payload="first",
|
||||
)
|
||||
second_message = TransportTextMessage(
|
||||
payload="second",
|
||||
)
|
||||
|
||||
async def scenario() -> None:
|
||||
await manager.subscribe(
|
||||
"trades:BTC",
|
||||
first_message,
|
||||
)
|
||||
await manager.subscribe(
|
||||
"trades:BTC",
|
||||
second_message,
|
||||
)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert transport.messages == [
|
||||
"first",
|
||||
]
|
||||
assert manager.subscription_keys == (
|
||||
"trades:BTC",
|
||||
)
|
||||
|
||||
|
||||
def test_failed_subscribe_remains_registered_as_pending() -> None:
|
||||
manager, transport = create_manager()
|
||||
transport.send_error = RuntimeError("send failed")
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="send failed",
|
||||
):
|
||||
asyncio.run(
|
||||
manager.subscribe(
|
||||
"trades:BTC",
|
||||
TransportTextMessage(
|
||||
payload="subscribe",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert manager.subscription_keys == (
|
||||
"trades:BTC",
|
||||
)
|
||||
|
||||
|
||||
def test_pending_subscription_can_be_retried() -> None:
|
||||
manager, transport = create_manager()
|
||||
transport.send_error = RuntimeError("send failed")
|
||||
|
||||
async def scenario() -> None:
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="send failed",
|
||||
):
|
||||
await manager.subscribe(
|
||||
"trades:BTC",
|
||||
TransportTextMessage(
|
||||
payload="first-attempt",
|
||||
),
|
||||
)
|
||||
|
||||
transport.send_error = None
|
||||
|
||||
await manager.subscribe(
|
||||
"trades:BTC",
|
||||
TransportTextMessage(
|
||||
payload="second-attempt",
|
||||
),
|
||||
)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert transport.messages == [
|
||||
"second-attempt",
|
||||
]
|
||||
assert manager.subscription_keys == (
|
||||
"trades:BTC",
|
||||
)
|
||||
|
||||
|
||||
def test_restore_sends_registered_messages_in_order() -> None:
|
||||
manager, transport = create_manager()
|
||||
|
||||
async def scenario() -> None:
|
||||
await manager.subscribe(
|
||||
"first",
|
||||
TransportTextMessage(
|
||||
payload="first-message",
|
||||
),
|
||||
)
|
||||
await manager.subscribe(
|
||||
"second",
|
||||
TransportBinaryMessage(
|
||||
payload=b"second-message",
|
||||
),
|
||||
)
|
||||
|
||||
transport.messages.clear()
|
||||
await manager.restore_subscriptions()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert transport.messages == [
|
||||
"first-message",
|
||||
b"second-message",
|
||||
]
|
||||
|
||||
|
||||
def test_restore_error_preserves_registry() -> None:
|
||||
manager, transport = create_manager()
|
||||
|
||||
async def prepare() -> None:
|
||||
await manager.subscribe(
|
||||
"trades:BTC",
|
||||
TransportTextMessage(
|
||||
payload="subscribe",
|
||||
),
|
||||
)
|
||||
|
||||
asyncio.run(prepare())
|
||||
transport.send_error = RuntimeError("restore failed")
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="restore failed",
|
||||
):
|
||||
asyncio.run(manager.restore_subscriptions())
|
||||
|
||||
assert manager.subscription_keys == (
|
||||
"trades:BTC",
|
||||
)
|
||||
|
||||
|
||||
def test_unsubscribe_is_explicitly_unsupported_by_default() -> None:
|
||||
manager, transport = create_manager()
|
||||
|
||||
async def scenario() -> None:
|
||||
await manager.subscribe(
|
||||
"trades:BTC",
|
||||
TransportTextMessage(
|
||||
payload="subscribe",
|
||||
),
|
||||
)
|
||||
await manager.unsubscribe(
|
||||
"trades:BTC",
|
||||
TransportTextMessage(
|
||||
payload="unsubscribe",
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
WebSocketUnsubscribeNotSupportedError,
|
||||
):
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert transport.messages == [
|
||||
"subscribe",
|
||||
]
|
||||
assert manager.subscription_keys == (
|
||||
"trades:BTC",
|
||||
)
|
||||
|
||||
|
||||
def test_supported_unsubscribe_sends_and_removes_subscription() -> None:
|
||||
manager, transport = create_manager(
|
||||
supports_unsubscribe=True,
|
||||
)
|
||||
|
||||
async def scenario() -> None:
|
||||
await manager.subscribe(
|
||||
"generic",
|
||||
TransportTextMessage(
|
||||
payload="subscribe",
|
||||
),
|
||||
)
|
||||
await manager.unsubscribe(
|
||||
"generic",
|
||||
TransportTextMessage(
|
||||
payload="unsubscribe",
|
||||
),
|
||||
)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert transport.messages == [
|
||||
"subscribe",
|
||||
"unsubscribe",
|
||||
]
|
||||
assert manager.subscription_keys == ()
|
||||
|
||||
|
||||
def test_supported_unsubscribe_is_idempotent_for_missing_key() -> None:
|
||||
manager, transport = create_manager(
|
||||
supports_unsubscribe=True,
|
||||
)
|
||||
|
||||
asyncio.run(
|
||||
manager.unsubscribe(
|
||||
"missing",
|
||||
TransportTextMessage(
|
||||
payload="unsubscribe",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert transport.messages == []
|
||||
assert manager.subscription_keys == ()
|
||||
|
||||
|
||||
def test_clear_removes_registry_without_sending_messages() -> None:
|
||||
manager, transport = create_manager()
|
||||
|
||||
async def scenario() -> None:
|
||||
await manager.subscribe(
|
||||
"trades:BTC",
|
||||
TransportTextMessage(
|
||||
payload="subscribe",
|
||||
),
|
||||
)
|
||||
|
||||
transport.messages.clear()
|
||||
await manager.clear_subscriptions()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert manager.subscription_keys == ()
|
||||
assert transport.messages == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"subscription_key",
|
||||
[
|
||||
"",
|
||||
" ",
|
||||
],
|
||||
)
|
||||
def test_rejects_empty_subscription_key(
|
||||
subscription_key: str,
|
||||
) -> None:
|
||||
manager, _ = create_manager()
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
asyncio.run(
|
||||
manager.subscribe(
|
||||
subscription_key,
|
||||
TransportTextMessage(
|
||||
payload="subscribe",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_rejects_non_string_subscription_key() -> None:
|
||||
manager, _ = create_manager()
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
asyncio.run(
|
||||
manager.subscribe(
|
||||
123, # type: ignore[arg-type]
|
||||
TransportTextMessage(
|
||||
payload="subscribe",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_rejects_unsupported_message_type() -> None:
|
||||
manager, _ = create_manager()
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
asyncio.run(
|
||||
manager.subscribe(
|
||||
"trades:BTC",
|
||||
object(), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_rejects_non_boolean_unsubscribe_capability() -> None:
|
||||
transport = RecordingTransport()
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
WebSocketSubscriptionManager(
|
||||
transport,
|
||||
supports_unsubscribe="yes", # type: ignore[arg-type]
|
||||
)
|
||||
@@ -28,6 +28,12 @@ from src.market_data.acquisition.runtime.reconnect import (
|
||||
ReconnectCoordinatorProtocol,
|
||||
ReconnectState,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_reconnect_recovery_coordinator import (
|
||||
RuntimeReconnectRecoveryProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_liveness_probe import (
|
||||
RuntimeLivenessProbeProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.runtime_events import (
|
||||
ReconnectCompletedEvent,
|
||||
ReconnectStartedEvent,
|
||||
@@ -123,11 +129,17 @@ class FakeSession:
|
||||
|
||||
|
||||
class FakeTransport:
|
||||
def __init__(self) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
probe_results: tuple[bool, ...] = (True,),
|
||||
) -> None:
|
||||
self.connect_calls = 0
|
||||
self.disconnect_calls = 0
|
||||
self.sent_messages: list[str | bytes] = []
|
||||
self.receive_calls = 0
|
||||
self.probe_calls = 0
|
||||
self._probe_results = list(probe_results)
|
||||
|
||||
async def connect(self) -> None:
|
||||
self.connect_calls += 1
|
||||
@@ -145,6 +157,14 @@ class FakeTransport:
|
||||
self.receive_calls += 1
|
||||
return ""
|
||||
|
||||
async def probe(self) -> bool:
|
||||
self.probe_calls += 1
|
||||
|
||||
if not self._probe_results:
|
||||
return True
|
||||
|
||||
return self._probe_results.pop(0)
|
||||
|
||||
|
||||
class FakeSubscriptionManager:
|
||||
def __init__(self) -> None:
|
||||
@@ -264,6 +284,25 @@ class FakeClock:
|
||||
def __call__(self) -> float:
|
||||
return self.value
|
||||
|
||||
def advance(
|
||||
self,
|
||||
seconds: float,
|
||||
) -> None:
|
||||
self.value += seconds
|
||||
|
||||
|
||||
class FakeUnixTimeClock:
|
||||
def __init__(
|
||||
self,
|
||||
value: int = RECOVERY_END_TIME_MS,
|
||||
) -> None:
|
||||
self.value = value
|
||||
self.calls = 0
|
||||
|
||||
def __call__(self) -> int:
|
||||
self.calls += 1
|
||||
return self.value
|
||||
|
||||
|
||||
class RecordingSleep:
|
||||
def __init__(self) -> None:
|
||||
@@ -285,6 +324,7 @@ class CompositionDependencies:
|
||||
message_adapter: FakeMessageAdapter
|
||||
recovery_document_source: StubTradesDocumentSource
|
||||
heartbeat_clock: FakeClock
|
||||
recovery_end_time_clock: FakeUnixTimeClock
|
||||
scheduler_sleep: RecordingSleep
|
||||
|
||||
|
||||
@@ -295,13 +335,16 @@ def create_composition(
|
||||
heartbeat_timeout_seconds: float = 10.0,
|
||||
scheduler_interval_seconds: float = 1.0,
|
||||
max_recovery_window_ms: int = 3_599_999,
|
||||
probe_results: tuple[bool, ...] = (True,),
|
||||
) -> tuple[
|
||||
TradeStreamRuntimeComposition,
|
||||
CompositionDependencies,
|
||||
]:
|
||||
dependencies = CompositionDependencies(
|
||||
session=FakeSession(),
|
||||
transport=FakeTransport(),
|
||||
transport=FakeTransport(
|
||||
probe_results=probe_results,
|
||||
),
|
||||
subscription_manager=FakeSubscriptionManager(),
|
||||
event_publisher=FakeEventPublisher(),
|
||||
message_adapter=FakeMessageAdapter(
|
||||
@@ -311,6 +354,7 @@ def create_composition(
|
||||
recovery_document,
|
||||
),
|
||||
heartbeat_clock=FakeClock(),
|
||||
recovery_end_time_clock=FakeUnixTimeClock(),
|
||||
scheduler_sleep=RecordingSleep(),
|
||||
)
|
||||
|
||||
@@ -323,10 +367,14 @@ def create_composition(
|
||||
recovery_document_source=(
|
||||
dependencies.recovery_document_source
|
||||
),
|
||||
symbols=(SYMBOL,),
|
||||
heartbeat_timeout_seconds=heartbeat_timeout_seconds,
|
||||
scheduler_interval_seconds=scheduler_interval_seconds,
|
||||
max_recovery_window_ms=max_recovery_window_ms,
|
||||
heartbeat_clock=dependencies.heartbeat_clock,
|
||||
recovery_end_time_clock=(
|
||||
dependencies.recovery_end_time_clock
|
||||
),
|
||||
scheduler_sleep=dependencies.scheduler_sleep,
|
||||
)
|
||||
|
||||
@@ -373,6 +421,14 @@ def test_components_implement_public_protocols() -> None:
|
||||
composition.reconnect_coordinator,
|
||||
ReconnectCoordinatorProtocol,
|
||||
)
|
||||
assert isinstance(
|
||||
composition.runtime_reconnect_recovery_coordinator,
|
||||
RuntimeReconnectRecoveryProtocol,
|
||||
)
|
||||
assert isinstance(
|
||||
composition.liveness_probe,
|
||||
RuntimeLivenessProbeProtocol,
|
||||
)
|
||||
assert isinstance(
|
||||
composition.heartbeat_monitor,
|
||||
HeartbeatMonitorProtocol,
|
||||
@@ -460,8 +516,27 @@ def test_runtime_components_share_lifecycle_dependencies() -> None:
|
||||
)
|
||||
assert (
|
||||
composition.runtime_supervisor._reconnect_coordinator
|
||||
is composition.runtime_reconnect_recovery_coordinator
|
||||
)
|
||||
assert (
|
||||
composition.runtime_reconnect_recovery_coordinator
|
||||
._reconnect_coordinator
|
||||
is composition.reconnect_coordinator
|
||||
)
|
||||
assert (
|
||||
composition.runtime_reconnect_recovery_coordinator
|
||||
._recovery_coordinator
|
||||
is composition.runtime_recovery_coordinator
|
||||
)
|
||||
assert (
|
||||
composition.runtime_reconnect_recovery_coordinator
|
||||
.live_processing_gate
|
||||
is composition.live_processing_gate
|
||||
)
|
||||
assert (
|
||||
composition.runtime_reconnect_recovery_coordinator.symbols
|
||||
== (SYMBOL,)
|
||||
)
|
||||
assert (
|
||||
composition.runtime_scheduler._heartbeat_monitor
|
||||
is composition.heartbeat_monitor
|
||||
@@ -470,6 +545,14 @@ def test_runtime_components_share_lifecycle_dependencies() -> None:
|
||||
composition.runtime_scheduler._runtime_supervisor
|
||||
is composition.runtime_supervisor
|
||||
)
|
||||
assert (
|
||||
composition.liveness_probe
|
||||
is dependencies.transport
|
||||
)
|
||||
assert (
|
||||
composition.runtime_scheduler._liveness_probe
|
||||
is composition.liveness_probe
|
||||
)
|
||||
|
||||
|
||||
def test_configuration_is_forwarded() -> None:
|
||||
@@ -507,6 +590,7 @@ def test_creation_has_no_runtime_side_effects() -> None:
|
||||
assert dependencies.transport.disconnect_calls == 0
|
||||
assert dependencies.transport.sent_messages == []
|
||||
assert dependencies.transport.receive_calls == 0
|
||||
assert dependencies.transport.probe_calls == 0
|
||||
|
||||
assert dependencies.subscription_manager.subscriptions == []
|
||||
assert dependencies.subscription_manager.unsubscriptions == []
|
||||
@@ -515,6 +599,7 @@ def test_creation_has_no_runtime_side_effects() -> None:
|
||||
|
||||
assert dependencies.event_publisher.events == []
|
||||
assert dependencies.recovery_document_source.calls == []
|
||||
assert dependencies.recovery_end_time_clock.calls == 0
|
||||
|
||||
assert composition.heartbeat_monitor.state is HeartbeatState.IDLE
|
||||
assert (
|
||||
@@ -528,6 +613,103 @@ def test_creation_has_no_runtime_side_effects() -> None:
|
||||
assert composition.runtime_scheduler.running is False
|
||||
|
||||
|
||||
def test_successful_probe_keeps_quiet_connection_alive() -> None:
|
||||
composition, dependencies = create_composition(
|
||||
heartbeat_timeout_seconds=10.0,
|
||||
probe_results=(True,),
|
||||
)
|
||||
composition.runtime_supervisor.start()
|
||||
dependencies.heartbeat_clock.advance(10.0)
|
||||
|
||||
timed_out = asyncio.run(
|
||||
composition.runtime_scheduler.run_once()
|
||||
)
|
||||
|
||||
assert timed_out is False
|
||||
assert dependencies.transport.probe_calls == 1
|
||||
assert dependencies.session.start_calls == 0
|
||||
assert dependencies.session.stop_calls == 0
|
||||
assert composition.heartbeat_monitor.last_activity_at == 110.0
|
||||
assert (
|
||||
composition.runtime_supervisor.state
|
||||
is RuntimeSupervisorState.RUNNING
|
||||
)
|
||||
|
||||
|
||||
def test_failed_probe_triggers_reconnect_after_timeout() -> None:
|
||||
composition, dependencies = create_composition(
|
||||
heartbeat_timeout_seconds=10.0,
|
||||
probe_results=(False,),
|
||||
)
|
||||
composition.runtime_supervisor.start()
|
||||
dependencies.heartbeat_clock.advance(10.0)
|
||||
|
||||
timed_out = asyncio.run(
|
||||
composition.runtime_scheduler.run_once()
|
||||
)
|
||||
|
||||
assert timed_out is True
|
||||
assert dependencies.transport.probe_calls == 1
|
||||
assert dependencies.session.stop_calls == 1
|
||||
assert dependencies.session.start_calls == 1
|
||||
assert dependencies.subscription_manager.restore_calls == 1
|
||||
assert dependencies.recovery_end_time_clock.calls == 1
|
||||
assert (
|
||||
composition.runtime_supervisor.state
|
||||
is RuntimeSupervisorState.RUNNING
|
||||
)
|
||||
|
||||
|
||||
def test_delayed_timeout_does_not_repeat_completed_transport_reconnect() -> None:
|
||||
async def scenario() -> tuple[
|
||||
TradeStreamRuntimeComposition,
|
||||
CompositionDependencies,
|
||||
bool,
|
||||
]:
|
||||
composition, dependencies = create_composition(
|
||||
heartbeat_timeout_seconds=10.0,
|
||||
probe_results=(False,),
|
||||
)
|
||||
composition.runtime_supervisor.start()
|
||||
observed_generation = (
|
||||
composition.runtime_reconnect_recovery_coordinator.generation
|
||||
)
|
||||
|
||||
await (
|
||||
composition.runtime_reconnect_recovery_coordinator
|
||||
.reconnect_after_transport_failure(
|
||||
observed_generation=observed_generation,
|
||||
)
|
||||
)
|
||||
dependencies.heartbeat_clock.advance(10.0)
|
||||
timed_out = await composition.runtime_scheduler.run_once()
|
||||
|
||||
return (
|
||||
composition,
|
||||
dependencies,
|
||||
timed_out,
|
||||
)
|
||||
|
||||
composition, dependencies, timed_out = asyncio.run(
|
||||
scenario()
|
||||
)
|
||||
|
||||
assert timed_out is True
|
||||
assert dependencies.transport.probe_calls == 1
|
||||
assert dependencies.session.stop_calls == 1
|
||||
assert dependencies.session.start_calls == 1
|
||||
assert dependencies.subscription_manager.restore_calls == 1
|
||||
assert dependencies.recovery_end_time_clock.calls == 1
|
||||
assert (
|
||||
composition.runtime_reconnect_recovery_coordinator.generation
|
||||
== 1
|
||||
)
|
||||
assert (
|
||||
composition.runtime_supervisor.state
|
||||
is RuntimeSupervisorState.RUNNING
|
||||
)
|
||||
|
||||
|
||||
def test_live_checkpoint_is_visible_to_runtime_recovery() -> None:
|
||||
checkpoint_trade = make_trade()
|
||||
|
||||
@@ -642,6 +824,35 @@ def test_reconnect_uses_composed_runtime_dependencies() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_reconnect_runs_recovery_after_subscription_restore() -> None:
|
||||
composition, dependencies = create_composition(
|
||||
recovery_document=[],
|
||||
)
|
||||
|
||||
composition.trade_stream_acquisition_service.handle_message(
|
||||
{
|
||||
"destination": "internal.trade",
|
||||
}
|
||||
)
|
||||
|
||||
asyncio.run(
|
||||
composition.runtime_reconnect_recovery_coordinator.reconnect()
|
||||
)
|
||||
|
||||
assert dependencies.session.stop_calls == 1
|
||||
assert dependencies.session.start_calls == 1
|
||||
assert dependencies.subscription_manager.restore_calls == 1
|
||||
assert dependencies.recovery_end_time_clock.calls == 1
|
||||
assert dependencies.recovery_document_source.calls == [
|
||||
(
|
||||
SYMBOL,
|
||||
CHECKPOINT_TIME_MS,
|
||||
RECOVERY_END_TIME_MS,
|
||||
None,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_separate_compositions_have_independent_state() -> None:
|
||||
first, _ = create_composition()
|
||||
second, _ = create_composition()
|
||||
@@ -690,6 +901,7 @@ def test_invalid_heartbeat_configuration_is_not_hidden(
|
||||
[],
|
||||
),
|
||||
heartbeat_clock=FakeClock(),
|
||||
recovery_end_time_clock=FakeUnixTimeClock(),
|
||||
scheduler_sleep=RecordingSleep(),
|
||||
)
|
||||
|
||||
@@ -705,9 +917,13 @@ def test_invalid_heartbeat_configuration_is_not_hidden(
|
||||
recovery_document_source=(
|
||||
dependencies.recovery_document_source
|
||||
),
|
||||
symbols=(SYMBOL,),
|
||||
heartbeat_timeout_seconds=heartbeat_timeout_seconds, # type: ignore[arg-type]
|
||||
scheduler_interval_seconds=1.0,
|
||||
heartbeat_clock=dependencies.heartbeat_clock,
|
||||
recovery_end_time_clock=(
|
||||
dependencies.recovery_end_time_clock
|
||||
),
|
||||
scheduler_sleep=dependencies.scheduler_sleep,
|
||||
)
|
||||
|
||||
|
||||
33
app/tests/unit/test_main.py
Normal file
33
app/tests/unit/test_main.py
Normal file
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import src.main as main_module
|
||||
|
||||
|
||||
def test_main_runs_application_composition(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
application = object()
|
||||
received_applications: list[object] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
main_module,
|
||||
"create_app",
|
||||
lambda: application,
|
||||
)
|
||||
|
||||
async def run(received_application: object) -> None:
|
||||
received_applications.append(
|
||||
received_application,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
main_module,
|
||||
"run_application",
|
||||
run,
|
||||
)
|
||||
|
||||
asyncio.run(main_module.main())
|
||||
|
||||
assert received_applications == [application]
|
||||
Reference in New Issue
Block a user