gitadmin/dzentra_bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build 060.27: implement Persistent Market Data Storage

Browse Source
This commit is contained in:
Sergey
2026-08-01 03:22:25 +03:00
parent cb8acfe5fe
commit 58e5a12a4d
54 changed files with 10243 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
app/tests/unit/market_data/acquisition/consistency/test_trade_stream_consistency_controller.py
View File

@@ -10,6 +10,9 @@ import pytest
from src.market_data.acquisition.consistency.trade_stream_consistency_controller import (
TradeStreamConsistencyController,
)
from src.market_data.acquisition.consistency.trade_observation_sink_protocol import (
TradeObservationSinkProtocol,
)
from src.market_data.acquisition.consistency.trade_stream_exceptions import (
TradeConsistencyError,
TradeOrderingError,
@@ -47,6 +50,7 @@ def _trade(
price: Decimal = Decimal("50000.00"),
quantity: Decimal = Decimal("0.25"),
aggressor_side: TradeAggressorSide = TradeAggressorSide.BUY,
source: str = "dzengi",
) -> Trade:
return Trade(
symbol=symbol,
@@ -62,10 +66,29 @@ def _trade(
tzinfo=timezone.utc,
),
aggressor_side=aggressor_side,
source="dzengi",
source=source,
)
class RecordingTradeObservationSink:
def __init__(
self,
*,
error: Exception | None = None,
) -> None:
self.error = error
self.observations: list[Trade] = []
def persist(
self,
trade: Trade,
) -> None:
self.observations.append(trade)
if self.error is not None:
raise self.error
def test_creates_state_for_first_symbol(
controller: TradeStreamConsistencyController,
state_store: TradeStreamStateStore,
@@ -179,4 +202,128 @@ def test_propagates_consistency_error(
trade_id=100,
price=Decimal("50001.00"),
),
)
)
def test_persists_trade_before_advancing_checkpoint(
state_store: TradeStreamStateStore,
) -> None:
sink = RecordingTradeObservationSink()
controller = TradeStreamConsistencyController(
state_store=state_store,
trade_observation_sink=sink,
)
trade = _trade()
result = controller.accept(trade)
state = state_store.get(trade.symbol)
assert result is trade
assert sink.observations == [trade]
assert state.last_trade is trade
def test_persistence_failure_leaves_checkpoint_unchanged(
state_store: TradeStreamStateStore,
) -> None:
storage_error = RuntimeError("storage failed")
sink = RecordingTradeObservationSink()
controller = TradeStreamConsistencyController(
state_store=state_store,
trade_observation_sink=sink,
)
first_trade = _trade(trade_id=100)
failed_trade = _trade(trade_id=101)
controller.accept(first_trade)
sink.error = storage_error
with pytest.raises(
RuntimeError,
match="storage failed",
) as error_info:
controller.accept(failed_trade)
state = state_store.get(first_trade.symbol)
assert error_info.value is storage_error
assert state.last_trade is first_trade
assert state.last_trade_id == first_trade.trade_id
assert failed_trade.trade_id not in state._trades
sink.error = None
assert controller.accept(failed_trade) is failed_trade
def test_valid_duplicate_is_persisted_without_checkpoint_advance(
state_store: TradeStreamStateStore,
) -> None:
sink = RecordingTradeObservationSink()
controller = TradeStreamConsistencyController(
state_store=state_store,
trade_observation_sink=sink,
)
websocket_trade = _trade(
source="dzengi_websocket_trade",
)
rest_duplicate = _trade(
source="dzengi",
)
controller.accept(websocket_trade)
result = controller.accept(rest_duplicate)
state = state_store.get(websocket_trade.symbol)
assert result is None
assert sink.observations == [
websocket_trade,
rest_duplicate,
]
assert state.last_trade is websocket_trade
assert state.last_trade_id == websocket_trade.trade_id
def test_invalid_trades_do_not_reach_persistence_sink(
state_store: TradeStreamStateStore,
) -> None:
sink = RecordingTradeObservationSink()
controller = TradeStreamConsistencyController(
state_store=state_store,
trade_observation_sink=sink,
)
first_trade = _trade(trade_id=100)
controller.accept(first_trade)
with pytest.raises(TradeOrderingError):
controller.accept(_trade(trade_id=99))
with pytest.raises(TradeConsistencyError):
controller.accept(
_trade(
trade_id=100,
price=Decimal("50001.00"),
)
)
assert sink.observations == [first_trade]
def test_rejects_invalid_trade_observation_sink(
state_store: TradeStreamStateStore,
) -> None:
with pytest.raises(
TypeError,
match="TradeObservationSinkProtocol",
):
TradeStreamConsistencyController(
state_store=state_store,
trade_observation_sink=object(), # type: ignore[arg-type]
)
def test_recording_sink_implements_public_protocol() -> None:
assert isinstance(
RecordingTradeObservationSink(),
TradeObservationSinkProtocol,
)

101
app/tests/unit/market_data/acquisition/runtime/test_trade_stream_production_runtime.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import json
import threading
from collections import deque
from collections.abc import Awaitable, Callable
from typing import cast
@@ -376,15 +377,20 @@ class FakeTradeStreamService:
subscribe_error: Exception | None = None,
handle_error: Exception | None = None,
subscribe_gate: asyncio.Event | None = None,
handle_entered: threading.Event | None = None,
handle_release: threading.Event | None = None,
) -> None:
self._calls = calls
self._subscribe_error = subscribe_error
self._handle_error = handle_error
self._subscribe_gate = subscribe_gate
self._handle_entered = handle_entered
self._handle_release = handle_release
self.subscribe_calls: list[tuple[str, ...]] = []
self.subscribe_correlation_ids: list[str | None] = []
self.documents: list[object] = []
self.handled = asyncio.Event()
self.handled = threading.Event()
self.handle_thread_ids: list[int] = []
self.subscribe_entered = asyncio.Event()
async def subscribe(
@@ -408,10 +414,17 @@ class FakeTradeStreamService:
self,
document: object,
) -> Trade | None:
self.handle_thread_ids.append(threading.get_ident())
self.documents.append(document)
self._calls.append("service.handle_message")
self.handled.set()
if self._handle_entered is not None:
self._handle_entered.set()
if self._handle_release is not None:
self._handle_release.wait()
if self._handle_error is not None:
raise self._handle_error
@@ -592,6 +605,8 @@ class RuntimeDependencies:
stop_error: Exception | None = None,
subscribe_error: Exception | None = None,
handle_error: Exception | None = None,
handle_entered: threading.Event | None = None,
handle_release: threading.Event | None = None,
clear_error: Exception | None = None,
start_gate: asyncio.Event | None = None,
connected_publish_gate: asyncio.Event | None = None,
@@ -633,6 +648,8 @@ class RuntimeDependencies:
subscribe_error=subscribe_error,
handle_error=handle_error,
subscribe_gate=subscribe_gate,
handle_entered=handle_entered,
handle_release=handle_release,
)
self.live_processing_gate = RuntimeLiveProcessingGate()
self.reconnect_recovery = FakeReconnectRecoveryCoordinator(
@@ -709,6 +726,16 @@ async def wait_until(
raise AssertionError("condition was not reached")
async def wait_for_thread_event(
event: threading.Event,
) -> None:
reached = await asyncio.to_thread(
event.wait,
1.0,
)
assert reached is True
def test_implements_public_protocol_and_uses_slots() -> None:
dependencies = RuntimeDependencies()
@@ -944,7 +971,7 @@ def test_decodes_and_forwards_market_messages(
dependencies.runtime.run(),
)
await dependencies.service.handled.wait()
await wait_for_thread_event(dependencies.service.handled)
await dependencies.runtime.stop()
await runtime_task
@@ -1295,7 +1322,10 @@ def test_transport_error_runs_recovery_and_resumes_receive_loop() -> None:
dependencies.runtime.run(),
)
await dependencies.service.handled.wait()
await wait_for_thread_event(dependencies.service.handled)
await wait_until(
lambda: dependencies.transport.receive_calls == 3
)
assert dependencies.runtime.running is True
await dependencies.runtime.stop()
@@ -1342,7 +1372,7 @@ def test_buffered_market_waits_for_reconnect_recovery() -> None:
assert dependencies.service.documents == []
reconnect_release.set()
await dependencies.service.handled.wait()
await wait_for_thread_event(dependencies.service.handled)
await dependencies.runtime.stop()
await runtime_task
@@ -1568,6 +1598,69 @@ def test_market_handler_error_is_terminal_and_not_wrapped() -> None:
assert dependencies.runtime.state is (
TradeStreamProductionRuntimeState.FAILED
)
assert dependencies.live_processing_gate.failed is True
def test_market_processing_runs_outside_event_loop_thread() -> None:
async def scenario() -> tuple[RuntimeDependencies, int]:
event_loop_thread_id = threading.get_ident()
dependencies = RuntimeDependencies(
incoming=(MARKET_MESSAGE,),
)
runtime_task = asyncio.create_task(
dependencies.runtime.run(),
)
await wait_for_thread_event(dependencies.service.handled)
await dependencies.runtime.stop()
await runtime_task
return dependencies, event_loop_thread_id
dependencies, event_loop_thread_id = asyncio.run(scenario())
assert dependencies.service.handle_thread_ids
assert dependencies.service.handle_thread_ids[0] != event_loop_thread_id
def test_stop_waits_for_inflight_market_processing() -> None:
async def scenario() -> RuntimeDependencies:
handle_entered = threading.Event()
handle_release = threading.Event()
dependencies = RuntimeDependencies(
incoming=(MARKET_MESSAGE,),
handle_entered=handle_entered,
handle_release=handle_release,
)
runtime_task = asyncio.create_task(
dependencies.runtime.run(),
)
await wait_for_thread_event(handle_entered)
stop_task = asyncio.create_task(
dependencies.runtime.stop(),
)
try:
await asyncio.sleep(0)
await asyncio.sleep(0)
assert stop_task.done() is False
assert dependencies.live_processing_gate.locked is True
assert dependencies.runtime._market_processing_task is not None
finally:
handle_release.set()
await stop_task
await runtime_task
return dependencies
dependencies = asyncio.run(scenario())
assert dependencies.runtime.state is (
TradeStreamProductionRuntimeState.STOPPED
)
assert dependencies.runtime._market_processing_task is None
def test_connect_failure_publishes_event_and_rolls_back() -> None:

138
app/tests/unit/market_data/acquisition/test_trade_stream_runtime_composition.py
View File

@@ -13,6 +13,9 @@ import pytest
from src.market_data.acquisition.adapters.dzengi.rest import (
DzengiTradesDocumentSource,
)
from src.market_data.acquisition.consistency.trade_observation_sink_protocol import (
TradeObservationSinkProtocol,
)
from src.market_data.acquisition.models.trade import (
Trade,
TradeAggressorSide,
@@ -315,6 +318,25 @@ class RecordingSleep:
self.calls.append(seconds)
class RecordingTradeObservationSink:
def __init__(
self,
*,
fail_on_trade_id: int | None = None,
) -> None:
self._fail_on_trade_id = fail_on_trade_id
self.observations: list[Trade] = []
def persist(
self,
trade: Trade,
) -> None:
self.observations.append(trade)
if trade.trade_id == self._fail_on_trade_id:
raise RuntimeError("storage failed")
@dataclass(slots=True)
class CompositionDependencies:
session: FakeSession
@@ -336,6 +358,7 @@ def create_composition(
scheduler_interval_seconds: float = 1.0,
max_recovery_window_ms: int = 3_599_999,
probe_results: tuple[bool, ...] = (True,),
trade_observation_sink: TradeObservationSinkProtocol | None = None,
) -> tuple[
TradeStreamRuntimeComposition,
CompositionDependencies,
@@ -370,6 +393,7 @@ def create_composition(
symbols=(SYMBOL,),
heartbeat_timeout_seconds=heartbeat_timeout_seconds,
scheduler_interval_seconds=scheduler_interval_seconds,
trade_observation_sink=trade_observation_sink,
max_recovery_window_ms=max_recovery_window_ms,
heartbeat_clock=dependencies.heartbeat_clock,
recovery_end_time_clock=(
@@ -491,6 +515,120 @@ def test_live_stream_and_recovery_share_consistency_state() -> None:
)
def test_live_and_recovery_share_optional_persistence_sink() -> None:
sink = RecordingTradeObservationSink()
recovered_trade_id = 101
composition, _ = create_composition(
trade_observation_sink=sink,
recovery_document=[
make_raw_trade(
trade_id=recovered_trade_id,
timestamp=CHECKPOINT_TIME_MS + 1_000,
),
],
)
live_trade = (
composition.trade_stream_acquisition_service.handle_message(
{
"destination": "internal.trade",
}
)
)
recovery_result = composition.runtime_recovery_coordinator.recover(
symbol=SYMBOL,
recovery_end_time=RECOVERY_END_TIME_MS,
)
assert composition.trade_observation_sink is sink
assert isinstance(sink, TradeObservationSinkProtocol)
assert sink.observations[0] is live_trade
assert sink.observations[1] is recovery_result.last_trade
assert [
trade.trade_id
for trade in sink.observations
] == [100, recovered_trade_id]
def test_recovery_duplicate_updates_persistence_without_checkpoint_change(
) -> None:
live_trade = Trade(
symbol=SYMBOL,
trade_id=100,
price=Decimal("64556.00"),
quantity=Decimal("0.003"),
executed_at=CHECKPOINT_TIME,
aggressor_side=TradeAggressorSide.BUY,
source="dzengi_websocket_trade",
)
sink = RecordingTradeObservationSink()
composition, _ = create_composition(
trade=live_trade,
trade_observation_sink=sink,
recovery_document=[
make_raw_trade(
trade_id=live_trade.trade_id,
timestamp=CHECKPOINT_TIME_MS,
),
],
)
composition.trade_stream_acquisition_service.handle_message(
{
"destination": "internal.trade",
}
)
result = composition.runtime_recovery_coordinator.recover(
symbol=SYMBOL,
recovery_end_time=RECOVERY_END_TIME_MS,
)
state = composition.state_store.get(SYMBOL)
assert result.is_empty is True
assert len(sink.observations) == 2
assert sink.observations[0] is live_trade
assert sink.observations[1].source == "dzengi"
assert state.last_trade is live_trade
def test_recovery_persistence_failure_does_not_advance_checkpoint() -> None:
sink = RecordingTradeObservationSink(
fail_on_trade_id=101,
)
composition, _ = create_composition(
trade_observation_sink=sink,
recovery_document=[
make_raw_trade(
trade_id=101,
timestamp=CHECKPOINT_TIME_MS + 1_000,
),
],
)
live_trade = (
composition.trade_stream_acquisition_service.handle_message(
{
"destination": "internal.trade",
}
)
)
with pytest.raises(
RuntimeError,
match="storage failed",
):
composition.runtime_recovery_coordinator.recover(
symbol=SYMBOL,
recovery_end_time=RECOVERY_END_TIME_MS,
)
state = composition.state_store.get(SYMBOL)
assert state.last_trade is live_trade
assert state.last_trade_id == 100
assert [trade.trade_id for trade in sink.observations] == [100, 101]
def test_runtime_components_share_lifecycle_dependencies() -> None:
composition, dependencies = create_composition()