Build 060.27: implement Persistent Market Data Storage

2026-08-01 03:22:25 +03:00
parent cb8acfe5fe
commit 58e5a12a4d
54 changed files with 10243 additions and 101 deletions
--- a/app/tests/unit/bootstrap/test_application.py
+++ b/app/tests/unit/bootstrap/test_application.py
@@ -1,6 +1,7 @@
 from __future__ import annotations

 import asyncio
+import threading

 import pytest

@@ -46,12 +47,14 @@ class FakeDispatcher:
        *,
        return_immediately: bool = False,
        error: BaseException | None = None,
+        calls: list[str] | None = None,
    ) -> None:
        self.started = asyncio.Event()
        self.release = asyncio.Event()
        self.cancelled = asyncio.Event()
        self.return_immediately = return_immediately
        self.error = error
+        self.calls = calls
        self.close_bot_session_values: list[bool] = []

    async def start_polling(
@@ -61,6 +64,8 @@ class FakeDispatcher:
        close_bot_session: bool,
    ) -> None:
        del bot
+        if self.calls is not None:
+            self.calls.append("polling.start")
        self.close_bot_session_values.append(close_bot_session)
        self.started.set()

@@ -82,6 +87,7 @@ class FakeRuntime:
        return_immediately: bool = False,
        error: BaseException | None = None,
        stop_error: BaseException | None = None,
+        calls: list[str] | None = None,
    ) -> None:
        self.started = asyncio.Event()
        self.release = asyncio.Event()
@@ -89,6 +95,7 @@ class FakeRuntime:
        self.return_immediately = return_immediately
        self.error = error
        self.stop_error = stop_error
+        self.calls = calls
        self.stop_calls = 0

    @property
@@ -104,6 +111,8 @@ class FakeRuntime:
        return self.state is TradeStreamProductionRuntimeState.RUNNING

    async def run(self) -> None:
+        if self.calls is not None:
+            self.calls.append("runtime.run")
        self.started.set()

        if not self.return_immediately:
@@ -114,6 +123,8 @@ class FakeRuntime:

    async def stop(self) -> None:
        self.stop_calls += 1
+        if self.calls is not None:
+            self.calls.append("runtime.stop")
        self.release.set()
        self.stopped.set()

@@ -136,19 +147,289 @@ class BlockingStopRuntime(FakeRuntime):
        self.stopped.set()


+class FakeStorageLifecycle:
+    def __init__(
+        self,
+        *,
+        calls: list[str] | None = None,
+        start_error: BaseException | None = None,
+        stop_error: BaseException | None = None,
+        start_release: threading.Event | None = None,
+        stop_release: threading.Event | None = None,
+    ) -> None:
+        self.calls = calls
+        self.start_error = start_error
+        self.stop_error = stop_error
+        self.start_release = start_release
+        self.stop_release = stop_release
+        self.start_entered = threading.Event()
+        self.stop_entered = threading.Event()
+        self.start_calls = 0
+        self.stop_calls = 0
+        self.started = False
+
+    def start(self) -> None:
+        self.start_calls += 1
+        if self.calls is not None:
+            self.calls.append("storage.start")
+        self.start_entered.set()
+
+        if self.start_release is not None:
+            if not self.start_release.wait(timeout=2.0):
+                raise TimeoutError("storage startup was not released")
+
+        if self.start_error is not None:
+            raise self.start_error
+
+        self.started = True
+
+    def stop(self) -> None:
+        self.stop_calls += 1
+        if self.calls is not None:
+            self.calls.append("storage.stop")
+        self.started = False
+        self.stop_entered.set()
+
+        if self.stop_release is not None:
+            if not self.stop_release.wait(timeout=2.0):
+                raise TimeoutError("storage shutdown was not released")
+
+        if self.stop_error is not None:
+            raise self.stop_error
+
+
 def make_application(
    *,
    dispatcher: FakeDispatcher,
    runtime: FakeRuntime | None,
    bot: FakeBot | None = None,
+    storage_lifecycle: FakeStorageLifecycle | None = None,
 ) -> ApplicationComposition:
    return ApplicationComposition(
        bot=bot or FakeBot(),  # type: ignore[arg-type]
        dispatcher=dispatcher,  # type: ignore[arg-type]
        trade_stream_runtime=runtime,
+        market_data_storage_lifecycle=storage_lifecycle,
    )


+def test_storage_lifecycle_wraps_root_runtime_tasks() -> None:
+    async def scenario() -> None:
+        calls: list[str] = []
+        storage = FakeStorageLifecycle(calls=calls)
+        dispatcher = FakeDispatcher(
+            return_immediately=True,
+            calls=calls,
+        )
+        runtime = FakeRuntime(calls=calls)
+
+        await run_application(
+            make_application(
+                dispatcher=dispatcher,
+                runtime=runtime,
+                storage_lifecycle=storage,
+            )
+        )
+
+        assert storage.start_calls == 1
+        assert storage.stop_calls == 1
+        assert calls.index("storage.start") < calls.index(
+            "polling.start"
+        )
+        assert calls.index("storage.start") < calls.index(
+            "runtime.run"
+        )
+        assert calls.index("runtime.stop") < calls.index(
+            "storage.stop"
+        )
+
+    asyncio.run(scenario())
+
+
+def test_storage_startup_failure_prevents_root_task_start() -> None:
+    async def scenario() -> None:
+        expected = RuntimeError("storage startup failed")
+        storage = FakeStorageLifecycle(start_error=expected)
+        dispatcher = FakeDispatcher()
+        runtime = FakeRuntime()
+        bot = FakeBot()
+
+        with pytest.raises(RuntimeError) as error_info:
+            await run_application(
+                make_application(
+                    dispatcher=dispatcher,
+                    runtime=runtime,
+                    bot=bot,
+                    storage_lifecycle=storage,
+                )
+            )
+
+        assert error_info.value is expected
+        assert dispatcher.started.is_set() is False
+        assert runtime.started.is_set() is False
+        assert runtime.stop_calls == 1
+        assert storage.stop_calls == 1
+        assert bot.session.close_calls == 1
+
+    asyncio.run(scenario())
+
+
+def test_cancellation_waits_for_storage_startup_before_cleanup() -> None:
+    async def scenario() -> None:
+        start_release = threading.Event()
+        storage = FakeStorageLifecycle(
+            start_release=start_release,
+        )
+        dispatcher = FakeDispatcher()
+        runtime = FakeRuntime()
+        bot = FakeBot()
+        task = asyncio.create_task(
+            run_application(
+                make_application(
+                    dispatcher=dispatcher,
+                    runtime=runtime,
+                    bot=bot,
+                    storage_lifecycle=storage,
+                )
+            )
+        )
+
+        entered = await asyncio.to_thread(
+            storage.start_entered.wait,
+            1.0,
+        )
+        assert entered is True
+        task.cancel()
+
+        try:
+            await asyncio.sleep(0)
+            await asyncio.sleep(0)
+            assert task.done() is False
+        finally:
+            start_release.set()
+
+        with pytest.raises(asyncio.CancelledError):
+            await task
+
+        assert dispatcher.started.is_set() is False
+        assert runtime.started.is_set() is False
+        assert storage.stop_calls == 1
+        assert bot.session.close_calls == 1
+        await asyncio.sleep(0)
+        assert not {
+            child.get_name()
+            for child in asyncio.all_tasks()
+            if child is not asyncio.current_task()
+            and not child.done()
+            and child.get_name()
+            in {
+                "application-shutdown",
+                "market-data-storage-shutdown",
+                "market-data-storage-startup",
+                "telegram-polling",
+                "trade-stream-runtime",
+            }
+        }
+
+    asyncio.run(scenario())
+
+
+def test_storage_shutdown_error_is_reported_and_bot_still_closes() -> None:
+    async def scenario() -> None:
+        expected = RuntimeError("storage close failed")
+        storage = FakeStorageLifecycle(stop_error=expected)
+        dispatcher = FakeDispatcher(return_immediately=True)
+        bot = FakeBot()
+
+        with pytest.raises(RuntimeError) as error_info:
+            await run_application(
+                make_application(
+                    dispatcher=dispatcher,
+                    runtime=None,
+                    bot=bot,
+                    storage_lifecycle=storage,
+                )
+            )
+
+        assert error_info.value is expected
+        assert storage.stop_calls == 1
+        assert bot.session.close_calls == 1
+
+    asyncio.run(scenario())
+
+
+def test_storage_shutdown_error_does_not_replace_runtime_error() -> None:
+    async def scenario() -> None:
+        runtime_error = RuntimeError("runtime failed")
+        storage_error = RuntimeError("storage close failed")
+        storage = FakeStorageLifecycle(stop_error=storage_error)
+        dispatcher = FakeDispatcher()
+        runtime = FakeRuntime(
+            return_immediately=True,
+            error=runtime_error,
+        )
+
+        with pytest.raises(RuntimeError) as error_info:
+            await run_application(
+                make_application(
+                    dispatcher=dispatcher,
+                    runtime=runtime,
+                    storage_lifecycle=storage,
+                )
+            )
+
+        assert error_info.value is runtime_error
+        assert any(
+            "cleanup also failed" in note
+            for note in getattr(runtime_error, "__notes__", ())
+        )
+
+    asyncio.run(scenario())
+
+
+def test_repeated_cancellation_does_not_interrupt_storage_shutdown(
+) -> None:
+    async def scenario() -> None:
+        stop_release = threading.Event()
+        storage = FakeStorageLifecycle(
+            stop_release=stop_release,
+        )
+        dispatcher = FakeDispatcher(return_immediately=True)
+        bot = FakeBot()
+        task = asyncio.create_task(
+            run_application(
+                make_application(
+                    dispatcher=dispatcher,
+                    runtime=None,
+                    bot=bot,
+                    storage_lifecycle=storage,
+                )
+            )
+        )
+
+        entered = await asyncio.to_thread(
+            storage.stop_entered.wait,
+            1.0,
+        )
+        assert entered is True
+        task.cancel()
+        task.cancel()
+
+        try:
+            await asyncio.sleep(0)
+            assert task.done() is False
+        finally:
+            stop_release.set()
+
+        with pytest.raises(asyncio.CancelledError):
+            await task
+
+        assert storage.stop_calls == 1
+        assert bot.session.close_calls == 1
+
+    asyncio.run(scenario())
+
+
 def test_disabled_runtime_runs_only_polling_and_closes_bot() -> None:
    async def scenario() -> None:
        dispatcher = FakeDispatcher(return_immediately=True)