Build 060.27: implement Persistent Market Data Storage
This commit is contained in:
@@ -114,7 +114,11 @@ def load_live_trade_stream_test_config(
|
||||
def build_live_trade_stream_settings(
|
||||
config: LiveTradeStreamTestConfig,
|
||||
) -> Settings:
|
||||
from src.core.config import Settings, TradeStreamSettings
|
||||
from src.core.config import (
|
||||
MarketDataStorageSettings,
|
||||
Settings,
|
||||
TradeStreamSettings,
|
||||
)
|
||||
|
||||
return Settings(
|
||||
bot_token="live-verification-does-not-use-telegram",
|
||||
@@ -147,6 +151,12 @@ def build_live_trade_stream_settings(
|
||||
db_name="live-verification",
|
||||
db_user="live-verification",
|
||||
db_password="",
|
||||
market_data_storage=MarketDataStorageSettings(
|
||||
enabled=False,
|
||||
pool_min_size=1,
|
||||
pool_max_size=4,
|
||||
pool_timeout_seconds=10.0,
|
||||
),
|
||||
debug_enabled=False,
|
||||
journal_debug_enabled=False,
|
||||
)
|
||||
|
||||
316
app/tests/support/postgres_market_data.py
Normal file
316
app/tests/support/postgres_market_data.py
Normal file
@@ -0,0 +1,316 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import psycopg
|
||||
from psycopg.conninfo import conninfo_to_dict, make_conninfo
|
||||
|
||||
|
||||
POSTGRES_TEST_FLAG = "DZENTRA_RUN_POSTGRES_TESTS"
|
||||
POSTGRES_TEST_DSN = "DZENTRA_TEST_POSTGRES_DSN"
|
||||
POSTGRES_TEST_APPLICATION_NAME = "dzentra-storage-integration"
|
||||
POSTGRES_TEST_CONTROL_APPLICATION_NAME = (
|
||||
"dzentra-storage-integration-control"
|
||||
)
|
||||
POSTGRES_TEST_DATABASE_PREFIX = "dzentra_test_"
|
||||
POSTGRES_TEST_ADVISORY_LOCK_ID = 0x445A454E54524154
|
||||
|
||||
_SAFE_DATABASE_NAME = re.compile(r"^dzentra_test_[A-Za-z0-9_]+$")
|
||||
_LOCAL_HOST_NAMES = frozenset({"localhost"})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PostgresTestSettings:
|
||||
"""Явные настройки подключения к одноразовой локальной базе."""
|
||||
|
||||
dsn: str
|
||||
database_name: str
|
||||
|
||||
|
||||
def load_postgres_test_settings(
|
||||
environ: Mapping[str, str],
|
||||
) -> PostgresTestSettings | None:
|
||||
"""Загрузить явно включаемый DSN без настроек основной базы."""
|
||||
enabled = str(environ.get(POSTGRES_TEST_FLAG, "")).strip()
|
||||
|
||||
if not enabled:
|
||||
return None
|
||||
|
||||
if enabled != "1":
|
||||
raise ValueError(f"{POSTGRES_TEST_FLAG} must be exactly '1'")
|
||||
|
||||
raw_dsn = str(environ.get(POSTGRES_TEST_DSN, "")).strip()
|
||||
|
||||
if not raw_dsn:
|
||||
raise ValueError(
|
||||
f"{POSTGRES_TEST_DSN} is required when {POSTGRES_TEST_FLAG}=1"
|
||||
)
|
||||
|
||||
try:
|
||||
parameters = conninfo_to_dict(raw_dsn)
|
||||
except Exception as error:
|
||||
raise ValueError(
|
||||
f"{POSTGRES_TEST_DSN} is not a valid PostgreSQL DSN"
|
||||
) from error
|
||||
|
||||
database_name = str(parameters.get("dbname", "")).strip()
|
||||
|
||||
if not _SAFE_DATABASE_NAME.fullmatch(database_name):
|
||||
raise ValueError(
|
||||
f"{POSTGRES_TEST_DSN} database name must start with "
|
||||
f"{POSTGRES_TEST_DATABASE_PREFIX!r} and contain only safe characters"
|
||||
)
|
||||
|
||||
_validate_local_endpoint(parameters)
|
||||
|
||||
try:
|
||||
normalized_dsn = make_conninfo(
|
||||
raw_dsn,
|
||||
application_name=POSTGRES_TEST_APPLICATION_NAME,
|
||||
connect_timeout=5,
|
||||
)
|
||||
except Exception as error:
|
||||
raise ValueError(f"{POSTGRES_TEST_DSN} could not be normalized") from error
|
||||
|
||||
return PostgresTestSettings(
|
||||
dsn=normalized_dsn,
|
||||
database_name=database_name,
|
||||
)
|
||||
|
||||
|
||||
def connect_postgres_test_database(
|
||||
settings: PostgresTestSettings,
|
||||
*,
|
||||
autocommit: bool = True,
|
||||
) -> psycopg.Connection[Any]:
|
||||
"""Подключиться после прохождения явных проверок безопасности."""
|
||||
if not isinstance(settings, PostgresTestSettings):
|
||||
raise TypeError("settings must be PostgresTestSettings")
|
||||
|
||||
control_dsn = make_conninfo(
|
||||
settings.dsn,
|
||||
application_name=POSTGRES_TEST_CONTROL_APPLICATION_NAME,
|
||||
)
|
||||
connection = psycopg.connect(
|
||||
control_dsn,
|
||||
autocommit=autocommit,
|
||||
)
|
||||
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT current_database()")
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row != (settings.database_name,):
|
||||
connection.close()
|
||||
raise RuntimeError(
|
||||
"Connected PostgreSQL database does not match the validated "
|
||||
"disposable test database."
|
||||
)
|
||||
|
||||
return connection
|
||||
|
||||
|
||||
def reset_postgres_test_database(
|
||||
connection: psycopg.Connection[Any],
|
||||
*,
|
||||
expected_database_name: str,
|
||||
) -> None:
|
||||
"""Удалить только объекты Build 060.27 из проверенной тестовой базы."""
|
||||
if not connection.autocommit:
|
||||
raise ValueError("test database reset requires autocommit")
|
||||
|
||||
if not _SAFE_DATABASE_NAME.fullmatch(expected_database_name):
|
||||
raise ValueError("expected database name is not a safe test database")
|
||||
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT current_database(), current_setting('application_name')
|
||||
"""
|
||||
)
|
||||
identity = cursor.fetchone()
|
||||
|
||||
if identity != (
|
||||
expected_database_name,
|
||||
POSTGRES_TEST_CONTROL_APPLICATION_NAME,
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Refusing destructive reset because the current PostgreSQL "
|
||||
"connection is not the validated test control connection."
|
||||
)
|
||||
|
||||
cursor.execute("DROP SCHEMA IF EXISTS market_data CASCADE")
|
||||
cursor.execute(
|
||||
"DROP TABLE IF EXISTS public.storage_schema_migrations"
|
||||
)
|
||||
|
||||
|
||||
def acquire_postgres_test_lock(
|
||||
connection: psycopg.Connection[Any],
|
||||
) -> bool:
|
||||
"""Не допустить две разрушающие сессии в одной тестовой базе."""
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT pg_try_advisory_lock(%s)",
|
||||
(POSTGRES_TEST_ADVISORY_LOCK_ID,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
|
||||
return row == (True,)
|
||||
|
||||
|
||||
def release_postgres_test_lock(
|
||||
connection: psycopg.Connection[Any],
|
||||
) -> None:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT pg_advisory_unlock(%s)",
|
||||
(POSTGRES_TEST_ADVISORY_LOCK_ID,),
|
||||
)
|
||||
|
||||
|
||||
def count_other_test_connections(
|
||||
connection: psycopg.Connection[Any],
|
||||
) -> int:
|
||||
"""Посчитать оставшиеся соединения стенда и пула с тестовой базой."""
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM pg_stat_activity
|
||||
WHERE datname = current_database()
|
||||
AND application_name = %s
|
||||
""",
|
||||
(POSTGRES_TEST_APPLICATION_NAME,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
|
||||
if (
|
||||
not isinstance(row, tuple)
|
||||
or len(row) != 1
|
||||
or isinstance(row[0], bool)
|
||||
or not isinstance(row[0], int)
|
||||
):
|
||||
raise RuntimeError("PostgreSQL returned an invalid connection count")
|
||||
|
||||
return row[0]
|
||||
|
||||
|
||||
def wait_for_postgres_advisory_lock_waiters(
|
||||
connection: psycopg.Connection[Any],
|
||||
*,
|
||||
lock_id: int,
|
||||
expected_count: int,
|
||||
timeout_seconds: float = 5.0,
|
||||
) -> None:
|
||||
"""Дождаться подтверждения конкуренции всех сессий за блокировку."""
|
||||
if (
|
||||
isinstance(lock_id, bool)
|
||||
or not isinstance(lock_id, int)
|
||||
or lock_id < 0
|
||||
or lock_id > 0x7FFF_FFFF_FFFF_FFFF
|
||||
):
|
||||
raise ValueError("lock_id must be a non-negative signed BIGINT")
|
||||
|
||||
if (
|
||||
isinstance(expected_count, bool)
|
||||
or not isinstance(expected_count, int)
|
||||
or expected_count <= 0
|
||||
):
|
||||
raise ValueError("expected_count must be a positive integer")
|
||||
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError("timeout_seconds must be positive")
|
||||
|
||||
class_id = (lock_id >> 32) & 0xFFFF_FFFF
|
||||
object_id = lock_id & 0xFFFF_FFFF
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
|
||||
while True:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM pg_catalog.pg_locks
|
||||
WHERE locktype = 'advisory'
|
||||
AND classid = %s
|
||||
AND objid = %s
|
||||
AND objsubid = 1
|
||||
AND NOT granted
|
||||
""",
|
||||
(class_id, object_id),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row == (expected_count,):
|
||||
return
|
||||
|
||||
if time.monotonic() >= deadline:
|
||||
observed_count = row[0] if isinstance(row, tuple) and row else row
|
||||
raise TimeoutError(
|
||||
"PostgreSQL did not observe all advisory-lock callers; "
|
||||
f"expected {expected_count}, observed {observed_count!r}."
|
||||
)
|
||||
|
||||
time.sleep(0.01)
|
||||
|
||||
|
||||
def _validate_local_endpoint(parameters: Mapping[str, object]) -> None:
|
||||
service = str(parameters.get("service", "")).strip()
|
||||
host = str(parameters.get("host", "")).strip()
|
||||
hostaddr = str(parameters.get("hostaddr", "")).strip()
|
||||
|
||||
if service:
|
||||
raise ValueError(
|
||||
f"{POSTGRES_TEST_DSN} must not use a PostgreSQL service"
|
||||
)
|
||||
|
||||
if not host and not hostaddr:
|
||||
raise ValueError(
|
||||
f"{POSTGRES_TEST_DSN} must contain an explicit local host or "
|
||||
"hostaddr"
|
||||
)
|
||||
|
||||
for endpoint_host in _split_postgres_endpoints(host):
|
||||
if not endpoint_host.startswith("/") and not _is_local_host(
|
||||
endpoint_host
|
||||
):
|
||||
raise ValueError(
|
||||
f"{POSTGRES_TEST_DSN} must target localhost or a local socket"
|
||||
)
|
||||
|
||||
for endpoint_address in _split_postgres_endpoints(hostaddr):
|
||||
if not _is_loopback_address(endpoint_address):
|
||||
raise ValueError(
|
||||
f"{POSTGRES_TEST_DSN} hostaddr must be a loopback address"
|
||||
)
|
||||
|
||||
|
||||
def _split_postgres_endpoints(value: str) -> tuple[str, ...]:
|
||||
if not value:
|
||||
return ()
|
||||
|
||||
endpoints = tuple(part.strip() for part in value.split(","))
|
||||
|
||||
if any(not endpoint for endpoint in endpoints):
|
||||
raise ValueError(
|
||||
f"{POSTGRES_TEST_DSN} must not contain an implicit endpoint"
|
||||
)
|
||||
|
||||
return endpoints
|
||||
|
||||
|
||||
def _is_local_host(value: str) -> bool:
|
||||
return value.lower() in _LOCAL_HOST_NAMES or _is_loopback_address(value)
|
||||
|
||||
|
||||
def _is_loopback_address(value: str) -> bool:
|
||||
try:
|
||||
return ipaddress.ip_address(value).is_loopback
|
||||
except ValueError:
|
||||
return False
|
||||
@@ -7,10 +7,17 @@ from typing import Any, Protocol
|
||||
from src.bootstrap.trade_stream_runtime import (
|
||||
build_trade_stream_production_runtime,
|
||||
)
|
||||
from src.core.config import Settings, TradeStreamSettings
|
||||
from src.core.config import (
|
||||
MarketDataStorageSettings,
|
||||
Settings,
|
||||
TradeStreamSettings,
|
||||
)
|
||||
from src.market_data.acquisition.consistency.trade_stream_state_store import (
|
||||
TradeStreamStateStore,
|
||||
)
|
||||
from src.market_data.acquisition.consistency.trade_observation_sink_protocol import (
|
||||
TradeObservationSinkProtocol,
|
||||
)
|
||||
from src.market_data.acquisition.runtime.trade_stream_production_runtime import (
|
||||
TradeStreamProductionRuntime,
|
||||
TradeStreamProductionRuntimeState,
|
||||
@@ -23,6 +30,8 @@ RUNTIME_CLEANUP_TIMEOUT_SECONDS = 5.0
|
||||
|
||||
OWNED_TASK_NAMES = frozenset(
|
||||
{
|
||||
"market-data-storage-shutdown",
|
||||
"market-data-storage-startup",
|
||||
"trade-stream-receive",
|
||||
"trade-stream-runtime",
|
||||
"trade-stream-runtime-recovery",
|
||||
@@ -98,6 +107,12 @@ def make_settings(
|
||||
db_name="integration",
|
||||
db_user="integration",
|
||||
db_password="",
|
||||
market_data_storage=MarketDataStorageSettings(
|
||||
enabled=False,
|
||||
pool_min_size=1,
|
||||
pool_max_size=4,
|
||||
pool_timeout_seconds=10.0,
|
||||
),
|
||||
debug_enabled=False,
|
||||
journal_debug_enabled=False,
|
||||
)
|
||||
@@ -111,6 +126,7 @@ def build_runtime(
|
||||
close_timeout_seconds: float = 0.2,
|
||||
heartbeat_timeout_seconds: float = 60.0,
|
||||
scheduler_interval_seconds: float = 60.0,
|
||||
trade_observation_sink: TradeObservationSinkProtocol | None = None,
|
||||
) -> TradeStreamProductionRuntime:
|
||||
runtime = build_trade_stream_production_runtime(
|
||||
make_settings(
|
||||
@@ -120,7 +136,8 @@ def build_runtime(
|
||||
close_timeout_seconds=close_timeout_seconds,
|
||||
heartbeat_timeout_seconds=heartbeat_timeout_seconds,
|
||||
scheduler_interval_seconds=scheduler_interval_seconds,
|
||||
)
|
||||
),
|
||||
trade_observation_sink=trade_observation_sink,
|
||||
)
|
||||
|
||||
assert runtime is not None
|
||||
|
||||
Reference in New Issue
Block a user