Build 060.27: implement Persistent Market Data Storage
This commit is contained in:
@@ -16,3 +16,11 @@ class InstrumentStoreError(StorageError):
|
||||
# Ошибка хранилища канонических котировок.
|
||||
class QuoteStoreError(StorageError):
|
||||
"""Quote store contract or operation error."""
|
||||
|
||||
|
||||
class PostgresConnectionPoolError(StorageError):
|
||||
"""PostgreSQL connection pool lifecycle error."""
|
||||
|
||||
|
||||
class StorageMigrationError(StorageError):
|
||||
"""Versioned storage migration error."""
|
||||
|
||||
429
app/src/storage/migrations.py
Normal file
429
app/src/storage/migrations.py
Normal file
@@ -0,0 +1,429 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from src.storage.exceptions import StorageMigrationError
|
||||
|
||||
|
||||
STORAGE_MIGRATION_ADVISORY_LOCK_ID = 0x445A454E545241
|
||||
|
||||
_CREATE_HISTORY_TABLE_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS public.storage_schema_migrations (
|
||||
version INTEGER PRIMARY KEY,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
"""
|
||||
|
||||
_SELECT_APPLIED_MIGRATIONS_SQL = """
|
||||
SELECT version, name
|
||||
FROM public.storage_schema_migrations
|
||||
ORDER BY version
|
||||
"""
|
||||
|
||||
_INSERT_APPLIED_MIGRATION_SQL = """
|
||||
INSERT INTO public.storage_schema_migrations (version, name)
|
||||
VALUES (%s, %s)
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StorageMigration:
|
||||
"""Одна неизменяемая упорядоченная миграция схемы хранилища."""
|
||||
|
||||
version: int
|
||||
name: str
|
||||
statements: tuple[str, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if (
|
||||
isinstance(self.version, bool)
|
||||
or not isinstance(self.version, int)
|
||||
or self.version <= 0
|
||||
):
|
||||
raise ValueError("migration version must be a positive integer")
|
||||
|
||||
normalized_name = str(self.name or "").strip()
|
||||
|
||||
if not normalized_name:
|
||||
raise ValueError("migration name must not be empty")
|
||||
|
||||
if not isinstance(self.statements, tuple) or not self.statements:
|
||||
raise ValueError("migration statements must be a non-empty tuple")
|
||||
|
||||
if any(not str(statement or "").strip() for statement in self.statements):
|
||||
raise ValueError("migration statements must not be empty")
|
||||
|
||||
object.__setattr__(self, "name", normalized_name)
|
||||
|
||||
|
||||
STORAGE_MIGRATIONS = (
|
||||
StorageMigration(
|
||||
version=1,
|
||||
name="create_market_data_schema",
|
||||
statements=(
|
||||
"CREATE SCHEMA IF NOT EXISTS market_data",
|
||||
),
|
||||
),
|
||||
StorageMigration(
|
||||
version=2,
|
||||
name="create_canonical_trades",
|
||||
statements=(
|
||||
"""
|
||||
CREATE TABLE market_data.trades (
|
||||
venue TEXT NOT NULL,
|
||||
symbol TEXT NOT NULL,
|
||||
trade_id INTEGER NOT NULL,
|
||||
executed_at TIMESTAMPTZ NOT NULL,
|
||||
price NUMERIC NOT NULL,
|
||||
quantity NUMERIC NOT NULL,
|
||||
aggressor_side TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
first_observed_at TIMESTAMPTZ NOT NULL,
|
||||
last_observed_at TIMESTAMPTZ NOT NULL,
|
||||
canonical_schema_version INTEGER NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY (venue, symbol, trade_id, executed_at),
|
||||
CHECK (trade_id BETWEEN -2147483648 AND 2147483647),
|
||||
CHECK (BTRIM(venue) <> ''),
|
||||
CHECK (BTRIM(symbol) <> ''),
|
||||
CHECK (BTRIM(source) <> ''),
|
||||
CHECK (price > 0),
|
||||
CHECK (quantity > 0),
|
||||
CHECK (aggressor_side IN ('buy', 'sell')),
|
||||
CHECK (canonical_schema_version > 0),
|
||||
CHECK (last_observed_at >= first_observed_at)
|
||||
) PARTITION BY RANGE (executed_at)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE market_data.trades_default
|
||||
PARTITION OF market_data.trades DEFAULT
|
||||
""",
|
||||
"""
|
||||
CREATE INDEX trades_event_order_idx
|
||||
ON market_data.trades (
|
||||
venue,
|
||||
symbol,
|
||||
executed_at,
|
||||
trade_id
|
||||
)
|
||||
""",
|
||||
),
|
||||
),
|
||||
StorageMigration(
|
||||
version=3,
|
||||
name="create_canonical_quotes",
|
||||
statements=(
|
||||
"""
|
||||
CREATE TABLE market_data.quotes (
|
||||
venue TEXT NOT NULL,
|
||||
symbol TEXT NOT NULL,
|
||||
received_at TIMESTAMPTZ NOT NULL,
|
||||
exchange_timestamp TIMESTAMPTZ,
|
||||
last_price NUMERIC NOT NULL,
|
||||
bid_price NUMERIC NOT NULL,
|
||||
ask_price NUMERIC NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
canonical_schema_version INTEGER NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY (venue, symbol, received_at),
|
||||
CHECK (BTRIM(venue) <> ''),
|
||||
CHECK (BTRIM(symbol) <> ''),
|
||||
CHECK (BTRIM(source) <> ''),
|
||||
CHECK (last_price > 0),
|
||||
CHECK (bid_price > 0),
|
||||
CHECK (ask_price > 0),
|
||||
CHECK (bid_price <= ask_price),
|
||||
CHECK (canonical_schema_version > 0)
|
||||
) PARTITION BY RANGE (received_at)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE market_data.quotes_default
|
||||
PARTITION OF market_data.quotes DEFAULT
|
||||
""",
|
||||
),
|
||||
),
|
||||
StorageMigration(
|
||||
version=4,
|
||||
name="create_canonical_candle_revisions",
|
||||
statements=(
|
||||
"""
|
||||
CREATE TABLE market_data.candle_revisions (
|
||||
venue TEXT NOT NULL,
|
||||
symbol TEXT NOT NULL,
|
||||
interval TEXT NOT NULL,
|
||||
open_time TIMESTAMPTZ NOT NULL,
|
||||
observed_at TIMESTAMPTZ NOT NULL,
|
||||
open_price NUMERIC NOT NULL,
|
||||
high_price NUMERIC NOT NULL,
|
||||
low_price NUMERIC NOT NULL,
|
||||
close_price NUMERIC NOT NULL,
|
||||
volume NUMERIC NOT NULL,
|
||||
is_final BOOLEAN NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
canonical_schema_version INTEGER NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY (
|
||||
venue,
|
||||
symbol,
|
||||
interval,
|
||||
open_time,
|
||||
observed_at
|
||||
),
|
||||
CHECK (BTRIM(venue) <> ''),
|
||||
CHECK (BTRIM(symbol) <> ''),
|
||||
CHECK (BTRIM(interval) <> ''),
|
||||
CHECK (BTRIM(source) <> ''),
|
||||
CHECK (open_price > 0),
|
||||
CHECK (high_price > 0),
|
||||
CHECK (low_price > 0),
|
||||
CHECK (close_price > 0),
|
||||
CHECK (volume >= 0),
|
||||
CHECK (low_price <= high_price),
|
||||
CHECK (open_price BETWEEN low_price AND high_price),
|
||||
CHECK (close_price BETWEEN low_price AND high_price),
|
||||
CHECK (canonical_schema_version > 0),
|
||||
CHECK (observed_at >= open_time)
|
||||
) PARTITION BY RANGE (open_time)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE market_data.candle_revisions_default
|
||||
PARTITION OF market_data.candle_revisions DEFAULT
|
||||
""",
|
||||
),
|
||||
),
|
||||
StorageMigration(
|
||||
version=5,
|
||||
name="add_trade_observation_sources",
|
||||
statements=(
|
||||
"""
|
||||
ALTER TABLE market_data.trades
|
||||
ADD COLUMN observation_sources TEXT[]
|
||||
""",
|
||||
"""
|
||||
UPDATE market_data.trades
|
||||
SET observation_sources = ARRAY[source]
|
||||
""",
|
||||
"""
|
||||
ALTER TABLE market_data.trades
|
||||
ALTER COLUMN observation_sources SET NOT NULL
|
||||
""",
|
||||
"""
|
||||
ALTER TABLE market_data.trades
|
||||
ADD CONSTRAINT trades_observation_sources_not_empty
|
||||
CHECK (
|
||||
CARDINALITY(observation_sources) > 0
|
||||
AND ARRAY_POSITION(observation_sources, NULL) IS NULL
|
||||
)
|
||||
""",
|
||||
),
|
||||
),
|
||||
StorageMigration(
|
||||
version=6,
|
||||
name="add_quote_and_candle_observation_sources",
|
||||
statements=(
|
||||
"""
|
||||
ALTER TABLE market_data.quotes
|
||||
ADD COLUMN observation_sources TEXT[]
|
||||
""",
|
||||
"""
|
||||
UPDATE market_data.quotes
|
||||
SET observation_sources = ARRAY[source]
|
||||
""",
|
||||
"""
|
||||
ALTER TABLE market_data.quotes
|
||||
ALTER COLUMN observation_sources SET NOT NULL
|
||||
""",
|
||||
"""
|
||||
ALTER TABLE market_data.quotes
|
||||
ADD CONSTRAINT quotes_observation_sources_not_empty
|
||||
CHECK (
|
||||
CARDINALITY(observation_sources) > 0
|
||||
AND ARRAY_POSITION(observation_sources, NULL) IS NULL
|
||||
)
|
||||
""",
|
||||
"""
|
||||
ALTER TABLE market_data.candle_revisions
|
||||
ADD COLUMN observation_sources TEXT[]
|
||||
""",
|
||||
"""
|
||||
UPDATE market_data.candle_revisions
|
||||
SET observation_sources = ARRAY[source]
|
||||
""",
|
||||
"""
|
||||
ALTER TABLE market_data.candle_revisions
|
||||
ALTER COLUMN observation_sources SET NOT NULL
|
||||
""",
|
||||
"""
|
||||
ALTER TABLE market_data.candle_revisions
|
||||
ADD CONSTRAINT candle_revisions_observation_sources_not_empty
|
||||
CHECK (
|
||||
CARDINALITY(observation_sources) > 0
|
||||
AND ARRAY_POSITION(observation_sources, NULL) IS NULL
|
||||
)
|
||||
""",
|
||||
),
|
||||
),
|
||||
StorageMigration(
|
||||
version=7,
|
||||
name="create_market_data_partition_registry",
|
||||
statements=(
|
||||
"""
|
||||
CREATE TABLE market_data.partition_registry (
|
||||
data_type TEXT NOT NULL,
|
||||
partition_name TEXT NOT NULL,
|
||||
range_start TIMESTAMPTZ NOT NULL,
|
||||
range_end TIMESTAMPTZ NOT NULL,
|
||||
partition_bound TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (data_type, range_start),
|
||||
UNIQUE (partition_name),
|
||||
CHECK (
|
||||
data_type IN (
|
||||
'trades',
|
||||
'quotes',
|
||||
'candle_revisions'
|
||||
)
|
||||
),
|
||||
CHECK (BTRIM(partition_name) <> ''),
|
||||
CHECK (BTRIM(partition_bound) <> ''),
|
||||
CHECK (range_end > range_start)
|
||||
)
|
||||
""",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
StorageConnectionProvider = Callable[[], AbstractContextManager[Any]]
|
||||
|
||||
|
||||
class StorageMigrationRunner:
|
||||
"""Применяет миграции PostgreSQL в одной заблокированной транзакции."""
|
||||
|
||||
__slots__ = (
|
||||
"_connection_provider",
|
||||
"_migrations",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
connection_provider: StorageConnectionProvider,
|
||||
migrations: Iterable[StorageMigration] = STORAGE_MIGRATIONS,
|
||||
) -> None:
|
||||
if not callable(connection_provider):
|
||||
raise TypeError("connection_provider must be callable")
|
||||
|
||||
normalized_migrations = tuple(migrations)
|
||||
versions = tuple(
|
||||
migration.version
|
||||
for migration in normalized_migrations
|
||||
)
|
||||
|
||||
if len(set(versions)) != len(versions):
|
||||
raise ValueError("migration versions must be unique")
|
||||
|
||||
if versions != tuple(sorted(versions)):
|
||||
raise ValueError("migrations must be ordered by version")
|
||||
|
||||
self._connection_provider = connection_provider
|
||||
self._migrations = normalized_migrations
|
||||
|
||||
def run(self) -> tuple[int, ...]:
|
||||
"""Применить ожидающие миграции и вернуть их версии."""
|
||||
try:
|
||||
with self._connection_provider() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT pg_advisory_xact_lock(%s)",
|
||||
(STORAGE_MIGRATION_ADVISORY_LOCK_ID,),
|
||||
)
|
||||
cursor.execute(_CREATE_HISTORY_TABLE_SQL)
|
||||
cursor.execute(_SELECT_APPLIED_MIGRATIONS_SQL)
|
||||
|
||||
applied = {
|
||||
int(version): str(name)
|
||||
for version, name in cursor.fetchall()
|
||||
}
|
||||
configured = {
|
||||
migration.version: migration
|
||||
for migration in self._migrations
|
||||
}
|
||||
|
||||
self._validate_applied(
|
||||
applied=applied,
|
||||
configured=configured,
|
||||
)
|
||||
|
||||
applied_now: list[int] = []
|
||||
|
||||
for migration in self._migrations:
|
||||
if migration.version in applied:
|
||||
continue
|
||||
|
||||
for statement in migration.statements:
|
||||
cursor.execute(statement)
|
||||
|
||||
cursor.execute(
|
||||
_INSERT_APPLIED_MIGRATION_SQL,
|
||||
(
|
||||
migration.version,
|
||||
migration.name,
|
||||
),
|
||||
)
|
||||
applied_now.append(migration.version)
|
||||
|
||||
return tuple(applied_now)
|
||||
except StorageMigrationError:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise StorageMigrationError(
|
||||
"Failed to apply storage schema migrations."
|
||||
) from error
|
||||
|
||||
@staticmethod
|
||||
def _validate_applied(
|
||||
*,
|
||||
applied: dict[int, str],
|
||||
configured: dict[int, StorageMigration],
|
||||
) -> None:
|
||||
for version, applied_name in applied.items():
|
||||
migration = configured.get(version)
|
||||
|
||||
if migration is None:
|
||||
raise StorageMigrationError(
|
||||
"Database contains unknown storage migration "
|
||||
f"version {version}."
|
||||
)
|
||||
|
||||
if migration.name != applied_name:
|
||||
raise StorageMigrationError(
|
||||
"Storage migration name mismatch for version "
|
||||
f"{version}: database={applied_name!r}, "
|
||||
f"configured={migration.name!r}."
|
||||
)
|
||||
|
||||
configured_versions = tuple(configured)
|
||||
applied_versions = tuple(sorted(applied))
|
||||
expected_prefix = configured_versions[: len(applied_versions)]
|
||||
|
||||
if applied_versions != expected_prefix:
|
||||
raise StorageMigrationError(
|
||||
"Applied storage migrations must form an ordered "
|
||||
"prefix of configured migrations."
|
||||
)
|
||||
|
||||
|
||||
def run_storage_migrations(
|
||||
connection_provider: StorageConnectionProvider | None = None,
|
||||
) -> tuple[int, ...]:
|
||||
"""Запустить миграции через явного поставщика соединений."""
|
||||
if connection_provider is None:
|
||||
from src.storage.session import get_connection
|
||||
|
||||
connection_provider = get_connection
|
||||
|
||||
return StorageMigrationRunner(
|
||||
connection_provider=connection_provider,
|
||||
).run()
|
||||
156
app/src/storage/postgres_pool.py
Normal file
156
app/src/storage/postgres_pool.py
Normal file
@@ -0,0 +1,156 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any
|
||||
|
||||
from src.storage.exceptions import PostgresConnectionPoolError
|
||||
|
||||
|
||||
PostgresPoolFactory = Callable[..., Any]
|
||||
|
||||
|
||||
def _default_pool_factory(**kwargs: Any) -> Any:
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
return ConnectionPool(**kwargs)
|
||||
|
||||
|
||||
class PostgresConnectionPool:
|
||||
"""Явная обёртка управляемого жизненного цикла пула psycopg."""
|
||||
|
||||
__slots__ = (
|
||||
"_conninfo",
|
||||
"_min_size",
|
||||
"_max_size",
|
||||
"_name",
|
||||
"_pool",
|
||||
"_pool_factory",
|
||||
"_timeout_seconds",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
conninfo: str,
|
||||
min_size: int = 1,
|
||||
max_size: int = 4,
|
||||
timeout_seconds: float = 10.0,
|
||||
name: str = "dzentra-storage",
|
||||
pool_factory: PostgresPoolFactory = _default_pool_factory,
|
||||
) -> None:
|
||||
normalized_conninfo = str(conninfo or "").strip()
|
||||
normalized_name = str(name or "").strip()
|
||||
|
||||
if not normalized_conninfo:
|
||||
raise ValueError("conninfo must not be empty")
|
||||
|
||||
if (
|
||||
isinstance(min_size, bool)
|
||||
or not isinstance(min_size, int)
|
||||
or min_size <= 0
|
||||
):
|
||||
raise ValueError("min_size must be a positive integer")
|
||||
|
||||
if (
|
||||
isinstance(max_size, bool)
|
||||
or not isinstance(max_size, int)
|
||||
or max_size < min_size
|
||||
):
|
||||
raise ValueError(
|
||||
"max_size must be an integer not smaller than min_size"
|
||||
)
|
||||
|
||||
if (
|
||||
isinstance(timeout_seconds, bool)
|
||||
or not isinstance(timeout_seconds, (int, float))
|
||||
or not math.isfinite(float(timeout_seconds))
|
||||
or timeout_seconds <= 0
|
||||
):
|
||||
raise ValueError("timeout_seconds must be positive and finite")
|
||||
|
||||
if not normalized_name:
|
||||
raise ValueError("name must not be empty")
|
||||
|
||||
if not callable(pool_factory):
|
||||
raise TypeError("pool_factory must be callable")
|
||||
|
||||
self._conninfo = normalized_conninfo
|
||||
self._min_size = min_size
|
||||
self._max_size = max_size
|
||||
self._timeout_seconds = float(timeout_seconds)
|
||||
self._name = normalized_name
|
||||
self._pool_factory = pool_factory
|
||||
self._pool: Any | None = None
|
||||
|
||||
@property
|
||||
def is_open(self) -> bool:
|
||||
return self._pool is not None
|
||||
|
||||
def open(self) -> None:
|
||||
"""Открыть пул и проверить создание минимального числа соединений."""
|
||||
if self._pool is not None:
|
||||
return
|
||||
|
||||
pool: Any = None
|
||||
|
||||
try:
|
||||
pool = self._pool_factory(
|
||||
conninfo=self._conninfo,
|
||||
min_size=self._min_size,
|
||||
max_size=self._max_size,
|
||||
timeout=self._timeout_seconds,
|
||||
kwargs={"autocommit": False},
|
||||
name=self._name,
|
||||
open=False,
|
||||
)
|
||||
pool.open(
|
||||
wait=True,
|
||||
timeout=self._timeout_seconds,
|
||||
)
|
||||
except BaseException as error:
|
||||
if pool is not None:
|
||||
try:
|
||||
pool.close(timeout=self._timeout_seconds)
|
||||
except Exception as cleanup_error:
|
||||
error.add_note(
|
||||
"PostgreSQL pool cleanup also failed: "
|
||||
f"{type(cleanup_error).__name__}."
|
||||
)
|
||||
|
||||
if not isinstance(error, Exception):
|
||||
raise
|
||||
|
||||
raise PostgresConnectionPoolError(
|
||||
"Failed to open PostgreSQL connection pool."
|
||||
) from error
|
||||
|
||||
self._pool = pool
|
||||
|
||||
def connection(self) -> AbstractContextManager[Any]:
|
||||
"""Выдать одно транзакционное соединение из открытого пула."""
|
||||
pool = self._pool
|
||||
|
||||
if pool is None:
|
||||
raise PostgresConnectionPoolError(
|
||||
"PostgreSQL connection pool is not open."
|
||||
)
|
||||
|
||||
return pool.connection(timeout=self._timeout_seconds)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Закрыть пул; повторное закрытие ничего не делает."""
|
||||
pool = self._pool
|
||||
|
||||
if pool is None:
|
||||
return
|
||||
|
||||
self._pool = None
|
||||
|
||||
try:
|
||||
pool.close(timeout=self._timeout_seconds)
|
||||
except Exception as error:
|
||||
raise PostgresConnectionPoolError(
|
||||
"Failed to close PostgreSQL connection pool."
|
||||
) from error
|
||||
Reference in New Issue
Block a user