Build 060.30: finalize Market Data Acquisition documentation

2026-08-03 23:30:41 +03:00
parent 8c98de9acc
commit 64a5bdd04c
61 changed files with 13980 additions and 1873 deletions
--- a/infra/compose/docker-compose.exchange-auth.yml
+++ b/infra/compose/docker-compose.exchange-auth.yml
@@ -0,0 +1,14 @@
+services:
+  bot:
+    environment:
+      EXCHANGE_API_KEY_FILE: /run/secrets/exchange_api_key
+      EXCHANGE_API_SECRET_FILE: /run/secrets/exchange_api_secret
+    secrets:
+      - exchange_api_key
+      - exchange_api_secret
+
+secrets:
+  exchange_api_key:
+    file: ${EXCHANGE_API_KEY_SOURCE_FILE:?EXCHANGE_API_KEY_SOURCE_FILE must be set}
+  exchange_api_secret:
+    file: ${EXCHANGE_API_SECRET_SOURCE_FILE:?EXCHANGE_API_SECRET_SOURCE_FILE must be set}
--- a/infra/compose/docker-compose.yml
+++ b/infra/compose/docker-compose.yml
@@ -1,33 +1,126 @@
+name: ${DZENTRA_COMPOSE_PROJECT_NAME:?DZENTRA_COMPOSE_PROJECT_NAME must be set}
+
 services:
  postgres:
-    image: postgres:17
-    container_name: dzentra_postgres
-    restart: unless-stopped
+    build:
+      context: ../..
+      dockerfile: infra/docker/postgres/Dockerfile
+    image: ${DZENTRA_POSTGRES_IMAGE:-dzentra-postgres:16.14-hardened}
+    pull_policy: build
+    restart: "no"
    environment:
-      POSTGRES_DB: dzentra_bot
-      POSTGRES_USER: dzentra_bot
-      POSTGRES_PASSWORD: change_me
-    ports:
-      - "5432:5432"
+      POSTGRES_DB: ${DB_NAME:-dzentra_bot}
+      POSTGRES_USER: ${POSTGRES_ADMIN_USER:-dzentra_admin}
+      POSTGRES_PASSWORD_FILE: /run/secrets/postgres_admin_password
+      APP_DB_USER: ${DB_USER:-dzentra_bot}
+    secrets:
+      - postgres_admin_password
+      - db_password
    volumes:
      - dzentra_postgres_data:/var/lib/postgresql/data
    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U dzentra_bot -d dzentra_bot"]
+      test:
+        - CMD
+        - /usr/local/bin/dzentra-postgres-healthcheck
      interval: 5s
      timeout: 3s
-      retries: 10
+      retries: 3
+      start_period: 10s
+    networks:
+      - database
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    cap_add:
+      - CHOWN
+      - DAC_OVERRIDE
+      - FOWNER
+      - SETGID
+      - SETUID
+    read_only: true
+    tmpfs:
+      - /tmp:rw,noexec,nosuid,nodev,size=64m
+      - /var/run/postgresql:rw,nosuid,nodev,size=16m
+    stop_grace_period: ${POSTGRES_STOP_GRACE_PERIOD:?POSTGRES_STOP_GRACE_PERIOD must be set}

  bot:
    build:
      context: ../..
      dockerfile: infra/docker/Dockerfile
-    container_name: dzentra_bot
-    restart: unless-stopped
-    env_file:
-      - ../../app/.env
+    image: ${DZENTRA_BOT_IMAGE:-dzentra-bot:local}
+    pull_policy: build
+    restart: "no"
+    init: true
+    user: "10001:10001"
+    environment:
+      BOT_TOKEN_FILE: /run/secrets/bot_token
+      BOT_PARSE_MODE: ${BOT_PARSE_MODE:-HTML}
+      APP_ENV: ${APP_ENV:-prod}
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
+      TZ: ${TZ:-Europe/Minsk}
+      DEBUG_ENABLED: ${DEBUG_ENABLED:-false}
+      DZENTRA_RUNTIME_ENV_FILE: /var/lib/dzentra/runtime.env
+      DB_HOST: postgres
+      DB_PORT: "5432"
+      DB_NAME: ${DB_NAME:-dzentra_bot}
+      DB_USER: ${DB_USER:-dzentra_bot}
+      DB_PASSWORD_FILE: /run/secrets/db_password
+      MARKET_DATA_STORAGE_ENABLED: ${MARKET_DATA_STORAGE_ENABLED:-false}
+      MARKET_DATA_STORAGE_POOL_MIN_SIZE: ${MARKET_DATA_STORAGE_POOL_MIN_SIZE:-1}
+      MARKET_DATA_STORAGE_POOL_MAX_SIZE: ${MARKET_DATA_STORAGE_POOL_MAX_SIZE:-4}
+      MARKET_DATA_STORAGE_POOL_TIMEOUT_SECONDS: ${MARKET_DATA_STORAGE_POOL_TIMEOUT_SECONDS:-10}
+      EXCHANGE_ENABLED: ${EXCHANGE_ENABLED:-false}
+      EXCHANGE_NAME: ${EXCHANGE_NAME:-dzengi}
+      EXCHANGE_BASE_URL: ${EXCHANGE_BASE_URL:-}
+      EXCHANGE_WS_URL: ${EXCHANGE_WS_URL:-}
+      EXCHANGE_TIMEOUT_SEC: ${EXCHANGE_TIMEOUT_SEC:-10}
+      EXCHANGE_TESTNET: ${EXCHANGE_TESTNET:-false}
+      DEFAULT_SYMBOL: ${DEFAULT_SYMBOL:-ETH/USD_LEVERAGE}
+      TRADE_STREAM_ENABLED: ${TRADE_STREAM_ENABLED:-false}
+      TRADE_STREAM_WS_URL: ${TRADE_STREAM_WS_URL:-}
+      TRADE_STREAM_SYMBOLS: ${TRADE_STREAM_SYMBOLS:-}
+      TRADE_STREAM_OPEN_TIMEOUT_SECONDS: ${TRADE_STREAM_OPEN_TIMEOUT_SECONDS:-10}
+      TRADE_STREAM_PROBE_TIMEOUT_SECONDS: ${TRADE_STREAM_PROBE_TIMEOUT_SECONDS:-20}
+      TRADE_STREAM_CLOSE_TIMEOUT_SECONDS: ${TRADE_STREAM_CLOSE_TIMEOUT_SECONDS:-10}
+      TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS: ${TRADE_STREAM_HEARTBEAT_TIMEOUT_SECONDS:-30}
+      TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS: ${TRADE_STREAM_SCHEDULER_INTERVAL_SECONDS:-5}
+      TRADE_STREAM_RECOVERY_WINDOW_MS: ${TRADE_STREAM_RECOVERY_WINDOW_MS:-3599999}
+      TRADE_STREAM_SUBSCRIPTION_ACK_TIMEOUT_SECONDS: ${TRADE_STREAM_SUBSCRIPTION_ACK_TIMEOUT_SECONDS:-10}
+      TRADE_STREAM_STARTUP_MARKET_BUFFER_CAPACITY: ${TRADE_STREAM_STARTUP_MARKET_BUFFER_CAPACITY:-10000}
+    secrets:
+      - bot_token
+      - db_password
+    volumes:
+      - dzentra_runtime_config:/var/lib/dzentra
    depends_on:
      postgres:
        condition: service_healthy
+    networks:
+      - database
+      - egress
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    read_only: true
+    tmpfs:
+      - /tmp:rw,noexec,nosuid,nodev,size=64m
+    stop_grace_period: ${BOT_STOP_GRACE_PERIOD:?BOT_STOP_GRACE_PERIOD must be set}

 volumes:
-  dzentra_postgres_data:
+  dzentra_postgres_data:
+  dzentra_runtime_config:
+
+networks:
+  database:
+    internal: true
+  egress:
+
+secrets:
+  postgres_admin_password:
+    file: ${POSTGRES_ADMIN_PASSWORD_SECRET_FILE:?POSTGRES_ADMIN_PASSWORD_SECRET_FILE must be set}
+  bot_token:
+    file: ${BOT_TOKEN_SECRET_FILE:?BOT_TOKEN_SECRET_FILE must be set}
+  db_password:
+    file: ${DB_PASSWORD_SECRET_FILE:?DB_PASSWORD_SECRET_FILE must be set}
--- a/infra/docker/Dockerfile
+++ b/infra/docker/Dockerfile
@@ -1,12 +1,41 @@
-FROM python:3.12-slim
+FROM python:3.12.13-slim-bookworm@sha256:d50fb7611f86d04a3b0471b46d7557818d88983fc3136726336b2a4c657aa30b

 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1
+ENV PIP_NO_CACHE_DIR=1
+ENV PIP_ROOT_USER_ACTION=ignore

-WORKDIR /project
-
-COPY app /project/app
-RUN pip install --no-cache-dir -r /project/app/requirements.txt
+RUN groupadd --gid 10001 dzentra \
+    && useradd --uid 10001 \
+        --gid dzentra \
+        --no-create-home \
+        --home-dir /nonexistent \
+        --shell /usr/sbin/nologin \
+        dzentra \
+    && install -d \
+        -m 0700 \
+        -o dzentra \
+        -g dzentra \
+        /var/lib/dzentra \
+    && install \
+        -m 0600 \
+        -o dzentra \
+        -g dzentra \
+        /dev/null \
+        /var/lib/dzentra/runtime.env

 WORKDIR /project/app
+
+COPY app/requirements.lock ./requirements.lock
+RUN python -m pip install \
+    --require-hashes \
+    --requirement requirements.lock
+
+COPY --chown=10001:10001 app/src ./src
+
+USER 10001:10001
+
+STOPSIGNAL SIGINT
+
 CMD ["python", "-m", "src.main"]
--- a/infra/docker/postgres/Dockerfile
+++ b/infra/docker/postgres/Dockerfile
@@ -0,0 +1,9 @@
+FROM postgres:16.14-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777
+
+COPY --chmod=0555 \
+    infra/docker/postgres/init-application-role.sh \
+    /docker-entrypoint-initdb.d/20-init-application-role.sh
+
+COPY --chmod=0555 \
+    infra/docker/postgres/healthcheck-application-role.sh \
+    /usr/local/bin/dzentra-postgres-healthcheck
--- a/infra/docker/postgres/healthcheck-application-role.sh
+++ b/infra/docker/postgres/healthcheck-application-role.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+set -eu
+
+
+role_status="$(
+    psql \
+        --no-psqlrc \
+        --set ON_ERROR_STOP=1 \
+        --tuples-only \
+        --no-align \
+        --username "$POSTGRES_USER" \
+        --dbname "$POSTGRES_DB" <<'SQL'
+\getenv app_user APP_DB_USER
+
+SELECT CASE
+    WHEN :'app_user' <> current_user
+        AND EXISTS (
+            SELECT 1
+            FROM pg_catalog.pg_roles
+            WHERE rolname = :'app_user'
+                AND rolcanlogin
+                AND NOT rolsuper
+                AND NOT rolcreatedb
+                AND NOT rolcreaterole
+                AND rolinherit
+                AND NOT rolreplication
+                AND NOT rolbypassrls
+        )
+        AND pg_catalog.pg_get_userbyid((
+            SELECT datdba
+            FROM pg_catalog.pg_database
+            WHERE datname = current_database()
+        )) = :'app_user'
+        AND NOT EXISTS (
+            SELECT 1
+            FROM pg_catalog.pg_auth_members AS membership
+            JOIN pg_catalog.pg_roles AS member_role
+                ON member_role.oid = membership.member
+            WHERE member_role.rolname = :'app_user'
+        )
+    THEN 1
+    ELSE 0
+END;
+SQL
+)"
+
+test "$role_status" = "1"
--- a/infra/docker/postgres/init-application-role.sh
+++ b/infra/docker/postgres/init-application-role.sh
@@ -0,0 +1,130 @@
+#!/bin/sh
+
+set -eu
+
+
+read_secret_file() {
+    secret_path=$1
+
+    if [ ! -r "$secret_path" ]; then
+        echo "Не удалось прочитать обязательный PostgreSQL secret file" >&2
+        return 1
+    fi
+
+    # Маркер сохраняет переводы строк внутри command substitution.
+    secret_with_marker="$(cat "$secret_path"; printf '\001')"
+    secret=${secret_with_marker%?}
+    carriage_return="$(printf '\r')"
+    newline='
+'
+
+    case "$secret" in
+        *"${carriage_return}${newline}")
+            secret=${secret%"${carriage_return}${newline}"}
+            ;;
+        *"${newline}")
+            secret=${secret%"${newline}"}
+            ;;
+        *"${carriage_return}")
+            secret=${secret%"${carriage_return}"}
+            ;;
+    esac
+
+    if [ -z "$(printf '%s' "$secret" | tr -d '[:space:]')" ]; then
+        echo "PostgreSQL secret file содержит пустое значение" >&2
+        return 1
+    fi
+
+    printf '%s' "$secret"
+}
+
+
+if [ -z "${APP_DB_USER:-}" ]; then
+    echo "APP_DB_USER должен содержать имя роли приложения" >&2
+    exit 1
+fi
+
+if [ "$APP_DB_USER" = "$POSTGRES_USER" ]; then
+    echo "APP_DB_USER не должен совпадать с PostgreSQL admin role" >&2
+    exit 1
+fi
+
+app_db_password="$(read_secret_file /run/secrets/db_password)"
+
+# Официальный entrypoint уже прочитал secret file и применил собственную
+# нормализацию конечных переводов строк. Сравниваем именно фактический пароль.
+if [ -z "$(printf '%s' "${POSTGRES_PASSWORD:-}" | tr -d '[:space:]')" ]; then
+    echo "POSTGRES_PASSWORD должен содержать пароль admin role" >&2
+    exit 1
+fi
+
+postgres_admin_password=$POSTGRES_PASSWORD
+
+if [ "$app_db_password" = "$postgres_admin_password" ]; then
+    echo "Пароли PostgreSQL admin и application roles должны различаться" >&2
+    exit 1
+fi
+
+export DZENTRA_APP_DB_PASSWORD="$app_db_password"
+
+psql \
+    --no-psqlrc \
+    --single-transaction \
+    --set ON_ERROR_STOP=1 \
+    --username "$POSTGRES_USER" \
+    --dbname "$POSTGRES_DB" <<'SQL'
+\getenv app_user APP_DB_USER
+\getenv app_password DZENTRA_APP_DB_PASSWORD
+
+SELECT format(
+    'CREATE ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB '
+    'NOCREATEROLE INHERIT NOREPLICATION NOBYPASSRLS',
+    :'app_user',
+    :'app_password'
+)
+WHERE NOT EXISTS (
+    SELECT 1
+    FROM pg_catalog.pg_roles
+    WHERE rolname = :'app_user'
+)
+\gexec
+
+SELECT format(
+    'ALTER ROLE %I WITH LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB '
+    'NOCREATEROLE INHERIT NOREPLICATION NOBYPASSRLS',
+    :'app_user',
+    :'app_password'
+)
+\gexec
+
+SELECT format(
+    'ALTER DATABASE %I OWNER TO %I',
+    current_database(),
+    :'app_user'
+)
+\gexec
+
+SELECT format(
+    'REVOKE ALL ON DATABASE %I FROM PUBLIC',
+    current_database()
+)
+\gexec
+
+SELECT format(
+    'GRANT CONNECT, TEMPORARY ON DATABASE %I TO %I',
+    current_database(),
+    :'app_user'
+)
+\gexec
+
+REVOKE CREATE ON SCHEMA public FROM PUBLIC;
+SELECT format(
+    'GRANT USAGE, CREATE ON SCHEMA public TO %I',
+    :'app_user'
+)
+\gexec
+SQL
+
+unset DZENTRA_APP_DB_PASSWORD
+unset app_db_password
+unset postgres_admin_password