Build 060.29: implement Market Data Access and Replay

app/tests/unit/storage/test_migrations.py

@@ -7,6 +7,7 @@ import pytest
 
 from src.storage.exceptions import StorageMigrationError
 from src.storage.migrations import (
+    MARKET_DATA_PARTITION_ADVISORY_LOCK_ID,
     STORAGE_MIGRATION_ADVISORY_LOCK_ID,
     STORAGE_MIGRATIONS,
     StorageMigration,
@@ -106,6 +107,7 @@ def test_default_migrations_have_stable_order_and_names() -> None:
         (6, "add_quote_and_candle_observation_sources"),
         (7, "create_market_data_partition_registry"),
         (8, "create_trade_stream_checkpoints"),
+        (9, "add_global_market_data_replay_sequence"),
     )
 
 
@@ -158,12 +160,73 @@ def test_default_schema_defines_partitions_identities_and_constraints() -> None:
     assert "CHECK (checkpoint_schema_version > 0)" in sql
 
 
+def test_replay_sequence_migration_has_atomic_global_order_contract() -> None:
+    migration = STORAGE_MIGRATIONS[-1]
+    statements = tuple(
+        " ".join(statement.split())
+        for statement in migration.statements
+    )
+    sql = "\n".join(statements)
+
+    assert migration.version == 9
+    assert migration.name == "add_global_market_data_replay_sequence"
+    assert statements[:4] == (
+        (
+            "SELECT pg_advisory_xact_lock("
+            f"{MARKET_DATA_PARTITION_ADVISORY_LOCK_ID}"
+            ")"
+        ),
+        "LOCK TABLE market_data.trades IN ACCESS EXCLUSIVE MODE",
+        "LOCK TABLE market_data.quotes IN ACCESS EXCLUSIVE MODE",
+        (
+            "LOCK TABLE market_data.candle_revisions "
+            "IN ACCESS EXCLUSIVE MODE"
+        ),
+    )
+    assert "CREATE SEQUENCE market_data.replay_sequence AS BIGINT" in sql
+    assert "MINVALUE 1" in sql
+    assert "CACHE 1" in sql
+    assert "NO CYCLE" in sql
+    assert "OWNED BY NONE" in sql
+    assert sql.count("ADD COLUMN replay_sequence BIGINT") == 3
+    assert "CREATE TABLE market_data.replay_sequence" not in sql
+    assert "CREATE TEMPORARY TABLE market_data_replay_sequence_backfill" in sql
+    assert "ON COMMIT DROP" in sql
+    assert "ROW_NUMBER() OVER" in sql
+    assert "UNION ALL" in sql
+    assert "event_time, data_type_rank, venue COLLATE \"C\"" in sql
+    assert "symbol COLLATE \"C\"" in sql
+    assert "candle_interval COLLATE \"C\"" in sql
+    assert "ctid" not in sql.lower()
+    assert sql.count("SET replay_sequence = backfill.replay_sequence") == 3
+    assert "target.trade_id = backfill.trade_id" in sql
+    assert "target.received_at = backfill.received_at" in sql
+    assert "target.interval = backfill.interval" in sql
+    assert "target.open_time = backfill.open_time" in sql
+    assert "target.observed_at = backfill.observed_at" in sql
+    assert "SELECT pg_catalog.setval(" in sql
+    assert "EXISTS ( SELECT 1 FROM market_data_replay_sequence_backfill )" in sql
+    assert sql.count("SET DEFAULT nextval(") == 3
+    assert sql.count("ALTER COLUMN replay_sequence SET NOT NULL") == 3
+    assert sql.count("CHECK (replay_sequence > 0)") == 3
+    assert "CREATE FUNCTION market_data.reject_replay_sequence_change()" in sql
+    assert sql.count("BEFORE UPDATE OF replay_sequence") == 3
+    assert "CREATE INDEX trades_history_keyset_idx" in sql
+    assert "CREATE INDEX quotes_history_keyset_idx" in sql
+    assert "CREATE INDEX candle_revisions_history_keyset_idx" in sql
+    assert "CREATE INDEX candle_revisions_replay_keyset_idx" in sql
+    assert "executed_at, replay_sequence" in sql
+    assert "received_at, replay_sequence" in sql
+    assert "interval, open_time, replay_sequence" in sql
+    assert "interval, observed_at, replay_sequence" in sql
+
+
 def test_run_locks_and_applies_every_pending_migration_in_order() -> None:
     runner, cursor, connection, provider = _runner()
 
     result = runner.run()
 
-    assert result == (1, 2, 3, 4, 5, 6, 7, 8)
+    assert result == (1, 2, 3, 4, 5, 6, 7, 8, 9)
     assert provider.calls == 1
     assert connection.entered == 1
     assert connection.exited == 1
@@ -180,7 +243,7 @@ def test_run_locks_and_applies_every_pending_migration_in_order() -> None:
         )
         and isinstance(parameters, tuple)
     )
-    assert inserted_versions == (1, 2, 3, 4, 5, 6, 7, 8)
+    assert inserted_versions == (1, 2, 3, 4, 5, 6, 7, 8, 9)
 
 
 def test_run_skips_already_applied_migrations() -> None:
@@ -209,7 +272,7 @@ def test_run_applies_only_migrations_after_existing_prefix() -> None:
 
     result = runner.run()
 
-    assert result == (3, 4, 5, 6, 7, 8)
+    assert result == (3, 4, 5, 6, 7, 8, 9)
     inserted_versions = tuple(
         parameters[0]
         for statement, parameters in cursor.calls
@@ -218,7 +281,7 @@ def test_run_applies_only_migrations_after_existing_prefix() -> None:
         )
         and isinstance(parameters, tuple)
     )
-    assert inserted_versions == (3, 4, 5, 6, 7, 8)
+    assert inserted_versions == (3, 4, 5, 6, 7, 8, 9)
 
 
 def test_run_rejects_unknown_applied_version() -> None: