# app/scripts/check_trade_backfill_api.py from __future__ import annotations import argparse import json import time from datetime import UTC, datetime from typing import Any from urllib.error import HTTPError, URLError from urllib.parse import urlencode from urllib.request import Request, urlopen from src.core.config import load_settings DEFAULT_ENDPOINT_PATH = "/api/v2/aggTrades" DEFAULT_LIMIT = 20 DEFAULT_REQUEST_DELAY_SECONDS = 0.2 MAX_RESPONSE_PREVIEW_ITEMS = 10 STATUS_PASS = "PASS" STATUS_FAIL = "FAIL" STATUS_INCONCLUSIVE = "INCONCLUSIVE" STATUS_ERROR = "ERROR" STATUS_SKIPPED = "SKIPPED" def parse_arguments() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( "Исследование REST API aggTrades для проектирования " "подсистемы Trade Recovery." ) ) parser.add_argument( "--symbol", help=( "Торговый символ. По умолчанию используется DEFAULT_SYMBOL " "из конфигурации проекта." ), ) parser.add_argument( "--limit", type=int, default=DEFAULT_LIMIT, help=( "Базовый размер выборки для диагностических запросов " f"(по умолчанию: {DEFAULT_LIMIT})." ), ) parser.add_argument( "--delay", type=float, default=DEFAULT_REQUEST_DELAY_SECONDS, help=( "Пауза между HTTP-запросами в секундах " f"(по умолчанию: {DEFAULT_REQUEST_DELAY_SECONDS})." ), ) parser.add_argument( "--endpoint-path", default=DEFAULT_ENDPOINT_PATH, help=( "Путь REST endpoint " f"(по умолчанию: {DEFAULT_ENDPOINT_PATH})." ), ) parser.add_argument( "--json", action="store_true", help="Вывести итоговый отчёт только в формате JSON.", ) parser.add_argument( "--verbose", action="store_true", help="Показывать расширенные диагностические данные.", ) arguments = parser.parse_args() if arguments.limit <= 0: parser.error("--limit должен быть положительным целым числом.") if arguments.delay < 0: parser.error("--delay не может быть отрицательным.") if not str(arguments.endpoint_path).strip(): parser.error("--endpoint-path не может быть пустым.") return arguments def build_rest_url( base_url: str, endpoint_path: str, ) -> str: normalized_base_url = base_url.strip().rstrip("/") if not normalized_base_url: raise RuntimeError( "EXCHANGE_BASE_URL не задан в конфигурации проекта." ) if not normalized_base_url.startswith(("http://", "https://")): raise RuntimeError( "EXCHANGE_BASE_URL должен начинаться с http:// или https://." ) return ( f"{normalized_base_url}/" f"{endpoint_path.strip().lstrip('/')}" ) def build_headers( api_key: str, ) -> dict[str, str]: headers = { "Accept": "application/json", "Content-Type": "application/json", "User-Agent": "Dzentra-Trade-Backfill-Diagnostics/1.0", } normalized_api_key = api_key.strip() if normalized_api_key: headers["X-MBX-APIKEY"] = normalized_api_key return headers def utc_timestamp() -> str: return datetime.now(UTC).isoformat() def format_json(document: object) -> str: return json.dumps( document, ensure_ascii=False, indent=2, sort_keys=True, ) def print_separator( character: str = "=", width: int = 80, ) -> None: print(character * width) def print_section(title: str) -> None: print() print_separator() print(title) print_separator() print() def print_subsection(title: str) -> None: print() print_separator("-") print(title) print_separator("-") print() def build_result( name: str, status: str, summary: str, *, details: list[str] | None = None, evidence: dict[str, Any] | None = None, ) -> dict[str, Any]: return { "name": name, "status": status, "summary": summary, "details": details or [], "evidence": evidence or {}, } def print_result( result: dict[str, Any], *, verbose: bool, ) -> None: print(f"Status: {result['status']}") print(f"Conclusion: {result['summary']}") details = result.get("details") or [] if details: print() print("Details:") for detail in details: print(f" - {detail}") evidence = result.get("evidence") or {} if verbose and evidence: print() print("Evidence:") print(format_json(evidence)) def request_json( url: str, headers: dict[str, str], timeout_seconds: float, *, params: dict[str, Any] | None = None, ) -> dict[str, Any]: query = urlencode( { key: value for key, value in (params or {}).items() if value is not None } ) request_url = f"{url}?{query}" if query else url request = Request( request_url, headers=headers, method="GET", ) started_at = time.monotonic() try: with urlopen( request, timeout=timeout_seconds, ) as response: raw_body = response.read().decode( "utf-8", errors="replace", ) elapsed_ms = ( time.monotonic() - started_at ) * 1000.0 try: payload = json.loads(raw_body) except json.JSONDecodeError as exc: return { "ok": False, "url": request_url, "status_code": response.status, "elapsed_ms": elapsed_ms, "payload": None, "raw_body": raw_body, "error_type": type(exc).__name__, "error": "REST API вернул невалидный JSON.", } return { "ok": True, "url": request_url, "status_code": response.status, "elapsed_ms": elapsed_ms, "payload": payload, "raw_body": raw_body, "error_type": None, "error": None, } except HTTPError as exc: elapsed_ms = ( time.monotonic() - started_at ) * 1000.0 raw_body = exc.read().decode( "utf-8", errors="replace", ) try: payload: object = json.loads(raw_body) except json.JSONDecodeError: payload = raw_body return { "ok": False, "url": request_url, "status_code": exc.code, "elapsed_ms": elapsed_ms, "payload": payload, "raw_body": raw_body, "error_type": type(exc).__name__, "error": str(exc), } except URLError as exc: elapsed_ms = ( time.monotonic() - started_at ) * 1000.0 return { "ok": False, "url": request_url, "status_code": None, "elapsed_ms": elapsed_ms, "payload": None, "raw_body": "", "error_type": type(exc).__name__, "error": str(exc.reason), } except TimeoutError as exc: elapsed_ms = ( time.monotonic() - started_at ) * 1000.0 return { "ok": False, "url": request_url, "status_code": None, "elapsed_ms": elapsed_ms, "payload": None, "raw_body": "", "error_type": type(exc).__name__, "error": str(exc), } def execute_request( url: str, headers: dict[str, str], timeout_seconds: float, request_delay_seconds: float, *, params: dict[str, Any] | None = None, ) -> dict[str, Any]: response = request_json( url, headers, timeout_seconds, params=params, ) if request_delay_seconds > 0: time.sleep(request_delay_seconds) return response def response_preview( response: dict[str, Any], ) -> object: payload = response.get("payload") if not isinstance(payload, list): return payload if len(payload) <= MAX_RESPONSE_PREVIEW_ITEMS: return payload return payload[:MAX_RESPONSE_PREVIEW_ITEMS] def response_evidence( response: dict[str, Any], ) -> dict[str, Any]: return { "url": response.get("url"), "status_code": response.get("status_code"), "elapsed_ms": round( float(response.get("elapsed_ms") or 0.0), 3, ), "payload_preview": response_preview(response), "error_type": response.get("error_type"), "error": response.get("error"), } def extract_trade_id( document: dict[str, Any], ) -> int: for field_name in ( "a", "id", "tradeId", "trade_id", ): value = document.get(field_name) if value is None or isinstance(value, bool): continue try: return int(value) except (TypeError, ValueError): continue raise ValueError( "Не удалось определить trade_id в документе сделки: " f"{document!r}" ) def extract_trade_timestamp_ms( document: dict[str, Any], ) -> int | None: for field_name in ( "T", "time", "timestamp", "executedAt", "executed_at", ): value = document.get(field_name) if value is None or isinstance(value, bool): continue try: return int(value) except (TypeError, ValueError): continue return None def normalize_trade_document( document: object, ) -> dict[str, Any]: if not isinstance(document, dict): raise ValueError( "Элемент ответа aggTrades должен быть JSON object, " f"получено: {type(document).__name__}." ) return { "trade_id": extract_trade_id(document), "timestamp_ms": extract_trade_timestamp_ms(document), "raw": document, } def normalize_trade_list( payload: object, ) -> list[dict[str, Any]]: if not isinstance(payload, list): raise ValueError( "Ответ aggTrades должен быть JSON array, " f"получено: {type(payload).__name__}." ) return [ normalize_trade_document(document) for document in payload ] def trade_ids( trades: list[dict[str, Any]], ) -> list[int]: return [ int(trade["trade_id"]) for trade in trades ] def trade_timestamps_ms( trades: list[dict[str, Any]], ) -> list[int | None]: return [ trade.get("timestamp_ms") for trade in trades ] def describe_trade_range( trades: list[dict[str, Any]], ) -> dict[str, Any]: ids = trade_ids(trades) timestamps = trade_timestamps_ms(trades) return { "count": len(trades), "first_trade_id": ids[0] if ids else None, "last_trade_id": ids[-1] if ids else None, "trade_ids": ids, "first_timestamp_ms": timestamps[0] if timestamps else None, "last_timestamp_ms": timestamps[-1] if timestamps else None, } def detect_order(values: list[int]) -> str: if len(values) < 2: return "UNKNOWN" if all( current < following for current, following in zip(values, values[1:]) ): return "ASCENDING" if all( current > following for current, following in zip(values, values[1:]) ): return "DESCENDING" return "UNORDERED" def is_non_decreasing(values: list[int]) -> bool: return all( current <= following for current, following in zip(values, values[1:]) ) def is_non_increasing(values: list[int]) -> bool: return all( current >= following for current, following in zip(values, values[1:]) ) def response_matches_trade_ids( trades: list[dict[str, Any]], expected_ids: list[int], ) -> bool: return trade_ids(trades) == expected_ids def request_trades( url: str, headers: dict[str, str], timeout_seconds: float, request_delay_seconds: float, *, symbol: str, from_id: int | None = None, start_time: int | None = None, end_time: int | None = None, limit: int | None = None, ) -> tuple[dict[str, Any], list[dict[str, Any]] | None]: response = execute_request( url, headers, timeout_seconds, request_delay_seconds, params={ "symbol": symbol, "fromId": from_id, "startTime": start_time, "endTime": end_time, "limit": limit, }, ) if not response["ok"]: return response, None try: trades = normalize_trade_list(response["payload"]) except ValueError as exc: response = { **response, "ok": False, "error_type": type(exc).__name__, "error": str(exc), } return response, None return response, trades def test_endpoint_accessibility( url: str, headers: dict[str, str], timeout_seconds: float, request_delay_seconds: float, *, symbol: str, limit: int, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: response, trades = request_trades( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, limit=limit, ) if not response["ok"] or trades is None: return ( build_result( "Endpoint accessibility", STATUS_ERROR, "Endpoint aggTrades недоступен или вернул некорректный ответ.", evidence=response_evidence(response), ), [], ) if not trades: return ( build_result( "Endpoint accessibility", STATUS_INCONCLUSIVE, "Endpoint доступен, но не вернул сделок для выбранного символа.", details=[ "Проверьте торговую активность и корректность символа.", ], evidence=response_evidence(response), ), [], ) return ( build_result( "Endpoint accessibility", STATUS_PASS, "Endpoint aggTrades доступен и возвращает сделки.", details=[ f"Получено сделок: {len(trades)}.", f"HTTP status: {response['status_code']}.", ], evidence={ **response_evidence(response), "trade_range": describe_trade_range(trades), }, ), trades, ) def test_from_id_support( url: str, headers: dict[str, str], timeout_seconds: float, request_delay_seconds: float, *, symbol: str, baseline_trades: list[dict[str, Any]], limit: int, ) -> tuple[dict[str, Any], list[dict[str, Any]], int | None]: baseline_ids = trade_ids(baseline_trades) candidate_indexes = sorted( { min(5, len(baseline_ids) - 1), len(baseline_ids) // 2, len(baseline_ids) - 1, } ) observations: list[dict[str, Any]] = [] selected_trades: list[dict[str, Any]] = [] selected_from_id: int | None = None for index in candidate_indexes: requested_from_id = baseline_ids[index] response, returned_trades = request_trades( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, from_id=requested_from_id, limit=limit, ) returned_ids = trade_ids(returned_trades or []) matches_baseline = returned_ids == baseline_ids observations.append( { "baseline_index": index, "requested_from_id": requested_from_id, "ok": response["ok"], "status_code": response["status_code"], "returned_count": len(returned_ids), "first_trade_id": returned_ids[0] if returned_ids else None, "last_trade_id": returned_ids[-1] if returned_ids else None, "contains_requested_from_id": requested_from_id in returned_ids, "matches_latest_page": matches_baseline, "trade_ids": returned_ids, "error": response.get("error"), } ) if response["ok"] and returned_trades and not matches_baseline: selected_trades = returned_trades selected_from_id = requested_from_id successful = [item for item in observations if item["ok"]] changed = [ item for item in successful if not item["matches_latest_page"] ] if not successful: return ( build_result( "fromId support", STATUS_FAIL, "Все запросы с историческими fromId завершились ошибкой.", evidence={"observations": observations}, ), [], None, ) if not changed: return ( build_result( "fromId support", STATUS_INCONCLUSIVE, "Исторические значения fromId не изменили latest page; влияние параметра не подтверждено.", details=[ "Сервер мог проигнорировать fromId или применить fallback к последней странице.", ], evidence={"observations": observations}, ), [], None, ) return ( build_result( "fromId support", STATUS_PASS, "Исторический fromId изменяет возвращаемый диапазон сделок.", evidence={"observations": observations}, ), selected_trades, selected_from_id, ) def test_deep_historical_from_id( url: str, headers: dict[str, str], timeout_seconds: float, request_delay_seconds: float, *, symbol: str, baseline_trades: list[dict[str, Any]], limit: int, ) -> tuple[dict[str, Any], list[dict[str, Any]], int | None]: baseline_ids = trade_ids(baseline_trades) deep_response, deep_trades = request_trades( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, limit=1000, ) if not deep_response["ok"] or not deep_trades: return ( build_result( "Deep historical fromId", STATUS_ERROR, "Не удалось получить глубокую историческую выборку limit=1000.", evidence=response_evidence(deep_response), ), [], None, ) deep_ids = trade_ids(deep_trades) candidates = [trade_id for trade_id in reversed(deep_ids) if trade_id not in baseline_ids] if not candidates: return ( build_result( "Deep historical fromId", STATUS_INCONCLUSIVE, "В выборке limit=1000 не найден trade_id вне latest page.", evidence={ "latest_page_count": len(baseline_ids), "deep_page_count": len(deep_ids), "deep_first_trade_id": deep_ids[0] if deep_ids else None, "deep_last_trade_id": deep_ids[-1] if deep_ids else None, }, ), [], None, ) requested_from_id = candidates[0] response, returned_trades = request_trades( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, from_id=requested_from_id, limit=limit, ) returned_ids = trade_ids(returned_trades or []) matches_latest_page = returned_ids == baseline_ids contains_requested_from_id = requested_from_id in returned_ids evidence = { "requested_from_id": requested_from_id, "deep_page_count": len(deep_ids), "deep_first_trade_id": deep_ids[0] if deep_ids else None, "deep_last_trade_id": deep_ids[-1] if deep_ids else None, "ok": response["ok"], "status_code": response["status_code"], "returned_count": len(returned_ids), "first_trade_id": returned_ids[0] if returned_ids else None, "last_trade_id": returned_ids[-1] if returned_ids else None, "contains_requested_from_id": contains_requested_from_id, "matches_latest_page": matches_latest_page, "trade_ids": returned_ids, "error": response.get("error"), "payload": response.get("payload"), } if not response["ok"]: return ( build_result( "Deep historical fromId", STATUS_FAIL, "Запрос с глубоким историческим fromId завершился ошибкой.", evidence=evidence, ), [], requested_from_id, ) if matches_latest_page: return ( build_result( "Deep historical fromId", STATUS_FAIL, "Глубокий исторический fromId не изменил latest page.", details=[ "Практически подтверждено, что fromId игнорируется или приводит к fallback на latest page.", ], evidence=evidence, ), returned_trades or [], requested_from_id, ) return ( build_result( "Deep historical fromId", STATUS_PASS, "Глубокий исторический fromId изменяет возвращаемый диапазон сделок.", details=[ "Параметр fromId пригоден для дальнейшего исследования как курсор исторической навигации.", ], evidence=evidence, ), returned_trades or [], requested_from_id, ) def test_from_id_semantics( *, requested_from_id: int, trades: list[dict[str, Any]], ) -> dict[str, Any]: if not trades: return build_result( "fromId semantics", STATUS_INCONCLUSIVE, "Невозможно определить семантику fromId без сделок.", ) ids = trade_ids(trades) order = detect_order(ids) if requested_from_id not in ids: return build_result( "fromId semantics", STATUS_INCONCLUSIVE, "Запрошенный fromId отсутствует в ответе.", evidence={ "requested_from_id": requested_from_id, "order": order, "trade_ids": ids, }, ) position = ids.index(requested_from_id) return build_result( "fromId semantics", STATUS_PASS, "Запрошенный fromId включён в ответ.", details=[ f"Позиция requested_from_id в ответе: {position}.", f"Порядок ответа: {order}.", ], evidence={ "requested_from_id": requested_from_id, "position": position, "order": order, "trade_ids": ids, }, ) def test_ordering( trades: list[dict[str, Any]], ) -> dict[str, Any]: ids = trade_ids(trades) order = detect_order(ids) if order == "UNKNOWN": return build_result( "Ordering", STATUS_INCONCLUSIVE, "Недостаточно сделок для определения порядка.", ) if order == "UNORDERED": return build_result( "Ordering", STATUS_FAIL, "REST возвращает сделки в немонотонном порядке trade_id.", evidence={"order": order, "trade_ids": ids}, ) return build_result( "Ordering", STATUS_PASS, f"REST возвращает сделки в порядке {order} по trade_id.", evidence={"order": order, "trade_ids": ids}, ) def test_trade_id_spacing( trades: list[dict[str, Any]], ) -> dict[str, Any]: ids = trade_ids(trades) if len(ids) < 2: return build_result( "Trade ID spacing", STATUS_INCONCLUSIVE, "Недостаточно сделок для анализа расстояний между trade_id.", ) deltas = [ abs(following - current) for current, following in zip(ids, ids[1:]) ] return build_result( "Trade ID spacing", STATUS_PASS, "Расстояния между соседними trade_id измерены без предположения об арифметической непрерывности.", details=[ f"Минимальный delta: {min(deltas)}.", f"Максимальный delta: {max(deltas)}.", "Ненулевые gaps допустимы и не интерпретируются как потеря сделок.", ], evidence={ "trade_ids": ids, "absolute_deltas": deltas, }, ) def test_limit_behavior( url: str, headers: dict[str, str], timeout_seconds: float, request_delay_seconds: float, *, symbol: str, ) -> dict[str, Any]: requested_limits = (1, 5, 20, 50, 100, 500, 1000, 1001) observations: list[dict[str, Any]] = [] for requested_limit in requested_limits: response, trades = request_trades( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, limit=requested_limit, ) observations.append( { "requested_limit": requested_limit, "ok": response["ok"], "status_code": response["status_code"], "returned_count": len(trades or []), "error": response.get("error"), "payload": response.get("payload") if not response["ok"] else None, } ) overflow = [ item for item in observations if item["ok"] and item["returned_count"] > item["requested_limit"] ] if overflow: return build_result( "Limit behavior", STATUS_FAIL, "API вернул больше сделок, чем запрошено через limit.", evidence={"observations": observations}, ) accepted = [item for item in observations if item["ok"]] rejected = [item for item in observations if not item["ok"]] return build_result( "Limit behavior", STATUS_PASS, "Поведение limit измерено, включая предполагаемую верхнюю границу.", details=[ f"Принятые значения: {[item['requested_limit'] for item in accepted]}.", f"Отклонённые значения: {[item['requested_limit'] for item in rejected]}.", ], evidence={"observations": observations}, ) def test_invalid_from_id_fallback( url: str, headers: dict[str, str], timeout_seconds: float, request_delay_seconds: float, *, symbol: str, baseline_trades: list[dict[str, Any]], limit: int, ) -> dict[str, Any]: baseline_ids = trade_ids(baseline_trades) latest_trade_id = max(baseline_ids) cases = { "negative": -1, "future": latest_trade_id + 1_000_000, } observations: dict[str, Any] = {} for name, requested_from_id in cases.items(): response, trades = request_trades( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, from_id=requested_from_id, limit=limit, ) returned_ids = trade_ids(trades or []) observations[name] = { "requested_from_id": requested_from_id, "ok": response["ok"], "status_code": response["status_code"], "matches_latest_page": returned_ids == baseline_ids, "trade_ids": returned_ids, "error": response.get("error"), "payload": response.get("payload") if not response["ok"] else None, } fallback_cases = [ name for name, observation in observations.items() if observation["ok"] and observation["matches_latest_page"] ] if len(fallback_cases) == len(cases): return build_result( "Invalid fromId fallback", STATUS_PASS, "Некорректные fromId приводят к fallback на latest page.", evidence={"observations": observations}, ) return build_result( "Invalid fromId fallback", STATUS_INCONCLUSIVE, "Поведение некорректных fromId неоднородно или не совпадает с latest page.", evidence={"observations": observations}, ) def test_repeatability( url: str, headers: dict[str, str], timeout_seconds: float, request_delay_seconds: float, *, symbol: str, from_id: int | None, limit: int, ) -> dict[str, Any]: first_response, first_trades = request_trades( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, from_id=from_id, limit=limit, ) second_response, second_trades = request_trades( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, from_id=from_id, limit=limit, ) if ( not first_response["ok"] or not second_response["ok"] or first_trades is None or second_trades is None ): return build_result( "Repeatability", STATUS_INCONCLUSIVE, "Не удалось выполнить два идентичных запроса.", evidence={ "first": response_evidence(first_response), "second": response_evidence(second_response), }, ) first_ids = trade_ids(first_trades) second_ids = trade_ids(second_trades) if first_ids == second_ids: return build_result( "Repeatability", STATUS_PASS, "Идентичные запросы возвращают одинаковую последовательность trade_id.", evidence={ "from_id": from_id, "first_trade_ids": first_ids, "second_trade_ids": second_ids, }, ) return build_result( "Repeatability", STATUS_FAIL, "Идентичные запросы возвращают разные последовательности trade_id.", evidence={ "from_id": from_id, "first_trade_ids": first_ids, "second_trade_ids": second_ids, }, ) def test_time_filters( url: str, headers: dict[str, str], timeout_seconds: float, request_delay_seconds: float, *, symbol: str, baseline_trades: list[dict[str, Any]], limit: int, ) -> dict[str, Any]: timestamps = [ timestamp for timestamp in trade_timestamps_ms(baseline_trades) if timestamp is not None ] if len(timestamps) < 3: return build_result( "Time filters", STATUS_SKIPPED, "Недостаточно временных меток для проверки фильтров.", ) lower_bound = sorted(timestamps)[len(timestamps) // 3] upper_bound = sorted(timestamps)[(len(timestamps) * 2) // 3] cases = { "startTime": { "start_time": lower_bound, "end_time": None, }, "endTime": { "start_time": None, "end_time": upper_bound, }, "startTime + endTime": { "start_time": lower_bound, "end_time": upper_bound, }, } observations: dict[str, Any] = {} for name, values in cases.items(): response, trades = request_trades( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, start_time=values["start_time"], end_time=values["end_time"], limit=limit, ) returned_timestamps = [ timestamp for timestamp in trade_timestamps_ms(trades or []) if timestamp is not None ] start_respected = ( values["start_time"] is None or all( timestamp >= values["start_time"] for timestamp in returned_timestamps ) ) end_respected = ( values["end_time"] is None or all( timestamp <= values["end_time"] for timestamp in returned_timestamps ) ) observations[name] = { "ok": response["ok"], "status_code": response["status_code"], "start_time": values["start_time"], "end_time": values["end_time"], "returned_count": len(trades or []), "first_timestamp_ms": returned_timestamps[0] if returned_timestamps else None, "last_timestamp_ms": returned_timestamps[-1] if returned_timestamps else None, "start_time_respected": start_respected, "end_time_respected": end_respected, "error": response.get("error"), "payload": response.get("payload") if not response["ok"] else None, } valid = [ name for name, observation in observations.items() if observation["ok"] and observation["start_time_respected"] and observation["end_time_respected"] ] if valid: return build_result( "Time filters", STATUS_PASS, "Как минимум один временной фильтр фактически ограничивает timestamps ответа.", details=[f"Подтверждённые варианты: {', '.join(valid)}."], evidence={"observations": observations}, ) return build_result( "Time filters", STATUS_INCONCLUSIVE, "Временные фильтры не подтверждены по содержимому ответа.", evidence={"observations": observations}, ) def test_timestamp_monotonicity( trades: list[dict[str, Any]], ) -> dict[str, Any]: ids = trade_ids(trades) timestamps = trade_timestamps_ms(trades) if any(timestamp is None for timestamp in timestamps): return build_result( "Timestamp monotonicity", STATUS_INCONCLUSIVE, "Не у всех сделок удалось извлечь временную метку.", evidence={ "timestamps_ms": timestamps, "trade_ids": ids, }, ) normalized_timestamps: list[int] = [ timestamp for timestamp in timestamps if timestamp is not None ] order = detect_order(ids) if order == "ASCENDING": consistent = is_non_decreasing(normalized_timestamps) elif order == "DESCENDING": consistent = is_non_increasing(normalized_timestamps) else: return build_result( "Timestamp monotonicity", STATUS_INCONCLUSIVE, "Невозможно сопоставить timestamps с немонотонным порядком trade_id.", evidence={ "order": order, "trade_ids": ids, "timestamps_ms": normalized_timestamps, }, ) if consistent: return build_result( "Timestamp monotonicity", STATUS_PASS, f"Временные метки согласованы с порядком {order} по trade_id.", evidence={ "order": order, "trade_ids": ids, "timestamps_ms": normalized_timestamps, }, ) return build_result( "Timestamp monotonicity", STATUS_FAIL, f"Временные метки не согласованы с порядком {order} по trade_id.", evidence={ "order": order, "trade_ids": ids, "timestamps_ms": normalized_timestamps, }, ) def build_architectural_conclusions( results: list[dict[str, Any]], ) -> list[str]: statuses = { result["name"]: result["status"] for result in results } conclusions: list[str] = [] if statuses.get("Deep historical fromId") == STATUS_PASS: conclusions.append( "Глубокий исторический fromId изменяет выборку и может рассматриваться как кандидат на курсор Recovery." ) elif statuses.get("Deep historical fromId") == STATUS_FAIL: conclusions.append( "Даже глубокий исторический fromId не изменяет latest page; Recovery нельзя проектировать вокруг fromId без дополнительного подтверждения контракта биржи." ) elif statuses.get("fromId support") == STATUS_PASS: conclusions.append( "fromId влияет на историческую выборку и может рассматриваться как кандидат на курсор Recovery." ) else: conclusions.append( "Использование fromId в Recovery пока не подтверждено." ) if statuses.get("Ordering") == STATUS_PASS: conclusions.append( "REST-выдача имеет определённый монотонный порядок, который должен быть нормализован перед передачей в consistency layer." ) conclusions.append( "Recovery не должен предполагать арифметическую непрерывность trade_id; gaps допустимы." ) if statuses.get("Repeatability") == STATUS_PASS: conclusions.append( "Повторный запрос одного диапазона детерминирован по trade_id." ) if statuses.get("Invalid fromId fallback") == STATUS_PASS: conclusions.append( "Некорректный fromId нельзя использовать как сигнал пустого диапазона: сервер возвращает latest page." ) return conclusions def print_summary( results: list[dict[str, Any]], architectural_conclusions: list[str], ) -> None: print_section("Diagnostic summary") counters = { STATUS_PASS: 0, STATUS_FAIL: 0, STATUS_INCONCLUSIVE: 0, STATUS_ERROR: 0, STATUS_SKIPPED: 0, } for result in results: counters[result["status"]] += 1 for status, count in counters.items(): print(f"{status}: {count}") print() print("Architectural conclusions:") for conclusion in architectural_conclusions: print(f" - {conclusion}") def run_diagnostics( *, symbol: str, endpoint_path: str, limit: int, request_delay_seconds: float, verbose: bool, json_output: bool, ) -> dict[str, Any]: settings = load_settings() url = build_rest_url( settings.exchange_base_url, endpoint_path, ) headers = build_headers(settings.exchange_api_key) timeout_seconds = float(settings.exchange_timeout_sec) results: list[dict[str, Any]] = [] if not json_output: print_section("Trade Backfill API Diagnostics") print(f"Generated at: {utc_timestamp()}") print(f"Exchange: {settings.exchange_name}") print(f"REST URL: {url}") print(f"Symbol: {symbol}") print(f"Base limit: {limit}") print(f"Timeout: {timeout_seconds:.1f} seconds") print(f"Request delay: {request_delay_seconds:.3f} seconds") endpoint_result, baseline_trades = test_endpoint_accessibility( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, limit=max(limit, 20), ) results.append(endpoint_result) if not json_output: print_subsection(endpoint_result["name"]) print_result(endpoint_result, verbose=verbose) if not baseline_trades: conclusions = build_architectural_conclusions(results) report = { "generated_at": utc_timestamp(), "exchange": settings.exchange_name, "url": url, "symbol": symbol, "results": results, "architectural_conclusions": conclusions, } if json_output: print(format_json(report)) else: print_summary(results, conclusions) return report baseline_ids = trade_ids(baseline_trades) from_id_result, historical_trades, historical_from_id = test_from_id_support( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, baseline_trades=baseline_trades, limit=limit, ) results.append(from_id_result) if not json_output: print_subsection(from_id_result["name"]) print_result(from_id_result, verbose=verbose) deep_from_id_result, deep_historical_trades, deep_historical_from_id = ( test_deep_historical_from_id( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, baseline_trades=baseline_trades, limit=limit, ) ) results.append(deep_from_id_result) if not json_output: print_subsection(deep_from_id_result["name"]) print_result(deep_from_id_result, verbose=verbose) selected_trades = ( deep_historical_trades if deep_from_id_result["status"] == STATUS_PASS else historical_trades or baseline_trades ) selected_from_id = ( deep_historical_from_id if deep_from_id_result["status"] == STATUS_PASS else historical_from_id ) dependent_tests: list[dict[str, Any]] = [ test_ordering(selected_trades), test_trade_id_spacing(selected_trades), test_timestamp_monotonicity(selected_trades), test_limit_behavior( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, ), test_invalid_from_id_fallback( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, baseline_trades=baseline_trades, limit=limit, ), test_repeatability( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, from_id=( baseline_ids[len(baseline_ids) // 2] if from_id_result["status"] == STATUS_PASS else None ), limit=limit, ), test_time_filters( url, headers, timeout_seconds, request_delay_seconds, symbol=symbol, baseline_trades=baseline_trades, limit=limit, ), ] if selected_from_id is not None and selected_trades: dependent_tests.insert( 0, test_from_id_semantics( requested_from_id=selected_from_id, trades=selected_trades, ), ) else: dependent_tests.insert( 0, build_result( "fromId semantics", STATUS_SKIPPED, "Проверка пропущена: влияние fromId не подтверждено.", ), ) for result in dependent_tests: results.append(result) if not json_output: print_subsection(result["name"]) print_result(result, verbose=verbose) conclusions = build_architectural_conclusions(results) report = { "generated_at": utc_timestamp(), "exchange": settings.exchange_name, "url": url, "symbol": symbol, "results": results, "architectural_conclusions": conclusions, } if json_output: print(format_json(report)) else: print_summary(results, conclusions) return report def main() -> None: arguments = parse_arguments() settings = load_settings() symbol = ( arguments.symbol.strip() if arguments.symbol else settings.default_symbol ) try: run_diagnostics( symbol=symbol, endpoint_path=arguments.endpoint_path, limit=arguments.limit, request_delay_seconds=arguments.delay, verbose=arguments.verbose, json_output=arguments.json, ) except KeyboardInterrupt: print() print("Stopped by user.") except Exception as exc: print() print( "Trade backfill diagnostic failed: " f"{type(exc).__name__}: {exc}" ) raise if __name__ == "__main__": main()