"""
V25 Backtest + Calibration Training Script
==========================================
Runs a full backtest on historical football matches, measures model accuracy
by market / confidence band / league, and trains isotonic calibration models
for MS, OU15, OU25, and BTTS markets.

Usage:
    venv/bin/python scripts/run_backtest_and_calibrate.py
"""

from __future__ import annotations

import os
import sys
import json
import pickle
import time
from collections import defaultdict
from datetime import datetime
from typing import Dict, List, Optional, Tuple, Any

import numpy as np
import pandas as pd
import psycopg2
from psycopg2.extras import RealDictCursor

# ---------------------------------------------------------------------------
# Path setup — works whether executed from ai-engine/ or project root
# ---------------------------------------------------------------------------
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
AI_ENGINE_DIR = os.path.dirname(SCRIPT_DIR)
sys.path.insert(0, AI_ENGINE_DIR)

from data.db import get_clean_dsn
from models.calibration import Calibrator

# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
QUALIFIED_LEAGUES_PATH = os.path.join(AI_ENGINE_DIR, "..", "qualified_leagues.json")
CALIBRATION_DIR = os.path.join(AI_ENGINE_DIR, "models", "calibration")
REPORTS_DIR = os.path.join(AI_ENGINE_DIR, "reports")
MAX_MATCHES = 3000          # target upper bound
PROGRESS_INTERVAL = 100     # print every N matches

os.makedirs(CALIBRATION_DIR, exist_ok=True)
os.makedirs(REPORTS_DIR, exist_ok=True)

# Mapping: Turkish category name -> internal feature key
ODDS_CATEGORY_MAP = {
    "Maç Sonucu": {
        "1": "odds_ms_h",
        "X": "odds_ms_d",
        "2": "odds_ms_a",
    },
    "1,5 Alt/Üst": {
        "Üst": "odds_ou15_o",
        "Alt": "odds_ou15_u",
    },
    "2,5 Alt/Üst": {
        "Üst": "odds_ou25_o",
        "Alt": "odds_ou25_u",
    },
    "3,5 Alt/Üst": {
        "Üst": "odds_ou35_o",
        "Alt": "odds_ou35_u",
    },
    "0,5 Alt/Üst": {
        "Üst": "odds_ou05_o",
        "Alt": "odds_ou05_u",
    },
    "Karşılıklı Gol": {
        "Var": "odds_btts_y",
        "Yok": "odds_btts_n",
    },
    "1. Yarı Sonucu": {
        "1": "odds_ht_ms_h",
        "X": "odds_ht_ms_d",
        "2": "odds_ht_ms_a",
    },
    "1. Yarı 0,5 Alt/Üst": {
        "Üst": "odds_ht_ou05_o",
        "Alt": "odds_ht_ou05_u",
    },
    "1. Yarı 1,5 Alt/Üst": {
        "Üst": "odds_ht_ou15_o",
        "Alt": "odds_ht_ou15_u",
    },
}

# Top 5 leagues by name for individual breakdown (will be matched by league_id)
TOP5_LEAGUE_NAMES = {
    "Premier League",
    "La Liga",
    "Bundesliga",
    "Serie A",
    "Ligue 1",
}

# ============================================================================
# STEP 1 — Load qualified league IDs
# ============================================================================

def load_qualified_leagues() -> List[str]:
    path = os.path.abspath(QUALIFIED_LEAGUES_PATH)
    with open(path, "r") as f:
        leagues = json.load(f)
    print(f"[Step 1] Loaded {len(leagues)} qualified league IDs.")
    return [str(lid) for lid in leagues]


# ============================================================================
# STEP 1b — Fetch matches + pre-computed features in batch
# ============================================================================

def fetch_matches(conn, league_ids: List[str]) -> pd.DataFrame:
    """
    Single batch query: matches + football_ai_features + league name.
    Only returns matches that also have odds data (inner join on odd_categories).
    Returns a DataFrame with one row per match.
    """
    print("[Step 1b] Fetching matches with pre-computed features and odds ...")
    cur = conn.cursor(cursor_factory=RealDictCursor)

    cur.execute(
        """
        SELECT
            m.id                AS match_id,
            m.league_id,
            l.name              AS league_name,
            m.score_home,
            m.score_away,
            m.mst_utc,
            -- From football_ai_features
            f.home_elo          AS home_overall_elo,
            f.away_elo          AS away_overall_elo,
            f.elo_diff,
            f.home_home_elo,
            f.away_away_elo,
            f.home_form_elo,
            f.away_form_elo,
            f.home_goals_avg_5  AS home_goals_avg,
            f.away_goals_avg_5  AS away_goals_avg,
            f.home_conceded_avg_5 AS home_conceded_avg,
            f.away_conceded_avg_5 AS away_conceded_avg,
            f.home_clean_sheet_rate,
            f.away_clean_sheet_rate,
            f.home_scoring_rate,
            f.away_scoring_rate,
            f.home_win_streak   AS home_winning_streak,
            f.away_win_streak   AS away_winning_streak,
            f.home_avg_possession,
            f.away_avg_possession,
            f.home_avg_shots_on_target,
            f.away_avg_shots_on_target,
            f.home_shot_conversion,
            f.away_shot_conversion,
            f.home_avg_corners,
            f.away_avg_corners,
            f.h2h_total         AS h2h_total_matches,
            f.h2h_home_win_rate,
            f.h2h_avg_goals,
            f.h2h_over25_rate,
            f.h2h_btts_rate,
            f.league_avg_goals,
            f.league_home_win_pct AS league_home_win_rate,
            f.league_over25_pct   AS league_ou25_rate,
            f.referee_avg_cards   AS referee_cards_total,
            f.referee_home_bias,
            f.referee_avg_goals,
            f.missing_players_impact AS home_missing_impact,
            f.implied_home,
            f.implied_draw,
            f.implied_away
        FROM matches m
        JOIN football_ai_features f ON f.match_id = m.id
        -- Only matches that have odds data
        JOIN (SELECT DISTINCT match_id FROM odd_categories WHERE sport = 'football') oc
            ON oc.match_id = m.id
        LEFT JOIN leagues l ON l.id = m.league_id
        WHERE m.status = 'FT'
          AND m.score_home IS NOT NULL
          AND m.score_away IS NOT NULL
          AND m.league_id = ANY(%s)
        ORDER BY m.mst_utc DESC
        LIMIT %s
        """,
        (league_ids, MAX_MATCHES),
    )

    rows = cur.fetchall()
    cur.close()
    df = pd.DataFrame([dict(r) for r in rows])
    print(f"[Step 1b] Fetched {len(df)} matches with features + odds coverage.")
    return df


# ============================================================================
# STEP 1c — Fetch all odds for the matched match IDs in one query
# ============================================================================

def fetch_odds_bulk(conn, match_ids: List[str]) -> Dict[str, Dict[str, float]]:
    """
    Returns {match_id: {feature_key: odd_value, ...}} for all known categories.
    """
    print(f"[Step 1c] Fetching odds for {len(match_ids)} matches ...")
    cur = conn.cursor(cursor_factory=RealDictCursor)

    # Build a set of known category names
    known_cats = tuple(ODDS_CATEGORY_MAP.keys())

    cur.execute(
        """
        SELECT oc.match_id, oc.name AS cat_name, os.name AS sel_name, os.odd_value
        FROM odd_categories oc
        JOIN odd_selections os ON os.odd_category_db_id = oc.db_id
        WHERE oc.match_id = ANY(%s)
          AND oc.name = ANY(%s)
          AND oc.sport = 'football'
          AND os.odd_value IS NOT NULL
          AND os.odd_value ~ '^[0-9]+(\.[0-9]+)?$'
        """,
        (match_ids, list(known_cats)),
    )

    rows = cur.fetchall()
    cur.close()

    # Build nested dict: match_id -> {feature_key -> value}
    odds_map: Dict[str, Dict[str, float]] = defaultdict(dict)
    for r in rows:
        cat_name = r["cat_name"]
        sel_name = r["sel_name"]
        if cat_name in ODDS_CATEGORY_MAP and sel_name in ODDS_CATEGORY_MAP[cat_name]:
            feat_key = ODDS_CATEGORY_MAP[cat_name][sel_name]
            try:
                val = float(r["odd_value"])
                if val > 1.0:
                    # Keep first encountered (most recent or primary bookmaker)
                    if feat_key not in odds_map[r["match_id"]]:
                        odds_map[r["match_id"]][feat_key] = val
            except (TypeError, ValueError):
                pass

    print(f"[Step 1c] Odds loaded for {len(odds_map)} matches.")
    return dict(odds_map)


# ============================================================================
# STEP 2 — Build 114-feature vector per match
# ============================================================================

def load_feature_cols() -> List[str]:
    path = os.path.join(AI_ENGINE_DIR, "models", "v25", "feature_cols.json")
    with open(path, "r") as f:
        return json.load(f)


def build_feature_vector(
    match_row: pd.Series,
    odds: Dict[str, float],
    feature_cols: List[str],
) -> Dict[str, float]:
    """
    Construct the full feature dict for one match.
    Falls back to 0.0 for any missing feature.
    """
    feat: Dict[str, float] = {col: 0.0 for col in feature_cols}

    # ---- Direct columns from match row ----
    direct_map = {
        "home_overall_elo": "home_overall_elo",
        "away_overall_elo": "away_overall_elo",
        "elo_diff": "elo_diff",
        "home_home_elo": "home_home_elo",
        "away_away_elo": "away_away_elo",
        "home_form_elo": "home_form_elo",
        "away_form_elo": "away_form_elo",
        "home_goals_avg": "home_goals_avg",
        "away_goals_avg": "away_goals_avg",
        "home_conceded_avg": "home_conceded_avg",
        "away_conceded_avg": "away_conceded_avg",
        "home_clean_sheet_rate": "home_clean_sheet_rate",
        "away_clean_sheet_rate": "away_clean_sheet_rate",
        "home_scoring_rate": "home_scoring_rate",
        "away_scoring_rate": "away_scoring_rate",
        "home_winning_streak": "home_winning_streak",
        "away_winning_streak": "away_winning_streak",
        "home_avg_possession": "home_avg_possession",
        "away_avg_possession": "away_avg_possession",
        "home_avg_shots_on_target": "home_avg_shots_on_target",
        "away_avg_shots_on_target": "away_avg_shots_on_target",
        "home_shot_conversion": "home_shot_conversion",
        "away_shot_conversion": "away_shot_conversion",
        "home_avg_corners": "home_avg_corners",
        "away_avg_corners": "away_avg_corners",
        "h2h_total_matches": "h2h_total_matches",
        "h2h_home_win_rate": "h2h_home_win_rate",
        "h2h_avg_goals": "h2h_avg_goals",
        "h2h_over25_rate": "h2h_over25_rate",
        "h2h_btts_rate": "h2h_btts_rate",
        "league_avg_goals": "league_avg_goals",
        "league_home_win_rate": "league_home_win_rate",
        "league_ou25_rate": "league_ou25_rate",
        "referee_cards_total": "referee_cards_total",
        "referee_home_bias": "referee_home_bias",
        "referee_avg_goals": "referee_avg_goals",
        "home_missing_impact": "home_missing_impact",
        "implied_home": "implied_home",
        "implied_draw": "implied_draw",
        "implied_away": "implied_away",
    }

    for src_col, feat_col in direct_map.items():
        if feat_col in feat and src_col in match_row.index:
            val = match_row.get(src_col)
            if val is not None and not (isinstance(val, float) and np.isnan(val)):
                feat[feat_col] = float(val)

    # ---- Derived elo features ----
    if feat.get("home_form_elo", 0) and feat.get("away_form_elo", 0):
        feat["form_elo_diff"] = feat["home_form_elo"] - feat["away_form_elo"]

    # ---- Odds features from relational tables ----
    odds_features = [
        "odds_ms_h", "odds_ms_d", "odds_ms_a",
        "odds_ht_ms_h", "odds_ht_ms_d", "odds_ht_ms_a",
        "odds_ou05_o", "odds_ou05_u",
        "odds_ou15_o", "odds_ou15_u",
        "odds_ou25_o", "odds_ou25_u",
        "odds_ou35_o", "odds_ou35_u",
        "odds_ht_ou05_o", "odds_ht_ou05_u",
        "odds_ht_ou15_o", "odds_ht_ou15_u",
        "odds_btts_y", "odds_btts_n",
    ]
    for ok in odds_features:
        if ok in odds:
            feat[ok] = odds[ok]
            presence_key = f"{ok}_present"
            if presence_key in feat:
                feat[presence_key] = 1.0

    # Recompute implied probabilities from odds if available and not already set
    if feat.get("odds_ms_h", 0) > 1 and feat.get("odds_ms_d", 0) > 1 and feat.get("odds_ms_a", 0) > 1:
        raw_h = 1.0 / feat["odds_ms_h"]
        raw_d = 1.0 / feat["odds_ms_d"]
        raw_a = 1.0 / feat["odds_ms_a"]
        total = raw_h + raw_d + raw_a
        if total > 0:
            feat["implied_home"] = raw_h / total
            feat["implied_draw"] = raw_d / total
            feat["implied_away"] = raw_a / total

    # ---- Derived match metadata ----
    mst = match_row.get("mst_utc")
    if mst is not None:
        try:
            ts_s = int(mst) / 1000  # stored as epoch ms
            dt = datetime.utcfromtimestamp(ts_s)
            if "match_month" in feat:
                feat["match_month"] = float(dt.month)
            # Season markers: Sept-Oct = start, April-May = end
            if "is_season_start" in feat:
                feat["is_season_start"] = 1.0 if dt.month in (8, 9, 10) else 0.0
            if "is_season_end" in feat:
                feat["is_season_end"] = 1.0 if dt.month in (4, 5) else 0.0
        except Exception:
            pass

    # ---- Interaction features ----
    if "attack_vs_defense_home" in feat:
        feat["attack_vs_defense_home"] = feat.get("home_goals_avg", 0) - feat.get("away_conceded_avg", 0)
    if "attack_vs_defense_away" in feat:
        feat["attack_vs_defense_away"] = feat.get("away_goals_avg", 0) - feat.get("home_conceded_avg", 0)
    if "form_momentum_interaction" in feat:
        feat["form_momentum_interaction"] = (
            feat.get("home_momentum_score", 0) * feat.get("home_goals_avg", 0)
            - feat.get("away_momentum_score", 0) * feat.get("away_goals_avg", 0)
        )
    if "elo_form_consistency" in feat:
        feat["elo_form_consistency"] = feat.get("elo_diff", 0) * feat.get("home_goals_avg", 0)

    return feat


# ============================================================================
# STEP 3 — Run V25 predictions
# ============================================================================

def load_predictor():
    from models.v25_ensemble import get_v25_predictor
    print("[Step 3] Loading V25 predictor ...")
    pred = get_v25_predictor()
    print("[Step 3] V25 predictor ready.")
    return pred


# ============================================================================
# STEP 4 — Compute actual outcomes from scores
# ============================================================================

def compute_actuals(score_home: int, score_away: int) -> Dict[str, Any]:
    total = score_home + score_away
    return {
        "ms_actual": "1" if score_home > score_away else ("X" if score_home == score_away else "2"),
        "ou15_actual": "Over" if total >= 2 else "Under",
        "ou25_actual": "Over" if total >= 3 else "Under",
        "btts_actual": "Yes" if score_home > 0 and score_away > 0 else "No",
    }


# ============================================================================
# STEP 5 — Accuracy helpers
# ============================================================================

def confidence_band(prob: float) -> str:
    if prob < 0.50:
        return "<50%"
    elif prob < 0.65:
        return "50-65%"
    elif prob < 0.75:
        return "65-75%"
    else:
        return "75%+"


def pick_from_ms(home_prob: float, draw_prob: float, away_prob: float) -> Tuple[str, float]:
    picks = {"1": home_prob, "X": draw_prob, "2": away_prob}
    best = max(picks, key=picks.__getitem__)
    return best, picks[best]


def pick_from_binary(yes_prob: float, no_prob: float, yes_label: str, no_label: str) -> Tuple[str, float]:
    if yes_prob >= no_prob:
        return yes_label, yes_prob
    return no_label, no_prob


# ============================================================================
# MAIN
# ============================================================================

def main():
    t_start = time.time()
    print("=" * 70)
    print("  V25 Backtest + Calibration Training")
    print(f"  Run at: {datetime.utcnow().isoformat()} UTC")
    print("=" * 70)

    # ------------------------------------------------------------------
    # Step 1 — Load qualified leagues
    # ------------------------------------------------------------------
    league_ids = load_qualified_leagues()

    # ------------------------------------------------------------------
    # Step 1b — Fetch matches with features
    # ------------------------------------------------------------------
    conn = psycopg2.connect(get_clean_dsn())
    try:
        matches_df = fetch_matches(conn, league_ids)

        if matches_df.empty:
            print("[ERROR] No matches found. Check DB connection and league IDs.")
            return

        match_ids = matches_df["match_id"].tolist()

        # ------------------------------------------------------------------
        # Step 1c — Fetch odds in bulk
        # ------------------------------------------------------------------
        odds_map = fetch_odds_bulk(conn, match_ids)
    finally:
        conn.close()

    # ------------------------------------------------------------------
    # Step 2 — Build feature vectors
    # ------------------------------------------------------------------
    print(f"\n[Step 2] Building feature vectors for {len(matches_df)} matches ...")
    feature_cols = load_feature_cols()

    # ------------------------------------------------------------------
    # Step 3 — Load V25 predictor
    # ------------------------------------------------------------------
    predictor = load_predictor()

    # ------------------------------------------------------------------
    # Main loop — predict each match, collect results
    # ------------------------------------------------------------------
    print(f"\n[Loop] Running predictions ...")

    # Storage for calibration training
    calib_data: Dict[str, List[Tuple[float, int]]] = {
        "ms_home": [],   # (prob, 1 if home win)
        "ms_draw": [],
        "ms_away": [],
        "ou15": [],
        "ou25": [],
        "btts": [],
    }

    # Storage for accuracy reporting
    records = []

    skipped = 0
    processed = 0

    for idx, row in matches_df.iterrows():
        match_id = row["match_id"]
        score_home = row.get("score_home")
        score_away = row.get("score_away")

        # Validate scores
        try:
            score_home = int(score_home)
            score_away = int(score_away)
        except (TypeError, ValueError):
            skipped += 1
            continue

        # Build features
        match_odds = odds_map.get(match_id, {})
        feat = build_feature_vector(row, match_odds, feature_cols)

        # Run predictions
        try:
            home_prob, draw_prob, away_prob = predictor.predict_ms(feat)
            over25_prob, under25_prob = predictor.predict_ou25(feat)
            btts_yes_prob, btts_no_prob = predictor.predict_btts(feat)

            # ou15 is loaded via predict_market (returns np.ndarray for binary)
            ou15_arr = predictor.predict_market("ou15", feat)
            if ou15_arr is not None and len(ou15_arr) > 0:
                over15_prob = float(ou15_arr[0])
                under15_prob = 1.0 - over15_prob
            else:
                over15_prob = 0.5
                under15_prob = 0.5

        except Exception as e:
            skipped += 1
            continue

        # Compute actuals
        actuals = compute_actuals(score_home, score_away)

        # MS picks
        ms_pick, ms_conf = pick_from_ms(home_prob, draw_prob, away_prob)
        ms_correct = int(ms_pick == actuals["ms_actual"])

        # OU15
        ou15_pick, ou15_conf = pick_from_binary(over15_prob, under15_prob, "Over", "Under")
        ou15_correct = int(ou15_pick == actuals["ou15_actual"])

        # OU25
        ou25_pick, ou25_conf = pick_from_binary(over25_prob, under25_prob, "Over", "Under")
        ou25_correct = int(ou25_pick == actuals["ou25_actual"])

        # BTTS
        btts_pick, btts_conf = pick_from_binary(btts_yes_prob, btts_no_prob, "Yes", "No")
        btts_correct = int(btts_pick == actuals["btts_actual"])

        # Collect calibration data
        calib_data["ms_home"].append((home_prob, int(actuals["ms_actual"] == "1")))
        calib_data["ms_draw"].append((draw_prob, int(actuals["ms_actual"] == "X")))
        calib_data["ms_away"].append((away_prob, int(actuals["ms_actual"] == "2")))
        calib_data["ou15"].append((over15_prob, int(actuals["ou15_actual"] == "Over")))
        calib_data["ou25"].append((over25_prob, int(actuals["ou25_actual"] == "Over")))
        calib_data["btts"].append((btts_yes_prob, int(actuals["btts_actual"] == "Yes")))

        # Determine league group
        league_name = str(row.get("league_name", "Other") or "Other")
        league_group = league_name if league_name in TOP5_LEAGUE_NAMES else "Other"

        records.append({
            "match_id": match_id,
            "league_name": league_name,
            "league_group": league_group,
            "score_home": score_home,
            "score_away": score_away,
            # MS
            "ms_pick": ms_pick,
            "ms_actual": actuals["ms_actual"],
            "ms_conf": ms_conf,
            "ms_conf_band": confidence_band(ms_conf),
            "ms_correct": ms_correct,
            "ms_home_prob": home_prob,
            "ms_draw_prob": draw_prob,
            "ms_away_prob": away_prob,
            # OU15
            "ou15_pick": ou15_pick,
            "ou15_actual": actuals["ou15_actual"],
            "ou15_conf": ou15_conf,
            "ou15_conf_band": confidence_band(ou15_conf),
            "ou15_correct": ou15_correct,
            "ou15_over_prob": over15_prob,
            # OU25
            "ou25_pick": ou25_pick,
            "ou25_actual": actuals["ou25_actual"],
            "ou25_conf": ou25_conf,
            "ou25_conf_band": confidence_band(ou25_conf),
            "ou25_correct": ou25_correct,
            "ou25_over_prob": over25_prob,
            # BTTS
            "btts_pick": btts_pick,
            "btts_actual": actuals["btts_actual"],
            "btts_conf": btts_conf,
            "btts_conf_band": confidence_band(btts_conf),
            "btts_correct": btts_correct,
            "btts_yes_prob": btts_yes_prob,
        })

        processed += 1
        if processed % PROGRESS_INTERVAL == 0:
            elapsed = time.time() - t_start
            print(f"  [Progress] {processed}/{len(matches_df)} matches | "
                  f"skipped={skipped} | elapsed={elapsed:.1f}s")

    print(f"\n[Loop] Done. Processed={processed}, Skipped={skipped}")

    if not records:
        print("[ERROR] No records to analyze. Exiting.")
        return

    results_df = pd.DataFrame(records)

    # ------------------------------------------------------------------
    # Step 5 — Accuracy report
    # ------------------------------------------------------------------
    print("\n" + "=" * 70)
    print("  ACCURACY REPORT")
    print("=" * 70)

    markets = [
        ("MS",   "ms_correct",   "ms_conf",   "ms_conf_band",   "ms_pick"),
        ("OU15", "ou15_correct", "ou15_conf", "ou15_conf_band", "ou15_pick"),
        ("OU25", "ou25_correct", "ou25_conf", "ou25_conf_band", "ou25_pick"),
        ("BTTS", "btts_correct", "btts_conf", "btts_conf_band", "btts_pick"),
    ]

    summary: Dict[str, Any] = {
        "generated_at": datetime.utcnow().isoformat(),
        "matches_processed": processed,
        "matches_skipped": skipped,
        "markets": {},
    }

    for market_label, correct_col, conf_col, band_col, pick_col in markets:
        print(f"\n--- {market_label} ---")
        sub = results_df[[correct_col, conf_col, band_col, pick_col, "league_group"]].copy()
        total = len(sub)
        overall_acc = sub[correct_col].mean() * 100
        print(f"  Overall accuracy: {overall_acc:.1f}% ({sub[correct_col].sum()}/{total})")

        market_summary = {
            "overall_accuracy": round(overall_acc, 2),
            "total_matches": total,
            "by_confidence_band": {},
            "by_league": {},
            "by_pick_direction": {},
        }

        # By confidence band
        print(f"  By confidence band:")
        bands = ["<50%", "50-65%", "65-75%", "75%+"]
        for band in bands:
            mask = sub[band_col] == band
            n = mask.sum()
            if n > 0:
                acc = sub.loc[mask, correct_col].mean() * 100
                mean_conf = sub.loc[mask, conf_col].mean() * 100
                print(f"    {band:8s}: {acc:5.1f}% acc | {n:4d} matches | "
                      f"mean_conf={mean_conf:.1f}%")
                market_summary["by_confidence_band"][band] = {
                    "accuracy": round(acc, 2),
                    "count": int(n),
                    "mean_confidence": round(mean_conf, 2),
                }

        # By league group
        print(f"  By league:")
        league_groups = list(results_df["league_group"].unique())
        # Sort: named leagues first, then Other
        named = sorted([g for g in league_groups if g != "Other"])
        ordered = named + (["Other"] if "Other" in league_groups else [])
        for lg in ordered:
            mask = sub["league_group"] == lg
            n = mask.sum()
            if n > 0:
                acc = sub.loc[mask, correct_col].mean() * 100
                print(f"    {lg[:20]:20s}: {acc:5.1f}% ({n} matches)")
                market_summary["by_league"][lg] = {
                    "accuracy": round(acc, 2),
                    "count": int(n),
                }

        # By pick direction
        print(f"  By pick direction:")
        for pick_val in sorted(sub[pick_col].unique()):
            mask = sub[pick_col] == pick_val
            n = mask.sum()
            if n > 0:
                acc = sub.loc[mask, correct_col].mean() * 100
                mean_conf = sub.loc[mask, conf_col].mean() * 100
                print(f"    {pick_val:8s}: {acc:5.1f}% acc | {n:4d} matches | "
                      f"mean_conf={mean_conf:.1f}%")
                market_summary["by_pick_direction"][pick_val] = {
                    "accuracy": round(acc, 2),
                    "count": int(n),
                    "mean_confidence": round(mean_conf, 2),
                }

        summary["markets"][market_label] = market_summary

    # ------------------------------------------------------------------
    # Step 6 — Train calibration models
    # ------------------------------------------------------------------
    print("\n" + "=" * 70)
    print("  CALIBRATION TRAINING")
    print("=" * 70)

    calibrator = Calibrator()

    # Market config: market_key -> (label for prob, label for actual binary)
    calib_market_map = {
        "ms_home": "ms_home",
        "ms_draw": "ms_draw",
        "ms_away": "ms_away",
        "ou15": "ou15",
        "ou25": "ou25",
        "btts": "btts",
    }

    calibration_results: Dict[str, Dict] = {}

    for market_key in calib_market_map:
        pairs = calib_data[market_key]
        if len(pairs) < 100:
            print(f"[Calib] {market_key}: only {len(pairs)} samples — skipping.")
            continue

        probs = np.array([p for p, _ in pairs])
        actuals_bin = np.array([a for _, a in pairs])

        # Build a tiny DataFrame to use Calibrator.train_calibration
        calib_df = pd.DataFrame({
            "prob": probs,
            "actual": actuals_bin,
        })

        metrics = calibrator.train_calibration(
            df=calib_df,
            market=market_key,
            prob_col="prob",
            actual_col="actual",
            min_samples=100,
            save=True,
        )
        calibration_results[market_key] = metrics.to_dict()
        print(f"  [Calib] {market_key}: Brier={metrics.brier_score:.4f} | "
              f"ECE={metrics.calibration_error:.4f} | n={metrics.sample_count}")

    # ------------------------------------------------------------------
    # Step 7 — Save results
    # ------------------------------------------------------------------
    output_path = os.path.join(REPORTS_DIR, "backtest_results.json")
    full_report = {
        **summary,
        "calibration": calibration_results,
        "runtime_seconds": round(time.time() - t_start, 1),
    }

    with open(output_path, "w") as f:
        json.dump(full_report, f, indent=2)
    print(f"\n[Step 7] Report saved to {output_path}")

    # ------------------------------------------------------------------
    # Final summary table
    # ------------------------------------------------------------------
    print("\n" + "=" * 70)
    print("  FINAL SUMMARY TABLE")
    print("=" * 70)
    print(f"{'Market':<8} {'Overall Acc':>12} {'Matches':>8} "
          f"{'Best Band (acc)':>18}")
    print("-" * 70)
    for market_label, _, _, _, _ in markets:
        ms = summary["markets"].get(market_label, {})
        overall = ms.get("overall_accuracy", 0)
        total_m = ms.get("total_matches", 0)
        bands_d = ms.get("by_confidence_band", {})
        # Find best accuracy band with >= 50 matches
        best_band = "-"
        best_acc = 0.0
        for band, bdata in bands_d.items():
            if bdata["count"] >= 50 and bdata["accuracy"] > best_acc:
                best_acc = bdata["accuracy"]
                best_band = f"{band} ({best_acc:.1f}%)"
        print(f"{market_label:<8} {overall:>11.1f}% {total_m:>8d} {best_band:>18s}")

    elapsed_total = time.time() - t_start
    print(f"\nTotal runtime: {elapsed_total:.1f}s")
    print("=" * 70)


if __name__ == "__main__":
    main()
