""" V25 Backtest + Calibration Training Script ========================================== Runs a full backtest on historical football matches, measures model accuracy by market / confidence band / league, and trains isotonic calibration models for MS, OU15, OU25, and BTTS markets. Usage: venv/bin/python scripts/run_backtest_and_calibrate.py """ from __future__ import annotations import os import sys import json import pickle import time from collections import defaultdict from datetime import datetime from typing import Dict, List, Optional, Tuple, Any import numpy as np import pandas as pd import psycopg2 from psycopg2.extras import RealDictCursor # --------------------------------------------------------------------------- # Path setup — works whether executed from ai-engine/ or project root # --------------------------------------------------------------------------- SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) AI_ENGINE_DIR = os.path.dirname(SCRIPT_DIR) sys.path.insert(0, AI_ENGINE_DIR) from data.db import get_clean_dsn from models.calibration import Calibrator # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- QUALIFIED_LEAGUES_PATH = os.path.join(AI_ENGINE_DIR, "..", "qualified_leagues.json") CALIBRATION_DIR = os.path.join(AI_ENGINE_DIR, "models", "calibration") REPORTS_DIR = os.path.join(AI_ENGINE_DIR, "reports") MAX_MATCHES = 3000 # target upper bound PROGRESS_INTERVAL = 100 # print every N matches os.makedirs(CALIBRATION_DIR, exist_ok=True) os.makedirs(REPORTS_DIR, exist_ok=True) # Mapping: Turkish category name -> internal feature key ODDS_CATEGORY_MAP = { "Maç Sonucu": { "1": "odds_ms_h", "X": "odds_ms_d", "2": "odds_ms_a", }, "1,5 Alt/Üst": { "Üst": "odds_ou15_o", "Alt": "odds_ou15_u", }, "2,5 Alt/Üst": { "Üst": "odds_ou25_o", "Alt": "odds_ou25_u", }, "3,5 Alt/Üst": { "Üst": "odds_ou35_o", "Alt": "odds_ou35_u", }, "0,5 Alt/Üst": { "Üst": "odds_ou05_o", "Alt": "odds_ou05_u", }, "Karşılıklı Gol": { "Var": "odds_btts_y", "Yok": "odds_btts_n", }, "1. Yarı Sonucu": { "1": "odds_ht_ms_h", "X": "odds_ht_ms_d", "2": "odds_ht_ms_a", }, "1. Yarı 0,5 Alt/Üst": { "Üst": "odds_ht_ou05_o", "Alt": "odds_ht_ou05_u", }, "1. Yarı 1,5 Alt/Üst": { "Üst": "odds_ht_ou15_o", "Alt": "odds_ht_ou15_u", }, } # Top 5 leagues by name for individual breakdown (will be matched by league_id) TOP5_LEAGUE_NAMES = { "Premier League", "La Liga", "Bundesliga", "Serie A", "Ligue 1", } # ============================================================================ # STEP 1 — Load qualified league IDs # ============================================================================ def load_qualified_leagues() -> List[str]: path = os.path.abspath(QUALIFIED_LEAGUES_PATH) with open(path, "r") as f: leagues = json.load(f) print(f"[Step 1] Loaded {len(leagues)} qualified league IDs.") return [str(lid) for lid in leagues] # ============================================================================ # STEP 1b — Fetch matches + pre-computed features in batch # ============================================================================ def fetch_matches(conn, league_ids: List[str]) -> pd.DataFrame: """ Single batch query: matches + football_ai_features + league name. Only returns matches that also have odds data (inner join on odd_categories). Returns a DataFrame with one row per match. """ print("[Step 1b] Fetching matches with pre-computed features and odds ...") cur = conn.cursor(cursor_factory=RealDictCursor) cur.execute( """ SELECT m.id AS match_id, m.league_id, l.name AS league_name, m.score_home, m.score_away, m.mst_utc, -- From football_ai_features f.home_elo AS home_overall_elo, f.away_elo AS away_overall_elo, f.elo_diff, f.home_home_elo, f.away_away_elo, f.home_form_elo, f.away_form_elo, f.home_goals_avg_5 AS home_goals_avg, f.away_goals_avg_5 AS away_goals_avg, f.home_conceded_avg_5 AS home_conceded_avg, f.away_conceded_avg_5 AS away_conceded_avg, f.home_clean_sheet_rate, f.away_clean_sheet_rate, f.home_scoring_rate, f.away_scoring_rate, f.home_win_streak AS home_winning_streak, f.away_win_streak AS away_winning_streak, f.home_avg_possession, f.away_avg_possession, f.home_avg_shots_on_target, f.away_avg_shots_on_target, f.home_shot_conversion, f.away_shot_conversion, f.home_avg_corners, f.away_avg_corners, f.h2h_total AS h2h_total_matches, f.h2h_home_win_rate, f.h2h_avg_goals, f.h2h_over25_rate, f.h2h_btts_rate, f.league_avg_goals, f.league_home_win_pct AS league_home_win_rate, f.league_over25_pct AS league_ou25_rate, f.referee_avg_cards AS referee_cards_total, f.referee_home_bias, f.referee_avg_goals, f.missing_players_impact AS home_missing_impact, f.implied_home, f.implied_draw, f.implied_away FROM matches m JOIN football_ai_features f ON f.match_id = m.id -- Only matches that have odds data JOIN (SELECT DISTINCT match_id FROM odd_categories WHERE sport = 'football') oc ON oc.match_id = m.id LEFT JOIN leagues l ON l.id = m.league_id WHERE m.status = 'FT' AND m.score_home IS NOT NULL AND m.score_away IS NOT NULL AND m.league_id = ANY(%s) ORDER BY m.mst_utc DESC LIMIT %s """, (league_ids, MAX_MATCHES), ) rows = cur.fetchall() cur.close() df = pd.DataFrame([dict(r) for r in rows]) print(f"[Step 1b] Fetched {len(df)} matches with features + odds coverage.") return df # ============================================================================ # STEP 1c — Fetch all odds for the matched match IDs in one query # ============================================================================ def fetch_odds_bulk(conn, match_ids: List[str]) -> Dict[str, Dict[str, float]]: """ Returns {match_id: {feature_key: odd_value, ...}} for all known categories. """ print(f"[Step 1c] Fetching odds for {len(match_ids)} matches ...") cur = conn.cursor(cursor_factory=RealDictCursor) # Build a set of known category names known_cats = tuple(ODDS_CATEGORY_MAP.keys()) cur.execute( """ SELECT oc.match_id, oc.name AS cat_name, os.name AS sel_name, os.odd_value FROM odd_categories oc JOIN odd_selections os ON os.odd_category_db_id = oc.db_id WHERE oc.match_id = ANY(%s) AND oc.name = ANY(%s) AND oc.sport = 'football' AND os.odd_value IS NOT NULL AND os.odd_value ~ '^[0-9]+(\.[0-9]+)?$' """, (match_ids, list(known_cats)), ) rows = cur.fetchall() cur.close() # Build nested dict: match_id -> {feature_key -> value} odds_map: Dict[str, Dict[str, float]] = defaultdict(dict) for r in rows: cat_name = r["cat_name"] sel_name = r["sel_name"] if cat_name in ODDS_CATEGORY_MAP and sel_name in ODDS_CATEGORY_MAP[cat_name]: feat_key = ODDS_CATEGORY_MAP[cat_name][sel_name] try: val = float(r["odd_value"]) if val > 1.0: # Keep first encountered (most recent or primary bookmaker) if feat_key not in odds_map[r["match_id"]]: odds_map[r["match_id"]][feat_key] = val except (TypeError, ValueError): pass print(f"[Step 1c] Odds loaded for {len(odds_map)} matches.") return dict(odds_map) # ============================================================================ # STEP 2 — Build 114-feature vector per match # ============================================================================ def load_feature_cols() -> List[str]: path = os.path.join(AI_ENGINE_DIR, "models", "v25", "feature_cols.json") with open(path, "r") as f: return json.load(f) def build_feature_vector( match_row: pd.Series, odds: Dict[str, float], feature_cols: List[str], ) -> Dict[str, float]: """ Construct the full feature dict for one match. Falls back to 0.0 for any missing feature. """ feat: Dict[str, float] = {col: 0.0 for col in feature_cols} # ---- Direct columns from match row ---- direct_map = { "home_overall_elo": "home_overall_elo", "away_overall_elo": "away_overall_elo", "elo_diff": "elo_diff", "home_home_elo": "home_home_elo", "away_away_elo": "away_away_elo", "home_form_elo": "home_form_elo", "away_form_elo": "away_form_elo", "home_goals_avg": "home_goals_avg", "away_goals_avg": "away_goals_avg", "home_conceded_avg": "home_conceded_avg", "away_conceded_avg": "away_conceded_avg", "home_clean_sheet_rate": "home_clean_sheet_rate", "away_clean_sheet_rate": "away_clean_sheet_rate", "home_scoring_rate": "home_scoring_rate", "away_scoring_rate": "away_scoring_rate", "home_winning_streak": "home_winning_streak", "away_winning_streak": "away_winning_streak", "home_avg_possession": "home_avg_possession", "away_avg_possession": "away_avg_possession", "home_avg_shots_on_target": "home_avg_shots_on_target", "away_avg_shots_on_target": "away_avg_shots_on_target", "home_shot_conversion": "home_shot_conversion", "away_shot_conversion": "away_shot_conversion", "home_avg_corners": "home_avg_corners", "away_avg_corners": "away_avg_corners", "h2h_total_matches": "h2h_total_matches", "h2h_home_win_rate": "h2h_home_win_rate", "h2h_avg_goals": "h2h_avg_goals", "h2h_over25_rate": "h2h_over25_rate", "h2h_btts_rate": "h2h_btts_rate", "league_avg_goals": "league_avg_goals", "league_home_win_rate": "league_home_win_rate", "league_ou25_rate": "league_ou25_rate", "referee_cards_total": "referee_cards_total", "referee_home_bias": "referee_home_bias", "referee_avg_goals": "referee_avg_goals", "home_missing_impact": "home_missing_impact", "implied_home": "implied_home", "implied_draw": "implied_draw", "implied_away": "implied_away", } for src_col, feat_col in direct_map.items(): if feat_col in feat and src_col in match_row.index: val = match_row.get(src_col) if val is not None and not (isinstance(val, float) and np.isnan(val)): feat[feat_col] = float(val) # ---- Derived elo features ---- if feat.get("home_form_elo", 0) and feat.get("away_form_elo", 0): feat["form_elo_diff"] = feat["home_form_elo"] - feat["away_form_elo"] # ---- Odds features from relational tables ---- odds_features = [ "odds_ms_h", "odds_ms_d", "odds_ms_a", "odds_ht_ms_h", "odds_ht_ms_d", "odds_ht_ms_a", "odds_ou05_o", "odds_ou05_u", "odds_ou15_o", "odds_ou15_u", "odds_ou25_o", "odds_ou25_u", "odds_ou35_o", "odds_ou35_u", "odds_ht_ou05_o", "odds_ht_ou05_u", "odds_ht_ou15_o", "odds_ht_ou15_u", "odds_btts_y", "odds_btts_n", ] for ok in odds_features: if ok in odds: feat[ok] = odds[ok] presence_key = f"{ok}_present" if presence_key in feat: feat[presence_key] = 1.0 # Recompute implied probabilities from odds if available and not already set if feat.get("odds_ms_h", 0) > 1 and feat.get("odds_ms_d", 0) > 1 and feat.get("odds_ms_a", 0) > 1: raw_h = 1.0 / feat["odds_ms_h"] raw_d = 1.0 / feat["odds_ms_d"] raw_a = 1.0 / feat["odds_ms_a"] total = raw_h + raw_d + raw_a if total > 0: feat["implied_home"] = raw_h / total feat["implied_draw"] = raw_d / total feat["implied_away"] = raw_a / total # ---- Derived match metadata ---- mst = match_row.get("mst_utc") if mst is not None: try: ts_s = int(mst) / 1000 # stored as epoch ms dt = datetime.utcfromtimestamp(ts_s) if "match_month" in feat: feat["match_month"] = float(dt.month) # Season markers: Sept-Oct = start, April-May = end if "is_season_start" in feat: feat["is_season_start"] = 1.0 if dt.month in (8, 9, 10) else 0.0 if "is_season_end" in feat: feat["is_season_end"] = 1.0 if dt.month in (4, 5) else 0.0 except Exception: pass # ---- Interaction features ---- if "attack_vs_defense_home" in feat: feat["attack_vs_defense_home"] = feat.get("home_goals_avg", 0) - feat.get("away_conceded_avg", 0) if "attack_vs_defense_away" in feat: feat["attack_vs_defense_away"] = feat.get("away_goals_avg", 0) - feat.get("home_conceded_avg", 0) if "form_momentum_interaction" in feat: feat["form_momentum_interaction"] = ( feat.get("home_momentum_score", 0) * feat.get("home_goals_avg", 0) - feat.get("away_momentum_score", 0) * feat.get("away_goals_avg", 0) ) if "elo_form_consistency" in feat: feat["elo_form_consistency"] = feat.get("elo_diff", 0) * feat.get("home_goals_avg", 0) return feat # ============================================================================ # STEP 3 — Run V25 predictions # ============================================================================ def load_predictor(): from models.v25_ensemble import get_v25_predictor print("[Step 3] Loading V25 predictor ...") pred = get_v25_predictor() print("[Step 3] V25 predictor ready.") return pred # ============================================================================ # STEP 4 — Compute actual outcomes from scores # ============================================================================ def compute_actuals(score_home: int, score_away: int) -> Dict[str, Any]: total = score_home + score_away return { "ms_actual": "1" if score_home > score_away else ("X" if score_home == score_away else "2"), "ou15_actual": "Over" if total >= 2 else "Under", "ou25_actual": "Over" if total >= 3 else "Under", "btts_actual": "Yes" if score_home > 0 and score_away > 0 else "No", } # ============================================================================ # STEP 5 — Accuracy helpers # ============================================================================ def confidence_band(prob: float) -> str: if prob < 0.50: return "<50%" elif prob < 0.65: return "50-65%" elif prob < 0.75: return "65-75%" else: return "75%+" def pick_from_ms(home_prob: float, draw_prob: float, away_prob: float) -> Tuple[str, float]: picks = {"1": home_prob, "X": draw_prob, "2": away_prob} best = max(picks, key=picks.__getitem__) return best, picks[best] def pick_from_binary(yes_prob: float, no_prob: float, yes_label: str, no_label: str) -> Tuple[str, float]: if yes_prob >= no_prob: return yes_label, yes_prob return no_label, no_prob # ============================================================================ # MAIN # ============================================================================ def main(): t_start = time.time() print("=" * 70) print(" V25 Backtest + Calibration Training") print(f" Run at: {datetime.utcnow().isoformat()} UTC") print("=" * 70) # ------------------------------------------------------------------ # Step 1 — Load qualified leagues # ------------------------------------------------------------------ league_ids = load_qualified_leagues() # ------------------------------------------------------------------ # Step 1b — Fetch matches with features # ------------------------------------------------------------------ conn = psycopg2.connect(get_clean_dsn()) try: matches_df = fetch_matches(conn, league_ids) if matches_df.empty: print("[ERROR] No matches found. Check DB connection and league IDs.") return match_ids = matches_df["match_id"].tolist() # ------------------------------------------------------------------ # Step 1c — Fetch odds in bulk # ------------------------------------------------------------------ odds_map = fetch_odds_bulk(conn, match_ids) finally: conn.close() # ------------------------------------------------------------------ # Step 2 — Build feature vectors # ------------------------------------------------------------------ print(f"\n[Step 2] Building feature vectors for {len(matches_df)} matches ...") feature_cols = load_feature_cols() # ------------------------------------------------------------------ # Step 3 — Load V25 predictor # ------------------------------------------------------------------ predictor = load_predictor() # ------------------------------------------------------------------ # Main loop — predict each match, collect results # ------------------------------------------------------------------ print(f"\n[Loop] Running predictions ...") # Storage for calibration training calib_data: Dict[str, List[Tuple[float, int]]] = { "ms_home": [], # (prob, 1 if home win) "ms_draw": [], "ms_away": [], "ou15": [], "ou25": [], "btts": [], } # Storage for accuracy reporting records = [] skipped = 0 processed = 0 for idx, row in matches_df.iterrows(): match_id = row["match_id"] score_home = row.get("score_home") score_away = row.get("score_away") # Validate scores try: score_home = int(score_home) score_away = int(score_away) except (TypeError, ValueError): skipped += 1 continue # Build features match_odds = odds_map.get(match_id, {}) feat = build_feature_vector(row, match_odds, feature_cols) # Run predictions try: home_prob, draw_prob, away_prob = predictor.predict_ms(feat) over25_prob, under25_prob = predictor.predict_ou25(feat) btts_yes_prob, btts_no_prob = predictor.predict_btts(feat) # ou15 is loaded via predict_market (returns np.ndarray for binary) ou15_arr = predictor.predict_market("ou15", feat) if ou15_arr is not None and len(ou15_arr) > 0: over15_prob = float(ou15_arr[0]) under15_prob = 1.0 - over15_prob else: over15_prob = 0.5 under15_prob = 0.5 except Exception as e: skipped += 1 continue # Compute actuals actuals = compute_actuals(score_home, score_away) # MS picks ms_pick, ms_conf = pick_from_ms(home_prob, draw_prob, away_prob) ms_correct = int(ms_pick == actuals["ms_actual"]) # OU15 ou15_pick, ou15_conf = pick_from_binary(over15_prob, under15_prob, "Over", "Under") ou15_correct = int(ou15_pick == actuals["ou15_actual"]) # OU25 ou25_pick, ou25_conf = pick_from_binary(over25_prob, under25_prob, "Over", "Under") ou25_correct = int(ou25_pick == actuals["ou25_actual"]) # BTTS btts_pick, btts_conf = pick_from_binary(btts_yes_prob, btts_no_prob, "Yes", "No") btts_correct = int(btts_pick == actuals["btts_actual"]) # Collect calibration data calib_data["ms_home"].append((home_prob, int(actuals["ms_actual"] == "1"))) calib_data["ms_draw"].append((draw_prob, int(actuals["ms_actual"] == "X"))) calib_data["ms_away"].append((away_prob, int(actuals["ms_actual"] == "2"))) calib_data["ou15"].append((over15_prob, int(actuals["ou15_actual"] == "Over"))) calib_data["ou25"].append((over25_prob, int(actuals["ou25_actual"] == "Over"))) calib_data["btts"].append((btts_yes_prob, int(actuals["btts_actual"] == "Yes"))) # Determine league group league_name = str(row.get("league_name", "Other") or "Other") league_group = league_name if league_name in TOP5_LEAGUE_NAMES else "Other" records.append({ "match_id": match_id, "league_name": league_name, "league_group": league_group, "score_home": score_home, "score_away": score_away, # MS "ms_pick": ms_pick, "ms_actual": actuals["ms_actual"], "ms_conf": ms_conf, "ms_conf_band": confidence_band(ms_conf), "ms_correct": ms_correct, "ms_home_prob": home_prob, "ms_draw_prob": draw_prob, "ms_away_prob": away_prob, # OU15 "ou15_pick": ou15_pick, "ou15_actual": actuals["ou15_actual"], "ou15_conf": ou15_conf, "ou15_conf_band": confidence_band(ou15_conf), "ou15_correct": ou15_correct, "ou15_over_prob": over15_prob, # OU25 "ou25_pick": ou25_pick, "ou25_actual": actuals["ou25_actual"], "ou25_conf": ou25_conf, "ou25_conf_band": confidence_band(ou25_conf), "ou25_correct": ou25_correct, "ou25_over_prob": over25_prob, # BTTS "btts_pick": btts_pick, "btts_actual": actuals["btts_actual"], "btts_conf": btts_conf, "btts_conf_band": confidence_band(btts_conf), "btts_correct": btts_correct, "btts_yes_prob": btts_yes_prob, }) processed += 1 if processed % PROGRESS_INTERVAL == 0: elapsed = time.time() - t_start print(f" [Progress] {processed}/{len(matches_df)} matches | " f"skipped={skipped} | elapsed={elapsed:.1f}s") print(f"\n[Loop] Done. Processed={processed}, Skipped={skipped}") if not records: print("[ERROR] No records to analyze. Exiting.") return results_df = pd.DataFrame(records) # ------------------------------------------------------------------ # Step 5 — Accuracy report # ------------------------------------------------------------------ print("\n" + "=" * 70) print(" ACCURACY REPORT") print("=" * 70) markets = [ ("MS", "ms_correct", "ms_conf", "ms_conf_band", "ms_pick"), ("OU15", "ou15_correct", "ou15_conf", "ou15_conf_band", "ou15_pick"), ("OU25", "ou25_correct", "ou25_conf", "ou25_conf_band", "ou25_pick"), ("BTTS", "btts_correct", "btts_conf", "btts_conf_band", "btts_pick"), ] summary: Dict[str, Any] = { "generated_at": datetime.utcnow().isoformat(), "matches_processed": processed, "matches_skipped": skipped, "markets": {}, } for market_label, correct_col, conf_col, band_col, pick_col in markets: print(f"\n--- {market_label} ---") sub = results_df[[correct_col, conf_col, band_col, pick_col, "league_group"]].copy() total = len(sub) overall_acc = sub[correct_col].mean() * 100 print(f" Overall accuracy: {overall_acc:.1f}% ({sub[correct_col].sum()}/{total})") market_summary = { "overall_accuracy": round(overall_acc, 2), "total_matches": total, "by_confidence_band": {}, "by_league": {}, "by_pick_direction": {}, } # By confidence band print(f" By confidence band:") bands = ["<50%", "50-65%", "65-75%", "75%+"] for band in bands: mask = sub[band_col] == band n = mask.sum() if n > 0: acc = sub.loc[mask, correct_col].mean() * 100 mean_conf = sub.loc[mask, conf_col].mean() * 100 print(f" {band:8s}: {acc:5.1f}% acc | {n:4d} matches | " f"mean_conf={mean_conf:.1f}%") market_summary["by_confidence_band"][band] = { "accuracy": round(acc, 2), "count": int(n), "mean_confidence": round(mean_conf, 2), } # By league group print(f" By league:") league_groups = list(results_df["league_group"].unique()) # Sort: named leagues first, then Other named = sorted([g for g in league_groups if g != "Other"]) ordered = named + (["Other"] if "Other" in league_groups else []) for lg in ordered: mask = sub["league_group"] == lg n = mask.sum() if n > 0: acc = sub.loc[mask, correct_col].mean() * 100 print(f" {lg[:20]:20s}: {acc:5.1f}% ({n} matches)") market_summary["by_league"][lg] = { "accuracy": round(acc, 2), "count": int(n), } # By pick direction print(f" By pick direction:") for pick_val in sorted(sub[pick_col].unique()): mask = sub[pick_col] == pick_val n = mask.sum() if n > 0: acc = sub.loc[mask, correct_col].mean() * 100 mean_conf = sub.loc[mask, conf_col].mean() * 100 print(f" {pick_val:8s}: {acc:5.1f}% acc | {n:4d} matches | " f"mean_conf={mean_conf:.1f}%") market_summary["by_pick_direction"][pick_val] = { "accuracy": round(acc, 2), "count": int(n), "mean_confidence": round(mean_conf, 2), } summary["markets"][market_label] = market_summary # ------------------------------------------------------------------ # Step 6 — Train calibration models # ------------------------------------------------------------------ print("\n" + "=" * 70) print(" CALIBRATION TRAINING") print("=" * 70) calibrator = Calibrator() # Market config: market_key -> (label for prob, label for actual binary) calib_market_map = { "ms_home": "ms_home", "ms_draw": "ms_draw", "ms_away": "ms_away", "ou15": "ou15", "ou25": "ou25", "btts": "btts", } calibration_results: Dict[str, Dict] = {} for market_key in calib_market_map: pairs = calib_data[market_key] if len(pairs) < 100: print(f"[Calib] {market_key}: only {len(pairs)} samples — skipping.") continue probs = np.array([p for p, _ in pairs]) actuals_bin = np.array([a for _, a in pairs]) # Build a tiny DataFrame to use Calibrator.train_calibration calib_df = pd.DataFrame({ "prob": probs, "actual": actuals_bin, }) metrics = calibrator.train_calibration( df=calib_df, market=market_key, prob_col="prob", actual_col="actual", min_samples=100, save=True, ) calibration_results[market_key] = metrics.to_dict() print(f" [Calib] {market_key}: Brier={metrics.brier_score:.4f} | " f"ECE={metrics.calibration_error:.4f} | n={metrics.sample_count}") # ------------------------------------------------------------------ # Step 7 — Save results # ------------------------------------------------------------------ output_path = os.path.join(REPORTS_DIR, "backtest_results.json") full_report = { **summary, "calibration": calibration_results, "runtime_seconds": round(time.time() - t_start, 1), } with open(output_path, "w") as f: json.dump(full_report, f, indent=2) print(f"\n[Step 7] Report saved to {output_path}") # ------------------------------------------------------------------ # Final summary table # ------------------------------------------------------------------ print("\n" + "=" * 70) print(" FINAL SUMMARY TABLE") print("=" * 70) print(f"{'Market':<8} {'Overall Acc':>12} {'Matches':>8} " f"{'Best Band (acc)':>18}") print("-" * 70) for market_label, _, _, _, _ in markets: ms = summary["markets"].get(market_label, {}) overall = ms.get("overall_accuracy", 0) total_m = ms.get("total_matches", 0) bands_d = ms.get("by_confidence_band", {}) # Find best accuracy band with >= 50 matches best_band = "-" best_acc = 0.0 for band, bdata in bands_d.items(): if bdata["count"] >= 50 and bdata["accuracy"] > best_acc: best_acc = bdata["accuracy"] best_band = f"{band} ({best_acc:.1f}%)" print(f"{market_label:<8} {overall:>11.1f}% {total_m:>8d} {best_band:>18s}") elapsed_total = time.time() - t_start print(f"\nTotal runtime: {elapsed_total:.1f}s") print("=" * 70) if __name__ == "__main__": main()