"""
Filter Optimizer
================
Grid-search over filter thresholds (per market) using the existing
diagnostic_backtest CSV. Finds the (confidence, edge, odds, reliability)
combination that maximizes ROI while keeping bet volume reasonable.

No re-prediction needed — pure offline simulation on the bets already
captured. Output: per-market optimal thresholds + projected ROI lift +
JSON patch ready to drop into config/market_thresholds.json.

Usage:
  python scripts/optimize_filters.py
  python scripts/optimize_filters.py --csv reports/diagnostic_backtest_X.csv
  python scripts/optimize_filters.py --min-bets 20 --apply
"""

import argparse
import json
import os
import sys
import glob
import itertools
from typing import List, Dict, Tuple, Optional
import pandas as pd
import numpy as np

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
AI_ENGINE_DIR = os.path.dirname(SCRIPT_DIR)
sys.path.insert(0, AI_ENGINE_DIR)

REPORTS_DIR = os.path.join(AI_ENGINE_DIR, "reports")
CONFIG_PATH = os.path.join(AI_ENGINE_DIR, "config", "market_thresholds.json")


def latest_csv() -> Optional[str]:
    files = sorted(glob.glob(os.path.join(REPORTS_DIR, "diagnostic_backtest_*.csv")),
                   key=os.path.getmtime, reverse=True)
    return files[0] if files else None


def load_backtest(path: str) -> pd.DataFrame:
    df = pd.read_csv(path)
    # Keep only playable + settled bets — these are what the SYSTEM
    # actually placed and got an outcome on.
    pdf = df[(df["playable"] == True) & (df["won"].notna())].copy()
    pdf["won"] = pdf["won"].astype(bool)
    pdf["calibrated_confidence"] = pdf["calibrated_confidence"].fillna(0)
    pdf["ev_edge"] = pdf["ev_edge"].fillna(0)
    pdf["odds"] = pdf["odds"].fillna(0)
    pdf["odds_reliability"] = pdf["odds_reliability"].fillna(0.35)
    return pdf


def evaluate(pdf: pd.DataFrame, mask) -> Dict:
    kept = pdf[mask]
    if len(kept) == 0:
        return {"n": 0, "hit_pct": 0, "profit": 0, "staked": 0, "roi_pct": 0}
    wins = kept["won"].sum()
    profit = kept["unit_profit"].sum()
    staked = kept["stake_units"].sum()
    return {
        "n": int(len(kept)),
        "hit_pct": round(100.0 * wins / len(kept), 2),
        "profit": round(profit, 3),
        "staked": round(staked, 3),
        "roi_pct": round(100.0 * profit / staked, 2) if staked else 0,
    }


def grid_search_market(
    market_df: pd.DataFrame,
    market: str,
    min_bets: int = 15,
) -> List[Dict]:
    """Try a wide grid of (min_conf, min_edge, max_edge, min_odds, max_odds,
    min_reliability) combinations. Return all candidates with n >= min_bets,
    sorted by ROI descending."""

    conf_options = [0, 45, 50, 55, 60, 65, 70]
    min_edge_options = [-1.0, -0.05, 0.0, 0.03, 0.05, 0.08]
    max_edge_options = [10.0, 0.30, 0.20, 0.15, 0.10]
    min_odds_options = [1.20, 1.30, 1.40, 1.50, 1.60, 1.80]
    max_odds_options = [10.0, 3.0, 2.5, 2.2, 2.0]
    rel_options = [0.0, 0.30, 0.45, 0.55]
    consensus_options = ["any", "agree_or_null"]

    candidates: List[Dict] = []
    for mc, mine, maxe, mino, maxo, mrel, cons in itertools.product(
        conf_options, min_edge_options, max_edge_options,
        min_odds_options, max_odds_options, rel_options, consensus_options,
    ):
        if mine >= maxe or mino >= maxo:
            continue
        mask = (
            (market_df["calibrated_confidence"] >= mc)
            & (market_df["ev_edge"] >= mine)
            & (market_df["ev_edge"] <= maxe)
            & (market_df["odds"] >= mino)
            & (market_df["odds"] <= maxo)
            & (market_df["odds_reliability"] >= mrel)
        )
        if cons == "agree_or_null":
            mask &= market_df["v27_consensus"] != "DISAGREE"
        result = evaluate(market_df, mask)
        if result["n"] >= min_bets:
            candidates.append({
                "market": market,
                "min_conf": mc,
                "min_edge": mine,
                "max_edge": maxe,
                "min_odds": mino,
                "max_odds": maxo,
                "min_reliability": mrel,
                "consensus": cons,
                **result,
            })
    candidates.sort(key=lambda r: (r["roi_pct"], r["n"]), reverse=True)
    return candidates


def baseline(pdf: pd.DataFrame, market: str) -> Dict:
    m = pdf[pdf["market"] == market]
    return evaluate(m, pd.Series([True] * len(m), index=m.index))


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--csv", default=None, help="Override CSV path")
    parser.add_argument("--min-bets", type=int, default=15,
                        help="Min bet count to consider a config valid")
    parser.add_argument("--top-k", type=int, default=3,
                        help="Show top K configs per market")
    parser.add_argument("--apply", action="store_true",
                        help="Patch config/market_thresholds.json with winners")
    args = parser.parse_args()

    csv_path = args.csv or latest_csv()
    if not csv_path or not os.path.exists(csv_path):
        print("No backtest CSV found.")
        return
    print(f"Loading: {csv_path}")
    pdf = load_backtest(csv_path)
    print(f"Playable + settled bets: {len(pdf)}")

    markets = sorted(pdf["market"].dropna().unique())
    print(f"Markets: {markets}\n")

    all_winners: Dict[str, Dict] = {}

    for market in markets:
        market_df = pdf[pdf["market"] == market]
        n_total = len(market_df)
        base = baseline(pdf, market)

        print(f"\n{'=' * 78}")
        print(f"MARKET: {market}   (n={n_total} baseline_roi={base['roi_pct']}%)")
        print(f"{'=' * 78}")

        if n_total < args.min_bets * 2:
            print(f"  Sample too small to grid-search reliably (n={n_total}). Skip.")
            continue

        candidates = grid_search_market(market_df, market, args.min_bets)
        if not candidates:
            print(f"  No config kept >= {args.min_bets} bets. Skip.")
            continue

        # Pareto-ish: show top-K by ROI but also one that keeps higher bet count
        winners = candidates[:args.top_k]
        keep_high_volume = None
        for c in candidates:
            if c["n"] >= max(40, n_total // 3) and c["roi_pct"] > base["roi_pct"]:
                keep_high_volume = c
                break

        print(f"  {'rank':<5}{'n':>5}{'hit%':>7}{'roi%':>8}  "
              f"{'min_conf':>9}{'min_edge':>10}{'max_edge':>10}"
              f"{'min_odds':>10}{'max_odds':>10}{'min_rel':>9}{'cons':>15}")
        for i, w in enumerate(winners, 1):
            print(f"  {i:<5}{w['n']:>5}{w['hit_pct']:>7}{w['roi_pct']:>+8}"
                  f"  {w['min_conf']:>9}{w['min_edge']:>+10.3f}{w['max_edge']:>+10.3f}"
                  f"{w['min_odds']:>10.2f}{w['max_odds']:>10.2f}"
                  f"{w['min_reliability']:>9.2f}{w['consensus']:>15}")
        if keep_high_volume and keep_high_volume not in winners:
            print(f"  high  {keep_high_volume['n']:>5}{keep_high_volume['hit_pct']:>7}"
                  f"{keep_high_volume['roi_pct']:>+8}"
                  f"  {keep_high_volume['min_conf']:>9}"
                  f"{keep_high_volume['min_edge']:>+10.3f}"
                  f"{keep_high_volume['max_edge']:>+10.3f}"
                  f"{keep_high_volume['min_odds']:>10.2f}"
                  f"{keep_high_volume['max_odds']:>10.2f}"
                  f"{keep_high_volume['min_reliability']:>9.2f}"
                  f"{keep_high_volume['consensus']:>15}")

        # Pick a "good" recommendation: best ROI with n >= min_bets
        # If best ROI is still negative, flag the market as unprofitable.
        best = winners[0]
        all_winners[market] = best
        if best["roi_pct"] <= 0:
            print(f"  ⚠️  Best config still loses money (ROI={best['roi_pct']}%) "
                  f"— consider muting this market entirely.")
        else:
            print(f"  ✅  Best config: ROI={best['roi_pct']}% on {best['n']} bets "
                  f"(vs baseline {base['roi_pct']}% on {n_total}).")

    # ─── Aggregate impact ────────────────────────────────────────────────
    print(f"\n{'=' * 78}")
    print("AGGREGATE IMPACT (if we apply each market's best config)")
    print(f"{'=' * 78}")
    total_old_bets = total_old_profit = total_old_staked = 0
    total_new_bets = total_new_profit = total_new_staked = 0
    for market, win in all_winners.items():
        base = baseline(pdf, market)
        total_old_bets += base["n"]
        total_old_profit += base["profit"]
        total_old_staked += base["staked"]
        total_new_bets += win["n"]
        total_new_profit += win["profit"]
        total_new_staked += win["staked"]
    base_roi = 100.0 * total_old_profit / total_old_staked if total_old_staked else 0
    new_roi = 100.0 * total_new_profit / total_new_staked if total_new_staked else 0
    print(f"  Baseline:   {total_old_bets:>4} bets, "
          f"profit={total_old_profit:+.2f}u, ROI={base_roi:+.2f}%")
    print(f"  Optimized:  {total_new_bets:>4} bets, "
          f"profit={total_new_profit:+.2f}u, ROI={new_roi:+.2f}%")
    print(f"  Δ:          {total_new_bets - total_old_bets:+d} bets, "
          f"{total_new_profit - total_old_profit:+.2f}u, "
          f"{new_roi - base_roi:+.2f}pp")

    # ─── Write JSON patch ────────────────────────────────────────────────
    patch_path = os.path.join(REPORTS_DIR, "filter_optimization_patch.json")
    patch = {market: {
        "min_calibrated_confidence": win["min_conf"],
        "min_ev_edge": win["min_edge"],
        "max_ev_edge": win["max_edge"],
        "min_odds": win["min_odds"],
        "max_odds": win["max_odds"],
        "min_odds_reliability": win["min_reliability"],
        "require_v27_agree": win["consensus"] == "agree_or_null",
        "expected_n_bets": win["n"],
        "expected_hit_pct": win["hit_pct"],
        "expected_roi_pct": win["roi_pct"],
    } for market, win in all_winners.items()}
    with open(patch_path, "w", encoding="utf-8") as f:
        json.dump(patch, f, indent=2)
    print(f"\nPatch saved: {patch_path}")

    if args.apply:
        print("\n--apply flag set. Patching not implemented yet — "
              "review the patch JSON and update config/market_thresholds.json manually.")

if __name__ == "__main__":
    main()
