""" Backtest ROI Engine =================== Simulates the NEW "Skip Logic" on historical predictions. Answers: "What if we only played the bets the model was confident about?" Usage: python ai-engine/scripts/backtest_roi.py """ import os import sys import json import psycopg2 from psycopg2.extras import RealDictCursor from typing import Dict, List, Any from dotenv import load_dotenv # Load .env from project root (2 levels up from this script) project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) load_dotenv(os.path.join(project_root, ".env")) def get_clean_dsn() -> str: """Return a psycopg2-compatible DSN from DATABASE_URL.""" # HARDCODED FOR BACKTEST (Bypassing dotenv issues) return "postgresql://suggestbet:SuGGesT2026SecuRe@localhost:15432/boilerplate_db" # ─── Configuration (Matching the NEW BetRecommender Logic) ───────── # Minimum confidence to even consider a bet (Hard Gate) MIN_CONF_THRESHOLDS = { "MS": 45.0, "DC": 40.0, "OU15": 50.0, "OU25": 45.0, "OU35": 45.0, "BTTS": 45.0, "HT": 40.0, } def get_market_type_from_key(key: str) -> str: """Map prediction keys to market types for thresholding.""" if key.startswith("ms_") or key in ["1", "X", "2"]: return "MS" if key.startswith("dc_") or key in ["1X", "X2", "12"]: return "DC" if key.startswith("ou15_") or key.startswith("1.5"): return "OU15" if key.startswith("ou25_") or key.startswith("2.5"): return "OU25" if key.startswith("ou35_") or key.startswith("3.5"): return "OU35" if key.startswith("btts_") or key in ["Var", "Yok"]: return "BTTS" if key.startswith("ht_") or key.startswith("İY"): return "HT" return "MS" def simulate_backtest(): print("🚀 Starting Backtest with NEW 'Skip Logic'...") print("="*60) dsn = get_clean_dsn() conn = psycopg2.connect(dsn) cur = conn.cursor(cursor_factory=RealDictCursor) # 1. Fetch PREDICTIONS that have a confidence score # We limit to last 1000 finished matches to keep it fast but representative cur.execute(""" SELECT p.match_id, p.prediction_json, m.score_home, m.score_away, m.status FROM predictions p JOIN matches m ON p.match_id = m.id WHERE m.status = 'FT' AND p.prediction_json IS NOT NULL ORDER BY m.mst_utc DESC LIMIT 2000 """) predictions = cur.fetchall() print(f"📊 Loaded {len(predictions)} historical predictions.") total_bets = 0 winning_bets = 0 skipped_bets = 0 total_profit = 0.0 # Assuming unit stake of 1.0 # 2. Process each prediction for pred_row in predictions: match_id = pred_row['match_id'] data = pred_row['prediction_json'] if isinstance(data, str): data = json.loads(data) # Real result home_score = pred_row['score_home'] or 0 away_score = pred_row['score_away'] or 0 total_goals = home_score + away_score # Extract prediction details from the JSON structure # The structure varies, but usually contains 'main_pick', 'bet_summary', or 'market_board' # Try to get the main pick recommendation main_pick = None main_pick_conf = 0.0 main_pick_odds = 0.0 # Navigate the V20+ JSON structure market_board = data.get("market_board", {}) # Check Main Pick if "main_pick" in data: mp = data["main_pick"] if isinstance(mp, dict): main_pick = mp.get("pick") main_pick_conf = mp.get("confidence", 0.0) main_pick_odds = mp.get("odds", 0.0) # If no main pick, try bet_summary if not main_pick and "bet_summary" in data: summary = data["bet_summary"] if isinstance(summary, list) and len(summary) > 0: # Take the highest confidence one best = max(summary, key=lambda x: x.get("confidence", 0)) main_pick = best.get("pick") main_pick_conf = best.get("confidence", 0.0) main_pick_odds = best.get("odds", 0.0) if not main_pick or not main_pick_conf: continue # ─── NEW LOGIC: APPLY FILTERS ─── # 1. Determine Market Type # Simple heuristic based on pick string pick_str = str(main_pick).upper() market_type = "MS" if "1X" in pick_str or "X2" in pick_str or "12" in pick_str: market_type = "DC" elif "ÜST" in pick_str or "ALT" in pick_str or "OVER" in pick_str or "UNDER" in pick_str: if "1.5" in pick_str: market_type = "OU15" elif "3.5" in pick_str: market_type = "OU35" else: market_type = "OU25" elif "VAR" in pick_str or "YOK" in pick_str or "BTTS" in pick_str: market_type = "BTTS" threshold = MIN_CONF_THRESHOLDS.get(market_type, 45.0) # 2. Check Confidence Gate if main_pick_conf < threshold: skipped_bets += 1 continue # 3. Check Value Gate (Edge) if main_pick_odds > 0: implied_prob = 1.0 / main_pick_odds my_prob = main_pick_conf / 100.0 edge = my_prob - implied_prob if edge < -0.03: # Negative value skipped_bets += 1 continue # ─── BET IS PLAYED ─── total_bets += 1 # Determine if WON is_won = False # Resolve MS (1, X, 2) if market_type == "MS": if main_pick == "1" and home_score > away_score: is_won = True elif main_pick == "X" and home_score == away_score: is_won = True elif main_pick == "2" and away_score > home_score: is_won = True elif main_pick == "MS 1" and home_score > away_score: is_won = True elif main_pick == "MS X" and home_score == away_score: is_won = True elif main_pick == "MS 2" and away_score > home_score: is_won = True # Resolve OU (Over/Under) elif market_type.startswith("OU"): line = 2.5 if "1.5" in pick_str: line = 1.5 elif "3.5" in pick_str: line = 3.5 is_over = total_goals > line is_under = total_goals < line # Simplification (usually line is X.5 so no draw) if "ÜST" in pick_str or "OVER" in pick_str: if is_over: is_won = True elif "ALT" in pick_str or "UNDER" in pick_str: if is_under: is_won = True # Resolve BTTS elif market_type == "BTTS": if home_score > 0 and away_score > 0: if "VAR" in pick_str: is_won = True else: if "YOK" in pick_str: is_won = True # Resolve DC (Double Chance) - Simplified elif market_type == "DC": if "1X" in pick_str and (home_score >= away_score): is_won = True elif "X2" in pick_str and (away_score >= home_score): is_won = True elif "12" in pick_str and (home_score != away_score): is_won = True if is_won: winning_bets += 1 profit = main_pick_odds - 1.0 total_profit += profit else: total_profit -= 1.0 # ─── REPORT ─── print("\n" + "="*60) print("📈 BACKTEST RESULTS (With NEW Skip Logic)") print("="*60) print(f"Total Historical Matches Analyzed: {len(predictions)}") print(f"🚫 Bets SKIPPED (Low Conf/Bad Value): {skipped_bets}") print(f"✅ Bets PLAYED: {total_bets}") if total_bets > 0: win_rate = (winning_bets / total_bets) * 100 roi = (total_profit / total_bets) * 100 print(f"🏆 Winning Bets: {winning_bets}") print(f"💀 Losing Bets: {total_bets - winning_bets}") print("-" * 40) print(f" Win Rate: {win_rate:.2f}%") print(f"💰 Total Profit (Units): {total_profit:.2f}") print(f"📊 ROI: {roi:.2f}%") if roi > 0: print("🟢 STRATEGY IS PROFITABLE!") else: print("🔴 STRATEGY IS LOSING (Adjust thresholds!)") else: print("⚠️ No bets were played. Thresholds might be too high.") cur.close() conn.close() if __name__ == "__main__": simulate_backtest()