232 lines
8.3 KiB
Python
232 lines
8.3 KiB
Python
"""
|
|
Backtest ROI Engine
|
|
===================
|
|
Simulates the NEW "Skip Logic" on historical predictions.
|
|
Answers: "What if we only played the bets the model was confident about?"
|
|
|
|
Usage:
|
|
python ai-engine/scripts/backtest_roi.py
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import psycopg2
|
|
from psycopg2.extras import RealDictCursor
|
|
from typing import Dict, List, Any
|
|
from dotenv import load_dotenv
|
|
|
|
# Load .env from project root (2 levels up from this script)
|
|
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
load_dotenv(os.path.join(project_root, ".env"))
|
|
|
|
def get_clean_dsn() -> str:
|
|
"""Return a psycopg2-compatible DSN from DATABASE_URL."""
|
|
# HARDCODED FOR BACKTEST (Bypassing dotenv issues)
|
|
return "postgresql://suggestbet:SuGGesT2026SecuRe@localhost:15432/boilerplate_db"
|
|
|
|
# ─── Configuration (Matching the NEW BetRecommender Logic) ─────────
|
|
# Minimum confidence to even consider a bet (Hard Gate)
|
|
MIN_CONF_THRESHOLDS = {
|
|
"MS": 45.0,
|
|
"DC": 40.0,
|
|
"OU15": 50.0,
|
|
"OU25": 45.0,
|
|
"OU35": 45.0,
|
|
"BTTS": 45.0,
|
|
"HT": 40.0,
|
|
}
|
|
|
|
def get_market_type_from_key(key: str) -> str:
|
|
"""Map prediction keys to market types for thresholding."""
|
|
if key.startswith("ms_") or key in ["1", "X", "2"]: return "MS"
|
|
if key.startswith("dc_") or key in ["1X", "X2", "12"]: return "DC"
|
|
if key.startswith("ou15_") or key.startswith("1.5"): return "OU15"
|
|
if key.startswith("ou25_") or key.startswith("2.5"): return "OU25"
|
|
if key.startswith("ou35_") or key.startswith("3.5"): return "OU35"
|
|
if key.startswith("btts_") or key in ["Var", "Yok"]: return "BTTS"
|
|
if key.startswith("ht_") or key.startswith("İY"): return "HT"
|
|
return "MS"
|
|
|
|
def simulate_backtest():
|
|
print("🚀 Starting Backtest with NEW 'Skip Logic'...")
|
|
print("="*60)
|
|
|
|
dsn = get_clean_dsn()
|
|
conn = psycopg2.connect(dsn)
|
|
cur = conn.cursor(cursor_factory=RealDictCursor)
|
|
|
|
# 1. Fetch PREDICTIONS that have a confidence score
|
|
# We limit to last 1000 finished matches to keep it fast but representative
|
|
cur.execute("""
|
|
SELECT p.match_id, p.prediction_json,
|
|
m.score_home, m.score_away, m.status
|
|
FROM predictions p
|
|
JOIN matches m ON p.match_id = m.id
|
|
WHERE m.status = 'FT'
|
|
AND p.prediction_json IS NOT NULL
|
|
ORDER BY m.mst_utc DESC
|
|
LIMIT 2000
|
|
""")
|
|
predictions = cur.fetchall()
|
|
|
|
print(f"📊 Loaded {len(predictions)} historical predictions.")
|
|
|
|
total_bets = 0
|
|
winning_bets = 0
|
|
skipped_bets = 0
|
|
total_profit = 0.0 # Assuming unit stake of 1.0
|
|
|
|
# 2. Process each prediction
|
|
for pred_row in predictions:
|
|
match_id = pred_row['match_id']
|
|
data = pred_row['prediction_json']
|
|
if isinstance(data, str):
|
|
data = json.loads(data)
|
|
|
|
# Real result
|
|
home_score = pred_row['score_home'] or 0
|
|
away_score = pred_row['score_away'] or 0
|
|
total_goals = home_score + away_score
|
|
|
|
# Extract prediction details from the JSON structure
|
|
# The structure varies, but usually contains 'main_pick', 'bet_summary', or 'market_board'
|
|
|
|
# Try to get the main pick recommendation
|
|
main_pick = None
|
|
main_pick_conf = 0.0
|
|
main_pick_odds = 0.0
|
|
|
|
# Navigate the V20+ JSON structure
|
|
market_board = data.get("market_board", {})
|
|
|
|
# Check Main Pick
|
|
if "main_pick" in data:
|
|
mp = data["main_pick"]
|
|
if isinstance(mp, dict):
|
|
main_pick = mp.get("pick")
|
|
main_pick_conf = mp.get("confidence", 0.0)
|
|
main_pick_odds = mp.get("odds", 0.0)
|
|
|
|
# If no main pick, try bet_summary
|
|
if not main_pick and "bet_summary" in data:
|
|
summary = data["bet_summary"]
|
|
if isinstance(summary, list) and len(summary) > 0:
|
|
# Take the highest confidence one
|
|
best = max(summary, key=lambda x: x.get("confidence", 0))
|
|
main_pick = best.get("pick")
|
|
main_pick_conf = best.get("confidence", 0.0)
|
|
main_pick_odds = best.get("odds", 0.0)
|
|
|
|
if not main_pick or not main_pick_conf:
|
|
continue
|
|
|
|
# ─── NEW LOGIC: APPLY FILTERS ───
|
|
# 1. Determine Market Type
|
|
# Simple heuristic based on pick string
|
|
pick_str = str(main_pick).upper()
|
|
market_type = "MS"
|
|
if "1X" in pick_str or "X2" in pick_str or "12" in pick_str: market_type = "DC"
|
|
elif "ÜST" in pick_str or "ALT" in pick_str or "OVER" in pick_str or "UNDER" in pick_str:
|
|
if "1.5" in pick_str: market_type = "OU15"
|
|
elif "3.5" in pick_str: market_type = "OU35"
|
|
else: market_type = "OU25"
|
|
elif "VAR" in pick_str or "YOK" in pick_str or "BTTS" in pick_str: market_type = "BTTS"
|
|
|
|
threshold = MIN_CONF_THRESHOLDS.get(market_type, 45.0)
|
|
|
|
# 2. Check Confidence Gate
|
|
if main_pick_conf < threshold:
|
|
skipped_bets += 1
|
|
continue
|
|
|
|
# 3. Check Value Gate (Edge)
|
|
if main_pick_odds > 0:
|
|
implied_prob = 1.0 / main_pick_odds
|
|
my_prob = main_pick_conf / 100.0
|
|
edge = my_prob - implied_prob
|
|
if edge < -0.03: # Negative value
|
|
skipped_bets += 1
|
|
continue
|
|
|
|
# ─── BET IS PLAYED ───
|
|
total_bets += 1
|
|
|
|
# Determine if WON
|
|
is_won = False
|
|
|
|
# Resolve MS (1, X, 2)
|
|
if market_type == "MS":
|
|
if main_pick == "1" and home_score > away_score: is_won = True
|
|
elif main_pick == "X" and home_score == away_score: is_won = True
|
|
elif main_pick == "2" and away_score > home_score: is_won = True
|
|
elif main_pick == "MS 1" and home_score > away_score: is_won = True
|
|
elif main_pick == "MS X" and home_score == away_score: is_won = True
|
|
elif main_pick == "MS 2" and away_score > home_score: is_won = True
|
|
|
|
# Resolve OU (Over/Under)
|
|
elif market_type.startswith("OU"):
|
|
line = 2.5
|
|
if "1.5" in pick_str: line = 1.5
|
|
elif "3.5" in pick_str: line = 3.5
|
|
|
|
is_over = total_goals > line
|
|
is_under = total_goals < line # Simplification (usually line is X.5 so no draw)
|
|
|
|
if "ÜST" in pick_str or "OVER" in pick_str:
|
|
if is_over: is_won = True
|
|
elif "ALT" in pick_str or "UNDER" in pick_str:
|
|
if is_under: is_won = True
|
|
|
|
# Resolve BTTS
|
|
elif market_type == "BTTS":
|
|
if home_score > 0 and away_score > 0:
|
|
if "VAR" in pick_str: is_won = True
|
|
else:
|
|
if "YOK" in pick_str: is_won = True
|
|
|
|
# Resolve DC (Double Chance) - Simplified
|
|
elif market_type == "DC":
|
|
if "1X" in pick_str and (home_score >= away_score): is_won = True
|
|
elif "X2" in pick_str and (away_score >= home_score): is_won = True
|
|
elif "12" in pick_str and (home_score != away_score): is_won = True
|
|
|
|
if is_won:
|
|
winning_bets += 1
|
|
profit = main_pick_odds - 1.0
|
|
total_profit += profit
|
|
else:
|
|
total_profit -= 1.0
|
|
|
|
# ─── REPORT ───
|
|
print("\n" + "="*60)
|
|
print("📈 BACKTEST RESULTS (With NEW Skip Logic)")
|
|
print("="*60)
|
|
print(f"Total Historical Matches Analyzed: {len(predictions)}")
|
|
print(f"🚫 Bets SKIPPED (Low Conf/Bad Value): {skipped_bets}")
|
|
print(f"✅ Bets PLAYED: {total_bets}")
|
|
|
|
if total_bets > 0:
|
|
win_rate = (winning_bets / total_bets) * 100
|
|
roi = (total_profit / total_bets) * 100
|
|
|
|
print(f"🏆 Winning Bets: {winning_bets}")
|
|
print(f"💀 Losing Bets: {total_bets - winning_bets}")
|
|
print("-" * 40)
|
|
print(f" Win Rate: {win_rate:.2f}%")
|
|
print(f"💰 Total Profit (Units): {total_profit:.2f}")
|
|
print(f"📊 ROI: {roi:.2f}%")
|
|
|
|
if roi > 0:
|
|
print("🟢 STRATEGY IS PROFITABLE!")
|
|
else:
|
|
print("🔴 STRATEGY IS LOSING (Adjust thresholds!)")
|
|
else:
|
|
print("⚠️ No bets were played. Thresholds might be too high.")
|
|
|
|
cur.close()
|
|
conn.close()
|
|
|
|
if __name__ == "__main__":
|
|
simulate_backtest()
|