gg
This commit is contained in:
@@ -1,231 +0,0 @@
|
||||
"""
|
||||
Backtest ROI Engine
|
||||
===================
|
||||
Simulates the NEW "Skip Logic" on historical predictions.
|
||||
Answers: "What if we only played the bets the model was confident about?"
|
||||
|
||||
Usage:
|
||||
python ai-engine/scripts/backtest_roi.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
from typing import Dict, List, Any
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load .env from project root (2 levels up from this script)
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
load_dotenv(os.path.join(project_root, ".env"))
|
||||
|
||||
def get_clean_dsn() -> str:
|
||||
"""Return a psycopg2-compatible DSN from DATABASE_URL."""
|
||||
# HARDCODED FOR BACKTEST (Bypassing dotenv issues)
|
||||
return "postgresql://suggestbet:SuGGesT2026SecuRe@localhost:15432/boilerplate_db"
|
||||
|
||||
# ─── Configuration (Matching the NEW BetRecommender Logic) ─────────
|
||||
# Minimum confidence to even consider a bet (Hard Gate)
|
||||
MIN_CONF_THRESHOLDS = {
|
||||
"MS": 45.0,
|
||||
"DC": 40.0,
|
||||
"OU15": 50.0,
|
||||
"OU25": 45.0,
|
||||
"OU35": 45.0,
|
||||
"BTTS": 45.0,
|
||||
"HT": 40.0,
|
||||
}
|
||||
|
||||
def get_market_type_from_key(key: str) -> str:
|
||||
"""Map prediction keys to market types for thresholding."""
|
||||
if key.startswith("ms_") or key in ["1", "X", "2"]: return "MS"
|
||||
if key.startswith("dc_") or key in ["1X", "X2", "12"]: return "DC"
|
||||
if key.startswith("ou15_") or key.startswith("1.5"): return "OU15"
|
||||
if key.startswith("ou25_") or key.startswith("2.5"): return "OU25"
|
||||
if key.startswith("ou35_") or key.startswith("3.5"): return "OU35"
|
||||
if key.startswith("btts_") or key in ["Var", "Yok"]: return "BTTS"
|
||||
if key.startswith("ht_") or key.startswith("İY"): return "HT"
|
||||
return "MS"
|
||||
|
||||
def simulate_backtest():
|
||||
print("🚀 Starting Backtest with NEW 'Skip Logic'...")
|
||||
print("="*60)
|
||||
|
||||
dsn = get_clean_dsn()
|
||||
conn = psycopg2.connect(dsn)
|
||||
cur = conn.cursor(cursor_factory=RealDictCursor)
|
||||
|
||||
# 1. Fetch PREDICTIONS that have a confidence score
|
||||
# We limit to last 1000 finished matches to keep it fast but representative
|
||||
cur.execute("""
|
||||
SELECT p.match_id, p.prediction_json,
|
||||
m.score_home, m.score_away, m.status
|
||||
FROM predictions p
|
||||
JOIN matches m ON p.match_id = m.id
|
||||
WHERE m.status = 'FT'
|
||||
AND p.prediction_json IS NOT NULL
|
||||
ORDER BY m.mst_utc DESC
|
||||
LIMIT 2000
|
||||
""")
|
||||
predictions = cur.fetchall()
|
||||
|
||||
print(f"📊 Loaded {len(predictions)} historical predictions.")
|
||||
|
||||
total_bets = 0
|
||||
winning_bets = 0
|
||||
skipped_bets = 0
|
||||
total_profit = 0.0 # Assuming unit stake of 1.0
|
||||
|
||||
# 2. Process each prediction
|
||||
for pred_row in predictions:
|
||||
match_id = pred_row['match_id']
|
||||
data = pred_row['prediction_json']
|
||||
if isinstance(data, str):
|
||||
data = json.loads(data)
|
||||
|
||||
# Real result
|
||||
home_score = pred_row['score_home'] or 0
|
||||
away_score = pred_row['score_away'] or 0
|
||||
total_goals = home_score + away_score
|
||||
|
||||
# Extract prediction details from the JSON structure
|
||||
# The structure varies, but usually contains 'main_pick', 'bet_summary', or 'market_board'
|
||||
|
||||
# Try to get the main pick recommendation
|
||||
main_pick = None
|
||||
main_pick_conf = 0.0
|
||||
main_pick_odds = 0.0
|
||||
|
||||
# Navigate the V20+ JSON structure
|
||||
market_board = data.get("market_board", {})
|
||||
|
||||
# Check Main Pick
|
||||
if "main_pick" in data:
|
||||
mp = data["main_pick"]
|
||||
if isinstance(mp, dict):
|
||||
main_pick = mp.get("pick")
|
||||
main_pick_conf = mp.get("confidence", 0.0)
|
||||
main_pick_odds = mp.get("odds", 0.0)
|
||||
|
||||
# If no main pick, try bet_summary
|
||||
if not main_pick and "bet_summary" in data:
|
||||
summary = data["bet_summary"]
|
||||
if isinstance(summary, list) and len(summary) > 0:
|
||||
# Take the highest confidence one
|
||||
best = max(summary, key=lambda x: x.get("confidence", 0))
|
||||
main_pick = best.get("pick")
|
||||
main_pick_conf = best.get("confidence", 0.0)
|
||||
main_pick_odds = best.get("odds", 0.0)
|
||||
|
||||
if not main_pick or not main_pick_conf:
|
||||
continue
|
||||
|
||||
# ─── NEW LOGIC: APPLY FILTERS ───
|
||||
# 1. Determine Market Type
|
||||
# Simple heuristic based on pick string
|
||||
pick_str = str(main_pick).upper()
|
||||
market_type = "MS"
|
||||
if "1X" in pick_str or "X2" in pick_str or "12" in pick_str: market_type = "DC"
|
||||
elif "ÜST" in pick_str or "ALT" in pick_str or "OVER" in pick_str or "UNDER" in pick_str:
|
||||
if "1.5" in pick_str: market_type = "OU15"
|
||||
elif "3.5" in pick_str: market_type = "OU35"
|
||||
else: market_type = "OU25"
|
||||
elif "VAR" in pick_str or "YOK" in pick_str or "BTTS" in pick_str: market_type = "BTTS"
|
||||
|
||||
threshold = MIN_CONF_THRESHOLDS.get(market_type, 45.0)
|
||||
|
||||
# 2. Check Confidence Gate
|
||||
if main_pick_conf < threshold:
|
||||
skipped_bets += 1
|
||||
continue
|
||||
|
||||
# 3. Check Value Gate (Edge)
|
||||
if main_pick_odds > 0:
|
||||
implied_prob = 1.0 / main_pick_odds
|
||||
my_prob = main_pick_conf / 100.0
|
||||
edge = my_prob - implied_prob
|
||||
if edge < -0.03: # Negative value
|
||||
skipped_bets += 1
|
||||
continue
|
||||
|
||||
# ─── BET IS PLAYED ───
|
||||
total_bets += 1
|
||||
|
||||
# Determine if WON
|
||||
is_won = False
|
||||
|
||||
# Resolve MS (1, X, 2)
|
||||
if market_type == "MS":
|
||||
if main_pick == "1" and home_score > away_score: is_won = True
|
||||
elif main_pick == "X" and home_score == away_score: is_won = True
|
||||
elif main_pick == "2" and away_score > home_score: is_won = True
|
||||
elif main_pick == "MS 1" and home_score > away_score: is_won = True
|
||||
elif main_pick == "MS X" and home_score == away_score: is_won = True
|
||||
elif main_pick == "MS 2" and away_score > home_score: is_won = True
|
||||
|
||||
# Resolve OU (Over/Under)
|
||||
elif market_type.startswith("OU"):
|
||||
line = 2.5
|
||||
if "1.5" in pick_str: line = 1.5
|
||||
elif "3.5" in pick_str: line = 3.5
|
||||
|
||||
is_over = total_goals > line
|
||||
is_under = total_goals < line # Simplification (usually line is X.5 so no draw)
|
||||
|
||||
if "ÜST" in pick_str or "OVER" in pick_str:
|
||||
if is_over: is_won = True
|
||||
elif "ALT" in pick_str or "UNDER" in pick_str:
|
||||
if is_under: is_won = True
|
||||
|
||||
# Resolve BTTS
|
||||
elif market_type == "BTTS":
|
||||
if home_score > 0 and away_score > 0:
|
||||
if "VAR" in pick_str: is_won = True
|
||||
else:
|
||||
if "YOK" in pick_str: is_won = True
|
||||
|
||||
# Resolve DC (Double Chance) - Simplified
|
||||
elif market_type == "DC":
|
||||
if "1X" in pick_str and (home_score >= away_score): is_won = True
|
||||
elif "X2" in pick_str and (away_score >= home_score): is_won = True
|
||||
elif "12" in pick_str and (home_score != away_score): is_won = True
|
||||
|
||||
if is_won:
|
||||
winning_bets += 1
|
||||
profit = main_pick_odds - 1.0
|
||||
total_profit += profit
|
||||
else:
|
||||
total_profit -= 1.0
|
||||
|
||||
# ─── REPORT ───
|
||||
print("\n" + "="*60)
|
||||
print("📈 BACKTEST RESULTS (With NEW Skip Logic)")
|
||||
print("="*60)
|
||||
print(f"Total Historical Matches Analyzed: {len(predictions)}")
|
||||
print(f"🚫 Bets SKIPPED (Low Conf/Bad Value): {skipped_bets}")
|
||||
print(f"✅ Bets PLAYED: {total_bets}")
|
||||
|
||||
if total_bets > 0:
|
||||
win_rate = (winning_bets / total_bets) * 100
|
||||
roi = (total_profit / total_bets) * 100
|
||||
|
||||
print(f"🏆 Winning Bets: {winning_bets}")
|
||||
print(f"💀 Losing Bets: {total_bets - winning_bets}")
|
||||
print("-" * 40)
|
||||
print(f" Win Rate: {win_rate:.2f}%")
|
||||
print(f"💰 Total Profit (Units): {total_profit:.2f}")
|
||||
print(f"📊 ROI: {roi:.2f}%")
|
||||
|
||||
if roi > 0:
|
||||
print("🟢 STRATEGY IS PROFITABLE!")
|
||||
else:
|
||||
print("🔴 STRATEGY IS LOSING (Adjust thresholds!)")
|
||||
else:
|
||||
print("⚠️ No bets were played. Thresholds might be too high.")
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
simulate_backtest()
|
||||
Reference in New Issue
Block a user