This commit is contained in:
@@ -0,0 +1,428 @@
|
||||
"""
|
||||
XGBoost Training Data Extraction (Basketball)
|
||||
==============================================
|
||||
Batch feature extraction for top-league basketball matches.
|
||||
Extracts features + labels per match for XGBoost model training.
|
||||
|
||||
Usage:
|
||||
python3 scripts/extract_basketball_data.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import csv
|
||||
import math
|
||||
import time
|
||||
from datetime import datetime
|
||||
from collections import defaultdict
|
||||
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# =============================================================================
|
||||
# CONFIG
|
||||
# =============================================================================
|
||||
AI_ENGINE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, AI_ENGINE_DIR)
|
||||
|
||||
TOP_LEAGUES_PATH = os.path.join(AI_ENGINE_DIR, "..", "basketball_top_leagues.json")
|
||||
OUTPUT_CSV = os.path.join(AI_ENGINE_DIR, "data", "basketball_training_data.csv")
|
||||
|
||||
os.makedirs(os.path.dirname(OUTPUT_CSV), exist_ok=True)
|
||||
|
||||
|
||||
def get_conn():
|
||||
db_url = os.getenv("DATABASE_URL", "").split("?schema=")[0]
|
||||
return psycopg2.connect(db_url)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# FEATURE COLUMNS (ORDER MATTERS — matches CSV header)
|
||||
# =============================================================================
|
||||
FEATURE_COLS = [
|
||||
# Match identifiers
|
||||
"match_id", "home_team_id", "away_team_id", "league_id", "mst_utc",
|
||||
|
||||
# Form Features (8)
|
||||
"home_points_avg", "home_conceded_avg",
|
||||
"away_points_avg", "away_conceded_avg",
|
||||
"home_winning_streak", "away_winning_streak",
|
||||
"home_win_rate", "away_win_rate",
|
||||
|
||||
# H2H Features (4)
|
||||
"h2h_total_matches", "h2h_home_win_rate",
|
||||
"h2h_avg_points", "h2h_over140_rate",
|
||||
|
||||
# Odds Features (6)
|
||||
"odds_ml_h", "odds_ml_a",
|
||||
"odds_tot_o", "odds_tot_u", "odds_tot_line",
|
||||
"odds_spread_h", "odds_spread_a", "odds_spread_line",
|
||||
|
||||
# Labels
|
||||
"score_home", "score_away", "total_points",
|
||||
"label_ml", # 0=Home, 1=Away
|
||||
"label_tot", # 0=Under, 1=Over (dynamic line)
|
||||
"label_spread", # 0=Away Cover, 1=Home Cover (dynamic line)
|
||||
]
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# BATCH LOADERS — Pre-load data to avoid N+1 queries
|
||||
# =============================================================================
|
||||
|
||||
class BatchDataLoader:
|
||||
"""Pre-loads all necessary data in bulk, then serves features per match."""
|
||||
|
||||
def __init__(self, conn, top_league_ids: list):
|
||||
self.conn = conn
|
||||
self.cur = conn.cursor(cursor_factory=RealDictCursor)
|
||||
self.top_league_ids = top_league_ids
|
||||
|
||||
# Pre-loaded data caches
|
||||
self.matches = []
|
||||
self.odds_cache = {} # match_id → {ml_h, ml_a, ...}
|
||||
self.form_cache = {} # (team_id, match_id) → form features
|
||||
self.h2h_cache = {} # (home_id, away_id, match_id) → h2h features
|
||||
|
||||
def load_all(self):
|
||||
"""Load all data in batch."""
|
||||
t0 = time.time()
|
||||
|
||||
self._load_matches()
|
||||
print(f" ✅ Matches: {len(self.matches)} ({time.time()-t0:.1f}s)", flush=True)
|
||||
|
||||
t1 = time.time()
|
||||
self._load_odds()
|
||||
print(f" ✅ Odds: {len(self.odds_cache)} matches ({time.time()-t1:.1f}s)", flush=True)
|
||||
|
||||
t3 = time.time()
|
||||
self._load_team_history()
|
||||
print(f" ✅ Team History & Stats cache built ({time.time()-t3:.1f}s)", flush=True)
|
||||
|
||||
print(f" 📊 Total load time: {time.time()-t0:.1f}s", flush=True)
|
||||
|
||||
def _load_matches(self):
|
||||
query = """
|
||||
SELECT
|
||||
id,
|
||||
mst_utc,
|
||||
league_id,
|
||||
home_team_id,
|
||||
away_team_id,
|
||||
score_home,
|
||||
score_away,
|
||||
status
|
||||
FROM matches
|
||||
WHERE sport = 'basketball'
|
||||
AND status = 'FT'
|
||||
AND score_home IS NOT NULL
|
||||
AND score_away IS NOT NULL
|
||||
AND mst_utc > 1640995200000 -- Since Jan 1, 2022
|
||||
"""
|
||||
if self.top_league_ids:
|
||||
format_strings = ",".join(["%s"] * len(self.top_league_ids))
|
||||
query += f" AND league_id IN ({format_strings})"
|
||||
self.cur.execute(query + " ORDER BY mst_utc ASC", tuple(self.top_league_ids))
|
||||
else:
|
||||
self.cur.execute(query + " ORDER BY mst_utc ASC")
|
||||
|
||||
self.matches = self.cur.fetchall()
|
||||
|
||||
def _load_odds(self):
|
||||
query = """
|
||||
SELECT match_id, name as category_name, db_id as category_id
|
||||
FROM odd_categories
|
||||
WHERE match_id IN (
|
||||
SELECT id FROM matches WHERE sport = 'basketball' AND status = 'FT'
|
||||
)
|
||||
"""
|
||||
self.cur.execute(query)
|
||||
cats = self.cur.fetchall()
|
||||
|
||||
# map cat -> match
|
||||
cat_to_match = {c['category_id']: c['match_id'] for c in cats}
|
||||
|
||||
query2 = """
|
||||
SELECT odd_category_db_id, name, odd_value
|
||||
FROM odd_selections
|
||||
WHERE odd_category_db_id IN %(cat_ids)s
|
||||
"""
|
||||
cat_ids = tuple(cat_to_match.keys())
|
||||
if not cat_ids:
|
||||
return
|
||||
|
||||
cat_id_to_name = {c['category_id']: c['category_name'] for c in cats}
|
||||
|
||||
chunk_size = 50000
|
||||
cats_list = list(cat_ids)
|
||||
total_chunks = len(cats_list) // chunk_size + 1
|
||||
print(f" Fetching {len(cats_list)} categories in {total_chunks} chunks...", flush=True)
|
||||
|
||||
for idx, i in enumerate(range(0, len(cats_list), chunk_size)):
|
||||
chunk = tuple(cats_list[i:i+chunk_size])
|
||||
self.cur.execute("SELECT odd_category_db_id, name, odd_value FROM odd_selections WHERE odd_category_db_id IN %s", (chunk,))
|
||||
rows = self.cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
c_id = row['odd_category_db_id']
|
||||
m_id = cat_to_match[c_id]
|
||||
c_name = cat_id_to_name.get(c_id, "")
|
||||
|
||||
if m_id not in self.odds_cache:
|
||||
self.odds_cache[m_id] = {}
|
||||
|
||||
self._parse_single_odd(m_id, c_name, str(row['name']), float(row['odd_value']))
|
||||
print(f" Processed chunk {idx+1}/{total_chunks} ({len(rows)} selections).", flush=True)
|
||||
|
||||
def _parse_single_odd(self, match_id, category_name, sel_name, odd_value):
|
||||
if odd_value <= 1.0: return
|
||||
cat_lower = category_name.lower()
|
||||
sel_lower = sel_name.lower()
|
||||
|
||||
target = self.odds_cache[match_id]
|
||||
|
||||
# ML
|
||||
if cat_lower in ("maç sonucu (uzt. dahil)", "mac sonucu (uzt. dahil)", "maç sonucu", "mac sonucu"):
|
||||
if sel_lower == "1": target["ml_h"] = odd_value
|
||||
elif sel_lower == "2": target["ml_a"] = odd_value
|
||||
|
||||
# Totals
|
||||
if "alt/üst" in cat_lower or "alt/ust" in cat_lower:
|
||||
# Extract line
|
||||
line = None
|
||||
try:
|
||||
left = cat_lower.find("(")
|
||||
right = cat_lower.find(")", left + 1)
|
||||
if left > -1 and right > -1:
|
||||
line = float(cat_lower[left+1:right].replace(",", "."))
|
||||
except: pass
|
||||
|
||||
if line and "tot_line" not in target:
|
||||
target["tot_line"] = line
|
||||
|
||||
if "üst" in sel_lower or "ust" in sel_lower or "over" in sel_lower:
|
||||
target.setdefault("tot_o", odd_value)
|
||||
elif "alt" in sel_lower or "under" in sel_lower:
|
||||
target.setdefault("tot_u", odd_value)
|
||||
|
||||
# Spread
|
||||
if "hnd. ms" in cat_lower or "hand. ms" in cat_lower or "hnd ms" in cat_lower:
|
||||
line = None
|
||||
try:
|
||||
left = cat_lower.find("(")
|
||||
right = cat_lower.find(")", left + 1)
|
||||
if left > -1 and right > -1:
|
||||
payload = cat_lower[left+1:right].replace(",", ".")
|
||||
if ":" in payload:
|
||||
home_hcp = float(payload.split(":")[0])
|
||||
away_hcp = float(payload.split(":")[1])
|
||||
if abs(home_hcp) < 1e-6 and away_hcp > 0: line = -away_hcp
|
||||
elif home_hcp > 0 and abs(away_hcp) < 1e-6: line = home_hcp
|
||||
elif abs(home_hcp - away_hcp) < 1e-6 and home_hcp > 0: line = 0.0
|
||||
except: pass
|
||||
|
||||
if line is not None and "spread_line" not in target:
|
||||
target["spread_line"] = line
|
||||
|
||||
if sel_lower == "1": target.setdefault("spread_h", odd_value)
|
||||
elif sel_lower == "2": target.setdefault("spread_a", odd_value)
|
||||
|
||||
|
||||
def _load_team_history(self):
|
||||
# We need historical form (avg points scored/conceded, win rate).
|
||||
team_matches = defaultdict(list)
|
||||
for m in self.matches:
|
||||
# m has id, mst_utc, home_team_id, away_team_id, score_home, score_away
|
||||
team_matches[m['home_team_id']].append((m['mst_utc'], m['score_home'], m['score_away'], 'H'))
|
||||
team_matches[m['away_team_id']].append((m['mst_utc'], m['score_away'], m['score_home'], 'A'))
|
||||
|
||||
for team_id, hist in team_matches.items():
|
||||
hist.sort(key=lambda x: x[0]) # Sort by time
|
||||
|
||||
for i, (mst_utc, scored, conceded, location) in enumerate(hist):
|
||||
# Filter past matches
|
||||
past = [x for x in hist[:i] if x[0] < mst_utc]
|
||||
if not past:
|
||||
self.form_cache[(team_id, mst_utc)] = {
|
||||
"points_avg": 80.0,
|
||||
"conceded_avg": 80.0,
|
||||
"winning_streak": 0,
|
||||
"win_rate": 0.5
|
||||
}
|
||||
continue
|
||||
|
||||
last_5 = past[-5:]
|
||||
|
||||
pts = sum(x[1] for x in last_5) / len(last_5)
|
||||
conc = sum(x[2] for x in last_5) / len(last_5)
|
||||
|
||||
wins = sum(1 for x in past if x[1] > x[2])
|
||||
win_rate = wins / len(past) if len(past) > 0 else 0.5
|
||||
|
||||
streak = 0
|
||||
for x in reversed(past):
|
||||
if x[1] > x[2]: streak += 1
|
||||
else: break
|
||||
|
||||
self.form_cache[(team_id, mst_utc)] = {
|
||||
"points_avg": pts,
|
||||
"conceded_avg": conc,
|
||||
"winning_streak": streak,
|
||||
"win_rate": win_rate
|
||||
}
|
||||
|
||||
# Build H2H
|
||||
h2h_map = defaultdict(list)
|
||||
for m in self.matches:
|
||||
pair = tuple(sorted([str(m['home_team_id']), str(m['away_team_id'])]))
|
||||
tgt = m['home_team_id']
|
||||
h_win = 1 if m['score_home'] > m['score_away'] else 0
|
||||
if tgt != pair[0]: # Ensure orientation is relative to pair[0] usually, but let's just do directional
|
||||
pass
|
||||
directional_pair = (str(m['home_team_id']), str(m['away_team_id']))
|
||||
h2h_map[directional_pair].append((m['mst_utc'], m['score_home'], m['score_away']))
|
||||
|
||||
for (h_id, a_id), hist in h2h_map.items():
|
||||
hist.sort(key=lambda x: x[0])
|
||||
for i, (mst_utc, sh, sa) in enumerate(hist):
|
||||
past = [x for x in hist[:i] if x[0] < mst_utc]
|
||||
|
||||
if not past:
|
||||
self.h2h_cache[(h_id, a_id, mst_utc)] = {
|
||||
"total": 0, "home_win_rate": 0.5,
|
||||
"avg_points": 160.0, "over140_rate": 0.5
|
||||
}
|
||||
else:
|
||||
home_wins = sum(1 for x in past if x[1] > x[2])
|
||||
total_pts = sum(x[1] + x[2] for x in past)
|
||||
over140 = sum(1 for x in past if x[1] + x[2] > 140)
|
||||
|
||||
self.h2h_cache[(h_id, a_id, mst_utc)] = {
|
||||
"total": len(past),
|
||||
"home_win_rate": home_wins / len(past),
|
||||
"avg_points": total_pts / len(past),
|
||||
"over140_rate": over140 / len(past)
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# FEATURE EXTRACTION PIPELINE
|
||||
# =============================================================================
|
||||
|
||||
def process_matches(loader: BatchDataLoader):
|
||||
"""Processes loaded matches, maps to features, handles implicit fallbacks, saves to CSV."""
|
||||
f = open(OUTPUT_CSV, "w", newline='')
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(FEATURE_COLS)
|
||||
|
||||
extracted_count = 0
|
||||
missing_odds_count = 0
|
||||
|
||||
for match in loader.matches:
|
||||
mid = str(match['id'])
|
||||
mst = int(match['mst_utc'])
|
||||
hid = str(match['home_team_id'])
|
||||
aid = str(match['away_team_id'])
|
||||
|
||||
# True Results
|
||||
s_home = int(match['score_home'])
|
||||
s_away = int(match['score_away'])
|
||||
total_pts = s_home + s_away
|
||||
|
||||
c_odds = loader.odds_cache.get(mid, {})
|
||||
c_form_h = loader.form_cache.get((hid, mst), {})
|
||||
c_form_a = loader.form_cache.get((aid, mst), {})
|
||||
c_h2h = loader.h2h_cache.get((hid, aid, mst), {})
|
||||
|
||||
# Basic validation: ensure we have at least ML odds
|
||||
if "ml_h" not in c_odds or "ml_a" not in c_odds:
|
||||
missing_odds_count += 1
|
||||
continue
|
||||
|
||||
# Target Variables (Labels)
|
||||
label_ml = 0 if s_home > s_away else 1 # Home Win vs Away Win
|
||||
|
||||
# Totals label (evaluate against dynamic line)
|
||||
line_tot = c_odds.get("tot_line", 160.0)
|
||||
label_tot = 1 if total_pts > line_tot else 0 # Over = 1, Under = 0
|
||||
|
||||
# Spread label (evaluate against dynamic line)
|
||||
# Home Spread Coverage. Example: line= -5.5. s_home + line = s_home - 5.5.
|
||||
line_spread = c_odds.get("spread_line", 0.0)
|
||||
hc_score = float(s_home) + float(line_spread)
|
||||
label_spread = 1 if hc_score > float(s_away) else 0 # Spread Coverage: 1=Home, 0=Away
|
||||
|
||||
# Compile Row
|
||||
row = [
|
||||
# Identifiers
|
||||
mid, hid, aid, match.get('league_id', ''), mst,
|
||||
|
||||
# Form cache
|
||||
c_form_h.get("points_avg", 80), c_form_h.get("conceded_avg", 80),
|
||||
c_form_a.get("points_avg", 80), c_form_a.get("conceded_avg", 80),
|
||||
c_form_h.get("winning_streak", 0), c_form_a.get("winning_streak", 0),
|
||||
c_form_h.get("win_rate", 0), c_form_a.get("win_rate", 0),
|
||||
|
||||
# H2H cache
|
||||
c_h2h.get("total", 0), c_h2h.get("home_win_rate", 0.5),
|
||||
c_h2h.get("avg_points", 160.0), c_h2h.get("over140_rate", 0.5),
|
||||
|
||||
# Odds
|
||||
c_odds.get("ml_h", 1.9), c_odds.get("ml_a", 1.9),
|
||||
c_odds.get("tot_o", 1.9), c_odds.get("tot_u", 1.9), line_tot,
|
||||
c_odds.get("spread_h", 1.9), c_odds.get("spread_a", 1.9), line_spread,
|
||||
|
||||
# Labels
|
||||
s_home, s_away, total_pts,
|
||||
label_ml,
|
||||
label_tot,
|
||||
label_spread,
|
||||
]
|
||||
|
||||
# Safeguard length
|
||||
if len(row) != len(FEATURE_COLS):
|
||||
print(f"Error: Row length mismatch {len(row)} != {len(FEATURE_COLS)}")
|
||||
sys.exit(1)
|
||||
|
||||
writer.writerow(row)
|
||||
extracted_count += 1
|
||||
|
||||
f.close()
|
||||
|
||||
print("\nExtraction Summary")
|
||||
print("=========================")
|
||||
print(f"Total Matches in Scope: {len(loader.matches)}")
|
||||
print(f"Filtered (Missing ML Odds): {missing_odds_count}")
|
||||
print(f"✅ Successfully Extracted: {extracted_count}")
|
||||
print(f"📂 Saved to: {OUTPUT_CSV}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
t_start = time.time()
|
||||
|
||||
# Load leagues
|
||||
if not os.path.exists(TOP_LEAGUES_PATH):
|
||||
print(f"Error: file not found {TOP_LEAGUES_PATH}")
|
||||
sys.exit(1)
|
||||
|
||||
with open(TOP_LEAGUES_PATH, "r") as f:
|
||||
top_leagues = json.load(f)
|
||||
|
||||
print(f"🏀 Extracting Basketball Training Data (XGBoost)")
|
||||
print(f"==================================================")
|
||||
print(f"Loaded {len(top_leagues)} top leagues.")
|
||||
|
||||
conn = get_conn()
|
||||
loader = BatchDataLoader(conn, top_leagues)
|
||||
|
||||
# 1. Pre-load everything into memory
|
||||
loader.load_all()
|
||||
|
||||
# 2. Extract and match features, then write CSV
|
||||
process_matches(loader)
|
||||
|
||||
conn.close()
|
||||
print(f"Total Script Run Time: {time.time()-t_start:.1f}s")
|
||||
Reference in New Issue
Block a user