/** * =================================================== * BACKTEST ACCURACY — V30 Prediction System * =================================================== * Tests historical predictions against actual outcomes. * Uses the running AI Engine's /v20plus/analyze/{match_id} * endpoint which extracts features from DB internally. * * Usage: npx ts-node --transpile-only -r tsconfig-paths/register src/scripts/backtest-accuracy.ts */ import { PrismaClient } from "@prisma/client"; import axios from "axios"; const prisma = new PrismaClient(); // ═══════════════════════════════════════════════════════ // Configuration // ═══════════════════════════════════════════════════════ const AI_ENGINE_URL = process.env.AI_ENGINE_URL || "http://127.0.0.1:3005"; const CONCURRENT_REQUESTS = 5; const MAX_MATCHES = 1000; // ═══════════════════════════════════════════════════════ // Types // ═══════════════════════════════════════════════════════ interface TestMatch { id: string; scoreHome: number; scoreAway: number; htScoreHome: number | null; htScoreAway: number | null; } interface BacktestResult { matchId: string; actual: { ms: string; ou25: string; btts: string; htft: string }; predicted: { ms: string; ou25: string; btts: string }; probabilities: { home: number; draw: number; away: number; over: number; under: number; bttsYes: number; bttsNo: number; }; mainPickCorrect: boolean; } // ═══════════════════════════════════════════════════════ // Helpers // ═══════════════════════════════════════════════════════ function determineActualOutcome( scoreHome: number, scoreAway: number, htScoreHome: number | null, htScoreAway: number | null, ): { ms: string; ou25: string; btts: string; htft: string } { const ms = scoreHome > scoreAway ? "1" : scoreHome < scoreAway ? "2" : "X"; const ou25 = scoreHome + scoreAway > 2.5 ? "Over" : "Under"; const btts = scoreHome > 0 && scoreAway > 0 ? "Yes" : "No"; let htft = "unknown"; if (htScoreHome !== null && htScoreAway !== null) { const htResult = htScoreHome > htScoreAway ? "1" : htScoreHome < htScoreAway ? "2" : "X"; htft = `${htResult}/${ms}`; } return { ms, ou25, btts, htft }; } function extractPrediction(response: unknown): { ms: string; ou25: string; btts: string; probs: BacktestResult["probabilities"]; mainPick: string; mainMarket: string; } { const data = response as Record; const predictions = data?.predictions as Record | undefined; const mainPickObj = data?.main_pick as Record | undefined; const mainPick = typeof mainPickObj?.pick === "string" ? mainPickObj.pick : ""; const mainMarket = typeof mainPickObj?.market === "string" ? mainPickObj.market : ""; // Extract MS from probabilities or main pick const msProbs = (predictions?.ms || data?.ms || {}) as Record< string, unknown >; const homeProb = typeof msProbs["1"] === "number" ? msProbs["1"] : typeof msProbs.home_prob === "number" ? msProbs.home_prob : 0; const drawProb = typeof msProbs["X"] === "number" ? msProbs["X"] : typeof msProbs.draw_prob === "number" ? msProbs.draw_prob : 0; const awayProb = typeof msProbs["2"] === "number" ? msProbs["2"] : typeof msProbs.away_prob === "number" ? msProbs.away_prob : 0; let ms = "1"; if (drawProb > homeProb && drawProb > awayProb) ms = "X"; else if (awayProb > homeProb) ms = "2"; // Extract OU25 const ou25Probs = (predictions?.ou25 || data?.ou25 || {}) as Record< string, unknown >; const overProb = typeof ou25Probs.Over === "number" ? ou25Probs.Over : typeof ou25Probs.over_prob === "number" ? ou25Probs.over_prob : 0; const underProb = typeof ou25Probs.Under === "number" ? ou25Probs.Under : typeof ou25Probs.under_prob === "number" ? ou25Probs.under_prob : 0; const ou25 = overProb > underProb ? "Over" : "Under"; // Extract BTTS const bttsProbs = (predictions?.btts || data?.btts || {}) as Record< string, unknown >; const bttsYes = typeof bttsProbs.Yes === "number" ? bttsProbs.Yes : typeof bttsProbs.yes_prob === "number" ? bttsProbs.yes_prob : 0; const bttsNo = typeof bttsProbs.No === "number" ? bttsProbs.No : typeof bttsProbs.no_prob === "number" ? bttsProbs.no_prob : 0; const btts = bttsYes > bttsNo ? "Yes" : "No"; return { ms, ou25, btts, probs: { home: homeProb, draw: drawProb, away: awayProb, over: overProb, under: underProb, bttsYes, bttsNo, }, mainPick, mainMarket, }; } async function processBatch(batch: TestMatch[]): Promise { const results: BacktestResult[] = []; const promises = batch.map(async (match) => { try { const response = await axios.post( `${AI_ENGINE_URL}/v20plus/analyze/${match.id}`, {}, { timeout: 15000 }, ); const actual = determineActualOutcome( match.scoreHome, match.scoreAway, match.htScoreHome, match.htScoreAway, ); const pred = extractPrediction(response.data); // Check main pick let mainPickCorrect = false; if (pred.mainMarket === "MS") { mainPickCorrect = pred.mainPick === actual.ms; } else if (pred.mainMarket === "OU25") { mainPickCorrect = pred.mainPick === actual.ou25; } else if (pred.mainMarket === "BTTS") { mainPickCorrect = pred.mainPick === actual.btts; } results.push({ matchId: match.id, actual, predicted: { ms: pred.ms, ou25: pred.ou25, btts: pred.btts }, probabilities: pred.probs, mainPickCorrect, }); } catch { // Skip failed matches silently } }); await Promise.all(promises); return results; } // ═══════════════════════════════════════════════════════ // Main Backtest // ═══════════════════════════════════════════════════════ async function runBacktest(): Promise { console.log("🎯 BACKTEST ACCURACY — V30 Betting Engine"); console.log("════════════════════════════════════════════════════════"); // 1. Health check try { const health = await axios.get(`${AI_ENGINE_URL}/health`, { timeout: 5000, }); console.log(`✅ AI Engine: ${JSON.stringify(health.data)}`); } catch { console.error("❌ AI Engine not reachable at", AI_ENGINE_URL); process.exit(1); } // 2. Load finished matches with features console.log("\n📥 Loading test matches..."); const matches = await prisma.$queryRaw` SELECT m.id, m.score_home AS "scoreHome", m.score_away AS "scoreAway", m.ht_score_home AS "htScoreHome", m.ht_score_away AS "htScoreAway" FROM matches m JOIN match_ai_features maf ON maf.match_id = m.id WHERE m.status = 'FT' AND m.score_home IS NOT NULL AND m.score_away IS NOT NULL AND m.sport = 'football' AND maf.home_elo != 1500 AND maf.implied_home != 0.33 ORDER BY m.mst_utc DESC LIMIT ${MAX_MATCHES} `; console.log(` 📊 Test matches: ${matches.length}`); // 3. Run predictions in batches console.log("\n🤖 Running predictions..."); const allResults: BacktestResult[] = []; let processed = 0; for (let i = 0; i < matches.length; i += CONCURRENT_REQUESTS) { const batch = matches.slice(i, i + CONCURRENT_REQUESTS); const batchResults = await processBatch(batch); allResults.push(...batchResults); processed += batch.length; if (processed % 50 === 0 || processed === matches.length) { const currentMsAcc = allResults.length > 0 ? ( (allResults.filter((r) => r.predicted.ms === r.actual.ms).length / allResults.length) * 100 ).toFixed(1) : "0"; console.log( ` 📊 ${processed}/${matches.length} — Success: ${allResults.length} — MS Acc: ${currentMsAcc}%`, ); } } // 4. Calculate metrics const total = allResults.length; if (total === 0) { console.error("❌ No results to analyze"); process.exit(1); } const msCorrect = allResults.filter( (r) => r.predicted.ms === r.actual.ms, ).length; const ou25Correct = allResults.filter( (r) => r.predicted.ou25 === r.actual.ou25, ).length; const bttsCorrect = allResults.filter( (r) => r.predicted.btts === r.actual.btts, ).length; const mainPickCorrect = allResults.filter((r) => r.mainPickCorrect).length; // Actual distribution const actHome = allResults.filter((r) => r.actual.ms === "1").length; const actDraw = allResults.filter((r) => r.actual.ms === "X").length; const actAway = allResults.filter((r) => r.actual.ms === "2").length; // Predicted distribution const predHome = allResults.filter((r) => r.predicted.ms === "1").length; const predDraw = allResults.filter((r) => r.predicted.ms === "X").length; const predAway = allResults.filter((r) => r.predicted.ms === "2").length; // Confidence calibration (based on max probability) const buckets: Record = { "33-40%": { correct: 0, total: 0 }, "40-50%": { correct: 0, total: 0 }, "50-60%": { correct: 0, total: 0 }, "60-70%": { correct: 0, total: 0 }, "70%+": { correct: 0, total: 0 }, }; for (const r of allResults) { const maxProb = Math.max( r.probabilities.home, r.probabilities.draw, r.probabilities.away, ); const key = maxProb >= 0.7 ? "70%+" : maxProb >= 0.6 ? "60-70%" : maxProb >= 0.5 ? "50-60%" : maxProb >= 0.4 ? "40-50%" : "33-40%"; buckets[key].total++; if (r.predicted.ms === r.actual.ms) buckets[key].correct++; } // 5. Print Report console.log("\n════════════════════════════════════════════════════════"); console.log("📊 BACKTEST ACCURACY REPORT"); console.log("════════════════════════════════════════════════════════"); console.log(` Total Matches Analyzed: ${total}`); console.log(""); console.log(" 🎯 Market Accuracy:"); console.log( ` ⚽ Match Result (MS): ${((msCorrect / total) * 100).toFixed(2)}% (${msCorrect}/${total})`, ); console.log( ` 📈 Over/Under 2.5: ${((ou25Correct / total) * 100).toFixed(2)}% (${ou25Correct}/${total})`, ); console.log( ` 🤝 Both Teams Score: ${((bttsCorrect / total) * 100).toFixed(2)}% (${bttsCorrect}/${total})`, ); console.log( ` 🏆 Main Pick Success: ${((mainPickCorrect / total) * 100).toFixed(2)}% (${mainPickCorrect}/${total})`, ); console.log("\n 📊 MS Distribution:"); console.log( ` Actual: 1: ${actHome} (${((actHome / total) * 100).toFixed(1)}%) | X: ${actDraw} (${((actDraw / total) * 100).toFixed(1)}%) | 2: ${actAway} (${((actAway / total) * 100).toFixed(1)}%)`, ); console.log( ` Predicted: 1: ${predHome} (${((predHome / total) * 100).toFixed(1)}%) | X: ${predDraw} (${((predDraw / total) * 100).toFixed(1)}%) | 2: ${predAway} (${((predAway / total) * 100).toFixed(1)}%)`, ); console.log("\n 📊 Confidence Calibration:"); for (const [range, bucket] of Object.entries(buckets)) { if (bucket.total === 0) continue; const acc = (bucket.correct / bucket.total) * 100; const bar = "█".repeat(Math.round(acc / 3)); console.log( ` ${range.padEnd(8)} : ${acc.toFixed(1)}% acc (n=${bucket.total}) ${bar}`, ); } // 6. Per-market deep dive console.log("\n 📊 OU25 Breakdown:"); const actOver = allResults.filter((r) => r.actual.ou25 === "Over").length; const actUnder = total - actOver; const predOver = allResults.filter((r) => r.predicted.ou25 === "Over").length; const predUnder = total - predOver; console.log( ` Actual: Over: ${actOver} (${((actOver / total) * 100).toFixed(1)}%) | Under: ${actUnder} (${((actUnder / total) * 100).toFixed(1)}%)`, ); console.log( ` Predicted: Over: ${predOver} (${((predOver / total) * 100).toFixed(1)}%) | Under: ${predUnder} (${((predUnder / total) * 100).toFixed(1)}%)`, ); console.log("\n 📊 BTTS Breakdown:"); const actBttsYes = allResults.filter((r) => r.actual.btts === "Yes").length; const actBttsNo = total - actBttsYes; const predBttsYes = allResults.filter( (r) => r.predicted.btts === "Yes", ).length; const predBttsNo = total - predBttsYes; console.log( ` Actual: Yes: ${actBttsYes} (${((actBttsYes / total) * 100).toFixed(1)}%) | No: ${actBttsNo} (${((actBttsNo / total) * 100).toFixed(1)}%)`, ); console.log( ` Predicted: Yes: ${predBttsYes} (${((predBttsYes / total) * 100).toFixed(1)}%) | No: ${predBttsNo} (${((predBttsNo / total) * 100).toFixed(1)}%)`, ); console.log("════════════════════════════════════════════════════════"); console.log("✅ Backtest complete!"); await prisma.$disconnect(); } runBacktest().catch((err: unknown) => { console.error("❌ Backtest failed:", err); void prisma.$disconnect(); process.exit(1); });