420 lines
14 KiB
TypeScript
420 lines
14 KiB
TypeScript
/**
|
|
* ===================================================
|
|
* BACKTEST ACCURACY — V30 Prediction System
|
|
* ===================================================
|
|
* Tests historical predictions against actual outcomes.
|
|
* Uses the running AI Engine's /v20plus/analyze/{match_id}
|
|
* endpoint which extracts features from DB internally.
|
|
*
|
|
* Usage: npx ts-node --transpile-only -r tsconfig-paths/register src/scripts/backtest-accuracy.ts
|
|
*/
|
|
|
|
import { PrismaClient } from "@prisma/client";
|
|
import axios from "axios";
|
|
|
|
const prisma = new PrismaClient();
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Configuration
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
const AI_ENGINE_URL = process.env.AI_ENGINE_URL || "http://127.0.0.1:3005";
|
|
const CONCURRENT_REQUESTS = 5;
|
|
const MAX_MATCHES = 1000;
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Types
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
interface TestMatch {
|
|
id: string;
|
|
scoreHome: number;
|
|
scoreAway: number;
|
|
htScoreHome: number | null;
|
|
htScoreAway: number | null;
|
|
}
|
|
|
|
interface BacktestResult {
|
|
matchId: string;
|
|
actual: { ms: string; ou25: string; btts: string; htft: string };
|
|
predicted: { ms: string; ou25: string; btts: string };
|
|
probabilities: {
|
|
home: number;
|
|
draw: number;
|
|
away: number;
|
|
over: number;
|
|
under: number;
|
|
bttsYes: number;
|
|
bttsNo: number;
|
|
};
|
|
mainPickCorrect: boolean;
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Helpers
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
function determineActualOutcome(
|
|
scoreHome: number,
|
|
scoreAway: number,
|
|
htScoreHome: number | null,
|
|
htScoreAway: number | null,
|
|
): { ms: string; ou25: string; btts: string; htft: string } {
|
|
const ms = scoreHome > scoreAway ? "1" : scoreHome < scoreAway ? "2" : "X";
|
|
const ou25 = scoreHome + scoreAway > 2.5 ? "Over" : "Under";
|
|
const btts = scoreHome > 0 && scoreAway > 0 ? "Yes" : "No";
|
|
|
|
let htft = "unknown";
|
|
if (htScoreHome !== null && htScoreAway !== null) {
|
|
const htResult =
|
|
htScoreHome > htScoreAway ? "1" : htScoreHome < htScoreAway ? "2" : "X";
|
|
htft = `${htResult}/${ms}`;
|
|
}
|
|
|
|
return { ms, ou25, btts, htft };
|
|
}
|
|
|
|
function extractPrediction(response: unknown): {
|
|
ms: string;
|
|
ou25: string;
|
|
btts: string;
|
|
probs: BacktestResult["probabilities"];
|
|
mainPick: string;
|
|
mainMarket: string;
|
|
} {
|
|
const data = response as Record<string, unknown>;
|
|
const predictions = data?.predictions as Record<string, unknown> | undefined;
|
|
|
|
const mainPickObj = data?.main_pick as Record<string, unknown> | undefined;
|
|
const mainPick =
|
|
typeof mainPickObj?.pick === "string" ? mainPickObj.pick : "";
|
|
const mainMarket =
|
|
typeof mainPickObj?.market === "string" ? mainPickObj.market : "";
|
|
|
|
// Extract MS from probabilities or main pick
|
|
const msProbs = (predictions?.ms || data?.ms || {}) as Record<
|
|
string,
|
|
unknown
|
|
>;
|
|
const homeProb =
|
|
typeof msProbs["1"] === "number"
|
|
? msProbs["1"]
|
|
: typeof msProbs.home_prob === "number"
|
|
? msProbs.home_prob
|
|
: 0;
|
|
const drawProb =
|
|
typeof msProbs["X"] === "number"
|
|
? msProbs["X"]
|
|
: typeof msProbs.draw_prob === "number"
|
|
? msProbs.draw_prob
|
|
: 0;
|
|
const awayProb =
|
|
typeof msProbs["2"] === "number"
|
|
? msProbs["2"]
|
|
: typeof msProbs.away_prob === "number"
|
|
? msProbs.away_prob
|
|
: 0;
|
|
|
|
let ms = "1";
|
|
if (drawProb > homeProb && drawProb > awayProb) ms = "X";
|
|
else if (awayProb > homeProb) ms = "2";
|
|
|
|
// Extract OU25
|
|
const ou25Probs = (predictions?.ou25 || data?.ou25 || {}) as Record<
|
|
string,
|
|
unknown
|
|
>;
|
|
const overProb =
|
|
typeof ou25Probs.Over === "number"
|
|
? ou25Probs.Over
|
|
: typeof ou25Probs.over_prob === "number"
|
|
? ou25Probs.over_prob
|
|
: 0;
|
|
const underProb =
|
|
typeof ou25Probs.Under === "number"
|
|
? ou25Probs.Under
|
|
: typeof ou25Probs.under_prob === "number"
|
|
? ou25Probs.under_prob
|
|
: 0;
|
|
const ou25 = overProb > underProb ? "Over" : "Under";
|
|
|
|
// Extract BTTS
|
|
const bttsProbs = (predictions?.btts || data?.btts || {}) as Record<
|
|
string,
|
|
unknown
|
|
>;
|
|
const bttsYes =
|
|
typeof bttsProbs.Yes === "number"
|
|
? bttsProbs.Yes
|
|
: typeof bttsProbs.yes_prob === "number"
|
|
? bttsProbs.yes_prob
|
|
: 0;
|
|
const bttsNo =
|
|
typeof bttsProbs.No === "number"
|
|
? bttsProbs.No
|
|
: typeof bttsProbs.no_prob === "number"
|
|
? bttsProbs.no_prob
|
|
: 0;
|
|
const btts = bttsYes > bttsNo ? "Yes" : "No";
|
|
|
|
return {
|
|
ms,
|
|
ou25,
|
|
btts,
|
|
probs: {
|
|
home: homeProb,
|
|
draw: drawProb,
|
|
away: awayProb,
|
|
over: overProb,
|
|
under: underProb,
|
|
bttsYes,
|
|
bttsNo,
|
|
},
|
|
mainPick,
|
|
mainMarket,
|
|
};
|
|
}
|
|
|
|
async function processBatch(batch: TestMatch[]): Promise<BacktestResult[]> {
|
|
const results: BacktestResult[] = [];
|
|
|
|
const promises = batch.map(async (match) => {
|
|
try {
|
|
const response = await axios.post(
|
|
`${AI_ENGINE_URL}/v20plus/analyze/${match.id}`,
|
|
{},
|
|
{ timeout: 15000 },
|
|
);
|
|
|
|
const actual = determineActualOutcome(
|
|
match.scoreHome,
|
|
match.scoreAway,
|
|
match.htScoreHome,
|
|
match.htScoreAway,
|
|
);
|
|
|
|
const pred = extractPrediction(response.data);
|
|
|
|
// Check main pick
|
|
let mainPickCorrect = false;
|
|
if (pred.mainMarket === "MS") {
|
|
mainPickCorrect = pred.mainPick === actual.ms;
|
|
} else if (pred.mainMarket === "OU25") {
|
|
mainPickCorrect = pred.mainPick === actual.ou25;
|
|
} else if (pred.mainMarket === "BTTS") {
|
|
mainPickCorrect = pred.mainPick === actual.btts;
|
|
}
|
|
|
|
results.push({
|
|
matchId: match.id,
|
|
actual,
|
|
predicted: { ms: pred.ms, ou25: pred.ou25, btts: pred.btts },
|
|
probabilities: pred.probs,
|
|
mainPickCorrect,
|
|
});
|
|
} catch {
|
|
// Skip failed matches silently
|
|
}
|
|
});
|
|
|
|
await Promise.all(promises);
|
|
return results;
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// Main Backtest
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
async function runBacktest(): Promise<void> {
|
|
console.log("🎯 BACKTEST ACCURACY — V30 Betting Engine");
|
|
console.log("════════════════════════════════════════════════════════");
|
|
|
|
// 1. Health check
|
|
try {
|
|
const health = await axios.get(`${AI_ENGINE_URL}/health`, {
|
|
timeout: 5000,
|
|
});
|
|
console.log(`✅ AI Engine: ${JSON.stringify(health.data)}`);
|
|
} catch {
|
|
console.error("❌ AI Engine not reachable at", AI_ENGINE_URL);
|
|
process.exit(1);
|
|
}
|
|
|
|
// 2. Load finished matches with features
|
|
console.log("\n📥 Loading test matches...");
|
|
const matches = await prisma.$queryRaw<TestMatch[]>`
|
|
SELECT m.id, m.score_home AS "scoreHome", m.score_away AS "scoreAway",
|
|
m.ht_score_home AS "htScoreHome", m.ht_score_away AS "htScoreAway"
|
|
FROM matches m
|
|
JOIN match_ai_features maf ON maf.match_id = m.id
|
|
WHERE m.status = 'FT'
|
|
AND m.score_home IS NOT NULL
|
|
AND m.score_away IS NOT NULL
|
|
AND m.sport = 'football'
|
|
AND maf.home_elo != 1500
|
|
AND maf.implied_home != 0.33
|
|
ORDER BY m.mst_utc DESC
|
|
LIMIT ${MAX_MATCHES}
|
|
`;
|
|
console.log(` 📊 Test matches: ${matches.length}`);
|
|
|
|
// 3. Run predictions in batches
|
|
console.log("\n🤖 Running predictions...");
|
|
const allResults: BacktestResult[] = [];
|
|
let processed = 0;
|
|
|
|
for (let i = 0; i < matches.length; i += CONCURRENT_REQUESTS) {
|
|
const batch = matches.slice(i, i + CONCURRENT_REQUESTS);
|
|
const batchResults = await processBatch(batch);
|
|
allResults.push(...batchResults);
|
|
processed += batch.length;
|
|
|
|
if (processed % 50 === 0 || processed === matches.length) {
|
|
const currentMsAcc =
|
|
allResults.length > 0
|
|
? (
|
|
(allResults.filter((r) => r.predicted.ms === r.actual.ms).length /
|
|
allResults.length) *
|
|
100
|
|
).toFixed(1)
|
|
: "0";
|
|
console.log(
|
|
` 📊 ${processed}/${matches.length} — Success: ${allResults.length} — MS Acc: ${currentMsAcc}%`,
|
|
);
|
|
}
|
|
}
|
|
|
|
// 4. Calculate metrics
|
|
const total = allResults.length;
|
|
if (total === 0) {
|
|
console.error("❌ No results to analyze");
|
|
process.exit(1);
|
|
}
|
|
|
|
const msCorrect = allResults.filter(
|
|
(r) => r.predicted.ms === r.actual.ms,
|
|
).length;
|
|
const ou25Correct = allResults.filter(
|
|
(r) => r.predicted.ou25 === r.actual.ou25,
|
|
).length;
|
|
const bttsCorrect = allResults.filter(
|
|
(r) => r.predicted.btts === r.actual.btts,
|
|
).length;
|
|
const mainPickCorrect = allResults.filter((r) => r.mainPickCorrect).length;
|
|
|
|
// Actual distribution
|
|
const actHome = allResults.filter((r) => r.actual.ms === "1").length;
|
|
const actDraw = allResults.filter((r) => r.actual.ms === "X").length;
|
|
const actAway = allResults.filter((r) => r.actual.ms === "2").length;
|
|
|
|
// Predicted distribution
|
|
const predHome = allResults.filter((r) => r.predicted.ms === "1").length;
|
|
const predDraw = allResults.filter((r) => r.predicted.ms === "X").length;
|
|
const predAway = allResults.filter((r) => r.predicted.ms === "2").length;
|
|
|
|
// Confidence calibration (based on max probability)
|
|
const buckets: Record<string, { correct: number; total: number }> = {
|
|
"33-40%": { correct: 0, total: 0 },
|
|
"40-50%": { correct: 0, total: 0 },
|
|
"50-60%": { correct: 0, total: 0 },
|
|
"60-70%": { correct: 0, total: 0 },
|
|
"70%+": { correct: 0, total: 0 },
|
|
};
|
|
|
|
for (const r of allResults) {
|
|
const maxProb = Math.max(
|
|
r.probabilities.home,
|
|
r.probabilities.draw,
|
|
r.probabilities.away,
|
|
);
|
|
const key =
|
|
maxProb >= 0.7
|
|
? "70%+"
|
|
: maxProb >= 0.6
|
|
? "60-70%"
|
|
: maxProb >= 0.5
|
|
? "50-60%"
|
|
: maxProb >= 0.4
|
|
? "40-50%"
|
|
: "33-40%";
|
|
buckets[key].total++;
|
|
if (r.predicted.ms === r.actual.ms) buckets[key].correct++;
|
|
}
|
|
|
|
// 5. Print Report
|
|
console.log("\n════════════════════════════════════════════════════════");
|
|
console.log("📊 BACKTEST ACCURACY REPORT");
|
|
console.log("════════════════════════════════════════════════════════");
|
|
console.log(` Total Matches Analyzed: ${total}`);
|
|
console.log("");
|
|
console.log(" 🎯 Market Accuracy:");
|
|
console.log(
|
|
` ⚽ Match Result (MS): ${((msCorrect / total) * 100).toFixed(2)}% (${msCorrect}/${total})`,
|
|
);
|
|
console.log(
|
|
` 📈 Over/Under 2.5: ${((ou25Correct / total) * 100).toFixed(2)}% (${ou25Correct}/${total})`,
|
|
);
|
|
console.log(
|
|
` 🤝 Both Teams Score: ${((bttsCorrect / total) * 100).toFixed(2)}% (${bttsCorrect}/${total})`,
|
|
);
|
|
console.log(
|
|
` 🏆 Main Pick Success: ${((mainPickCorrect / total) * 100).toFixed(2)}% (${mainPickCorrect}/${total})`,
|
|
);
|
|
|
|
console.log("\n 📊 MS Distribution:");
|
|
console.log(
|
|
` Actual: 1: ${actHome} (${((actHome / total) * 100).toFixed(1)}%) | X: ${actDraw} (${((actDraw / total) * 100).toFixed(1)}%) | 2: ${actAway} (${((actAway / total) * 100).toFixed(1)}%)`,
|
|
);
|
|
console.log(
|
|
` Predicted: 1: ${predHome} (${((predHome / total) * 100).toFixed(1)}%) | X: ${predDraw} (${((predDraw / total) * 100).toFixed(1)}%) | 2: ${predAway} (${((predAway / total) * 100).toFixed(1)}%)`,
|
|
);
|
|
|
|
console.log("\n 📊 Confidence Calibration:");
|
|
for (const [range, bucket] of Object.entries(buckets)) {
|
|
if (bucket.total === 0) continue;
|
|
const acc = (bucket.correct / bucket.total) * 100;
|
|
const bar = "█".repeat(Math.round(acc / 3));
|
|
console.log(
|
|
` ${range.padEnd(8)} : ${acc.toFixed(1)}% acc (n=${bucket.total}) ${bar}`,
|
|
);
|
|
}
|
|
|
|
// 6. Per-market deep dive
|
|
console.log("\n 📊 OU25 Breakdown:");
|
|
const actOver = allResults.filter((r) => r.actual.ou25 === "Over").length;
|
|
const actUnder = total - actOver;
|
|
const predOver = allResults.filter((r) => r.predicted.ou25 === "Over").length;
|
|
const predUnder = total - predOver;
|
|
console.log(
|
|
` Actual: Over: ${actOver} (${((actOver / total) * 100).toFixed(1)}%) | Under: ${actUnder} (${((actUnder / total) * 100).toFixed(1)}%)`,
|
|
);
|
|
console.log(
|
|
` Predicted: Over: ${predOver} (${((predOver / total) * 100).toFixed(1)}%) | Under: ${predUnder} (${((predUnder / total) * 100).toFixed(1)}%)`,
|
|
);
|
|
|
|
console.log("\n 📊 BTTS Breakdown:");
|
|
const actBttsYes = allResults.filter((r) => r.actual.btts === "Yes").length;
|
|
const actBttsNo = total - actBttsYes;
|
|
const predBttsYes = allResults.filter(
|
|
(r) => r.predicted.btts === "Yes",
|
|
).length;
|
|
const predBttsNo = total - predBttsYes;
|
|
console.log(
|
|
` Actual: Yes: ${actBttsYes} (${((actBttsYes / total) * 100).toFixed(1)}%) | No: ${actBttsNo} (${((actBttsNo / total) * 100).toFixed(1)}%)`,
|
|
);
|
|
console.log(
|
|
` Predicted: Yes: ${predBttsYes} (${((predBttsYes / total) * 100).toFixed(1)}%) | No: ${predBttsNo} (${((predBttsNo / total) * 100).toFixed(1)}%)`,
|
|
);
|
|
|
|
console.log("════════════════════════════════════════════════════════");
|
|
console.log("✅ Backtest complete!");
|
|
|
|
await prisma.$disconnect();
|
|
}
|
|
|
|
runBacktest().catch((err: unknown) => {
|
|
console.error("❌ Backtest failed:", err);
|
|
void prisma.$disconnect();
|
|
process.exit(1);
|
|
});
|