This commit is contained in:
@@ -0,0 +1,419 @@
|
||||
/**
|
||||
* ===================================================
|
||||
* BACKTEST ACCURACY — V30 Prediction System
|
||||
* ===================================================
|
||||
* Tests historical predictions against actual outcomes.
|
||||
* Uses the running AI Engine's /v20plus/analyze/{match_id}
|
||||
* endpoint which extracts features from DB internally.
|
||||
*
|
||||
* Usage: npx ts-node --transpile-only -r tsconfig-paths/register src/scripts/backtest-accuracy.ts
|
||||
*/
|
||||
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import axios from 'axios';
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
// ═══════════════════════════════════════════════════════
|
||||
// Configuration
|
||||
// ═══════════════════════════════════════════════════════
|
||||
|
||||
const AI_ENGINE_URL = process.env.AI_ENGINE_URL || 'http://127.0.0.1:3005';
|
||||
const CONCURRENT_REQUESTS = 5;
|
||||
const MAX_MATCHES = 1000;
|
||||
|
||||
// ═══════════════════════════════════════════════════════
|
||||
// Types
|
||||
// ═══════════════════════════════════════════════════════
|
||||
|
||||
interface TestMatch {
|
||||
id: string;
|
||||
scoreHome: number;
|
||||
scoreAway: number;
|
||||
htScoreHome: number | null;
|
||||
htScoreAway: number | null;
|
||||
}
|
||||
|
||||
interface BacktestResult {
|
||||
matchId: string;
|
||||
actual: { ms: string; ou25: string; btts: string; htft: string };
|
||||
predicted: { ms: string; ou25: string; btts: string };
|
||||
probabilities: {
|
||||
home: number;
|
||||
draw: number;
|
||||
away: number;
|
||||
over: number;
|
||||
under: number;
|
||||
bttsYes: number;
|
||||
bttsNo: number;
|
||||
};
|
||||
mainPickCorrect: boolean;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════
|
||||
// Helpers
|
||||
// ═══════════════════════════════════════════════════════
|
||||
|
||||
function determineActualOutcome(
|
||||
scoreHome: number,
|
||||
scoreAway: number,
|
||||
htScoreHome: number | null,
|
||||
htScoreAway: number | null,
|
||||
): { ms: string; ou25: string; btts: string; htft: string } {
|
||||
const ms = scoreHome > scoreAway ? '1' : scoreHome < scoreAway ? '2' : 'X';
|
||||
const ou25 = scoreHome + scoreAway > 2.5 ? 'Over' : 'Under';
|
||||
const btts = scoreHome > 0 && scoreAway > 0 ? 'Yes' : 'No';
|
||||
|
||||
let htft = 'unknown';
|
||||
if (htScoreHome !== null && htScoreAway !== null) {
|
||||
const htResult =
|
||||
htScoreHome > htScoreAway ? '1' : htScoreHome < htScoreAway ? '2' : 'X';
|
||||
htft = `${htResult}/${ms}`;
|
||||
}
|
||||
|
||||
return { ms, ou25, btts, htft };
|
||||
}
|
||||
|
||||
function extractPrediction(response: unknown): {
|
||||
ms: string;
|
||||
ou25: string;
|
||||
btts: string;
|
||||
probs: BacktestResult['probabilities'];
|
||||
mainPick: string;
|
||||
mainMarket: string;
|
||||
} {
|
||||
const data = response as Record<string, unknown>;
|
||||
const predictions = data?.predictions as Record<string, unknown> | undefined;
|
||||
|
||||
const mainPickObj = data?.main_pick as Record<string, unknown> | undefined;
|
||||
const mainPick =
|
||||
typeof mainPickObj?.pick === 'string' ? mainPickObj.pick : '';
|
||||
const mainMarket =
|
||||
typeof mainPickObj?.market === 'string' ? mainPickObj.market : '';
|
||||
|
||||
// Extract MS from probabilities or main pick
|
||||
const msProbs = (predictions?.ms || data?.ms || {}) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
const homeProb =
|
||||
typeof msProbs['1'] === 'number'
|
||||
? msProbs['1']
|
||||
: typeof msProbs.home_prob === 'number'
|
||||
? msProbs.home_prob
|
||||
: 0;
|
||||
const drawProb =
|
||||
typeof msProbs['X'] === 'number'
|
||||
? msProbs['X']
|
||||
: typeof msProbs.draw_prob === 'number'
|
||||
? msProbs.draw_prob
|
||||
: 0;
|
||||
const awayProb =
|
||||
typeof msProbs['2'] === 'number'
|
||||
? msProbs['2']
|
||||
: typeof msProbs.away_prob === 'number'
|
||||
? msProbs.away_prob
|
||||
: 0;
|
||||
|
||||
let ms = '1';
|
||||
if (drawProb > homeProb && drawProb > awayProb) ms = 'X';
|
||||
else if (awayProb > homeProb) ms = '2';
|
||||
|
||||
// Extract OU25
|
||||
const ou25Probs = (predictions?.ou25 || data?.ou25 || {}) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
const overProb =
|
||||
typeof ou25Probs.Over === 'number'
|
||||
? ou25Probs.Over
|
||||
: typeof ou25Probs.over_prob === 'number'
|
||||
? ou25Probs.over_prob
|
||||
: 0;
|
||||
const underProb =
|
||||
typeof ou25Probs.Under === 'number'
|
||||
? ou25Probs.Under
|
||||
: typeof ou25Probs.under_prob === 'number'
|
||||
? ou25Probs.under_prob
|
||||
: 0;
|
||||
const ou25 = overProb > underProb ? 'Over' : 'Under';
|
||||
|
||||
// Extract BTTS
|
||||
const bttsProbs = (predictions?.btts || data?.btts || {}) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
const bttsYes =
|
||||
typeof bttsProbs.Yes === 'number'
|
||||
? bttsProbs.Yes
|
||||
: typeof bttsProbs.yes_prob === 'number'
|
||||
? bttsProbs.yes_prob
|
||||
: 0;
|
||||
const bttsNo =
|
||||
typeof bttsProbs.No === 'number'
|
||||
? bttsProbs.No
|
||||
: typeof bttsProbs.no_prob === 'number'
|
||||
? bttsProbs.no_prob
|
||||
: 0;
|
||||
const btts = bttsYes > bttsNo ? 'Yes' : 'No';
|
||||
|
||||
return {
|
||||
ms,
|
||||
ou25,
|
||||
btts,
|
||||
probs: {
|
||||
home: homeProb,
|
||||
draw: drawProb,
|
||||
away: awayProb,
|
||||
over: overProb,
|
||||
under: underProb,
|
||||
bttsYes,
|
||||
bttsNo,
|
||||
},
|
||||
mainPick,
|
||||
mainMarket,
|
||||
};
|
||||
}
|
||||
|
||||
async function processBatch(batch: TestMatch[]): Promise<BacktestResult[]> {
|
||||
const results: BacktestResult[] = [];
|
||||
|
||||
const promises = batch.map(async (match) => {
|
||||
try {
|
||||
const response = await axios.post(
|
||||
`${AI_ENGINE_URL}/v20plus/analyze/${match.id}`,
|
||||
{},
|
||||
{ timeout: 15000 },
|
||||
);
|
||||
|
||||
const actual = determineActualOutcome(
|
||||
match.scoreHome,
|
||||
match.scoreAway,
|
||||
match.htScoreHome,
|
||||
match.htScoreAway,
|
||||
);
|
||||
|
||||
const pred = extractPrediction(response.data);
|
||||
|
||||
// Check main pick
|
||||
let mainPickCorrect = false;
|
||||
if (pred.mainMarket === 'MS') {
|
||||
mainPickCorrect = pred.mainPick === actual.ms;
|
||||
} else if (pred.mainMarket === 'OU25') {
|
||||
mainPickCorrect = pred.mainPick === actual.ou25;
|
||||
} else if (pred.mainMarket === 'BTTS') {
|
||||
mainPickCorrect = pred.mainPick === actual.btts;
|
||||
}
|
||||
|
||||
results.push({
|
||||
matchId: match.id,
|
||||
actual,
|
||||
predicted: { ms: pred.ms, ou25: pred.ou25, btts: pred.btts },
|
||||
probabilities: pred.probs,
|
||||
mainPickCorrect,
|
||||
});
|
||||
} catch {
|
||||
// Skip failed matches silently
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(promises);
|
||||
return results;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════
|
||||
// Main Backtest
|
||||
// ═══════════════════════════════════════════════════════
|
||||
|
||||
async function runBacktest(): Promise<void> {
|
||||
console.log('🎯 BACKTEST ACCURACY — V30 Betting Engine');
|
||||
console.log('════════════════════════════════════════════════════════');
|
||||
|
||||
// 1. Health check
|
||||
try {
|
||||
const health = await axios.get(`${AI_ENGINE_URL}/health`, {
|
||||
timeout: 5000,
|
||||
});
|
||||
console.log(`✅ AI Engine: ${JSON.stringify(health.data)}`);
|
||||
} catch {
|
||||
console.error('❌ AI Engine not reachable at', AI_ENGINE_URL);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// 2. Load finished matches with features
|
||||
console.log('\n📥 Loading test matches...');
|
||||
const matches = await prisma.$queryRaw<TestMatch[]>`
|
||||
SELECT m.id, m.score_home AS "scoreHome", m.score_away AS "scoreAway",
|
||||
m.ht_score_home AS "htScoreHome", m.ht_score_away AS "htScoreAway"
|
||||
FROM matches m
|
||||
JOIN match_ai_features maf ON maf.match_id = m.id
|
||||
WHERE m.status = 'FT'
|
||||
AND m.score_home IS NOT NULL
|
||||
AND m.score_away IS NOT NULL
|
||||
AND m.sport = 'football'
|
||||
AND maf.home_elo != 1500
|
||||
AND maf.implied_home != 0.33
|
||||
ORDER BY m.mst_utc DESC
|
||||
LIMIT ${MAX_MATCHES}
|
||||
`;
|
||||
console.log(` 📊 Test matches: ${matches.length}`);
|
||||
|
||||
// 3. Run predictions in batches
|
||||
console.log('\n🤖 Running predictions...');
|
||||
const allResults: BacktestResult[] = [];
|
||||
let processed = 0;
|
||||
|
||||
for (let i = 0; i < matches.length; i += CONCURRENT_REQUESTS) {
|
||||
const batch = matches.slice(i, i + CONCURRENT_REQUESTS);
|
||||
const batchResults = await processBatch(batch);
|
||||
allResults.push(...batchResults);
|
||||
processed += batch.length;
|
||||
|
||||
if (processed % 50 === 0 || processed === matches.length) {
|
||||
const currentMsAcc =
|
||||
allResults.length > 0
|
||||
? (
|
||||
(allResults.filter((r) => r.predicted.ms === r.actual.ms).length /
|
||||
allResults.length) *
|
||||
100
|
||||
).toFixed(1)
|
||||
: '0';
|
||||
console.log(
|
||||
` 📊 ${processed}/${matches.length} — Success: ${allResults.length} — MS Acc: ${currentMsAcc}%`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Calculate metrics
|
||||
const total = allResults.length;
|
||||
if (total === 0) {
|
||||
console.error('❌ No results to analyze');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const msCorrect = allResults.filter(
|
||||
(r) => r.predicted.ms === r.actual.ms,
|
||||
).length;
|
||||
const ou25Correct = allResults.filter(
|
||||
(r) => r.predicted.ou25 === r.actual.ou25,
|
||||
).length;
|
||||
const bttsCorrect = allResults.filter(
|
||||
(r) => r.predicted.btts === r.actual.btts,
|
||||
).length;
|
||||
const mainPickCorrect = allResults.filter((r) => r.mainPickCorrect).length;
|
||||
|
||||
// Actual distribution
|
||||
const actHome = allResults.filter((r) => r.actual.ms === '1').length;
|
||||
const actDraw = allResults.filter((r) => r.actual.ms === 'X').length;
|
||||
const actAway = allResults.filter((r) => r.actual.ms === '2').length;
|
||||
|
||||
// Predicted distribution
|
||||
const predHome = allResults.filter((r) => r.predicted.ms === '1').length;
|
||||
const predDraw = allResults.filter((r) => r.predicted.ms === 'X').length;
|
||||
const predAway = allResults.filter((r) => r.predicted.ms === '2').length;
|
||||
|
||||
// Confidence calibration (based on max probability)
|
||||
const buckets: Record<string, { correct: number; total: number }> = {
|
||||
'33-40%': { correct: 0, total: 0 },
|
||||
'40-50%': { correct: 0, total: 0 },
|
||||
'50-60%': { correct: 0, total: 0 },
|
||||
'60-70%': { correct: 0, total: 0 },
|
||||
'70%+': { correct: 0, total: 0 },
|
||||
};
|
||||
|
||||
for (const r of allResults) {
|
||||
const maxProb = Math.max(
|
||||
r.probabilities.home,
|
||||
r.probabilities.draw,
|
||||
r.probabilities.away,
|
||||
);
|
||||
const key =
|
||||
maxProb >= 0.7
|
||||
? '70%+'
|
||||
: maxProb >= 0.6
|
||||
? '60-70%'
|
||||
: maxProb >= 0.5
|
||||
? '50-60%'
|
||||
: maxProb >= 0.4
|
||||
? '40-50%'
|
||||
: '33-40%';
|
||||
buckets[key].total++;
|
||||
if (r.predicted.ms === r.actual.ms) buckets[key].correct++;
|
||||
}
|
||||
|
||||
// 5. Print Report
|
||||
console.log('\n════════════════════════════════════════════════════════');
|
||||
console.log('📊 BACKTEST ACCURACY REPORT');
|
||||
console.log('════════════════════════════════════════════════════════');
|
||||
console.log(` Total Matches Analyzed: ${total}`);
|
||||
console.log('');
|
||||
console.log(' 🎯 Market Accuracy:');
|
||||
console.log(
|
||||
` ⚽ Match Result (MS): ${((msCorrect / total) * 100).toFixed(2)}% (${msCorrect}/${total})`,
|
||||
);
|
||||
console.log(
|
||||
` 📈 Over/Under 2.5: ${((ou25Correct / total) * 100).toFixed(2)}% (${ou25Correct}/${total})`,
|
||||
);
|
||||
console.log(
|
||||
` 🤝 Both Teams Score: ${((bttsCorrect / total) * 100).toFixed(2)}% (${bttsCorrect}/${total})`,
|
||||
);
|
||||
console.log(
|
||||
` 🏆 Main Pick Success: ${((mainPickCorrect / total) * 100).toFixed(2)}% (${mainPickCorrect}/${total})`,
|
||||
);
|
||||
|
||||
console.log('\n 📊 MS Distribution:');
|
||||
console.log(
|
||||
` Actual: 1: ${actHome} (${((actHome / total) * 100).toFixed(1)}%) | X: ${actDraw} (${((actDraw / total) * 100).toFixed(1)}%) | 2: ${actAway} (${((actAway / total) * 100).toFixed(1)}%)`,
|
||||
);
|
||||
console.log(
|
||||
` Predicted: 1: ${predHome} (${((predHome / total) * 100).toFixed(1)}%) | X: ${predDraw} (${((predDraw / total) * 100).toFixed(1)}%) | 2: ${predAway} (${((predAway / total) * 100).toFixed(1)}%)`,
|
||||
);
|
||||
|
||||
console.log('\n 📊 Confidence Calibration:');
|
||||
for (const [range, bucket] of Object.entries(buckets)) {
|
||||
if (bucket.total === 0) continue;
|
||||
const acc = (bucket.correct / bucket.total) * 100;
|
||||
const bar = '█'.repeat(Math.round(acc / 3));
|
||||
console.log(
|
||||
` ${range.padEnd(8)} : ${acc.toFixed(1)}% acc (n=${bucket.total}) ${bar}`,
|
||||
);
|
||||
}
|
||||
|
||||
// 6. Per-market deep dive
|
||||
console.log('\n 📊 OU25 Breakdown:');
|
||||
const actOver = allResults.filter((r) => r.actual.ou25 === 'Over').length;
|
||||
const actUnder = total - actOver;
|
||||
const predOver = allResults.filter((r) => r.predicted.ou25 === 'Over').length;
|
||||
const predUnder = total - predOver;
|
||||
console.log(
|
||||
` Actual: Over: ${actOver} (${((actOver / total) * 100).toFixed(1)}%) | Under: ${actUnder} (${((actUnder / total) * 100).toFixed(1)}%)`,
|
||||
);
|
||||
console.log(
|
||||
` Predicted: Over: ${predOver} (${((predOver / total) * 100).toFixed(1)}%) | Under: ${predUnder} (${((predUnder / total) * 100).toFixed(1)}%)`,
|
||||
);
|
||||
|
||||
console.log('\n 📊 BTTS Breakdown:');
|
||||
const actBttsYes = allResults.filter((r) => r.actual.btts === 'Yes').length;
|
||||
const actBttsNo = total - actBttsYes;
|
||||
const predBttsYes = allResults.filter(
|
||||
(r) => r.predicted.btts === 'Yes',
|
||||
).length;
|
||||
const predBttsNo = total - predBttsYes;
|
||||
console.log(
|
||||
` Actual: Yes: ${actBttsYes} (${((actBttsYes / total) * 100).toFixed(1)}%) | No: ${actBttsNo} (${((actBttsNo / total) * 100).toFixed(1)}%)`,
|
||||
);
|
||||
console.log(
|
||||
` Predicted: Yes: ${predBttsYes} (${((predBttsYes / total) * 100).toFixed(1)}%) | No: ${predBttsNo} (${((predBttsNo / total) * 100).toFixed(1)}%)`,
|
||||
);
|
||||
|
||||
console.log('════════════════════════════════════════════════════════');
|
||||
console.log('✅ Backtest complete!');
|
||||
|
||||
await prisma.$disconnect();
|
||||
}
|
||||
|
||||
runBacktest().catch((err: unknown) => {
|
||||
console.error('❌ Backtest failed:', err);
|
||||
void prisma.$disconnect();
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user