/** * Read-only analysis of prediction patterns for the last N finished football matches. * * Outputs systematic-bias indicators that inform the engine improvement brief: * 1. Surprise transparency rate (how often surprise_reasons is empty) * 2. Surprise miss rate (underdog won but is_surprise_risk was false) * 3. REJECT-all rate + actual outcome distribution on those matches * 4. Calibration shrinkage histogram (raw - calibrated per market) * 5. Trap-market frequency (band_rate << implied_prob despite high model_prob) * 6. Commentary "hafif favori" hit-rate vs actual result * 7. Live-blind cases (LIVE matches whose latest prediction was pre-match) * * Usage: * npx ts-node iddaai-be/scripts/analyze_prediction_patterns.ts [limit=200] */ import { PrismaClient } from "@prisma/client"; const prisma = new PrismaClient(); const LIMIT = parseInt(process.argv[2] || "200", 10); type Payload = Record; function readNum(v: any): number | null { const n = Number(v); return Number.isFinite(n) ? n : null; } function bucket(value: number, edges: number[]): string { for (let i = 0; i < edges.length; i++) { if (value < edges[i]) { const lo = i === 0 ? "-inf" : String(edges[i - 1]); return `[${lo}, ${edges[i]})`; } } return `[${edges[edges.length - 1]}, +inf)`; } async function main() { console.log(`\n=== PREDICTION PATTERN ANALYZER ===`); console.log(`Pulling the most recent ${LIMIT} football matches with predictions.\n`); const matches = await prisma.match.findMany({ where: { sport: "football", status: "FT", scoreHome: { not: null }, scoreAway: { not: null }, prediction: { isNot: null }, }, include: { prediction: true }, orderBy: { mstUtc: "desc" }, take: LIMIT, }); console.log(`Found ${matches.length} matches.\n`); // Counters let surpriseEmpty = 0; let surpriseFilled = 0; let upsetMatches = 0; // underdog (per odds) actually won let upsetMissedBySystem = 0; // upset happened, is_surprise_risk false let upsetCaughtBySystem = 0; let rejectAllCount = 0; const rejectAllOutcomes = { homeWin: 0, draw: 0, awayWin: 0 }; const shrinkageByMarket = new Map(); // raw - calibrated per market let trapMarketCount = 0; // band_rate < implied_prob - 0.10 AND main_pick selected anyway let trapMarketSampled = 0; let hafifFavoriUseCount = 0; // commentary said "hafif favori" let hafifFavoriCorrectCount = 0; // and that favorite actually won for (const m of matches) { const payload = (m.prediction?.predictionJson as Payload) || {}; // 1. Surprise transparency const risk = payload.risk || {}; const surpriseReasons = Array.isArray(risk.surprise_reasons) ? risk.surprise_reasons : []; if (surpriseReasons.length === 0) surpriseEmpty++; else surpriseFilled++; // 2. Upset detection vs reality const finalHome = m.scoreHome ?? 0; const finalAway = m.scoreAway ?? 0; const actualWinner = finalHome > finalAway ? "H" : finalHome < finalAway ? "A" : "D"; const oddsSnap = payload.bet_summary && Array.isArray(payload.bet_summary) ? payload.bet_summary.find((b: any) => b.market === "MS") : null; const msMain = payload.main_pick || {}; // Crude favorite-side detection: scan bet_summary or market_board for MS implied probs const msBoard = (payload.market_board || {}).MS || {}; let favSide: "H" | "A" | "D" | null = null; const implH = readNum(msBoard?.probs?.["1"]); const implA = readNum(msBoard?.probs?.["2"]); if (implH !== null && implA !== null) { favSide = implH > implA ? "H" : implA > implH ? "A" : null; } if (favSide && actualWinner !== favSide && actualWinner !== "D") { upsetMatches++; if (risk.is_surprise_risk === true) upsetCaughtBySystem++; else upsetMissedBySystem++; } // 3. REJECT-all matches const brain = payload.betting_brain || {}; if ((brain.decision || "NO_BET").toUpperCase() === "NO_BET" && brain.approved_count === 0) { rejectAllCount++; if (actualWinner === "H") rejectAllOutcomes.homeWin++; else if (actualWinner === "A") rejectAllOutcomes.awayWin++; else rejectAllOutcomes.draw++; } // 4. Calibration shrinkage by market const summary: any[] = Array.isArray(payload.bet_summary) ? payload.bet_summary : []; for (const row of summary) { const market = String(row.market || "OTHER"); const raw = readNum(row.raw_confidence); const cal = readNum(row.calibrated_confidence); if (raw !== null && cal !== null) { const arr = shrinkageByMarket.get(market) || []; arr.push(raw - cal); shrinkageByMarket.set(market, arr); } } // 5. Trap market: model says high prob but band_rate is much lower than implied const bb = (msMain.betting_brain || {}) as any; const triple = bb.triple_value || null; if (triple && typeof triple === "object") { trapMarketSampled++; const bandRate = readNum(triple.band_rate); const implied = readNum(triple.implied_prob); if (bandRate !== null && implied !== null && implied - bandRate > 0.10) { trapMarketCount++; } } // 6. "hafif favori" usage vs reality const commentary = payload.match_commentary || {}; const summaryText = String(commentary.summary || ""); if (summaryText.includes("hafif favori")) { hafifFavoriUseCount++; // If summary mentions home name first then says hafif favori, assume home favorite const home = (payload.match_info || {}).home_team || ""; const sayingHomeFav = summaryText.indexOf(home) >= 0 && summaryText.indexOf(home) < summaryText.indexOf("hafif favori"); const predictedSide = sayingHomeFav ? "H" : "A"; if (predictedSide === actualWinner) hafifFavoriCorrectCount++; } } // ─── Output ───────────────────────────────────────────────── console.log(`\n--- 1. SURPRISE TRANSPARENCY ---`); console.log(` Empty surprise_reasons: ${surpriseEmpty}/${matches.length} (${((surpriseEmpty / matches.length) * 100).toFixed(1)}%)`); console.log(` Filled surprise_reasons: ${surpriseFilled}/${matches.length}`); console.log(`\n--- 2. UPSET DETECTION ---`); console.log(` Actual upsets (underdog wins): ${upsetMatches}/${matches.length}`); console.log(` Caught by is_surprise_risk: ${upsetCaughtBySystem}`); console.log(` MISSED (no surprise flag): ${upsetMissedBySystem}`); if (upsetMatches > 0) { console.log(` Miss rate: ${((upsetMissedBySystem / upsetMatches) * 100).toFixed(1)}%`); } console.log(`\n--- 3. REJECT-ALL MATCHES ---`); console.log(` Count: ${rejectAllCount}/${matches.length} (${((rejectAllCount / matches.length) * 100).toFixed(1)}%)`); console.log(` Outcome distribution on those matches:`); console.log(` Home wins: ${rejectAllOutcomes.homeWin}`); console.log(` Draws: ${rejectAllOutcomes.draw}`); console.log(` Away wins: ${rejectAllOutcomes.awayWin}`); console.log(`\n--- 4. CALIBRATION SHRINKAGE (raw - calibrated) BY MARKET ---`); const buckets = [-5, 0, 5, 10, 15, 20]; for (const [market, arr] of Array.from(shrinkageByMarket.entries()).sort()) { const sorted = [...arr].sort((a, b) => a - b); const median = sorted[Math.floor(sorted.length / 2)] ?? 0; const p90 = sorted[Math.floor(sorted.length * 0.9)] ?? 0; const avg = arr.reduce((s, v) => s + v, 0) / arr.length; console.log(` ${market.padEnd(10)} n=${String(arr.length).padStart(4)} avg=${avg.toFixed(2).padStart(6)} median=${median.toFixed(2).padStart(6)} p90=${p90.toFixed(2).padStart(6)}`); } console.log(`\n--- 5. TRAP MARKET PREVALENCE (main_pick) ---`); console.log(` Sampled main_picks with triple_value: ${trapMarketSampled}`); console.log(` Trap candidates (implied - band_rate > 0.10): ${trapMarketCount}`); if (trapMarketSampled > 0) { console.log(` Trap rate: ${((trapMarketCount / trapMarketSampled) * 100).toFixed(1)}%`); } console.log(`\n--- 6. "hafif favori" COMMENTARY ACCURACY ---`); console.log(` Used in commentary: ${hafifFavoriUseCount}`); console.log(` Correctly predicted winner: ${hafifFavoriCorrectCount}`); if (hafifFavoriUseCount > 0) { console.log(` Accuracy: ${((hafifFavoriCorrectCount / hafifFavoriUseCount) * 100).toFixed(1)}%`); } console.log(`\n=== DONE ===\n`); void bucket; } main() .catch((e) => { console.error(e); process.exit(1); }) .finally(async () => { await prisma.$disconnect(); });