This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
/**
|
||||
* Read-only analysis of prediction patterns for the last N finished football matches.
|
||||
*
|
||||
* Outputs systematic-bias indicators that inform the engine improvement brief:
|
||||
* 1. Surprise transparency rate (how often surprise_reasons is empty)
|
||||
* 2. Surprise miss rate (underdog won but is_surprise_risk was false)
|
||||
* 3. REJECT-all rate + actual outcome distribution on those matches
|
||||
* 4. Calibration shrinkage histogram (raw - calibrated per market)
|
||||
* 5. Trap-market frequency (band_rate << implied_prob despite high model_prob)
|
||||
* 6. Commentary "hafif favori" hit-rate vs actual result
|
||||
* 7. Live-blind cases (LIVE matches whose latest prediction was pre-match)
|
||||
*
|
||||
* Usage:
|
||||
* npx ts-node iddaai-be/scripts/analyze_prediction_patterns.ts [limit=200]
|
||||
*/
|
||||
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
const LIMIT = parseInt(process.argv[2] || "200", 10);
|
||||
|
||||
type Payload = Record<string, any>;
|
||||
|
||||
function readNum(v: any): number | null {
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
|
||||
function bucket(value: number, edges: number[]): string {
|
||||
for (let i = 0; i < edges.length; i++) {
|
||||
if (value < edges[i]) {
|
||||
const lo = i === 0 ? "-inf" : String(edges[i - 1]);
|
||||
return `[${lo}, ${edges[i]})`;
|
||||
}
|
||||
}
|
||||
return `[${edges[edges.length - 1]}, +inf)`;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log(`\n=== PREDICTION PATTERN ANALYZER ===`);
|
||||
console.log(`Pulling the most recent ${LIMIT} football matches with predictions.\n`);
|
||||
|
||||
const matches = await prisma.match.findMany({
|
||||
where: {
|
||||
sport: "football",
|
||||
status: "FT",
|
||||
scoreHome: { not: null },
|
||||
scoreAway: { not: null },
|
||||
prediction: { isNot: null },
|
||||
},
|
||||
include: { prediction: true },
|
||||
orderBy: { mstUtc: "desc" },
|
||||
take: LIMIT,
|
||||
});
|
||||
|
||||
console.log(`Found ${matches.length} matches.\n`);
|
||||
|
||||
// Counters
|
||||
let surpriseEmpty = 0;
|
||||
let surpriseFilled = 0;
|
||||
|
||||
let upsetMatches = 0; // underdog (per odds) actually won
|
||||
let upsetMissedBySystem = 0; // upset happened, is_surprise_risk false
|
||||
let upsetCaughtBySystem = 0;
|
||||
|
||||
let rejectAllCount = 0;
|
||||
const rejectAllOutcomes = { homeWin: 0, draw: 0, awayWin: 0 };
|
||||
|
||||
const shrinkageByMarket = new Map<string, number[]>(); // raw - calibrated per market
|
||||
let trapMarketCount = 0; // band_rate < implied_prob - 0.10 AND main_pick selected anyway
|
||||
let trapMarketSampled = 0;
|
||||
|
||||
let hafifFavoriUseCount = 0; // commentary said "hafif favori"
|
||||
let hafifFavoriCorrectCount = 0; // and that favorite actually won
|
||||
|
||||
for (const m of matches) {
|
||||
const payload = (m.prediction?.predictionJson as Payload) || {};
|
||||
|
||||
// 1. Surprise transparency
|
||||
const risk = payload.risk || {};
|
||||
const surpriseReasons = Array.isArray(risk.surprise_reasons) ? risk.surprise_reasons : [];
|
||||
if (surpriseReasons.length === 0) surpriseEmpty++;
|
||||
else surpriseFilled++;
|
||||
|
||||
// 2. Upset detection vs reality
|
||||
const finalHome = m.scoreHome ?? 0;
|
||||
const finalAway = m.scoreAway ?? 0;
|
||||
const actualWinner = finalHome > finalAway ? "H" : finalHome < finalAway ? "A" : "D";
|
||||
|
||||
const oddsSnap = payload.bet_summary && Array.isArray(payload.bet_summary)
|
||||
? payload.bet_summary.find((b: any) => b.market === "MS")
|
||||
: null;
|
||||
const msMain = payload.main_pick || {};
|
||||
// Crude favorite-side detection: scan bet_summary or market_board for MS implied probs
|
||||
const msBoard = (payload.market_board || {}).MS || {};
|
||||
let favSide: "H" | "A" | "D" | null = null;
|
||||
const implH = readNum(msBoard?.probs?.["1"]);
|
||||
const implA = readNum(msBoard?.probs?.["2"]);
|
||||
if (implH !== null && implA !== null) {
|
||||
favSide = implH > implA ? "H" : implA > implH ? "A" : null;
|
||||
}
|
||||
if (favSide && actualWinner !== favSide && actualWinner !== "D") {
|
||||
upsetMatches++;
|
||||
if (risk.is_surprise_risk === true) upsetCaughtBySystem++;
|
||||
else upsetMissedBySystem++;
|
||||
}
|
||||
|
||||
// 3. REJECT-all matches
|
||||
const brain = payload.betting_brain || {};
|
||||
if ((brain.decision || "NO_BET").toUpperCase() === "NO_BET" && brain.approved_count === 0) {
|
||||
rejectAllCount++;
|
||||
if (actualWinner === "H") rejectAllOutcomes.homeWin++;
|
||||
else if (actualWinner === "A") rejectAllOutcomes.awayWin++;
|
||||
else rejectAllOutcomes.draw++;
|
||||
}
|
||||
|
||||
// 4. Calibration shrinkage by market
|
||||
const summary: any[] = Array.isArray(payload.bet_summary) ? payload.bet_summary : [];
|
||||
for (const row of summary) {
|
||||
const market = String(row.market || "OTHER");
|
||||
const raw = readNum(row.raw_confidence);
|
||||
const cal = readNum(row.calibrated_confidence);
|
||||
if (raw !== null && cal !== null) {
|
||||
const arr = shrinkageByMarket.get(market) || [];
|
||||
arr.push(raw - cal);
|
||||
shrinkageByMarket.set(market, arr);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Trap market: model says high prob but band_rate is much lower than implied
|
||||
const bb = (msMain.betting_brain || {}) as any;
|
||||
const triple = bb.triple_value || null;
|
||||
if (triple && typeof triple === "object") {
|
||||
trapMarketSampled++;
|
||||
const bandRate = readNum(triple.band_rate);
|
||||
const implied = readNum(triple.implied_prob);
|
||||
if (bandRate !== null && implied !== null && implied - bandRate > 0.10) {
|
||||
trapMarketCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// 6. "hafif favori" usage vs reality
|
||||
const commentary = payload.match_commentary || {};
|
||||
const summaryText = String(commentary.summary || "");
|
||||
if (summaryText.includes("hafif favori")) {
|
||||
hafifFavoriUseCount++;
|
||||
// If summary mentions home name first then says hafif favori, assume home favorite
|
||||
const home = (payload.match_info || {}).home_team || "";
|
||||
const sayingHomeFav = summaryText.indexOf(home) >= 0 && summaryText.indexOf(home) < summaryText.indexOf("hafif favori");
|
||||
const predictedSide = sayingHomeFav ? "H" : "A";
|
||||
if (predictedSide === actualWinner) hafifFavoriCorrectCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Output ─────────────────────────────────────────────────
|
||||
console.log(`\n--- 1. SURPRISE TRANSPARENCY ---`);
|
||||
console.log(` Empty surprise_reasons: ${surpriseEmpty}/${matches.length} (${((surpriseEmpty / matches.length) * 100).toFixed(1)}%)`);
|
||||
console.log(` Filled surprise_reasons: ${surpriseFilled}/${matches.length}`);
|
||||
|
||||
console.log(`\n--- 2. UPSET DETECTION ---`);
|
||||
console.log(` Actual upsets (underdog wins): ${upsetMatches}/${matches.length}`);
|
||||
console.log(` Caught by is_surprise_risk: ${upsetCaughtBySystem}`);
|
||||
console.log(` MISSED (no surprise flag): ${upsetMissedBySystem}`);
|
||||
if (upsetMatches > 0) {
|
||||
console.log(` Miss rate: ${((upsetMissedBySystem / upsetMatches) * 100).toFixed(1)}%`);
|
||||
}
|
||||
|
||||
console.log(`\n--- 3. REJECT-ALL MATCHES ---`);
|
||||
console.log(` Count: ${rejectAllCount}/${matches.length} (${((rejectAllCount / matches.length) * 100).toFixed(1)}%)`);
|
||||
console.log(` Outcome distribution on those matches:`);
|
||||
console.log(` Home wins: ${rejectAllOutcomes.homeWin}`);
|
||||
console.log(` Draws: ${rejectAllOutcomes.draw}`);
|
||||
console.log(` Away wins: ${rejectAllOutcomes.awayWin}`);
|
||||
|
||||
console.log(`\n--- 4. CALIBRATION SHRINKAGE (raw - calibrated) BY MARKET ---`);
|
||||
const buckets = [-5, 0, 5, 10, 15, 20];
|
||||
for (const [market, arr] of Array.from(shrinkageByMarket.entries()).sort()) {
|
||||
const sorted = [...arr].sort((a, b) => a - b);
|
||||
const median = sorted[Math.floor(sorted.length / 2)] ?? 0;
|
||||
const p90 = sorted[Math.floor(sorted.length * 0.9)] ?? 0;
|
||||
const avg = arr.reduce((s, v) => s + v, 0) / arr.length;
|
||||
console.log(` ${market.padEnd(10)} n=${String(arr.length).padStart(4)} avg=${avg.toFixed(2).padStart(6)} median=${median.toFixed(2).padStart(6)} p90=${p90.toFixed(2).padStart(6)}`);
|
||||
}
|
||||
|
||||
console.log(`\n--- 5. TRAP MARKET PREVALENCE (main_pick) ---`);
|
||||
console.log(` Sampled main_picks with triple_value: ${trapMarketSampled}`);
|
||||
console.log(` Trap candidates (implied - band_rate > 0.10): ${trapMarketCount}`);
|
||||
if (trapMarketSampled > 0) {
|
||||
console.log(` Trap rate: ${((trapMarketCount / trapMarketSampled) * 100).toFixed(1)}%`);
|
||||
}
|
||||
|
||||
console.log(`\n--- 6. "hafif favori" COMMENTARY ACCURACY ---`);
|
||||
console.log(` Used in commentary: ${hafifFavoriUseCount}`);
|
||||
console.log(` Correctly predicted winner: ${hafifFavoriCorrectCount}`);
|
||||
if (hafifFavoriUseCount > 0) {
|
||||
console.log(` Accuracy: ${((hafifFavoriCorrectCount / hafifFavoriUseCount) * 100).toFixed(1)}%`);
|
||||
}
|
||||
|
||||
console.log(`\n=== DONE ===\n`);
|
||||
void bucket;
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
})
|
||||
.finally(async () => {
|
||||
await prisma.$disconnect();
|
||||
});
|
||||
Reference in New Issue
Block a user