iddaai-be/scripts/analyze_prediction_patterns.ts

/**
 * Read-only analysis of prediction patterns for the last N finished football matches.
 *
 * Outputs systematic-bias indicators that inform the engine improvement brief:
 *   1. Surprise transparency rate (how often surprise_reasons is empty)
 *   2. Surprise miss rate (underdog won but is_surprise_risk was false)
 *   3. REJECT-all rate + actual outcome distribution on those matches
 *   4. Calibration shrinkage histogram (raw - calibrated per market)
 *   5. Trap-market frequency (band_rate << implied_prob despite high model_prob)
 *   6. Commentary "hafif favori" hit-rate vs actual result
 *   7. Live-blind cases (LIVE matches whose latest prediction was pre-match)
 *
 * Usage:
 *   npx ts-node iddaai-be/scripts/analyze_prediction_patterns.ts [limit=200]
 */

import { PrismaClient } from "@prisma/client";

const prisma = new PrismaClient();
const LIMIT = parseInt(process.argv[2] || "200", 10);

type Payload = Record<string, any>;

function readNum(v: any): number | null {
  const n = Number(v);
  return Number.isFinite(n) ? n : null;
}

function bucket(value: number, edges: number[]): string {
  for (let i = 0; i < edges.length; i++) {
    if (value < edges[i]) {
      const lo = i === 0 ? "-inf" : String(edges[i - 1]);
      return `[${lo}, ${edges[i]})`;
    }
  }
  return `[${edges[edges.length - 1]}, +inf)`;
}

async function main() {
  console.log(`\n=== PREDICTION PATTERN ANALYZER ===`);
  console.log(`Pulling the most recent ${LIMIT} football matches with predictions.\n`);

  const matches = await prisma.match.findMany({
    where: {
      sport: "football",
      status: "FT",
      scoreHome: { not: null },
      scoreAway: { not: null },
      prediction: { isNot: null },
    },
    include: { prediction: true },
    orderBy: { mstUtc: "desc" },
    take: LIMIT,
  });

  console.log(`Found ${matches.length} matches.\n`);

  // Counters
  let surpriseEmpty = 0;
  let surpriseFilled = 0;

  let upsetMatches = 0; // underdog (per odds) actually won
  let upsetMissedBySystem = 0; // upset happened, is_surprise_risk false
  let upsetCaughtBySystem = 0;

  let rejectAllCount = 0;
  const rejectAllOutcomes = { homeWin: 0, draw: 0, awayWin: 0 };

  const shrinkageByMarket = new Map<string, number[]>(); // raw - calibrated per market
  let trapMarketCount = 0; // band_rate < implied_prob - 0.10 AND main_pick selected anyway
  let trapMarketSampled = 0;

  let hafifFavoriUseCount = 0; // commentary said "hafif favori"
  let hafifFavoriCorrectCount = 0; // and that favorite actually won

  for (const m of matches) {
    const payload = (m.prediction?.predictionJson as Payload) || {};

    // 1. Surprise transparency
    const risk = payload.risk || {};
    const surpriseReasons = Array.isArray(risk.surprise_reasons) ? risk.surprise_reasons : [];
    if (surpriseReasons.length === 0) surpriseEmpty++;
    else surpriseFilled++;

    // 2. Upset detection vs reality
    const finalHome = m.scoreHome ?? 0;
    const finalAway = m.scoreAway ?? 0;
    const actualWinner = finalHome > finalAway ? "H" : finalHome < finalAway ? "A" : "D";

    const oddsSnap = payload.bet_summary && Array.isArray(payload.bet_summary)
      ? payload.bet_summary.find((b: any) => b.market === "MS")
      : null;
    const msMain = payload.main_pick || {};
    // Crude favorite-side detection: scan bet_summary or market_board for MS implied probs
    const msBoard = (payload.market_board || {}).MS || {};
    let favSide: "H" | "A" | "D" | null = null;
    const implH = readNum(msBoard?.probs?.["1"]);
    const implA = readNum(msBoard?.probs?.["2"]);
    if (implH !== null && implA !== null) {
      favSide = implH > implA ? "H" : implA > implH ? "A" : null;
    }
    if (favSide && actualWinner !== favSide && actualWinner !== "D") {
      upsetMatches++;
      if (risk.is_surprise_risk === true) upsetCaughtBySystem++;
      else upsetMissedBySystem++;
    }

    // 3. REJECT-all matches
    const brain = payload.betting_brain || {};
    if ((brain.decision || "NO_BET").toUpperCase() === "NO_BET" && brain.approved_count === 0) {
      rejectAllCount++;
      if (actualWinner === "H") rejectAllOutcomes.homeWin++;
      else if (actualWinner === "A") rejectAllOutcomes.awayWin++;
      else rejectAllOutcomes.draw++;
    }

    // 4. Calibration shrinkage by market
    const summary: any[] = Array.isArray(payload.bet_summary) ? payload.bet_summary : [];
    for (const row of summary) {
      const market = String(row.market || "OTHER");
      const raw = readNum(row.raw_confidence);
      const cal = readNum(row.calibrated_confidence);
      if (raw !== null && cal !== null) {
        const arr = shrinkageByMarket.get(market) || [];
        arr.push(raw - cal);
        shrinkageByMarket.set(market, arr);
      }
    }

    // 5. Trap market: model says high prob but band_rate is much lower than implied
    const bb = (msMain.betting_brain || {}) as any;
    const triple = bb.triple_value || null;
    if (triple && typeof triple === "object") {
      trapMarketSampled++;
      const bandRate = readNum(triple.band_rate);
      const implied = readNum(triple.implied_prob);
      if (bandRate !== null && implied !== null && implied - bandRate > 0.10) {
        trapMarketCount++;
      }
    }

    // 6. "hafif favori" usage vs reality
    const commentary = payload.match_commentary || {};
    const summaryText = String(commentary.summary || "");
    if (summaryText.includes("hafif favori")) {
      hafifFavoriUseCount++;
      // If summary mentions home name first then says hafif favori, assume home favorite
      const home = (payload.match_info || {}).home_team || "";
      const sayingHomeFav = summaryText.indexOf(home) >= 0 && summaryText.indexOf(home) < summaryText.indexOf("hafif favori");
      const predictedSide = sayingHomeFav ? "H" : "A";
      if (predictedSide === actualWinner) hafifFavoriCorrectCount++;
    }
  }

  // ─── Output ─────────────────────────────────────────────────
  console.log(`\n--- 1. SURPRISE TRANSPARENCY ---`);
  console.log(`  Empty surprise_reasons:  ${surpriseEmpty}/${matches.length} (${((surpriseEmpty / matches.length) * 100).toFixed(1)}%)`);
  console.log(`  Filled surprise_reasons: ${surpriseFilled}/${matches.length}`);

  console.log(`\n--- 2. UPSET DETECTION ---`);
  console.log(`  Actual upsets (underdog wins): ${upsetMatches}/${matches.length}`);
  console.log(`  Caught by is_surprise_risk:    ${upsetCaughtBySystem}`);
  console.log(`  MISSED (no surprise flag):     ${upsetMissedBySystem}`);
  if (upsetMatches > 0) {
    console.log(`  Miss rate: ${((upsetMissedBySystem / upsetMatches) * 100).toFixed(1)}%`);
  }

  console.log(`\n--- 3. REJECT-ALL MATCHES ---`);
  console.log(`  Count: ${rejectAllCount}/${matches.length} (${((rejectAllCount / matches.length) * 100).toFixed(1)}%)`);
  console.log(`  Outcome distribution on those matches:`);
  console.log(`    Home wins: ${rejectAllOutcomes.homeWin}`);
  console.log(`    Draws:     ${rejectAllOutcomes.draw}`);
  console.log(`    Away wins: ${rejectAllOutcomes.awayWin}`);

  console.log(`\n--- 4. CALIBRATION SHRINKAGE (raw - calibrated) BY MARKET ---`);
  const buckets = [-5, 0, 5, 10, 15, 20];
  for (const [market, arr] of Array.from(shrinkageByMarket.entries()).sort()) {
    const sorted = [...arr].sort((a, b) => a - b);
    const median = sorted[Math.floor(sorted.length / 2)] ?? 0;
    const p90 = sorted[Math.floor(sorted.length * 0.9)] ?? 0;
    const avg = arr.reduce((s, v) => s + v, 0) / arr.length;
    console.log(`  ${market.padEnd(10)} n=${String(arr.length).padStart(4)}  avg=${avg.toFixed(2).padStart(6)}  median=${median.toFixed(2).padStart(6)}  p90=${p90.toFixed(2).padStart(6)}`);
  }

  console.log(`\n--- 5. TRAP MARKET PREVALENCE (main_pick) ---`);
  console.log(`  Sampled main_picks with triple_value: ${trapMarketSampled}`);
  console.log(`  Trap candidates (implied - band_rate > 0.10): ${trapMarketCount}`);
  if (trapMarketSampled > 0) {
    console.log(`  Trap rate: ${((trapMarketCount / trapMarketSampled) * 100).toFixed(1)}%`);
  }

  console.log(`\n--- 6. "hafif favori" COMMENTARY ACCURACY ---`);
  console.log(`  Used in commentary: ${hafifFavoriUseCount}`);
  console.log(`  Correctly predicted winner: ${hafifFavoriCorrectCount}`);
  if (hafifFavoriUseCount > 0) {
    console.log(`  Accuracy: ${((hafifFavoriCorrectCount / hafifFavoriUseCount) * 100).toFixed(1)}%`);
  }

  console.log(`\n=== DONE ===\n`);
  void bucket;
}

main()
  .catch((e) => {
    console.error(e);
    process.exit(1);
  })
  .finally(async () => {
    await prisma.$disconnect();
  });