vv

src/modules/admin/admin.controller.ts

@@ -394,4 +394,202 @@ export class AdminController {
       predictions: totalPredictions,
     });
   }
+
+  // ================== Model Performance (Forward-Test) ==================
+
+  @Get("model-performance")
+  @ApiOperation({
+    summary:
+      "Per-market calibration (model% vs actual%), ROI and decision rationale " +
+      "from settled prediction_runs. Powers the admin Model Performance page.",
+  })
+  @SwaggerResponse({ status: 200, schema: { type: "object" } })
+  async getModelPerformance(
+    @Query("days") daysRaw?: string,
+  ): Promise<ApiResponse<ModelPerformanceResult>> {
+    const days = Math.min(Math.max(Number(daysRaw) || 90, 1), 1000);
+    const sinceMs = Date.now() - days * 24 * 60 * 60 * 1000;
+
+    // Pull settled rows in window. markets_settled holds one entry per market.
+    const rows = await this.prisma.$queryRawUnsafe<
+      Array<{ payload_summary: unknown; generated_at: Date }>
+    >(
+      `
+        SELECT pr.payload_summary, pr.generated_at
+        FROM prediction_runs pr
+        WHERE pr.eventual_outcome IS NOT NULL
+          AND pr.generated_at >= $1
+          AND pr.payload_summary -> 'settlement' -> 'markets_settled' IS NOT NULL
+        ORDER BY pr.generated_at DESC
+        LIMIT 50000
+      `,
+      new Date(sinceMs),
+    );
+
+    // ── Aggregate per market ──────────────────────────────────────────
+    type Acc = {
+      market: string;
+      n: number;
+      wins: number;
+      sumShown: number; // Σ shown_confidence (0–100)
+      shownCount: number;
+      // 10-bin reliability for ECE
+      bins: Array<{ sumP: number; sumY: number; n: number }>;
+      // betting (only playable BET rows with odds)
+      betN: number;
+      betWins: number;
+      betProfit: number;
+      betStake: number;
+      // rationale tally
+      actions: Record<string, number>;
+      tiers: Record<string, number>;
+    };
+    const acc = new Map<string, Acc>();
+    const ensure = (mk: string): Acc => {
+      let a = acc.get(mk);
+      if (!a) {
+        a = {
+          market: mk,
+          n: 0,
+          wins: 0,
+          sumShown: 0,
+          shownCount: 0,
+          bins: Array.from({ length: 10 }, () => ({ sumP: 0, sumY: 0, n: 0 })),
+          betN: 0,
+          betWins: 0,
+          betProfit: 0,
+          betStake: 0,
+          actions: {},
+          tiers: {},
+        };
+        acc.set(mk, a);
+      }
+      return a;
+    };
+
+    let totalSettledMarkets = 0;
+    for (const row of rows) {
+      const summary =
+        row.payload_summary && typeof row.payload_summary === "object"
+          ? (row.payload_summary as Record<string, unknown>)
+          : {};
+      const settlement =
+        summary.settlement && typeof summary.settlement === "object"
+          ? (summary.settlement as Record<string, unknown>)
+          : {};
+      const markets = Array.isArray(settlement.markets_settled)
+        ? (settlement.markets_settled as Array<Record<string, unknown>>)
+        : [];
+
+      for (const m of markets) {
+        const market = typeof m.market === "string" ? m.market : "";
+        if (!market) continue;
+        const won = m.won === true;
+        const shown =
+          m.shown_confidence != null ? Number(m.shown_confidence) : null;
+        const a = ensure(market);
+        a.n += 1;
+        if (won) a.wins += 1;
+        totalSettledMarkets += 1;
+
+        if (shown != null && Number.isFinite(shown)) {
+          a.sumShown += shown;
+          a.shownCount += 1;
+          const p = Math.min(Math.max(shown / 100, 0), 0.999999);
+          const bi = Math.min(9, Math.floor(p * 10));
+          a.bins[bi].sumP += p;
+          a.bins[bi].sumY += won ? 1 : 0;
+          a.bins[bi].n += 1;
+        }
+
+        const odds = m.odds != null ? Number(m.odds) : null;
+        const isBet = m.playable === true && m.action === "BET";
+        if (isBet && odds != null && Number.isFinite(odds) && odds > 1.01) {
+          a.betN += 1;
+          if (won) {
+            a.betWins += 1;
+            a.betProfit += odds - 1;
+          } else {
+            a.betProfit -= 1;
+          }
+          a.betStake += 1;
+        }
+
+        const action = typeof m.action === "string" ? m.action : "—";
+        a.actions[action] = (a.actions[action] ?? 0) + 1;
+        const tier = typeof m.value_tier === "string" ? m.value_tier : "—";
+        a.tiers[tier] = (a.tiers[tier] ?? 0) + 1;
+      }
+    }
+
+    const markets = Array.from(acc.values())
+      .map((a) => {
+        const actualPct = a.n > 0 ? (a.wins / a.n) * 100 : 0;
+        const shownPct = a.shownCount > 0 ? a.sumShown / a.shownCount : 0;
+        // ECE: Σ |acc_bin - conf_bin| * (n_bin / N)
+        let ece = 0;
+        for (const b of a.bins) {
+          if (b.n === 0) continue;
+          const conf = b.sumP / b.n;
+          const acc2 = b.sumY / b.n;
+          ece += Math.abs(acc2 - conf) * (b.n / a.shownCount || 0);
+        }
+        return {
+          market: a.market,
+          samples: a.n,
+          shown_pct: Number(shownPct.toFixed(1)),
+          actual_pct: Number(actualPct.toFixed(1)),
+          gap: Number((shownPct - actualPct).toFixed(1)),
+          ece: Number((ece * 100).toFixed(1)),
+          calibration: (Math.abs(shownPct - actualPct) <= 4
+            ? "good"
+            : shownPct > actualPct
+              ? "overconfident"
+              : "underconfident") as
+            | "good"
+            | "overconfident"
+            | "underconfident",
+          bet_count: a.betN,
+          bet_hit_pct:
+            a.betN > 0 ? Number(((a.betWins / a.betN) * 100).toFixed(1)) : 0,
+          bet_roi_pct:
+            a.betStake > 0
+              ? Number(((a.betProfit / a.betStake) * 100).toFixed(1))
+              : 0,
+          actions: a.actions,
+          tiers: a.tiers,
+        };
+      })
+      .sort((x, y) => y.samples - x.samples);
+
+    const result: ModelPerformanceResult = {
+      window_days: days,
+      settled_runs: Number(rows.length),
+      settled_markets: totalSettledMarkets,
+      generated_at: new Date().toISOString(),
+      markets,
+    };
+    return createSuccessResponse(result);
+  }
+}
+
+interface ModelPerformanceResult {
+  window_days: number;
+  settled_runs: number;
+  settled_markets: number;
+  generated_at: string;
+  markets: Array<{
+    market: string;
+    samples: number;
+    shown_pct: number;
+    actual_pct: number;
+    gap: number;
+    ece: number;
+    calibration: "good" | "overconfident" | "underconfident";
+    bet_count: number;
+    bet_hit_pct: number;
+    bet_roi_pct: number;
+    actions: Record<string, number>;
+    tiers: Record<string, number>;
+  }>;
 }

src/modules/predictions/dto/index.ts

@@ -27,6 +27,20 @@ export class MatchInfoDto {
   @ApiProperty({ required: false, default: false })
   is_top_league?: boolean;
 
+  @ApiProperty({
+    required: false,
+    nullable: true,
+    description:
+      "Backtest-derived per-league confidence (ROI + sample size). " +
+      "null when the league has too little data to judge.",
+  })
+  league_confidence?: {
+    label: "high" | "medium" | "low";
+    bet_roi: number;
+    bet_n: number;
+    hit: number;
+  } | null;
+
   @ApiProperty({
     required: false,
     enum: ["football", "basketball"],

src/modules/predictions/predictions.service.ts

@@ -1611,7 +1611,69 @@ export class PredictionsService implements OnModuleInit, OnModuleDestroy {
         }))
       : [];
 
+    // ── Forward-test capture (V31e) ────────────────────────────────────
+    // Persist EVERY market's probability + the rationale behind each pick so
+    // the settlement job can later score each market against reality and the
+    // admin "Model Performance" page can show per-market calibration
+    // (model% → actual%) and decision reasons. Compact projection only.
+    // Some runtime fields (betting_brain, is_underdog_reference) are present
+    // in the AI payload but not declared on the DTO — read them via a cast.
+    const marketsFull = Array.isArray(payload.bet_summary)
+      ? payload.bet_summary.map((item) => {
+          const loose = item as unknown as Record<string, unknown>;
+          const bb = (loose.betting_brain ?? {}) as Record<string, unknown>;
+          return {
+            market: item.market,
+            pick: item.pick,
+            odds: item.odds ?? null,
+            model_probability: item.model_probability ?? null,
+            calibrated_confidence: item.calibrated_confidence ?? null,
+            raw_confidence: item.raw_confidence ?? null,
+            calibrated_probability: item.calibrated_probability ?? null,
+            implied_prob: item.implied_prob ?? null,
+            ev_edge: item.ev_edge ?? 0,
+            playable: item.playable ?? false,
+            bet_grade: item.bet_grade ?? "PASS",
+            signal_tier: item.signal_tier ?? null,
+            stake_units: item.stake_units ?? 0,
+            is_underdog_reference: Boolean(loose.is_underdog_reference),
+            action: (bb.action as string | undefined) ?? null,
+            value_tier: (bb.value_tier as string | undefined) ?? null,
+            model_market_gap: (bb.model_market_gap as number | undefined) ?? null,
+            trap_market_flag: Boolean(bb.trap_market_flag),
+            vetoes: Array.isArray(bb.vetoes)
+              ? (bb.vetoes as unknown[]).slice(0, 6)
+              : [],
+            positives: Array.isArray(bb.positives)
+              ? (bb.positives as unknown[]).slice(0, 6)
+              : [],
+            reasons: Array.isArray(item.reasons) ? item.reasons.slice(0, 6) : [],
+          };
+        })
+      : [];
+
+    // Per-outcome probability distribution for each market (graph bars).
+    const marketBoardProbs =
+      payload.market_board && typeof payload.market_board === "object"
+        ? Object.fromEntries(
+            Object.entries(
+              payload.market_board as Record<string, { probs?: unknown }>,
+            ).map(([mkt, entry]) => [
+              mkt,
+              entry && typeof entry === "object" ? (entry.probs ?? null) : null,
+            ]),
+          )
+        : {};
+
     return {
+      markets_full: marketsFull,
+      market_board_probs: marketBoardProbs,
+      betting_brain_version:
+        (
+          (payload as unknown as Record<string, unknown>).betting_brain as
+            | { version?: string }
+            | undefined
+        )?.version ?? null,
       model_version: payload.model_version,
       calibration_version: payload.calibration_version ?? null,
       shadow_engine_version: payload.shadow_engine_version ?? null,

src/tasks/data-fetcher.task.ts

@@ -399,6 +399,12 @@ export class DataFetcherTask {
         const closingOddsSnapshot = await this.getClosingOddsSnapshot(
           row.matchId,
         );
+        // ── Per-market settlement (V31e forward-test) ────────────────
+        // Score EVERY captured market (not just main_pick) against reality,
+        // so the admin Model Performance page can compute per-market
+        // calibration (model% → actual%) and ROI. won=null → push (skip).
+        const marketsSettled = this.settleAllMarkets(row);
+
         const settlementSummary = {
           settled_at: new Date().toISOString(),
           model_version: row.engineVersion,
@@ -413,6 +419,7 @@ export class DataFetcherTask {
             away: row.htScoreAway,
           },
           closing_odds_snapshot: closingOddsSnapshot,
+          markets_settled: marketsSettled,
         };
 
         await this.prisma.$executeRawUnsafe(
@@ -538,6 +545,64 @@ export class DataFetcherTask {
     };
   }
 
+  /**
+   * V31e forward-test: settle EVERY captured market (payload_summary.markets_full)
+   * against the final score. Produces one compact record per market with its
+   * shown probability, the real outcome, and flat profit — the raw material for
+   * per-market calibration (model% vs actual%) on the admin dashboard.
+   */
+  private settleAllMarkets(
+    row: PendingPredictionRunForSettlement,
+  ): Array<Record<string, unknown>> {
+    const summary = this.asRecord(row.payloadSummary);
+    const markets = Array.isArray(summary.markets_full)
+      ? (summary.markets_full as unknown[])
+      : [];
+    const out: Array<Record<string, unknown>> = [];
+
+    for (const raw of markets) {
+      const m = this.asRecord(raw);
+      const market = typeof m.market === "string" ? m.market : "";
+      const pick = typeof m.pick === "string" ? m.pick : "";
+      if (!market || !pick) continue;
+
+      const won = this.isPredictionPickWon({
+        market,
+        pick,
+        scoreHome: row.scoreHome,
+        scoreAway: row.scoreAway,
+        htScoreHome: row.htScoreHome,
+        htScoreAway: row.htScoreAway,
+      });
+      if (won === null) continue; // push / unresolvable → exclude from stats
+
+      const odds = Number(m.odds || 0);
+      const hasOdds = Number.isFinite(odds) && odds > 1.01;
+      out.push({
+        market,
+        pick,
+        won,
+        // shown probability for calibration (0–100). Prefer calibrated_confidence.
+        shown_confidence:
+          m.calibrated_confidence != null
+            ? Number(m.calibrated_confidence)
+            : m.model_probability != null
+              ? Number(m.model_probability) * 100
+              : null,
+        model_probability:
+          m.model_probability != null ? Number(m.model_probability) : null,
+        odds: hasOdds ? odds : null,
+        playable: m.playable === true,
+        bet_grade: typeof m.bet_grade === "string" ? m.bet_grade : null,
+        action: typeof m.action === "string" ? m.action : null,
+        value_tier: typeof m.value_tier === "string" ? m.value_tier : null,
+        // flat 1u profit if a real price existed (for per-market ROI)
+        flat_profit: hasOdds ? Number((won ? odds - 1 : -1).toFixed(4)) : null,
+      });
+    }
+    return out;
+  }
+
   private isPredictionPickWon(input: {
     market: string;
     pick: string;