gg

src/scripts/backfill-prediction-runs.ts

@@ -0,0 +1,234 @@
+/**
+ * ===================================================
+ * HISTORICAL PREDICTION BACKFILL
+ * ===================================================
+ * Replays v28-pro-max (or any engine) against historical
+ * finished matches to populate prediction_runs with
+ * payload_summary + odds_snapshot + eventual_outcome +
+ * unit_profit, so we can compute true ROI / calibration.
+ *
+ * Requires ENABLE_BACKFILL=true to prevent accidental run.
+ *
+ * Usage:
+ *   ENABLE_BACKFILL=true npx ts-node --transpile-only \
+ *     -r tsconfig-paths/register src/scripts/backfill-prediction-runs.ts \
+ *     --from 2023-05-01 --to 2025-12-31 --limit 50000 --sport football
+ */
+
+import { PrismaClient } from "@prisma/client";
+import axios from "axios";
+import {
+  resolveOutcomeForPick,
+  computeUnitProfit,
+  type MatchResult,
+} from "../tasks/prediction-settlement.market-resolver";
+
+const prisma = new PrismaClient();
+const AI_ENGINE_URL = process.env.AI_ENGINE_URL || "http://127.0.0.1:3005";
+const REQUEST_DELAY_MS = Number(process.env.BACKFILL_DELAY_MS ?? 300);
+const CONCURRENCY = Number(process.env.BACKFILL_CONCURRENCY ?? 2);
+
+function parseArgs() {
+  const args = process.argv.slice(2);
+  const opts: Record<string, string> = {};
+  for (let i = 0; i < args.length; i += 2) {
+    opts[args[i].replace(/^--/, "")] = args[i + 1];
+  }
+  return {
+    from: opts.from ?? "2023-05-01",
+    to: opts.to ?? new Date().toISOString().slice(0, 10),
+    limit: Number(opts.limit ?? 50000),
+    sport: (opts.sport ?? "football") as "football" | "basketball",
+    engine: opts.engine ?? "v28-pro-max",
+  };
+}
+
+function sleep(ms: number) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+async function runOne(matchId: string, engine: string): Promise<any | null> {
+  try {
+    const resp = await axios.post(
+      `${AI_ENGINE_URL}/v20plus/analyze/${matchId}`,
+      { engine_version: engine },
+      { timeout: 25000 },
+    );
+    return resp.data;
+  } catch (err: any) {
+    console.error(
+      `  ✗ ${matchId} ${err?.response?.status ?? ""} ${err?.message ?? ""}`,
+    );
+    return null;
+  }
+}
+
+function buildPayloadSummary(payload: any): Record<string, any> {
+  const main = payload?.main_pick ?? null;
+  const value = payload?.value_pick ?? null;
+  const advice = payload?.bet_advice ?? {};
+  return {
+    model_version: payload?.model_version,
+    decision_trace_id: payload?.decision_trace_id ?? null,
+    main_pick: main
+      ? {
+          market: main.market,
+          pick: main.pick,
+          playable: main.playable,
+          bet_grade: main.bet_grade,
+          calibrated_confidence: main.calibrated_confidence,
+          ev_edge: main.ev_edge ?? 0,
+          stake_units: main.stake_units,
+          odds: main.odds ?? null,
+        }
+      : null,
+    value_pick: value
+      ? {
+          market: value.market,
+          pick: value.pick,
+          playable: value.playable,
+          calibrated_confidence: value.calibrated_confidence,
+          ev_edge: value.ev_edge ?? 0,
+          odds: value.odds ?? null,
+        }
+      : null,
+    bet_advice: {
+      playable: advice.playable ?? false,
+      suggested_stake_units: advice.suggested_stake_units ?? 0,
+    },
+  };
+}
+
+function settleFromPayload(
+  payload: any,
+  result: MatchResult,
+): { outcome: string; unitProfit: number } {
+  const advice = payload?.bet_advice ?? {};
+  if (advice.playable !== true) return { outcome: "NO_BET", unitProfit: 0 };
+
+  const main = payload?.main_pick;
+  if (!main || !main.playable) return { outcome: "NO_BET", unitProfit: 0 };
+
+  const pickRef = {
+    market: String(main.market),
+    pick: String(main.pick),
+    stake_units: Number(main.stake_units ?? advice.suggested_stake_units ?? 1),
+    odds: Number(main.odds ?? 0) || null,
+  };
+  const won = resolveOutcomeForPick(pickRef, result);
+  if (won === null) {
+    return { outcome: "NO_BET", unitProfit: 0 };
+  }
+  return {
+    outcome: `${won ? "WON" : "LOST"}:${pickRef.market}:${pickRef.pick}`,
+    unitProfit: computeUnitProfit(won, pickRef.stake_units, pickRef.odds),
+  };
+}
+
+async function main() {
+  if (process.env.ENABLE_BACKFILL !== "true") {
+    console.error(
+      "✗ Backfill is gated. Re-run with ENABLE_BACKFILL=true to proceed.",
+    );
+    process.exit(1);
+  }
+
+  const opts = parseArgs();
+  console.log("📦 Backfill prediction_runs");
+  console.log(`   Range: ${opts.from} → ${opts.to}`);
+  console.log(`   Sport: ${opts.sport}`);
+  console.log(`   Engine: ${opts.engine}`);
+  console.log(`   Limit: ${opts.limit}`);
+  console.log(`   AI engine: ${AI_ENGINE_URL}`);
+
+  const fromMs = BigInt(new Date(opts.from).getTime());
+  const toMs = BigInt(new Date(opts.to).getTime());
+
+  const matches = await prisma.match.findMany({
+    where: {
+      sport: opts.sport as any,
+      status: "FT",
+      scoreHome: { not: null },
+      scoreAway: { not: null },
+      mstUtc: { gte: fromMs, lte: toMs },
+    },
+    orderBy: { mstUtc: "asc" },
+    take: opts.limit,
+    select: {
+      id: true,
+      mstUtc: true,
+      scoreHome: true,
+      scoreAway: true,
+      htScoreHome: true,
+      htScoreAway: true,
+    },
+  });
+
+  console.log(`\n   ${matches.length} finished matches in range\n`);
+
+  let predicted = 0;
+  let written = 0;
+  let skipped = 0;
+
+  for (let i = 0; i < matches.length; i += CONCURRENCY) {
+    const batch = matches.slice(i, i + CONCURRENCY);
+    await Promise.all(
+      batch.map(async (m) => {
+        const existing = await prisma.predictionRun.findFirst({
+          where: { matchId: m.id, engineVersion: opts.engine },
+          select: { id: true },
+        });
+        if (existing) {
+          skipped += 1;
+          return;
+        }
+
+        const payload = await runOne(m.id, opts.engine);
+        if (!payload) return;
+        predicted += 1;
+
+        const summary = buildPayloadSummary(payload);
+        const result: MatchResult = {
+          scoreHome: m.scoreHome!,
+          scoreAway: m.scoreAway!,
+          htScoreHome: m.htScoreHome,
+          htScoreAway: m.htScoreAway,
+        };
+        const settled = settleFromPayload(payload, result);
+
+        await prisma.predictionRun.create({
+          data: {
+            matchId: m.id,
+            engineVersion: payload?.model_version ?? opts.engine,
+            decisionTraceId: payload?.decision_trace_id ?? null,
+            oddsSnapshot: payload?.odds_snapshot ?? null,
+            payloadSummary: summary,
+            eventualOutcome: settled.outcome,
+            unitProfit: settled.unitProfit,
+          },
+        });
+        written += 1;
+      }),
+    );
+
+    if ((i / CONCURRENCY) % 25 === 0) {
+      console.log(
+        `   [${i + batch.length}/${matches.length}] predicted=${predicted} written=${written} skipped=${skipped}`,
+      );
+    }
+    await sleep(REQUEST_DELAY_MS);
+  }
+
+  console.log("\n✅ Backfill complete");
+  console.log(`   Predicted: ${predicted}`);
+  console.log(`   Written:   ${written}`);
+  console.log(`   Skipped:   ${skipped}`);
+
+  await prisma.$disconnect();
+}
+
+main().catch(async (err) => {
+  console.error(err);
+  await prisma.$disconnect();
+  process.exit(1);
+});

src/scripts/print-backtest-report.ts

@@ -0,0 +1,208 @@
+/**
+ * ===================================================
+ * BACKTEST REPORT
+ * ===================================================
+ * Reads prediction_runs (with eventual_outcome + unit_profit
+ * already filled by settlement / backfill) and prints
+ * per-engine x per-market ROI tables. Also writes JSON.
+ *
+ * Usage:
+ *   npx ts-node --transpile-only -r tsconfig-paths/register \
+ *     src/scripts/print-backtest-report.ts [--engine v28-pro-max]
+ */
+
+import { PrismaClient, Prisma } from "@prisma/client";
+import * as fs from "fs";
+import * as path from "path";
+
+const prisma = new PrismaClient();
+
+interface RawRow {
+  engine_version: string;
+  market: string | null;
+  bet_grade: string | null;
+  outcome_kind: "WON" | "LOST" | "NO_BET";
+  unit_profit: number | null;
+  count: bigint;
+}
+
+interface AggregatedBucket {
+  engine: string;
+  market: string;
+  betGrade: string;
+  totalBets: number;
+  wins: number;
+  losses: number;
+  noBets: number;
+  totalProfit: number;
+  roi: number;
+  winRate: number;
+}
+
+function parseArgs() {
+  const args = process.argv.slice(2);
+  const opts: Record<string, string> = {};
+  for (let i = 0; i < args.length; i += 2) {
+    opts[args[i].replace(/^--/, "")] = args[i + 1];
+  }
+  return { engineFilter: opts.engine ?? null };
+}
+
+async function loadRows(engineFilter: string | null): Promise<RawRow[]> {
+  const engineClause = engineFilter
+    ? Prisma.sql`AND pr.engine_version = ${engineFilter}`
+    : Prisma.sql``;
+
+  return prisma.$queryRaw<RawRow[]>(Prisma.sql`
+    SELECT pr.engine_version,
+           pr.payload_summary->'main_pick'->>'market' AS market,
+           pr.payload_summary->'main_pick'->>'bet_grade' AS bet_grade,
+           CASE
+             WHEN pr.eventual_outcome LIKE 'WON:%' THEN 'WON'
+             WHEN pr.eventual_outcome LIKE 'LOST:%' THEN 'LOST'
+             ELSE 'NO_BET'
+           END AS outcome_kind,
+           pr.unit_profit,
+           1::bigint AS count
+    FROM prediction_runs pr
+    WHERE pr.eventual_outcome IS NOT NULL
+    ${engineClause}
+  `);
+}
+
+function aggregate(rows: RawRow[]): AggregatedBucket[] {
+  const map = new Map<string, AggregatedBucket>();
+  for (const r of rows) {
+    const market = r.market ?? "(none)";
+    const betGrade = r.bet_grade ?? "(none)";
+    const key = `${r.engine_version}|${market}|${betGrade}`;
+    if (!map.has(key)) {
+      map.set(key, {
+        engine: r.engine_version,
+        market,
+        betGrade,
+        totalBets: 0,
+        wins: 0,
+        losses: 0,
+        noBets: 0,
+        totalProfit: 0,
+        roi: 0,
+        winRate: 0,
+      });
+    }
+    const b = map.get(key)!;
+    if (r.outcome_kind === "WON") b.wins += 1;
+    else if (r.outcome_kind === "LOST") b.losses += 1;
+    else b.noBets += 1;
+    if (r.outcome_kind !== "NO_BET") b.totalBets += 1;
+    b.totalProfit += Number(r.unit_profit ?? 0);
+  }
+
+  const out = Array.from(map.values());
+  for (const b of out) {
+    b.roi = b.totalBets > 0 ? b.totalProfit / b.totalBets : 0;
+    b.winRate = b.totalBets > 0 ? b.wins / b.totalBets : 0;
+  }
+  return out.sort((a, b) => b.totalBets - a.totalBets);
+}
+
+function printTable(buckets: AggregatedBucket[]) {
+  const header = [
+    "engine".padEnd(20),
+    "market".padEnd(10),
+    "grade".padEnd(6),
+    "bets".padStart(7),
+    "wins".padStart(7),
+    "losses".padStart(7),
+    "noBet".padStart(7),
+    "winRate".padStart(8),
+    "profit".padStart(10),
+    "ROI".padStart(8),
+  ].join(" │ ");
+  console.log(header);
+  console.log("─".repeat(header.length));
+  for (const b of buckets) {
+    console.log(
+      [
+        b.engine.padEnd(20),
+        b.market.padEnd(10),
+        b.betGrade.padEnd(6),
+        String(b.totalBets).padStart(7),
+        String(b.wins).padStart(7),
+        String(b.losses).padStart(7),
+        String(b.noBets).padStart(7),
+        `${(b.winRate * 100).toFixed(1)}%`.padStart(8),
+        b.totalProfit.toFixed(2).padStart(10),
+        `${(b.roi * 100).toFixed(2)}%`.padStart(8),
+      ].join(" │ "),
+    );
+  }
+}
+
+function engineSummary(buckets: AggregatedBucket[]) {
+  const byEngine = new Map<string, AggregatedBucket>();
+  for (const b of buckets) {
+    if (!byEngine.has(b.engine)) {
+      byEngine.set(b.engine, {
+        engine: b.engine,
+        market: "ALL",
+        betGrade: "ALL",
+        totalBets: 0,
+        wins: 0,
+        losses: 0,
+        noBets: 0,
+        totalProfit: 0,
+        roi: 0,
+        winRate: 0,
+      });
+    }
+    const acc = byEngine.get(b.engine)!;
+    acc.totalBets += b.totalBets;
+    acc.wins += b.wins;
+    acc.losses += b.losses;
+    acc.noBets += b.noBets;
+    acc.totalProfit += b.totalProfit;
+  }
+  const out = Array.from(byEngine.values());
+  for (const e of out) {
+    e.roi = e.totalBets > 0 ? e.totalProfit / e.totalBets : 0;
+    e.winRate = e.totalBets > 0 ? e.wins / e.totalBets : 0;
+  }
+  return out;
+}
+
+async function main() {
+  const { engineFilter } = parseArgs();
+  console.log("📊 Backtest report");
+  if (engineFilter) console.log(`   filter engine = ${engineFilter}`);
+
+  const rows = await loadRows(engineFilter);
+  console.log(`   ${rows.length} resolved prediction_runs loaded\n`);
+
+  const buckets = aggregate(rows);
+  const enginesAll = engineSummary(buckets);
+
+  console.log("=== Per engine (all markets) ===");
+  printTable(enginesAll);
+  console.log("\n=== Per engine × market × bet_grade ===");
+  printTable(buckets);
+
+  const ts = new Date().toISOString().replace(/[:.]/g, "-");
+  const outDir = path.join(process.cwd(), "reports");
+  if (!fs.existsSync(outDir)) fs.mkdirSync(outDir);
+  const outFile = path.join(outDir, `backtest-${ts}.json`);
+  fs.writeFileSync(
+    outFile,
+    JSON.stringify({ engines: enginesAll, buckets }, null, 2),
+    "utf8",
+  );
+  console.log(`\n💾 Saved ${outFile}`);
+
+  await prisma.$disconnect();
+}
+
+main().catch(async (err) => {
+  console.error(err);
+  await prisma.$disconnect();
+  process.exit(1);
+});