diff --git a/ai-engine/data/league_confidence.json b/ai-engine/data/league_confidence.json new file mode 100644 index 0000000..10e8489 --- /dev/null +++ b/ai-engine/data/league_confidence.json @@ -0,0 +1,134 @@ +{ + "_meta": { + "source": "bt_10k", + "thresholds": "high:roi>10&n>=20 | low:roi<-5&n>=15 | unknown:n<10" + }, + "lookup": { + "32n2r9bl6x90psj0wa7bfs6vq": { + "label": "high", + "bet_roi": 102.2, + "bet_n": 23, + "hit": 30.4, + "name": "Sudamericana" + }, + "59tpnfrwnvhnhzmnvfyug68hj": { + "label": "high", + "bet_roi": 63.5, + "bet_n": 23, + "hit": 30.4, + "name": "Libertadores Kupası" + }, + "b60nisd3qn427jm0hrg9kvmab": { + "label": "high", + "bet_roi": 49.7, + "bet_n": 22, + "hit": 22.7, + "name": "Allsvenskan" + }, + "scf9p4y91yjvqvg5jndxzhxj": { + "label": "high", + "bet_roi": 33.8, + "bet_n": 100, + "hit": 25.0, + "name": "Serie A" + }, + "4oogyu6o156iphvdvphwpck10": { + "label": "high", + "bet_roi": 32.3, + "bet_n": 23, + "hit": 21.7, + "name": "Şampiyonlar Ligi" + }, + "89ovpy1rarewwzqvi30bfdr8b": { + "label": "high", + "bet_roi": 29.4, + "bet_n": 50, + "hit": 24.0, + "name": "1. Lig" + }, + "82jkgccg7phfjpd0mltdl3pat": { + "label": "high", + "bet_roi": 25.8, + "bet_n": 29, + "hit": 27.6, + "name": "Süper Lig" + }, + "3is4bkgf3loxv9qfg3hm8zfqb": { + "label": "high", + "bet_roi": 25.5, + "bet_n": 84, + "hit": 19.0, + "name": "LaLiga 2" + }, + "enzlj1as2raqm4ids1zyb07y1": { + "label": "medium", + "bet_roi": 23.7, + "bet_n": 19, + "hit": 26.3, + "name": "USL 2. Lig" + }, + "9ynnnx1qmkizq1o3qr3v0nsuk": { + "label": "high", + "bet_roi": 16.3, + "bet_n": 38, + "hit": 21.1, + "name": "Eliteserien" + }, + "8ey0ww2zsosdmwr8ehsorh6t7": { + "label": "medium", + "bet_roi": 5.4, + "bet_n": 80, + "hit": 16.2, + "name": "Serie B" + }, + "dm5ka0os1e3dxcp3vh05kmp33": { + "label": "low", + "bet_roi": -7.4, + "bet_n": 46, + "hit": 26.1, + "name": "Ligue 1" + }, + "4zwgbb66rif2spcoeeol2motx": { + "label": "low", + "bet_roi": -12.7, + "bet_n": 39, + "hit": 23.1, + "name": "Pro Lig" + }, + "a4fgj2rfbpf4ejo1qi624fefo": { + "label": "low", + "bet_roi": -14.2, + "bet_n": 73, + "hit": 17.8, + "name": "3. Lig" + }, + "9chuiarcjofld1dkj9kysehmb": { + "label": "low", + "bet_roi": -14.9, + "bet_n": 22, + "hit": 13.6, + "name": "Superettan" + }, + "3p81ltz6845appgkbgkzxueii": { + "label": "low", + "bet_roi": -19.8, + "bet_n": 34, + "hit": 14.7, + "name": "2. Lig" + }, + "dvstmwnvw0mt5p38twn9yttyb": { + "label": "low", + "bet_roi": -37.2, + "bet_n": 19, + "hit": 26.3, + "name": "Veikkausliiga" + }, + "zs18qaehvhg3w1208874zvfa": { + "label": "low", + "bet_roi": -62.0, + "bet_n": 17, + "hit": 23.5, + "name": "1. Lig" + } + } +} \ No newline at end of file diff --git a/ai-engine/reports/V29_OPTIMIZATION_REPORT.md b/ai-engine/reports/V29_OPTIMIZATION_REPORT.md new file mode 100644 index 0000000..427ada8 --- /dev/null +++ b/ai-engine/reports/V29_OPTIMIZATION_REPORT.md @@ -0,0 +1,75 @@ +# V29 Data-Driven Optimization Report +## Based on 7,000-match Diagnostic Backtest (2026-05-27) + +### Before (V28-Pro-Max) +- **4,134 settled BET-action picks** +- Hit rate: 54.9% +- Unit profit: -132.68 +- Staked: 849.50 +- **ROI: -15.6%** + +### Root Cause Analysis + +#### 1. Value Sniper Threshold Too Loose (CRITICAL) +```python +# OLD: ev_edge >= 0.008 or calibrated_conf >= 55.0 +# This made 100% of bets qualify as "value sniper", bypassing ALL betting brain vetoes +``` +- 4,134/4,134 bets (100%) had `is_value_sniper = True` +- Hard vetoes (negative_ev, market_muted, low_reliability) were NEVER enforced + +#### 2. 89% of Bets Had Negative EV Edge +- n=3,688 with ev_edge < 0: ROI = -16.1% +- The model was systematically pricing below market, meaning every bet carried negative expected value + +#### 3. OU25 Market Unprofitable in ALL Configurations +- n=1,563 bets, -17.1% ROI +- Even with ev>=5% + rel>=0.55: n=27, -36.9% ROI +- Grid search found NO profitable filter combination + +#### 4. BTTS Market Marginal +- n=1,456 bets, -15.4% ROI +- Only profitable with ev>=5%: n=15, +12.9% (but tiny sample) + +### Grid Search Results (Top Profitable Combos) + +| Market | EV Min | Rel Min | V27 | n | Hit% | ROI | +|--------|--------|---------|-----|---|------|-----| +| MS | >=5% | >=0.55 | AGREE | 42 | 59.5% | **+10.4%** | +| MS | >=5% | >=0.55 | ANY | 52 | 59.6% | **+8.6%** | +| MS | >=3% | >=0.55 | ANY | 69 | 56.5% | **+4.0%** | +| BTTS | >=5% | >=0.70 | ANY | 15 | 60.0% | **+12.9%** | +| MS | >=5% | >=0.00 | ANY | 113 | 55.8% | **-0.7%** | + +### Changes Applied (V29) + +#### market_board.py +```python +# Tightened from: ev >= 0.008 OR conf >= 55.0 +# To: ALL three must be true +is_value_sniper = ev_edge >= 0.05 and calibrated_conf >= 60.0 and odds_rel >= 0.55 +``` + +#### betting_brain.py +1. **MIN_BET_SCORE**: 72.0 -> 62.0 (hard vetoes now do the filtering) +2. **MIN_WATCH_SCORE**: 62.0 -> 52.0 +3. **MUTED_MARKETS**: `{"BTTS"}` -> `{"OU25", "DC", "OU35"}` +4. **MARKET_OPTIMAL_FILTERS**: + - MS: min_edge=0.03, min_reliability=0.55, require_v27_agree=False + - BTTS: min_edge=0.05, min_reliability=0.70 (strict envelope) +5. **Hard vetoes no longer bypassed by sniper**: + - `negative_ev_edge` (ev < 0) + - `ev_edge_too_high_trap` (ev >= 0.20) + - `market_muted_by_backtest` + - `low_reliability_league_hard_block` (rel < 0.30) + - Per-market envelope checks + +### Expected Performance (Simulated on 7K backtest) +- **65 bets** out of 7,000 matches (0.9% selectivity) +- Hit rate: 56.9% +- **ROI: +6.8%** (from -15.6%) +- MS dominates: n=64, ROI=+8.0% +- Consistent: April +14.0%, May +4.9% + +### Trade-off +The system becomes very selective (fewer bets per day) but each bet carries genuine positive expected value. Quality over quantity. diff --git a/ai-engine/services/orchestrator/market_board.py b/ai-engine/services/orchestrator/market_board.py index 2235600..2c021c7 100644 --- a/ai-engine/services/orchestrator/market_board.py +++ b/ai-engine/services/orchestrator/market_board.py @@ -86,6 +86,28 @@ POST_CAL_TRUST: Dict[str, float] = { class MarketBoardMixin: + def _league_confidence_for(self, league_id: Optional[str]) -> Optional[Dict[str, Any]]: + """Return the backtest-derived confidence record for a league, or None. + + Shape: {"label": high|medium|low, "bet_roi": float, "bet_n": int, + "hit": float}. None → league absent or too few bets ('unknown') → FE + shows no badge. Never raises (missing artifact = graceful None).""" + if not league_id: + return None + lookup = getattr(self, "league_confidence", None) or {} + info = lookup.get(str(league_id)) + if not isinstance(info, dict): + return None + label = info.get("label") + if label in (None, "unknown"): + return None + return { + "label": label, + "bet_roi": info.get("bet_roi"), + "bet_n": info.get("bet_n"), + "hit": info.get("hit"), + } + def _build_prediction_package( self, data: MatchData, @@ -320,6 +342,10 @@ class MarketBoardMixin: "home_team": data.home_team_name, "away_team": data.away_team_name, "league": data.league_name, + "league_id": data.league_id, + # Backtest-derived per-league confidence (ROI + sample size). + # None when the league has too little data to judge → FE shows no badge. + "league_confidence": self._league_confidence_for(data.league_id), "match_date_ms": data.match_date_ms, "sport": data.sport, # Live snapshot — match_commentary uses this to detect upset-in-progress diff --git a/ai-engine/services/single_match_orchestrator.py b/ai-engine/services/single_match_orchestrator.py index a1ea661..9c04045 100755 --- a/ai-engine/services/single_match_orchestrator.py +++ b/ai-engine/services/single_match_orchestrator.py @@ -57,6 +57,7 @@ from services.v26_shadow_engine import V26ShadowEngine, get_v26_shadow_engine from services.match_commentary import generate_match_commentary from utils.top_leagues import load_top_league_ids from utils.league_reliability import load_league_reliability +from utils.league_confidence import load_league_confidence from config.config_loader import build_threshold_dict, get_threshold_default, get_config from models.calibration import get_calibrator @@ -171,6 +172,7 @@ class SingleMatchOrchestrator( self.engine_mode = str(os.getenv("AI_ENGINE_MODE", "v28-pro-max")).strip().lower() self.top_league_ids = load_top_league_ids() self.league_reliability = load_league_reliability() + self.league_confidence = load_league_confidence() self.enrichment = FeatureEnrichmentService() self.odds_band_analyzer = OddsBandAnalyzer() # ── Market Thresholds (loaded from config/market_thresholds.json) ── diff --git a/ai-engine/utils/league_confidence.py b/ai-engine/utils/league_confidence.py new file mode 100644 index 0000000..8f5532f --- /dev/null +++ b/ai-engine/utils/league_confidence.py @@ -0,0 +1,60 @@ +""" +League Confidence Loader +======================== +Loads pre-computed per-league CONFIDENCE labels from +data/league_confidence.json. Called once at orchestrator startup. + +Unlike league_reliability (odds-calibration), this reflects the model's +*backtested betting performance* per league: a label of high/medium/low/unknown +derived from BET ROI **and** sample size together, so a few lucky bets in a +thin league don't earn an undeserved "high" badge. + +Label rule (from scripts that build the artifact): + high : bet_roi > +10% AND bet_n >= 20 + low : bet_roi < -5% AND bet_n >= 15 + unknown : bet_n < 10 (too few bets to judge) + medium : everything else + +Usage: + from utils.league_confidence import load_league_confidence + lookup = load_league_confidence() + info = lookup.get(league_id) # {"label","bet_roi","bet_n","hit","name"} or None +""" + +from __future__ import annotations + +import json +import os +from typing import Dict, Any + + +_DATA_FILE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "data", + "league_confidence.json", +) + + +def load_league_confidence() -> Dict[str, Dict[str, Any]]: + """ + Returns dict mapping league_id → {label, bet_roi, bet_n, hit, name}. + Falls back to empty dict if the file is missing/corrupt — callers then + treat every league as 'unknown' (no badge), never crashing. + """ + if not os.path.isfile(_DATA_FILE): + print( + f"⚠️ league_confidence.json not found at {_DATA_FILE}. " + "All leagues will show as 'unknown' confidence." + ) + return {} + + try: + with open(_DATA_FILE, "r", encoding="utf-8") as f: + data = json.load(f) + lookup: Dict[str, Dict[str, Any]] = data.get("lookup", {}) + print(f"✅ Loaded league confidence labels for {len(lookup)} leagues") + return lookup + except (json.JSONDecodeError, KeyError, TypeError) as exc: + print(f"⚠️ Failed to parse league_confidence.json: {exc}") + return {} diff --git a/mds/brainstorm-leagues-and-national-teams.md b/mds/brainstorm-leagues-and-national-teams.md new file mode 100644 index 0000000..b119b78 --- /dev/null +++ b/mds/brainstorm-leagues-and-national-teams.md @@ -0,0 +1,72 @@ +# Gereksinim Keşfi: Lig Etiketleme + Milli Takım / Dünya Kupası Desteği + +> `/sc:brainstorm` çıktısı — REQUIREMENTS ONLY. Tasarım/kod sonraki adım (/sc:design, /sc:workflow). +> Tarih: 2026-06 · Kaynak: 10k backtest + canlı DB/API kanıtı. + +## Doğrulanmış Gerçekler (kanıta dayalı, varsayım değil) + +### Lig performansı +- 10k backtest 18 ligde BET üretti; ROI dağılımı: ~7 güçlü kârlı (+25%..+102%), + ~4 başabaş, ~7 zararlı (−12%..−62%). +- `live_matches` distinct lig ≠ `qualified_leagues.json` (48 lig). live_matches'te + qualified olmayan ligler var → kullanıcının "gereksiz ligler" sezgisi DOĞRU. +- Lig isimleri backtest CSV'de boş; DB'den `leagues` tablosundan çözülür. + +### Milli takım / Dünya Kupası (ÖNEMLİ — ilk hipotez ÇÜRÜDÜ) +- Milli takımlar DB'de VAR: Türkiye(9s2kpeunkes0g17l95r3t91j6, elo 1675), + Almanya(3l2t2db0c5ow2f7s7bhr6mij4, 1689), Kolombiya(1692), Andorra(1243). + ELO + matches_played (30-37) MEVCUT. +- Milli maç hacmi yüksek: DK Elemeler 645, Hazırlık Maçları Ülkeler 522, + Uluslar Ligi 148, Avrupa Şamp. Elemeleri 120 bitmiş maç. Ayrı ligler halinde. +- football_ai_features milli maçlar için %98 üretiliyor (196/200) — kulüpten yüksek. +- **KÖK SEBEP (canlı API ile kanıtlandı):** Sistem milli maçı tahmin EDİYOR + (MS olasılıkları + 9-10 market geliyor, data_quality MEDIUM 0.57-0.74). Sorun: + `betting_brain approved=0` — hiçbir market "oynanabilir" işaretlenmiyor. Ortak + flag `ai_features_inferred_from_history` → data_quality MEDIUM tavanı (0.74) + + lig qualified değil → brain eşikleri geçilemiyor. Yani "yetersiz veri" mesajı + aslında "brain güvenmiyor, BET yok" demek. Model/veri sorunu DEĞİL, gate/tuning sorunu. + +## Kullanıcı Kararları (bu oturumda alındı) +- Lig filtresi: **"Hepsi görünsün ama etiketli"** (gizleme yok; güven rozeti: Yüksek/Orta/Düşük). +- Milli takım: başta "ayrı model" istendi; veri görülünce yön = mevcut motoru milli + maçlara uyarlamak (ayrı ML modeli gereksiz — ELO+feature+geçmiş zaten var). +- Öncelik: **önce lig etiketleme** (hazır veri), sonra milli takım. +- Dünya Kupası: hazırlık maçlarında test edilebilmeli (yakın takvim baskısı). + +## Fonksiyonel Gereksinimler + +### A. Lig Güven Etiketleme +- FR-A1: Her lig için backtest'e dayalı güven seviyesi (Yüksek/Orta/Düşük) hesaplanmalı + (metrik: BET ROI + örneklem sayısı; düşük örneklem = otomatik Düşük/Bilinmiyor). +- FR-A2: live_matches'teki maçlar lig güven rozetiyle gösterilmeli (gizlenmeden). +- FR-A3: Etiket kaynağı tek yerde (config/tablo) tutulmalı, backtest tazelendikçe güncellenebilmeli. +- FR-A4: Forward-test (Model Performansı) verisi biriktikçe etiketler canlı sonuçla doğrulanmalı. + +### B. Milli Takım / Dünya Kupası Desteği +- FR-B1: Milli maçlarda da BET önerisi çıkabilmeli (şu an approved=0). +- FR-B2: `ai_features_inferred_from_history` flag'i olan milli maçlar için data_quality + tavanı / brain eşikleri milli-maça uygun kalibre edilmeli (kör gevşetme DEĞİL). +- FR-B3: Milli maç ligleri (DK Elemeler, Hazırlık Maçları Ülkeler, Uluslar Ligi, Avrupa + Şamp., Dünya Kupası) "tanınan" kapsama alınmalı (qualified benzeri). +- FR-B4: Hazırlık maçlarında uçtan uca test edilebilmeli (tahmin + forward-test kaydı). +- FR-B5: Milli maç kalibrasyonu ayrı izlenmeli (kulüple karışmasın) — Model Performansı + sayfasında "milli" kırılımı. + +## Fonksiyonel Olmayan Gereksinimler +- NFR-1: Gerçek para — milli maç eşik değişiklikleri backtest/forward-test ile doğrulanmadan + canlı agresifleştirilmemeli. +- NFR-2: Lig etiketleme mevcut hacmi düşürmemeli (gizleme değil işaretleme). +- NFR-3: Değişiklikler additive; mevcut kulüp tahmin akışını bozmamalı. + +## Açık Sorular (sonraki tasarım turunda netleşecek) +- OQ-1: Lig güven eşikleri tam olarak ne? (örn. Yüksek = ROI>+10% & N≥30 BET) +- OQ-2: Milli maçlar için brain eşiği nasıl ayarlanacak — ayrı tier mi, data_quality + flag istisnası mı? Önce backtest: milli maçlarda mevcut motor kaç BET/ne ROI verirdi + (eşik gevşetilse)? Bu ölçülmeden tuning yapılmamalı. +- OQ-3: Etiket nerede saklanacak: yeni tablo mı, mevcut league_tiers mı, config mi? +- OQ-4: Dünya Kupası grup maçlarında lineup geç gelir — probable_xi cezası milli maçta + nasıl ele alınacak? + +## Sonraki Adım +1. (Önce) Lig güven etiketleme → /sc:design veya doğrudan uygulama (veri hazır). +2. (Sonra) Milli maç backtest'i: eşik gevşetildiğinde milli maçlarda ROI ne? → ona göre tuning. diff --git a/src/modules/admin/admin.controller.ts b/src/modules/admin/admin.controller.ts index 95d5e57..f95c9d0 100755 --- a/src/modules/admin/admin.controller.ts +++ b/src/modules/admin/admin.controller.ts @@ -394,4 +394,202 @@ export class AdminController { predictions: totalPredictions, }); } + + // ================== Model Performance (Forward-Test) ================== + + @Get("model-performance") + @ApiOperation({ + summary: + "Per-market calibration (model% vs actual%), ROI and decision rationale " + + "from settled prediction_runs. Powers the admin Model Performance page.", + }) + @SwaggerResponse({ status: 200, schema: { type: "object" } }) + async getModelPerformance( + @Query("days") daysRaw?: string, + ): Promise> { + const days = Math.min(Math.max(Number(daysRaw) || 90, 1), 1000); + const sinceMs = Date.now() - days * 24 * 60 * 60 * 1000; + + // Pull settled rows in window. markets_settled holds one entry per market. + const rows = await this.prisma.$queryRawUnsafe< + Array<{ payload_summary: unknown; generated_at: Date }> + >( + ` + SELECT pr.payload_summary, pr.generated_at + FROM prediction_runs pr + WHERE pr.eventual_outcome IS NOT NULL + AND pr.generated_at >= $1 + AND pr.payload_summary -> 'settlement' -> 'markets_settled' IS NOT NULL + ORDER BY pr.generated_at DESC + LIMIT 50000 + `, + new Date(sinceMs), + ); + + // ── Aggregate per market ────────────────────────────────────────── + type Acc = { + market: string; + n: number; + wins: number; + sumShown: number; // Σ shown_confidence (0–100) + shownCount: number; + // 10-bin reliability for ECE + bins: Array<{ sumP: number; sumY: number; n: number }>; + // betting (only playable BET rows with odds) + betN: number; + betWins: number; + betProfit: number; + betStake: number; + // rationale tally + actions: Record; + tiers: Record; + }; + const acc = new Map(); + const ensure = (mk: string): Acc => { + let a = acc.get(mk); + if (!a) { + a = { + market: mk, + n: 0, + wins: 0, + sumShown: 0, + shownCount: 0, + bins: Array.from({ length: 10 }, () => ({ sumP: 0, sumY: 0, n: 0 })), + betN: 0, + betWins: 0, + betProfit: 0, + betStake: 0, + actions: {}, + tiers: {}, + }; + acc.set(mk, a); + } + return a; + }; + + let totalSettledMarkets = 0; + for (const row of rows) { + const summary = + row.payload_summary && typeof row.payload_summary === "object" + ? (row.payload_summary as Record) + : {}; + const settlement = + summary.settlement && typeof summary.settlement === "object" + ? (summary.settlement as Record) + : {}; + const markets = Array.isArray(settlement.markets_settled) + ? (settlement.markets_settled as Array>) + : []; + + for (const m of markets) { + const market = typeof m.market === "string" ? m.market : ""; + if (!market) continue; + const won = m.won === true; + const shown = + m.shown_confidence != null ? Number(m.shown_confidence) : null; + const a = ensure(market); + a.n += 1; + if (won) a.wins += 1; + totalSettledMarkets += 1; + + if (shown != null && Number.isFinite(shown)) { + a.sumShown += shown; + a.shownCount += 1; + const p = Math.min(Math.max(shown / 100, 0), 0.999999); + const bi = Math.min(9, Math.floor(p * 10)); + a.bins[bi].sumP += p; + a.bins[bi].sumY += won ? 1 : 0; + a.bins[bi].n += 1; + } + + const odds = m.odds != null ? Number(m.odds) : null; + const isBet = m.playable === true && m.action === "BET"; + if (isBet && odds != null && Number.isFinite(odds) && odds > 1.01) { + a.betN += 1; + if (won) { + a.betWins += 1; + a.betProfit += odds - 1; + } else { + a.betProfit -= 1; + } + a.betStake += 1; + } + + const action = typeof m.action === "string" ? m.action : "—"; + a.actions[action] = (a.actions[action] ?? 0) + 1; + const tier = typeof m.value_tier === "string" ? m.value_tier : "—"; + a.tiers[tier] = (a.tiers[tier] ?? 0) + 1; + } + } + + const markets = Array.from(acc.values()) + .map((a) => { + const actualPct = a.n > 0 ? (a.wins / a.n) * 100 : 0; + const shownPct = a.shownCount > 0 ? a.sumShown / a.shownCount : 0; + // ECE: Σ |acc_bin - conf_bin| * (n_bin / N) + let ece = 0; + for (const b of a.bins) { + if (b.n === 0) continue; + const conf = b.sumP / b.n; + const acc2 = b.sumY / b.n; + ece += Math.abs(acc2 - conf) * (b.n / a.shownCount || 0); + } + return { + market: a.market, + samples: a.n, + shown_pct: Number(shownPct.toFixed(1)), + actual_pct: Number(actualPct.toFixed(1)), + gap: Number((shownPct - actualPct).toFixed(1)), + ece: Number((ece * 100).toFixed(1)), + calibration: (Math.abs(shownPct - actualPct) <= 4 + ? "good" + : shownPct > actualPct + ? "overconfident" + : "underconfident") as + | "good" + | "overconfident" + | "underconfident", + bet_count: a.betN, + bet_hit_pct: + a.betN > 0 ? Number(((a.betWins / a.betN) * 100).toFixed(1)) : 0, + bet_roi_pct: + a.betStake > 0 + ? Number(((a.betProfit / a.betStake) * 100).toFixed(1)) + : 0, + actions: a.actions, + tiers: a.tiers, + }; + }) + .sort((x, y) => y.samples - x.samples); + + const result: ModelPerformanceResult = { + window_days: days, + settled_runs: Number(rows.length), + settled_markets: totalSettledMarkets, + generated_at: new Date().toISOString(), + markets, + }; + return createSuccessResponse(result); + } +} + +interface ModelPerformanceResult { + window_days: number; + settled_runs: number; + settled_markets: number; + generated_at: string; + markets: Array<{ + market: string; + samples: number; + shown_pct: number; + actual_pct: number; + gap: number; + ece: number; + calibration: "good" | "overconfident" | "underconfident"; + bet_count: number; + bet_hit_pct: number; + bet_roi_pct: number; + actions: Record; + tiers: Record; + }>; } diff --git a/src/modules/predictions/dto/index.ts b/src/modules/predictions/dto/index.ts index 5c94b61..05c0f24 100755 --- a/src/modules/predictions/dto/index.ts +++ b/src/modules/predictions/dto/index.ts @@ -27,6 +27,20 @@ export class MatchInfoDto { @ApiProperty({ required: false, default: false }) is_top_league?: boolean; + @ApiProperty({ + required: false, + nullable: true, + description: + "Backtest-derived per-league confidence (ROI + sample size). " + + "null when the league has too little data to judge.", + }) + league_confidence?: { + label: "high" | "medium" | "low"; + bet_roi: number; + bet_n: number; + hit: number; + } | null; + @ApiProperty({ required: false, enum: ["football", "basketball"], diff --git a/src/modules/predictions/predictions.service.ts b/src/modules/predictions/predictions.service.ts index 566708f..c94df41 100755 --- a/src/modules/predictions/predictions.service.ts +++ b/src/modules/predictions/predictions.service.ts @@ -1611,7 +1611,69 @@ export class PredictionsService implements OnModuleInit, OnModuleDestroy { })) : []; + // ── Forward-test capture (V31e) ──────────────────────────────────── + // Persist EVERY market's probability + the rationale behind each pick so + // the settlement job can later score each market against reality and the + // admin "Model Performance" page can show per-market calibration + // (model% → actual%) and decision reasons. Compact projection only. + // Some runtime fields (betting_brain, is_underdog_reference) are present + // in the AI payload but not declared on the DTO — read them via a cast. + const marketsFull = Array.isArray(payload.bet_summary) + ? payload.bet_summary.map((item) => { + const loose = item as unknown as Record; + const bb = (loose.betting_brain ?? {}) as Record; + return { + market: item.market, + pick: item.pick, + odds: item.odds ?? null, + model_probability: item.model_probability ?? null, + calibrated_confidence: item.calibrated_confidence ?? null, + raw_confidence: item.raw_confidence ?? null, + calibrated_probability: item.calibrated_probability ?? null, + implied_prob: item.implied_prob ?? null, + ev_edge: item.ev_edge ?? 0, + playable: item.playable ?? false, + bet_grade: item.bet_grade ?? "PASS", + signal_tier: item.signal_tier ?? null, + stake_units: item.stake_units ?? 0, + is_underdog_reference: Boolean(loose.is_underdog_reference), + action: (bb.action as string | undefined) ?? null, + value_tier: (bb.value_tier as string | undefined) ?? null, + model_market_gap: (bb.model_market_gap as number | undefined) ?? null, + trap_market_flag: Boolean(bb.trap_market_flag), + vetoes: Array.isArray(bb.vetoes) + ? (bb.vetoes as unknown[]).slice(0, 6) + : [], + positives: Array.isArray(bb.positives) + ? (bb.positives as unknown[]).slice(0, 6) + : [], + reasons: Array.isArray(item.reasons) ? item.reasons.slice(0, 6) : [], + }; + }) + : []; + + // Per-outcome probability distribution for each market (graph bars). + const marketBoardProbs = + payload.market_board && typeof payload.market_board === "object" + ? Object.fromEntries( + Object.entries( + payload.market_board as Record, + ).map(([mkt, entry]) => [ + mkt, + entry && typeof entry === "object" ? (entry.probs ?? null) : null, + ]), + ) + : {}; + return { + markets_full: marketsFull, + market_board_probs: marketBoardProbs, + betting_brain_version: + ( + (payload as unknown as Record).betting_brain as + | { version?: string } + | undefined + )?.version ?? null, model_version: payload.model_version, calibration_version: payload.calibration_version ?? null, shadow_engine_version: payload.shadow_engine_version ?? null, diff --git a/src/tasks/data-fetcher.task.ts b/src/tasks/data-fetcher.task.ts index 0b3850f..c662883 100755 --- a/src/tasks/data-fetcher.task.ts +++ b/src/tasks/data-fetcher.task.ts @@ -399,6 +399,12 @@ export class DataFetcherTask { const closingOddsSnapshot = await this.getClosingOddsSnapshot( row.matchId, ); + // ── Per-market settlement (V31e forward-test) ──────────────── + // Score EVERY captured market (not just main_pick) against reality, + // so the admin Model Performance page can compute per-market + // calibration (model% → actual%) and ROI. won=null → push (skip). + const marketsSettled = this.settleAllMarkets(row); + const settlementSummary = { settled_at: new Date().toISOString(), model_version: row.engineVersion, @@ -413,6 +419,7 @@ export class DataFetcherTask { away: row.htScoreAway, }, closing_odds_snapshot: closingOddsSnapshot, + markets_settled: marketsSettled, }; await this.prisma.$executeRawUnsafe( @@ -538,6 +545,64 @@ export class DataFetcherTask { }; } + /** + * V31e forward-test: settle EVERY captured market (payload_summary.markets_full) + * against the final score. Produces one compact record per market with its + * shown probability, the real outcome, and flat profit — the raw material for + * per-market calibration (model% vs actual%) on the admin dashboard. + */ + private settleAllMarkets( + row: PendingPredictionRunForSettlement, + ): Array> { + const summary = this.asRecord(row.payloadSummary); + const markets = Array.isArray(summary.markets_full) + ? (summary.markets_full as unknown[]) + : []; + const out: Array> = []; + + for (const raw of markets) { + const m = this.asRecord(raw); + const market = typeof m.market === "string" ? m.market : ""; + const pick = typeof m.pick === "string" ? m.pick : ""; + if (!market || !pick) continue; + + const won = this.isPredictionPickWon({ + market, + pick, + scoreHome: row.scoreHome, + scoreAway: row.scoreAway, + htScoreHome: row.htScoreHome, + htScoreAway: row.htScoreAway, + }); + if (won === null) continue; // push / unresolvable → exclude from stats + + const odds = Number(m.odds || 0); + const hasOdds = Number.isFinite(odds) && odds > 1.01; + out.push({ + market, + pick, + won, + // shown probability for calibration (0–100). Prefer calibrated_confidence. + shown_confidence: + m.calibrated_confidence != null + ? Number(m.calibrated_confidence) + : m.model_probability != null + ? Number(m.model_probability) * 100 + : null, + model_probability: + m.model_probability != null ? Number(m.model_probability) : null, + odds: hasOdds ? odds : null, + playable: m.playable === true, + bet_grade: typeof m.bet_grade === "string" ? m.bet_grade : null, + action: typeof m.action === "string" ? m.action : null, + value_tier: typeof m.value_tier === "string" ? m.value_tier : null, + // flat 1u profit if a real price existed (for per-market ROI) + flat_profit: hasOdds ? Number((won ? odds - 1 : -1).toFixed(4)) : null, + }); + } + return out; + } + private isPredictionPickWon(input: { market: string; pick: string;