@@ -0,0 +1,134 @@
|
||||
{
|
||||
"_meta": {
|
||||
"source": "bt_10k",
|
||||
"thresholds": "high:roi>10&n>=20 | low:roi<-5&n>=15 | unknown:n<10"
|
||||
},
|
||||
"lookup": {
|
||||
"32n2r9bl6x90psj0wa7bfs6vq": {
|
||||
"label": "high",
|
||||
"bet_roi": 102.2,
|
||||
"bet_n": 23,
|
||||
"hit": 30.4,
|
||||
"name": "Sudamericana"
|
||||
},
|
||||
"59tpnfrwnvhnhzmnvfyug68hj": {
|
||||
"label": "high",
|
||||
"bet_roi": 63.5,
|
||||
"bet_n": 23,
|
||||
"hit": 30.4,
|
||||
"name": "Libertadores Kupası"
|
||||
},
|
||||
"b60nisd3qn427jm0hrg9kvmab": {
|
||||
"label": "high",
|
||||
"bet_roi": 49.7,
|
||||
"bet_n": 22,
|
||||
"hit": 22.7,
|
||||
"name": "Allsvenskan"
|
||||
},
|
||||
"scf9p4y91yjvqvg5jndxzhxj": {
|
||||
"label": "high",
|
||||
"bet_roi": 33.8,
|
||||
"bet_n": 100,
|
||||
"hit": 25.0,
|
||||
"name": "Serie A"
|
||||
},
|
||||
"4oogyu6o156iphvdvphwpck10": {
|
||||
"label": "high",
|
||||
"bet_roi": 32.3,
|
||||
"bet_n": 23,
|
||||
"hit": 21.7,
|
||||
"name": "Şampiyonlar Ligi"
|
||||
},
|
||||
"89ovpy1rarewwzqvi30bfdr8b": {
|
||||
"label": "high",
|
||||
"bet_roi": 29.4,
|
||||
"bet_n": 50,
|
||||
"hit": 24.0,
|
||||
"name": "1. Lig"
|
||||
},
|
||||
"82jkgccg7phfjpd0mltdl3pat": {
|
||||
"label": "high",
|
||||
"bet_roi": 25.8,
|
||||
"bet_n": 29,
|
||||
"hit": 27.6,
|
||||
"name": "Süper Lig"
|
||||
},
|
||||
"3is4bkgf3loxv9qfg3hm8zfqb": {
|
||||
"label": "high",
|
||||
"bet_roi": 25.5,
|
||||
"bet_n": 84,
|
||||
"hit": 19.0,
|
||||
"name": "LaLiga 2"
|
||||
},
|
||||
"enzlj1as2raqm4ids1zyb07y1": {
|
||||
"label": "medium",
|
||||
"bet_roi": 23.7,
|
||||
"bet_n": 19,
|
||||
"hit": 26.3,
|
||||
"name": "USL 2. Lig"
|
||||
},
|
||||
"9ynnnx1qmkizq1o3qr3v0nsuk": {
|
||||
"label": "high",
|
||||
"bet_roi": 16.3,
|
||||
"bet_n": 38,
|
||||
"hit": 21.1,
|
||||
"name": "Eliteserien"
|
||||
},
|
||||
"8ey0ww2zsosdmwr8ehsorh6t7": {
|
||||
"label": "medium",
|
||||
"bet_roi": 5.4,
|
||||
"bet_n": 80,
|
||||
"hit": 16.2,
|
||||
"name": "Serie B"
|
||||
},
|
||||
"dm5ka0os1e3dxcp3vh05kmp33": {
|
||||
"label": "low",
|
||||
"bet_roi": -7.4,
|
||||
"bet_n": 46,
|
||||
"hit": 26.1,
|
||||
"name": "Ligue 1"
|
||||
},
|
||||
"4zwgbb66rif2spcoeeol2motx": {
|
||||
"label": "low",
|
||||
"bet_roi": -12.7,
|
||||
"bet_n": 39,
|
||||
"hit": 23.1,
|
||||
"name": "Pro Lig"
|
||||
},
|
||||
"a4fgj2rfbpf4ejo1qi624fefo": {
|
||||
"label": "low",
|
||||
"bet_roi": -14.2,
|
||||
"bet_n": 73,
|
||||
"hit": 17.8,
|
||||
"name": "3. Lig"
|
||||
},
|
||||
"9chuiarcjofld1dkj9kysehmb": {
|
||||
"label": "low",
|
||||
"bet_roi": -14.9,
|
||||
"bet_n": 22,
|
||||
"hit": 13.6,
|
||||
"name": "Superettan"
|
||||
},
|
||||
"3p81ltz6845appgkbgkzxueii": {
|
||||
"label": "low",
|
||||
"bet_roi": -19.8,
|
||||
"bet_n": 34,
|
||||
"hit": 14.7,
|
||||
"name": "2. Lig"
|
||||
},
|
||||
"dvstmwnvw0mt5p38twn9yttyb": {
|
||||
"label": "low",
|
||||
"bet_roi": -37.2,
|
||||
"bet_n": 19,
|
||||
"hit": 26.3,
|
||||
"name": "Veikkausliiga"
|
||||
},
|
||||
"zs18qaehvhg3w1208874zvfa": {
|
||||
"label": "low",
|
||||
"bet_roi": -62.0,
|
||||
"bet_n": 17,
|
||||
"hit": 23.5,
|
||||
"name": "1. Lig"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
# V29 Data-Driven Optimization Report
|
||||
## Based on 7,000-match Diagnostic Backtest (2026-05-27)
|
||||
|
||||
### Before (V28-Pro-Max)
|
||||
- **4,134 settled BET-action picks**
|
||||
- Hit rate: 54.9%
|
||||
- Unit profit: -132.68
|
||||
- Staked: 849.50
|
||||
- **ROI: -15.6%**
|
||||
|
||||
### Root Cause Analysis
|
||||
|
||||
#### 1. Value Sniper Threshold Too Loose (CRITICAL)
|
||||
```python
|
||||
# OLD: ev_edge >= 0.008 or calibrated_conf >= 55.0
|
||||
# This made 100% of bets qualify as "value sniper", bypassing ALL betting brain vetoes
|
||||
```
|
||||
- 4,134/4,134 bets (100%) had `is_value_sniper = True`
|
||||
- Hard vetoes (negative_ev, market_muted, low_reliability) were NEVER enforced
|
||||
|
||||
#### 2. 89% of Bets Had Negative EV Edge
|
||||
- n=3,688 with ev_edge < 0: ROI = -16.1%
|
||||
- The model was systematically pricing below market, meaning every bet carried negative expected value
|
||||
|
||||
#### 3. OU25 Market Unprofitable in ALL Configurations
|
||||
- n=1,563 bets, -17.1% ROI
|
||||
- Even with ev>=5% + rel>=0.55: n=27, -36.9% ROI
|
||||
- Grid search found NO profitable filter combination
|
||||
|
||||
#### 4. BTTS Market Marginal
|
||||
- n=1,456 bets, -15.4% ROI
|
||||
- Only profitable with ev>=5%: n=15, +12.9% (but tiny sample)
|
||||
|
||||
### Grid Search Results (Top Profitable Combos)
|
||||
|
||||
| Market | EV Min | Rel Min | V27 | n | Hit% | ROI |
|
||||
|--------|--------|---------|-----|---|------|-----|
|
||||
| MS | >=5% | >=0.55 | AGREE | 42 | 59.5% | **+10.4%** |
|
||||
| MS | >=5% | >=0.55 | ANY | 52 | 59.6% | **+8.6%** |
|
||||
| MS | >=3% | >=0.55 | ANY | 69 | 56.5% | **+4.0%** |
|
||||
| BTTS | >=5% | >=0.70 | ANY | 15 | 60.0% | **+12.9%** |
|
||||
| MS | >=5% | >=0.00 | ANY | 113 | 55.8% | **-0.7%** |
|
||||
|
||||
### Changes Applied (V29)
|
||||
|
||||
#### market_board.py
|
||||
```python
|
||||
# Tightened from: ev >= 0.008 OR conf >= 55.0
|
||||
# To: ALL three must be true
|
||||
is_value_sniper = ev_edge >= 0.05 and calibrated_conf >= 60.0 and odds_rel >= 0.55
|
||||
```
|
||||
|
||||
#### betting_brain.py
|
||||
1. **MIN_BET_SCORE**: 72.0 -> 62.0 (hard vetoes now do the filtering)
|
||||
2. **MIN_WATCH_SCORE**: 62.0 -> 52.0
|
||||
3. **MUTED_MARKETS**: `{"BTTS"}` -> `{"OU25", "DC", "OU35"}`
|
||||
4. **MARKET_OPTIMAL_FILTERS**:
|
||||
- MS: min_edge=0.03, min_reliability=0.55, require_v27_agree=False
|
||||
- BTTS: min_edge=0.05, min_reliability=0.70 (strict envelope)
|
||||
5. **Hard vetoes no longer bypassed by sniper**:
|
||||
- `negative_ev_edge` (ev < 0)
|
||||
- `ev_edge_too_high_trap` (ev >= 0.20)
|
||||
- `market_muted_by_backtest`
|
||||
- `low_reliability_league_hard_block` (rel < 0.30)
|
||||
- Per-market envelope checks
|
||||
|
||||
### Expected Performance (Simulated on 7K backtest)
|
||||
- **65 bets** out of 7,000 matches (0.9% selectivity)
|
||||
- Hit rate: 56.9%
|
||||
- **ROI: +6.8%** (from -15.6%)
|
||||
- MS dominates: n=64, ROI=+8.0%
|
||||
- Consistent: April +14.0%, May +4.9%
|
||||
|
||||
### Trade-off
|
||||
The system becomes very selective (fewer bets per day) but each bet carries genuine positive expected value. Quality over quantity.
|
||||
@@ -86,6 +86,28 @@ POST_CAL_TRUST: Dict[str, float] = {
|
||||
|
||||
|
||||
class MarketBoardMixin:
|
||||
def _league_confidence_for(self, league_id: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||
"""Return the backtest-derived confidence record for a league, or None.
|
||||
|
||||
Shape: {"label": high|medium|low, "bet_roi": float, "bet_n": int,
|
||||
"hit": float}. None → league absent or too few bets ('unknown') → FE
|
||||
shows no badge. Never raises (missing artifact = graceful None)."""
|
||||
if not league_id:
|
||||
return None
|
||||
lookup = getattr(self, "league_confidence", None) or {}
|
||||
info = lookup.get(str(league_id))
|
||||
if not isinstance(info, dict):
|
||||
return None
|
||||
label = info.get("label")
|
||||
if label in (None, "unknown"):
|
||||
return None
|
||||
return {
|
||||
"label": label,
|
||||
"bet_roi": info.get("bet_roi"),
|
||||
"bet_n": info.get("bet_n"),
|
||||
"hit": info.get("hit"),
|
||||
}
|
||||
|
||||
def _build_prediction_package(
|
||||
self,
|
||||
data: MatchData,
|
||||
@@ -320,6 +342,10 @@ class MarketBoardMixin:
|
||||
"home_team": data.home_team_name,
|
||||
"away_team": data.away_team_name,
|
||||
"league": data.league_name,
|
||||
"league_id": data.league_id,
|
||||
# Backtest-derived per-league confidence (ROI + sample size).
|
||||
# None when the league has too little data to judge → FE shows no badge.
|
||||
"league_confidence": self._league_confidence_for(data.league_id),
|
||||
"match_date_ms": data.match_date_ms,
|
||||
"sport": data.sport,
|
||||
# Live snapshot — match_commentary uses this to detect upset-in-progress
|
||||
|
||||
@@ -57,6 +57,7 @@ from services.v26_shadow_engine import V26ShadowEngine, get_v26_shadow_engine
|
||||
from services.match_commentary import generate_match_commentary
|
||||
from utils.top_leagues import load_top_league_ids
|
||||
from utils.league_reliability import load_league_reliability
|
||||
from utils.league_confidence import load_league_confidence
|
||||
from config.config_loader import build_threshold_dict, get_threshold_default, get_config
|
||||
from models.calibration import get_calibrator
|
||||
|
||||
@@ -171,6 +172,7 @@ class SingleMatchOrchestrator(
|
||||
self.engine_mode = str(os.getenv("AI_ENGINE_MODE", "v28-pro-max")).strip().lower()
|
||||
self.top_league_ids = load_top_league_ids()
|
||||
self.league_reliability = load_league_reliability()
|
||||
self.league_confidence = load_league_confidence()
|
||||
self.enrichment = FeatureEnrichmentService()
|
||||
self.odds_band_analyzer = OddsBandAnalyzer()
|
||||
# ── Market Thresholds (loaded from config/market_thresholds.json) ──
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""
|
||||
League Confidence Loader
|
||||
========================
|
||||
Loads pre-computed per-league CONFIDENCE labels from
|
||||
data/league_confidence.json. Called once at orchestrator startup.
|
||||
|
||||
Unlike league_reliability (odds-calibration), this reflects the model's
|
||||
*backtested betting performance* per league: a label of high/medium/low/unknown
|
||||
derived from BET ROI **and** sample size together, so a few lucky bets in a
|
||||
thin league don't earn an undeserved "high" badge.
|
||||
|
||||
Label rule (from scripts that build the artifact):
|
||||
high : bet_roi > +10% AND bet_n >= 20
|
||||
low : bet_roi < -5% AND bet_n >= 15
|
||||
unknown : bet_n < 10 (too few bets to judge)
|
||||
medium : everything else
|
||||
|
||||
Usage:
|
||||
from utils.league_confidence import load_league_confidence
|
||||
lookup = load_league_confidence()
|
||||
info = lookup.get(league_id) # {"label","bet_roi","bet_n","hit","name"} or None
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
_DATA_FILE = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
"..",
|
||||
"data",
|
||||
"league_confidence.json",
|
||||
)
|
||||
|
||||
|
||||
def load_league_confidence() -> Dict[str, Dict[str, Any]]:
|
||||
"""
|
||||
Returns dict mapping league_id → {label, bet_roi, bet_n, hit, name}.
|
||||
Falls back to empty dict if the file is missing/corrupt — callers then
|
||||
treat every league as 'unknown' (no badge), never crashing.
|
||||
"""
|
||||
if not os.path.isfile(_DATA_FILE):
|
||||
print(
|
||||
f"⚠️ league_confidence.json not found at {_DATA_FILE}. "
|
||||
"All leagues will show as 'unknown' confidence."
|
||||
)
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(_DATA_FILE, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
lookup: Dict[str, Dict[str, Any]] = data.get("lookup", {})
|
||||
print(f"✅ Loaded league confidence labels for {len(lookup)} leagues")
|
||||
return lookup
|
||||
except (json.JSONDecodeError, KeyError, TypeError) as exc:
|
||||
print(f"⚠️ Failed to parse league_confidence.json: {exc}")
|
||||
return {}
|
||||
@@ -0,0 +1,72 @@
|
||||
# Gereksinim Keşfi: Lig Etiketleme + Milli Takım / Dünya Kupası Desteği
|
||||
|
||||
> `/sc:brainstorm` çıktısı — REQUIREMENTS ONLY. Tasarım/kod sonraki adım (/sc:design, /sc:workflow).
|
||||
> Tarih: 2026-06 · Kaynak: 10k backtest + canlı DB/API kanıtı.
|
||||
|
||||
## Doğrulanmış Gerçekler (kanıta dayalı, varsayım değil)
|
||||
|
||||
### Lig performansı
|
||||
- 10k backtest 18 ligde BET üretti; ROI dağılımı: ~7 güçlü kârlı (+25%..+102%),
|
||||
~4 başabaş, ~7 zararlı (−12%..−62%).
|
||||
- `live_matches` distinct lig ≠ `qualified_leagues.json` (48 lig). live_matches'te
|
||||
qualified olmayan ligler var → kullanıcının "gereksiz ligler" sezgisi DOĞRU.
|
||||
- Lig isimleri backtest CSV'de boş; DB'den `leagues` tablosundan çözülür.
|
||||
|
||||
### Milli takım / Dünya Kupası (ÖNEMLİ — ilk hipotez ÇÜRÜDÜ)
|
||||
- Milli takımlar DB'de VAR: Türkiye(9s2kpeunkes0g17l95r3t91j6, elo 1675),
|
||||
Almanya(3l2t2db0c5ow2f7s7bhr6mij4, 1689), Kolombiya(1692), Andorra(1243).
|
||||
ELO + matches_played (30-37) MEVCUT.
|
||||
- Milli maç hacmi yüksek: DK Elemeler 645, Hazırlık Maçları Ülkeler 522,
|
||||
Uluslar Ligi 148, Avrupa Şamp. Elemeleri 120 bitmiş maç. Ayrı ligler halinde.
|
||||
- football_ai_features milli maçlar için %98 üretiliyor (196/200) — kulüpten yüksek.
|
||||
- **KÖK SEBEP (canlı API ile kanıtlandı):** Sistem milli maçı tahmin EDİYOR
|
||||
(MS olasılıkları + 9-10 market geliyor, data_quality MEDIUM 0.57-0.74). Sorun:
|
||||
`betting_brain approved=0` — hiçbir market "oynanabilir" işaretlenmiyor. Ortak
|
||||
flag `ai_features_inferred_from_history` → data_quality MEDIUM tavanı (0.74) +
|
||||
lig qualified değil → brain eşikleri geçilemiyor. Yani "yetersiz veri" mesajı
|
||||
aslında "brain güvenmiyor, BET yok" demek. Model/veri sorunu DEĞİL, gate/tuning sorunu.
|
||||
|
||||
## Kullanıcı Kararları (bu oturumda alındı)
|
||||
- Lig filtresi: **"Hepsi görünsün ama etiketli"** (gizleme yok; güven rozeti: Yüksek/Orta/Düşük).
|
||||
- Milli takım: başta "ayrı model" istendi; veri görülünce yön = mevcut motoru milli
|
||||
maçlara uyarlamak (ayrı ML modeli gereksiz — ELO+feature+geçmiş zaten var).
|
||||
- Öncelik: **önce lig etiketleme** (hazır veri), sonra milli takım.
|
||||
- Dünya Kupası: hazırlık maçlarında test edilebilmeli (yakın takvim baskısı).
|
||||
|
||||
## Fonksiyonel Gereksinimler
|
||||
|
||||
### A. Lig Güven Etiketleme
|
||||
- FR-A1: Her lig için backtest'e dayalı güven seviyesi (Yüksek/Orta/Düşük) hesaplanmalı
|
||||
(metrik: BET ROI + örneklem sayısı; düşük örneklem = otomatik Düşük/Bilinmiyor).
|
||||
- FR-A2: live_matches'teki maçlar lig güven rozetiyle gösterilmeli (gizlenmeden).
|
||||
- FR-A3: Etiket kaynağı tek yerde (config/tablo) tutulmalı, backtest tazelendikçe güncellenebilmeli.
|
||||
- FR-A4: Forward-test (Model Performansı) verisi biriktikçe etiketler canlı sonuçla doğrulanmalı.
|
||||
|
||||
### B. Milli Takım / Dünya Kupası Desteği
|
||||
- FR-B1: Milli maçlarda da BET önerisi çıkabilmeli (şu an approved=0).
|
||||
- FR-B2: `ai_features_inferred_from_history` flag'i olan milli maçlar için data_quality
|
||||
tavanı / brain eşikleri milli-maça uygun kalibre edilmeli (kör gevşetme DEĞİL).
|
||||
- FR-B3: Milli maç ligleri (DK Elemeler, Hazırlık Maçları Ülkeler, Uluslar Ligi, Avrupa
|
||||
Şamp., Dünya Kupası) "tanınan" kapsama alınmalı (qualified benzeri).
|
||||
- FR-B4: Hazırlık maçlarında uçtan uca test edilebilmeli (tahmin + forward-test kaydı).
|
||||
- FR-B5: Milli maç kalibrasyonu ayrı izlenmeli (kulüple karışmasın) — Model Performansı
|
||||
sayfasında "milli" kırılımı.
|
||||
|
||||
## Fonksiyonel Olmayan Gereksinimler
|
||||
- NFR-1: Gerçek para — milli maç eşik değişiklikleri backtest/forward-test ile doğrulanmadan
|
||||
canlı agresifleştirilmemeli.
|
||||
- NFR-2: Lig etiketleme mevcut hacmi düşürmemeli (gizleme değil işaretleme).
|
||||
- NFR-3: Değişiklikler additive; mevcut kulüp tahmin akışını bozmamalı.
|
||||
|
||||
## Açık Sorular (sonraki tasarım turunda netleşecek)
|
||||
- OQ-1: Lig güven eşikleri tam olarak ne? (örn. Yüksek = ROI>+10% & N≥30 BET)
|
||||
- OQ-2: Milli maçlar için brain eşiği nasıl ayarlanacak — ayrı tier mi, data_quality
|
||||
flag istisnası mı? Önce backtest: milli maçlarda mevcut motor kaç BET/ne ROI verirdi
|
||||
(eşik gevşetilse)? Bu ölçülmeden tuning yapılmamalı.
|
||||
- OQ-3: Etiket nerede saklanacak: yeni tablo mı, mevcut league_tiers mı, config mi?
|
||||
- OQ-4: Dünya Kupası grup maçlarında lineup geç gelir — probable_xi cezası milli maçta
|
||||
nasıl ele alınacak?
|
||||
|
||||
## Sonraki Adım
|
||||
1. (Önce) Lig güven etiketleme → /sc:design veya doğrudan uygulama (veri hazır).
|
||||
2. (Sonra) Milli maç backtest'i: eşik gevşetildiğinde milli maçlarda ROI ne? → ona göre tuning.
|
||||
@@ -394,4 +394,202 @@ export class AdminController {
|
||||
predictions: totalPredictions,
|
||||
});
|
||||
}
|
||||
|
||||
// ================== Model Performance (Forward-Test) ==================
|
||||
|
||||
@Get("model-performance")
|
||||
@ApiOperation({
|
||||
summary:
|
||||
"Per-market calibration (model% vs actual%), ROI and decision rationale " +
|
||||
"from settled prediction_runs. Powers the admin Model Performance page.",
|
||||
})
|
||||
@SwaggerResponse({ status: 200, schema: { type: "object" } })
|
||||
async getModelPerformance(
|
||||
@Query("days") daysRaw?: string,
|
||||
): Promise<ApiResponse<ModelPerformanceResult>> {
|
||||
const days = Math.min(Math.max(Number(daysRaw) || 90, 1), 1000);
|
||||
const sinceMs = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||
|
||||
// Pull settled rows in window. markets_settled holds one entry per market.
|
||||
const rows = await this.prisma.$queryRawUnsafe<
|
||||
Array<{ payload_summary: unknown; generated_at: Date }>
|
||||
>(
|
||||
`
|
||||
SELECT pr.payload_summary, pr.generated_at
|
||||
FROM prediction_runs pr
|
||||
WHERE pr.eventual_outcome IS NOT NULL
|
||||
AND pr.generated_at >= $1
|
||||
AND pr.payload_summary -> 'settlement' -> 'markets_settled' IS NOT NULL
|
||||
ORDER BY pr.generated_at DESC
|
||||
LIMIT 50000
|
||||
`,
|
||||
new Date(sinceMs),
|
||||
);
|
||||
|
||||
// ── Aggregate per market ──────────────────────────────────────────
|
||||
type Acc = {
|
||||
market: string;
|
||||
n: number;
|
||||
wins: number;
|
||||
sumShown: number; // Σ shown_confidence (0–100)
|
||||
shownCount: number;
|
||||
// 10-bin reliability for ECE
|
||||
bins: Array<{ sumP: number; sumY: number; n: number }>;
|
||||
// betting (only playable BET rows with odds)
|
||||
betN: number;
|
||||
betWins: number;
|
||||
betProfit: number;
|
||||
betStake: number;
|
||||
// rationale tally
|
||||
actions: Record<string, number>;
|
||||
tiers: Record<string, number>;
|
||||
};
|
||||
const acc = new Map<string, Acc>();
|
||||
const ensure = (mk: string): Acc => {
|
||||
let a = acc.get(mk);
|
||||
if (!a) {
|
||||
a = {
|
||||
market: mk,
|
||||
n: 0,
|
||||
wins: 0,
|
||||
sumShown: 0,
|
||||
shownCount: 0,
|
||||
bins: Array.from({ length: 10 }, () => ({ sumP: 0, sumY: 0, n: 0 })),
|
||||
betN: 0,
|
||||
betWins: 0,
|
||||
betProfit: 0,
|
||||
betStake: 0,
|
||||
actions: {},
|
||||
tiers: {},
|
||||
};
|
||||
acc.set(mk, a);
|
||||
}
|
||||
return a;
|
||||
};
|
||||
|
||||
let totalSettledMarkets = 0;
|
||||
for (const row of rows) {
|
||||
const summary =
|
||||
row.payload_summary && typeof row.payload_summary === "object"
|
||||
? (row.payload_summary as Record<string, unknown>)
|
||||
: {};
|
||||
const settlement =
|
||||
summary.settlement && typeof summary.settlement === "object"
|
||||
? (summary.settlement as Record<string, unknown>)
|
||||
: {};
|
||||
const markets = Array.isArray(settlement.markets_settled)
|
||||
? (settlement.markets_settled as Array<Record<string, unknown>>)
|
||||
: [];
|
||||
|
||||
for (const m of markets) {
|
||||
const market = typeof m.market === "string" ? m.market : "";
|
||||
if (!market) continue;
|
||||
const won = m.won === true;
|
||||
const shown =
|
||||
m.shown_confidence != null ? Number(m.shown_confidence) : null;
|
||||
const a = ensure(market);
|
||||
a.n += 1;
|
||||
if (won) a.wins += 1;
|
||||
totalSettledMarkets += 1;
|
||||
|
||||
if (shown != null && Number.isFinite(shown)) {
|
||||
a.sumShown += shown;
|
||||
a.shownCount += 1;
|
||||
const p = Math.min(Math.max(shown / 100, 0), 0.999999);
|
||||
const bi = Math.min(9, Math.floor(p * 10));
|
||||
a.bins[bi].sumP += p;
|
||||
a.bins[bi].sumY += won ? 1 : 0;
|
||||
a.bins[bi].n += 1;
|
||||
}
|
||||
|
||||
const odds = m.odds != null ? Number(m.odds) : null;
|
||||
const isBet = m.playable === true && m.action === "BET";
|
||||
if (isBet && odds != null && Number.isFinite(odds) && odds > 1.01) {
|
||||
a.betN += 1;
|
||||
if (won) {
|
||||
a.betWins += 1;
|
||||
a.betProfit += odds - 1;
|
||||
} else {
|
||||
a.betProfit -= 1;
|
||||
}
|
||||
a.betStake += 1;
|
||||
}
|
||||
|
||||
const action = typeof m.action === "string" ? m.action : "—";
|
||||
a.actions[action] = (a.actions[action] ?? 0) + 1;
|
||||
const tier = typeof m.value_tier === "string" ? m.value_tier : "—";
|
||||
a.tiers[tier] = (a.tiers[tier] ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
const markets = Array.from(acc.values())
|
||||
.map((a) => {
|
||||
const actualPct = a.n > 0 ? (a.wins / a.n) * 100 : 0;
|
||||
const shownPct = a.shownCount > 0 ? a.sumShown / a.shownCount : 0;
|
||||
// ECE: Σ |acc_bin - conf_bin| * (n_bin / N)
|
||||
let ece = 0;
|
||||
for (const b of a.bins) {
|
||||
if (b.n === 0) continue;
|
||||
const conf = b.sumP / b.n;
|
||||
const acc2 = b.sumY / b.n;
|
||||
ece += Math.abs(acc2 - conf) * (b.n / a.shownCount || 0);
|
||||
}
|
||||
return {
|
||||
market: a.market,
|
||||
samples: a.n,
|
||||
shown_pct: Number(shownPct.toFixed(1)),
|
||||
actual_pct: Number(actualPct.toFixed(1)),
|
||||
gap: Number((shownPct - actualPct).toFixed(1)),
|
||||
ece: Number((ece * 100).toFixed(1)),
|
||||
calibration: (Math.abs(shownPct - actualPct) <= 4
|
||||
? "good"
|
||||
: shownPct > actualPct
|
||||
? "overconfident"
|
||||
: "underconfident") as
|
||||
| "good"
|
||||
| "overconfident"
|
||||
| "underconfident",
|
||||
bet_count: a.betN,
|
||||
bet_hit_pct:
|
||||
a.betN > 0 ? Number(((a.betWins / a.betN) * 100).toFixed(1)) : 0,
|
||||
bet_roi_pct:
|
||||
a.betStake > 0
|
||||
? Number(((a.betProfit / a.betStake) * 100).toFixed(1))
|
||||
: 0,
|
||||
actions: a.actions,
|
||||
tiers: a.tiers,
|
||||
};
|
||||
})
|
||||
.sort((x, y) => y.samples - x.samples);
|
||||
|
||||
const result: ModelPerformanceResult = {
|
||||
window_days: days,
|
||||
settled_runs: Number(rows.length),
|
||||
settled_markets: totalSettledMarkets,
|
||||
generated_at: new Date().toISOString(),
|
||||
markets,
|
||||
};
|
||||
return createSuccessResponse(result);
|
||||
}
|
||||
}
|
||||
|
||||
interface ModelPerformanceResult {
|
||||
window_days: number;
|
||||
settled_runs: number;
|
||||
settled_markets: number;
|
||||
generated_at: string;
|
||||
markets: Array<{
|
||||
market: string;
|
||||
samples: number;
|
||||
shown_pct: number;
|
||||
actual_pct: number;
|
||||
gap: number;
|
||||
ece: number;
|
||||
calibration: "good" | "overconfident" | "underconfident";
|
||||
bet_count: number;
|
||||
bet_hit_pct: number;
|
||||
bet_roi_pct: number;
|
||||
actions: Record<string, number>;
|
||||
tiers: Record<string, number>;
|
||||
}>;
|
||||
}
|
||||
|
||||
@@ -27,6 +27,20 @@ export class MatchInfoDto {
|
||||
@ApiProperty({ required: false, default: false })
|
||||
is_top_league?: boolean;
|
||||
|
||||
@ApiProperty({
|
||||
required: false,
|
||||
nullable: true,
|
||||
description:
|
||||
"Backtest-derived per-league confidence (ROI + sample size). " +
|
||||
"null when the league has too little data to judge.",
|
||||
})
|
||||
league_confidence?: {
|
||||
label: "high" | "medium" | "low";
|
||||
bet_roi: number;
|
||||
bet_n: number;
|
||||
hit: number;
|
||||
} | null;
|
||||
|
||||
@ApiProperty({
|
||||
required: false,
|
||||
enum: ["football", "basketball"],
|
||||
|
||||
@@ -1611,7 +1611,69 @@ export class PredictionsService implements OnModuleInit, OnModuleDestroy {
|
||||
}))
|
||||
: [];
|
||||
|
||||
// ── Forward-test capture (V31e) ────────────────────────────────────
|
||||
// Persist EVERY market's probability + the rationale behind each pick so
|
||||
// the settlement job can later score each market against reality and the
|
||||
// admin "Model Performance" page can show per-market calibration
|
||||
// (model% → actual%) and decision reasons. Compact projection only.
|
||||
// Some runtime fields (betting_brain, is_underdog_reference) are present
|
||||
// in the AI payload but not declared on the DTO — read them via a cast.
|
||||
const marketsFull = Array.isArray(payload.bet_summary)
|
||||
? payload.bet_summary.map((item) => {
|
||||
const loose = item as unknown as Record<string, unknown>;
|
||||
const bb = (loose.betting_brain ?? {}) as Record<string, unknown>;
|
||||
return {
|
||||
market: item.market,
|
||||
pick: item.pick,
|
||||
odds: item.odds ?? null,
|
||||
model_probability: item.model_probability ?? null,
|
||||
calibrated_confidence: item.calibrated_confidence ?? null,
|
||||
raw_confidence: item.raw_confidence ?? null,
|
||||
calibrated_probability: item.calibrated_probability ?? null,
|
||||
implied_prob: item.implied_prob ?? null,
|
||||
ev_edge: item.ev_edge ?? 0,
|
||||
playable: item.playable ?? false,
|
||||
bet_grade: item.bet_grade ?? "PASS",
|
||||
signal_tier: item.signal_tier ?? null,
|
||||
stake_units: item.stake_units ?? 0,
|
||||
is_underdog_reference: Boolean(loose.is_underdog_reference),
|
||||
action: (bb.action as string | undefined) ?? null,
|
||||
value_tier: (bb.value_tier as string | undefined) ?? null,
|
||||
model_market_gap: (bb.model_market_gap as number | undefined) ?? null,
|
||||
trap_market_flag: Boolean(bb.trap_market_flag),
|
||||
vetoes: Array.isArray(bb.vetoes)
|
||||
? (bb.vetoes as unknown[]).slice(0, 6)
|
||||
: [],
|
||||
positives: Array.isArray(bb.positives)
|
||||
? (bb.positives as unknown[]).slice(0, 6)
|
||||
: [],
|
||||
reasons: Array.isArray(item.reasons) ? item.reasons.slice(0, 6) : [],
|
||||
};
|
||||
})
|
||||
: [];
|
||||
|
||||
// Per-outcome probability distribution for each market (graph bars).
|
||||
const marketBoardProbs =
|
||||
payload.market_board && typeof payload.market_board === "object"
|
||||
? Object.fromEntries(
|
||||
Object.entries(
|
||||
payload.market_board as Record<string, { probs?: unknown }>,
|
||||
).map(([mkt, entry]) => [
|
||||
mkt,
|
||||
entry && typeof entry === "object" ? (entry.probs ?? null) : null,
|
||||
]),
|
||||
)
|
||||
: {};
|
||||
|
||||
return {
|
||||
markets_full: marketsFull,
|
||||
market_board_probs: marketBoardProbs,
|
||||
betting_brain_version:
|
||||
(
|
||||
(payload as unknown as Record<string, unknown>).betting_brain as
|
||||
| { version?: string }
|
||||
| undefined
|
||||
)?.version ?? null,
|
||||
model_version: payload.model_version,
|
||||
calibration_version: payload.calibration_version ?? null,
|
||||
shadow_engine_version: payload.shadow_engine_version ?? null,
|
||||
|
||||
@@ -399,6 +399,12 @@ export class DataFetcherTask {
|
||||
const closingOddsSnapshot = await this.getClosingOddsSnapshot(
|
||||
row.matchId,
|
||||
);
|
||||
// ── Per-market settlement (V31e forward-test) ────────────────
|
||||
// Score EVERY captured market (not just main_pick) against reality,
|
||||
// so the admin Model Performance page can compute per-market
|
||||
// calibration (model% → actual%) and ROI. won=null → push (skip).
|
||||
const marketsSettled = this.settleAllMarkets(row);
|
||||
|
||||
const settlementSummary = {
|
||||
settled_at: new Date().toISOString(),
|
||||
model_version: row.engineVersion,
|
||||
@@ -413,6 +419,7 @@ export class DataFetcherTask {
|
||||
away: row.htScoreAway,
|
||||
},
|
||||
closing_odds_snapshot: closingOddsSnapshot,
|
||||
markets_settled: marketsSettled,
|
||||
};
|
||||
|
||||
await this.prisma.$executeRawUnsafe(
|
||||
@@ -538,6 +545,64 @@ export class DataFetcherTask {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* V31e forward-test: settle EVERY captured market (payload_summary.markets_full)
|
||||
* against the final score. Produces one compact record per market with its
|
||||
* shown probability, the real outcome, and flat profit — the raw material for
|
||||
* per-market calibration (model% vs actual%) on the admin dashboard.
|
||||
*/
|
||||
private settleAllMarkets(
|
||||
row: PendingPredictionRunForSettlement,
|
||||
): Array<Record<string, unknown>> {
|
||||
const summary = this.asRecord(row.payloadSummary);
|
||||
const markets = Array.isArray(summary.markets_full)
|
||||
? (summary.markets_full as unknown[])
|
||||
: [];
|
||||
const out: Array<Record<string, unknown>> = [];
|
||||
|
||||
for (const raw of markets) {
|
||||
const m = this.asRecord(raw);
|
||||
const market = typeof m.market === "string" ? m.market : "";
|
||||
const pick = typeof m.pick === "string" ? m.pick : "";
|
||||
if (!market || !pick) continue;
|
||||
|
||||
const won = this.isPredictionPickWon({
|
||||
market,
|
||||
pick,
|
||||
scoreHome: row.scoreHome,
|
||||
scoreAway: row.scoreAway,
|
||||
htScoreHome: row.htScoreHome,
|
||||
htScoreAway: row.htScoreAway,
|
||||
});
|
||||
if (won === null) continue; // push / unresolvable → exclude from stats
|
||||
|
||||
const odds = Number(m.odds || 0);
|
||||
const hasOdds = Number.isFinite(odds) && odds > 1.01;
|
||||
out.push({
|
||||
market,
|
||||
pick,
|
||||
won,
|
||||
// shown probability for calibration (0–100). Prefer calibrated_confidence.
|
||||
shown_confidence:
|
||||
m.calibrated_confidence != null
|
||||
? Number(m.calibrated_confidence)
|
||||
: m.model_probability != null
|
||||
? Number(m.model_probability) * 100
|
||||
: null,
|
||||
model_probability:
|
||||
m.model_probability != null ? Number(m.model_probability) : null,
|
||||
odds: hasOdds ? odds : null,
|
||||
playable: m.playable === true,
|
||||
bet_grade: typeof m.bet_grade === "string" ? m.bet_grade : null,
|
||||
action: typeof m.action === "string" ? m.action : null,
|
||||
value_tier: typeof m.value_tier === "string" ? m.value_tier : null,
|
||||
// flat 1u profit if a real price existed (for per-market ROI)
|
||||
flat_profit: hasOdds ? Number((won ? odds - 1 : -1).toFixed(4)) : null,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private isPredictionPickWon(input: {
|
||||
market: string;
|
||||
pick: string;
|
||||
|
||||
Reference in New Issue
Block a user