main

ai-engine/models/calibration.py

@@ -91,22 +91,26 @@ class Calibrator:
     def __init__(self):
         self.calibrators: Dict[str, IsotonicRegression] = {}
         self.metrics: Dict[str, CalibrationMetrics] = {}
+        # Less aggressive shrinkage — only meaningful overconfident bands are pulled.
+        # Default raised from ~0.85-0.90 to 0.95+ since the orchestrator and config
+        # already apply market-level multipliers; double-shrinkage was the root cause
+        # of 24-35pt avg calibrated-vs-raw drops in production traces.
         self.heuristic_fallback: Dict[str, float] = {
-            "ms": 0.90,
-            "ms_home": 0.90,
-            "ms_home_heavy_fav": 0.95,
-            "ms_home_fav": 0.90,
-            "ms_home_balanced": 0.85,
-            "ms_home_underdog": 0.80,
-            "ms_draw": 0.90,
-            "ms_away": 0.90,
-            "ou15": 0.90,
-            "ou25": 0.90,
-            "ou35": 0.90,
-            "btts": 0.90,
-            "ht_ft": 0.85,
-            "dc": 0.93,
-            "ht": 0.85,
+            "ms": 0.96,
+            "ms_home": 0.96,
+            "ms_home_heavy_fav": 0.98,
+            "ms_home_fav": 0.96,
+            "ms_home_balanced": 0.94,
+            "ms_home_underdog": 0.92,
+            "ms_draw": 0.94,
+            "ms_away": 0.96,
+            "ou15": 0.96,
+            "ou25": 0.96,
+            "ou35": 0.94,
+            "btts": 0.96,
+            "ht_ft": 0.92,
+            "dc": 0.97,
+            "ht": 0.92,
         }
         self._load_calibrators()
         
@@ -139,21 +143,32 @@ class Calibrator:
                 except Exception as e:
                     print(f"[Calibrator] Warning: Failed to load metrics for {market}: {e}")
 
+    # Below this sample count, blend isotonic with raw_prob to dampen overfit jumps.
+    # Above this count, trust isotonic fully.
+    TRUSTED_SAMPLE_FLOOR = 30
+    TRUSTED_SAMPLE_CEILING = 200
+    # Hard cap on how far calibration can move probability in either direction.
+    MAX_DELTA = 0.20
+
     def calibrate(self, market_type: str, raw_prob: float, odds_val: Optional[float] = None) -> float:
         """
-        Calibrate a raw probability using Isotonic Regression.
-        
+        Calibrate a raw probability using Isotonic Regression with safeguards.
+
         Args:
             market_type (str): 'ms_home', 'ou25', 'btts', 'ht_ft', etc.
             raw_prob (float): The raw probability from XGBoost (0.0 - 1.0)
             odds_val (float, optional): The pre-match odds, used for context-aware bucket mapping
-            
+
         Returns:
             float: Calibrated probability (0.0 - 1.0)
+
+        Safeguards:
+          * Low-sample trained models are blended with raw_prob to dampen overfit.
+          * MAX_DELTA caps the per-call adjustment (prevents 40pp swings).
         """
         # Normalize market type
         market_key = market_type.lower().replace("-", "_")
-        
+
         # Route to bucket if ms_home and odds provided
         if market_key == "ms_home" and odds_val is not None and odds_val > 1.0:
             if odds_val <= 1.40:
@@ -164,20 +179,42 @@ class Calibrator:
                 bucket_key = "ms_home_balanced"
             else:
                 bucket_key = "ms_home_underdog"
-                
+
             if bucket_key in self.calibrators:
                 market_key = bucket_key
-        
-        # If we have a trained Isotonic Regression model, use it
+
+        # If we have a trained Isotonic Regression model, use it (with safeguards)
         if market_key in self.calibrators:
             try:
-                calibrated = self.calibrators[market_key].predict([raw_prob])[0]
-                # Ensure output is valid probability
-                return float(np.clip(calibrated, 0.01, 0.99))
+                iso_pred = float(self.calibrators[market_key].predict([raw_prob])[0])
+
+                # Sample-count weighted blend with raw probability.
+                # Sparse models barely move probability; mature models dominate.
+                metrics = self.metrics.get(market_key)
+                n_samples = metrics.sample_count if metrics else 0
+                if n_samples >= self.TRUSTED_SAMPLE_CEILING:
+                    iso_weight = 1.0
+                elif n_samples <= self.TRUSTED_SAMPLE_FLOOR:
+                    # Very sparse: at least 30% trust to surface the signal
+                    iso_weight = max(0.30, n_samples / self.TRUSTED_SAMPLE_CEILING)
+                else:
+                    # Linearly ramp 30% → 100% between floor and ceiling
+                    span = self.TRUSTED_SAMPLE_CEILING - self.TRUSTED_SAMPLE_FLOOR
+                    iso_weight = 0.30 + 0.70 * (n_samples - self.TRUSTED_SAMPLE_FLOOR) / span
+                blended = iso_weight * iso_pred + (1.0 - iso_weight) * raw_prob
+
+                # Cap delta to avoid huge swings on noisy calibrators
+                delta = blended - raw_prob
+                if delta > self.MAX_DELTA:
+                    blended = raw_prob + self.MAX_DELTA
+                elif delta < -self.MAX_DELTA:
+                    blended = raw_prob - self.MAX_DELTA
+
+                return float(np.clip(blended, 0.01, 0.99))
             except Exception as e:
                 print(f"[Calibrator] Warning: Isotonic failed for {market_key}: {e}")
                 # Fall through to heuristic
-        
+
         # Fallback to heuristic calibration
         return self._heuristic_calibrate(market_key, raw_prob)
     

ai-engine/models/v20_ensemble.py

@@ -139,7 +139,7 @@ class FullMatchPrediction:
     ht_confidence: float = 0.0
     
     # === SKOR TAHMİNLERİ ===
-    score: ScorePrediction = None
+    score: Optional[ScorePrediction] = None
     predicted_ft_score: str = "1-1"
     predicted_ht_score: str = "0-0"
     ft_scores_top5: List[Dict] = field(default_factory=list)
@@ -161,7 +161,13 @@ class FullMatchPrediction:
     upset_score: int = 0  # 0-100 arası sürpriz skoru
     upset_level: str = "LOW"  # LOW, MEDIUM, HIGH, EXTREME
     upset_reasons: List[str] = field(default_factory=list)
-    
+
+    # === SÜRPRİZ PROFİLİ ===
+    surprise_score: float = 0.0       # 0-100 overall surprise risk score
+    surprise_comment: str = ""        # Human-readable surprise commentary
+    surprise_reasons: List[str] = field(default_factory=list)  # Flagged risk reasons
+    surprise_breakdown: List[Dict[str, Any]] = field(default_factory=list)  # Per-factor {code, points, label}
+
     # === ENGINE KATKILARI ===
     team_confidence: float = 0.0
     player_confidence: float = 0.0
@@ -412,18 +418,19 @@ class V20EnsemblePredictor:
         
         # Calculators
         print("⚙️  Loading market calculators...")
-        self.match_result_calc = MatchResultCalculator(self.config)
-        self.over_under_calc = OverUnderCalculator(self.config)
-        self.half_time_calc = HalfTimeCalculator(self.config)
-        self.score_calc = ScoreCalculator(self.config)
+        cfg: Any = self.config
+        self.match_result_calc = MatchResultCalculator(cfg)
+        self.over_under_calc = OverUnderCalculator(cfg)
+        self.half_time_calc = HalfTimeCalculator(cfg)
+        self.score_calc = ScoreCalculator(cfg)
         print("   ✅ Score Calculator (XGBoost FT+HT) loaded")
-        self.other_markets_calc = OtherMarketsCalculator(self.config)
-        self.risk_assessor = RiskAssessor(self.config)
-        self.bet_recommender = BetRecommender(self.config)
+        self.other_markets_calc = OtherMarketsCalculator(cfg)
+        self.risk_assessor = RiskAssessor(cfg)
+        self.bet_recommender = BetRecommender(cfg)
         
         # Expert Recommender (New Logic)
         from core.calculators.expert_recommender import ExpertRecommender
-        self.expert_recommender = ExpertRecommender(self.config)
+        self.expert_recommender = ExpertRecommender(cfg)
         
         # XGBoost Integration
         print("🤖 Loading XGBoost models...")
@@ -551,7 +558,7 @@ class V20EnsemblePredictor:
                     features = features.copy()
                     features[col] = 0.0
 
-        return features[expected]
+        return features[expected]  # type: ignore[return-value]
 
     def _favorite_profile_from_odds(self, odds_data: Dict[str, float]) -> Tuple[str, float]:
         """
@@ -838,10 +845,10 @@ class V20EnsemblePredictor:
                 home_team_name: str,
                 away_team_name: str,
                 match_date_ms: int,
-                odds_data: Dict[str, float] = None,
-                home_lineup: List[str] = None,
-                away_lineup: List[str] = None,
-                referee_name: str = None,
+                odds_data: Optional[Dict[str, float]] = None,
+                home_lineup: Optional[List[str]] = None,
+                away_lineup: Optional[List[str]] = None,
+                referee_name: Optional[str] = None,
                 home_goals_avg: float = 1.5,
                 home_conceded_avg: float = 1.2,
                 away_goals_avg: float = 1.2,
@@ -849,9 +856,9 @@ class V20EnsemblePredictor:
                 home_position: int = 10,
                 away_position: int = 10,
                 league_name: str = "",
-                league_id: str = None,
+                league_id: Optional[str] = None,
                 sport: str = "football",
-                sidelined_data: Dict = None) -> FullMatchPrediction:
+                sidelined_data: Optional[Dict] = None) -> FullMatchPrediction:
         """
         Generate complete V20 ensemble prediction.
         
@@ -895,8 +902,8 @@ class V20EnsemblePredictor:
         
         referee_pred = self.referee_engine.predict(
             match_id=match_id,
-            referee_name=referee_name,
-            league_id=league_id
+            referee_name=referee_name or "",
+            league_id=league_id or ""
         )
         
         upset_factors = self.upset_engine.calculate_upset_potential(
@@ -935,9 +942,9 @@ class V20EnsemblePredictor:
             away_position=away_position,
             match_date_ms=match_date_ms,
             odds_data=odds_data,
-            referee_name=referee_name,
-            home_form_score=team_pred.home_form_score if hasattr(team_pred, 'home_form_score') else 50.0,
-            away_form_score=team_pred.away_form_score if hasattr(team_pred, 'away_form_score') else 50.0,
+            referee_name=referee_name or "",
+            home_form_score=getattr(team_pred, 'home_form_score', 50.0),
+            away_form_score=getattr(team_pred, 'away_form_score', 50.0),
             favorite_side=favorite_side,
             favorite_odds=favorite_odds
         )
@@ -1105,7 +1112,7 @@ class V20EnsemblePredictor:
             
         best_bet = _map_dto(rec_result.best_bet)
         alt_bet = _map_dto(rec_result.alternative_bet)
-        recommended = [_map_dto(r) for r in rec_result.recommended_bets]
+        recommended = [m for m in (_map_dto(r) for r in rec_result.recommended_bets) if m is not None]
         
         # Analysis Details
         analysis_details = {
@@ -1187,13 +1194,13 @@ class V20EnsemblePredictor:
             
             # Others
             total_corners_pred=other_result.total_corners_pred,
-            corner_pick=other_result.corner_pick,
+            corner_pick=other_result.corner_pick or "",
             total_cards_pred=other_result.total_cards_pred,
-            card_pick=other_result.card_pick,
+            card_pick=other_result.card_pick or "",
             cards_over_prob=other_result.cards_over_prob,
             cards_under_prob=other_result.cards_under_prob,
             cards_confidence=other_result.cards_confidence,
-            handicap_pick=other_result.handicap_pick,
+            handicap_pick=other_result.handicap_pick or "",
             handicap_home_prob=other_result.handicap_home_prob,
             handicap_draw_prob=other_result.handicap_draw_prob,
             handicap_away_prob=other_result.handicap_away_prob,

ai-engine/models/v25_ensemble.py

@@ -228,15 +228,13 @@ class V25Predictor:
         print(f"[V25] Using fallback feature columns ({len(V25Predictor._FALLBACK_FEATURE_COLS)} features)")
         return V25Predictor._FALLBACK_FEATURE_COLS
 
-    FEATURE_COLS = _load_feature_cols.__func__()
-    
     # Model weights for ensemble
     DEFAULT_WEIGHTS = {
         'xgb': 0.50,
         'lgb': 0.50,
     }
-    
-    def __init__(self, models_dir: str = None):
+
+    def __init__(self, models_dir: Optional[str] = None):
         """
         Initialize V25 Predictor.
         
@@ -246,6 +244,7 @@ class V25Predictor:
         self.models_dir = models_dir or MODELS_DIR
         self.models = {}  # market -> {'xgb': model, 'lgb': model}
         self._loaded = False
+        self.FEATURE_COLS = self._load_feature_cols()
     
     # All trained market models available in V25
     ALL_MARKETS = [
@@ -412,7 +411,7 @@ class V25Predictor:
         
         return float(avg_prob), float(1 - avg_prob)
     
-    def predict_market(self, market: str, features: Dict[str, float]) -> np.ndarray:
+    def predict_market(self, market: str, features: Dict[str, float]) -> Optional[np.ndarray]:
         """
         Generic prediction for any loaded market.
         
@@ -510,15 +509,15 @@ class V25Predictor:
         
         # Determine picks
         ms_probs = {'1': home_prob, 'X': draw_prob, '2': away_prob}
-        ms_pick = max(ms_probs, key=ms_probs.get)
+        ms_pick = max(ms_probs, key=ms_probs.__getitem__)
         ms_confidence = ms_probs[ms_pick] * 100
         
         ou25_probs = {'Over': over_prob, 'Under': under_prob}
-        ou25_pick = max(ou25_probs, key=ou25_probs.get)
+        ou25_pick = max(ou25_probs, key=ou25_probs.__getitem__)
         ou25_confidence = ou25_probs[ou25_pick] * 100
         
         btts_probs = {'Yes': btts_yes_prob, 'No': btts_no_prob}
-        btts_pick = max(btts_probs, key=btts_probs.get)
+        btts_pick = max(btts_probs, key=btts_probs.__getitem__)
         btts_confidence = btts_probs[btts_pick] * 100
         
         # Create prediction