from __future__ import annotations import argparse import csv import json import sys from collections import defaultdict from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, Iterable, Optional import psycopg2 from psycopg2.extras import RealDictCursor AI_ENGINE_DIR = Path(__file__).resolve().parents[1] if str(AI_ENGINE_DIR) not in sys.path: sys.path.insert(0, str(AI_ENGINE_DIR)) from services.single_match_orchestrator import SingleMatchOrchestrator from utils.top_leagues import load_top_league_ids MARKET_ORDER = [ "MS", "DC", "OU15", "OU25", "OU35", "BTTS", "HT", "HT_OU05", "HT_OU15", "HTFT", "OE", "CARDS", "HCAP", ] @dataclass class MatchContext: match_id: str match_date_ms: int league_id: Optional[str] league: str home_team: str away_team: str final_home: int final_away: int ht_home: Optional[int] ht_away: Optional[int] total_cards: Optional[float] @property def match_name(self) -> str: return f"{self.home_team} vs {self.away_team}" @property def final_score(self) -> str: return f"{self.final_home}-{self.final_away}" @property def ht_score(self) -> Optional[str]: if self.ht_home is None or self.ht_away is None: return None return f"{self.ht_home}-{self.ht_away}" @property def total_goals(self) -> int: return self.final_home + self.final_away @property def total_ht_goals(self) -> Optional[int]: if self.ht_home is None or self.ht_away is None: return None return self.ht_home + self.ht_away def _resolve_dsn() -> str: env_path = AI_ENGINE_DIR / ".env" if env_path.exists(): for line in env_path.read_text(encoding="utf-8").splitlines(): if line.startswith("DATABASE_URL="): return line.split("=", 1)[1].strip().split("?schema=")[0] raise SystemExit("DATABASE_URL not found in ai-engine/.env") def _fetch_matches( dsn: str, limit: int, top_league_ids: Optional[list[str]] = None, ) -> list[MatchContext]: query = """ SELECT m.id, m.mst_utc, m.league_id, COALESCE(l.name, 'Unknown League') AS league, COALESCE(ht.name, 'Home') AS home_team, COALESCE(at.name, 'Away') AS away_team, COALESCE(m.score_home, 0) AS score_home, COALESCE(m.score_away, 0) AS score_away, m.ht_score_home, m.ht_score_away, cards.total_cards FROM matches m LEFT JOIN leagues l ON l.id = m.league_id LEFT JOIN teams ht ON ht.id = m.home_team_id LEFT JOIN teams at ON at.id = m.away_team_id LEFT JOIN ( SELECT mpe.match_id, SUM( CASE WHEN mpe.event_type::text LIKE '%%yellow_card%%' THEN 1 WHEN mpe.event_type::text LIKE '%%red_card%%' THEN 2 ELSE 1 END )::float AS total_cards FROM match_player_events mpe WHERE mpe.event_type::text LIKE '%%card%%' GROUP BY mpe.match_id ) cards ON cards.match_id = m.id WHERE m.status = 'FT' AND m.sport = 'football' AND m.score_home IS NOT NULL AND m.score_away IS NOT NULL """ params: list[Any] = [] if top_league_ids: query += " AND m.league_id = ANY(%s)" params.append(top_league_ids) query += """ ORDER BY m.mst_utc DESC LIMIT %s """ params.append(limit) with psycopg2.connect(dsn) as conn: with conn.cursor(cursor_factory=RealDictCursor) as cur: cur.execute(query, params) rows = cur.fetchall() results: list[MatchContext] = [] for row in rows: results.append( MatchContext( match_id=str(row["id"]), match_date_ms=int(row["mst_utc"] or 0), league_id=str(row["league_id"]) if row.get("league_id") else None, league=str(row["league"] or "Unknown League"), home_team=str(row["home_team"] or "Home"), away_team=str(row["away_team"] or "Away"), final_home=int(row["score_home"] or 0), final_away=int(row["score_away"] or 0), ht_home=( int(row["ht_score_home"]) if row.get("ht_score_home") is not None else None ), ht_away=( int(row["ht_score_away"]) if row.get("ht_score_away") is not None else None ), total_cards=( float(row["total_cards"]) if row.get("total_cards") is not None else None ), ) ) return results def _odds_band(odds: float) -> str: if odds < 1.5: return "<1.50" if odds < 1.8: return "1.50-1.79" if odds < 2.1: return "1.80-2.09" if odds < 2.5: return "2.10-2.49" return "2.50+" def _confidence_band(confidence: float) -> str: if confidence < 55.0: return "<55" if confidence < 65.0: return "55-64.9" if confidence < 75.0: return "65-74.9" return "75+" def _edge_band(edge: float) -> str: if edge < 0.03: return "<0.03" if edge < 0.06: return "0.03-0.059" if edge < 0.10: return "0.06-0.099" return "0.10+" def _top_n_buckets(rows: Iterable[tuple[str, float]], limit: int = 10) -> list[dict[str, Any]]: ranked = sorted(rows, key=lambda item: (-item[1], item[0])) return [ {"label": label, "count": int(count)} for label, count in ranked[:limit] ] def _summarize_v26_losses(csv_rows: list[Dict[str, Any]]) -> Dict[str, Any]: losses = [ row for row in csv_rows if row.get("model") == "v26.shadow" and bool(row.get("counted_in_roi")) and row.get("result") == "LOST" ] by_market: Dict[str, float] = defaultdict(float) by_league: Dict[str, float] = defaultdict(float) by_pick: Dict[str, float] = defaultdict(float) by_odds_band: Dict[str, float] = defaultdict(float) by_conf_band: Dict[str, float] = defaultdict(float) by_edge_band: Dict[str, float] = defaultdict(float) for row in losses: market = str(row.get("market") or "UNKNOWN") league = str(row.get("league") or "Unknown League") pick = str(row.get("pick") or "") odds = _safe_float(row.get("odds")) confidence = _safe_float(row.get("confidence")) edge = _safe_float(row.get("edge")) by_market[market] += 1 by_league[league] += 1 by_pick[f"{market} {pick}".strip()] += 1 by_odds_band[_odds_band(odds)] += 1 by_conf_band[_confidence_band(confidence)] += 1 by_edge_band[_edge_band(edge)] += 1 return { "lost_bets": len(losses), "by_market": _top_n_buckets(by_market.items(), limit=20), "by_league": _top_n_buckets(by_league.items(), limit=15), "by_pick": _top_n_buckets(by_pick.items(), limit=15), "by_odds_band": _top_n_buckets(by_odds_band.items(), limit=10), "by_confidence_band": _top_n_buckets(by_conf_band.items(), limit=10), "by_edge_band": _top_n_buckets(by_edge_band.items(), limit=10), } def _safe_float(value: Any) -> float: try: return float(value) except (TypeError, ValueError): return 0.0 def _normalize_text(value: Any) -> str: text = str(value or "").strip().upper() return ( text.replace("İ", "I") .replace("İ", "I") .replace("Ş", "S") .replace("Ğ", "G") .replace("Ü", "U") .replace("Ö", "O") .replace("Ç", "C") ) def _outcome_symbol(home: int, away: int) -> str: if home > away: return "1" if home < away: return "2" return "X" def _resolve_pick( market: str, pick: str, context: MatchContext, ) -> Dict[str, Any]: market_code = _normalize_text(market).replace("/", "") pick_text = str(pick or "").strip() pick_norm = _normalize_text(pick_text) if not market_code or not pick_norm: return {"result": "UNRESOLVED", "won": None, "note": "pick_missing"} if market_code == "HTFT": market_code = "HTFT" if market_code == "HTFT" or market_code == "HTFT": if context.ht_home is None or context.ht_away is None: return {"result": "UNRESOLVED", "won": None, "note": "ht_score_missing"} if "/" not in pick_text: return {"result": "UNRESOLVED", "won": None, "note": "htft_pick_invalid"} ht_pick, ft_pick = pick_text.split("/", 1) actual = f"{_outcome_symbol(context.ht_home, context.ht_away)}/{_outcome_symbol(context.final_home, context.final_away)}" won = f"{_normalize_text(ht_pick)}/{_normalize_text(ft_pick)}" == actual return {"result": "WON" if won else "LOST", "won": won, "note": f"actual={actual}"} if market_code == "MS": actual = _outcome_symbol(context.final_home, context.final_away) won = pick_norm in {actual, f"MS {actual}"} return {"result": "WON" if won else "LOST", "won": won, "note": f"actual={actual}"} if market_code == "DC": actual = _outcome_symbol(context.final_home, context.final_away) winning = { "1X": {"1", "X"}, "X2": {"X", "2"}, "12": {"1", "2"}, } won = actual in winning.get(pick_norm, set()) return {"result": "WON" if won else "LOST", "won": won, "note": f"actual={actual}"} if market_code in {"OU15", "OU25", "OU35", "HTOU05", "HTOU15", "HT_OU05", "HT_OU15"}: if market_code in {"HTOU05", "HTOU15", "HT_OU05", "HT_OU15"}: if context.total_ht_goals is None: return {"result": "UNRESOLVED", "won": None, "note": "ht_score_missing"} total = context.total_ht_goals line = 0.5 if "05" in market_code else 1.5 else: total = context.total_goals line = {"OU15": 1.5, "OU25": 2.5, "OU35": 3.5}[market_code] if "UST" in pick_norm or "OVER" in pick_norm: won = total > line side = "OVER" elif "ALT" in pick_norm or "UNDER" in pick_norm: won = total < line side = "UNDER" else: return {"result": "UNRESOLVED", "won": None, "note": "ou_side_unknown"} return { "result": "WON" if won else "LOST", "won": won, "note": f"actual_total={total} side={side} line={line}", } if market_code == "BTTS": both_scored = context.final_home > 0 and context.final_away > 0 if "VAR" in pick_norm or "YES" in pick_norm: won = both_scored side = "YES" elif "YOK" in pick_norm or pick_norm.endswith("NO") or pick_norm == "NO": won = not both_scored side = "NO" else: return {"result": "UNRESOLVED", "won": None, "note": "btts_side_unknown"} return { "result": "WON" if won else "LOST", "won": won, "note": f"actual_btts={'YES' if both_scored else 'NO'} side={side}", } if market_code == "HT": if context.ht_home is None or context.ht_away is None: return {"result": "UNRESOLVED", "won": None, "note": "ht_score_missing"} actual = _outcome_symbol(context.ht_home, context.ht_away) won = pick_norm == actual return {"result": "WON" if won else "LOST", "won": won, "note": f"actual={actual}"} if market_code == "OE": actual = "EVEN" if context.total_goals % 2 == 0 else "ODD" if pick_norm in {"CIFT", "EVEN"}: wanted = "EVEN" elif pick_norm in {"TEK", "ODD"}: wanted = "ODD" else: return {"result": "UNRESOLVED", "won": None, "note": "oe_pick_unknown"} won = actual == wanted return {"result": "WON" if won else "LOST", "won": won, "note": f"actual={actual}"} if market_code == "CARDS": if context.total_cards is None: return {"result": "UNRESOLVED", "won": None, "note": "cards_missing"} if "UST" in pick_norm or "OVER" in pick_norm: won = context.total_cards > 4.5 side = "OVER" elif "ALT" in pick_norm or "UNDER" in pick_norm: won = context.total_cards < 4.5 side = "UNDER" else: return {"result": "UNRESOLVED", "won": None, "note": "cards_side_unknown"} return { "result": "WON" if won else "LOST", "won": won, "note": f"actual_cards={context.total_cards:.1f} side={side} line=4.5", } if market_code == "HCAP": adjusted_home = context.final_home - 1.0 adjusted_away = float(context.final_away) if adjusted_home > adjusted_away: actual = "1" elif adjusted_home < adjusted_away: actual = "2" else: actual = "X" won = pick_norm == actual return { "result": "WON" if won else "LOST", "won": won, "note": f"actual={actual} line_home=-1.0", } return {"result": "UNRESOLVED", "won": None, "note": "market_not_supported"} def _evaluate_row( market: str, pick: str, odds: Any, playable: bool, stake_units: Any, context: MatchContext, ) -> Dict[str, Any]: resolution = _resolve_pick(market, pick, context) odds_value = _safe_float(odds) stake_value = _safe_float(stake_units) counted = bool(playable and odds_value > 1.01 and resolution["result"] in {"WON", "LOST"}) flat_profit = 0.0 stake_profit = 0.0 if counted: flat_profit = (odds_value - 1.0) if resolution["result"] == "WON" else -1.0 stake_profit = flat_profit * (stake_value if stake_value > 0 else 1.0) return { "result": resolution["result"], "won": resolution["won"], "resolution_note": resolution["note"], "counted_in_roi": counted, "profit_flat": round(flat_profit, 4), "profit_stake": round(stake_profit, 4), } def _summarize_bucket(bucket: Dict[str, float]) -> Dict[str, Any]: played = int(bucket["played"]) won = int(bucket["won"]) lost = int(bucket["lost"]) unresolved = int(bucket["unresolved"]) profit = round(bucket["profit"], 4) roi = round((profit / played) * 100.0, 2) if played else 0.0 win_rate = round((won / played) * 100.0, 2) if played else 0.0 return { "played": played, "won": won, "lost": lost, "unresolved": unresolved, "profit_flat": profit, "roi_flat_pct": roi, "win_rate_pct": win_rate, } def _format_date(ms: int) -> str: if ms <= 0: return "-" dt = datetime.fromtimestamp(ms / 1000, tz=timezone.utc) return dt.strftime("%Y-%m-%d") def _build_markdown_report(report: Dict[str, Any]) -> str: lines: list[str] = [] lines.append("# v25 vs v26.shadow ROI Report") lines.append("") lines.append(f"- Sample: last {report['sample_size']} finished football matches") if report.get("top_leagues_only"): lines.append("- Filter: top leagues only") lines.append("- ROI calculation: flat `1 unit` per playable and resolvable bet") lines.append(f"- Generated at: {report['generated_at']}") lines.append("") lines.append("## Overall Summary") lines.append("") lines.append("| Model | Played | Won | Lost | Win Rate | Profit | ROI | Main Pick ROI | Main Pick W/L |") lines.append("|---|---:|---:|---:|---:|---:|---:|---:|---|") for model_name, payload in report["summary"]["models"].items(): main = payload["main_pick"] lines.append( f"| {model_name} | {payload['all_playable']['played']} | {payload['all_playable']['won']} | " f"{payload['all_playable']['lost']} | {payload['all_playable']['win_rate_pct']}% | " f"{payload['all_playable']['profit_flat']:+.2f} | {payload['all_playable']['roi_flat_pct']:+.2f}% | " f"{main['roi_flat_pct']:+.2f}% | {main['won']}/{main['played']} |" ) lines.append("") lines.append("## Market Summary") lines.append("") lines.append("| Model | Market | Played | Won | Lost | Profit | ROI |") lines.append("|---|---|---:|---:|---:|---:|---:|") for model_name, markets in report["summary"]["markets"].items(): for market_name in MARKET_ORDER: payload = markets.get(market_name) if not payload or payload["played"] == 0: continue lines.append( f"| {model_name} | {market_name} | {payload['played']} | {payload['won']} | {payload['lost']} | " f"{payload['profit_flat']:+.2f} | {payload['roi_flat_pct']:+.2f}% |" ) lines.append("") loss_summary = report["summary"].get("v26_loss_analysis", {}) if loss_summary: lines.append("## v26 Loss Analysis") lines.append("") lines.append(f"- Lost bets: {loss_summary.get('lost_bets', 0)}") lines.append("") lines.append("| Bucket | Top Items |") lines.append("|---|---|") for label, key in ( ("By market", "by_market"), ("By league", "by_league"), ("By pick", "by_pick"), ("By odds band", "by_odds_band"), ("By confidence band", "by_confidence_band"), ("By edge band", "by_edge_band"), ): items = loss_summary.get(key) or [] rendered = ", ".join(f"{item['label']} ({item['count']})" for item in items[:6]) or "-" lines.append(f"| {label} | {rendered} |") lines.append("") lines.append("## Match By Match") lines.append("") lines.append("| Date | Match | Score | v25 Main | v25 Played Picks | v25 Profit | v26 Main | v26 Played Picks | v26 Profit |") lines.append("|---|---|---|---|---|---:|---|---|---:|") for match in report["matches"]: v25 = match["models"]["v25"] v26 = match["models"]["v26.shadow"] lines.append( f"| {_format_date(match['match_date_ms'])} | {match['match_name']} | {match['final_score']} | " f"{v25['main_pick']['summary']} | {v25['played_picks_summary']} | {v25['profit_flat']:+.2f} | " f"{v26['main_pick']['summary']} | {v26['played_picks_summary']} | {v26['profit_flat']:+.2f} |" ) lines.append("") return "\n".join(lines) def main() -> None: parser = argparse.ArgumentParser( description="Detailed ROI backtest for v25 vs v26.shadow.", ) parser.add_argument("--limit", type=int, default=60, help="Number of finished matches to analyze.") parser.add_argument( "--top-leagues-only", action="store_true", help="Only analyze matches whose league_id exists in top_leagues.json.", ) args = parser.parse_args() dsn = _resolve_dsn() top_league_ids = sorted(load_top_league_ids()) if args.top_leagues_only else None matches = _fetch_matches(dsn, max(1, args.limit), top_league_ids=top_league_ids) orchestrator = SingleMatchOrchestrator() report_matches: list[Dict[str, Any]] = [] model_aggregate: Dict[str, Dict[str, float]] = { "v25": defaultdict(float), "v26.shadow": defaultdict(float), } main_pick_aggregate: Dict[str, Dict[str, float]] = { "v25": defaultdict(float), "v26.shadow": defaultdict(float), } market_aggregate: Dict[str, Dict[str, Dict[str, float]]] = { "v25": defaultdict(lambda: defaultdict(float)), "v26.shadow": defaultdict(lambda: defaultdict(float)), } csv_rows: list[Dict[str, Any]] = [] for context in matches: match_payload = { "match_id": context.match_id, "match_name": context.match_name, "league": context.league, "match_date_ms": context.match_date_ms, "final_score": context.final_score, "ht_score": context.ht_score, "total_cards": context.total_cards, "models": {}, } for model_name, mode in (("v25", "v25"), ("v26.shadow", "v26")): orchestrator.engine_mode = mode package = orchestrator.analyze_match(context.match_id) or {} rows = package.get("bet_summary") or [] evaluated_rows: list[Dict[str, Any]] = [] match_profit = 0.0 for row in rows: market = str(row.get("market") or "") pick = str(row.get("pick") or "") evaluation = _evaluate_row( market=market, pick=pick, odds=row.get("odds"), playable=bool(row.get("playable")), stake_units=row.get("stake_units"), context=context, ) combined = { "market": market, "pick": pick, "playable": bool(row.get("playable")), "bet_grade": row.get("bet_grade"), "odds": round(_safe_float(row.get("odds")), 2), "calibrated_confidence": round(_safe_float(row.get("calibrated_confidence")), 1), "edge": round(_safe_float(row.get("ev_edge", row.get("edge"))), 4), "stake_units": round(_safe_float(row.get("stake_units")), 2), **evaluation, } evaluated_rows.append(combined) if combined["counted_in_roi"]: bucket = market_aggregate[model_name][market] bucket["played"] += 1 if combined["result"] == "WON": bucket["won"] += 1 else: bucket["lost"] += 1 bucket["profit"] += combined["profit_flat"] model_bucket = model_aggregate[model_name] model_bucket["played"] += 1 if combined["result"] == "WON": model_bucket["won"] += 1 else: model_bucket["lost"] += 1 model_bucket["profit"] += combined["profit_flat"] match_profit += combined["profit_flat"] elif combined["playable"]: model_aggregate[model_name]["unresolved"] += 1 market_aggregate[model_name][market]["unresolved"] += 1 csv_rows.append( { "match_id": context.match_id, "date": _format_date(context.match_date_ms), "league": context.league, "match": context.match_name, "final_score": context.final_score, "ht_score": context.ht_score or "", "model": model_name, "market": market, "pick": pick, "playable": combined["playable"], "bet_grade": combined["bet_grade"], "odds": combined["odds"], "confidence": combined["calibrated_confidence"], "edge": combined["edge"], "result": combined["result"], "counted_in_roi": combined["counted_in_roi"], "profit_flat": combined["profit_flat"], "resolution_note": combined["resolution_note"], } ) main_pick = package.get("main_pick") or {} main_eval = _evaluate_row( market=str(main_pick.get("market") or ""), pick=str(main_pick.get("pick") or ""), odds=main_pick.get("odds"), playable=bool(main_pick.get("playable")), stake_units=main_pick.get("stake_units"), context=context, ) main_pick_summary = { "market": main_pick.get("market"), "pick": main_pick.get("pick"), "playable": bool(main_pick.get("playable")), "odds": round(_safe_float(main_pick.get("odds")), 2), "confidence": round( _safe_float( main_pick.get("calibrated_confidence", main_pick.get("confidence")) ), 1, ), "edge": round(_safe_float(main_pick.get("ev_edge", main_pick.get("edge"))), 4), **main_eval, } if main_pick_summary["counted_in_roi"]: summary_suffix = ( f"{main_pick_summary['result']}, played, {main_pick_summary['profit_flat']:+.2f}" ) elif main_pick_summary.get("market") and main_pick_summary.get("pick"): summary_suffix = f"{main_pick_summary['result']}, not played" else: summary_suffix = "" if main_pick_summary["counted_in_roi"]: bucket = main_pick_aggregate[model_name] bucket["played"] += 1 if main_pick_summary["result"] == "WON": bucket["won"] += 1 else: bucket["lost"] += 1 bucket["profit"] += main_pick_summary["profit_flat"] elif main_pick_summary["playable"]: main_pick_aggregate[model_name]["unresolved"] += 1 main_pick_summary["summary"] = ( f"{main_pick_summary['market']} {main_pick_summary['pick']} " f"({summary_suffix})" if main_pick_summary.get("market") and main_pick_summary.get("pick") else "No main pick" ) played_rows = [row for row in evaluated_rows if row["counted_in_roi"]] played_picks_summary = ( "; ".join( f"{row['market']} {row['pick']}={row['result']} ({row['profit_flat']:+.2f})" for row in played_rows ) if played_rows else "-" ) match_payload["models"][model_name] = { "main_pick": main_pick_summary, "profit_flat": round(match_profit, 4), "played_picks_summary": played_picks_summary, "played_picks": played_rows, "all_picks": evaluated_rows, } report_matches.append(match_payload) summary = { "models": { model_name: { "all_playable": _summarize_bucket(model_aggregate[model_name]), "main_pick": _summarize_bucket(main_pick_aggregate[model_name]), } for model_name in ("v25", "v26.shadow") }, "markets": { model_name: { market_name: _summarize_bucket(bucket) for market_name, bucket in sorted( market_aggregate[model_name].items(), key=lambda item: ( MARKET_ORDER.index(item[0]) if item[0] in MARKET_ORDER else 999, item[0], ), ) } for model_name in ("v25", "v26.shadow") }, "v26_loss_analysis": _summarize_v26_losses(csv_rows), } report = { "generated_at": datetime.now(timezone.utc).isoformat(), "sample_size": len(report_matches), "top_leagues_only": bool(args.top_leagues_only), "summary": summary, "matches": report_matches, } report_dir = AI_ENGINE_DIR / "reports" json_path = report_dir / "backtest_v26_shadow_roi_detail.json" csv_path = report_dir / "backtest_v26_shadow_roi_picks.csv" md_path = report_dir / "backtest_v26_shadow_roi_report.md" json_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") with csv_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter( handle, fieldnames=[ "match_id", "date", "league", "match", "final_score", "ht_score", "model", "market", "pick", "playable", "bet_grade", "odds", "confidence", "edge", "result", "counted_in_roi", "profit_flat", "resolution_note", ], ) writer.writeheader() writer.writerows(csv_rows) md_path.write_text(_build_markdown_report(report), encoding="utf-8") print(f"[OK] JSON report written to {json_path}") print(f"[OK] CSV report written to {csv_path}") print(f"[OK] Markdown report written to {md_path}") if __name__ == "__main__": main()