import { Injectable, Logger } from "@nestjs/common"; import { Cron } from "@nestjs/schedule"; import { HttpService } from "@nestjs/axios"; import { PrismaService } from "../database/prisma.service"; import { firstValueFrom } from "rxjs"; import { FeederScraperService } from "../modules/feeder/feeder-scraper.service"; import * as fs from "fs"; import * as path from "path"; import { Prisma } from "@prisma/client"; import { SidelinedResponse } from "../modules/feeder/feeder.types"; import { deriveStoredMatchStatus, FINISHED_STATE_VALUES_FOR_DB, FINISHED_STATUS_VALUES_FOR_DB, LIVE_STATE_VALUES_FOR_DB, LIVE_STATUS_VALUES_FOR_DB, } from "../common/utils/match-status.util"; import { getDateStringInTimeZone, getDayBoundsForTimeZone, getShiftedDateStringInTimeZone, } from "../common/utils/timezone.util"; import { TaskLockService } from "./task-lock.service"; // ──────────────────────────────────────────────────────────────── // Types // ──────────────────────────────────────────────────────────────── interface LiveScoreTeamPayload { id: string; name: string; slug: string | null; } interface LiveScoreCompetitionPayload { id: string; name: string; slug: string | null; country: { id: string; name: string; } | null; } interface LiveScorePayloadMatch { id: string; matchName: string; matchSlug: string; competitionId: string | null; mstUtc: number | null; state: string | null; substate: string | null; status: string | null; statusBoxContent: string | null; // ERT = Erteledendi iddaaCode: string | null; homeTeam: LiveScoreTeamPayload; awayTeam: LiveScoreTeamPayload; homeScore: number | null; awayScore: number | null; score: { home: number | null; away: number | null; } | null; } type LiveMatchOddsTarget = Prisma.LiveMatchGetPayload<{ include: { homeTeam: { select: { name: true } }; awayTeam: { select: { name: true } }; }; }>; interface LiveLineupsJson { home: { xi: unknown[]; subs: unknown[] }; away: { xi: unknown[]; subs: unknown[] }; } interface PendingPredictionRunForSettlement { id: bigint; matchId: string; engineVersion: string; payloadSummary: unknown; scoreHome: number | null; scoreAway: number | null; htScoreHome: number | null; htScoreAway: number | null; } type SportType = "football" | "basketball"; // ──────────────────────────────────────────────────────────────── // Service // ──────────────────────────────────────────────────────────────── @Injectable() export class DataFetcherTask { private readonly logger = new Logger(DataFetcherTask.name); private readonly timeZone = "Europe/Istanbul"; constructor( private readonly httpService: HttpService, private readonly prisma: PrismaService, private readonly scraper: FeederScraperService, private readonly taskLock: TaskLockService, ) {} // ──────────────────────────────────────────────────────────── // CRON 1: Main sync — every 15 minutes // Phases: match list → live scores → odds → lineups // ──────────────────────────────────────────────────────────── @Cron("*/15 * * * *") async syncLiveMatches(): Promise { if (this.shouldSkipInHistoricalMode("syncLiveMatches")) return; await this.taskLock.runWithLease( "syncLiveMatches", 30 * 60 * 1000, async () => { await this.runLiveSync(); }, this.logger, ); } // ──────────────────────────────────────────────────────────── // CRON 2: Daily cleanup + full sync — 07:00 Istanbul // Preserve yesterday as a fallback until the 08:00 archive job completes. // ──────────────────────────────────────────────────────────── @Cron("0 7 * * *", { timeZone: "Europe/Istanbul" }) async cleanAndFullSync(): Promise { if (this.shouldSkipInHistoricalMode("cleanAndFullSync")) return; await this.taskLock.runWithLease( "cleanAndFullSync", 2 * 60 * 60 * 1000, async () => { this.logger.log( "cleanAndFullSync: Pruning stale live_matches while preserving yesterday for archive fallback...", ); try { const yesterdayDate = getShiftedDateStringInTimeZone( -1, this.timeZone, ); const { startMs: yesterdayStartMs } = getDayBoundsForTimeZone( yesterdayDate, this.timeZone, ); const cutoffDate = new Date(yesterdayStartMs); const deleted = await this.prisma.liveMatch.deleteMany({ where: { OR: [ { mstUtc: { lt: BigInt(yesterdayStartMs) } }, { AND: [ { mstUtc: null }, { updatedAt: { lt: cutoffDate } }, { OR: [ { status: { in: FINISHED_STATUS_VALUES_FOR_DB } }, { state: { in: FINISHED_STATE_VALUES_FOR_DB } }, ], }, ], }, ], }, }); this.logger.log( `Pruned ${deleted.count} stale live matches. Starting full sync...`, ); } catch (error: unknown) { const message = error instanceof Error ? error.message : String(error); this.logger.error(`Stale live_match cleanup failed: ${message}`); return; } await this.runLiveSync(); }, this.logger, ); } // ──────────────────────────────────────────────────────────── // Phase 1: Fetch match list for all sports // ──────────────────────────────────────────────────────────── private async runLiveSync(): Promise { if (this.shouldSkipInHistoricalMode("syncLiveMatches")) return; this.logger.log("syncLiveMatches START"); const today = getDateStringInTimeZone(new Date(), this.timeZone); const tomorrow = getShiftedDateStringInTimeZone(1, this.timeZone); await this.syncMatchList(today); await this.syncMatchList(tomorrow); await this.updateLiveScores(); await this.settlePredictionRuns(); await this.fetchOddsForMatches(); await this.fillMissingLineups(); this.logger.log("syncLiveMatches END"); } private async syncMatchList(date: string): Promise { // Football const footballLeagues = this.loadLeagueFilterSet("qualified_leagues.json"); if (footballLeagues && footballLeagues.size > 0) { await this.fetchMatchesForSport("football", date, footballLeagues); } else { this.logger.warn( "qualified_leagues.json is missing/empty — writing ALL football matches", ); await this.fetchMatchesForSport("football", date, new Set()); } // Basketball const basketballLeagues = this.loadLeagueFilterSet( "basketball_top_leagues.json", ); if (!basketballLeagues || basketballLeagues.size === 0) { this.logger.error( "basketball_top_leagues.json is missing/empty. Basketball ingest skipped to protect data quality.", ); } else { await this.fetchMatchesForSport("basketball", date, basketballLeagues); // Clean up basketball matches that are NOT in configured leagues await this.prisma.liveMatch.deleteMany({ where: { sport: "basketball", OR: [ { leagueId: null }, { leagueId: { notIn: Array.from(basketballLeagues) } }, ], }, }); } } // ──────────────────────────────────────────────────────────── // Phase 2: Live score updates (merged from live-updater.task) // ──────────────────────────────────────────────────────────── private async updateLiveScores(): Promise { try { const liveMatches = await this.prisma.liveMatch.findMany({ where: { OR: [ { state: { in: LIVE_STATE_VALUES_FOR_DB } }, { status: { in: LIVE_STATUS_VALUES_FOR_DB } }, ], }, select: { id: true, matchSlug: true }, }); if (liveMatches.length === 0) { this.logger.debug("No live matches to update scores for"); return; } this.logger.log( `LIVE Updating scores for ${liveMatches.length} live matches`, ); for (const match of liveMatches) { try { const url = `https://www.mackolik.com/ajax/football/match-info?matchId=${match.id}`; const response = await firstValueFrom( this.httpService.get(url, { timeout: 5000 }), ); if (response.data?.data) { const matchData = response.data.data; const scoreHome = matchData.homeScore ?? null; const scoreAway = matchData.awayScore ?? null; const storedStatus = deriveStoredMatchStatus({ state: matchData.state, status: matchData.status, substate: matchData.substate, scoreHome, scoreAway, }); await this.prisma.liveMatch.update({ where: { id: match.id }, data: { scoreHome, scoreAway, state: matchData.state || null, substate: matchData.substate || null, status: storedStatus, updatedAt: new Date(), }, }); } } catch { // Individual match update failed, continue with others } } this.logger.log("LIVE Live score update complete"); } catch (error: unknown) { const message = error instanceof Error ? error.message : String(error); this.logger.error(`Live score update failed: ${message}`); } } private async settlePredictionRuns(): Promise { try { const rows = await this.prisma.$queryRawUnsafe< PendingPredictionRunForSettlement[] >(` SELECT pr.id, pr.match_id AS "matchId", pr.engine_version AS "engineVersion", pr.payload_summary AS "payloadSummary", m.score_home AS "scoreHome", m.score_away AS "scoreAway", m.ht_score_home AS "htScoreHome", m.ht_score_away AS "htScoreAway" FROM prediction_runs pr JOIN matches m ON m.id = pr.match_id WHERE pr.eventual_outcome IS NULL AND m.sport = 'football' AND m.status = 'FT' AND m.score_home IS NOT NULL AND m.score_away IS NOT NULL ORDER BY pr.generated_at ASC LIMIT 500 `); if (rows.length === 0) return; let settled = 0; for (const row of rows) { const result = this.resolvePredictionRunSettlement(row); if (!result) continue; const closingOddsSnapshot = await this.getClosingOddsSnapshot(row.matchId); const settlementSummary = { settled_at: new Date().toISOString(), model_version: row.engineVersion, outcome: result.outcome, unit_profit: result.unitProfit, final_score: { home: row.scoreHome, away: row.scoreAway, }, halftime_score: { home: row.htScoreHome, away: row.htScoreAway, }, closing_odds_snapshot: closingOddsSnapshot, }; await this.prisma.$executeRawUnsafe( ` UPDATE prediction_runs SET eventual_outcome = $1, unit_profit = $2, payload_summary = payload_summary || jsonb_build_object('settlement', $3::jsonb) WHERE id = $4 `, result.outcome, result.unitProfit, JSON.stringify(settlementSummary), row.id, ); settled++; } if (settled > 0) { this.logger.log(`Settled ${settled} prediction run(s)`); } } catch (error: unknown) { const message = error instanceof Error ? error.message : String(error); this.logger.warn(`Prediction run settlement skipped: ${message}`); } } private async getClosingOddsSnapshot( matchId: string, ): Promise> { const liveMatch = await this.prisma.liveMatch.findUnique({ where: { id: matchId }, select: { odds: true, oddsUpdatedAt: true, status: true, state: true, scoreHome: true, scoreAway: true, }, }); if (liveMatch?.odds) { return { source: "live_match", odds: liveMatch.odds, odds_updated_at: liveMatch.oddsUpdatedAt?.toISOString() ?? null, status: liveMatch.status ?? null, state: liveMatch.state ?? null, score_home: liveMatch.scoreHome, score_away: liveMatch.scoreAway, }; } const categories = await this.prisma.oddCategory.findMany({ where: { matchId }, select: { name: true, selections: { select: { name: true, oddValue: true, position: true, updatedAt: true, }, orderBy: { position: "asc" }, take: 12, }, }, orderBy: { name: "asc" }, take: 24, }); return { source: "odd_selections", category_count: categories.length, categories: categories.map((category) => ({ name: category.name, selections: category.selections.map((selection) => ({ name: selection.name, odd_value: selection.oddValue, position: selection.position, updated_at: selection.updatedAt?.toISOString() ?? null, })), })), }; } private resolvePredictionRunSettlement( row: PendingPredictionRunForSettlement, ): { outcome: string; unitProfit: number } | null { const summary = this.asRecord(row.payloadSummary); const mainPick = this.asRecord(summary.main_pick); const market = String(mainPick.market || ""); const pick = String(mainPick.pick || ""); const playable = mainPick.playable === true; const odds = Number(mainPick.odds || 0); if (!market || !pick || !playable || !Number.isFinite(odds) || odds <= 1.01) { return { outcome: "NO_BET", unitProfit: 0 }; } const won = this.isPredictionPickWon({ market, pick, scoreHome: row.scoreHome, scoreAway: row.scoreAway, htScoreHome: row.htScoreHome, htScoreAway: row.htScoreAway, }); if (won === null) return null; return { outcome: `${won ? "WON" : "LOST"}:${market}:${pick}`, unitProfit: Number((won ? odds - 1 : -1).toFixed(4)), }; } private isPredictionPickWon(input: { market: string; pick: string; scoreHome: number | null; scoreAway: number | null; htScoreHome: number | null; htScoreAway: number | null; }): boolean | null { const market = input.market.toUpperCase(); const pick = this.normalizePick(input.pick); const scoreHome = input.scoreHome; const scoreAway = input.scoreAway; if (scoreHome === null || scoreAway === null) return null; if (market === "MS") { if (pick === "1") return scoreHome > scoreAway; if (pick === "X" || pick === "0") return scoreHome === scoreAway; if (pick === "2") return scoreAway > scoreHome; return null; } if (market === "DC") { const normalized = pick.replace("-", ""); if (normalized === "1X") return scoreHome >= scoreAway; if (normalized === "X2") return scoreAway >= scoreHome; if (normalized === "12") return scoreHome !== scoreAway; return null; } if (market === "BTTS") { const bothScored = scoreHome > 0 && scoreAway > 0; if (pick.includes("VAR") || pick.includes("YES") || pick === "Y") { return bothScored; } if (pick.includes("YOK") || pick.includes("NO") || pick === "N") { return !bothScored; } return null; } const goalLine = this.goalLineForMarket(market); if (goalLine !== null) { const total = market.startsWith("HT_") ? this.nullableSum(input.htScoreHome, input.htScoreAway) : scoreHome + scoreAway; if (total === null) return null; if (this.isOverPick(pick)) return total > goalLine; return total < goalLine; } if (market === "HT") { const htHome = input.htScoreHome; const htAway = input.htScoreAway; if (htHome === null || htAway === null) return null; if (pick === "1") return htHome > htAway; if (pick === "X" || pick === "0") return htHome === htAway; if (pick === "2") return htAway > htHome; } if (market === "HTFT") { const htHome = input.htScoreHome; const htAway = input.htScoreAway; if (htHome === null || htAway === null || !pick.includes("/")) return null; const [htPick, ftPick] = pick.split("/"); return ( this.isResultPickWon(htPick, htHome, htAway) === true && this.isResultPickWon(ftPick, scoreHome, scoreAway) === true ); } return null; } private isResultPickWon( pick: string, homeScore: number, awayScore: number, ): boolean | null { if (pick === "1") return homeScore > awayScore; if (pick === "X" || pick === "0") return homeScore === awayScore; if (pick === "2") return awayScore > homeScore; return null; } private goalLineForMarket(market: string): number | null { if (market === "OU15") return 1.5; if (market === "OU25") return 2.5; if (market === "OU35") return 3.5; if (market === "HT_OU05") return 0.5; if (market === "HT_OU15") return 1.5; return null; } private nullableSum(a: number | null, b: number | null): number | null { if (a === null || b === null) return null; return a + b; } private normalizePick(value: string): string { return value .trim() .toUpperCase() .replace(/İ/g, "I") .replace(/Ü/g, "U") .replace(/Ş/g, "S") .replace(/Ğ/g, "G") .replace(/Ö/g, "O") .replace(/Ç/g, "C"); } private isOverPick(pick: string): boolean { return pick.includes("UST") || pick.includes("OVER"); } private asRecord(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; } // Phase 3: Odds + referee + lineups + sidelined private async fetchOddsForMatches(): Promise { this.logger.log("MONEY Fetching odds for live matches..."); try { // Load both league filters (data-driven qualified leagues) const topLeagueIds: string[] = []; const footballLeagues = this.loadLeagueFilterSet( "qualified_leagues.json", ); if (footballLeagues) topLeagueIds.push(...footballLeagues); const basketballLeagues = this.loadLeagueFilterSet( "basketball_top_leagues.json", ); if (basketballLeagues) topLeagueIds.push(...basketballLeagues); const allowedLeagueIds = Array.from(new Set(topLeagueIds)); // Get matches needing odds (from 12 hours ago onward) // CRITICAL: Only fetch odds/lineups for NOT STARTED matches. // Once a match goes live, odds selections disappear or change, // corrupting the pre-match data the AI model relies on. const twelveHoursAgo = new Date(Date.now() - 12 * 60 * 60 * 1000); const matchesToFetch = await this.prisma.liveMatch.findMany({ where: { mstUtc: { gte: BigInt(twelveHoursAgo.getTime()) }, ...(allowedLeagueIds.length > 0 ? { leagueId: { in: allowedLeagueIds } } : {}), // Exclude live and finished matches — preserve their pre-match odds NOT: { OR: [ { status: { in: FINISHED_STATUS_VALUES_FOR_DB } }, { state: { in: FINISHED_STATE_VALUES_FOR_DB } }, { status: { in: LIVE_STATUS_VALUES_FOR_DB } }, { state: { in: LIVE_STATE_VALUES_FOR_DB } }, ], }, }, include: { homeTeam: { select: { name: true } }, awayTeam: { select: { name: true } }, }, orderBy: [{ oddsUpdatedAt: "asc" }, { mstUtc: "asc" }], take: 1000, }); if (matchesToFetch.length === 0) { this.logger.log("MONEY No matches to fetch odds for"); return; } this.logger.log( `MONEY Fetching odds for ${matchesToFetch.length} matches`, ); let successCount = 0; let errorCount = 0; const failedMatches: LiveMatchOddsTarget[] = []; // Initial pass for (const match of matchesToFetch) { try { await this.processMatchOdds(match); successCount++; await this.delay(500); } catch (err: unknown) { errorCount++; const isRetryable = this.isRetryableError(err); if (isRetryable) { failedMatches.push(match); } else { const message = err instanceof Error ? err.message : String(err); this.logger.warn( `Match ${match.id} odds fetch failed (Non-retryable): ${message}`, ); } } } // Retry failed matches (502/Timeout) if (failedMatches.length > 0) { this.logger.warn( `Retrying ${failedMatches.length} failed matches (502/Timeout)...`, ); for (const match of failedMatches) { await this.delay(2000); try { await this.processMatchOdds(match); successCount++; this.logger.log(`SUCCESS Retry successful for match ${match.id}`); } catch (retryErr: unknown) { const message = retryErr instanceof Error ? retryErr.message : String(retryErr); this.logger.error( `❌ Retry failed for match ${match.id}: ${message}`, ); } } } this.logger.log( `MONEY Odds complete: ${successCount} success, ${errorCount} errors (initially)`, ); } catch (error: unknown) { const message = error instanceof Error ? error.message : String(error); this.logger.error(`Odds fetch job failed: ${message}`); } } // ──────────────────────────────────────────────────────────── // Phase 4: Fill missing lineups (backup) // ──────────────────────────────────────────────────────────── private async fillMissingLineups(): Promise { try { const matchesToUpdate = await this.prisma.liveMatch.findMany({ where: { sport: "football", NOT: { OR: [ { status: { in: FINISHED_STATUS_VALUES_FOR_DB } }, { state: { in: FINISHED_STATE_VALUES_FOR_DB } }, { AND: [ { scoreHome: { not: null } }, { scoreAway: { not: null } }, { NOT: { OR: [ { status: { in: LIVE_STATUS_VALUES_FOR_DB } }, { state: { in: LIVE_STATE_VALUES_FOR_DB } }, ], }, }, ], }, ], }, }, select: { id: true, matchSlug: true, lineups: true, sport: true }, take: 30, }); // Only football matches without lineups const toUpdate = matchesToUpdate.filter( (m) => !m.lineups && m.sport === "football", ); if (toUpdate.length === 0) { this.logger.debug("👕 All lineups already filled"); return; } this.logger.log(`👕 Filling lineups for ${toUpdate.length} matches...`); for (const match of toUpdate) { try { const [formation, substitutions] = await Promise.all([ this.scraper.fetchStartingFormation(match.id), this.scraper.fetchSubstitutions(match.id), ]); const sidelined = match.matchSlug ? await this.scraper.fetchSidelinedPlayers( match.id, match.matchSlug, ) : null; // Normalize to same home.xi/away.xi format used by processMatchOdds let normalizedLineups: Record | null = null; if (formation || substitutions) { normalizedLineups = { home: { xi: formation?.stats?.home || [], subs: substitutions?.stats?.home || [], }, away: { xi: formation?.stats?.away || [], subs: substitutions?.stats?.away || [], }, }; } await this.prisma.liveMatch.update({ where: { id: match.id }, data: { lineups: normalizedLineups ? JSON.parse(JSON.stringify(normalizedLineups)) : Prisma.JsonNull, sidelined: sidelined ? JSON.parse(JSON.stringify(sidelined)) : Prisma.JsonNull, updatedAt: new Date(), }, }); this.logger.log(`👕 Lineups filled for match ${match.id}`); await this.delay(500); } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); this.logger.warn(`Lineup fill failed for ${match.id}: ${message}`); } } } catch (error: unknown) { const message = error instanceof Error ? error.message : String(error); this.logger.error(`Lineup fill error: ${message}`); } } // ──────────────────────────────────────────────────────────── // Unified match fetcher — DRY for football + basketball // ──────────────────────────────────────────────────────────── private async fetchMatchesForSport( sport: SportType, date: string, topLeagueIds: Set, ): Promise { const sportParam = sport === "basketball" ? "Basketball" : "Soccer"; const url = `https://www.mackolik.com/perform/p0/ajax/components/competition/livescores/json?sports[]=${sportParam}&matchDate=${date}`; try { const response = await firstValueFrom( this.httpService.get(url, { timeout: 20000 }), ); const payload = this.parseLiveScoresPayload(response.data); if (!payload) { this.logger.warn(`[${sport}] No valid data from API for ${date}`); return; } const allMatches = payload.matches; const competitions = payload.competitions; // Apply league filter const targetMatches = topLeagueIds.size > 0 ? allMatches.filter( (m) => !!m.competitionId && topLeagueIds.has(String(m.competitionId)), ) : allMatches; if (targetMatches.length === 0) { this.logger.log(`[${sport}] No matches found for ${date}`); return; } this.logger.log( `[${sport}] Processing ${targetMatches.length}/${allMatches.length} matches for ${date}`, ); // Local caches to avoid N+1 redundant upserts const processedCountries = new Set(); const processedLeagues = new Set(); const processedTeams = new Set(); const integrityChecked = new Map(); let upsertCount = 0; let skippedCount = 0; for (const match of targetMatches) { try { const homeTeamId = match.homeTeam.id; const awayTeamId = match.awayTeam.id; const leagueId = match.competitionId; // Sport integrity check (cached) const ensureIntegrity = async ( entityType: "league" | "team" | "liveMatch", entityId: string | null, ): Promise => { if (!entityId) return true; const cacheKey = `${sport}:${entityType}:${entityId}`; if (!integrityChecked.has(cacheKey)) { integrityChecked.set( cacheKey, await this.ensureLiveEntitySportIntegrity( entityType, entityId, sport, ), ); } return integrityChecked.get(cacheKey) === true; }; if ( !(await ensureIntegrity("liveMatch", String(match.id))) || !(await ensureIntegrity("league", leagueId)) || !(await ensureIntegrity("team", homeTeamId)) || !(await ensureIntegrity("team", awayTeamId)) ) { skippedCount++; continue; } // Resolve competition details const compInfo = this.resolveCompetition(competitions, leagueId); const countryId = compInfo?.country?.id || null; // 1. Upsert Country (cached) if ( countryId && compInfo?.country?.name && !processedCountries.has(countryId) ) { await this.prisma.country .upsert({ where: { id: countryId }, update: { name: compInfo.country.name }, create: { id: countryId, name: compInfo.country.name }, }) .catch((e: Error) => this.logger.warn( `[${sport}] Country upsert failed: ${e.message}`, ), ); processedCountries.add(countryId); } // 2. Upsert League (cached) if ( leagueId && compInfo?.name && !processedLeagues.has(String(leagueId)) ) { await this.prisma.league .upsert({ where: { id: leagueId }, update: { name: compInfo.name, countryId: countryId, sport: sport, }, create: { id: leagueId, name: compInfo.name, countryId: countryId, sport: sport, competitionSlug: compInfo.slug || null, }, }) .catch((e: Error) => this.logger.warn( `[${sport}] League upsert failed: ${e.message}`, ), ); processedLeagues.add(String(leagueId)); } // 3. Upsert Home Team (cached) if ( homeTeamId && match.homeTeam?.name && !processedTeams.has(homeTeamId) ) { await this.prisma.team .upsert({ where: { id: homeTeamId }, update: { name: match.homeTeam.name, slug: match.homeTeam.slug || null, sport: sport, }, create: { id: homeTeamId, name: match.homeTeam.name, slug: match.homeTeam.slug || null, sport: sport, }, }) .catch((e: Error) => this.logger.warn( `[${sport}] Home team upsert failed: ${e.message}`, ), ); processedTeams.add(homeTeamId); } // 4. Upsert Away Team (cached) if ( awayTeamId && match.awayTeam?.name && !processedTeams.has(awayTeamId) ) { await this.prisma.team .upsert({ where: { id: awayTeamId }, update: { name: match.awayTeam.name, slug: match.awayTeam.slug || null, sport: sport, }, create: { id: awayTeamId, name: match.awayTeam.name, slug: match.awayTeam.slug || null, sport: sport, }, }) .catch((e: Error) => this.logger.warn( `[${sport}] Away team upsert failed: ${e.message}`, ), ); processedTeams.add(awayTeamId); } // Safe score parsing const sHome = this.asInt(match.homeScore ?? match.score?.home); const sAway = this.asInt(match.awayScore ?? match.score?.away); const storedStatus = deriveStoredMatchStatus({ state: match.state, status: match.status, substate: match.substate, statusBoxContent: match.statusBoxContent, scoreHome: sHome, scoreAway: sAway, score: match.score, }); // Handle postponed matches (ERT = Erteledendi) if (match.statusBoxContent === "ERT") { await this.prisma.liveMatch .updateMany({ where: { id: String(match.id) }, data: { status: "POSTPONED", state: "postponed", substate: "postponed", updatedAt: new Date(), }, }) .catch(() => {}); this.logger.debug( `[${sport}] Marked as POSTPONED: ${match.matchName}`, ); skippedCount++; continue; } // 5. Upsert LiveMatch await this.prisma.liveMatch.upsert({ where: { id: String(match.id) }, update: { leagueId: leagueId, state: match.state || null, substate: match.substate || null, status: storedStatus, scoreHome: sHome, scoreAway: sAway, homeTeamId: homeTeamId, awayTeamId: awayTeamId, updatedAt: new Date(), }, create: { id: String(match.id), matchName: match.matchName, matchSlug: match.matchSlug, sport: sport, leagueId: leagueId, state: match.state || null, substate: match.substate || null, status: storedStatus, mstUtc: BigInt(match.mstUtc || Date.now()), scoreHome: sHome, scoreAway: sAway, homeTeamId: homeTeamId, awayTeamId: awayTeamId, }, }); upsertCount++; // Progress logging every 100 matches if ( (upsertCount + skippedCount) % 100 === 0 || upsertCount + skippedCount === targetMatches.length ) { this.logger.log( `[${sport}] ⏳ Progress: ${upsertCount + skippedCount}/${targetMatches.length} (Saved: ${upsertCount}, Skipped: ${skippedCount})`, ); } } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); this.logger.warn(`[${sport}] Match ${match.id} failed: ${message}`); } } this.logger.log( `[${sport}] [${date}] Done: ${upsertCount} saved, ${skippedCount} skipped`, ); } catch (error: unknown) { const message = error instanceof Error ? error.message : String(error); this.logger.error(`[${sport}] Fetch failed for ${date}: ${message}`); } } // ──────────────────────────────────────────────────────────── // processMatchOdds — odds + referee + lineups + sidelined // (Preserved from original — no logic changes) // ──────────────────────────────────────────────────────────── private async processMatchOdds(match: LiveMatchOddsTarget): Promise { const matchSlug = match.matchSlug || "match"; const sport = String(match.sport || "football").toLowerCase(); const sportPath = sport === "basketball" ? "basketbol/mac" : "mac"; const httpHeaders = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", }; let odds: Record> = {}; let refereeName: string | null = null; let lineups: LiveLineupsJson | null = null; let sidelined: SidelinedResponse | null = null; // 1. Fetch Odds from İddaa page const oddsUrl = `https://www.mackolik.com/${sportPath}/${matchSlug}/iddaa/${match.id}`; try { const response = await firstValueFrom( this.httpService.get(oddsUrl, { timeout: 10000, headers: httpHeaders, maxRedirects: 5, }), ); odds = this.extractOddsFromHtml( typeof response.data === "string" ? response.data : "", ); } catch (e: unknown) { if (this.isRetryableError(e)) throw e; const message = e instanceof Error ? e.message : String(e); this.logger.warn(`Odds fetch failed for ${match.id}: ${message}`); } if (sport === "football") { // 2. Fetch Referee from main match page if (!refereeName) { const mainUrl = `https://www.mackolik.com/mac/${matchSlug}/${match.id}`; try { const mainResp = await firstValueFrom( this.httpService.get(mainUrl, { timeout: 10000, headers: httpHeaders, maxRedirects: 5, }), ); refereeName = this.extractRefereeFromHtml( typeof mainResp.data === "string" ? mainResp.data : "", ); } catch { // Non-critical — referee is optional } } // 3. Fetch Lineups & Sidelined Players const now = Date.now(); const matchTime = Number(match.mstUtc); const diffHours = (matchTime - now) / (1000 * 60 * 60); // Fetch if between -3 hours (started) and +24 hours (upcoming) if (diffHours < 24 && diffHours > -3) { // Lineups try { const [startingFormation, substitutions] = await Promise.all([ this.scraper.fetchStartingFormation(match.id), this.scraper.fetchSubstitutions(match.id), ]); if (startingFormation || substitutions) { lineups = { home: { xi: startingFormation?.stats?.home || [], subs: substitutions?.stats?.home || [], }, away: { xi: startingFormation?.stats?.away || [], subs: substitutions?.stats?.away || [], }, }; this.logger.log(`👥 Lineups found for ${match.matchName}`); } else { this.logger.debug(`No lineups (yet) for ${match.matchName}`); } } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); this.logger.warn(`Lineup fetch failed for ${match.id}: ${message}`); } // Sidelined Players (Injuries/Suspensions) try { sidelined = await this.scraper.fetchSidelinedPlayers( match.id, matchSlug, ); if (sidelined) { if (sidelined.homeTeam) { sidelined.homeTeam.teamName = match.homeTeam?.name || ""; } if (sidelined.awayTeam) { sidelined.awayTeam.teamName = match.awayTeam?.name || ""; } if ( sidelined.homeTeam?.totalSidelined > 0 || sidelined.awayTeam?.totalSidelined > 0 ) { this.logger.log( `🚑 Sidelined: ${sidelined.homeTeam.totalSidelined}(H) - ${sidelined.awayTeam.totalSidelined}(A) for ${match.matchName}`, ); } } } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); this.logger.warn( `Sidelined fetch failed for ${match.id}: ${message}`, ); } } } // Guard: If match already has pre-match odds and is now live/finished, // do NOT overwrite odds/lineups/sidelined — the model needs stable pre-match data. const matchState = match.state?.toLowerCase() ?? ""; const matchStatus = match.status?.toLowerCase() ?? ""; const liveStates = LIVE_STATE_VALUES_FOR_DB.map((s) => s.toLowerCase()); const liveStatuses = LIVE_STATUS_VALUES_FOR_DB.map((s) => s.toLowerCase()); const finishedStates = FINISHED_STATE_VALUES_FOR_DB.map((s) => s.toLowerCase(), ); const finishedStatuses = FINISHED_STATUS_VALUES_FOR_DB.map((s) => s.toLowerCase(), ); const isLiveOrFinished = liveStates.includes(matchState) || liveStatuses.includes(matchStatus) || finishedStates.includes(matchState) || finishedStatuses.includes(matchStatus); const existingOdds = match.odds as Record | null; const hasExistingOdds = !!existingOdds && typeof existingOdds === "object" && Object.keys(existingOdds).length > 0; if (isLiveOrFinished && hasExistingOdds) { // Match is live/finished and already has pre-match odds — skip data update this.logger.debug( `🛡️ Preserving pre-match data for ${match.matchName} (status: ${matchStatus}, state: ${matchState})`, ); await this.prisma.liveMatch.update({ where: { id: match.id }, data: { oddsUpdatedAt: new Date() }, }); return; } // Update odds/lineups/sidelined for pre-match (NS) matches await this.prisma.liveMatch.update({ where: { id: match.id }, data: { odds: Object.keys(odds).length > 0 ? odds : undefined, oddsUpdatedAt: new Date(), refereeName: refereeName ?? undefined, lineups: (lineups as unknown as Prisma.InputJsonValue) ?? undefined, sidelined: (sidelined as unknown as Prisma.InputJsonValue) ?? undefined, }, }); if ( Object.keys(odds).length > 0 || refereeName || lineups || (sidelined && (sidelined.homeTeam.totalSidelined > 0 || sidelined.awayTeam.totalSidelined > 0)) ) { this.logger.log( `SUCCESS Loop update: ${match.matchName} | Odds: ${Object.keys(odds).length} | Ref: ${refereeName || "N/A"} | Lineups: ${lineups ? "Yes" : "No"} | Sidelined: ${sidelined ? "Yes" : "No"}`, ); } else { this.logger.debug( `❕ No detailed data for ${match.matchName}, marked check.`, ); } } // ──────────────────────────────────────────────────────────── // HTML Extraction Helpers (preserved — no logic changes) // ──────────────────────────────────────────────────────────── /** * Extract odds from Mackolik HTML page * Returns structured odds object: { "MS": {"1": 2.10, "X": 3.40}, "AU25": {"Alt": 2.05, "Üst": 1.75} } */ private extractOddsFromHtml( html: string, ): Record> { const odds: Record> = {}; if (!html) return odds; try { const settingsPattern = /data-settings="([^"]+)"/g; let match; while ((match = settingsPattern.exec(html)) !== null) { try { const decoded = match[1] .replace(/"/g, '"') .replace(/&/g, "&") .replace(/</g, "<") .replace(/>/g, ">"); const parsed = JSON.parse(decoded) as unknown; if (!this.isRecord(parsed)) continue; const iddaaEventId = this.isRecord(parsed.iddaaEventId) ? parsed.iddaaEventId : null; const marketCollection = iddaaEventId && this.isRecord(iddaaEventId.marketCollection) ? iddaaEventId.marketCollection : null; if (marketCollection) { for (const marketValue of Object.values(marketCollection)) { if (!this.isRecord(marketValue)) continue; const marketName = this.asString(marketValue.name)?.trim(); if (!marketName) continue; // First-come-first-served: Skip if already populated if ( odds[marketName] && Object.keys(odds[marketName]).length > 0 ) { continue; } odds[marketName] = {}; const selectionCollection = this.isRecord( marketValue.selectionCollection, ) ? marketValue.selectionCollection : {}; for (const selectionValue of Object.values(selectionCollection)) { if (!this.isRecord(selectionValue)) continue; const selName = this.asString(selectionValue.name) || this.asString(selectionValue.outcome); const selOdd = Number.parseFloat( this.asString(selectionValue.odd) || "", ); if (selName && !isNaN(selOdd)) { odds[marketName][selName] = selOdd; } } } } } catch { // JSON parse error, skip } } } catch { this.logger.warn("Failed to extract odds from HTML"); } return odds; } /** * Normalize odds category names to short codes */ private normalizeOddsCategory(name: string): string | null { if (!name) return null; const lower = name.toLowerCase(); // Specific & Compound names FIRST if (lower.includes("ilk yarı/maç sonucu")) return "HTFT"; if (lower.includes("1. yarı sonucu")) return "HT"; if (lower.includes("çifte ÅŸans")) return "CS"; // General names LATER if (lower.includes("maç sonucu") && !lower.includes("handikap")) return "MS"; if (lower.includes("karşılıklı gol")) return "KG"; if (lower.includes("2,5 alt/üst") || lower.includes("2.5")) return "AU25"; if (lower.includes("1,5 alt/üst") || lower.includes("1.5")) return "AU15"; if (lower.includes("3,5 alt/üst") || lower.includes("3.5")) return "AU35"; return null; } /** * Extract referee name from match page HTML */ private extractRefereeFromHtml(html: string): string | null { try { // Strategy 1: Mackolik officials section — head referee in '--main' list item const mainOfficialPattern = /official-list-item--main[^>]*>\s*(?:<[^>]*>\s*)*?]*official-name[^>]*>\s*([^<]+)/i; const mainMatch = mainOfficialPattern.exec(html); if (mainMatch?.[1]) { const name = mainMatch[1].trim(); if (name.length > 2 && name.length < 100) return name; } // Strategy 2: Any official-name followed by "Orta Hakem" const ortaHakemPattern = /official-name[^>]*>\s*([^<]+)<[\s\S]*?Orta\s*Hakem/i; const ortaMatch = ortaHakemPattern.exec(html); if (ortaMatch?.[1]) { const name = ortaMatch[1].trim(); if (name.length > 2 && name.length < 100) return name; } // Strategy 3: Generic fallback patterns const fallbackPatterns = [ /Hakem:\s*([^<]+)/i, /"refereeName":"([^"]+)"/i, ]; for (const pattern of fallbackPatterns) { const result = pattern.exec(html); if (result?.[1]) { const name = result[1].trim(); if (name.length > 2 && name.length < 100) return name; } } } catch { // Ignore extraction errors } return null; } // ──────────────────────────────────────────────────────────── // Low-level Helpers (preserved — no logic changes) // ──────────────────────────────────────────────────────────── private shouldSkipInHistoricalMode(jobName: string): boolean { if (process.env.FEEDER_MODE === "historical") { this.logger.debug(`Skipping ${jobName} in historical feeder mode`); return true; } return false; } private isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } private asString(value: unknown): string | null { if (typeof value === "string") return value; if (typeof value === "number" && Number.isFinite(value)) return String(value); return null; } private asInt(value: unknown): number | null { if (typeof value === "number" && Number.isFinite(value)) { return Math.trunc(value); } if (typeof value === "string" && value.trim().length > 0) { const parsed = parseInt(value, 10); return Number.isNaN(parsed) ? null : parsed; } return null; } private parseTeam(value: unknown): LiveScoreTeamPayload | null { if (!this.isRecord(value)) return null; const id = this.asString(value.id); if (!id) return null; return { id, name: this.asString(value.name) || "Unknown", slug: this.asString(value.slug), }; } private parseCompetition(value: unknown): LiveScoreCompetitionPayload | null { if (!this.isRecord(value)) return null; const id = this.asString(value.id); const name = this.asString(value.name); if (!id || !name) return null; const rawCountry = this.isRecord(value.country) ? value.country : null; const countryId = rawCountry ? this.asString(rawCountry.id) : null; const countryName = rawCountry ? this.asString(rawCountry.name) : null; return { id, name, slug: this.asString(value.slug), country: countryId && countryName ? { id: countryId, name: countryName } : null, }; } private parseLiveScoreMatch(value: unknown): LiveScorePayloadMatch | null { if (!this.isRecord(value)) return null; const id = this.asString(value.id); const homeTeam = this.parseTeam(value.homeTeam); const awayTeam = this.parseTeam(value.awayTeam); if (!id || !homeTeam || !awayTeam) return null; const score = this.isRecord(value.score) ? { home: this.asInt(value.score.home), away: this.asInt(value.score.away), } : null; return { id, matchName: this.asString(value.matchName) || `${homeTeam.name} - ${awayTeam.name}`, matchSlug: this.asString(value.matchSlug) || "", competitionId: this.asString(value.competitionId), mstUtc: this.asInt(value.mstUtc), state: this.asString(value.state), substate: this.asString(value.substate), status: this.asString(value.status), statusBoxContent: this.asString(value.statusBoxContent), iddaaCode: this.asString(value.iddaaCode), homeTeam, awayTeam, homeScore: this.asInt(value.homeScore), awayScore: this.asInt(value.awayScore), score, }; } private parseLiveScoresPayload(raw: unknown): { matches: LiveScorePayloadMatch[]; competitions: Record; } | null { if (!this.isRecord(raw)) return null; if (this.asString(raw.status) !== "success") return null; const data = this.isRecord(raw.data) ? raw.data : null; if (!data) return null; const rawMatches = this.isRecord(data.matches) ? data.matches : {}; const rawCompetitions = this.isRecord(data.competitions) ? data.competitions : {}; const matches: LiveScorePayloadMatch[] = []; for (const rawMatch of Object.values(rawMatches)) { const parsed = this.parseLiveScoreMatch(rawMatch); if (parsed) matches.push(parsed); } const competitions: Record = {}; for (const [key, rawCompetition] of Object.entries(rawCompetitions)) { const parsed = this.parseCompetition(rawCompetition); if (parsed) { competitions[key] = parsed; } } return { matches, competitions }; } private loadLeagueFilterSet(fileName: string): Set | null { try { const filePath = path.join(process.cwd(), fileName); const raw = JSON.parse(fs.readFileSync(filePath, "utf-8")) as unknown; if (!Array.isArray(raw)) { this.logger.error(`${fileName} is not a JSON array`); return null; } const ids = raw .map((value) => this.asString(value)) .filter((value): value is string => !!value); return new Set(ids); } catch (e: unknown) { const message = e instanceof Error ? e.message : String(e); this.logger.error(`Failed to load ${fileName}: ${message}`); return null; } } private resolveCompetition( competitions: Record, leagueId: string | null, ): LiveScoreCompetitionPayload | null { if (!leagueId) return null; if (competitions[leagueId]) return competitions[leagueId]; for (const comp of Object.values(competitions)) { if (comp.id === leagueId) return comp; } return null; } private async ensureLiveEntitySportIntegrity( entityType: "league" | "team" | "liveMatch", id: string, expectedSport: SportType, ): Promise { if (!id) return true; let existingSport: string | null | undefined; if (entityType === "league") { const existing = await this.prisma.league.findUnique({ where: { id }, select: { sport: true }, }); existingSport = existing?.sport; } else if (entityType === "team") { const existing = await this.prisma.team.findUnique({ where: { id }, select: { sport: true }, }); existingSport = existing?.sport; } else { const existing = await this.prisma.liveMatch.findUnique({ where: { id }, select: { sport: true }, }); existingSport = existing?.sport; } if (existingSport && existingSport !== expectedSport) { this.logger.error( `Sport integrity violation on ${entityType}:${id}. Existing=${existingSport}, incoming=${expectedSport}. Skipping write to prevent cross-sport overwrite.`, ); return false; } return true; } private isRetryableError(err: unknown): boolean { if (!err || typeof err !== "object") return false; const errorObj = err as Record; const response = errorObj.response as Record | undefined; return ( response?.status === 502 || errorObj.code === "ECONNABORTED" || errorObj.code === "ETIMEDOUT" ); } private delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } }