1884 lines
69 KiB
TypeScript
Executable File
1884 lines
69 KiB
TypeScript
Executable File
import { Injectable, Logger } from "@nestjs/common";
|
||
import { Cron } from "@nestjs/schedule";
|
||
import { HttpService } from "@nestjs/axios";
|
||
import { PrismaService } from "../database/prisma.service";
|
||
import { firstValueFrom } from "rxjs";
|
||
import { FeederScraperService } from "../modules/feeder/feeder-scraper.service";
|
||
import * as fs from "fs";
|
||
import * as path from "path";
|
||
import { Prisma } from "@prisma/client";
|
||
import { SidelinedResponse } from "../modules/feeder/feeder.types";
|
||
import {
|
||
deriveStoredMatchStatus,
|
||
FINISHED_STATE_VALUES_FOR_DB,
|
||
FINISHED_STATUS_VALUES_FOR_DB,
|
||
LIVE_STATE_VALUES_FOR_DB,
|
||
LIVE_STATUS_VALUES_FOR_DB,
|
||
} from "../common/utils/match-status.util";
|
||
import {
|
||
getDateStringInTimeZone,
|
||
getDayBoundsForTimeZone,
|
||
getShiftedDateStringInTimeZone,
|
||
} from "../common/utils/timezone.util";
|
||
import { TaskLockService } from "./task-lock.service";
|
||
import { FeederService } from "../modules/feeder/feeder.service";
|
||
|
||
|
||
// Types
|
||
|
||
interface LiveScoreTeamPayload {
|
||
id: string;
|
||
name: string;
|
||
slug: string | null;
|
||
}
|
||
|
||
interface LiveScoreCompetitionPayload {
|
||
id: string;
|
||
name: string;
|
||
slug: string | null;
|
||
country: {
|
||
id: string;
|
||
name: string;
|
||
} | null;
|
||
}
|
||
|
||
interface LiveScorePayloadMatch {
|
||
id: string;
|
||
matchName: string;
|
||
matchSlug: string;
|
||
competitionId: string | null;
|
||
mstUtc: number | null;
|
||
state: string | null;
|
||
substate: string | null;
|
||
status: string | null;
|
||
statusBoxContent: string | null; // ERT = Erteledendi
|
||
iddaaCode: string | null;
|
||
homeTeam: LiveScoreTeamPayload;
|
||
awayTeam: LiveScoreTeamPayload;
|
||
homeScore: number | null;
|
||
awayScore: number | null;
|
||
score: {
|
||
home: number | null;
|
||
away: number | null;
|
||
ht?: {
|
||
home: number | null;
|
||
away: number | null;
|
||
} | null;
|
||
} | null;
|
||
}
|
||
|
||
type LiveMatchOddsTarget = Prisma.LiveMatchGetPayload<{
|
||
include: {
|
||
homeTeam: { select: { name: true } };
|
||
awayTeam: { select: { name: true } };
|
||
};
|
||
}>;
|
||
|
||
interface LiveLineupsJson {
|
||
home: { xi: unknown[]; subs: unknown[] };
|
||
away: { xi: unknown[]; subs: unknown[] };
|
||
}
|
||
|
||
interface PendingPredictionRunForSettlement {
|
||
id: bigint;
|
||
matchId: string;
|
||
engineVersion: string;
|
||
payloadSummary: unknown;
|
||
scoreHome: number | null;
|
||
scoreAway: number | null;
|
||
htScoreHome: number | null;
|
||
htScoreAway: number | null;
|
||
}
|
||
|
||
type SportType = "football" | "basketball";
|
||
|
||
|
||
// Service
|
||
|
||
|
||
@Injectable()
|
||
export class DataFetcherTask {
|
||
private readonly logger = new Logger(DataFetcherTask.name);
|
||
private readonly timeZone = "Europe/Istanbul";
|
||
|
||
constructor(
|
||
private readonly httpService: HttpService,
|
||
private readonly prisma: PrismaService,
|
||
private readonly scraper: FeederScraperService,
|
||
private readonly taskLock: TaskLockService,
|
||
private readonly feeder: FeederService,
|
||
) { }
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// CRON 1: Main sync — every 15 minutes
|
||
// Phases: match list → live scores → odds → lineups
|
||
// ────────────────────────────────────────────────────────────
|
||
|
||
@Cron("*/15 * * * *")
|
||
async syncLiveMatches(): Promise<void> {
|
||
if (this.shouldSkipInHistoricalMode("syncLiveMatches")) return;
|
||
await this.taskLock.runWithLease(
|
||
"syncLiveMatches",
|
||
30 * 60 * 1000,
|
||
async () => {
|
||
await this.runLiveSync();
|
||
},
|
||
this.logger,
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// CRON 2: Daily cleanup + full sync — 07:00 Istanbul
|
||
// Preserve yesterday as a fallback until the 08:00 archive job completes.
|
||
// ────────────────────────────────────────────────────────────
|
||
|
||
@Cron("0 7 * * *", { timeZone: "Europe/Istanbul" })
|
||
async cleanAndFullSync(): Promise<void> {
|
||
if (this.shouldSkipInHistoricalMode("cleanAndFullSync")) return;
|
||
await this.taskLock.runWithLease(
|
||
"cleanAndFullSync",
|
||
2 * 60 * 60 * 1000,
|
||
async () => {
|
||
this.logger.log(
|
||
"cleanAndFullSync: Pruning stale live_matches while preserving yesterday for archive fallback...",
|
||
);
|
||
|
||
try {
|
||
const yesterdayDate = getShiftedDateStringInTimeZone(
|
||
-1,
|
||
this.timeZone,
|
||
);
|
||
const { startMs: yesterdayStartMs } = getDayBoundsForTimeZone(
|
||
yesterdayDate,
|
||
this.timeZone,
|
||
);
|
||
const cutoffDate = new Date(yesterdayStartMs);
|
||
|
||
const deleted = await this.prisma.liveMatch.deleteMany({
|
||
where: {
|
||
OR: [
|
||
{ mstUtc: { lt: BigInt(yesterdayStartMs) } },
|
||
{
|
||
AND: [
|
||
{ mstUtc: null },
|
||
{ updatedAt: { lt: cutoffDate } },
|
||
{
|
||
OR: [
|
||
{ status: { in: FINISHED_STATUS_VALUES_FOR_DB } },
|
||
{ state: { in: FINISHED_STATE_VALUES_FOR_DB } },
|
||
],
|
||
},
|
||
],
|
||
},
|
||
],
|
||
},
|
||
});
|
||
|
||
this.logger.log(
|
||
`Pruned ${deleted.count} stale live matches. Starting full sync...`,
|
||
);
|
||
} catch (error: unknown) {
|
||
const message =
|
||
error instanceof Error ? error.message : String(error);
|
||
this.logger.error(`Stale live_match cleanup failed: ${message}`);
|
||
return;
|
||
}
|
||
|
||
await this.runLiveSync();
|
||
},
|
||
this.logger,
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// Phase 1: Fetch match list for all sports
|
||
// ────────────────────────────────────────────────────────────
|
||
|
||
private async runLiveSync(): Promise<void> {
|
||
if (this.shouldSkipInHistoricalMode("syncLiveMatches")) return;
|
||
|
||
this.logger.log("syncLiveMatches START");
|
||
|
||
// 4-day forward window: pull today .. +3 days so upcoming matches enter
|
||
// live_matches early and their odds are refreshed every cycle. That rolling
|
||
// refresh is what lets the odds-movement / steam monitor (and forward CLV)
|
||
// see a real opening→closing range. Finished/live matches are already
|
||
// excluded from odds re-fetch in fetchOddsForMatches(), so closed-match
|
||
// odds and their ranges are never re-pulled.
|
||
const SYNC_DAYS_AHEAD = 4;
|
||
const today = getDateStringInTimeZone(new Date(), this.timeZone);
|
||
await this.syncMatchList(today);
|
||
for (let dayOffset = 1; dayOffset < SYNC_DAYS_AHEAD; dayOffset++) {
|
||
await this.syncMatchList(
|
||
getShiftedDateStringInTimeZone(dayOffset, this.timeZone),
|
||
);
|
||
}
|
||
await this.updateLiveScores();
|
||
await this.archiveNewlyFinishedMatches(today);
|
||
await this.settlePredictionRuns();
|
||
await this.fetchOddsForMatches();
|
||
await this.fillMissingLineups();
|
||
|
||
this.logger.log("syncLiveMatches END");
|
||
}
|
||
|
||
private async archiveNewlyFinishedMatches(todayStr: string): Promise<void> {
|
||
try {
|
||
const finishedLive = await this.prisma.liveMatch.findMany({
|
||
where: {
|
||
OR: [
|
||
{ status: { in: FINISHED_STATUS_VALUES_FOR_DB } },
|
||
{ state: { in: FINISHED_STATE_VALUES_FOR_DB } },
|
||
],
|
||
scoreHome: { not: null },
|
||
scoreAway: { not: null },
|
||
},
|
||
select: { id: true },
|
||
});
|
||
|
||
if (finishedLive.length === 0) return;
|
||
|
||
const ids = finishedLive.map((m) => m.id);
|
||
const alreadyArchived = await this.prisma.match.findMany({
|
||
where: { id: { in: ids } },
|
||
select: { id: true },
|
||
});
|
||
|
||
const archivedSet = new Set(alreadyArchived.map((m) => m.id));
|
||
const newCount = ids.filter((id) => !archivedSet.has(id)).length;
|
||
|
||
if (newCount === 0) return;
|
||
|
||
this.logger.log(
|
||
`${newCount} finished match(es) not yet archived — running feeder for ${todayStr}`,
|
||
);
|
||
await this.feeder.archiveCompletedMatchesForDate(todayStr);
|
||
} catch (error: unknown) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
this.logger.error(`archiveNewlyFinishedMatches failed: ${message}`);
|
||
}
|
||
}
|
||
|
||
private async syncMatchList(date: string): Promise<void> {
|
||
// Football
|
||
const footballLeagues = this.loadLeagueFilterSet("qualified_leagues.json");
|
||
if (footballLeagues && footballLeagues.size > 0) {
|
||
await this.fetchMatchesForSport("football", date, footballLeagues);
|
||
} else {
|
||
this.logger.warn(
|
||
"qualified_leagues.json is missing/empty — writing ALL football matches",
|
||
);
|
||
await this.fetchMatchesForSport("football", date, new Set());
|
||
}
|
||
|
||
// Basketball
|
||
const basketballLeagues = this.loadLeagueFilterSet(
|
||
"basketball_top_leagues.json",
|
||
);
|
||
if (!basketballLeagues || basketballLeagues.size === 0) {
|
||
this.logger.error(
|
||
"basketball_top_leagues.json is missing/empty. Basketball ingest skipped to protect data quality.",
|
||
);
|
||
} else {
|
||
await this.fetchMatchesForSport("basketball", date, basketballLeagues);
|
||
|
||
// Clean up basketball matches that are NOT in configured leagues
|
||
await this.prisma.liveMatch.deleteMany({
|
||
where: {
|
||
sport: "basketball",
|
||
OR: [
|
||
{ leagueId: null },
|
||
{ leagueId: { notIn: Array.from(basketballLeagues) } },
|
||
],
|
||
},
|
||
});
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// Phase 2: Live score updates (merged from live-updater.task)
|
||
// ────────────────────────────────────────────────────────────
|
||
|
||
private async updateLiveScores(): Promise<void> {
|
||
try {
|
||
const liveMatches = await this.prisma.liveMatch.findMany({
|
||
where: {
|
||
OR: [
|
||
{ state: { in: LIVE_STATE_VALUES_FOR_DB } },
|
||
{ status: { in: LIVE_STATUS_VALUES_FOR_DB } },
|
||
],
|
||
},
|
||
select: { id: true, matchSlug: true },
|
||
});
|
||
|
||
if (liveMatches.length === 0) {
|
||
this.logger.debug("No live matches to update scores for");
|
||
return;
|
||
}
|
||
|
||
this.logger.log(
|
||
`LIVE Updating scores for ${liveMatches.length} live matches`,
|
||
);
|
||
|
||
for (const match of liveMatches) {
|
||
try {
|
||
const url = `https://www.mackolik.com/ajax/football/match-info?matchId=${match.id}`;
|
||
const response = await firstValueFrom(
|
||
this.httpService.get(url, { timeout: 5000 }),
|
||
);
|
||
|
||
if (response.data?.data) {
|
||
const matchData = response.data.data;
|
||
const scoreHome = matchData.homeScore ?? null;
|
||
const scoreAway = matchData.awayScore ?? null;
|
||
const htScoreHome = this.asInt(
|
||
matchData.score?.ht?.home ??
|
||
matchData.htHomeScore ??
|
||
matchData.homeHtScore,
|
||
);
|
||
const htScoreAway = this.asInt(
|
||
matchData.score?.ht?.away ??
|
||
matchData.htAwayScore ??
|
||
matchData.awayHtScore,
|
||
);
|
||
const storedStatus = deriveStoredMatchStatus({
|
||
state: matchData.state,
|
||
status: matchData.status,
|
||
substate: matchData.substate,
|
||
scoreHome,
|
||
scoreAway,
|
||
});
|
||
await this.prisma.liveMatch.update({
|
||
where: { id: match.id },
|
||
data: {
|
||
scoreHome,
|
||
scoreAway,
|
||
htScoreHome,
|
||
htScoreAway,
|
||
state: matchData.state || null,
|
||
substate: matchData.substate || null,
|
||
status: storedStatus,
|
||
updatedAt: new Date(),
|
||
},
|
||
});
|
||
}
|
||
} catch {
|
||
// Individual match update failed, continue with others
|
||
}
|
||
}
|
||
|
||
this.logger.log("LIVE Live score update complete");
|
||
} catch (error: unknown) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
this.logger.error(`Live score update failed: ${message}`);
|
||
}
|
||
}
|
||
|
||
private async settlePredictionRuns(): Promise<void> {
|
||
try {
|
||
const rows = await this.prisma.$queryRawUnsafe<
|
||
PendingPredictionRunForSettlement[]
|
||
>(`
|
||
SELECT
|
||
pr.id,
|
||
pr.match_id AS "matchId",
|
||
pr.engine_version AS "engineVersion",
|
||
pr.payload_summary AS "payloadSummary",
|
||
m.score_home AS "scoreHome",
|
||
m.score_away AS "scoreAway",
|
||
m.ht_score_home AS "htScoreHome",
|
||
m.ht_score_away AS "htScoreAway"
|
||
FROM prediction_runs pr
|
||
JOIN matches m ON m.id = pr.match_id
|
||
WHERE pr.eventual_outcome IS NULL
|
||
AND m.sport = 'football'
|
||
AND m.status = 'FT'
|
||
AND m.score_home IS NOT NULL
|
||
AND m.score_away IS NOT NULL
|
||
ORDER BY pr.generated_at ASC
|
||
LIMIT 500
|
||
`);
|
||
|
||
if (rows.length === 0) return;
|
||
|
||
let settled = 0;
|
||
for (const row of rows) {
|
||
const result = this.resolvePredictionRunSettlement(row);
|
||
if (!result) continue;
|
||
const closingOddsSnapshot = await this.getClosingOddsSnapshot(
|
||
row.matchId,
|
||
);
|
||
// ── Per-market settlement (V31e forward-test) ────────────────
|
||
// Score EVERY captured market (not just main_pick) against reality,
|
||
// so the admin Model Performance page can compute per-market
|
||
// calibration (model% → actual%) and ROI. won=null → push (skip).
|
||
const marketsSettled = this.settleAllMarkets(row);
|
||
|
||
const settlementSummary = {
|
||
settled_at: new Date().toISOString(),
|
||
model_version: row.engineVersion,
|
||
outcome: result.outcome,
|
||
unit_profit: result.unitProfit,
|
||
final_score: {
|
||
home: row.scoreHome,
|
||
away: row.scoreAway,
|
||
},
|
||
halftime_score: {
|
||
home: row.htScoreHome,
|
||
away: row.htScoreAway,
|
||
},
|
||
closing_odds_snapshot: closingOddsSnapshot,
|
||
markets_settled: marketsSettled,
|
||
};
|
||
|
||
await this.prisma.$executeRawUnsafe(
|
||
`
|
||
UPDATE prediction_runs
|
||
SET eventual_outcome = $1,
|
||
unit_profit = $2,
|
||
payload_summary = payload_summary || jsonb_build_object('settlement', $3::jsonb)
|
||
WHERE id = $4
|
||
`,
|
||
result.outcome,
|
||
result.unitProfit,
|
||
JSON.stringify(settlementSummary),
|
||
row.id,
|
||
);
|
||
settled++;
|
||
}
|
||
|
||
if (settled > 0) {
|
||
this.logger.log(`Settled ${settled} prediction run(s)`);
|
||
}
|
||
} catch (error: unknown) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
this.logger.warn(`Prediction run settlement skipped: ${message}`);
|
||
}
|
||
}
|
||
|
||
private async getClosingOddsSnapshot(
|
||
matchId: string,
|
||
): Promise<Record<string, unknown>> {
|
||
const liveMatch = await this.prisma.liveMatch.findUnique({
|
||
where: { id: matchId },
|
||
select: {
|
||
odds: true,
|
||
oddsUpdatedAt: true,
|
||
status: true,
|
||
state: true,
|
||
scoreHome: true,
|
||
scoreAway: true,
|
||
},
|
||
});
|
||
|
||
if (liveMatch?.odds) {
|
||
return {
|
||
source: "live_match",
|
||
odds: liveMatch.odds,
|
||
odds_updated_at: liveMatch.oddsUpdatedAt?.toISOString() ?? null,
|
||
status: liveMatch.status ?? null,
|
||
state: liveMatch.state ?? null,
|
||
score_home: liveMatch.scoreHome,
|
||
score_away: liveMatch.scoreAway,
|
||
};
|
||
}
|
||
|
||
const categories = await this.prisma.oddCategory.findMany({
|
||
where: { matchId },
|
||
select: {
|
||
name: true,
|
||
selections: {
|
||
select: {
|
||
name: true,
|
||
oddValue: true,
|
||
position: true,
|
||
updatedAt: true,
|
||
},
|
||
orderBy: { position: "asc" },
|
||
take: 12,
|
||
},
|
||
},
|
||
orderBy: { name: "asc" },
|
||
take: 24,
|
||
});
|
||
|
||
return {
|
||
source: "odd_selections",
|
||
category_count: categories.length,
|
||
categories: categories.map((category) => ({
|
||
name: category.name,
|
||
selections: category.selections.map((selection) => ({
|
||
name: selection.name,
|
||
odd_value: selection.oddValue,
|
||
position: selection.position,
|
||
updated_at: selection.updatedAt?.toISOString() ?? null,
|
||
})),
|
||
})),
|
||
};
|
||
}
|
||
|
||
private resolvePredictionRunSettlement(
|
||
row: PendingPredictionRunForSettlement,
|
||
): { outcome: string; unitProfit: number } | null {
|
||
const summary = this.asRecord(row.payloadSummary);
|
||
const mainPick = this.asRecord(summary.main_pick);
|
||
const market = typeof mainPick.market === "string" ? mainPick.market : "";
|
||
const pick = typeof mainPick.pick === "string" ? mainPick.pick : "";
|
||
const playable = mainPick.playable === true;
|
||
const odds = Number(mainPick.odds || 0);
|
||
|
||
if (
|
||
!market ||
|
||
!pick ||
|
||
!playable ||
|
||
!Number.isFinite(odds) ||
|
||
odds <= 1.01
|
||
) {
|
||
return { outcome: "NO_BET", unitProfit: 0 };
|
||
}
|
||
|
||
const won = this.isPredictionPickWon({
|
||
market,
|
||
pick,
|
||
scoreHome: row.scoreHome,
|
||
scoreAway: row.scoreAway,
|
||
htScoreHome: row.htScoreHome,
|
||
htScoreAway: row.htScoreAway,
|
||
});
|
||
|
||
if (won === null) return null;
|
||
|
||
return {
|
||
outcome: `${won ? "WON" : "LOST"}:${market}:${pick}`,
|
||
unitProfit: Number((won ? odds - 1 : -1).toFixed(4)),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* V31e forward-test: settle EVERY captured market (payload_summary.markets_full)
|
||
* against the final score. Produces one compact record per market with its
|
||
* shown probability, the real outcome, and flat profit — the raw material for
|
||
* per-market calibration (model% vs actual%) on the admin dashboard.
|
||
*/
|
||
private settleAllMarkets(
|
||
row: PendingPredictionRunForSettlement,
|
||
): Array<Record<string, unknown>> {
|
||
const summary = this.asRecord(row.payloadSummary);
|
||
const markets = Array.isArray(summary.markets_full)
|
||
? (summary.markets_full as unknown[])
|
||
: [];
|
||
const out: Array<Record<string, unknown>> = [];
|
||
|
||
for (const raw of markets) {
|
||
const m = this.asRecord(raw);
|
||
const market = typeof m.market === "string" ? m.market : "";
|
||
const pick = typeof m.pick === "string" ? m.pick : "";
|
||
if (!market || !pick) continue;
|
||
|
||
const won = this.isPredictionPickWon({
|
||
market,
|
||
pick,
|
||
scoreHome: row.scoreHome,
|
||
scoreAway: row.scoreAway,
|
||
htScoreHome: row.htScoreHome,
|
||
htScoreAway: row.htScoreAway,
|
||
});
|
||
if (won === null) continue; // push / unresolvable → exclude from stats
|
||
|
||
const odds = Number(m.odds || 0);
|
||
const hasOdds = Number.isFinite(odds) && odds > 1.01;
|
||
out.push({
|
||
market,
|
||
pick,
|
||
won,
|
||
// shown probability for calibration (0–100). Prefer calibrated_confidence.
|
||
shown_confidence:
|
||
m.calibrated_confidence != null
|
||
? Number(m.calibrated_confidence)
|
||
: m.model_probability != null
|
||
? Number(m.model_probability) * 100
|
||
: null,
|
||
model_probability:
|
||
m.model_probability != null ? Number(m.model_probability) : null,
|
||
odds: hasOdds ? odds : null,
|
||
playable: m.playable === true,
|
||
bet_grade: typeof m.bet_grade === "string" ? m.bet_grade : null,
|
||
action: typeof m.action === "string" ? m.action : null,
|
||
value_tier: typeof m.value_tier === "string" ? m.value_tier : null,
|
||
// flat 1u profit if a real price existed (for per-market ROI)
|
||
flat_profit: hasOdds ? Number((won ? odds - 1 : -1).toFixed(4)) : null,
|
||
});
|
||
}
|
||
return out;
|
||
}
|
||
|
||
private isPredictionPickWon(input: {
|
||
market: string;
|
||
pick: string;
|
||
scoreHome: number | null;
|
||
scoreAway: number | null;
|
||
htScoreHome: number | null;
|
||
htScoreAway: number | null;
|
||
}): boolean | null {
|
||
const market = input.market.toUpperCase();
|
||
const pick = this.normalizePick(input.pick);
|
||
const scoreHome = input.scoreHome;
|
||
const scoreAway = input.scoreAway;
|
||
if (scoreHome === null || scoreAway === null) return null;
|
||
|
||
if (market === "MS") {
|
||
if (pick === "1") return scoreHome > scoreAway;
|
||
if (pick === "X" || pick === "0") return scoreHome === scoreAway;
|
||
if (pick === "2") return scoreAway > scoreHome;
|
||
return null;
|
||
}
|
||
|
||
if (market === "DC") {
|
||
const normalized = pick.replace("-", "");
|
||
if (normalized === "1X") return scoreHome >= scoreAway;
|
||
if (normalized === "X2") return scoreAway >= scoreHome;
|
||
if (normalized === "12") return scoreHome !== scoreAway;
|
||
return null;
|
||
}
|
||
|
||
if (market === "BTTS") {
|
||
const bothScored = scoreHome > 0 && scoreAway > 0;
|
||
if (pick.includes("VAR") || pick.includes("YES") || pick === "Y") {
|
||
return bothScored;
|
||
}
|
||
if (pick.includes("YOK") || pick.includes("NO") || pick === "N") {
|
||
return !bothScored;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
const goalLine = this.goalLineForMarket(market);
|
||
if (goalLine !== null) {
|
||
const total = market.startsWith("HT_")
|
||
? this.nullableSum(input.htScoreHome, input.htScoreAway)
|
||
: scoreHome + scoreAway;
|
||
if (total === null) return null;
|
||
if (this.isOverPick(pick)) return total > goalLine;
|
||
return total < goalLine;
|
||
}
|
||
|
||
if (market === "HT") {
|
||
const htHome = input.htScoreHome;
|
||
const htAway = input.htScoreAway;
|
||
if (htHome === null || htAway === null) return null;
|
||
if (pick === "1") return htHome > htAway;
|
||
if (pick === "X" || pick === "0") return htHome === htAway;
|
||
if (pick === "2") return htAway > htHome;
|
||
}
|
||
|
||
if (market === "HTFT") {
|
||
const htHome = input.htScoreHome;
|
||
const htAway = input.htScoreAway;
|
||
if (htHome === null || htAway === null || !pick.includes("/"))
|
||
return null;
|
||
const [htPick, ftPick] = pick.split("/");
|
||
return (
|
||
this.isResultPickWon(htPick, htHome, htAway) === true &&
|
||
this.isResultPickWon(ftPick, scoreHome, scoreAway) === true
|
||
);
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
private isResultPickWon(
|
||
pick: string,
|
||
homeScore: number,
|
||
awayScore: number,
|
||
): boolean | null {
|
||
if (pick === "1") return homeScore > awayScore;
|
||
if (pick === "X" || pick === "0") return homeScore === awayScore;
|
||
if (pick === "2") return awayScore > homeScore;
|
||
return null;
|
||
}
|
||
|
||
private goalLineForMarket(market: string): number | null {
|
||
if (market === "OU15") return 1.5;
|
||
if (market === "OU25") return 2.5;
|
||
if (market === "OU35") return 3.5;
|
||
if (market === "HT_OU05") return 0.5;
|
||
if (market === "HT_OU15") return 1.5;
|
||
return null;
|
||
}
|
||
|
||
private nullableSum(a: number | null, b: number | null): number | null {
|
||
if (a === null || b === null) return null;
|
||
return a + b;
|
||
}
|
||
|
||
private normalizePick(value: string): string {
|
||
return value
|
||
.trim()
|
||
.toUpperCase()
|
||
.replace(/İ/g, "I")
|
||
.replace(/Ü/g, "U")
|
||
.replace(/Ş/g, "S")
|
||
.replace(/Ğ/g, "G")
|
||
.replace(/Ö/g, "O")
|
||
.replace(/Ç/g, "C");
|
||
}
|
||
|
||
private isOverPick(pick: string): boolean {
|
||
return pick.includes("UST") || pick.includes("OVER");
|
||
}
|
||
|
||
private asRecord(value: unknown): Record<string, unknown> {
|
||
return value && typeof value === "object" && !Array.isArray(value)
|
||
? (value as Record<string, unknown>)
|
||
: {};
|
||
}
|
||
|
||
// Phase 3: Odds + referee + lineups + sidelined
|
||
|
||
private async fetchOddsForMatches(): Promise<void> {
|
||
this.logger.log("MONEY Fetching odds for live matches...");
|
||
|
||
try {
|
||
// Load both league filters (data-driven qualified leagues)
|
||
const topLeagueIds: string[] = [];
|
||
|
||
const footballLeagues = this.loadLeagueFilterSet(
|
||
"qualified_leagues.json",
|
||
);
|
||
if (footballLeagues) topLeagueIds.push(...footballLeagues);
|
||
|
||
const basketballLeagues = this.loadLeagueFilterSet(
|
||
"basketball_top_leagues.json",
|
||
);
|
||
if (basketballLeagues) topLeagueIds.push(...basketballLeagues);
|
||
|
||
const allowedLeagueIds = Array.from(new Set(topLeagueIds));
|
||
|
||
// Get matches needing odds (from 12 hours ago onward)
|
||
// CRITICAL: Only fetch odds/lineups for NOT STARTED matches.
|
||
// Once a match goes live, odds selections disappear or change,
|
||
// corrupting the pre-match data the AI model relies on.
|
||
const twelveHoursAgo = new Date(Date.now() - 12 * 60 * 60 * 1000);
|
||
|
||
const matchesToFetch = await this.prisma.liveMatch.findMany({
|
||
where: {
|
||
mstUtc: { gte: BigInt(twelveHoursAgo.getTime()) },
|
||
...(allowedLeagueIds.length > 0
|
||
? { leagueId: { in: allowedLeagueIds } }
|
||
: {}),
|
||
// Exclude live and finished matches — preserve their pre-match odds
|
||
NOT: {
|
||
OR: [
|
||
{ status: { in: FINISHED_STATUS_VALUES_FOR_DB } },
|
||
{ state: { in: FINISHED_STATE_VALUES_FOR_DB } },
|
||
{ status: { in: LIVE_STATUS_VALUES_FOR_DB } },
|
||
{ state: { in: LIVE_STATE_VALUES_FOR_DB } },
|
||
],
|
||
},
|
||
},
|
||
include: {
|
||
homeTeam: { select: { name: true } },
|
||
awayTeam: { select: { name: true } },
|
||
},
|
||
orderBy: [{ oddsUpdatedAt: "asc" }, { mstUtc: "asc" }],
|
||
take: 1000,
|
||
});
|
||
|
||
if (matchesToFetch.length === 0) {
|
||
this.logger.log("MONEY No matches to fetch odds for");
|
||
return;
|
||
}
|
||
|
||
this.logger.log(
|
||
`MONEY Fetching odds for ${matchesToFetch.length} matches`,
|
||
);
|
||
|
||
let successCount = 0;
|
||
let errorCount = 0;
|
||
const failedMatches: LiveMatchOddsTarget[] = [];
|
||
|
||
// Initial pass
|
||
for (const match of matchesToFetch) {
|
||
try {
|
||
await this.processMatchOdds(match);
|
||
successCount++;
|
||
await this.delay(500);
|
||
} catch (err: unknown) {
|
||
errorCount++;
|
||
const isRetryable = this.isRetryableError(err);
|
||
if (isRetryable) {
|
||
failedMatches.push(match);
|
||
} else {
|
||
const message = err instanceof Error ? err.message : String(err);
|
||
this.logger.warn(
|
||
`Match ${match.id} odds fetch failed (Non-retryable): ${message}`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Retry failed matches (502/Timeout)
|
||
if (failedMatches.length > 0) {
|
||
this.logger.warn(
|
||
`Retrying ${failedMatches.length} failed matches (502/Timeout)...`,
|
||
);
|
||
|
||
for (const match of failedMatches) {
|
||
await this.delay(2000);
|
||
try {
|
||
await this.processMatchOdds(match);
|
||
successCount++;
|
||
this.logger.log(`SUCCESS Retry successful for match ${match.id}`);
|
||
} catch (retryErr: unknown) {
|
||
const message =
|
||
retryErr instanceof Error ? retryErr.message : String(retryErr);
|
||
this.logger.error(
|
||
`⌠Retry failed for match ${match.id}: ${message}`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
this.logger.log(
|
||
`MONEY Odds complete: ${successCount} success, ${errorCount} errors (initially)`,
|
||
);
|
||
} catch (error: unknown) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
this.logger.error(`Odds fetch job failed: ${message}`);
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// Phase 4: Fill missing lineups (backup)
|
||
// ────────────────────────────────────────────────────────────
|
||
|
||
private async fillMissingLineups(): Promise<void> {
|
||
try {
|
||
const matchesToUpdate = await this.prisma.liveMatch.findMany({
|
||
where: {
|
||
sport: "football",
|
||
NOT: {
|
||
OR: [
|
||
{ status: { in: FINISHED_STATUS_VALUES_FOR_DB } },
|
||
{ state: { in: FINISHED_STATE_VALUES_FOR_DB } },
|
||
{
|
||
AND: [
|
||
{ scoreHome: { not: null } },
|
||
{ scoreAway: { not: null } },
|
||
{
|
||
NOT: {
|
||
OR: [
|
||
{ status: { in: LIVE_STATUS_VALUES_FOR_DB } },
|
||
{ state: { in: LIVE_STATE_VALUES_FOR_DB } },
|
||
],
|
||
},
|
||
},
|
||
],
|
||
},
|
||
],
|
||
},
|
||
},
|
||
select: { id: true, matchSlug: true, lineups: true, sport: true },
|
||
take: 30,
|
||
});
|
||
|
||
// Only football matches without lineups
|
||
const toUpdate = matchesToUpdate.filter(
|
||
(m) => !m.lineups && m.sport === "football",
|
||
);
|
||
|
||
if (toUpdate.length === 0) {
|
||
this.logger.debug("👕 All lineups already filled");
|
||
return;
|
||
}
|
||
|
||
this.logger.log(`👕 Filling lineups for ${toUpdate.length} matches...`);
|
||
|
||
for (const match of toUpdate) {
|
||
try {
|
||
const [formation, substitutions] = await Promise.all([
|
||
this.scraper.fetchStartingFormation(match.id),
|
||
this.scraper.fetchSubstitutions(match.id),
|
||
]);
|
||
const sidelined = match.matchSlug
|
||
? await this.scraper.fetchSidelinedPlayers(
|
||
match.id,
|
||
match.matchSlug,
|
||
)
|
||
: null;
|
||
|
||
// Normalize to same home.xi/away.xi format used by processMatchOdds
|
||
let normalizedLineups: Record<string, unknown> | null = null;
|
||
if (formation || substitutions) {
|
||
normalizedLineups = {
|
||
home: {
|
||
xi: formation?.stats?.home || [],
|
||
subs: substitutions?.stats?.home || [],
|
||
},
|
||
away: {
|
||
xi: formation?.stats?.away || [],
|
||
subs: substitutions?.stats?.away || [],
|
||
},
|
||
};
|
||
}
|
||
|
||
await this.prisma.liveMatch.update({
|
||
where: { id: match.id },
|
||
data: {
|
||
lineups: normalizedLineups
|
||
? JSON.parse(JSON.stringify(normalizedLineups))
|
||
: Prisma.JsonNull,
|
||
sidelined: sidelined
|
||
? JSON.parse(JSON.stringify(sidelined))
|
||
: Prisma.JsonNull,
|
||
updatedAt: new Date(),
|
||
},
|
||
});
|
||
|
||
this.logger.log(`👕 Lineups filled for match ${match.id}`);
|
||
await this.delay(500);
|
||
} catch (err: unknown) {
|
||
const message = err instanceof Error ? err.message : String(err);
|
||
this.logger.warn(`Lineup fill failed for ${match.id}: ${message}`);
|
||
}
|
||
}
|
||
} catch (error: unknown) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
this.logger.error(`Lineup fill error: ${message}`);
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// Unified match fetcher — DRY for football + basketball
|
||
// ────────────────────────────────────────────────────────────
|
||
|
||
private async fetchMatchesForSport(
|
||
sport: SportType,
|
||
date: string,
|
||
topLeagueIds: Set<string>,
|
||
): Promise<void> {
|
||
const sportParam = sport === "basketball" ? "Basketball" : "Soccer";
|
||
const url = `https://www.mackolik.com/perform/p0/ajax/components/competition/livescores/json?sports[]=${sportParam}&matchDate=${date}`;
|
||
|
||
try {
|
||
const response = await firstValueFrom(
|
||
this.httpService.get(url, { timeout: 20000 }),
|
||
);
|
||
|
||
const payload = this.parseLiveScoresPayload(response.data);
|
||
if (!payload) {
|
||
this.logger.warn(`[${sport}] No valid data from API for ${date}`);
|
||
return;
|
||
}
|
||
|
||
const allMatches = payload.matches;
|
||
const competitions = payload.competitions;
|
||
|
||
// Apply league filter
|
||
const targetMatches =
|
||
topLeagueIds.size > 0
|
||
? allMatches.filter(
|
||
(m) =>
|
||
!!m.competitionId && topLeagueIds.has(String(m.competitionId)),
|
||
)
|
||
: allMatches;
|
||
|
||
if (targetMatches.length === 0) {
|
||
this.logger.log(`[${sport}] No matches found for ${date}`);
|
||
return;
|
||
}
|
||
|
||
this.logger.log(
|
||
`[${sport}] Processing ${targetMatches.length}/${allMatches.length} matches for ${date}`,
|
||
);
|
||
|
||
// Local caches to avoid N+1 redundant upserts
|
||
const processedCountries = new Set<string>();
|
||
const processedLeagues = new Set<string>();
|
||
const processedTeams = new Set<string>();
|
||
const integrityChecked = new Map<string, boolean>();
|
||
|
||
let upsertCount = 0;
|
||
let skippedCount = 0;
|
||
|
||
for (const match of targetMatches) {
|
||
try {
|
||
const homeTeamId = match.homeTeam.id;
|
||
const awayTeamId = match.awayTeam.id;
|
||
const leagueId = match.competitionId;
|
||
|
||
// Sport integrity check (cached)
|
||
const ensureIntegrity = async (
|
||
entityType: "league" | "team" | "liveMatch",
|
||
entityId: string | null,
|
||
): Promise<boolean> => {
|
||
if (!entityId) return true;
|
||
const cacheKey = `${sport}:${entityType}:${entityId}`;
|
||
if (!integrityChecked.has(cacheKey)) {
|
||
integrityChecked.set(
|
||
cacheKey,
|
||
await this.ensureLiveEntitySportIntegrity(
|
||
entityType,
|
||
entityId,
|
||
sport,
|
||
),
|
||
);
|
||
}
|
||
return integrityChecked.get(cacheKey) === true;
|
||
};
|
||
|
||
if (
|
||
!(await ensureIntegrity("liveMatch", String(match.id))) ||
|
||
!(await ensureIntegrity("league", leagueId)) ||
|
||
!(await ensureIntegrity("team", homeTeamId)) ||
|
||
!(await ensureIntegrity("team", awayTeamId))
|
||
) {
|
||
skippedCount++;
|
||
continue;
|
||
}
|
||
|
||
// Resolve competition details
|
||
const compInfo = this.resolveCompetition(competitions, leagueId);
|
||
const countryId = compInfo?.country?.id || null;
|
||
|
||
// 1. Upsert Country (cached)
|
||
if (
|
||
countryId &&
|
||
compInfo?.country?.name &&
|
||
!processedCountries.has(countryId)
|
||
) {
|
||
await this.prisma.country
|
||
.upsert({
|
||
where: { id: countryId },
|
||
update: { name: compInfo.country.name },
|
||
create: { id: countryId, name: compInfo.country.name },
|
||
})
|
||
.catch((e: Error) =>
|
||
this.logger.warn(
|
||
`[${sport}] Country upsert failed: ${e.message}`,
|
||
),
|
||
);
|
||
processedCountries.add(countryId);
|
||
}
|
||
|
||
// 2. Upsert League (cached)
|
||
if (
|
||
leagueId &&
|
||
compInfo?.name &&
|
||
!processedLeagues.has(String(leagueId))
|
||
) {
|
||
await this.prisma.league
|
||
.upsert({
|
||
where: { id: leagueId },
|
||
update: {
|
||
name: compInfo.name,
|
||
countryId: countryId,
|
||
sport: sport,
|
||
},
|
||
create: {
|
||
id: leagueId,
|
||
name: compInfo.name,
|
||
countryId: countryId,
|
||
sport: sport,
|
||
competitionSlug: compInfo.slug || null,
|
||
},
|
||
})
|
||
.catch((e: Error) =>
|
||
this.logger.warn(
|
||
`[${sport}] League upsert failed: ${e.message}`,
|
||
),
|
||
);
|
||
processedLeagues.add(String(leagueId));
|
||
}
|
||
|
||
// 3. Upsert Home Team (cached)
|
||
if (
|
||
homeTeamId &&
|
||
match.homeTeam?.name &&
|
||
!processedTeams.has(homeTeamId)
|
||
) {
|
||
await this.prisma.team
|
||
.upsert({
|
||
where: { id: homeTeamId },
|
||
update: {
|
||
name: match.homeTeam.name,
|
||
slug: match.homeTeam.slug || null,
|
||
sport: sport,
|
||
},
|
||
create: {
|
||
id: homeTeamId,
|
||
name: match.homeTeam.name,
|
||
slug: match.homeTeam.slug || null,
|
||
sport: sport,
|
||
},
|
||
})
|
||
.catch((e: Error) =>
|
||
this.logger.warn(
|
||
`[${sport}] Home team upsert failed: ${e.message}`,
|
||
),
|
||
);
|
||
processedTeams.add(homeTeamId);
|
||
}
|
||
|
||
// 4. Upsert Away Team (cached)
|
||
if (
|
||
awayTeamId &&
|
||
match.awayTeam?.name &&
|
||
!processedTeams.has(awayTeamId)
|
||
) {
|
||
await this.prisma.team
|
||
.upsert({
|
||
where: { id: awayTeamId },
|
||
update: {
|
||
name: match.awayTeam.name,
|
||
slug: match.awayTeam.slug || null,
|
||
sport: sport,
|
||
},
|
||
create: {
|
||
id: awayTeamId,
|
||
name: match.awayTeam.name,
|
||
slug: match.awayTeam.slug || null,
|
||
sport: sport,
|
||
},
|
||
})
|
||
.catch((e: Error) =>
|
||
this.logger.warn(
|
||
`[${sport}] Away team upsert failed: ${e.message}`,
|
||
),
|
||
);
|
||
processedTeams.add(awayTeamId);
|
||
}
|
||
|
||
// Safe score parsing
|
||
const sHome = this.asInt(match.homeScore ?? match.score?.home);
|
||
const sAway = this.asInt(match.awayScore ?? match.score?.away);
|
||
const sHtHome = this.asInt(match.score?.ht?.home);
|
||
const sHtAway = this.asInt(match.score?.ht?.away);
|
||
const storedStatus = deriveStoredMatchStatus({
|
||
state: match.state,
|
||
status: match.status,
|
||
substate: match.substate,
|
||
statusBoxContent: match.statusBoxContent,
|
||
scoreHome: sHome,
|
||
scoreAway: sAway,
|
||
score: match.score,
|
||
});
|
||
|
||
// Handle postponed matches (ERT = Erteledendi)
|
||
if (match.statusBoxContent === "ERT") {
|
||
await this.prisma.liveMatch
|
||
.updateMany({
|
||
where: { id: String(match.id) },
|
||
data: {
|
||
status: "POSTPONED",
|
||
state: "postponed",
|
||
substate: "postponed",
|
||
updatedAt: new Date(),
|
||
},
|
||
})
|
||
.catch(() => { });
|
||
this.logger.debug(
|
||
`[${sport}] Marked as POSTPONED: ${match.matchName}`,
|
||
);
|
||
skippedCount++;
|
||
continue;
|
||
}
|
||
|
||
// 5. Upsert LiveMatch
|
||
await this.prisma.liveMatch.upsert({
|
||
where: { id: String(match.id) },
|
||
update: {
|
||
leagueId: leagueId,
|
||
state: match.state || null,
|
||
substate: match.substate || null,
|
||
status: storedStatus,
|
||
scoreHome: sHome,
|
||
scoreAway: sAway,
|
||
...(sHtHome !== null && { htScoreHome: sHtHome }),
|
||
...(sHtAway !== null && { htScoreAway: sHtAway }),
|
||
homeTeamId: homeTeamId,
|
||
awayTeamId: awayTeamId,
|
||
updatedAt: new Date(),
|
||
},
|
||
create: {
|
||
id: String(match.id),
|
||
matchName: match.matchName,
|
||
matchSlug: match.matchSlug,
|
||
sport: sport,
|
||
leagueId: leagueId,
|
||
state: match.state || null,
|
||
substate: match.substate || null,
|
||
status: storedStatus,
|
||
mstUtc: BigInt(match.mstUtc || Date.now()),
|
||
scoreHome: sHome,
|
||
scoreAway: sAway,
|
||
htScoreHome: sHtHome,
|
||
htScoreAway: sHtAway,
|
||
homeTeamId: homeTeamId,
|
||
awayTeamId: awayTeamId,
|
||
},
|
||
});
|
||
upsertCount++;
|
||
|
||
// Progress logging every 100 matches
|
||
if (
|
||
(upsertCount + skippedCount) % 100 === 0 ||
|
||
upsertCount + skippedCount === targetMatches.length
|
||
) {
|
||
this.logger.log(
|
||
`[${sport}] â³ Progress: ${upsertCount + skippedCount}/${targetMatches.length} (Saved: ${upsertCount}, Skipped: ${skippedCount})`,
|
||
);
|
||
}
|
||
} catch (err: unknown) {
|
||
const message = err instanceof Error ? err.message : String(err);
|
||
this.logger.warn(`[${sport}] Match ${match.id} failed: ${message}`);
|
||
}
|
||
}
|
||
|
||
this.logger.log(
|
||
`[${sport}] [${date}] Done: ${upsertCount} saved, ${skippedCount} skipped`,
|
||
);
|
||
} catch (error: unknown) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
this.logger.error(`[${sport}] Fetch failed for ${date}: ${message}`);
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// processMatchOdds — odds + referee + lineups + sidelined
|
||
// (Preserved from original — no logic changes)
|
||
// ────────────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Persist live odds movement into the structured odd_selections / odds_history
|
||
* tables (the live odds elsewhere are only a JSON blob on live_matches). This
|
||
* is what makes opening→closing ranges and steam queryable in SQL. Mirrors the
|
||
* historical feeder's change-detection (feeder-persistence.service.ts) but for
|
||
* the live HTML odds format. Change-only writes; ALL markets in the payload
|
||
* are tracked (so anomalies/steam on any bet type are captured).
|
||
*/
|
||
private async persistOddsHistory(
|
||
matchId: string,
|
||
odds: Record<string, Record<string, number>>,
|
||
): Promise<void> {
|
||
try {
|
||
const cats = await this.prisma.oddCategory.findMany({
|
||
where: { matchId },
|
||
include: { selections: true },
|
||
});
|
||
for (const [catName, sels] of Object.entries(odds)) {
|
||
let category = cats.find((c) => c.name === catName);
|
||
if (!category) {
|
||
category = await this.prisma.oddCategory.create({
|
||
data: { matchId, name: catName },
|
||
include: { selections: true },
|
||
});
|
||
cats.push(category);
|
||
}
|
||
for (const [selName, value] of Object.entries(sels)) {
|
||
const sVal = String(value);
|
||
if (!selName || !sVal || !(value > 0)) continue;
|
||
const existing = category.selections.find((s) => s.name === selName);
|
||
if (existing) {
|
||
if (existing.oddValue !== sVal) {
|
||
const oldVal = parseFloat(existing.oddValue || "0");
|
||
const newVal = parseFloat(sVal);
|
||
if (!isNaN(oldVal) && !isNaN(newVal) && oldVal > 0) {
|
||
await this.prisma.oddsHistory.create({
|
||
data: {
|
||
selectionId: existing.dbId,
|
||
matchId,
|
||
previousValue: oldVal,
|
||
newValue: newVal,
|
||
},
|
||
});
|
||
}
|
||
await this.prisma.oddSelection.update({
|
||
where: { dbId: existing.dbId },
|
||
data: { oddValue: sVal },
|
||
});
|
||
existing.oddValue = sVal;
|
||
}
|
||
} else {
|
||
const created = await this.prisma.oddSelection.create({
|
||
data: {
|
||
categoryId: category.dbId,
|
||
name: selName,
|
||
oddValue: sVal,
|
||
openingValue: sVal,
|
||
},
|
||
});
|
||
category.selections.push(created);
|
||
}
|
||
}
|
||
}
|
||
} catch (err: unknown) {
|
||
const message = err instanceof Error ? err.message : String(err);
|
||
this.logger.debug(
|
||
`odds_history persist skipped for ${matchId}: ${message}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
private async processMatchOdds(match: LiveMatchOddsTarget): Promise<void> {
|
||
const matchSlug = match.matchSlug || "match";
|
||
const sport = String(match.sport || "football").toLowerCase();
|
||
const sportPath = sport === "basketball" ? "basketbol/mac" : "mac";
|
||
const httpHeaders = {
|
||
"User-Agent":
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||
};
|
||
|
||
let odds: Record<string, Record<string, number>> = {};
|
||
let refereeName: string | null = null;
|
||
let lineups: LiveLineupsJson | null = null;
|
||
let sidelined: SidelinedResponse | null = null;
|
||
|
||
// 1. Fetch Odds from Ä°ddaa page
|
||
const oddsUrl = `https://www.mackolik.com/${sportPath}/${matchSlug}/iddaa/${match.id}`;
|
||
try {
|
||
const response = await firstValueFrom(
|
||
this.httpService.get(oddsUrl, {
|
||
timeout: 10000,
|
||
headers: httpHeaders,
|
||
maxRedirects: 5,
|
||
}),
|
||
);
|
||
odds = this.extractOddsFromHtml(
|
||
typeof response.data === "string" ? response.data : "",
|
||
);
|
||
} catch (e: unknown) {
|
||
if (this.isRetryableError(e)) throw e;
|
||
const message = e instanceof Error ? e.message : String(e);
|
||
this.logger.warn(`Odds fetch failed for ${match.id}: ${message}`);
|
||
}
|
||
|
||
if (sport === "football") {
|
||
// 2. Fetch Referee from main match page
|
||
if (!refereeName) {
|
||
const mainUrl = `https://www.mackolik.com/mac/${matchSlug}/${match.id}`;
|
||
try {
|
||
const mainResp = await firstValueFrom(
|
||
this.httpService.get(mainUrl, {
|
||
timeout: 10000,
|
||
headers: httpHeaders,
|
||
maxRedirects: 5,
|
||
}),
|
||
);
|
||
refereeName = this.extractRefereeFromHtml(
|
||
typeof mainResp.data === "string" ? mainResp.data : "",
|
||
);
|
||
} catch {
|
||
// Non-critical — referee is optional
|
||
}
|
||
}
|
||
|
||
// 3. Fetch Lineups & Sidelined Players
|
||
const now = Date.now();
|
||
const matchTime = Number(match.mstUtc);
|
||
const diffHours = (matchTime - now) / (1000 * 60 * 60);
|
||
|
||
// Fetch if between -3 hours (started) and +24 hours (upcoming)
|
||
if (diffHours < 24 && diffHours > -3) {
|
||
// Lineups
|
||
try {
|
||
const [startingFormation, substitutions] = await Promise.all([
|
||
this.scraper.fetchStartingFormation(match.id),
|
||
this.scraper.fetchSubstitutions(match.id),
|
||
]);
|
||
|
||
if (startingFormation || substitutions) {
|
||
lineups = {
|
||
home: {
|
||
xi: startingFormation?.stats?.home || [],
|
||
subs: substitutions?.stats?.home || [],
|
||
},
|
||
away: {
|
||
xi: startingFormation?.stats?.away || [],
|
||
subs: substitutions?.stats?.away || [],
|
||
},
|
||
};
|
||
this.logger.log(`👥 Lineups found for ${match.matchName}`);
|
||
} else {
|
||
this.logger.debug(`No lineups (yet) for ${match.matchName}`);
|
||
}
|
||
} catch (err: unknown) {
|
||
const message = err instanceof Error ? err.message : String(err);
|
||
this.logger.warn(`Lineup fetch failed for ${match.id}: ${message}`);
|
||
}
|
||
|
||
// Sidelined Players (Injuries/Suspensions)
|
||
try {
|
||
sidelined = await this.scraper.fetchSidelinedPlayers(
|
||
match.id,
|
||
matchSlug,
|
||
);
|
||
if (sidelined) {
|
||
if (sidelined.homeTeam) {
|
||
sidelined.homeTeam.teamName = match.homeTeam?.name || "";
|
||
}
|
||
if (sidelined.awayTeam) {
|
||
sidelined.awayTeam.teamName = match.awayTeam?.name || "";
|
||
}
|
||
|
||
if (
|
||
sidelined.homeTeam?.totalSidelined > 0 ||
|
||
sidelined.awayTeam?.totalSidelined > 0
|
||
) {
|
||
this.logger.log(
|
||
`🚑 Sidelined: ${sidelined.homeTeam.totalSidelined}(H) - ${sidelined.awayTeam.totalSidelined}(A) for ${match.matchName}`,
|
||
);
|
||
}
|
||
}
|
||
} catch (err: unknown) {
|
||
const message = err instanceof Error ? err.message : String(err);
|
||
this.logger.warn(
|
||
`Sidelined fetch failed for ${match.id}: ${message}`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Guard: If match already has pre-match odds and is now live/finished,
|
||
// do NOT overwrite odds/lineups/sidelined — the model needs stable pre-match data.
|
||
const matchState = match.state?.toLowerCase() ?? "";
|
||
const matchStatus = match.status?.toLowerCase() ?? "";
|
||
const liveStates = LIVE_STATE_VALUES_FOR_DB.map((s) => s.toLowerCase());
|
||
const liveStatuses = LIVE_STATUS_VALUES_FOR_DB.map((s) => s.toLowerCase());
|
||
const finishedStates = FINISHED_STATE_VALUES_FOR_DB.map((s) =>
|
||
s.toLowerCase(),
|
||
);
|
||
const finishedStatuses = FINISHED_STATUS_VALUES_FOR_DB.map((s) =>
|
||
s.toLowerCase(),
|
||
);
|
||
|
||
const isLiveOrFinished =
|
||
liveStates.includes(matchState) ||
|
||
liveStatuses.includes(matchStatus) ||
|
||
finishedStates.includes(matchState) ||
|
||
finishedStatuses.includes(matchStatus);
|
||
|
||
const existingOdds = match.odds as Record<string, unknown> | null;
|
||
const hasExistingOdds =
|
||
!!existingOdds &&
|
||
typeof existingOdds === "object" &&
|
||
Object.keys(existingOdds).length > 0;
|
||
|
||
if (isLiveOrFinished && hasExistingOdds) {
|
||
// Match is live/finished and already has pre-match odds — skip data update
|
||
this.logger.debug(
|
||
`🛡️ Preserving pre-match data for ${match.matchName} (status: ${matchStatus}, state: ${matchState})`,
|
||
);
|
||
await this.prisma.liveMatch.update({
|
||
where: { id: match.id },
|
||
data: { oddsUpdatedAt: new Date() },
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Update odds/lineups/sidelined for pre-match (NS) matches
|
||
await this.prisma.liveMatch.update({
|
||
where: { id: match.id },
|
||
data: {
|
||
odds: Object.keys(odds).length > 0 ? odds : undefined,
|
||
oddsUpdatedAt: new Date(),
|
||
refereeName: refereeName ?? undefined,
|
||
lineups: (lineups as unknown as Prisma.InputJsonValue) ?? undefined,
|
||
sidelined: (sidelined as unknown as Prisma.InputJsonValue) ?? undefined,
|
||
},
|
||
});
|
||
|
||
// Fill the structured odds_history table from this pre-match odds refresh,
|
||
// so opening→closing ranges & steam are captured in the DB (not just the
|
||
// live_matches JSON blob). Runs only for pre-match matches reached here.
|
||
if (Object.keys(odds).length > 0) {
|
||
await this.persistOddsHistory(match.id, odds);
|
||
}
|
||
|
||
if (
|
||
Object.keys(odds).length > 0 ||
|
||
refereeName ||
|
||
lineups ||
|
||
(sidelined &&
|
||
(sidelined.homeTeam.totalSidelined > 0 ||
|
||
sidelined.awayTeam.totalSidelined > 0))
|
||
) {
|
||
this.logger.log(
|
||
`SUCCESS Loop update: ${match.matchName} | Odds: ${Object.keys(odds).length} | Ref: ${refereeName || "N/A"} | Lineups: ${lineups ? "Yes" : "No"} | Sidelined: ${sidelined ? "Yes" : "No"}`,
|
||
);
|
||
} else {
|
||
this.logger.debug(
|
||
`â• No detailed data for ${match.matchName}, marked check.`,
|
||
);
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// HTML Extraction Helpers (preserved — no logic changes)
|
||
// ────────────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Extract odds from Mackolik HTML page
|
||
* Returns structured odds object: { "MS": {"1": 2.10, "X": 3.40}, "AU25": {"Alt": 2.05, "Ãœst": 1.75} }
|
||
*/
|
||
private extractOddsFromHtml(
|
||
html: string,
|
||
): Record<string, Record<string, number>> {
|
||
const odds: Record<string, Record<string, number>> = {};
|
||
if (!html) return odds;
|
||
|
||
try {
|
||
const settingsPattern = /data-settings="([^"]+)"/g;
|
||
let match;
|
||
|
||
while ((match = settingsPattern.exec(html)) !== null) {
|
||
try {
|
||
const decoded = match[1]
|
||
.replace(/"/g, '"')
|
||
.replace(/&/g, "&")
|
||
.replace(/</g, "<")
|
||
.replace(/>/g, ">");
|
||
|
||
const parsed = JSON.parse(decoded) as unknown;
|
||
if (!this.isRecord(parsed)) continue;
|
||
|
||
const iddaaEventId = this.isRecord(parsed.iddaaEventId)
|
||
? parsed.iddaaEventId
|
||
: null;
|
||
const marketCollection =
|
||
iddaaEventId && this.isRecord(iddaaEventId.marketCollection)
|
||
? iddaaEventId.marketCollection
|
||
: null;
|
||
|
||
if (marketCollection) {
|
||
for (const marketValue of Object.values(marketCollection)) {
|
||
if (!this.isRecord(marketValue)) continue;
|
||
const marketName = this.asString(marketValue.name)?.trim();
|
||
if (!marketName) continue;
|
||
|
||
// First-come-first-served: Skip if already populated
|
||
if (
|
||
odds[marketName] &&
|
||
Object.keys(odds[marketName]).length > 0
|
||
) {
|
||
continue;
|
||
}
|
||
|
||
odds[marketName] = {};
|
||
|
||
const selectionCollection = this.isRecord(
|
||
marketValue.selectionCollection,
|
||
)
|
||
? marketValue.selectionCollection
|
||
: {};
|
||
for (const selectionValue of Object.values(selectionCollection)) {
|
||
if (!this.isRecord(selectionValue)) continue;
|
||
|
||
const selName =
|
||
this.asString(selectionValue.name) ||
|
||
this.asString(selectionValue.outcome);
|
||
const selOdd = Number.parseFloat(
|
||
this.asString(selectionValue.odd) || "",
|
||
);
|
||
if (selName && !isNaN(selOdd)) {
|
||
odds[marketName][selName] = selOdd;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} catch {
|
||
// JSON parse error, skip
|
||
}
|
||
}
|
||
} catch {
|
||
this.logger.warn("Failed to extract odds from HTML");
|
||
}
|
||
|
||
return odds;
|
||
}
|
||
|
||
/**
|
||
* Normalize odds category names to short codes
|
||
*/
|
||
private normalizeOddsCategory(name: string): string | null {
|
||
if (!name) return null;
|
||
const lower = name.toLowerCase();
|
||
|
||
// Specific & Compound names FIRST
|
||
if (lower.includes("ilk yarı/maç sonucu")) return "HTFT";
|
||
if (lower.includes("1. yarı sonucu")) return "HT";
|
||
if (lower.includes("çifte şans")) return "CS";
|
||
|
||
// General names LATER
|
||
if (lower.includes("maç sonucu") && !lower.includes("handikap"))
|
||
return "MS";
|
||
if (lower.includes("karşılıklı gol")) return "KG";
|
||
if (lower.includes("2,5 alt/üst") || lower.includes("2.5")) return "AU25";
|
||
if (lower.includes("1,5 alt/üst") || lower.includes("1.5")) return "AU15";
|
||
if (lower.includes("3,5 alt/üst") || lower.includes("3.5")) return "AU35";
|
||
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* Extract referee name from match page HTML
|
||
*/
|
||
private extractRefereeFromHtml(html: string): string | null {
|
||
try {
|
||
// Strategy 1: Mackolik officials section — head referee in '--main' list item
|
||
const mainOfficialPattern =
|
||
/official-list-item--main[^>]*>\s*(?:<[^>]*>\s*)*?<span[^>]*official-name[^>]*>\s*([^<]+)/i;
|
||
const mainMatch = mainOfficialPattern.exec(html);
|
||
if (mainMatch?.[1]) {
|
||
const name = mainMatch[1].trim();
|
||
if (name.length > 2 && name.length < 100) return name;
|
||
}
|
||
|
||
// Strategy 2: Any official-name followed by "Orta Hakem"
|
||
const ortaHakemPattern =
|
||
/official-name[^>]*>\s*([^<]+)<[\s\S]*?Orta\s*Hakem/i;
|
||
const ortaMatch = ortaHakemPattern.exec(html);
|
||
if (ortaMatch?.[1]) {
|
||
const name = ortaMatch[1].trim();
|
||
if (name.length > 2 && name.length < 100) return name;
|
||
}
|
||
|
||
// Strategy 3: Generic fallback patterns
|
||
const fallbackPatterns = [
|
||
/Hakem:\s*([^<]+)/i,
|
||
/"refereeName":"([^"]+)"/i,
|
||
];
|
||
for (const pattern of fallbackPatterns) {
|
||
const result = pattern.exec(html);
|
||
if (result?.[1]) {
|
||
const name = result[1].trim();
|
||
if (name.length > 2 && name.length < 100) return name;
|
||
}
|
||
}
|
||
} catch {
|
||
// Ignore extraction errors
|
||
}
|
||
return null;
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────
|
||
// Low-level Helpers (preserved — no logic changes)
|
||
// ────────────────────────────────────────────────────────────
|
||
|
||
private shouldSkipInHistoricalMode(jobName: string): boolean {
|
||
if (process.env.FEEDER_MODE === "historical") {
|
||
this.logger.debug(`Skipping ${jobName} in historical feeder mode`);
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
private isRecord(value: unknown): value is Record<string, unknown> {
|
||
return !!value && typeof value === "object" && !Array.isArray(value);
|
||
}
|
||
|
||
private asString(value: unknown): string | null {
|
||
if (typeof value === "string") return value;
|
||
if (typeof value === "number" && Number.isFinite(value))
|
||
return String(value);
|
||
return null;
|
||
}
|
||
|
||
private asInt(value: unknown): number | null {
|
||
if (typeof value === "number" && Number.isFinite(value)) {
|
||
return Math.trunc(value);
|
||
}
|
||
if (typeof value === "string" && value.trim().length > 0) {
|
||
const parsed = parseInt(value, 10);
|
||
return Number.isNaN(parsed) ? null : parsed;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
private parseTeam(value: unknown): LiveScoreTeamPayload | null {
|
||
if (!this.isRecord(value)) return null;
|
||
|
||
const id = this.asString(value.id);
|
||
if (!id) return null;
|
||
|
||
return {
|
||
id,
|
||
name: this.asString(value.name) || "Unknown",
|
||
slug: this.asString(value.slug),
|
||
};
|
||
}
|
||
|
||
private parseCompetition(value: unknown): LiveScoreCompetitionPayload | null {
|
||
if (!this.isRecord(value)) return null;
|
||
|
||
const id = this.asString(value.id);
|
||
const name = this.asString(value.name);
|
||
if (!id || !name) return null;
|
||
|
||
const rawCountry = this.isRecord(value.country) ? value.country : null;
|
||
const countryId = rawCountry ? this.asString(rawCountry.id) : null;
|
||
const countryName = rawCountry ? this.asString(rawCountry.name) : null;
|
||
|
||
return {
|
||
id,
|
||
name,
|
||
slug: this.asString(value.slug),
|
||
country:
|
||
countryId && countryName ? { id: countryId, name: countryName } : null,
|
||
};
|
||
}
|
||
|
||
private parseLiveScoreMatch(value: unknown): LiveScorePayloadMatch | null {
|
||
if (!this.isRecord(value)) return null;
|
||
|
||
const id = this.asString(value.id);
|
||
const homeTeam = this.parseTeam(value.homeTeam);
|
||
const awayTeam = this.parseTeam(value.awayTeam);
|
||
if (!id || !homeTeam || !awayTeam) return null;
|
||
|
||
const score = this.isRecord(value.score)
|
||
? {
|
||
home: this.asInt(value.score.home),
|
||
away: this.asInt(value.score.away),
|
||
}
|
||
: null;
|
||
|
||
return {
|
||
id,
|
||
matchName:
|
||
this.asString(value.matchName) || `${homeTeam.name} - ${awayTeam.name}`,
|
||
matchSlug: this.asString(value.matchSlug) || "",
|
||
competitionId: this.asString(value.competitionId),
|
||
mstUtc: this.asInt(value.mstUtc),
|
||
state: this.asString(value.state),
|
||
substate: this.asString(value.substate),
|
||
status: this.asString(value.status),
|
||
statusBoxContent: this.asString(value.statusBoxContent),
|
||
iddaaCode: this.asString(value.iddaaCode),
|
||
homeTeam,
|
||
awayTeam,
|
||
homeScore: this.asInt(value.homeScore),
|
||
awayScore: this.asInt(value.awayScore),
|
||
score,
|
||
};
|
||
}
|
||
|
||
private parseLiveScoresPayload(raw: unknown): {
|
||
matches: LiveScorePayloadMatch[];
|
||
competitions: Record<string, LiveScoreCompetitionPayload>;
|
||
} | null {
|
||
if (!this.isRecord(raw)) return null;
|
||
if (this.asString(raw.status) !== "success") return null;
|
||
|
||
const data = this.isRecord(raw.data) ? raw.data : null;
|
||
if (!data) return null;
|
||
|
||
const rawMatches = this.isRecord(data.matches) ? data.matches : {};
|
||
const rawCompetitions = this.isRecord(data.competitions)
|
||
? data.competitions
|
||
: {};
|
||
|
||
const matches: LiveScorePayloadMatch[] = [];
|
||
for (const rawMatch of Object.values(rawMatches)) {
|
||
const parsed = this.parseLiveScoreMatch(rawMatch);
|
||
if (parsed) matches.push(parsed);
|
||
}
|
||
|
||
const competitions: Record<string, LiveScoreCompetitionPayload> = {};
|
||
for (const [key, rawCompetition] of Object.entries(rawCompetitions)) {
|
||
const parsed = this.parseCompetition(rawCompetition);
|
||
if (parsed) {
|
||
competitions[key] = parsed;
|
||
}
|
||
}
|
||
|
||
return { matches, competitions };
|
||
}
|
||
|
||
private loadLeagueFilterSet(fileName: string): Set<string> | null {
|
||
try {
|
||
const filePath = path.join(process.cwd(), fileName);
|
||
const raw = JSON.parse(fs.readFileSync(filePath, "utf-8")) as unknown;
|
||
if (!Array.isArray(raw)) {
|
||
this.logger.error(`${fileName} is not a JSON array`);
|
||
return null;
|
||
}
|
||
|
||
const ids = raw
|
||
.map((value) => this.asString(value))
|
||
.filter((value): value is string => !!value);
|
||
return new Set(ids);
|
||
} catch (e: unknown) {
|
||
const message = e instanceof Error ? e.message : String(e);
|
||
this.logger.error(`Failed to load ${fileName}: ${message}`);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private resolveCompetition(
|
||
competitions: Record<string, LiveScoreCompetitionPayload>,
|
||
leagueId: string | null,
|
||
): LiveScoreCompetitionPayload | null {
|
||
if (!leagueId) return null;
|
||
if (competitions[leagueId]) return competitions[leagueId];
|
||
for (const comp of Object.values(competitions)) {
|
||
if (comp.id === leagueId) return comp;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
private async ensureLiveEntitySportIntegrity(
|
||
entityType: "league" | "team" | "liveMatch",
|
||
id: string,
|
||
expectedSport: SportType,
|
||
): Promise<boolean> {
|
||
if (!id) return true;
|
||
|
||
let existingSport: string | null | undefined;
|
||
|
||
if (entityType === "league") {
|
||
const existing = await this.prisma.league.findUnique({
|
||
where: { id },
|
||
select: { sport: true },
|
||
});
|
||
existingSport = existing?.sport;
|
||
} else if (entityType === "team") {
|
||
const existing = await this.prisma.team.findUnique({
|
||
where: { id },
|
||
select: { sport: true },
|
||
});
|
||
existingSport = existing?.sport;
|
||
} else {
|
||
const existing = await this.prisma.liveMatch.findUnique({
|
||
where: { id },
|
||
select: { sport: true },
|
||
});
|
||
existingSport = existing?.sport;
|
||
}
|
||
|
||
if (existingSport && existingSport !== expectedSport) {
|
||
this.logger.error(
|
||
`Sport integrity violation on ${entityType}:${id}. Existing=${existingSport}, incoming=${expectedSport}. Skipping write to prevent cross-sport overwrite.`,
|
||
);
|
||
return false;
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
private isRetryableError(err: unknown): boolean {
|
||
if (!err || typeof err !== "object") return false;
|
||
const errorObj = err as Record<string, unknown>;
|
||
const response = errorObj.response as Record<string, unknown> | undefined;
|
||
return (
|
||
response?.status === 502 ||
|
||
errorObj.code === "ECONNABORTED" ||
|
||
errorObj.code === "ETIMEDOUT"
|
||
);
|
||
}
|
||
|
||
private delay(ms: number): Promise<void> {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
}
|