1692 lines
63 KiB
TypeScript
Executable File
1692 lines
63 KiB
TypeScript
Executable File
import { Injectable, Logger } from "@nestjs/common";
|
|
import { Cron } from "@nestjs/schedule";
|
|
import { HttpService } from "@nestjs/axios";
|
|
import { PrismaService } from "../database/prisma.service";
|
|
import { firstValueFrom } from "rxjs";
|
|
import { FeederScraperService } from "../modules/feeder/feeder-scraper.service";
|
|
import * as fs from "fs";
|
|
import * as path from "path";
|
|
import { Prisma } from "@prisma/client";
|
|
import { SidelinedResponse } from "../modules/feeder/feeder.types";
|
|
import {
|
|
deriveStoredMatchStatus,
|
|
FINISHED_STATE_VALUES_FOR_DB,
|
|
FINISHED_STATUS_VALUES_FOR_DB,
|
|
LIVE_STATE_VALUES_FOR_DB,
|
|
LIVE_STATUS_VALUES_FOR_DB,
|
|
} from "../common/utils/match-status.util";
|
|
import {
|
|
getDateStringInTimeZone,
|
|
getDayBoundsForTimeZone,
|
|
getShiftedDateStringInTimeZone,
|
|
} from "../common/utils/timezone.util";
|
|
import { TaskLockService } from "./task-lock.service";
|
|
|
|
// ────────────────────────────────────────────────────────────────
|
|
// Types
|
|
// ────────────────────────────────────────────────────────────────
|
|
|
|
interface LiveScoreTeamPayload {
|
|
id: string;
|
|
name: string;
|
|
slug: string | null;
|
|
}
|
|
|
|
interface LiveScoreCompetitionPayload {
|
|
id: string;
|
|
name: string;
|
|
slug: string | null;
|
|
country: {
|
|
id: string;
|
|
name: string;
|
|
} | null;
|
|
}
|
|
|
|
interface LiveScorePayloadMatch {
|
|
id: string;
|
|
matchName: string;
|
|
matchSlug: string;
|
|
competitionId: string | null;
|
|
mstUtc: number | null;
|
|
state: string | null;
|
|
substate: string | null;
|
|
status: string | null;
|
|
statusBoxContent: string | null; // ERT = Erteledendi
|
|
iddaaCode: string | null;
|
|
homeTeam: LiveScoreTeamPayload;
|
|
awayTeam: LiveScoreTeamPayload;
|
|
homeScore: number | null;
|
|
awayScore: number | null;
|
|
score: {
|
|
home: number | null;
|
|
away: number | null;
|
|
ht?: {
|
|
home: number | null;
|
|
away: number | null;
|
|
} | null;
|
|
} | null;
|
|
}
|
|
|
|
type LiveMatchOddsTarget = Prisma.LiveMatchGetPayload<{
|
|
include: {
|
|
homeTeam: { select: { name: true } };
|
|
awayTeam: { select: { name: true } };
|
|
};
|
|
}>;
|
|
|
|
interface LiveLineupsJson {
|
|
home: { xi: unknown[]; subs: unknown[] };
|
|
away: { xi: unknown[]; subs: unknown[] };
|
|
}
|
|
|
|
interface PendingPredictionRunForSettlement {
|
|
id: bigint;
|
|
matchId: string;
|
|
engineVersion: string;
|
|
payloadSummary: unknown;
|
|
scoreHome: number | null;
|
|
scoreAway: number | null;
|
|
htScoreHome: number | null;
|
|
htScoreAway: number | null;
|
|
}
|
|
|
|
type SportType = "football" | "basketball";
|
|
|
|
// ────────────────────────────────────────────────────────────────
|
|
// Service
|
|
// ────────────────────────────────────────────────────────────────
|
|
|
|
@Injectable()
|
|
export class DataFetcherTask {
|
|
private readonly logger = new Logger(DataFetcherTask.name);
|
|
private readonly timeZone = "Europe/Istanbul";
|
|
|
|
constructor(
|
|
private readonly httpService: HttpService,
|
|
private readonly prisma: PrismaService,
|
|
private readonly scraper: FeederScraperService,
|
|
private readonly taskLock: TaskLockService,
|
|
) {}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// CRON 1: Main sync — every 15 minutes
|
|
// Phases: match list → live scores → odds → lineups
|
|
// ────────────────────────────────────────────────────────────
|
|
|
|
@Cron("*/15 * * * *")
|
|
async syncLiveMatches(): Promise<void> {
|
|
if (this.shouldSkipInHistoricalMode("syncLiveMatches")) return;
|
|
await this.taskLock.runWithLease(
|
|
"syncLiveMatches",
|
|
30 * 60 * 1000,
|
|
async () => {
|
|
await this.runLiveSync();
|
|
},
|
|
this.logger,
|
|
);
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// CRON 2: Daily cleanup + full sync — 07:00 Istanbul
|
|
// Preserve yesterday as a fallback until the 08:00 archive job completes.
|
|
// ────────────────────────────────────────────────────────────
|
|
|
|
@Cron("0 7 * * *", { timeZone: "Europe/Istanbul" })
|
|
async cleanAndFullSync(): Promise<void> {
|
|
if (this.shouldSkipInHistoricalMode("cleanAndFullSync")) return;
|
|
await this.taskLock.runWithLease(
|
|
"cleanAndFullSync",
|
|
2 * 60 * 60 * 1000,
|
|
async () => {
|
|
this.logger.log(
|
|
"cleanAndFullSync: Pruning stale live_matches while preserving yesterday for archive fallback...",
|
|
);
|
|
|
|
try {
|
|
const yesterdayDate = getShiftedDateStringInTimeZone(
|
|
-1,
|
|
this.timeZone,
|
|
);
|
|
const { startMs: yesterdayStartMs } = getDayBoundsForTimeZone(
|
|
yesterdayDate,
|
|
this.timeZone,
|
|
);
|
|
const cutoffDate = new Date(yesterdayStartMs);
|
|
|
|
const deleted = await this.prisma.liveMatch.deleteMany({
|
|
where: {
|
|
OR: [
|
|
{ mstUtc: { lt: BigInt(yesterdayStartMs) } },
|
|
{
|
|
AND: [
|
|
{ mstUtc: null },
|
|
{ updatedAt: { lt: cutoffDate } },
|
|
{
|
|
OR: [
|
|
{ status: { in: FINISHED_STATUS_VALUES_FOR_DB } },
|
|
{ state: { in: FINISHED_STATE_VALUES_FOR_DB } },
|
|
],
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
this.logger.log(
|
|
`Pruned ${deleted.count} stale live matches. Starting full sync...`,
|
|
);
|
|
} catch (error: unknown) {
|
|
const message =
|
|
error instanceof Error ? error.message : String(error);
|
|
this.logger.error(`Stale live_match cleanup failed: ${message}`);
|
|
return;
|
|
}
|
|
|
|
await this.runLiveSync();
|
|
},
|
|
this.logger,
|
|
);
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// Phase 1: Fetch match list for all sports
|
|
// ────────────────────────────────────────────────────────────
|
|
|
|
private async runLiveSync(): Promise<void> {
|
|
if (this.shouldSkipInHistoricalMode("syncLiveMatches")) return;
|
|
|
|
this.logger.log("syncLiveMatches START");
|
|
|
|
const today = getDateStringInTimeZone(new Date(), this.timeZone);
|
|
const tomorrow = getShiftedDateStringInTimeZone(1, this.timeZone);
|
|
await this.syncMatchList(today);
|
|
await this.syncMatchList(tomorrow);
|
|
await this.updateLiveScores();
|
|
await this.settlePredictionRuns();
|
|
await this.fetchOddsForMatches();
|
|
await this.fillMissingLineups();
|
|
|
|
this.logger.log("syncLiveMatches END");
|
|
}
|
|
|
|
private async syncMatchList(date: string): Promise<void> {
|
|
// Football
|
|
const footballLeagues = this.loadLeagueFilterSet("qualified_leagues.json");
|
|
if (footballLeagues && footballLeagues.size > 0) {
|
|
await this.fetchMatchesForSport("football", date, footballLeagues);
|
|
} else {
|
|
this.logger.warn(
|
|
"qualified_leagues.json is missing/empty — writing ALL football matches",
|
|
);
|
|
await this.fetchMatchesForSport("football", date, new Set());
|
|
}
|
|
|
|
// Basketball
|
|
const basketballLeagues = this.loadLeagueFilterSet(
|
|
"basketball_top_leagues.json",
|
|
);
|
|
if (!basketballLeagues || basketballLeagues.size === 0) {
|
|
this.logger.error(
|
|
"basketball_top_leagues.json is missing/empty. Basketball ingest skipped to protect data quality.",
|
|
);
|
|
} else {
|
|
await this.fetchMatchesForSport("basketball", date, basketballLeagues);
|
|
|
|
// Clean up basketball matches that are NOT in configured leagues
|
|
await this.prisma.liveMatch.deleteMany({
|
|
where: {
|
|
sport: "basketball",
|
|
OR: [
|
|
{ leagueId: null },
|
|
{ leagueId: { notIn: Array.from(basketballLeagues) } },
|
|
],
|
|
},
|
|
});
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// Phase 2: Live score updates (merged from live-updater.task)
|
|
// ────────────────────────────────────────────────────────────
|
|
|
|
private async updateLiveScores(): Promise<void> {
|
|
try {
|
|
const liveMatches = await this.prisma.liveMatch.findMany({
|
|
where: {
|
|
OR: [
|
|
{ state: { in: LIVE_STATE_VALUES_FOR_DB } },
|
|
{ status: { in: LIVE_STATUS_VALUES_FOR_DB } },
|
|
],
|
|
},
|
|
select: { id: true, matchSlug: true },
|
|
});
|
|
|
|
if (liveMatches.length === 0) {
|
|
this.logger.debug("No live matches to update scores for");
|
|
return;
|
|
}
|
|
|
|
this.logger.log(
|
|
`LIVE Updating scores for ${liveMatches.length} live matches`,
|
|
);
|
|
|
|
for (const match of liveMatches) {
|
|
try {
|
|
const url = `https://www.mackolik.com/ajax/football/match-info?matchId=${match.id}`;
|
|
const response = await firstValueFrom(
|
|
this.httpService.get(url, { timeout: 5000 }),
|
|
);
|
|
|
|
if (response.data?.data) {
|
|
const matchData = response.data.data;
|
|
const scoreHome = matchData.homeScore ?? null;
|
|
const scoreAway = matchData.awayScore ?? null;
|
|
const htScoreHome = this.asInt(
|
|
matchData.score?.ht?.home ??
|
|
matchData.htHomeScore ??
|
|
matchData.homeHtScore,
|
|
);
|
|
const htScoreAway = this.asInt(
|
|
matchData.score?.ht?.away ??
|
|
matchData.htAwayScore ??
|
|
matchData.awayHtScore,
|
|
);
|
|
const storedStatus = deriveStoredMatchStatus({
|
|
state: matchData.state,
|
|
status: matchData.status,
|
|
substate: matchData.substate,
|
|
scoreHome,
|
|
scoreAway,
|
|
});
|
|
await this.prisma.liveMatch.update({
|
|
where: { id: match.id },
|
|
data: {
|
|
scoreHome,
|
|
scoreAway,
|
|
htScoreHome,
|
|
htScoreAway,
|
|
state: matchData.state || null,
|
|
substate: matchData.substate || null,
|
|
status: storedStatus,
|
|
updatedAt: new Date(),
|
|
},
|
|
});
|
|
}
|
|
} catch {
|
|
// Individual match update failed, continue with others
|
|
}
|
|
}
|
|
|
|
this.logger.log("LIVE Live score update complete");
|
|
} catch (error: unknown) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
this.logger.error(`Live score update failed: ${message}`);
|
|
}
|
|
}
|
|
|
|
private async settlePredictionRuns(): Promise<void> {
|
|
try {
|
|
const rows = await this.prisma.$queryRawUnsafe<
|
|
PendingPredictionRunForSettlement[]
|
|
>(`
|
|
SELECT
|
|
pr.id,
|
|
pr.match_id AS "matchId",
|
|
pr.engine_version AS "engineVersion",
|
|
pr.payload_summary AS "payloadSummary",
|
|
m.score_home AS "scoreHome",
|
|
m.score_away AS "scoreAway",
|
|
m.ht_score_home AS "htScoreHome",
|
|
m.ht_score_away AS "htScoreAway"
|
|
FROM prediction_runs pr
|
|
JOIN matches m ON m.id = pr.match_id
|
|
WHERE pr.eventual_outcome IS NULL
|
|
AND m.sport = 'football'
|
|
AND m.status = 'FT'
|
|
AND m.score_home IS NOT NULL
|
|
AND m.score_away IS NOT NULL
|
|
ORDER BY pr.generated_at ASC
|
|
LIMIT 500
|
|
`);
|
|
|
|
if (rows.length === 0) return;
|
|
|
|
let settled = 0;
|
|
for (const row of rows) {
|
|
const result = this.resolvePredictionRunSettlement(row);
|
|
if (!result) continue;
|
|
const closingOddsSnapshot = await this.getClosingOddsSnapshot(
|
|
row.matchId,
|
|
);
|
|
const settlementSummary = {
|
|
settled_at: new Date().toISOString(),
|
|
model_version: row.engineVersion,
|
|
outcome: result.outcome,
|
|
unit_profit: result.unitProfit,
|
|
final_score: {
|
|
home: row.scoreHome,
|
|
away: row.scoreAway,
|
|
},
|
|
halftime_score: {
|
|
home: row.htScoreHome,
|
|
away: row.htScoreAway,
|
|
},
|
|
closing_odds_snapshot: closingOddsSnapshot,
|
|
};
|
|
|
|
await this.prisma.$executeRawUnsafe(
|
|
`
|
|
UPDATE prediction_runs
|
|
SET eventual_outcome = $1,
|
|
unit_profit = $2,
|
|
payload_summary = payload_summary || jsonb_build_object('settlement', $3::jsonb)
|
|
WHERE id = $4
|
|
`,
|
|
result.outcome,
|
|
result.unitProfit,
|
|
JSON.stringify(settlementSummary),
|
|
row.id,
|
|
);
|
|
settled++;
|
|
}
|
|
|
|
if (settled > 0) {
|
|
this.logger.log(`Settled ${settled} prediction run(s)`);
|
|
}
|
|
} catch (error: unknown) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
this.logger.warn(`Prediction run settlement skipped: ${message}`);
|
|
}
|
|
}
|
|
|
|
private async getClosingOddsSnapshot(
|
|
matchId: string,
|
|
): Promise<Record<string, unknown>> {
|
|
const liveMatch = await this.prisma.liveMatch.findUnique({
|
|
where: { id: matchId },
|
|
select: {
|
|
odds: true,
|
|
oddsUpdatedAt: true,
|
|
status: true,
|
|
state: true,
|
|
scoreHome: true,
|
|
scoreAway: true,
|
|
},
|
|
});
|
|
|
|
if (liveMatch?.odds) {
|
|
return {
|
|
source: "live_match",
|
|
odds: liveMatch.odds,
|
|
odds_updated_at: liveMatch.oddsUpdatedAt?.toISOString() ?? null,
|
|
status: liveMatch.status ?? null,
|
|
state: liveMatch.state ?? null,
|
|
score_home: liveMatch.scoreHome,
|
|
score_away: liveMatch.scoreAway,
|
|
};
|
|
}
|
|
|
|
const categories = await this.prisma.oddCategory.findMany({
|
|
where: { matchId },
|
|
select: {
|
|
name: true,
|
|
selections: {
|
|
select: {
|
|
name: true,
|
|
oddValue: true,
|
|
position: true,
|
|
updatedAt: true,
|
|
},
|
|
orderBy: { position: "asc" },
|
|
take: 12,
|
|
},
|
|
},
|
|
orderBy: { name: "asc" },
|
|
take: 24,
|
|
});
|
|
|
|
return {
|
|
source: "odd_selections",
|
|
category_count: categories.length,
|
|
categories: categories.map((category) => ({
|
|
name: category.name,
|
|
selections: category.selections.map((selection) => ({
|
|
name: selection.name,
|
|
odd_value: selection.oddValue,
|
|
position: selection.position,
|
|
updated_at: selection.updatedAt?.toISOString() ?? null,
|
|
})),
|
|
})),
|
|
};
|
|
}
|
|
|
|
private resolvePredictionRunSettlement(
|
|
row: PendingPredictionRunForSettlement,
|
|
): { outcome: string; unitProfit: number } | null {
|
|
const summary = this.asRecord(row.payloadSummary);
|
|
const mainPick = this.asRecord(summary.main_pick);
|
|
const market = String(mainPick.market || "");
|
|
const pick = String(mainPick.pick || "");
|
|
const playable = mainPick.playable === true;
|
|
const odds = Number(mainPick.odds || 0);
|
|
|
|
if (
|
|
!market ||
|
|
!pick ||
|
|
!playable ||
|
|
!Number.isFinite(odds) ||
|
|
odds <= 1.01
|
|
) {
|
|
return { outcome: "NO_BET", unitProfit: 0 };
|
|
}
|
|
|
|
const won = this.isPredictionPickWon({
|
|
market,
|
|
pick,
|
|
scoreHome: row.scoreHome,
|
|
scoreAway: row.scoreAway,
|
|
htScoreHome: row.htScoreHome,
|
|
htScoreAway: row.htScoreAway,
|
|
});
|
|
|
|
if (won === null) return null;
|
|
|
|
return {
|
|
outcome: `${won ? "WON" : "LOST"}:${market}:${pick}`,
|
|
unitProfit: Number((won ? odds - 1 : -1).toFixed(4)),
|
|
};
|
|
}
|
|
|
|
private isPredictionPickWon(input: {
|
|
market: string;
|
|
pick: string;
|
|
scoreHome: number | null;
|
|
scoreAway: number | null;
|
|
htScoreHome: number | null;
|
|
htScoreAway: number | null;
|
|
}): boolean | null {
|
|
const market = input.market.toUpperCase();
|
|
const pick = this.normalizePick(input.pick);
|
|
const scoreHome = input.scoreHome;
|
|
const scoreAway = input.scoreAway;
|
|
if (scoreHome === null || scoreAway === null) return null;
|
|
|
|
if (market === "MS") {
|
|
if (pick === "1") return scoreHome > scoreAway;
|
|
if (pick === "X" || pick === "0") return scoreHome === scoreAway;
|
|
if (pick === "2") return scoreAway > scoreHome;
|
|
return null;
|
|
}
|
|
|
|
if (market === "DC") {
|
|
const normalized = pick.replace("-", "");
|
|
if (normalized === "1X") return scoreHome >= scoreAway;
|
|
if (normalized === "X2") return scoreAway >= scoreHome;
|
|
if (normalized === "12") return scoreHome !== scoreAway;
|
|
return null;
|
|
}
|
|
|
|
if (market === "BTTS") {
|
|
const bothScored = scoreHome > 0 && scoreAway > 0;
|
|
if (pick.includes("VAR") || pick.includes("YES") || pick === "Y") {
|
|
return bothScored;
|
|
}
|
|
if (pick.includes("YOK") || pick.includes("NO") || pick === "N") {
|
|
return !bothScored;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
const goalLine = this.goalLineForMarket(market);
|
|
if (goalLine !== null) {
|
|
const total = market.startsWith("HT_")
|
|
? this.nullableSum(input.htScoreHome, input.htScoreAway)
|
|
: scoreHome + scoreAway;
|
|
if (total === null) return null;
|
|
if (this.isOverPick(pick)) return total > goalLine;
|
|
return total < goalLine;
|
|
}
|
|
|
|
if (market === "HT") {
|
|
const htHome = input.htScoreHome;
|
|
const htAway = input.htScoreAway;
|
|
if (htHome === null || htAway === null) return null;
|
|
if (pick === "1") return htHome > htAway;
|
|
if (pick === "X" || pick === "0") return htHome === htAway;
|
|
if (pick === "2") return htAway > htHome;
|
|
}
|
|
|
|
if (market === "HTFT") {
|
|
const htHome = input.htScoreHome;
|
|
const htAway = input.htScoreAway;
|
|
if (htHome === null || htAway === null || !pick.includes("/"))
|
|
return null;
|
|
const [htPick, ftPick] = pick.split("/");
|
|
return (
|
|
this.isResultPickWon(htPick, htHome, htAway) === true &&
|
|
this.isResultPickWon(ftPick, scoreHome, scoreAway) === true
|
|
);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private isResultPickWon(
|
|
pick: string,
|
|
homeScore: number,
|
|
awayScore: number,
|
|
): boolean | null {
|
|
if (pick === "1") return homeScore > awayScore;
|
|
if (pick === "X" || pick === "0") return homeScore === awayScore;
|
|
if (pick === "2") return awayScore > homeScore;
|
|
return null;
|
|
}
|
|
|
|
private goalLineForMarket(market: string): number | null {
|
|
if (market === "OU15") return 1.5;
|
|
if (market === "OU25") return 2.5;
|
|
if (market === "OU35") return 3.5;
|
|
if (market === "HT_OU05") return 0.5;
|
|
if (market === "HT_OU15") return 1.5;
|
|
return null;
|
|
}
|
|
|
|
private nullableSum(a: number | null, b: number | null): number | null {
|
|
if (a === null || b === null) return null;
|
|
return a + b;
|
|
}
|
|
|
|
private normalizePick(value: string): string {
|
|
return value
|
|
.trim()
|
|
.toUpperCase()
|
|
.replace(/İ/g, "I")
|
|
.replace(/Ü/g, "U")
|
|
.replace(/Ş/g, "S")
|
|
.replace(/Ğ/g, "G")
|
|
.replace(/Ö/g, "O")
|
|
.replace(/Ç/g, "C");
|
|
}
|
|
|
|
private isOverPick(pick: string): boolean {
|
|
return pick.includes("UST") || pick.includes("OVER");
|
|
}
|
|
|
|
private asRecord(value: unknown): Record<string, unknown> {
|
|
return value && typeof value === "object" && !Array.isArray(value)
|
|
? (value as Record<string, unknown>)
|
|
: {};
|
|
}
|
|
|
|
// Phase 3: Odds + referee + lineups + sidelined
|
|
|
|
private async fetchOddsForMatches(): Promise<void> {
|
|
this.logger.log("MONEY Fetching odds for live matches...");
|
|
|
|
try {
|
|
// Load both league filters (data-driven qualified leagues)
|
|
const topLeagueIds: string[] = [];
|
|
|
|
const footballLeagues = this.loadLeagueFilterSet(
|
|
"qualified_leagues.json",
|
|
);
|
|
if (footballLeagues) topLeagueIds.push(...footballLeagues);
|
|
|
|
const basketballLeagues = this.loadLeagueFilterSet(
|
|
"basketball_top_leagues.json",
|
|
);
|
|
if (basketballLeagues) topLeagueIds.push(...basketballLeagues);
|
|
|
|
const allowedLeagueIds = Array.from(new Set(topLeagueIds));
|
|
|
|
// Get matches needing odds (from 12 hours ago onward)
|
|
// CRITICAL: Only fetch odds/lineups for NOT STARTED matches.
|
|
// Once a match goes live, odds selections disappear or change,
|
|
// corrupting the pre-match data the AI model relies on.
|
|
const twelveHoursAgo = new Date(Date.now() - 12 * 60 * 60 * 1000);
|
|
|
|
const matchesToFetch = await this.prisma.liveMatch.findMany({
|
|
where: {
|
|
mstUtc: { gte: BigInt(twelveHoursAgo.getTime()) },
|
|
...(allowedLeagueIds.length > 0
|
|
? { leagueId: { in: allowedLeagueIds } }
|
|
: {}),
|
|
// Exclude live and finished matches — preserve their pre-match odds
|
|
NOT: {
|
|
OR: [
|
|
{ status: { in: FINISHED_STATUS_VALUES_FOR_DB } },
|
|
{ state: { in: FINISHED_STATE_VALUES_FOR_DB } },
|
|
{ status: { in: LIVE_STATUS_VALUES_FOR_DB } },
|
|
{ state: { in: LIVE_STATE_VALUES_FOR_DB } },
|
|
],
|
|
},
|
|
},
|
|
include: {
|
|
homeTeam: { select: { name: true } },
|
|
awayTeam: { select: { name: true } },
|
|
},
|
|
orderBy: [{ oddsUpdatedAt: "asc" }, { mstUtc: "asc" }],
|
|
take: 1000,
|
|
});
|
|
|
|
if (matchesToFetch.length === 0) {
|
|
this.logger.log("MONEY No matches to fetch odds for");
|
|
return;
|
|
}
|
|
|
|
this.logger.log(
|
|
`MONEY Fetching odds for ${matchesToFetch.length} matches`,
|
|
);
|
|
|
|
let successCount = 0;
|
|
let errorCount = 0;
|
|
const failedMatches: LiveMatchOddsTarget[] = [];
|
|
|
|
// Initial pass
|
|
for (const match of matchesToFetch) {
|
|
try {
|
|
await this.processMatchOdds(match);
|
|
successCount++;
|
|
await this.delay(500);
|
|
} catch (err: unknown) {
|
|
errorCount++;
|
|
const isRetryable = this.isRetryableError(err);
|
|
if (isRetryable) {
|
|
failedMatches.push(match);
|
|
} else {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
this.logger.warn(
|
|
`Match ${match.id} odds fetch failed (Non-retryable): ${message}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Retry failed matches (502/Timeout)
|
|
if (failedMatches.length > 0) {
|
|
this.logger.warn(
|
|
`Retrying ${failedMatches.length} failed matches (502/Timeout)...`,
|
|
);
|
|
|
|
for (const match of failedMatches) {
|
|
await this.delay(2000);
|
|
try {
|
|
await this.processMatchOdds(match);
|
|
successCount++;
|
|
this.logger.log(`SUCCESS Retry successful for match ${match.id}`);
|
|
} catch (retryErr: unknown) {
|
|
const message =
|
|
retryErr instanceof Error ? retryErr.message : String(retryErr);
|
|
this.logger.error(
|
|
`⌠Retry failed for match ${match.id}: ${message}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
this.logger.log(
|
|
`MONEY Odds complete: ${successCount} success, ${errorCount} errors (initially)`,
|
|
);
|
|
} catch (error: unknown) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
this.logger.error(`Odds fetch job failed: ${message}`);
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// Phase 4: Fill missing lineups (backup)
|
|
// ────────────────────────────────────────────────────────────
|
|
|
|
private async fillMissingLineups(): Promise<void> {
|
|
try {
|
|
const matchesToUpdate = await this.prisma.liveMatch.findMany({
|
|
where: {
|
|
sport: "football",
|
|
NOT: {
|
|
OR: [
|
|
{ status: { in: FINISHED_STATUS_VALUES_FOR_DB } },
|
|
{ state: { in: FINISHED_STATE_VALUES_FOR_DB } },
|
|
{
|
|
AND: [
|
|
{ scoreHome: { not: null } },
|
|
{ scoreAway: { not: null } },
|
|
{
|
|
NOT: {
|
|
OR: [
|
|
{ status: { in: LIVE_STATUS_VALUES_FOR_DB } },
|
|
{ state: { in: LIVE_STATE_VALUES_FOR_DB } },
|
|
],
|
|
},
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
select: { id: true, matchSlug: true, lineups: true, sport: true },
|
|
take: 30,
|
|
});
|
|
|
|
// Only football matches without lineups
|
|
const toUpdate = matchesToUpdate.filter(
|
|
(m) => !m.lineups && m.sport === "football",
|
|
);
|
|
|
|
if (toUpdate.length === 0) {
|
|
this.logger.debug("👕 All lineups already filled");
|
|
return;
|
|
}
|
|
|
|
this.logger.log(`👕 Filling lineups for ${toUpdate.length} matches...`);
|
|
|
|
for (const match of toUpdate) {
|
|
try {
|
|
const [formation, substitutions] = await Promise.all([
|
|
this.scraper.fetchStartingFormation(match.id),
|
|
this.scraper.fetchSubstitutions(match.id),
|
|
]);
|
|
const sidelined = match.matchSlug
|
|
? await this.scraper.fetchSidelinedPlayers(
|
|
match.id,
|
|
match.matchSlug,
|
|
)
|
|
: null;
|
|
|
|
// Normalize to same home.xi/away.xi format used by processMatchOdds
|
|
let normalizedLineups: Record<string, unknown> | null = null;
|
|
if (formation || substitutions) {
|
|
normalizedLineups = {
|
|
home: {
|
|
xi: formation?.stats?.home || [],
|
|
subs: substitutions?.stats?.home || [],
|
|
},
|
|
away: {
|
|
xi: formation?.stats?.away || [],
|
|
subs: substitutions?.stats?.away || [],
|
|
},
|
|
};
|
|
}
|
|
|
|
await this.prisma.liveMatch.update({
|
|
where: { id: match.id },
|
|
data: {
|
|
lineups: normalizedLineups
|
|
? JSON.parse(JSON.stringify(normalizedLineups))
|
|
: Prisma.JsonNull,
|
|
sidelined: sidelined
|
|
? JSON.parse(JSON.stringify(sidelined))
|
|
: Prisma.JsonNull,
|
|
updatedAt: new Date(),
|
|
},
|
|
});
|
|
|
|
this.logger.log(`👕 Lineups filled for match ${match.id}`);
|
|
await this.delay(500);
|
|
} catch (err: unknown) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
this.logger.warn(`Lineup fill failed for ${match.id}: ${message}`);
|
|
}
|
|
}
|
|
} catch (error: unknown) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
this.logger.error(`Lineup fill error: ${message}`);
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// Unified match fetcher — DRY for football + basketball
|
|
// ────────────────────────────────────────────────────────────
|
|
|
|
private async fetchMatchesForSport(
|
|
sport: SportType,
|
|
date: string,
|
|
topLeagueIds: Set<string>,
|
|
): Promise<void> {
|
|
const sportParam = sport === "basketball" ? "Basketball" : "Soccer";
|
|
const url = `https://www.mackolik.com/perform/p0/ajax/components/competition/livescores/json?sports[]=${sportParam}&matchDate=${date}`;
|
|
|
|
try {
|
|
const response = await firstValueFrom(
|
|
this.httpService.get(url, { timeout: 20000 }),
|
|
);
|
|
|
|
const payload = this.parseLiveScoresPayload(response.data);
|
|
if (!payload) {
|
|
this.logger.warn(`[${sport}] No valid data from API for ${date}`);
|
|
return;
|
|
}
|
|
|
|
const allMatches = payload.matches;
|
|
const competitions = payload.competitions;
|
|
|
|
// Apply league filter
|
|
const targetMatches =
|
|
topLeagueIds.size > 0
|
|
? allMatches.filter(
|
|
(m) =>
|
|
!!m.competitionId && topLeagueIds.has(String(m.competitionId)),
|
|
)
|
|
: allMatches;
|
|
|
|
if (targetMatches.length === 0) {
|
|
this.logger.log(`[${sport}] No matches found for ${date}`);
|
|
return;
|
|
}
|
|
|
|
this.logger.log(
|
|
`[${sport}] Processing ${targetMatches.length}/${allMatches.length} matches for ${date}`,
|
|
);
|
|
|
|
// Local caches to avoid N+1 redundant upserts
|
|
const processedCountries = new Set<string>();
|
|
const processedLeagues = new Set<string>();
|
|
const processedTeams = new Set<string>();
|
|
const integrityChecked = new Map<string, boolean>();
|
|
|
|
let upsertCount = 0;
|
|
let skippedCount = 0;
|
|
|
|
for (const match of targetMatches) {
|
|
try {
|
|
const homeTeamId = match.homeTeam.id;
|
|
const awayTeamId = match.awayTeam.id;
|
|
const leagueId = match.competitionId;
|
|
|
|
// Sport integrity check (cached)
|
|
const ensureIntegrity = async (
|
|
entityType: "league" | "team" | "liveMatch",
|
|
entityId: string | null,
|
|
): Promise<boolean> => {
|
|
if (!entityId) return true;
|
|
const cacheKey = `${sport}:${entityType}:${entityId}`;
|
|
if (!integrityChecked.has(cacheKey)) {
|
|
integrityChecked.set(
|
|
cacheKey,
|
|
await this.ensureLiveEntitySportIntegrity(
|
|
entityType,
|
|
entityId,
|
|
sport,
|
|
),
|
|
);
|
|
}
|
|
return integrityChecked.get(cacheKey) === true;
|
|
};
|
|
|
|
if (
|
|
!(await ensureIntegrity("liveMatch", String(match.id))) ||
|
|
!(await ensureIntegrity("league", leagueId)) ||
|
|
!(await ensureIntegrity("team", homeTeamId)) ||
|
|
!(await ensureIntegrity("team", awayTeamId))
|
|
) {
|
|
skippedCount++;
|
|
continue;
|
|
}
|
|
|
|
// Resolve competition details
|
|
const compInfo = this.resolveCompetition(competitions, leagueId);
|
|
const countryId = compInfo?.country?.id || null;
|
|
|
|
// 1. Upsert Country (cached)
|
|
if (
|
|
countryId &&
|
|
compInfo?.country?.name &&
|
|
!processedCountries.has(countryId)
|
|
) {
|
|
await this.prisma.country
|
|
.upsert({
|
|
where: { id: countryId },
|
|
update: { name: compInfo.country.name },
|
|
create: { id: countryId, name: compInfo.country.name },
|
|
})
|
|
.catch((e: Error) =>
|
|
this.logger.warn(
|
|
`[${sport}] Country upsert failed: ${e.message}`,
|
|
),
|
|
);
|
|
processedCountries.add(countryId);
|
|
}
|
|
|
|
// 2. Upsert League (cached)
|
|
if (
|
|
leagueId &&
|
|
compInfo?.name &&
|
|
!processedLeagues.has(String(leagueId))
|
|
) {
|
|
await this.prisma.league
|
|
.upsert({
|
|
where: { id: leagueId },
|
|
update: {
|
|
name: compInfo.name,
|
|
countryId: countryId,
|
|
sport: sport,
|
|
},
|
|
create: {
|
|
id: leagueId,
|
|
name: compInfo.name,
|
|
countryId: countryId,
|
|
sport: sport,
|
|
competitionSlug: compInfo.slug || null,
|
|
},
|
|
})
|
|
.catch((e: Error) =>
|
|
this.logger.warn(
|
|
`[${sport}] League upsert failed: ${e.message}`,
|
|
),
|
|
);
|
|
processedLeagues.add(String(leagueId));
|
|
}
|
|
|
|
// 3. Upsert Home Team (cached)
|
|
if (
|
|
homeTeamId &&
|
|
match.homeTeam?.name &&
|
|
!processedTeams.has(homeTeamId)
|
|
) {
|
|
await this.prisma.team
|
|
.upsert({
|
|
where: { id: homeTeamId },
|
|
update: {
|
|
name: match.homeTeam.name,
|
|
slug: match.homeTeam.slug || null,
|
|
sport: sport,
|
|
},
|
|
create: {
|
|
id: homeTeamId,
|
|
name: match.homeTeam.name,
|
|
slug: match.homeTeam.slug || null,
|
|
sport: sport,
|
|
},
|
|
})
|
|
.catch((e: Error) =>
|
|
this.logger.warn(
|
|
`[${sport}] Home team upsert failed: ${e.message}`,
|
|
),
|
|
);
|
|
processedTeams.add(homeTeamId);
|
|
}
|
|
|
|
// 4. Upsert Away Team (cached)
|
|
if (
|
|
awayTeamId &&
|
|
match.awayTeam?.name &&
|
|
!processedTeams.has(awayTeamId)
|
|
) {
|
|
await this.prisma.team
|
|
.upsert({
|
|
where: { id: awayTeamId },
|
|
update: {
|
|
name: match.awayTeam.name,
|
|
slug: match.awayTeam.slug || null,
|
|
sport: sport,
|
|
},
|
|
create: {
|
|
id: awayTeamId,
|
|
name: match.awayTeam.name,
|
|
slug: match.awayTeam.slug || null,
|
|
sport: sport,
|
|
},
|
|
})
|
|
.catch((e: Error) =>
|
|
this.logger.warn(
|
|
`[${sport}] Away team upsert failed: ${e.message}`,
|
|
),
|
|
);
|
|
processedTeams.add(awayTeamId);
|
|
}
|
|
|
|
// Safe score parsing
|
|
const sHome = this.asInt(match.homeScore ?? match.score?.home);
|
|
const sAway = this.asInt(match.awayScore ?? match.score?.away);
|
|
const sHtHome = this.asInt(match.score?.ht?.home);
|
|
const sHtAway = this.asInt(match.score?.ht?.away);
|
|
const storedStatus = deriveStoredMatchStatus({
|
|
state: match.state,
|
|
status: match.status,
|
|
substate: match.substate,
|
|
statusBoxContent: match.statusBoxContent,
|
|
scoreHome: sHome,
|
|
scoreAway: sAway,
|
|
score: match.score,
|
|
});
|
|
|
|
// Handle postponed matches (ERT = Erteledendi)
|
|
if (match.statusBoxContent === "ERT") {
|
|
await this.prisma.liveMatch
|
|
.updateMany({
|
|
where: { id: String(match.id) },
|
|
data: {
|
|
status: "POSTPONED",
|
|
state: "postponed",
|
|
substate: "postponed",
|
|
updatedAt: new Date(),
|
|
},
|
|
})
|
|
.catch(() => {});
|
|
this.logger.debug(
|
|
`[${sport}] Marked as POSTPONED: ${match.matchName}`,
|
|
);
|
|
skippedCount++;
|
|
continue;
|
|
}
|
|
|
|
// 5. Upsert LiveMatch
|
|
await this.prisma.liveMatch.upsert({
|
|
where: { id: String(match.id) },
|
|
update: {
|
|
leagueId: leagueId,
|
|
state: match.state || null,
|
|
substate: match.substate || null,
|
|
status: storedStatus,
|
|
scoreHome: sHome,
|
|
scoreAway: sAway,
|
|
htScoreHome: sHtHome,
|
|
htScoreAway: sHtAway,
|
|
homeTeamId: homeTeamId,
|
|
awayTeamId: awayTeamId,
|
|
updatedAt: new Date(),
|
|
},
|
|
create: {
|
|
id: String(match.id),
|
|
matchName: match.matchName,
|
|
matchSlug: match.matchSlug,
|
|
sport: sport,
|
|
leagueId: leagueId,
|
|
state: match.state || null,
|
|
substate: match.substate || null,
|
|
status: storedStatus,
|
|
mstUtc: BigInt(match.mstUtc || Date.now()),
|
|
scoreHome: sHome,
|
|
scoreAway: sAway,
|
|
htScoreHome: sHtHome,
|
|
htScoreAway: sHtAway,
|
|
homeTeamId: homeTeamId,
|
|
awayTeamId: awayTeamId,
|
|
},
|
|
});
|
|
upsertCount++;
|
|
|
|
// Progress logging every 100 matches
|
|
if (
|
|
(upsertCount + skippedCount) % 100 === 0 ||
|
|
upsertCount + skippedCount === targetMatches.length
|
|
) {
|
|
this.logger.log(
|
|
`[${sport}] â³ Progress: ${upsertCount + skippedCount}/${targetMatches.length} (Saved: ${upsertCount}, Skipped: ${skippedCount})`,
|
|
);
|
|
}
|
|
} catch (err: unknown) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
this.logger.warn(`[${sport}] Match ${match.id} failed: ${message}`);
|
|
}
|
|
}
|
|
|
|
this.logger.log(
|
|
`[${sport}] [${date}] Done: ${upsertCount} saved, ${skippedCount} skipped`,
|
|
);
|
|
} catch (error: unknown) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
this.logger.error(`[${sport}] Fetch failed for ${date}: ${message}`);
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// processMatchOdds — odds + referee + lineups + sidelined
|
|
// (Preserved from original — no logic changes)
|
|
// ────────────────────────────────────────────────────────────
|
|
|
|
private async processMatchOdds(match: LiveMatchOddsTarget): Promise<void> {
|
|
const matchSlug = match.matchSlug || "match";
|
|
const sport = String(match.sport || "football").toLowerCase();
|
|
const sportPath = sport === "basketball" ? "basketbol/mac" : "mac";
|
|
const httpHeaders = {
|
|
"User-Agent":
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
};
|
|
|
|
let odds: Record<string, Record<string, number>> = {};
|
|
let refereeName: string | null = null;
|
|
let lineups: LiveLineupsJson | null = null;
|
|
let sidelined: SidelinedResponse | null = null;
|
|
|
|
// 1. Fetch Odds from Ä°ddaa page
|
|
const oddsUrl = `https://www.mackolik.com/${sportPath}/${matchSlug}/iddaa/${match.id}`;
|
|
try {
|
|
const response = await firstValueFrom(
|
|
this.httpService.get(oddsUrl, {
|
|
timeout: 10000,
|
|
headers: httpHeaders,
|
|
maxRedirects: 5,
|
|
}),
|
|
);
|
|
odds = this.extractOddsFromHtml(
|
|
typeof response.data === "string" ? response.data : "",
|
|
);
|
|
} catch (e: unknown) {
|
|
if (this.isRetryableError(e)) throw e;
|
|
const message = e instanceof Error ? e.message : String(e);
|
|
this.logger.warn(`Odds fetch failed for ${match.id}: ${message}`);
|
|
}
|
|
|
|
if (sport === "football") {
|
|
// 2. Fetch Referee from main match page
|
|
if (!refereeName) {
|
|
const mainUrl = `https://www.mackolik.com/mac/${matchSlug}/${match.id}`;
|
|
try {
|
|
const mainResp = await firstValueFrom(
|
|
this.httpService.get(mainUrl, {
|
|
timeout: 10000,
|
|
headers: httpHeaders,
|
|
maxRedirects: 5,
|
|
}),
|
|
);
|
|
refereeName = this.extractRefereeFromHtml(
|
|
typeof mainResp.data === "string" ? mainResp.data : "",
|
|
);
|
|
} catch {
|
|
// Non-critical — referee is optional
|
|
}
|
|
}
|
|
|
|
// 3. Fetch Lineups & Sidelined Players
|
|
const now = Date.now();
|
|
const matchTime = Number(match.mstUtc);
|
|
const diffHours = (matchTime - now) / (1000 * 60 * 60);
|
|
|
|
// Fetch if between -3 hours (started) and +24 hours (upcoming)
|
|
if (diffHours < 24 && diffHours > -3) {
|
|
// Lineups
|
|
try {
|
|
const [startingFormation, substitutions] = await Promise.all([
|
|
this.scraper.fetchStartingFormation(match.id),
|
|
this.scraper.fetchSubstitutions(match.id),
|
|
]);
|
|
|
|
if (startingFormation || substitutions) {
|
|
lineups = {
|
|
home: {
|
|
xi: startingFormation?.stats?.home || [],
|
|
subs: substitutions?.stats?.home || [],
|
|
},
|
|
away: {
|
|
xi: startingFormation?.stats?.away || [],
|
|
subs: substitutions?.stats?.away || [],
|
|
},
|
|
};
|
|
this.logger.log(`👥 Lineups found for ${match.matchName}`);
|
|
} else {
|
|
this.logger.debug(`No lineups (yet) for ${match.matchName}`);
|
|
}
|
|
} catch (err: unknown) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
this.logger.warn(`Lineup fetch failed for ${match.id}: ${message}`);
|
|
}
|
|
|
|
// Sidelined Players (Injuries/Suspensions)
|
|
try {
|
|
sidelined = await this.scraper.fetchSidelinedPlayers(
|
|
match.id,
|
|
matchSlug,
|
|
);
|
|
if (sidelined) {
|
|
if (sidelined.homeTeam) {
|
|
sidelined.homeTeam.teamName = match.homeTeam?.name || "";
|
|
}
|
|
if (sidelined.awayTeam) {
|
|
sidelined.awayTeam.teamName = match.awayTeam?.name || "";
|
|
}
|
|
|
|
if (
|
|
sidelined.homeTeam?.totalSidelined > 0 ||
|
|
sidelined.awayTeam?.totalSidelined > 0
|
|
) {
|
|
this.logger.log(
|
|
`🚑 Sidelined: ${sidelined.homeTeam.totalSidelined}(H) - ${sidelined.awayTeam.totalSidelined}(A) for ${match.matchName}`,
|
|
);
|
|
}
|
|
}
|
|
} catch (err: unknown) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
this.logger.warn(
|
|
`Sidelined fetch failed for ${match.id}: ${message}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Guard: If match already has pre-match odds and is now live/finished,
|
|
// do NOT overwrite odds/lineups/sidelined — the model needs stable pre-match data.
|
|
const matchState = match.state?.toLowerCase() ?? "";
|
|
const matchStatus = match.status?.toLowerCase() ?? "";
|
|
const liveStates = LIVE_STATE_VALUES_FOR_DB.map((s) => s.toLowerCase());
|
|
const liveStatuses = LIVE_STATUS_VALUES_FOR_DB.map((s) => s.toLowerCase());
|
|
const finishedStates = FINISHED_STATE_VALUES_FOR_DB.map((s) =>
|
|
s.toLowerCase(),
|
|
);
|
|
const finishedStatuses = FINISHED_STATUS_VALUES_FOR_DB.map((s) =>
|
|
s.toLowerCase(),
|
|
);
|
|
|
|
const isLiveOrFinished =
|
|
liveStates.includes(matchState) ||
|
|
liveStatuses.includes(matchStatus) ||
|
|
finishedStates.includes(matchState) ||
|
|
finishedStatuses.includes(matchStatus);
|
|
|
|
const existingOdds = match.odds as Record<string, unknown> | null;
|
|
const hasExistingOdds =
|
|
!!existingOdds &&
|
|
typeof existingOdds === "object" &&
|
|
Object.keys(existingOdds).length > 0;
|
|
|
|
if (isLiveOrFinished && hasExistingOdds) {
|
|
// Match is live/finished and already has pre-match odds — skip data update
|
|
this.logger.debug(
|
|
`🛡️ Preserving pre-match data for ${match.matchName} (status: ${matchStatus}, state: ${matchState})`,
|
|
);
|
|
await this.prisma.liveMatch.update({
|
|
where: { id: match.id },
|
|
data: { oddsUpdatedAt: new Date() },
|
|
});
|
|
return;
|
|
}
|
|
|
|
// Update odds/lineups/sidelined for pre-match (NS) matches
|
|
await this.prisma.liveMatch.update({
|
|
where: { id: match.id },
|
|
data: {
|
|
odds: Object.keys(odds).length > 0 ? odds : undefined,
|
|
oddsUpdatedAt: new Date(),
|
|
refereeName: refereeName ?? undefined,
|
|
lineups: (lineups as unknown as Prisma.InputJsonValue) ?? undefined,
|
|
sidelined: (sidelined as unknown as Prisma.InputJsonValue) ?? undefined,
|
|
},
|
|
});
|
|
|
|
if (
|
|
Object.keys(odds).length > 0 ||
|
|
refereeName ||
|
|
lineups ||
|
|
(sidelined &&
|
|
(sidelined.homeTeam.totalSidelined > 0 ||
|
|
sidelined.awayTeam.totalSidelined > 0))
|
|
) {
|
|
this.logger.log(
|
|
`SUCCESS Loop update: ${match.matchName} | Odds: ${Object.keys(odds).length} | Ref: ${refereeName || "N/A"} | Lineups: ${lineups ? "Yes" : "No"} | Sidelined: ${sidelined ? "Yes" : "No"}`,
|
|
);
|
|
} else {
|
|
this.logger.debug(
|
|
`â• No detailed data for ${match.matchName}, marked check.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// HTML Extraction Helpers (preserved — no logic changes)
|
|
// ────────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Extract odds from Mackolik HTML page
|
|
* Returns structured odds object: { "MS": {"1": 2.10, "X": 3.40}, "AU25": {"Alt": 2.05, "Ãœst": 1.75} }
|
|
*/
|
|
private extractOddsFromHtml(
|
|
html: string,
|
|
): Record<string, Record<string, number>> {
|
|
const odds: Record<string, Record<string, number>> = {};
|
|
if (!html) return odds;
|
|
|
|
try {
|
|
const settingsPattern = /data-settings="([^"]+)"/g;
|
|
let match;
|
|
|
|
while ((match = settingsPattern.exec(html)) !== null) {
|
|
try {
|
|
const decoded = match[1]
|
|
.replace(/"/g, '"')
|
|
.replace(/&/g, "&")
|
|
.replace(/</g, "<")
|
|
.replace(/>/g, ">");
|
|
|
|
const parsed = JSON.parse(decoded) as unknown;
|
|
if (!this.isRecord(parsed)) continue;
|
|
|
|
const iddaaEventId = this.isRecord(parsed.iddaaEventId)
|
|
? parsed.iddaaEventId
|
|
: null;
|
|
const marketCollection =
|
|
iddaaEventId && this.isRecord(iddaaEventId.marketCollection)
|
|
? iddaaEventId.marketCollection
|
|
: null;
|
|
|
|
if (marketCollection) {
|
|
for (const marketValue of Object.values(marketCollection)) {
|
|
if (!this.isRecord(marketValue)) continue;
|
|
const marketName = this.asString(marketValue.name)?.trim();
|
|
if (!marketName) continue;
|
|
|
|
// First-come-first-served: Skip if already populated
|
|
if (
|
|
odds[marketName] &&
|
|
Object.keys(odds[marketName]).length > 0
|
|
) {
|
|
continue;
|
|
}
|
|
|
|
odds[marketName] = {};
|
|
|
|
const selectionCollection = this.isRecord(
|
|
marketValue.selectionCollection,
|
|
)
|
|
? marketValue.selectionCollection
|
|
: {};
|
|
for (const selectionValue of Object.values(selectionCollection)) {
|
|
if (!this.isRecord(selectionValue)) continue;
|
|
|
|
const selName =
|
|
this.asString(selectionValue.name) ||
|
|
this.asString(selectionValue.outcome);
|
|
const selOdd = Number.parseFloat(
|
|
this.asString(selectionValue.odd) || "",
|
|
);
|
|
if (selName && !isNaN(selOdd)) {
|
|
odds[marketName][selName] = selOdd;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch {
|
|
// JSON parse error, skip
|
|
}
|
|
}
|
|
} catch {
|
|
this.logger.warn("Failed to extract odds from HTML");
|
|
}
|
|
|
|
return odds;
|
|
}
|
|
|
|
/**
|
|
* Normalize odds category names to short codes
|
|
*/
|
|
private normalizeOddsCategory(name: string): string | null {
|
|
if (!name) return null;
|
|
const lower = name.toLowerCase();
|
|
|
|
// Specific & Compound names FIRST
|
|
if (lower.includes("ilk yarı/maç sonucu")) return "HTFT";
|
|
if (lower.includes("1. yarı sonucu")) return "HT";
|
|
if (lower.includes("çifte şans")) return "CS";
|
|
|
|
// General names LATER
|
|
if (lower.includes("maç sonucu") && !lower.includes("handikap"))
|
|
return "MS";
|
|
if (lower.includes("karşılıklı gol")) return "KG";
|
|
if (lower.includes("2,5 alt/üst") || lower.includes("2.5")) return "AU25";
|
|
if (lower.includes("1,5 alt/üst") || lower.includes("1.5")) return "AU15";
|
|
if (lower.includes("3,5 alt/üst") || lower.includes("3.5")) return "AU35";
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Extract referee name from match page HTML
|
|
*/
|
|
private extractRefereeFromHtml(html: string): string | null {
|
|
try {
|
|
// Strategy 1: Mackolik officials section — head referee in '--main' list item
|
|
const mainOfficialPattern =
|
|
/official-list-item--main[^>]*>\s*(?:<[^>]*>\s*)*?<span[^>]*official-name[^>]*>\s*([^<]+)/i;
|
|
const mainMatch = mainOfficialPattern.exec(html);
|
|
if (mainMatch?.[1]) {
|
|
const name = mainMatch[1].trim();
|
|
if (name.length > 2 && name.length < 100) return name;
|
|
}
|
|
|
|
// Strategy 2: Any official-name followed by "Orta Hakem"
|
|
const ortaHakemPattern =
|
|
/official-name[^>]*>\s*([^<]+)<[\s\S]*?Orta\s*Hakem/i;
|
|
const ortaMatch = ortaHakemPattern.exec(html);
|
|
if (ortaMatch?.[1]) {
|
|
const name = ortaMatch[1].trim();
|
|
if (name.length > 2 && name.length < 100) return name;
|
|
}
|
|
|
|
// Strategy 3: Generic fallback patterns
|
|
const fallbackPatterns = [
|
|
/Hakem:\s*([^<]+)/i,
|
|
/"refereeName":"([^"]+)"/i,
|
|
];
|
|
for (const pattern of fallbackPatterns) {
|
|
const result = pattern.exec(html);
|
|
if (result?.[1]) {
|
|
const name = result[1].trim();
|
|
if (name.length > 2 && name.length < 100) return name;
|
|
}
|
|
}
|
|
} catch {
|
|
// Ignore extraction errors
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// ────────────────────────────────────────────────────────────
|
|
// Low-level Helpers (preserved — no logic changes)
|
|
// ────────────────────────────────────────────────────────────
|
|
|
|
private shouldSkipInHistoricalMode(jobName: string): boolean {
|
|
if (process.env.FEEDER_MODE === "historical") {
|
|
this.logger.debug(`Skipping ${jobName} in historical feeder mode`);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private isRecord(value: unknown): value is Record<string, unknown> {
|
|
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
}
|
|
|
|
private asString(value: unknown): string | null {
|
|
if (typeof value === "string") return value;
|
|
if (typeof value === "number" && Number.isFinite(value))
|
|
return String(value);
|
|
return null;
|
|
}
|
|
|
|
private asInt(value: unknown): number | null {
|
|
if (typeof value === "number" && Number.isFinite(value)) {
|
|
return Math.trunc(value);
|
|
}
|
|
if (typeof value === "string" && value.trim().length > 0) {
|
|
const parsed = parseInt(value, 10);
|
|
return Number.isNaN(parsed) ? null : parsed;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private parseTeam(value: unknown): LiveScoreTeamPayload | null {
|
|
if (!this.isRecord(value)) return null;
|
|
|
|
const id = this.asString(value.id);
|
|
if (!id) return null;
|
|
|
|
return {
|
|
id,
|
|
name: this.asString(value.name) || "Unknown",
|
|
slug: this.asString(value.slug),
|
|
};
|
|
}
|
|
|
|
private parseCompetition(value: unknown): LiveScoreCompetitionPayload | null {
|
|
if (!this.isRecord(value)) return null;
|
|
|
|
const id = this.asString(value.id);
|
|
const name = this.asString(value.name);
|
|
if (!id || !name) return null;
|
|
|
|
const rawCountry = this.isRecord(value.country) ? value.country : null;
|
|
const countryId = rawCountry ? this.asString(rawCountry.id) : null;
|
|
const countryName = rawCountry ? this.asString(rawCountry.name) : null;
|
|
|
|
return {
|
|
id,
|
|
name,
|
|
slug: this.asString(value.slug),
|
|
country:
|
|
countryId && countryName ? { id: countryId, name: countryName } : null,
|
|
};
|
|
}
|
|
|
|
private parseLiveScoreMatch(value: unknown): LiveScorePayloadMatch | null {
|
|
if (!this.isRecord(value)) return null;
|
|
|
|
const id = this.asString(value.id);
|
|
const homeTeam = this.parseTeam(value.homeTeam);
|
|
const awayTeam = this.parseTeam(value.awayTeam);
|
|
if (!id || !homeTeam || !awayTeam) return null;
|
|
|
|
const score = this.isRecord(value.score)
|
|
? {
|
|
home: this.asInt(value.score.home),
|
|
away: this.asInt(value.score.away),
|
|
}
|
|
: null;
|
|
|
|
return {
|
|
id,
|
|
matchName:
|
|
this.asString(value.matchName) || `${homeTeam.name} - ${awayTeam.name}`,
|
|
matchSlug: this.asString(value.matchSlug) || "",
|
|
competitionId: this.asString(value.competitionId),
|
|
mstUtc: this.asInt(value.mstUtc),
|
|
state: this.asString(value.state),
|
|
substate: this.asString(value.substate),
|
|
status: this.asString(value.status),
|
|
statusBoxContent: this.asString(value.statusBoxContent),
|
|
iddaaCode: this.asString(value.iddaaCode),
|
|
homeTeam,
|
|
awayTeam,
|
|
homeScore: this.asInt(value.homeScore),
|
|
awayScore: this.asInt(value.awayScore),
|
|
score,
|
|
};
|
|
}
|
|
|
|
private parseLiveScoresPayload(raw: unknown): {
|
|
matches: LiveScorePayloadMatch[];
|
|
competitions: Record<string, LiveScoreCompetitionPayload>;
|
|
} | null {
|
|
if (!this.isRecord(raw)) return null;
|
|
if (this.asString(raw.status) !== "success") return null;
|
|
|
|
const data = this.isRecord(raw.data) ? raw.data : null;
|
|
if (!data) return null;
|
|
|
|
const rawMatches = this.isRecord(data.matches) ? data.matches : {};
|
|
const rawCompetitions = this.isRecord(data.competitions)
|
|
? data.competitions
|
|
: {};
|
|
|
|
const matches: LiveScorePayloadMatch[] = [];
|
|
for (const rawMatch of Object.values(rawMatches)) {
|
|
const parsed = this.parseLiveScoreMatch(rawMatch);
|
|
if (parsed) matches.push(parsed);
|
|
}
|
|
|
|
const competitions: Record<string, LiveScoreCompetitionPayload> = {};
|
|
for (const [key, rawCompetition] of Object.entries(rawCompetitions)) {
|
|
const parsed = this.parseCompetition(rawCompetition);
|
|
if (parsed) {
|
|
competitions[key] = parsed;
|
|
}
|
|
}
|
|
|
|
return { matches, competitions };
|
|
}
|
|
|
|
private loadLeagueFilterSet(fileName: string): Set<string> | null {
|
|
try {
|
|
const filePath = path.join(process.cwd(), fileName);
|
|
const raw = JSON.parse(fs.readFileSync(filePath, "utf-8")) as unknown;
|
|
if (!Array.isArray(raw)) {
|
|
this.logger.error(`${fileName} is not a JSON array`);
|
|
return null;
|
|
}
|
|
|
|
const ids = raw
|
|
.map((value) => this.asString(value))
|
|
.filter((value): value is string => !!value);
|
|
return new Set(ids);
|
|
} catch (e: unknown) {
|
|
const message = e instanceof Error ? e.message : String(e);
|
|
this.logger.error(`Failed to load ${fileName}: ${message}`);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private resolveCompetition(
|
|
competitions: Record<string, LiveScoreCompetitionPayload>,
|
|
leagueId: string | null,
|
|
): LiveScoreCompetitionPayload | null {
|
|
if (!leagueId) return null;
|
|
if (competitions[leagueId]) return competitions[leagueId];
|
|
for (const comp of Object.values(competitions)) {
|
|
if (comp.id === leagueId) return comp;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private async ensureLiveEntitySportIntegrity(
|
|
entityType: "league" | "team" | "liveMatch",
|
|
id: string,
|
|
expectedSport: SportType,
|
|
): Promise<boolean> {
|
|
if (!id) return true;
|
|
|
|
let existingSport: string | null | undefined;
|
|
|
|
if (entityType === "league") {
|
|
const existing = await this.prisma.league.findUnique({
|
|
where: { id },
|
|
select: { sport: true },
|
|
});
|
|
existingSport = existing?.sport;
|
|
} else if (entityType === "team") {
|
|
const existing = await this.prisma.team.findUnique({
|
|
where: { id },
|
|
select: { sport: true },
|
|
});
|
|
existingSport = existing?.sport;
|
|
} else {
|
|
const existing = await this.prisma.liveMatch.findUnique({
|
|
where: { id },
|
|
select: { sport: true },
|
|
});
|
|
existingSport = existing?.sport;
|
|
}
|
|
|
|
if (existingSport && existingSport !== expectedSport) {
|
|
this.logger.error(
|
|
`Sport integrity violation on ${entityType}:${id}. Existing=${existingSport}, incoming=${expectedSport}. Skipping write to prevent cross-sport overwrite.`,
|
|
);
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
private isRetryableError(err: unknown): boolean {
|
|
if (!err || typeof err !== "object") return false;
|
|
const errorObj = err as Record<string, unknown>;
|
|
const response = errorObj.response as Record<string, unknown> | undefined;
|
|
return (
|
|
response?.status === 502 ||
|
|
errorObj.code === "ECONNABORTED" ||
|
|
errorObj.code === "ETIMEDOUT"
|
|
);
|
|
}
|
|
|
|
private delay(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
}
|