fahricansecer/iddaai-be
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cr

Browse Source
This commit is contained in:
Fahri Can
2026-04-16 17:21:48 +03:00
parent c8fa4c442d
commit c8e7e4e927
116 changed files with 3720 additions and 4197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/modules/feeder/feeder.service.ts
+100 -102
View File
@@ -3,10 +3,10 @@
* Main orchestration service for historical data scanning
*/
import { Injectable, Logger } from '@nestjs/common';
import { FeederScraperService } from './feeder-scraper.service';
import { FeederTransformerService } from './feeder-transformer.service';
import { FeederPersistenceService } from './feeder-persistence.service';
import { Injectable, Logger } from "@nestjs/common";
import { FeederScraperService } from "./feeder-scraper.service";
import { FeederTransformerService } from "./feeder-transformer.service";
import { FeederPersistenceService } from "./feeder-persistence.service";
import {
Sport,
MatchSummary,
@@ -23,7 +23,7 @@ import {
ParsedMarket,
DbEventPayload,
DbMarketPayload,
} from './feeder.types';
} from "./feeder.types";
interface ProcessDateOptions {
onlyCompletedMatches?: boolean;
@@ -37,10 +37,10 @@ export class FeederService {
// Configuration - Adjust these based on rate limiting behavior
private readonly CONCURRENCY_LIMIT = 20; // Increased for maximum speed on EC2
private readonly REQUEST_DELAY_MS = 50; // Minimal delay to respect basics
private readonly HISTORICAL_START_DATE = '2023-06-01'; // 2 years of data
private readonly SPORTS: Sport[] = ['football', 'basketball'];
private readonly HISTORICAL_START_DATE = "2023-06-01"; // 2 years of data
private readonly SPORTS: Sport[] = ["football", "basketball"];
private readonly MAX_RETRIES = 50;
private readonly DAILY_SYNC_TIME_ZONE = 'Europe/Istanbul';
private readonly DAILY_SYNC_TIME_ZONE = "Europe/Istanbul";
constructor(
private readonly scraperService: FeederScraperService,
@@ -56,38 +56,38 @@ export class FeederService {
}
private getYesterdayDateString(timeZone: string): string {
const formatter = new Intl.DateTimeFormat('en-CA', {
const formatter = new Intl.DateTimeFormat("en-CA", {
timeZone,
year: 'numeric',
month: '2-digit',
day: '2-digit',
year: "numeric",
month: "2-digit",
day: "2-digit",
});
const parts = formatter.formatToParts(new Date());
const year = Number(parts.find((part) => part.type === 'year')?.value);
const month = Number(parts.find((part) => part.type === 'month')?.value);
const day = Number(parts.find((part) => part.type === 'day')?.value);
const year = Number(parts.find((part) => part.type === "year")?.value);
const month = Number(parts.find((part) => part.type === "month")?.value);
const day = Number(parts.find((part) => part.type === "day")?.value);
const tzMidnightUtc = new Date(Date.UTC(year, month - 1, day));
tzMidnightUtc.setUTCDate(tzMidnightUtc.getUTCDate() - 1);
return tzMidnightUtc.toISOString().split('T')[0];
return tzMidnightUtc.toISOString().split("T")[0];
}
private getTimeZoneOffsetMs(date: Date, timeZone: string): number {
const formatter = new Intl.DateTimeFormat('en-US', {
const formatter = new Intl.DateTimeFormat("en-US", {
timeZone,
timeZoneName: 'shortOffset',
timeZoneName: "shortOffset",
});
const offsetLabel =
formatter.formatToParts(date).find((part) => part.type === 'timeZoneName')
?.value || 'GMT+0';
formatter.formatToParts(date).find((part) => part.type === "timeZoneName")
?.value || "GMT+0";
const match = offsetLabel.match(/GMT([+-])(\d{1,2})(?::?(\d{2}))?/);
if (!match) return 0;
const sign = match[1] === '-' ? -1 : 1;
const hours = Number(match[2] || '0');
const minutes = Number(match[3] || '0');
const sign = match[1] === "-" ? -1 : 1;
const hours = Number(match[2] || "0");
const minutes = Number(match[3] || "0");
return sign * (hours * 60 + minutes) * 60 * 1000;
}
@@ -96,17 +96,14 @@ export class FeederService {
dateString: string,
timeZone: string,
): { startTs: number; endTs: number } {
const [year, month, day] = dateString.split('-').map(Number);
const [year, month, day] = dateString.split("-").map(Number);
const startGuess = new Date(Date.UTC(year, month - 1, day, 0, 0, 0));
const nextDayGuess = new Date(
Date.UTC(year, month - 1, day + 1, 0, 0, 0),
);
const nextDayGuess = new Date(Date.UTC(year, month - 1, day + 1, 0, 0, 0));
const startOffsetMs = this.getTimeZoneOffsetMs(startGuess, timeZone);
const nextDayOffsetMs = this.getTimeZoneOffsetMs(nextDayGuess, timeZone);
const startMs =
Date.UTC(year, month - 1, day, 0, 0, 0) - startOffsetMs;
const startMs = Date.UTC(year, month - 1, day, 0, 0, 0) - startOffsetMs;
const nextDayStartMs =
Date.UTC(year, month - 1, day + 1, 0, 0, 0) - nextDayOffsetMs;
@@ -117,35 +114,39 @@ export class FeederService {
}
private parseScoreValue(value: unknown): number | null {
if (value === null || value === undefined || value === '') return null;
if (value === null || value === undefined || value === "") return null;
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
private isCompletedMatchSummary(match: MatchSummary): boolean {
if (match.statusBoxContent === 'ERT') return false;
if (match.statusBoxContent === "ERT") return false;
const normalizedState = String(match.state || '')
const normalizedState = String(match.state || "")
.trim()
.toLowerCase();
const normalizedStatus = String(match.status || '')
const normalizedStatus = String(match.status || "")
.trim()
.toLowerCase();
const normalizedSubstate = String(match.substate || '')
const normalizedSubstate = String(match.substate || "")
.trim()
.toLowerCase();
if (['postgame', 'post'].includes(normalizedState)) return true;
if (["postgame", "post"].includes(normalizedState)) return true;
if (
['played', 'finished', 'ft', 'afterpenalties', 'penalties'].includes(
["played", "finished", "ft", "afterpenalties", "penalties"].includes(
normalizedStatus,
)
) {
return true;
}
if (['postgame', 'post', 'played', 'finished', 'ft'].includes(normalizedSubstate)) {
if (
["postgame", "post", "played", "finished", "ft"].includes(
normalizedSubstate,
)
) {
return true;
}
@@ -167,7 +168,7 @@ export class FeederService {
targetLeagueIds: string[] = [],
): Promise<void> {
this.logger.log(
`🗓️ STARTING DAILY COMPLETED MATCH SYNC [Date: ${targetDateStr}] [Sports: ${sports.join(', ')}] ${targetLeagueIds.length > 0 ? `[Filter: ${targetLeagueIds.length} leagues]` : ''}`,
`🗓️ STARTING DAILY COMPLETED MATCH SYNC [Date: ${targetDateStr}] [Sports: ${sports.join(", ")}] ${targetLeagueIds.length > 0 ? `[Filter: ${targetLeagueIds.length} leagues]` : ""}`,
);
for (const sport of sports) {
@@ -191,7 +192,7 @@ export class FeederService {
targetLeagueIds: string[] = [], // NEW: Optional league filter
): Promise<void> {
this.logger.log(
`🚀 STARTING HISTORICAL SCAN [Target: ${sports.join(', ')}] ${targetLeagueIds.length > 0 ? `[Filter: ${targetLeagueIds.length} leagues]` : ''}`,
`🚀 STARTING HISTORICAL SCAN [Target: ${sports.join(", ")}] ${targetLeagueIds.length > 0 ? `[Filter: ${targetLeagueIds.length} leagues]` : ""}`,
);
const startDate = new Date(startDateStr);
@@ -201,7 +202,7 @@ export class FeederService {
// writing to live_matches. Historical scan should only fill matches table.
endDate.setDate(endDate.getDate() - 2);
const stateKey = `historical_scan_state_${sports.join('_')}${targetLeagueIds.length > 0 ? '_filtered' : ''}_desc`;
const stateKey = `historical_scan_state_${sports.join("_")}${targetLeagueIds.length > 0 ? "_filtered" : ""}_desc`;
let currentDate: Date | null = null;
// Resume from saved state
@@ -215,12 +216,12 @@ export class FeederService {
// For reverse scan, we resume from the *next* day backwards, i.e., resumeDate - 1 day
currentDate.setDate(currentDate.getDate() - 1);
this.logger.log(
`📍 Resuming from: ${currentDate.toISOString().split('T')[0]}`,
`📍 Resuming from: ${currentDate.toISOString().split("T")[0]}`,
);
}
}
} catch {
this.logger.warn('Could not read state, starting from beginning');
this.logger.warn("Could not read state, starting from beginning");
}
// Initialize currentDate to endDate if not resuming (or if resume failed)
@@ -231,7 +232,7 @@ export class FeederService {
}
this.logger.log(
`📊 Scanning (Reverse): ${currentDate.toISOString().split('T')[0]} ← ${startDate.toISOString().split('T')[0]}`,
`📊 Scanning (Reverse): ${currentDate.toISOString().split("T")[0]} ← ${startDate.toISOString().split("T")[0]}`,
);
let processedDays = 0;
@@ -239,7 +240,7 @@ export class FeederService {
// REVERSE LOOP: Iterate while currentDate is greater than or equal to startDate
while (currentDate >= startDate) {
const dateString = currentDate.toISOString().split('T')[0];
const dateString = currentDate.toISOString().split("T")[0];
for (const sport of sports) {
await this.processDate(dateString, sport, targetLeagueIds);
@@ -278,7 +279,7 @@ export class FeederService {
currentDate.setDate(currentDate.getDate() - 1);
}
this.logger.log('🎉 HISTORICAL SCAN COMPLETED');
this.logger.log("🎉 HISTORICAL SCAN COMPLETED");
}
// ============================================
@@ -308,9 +309,9 @@ export class FeederService {
break; // Success, exit loop
} catch (e: any) {
const is502 =
e.message?.includes('502') ||
e.message?.includes("502") ||
e.response?.status === 502 ||
e.message?.includes('Bad Gateway');
e.message?.includes("Bad Gateway");
if (is502 && i < 2) {
this.logger.warn(
@@ -341,10 +342,7 @@ export class FeederService {
// regardless of the matchDate query parameter. We must filter by mstUtc
// to ensure we only process matches that actually belong to the target date.
const { startTs: targetDateStartTs, endTs: targetDateEndTs } =
this.getDayBoundsForTimeZone(
dateString,
this.DAILY_SYNC_TIME_ZONE,
);
this.getDayBoundsForTimeZone(dateString, this.DAILY_SYNC_TIME_ZONE);
const dateFilteredMatches = allMatches.filter((m) => {
const matchTs = m.mstUtc;
@@ -518,14 +516,14 @@ export class FeederService {
// ============================================
async refreshMatch(
matchId: string,
scope: 'all' | 'lineups' | 'odds' = 'all',
scope: "all" | "lineups" | "odds" = "all",
): Promise<ProcessResult> {
this.logger.log(`🔄 Refreshing match (${scope}) for ${matchId}`);
const matchRecord = await this.persistenceService.getMatch(matchId);
if (!matchRecord) {
this.logger.warn(`[${matchId}] Refresh failed: Match not in DB`);
return { success: false, retryable: false, error: 'Match not found' };
return { success: false, retryable: false, error: "Match not found" };
}
// Construct MatchSummary from DB record
@@ -538,13 +536,13 @@ export class FeederService {
iddaaCode: matchRecord.iddaaCode,
homeTeam: {
id: matchRecord.homeTeamId,
name: matchRecord.homeTeam?.name || '',
slug: matchRecord.homeTeam?.slug || '',
name: matchRecord.homeTeam?.name || "",
slug: matchRecord.homeTeam?.slug || "",
},
awayTeam: {
id: matchRecord.awayTeamId,
name: matchRecord.awayTeam?.name || '',
slug: matchRecord.awayTeam?.slug || '',
name: matchRecord.awayTeam?.name || "",
slug: matchRecord.awayTeam?.slug || "",
},
score: {
home: matchRecord.scoreHome,
@@ -555,9 +553,9 @@ export class FeederService {
const dummyCompetitions: Record<string, Competition> = {
[summary.competitionId]: {
id: summary.competitionId,
name: 'Unknown',
competitionSlug: '',
country: { id: '', name: '' },
name: "Unknown",
competitionSlug: "",
country: { id: "", name: "" },
},
};
@@ -583,7 +581,7 @@ export class FeederService {
competitions: Record<string, Competition>,
sport: Sport,
force: boolean = false,
scope: 'all' | 'lineups' | 'odds' = 'all', // Add scope flag
scope: "all" | "lineups" | "odds" = "all", // Add scope flag
): Promise<ProcessResult> {
const matchId = matchSummary.id;
const homeTeamId = matchSummary.homeTeam?.id;
@@ -595,7 +593,7 @@ export class FeederService {
}
// Skip postponed matches (ERT = Erteledendi)
if (matchSummary.statusBoxContent === 'ERT') {
if (matchSummary.statusBoxContent === "ERT") {
this.logger.debug(`[${matchId}] Skipped: Postponed match (ERT)`);
return { success: false, retryable: false };
}
@@ -615,9 +613,9 @@ export class FeederService {
return await fn();
} catch (e: any) {
const is502 =
e.message?.includes('502') ||
e.message?.includes("502") ||
e.response?.status === 502 ||
e.message?.includes('Bad Gateway');
e.message?.includes("Bad Gateway");
if (i === retries - 1) throw e; // Last attempt failed
@@ -661,44 +659,44 @@ export class FeederService {
// 1. Fetch Match Header (score, status)
let headerData: ParsedMatchHeader | null = null;
if (scope === 'all') {
if (scope === "all") {
try {
headerData = await fetchResilient('Header', () =>
headerData = await fetchResilient("Header", () =>
this.scraperService.fetchMatchHeader(matchId),
);
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Header fetch failed: ${e.message}`);
}
}
// 2. Sport-specific data fetching
if (sport === 'basketball') {
if (sport === "basketball") {
// Basketball: Box Score (Always if all or lineups)
if (scope === 'all' || scope === 'lineups') {
if (scope === "all" || scope === "lineups") {
try {
const boxData = await fetchResilient('BoxScore', () =>
const boxData = await fetchResilient("BoxScore", () =>
this.scraperService.fetchBasketballBoxScore(matchId),
);
if (boxData) {
const homeParsed = this.scraperService.parseBasketballBoxScore(
boxData.views?.home?.html || '',
boxData.views?.home?.html || "",
);
const awayParsed = this.scraperService.parseBasketballBoxScore(
boxData.views?.away?.html || '',
boxData.views?.away?.html || "",
);
basketballTeamStats =
scope === 'all'
scope === "all"
? {
home: homeParsed.teamTotals,
away: awayParsed.teamTotals,
}
: null;
if (scope === 'all') {
if (scope === "all") {
try {
const details = await fetchResilient('QuarterScores', () =>
const details = await fetchResilient("QuarterScores", () =>
this.scraperService.fetchBasketballDetailsHeader(matchId),
);
if (details && basketballTeamStats) {
@@ -712,7 +710,7 @@ export class FeederService {
};
}
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(
`[${matchId}] Quarter scores fetch failed: ${e.message}`,
);
@@ -748,7 +746,7 @@ export class FeederService {
processPlayers(awayParsed, awayTeamId);
}
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Box score failed: ${e.message}`);
}
}
@@ -756,9 +754,9 @@ export class FeederService {
// Football: Events, Lineups, Stats, Officials
// Key Events
if (scope === 'all') {
if (scope === "all") {
try {
const eventsData = await fetchResilient('Events', () =>
const eventsData = await fetchResilient("Events", () =>
this.scraperService.fetchKeyEvents(matchId),
);
if (eventsData?.keyEvents) {
@@ -781,7 +779,7 @@ export class FeederService {
);
}
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Events failed: ${e.message}`);
}
@@ -850,20 +848,20 @@ export class FeederService {
*/
// Game Stats & Officials
if (scope === 'all') {
if (scope === "all") {
try {
const gameStats = await fetchResilient('Stats', () =>
const gameStats = await fetchResilient("Stats", () =>
this.scraperService.fetchGameStats(matchId),
);
stats = this.transformerService.transformGameStats(gameStats);
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Stats failed: ${e.message}`);
}
// Officials (from match page)
try {
const matchPageHtml = await fetchResilient('Officials', () =>
const matchPageHtml = await fetchResilient("Officials", () =>
this.scraperService.fetchMatchPage(
matchId,
matchSummary.matchSlug,
@@ -875,7 +873,7 @@ export class FeederService {
this.transformerService.parseOfficials(matchPageHtml);
}
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Officials failed: ${e.message}`);
}
}
@@ -883,31 +881,31 @@ export class FeederService {
// 3. Fetch Iddaa Odds (Always if all or odds)
let oddsArray: DbMarketPayload[] = [];
if (scope === 'all' || scope === 'odds') {
if (scope === "all" || scope === "odds") {
try {
let markets: ParsedMarket[] = [];
if (sport === 'basketball') {
if (sport === "basketball") {
markets =
((await fetchResilient('BucketOdds', () =>
((await fetchResilient("BucketOdds", () =>
this.scraperService.fetchBasketballMarkets(matchId),
)) as ParsedMarket[]) || [];
} else {
markets =
((await fetchResilient('IddaaOdds', () =>
((await fetchResilient("IddaaOdds", () =>
this.scraperService.fetchIddaaMarkets(matchId),
)) as ParsedMarket[]) || [];
}
// Logic is same since structure is ParsedMarket[]
oddsArray = this.transformerService.transformIddaaMarkets(markets);
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Odds failed: ${e.message}`);
}
}
// 4. Persist to Database
let saved = false;
if (scope === 'lineups') {
if (scope === "lineups") {
saved = await this.persistenceService.saveLineups(
matchId,
playersMap,
@@ -915,7 +913,7 @@ export class FeederService {
homeTeamId,
awayTeamId,
);
} else if (scope === 'odds') {
} else if (scope === "odds") {
saved = await this.persistenceService.saveOdds(matchId, oddsArray);
} else {
// Full Update
@@ -962,12 +960,12 @@ export class FeederService {
if (saved && hasCriticalError) {
// Collect missing components
const missingParts: string[] = [];
if (!stats) missingParts.push('Stats');
if (oddsArray.length === 0) missingParts.push('Odds');
if (officialsData.length === 0) missingParts.push('Officials');
if (!stats) missingParts.push("Stats");
if (oddsArray.length === 0) missingParts.push("Odds");
if (officialsData.length === 0) missingParts.push("Officials");
this.logger.warn(
`[${matchId}] Saved with MISSING DATA (502). Missing: [${missingParts.join(', ')}]. Scheduled for retry.`,
`[${matchId}] Saved with MISSING DATA (502). Missing: [${missingParts.join(", ")}]. Scheduled for retry.`,
);
return { success: false, retryable: true };
}
@@ -975,12 +973,12 @@ export class FeederService {
return { success: saved, retryable: !saved };
} catch (error: any) {
const isRetryable =
error.message.includes('502') ||
error.message.includes('504') ||
error.message.includes('ECONNABORTED') ||
error.message.includes('timeout') ||
error.message.includes('ETIMEDOUT') ||
error.message.includes('Unique constraint'); // Concurrency retry
error.message.includes("502") ||
error.message.includes("504") ||
error.message.includes("ECONNABORTED") ||
error.message.includes("timeout") ||
error.message.includes("ETIMEDOUT") ||
error.message.includes("Unique constraint"); // Concurrency retry
if (isRetryable) {
this.logger.warn(`[${matchId}] ${error.message} - Will retry`);