fahricansecer/iddaai-be
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cr

Browse Source
This commit is contained in:
Fahri Can
2026-04-16 17:21:48 +03:00
parent c8fa4c442d
commit c8e7e4e927
116 changed files with 3720 additions and 4197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/modules/feeder/feeder-persistence.service.ts
+39 -39
View File
@@ -5,8 +5,8 @@
* Database operations using Prisma
*/
import { Injectable, Logger } from '@nestjs/common';
import { PrismaService } from '../../database/prisma.service';
import { Injectable, Logger } from "@nestjs/common";
import { PrismaService } from "../../database/prisma.service";
import {
Sport,
MatchSummary,
@@ -20,8 +20,8 @@ import {
DbEventPayload,
DbMarketPayload,
BasketballTeamStats,
} from './feeder.types';
import { ImageUtils } from '../../common/utils/image.util';
} from "./feeder.types";
import { ImageUtils } from "../../common/utils/image.util";
@Injectable()
export class FeederPersistenceService {
@@ -33,7 +33,7 @@ export class FeederPersistenceService {
// HELPER FUNCTIONS
// ============================================
private safeString(value: any): string | null {
return value === null || value === undefined || value === ''
return value === null || value === undefined || value === ""
? null
: String(value);
}
@@ -51,12 +51,12 @@ export class FeederPersistenceService {
private mapPositionToEnum(position: string | null): any {
if (!position) return null;
const pos = position.toLowerCase();
if (pos.includes('kaleci') || pos.includes('goalkeeper'))
return 'goalkeeper';
if (pos.includes('defans') || pos.includes('defender')) return 'defender';
if (pos.includes('orta saha') || pos.includes('midfielder'))
return 'midfielder';
if (pos.includes('forvet') || pos.includes('striker')) return 'striker';
if (pos.includes("kaleci") || pos.includes("goalkeeper"))
return "goalkeeper";
if (pos.includes("defans") || pos.includes("defender")) return "defender";
if (pos.includes("orta saha") || pos.includes("midfielder"))
return "midfielder";
if (pos.includes("forvet") || pos.includes("striker")) return "striker";
return null;
}
@@ -93,7 +93,7 @@ export class FeederPersistenceService {
}
for (const s of market.selectionCollection) {
if (!s || s.odd === '-' || s.odd === '') continue;
if (!s || s.odd === "-" || s.odd === "") continue;
const sName = this.safeString(s.name);
const sValue = this.safeString(s.odd);
@@ -107,7 +107,7 @@ export class FeederPersistenceService {
if (existingSel) {
if (existingSel.oddValue !== sValue) {
const oldVal = parseFloat(existingSel.oddValue || '0');
const oldVal = parseFloat(existingSel.oddValue || "0");
const newVal = parseFloat(sValue);
if (!isNaN(oldVal) && !isNaN(newVal)) {
@@ -182,13 +182,13 @@ export class FeederPersistenceService {
const teamsToUpsert = [
{
id: homeTeamId,
name: matchSummary.homeTeam?.name || 'Unknown',
name: matchSummary.homeTeam?.name || "Unknown",
slug: matchSummary.homeTeam?.slug || homeTeamId,
sport: sport,
},
{
id: awayTeamId,
name: matchSummary.awayTeam?.name || 'Unknown',
name: matchSummary.awayTeam?.name || "Unknown",
slug: matchSummary.awayTeam?.slug || awayTeamId,
sport: sport,
},
@@ -221,18 +221,18 @@ export class FeederPersistenceService {
update: {},
create: {
id: countryId,
name: league.country.name || 'Unknown',
name: league.country.name || "Unknown",
},
});
} catch (error: any) {
if (error.code !== 'P2002') throw error;
if (error.code !== "P2002") throw error;
}
}
// 2. Save League (Handle ID changes by checking unique constraint)
let finalLeagueId = this.safeString(league.id);
if (finalLeagueId && countryId) {
const leagueName = league.name || 'Unknown';
const leagueName = league.name || "Unknown";
// Check if league exists by unique constraint (name + country + sport)
const existingLeague = await tx.league.findUnique({
@@ -311,32 +311,32 @@ export class FeederPersistenceService {
headerData?.htScoreAway ??
this.safeInt(matchSummary.score?.ht?.away);
let status = 'NS';
let status = "NS";
if (headerData?.matchStatus) {
if (
headerData.matchStatus === 'postGame' ||
headerData.matchStatus === 'post'
headerData.matchStatus === "postGame" ||
headerData.matchStatus === "post"
) {
status = 'FT';
status = "FT";
} else if (
headerData.matchStatus === 'live' ||
headerData.matchStatus === 'liveGame'
headerData.matchStatus === "live" ||
headerData.matchStatus === "liveGame"
) {
status = 'LIVE';
status = "LIVE";
}
}
// Handle Postponed Matches (ERT)
if (matchSummary.statusBoxContent === 'ERT') {
status = 'POSTPONED';
if (matchSummary.statusBoxContent === "ERT") {
status = "POSTPONED";
}
if (
status === 'NS' &&
status === "NS" &&
finalScoreHome !== null &&
finalScoreAway !== null
) {
status = 'FT';
status = "FT";
}
await tx.match.upsert({
@@ -455,7 +455,7 @@ export class FeederPersistenceService {
}
// 8. Save Team Stats (Football)
if (stats && sport === 'football') {
if (stats && sport === "football") {
const statsRows = [
{
matchId,
@@ -499,7 +499,7 @@ export class FeederPersistenceService {
}
// 8b. Save Team Stats (Basketball)
if (basketballTeamStats && sport === 'basketball') {
if (basketballTeamStats && sport === "basketball") {
const teams = [
{ id: homeTeamId, data: basketballTeamStats.home },
{ id: awayTeamId, data: basketballTeamStats.away },
@@ -558,7 +558,7 @@ export class FeederPersistenceService {
}
// 8c. Save Player Stats (Basketball)
if (basketballPlayerStats.length > 0 && sport === 'basketball') {
if (basketballPlayerStats.length > 0 && sport === "basketball") {
await tx.basketballPlayerStats.deleteMany({ where: { matchId } });
for (const p of basketballPlayerStats) {
@@ -592,12 +592,12 @@ export class FeederPersistenceService {
await this.saveOddsInTransaction(tx, matchId, oddsArray);
// 10. Save Officials
if (sport === 'football' && officialsData.length > 0) {
if (sport === "football" && officialsData.length > 0) {
await tx.matchOfficial.deleteMany({ where: { matchId } });
const processedOfficials = new Set<string>();
for (const o of officialsData) {
const roleName = o.role || 'Referee';
const roleName = o.role || "Referee";
const uniqueKey = `${o.name}_${roleName}`;
if (processedOfficials.has(uniqueKey)) continue;
@@ -798,10 +798,10 @@ export class FeederPersistenceService {
const history = await this.prisma.match.findMany({
where: {
OR: [{ homeTeamId: teamId }, { awayTeamId: teamId }],
status: 'FT',
status: "FT",
mstUtc: { lt: match.mstUtc },
},
orderBy: { mstUtc: 'desc' },
orderBy: { mstUtc: "desc" },
take: 5,
});
@@ -840,8 +840,8 @@ export class FeederPersistenceService {
return {
match_id: match.id,
home_team: match.homeTeam?.name || 'Unknown',
away_team: match.awayTeam?.name || 'Unknown',
home_team: match.homeTeam?.name || "Unknown",
away_team: match.awayTeam?.name || "Unknown",
home_team_id: match.homeTeamId,
away_team_id: match.awayTeamId,
league_id: match.leagueId,
@@ -934,7 +934,7 @@ export class FeederPersistenceService {
scoreHome: liveMatch.scoreHome,
scoreAway: liveMatch.scoreAway,
mstUtc: liveMatch.mstUtc,
sport: liveMatch.sport || 'football',
sport: liveMatch.sport || "football",
};
}
src/modules/feeder/feeder-scraper.service.ts
+105 -105
View File
@@ -3,9 +3,9 @@
* HTTP requests with exact headers from working curl commands
*/
import { Injectable, Logger } from '@nestjs/common';
import axios, { AxiosInstance } from 'axios';
import * as cheerio from 'cheerio';
import { Injectable, Logger } from "@nestjs/common";
import axios, { AxiosInstance } from "axios";
import * as cheerio from "cheerio";
import {
Sport,
SPORTS_CONFIG,
@@ -25,7 +25,7 @@ import {
SidelinedResponse,
SidelinedTeamData,
SidelinedPlayer,
} from './feeder.types';
} from "./feeder.types";
@Injectable()
export class FeederScraperService {
@@ -43,13 +43,13 @@ export class FeederScraperService {
this.axios.interceptors.response.use(
(response) => {
this.logger.debug(
`✅ [${response.config.url?.split('?')[0]}] Status: ${response.status}`,
`✅ [${response.config.url?.split("?")[0]}] Status: ${response.status}`,
);
return response;
},
(error) => {
const status = error.response?.status || 'N/A';
const url = error.config?.url?.split('?')[0] || 'Unknown';
const status = error.response?.status || "N/A";
const url = error.config?.url?.split("?")[0] || "Unknown";
this.logger.error(`❌ [${url}] Status: ${status} - ${error.message}`);
throw error;
},
@@ -72,7 +72,7 @@ export class FeederScraperService {
const response = await this.axios.get(url, {
params: {
'sports[]': sportParam,
"sports[]": sportParam,
matchDate: dateString,
},
});
@@ -80,11 +80,11 @@ export class FeederScraperService {
const payload = response.data as unknown;
if (
!payload ||
typeof payload !== 'object' ||
!('status' in payload) ||
!('data' in payload)
typeof payload !== "object" ||
!("status" in payload) ||
!("data" in payload)
) {
throw new Error('Historical source payload has invalid shape');
throw new Error("Historical source payload has invalid shape");
}
return payload as LivescoresApiResponse;
@@ -101,14 +101,14 @@ export class FeederScraperService {
const response = await this.axios.get(url, {
params: {
matchId,
sdapiLanguageCode: 'tr-mk',
ajaxViewName: 'match-details',
ajaxPartialViewName: 'match-details-status',
displayMode: 'all',
sdapiLanguageCode: "tr-mk",
ajaxViewName: "match-details",
ajaxPartialViewName: "match-details-status",
displayMode: "all",
},
});
return this.parseMatchHeader(response.data.data?.html || '');
return this.parseMatchHeader(response.data.data?.html || "");
}
private parseMatchHeader(html: string): ParsedMatchHeader {
@@ -116,7 +116,7 @@ export class FeederScraperService {
// Extract match-status from data attribute
const matchStatus =
($('[data-match-status]').attr('data-match-status') as any) || 'postGame';
($("[data-match-status]").attr("data-match-status") as any) || "postGame";
// Extract scores
const scoreHome = this.safeInt($('[data-slot="score-home"]').text().trim());
@@ -126,7 +126,7 @@ export class FeederScraperService {
let htScoreHome: number | null = null;
let htScoreAway: number | null = null;
const detailedScore = $('.p0c-soccer-match-details-header__detailed-score')
const detailedScore = $(".p0c-soccer-match-details-header__detailed-score")
.text()
.trim();
const htMatch = detailedScore.match(/\(İY\s*(\d+)\s*-\s*(\d+)\)/);
@@ -143,7 +143,7 @@ export class FeederScraperService {
// ============================================
async fetchKeyEvents(
matchId: string,
): Promise<KeyEventsResponse['data'] | null> {
): Promise<KeyEventsResponse["data"] | null> {
const url = `https://www.mackolik.com/ajax/football/key-events`;
this.logger.debug(`📡 [${matchId}] Fetching key events`);
@@ -151,7 +151,7 @@ export class FeederScraperService {
try {
const response = await this.axios.get<KeyEventsResponse>(url, {
params: {
ajaxViewName: 'events',
ajaxViewName: "events",
matchId,
seasonId: matchId, // Same as matchId
},
@@ -172,7 +172,7 @@ export class FeederScraperService {
// ============================================
async fetchStartingFormation(
matchId: string,
): Promise<MatchStatsResponse['data'] | null> {
): Promise<MatchStatsResponse["data"] | null> {
const url = `https://www.mackolik.com/ajax/football/match-stats`;
this.logger.debug(`📡 [${matchId}] Fetching starting formation`);
@@ -180,7 +180,7 @@ export class FeederScraperService {
try {
const response = await this.axios.get<MatchStatsResponse>(url, {
params: {
ajaxViewName: 'starting-formation',
ajaxViewName: "starting-formation",
matchId,
seasonId: matchId,
},
@@ -201,7 +201,7 @@ export class FeederScraperService {
// ============================================
async fetchSubstitutions(
matchId: string,
): Promise<MatchStatsResponse['data'] | null> {
): Promise<MatchStatsResponse["data"] | null> {
const url = `https://www.mackolik.com/ajax/football/match-stats`;
this.logger.debug(`📡 [${matchId}] Fetching substitutions`);
@@ -209,7 +209,7 @@ export class FeederScraperService {
try {
const response = await this.axios.get<MatchStatsResponse>(url, {
params: {
ajaxViewName: 'substitutions',
ajaxViewName: "substitutions",
matchId,
seasonId: matchId,
},
@@ -230,7 +230,7 @@ export class FeederScraperService {
// ============================================
async fetchGameStats(
matchId: string,
): Promise<GameStatsResponse['data'] | null> {
): Promise<GameStatsResponse["data"] | null> {
const url = `https://www.mackolik.com/ajax/soccer/match/gameStats`;
this.logger.debug(`📡 [${matchId}] Fetching game stats`);
@@ -253,7 +253,7 @@ export class FeederScraperService {
// ============================================
// MANAGER
// ============================================
async fetchManager(matchId: string): Promise<ManagerResponse['data'] | null> {
async fetchManager(matchId: string): Promise<ManagerResponse["data"] | null> {
const url = `https://www.mackolik.com/ajax/football/match-stats`;
this.logger.debug(`📡 [${matchId}] Fetching manager`);
@@ -261,7 +261,7 @@ export class FeederScraperService {
try {
const response = await this.axios.get<ManagerResponse>(url, {
params: {
ajaxViewName: 'manager',
ajaxViewName: "manager",
matchId,
seasonId: matchId,
},
@@ -287,10 +287,10 @@ export class FeederScraperService {
try {
const response = await this.axios.get<IddaaMarketsHtmlResponse>(url, {
params: { template: 'all' },
params: { template: "all" },
});
return this.parseIddaaMarketsHtml(response.data.data?.html || '');
return this.parseIddaaMarketsHtml(response.data.data?.html || "");
} catch (error: any) {
if (error.response?.status === 404) {
this.logger.warn(`[${matchId}] Iddaa markets not found (404)`);
@@ -306,30 +306,30 @@ export class FeederScraperService {
const $ = cheerio.load(html);
const markets: ParsedMarket[] = [];
$('.widget-iddaa-markets__market-item').each((_, marketEl) => {
$(".widget-iddaa-markets__market-item").each((_, marketEl) => {
const $market = $(marketEl);
const marketId = $market.attr('data-market') || '';
const marketId = $market.attr("data-market") || "";
const marketName = $market
.find('.widget-iddaa-markets__header-text')
.find(".widget-iddaa-markets__header-text")
.text()
.trim();
const iddaaCode = $market
.find('.widget-iddaa-markets__iddaa-code')
.find(".widget-iddaa-markets__iddaa-code")
.text()
.trim();
const mbc = $market.find('.widget-iddaa-markets__mbc').text().trim();
const mbc = $market.find(".widget-iddaa-markets__mbc").text().trim();
const selections: ParsedSelection[] = [];
$market.find('.widget-iddaa-markets__option').each((_, optionEl) => {
$market.find(".widget-iddaa-markets__option").each((_, optionEl) => {
const $option = $(optionEl);
selections.push({
shortcode: $option.attr('data-shortcode') || '',
outcomeNo: $option.attr('data-outcome-no') || '',
label: $option.find('.widget-iddaa-markets__label').text().trim(),
value: $option.find('.widget-iddaa-markets__value').text().trim(),
shortcode: $option.attr("data-shortcode") || "",
outcomeNo: $option.attr("data-outcome-no") || "",
label: $option.find(".widget-iddaa-markets__label").text().trim(),
value: $option.find(".widget-iddaa-markets__value").text().trim(),
});
});
@@ -347,7 +347,7 @@ export class FeederScraperService {
// ============================================
async fetchBasketballBoxScore(
matchId: string,
): Promise<BasketballBoxScoreResponse['data'] | null> {
): Promise<BasketballBoxScoreResponse["data"] | null> {
// Updated URL based on user request
const url = `https://www.mackolik.com/ajax/basketball/match/box-score`;
@@ -357,8 +357,8 @@ export class FeederScraperService {
const response = await this.axios.get<BasketballBoxScoreResponse>(url, {
params: { matchId },
headers: {
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': DEFAULT_HEADERS['User-Agent'],
"X-Requested-With": "XMLHttpRequest",
"User-Agent": DEFAULT_HEADERS["User-Agent"],
},
});
@@ -382,25 +382,25 @@ export class FeederScraperService {
const players: Partial<BasketballPlayerStats>[] = [];
// Parse individual players from widget rows
$('.widget-basketball-match-box-score__row').each((_, elem) => {
$(".widget-basketball-match-box-score__row").each((_, elem) => {
const row = $(elem);
// Skip if no player name found
const nameElem = row.find('.widget-basketball-match-box-score__player');
const nameElem = row.find(".widget-basketball-match-box-score__player");
if (!nameElem.length) return;
const name = nameElem.text().trim();
// Indices based on User HTML:
// 0: Name, 1: Min, 2: Pts, 3: Reb, 4: Ast, 5: 2FG, 6: 3FG, 7: FT, 8: Fouls, 9: Blk, 10: Stl, 11: TO
const values = row.find('td');
const values = row.find("td");
// Check if it's a valid player row (should have enough columns)
if (values.length < 10) return;
// Extract ID from link if possible
let playerId = '';
const link = nameElem.find('a').attr('href');
let playerId = "";
const link = nameElem.find("a").attr("href");
if (link) {
playerId = this.extractPlayerIdFromUrl(link) || '';
playerId = this.extractPlayerIdFromUrl(link) || "";
}
players.push({
@@ -410,16 +410,16 @@ export class FeederScraperService {
points: this.safeInt(values.eq(2).text().trim()) || 0,
rebounds: this.safeInt(values.eq(3).text().trim()) || 0,
assists: this.safeInt(values.eq(4).text().trim()) || 0,
fgMade: this.safeInt(values.eq(5).text().trim().split('/')[0]) || 0,
fgMade: this.safeInt(values.eq(5).text().trim().split("/")[0]) || 0,
fgAttempted:
this.safeInt(values.eq(5).text().trim().split('/')[1]) || 0,
this.safeInt(values.eq(5).text().trim().split("/")[1]) || 0,
threePtMade:
this.safeInt(values.eq(6).text().trim().split('/')[0]) || 0,
this.safeInt(values.eq(6).text().trim().split("/")[0]) || 0,
threePtAttempted:
this.safeInt(values.eq(6).text().trim().split('/')[1]) || 0,
ftMade: this.safeInt(values.eq(7).text().trim().split('/')[0]) || 0,
this.safeInt(values.eq(6).text().trim().split("/")[1]) || 0,
ftMade: this.safeInt(values.eq(7).text().trim().split("/")[0]) || 0,
ftAttempted:
this.safeInt(values.eq(7).text().trim().split('/')[1]) || 0,
this.safeInt(values.eq(7).text().trim().split("/")[1]) || 0,
fouls: this.safeInt(values.eq(8).text().trim()) || 0,
blocks: this.safeInt(values.eq(9).text().trim()) || 0,
steals: this.safeInt(values.eq(10).text().trim()) || 0,
@@ -428,7 +428,7 @@ export class FeederScraperService {
});
// Parse Team Totals from Footer
const footerRow = $('.widget-basketball-match-box-score__footer td');
const footerRow = $(".widget-basketball-match-box-score__footer td");
let teamTotals: any = {};
if (footerRow.length > 5) {
@@ -438,16 +438,16 @@ export class FeederScraperService {
points: this.safeInt(footerRow.eq(2).text().trim()) || 0,
rebounds: this.safeInt(footerRow.eq(3).text().trim()) || 0,
assists: this.safeInt(footerRow.eq(4).text().trim()) || 0,
fgMade: this.safeInt(footerRow.eq(5).text().trim().split('/')[0]) || 0,
fgMade: this.safeInt(footerRow.eq(5).text().trim().split("/")[0]) || 0,
fgAttempted:
this.safeInt(footerRow.eq(5).text().trim().split('/')[1]) || 0,
this.safeInt(footerRow.eq(5).text().trim().split("/")[1]) || 0,
threePtMade:
this.safeInt(footerRow.eq(6).text().trim().split('/')[0]) || 0,
this.safeInt(footerRow.eq(6).text().trim().split("/")[0]) || 0,
threePtAttempted:
this.safeInt(footerRow.eq(6).text().trim().split('/')[1]) || 0,
ftMade: this.safeInt(footerRow.eq(7).text().trim().split('/')[0]) || 0,
this.safeInt(footerRow.eq(6).text().trim().split("/")[1]) || 0,
ftMade: this.safeInt(footerRow.eq(7).text().trim().split("/")[0]) || 0,
ftAttempted:
this.safeInt(footerRow.eq(7).text().trim().split('/')[1]) || 0,
this.safeInt(footerRow.eq(7).text().trim().split("/")[1]) || 0,
fouls: this.safeInt(footerRow.eq(8).text().trim()) || 0,
blocks: this.safeInt(footerRow.eq(9).text().trim()) || 0,
steals: this.safeInt(footerRow.eq(10).text().trim()) || 0,
@@ -474,11 +474,11 @@ export class FeederScraperService {
// For HTML pages, we DON'T send X-Requested-With header
const response = await this.axios.get(url, {
headers: {
'User-Agent': DEFAULT_HEADERS['User-Agent'],
Referer: DEFAULT_HEADERS['Referer'],
'Accept-Language': DEFAULT_HEADERS['Accept-Language'],
"User-Agent": DEFAULT_HEADERS["User-Agent"],
Referer: DEFAULT_HEADERS["Referer"],
"Accept-Language": DEFAULT_HEADERS["Accept-Language"],
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
// NO X-Requested-With for HTML pages!
},
});
@@ -507,8 +507,8 @@ export class FeederScraperService {
const response = await this.axios.get(url, {
params: { matchId },
headers: {
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': DEFAULT_HEADERS['User-Agent'],
"X-Requested-With": "XMLHttpRequest",
"User-Agent": DEFAULT_HEADERS["User-Agent"],
},
});
@@ -532,12 +532,12 @@ export class FeederScraperService {
const $ = cheerio.load(html);
const rows = $(
'.widget-basketball-match-details-header__score-details tbody tr',
".widget-basketball-match-details-header__score-details tbody tr",
);
if (rows.length < 2) return null;
const parseRow = (row: any) => {
const cols = $(row).find('td');
const cols = $(row).find("td");
// Format: TeamName, Q1, Q2, Q3, Q4, Final
// Values are inside .widget-basketball-match-details-header__score-part (just the quarter score)
// or direct text if simple table.
@@ -545,7 +545,7 @@ export class FeederScraperService {
const getScore = (index: number) => {
const cell = cols.eq(index);
const part = cell.find(
'.widget-basketball-match-details-header__score-part',
".widget-basketball-match-details-header__score-part",
);
const val = part.length ? part.text() : cell.text();
return this.safeInt(val.trim());
@@ -580,10 +580,10 @@ export class FeederScraperService {
try {
const response = await this.axios.get<IddaaMarketsHtmlResponse>(url, {
params: { template: 'all' },
params: { template: "all" },
headers: {
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': DEFAULT_HEADERS['User-Agent'],
"X-Requested-With": "XMLHttpRequest",
"User-Agent": DEFAULT_HEADERS["User-Agent"],
},
});
@@ -602,7 +602,7 @@ export class FeederScraperService {
extractPlayerIdFromUrl(url: string | undefined): string | null {
if (!url) return null;
const parts = url.split('/');
const parts = url.split("/");
return parts[parts.length - 1] || null;
}
@@ -620,12 +620,12 @@ export class FeederScraperService {
try {
const response = await this.axios.get(url, {
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7',
Referer: 'https://www.mackolik.com',
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7",
Referer: "https://www.mackolik.com",
},
timeout: 10000,
});
@@ -652,24 +652,24 @@ export class FeederScraperService {
$: cheerio.CheerioAPI,
teamIndex: number,
): SidelinedTeamData {
const sidelinedWidgets = $('.widget-sidelined-players');
const sidelinedWidgets = $(".widget-sidelined-players");
if (sidelinedWidgets.length <= teamIndex) {
return { teamName: '', teamId: '', totalSidelined: 0, players: [] };
return { teamName: "", teamId: "", totalSidelined: 0, players: [] };
}
const widget = sidelinedWidgets.eq(teamIndex);
const teamCrest = widget.find('.widget-sidelined-players__header-crest');
const teamCrestSrc = teamCrest.attr('src') || '';
const teamId = teamCrestSrc.split('/').pop() || '';
const teamCrest = widget.find(".widget-sidelined-players__header-crest");
const teamCrestSrc = teamCrest.attr("src") || "";
const teamId = teamCrestSrc.split("/").pop() || "";
const teamName = widget
.find('.widget-sidelined-players__header-text')
.find(".widget-sidelined-players__header-text")
.text()
.trim();
const players: SidelinedPlayer[] = [];
widget.find('.widget-sidelined-players__item').each((_, element) => {
widget.find(".widget-sidelined-players__item").each((_, element) => {
const playerData = this._parsePlayerItem($, $(element));
if (playerData) {
players.push(playerData);
@@ -689,44 +689,44 @@ export class FeederScraperService {
$item: cheerio.Cheerio<any>,
): SidelinedPlayer | null {
try {
const nameElem = $item.find('.widget-sidelined-players__name');
const nameElem = $item.find(".widget-sidelined-players__name");
const playerName = nameElem.text().trim();
const playerUrl = nameElem.attr('href') || '';
const playerId = playerUrl.split('/').pop() || '';
const playerUrl = nameElem.attr("href") || "";
const playerId = playerUrl.split("/").pop() || "";
const positionElem = $item.find('.widget-sidelined-players__position');
const position = positionElem.attr('title') || '';
const positionElem = $item.find(".widget-sidelined-players__position");
const position = positionElem.attr("title") || "";
const positionShort = positionElem.text().trim();
const reasonImg = $item.find('.widget-sidelined-players__reason img');
const reasonIcon = reasonImg.attr('src') || '';
const reasonImg = $item.find(".widget-sidelined-players__reason img");
const reasonIcon = reasonImg.attr("src") || "";
const numbers = $item.find('.widget-sidelined-players__number');
const numbers = $item.find(".widget-sidelined-players__number");
// Use parseInt EXACTLY as in JS script (ignoring potential NaN for now, will handle via helper if needed but safer to stick to script logic first)
const matchesMissedText =
numbers.length > 0 ? numbers.eq(0).text().trim() : '';
numbers.length > 0 ? numbers.eq(0).text().trim() : "";
const matchesMissed = matchesMissedText
? parseInt(matchesMissedText, 10)
: null;
const averageText = numbers.length > 1 ? numbers.eq(1).text().trim() : '';
const averageText = numbers.length > 1 ? numbers.eq(1).text().trim() : "";
const average = averageText ? parseInt(averageText, 10) : null;
const description = $item
.find('.widget-sidelined-players__value')
.find(".widget-sidelined-players__value")
.text()
.trim();
const type = reasonIcon.includes('shortage_1.png')
? 'injury'
: reasonIcon.includes('suspension')
? 'suspension'
: 'other';
const type = reasonIcon.includes("shortage_1.png")
? "injury"
: reasonIcon.includes("suspension")
? "suspension"
: "other";
return {
playerId,
playerName,
playerUrl: playerUrl.startsWith('http')
playerUrl: playerUrl.startsWith("http")
? playerUrl
: `https://www.mackolik.com${playerUrl}`,
position,
@@ -735,7 +735,7 @@ export class FeederScraperService {
description,
matchesMissed: isNaN(matchesMissed as number) ? null : matchesMissed,
average: isNaN(average as number) ? null : average,
reasonIcon: reasonIcon.startsWith('http')
reasonIcon: reasonIcon.startsWith("http")
? reasonIcon
: `https://www.mackolik.com${reasonIcon}`, // Keep safer URL construction but stick closer to logic
};
src/modules/feeder/feeder-transformer.service.ts
+32 -32
View File
@@ -3,8 +3,8 @@
* Transforms raw API data into database-ready formats
*/
import { Injectable, Logger } from '@nestjs/common';
import * as cheerio from 'cheerio';
import { Injectable, Logger } from "@nestjs/common";
import * as cheerio from "cheerio";
import {
RawKeyEvent,
TransformedEvent,
@@ -18,7 +18,7 @@ import {
GameStatsResponse,
DbEventPayload,
DbMarketPayload,
} from './feeder.types';
} from "./feeder.types";
@Injectable()
export class FeederTransformerService {
@@ -28,7 +28,7 @@ export class FeederTransformerService {
// HELPER FUNCTIONS
// ============================================
private safeString(value: any): string | null {
return value === null || value === undefined || value === ''
return value === null || value === undefined || value === ""
? null
: String(value);
}
@@ -45,7 +45,7 @@ export class FeederTransformerService {
private extractPlayerIdFromUrl(url: string | undefined): string | null {
if (!url) return null;
const parts = url.split('/');
const parts = url.split("/");
return parts[parts.length - 1] || null;
}
@@ -59,7 +59,7 @@ export class FeederTransformerService {
matchId: string,
): TransformedEvent[] {
return rawEvents.map((e) => {
const playerId = this.extractPlayerIdFromUrl(e.playerUrl) || '';
const playerId = this.extractPlayerIdFromUrl(e.playerUrl) || "";
const assistPlayerId = e.assistPlayerUrl
? this.extractPlayerIdFromUrl(e.assistPlayerUrl)
: null;
@@ -68,16 +68,16 @@ export class FeederTransformerService {
: null;
// Determine event type
let eventType: 'goal' | 'card' | 'substitute' | 'other' = 'other';
if (e.type === 'goal') eventType = 'goal';
else if (e.type === 'card') eventType = 'card';
else if (e.type === 'substitute') eventType = 'substitute';
let eventType: "goal" | "card" | "substitute" | "other" = "other";
if (e.type === "goal") eventType = "goal";
else if (e.type === "card") eventType = "card";
else if (e.type === "substitute") eventType = "substitute";
return {
matchId,
playerId,
playerName: e.playerName,
teamId: e.position === 'home' ? homeTeamId : awayTeamId,
teamId: e.position === "home" ? homeTeamId : awayTeamId,
eventType,
eventSubtype: e.subType || null,
timeMinute: e.timeMin,
@@ -136,7 +136,7 @@ export class FeederTransformerService {
// GAME STATS TRANSFORMER
// ============================================
transformGameStats(
data: GameStatsResponse['data'] | null,
data: GameStatsResponse["data"] | null,
): TransformedMatchStats | null {
if (!data || !data.home) return null;
@@ -173,20 +173,20 @@ export class FeederTransformerService {
// MATCH STATE TO STATUS MAPPER
// ============================================
mapMatchStateToStatus(state: MatchState | undefined): string {
if (!state) return 'NS';
if (!state) return "NS";
switch (state) {
case 'postGame':
case 'post':
return 'FT';
case 'preGame':
case 'pre':
return 'NS';
case 'live':
case 'liveGame':
return 'LIVE';
case "postGame":
case "post":
return "FT";
case "preGame":
case "pre":
return "NS";
case "live":
case "liveGame":
return "LIVE";
default:
return 'NS';
return "NS";
}
}
@@ -200,28 +200,28 @@ export class FeederTransformerService {
const officials: MatchOfficial[] = [];
// Try standard officials component
$('.p0c-match-officials__official-list-item').each((_, elem) => {
$(".p0c-match-officials__official-list-item").each((_, elem) => {
const name = $(elem)
.find('.p0c-match-officials__official-name')
.find(".p0c-match-officials__official-name")
.text()
.trim();
const role = $(elem)
.find('.p0c-match-officials__official-group-title')
.find(".p0c-match-officials__official-group-title")
.text()
.trim();
if (name) {
officials.push({ name, role: role || 'Referee' });
officials.push({ name, role: role || "Referee" });
}
});
// Fallback: look for referee info in match info section
if (officials.length === 0) {
// Try alternative selectors
$('.widget-match-info__referee-name, .referee-name').each((_, elem) => {
$(".widget-match-info__referee-name, .referee-name").each((_, elem) => {
const name = $(elem).text().trim();
if (name) {
officials.push({ name, role: 'Referee' });
officials.push({ name, role: "Referee" });
}
});
}
@@ -331,8 +331,8 @@ export class FeederTransformerService {
(
e,
): e is TransformedEvent & {
eventType: 'goal' | 'card' | 'substitute';
} => e.eventType !== 'other' && !!e.playerId,
eventType: "goal" | "card" | "substitute";
} => e.eventType !== "other" && !!e.playerId,
)
.map((e) => ({
match_id: e.matchId,
@@ -354,6 +354,6 @@ export class FeederTransformerService {
// BASKETBALL PLAYER ID GENERATOR
// ============================================
generateBasketballPlayerId(teamId: string, playerName: string): string {
return `${teamId}-${playerName.replace(/[^a-zA-Z0-9]/g, '-').toLowerCase()}`;
return `${teamId}-${playerName.replace(/[^a-zA-Z0-9]/g, "-").toLowerCase()}`;
}
}
src/modules/feeder/feeder.module.ts
+6 -6
View File
@@ -2,12 +2,12 @@
* Feeder Module - Senior Level Implementation
*/
import { Module } from '@nestjs/common';
import { FeederService } from './feeder.service';
import { FeederScraperService } from './feeder-scraper.service';
import { FeederTransformerService } from './feeder-transformer.service';
import { FeederPersistenceService } from './feeder-persistence.service';
import { DatabaseModule } from '../../database/database.module';
import { Module } from "@nestjs/common";
import { FeederService } from "./feeder.service";
import { FeederScraperService } from "./feeder-scraper.service";
import { FeederTransformerService } from "./feeder-transformer.service";
import { FeederPersistenceService } from "./feeder-persistence.service";
import { DatabaseModule } from "../../database/database.module";
@Module({
imports: [DatabaseModule],
src/modules/feeder/feeder.service.ts
+100 -102
View File
@@ -3,10 +3,10 @@
* Main orchestration service for historical data scanning
*/
import { Injectable, Logger } from '@nestjs/common';
import { FeederScraperService } from './feeder-scraper.service';
import { FeederTransformerService } from './feeder-transformer.service';
import { FeederPersistenceService } from './feeder-persistence.service';
import { Injectable, Logger } from "@nestjs/common";
import { FeederScraperService } from "./feeder-scraper.service";
import { FeederTransformerService } from "./feeder-transformer.service";
import { FeederPersistenceService } from "./feeder-persistence.service";
import {
Sport,
MatchSummary,
@@ -23,7 +23,7 @@ import {
ParsedMarket,
DbEventPayload,
DbMarketPayload,
} from './feeder.types';
} from "./feeder.types";
interface ProcessDateOptions {
onlyCompletedMatches?: boolean;
@@ -37,10 +37,10 @@ export class FeederService {
// Configuration - Adjust these based on rate limiting behavior
private readonly CONCURRENCY_LIMIT = 20; // Increased for maximum speed on EC2
private readonly REQUEST_DELAY_MS = 50; // Minimal delay to respect basics
private readonly HISTORICAL_START_DATE = '2023-06-01'; // 2 years of data
private readonly SPORTS: Sport[] = ['football', 'basketball'];
private readonly HISTORICAL_START_DATE = "2023-06-01"; // 2 years of data
private readonly SPORTS: Sport[] = ["football", "basketball"];
private readonly MAX_RETRIES = 50;
private readonly DAILY_SYNC_TIME_ZONE = 'Europe/Istanbul';
private readonly DAILY_SYNC_TIME_ZONE = "Europe/Istanbul";
constructor(
private readonly scraperService: FeederScraperService,
@@ -56,38 +56,38 @@ export class FeederService {
}
private getYesterdayDateString(timeZone: string): string {
const formatter = new Intl.DateTimeFormat('en-CA', {
const formatter = new Intl.DateTimeFormat("en-CA", {
timeZone,
year: 'numeric',
month: '2-digit',
day: '2-digit',
year: "numeric",
month: "2-digit",
day: "2-digit",
});
const parts = formatter.formatToParts(new Date());
const year = Number(parts.find((part) => part.type === 'year')?.value);
const month = Number(parts.find((part) => part.type === 'month')?.value);
const day = Number(parts.find((part) => part.type === 'day')?.value);
const year = Number(parts.find((part) => part.type === "year")?.value);
const month = Number(parts.find((part) => part.type === "month")?.value);
const day = Number(parts.find((part) => part.type === "day")?.value);
const tzMidnightUtc = new Date(Date.UTC(year, month - 1, day));
tzMidnightUtc.setUTCDate(tzMidnightUtc.getUTCDate() - 1);
return tzMidnightUtc.toISOString().split('T')[0];
return tzMidnightUtc.toISOString().split("T")[0];
}
private getTimeZoneOffsetMs(date: Date, timeZone: string): number {
const formatter = new Intl.DateTimeFormat('en-US', {
const formatter = new Intl.DateTimeFormat("en-US", {
timeZone,
timeZoneName: 'shortOffset',
timeZoneName: "shortOffset",
});
const offsetLabel =
formatter.formatToParts(date).find((part) => part.type === 'timeZoneName')
?.value || 'GMT+0';
formatter.formatToParts(date).find((part) => part.type === "timeZoneName")
?.value || "GMT+0";
const match = offsetLabel.match(/GMT([+-])(\d{1,2})(?::?(\d{2}))?/);
if (!match) return 0;
const sign = match[1] === '-' ? -1 : 1;
const hours = Number(match[2] || '0');
const minutes = Number(match[3] || '0');
const sign = match[1] === "-" ? -1 : 1;
const hours = Number(match[2] || "0");
const minutes = Number(match[3] || "0");
return sign * (hours * 60 + minutes) * 60 * 1000;
}
@@ -96,17 +96,14 @@ export class FeederService {
dateString: string,
timeZone: string,
): { startTs: number; endTs: number } {
const [year, month, day] = dateString.split('-').map(Number);
const [year, month, day] = dateString.split("-").map(Number);
const startGuess = new Date(Date.UTC(year, month - 1, day, 0, 0, 0));
const nextDayGuess = new Date(
Date.UTC(year, month - 1, day + 1, 0, 0, 0),
);
const nextDayGuess = new Date(Date.UTC(year, month - 1, day + 1, 0, 0, 0));
const startOffsetMs = this.getTimeZoneOffsetMs(startGuess, timeZone);
const nextDayOffsetMs = this.getTimeZoneOffsetMs(nextDayGuess, timeZone);
const startMs =
Date.UTC(year, month - 1, day, 0, 0, 0) - startOffsetMs;
const startMs = Date.UTC(year, month - 1, day, 0, 0, 0) - startOffsetMs;
const nextDayStartMs =
Date.UTC(year, month - 1, day + 1, 0, 0, 0) - nextDayOffsetMs;
@@ -117,35 +114,39 @@ export class FeederService {
}
private parseScoreValue(value: unknown): number | null {
if (value === null || value === undefined || value === '') return null;
if (value === null || value === undefined || value === "") return null;
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
private isCompletedMatchSummary(match: MatchSummary): boolean {
if (match.statusBoxContent === 'ERT') return false;
if (match.statusBoxContent === "ERT") return false;
const normalizedState = String(match.state || '')
const normalizedState = String(match.state || "")
.trim()
.toLowerCase();
const normalizedStatus = String(match.status || '')
const normalizedStatus = String(match.status || "")
.trim()
.toLowerCase();
const normalizedSubstate = String(match.substate || '')
const normalizedSubstate = String(match.substate || "")
.trim()
.toLowerCase();
if (['postgame', 'post'].includes(normalizedState)) return true;
if (["postgame", "post"].includes(normalizedState)) return true;
if (
['played', 'finished', 'ft', 'afterpenalties', 'penalties'].includes(
["played", "finished", "ft", "afterpenalties", "penalties"].includes(
normalizedStatus,
)
) {
return true;
}
if (['postgame', 'post', 'played', 'finished', 'ft'].includes(normalizedSubstate)) {
if (
["postgame", "post", "played", "finished", "ft"].includes(
normalizedSubstate,
)
) {
return true;
}
@@ -167,7 +168,7 @@ export class FeederService {
targetLeagueIds: string[] = [],
): Promise<void> {
this.logger.log(
`🗓️ STARTING DAILY COMPLETED MATCH SYNC [Date: ${targetDateStr}] [Sports: ${sports.join(', ')}] ${targetLeagueIds.length > 0 ? `[Filter: ${targetLeagueIds.length} leagues]` : ''}`,
`🗓️ STARTING DAILY COMPLETED MATCH SYNC [Date: ${targetDateStr}] [Sports: ${sports.join(", ")}] ${targetLeagueIds.length > 0 ? `[Filter: ${targetLeagueIds.length} leagues]` : ""}`,
);
for (const sport of sports) {
@@ -191,7 +192,7 @@ export class FeederService {
targetLeagueIds: string[] = [], // NEW: Optional league filter
): Promise<void> {
this.logger.log(
`🚀 STARTING HISTORICAL SCAN [Target: ${sports.join(', ')}] ${targetLeagueIds.length > 0 ? `[Filter: ${targetLeagueIds.length} leagues]` : ''}`,
`🚀 STARTING HISTORICAL SCAN [Target: ${sports.join(", ")}] ${targetLeagueIds.length > 0 ? `[Filter: ${targetLeagueIds.length} leagues]` : ""}`,
);
const startDate = new Date(startDateStr);
@@ -201,7 +202,7 @@ export class FeederService {
// writing to live_matches. Historical scan should only fill matches table.
endDate.setDate(endDate.getDate() - 2);
const stateKey = `historical_scan_state_${sports.join('_')}${targetLeagueIds.length > 0 ? '_filtered' : ''}_desc`;
const stateKey = `historical_scan_state_${sports.join("_")}${targetLeagueIds.length > 0 ? "_filtered" : ""}_desc`;
let currentDate: Date | null = null;
// Resume from saved state
@@ -215,12 +216,12 @@ export class FeederService {
// For reverse scan, we resume from the *next* day backwards, i.e., resumeDate - 1 day
currentDate.setDate(currentDate.getDate() - 1);
this.logger.log(
`📍 Resuming from: ${currentDate.toISOString().split('T')[0]}`,
`📍 Resuming from: ${currentDate.toISOString().split("T")[0]}`,
);
}
}
} catch {
this.logger.warn('Could not read state, starting from beginning');
this.logger.warn("Could not read state, starting from beginning");
}
// Initialize currentDate to endDate if not resuming (or if resume failed)
@@ -231,7 +232,7 @@ export class FeederService {
}
this.logger.log(
`📊 Scanning (Reverse): ${currentDate.toISOString().split('T')[0]} ← ${startDate.toISOString().split('T')[0]}`,
`📊 Scanning (Reverse): ${currentDate.toISOString().split("T")[0]} ← ${startDate.toISOString().split("T")[0]}`,
);
let processedDays = 0;
@@ -239,7 +240,7 @@ export class FeederService {
// REVERSE LOOP: Iterate while currentDate is greater than or equal to startDate
while (currentDate >= startDate) {
const dateString = currentDate.toISOString().split('T')[0];
const dateString = currentDate.toISOString().split("T")[0];
for (const sport of sports) {
await this.processDate(dateString, sport, targetLeagueIds);
@@ -278,7 +279,7 @@ export class FeederService {
currentDate.setDate(currentDate.getDate() - 1);
}
this.logger.log('🎉 HISTORICAL SCAN COMPLETED');
this.logger.log("🎉 HISTORICAL SCAN COMPLETED");
}
// ============================================
@@ -308,9 +309,9 @@ export class FeederService {
break; // Success, exit loop
} catch (e: any) {
const is502 =
e.message?.includes('502') ||
e.message?.includes("502") ||
e.response?.status === 502 ||
e.message?.includes('Bad Gateway');
e.message?.includes("Bad Gateway");
if (is502 && i < 2) {
this.logger.warn(
@@ -341,10 +342,7 @@ export class FeederService {
// regardless of the matchDate query parameter. We must filter by mstUtc
// to ensure we only process matches that actually belong to the target date.
const { startTs: targetDateStartTs, endTs: targetDateEndTs } =
this.getDayBoundsForTimeZone(
dateString,
this.DAILY_SYNC_TIME_ZONE,
);
this.getDayBoundsForTimeZone(dateString, this.DAILY_SYNC_TIME_ZONE);
const dateFilteredMatches = allMatches.filter((m) => {
const matchTs = m.mstUtc;
@@ -518,14 +516,14 @@ export class FeederService {
// ============================================
async refreshMatch(
matchId: string,
scope: 'all' | 'lineups' | 'odds' = 'all',
scope: "all" | "lineups" | "odds" = "all",
): Promise<ProcessResult> {
this.logger.log(`🔄 Refreshing match (${scope}) for ${matchId}`);
const matchRecord = await this.persistenceService.getMatch(matchId);
if (!matchRecord) {
this.logger.warn(`[${matchId}] Refresh failed: Match not in DB`);
return { success: false, retryable: false, error: 'Match not found' };
return { success: false, retryable: false, error: "Match not found" };
}
// Construct MatchSummary from DB record
@@ -538,13 +536,13 @@ export class FeederService {
iddaaCode: matchRecord.iddaaCode,
homeTeam: {
id: matchRecord.homeTeamId,
name: matchRecord.homeTeam?.name || '',
slug: matchRecord.homeTeam?.slug || '',
name: matchRecord.homeTeam?.name || "",
slug: matchRecord.homeTeam?.slug || "",
},
awayTeam: {
id: matchRecord.awayTeamId,
name: matchRecord.awayTeam?.name || '',
slug: matchRecord.awayTeam?.slug || '',
name: matchRecord.awayTeam?.name || "",
slug: matchRecord.awayTeam?.slug || "",
},
score: {
home: matchRecord.scoreHome,
@@ -555,9 +553,9 @@ export class FeederService {
const dummyCompetitions: Record<string, Competition> = {
[summary.competitionId]: {
id: summary.competitionId,
name: 'Unknown',
competitionSlug: '',
country: { id: '', name: '' },
name: "Unknown",
competitionSlug: "",
country: { id: "", name: "" },
},
};
@@ -583,7 +581,7 @@ export class FeederService {
competitions: Record<string, Competition>,
sport: Sport,
force: boolean = false,
scope: 'all' | 'lineups' | 'odds' = 'all', // Add scope flag
scope: "all" | "lineups" | "odds" = "all", // Add scope flag
): Promise<ProcessResult> {
const matchId = matchSummary.id;
const homeTeamId = matchSummary.homeTeam?.id;
@@ -595,7 +593,7 @@ export class FeederService {
}
// Skip postponed matches (ERT = Erteledendi)
if (matchSummary.statusBoxContent === 'ERT') {
if (matchSummary.statusBoxContent === "ERT") {
this.logger.debug(`[${matchId}] Skipped: Postponed match (ERT)`);
return { success: false, retryable: false };
}
@@ -615,9 +613,9 @@ export class FeederService {
return await fn();
} catch (e: any) {
const is502 =
e.message?.includes('502') ||
e.message?.includes("502") ||
e.response?.status === 502 ||
e.message?.includes('Bad Gateway');
e.message?.includes("Bad Gateway");
if (i === retries - 1) throw e; // Last attempt failed
@@ -661,44 +659,44 @@ export class FeederService {
// 1. Fetch Match Header (score, status)
let headerData: ParsedMatchHeader | null = null;
if (scope === 'all') {
if (scope === "all") {
try {
headerData = await fetchResilient('Header', () =>
headerData = await fetchResilient("Header", () =>
this.scraperService.fetchMatchHeader(matchId),
);
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Header fetch failed: ${e.message}`);
}
}
// 2. Sport-specific data fetching
if (sport === 'basketball') {
if (sport === "basketball") {
// Basketball: Box Score (Always if all or lineups)
if (scope === 'all' || scope === 'lineups') {
if (scope === "all" || scope === "lineups") {
try {
const boxData = await fetchResilient('BoxScore', () =>
const boxData = await fetchResilient("BoxScore", () =>
this.scraperService.fetchBasketballBoxScore(matchId),
);
if (boxData) {
const homeParsed = this.scraperService.parseBasketballBoxScore(
boxData.views?.home?.html || '',
boxData.views?.home?.html || "",
);
const awayParsed = this.scraperService.parseBasketballBoxScore(
boxData.views?.away?.html || '',
boxData.views?.away?.html || "",
);
basketballTeamStats =
scope === 'all'
scope === "all"
? {
home: homeParsed.teamTotals,
away: awayParsed.teamTotals,
}
: null;
if (scope === 'all') {
if (scope === "all") {
try {
const details = await fetchResilient('QuarterScores', () =>
const details = await fetchResilient("QuarterScores", () =>
this.scraperService.fetchBasketballDetailsHeader(matchId),
);
if (details && basketballTeamStats) {
@@ -712,7 +710,7 @@ export class FeederService {
};
}
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(
`[${matchId}] Quarter scores fetch failed: ${e.message}`,
);
@@ -748,7 +746,7 @@ export class FeederService {
processPlayers(awayParsed, awayTeamId);
}
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Box score failed: ${e.message}`);
}
}
@@ -756,9 +754,9 @@ export class FeederService {
// Football: Events, Lineups, Stats, Officials
// Key Events
if (scope === 'all') {
if (scope === "all") {
try {
const eventsData = await fetchResilient('Events', () =>
const eventsData = await fetchResilient("Events", () =>
this.scraperService.fetchKeyEvents(matchId),
);
if (eventsData?.keyEvents) {
@@ -781,7 +779,7 @@ export class FeederService {
);
}
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Events failed: ${e.message}`);
}
@@ -850,20 +848,20 @@ export class FeederService {
*/
// Game Stats & Officials
if (scope === 'all') {
if (scope === "all") {
try {
const gameStats = await fetchResilient('Stats', () =>
const gameStats = await fetchResilient("Stats", () =>
this.scraperService.fetchGameStats(matchId),
);
stats = this.transformerService.transformGameStats(gameStats);
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Stats failed: ${e.message}`);
}
// Officials (from match page)
try {
const matchPageHtml = await fetchResilient('Officials', () =>
const matchPageHtml = await fetchResilient("Officials", () =>
this.scraperService.fetchMatchPage(
matchId,
matchSummary.matchSlug,
@@ -875,7 +873,7 @@ export class FeederService {
this.transformerService.parseOfficials(matchPageHtml);
}
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Officials failed: ${e.message}`);
}
}
@@ -883,31 +881,31 @@ export class FeederService {
// 3. Fetch Iddaa Odds (Always if all or odds)
let oddsArray: DbMarketPayload[] = [];
if (scope === 'all' || scope === 'odds') {
if (scope === "all" || scope === "odds") {
try {
let markets: ParsedMarket[] = [];
if (sport === 'basketball') {
if (sport === "basketball") {
markets =
((await fetchResilient('BucketOdds', () =>
((await fetchResilient("BucketOdds", () =>
this.scraperService.fetchBasketballMarkets(matchId),
)) as ParsedMarket[]) || [];
} else {
markets =
((await fetchResilient('IddaaOdds', () =>
((await fetchResilient("IddaaOdds", () =>
this.scraperService.fetchIddaaMarkets(matchId),
)) as ParsedMarket[]) || [];
}
// Logic is same since structure is ParsedMarket[]
oddsArray = this.transformerService.transformIddaaMarkets(markets);
} catch (e: any) {
if (e.message?.includes('502')) hasCriticalError = true;
if (e.message?.includes("502")) hasCriticalError = true;
this.logger.warn(`[${matchId}] Odds failed: ${e.message}`);
}
}
// 4. Persist to Database
let saved = false;
if (scope === 'lineups') {
if (scope === "lineups") {
saved = await this.persistenceService.saveLineups(
matchId,
playersMap,
@@ -915,7 +913,7 @@ export class FeederService {
homeTeamId,
awayTeamId,
);
} else if (scope === 'odds') {
} else if (scope === "odds") {
saved = await this.persistenceService.saveOdds(matchId, oddsArray);
} else {
// Full Update
@@ -962,12 +960,12 @@ export class FeederService {
if (saved && hasCriticalError) {
// Collect missing components
const missingParts: string[] = [];
if (!stats) missingParts.push('Stats');
if (oddsArray.length === 0) missingParts.push('Odds');
if (officialsData.length === 0) missingParts.push('Officials');
if (!stats) missingParts.push("Stats");
if (oddsArray.length === 0) missingParts.push("Odds");
if (officialsData.length === 0) missingParts.push("Officials");
this.logger.warn(
`[${matchId}] Saved with MISSING DATA (502). Missing: [${missingParts.join(', ')}]. Scheduled for retry.`,
`[${matchId}] Saved with MISSING DATA (502). Missing: [${missingParts.join(", ")}]. Scheduled for retry.`,
);
return { success: false, retryable: true };
}
@@ -975,12 +973,12 @@ export class FeederService {
return { success: saved, retryable: !saved };
} catch (error: any) {
const isRetryable =
error.message.includes('502') ||
error.message.includes('504') ||
error.message.includes('ECONNABORTED') ||
error.message.includes('timeout') ||
error.message.includes('ETIMEDOUT') ||
error.message.includes('Unique constraint'); // Concurrency retry
error.message.includes("502") ||
error.message.includes("504") ||
error.message.includes("ECONNABORTED") ||
error.message.includes("timeout") ||
error.message.includes("ETIMEDOUT") ||
error.message.includes("Unique constraint"); // Concurrency retry
if (isRetryable) {
this.logger.warn(`[${matchId}] ${error.message} - Will retry`);
src/modules/feeder/feeder.types.ts
+88 -88
View File
@@ -6,27 +6,27 @@
// ============================================
// SPORT TYPES
// ============================================
export type Sport = 'football' | 'basketball';
export type Sport = "football" | "basketball";
export const SPORTS_CONFIG: Record<
Sport,
{ sportParam: string; iddaaUrlPath: string }
> = {
football: { sportParam: 'Soccer', iddaaUrlPath: 'mac' },
basketball: { sportParam: 'Basketball', iddaaUrlPath: 'basketbol/mac' },
football: { sportParam: "Soccer", iddaaUrlPath: "mac" },
basketball: { sportParam: "Basketball", iddaaUrlPath: "basketbol/mac" },
};
// ============================================
// MATCH STATUS TYPES
// ============================================
export type MatchStatus = 'Cancelled' | 'Played' | 'Playing' | 'Scheduled';
export type MatchStatus = "Cancelled" | "Played" | "Playing" | "Scheduled";
export type MatchState =
| 'preGame'
| 'postGame'
| 'live'
| 'liveGame'
| 'pre'
| 'post';
| "preGame"
| "postGame"
| "live"
| "liveGame"
| "pre"
| "post";
// ============================================
// LIVESCORES API RESPONSE
@@ -115,9 +115,9 @@ export interface KeyEventsResponse {
}
export interface RawKeyEvent {
type: 'goal' | 'card' | 'substitute' | 'penalty-missed';
subType: 'goal' | 'penalty-goal' | 'yc' | 'rc' | 'pm' | 'ps' | null;
position: 'home' | 'away';
type: "goal" | "card" | "substitute" | "penalty-missed";
subType: "goal" | "penalty-goal" | "yc" | "rc" | "pm" | "ps" | null;
position: "home" | "away";
periodId: number; // 1 = 1st half, 2 = 2nd half
timeMin: string;
seconds: number | null;
@@ -135,7 +135,7 @@ export interface TransformedEvent {
playerId: string;
playerName: string;
teamId: string;
eventType: 'goal' | 'card' | 'substitute' | 'other';
eventType: "goal" | "card" | "substitute" | "other";
eventSubtype: string | null;
timeMinute: string;
timeSeconds: number | null;
@@ -145,7 +145,7 @@ export interface TransformedEvent {
scoreAfter: string | null;
playerOutId: string | null;
playerOutName: string | null;
position: 'home' | 'away';
position: "home" | "away";
}
// ============================================
@@ -170,18 +170,18 @@ export interface RawPlayerStats {
personId: string;
matchName: string;
shirtNumber: number | null;
position: 'goalkeeper' | 'defender' | 'midfielder' | 'striker' | 'Coach' | '';
position: "goalkeeper" | "defender" | "midfielder" | "striker" | "Coach" | "";
events: PlayerEvent[] | null;
}
export interface PlayerEvent {
name:
| 'goal'
| 'yellow-card'
| 'red-card'
| 'sub-off'
| 'sub-on'
| 'penalty-missed';
| "goal"
| "yellow-card"
| "red-card"
| "sub-off"
| "sub-on"
| "penalty-missed";
timeMin: string;
count: number;
}
@@ -270,7 +270,7 @@ export interface IddaaMarket {
export interface IddaaOutcome {
outcome: string; // The odds value (e.g., "1.78")
handicap: string | null;
state: 'active' | 'suspended';
state: "active" | "suspended";
label: string; // "1", "X", "2", "Alt", "Üst", etc.
}
@@ -371,7 +371,7 @@ export interface DbEventPayload {
match_id: string;
player_id: string;
team_id: string;
event_type: 'goal' | 'card' | 'substitute';
event_type: "goal" | "card" | "substitute";
event_subtype: string | null;
time_minute: string;
time_seconds: number | null;
@@ -379,7 +379,7 @@ export interface DbEventPayload {
assist_player_id: string | null;
score_after: string | null;
player_out_id: string | null;
position: 'home' | 'away';
position: "home" | "away";
}
export interface DbMarketSelectionPayload {
@@ -402,74 +402,74 @@ export interface DbMarketPayload {
// ============================================
export const MARKET_MAPPING: Record<string, string> = {
// Ana Bahisler
'1': 'Maç Sonucu',
'3': 'Çifte Şans',
'6-11': 'Handikaplı MS (0:1)',
'6-22': 'Handikaplı MS (0:2)',
'611': 'Handikaplı MS (1:0)',
'622': 'Handikaplı MS (2:0)',
'14': 'İlk Yarı / Maç Sonucu',
'15': 'Maç Skoru',
"1": "Maç Sonucu",
"3": "Çifte Şans",
"6-11": "Handikaplı MS (0:1)",
"6-22": "Handikaplı MS (0:2)",
"611": "Handikaplı MS (1:0)",
"622": "Handikaplı MS (2:0)",
"14": "İlk Yarı / Maç Sonucu",
"15": "Maç Skoru",
// Gol Alt/Üst
'180.5': '0.5 Alt/Üst',
'181.5': '1.5 Alt/Üst',
'182.5': '2.5 Alt/Üst',
'183.5': '3.5 Alt/Üst',
'184.5': '4.5 Alt/Üst',
'185.5': '5.5 Alt/Üst',
"180.5": "0.5 Alt/Üst",
"181.5": "1.5 Alt/Üst",
"182.5": "2.5 Alt/Üst",
"183.5": "3.5 Alt/Üst",
"184.5": "4.5 Alt/Üst",
"185.5": "5.5 Alt/Üst",
// Diğer Gol Bahisleri
'11': 'Karşılıklı Gol',
'12': 'Tek / Çift',
'24': 'İlk Golü Kim Atar',
'26': 'Toplam Gol Aralığı',
'32': 'En Çok Gol Olacak Yarı',
"11": "Karşılıklı Gol",
"12": "Tek / Çift",
"24": "İlk Golü Kim Atar",
"26": "Toplam Gol Aralığı",
"32": "En Çok Gol Olacak Yarı",
// Yarı Bahisleri
'4': '1. Yarı Sonucu',
'5': '1. Yarı Çifte Şans',
'54': '2. Yarı Sonucu',
'190.5': '1. Yarı 0.5 Alt/Üst',
'191.5': '1. Yarı 1.5 Alt/Üst',
'192.5': '1. Yarı 2.5 Alt/Üst',
'39': '1. Yarı Karşılıklı Gol',
"4": "1. Yarı Sonucu",
"5": "1. Yarı Çifte Şans",
"54": "2. Yarı Sonucu",
"190.5": "1. Yarı 0.5 Alt/Üst",
"191.5": "1. Yarı 1.5 Alt/Üst",
"192.5": "1. Yarı 2.5 Alt/Üst",
"39": "1. Yarı Karşılıklı Gol",
// Takım Bahisleri
'280.5': 'Ev Sahibi 0.5 Alt/Üst',
'281.5': 'Ev Sahibi 1.5 Alt/Üst',
'282.5': 'Ev Sahibi 2.5 Alt/Üst',
'283.5': 'Ev Sahibi 3.5 Alt/Üst',
'290.5': 'Deplasman 0.5 Alt/Üst',
'291.5': 'Deplasman 1.5 Alt/Üst',
'292.5': 'Deplasman 2.5 Alt/Üst',
'400.5': '1. Yarı Ev Sahibi 0.5 Alt/Üst',
'430.5': '1. Yarı Deplasman 0.5 Alt/Üst',
'37': 'Ev Sahibi Gol Yemeden Kazanır',
'38': 'Deplasman Gol Yemeden Kazanır',
"280.5": "Ev Sahibi 0.5 Alt/Üst",
"281.5": "Ev Sahibi 1.5 Alt/Üst",
"282.5": "Ev Sahibi 2.5 Alt/Üst",
"283.5": "Ev Sahibi 3.5 Alt/Üst",
"290.5": "Deplasman 0.5 Alt/Üst",
"291.5": "Deplasman 1.5 Alt/Üst",
"292.5": "Deplasman 2.5 Alt/Üst",
"400.5": "1. Yarı Ev Sahibi 0.5 Alt/Üst",
"430.5": "1. Yarı Deplasman 0.5 Alt/Üst",
"37": "Ev Sahibi Gol Yemeden Kazanır",
"38": "Deplasman Gol Yemeden Kazanır",
// Korner & Kart
'47': 'En Çok Korner',
'48': '1. Yarı En Çok Korner',
'49': 'İlk Korner',
'43': 'Toplam Korner Aralığı',
'44': '1. Yarı Korner Aralığı',
'463.5': '1. Yarı 3.5 Korner Alt/Üst',
'464.5': '1. Yarı 4.5 Korner Alt/Üst',
'465.5': '1. Yarı 5.5 Korner Alt/Üst',
'53': 'Kırmızı Kart Olur mu?',
'384.5': '4.5 Kart Puanı Alt/Üst',
'385.5': '5.5 Kart Puanı Alt/Üst',
'386.5': '6.5 Kart Puanı Alt/Üst',
"47": "En Çok Korner",
"48": "1. Yarı En Çok Korner",
"49": "İlk Korner",
"43": "Toplam Korner Aralığı",
"44": "1. Yarı Korner Aralığı",
"463.5": "1. Yarı 3.5 Korner Alt/Üst",
"464.5": "1. Yarı 4.5 Korner Alt/Üst",
"465.5": "1. Yarı 5.5 Korner Alt/Üst",
"53": "Kırmızı Kart Olur mu?",
"384.5": "4.5 Kart Puanı Alt/Üst",
"385.5": "5.5 Kart Puanı Alt/Üst",
"386.5": "6.5 Kart Puanı Alt/Üst",
// Kombine
'301.5': 'MS ve 1.5 Alt/Üst',
'302.5': 'MS ve 2.5 Alt/Üst',
'303.5': 'MS ve 3.5 Alt/Üst',
'304.5': 'MS ve 4.5 Alt/Üst',
"301.5": "MS ve 1.5 Alt/Üst",
"302.5": "MS ve 2.5 Alt/Üst",
"303.5": "MS ve 3.5 Alt/Üst",
"304.5": "MS ve 4.5 Alt/Üst",
// İki Yarıyı da Kazanır (39 conflicts with 1. Yarı Karşılıklı Gol, keep that one)
'40': 'Deplasman İki Yarıyı da Kazanır',
"40": "Deplasman İki Yarıyı da Kazanır",
};
// ============================================
@@ -477,20 +477,20 @@ export const MARKET_MAPPING: Record<string, string> = {
// ============================================
export interface AxiosRequestConfig {
headers: {
'User-Agent': string;
"User-Agent": string;
Referer: string;
'X-Requested-With': string;
'Accept-Language'?: string;
"X-Requested-With": string;
"Accept-Language"?: string;
};
timeout: number;
}
export const DEFAULT_HEADERS = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
Referer: 'https://www.mackolik.com/',
'X-Requested-With': 'XMLHttpRequest',
'Accept-Language': 'tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7',
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
Referer: "https://www.mackolik.com/",
"X-Requested-With": "XMLHttpRequest",
"Accept-Language": "tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7",
};
export const DEFAULT_TIMEOUT = 30000;
@@ -516,7 +516,7 @@ export interface SidelinedPlayer {
playerUrl: string;
position: string;
positionShort: string;
type: 'injury' | 'suspension' | 'other';
type: "injury" | "suspension" | "other";
description: string;
matchesMissed: number | null;
average: number | null;