This commit is contained in:
2026-04-16 17:21:48 +03:00
parent c8fa4c442d
commit c8e7e4e927
116 changed files with 3720 additions and 4197 deletions
+105 -105
View File
@@ -3,9 +3,9 @@
* HTTP requests with exact headers from working curl commands
*/
import { Injectable, Logger } from '@nestjs/common';
import axios, { AxiosInstance } from 'axios';
import * as cheerio from 'cheerio';
import { Injectable, Logger } from "@nestjs/common";
import axios, { AxiosInstance } from "axios";
import * as cheerio from "cheerio";
import {
Sport,
SPORTS_CONFIG,
@@ -25,7 +25,7 @@ import {
SidelinedResponse,
SidelinedTeamData,
SidelinedPlayer,
} from './feeder.types';
} from "./feeder.types";
@Injectable()
export class FeederScraperService {
@@ -43,13 +43,13 @@ export class FeederScraperService {
this.axios.interceptors.response.use(
(response) => {
this.logger.debug(
`✅ [${response.config.url?.split('?')[0]}] Status: ${response.status}`,
`✅ [${response.config.url?.split("?")[0]}] Status: ${response.status}`,
);
return response;
},
(error) => {
const status = error.response?.status || 'N/A';
const url = error.config?.url?.split('?')[0] || 'Unknown';
const status = error.response?.status || "N/A";
const url = error.config?.url?.split("?")[0] || "Unknown";
this.logger.error(`❌ [${url}] Status: ${status} - ${error.message}`);
throw error;
},
@@ -72,7 +72,7 @@ export class FeederScraperService {
const response = await this.axios.get(url, {
params: {
'sports[]': sportParam,
"sports[]": sportParam,
matchDate: dateString,
},
});
@@ -80,11 +80,11 @@ export class FeederScraperService {
const payload = response.data as unknown;
if (
!payload ||
typeof payload !== 'object' ||
!('status' in payload) ||
!('data' in payload)
typeof payload !== "object" ||
!("status" in payload) ||
!("data" in payload)
) {
throw new Error('Historical source payload has invalid shape');
throw new Error("Historical source payload has invalid shape");
}
return payload as LivescoresApiResponse;
@@ -101,14 +101,14 @@ export class FeederScraperService {
const response = await this.axios.get(url, {
params: {
matchId,
sdapiLanguageCode: 'tr-mk',
ajaxViewName: 'match-details',
ajaxPartialViewName: 'match-details-status',
displayMode: 'all',
sdapiLanguageCode: "tr-mk",
ajaxViewName: "match-details",
ajaxPartialViewName: "match-details-status",
displayMode: "all",
},
});
return this.parseMatchHeader(response.data.data?.html || '');
return this.parseMatchHeader(response.data.data?.html || "");
}
private parseMatchHeader(html: string): ParsedMatchHeader {
@@ -116,7 +116,7 @@ export class FeederScraperService {
// Extract match-status from data attribute
const matchStatus =
($('[data-match-status]').attr('data-match-status') as any) || 'postGame';
($("[data-match-status]").attr("data-match-status") as any) || "postGame";
// Extract scores
const scoreHome = this.safeInt($('[data-slot="score-home"]').text().trim());
@@ -126,7 +126,7 @@ export class FeederScraperService {
let htScoreHome: number | null = null;
let htScoreAway: number | null = null;
const detailedScore = $('.p0c-soccer-match-details-header__detailed-score')
const detailedScore = $(".p0c-soccer-match-details-header__detailed-score")
.text()
.trim();
const htMatch = detailedScore.match(/\(İY\s*(\d+)\s*-\s*(\d+)\)/);
@@ -143,7 +143,7 @@ export class FeederScraperService {
// ============================================
async fetchKeyEvents(
matchId: string,
): Promise<KeyEventsResponse['data'] | null> {
): Promise<KeyEventsResponse["data"] | null> {
const url = `https://www.mackolik.com/ajax/football/key-events`;
this.logger.debug(`📡 [${matchId}] Fetching key events`);
@@ -151,7 +151,7 @@ export class FeederScraperService {
try {
const response = await this.axios.get<KeyEventsResponse>(url, {
params: {
ajaxViewName: 'events',
ajaxViewName: "events",
matchId,
seasonId: matchId, // Same as matchId
},
@@ -172,7 +172,7 @@ export class FeederScraperService {
// ============================================
async fetchStartingFormation(
matchId: string,
): Promise<MatchStatsResponse['data'] | null> {
): Promise<MatchStatsResponse["data"] | null> {
const url = `https://www.mackolik.com/ajax/football/match-stats`;
this.logger.debug(`📡 [${matchId}] Fetching starting formation`);
@@ -180,7 +180,7 @@ export class FeederScraperService {
try {
const response = await this.axios.get<MatchStatsResponse>(url, {
params: {
ajaxViewName: 'starting-formation',
ajaxViewName: "starting-formation",
matchId,
seasonId: matchId,
},
@@ -201,7 +201,7 @@ export class FeederScraperService {
// ============================================
async fetchSubstitutions(
matchId: string,
): Promise<MatchStatsResponse['data'] | null> {
): Promise<MatchStatsResponse["data"] | null> {
const url = `https://www.mackolik.com/ajax/football/match-stats`;
this.logger.debug(`📡 [${matchId}] Fetching substitutions`);
@@ -209,7 +209,7 @@ export class FeederScraperService {
try {
const response = await this.axios.get<MatchStatsResponse>(url, {
params: {
ajaxViewName: 'substitutions',
ajaxViewName: "substitutions",
matchId,
seasonId: matchId,
},
@@ -230,7 +230,7 @@ export class FeederScraperService {
// ============================================
async fetchGameStats(
matchId: string,
): Promise<GameStatsResponse['data'] | null> {
): Promise<GameStatsResponse["data"] | null> {
const url = `https://www.mackolik.com/ajax/soccer/match/gameStats`;
this.logger.debug(`📡 [${matchId}] Fetching game stats`);
@@ -253,7 +253,7 @@ export class FeederScraperService {
// ============================================
// MANAGER
// ============================================
async fetchManager(matchId: string): Promise<ManagerResponse['data'] | null> {
async fetchManager(matchId: string): Promise<ManagerResponse["data"] | null> {
const url = `https://www.mackolik.com/ajax/football/match-stats`;
this.logger.debug(`📡 [${matchId}] Fetching manager`);
@@ -261,7 +261,7 @@ export class FeederScraperService {
try {
const response = await this.axios.get<ManagerResponse>(url, {
params: {
ajaxViewName: 'manager',
ajaxViewName: "manager",
matchId,
seasonId: matchId,
},
@@ -287,10 +287,10 @@ export class FeederScraperService {
try {
const response = await this.axios.get<IddaaMarketsHtmlResponse>(url, {
params: { template: 'all' },
params: { template: "all" },
});
return this.parseIddaaMarketsHtml(response.data.data?.html || '');
return this.parseIddaaMarketsHtml(response.data.data?.html || "");
} catch (error: any) {
if (error.response?.status === 404) {
this.logger.warn(`[${matchId}] Iddaa markets not found (404)`);
@@ -306,30 +306,30 @@ export class FeederScraperService {
const $ = cheerio.load(html);
const markets: ParsedMarket[] = [];
$('.widget-iddaa-markets__market-item').each((_, marketEl) => {
$(".widget-iddaa-markets__market-item").each((_, marketEl) => {
const $market = $(marketEl);
const marketId = $market.attr('data-market') || '';
const marketId = $market.attr("data-market") || "";
const marketName = $market
.find('.widget-iddaa-markets__header-text')
.find(".widget-iddaa-markets__header-text")
.text()
.trim();
const iddaaCode = $market
.find('.widget-iddaa-markets__iddaa-code')
.find(".widget-iddaa-markets__iddaa-code")
.text()
.trim();
const mbc = $market.find('.widget-iddaa-markets__mbc').text().trim();
const mbc = $market.find(".widget-iddaa-markets__mbc").text().trim();
const selections: ParsedSelection[] = [];
$market.find('.widget-iddaa-markets__option').each((_, optionEl) => {
$market.find(".widget-iddaa-markets__option").each((_, optionEl) => {
const $option = $(optionEl);
selections.push({
shortcode: $option.attr('data-shortcode') || '',
outcomeNo: $option.attr('data-outcome-no') || '',
label: $option.find('.widget-iddaa-markets__label').text().trim(),
value: $option.find('.widget-iddaa-markets__value').text().trim(),
shortcode: $option.attr("data-shortcode") || "",
outcomeNo: $option.attr("data-outcome-no") || "",
label: $option.find(".widget-iddaa-markets__label").text().trim(),
value: $option.find(".widget-iddaa-markets__value").text().trim(),
});
});
@@ -347,7 +347,7 @@ export class FeederScraperService {
// ============================================
async fetchBasketballBoxScore(
matchId: string,
): Promise<BasketballBoxScoreResponse['data'] | null> {
): Promise<BasketballBoxScoreResponse["data"] | null> {
// Updated URL based on user request
const url = `https://www.mackolik.com/ajax/basketball/match/box-score`;
@@ -357,8 +357,8 @@ export class FeederScraperService {
const response = await this.axios.get<BasketballBoxScoreResponse>(url, {
params: { matchId },
headers: {
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': DEFAULT_HEADERS['User-Agent'],
"X-Requested-With": "XMLHttpRequest",
"User-Agent": DEFAULT_HEADERS["User-Agent"],
},
});
@@ -382,25 +382,25 @@ export class FeederScraperService {
const players: Partial<BasketballPlayerStats>[] = [];
// Parse individual players from widget rows
$('.widget-basketball-match-box-score__row').each((_, elem) => {
$(".widget-basketball-match-box-score__row").each((_, elem) => {
const row = $(elem);
// Skip if no player name found
const nameElem = row.find('.widget-basketball-match-box-score__player');
const nameElem = row.find(".widget-basketball-match-box-score__player");
if (!nameElem.length) return;
const name = nameElem.text().trim();
// Indices based on User HTML:
// 0: Name, 1: Min, 2: Pts, 3: Reb, 4: Ast, 5: 2FG, 6: 3FG, 7: FT, 8: Fouls, 9: Blk, 10: Stl, 11: TO
const values = row.find('td');
const values = row.find("td");
// Check if it's a valid player row (should have enough columns)
if (values.length < 10) return;
// Extract ID from link if possible
let playerId = '';
const link = nameElem.find('a').attr('href');
let playerId = "";
const link = nameElem.find("a").attr("href");
if (link) {
playerId = this.extractPlayerIdFromUrl(link) || '';
playerId = this.extractPlayerIdFromUrl(link) || "";
}
players.push({
@@ -410,16 +410,16 @@ export class FeederScraperService {
points: this.safeInt(values.eq(2).text().trim()) || 0,
rebounds: this.safeInt(values.eq(3).text().trim()) || 0,
assists: this.safeInt(values.eq(4).text().trim()) || 0,
fgMade: this.safeInt(values.eq(5).text().trim().split('/')[0]) || 0,
fgMade: this.safeInt(values.eq(5).text().trim().split("/")[0]) || 0,
fgAttempted:
this.safeInt(values.eq(5).text().trim().split('/')[1]) || 0,
this.safeInt(values.eq(5).text().trim().split("/")[1]) || 0,
threePtMade:
this.safeInt(values.eq(6).text().trim().split('/')[0]) || 0,
this.safeInt(values.eq(6).text().trim().split("/")[0]) || 0,
threePtAttempted:
this.safeInt(values.eq(6).text().trim().split('/')[1]) || 0,
ftMade: this.safeInt(values.eq(7).text().trim().split('/')[0]) || 0,
this.safeInt(values.eq(6).text().trim().split("/")[1]) || 0,
ftMade: this.safeInt(values.eq(7).text().trim().split("/")[0]) || 0,
ftAttempted:
this.safeInt(values.eq(7).text().trim().split('/')[1]) || 0,
this.safeInt(values.eq(7).text().trim().split("/")[1]) || 0,
fouls: this.safeInt(values.eq(8).text().trim()) || 0,
blocks: this.safeInt(values.eq(9).text().trim()) || 0,
steals: this.safeInt(values.eq(10).text().trim()) || 0,
@@ -428,7 +428,7 @@ export class FeederScraperService {
});
// Parse Team Totals from Footer
const footerRow = $('.widget-basketball-match-box-score__footer td');
const footerRow = $(".widget-basketball-match-box-score__footer td");
let teamTotals: any = {};
if (footerRow.length > 5) {
@@ -438,16 +438,16 @@ export class FeederScraperService {
points: this.safeInt(footerRow.eq(2).text().trim()) || 0,
rebounds: this.safeInt(footerRow.eq(3).text().trim()) || 0,
assists: this.safeInt(footerRow.eq(4).text().trim()) || 0,
fgMade: this.safeInt(footerRow.eq(5).text().trim().split('/')[0]) || 0,
fgMade: this.safeInt(footerRow.eq(5).text().trim().split("/")[0]) || 0,
fgAttempted:
this.safeInt(footerRow.eq(5).text().trim().split('/')[1]) || 0,
this.safeInt(footerRow.eq(5).text().trim().split("/")[1]) || 0,
threePtMade:
this.safeInt(footerRow.eq(6).text().trim().split('/')[0]) || 0,
this.safeInt(footerRow.eq(6).text().trim().split("/")[0]) || 0,
threePtAttempted:
this.safeInt(footerRow.eq(6).text().trim().split('/')[1]) || 0,
ftMade: this.safeInt(footerRow.eq(7).text().trim().split('/')[0]) || 0,
this.safeInt(footerRow.eq(6).text().trim().split("/")[1]) || 0,
ftMade: this.safeInt(footerRow.eq(7).text().trim().split("/")[0]) || 0,
ftAttempted:
this.safeInt(footerRow.eq(7).text().trim().split('/')[1]) || 0,
this.safeInt(footerRow.eq(7).text().trim().split("/")[1]) || 0,
fouls: this.safeInt(footerRow.eq(8).text().trim()) || 0,
blocks: this.safeInt(footerRow.eq(9).text().trim()) || 0,
steals: this.safeInt(footerRow.eq(10).text().trim()) || 0,
@@ -474,11 +474,11 @@ export class FeederScraperService {
// For HTML pages, we DON'T send X-Requested-With header
const response = await this.axios.get(url, {
headers: {
'User-Agent': DEFAULT_HEADERS['User-Agent'],
Referer: DEFAULT_HEADERS['Referer'],
'Accept-Language': DEFAULT_HEADERS['Accept-Language'],
"User-Agent": DEFAULT_HEADERS["User-Agent"],
Referer: DEFAULT_HEADERS["Referer"],
"Accept-Language": DEFAULT_HEADERS["Accept-Language"],
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
// NO X-Requested-With for HTML pages!
},
});
@@ -507,8 +507,8 @@ export class FeederScraperService {
const response = await this.axios.get(url, {
params: { matchId },
headers: {
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': DEFAULT_HEADERS['User-Agent'],
"X-Requested-With": "XMLHttpRequest",
"User-Agent": DEFAULT_HEADERS["User-Agent"],
},
});
@@ -532,12 +532,12 @@ export class FeederScraperService {
const $ = cheerio.load(html);
const rows = $(
'.widget-basketball-match-details-header__score-details tbody tr',
".widget-basketball-match-details-header__score-details tbody tr",
);
if (rows.length < 2) return null;
const parseRow = (row: any) => {
const cols = $(row).find('td');
const cols = $(row).find("td");
// Format: TeamName, Q1, Q2, Q3, Q4, Final
// Values are inside .widget-basketball-match-details-header__score-part (just the quarter score)
// or direct text if simple table.
@@ -545,7 +545,7 @@ export class FeederScraperService {
const getScore = (index: number) => {
const cell = cols.eq(index);
const part = cell.find(
'.widget-basketball-match-details-header__score-part',
".widget-basketball-match-details-header__score-part",
);
const val = part.length ? part.text() : cell.text();
return this.safeInt(val.trim());
@@ -580,10 +580,10 @@ export class FeederScraperService {
try {
const response = await this.axios.get<IddaaMarketsHtmlResponse>(url, {
params: { template: 'all' },
params: { template: "all" },
headers: {
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': DEFAULT_HEADERS['User-Agent'],
"X-Requested-With": "XMLHttpRequest",
"User-Agent": DEFAULT_HEADERS["User-Agent"],
},
});
@@ -602,7 +602,7 @@ export class FeederScraperService {
extractPlayerIdFromUrl(url: string | undefined): string | null {
if (!url) return null;
const parts = url.split('/');
const parts = url.split("/");
return parts[parts.length - 1] || null;
}
@@ -620,12 +620,12 @@ export class FeederScraperService {
try {
const response = await this.axios.get(url, {
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7',
Referer: 'https://www.mackolik.com',
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7",
Referer: "https://www.mackolik.com",
},
timeout: 10000,
});
@@ -652,24 +652,24 @@ export class FeederScraperService {
$: cheerio.CheerioAPI,
teamIndex: number,
): SidelinedTeamData {
const sidelinedWidgets = $('.widget-sidelined-players');
const sidelinedWidgets = $(".widget-sidelined-players");
if (sidelinedWidgets.length <= teamIndex) {
return { teamName: '', teamId: '', totalSidelined: 0, players: [] };
return { teamName: "", teamId: "", totalSidelined: 0, players: [] };
}
const widget = sidelinedWidgets.eq(teamIndex);
const teamCrest = widget.find('.widget-sidelined-players__header-crest');
const teamCrestSrc = teamCrest.attr('src') || '';
const teamId = teamCrestSrc.split('/').pop() || '';
const teamCrest = widget.find(".widget-sidelined-players__header-crest");
const teamCrestSrc = teamCrest.attr("src") || "";
const teamId = teamCrestSrc.split("/").pop() || "";
const teamName = widget
.find('.widget-sidelined-players__header-text')
.find(".widget-sidelined-players__header-text")
.text()
.trim();
const players: SidelinedPlayer[] = [];
widget.find('.widget-sidelined-players__item').each((_, element) => {
widget.find(".widget-sidelined-players__item").each((_, element) => {
const playerData = this._parsePlayerItem($, $(element));
if (playerData) {
players.push(playerData);
@@ -689,44 +689,44 @@ export class FeederScraperService {
$item: cheerio.Cheerio<any>,
): SidelinedPlayer | null {
try {
const nameElem = $item.find('.widget-sidelined-players__name');
const nameElem = $item.find(".widget-sidelined-players__name");
const playerName = nameElem.text().trim();
const playerUrl = nameElem.attr('href') || '';
const playerId = playerUrl.split('/').pop() || '';
const playerUrl = nameElem.attr("href") || "";
const playerId = playerUrl.split("/").pop() || "";
const positionElem = $item.find('.widget-sidelined-players__position');
const position = positionElem.attr('title') || '';
const positionElem = $item.find(".widget-sidelined-players__position");
const position = positionElem.attr("title") || "";
const positionShort = positionElem.text().trim();
const reasonImg = $item.find('.widget-sidelined-players__reason img');
const reasonIcon = reasonImg.attr('src') || '';
const reasonImg = $item.find(".widget-sidelined-players__reason img");
const reasonIcon = reasonImg.attr("src") || "";
const numbers = $item.find('.widget-sidelined-players__number');
const numbers = $item.find(".widget-sidelined-players__number");
// Use parseInt EXACTLY as in JS script (ignoring potential NaN for now, will handle via helper if needed but safer to stick to script logic first)
const matchesMissedText =
numbers.length > 0 ? numbers.eq(0).text().trim() : '';
numbers.length > 0 ? numbers.eq(0).text().trim() : "";
const matchesMissed = matchesMissedText
? parseInt(matchesMissedText, 10)
: null;
const averageText = numbers.length > 1 ? numbers.eq(1).text().trim() : '';
const averageText = numbers.length > 1 ? numbers.eq(1).text().trim() : "";
const average = averageText ? parseInt(averageText, 10) : null;
const description = $item
.find('.widget-sidelined-players__value')
.find(".widget-sidelined-players__value")
.text()
.trim();
const type = reasonIcon.includes('shortage_1.png')
? 'injury'
: reasonIcon.includes('suspension')
? 'suspension'
: 'other';
const type = reasonIcon.includes("shortage_1.png")
? "injury"
: reasonIcon.includes("suspension")
? "suspension"
: "other";
return {
playerId,
playerName,
playerUrl: playerUrl.startsWith('http')
playerUrl: playerUrl.startsWith("http")
? playerUrl
: `https://www.mackolik.com${playerUrl}`,
position,
@@ -735,7 +735,7 @@ export class FeederScraperService {
description,
matchesMissed: isNaN(matchesMissed as number) ? null : matchesMissed,
average: isNaN(average as number) ? null : average,
reasonIcon: reasonIcon.startsWith('http')
reasonIcon: reasonIcon.startsWith("http")
? reasonIcon
: `https://www.mackolik.com${reasonIcon}`, // Keep safer URL construction but stick closer to logic
};