main

src/modules/feeder/feeder.service.ts

@@ -385,21 +385,71 @@ export class FeederService {
         return;
       }
 
-      // 2. Filter out already existing matches to skip processing
+      // 2. Filter out already existing matches & patch incomplete ones
       const allIds = matchesToProcess.map((m) => m.id);
       const existingIds =
         await this.persistenceService.getExistingMatchIds(allIds);
       const totalCount = matchesToProcess.length;
 
+      // ── Patch incomplete existing matches ──────────────────────
+      // Find matches that ARE in DB but have missing data scopes
+      const allExistingInDb = await this.persistenceService.getMissingScopes(allIds);
+      if (allExistingInDb.size > 0) {
+        this.logger.log(
+          `[${sport}] [${dateString}] 🔧 Found ${allExistingInDb.size} existing matches with missing data. Patching...`,
+        );
+
+        for (const [matchId, missingScopes] of allExistingInDb) {
+          const matchSummary = matchesToProcess.find((m) => m.id === matchId);
+          if (!matchSummary) continue;
+
+          for (const scope of missingScopes) {
+            await this.delay(500);
+            try {
+              const patchScope: "all" | "lineups" | "odds" =
+                scope === "odds" ? "odds" : scope === "lineups" ? "lineups" : "all";
+
+              const result = await this.processSingleMatch(
+                matchSummary,
+                data.competitions,
+                sport,
+                true, // force
+                patchScope,
+              );
+
+              this.heartbeat();
+              if (result.success) {
+                this.logger.log(
+                  `[${sport}] ✅ Patched [${scope}] for ${matchId} ${matchSummary.homeTeam.name} vs ${matchSummary.awayTeam.name}`,
+                );
+              } else {
+                this.logger.warn(
+                  `[${sport}] ⚠️ Patch [${scope}] failed for ${matchId}`,
+                );
+              }
+            } catch (e: any) {
+              this.logger.warn(
+                `[${sport}] ❌ Patch [${scope}] exception for ${matchId}: ${e.message}`,
+              );
+            }
+          }
+        }
+      }
+      // ─────────────────────────────────────────────────────────────
+
+      // Now filter out COMPLETE existing matches (skip them)
       if (!refreshExistingMatches && existingIds.length > 0) {
+        // Re-check after patching - which ones are now complete?
+        const updatedExistingIds =
+          await this.persistenceService.getExistingMatchIds(allIds);
         matchesToProcess = matchesToProcess.filter(
-          (m) => !existingIds.includes(m.id),
+          (m) => !updatedExistingIds.includes(m.id),
         );
       }
 
       if (matchesToProcess.length === 0) {
         this.logger.log(
-          `[${sport}] [${dateString}] All ${totalCount} matches already exist. Skipping...`,
+          `[${sport}] [${dateString}] All ${totalCount} matches processed (${existingIds.length} existed, ${allExistingInDb.size} patched). Done.`,
         );
         return;
       }
@@ -410,7 +460,7 @@ export class FeederService {
         );
       } else {
         this.logger.log(
-          `[${sport}] [${dateString}] Processing ${matchesToProcess.length}/${totalCount} matches (Skipped ${existingIds.length} existing)`,
+          `[${sport}] [${dateString}] Processing ${matchesToProcess.length}/${totalCount} new matches (${existingIds.length} existing, ${allExistingInDb.size} patched)`,
         );
       }
 
@@ -474,7 +524,7 @@ export class FeederService {
               match,
               data.competitions,
               sport,
-              refreshExistingMatches,
+              true, // FORCE: re-fetch incomplete data
             );
             if (result.success) {
               successCount++;
@@ -778,8 +828,9 @@ export class FeederService {
         if (scope === "all" || scope === "lineups") {
           // Starting Formation
           try {
-            const formationData =
-              await this.scraperService.fetchStartingFormation(matchId);
+            const formationData = await fetchResilient("Formation", () =>
+              this.scraperService.fetchStartingFormation(matchId),
+            );
             if (formationData?.stats) {
               this.transformerService.processLineup(
                 formationData.stats.home || [],
@@ -805,8 +856,9 @@ export class FeederService {
 
           // Substitutes
           try {
-            const subsData =
-              await this.scraperService.fetchSubstitutions(matchId);
+            const subsData = await fetchResilient("Subs", () =>
+              this.scraperService.fetchSubstitutions(matchId),
+            );
             if (subsData?.stats) {
               this.transformerService.processLineup(
                 subsData.stats.home || [],
@@ -887,7 +939,37 @@ export class FeederService {
         }
       }
 
-      // 4. Persist to Database
+      // ── Pre-save completeness gate ──────────────────────────────
+      // If a 502 caused missing data, do NOT save. The data exists on
+      // the API and will be available shortly. Skip and retry instead.
+      const completedMatch = isMatchCompleted({
+        state: headerData?.matchStatus ?? matchSummary.state,
+        status: matchSummary.status,
+        substate: matchSummary.substate,
+        statusBoxContent: matchSummary.statusBoxContent,
+        scoreHome: headerData?.scoreHome ?? matchSummary.score?.home,
+        scoreAway: headerData?.scoreAway ?? matchSummary.score?.away,
+      });
+
+      const missingParts: string[] = [];
+      if (scope === "all" && completedMatch) {
+        if (sport === "football" && !stats) missingParts.push("Stats");
+        if (sport === "football" && participationData.length < 18)
+          missingParts.push("Lineups");
+        if (sport === "basketball" && !basketballTeamStats)
+          missingParts.push("BoxScore");
+        if (oddsArray.length === 0) missingParts.push("Odds");
+      }
+
+      // 502 caused missing data → do NOT save, retry later
+      if (hasCriticalError && missingParts.length > 0) {
+        this.logger.warn(
+          `[${matchId}] ⛔ SKIPPED SAVE: 502 errors caused missing [${missingParts.join(", ")}]. Will retry for complete data.`,
+        );
+        return { success: false, retryable: true };
+      }
+
+      // 4. SAVE
       let saved = false;
       if (scope === "lineups") {
         saved = await this.persistenceService.saveLineups(
@@ -941,34 +1023,11 @@ export class FeederService {
       */
       // ==========================================
 
-      const completedMatch = isMatchCompleted({
-        state: headerData?.matchStatus ?? matchSummary.state,
-        status: matchSummary.status,
-        substate: matchSummary.substate,
-        statusBoxContent: matchSummary.statusBoxContent,
-        scoreHome: headerData?.scoreHome ?? matchSummary.score?.home,
-        scoreAway: headerData?.scoreAway ?? matchSummary.score?.away,
-      });
-
-      const missingParts: string[] = [];
-      if (scope === "all" && completedMatch) {
-        if (sport === "football" && !stats) missingParts.push("Stats");
-        if (sport === "football" && participationData.length < 18)
-          missingParts.push("Lineups");
-        if (sport === "basketball" && !basketballTeamStats)
-          missingParts.push("BoxScore");
-        if (oddsArray.length === 0) missingParts.push("Odds");
-      }
-
-      if (saved && (hasCriticalError || missingParts.length > 0)) {
-        const reason = hasCriticalError
-          ? "missing data after upstream errors"
-          : "incomplete completed-match payload";
-
+      // No 502 but data genuinely missing → save anyway, log warning
+      if (saved && missingParts.length > 0) {
         this.logger.warn(
-          `[${matchId}] Saved with ${reason}. Missing: [${missingParts.join(", ")}]. Scheduled for retry.`,
+          `[${matchId}] Saved but data genuinely missing (no 502): [${missingParts.join(", ")}]`,
         );
-        return { success: false, retryable: true };
       }
 
       return { success: saved, retryable: !saved };