From 4c7930e9d28153d331287c03c41ed3b157d73ac9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fahri=20Can=20Se=C3=A7er?= <fahricansecer@gmail.com>
Date: Sat, 25 Apr 2026 11:20:30 +0300
Subject: [PATCH] feat: add watchdog timer to detect and recover from hung API
 requests

---
 src/modules/feeder/feeder.service.ts | 11 +++++++++++
 src/scripts/run-feeder.ts            | 27 ++++++++++++++++++++++++++-
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/src/modules/feeder/feeder.service.ts b/src/modules/feeder/feeder.service.ts
index 389e67a..db2164a 100755
--- a/src/modules/feeder/feeder.service.ts
+++ b/src/modules/feeder/feeder.service.ts
@@ -43,6 +43,14 @@ export class FeederService {
   private readonly MAX_RETRIES = 50;
   private readonly DAILY_SYNC_TIME_ZONE = "Europe/Istanbul";
 
+  /** Watchdog heartbeat – updated on every match/date activity */
+  public lastActivityAt: number = Date.now();
+
+  /** Call this to bump the heartbeat */
+  private heartbeat(): void {
+    this.lastActivityAt = Date.now();
+  }
+
   constructor(
     private readonly scraperService: FeederScraperService,
     private readonly transformerService: FeederTransformerService,
@@ -259,6 +267,7 @@ export class FeederService {
   ): Promise<void> {
     const { onlyCompletedMatches = false, refreshExistingMatches = false } =
       options;
+    this.heartbeat();
     this.logger.log(`[${sport}] 📅 Processing: ${dateString}`);
 
     try {
@@ -431,6 +440,7 @@ export class FeederService {
             refreshExistingMatches,
           );
 
+          this.heartbeat();
           if (result.success) {
             this.logger.log(
               `[${sport}] ✅ successful for ${match.id} ${match.homeTeam.name} vs ${match.awayTeam.name}`,
@@ -443,6 +453,7 @@ export class FeederService {
             failedMatches.push(match);
           }
         } catch (e: any) {
+          this.heartbeat();
           this.logger.warn(
             `[${sport}] Sequential error for ${match.id}: ${e.message}`,
           );
diff --git a/src/scripts/run-feeder.ts b/src/scripts/run-feeder.ts
index b238bb4..fd5e6ed 100755
--- a/src/scripts/run-feeder.ts
+++ b/src/scripts/run-feeder.ts
@@ -1,12 +1,19 @@
 /**
  * Run Full Historical Feeder
  * Usage: npm run feeder:historical
+ *
+ * Includes a watchdog that kills the process if no activity
+ * is detected for 5 minutes (stuck API request), letting PM2
+ * auto-restart and resume from DB.
  */
 
 import { NestFactory } from "@nestjs/core";
 import { FeederService } from "../modules/feeder/feeder.service";
 import { Logger } from "@nestjs/common";
 
+const WATCHDOG_INTERVAL_MS = 60_000; // Check every 1 minute
+const WATCHDOG_TIMEOUT_MS = 5 * 60_000; // Kill if no activity for 5 minutes
+
 async function bootstrap() {
   process.env.FEEDER_MODE = "historical";
 
@@ -21,8 +28,25 @@ async function bootstrap() {
     logger: ["log", "error", "warn"],
   });
 
+  const feederService = app.get(FeederService);
+
+  // ── Watchdog Timer ──────────────────────────────────────────
+  // If the feeder hangs on an API call for 5+ minutes, force-exit
+  // so PM2 can restart and resume from where it left off in DB.
+  const watchdog = setInterval(() => {
+    const idleMs = Date.now() - feederService.lastActivityAt;
+    if (idleMs > WATCHDOG_TIMEOUT_MS) {
+      logger.error(
+        `🐕 WATCHDOG: No activity for ${Math.round(idleMs / 1000)}s. Force-exiting for PM2 restart...`,
+      );
+      process.exit(1);
+    }
+  }, WATCHDOG_INTERVAL_MS);
+
+  // Don't let the watchdog timer keep the process alive after scan finishes
+  watchdog.unref();
+
   try {
-    const feederService = app.get(FeederService);
     const startDate = process.env.FEEDER_START_DATE || "2023-06-01";
     const sports = (process.env.FEEDER_SPORTS || "football,basketball")
       .split(",")
@@ -36,6 +60,7 @@ async function bootstrap() {
     logger.error(error.stack);
     process.exit(1);
   } finally {
+    clearInterval(watchdog);
     await app.close();
   }