feat: add watchdog timer to detect and recover from hung API requests

src/scripts/run-feeder.ts

@@ -1,12 +1,19 @@
 /**
  * Run Full Historical Feeder
  * Usage: npm run feeder:historical
+ *
+ * Includes a watchdog that kills the process if no activity
+ * is detected for 5 minutes (stuck API request), letting PM2
+ * auto-restart and resume from DB.
  */
 
 import { NestFactory } from "@nestjs/core";
 import { FeederService } from "../modules/feeder/feeder.service";
 import { Logger } from "@nestjs/common";
 
+const WATCHDOG_INTERVAL_MS = 60_000; // Check every 1 minute
+const WATCHDOG_TIMEOUT_MS = 5 * 60_000; // Kill if no activity for 5 minutes
+
 async function bootstrap() {
   process.env.FEEDER_MODE = "historical";
 
@@ -21,8 +28,25 @@ async function bootstrap() {
     logger: ["log", "error", "warn"],
   });
 
+  const feederService = app.get(FeederService);
+
+  // ── Watchdog Timer ──────────────────────────────────────────
+  // If the feeder hangs on an API call for 5+ minutes, force-exit
+  // so PM2 can restart and resume from where it left off in DB.
+  const watchdog = setInterval(() => {
+    const idleMs = Date.now() - feederService.lastActivityAt;
+    if (idleMs > WATCHDOG_TIMEOUT_MS) {
+      logger.error(
+        `🐕 WATCHDOG: No activity for ${Math.round(idleMs / 1000)}s. Force-exiting for PM2 restart...`,
+      );
+      process.exit(1);
+    }
+  }, WATCHDOG_INTERVAL_MS);
+
+  // Don't let the watchdog timer keep the process alive after scan finishes
+  watchdog.unref();
+
   try {
-    const feederService = app.get(FeederService);
     const startDate = process.env.FEEDER_START_DATE || "2023-06-01";
     const sports = (process.env.FEEDER_SPORTS || "football,basketball")
       .split(",")
@@ -36,6 +60,7 @@ async function bootstrap() {
     logger.error(error.stack);
     process.exit(1);
   } finally {
+    clearInterval(watchdog);
     await app.close();
   }