fix(predictions): cooldown fallback cascade + circuit breaker tuning
Deploy Iddaai Backend / build-and-deploy (push) Successful in 27s
Deploy Iddaai Backend / build-and-deploy (push) Successful in 27s
- Add 4-level fallback when AI circuit breaker fires cooldown: 1) In-memory cache (10min TTL) 2) DB stored prediction (no TTL filter) 3) DB cached prediction (with model version check) 4) Wait out cooldown + retry once (max 20s wait) - Raise circuit breaker threshold from 3 to 5 consecutive failures - Reduce cooldown duration from 30s to 15s for faster recovery - Add extractCooldownMs helper to parse remaining ms from error detail
This commit is contained in:
@@ -246,10 +246,21 @@ export class PredictionsService implements OnModuleInit, OnModuleDestroy {
|
||||
const status = requestError.status;
|
||||
const detail = requestError.detail || requestError.message;
|
||||
|
||||
// ── Cooldown fallback cascade: memCache → DB stored → DB cached → wait & retry ──
|
||||
if (
|
||||
status === HttpStatus.SERVICE_UNAVAILABLE &&
|
||||
this.hasCooldown(detail)
|
||||
) {
|
||||
// 1) In-memory cache (10min TTL)
|
||||
const memCached = this.predictionMemCache.get(matchId);
|
||||
if (memCached && Date.now() - memCached.timestamp < 10 * 60 * 1000) {
|
||||
this.logger.warn(
|
||||
`AI Engine cooldown for ${matchId}; returning mem-cached prediction`,
|
||||
);
|
||||
return memCached.payload;
|
||||
}
|
||||
|
||||
// 2) DB stored prediction (no TTL filter)
|
||||
const storedPrediction = await this.getStoredPrediction(matchId);
|
||||
if (storedPrediction) {
|
||||
this.logger.warn(
|
||||
@@ -257,6 +268,43 @@ export class PredictionsService implements OnModuleInit, OnModuleDestroy {
|
||||
);
|
||||
return this.enrichPredictionResponse(storedPrediction, matchContext);
|
||||
}
|
||||
|
||||
// 3) DB cached prediction (with model version check)
|
||||
const cachedPrediction = await this.getCachedPrediction(matchId);
|
||||
if (cachedPrediction) {
|
||||
this.logger.warn(
|
||||
`AI Engine cooldown for ${matchId}; returning cached prediction`,
|
||||
);
|
||||
return this.enrichPredictionResponse(cachedPrediction, matchContext);
|
||||
}
|
||||
|
||||
// 4) No cached data at all — wait out cooldown and retry once
|
||||
const cooldownMs = this.extractCooldownMs(detail);
|
||||
if (cooldownMs > 0 && cooldownMs <= 20000) {
|
||||
this.logger.warn(
|
||||
`AI Engine cooldown for ${matchId}; no cached data — waiting ${cooldownMs}ms and retrying...`,
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, cooldownMs + 500));
|
||||
try {
|
||||
const retryResponse =
|
||||
await this.aiEngineClient.post<MatchPredictionDto>(
|
||||
`/v20plus/analyze/${matchId}`,
|
||||
{ simulate: true, is_simulation: true, pre_match_only: true },
|
||||
);
|
||||
const retryPrediction = this.enrichPredictionResponse(
|
||||
retryResponse.data,
|
||||
matchContext,
|
||||
);
|
||||
await this.recordPredictionRun(matchId, retryResponse.data);
|
||||
await this.cachePrediction(matchId, retryPrediction);
|
||||
return retryPrediction;
|
||||
} catch (retryErr: unknown) {
|
||||
this.logger.error(
|
||||
`AI Engine retry after cooldown also failed for ${matchId}`,
|
||||
);
|
||||
// Fall through to error handling below
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.error(
|
||||
@@ -1243,6 +1291,19 @@ export class PredictionsService implements OnModuleInit, OnModuleDestroy {
|
||||
return false;
|
||||
}
|
||||
|
||||
private extractCooldownMs(detail: unknown): number {
|
||||
if (detail && typeof detail === "object" && "cooldownRemainingMs" in detail) {
|
||||
return Number((detail as Record<string, unknown>).cooldownRemainingMs) || 0;
|
||||
}
|
||||
|
||||
if (typeof detail === "string") {
|
||||
const match = detail.match(/cooldownRemainingMs[":\s]+(\d+)/);
|
||||
return match ? Number(match[1]) : 0;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
private async ensureSmartCouponDataReady(matchIds: string[]): Promise<void> {
|
||||
const uniqueMatchIds = [...new Set(matchIds.filter((id) => !!id))];
|
||||
if (uniqueMatchIds.length === 0) {
|
||||
|
||||
Reference in New Issue
Block a user