fix(simple_metrics): skip invalid interval_minutes and restart a malformed healthy_streak (SIENTIAPDE-2191)
- interval_minutes is untyped Mongo config: a string or null now warns and skips the adaptive state instead of failing every run (TypeError). - A hand-edited healthy_streak (null, text, inf) restarts the count and keeps escalated, instead of raising on every run. - README: step 4 runs on every schedule, so a MongoDB failure fails the run.
This commit is contained in:
@@ -85,7 +85,10 @@ def next_adaptive_state(state: dict[str, Any], breached: bool, runs: int) -> dic
|
||||
return {'escalated': True, 'healthy_streak': 0}
|
||||
if not state.get('escalated'):
|
||||
return {'escalated': False, 'healthy_streak': 0}
|
||||
streak = int(state.get('healthy_streak', 0)) + 1
|
||||
try:
|
||||
streak = int(state.get('healthy_streak', 0)) + 1
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
streak = 1 # hand-edited streak (null, text, inf): restart the count, stay escalated
|
||||
if streak >= runs:
|
||||
return {'escalated': False, 'healthy_streak': 0}
|
||||
return {'escalated': True, 'healthy_streak': streak}
|
||||
|
||||
@@ -192,11 +192,14 @@ class MongoActivities(SientiaMonitoring):
|
||||
|
||||
try:
|
||||
runs = recovery_runs(input_data['interval_minutes'], str(frequency_alert))
|
||||
except ValueError:
|
||||
# Config error, not a failure: the orchestrator ignores this value too (keeps
|
||||
# `frequency`) and already sends SIMPLE_METRICS_FREQUENCY_ALERT_BELOW_FLOOR for it.
|
||||
except (TypeError, ValueError):
|
||||
# Config error, not a failure: the orchestrator ignores an invalid `frequency_alert`
|
||||
# too (keeps `frequency`) and already sends SIMPLE_METRICS_FREQUENCY_ALERT_BELOW_FLOOR
|
||||
# for it. `interval_minutes` is untyped Mongo config as well (e.g. "60" or null).
|
||||
self.warning(
|
||||
f'Invalid frequency_alert={frequency_alert!r}: adaptive state skipped', metadata
|
||||
f'Invalid frequency_alert={frequency_alert!r} or '
|
||||
f'interval_minutes={input_data["interval_minutes"]!r}: adaptive state skipped',
|
||||
metadata,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user