fix(simple_metrics): fail the run explicitly when the adaptive state update fails (SIENTIAPDE-2191)
Drop the try/except around update_simple_metrics_adaptive_state in the workflow and the broad except in the activity, so Mongo failures fail the run and show up in the Temporal metrics (PR #59 review). Invalid frequency_alert stays a warning + skip (the orchestrator ignores it too) and a malformed adaptive_state is read as not escalated.
This commit is contained in:
PedroHMCosme
@@ -524,7 +524,7 @@ class ModelMetrics(SientiaMonitoring):
|
||||
@activity.defn(name='update_simple_metrics_adaptive_state')
|
||||
def update_simple_metrics_adaptive_state(self, input_data: dict[str, Any]) -> dict | None:
|
||||
"""
|
||||
Best-effort update of the adaptive frequency state on the schedule's `pipelines` doc.
|
||||
Update the adaptive frequency state on the schedule's `pipelines` doc.
|
||||
|
||||
The orchestrator derives the schedule frequency from `adaptive_state.escalated` and
|
||||
`frequency_alert`. State is read from Mongo, not from the workflow input, because the
|
||||
@@ -541,48 +541,62 @@ class ModelMetrics(SientiaMonitoring):
|
||||
- interval_minutes (int): Window size, used to compute the recovery runs.
|
||||
|
||||
Return:
|
||||
The new state, or `None` when adaptive mode is off, Mongo is not configured, or the
|
||||
read/parse/write failed.
|
||||
The new state, or `None` when adaptive mode is off, Mongo is not configured, the doc
|
||||
is missing or `frequency_alert` is invalid.
|
||||
|
||||
Raises:
|
||||
Exception: Mongo read/write failures, so the workflow run fails explicitly.
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
if self.mongo_repository is None:
|
||||
return None
|
||||
|
||||
# Mongo errors propagate on purpose: the run must fail explicitly so the problem shows
|
||||
# up in the Temporal metrics (PR #59 review). Metrics were already exported by then.
|
||||
query = {'schedule_name': metadata['schedule_name']}
|
||||
try:
|
||||
pipeline = self.mongo_repository.find_one(self._PIPELINES_COLLECTION, query, metadata)
|
||||
if not pipeline:
|
||||
# Likely misconfig: MONGODB_DATABASE_NAME differs from the orchestrator's, or the
|
||||
# schedule_name does not match its doc.
|
||||
self.warning(
|
||||
f'No pipelines doc for schedule {metadata["schedule_name"]}: '
|
||||
'adaptive state skipped',
|
||||
metadata,
|
||||
)
|
||||
return None
|
||||
if not pipeline.get('frequency_alert'):
|
||||
return None
|
||||
|
||||
runs = recovery_runs(input_data['interval_minutes'], pipeline['frequency_alert'])
|
||||
breached = bool(
|
||||
threshold_breaches(input_data['simple_metrics'], input_data.get('thresholds'))
|
||||
pipeline = self.mongo_repository.find_one(self._PIPELINES_COLLECTION, query, metadata)
|
||||
if not pipeline:
|
||||
# Likely misconfig: MONGODB_DATABASE_NAME differs from the orchestrator's, or the
|
||||
# schedule_name does not match its doc.
|
||||
self.warning(
|
||||
f'No pipelines doc for schedule {metadata["schedule_name"]}: '
|
||||
'adaptive state skipped',
|
||||
metadata,
|
||||
)
|
||||
state = pipeline.get('adaptive_state') or {'escalated': False, 'healthy_streak': 0}
|
||||
new_state = next_adaptive_state(state, breached, runs)
|
||||
if new_state == state:
|
||||
return new_state
|
||||
|
||||
update: dict[str, Any] = {'adaptive_state': new_state}
|
||||
if new_state['escalated'] != bool(state.get('escalated')):
|
||||
# BSON datetime, never a string: the orchestrator normalizes `updated_at` with
|
||||
# `.replace(tzinfo=UTC).strftime(...)` and a string would fail its whole tick.
|
||||
update['updated_at'] = now()
|
||||
self.mongo_repository.update_one(
|
||||
self._PIPELINES_COLLECTION, query, {'$set': update}, metadata
|
||||
)
|
||||
except Exception as e:
|
||||
self.error(f'Failed to update simple metrics adaptive state: {e}', metadata)
|
||||
return None
|
||||
frequency_alert = pipeline.get('frequency_alert')
|
||||
if not frequency_alert:
|
||||
return None
|
||||
|
||||
try:
|
||||
runs = recovery_runs(input_data['interval_minutes'], str(frequency_alert))
|
||||
except ValueError:
|
||||
# Config error, not a failure: the orchestrator ignores this value too (keeps
|
||||
# `frequency`) and already sends SIMPLE_METRICS_FREQUENCY_ALERT_BELOW_FLOOR for it.
|
||||
self.warning(
|
||||
f'Invalid frequency_alert={frequency_alert!r}: adaptive state skipped', metadata
|
||||
)
|
||||
return None
|
||||
|
||||
breached = bool(
|
||||
threshold_breaches(input_data['simple_metrics'], input_data.get('thresholds'))
|
||||
)
|
||||
state = pipeline.get('adaptive_state')
|
||||
if not isinstance(state, dict):
|
||||
# Missing or hand-edited garbage: not escalated, same as the orchestrator reads it.
|
||||
state = {'escalated': False, 'healthy_streak': 0}
|
||||
new_state = next_adaptive_state(state, breached, runs)
|
||||
if new_state == state:
|
||||
return new_state
|
||||
|
||||
update: dict[str, Any] = {'adaptive_state': new_state}
|
||||
if new_state['escalated'] != bool(state.get('escalated')):
|
||||
# BSON datetime, never a string: the orchestrator normalizes `updated_at` with
|
||||
# `.replace(tzinfo=UTC).strftime(...)` and a string would fail its whole tick.
|
||||
update['updated_at'] = now()
|
||||
self.mongo_repository.update_one(
|
||||
self._PIPELINES_COLLECTION, query, {'$set': update}, metadata
|
||||
)
|
||||
|
||||
self.info(f'Adaptive state of {query["schedule_name"]}: {state} -> {new_state}', metadata)
|
||||
return new_state
|
||||
|
||||
@@ -114,23 +114,20 @@ class SimpleMetrics:
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
try:
|
||||
await workflow.execute_activity_method(
|
||||
Activities.update_simple_metrics_adaptive_state,
|
||||
{
|
||||
**metadata,
|
||||
# Only a valid breach escalates: an empty or under-sampled window is sent as
|
||||
# no rows, which counts as a healthy run (spec decision 8). Always called, even
|
||||
# without thresholds (no breach), so an escalated doc recovers after they are
|
||||
# removed.
|
||||
'simple_metrics': (simple_metrics or []) if enough_samples else [],
|
||||
'thresholds': thresholds,
|
||||
'interval_minutes': interval_minutes,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
except Exception as e:
|
||||
# Best-effort: metrics are already persisted. retry_policy has maximum_attempts=1, so
|
||||
# a dispatch failure would otherwise fail the workflow (same as prediction_process).
|
||||
workflow.logger.warning(f'Failed to update adaptive state, continuing anyway: {e}')
|
||||
# No try/except: a failure here must fail the run so it shows up in the Temporal metrics
|
||||
# (metrics are already persisted above).
|
||||
await workflow.execute_activity_method(
|
||||
Activities.update_simple_metrics_adaptive_state,
|
||||
{
|
||||
**metadata,
|
||||
# Only a valid breach escalates: an empty or under-sampled window is sent as no
|
||||
# rows, which counts as a healthy run (spec decision 8). Always called, even
|
||||
# without thresholds (no breach), so an escalated doc recovers after they are
|
||||
# removed.
|
||||
'simple_metrics': (simple_metrics or []) if enough_samples else [],
|
||||
'thresholds': thresholds,
|
||||
'interval_minutes': interval_minutes,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user