feat(simple_metrics): warn when frequency_alert is below the 1m floor (SIENTIAPDE-2190)
This commit is contained in:
@@ -20,6 +20,7 @@ with workflow.unsafe.imports_passed_through():
|
||||
from orchestrator.utils.orchestrator_functions import (
|
||||
build_tag_config,
|
||||
drift,
|
||||
frequency_alert_warning,
|
||||
gather_read_tags,
|
||||
minimal_retrain,
|
||||
pi_web_api_scouter,
|
||||
@@ -409,6 +410,24 @@ class Formatters(SientiaMonitoring):
|
||||
schedules, current_schedules, to_update, to_create, namespace, metadata
|
||||
)
|
||||
|
||||
# Only schedules being created/updated, so this fires once per config change.
|
||||
changed = {**to_create[self.laborious_namespace], **to_update[self.laborious_namespace]}
|
||||
for schedule_name, schedule in changed.items():
|
||||
warning = frequency_alert_warning(schedule.get('frequency_alert'))
|
||||
if warning:
|
||||
self.send_notification(
|
||||
metadata={
|
||||
**metadata,
|
||||
'schedule_name': schedule_name,
|
||||
'model_id': schedule.get('model_id'),
|
||||
'model_name': schedule.get('model_name'),
|
||||
},
|
||||
notification_id='SIMPLE_METRICS_FREQUENCY_ALERT_BELOW_FLOOR',
|
||||
message=warning,
|
||||
block='create_schedule_config',
|
||||
level=NotificationLevel.WARNING,
|
||||
)
|
||||
|
||||
for namespace, schedules in current_schedule_config.items():
|
||||
for schedule_name in schedules:
|
||||
if schedule_name not in schedule_config[namespace]:
|
||||
|
||||
@@ -115,6 +115,27 @@ def simple_metrics(config: dict[str, Any]):
|
||||
}
|
||||
|
||||
|
||||
MIN_FREQUENCY_ALERT_SECONDS = 60
|
||||
|
||||
|
||||
def frequency_alert_warning(frequency_alert: str | None) -> str | None:
|
||||
"""
|
||||
Warning message when `frequency_alert` is below the 1-minute floor or does not parse,
|
||||
otherwise None. Warn only: the configured value is kept (SIENTIAPDE-1995 decision 7).
|
||||
"""
|
||||
if not frequency_alert:
|
||||
return None
|
||||
try:
|
||||
# str(): a non-string value (e.g. a number typed into the doc) must warn, not raise
|
||||
# AttributeError and fail the whole create_schedule_config activity.
|
||||
seconds = parse_frequency(str(frequency_alert))
|
||||
except ValueError:
|
||||
return f'frequency_alert={frequency_alert!r} is not a valid frequency'
|
||||
if seconds < MIN_FREQUENCY_ALERT_SECONDS:
|
||||
return f'frequency_alert={frequency_alert} is below the 1m floor; kept as configured'
|
||||
return None
|
||||
|
||||
|
||||
def minimal_retrain(config: dict[str, Any]):
|
||||
"""
|
||||
Build minimal retrain configuration from pipeline config.
|
||||
|
||||
Reference in New Issue
Block a user