d265e3328c
Drop the try/except around update_simple_metrics_adaptive_state in the workflow and the broad except in the activity, so Mongo failures fail the run and show up in the Temporal metrics (PR #59 review). Invalid frequency_alert stays a warning + skip (the orchestrator ignores it too) and a malformed adaptive_state is read as not escalated.
134 lines
5.5 KiB
Python
134 lines
5.5 KiB
Python
from temporalio import workflow
|
|
|
|
with workflow.unsafe.imports_passed_through():
|
|
from datetime import timedelta
|
|
from typing import Any
|
|
|
|
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
|
from sientia_do.temporal.policies import retry_policy
|
|
|
|
from laborious.activities.activities import Activities
|
|
|
|
|
|
@workflow.defn(name='simple_metrics')
|
|
class SimpleMetrics:
|
|
@workflow.run
|
|
async def run(self, input_data: dict[str, Any]):
|
|
"""
|
|
Execute the simple metrics workflow.
|
|
"""
|
|
metadata = {
|
|
'metadata': {
|
|
'model_id': input_data['model_id'],
|
|
'model_name': input_data['model_name'],
|
|
'workflow_name': 'simple_metrics',
|
|
'schedule_name': input_data['schedule_name'],
|
|
}
|
|
}
|
|
|
|
model_id = input_data['model_id']
|
|
interval_minutes = input_data['interval_minutes']
|
|
|
|
model_config = input_data['model_config']
|
|
target_name = model_config['target']
|
|
model_type = model_config.get('model_type')
|
|
thresholds = model_config.get('simple_metrics_thresholds')
|
|
min_samples = model_config.get('simple_metrics_min_samples')
|
|
# Untyped Mongo config: anything but a positive int (bool excluded) turns the guard off
|
|
# instead of raising in workflow code; falsy values mean "unset" and stay silent.
|
|
if type(min_samples) is not int or min_samples <= 0:
|
|
if min_samples:
|
|
workflow.logger.warning(
|
|
f'Invalid simple_metrics_min_samples={min_samples!r}: min samples guard off'
|
|
)
|
|
min_samples = None
|
|
|
|
# Built from internal Temporal workflow config, not user input (bandit B608).
|
|
query = f"""
|
|
select p."timestamp", p.prediction, ld.value as "target"
|
|
from "{input_data['schema']}"."{input_data['predictions_table_name']}" p
|
|
inner join "{input_data['schema']}"."{input_data['data_table_name']}" ld
|
|
on p."timestamp" = ld."timestamp"
|
|
where
|
|
p.model_id = '{model_id}' and
|
|
p.prediction is not null and
|
|
ld.variable = '{target_name}' and
|
|
ld.value is not null and
|
|
p."timestamp" >= NOW() - INTERVAL '{interval_minutes} minutes'
|
|
order by
|
|
p."timestamp" desc;
|
|
""" # nosec
|
|
|
|
target_data = await workflow.execute_activity_method(
|
|
Activities.load_custom_query,
|
|
{
|
|
**metadata,
|
|
'query': query,
|
|
'datetime_columns': ['timestamp'],
|
|
'orient': 'records',
|
|
},
|
|
retry_policy=retry_policy,
|
|
start_to_close_timeout=timedelta(seconds=300),
|
|
)
|
|
|
|
# Too few rows for a reliable value: metrics are still persisted, but a noisy short
|
|
# window must neither alert nor escalate (SIENTIAPDE-1995).
|
|
enough_samples = bool(target_data) and (not min_samples or len(target_data) >= min_samples)
|
|
if target_data and not enough_samples:
|
|
workflow.logger.info(
|
|
f'{len(target_data)} rows < simple_metrics_min_samples={min_samples}: '
|
|
'threshold alerting and escalation skipped'
|
|
)
|
|
|
|
simple_metrics = None
|
|
if target_data:
|
|
simple_metrics = await workflow.execute_local_activity_method(
|
|
Activities.calculate_simple_metrics,
|
|
{
|
|
**metadata,
|
|
'model_id': model_id,
|
|
'target_data': target_data,
|
|
'metrics': input_data.get('metrics', ['rmse', 'mse', 'mae', 'r2']),
|
|
'interval_minutes': interval_minutes,
|
|
'model_type': model_type,
|
|
'thresholds': thresholds if enough_samples else None,
|
|
},
|
|
retry_policy=retry_policy,
|
|
start_to_close_timeout=timedelta(seconds=300),
|
|
)
|
|
|
|
if simple_metrics:
|
|
await workflow.execute_activity_method(
|
|
Activities.export_data_to_postgres,
|
|
{
|
|
**metadata,
|
|
'data': simple_metrics,
|
|
'schema': input_data['schema'],
|
|
'table_name': input_data['target_table_name'],
|
|
'timestamp_conversion': {
|
|
'column': 'timestamp',
|
|
'format': DATETIME_FORMAT_WITH_TZ,
|
|
},
|
|
},
|
|
retry_policy=retry_policy,
|
|
start_to_close_timeout=timedelta(seconds=300),
|
|
)
|
|
|
|
# No try/except: a failure here must fail the run so it shows up in the Temporal metrics
|
|
# (metrics are already persisted above).
|
|
await workflow.execute_activity_method(
|
|
Activities.update_simple_metrics_adaptive_state,
|
|
{
|
|
**metadata,
|
|
# Only a valid breach escalates: an empty or under-sampled window is sent as no
|
|
# rows, which counts as a healthy run (spec decision 8). Always called, even
|
|
# without thresholds (no breach), so an escalated doc recovers after they are
|
|
# removed.
|
|
'simple_metrics': (simple_metrics or []) if enough_samples else [],
|
|
'thresholds': thresholds,
|
|
'interval_minutes': interval_minutes,
|
|
},
|
|
retry_policy=retry_policy,
|
|
start_to_close_timeout=timedelta(minutes=1),
|
|
)
|