From 3b55072ab4e538ac4a30f7e28d428ac55e3752ff Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 19 Aug 2026 10:57:42 -0300 Subject: [PATCH 01/29] chore(deps): bump sientia_model>=0.12.0, local pin to 0.13.1 RegressionMetrics class (needed for SIENTIAPDE-1986) was introduced in sientia-model-library 0.12.0. The previous local pin (@0.10.0) predates the class. Co-Authored-By: Claude Opus 4.6 --- requirements-local.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-local.txt b/requirements-local.txt index 73a9a47..aea28bd 100644 --- a/requirements-local.txt +++ b/requirements-local.txt @@ -4,7 +4,7 @@ sqlalchemy asyncua==1.0.6 redis git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.1 -git+ssh://git@github.com/Aignosi/sientia-model-library.git@0.10.0 +git+ssh://git@github.com/Aignosi/sientia-model-library.git@0.13.1 prometheus-client botocore boto3 diff --git a/requirements.txt b/requirements.txt index eff54bd..a0218de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ sqlalchemy asyncua==1.0.6 redis sientia_do>=1.12.1 -sientia_model>=0.8.2 +sientia_model>=0.12.0 prometheus-client botocore boto3 From 33b4ae840606fd5b85e46ba2581b1ef0aa3b81eb Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 19 Aug 2026 11:08:07 -0300 Subject: [PATCH 02/29] feat(simple_metrics): thread model_type from model_config to activity Extracts model_type from model_config (optional, defaults to None when absent) and passes it to calculate_simple_metrics. Required for the r2 lock - is_r2_supported() needs model_type to decide whether to compute r2. Updates workflow tests to expect the new key in the activity-call dict. --- laborious/workflows/simple_metrics.py | 2 ++ tests/laborious/workflows/test_simple_metrics.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/laborious/workflows/simple_metrics.py b/laborious/workflows/simple_metrics.py index 38b47c4..e8cd9fb 100644 --- a/laborious/workflows/simple_metrics.py +++ b/laborious/workflows/simple_metrics.py @@ -31,6 +31,7 @@ class SimpleMetrics: model_config = input_data['model_config'] target_name = model_config['target'] + model_type = model_config.get('model_type') query = f""" select p."timestamp", p.prediction, ld.value as "target" @@ -70,6 +71,7 @@ class SimpleMetrics: 'target_data': target_data, 'metrics': input_data.get('metrics', ['rmse', 'mse', 'mae', 'r2']), 'interval_minutes': interval_minutes, + 'model_type': model_type, }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=300), diff --git a/tests/laborious/workflows/test_simple_metrics.py b/tests/laborious/workflows/test_simple_metrics.py index dbda735..2423cc5 100644 --- a/tests/laborious/workflows/test_simple_metrics.py +++ b/tests/laborious/workflows/test_simple_metrics.py @@ -103,6 +103,7 @@ async def test_run(workflow_mock: AsyncMock, simple_metrics: SimpleMetrics): 'target_data': target_data, 'metrics': input_data['metrics'], 'interval_minutes': input_data['interval_minutes'], + 'model_type': None, }, retry_policy=ANY, start_to_close_timeout=ANY, @@ -209,6 +210,7 @@ async def test_run_default_metrics(workflow_mock: AsyncMock, simple_metrics: Sim 'target_data': target_data, 'metrics': ['rmse', 'mse', 'mae', 'r2'], # Default value 'interval_minutes': input_data['interval_minutes'], + 'model_type': None, }, retry_policy=ANY, start_to_close_timeout=ANY, From b7fdcebc15e836ae9781470c0f551cbdb4eff574 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 19 Aug 2026 11:13:08 -0300 Subject: [PATCH 03/29] feat(simple_metrics): rewrite calculate_simple_metrics to use RegressionMetrics Replaces manual numpy if/elif chain with RegressionMetrics from sientia_model. Fixes 3 known bugs in one pass: - NaN now handled via _align_dropna (was silently propagated) - r2 zero-variance uses sklearn r2_score (was divergent, pinned at 0.0) - Unknown metric names raise ValueError (were silently dropped) Also adds r2 lock: when model_type is known and non-linear, r2 is excluded from calculation with a WARNING notification. Drops the now-unused `numpy` import (the manual math it backed is gone, and no other method in this file references it). SIENTIAPDE-1986 --- laborious/activities/model_metrics.py | 78 +++++++++++++-------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index cb44805..b9b1841 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -6,7 +6,6 @@ with workflow.unsafe.imports_passed_through(): import warnings from typing import Any - import numpy as np import pandas as pd from pandas import DataFrame, Index, Series, to_datetime from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler @@ -16,6 +15,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.sientia_monitoring import SientiaMonitoring from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ from sientia_model.analytics.drift_analysis import DriftAnalysis, DriftInsufficientDataError + from sientia_model.metrics.regression import RegressionMetrics from laborious import metrics from laborious.utils.dataframe_debug import build_dataframe_debug_message @@ -365,62 +365,62 @@ class ModelMetrics(SientiaMonitoring): @activity.defn(name='calculate_simple_metrics') def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict]: """ - Calculate simple metrics for a model. Metrics available are: - - rmse - - mse - - mae - - r2 - - accuracy - - precision - - recall - - f1 + Calculate simple regression metrics for a model using RegressionMetrics. + Args: - input_data (dict[str, Any]): Input data containing: + input_data: Input data containing: - metadata (dict): Workflow execution metadata - model_id (str): ID of the MLFlow model - - target_data (pd.DataFrame): Target data for calculating metrics, containing target and prediction columns - - metrics (list[str]): List of metrics to calculate + - target_data (list[dict]): Target data with target, prediction, timestamp columns + - metrics (list[str]): Metric names to calculate + - interval_minutes (int): Window interval in minutes + - model_type (str | None): Model algorithm type (for r2 lock) Returns: - dict[Hashable, Any]: Dictionary containing the calculated metrics + list[dict]: Records with metric, value, model_id, timestamp, data_size, interval_minutes """ - metadata = input_data['metadata'] model_id = input_data['model_id'] target_data = DataFrame(input_data['target_data']) - metric_names = input_data['metrics'] + metric_names = list(input_data['metrics']) interval_minutes = input_data['interval_minutes'] + model_type = input_data.get('model_type') data_size = target_data.shape[0] - output_data = [] + # Filter r2 when model_type is known and unsupported + if ( + model_type + and 'r2' in metric_names + and not RegressionMetrics.is_r2_supported(model_type) + ): + metric_names = [m for m in metric_names if m != 'r2'] + self.warning( + f'r2 excluded for model {model_id}: not supported for model_type={model_type}', + metadata, + ) + self.send_notification( + metadata=metadata, + notification_id='SIMPLE_METRICS_R2_UNSUPPORTED', + message=f'r2 excluded: not a valid metric for model_type={model_type}', + block='model_metrics', + level=NotificationLevel.WARNING, + ) - diff = target_data['target'] - target_data['prediction'] - diff_squared = diff**2 + if not metric_names: + self.info(f'No metrics to calculate for model {model_id} after filtering', metadata) + return [] self.info(f'Calculating simple metrics for model {model_id}: {metric_names}', metadata) - for metric in metric_names: - if metric == 'rmse': - output_data.append({'metric': 'rmse', 'value': np.sqrt(np.mean(diff_squared))}) - elif metric == 'mse': - output_data.append({'metric': 'mse', 'value': np.mean(diff_squared)}) - elif metric == 'mae': - output_data.append({'metric': 'mae', 'value': np.mean(np.abs(diff))}) - elif metric == 'r2': - y_true = target_data['target'] - y_mean = np.mean(y_true) + # Build Series with DatetimeIndex for RegressionMetrics + timestamps = pd.to_datetime(target_data['timestamp']) + real_data = Series(target_data['target'].values, index=timestamps, dtype=float) + predictions = Series(target_data['prediction'].values, index=timestamps, dtype=float) - ss_res = np.sum(diff_squared) - ss_tot = np.sum((y_true - y_mean) ** 2) - - # Evita divisão por zero - if ss_tot == 0: - r2_score = 0.0 - else: - r2_score = 1 - (ss_res / ss_tot) - - output_data.append({'metric': 'r2', 'value': r2_score}) + regression = RegressionMetrics(real_data, predictions) + output_data = regression.calculate(metric_names) + # Wrap with metadata columns matching the existing output schema data = DataFrame(output_data) data['model_id'] = model_id data['timestamp'] = target_data['timestamp'].max() From 666188a7b30095ba3febc29e447128078a91ccaf Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 19 Aug 2026 11:17:59 -0300 Subject: [PATCH 04/29] test(simple_metrics): rewrite tests against RegressionMetrics dispatch Rewrites 8 existing tests to mock RegressionMetrics instead of verifying manual numpy math. Adds 2 new tests: - r2 excluded + warning when model_type is non-linear - r2 included when model_type is absent (backward compat) Replaces silent-discard test with ValueError propagation test. SIENTIAPDE-1986 --- .../activities/test_model_metrics.py | 251 ++++++++++++------ 1 file changed, 168 insertions(+), 83 deletions(-) diff --git a/tests/laborious/activities/test_model_metrics.py b/tests/laborious/activities/test_model_metrics.py index 4c41109..ede6a27 100644 --- a/tests/laborious/activities/test_model_metrics.py +++ b/tests/laborious/activities/test_model_metrics.py @@ -45,6 +45,23 @@ metadata = { } +def _mock_regression_metrics_class(calculate_return): + """Returns a patch context manager that mocks RegressionMetrics.""" + mock_instance = MagicMock() + mock_instance.calculate.return_value = calculate_return + mock_class = MagicMock(return_value=mock_instance) + mock_class.is_r2_supported = MagicMock(return_value=True) + mock_class.supported_metrics = MagicMock(return_value=['rmse', 'mse', 'mae', 'r2']) + return ( + patch( + 'laborious.activities.model_metrics.RegressionMetrics', + mock_class, + ), + mock_class, + mock_instance, + ) + + def test_calculate_drift_invalid_chunk_period(model_metrics_activity): # Arrange input_data = { @@ -693,7 +710,6 @@ def test_get_drift_metrics_dataframe_error( def test_calculate_simple_metrics_success_all_metrics(model_metrics_activity): - # Arrange target_data = DataFrame( { 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'], @@ -710,28 +726,28 @@ def test_calculate_simple_metrics_success_all_metrics(model_metrics_activity): 'interval_minutes': 5, } - # Act - result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) - - # Assert - assert len(result['metric']) == 4 - assert 'rmse' in result['metric'].values - assert 'mse' in result['metric'].values - assert 'mae' in result['metric'].values - assert 'r2' in result['metric'].values - assert all(model_id == 'test_model_id' for model_id in result['model_id'].values) - assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values) - assert all(data_size == 3 for data_size in result['data_size'].values) - assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values) - model_metrics_activity.info.assert_called_once_with( - "Calculating simple metrics for model test_model_id: ['rmse', 'mse', 'mae', 'r2']", - metadata['metadata'], + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [ + {'metric': 'rmse', 'value': 0.1}, + {'metric': 'mse', 'value': 0.01}, + {'metric': 'mae', 'value': 0.1}, + {'metric': 'r2', 'value': 0.99}, + ] ) - model_metrics_activity.debug.assert_called_once() + + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert len(result) == 4 + assert set(result['metric'].values) == {'rmse', 'mse', 'mae', 'r2'} + assert all(mid == 'test_model_id' for mid in result['model_id'].values) + assert all(ts == '2023-05-26 11:12:29' for ts in result['timestamp'].values) + assert all(ds == 3 for ds in result['data_size'].values) + assert all(im == 5 for im in result['interval_minutes'].values) + mock_instance.calculate.assert_called_once_with(['rmse', 'mse', 'mae', 'r2']) def test_calculate_simple_metrics_success_rmse_only(model_metrics_activity): - # Arrange target_data = DataFrame( { 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], @@ -748,23 +764,23 @@ def test_calculate_simple_metrics_success_rmse_only(model_metrics_activity): 'interval_minutes': 5, } - # Act - result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [{'metric': 'rmse', 'value': 0.1}] + ) - # Assert - assert len(result['metric']) == 1 + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert len(result) == 1 assert result['metric'].values[0] == 'rmse' assert result['model_id'].values[0] == 'test_model_id' assert result['timestamp'].values[0] == '2023-05-26 11:12:28' assert result['data_size'].values[0] == 2 assert result['interval_minutes'].values[0] == 5 - model_metrics_activity.info.assert_called_once_with( - "Calculating simple metrics for model test_model_id: ['rmse']", metadata['metadata'] - ) + mock_instance.calculate.assert_called_once_with(['rmse']) def test_calculate_simple_metrics_success_mse_only(model_metrics_activity): - # Arrange target_data = DataFrame( { 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], @@ -781,23 +797,23 @@ def test_calculate_simple_metrics_success_mse_only(model_metrics_activity): 'interval_minutes': 5, } - # Act - result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [{'metric': 'mse', 'value': 0.01}] + ) - # Assert - assert len(result['metric']) == 1 + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert len(result) == 1 assert result['metric'].values[0] == 'mse' assert result['model_id'].values[0] == 'test_model_id' assert result['timestamp'].values[0] == '2023-05-26 11:12:28' assert result['data_size'].values[0] == 2 assert result['interval_minutes'].values[0] == 5 - model_metrics_activity.info.assert_called_once_with( - "Calculating simple metrics for model test_model_id: ['mse']", metadata['metadata'] - ) + mock_instance.calculate.assert_called_once_with(['mse']) def test_calculate_simple_metrics_success_mae_only(model_metrics_activity): - # Arrange target_data = DataFrame( { 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], @@ -814,23 +830,23 @@ def test_calculate_simple_metrics_success_mae_only(model_metrics_activity): 'interval_minutes': 5, } - # Act - result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [{'metric': 'mae', 'value': 0.1}] + ) - # Assert - assert len(result['metric']) == 1 + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert len(result) == 1 assert result['metric'].values[0] == 'mae' assert result['model_id'].values[0] == 'test_model_id' assert result['timestamp'].values[0] == '2023-05-26 11:12:28' assert result['data_size'].values[0] == 2 assert result['interval_minutes'].values[0] == 5 - model_metrics_activity.info.assert_called_once_with( - "Calculating simple metrics for model test_model_id: ['mae']", metadata['metadata'] - ) + mock_instance.calculate.assert_called_once_with(['mae']) def test_calculate_simple_metrics_success_r2_only(model_metrics_activity): - # Arrange target_data = DataFrame( { 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], @@ -847,24 +863,24 @@ def test_calculate_simple_metrics_success_r2_only(model_metrics_activity): 'interval_minutes': 5, } - # Act - result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [{'metric': 'r2', 'value': 0.95}] + ) - # Assert - assert len(result['metric']) == 1 + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert len(result) == 1 assert result['metric'].values[0] == 'r2' assert result['model_id'].values[0] == 'test_model_id' assert result['timestamp'].values[0] == '2023-05-26 11:12:28' assert result['data_size'].values[0] == 2 assert result['interval_minutes'].values[0] == 5 - model_metrics_activity.info.assert_called_once_with( - "Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata'] - ) + mock_instance.calculate.assert_called_once_with(['r2']) -def test_calculate_simple_metrics_r2_zero_ss_tot(model_metrics_activity): - # Arrange - # All target values are the same, so ss_tot will be 0 +def test_calculate_simple_metrics_r2_zero_variance_delegates_to_lib(model_metrics_activity): + """r2 zero-variance is now the lib's responsibility. Activity just passes through.""" target_data = DataFrame( { 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], @@ -881,24 +897,18 @@ def test_calculate_simple_metrics_r2_zero_ss_tot(model_metrics_activity): 'interval_minutes': 5, } - # Act - result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) - - # Assert - assert len(result['metric']) == 1 - assert result['metric'].values[0] == 'r2' - assert result['value'].values[0] == 0.0 # Should return 0.0 when ss_tot == 0 - assert result['model_id'].values[0] == 'test_model_id' - assert result['timestamp'].values[0] == '2023-05-26 11:12:28' - assert result['data_size'].values[0] == 2 - assert result['interval_minutes'].values[0] == 5 - model_metrics_activity.info.assert_called_once_with( - "Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata'] + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [{'metric': 'r2', 'value': 0.0}] ) + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert result['value'].values[0] == 0.0 + mock_instance.calculate.assert_called_once_with(['r2']) + def test_calculate_simple_metrics_success_multiple_metrics_subset(model_metrics_activity): - # Arrange target_data = DataFrame( { 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'], @@ -915,23 +925,26 @@ def test_calculate_simple_metrics_success_multiple_metrics_subset(model_metrics_ 'interval_minutes': 5, } - # Act - result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) - - # Assert - assert len(result['metric']) == 2 - assert 'rmse' in result['metric'].values - assert 'mae' in result['metric'].values - assert all(model_id == 'test_model_id' for model_id in result['model_id'].values) - assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values) - assert all(data_size == 3 for data_size in result['data_size'].values) - assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values) - model_metrics_activity.info.assert_called_once_with( - "Calculating simple metrics for model test_model_id: ['rmse', 'mae']", metadata['metadata'] + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [ + {'metric': 'rmse', 'value': 0.1}, + {'metric': 'mae', 'value': 0.1}, + ] ) + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) -def test_calculate_simple_metrics_unknown_metric_ignored(model_metrics_activity): + assert len(result) == 2 + assert set(result['metric'].values) == {'rmse', 'mae'} + assert all(mid == 'test_model_id' for mid in result['model_id'].values) + assert all(ts == '2023-05-26 11:12:29' for ts in result['timestamp'].values) + assert all(ds == 3 for ds in result['data_size'].values) + assert all(im == 5 for im in result['interval_minutes'].values) + mock_instance.calculate.assert_called_once_with(['rmse', 'mae']) + + +def test_calculate_simple_metrics_unknown_metric_raises(model_metrics_activity): target_data = DataFrame( { 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], @@ -948,7 +961,79 @@ def test_calculate_simple_metrics_unknown_metric_ignored(model_metrics_activity) 'interval_minutes': 5, } - result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + mock_instance = MagicMock() + mock_instance.calculate.side_effect = ValueError('Unknown metric: unknown_metric') + mock_class = MagicMock(return_value=mock_instance) + mock_class.is_r2_supported = MagicMock(return_value=True) - assert len(result['metric']) == 1 + with patch('laborious.activities.model_metrics.RegressionMetrics', mock_class): + with raises(ValueError, match='Unknown metric'): + model_metrics_activity.calculate_simple_metrics(input_data) + + +def test_calculate_simple_metrics_r2_skipped_for_nonlinear_model(model_metrics_activity): + """When model_type is non-linear, r2 is excluded and a warning notification fires.""" + target_data = DataFrame( + { + 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], + 'target': [1.0, 2.0], + 'prediction': [1.1, 2.1], + } + ) + + input_data = { + **metadata, + 'model_id': 'test_model_id', + 'target_data': target_data.to_dict(), + 'metrics': ['rmse', 'r2'], + 'interval_minutes': 5, + 'model_type': 'XGBoost', + } + + mock_instance = MagicMock() + mock_instance.calculate.return_value = [{'metric': 'rmse', 'value': 0.1}] + mock_class = MagicMock(return_value=mock_instance) + mock_class.is_r2_supported = MagicMock(return_value=False) + + with patch('laborious.activities.model_metrics.RegressionMetrics', mock_class): + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert len(result) == 1 assert result['metric'].values[0] == 'rmse' + mock_class.is_r2_supported.assert_called_once_with('XGBoost') + mock_instance.calculate.assert_called_once_with(['rmse']) + model_metrics_activity.warning.assert_called_once() + model_metrics_activity.send_notification.assert_called_once() + call_kwargs = model_metrics_activity.send_notification.call_args.kwargs + assert call_kwargs['notification_id'] == 'SIMPLE_METRICS_R2_UNSUPPORTED' + assert call_kwargs['level'] == NotificationLevel.WARNING + + +def test_calculate_simple_metrics_no_model_type_includes_r2(model_metrics_activity): + """When model_type is None (legacy input), r2 is included without check.""" + target_data = DataFrame( + { + 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], + 'target': [1.0, 2.0], + 'prediction': [1.1, 2.1], + } + ) + + input_data = { + **metadata, + 'model_id': 'test_model_id', + 'target_data': target_data.to_dict(), + 'metrics': ['r2'], + 'interval_minutes': 5, + # no model_type key + } + + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [{'metric': 'r2', 'value': 0.95}] + ) + + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert result['metric'].values[0] == 'r2' + mock_class.is_r2_supported.assert_not_called() From 4e3b5756be4ec0db5903bbcb556db0fbe012c20d Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 19 Aug 2026 11:25:49 -0300 Subject: [PATCH 05/29] feat(simple_metrics): thread thresholds config from model_config to activity Reads optional simple_metrics_thresholds from model_config and passes to calculate_simple_metrics. Threshold schema: {rmse_max, r2_min, ...}. None when not configured (no alerting, no crash). Updates workflow tests to expect the new key in the activity-call dict. SIENTIAPDE-1986 Co-Authored-By: Claude Opus 4.6 --- laborious/workflows/simple_metrics.py | 2 ++ tests/laborious/workflows/test_simple_metrics.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/laborious/workflows/simple_metrics.py b/laborious/workflows/simple_metrics.py index e8cd9fb..3253517 100644 --- a/laborious/workflows/simple_metrics.py +++ b/laborious/workflows/simple_metrics.py @@ -32,6 +32,7 @@ class SimpleMetrics: model_config = input_data['model_config'] target_name = model_config['target'] model_type = model_config.get('model_type') + thresholds = model_config.get('simple_metrics_thresholds') query = f""" select p."timestamp", p.prediction, ld.value as "target" @@ -72,6 +73,7 @@ class SimpleMetrics: 'metrics': input_data.get('metrics', ['rmse', 'mse', 'mae', 'r2']), 'interval_minutes': interval_minutes, 'model_type': model_type, + 'thresholds': thresholds, }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=300), diff --git a/tests/laborious/workflows/test_simple_metrics.py b/tests/laborious/workflows/test_simple_metrics.py index 2423cc5..5d94f60 100644 --- a/tests/laborious/workflows/test_simple_metrics.py +++ b/tests/laborious/workflows/test_simple_metrics.py @@ -104,6 +104,7 @@ async def test_run(workflow_mock: AsyncMock, simple_metrics: SimpleMetrics): 'metrics': input_data['metrics'], 'interval_minutes': input_data['interval_minutes'], 'model_type': None, + 'thresholds': None, }, retry_policy=ANY, start_to_close_timeout=ANY, @@ -211,6 +212,7 @@ async def test_run_default_metrics(workflow_mock: AsyncMock, simple_metrics: Sim 'metrics': ['rmse', 'mse', 'mae', 'r2'], # Default value 'interval_minutes': input_data['interval_minutes'], 'model_type': None, + 'thresholds': None, }, retry_policy=ANY, start_to_close_timeout=ANY, From 5997210118e1a3f073c1a6939a24374062c51f2d Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 19 Aug 2026 11:29:14 -0300 Subject: [PATCH 06/29] feat(simple_metrics): add threshold-based alerting via send_notification Compares each computed metric against optional per-model thresholds from model_config.simple_metrics_thresholds. Convention: - {metric}_max: breach when value > threshold (rmse, mse, mae) - {metric}_min: breach when value < threshold (r2) Fires WARNING notification on breach. Missing thresholds = no alerting. Schema designed to be extensible for Card 2 (Drift) thresholds. SIENTIAPDE-1986 --- laborious/activities/model_metrics.py | 30 ++++ .../activities/test_model_metrics.py | 132 ++++++++++++++++++ 2 files changed, 162 insertions(+) diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index b9b1841..1bc0a6d 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -429,4 +429,34 @@ class ModelMetrics(SientiaMonitoring): self._debug_dataframe(f'Simple metrics dataframe: Size {data.shape}', data, metadata) + # Threshold alerting (optional — no crash when absent) + thresholds = input_data.get('thresholds') + if thresholds: + # Convention: _max thresholds breach when value > threshold, + # _min thresholds breach when value < threshold. + for row in output_data: + metric_name = row['metric'] + value = row['value'] + max_key = f'{metric_name}_max' + min_key = f'{metric_name}_min' + + breach_msg = None + if max_key in thresholds and value > thresholds[max_key]: + breach_msg = f'{metric_name}={value} exceeds {max_key}={thresholds[max_key]}' + elif min_key in thresholds and value < thresholds[min_key]: + breach_msg = f'{metric_name}={value} below {min_key}={thresholds[min_key]}' + + if breach_msg: + self.warning( + f'Threshold breach for model {model_id}: {breach_msg}', + metadata, + ) + self.send_notification( + metadata=metadata, + notification_id='SIMPLE_METRICS_THRESHOLD_BREACH', + message=f'Threshold breach for model {model_id}: {breach_msg}', + block='model_metrics', + level=NotificationLevel.WARNING, + ) + return data.to_dict(orient='records') diff --git a/tests/laborious/activities/test_model_metrics.py b/tests/laborious/activities/test_model_metrics.py index ede6a27..a71e9ba 100644 --- a/tests/laborious/activities/test_model_metrics.py +++ b/tests/laborious/activities/test_model_metrics.py @@ -1037,3 +1037,135 @@ def test_calculate_simple_metrics_no_model_type_includes_r2(model_metrics_activi assert result['metric'].values[0] == 'r2' mock_class.is_r2_supported.assert_not_called() + + +def test_calculate_simple_metrics_threshold_breach_rmse(model_metrics_activity): + """When rmse exceeds rmse_max, a WARNING notification fires.""" + target_data = DataFrame( + { + 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], + 'target': [1.0, 2.0], + 'prediction': [1.1, 2.1], + } + ) + + input_data = { + **metadata, + 'model_id': 'test_model_id', + 'target_data': target_data.to_dict(), + 'metrics': ['rmse'], + 'interval_minutes': 5, + 'thresholds': {'rmse_max': 0.05}, + } + + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [ + {'metric': 'rmse', 'value': 0.1}, + ] + ) + + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert len(result) == 1 + model_metrics_activity.send_notification.assert_called_once() + call_kwargs = model_metrics_activity.send_notification.call_args.kwargs + assert call_kwargs['notification_id'] == 'SIMPLE_METRICS_THRESHOLD_BREACH' + assert call_kwargs['level'] == NotificationLevel.WARNING + assert 'rmse' in call_kwargs['message'] + + +def test_calculate_simple_metrics_threshold_no_breach(model_metrics_activity): + """When rmse is below rmse_max, no notification fires.""" + target_data = DataFrame( + { + 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], + 'target': [1.0, 2.0], + 'prediction': [1.1, 2.1], + } + ) + + input_data = { + **metadata, + 'model_id': 'test_model_id', + 'target_data': target_data.to_dict(), + 'metrics': ['rmse'], + 'interval_minutes': 5, + 'thresholds': {'rmse_max': 1.0}, + } + + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [ + {'metric': 'rmse', 'value': 0.1}, + ] + ) + + with patcher: + model_metrics_activity.calculate_simple_metrics(input_data) + + model_metrics_activity.send_notification.assert_not_called() + + +def test_calculate_simple_metrics_threshold_r2_below_min(model_metrics_activity): + """When r2 drops below r2_min, a WARNING notification fires.""" + target_data = DataFrame( + { + 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], + 'target': [1.0, 2.0], + 'prediction': [1.1, 2.1], + } + ) + + input_data = { + **metadata, + 'model_id': 'test_model_id', + 'target_data': target_data.to_dict(), + 'metrics': ['r2'], + 'interval_minutes': 5, + 'thresholds': {'r2_min': 0.95}, + } + + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [ + {'metric': 'r2', 'value': 0.8}, + ] + ) + + with patcher: + model_metrics_activity.calculate_simple_metrics(input_data) + + model_metrics_activity.send_notification.assert_called_once() + call_kwargs = model_metrics_activity.send_notification.call_args.kwargs + assert 'r2' in call_kwargs['message'] + + +def test_calculate_simple_metrics_no_thresholds_no_alert(model_metrics_activity): + """When thresholds is None (not configured), no alerting, no crash.""" + target_data = DataFrame( + { + 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], + 'target': [1.0, 2.0], + 'prediction': [1.1, 2.1], + } + ) + + input_data = { + **metadata, + 'model_id': 'test_model_id', + 'target_data': target_data.to_dict(), + 'metrics': ['rmse'], + 'interval_minutes': 5, + # no thresholds key + } + + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [ + {'metric': 'rmse', 'value': 999.0}, + ] + ) + + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert len(result) == 1 + model_metrics_activity.send_notification.assert_not_called() From a0b916999f881ff820ca3becc929f72a4769bfaf Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 2 Sep 2026 08:22:22 -0300 Subject: [PATCH 07/29] test(e2e): fix S.1.1 expected values to match RegressionMetrics rounding RegressionMetrics rounds every metric to 2 decimals (documented in lib.md); the old test compared against unrounded closed-form values with a 1e-6 tolerance, which the rewritten activity can no longer satisfy. Round the expected values the same way before comparing. Confirmed via real Docker-backed Postgres/MinIO/Mongo + Temporal test environment: all 4 simple_metrics e2e scenarios pass. --- e2e/test_simple_metrics.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/e2e/test_simple_metrics.py b/e2e/test_simple_metrics.py index c232e65..900fd24 100644 --- a/e2e/test_simple_metrics.py +++ b/e2e/test_simple_metrics.py @@ -140,7 +140,8 @@ async def test_simple_metrics_happy_path_persists_all_metrics_and_columns( prediction/target pair set and written one row per metric. Every column expected by ``sientia_data.simple_metrics`` must be populated (except the nullable ``timestamp`` column) and the numerical values must match - closed-form expectations. + closed-form expectations, rounded to 2 decimals as ``RegressionMetrics`` + persists them (see ``sientia_model/metrics/regression.py``). """ client = temporal_test_env.client model_id = 511 @@ -154,13 +155,13 @@ async def test_simple_metrics_happy_path_persists_all_metrics_and_columns( ] diffs = [target - prediction for prediction, target in pairs] n = len(diffs) - expected_rmse = math.sqrt(sum(d * d for d in diffs) / n) - expected_mse = sum(d * d for d in diffs) / n - expected_mae = sum(abs(d) for d in diffs) / n + expected_rmse = round(math.sqrt(sum(d * d for d in diffs) / n), 2) + expected_mse = round(sum(d * d for d in diffs) / n, 2) + expected_mae = round(sum(abs(d) for d in diffs) / n, 2) target_mean = sum(t for _, t in pairs) / n ss_res = sum((target - prediction) ** 2 for prediction, target in pairs) ss_tot = sum((t - target_mean) ** 2 for _, t in pairs) - expected_r2 = 1.0 - (ss_res / ss_tot) + expected_r2 = round(1.0 - (ss_res / ss_tot), 2) _seed_predictions_and_targets(postgres_engine, model_id=model_id, pairs=pairs) From f7b457e21562f768c107037cc8fa251fbeb93121 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 2 Sep 2026 08:22:38 -0300 Subject: [PATCH 08/29] chore(deps): bump scikit-learn pin to 1.8.0 to match basic runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sientia-dev test cluster's "basic" plugin runtime declares scikit-learn==1.8.0, installed on top of this repo's own requirements.txt at container startup. The old scikit-learn==1.5.2 pin here got silently overwritten by that later install, but the son-approx model's pickled LinearRegression wrapper was recreated against a version needing sklearn 1.8's API — mismatched envs caused `ImportError: cannot import name 'validate_data' from sklearn.utils.validation` on model load (found while testing SIENTIAPDE-1986 against the real cluster). Matching the pin here to 1.8.0 removes the redundant conflicting install. --- requirements-local.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-local.txt b/requirements-local.txt index aea28bd..c970def 100644 --- a/requirements-local.txt +++ b/requirements-local.txt @@ -15,4 +15,4 @@ hyperopt shap pycurl scipy<1.14.0 -scikit-learn==1.5.2 \ No newline at end of file +scikit-learn==1.8.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a0218de..ecf3261 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,4 @@ hyperopt shap pycurl scipy<1.14.0 -scikit-learn==1.5.2 \ No newline at end of file +scikit-learn==1.8.0 \ No newline at end of file From e7c90e6ebb06624b2cfd98e08449df5f24ae76af Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 2 Sep 2026 08:51:15 -0300 Subject: [PATCH 09/29] chore(deps): sync requirements.txt with release/SIENTIAPDE-1646's trim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vitor trimmed release/SIENTIAPDE-1646's requirements.txt to drop explicit pins for packages already pulled transitively by sientia_do/ sientia_model (psycopg2-binary, sqlalchemy, redis, botocore, s3fs, pyarrow, kaleido, hyperopt, shap, pycurl) — confirmed all of these install anyway via sientia_do/sientia_model's own deps in the cluster pod logs. Fewer redundant top-level pins means fewer chances for a version mismatch like the scikit-learn one from earlier today. Also bumps sientia_model floor to >=0.14.0 to match, and mirrors the same trim in requirements-local.txt for local-dev parity — but keeps this branch's sientia-model-library@0.13.1 git pin there (release's own requirements-local.txt is stale at @0.10.0, predating the RegressionMetrics class this card depends on). --- requirements-local.txt | 12 +----------- requirements.txt | 12 +----------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/requirements-local.txt b/requirements-local.txt index c970def..5603103 100644 --- a/requirements-local.txt +++ b/requirements-local.txt @@ -1,18 +1,8 @@ temporalio -psycopg2-binary -sqlalchemy asyncua==1.0.6 -redis git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.1 git+ssh://git@github.com/Aignosi/sientia-model-library.git@0.13.1 -prometheus-client -botocore +prometheus-client boto3 -s3fs -pyarrow -kaleido -hyperopt -shap -pycurl scipy<1.14.0 scikit-learn==1.8.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ecf3261..de0de45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,8 @@ temporalio -psycopg2-binary -sqlalchemy asyncua==1.0.6 -redis sientia_do>=1.12.1 -sientia_model>=0.12.0 +sientia_model>=0.14.0 prometheus-client -botocore boto3 -s3fs -pyarrow -kaleido -hyperopt -shap -pycurl scipy<1.14.0 scikit-learn==1.8.0 \ No newline at end of file From 47c12cf0cc168059ac1742acb7310260b6ab2090 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 2 Sep 2026 10:25:07 -0300 Subject: [PATCH 10/29] chore(deps): re-apply the requirements.txt trim Vitor already validated Confirmed with Vitor: the trimmed dependency list (dropping psycopg2-binary, sqlalchemy, redis, botocore, s3fs, pyarrow, kaleido, hyperopt, shap, pycurl as explicit top-level pins) is intentional, already tested against the cluster's basic runtime, and meant to land on release/SIENTIAPDE-1646 -- it just hadn't been pushed to the real GitHub branch yet (only to a Gitea test mirror), which is why the merge in the previous commit restored the untrimmed list. Reverting that restoration; keeps this branch's sientia_model>=0.12.0 floor and explicit scikit-learn==1.8.0 pin. --- requirements-local.txt | 10 ---------- requirements.txt | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/requirements-local.txt b/requirements-local.txt index 02175fc..2d99485 100644 --- a/requirements-local.txt +++ b/requirements-local.txt @@ -1,18 +1,8 @@ temporalio -psycopg2-binary -sqlalchemy asyncua==1.0.6 -redis git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.1 git+ssh://git@github.com/Aignosi/sientia-model-library.git@0.13.1 prometheus-client -botocore boto3 -s3fs -pyarrow -kaleido -hyperopt -shap -pycurl scipy<1.14.0 scikit-learn==1.8.0 diff --git a/requirements.txt b/requirements.txt index 80c3c03..d14997d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,8 @@ temporalio -psycopg2-binary -sqlalchemy asyncua==1.0.6 -redis sientia_do>=1.12.1 sientia_model>=0.12.0 prometheus-client -botocore boto3 -s3fs -pyarrow -kaleido -hyperopt -shap -pycurl scipy<1.14.0 scikit-learn==1.8.0 From 9c89cfbf8efe7d6401d7cc9fa718f7c09cdbe8a4 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 7 Oct 2026 15:18:35 -0300 Subject: [PATCH 11/29] feat(simple_metrics): model-level, window-aware threshold breach alert (SIENTIAPDE-2189) --- laborious/activities/model_metrics.py | 14 +++++++++----- tests/laborious/activities/test_model_metrics.py | 4 ++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index 1bc0a6d..3e7a363 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -447,14 +447,18 @@ class ModelMetrics(SientiaMonitoring): breach_msg = f'{metric_name}={value} below {min_key}={thresholds[min_key]}' if breach_msg: - self.warning( - f'Threshold breach for model {model_id}: {breach_msg}', - metadata, + message = ( + f'Threshold breach for model {model_id} ' + f'(window {interval_minutes} min): {breach_msg}' ) + self.warning(message, metadata) self.send_notification( - metadata=metadata, + # Model-level trigger: the orchestrator dedups alerts on + # `:`, so one breach seen by several windows + # (one schedule each) becomes one alert (SIENTIAPDE-1995). + metadata={**metadata, 'schedule_name': f'{model_id}-simple_metrics'}, notification_id='SIMPLE_METRICS_THRESHOLD_BREACH', - message=f'Threshold breach for model {model_id}: {breach_msg}', + message=message, block='model_metrics', level=NotificationLevel.WARNING, ) diff --git a/tests/laborious/activities/test_model_metrics.py b/tests/laborious/activities/test_model_metrics.py index 741fda5..213f968 100644 --- a/tests/laborious/activities/test_model_metrics.py +++ b/tests/laborious/activities/test_model_metrics.py @@ -1074,6 +1074,10 @@ def test_calculate_simple_metrics_threshold_breach_rmse(model_metrics_activity): assert call_kwargs['notification_id'] == 'SIMPLE_METRICS_THRESHOLD_BREACH' assert call_kwargs['level'] == NotificationLevel.WARNING assert 'rmse' in call_kwargs['message'] + assert '(window 5 min)' in call_kwargs['message'] + # Model-level trigger so the orchestrator dedup merges windows (spec decision 9). + assert call_kwargs['metadata']['schedule_name'] == 'test_model_id-simple_metrics' + assert call_kwargs['metadata']['model_name'] == 'test_model' def test_calculate_simple_metrics_threshold_no_breach(model_metrics_activity): From ccf4dc86e2dcaef32874171474fffdc2504c6aa3 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 7 Oct 2026 15:26:57 -0300 Subject: [PATCH 12/29] feat(simple_metrics): skip threshold alerting below simple_metrics_min_samples (SIENTIAPDE-2189) --- laborious/workflows/simple_metrics.py | 12 +++- .../workflows/test_simple_metrics.py | 55 ++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/laborious/workflows/simple_metrics.py b/laborious/workflows/simple_metrics.py index 8819d7e..475f871 100644 --- a/laborious/workflows/simple_metrics.py +++ b/laborious/workflows/simple_metrics.py @@ -33,6 +33,7 @@ class SimpleMetrics: target_name = model_config['target'] model_type = model_config.get('model_type') thresholds = model_config.get('simple_metrics_thresholds') + min_samples = model_config.get('simple_metrics_min_samples') # Built from internal Temporal workflow config, not user input (bandit B608). query = f""" @@ -65,6 +66,15 @@ class SimpleMetrics: if not target_data: return + # Too few rows for a reliable value: metrics are still persisted, but a noisy short + # window must not alert (SIENTIAPDE-1995). + enough_samples = not min_samples or len(target_data) >= min_samples + if not enough_samples: + workflow.logger.info( + f'{len(target_data)} rows < simple_metrics_min_samples={min_samples}: ' + 'threshold alerting skipped' + ) + simple_metrics = await workflow.execute_local_activity_method( Activities.calculate_simple_metrics, { @@ -74,7 +84,7 @@ class SimpleMetrics: 'metrics': input_data.get('metrics', ['rmse', 'mse', 'mae', 'r2']), 'interval_minutes': interval_minutes, 'model_type': model_type, - 'thresholds': thresholds, + 'thresholds': thresholds if enough_samples else None, }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=300), diff --git a/tests/laborious/workflows/test_simple_metrics.py b/tests/laborious/workflows/test_simple_metrics.py index 5d94f60..c293685 100644 --- a/tests/laborious/workflows/test_simple_metrics.py +++ b/tests/laborious/workflows/test_simple_metrics.py @@ -1,4 +1,4 @@ -from unittest.mock import ANY, AsyncMock, call, patch +from unittest.mock import ANY, AsyncMock, MagicMock, call, patch from pytest import fixture, mark from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ @@ -217,3 +217,56 @@ async def test_run_default_metrics(workflow_mock: AsyncMock, simple_metrics: Sim retry_policy=ANY, start_to_close_timeout=ANY, ) + + +def _input(**model_config) -> dict: + return { + 'schedule_name': 'test_schedule', + 'model_name': 'test_model', + 'model_id': 'test_model_id', + 'interval_minutes': 60, + 'model_config': {'target': 'test_target', **model_config}, + 'schema': 'test_schema', + 'predictions_table_name': 'test_predictions_table', + 'data_table_name': 'test_data_table', + 'target_table_name': 'test_target_table', + 'metrics': ['rmse'], + } + + +_ROW = {'timestamp': '2026-10-06 10:00:00+0000', 'prediction': 1.0, 'target': 2.0} + + +@mark.asyncio +@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) +async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metrics): + workflow_mock.logger = MagicMock() + # load, export, and the adaptive-state call S2 (Task 8) adds. + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None, None]) + workflow_mock.execute_local_activity_method = AsyncMock( + return_value=[{'metric': 'rmse', 'value': 9.0}] + ) + + await simple_metrics.run( + _input(simple_metrics_thresholds={'rmse_max': 1.0}, simple_metrics_min_samples=5) + ) + + activity_input = workflow_mock.execute_local_activity_method.call_args.args[1] + assert activity_input['thresholds'] is None + + +@mark.asyncio +@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) +async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics): + workflow_mock.logger = MagicMock() + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 5, None, None]) + workflow_mock.execute_local_activity_method = AsyncMock( + return_value=[{'metric': 'rmse', 'value': 9.0}] + ) + + await simple_metrics.run( + _input(simple_metrics_thresholds={'rmse_max': 1.0}, simple_metrics_min_samples=5) + ) + + activity_input = workflow_mock.execute_local_activity_method.call_args.args[1] + assert activity_input['thresholds'] == {'rmse_max': 1.0} From 9f8237a46e0a9b3c2956c26ea33a9a8b919061f8 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Thu, 8 Oct 2026 08:43:01 -0300 Subject: [PATCH 13/29] fix(simple_metrics): ignore invalid simple_metrics_min_samples; tighten tests and docs (SIENTIAPDE-2189) --- README.md | 9 +++- laborious/workflows/simple_metrics.py | 8 ++++ .../workflows/test_simple_metrics.py | 44 ++++++++++++++----- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ede41bf..66f59be 100644 --- a/README.md +++ b/README.md @@ -487,7 +487,7 @@ The **SimpleMetrics** workflow calculates regression metrics (RMSE, MSE, MAE, R #### Execution Flow 1. **Data Loading**: Loads prediction vs target data via a JOIN query -2. **Metrics Calculation**: Calculates configured regression metrics +2. **Metrics Calculation**: Calculates configured regression metrics; alerts on `simple_metrics_thresholds` breaches (one alert per model, the message names the window) unless the window has fewer rows than `simple_metrics_min_samples` 3. **Data Export**: Exports metrics to PostgreSQL #### Architecture Diagram @@ -510,7 +510,12 @@ flowchart LR "data_table_name": "laborious_data", "target_table_name": "simple_metrics", "interval_minutes": 60, - "model_config": { "target": "temperature", "alias": "production" }, + "model_config": { + "target": "temperature", + "alias": "production", + "simple_metrics_thresholds": { "rmse_max": 1.5 }, + "simple_metrics_min_samples": 10 + }, "metrics": ["rmse", "mse", "mae", "r2"] } ``` diff --git a/laborious/workflows/simple_metrics.py b/laborious/workflows/simple_metrics.py index 475f871..fad9a89 100644 --- a/laborious/workflows/simple_metrics.py +++ b/laborious/workflows/simple_metrics.py @@ -34,6 +34,14 @@ class SimpleMetrics: model_type = model_config.get('model_type') thresholds = model_config.get('simple_metrics_thresholds') min_samples = model_config.get('simple_metrics_min_samples') + # Untyped Mongo config: anything but a positive int (bool excluded) turns the guard off + # instead of raising in workflow code; falsy values mean "unset" and stay silent. + if type(min_samples) is not int or min_samples <= 0: + if min_samples: + workflow.logger.warning( + f'Invalid simple_metrics_min_samples={min_samples!r}: min samples guard off' + ) + min_samples = None # Built from internal Temporal workflow config, not user input (bandit B608). query = f""" diff --git a/tests/laborious/workflows/test_simple_metrics.py b/tests/laborious/workflows/test_simple_metrics.py index c293685..bafd007 100644 --- a/tests/laborious/workflows/test_simple_metrics.py +++ b/tests/laborious/workflows/test_simple_metrics.py @@ -237,15 +237,16 @@ def _input(**model_config) -> dict: _ROW = {'timestamp': '2026-10-06 10:00:00+0000', 'prediction': 1.0, 'target': 2.0} +_METRICS = [{'metric': 'rmse', 'value': 9.0}] + + @mark.asyncio +@mark.parametrize('n_rows', [2, 4]) @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) -async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metrics): +async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metrics, n_rows): workflow_mock.logger = MagicMock() - # load, export, and the adaptive-state call S2 (Task 8) adds. - workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None, None]) - workflow_mock.execute_local_activity_method = AsyncMock( - return_value=[{'metric': 'rmse', 'value': 9.0}] - ) + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * n_rows, None]) + workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS) await simple_metrics.run( _input(simple_metrics_thresholds={'rmse_max': 1.0}, simple_metrics_min_samples=5) @@ -253,16 +254,19 @@ async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metr activity_input = workflow_mock.execute_local_activity_method.call_args.args[1] assert activity_input['thresholds'] is None + # Persist, don't alert. + assert workflow_mock.execute_activity_method.call_count == 2 + export_call = workflow_mock.execute_activity_method.call_args_list[1] + assert export_call.args[0] == Activities.export_data_to_postgres + assert export_call.args[1]['data'] == _METRICS @mark.asyncio @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics): workflow_mock.logger = MagicMock() - workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 5, None, None]) - workflow_mock.execute_local_activity_method = AsyncMock( - return_value=[{'metric': 'rmse', 'value': 9.0}] - ) + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 5, None]) + workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS) await simple_metrics.run( _input(simple_metrics_thresholds={'rmse_max': 1.0}, simple_metrics_min_samples=5) @@ -270,3 +274,23 @@ async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics activity_input = workflow_mock.execute_local_activity_method.call_args.args[1] assert activity_input['thresholds'] == {'rmse_max': 1.0} + assert workflow_mock.execute_activity_method.call_count == 2 + + +@mark.asyncio +@mark.parametrize( + ('min_samples', 'warns'), [(None, False), (0, False), ('5', True), (5.0, True), (True, True)] +) +@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) +async def test_run_guard_off_keeps_thresholds(workflow_mock, simple_metrics, min_samples, warns): + workflow_mock.logger = MagicMock() + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None]) + workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS) + + await simple_metrics.run( + _input(simple_metrics_thresholds={'rmse_max': 1.0}, simple_metrics_min_samples=min_samples) + ) + + activity_input = workflow_mock.execute_local_activity_method.call_args.args[1] + assert activity_input['thresholds'] == {'rmse_max': 1.0} + assert workflow_mock.logger.warning.call_count == int(warns) From f503fd06107d47d6ab110137721783b1dac89a35 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 7 Oct 2026 15:33:15 -0300 Subject: [PATCH 14/29] refactor(simple_metrics): extract threshold_breaches helper (SIENTIAPDE-2191) --- laborious/activities/model_metrics.py | 70 ++++++++++--------- .../activities/test_model_metrics.py | 23 +++++- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index 3e7a363..8b59f00 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -26,6 +26,28 @@ warnings.filterwarnings( ) +def threshold_breaches( + rows: list[dict[str, Any]], thresholds: dict[str, float] | None +) -> list[str]: + """ + Breach messages for metric rows outside their configured thresholds. + + Convention: `_max` breaches when value > threshold, `_min` when value < + threshold. Shared by the breach alert and the adaptive-state rule so both agree on what a + breach is. + """ + thresholds = thresholds or {} + breaches = [] + for row in rows: + metric_name, value = row['metric'], row['value'] + max_key, min_key = f'{metric_name}_max', f'{metric_name}_min' + if max_key in thresholds and value > thresholds[max_key]: + breaches.append(f'{metric_name}={value} exceeds {max_key}={thresholds[max_key]}') + elif min_key in thresholds and value < thresholds[min_key]: + breaches.append(f'{metric_name}={value} below {min_key}={thresholds[min_key]}') + return breaches + + class ModelMetrics(SientiaMonitoring): """ Metrics and statistical analysis activities for the Laborious pipeline. @@ -430,37 +452,21 @@ class ModelMetrics(SientiaMonitoring): self._debug_dataframe(f'Simple metrics dataframe: Size {data.shape}', data, metadata) # Threshold alerting (optional — no crash when absent) - thresholds = input_data.get('thresholds') - if thresholds: - # Convention: _max thresholds breach when value > threshold, - # _min thresholds breach when value < threshold. - for row in output_data: - metric_name = row['metric'] - value = row['value'] - max_key = f'{metric_name}_max' - min_key = f'{metric_name}_min' - - breach_msg = None - if max_key in thresholds and value > thresholds[max_key]: - breach_msg = f'{metric_name}={value} exceeds {max_key}={thresholds[max_key]}' - elif min_key in thresholds and value < thresholds[min_key]: - breach_msg = f'{metric_name}={value} below {min_key}={thresholds[min_key]}' - - if breach_msg: - message = ( - f'Threshold breach for model {model_id} ' - f'(window {interval_minutes} min): {breach_msg}' - ) - self.warning(message, metadata) - self.send_notification( - # Model-level trigger: the orchestrator dedups alerts on - # `:`, so one breach seen by several windows - # (one schedule each) becomes one alert (SIENTIAPDE-1995). - metadata={**metadata, 'schedule_name': f'{model_id}-simple_metrics'}, - notification_id='SIMPLE_METRICS_THRESHOLD_BREACH', - message=message, - block='model_metrics', - level=NotificationLevel.WARNING, - ) + for breach_msg in threshold_breaches(output_data, input_data.get('thresholds')): + message = ( + f'Threshold breach for model {model_id} ' + f'(window {interval_minutes} min): {breach_msg}' + ) + self.warning(message, metadata) + self.send_notification( + # Model-level trigger: the orchestrator dedups alerts on + # `:`, so one breach seen by several windows + # (one schedule each) becomes one alert (SIENTIAPDE-1995). + metadata={**metadata, 'schedule_name': f'{model_id}-simple_metrics'}, + notification_id='SIMPLE_METRICS_THRESHOLD_BREACH', + message=message, + block='model_metrics', + level=NotificationLevel.WARNING, + ) return data.to_dict(orient='records') diff --git a/tests/laborious/activities/test_model_metrics.py b/tests/laborious/activities/test_model_metrics.py index 213f968..e849757 100644 --- a/tests/laborious/activities/test_model_metrics.py +++ b/tests/laborious/activities/test_model_metrics.py @@ -7,7 +7,7 @@ from sientia_do.observability.metrics_controller import MetricsController from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ from sientia_model.analytics.drift_analysis import DriftInsufficientDataError -from laborious.activities.model_metrics import ModelMetrics +from laborious.activities.model_metrics import ModelMetrics, threshold_breaches @fixture @@ -1221,3 +1221,24 @@ def test_to_naive_utc_keeps_naive_timestamps(): assert result.dt.tz is None assert result.tolist() == [Timestamp('2024-01-01 06:00:00')] + + +@mark.parametrize( + 'rows, thresholds, expected_count', + [ + ([{'metric': 'rmse', 'value': 2.0}], {'rmse_max': 1.0}, 1), + ([{'metric': 'rmse', 'value': 0.5}], {'rmse_max': 1.0}, 0), + ([{'metric': 'r2', 'value': 0.5}], {'r2_min': 0.9}, 1), + ([{'metric': 'r2', 'value': 0.95}], {'r2_min': 0.9}, 0), + ([{'metric': 'mae', 'value': 99.0}], {'rmse_max': 1.0}, 0), + ([{'metric': 'rmse', 'value': 99.0}], None, 0), + ([], {'rmse_max': 1.0}, 0), + ( + [{'metric': 'rmse', 'value': 2.0}, {'metric': 'r2', 'value': 0.1}], + {'rmse_max': 1.0, 'r2_min': 0.9}, + 2, + ), + ], +) +def test_threshold_breaches(rows, thresholds, expected_count): + assert len(threshold_breaches(rows, thresholds)) == expected_count From cb97f7344fa4953718ee4df0a6a7df33bebd0fe1 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 7 Oct 2026 15:38:12 -0300 Subject: [PATCH 15/29] feat(simple_metrics): add adaptive-state hysteresis and recovery-run rules (SIENTIAPDE-2191) --- laborious/activities/model_metrics.py | 43 +++++++++++++ .../activities/test_model_metrics.py | 60 ++++++++++++++++++- 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index 8b59f00..d2271b9 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -4,6 +4,7 @@ with workflow.unsafe.imports_passed_through(): import time import traceback import warnings + from math import ceil from typing import Any import pandas as pd @@ -48,6 +49,48 @@ def threshold_breaches( return breaches +_MAX_RECOVERY_RUNS = 12 +_FREQUENCY_UNIT_SECONDS = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400} + + +def frequency_to_seconds(frequency: str) -> int: + """`'5m'` -> 300. Same `''` format the orchestrator schedules with.""" + # ponytail: copy of orchestrator/utils/converters.py:parse_frequency (no shared helper in + # sientia_do), same int() parsing so both repos accept the same strings, plus a > 0 guard; + # move it to sientia_do if a third repo needs it. + unit = _FREQUENCY_UNIT_SECONDS.get(frequency[-1:]) + if unit is None: + raise ValueError(f'Invalid frequency: {frequency!r}') + value = int(frequency[:-1]) + if value <= 0: + raise ValueError(f'Invalid frequency: {frequency!r}') + return value * unit + + +def recovery_runs(interval_minutes: int, frequency_alert: str) -> int: + """ + N of the hysteresis rule: healthy runs needed to renew the whole window without a breach, + capped at 12 (SIENTIAPDE-1995 decision 3). + """ + runs = ceil(interval_minutes * 60 / frequency_to_seconds(frequency_alert)) + return min(_MAX_RECOVERY_RUNS, max(1, runs)) + + +def next_adaptive_state(state: dict[str, Any], breached: bool, runs: int) -> dict[str, Any]: + """ + Hysteresis rule of the adaptive frequency (SIENTIAPDE-1995): escalate on the first breach, + recover after `runs` consecutive non-breach runs. + """ + if breached: + return {'escalated': True, 'healthy_streak': 0} + if not state.get('escalated'): + return {'escalated': False, 'healthy_streak': 0} + streak = int(state.get('healthy_streak', 0)) + 1 + if streak >= runs: + return {'escalated': False, 'healthy_streak': 0} + return {'escalated': True, 'healthy_streak': streak} + + class ModelMetrics(SientiaMonitoring): """ Metrics and statistical analysis activities for the Laborious pipeline. diff --git a/tests/laborious/activities/test_model_metrics.py b/tests/laborious/activities/test_model_metrics.py index e849757..c6490ad 100644 --- a/tests/laborious/activities/test_model_metrics.py +++ b/tests/laborious/activities/test_model_metrics.py @@ -7,7 +7,13 @@ from sientia_do.observability.metrics_controller import MetricsController from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ from sientia_model.analytics.drift_analysis import DriftInsufficientDataError -from laborious.activities.model_metrics import ModelMetrics, threshold_breaches +from laborious.activities.model_metrics import ( + ModelMetrics, + frequency_to_seconds, + next_adaptive_state, + recovery_runs, + threshold_breaches, +) @fixture @@ -1230,6 +1236,8 @@ def test_to_naive_utc_keeps_naive_timestamps(): ([{'metric': 'rmse', 'value': 0.5}], {'rmse_max': 1.0}, 0), ([{'metric': 'r2', 'value': 0.5}], {'r2_min': 0.9}, 1), ([{'metric': 'r2', 'value': 0.95}], {'r2_min': 0.9}, 0), + ([{'metric': 'rmse', 'value': 1.0}], {'rmse_max': 1.0}, 0), # equal is not a breach + ([{'metric': 'r2', 'value': 0.9}], {'r2_min': 0.9}, 0), ([{'metric': 'mae', 'value': 99.0}], {'rmse_max': 1.0}, 0), ([{'metric': 'rmse', 'value': 99.0}], None, 0), ([], {'rmse_max': 1.0}, 0), @@ -1242,3 +1250,53 @@ def test_to_naive_utc_keeps_naive_timestamps(): ) def test_threshold_breaches(rows, thresholds, expected_count): assert len(threshold_breaches(rows, thresholds)) == expected_count + + +def _state(escalated: bool, streak: int) -> dict: + return {'escalated': escalated, 'healthy_streak': streak} + + +@mark.parametrize( + 'state, breached, runs, expected', + [ + ({}, True, 3, _state(True, 0)), # first breach escalates + ({}, False, 3, _state(False, 0)), # missing state = healthy + (_state(False, 0), False, 3, _state(False, 0)), # healthy stays healthy + (_state(True, 0), False, 3, _state(True, 1)), # healthy run while escalated + (_state(True, 1), False, 3, _state(True, 2)), + (_state(True, 2), False, 3, _state(False, 0)), # N-th healthy run recovers + (_state(True, 2), True, 3, _state(True, 0)), # breach resets the streak + (_state(True, 0), False, 1, _state(False, 0)), # N=1 recovers on first healthy run + ], +) +def test_next_adaptive_state(state, breached, runs, expected): + assert next_adaptive_state(state, breached, runs) == expected + + +@mark.parametrize( + 'frequency, seconds', + [('30s', 30), ('5m', 300), ('2h', 7200), ('1d', 86400)], +) +def test_frequency_to_seconds(frequency, seconds): + assert frequency_to_seconds(frequency) == seconds + + +@mark.parametrize('frequency', ['', '5', 'm', '5x', 'abc', '0m']) +def test_frequency_to_seconds_rejects_invalid(frequency): + with raises(ValueError): + frequency_to_seconds(frequency) + + +@mark.parametrize( + 'interval_minutes, frequency_alert, expected', + [ + (60, '5m', 12), # 60/5 = 12, at the cap + (60, '15m', 4), + (60, '7m', 9), # ceil(8.57) + (60, '30s', 12), # 120 capped at 12 + (1440, '1h', 12), # 24 capped at 12 + (5, '1h', 1), # window shorter than one run: at least 1 + ], +) +def test_recovery_runs(interval_minutes, frequency_alert, expected): + assert recovery_runs(interval_minutes, frequency_alert) == expected From 34f56c7416c89b441b00f030bf70750191a40700 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 7 Oct 2026 15:45:38 -0300 Subject: [PATCH 16/29] feat(simple_metrics): write adaptive_state to the pipelines doc (SIENTIAPDE-2191) --- laborious/activities/model_metrics.py | 66 ++++++++- .../activities/test_model_metrics.py | 133 ++++++++++++++++++ 2 files changed, 198 insertions(+), 1 deletion(-) diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index d2271b9..2aa8fa6 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -14,7 +14,8 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.logger import Logger from sientia_do.observability.metrics_controller import MetricsController from sientia_do.observability.sientia_monitoring import SientiaMonitoring - from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + from sientia_do.repository.mongodb_repository_sync import MongoDBRepository + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now from sientia_model.analytics.drift_analysis import DriftAnalysis, DriftInsufficientDataError from sientia_model.metrics.regression import RegressionMetrics @@ -103,6 +104,12 @@ class ModelMetrics(SientiaMonitoring): _MAX_DEBUG_DATAFRAME_ROWS = 100 + # ponytail: same declaration as PredictionMetadata; the Activities composite sets the + # instance value in PredictionMetadata.__init__. A standalone ModelMetrics (unit tests) keeps + # None and the adaptive-state activity becomes a no-op. + mongo_repository: MongoDBRepository | None = None + _PIPELINES_COLLECTION = 'pipelines' + def __init__( self, logger: Logger, @@ -513,3 +520,60 @@ class ModelMetrics(SientiaMonitoring): ) return data.to_dict(orient='records') + + @activity.defn(name='update_simple_metrics_adaptive_state') + def update_simple_metrics_adaptive_state(self, input_data: dict[str, Any]) -> dict | None: + """ + Best-effort update of the adaptive frequency state on the schedule's `pipelines` doc. + + The orchestrator derives the schedule frequency from `adaptive_state.escalated` and + `frequency_alert`. State is read from Mongo, not from the workflow input, because the + input only refreshes on the orchestrator tick and would be stale while escalated. + `updated_at` is bumped only when `escalated` flips: that is what makes the orchestrator + rewrite the schedule, and a streak change alone does not change the frequency. + + Args: + input_data: Dict with: + - metadata (dict): Workflow metadata; `schedule_name` identifies the doc. + - simple_metrics (list[dict]): Rows from `calculate_simple_metrics`; empty for an + empty or under-sampled window, which counts as a non-breach run. + - thresholds (dict): `simple_metrics_thresholds` from `model_config`. + - interval_minutes (int): Window size, used to compute the recovery runs. + + Return: + The new state, or `None` when adaptive mode is off, Mongo is not configured, or the + read/parse/write failed. + """ + metadata = input_data['metadata'] + if self.mongo_repository is None: + return None + + query = {'schedule_name': metadata['schedule_name']} + try: + pipeline = self.mongo_repository.find_one(self._PIPELINES_COLLECTION, query, metadata) + if not pipeline or not pipeline.get('frequency_alert'): + return None + + runs = recovery_runs(input_data['interval_minutes'], pipeline['frequency_alert']) + breached = bool( + threshold_breaches(input_data['simple_metrics'], input_data.get('thresholds')) + ) + state = pipeline.get('adaptive_state') or {'escalated': False, 'healthy_streak': 0} + new_state = next_adaptive_state(state, breached, runs) + if new_state == state: + return new_state + + update: dict[str, Any] = {'adaptive_state': new_state} + if new_state['escalated'] != bool(state.get('escalated')): + # BSON datetime, never a string: the orchestrator normalizes `updated_at` with + # `.replace(tzinfo=UTC).strftime(...)` and a string would fail its whole tick. + update['updated_at'] = now() + self.mongo_repository.update_one( + self._PIPELINES_COLLECTION, query, {'$set': update}, metadata + ) + except Exception as e: + self.error(f'Failed to update simple metrics adaptive state: {e}', metadata) + return None + + self.info(f'Adaptive state of {query["schedule_name"]}: {state} -> {new_state}', metadata) + return new_state diff --git a/tests/laborious/activities/test_model_metrics.py b/tests/laborious/activities/test_model_metrics.py index c6490ad..a6a2919 100644 --- a/tests/laborious/activities/test_model_metrics.py +++ b/tests/laborious/activities/test_model_metrics.py @@ -1,3 +1,4 @@ +from datetime import datetime from unittest.mock import ANY, MagicMock, patch from pandas import DataFrame, Series, Timestamp @@ -1300,3 +1301,135 @@ def test_frequency_to_seconds_rejects_invalid(frequency): ) def test_recovery_runs(interval_minutes, frequency_alert, expected): assert recovery_runs(interval_minutes, frequency_alert) == expected + + +_SM_METADATA = {**metadata['metadata'], 'schedule_name': 'm1-simple_metrics-60m'} +_BREACH_INPUT = { + 'metadata': _SM_METADATA, + 'simple_metrics': [{'metric': 'rmse', 'value': 2.0}], + 'thresholds': {'rmse_max': 1.0}, + 'interval_minutes': 60, +} +_HEALTHY_INPUT = {**_BREACH_INPUT, 'simple_metrics': [{'metric': 'rmse', 'value': 0.5}]} +_EMPTY_INPUT = {**_BREACH_INPUT, 'simple_metrics': []} + + +@fixture +def mongo(model_metrics_activity): + repository = MagicMock() + model_metrics_activity.mongo_repository = repository + return repository + + +def _pipeline(**fields) -> dict: + return {'schedule_name': 'm1-simple_metrics-60m', 'frequency': '1h', **fields} + + +def _set_of(mongo: MagicMock) -> dict: + collection, query, update, _metadata = mongo.update_one.call_args.args + assert collection == 'pipelines' + assert query == {'schedule_name': 'm1-simple_metrics-60m'} + return update['$set'] + + +def test_adaptive_state_no_mongo_is_noop(model_metrics_activity): + assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None + + +def test_adaptive_state_missing_pipeline_doc_is_noop(model_metrics_activity, mongo): + mongo.find_one.return_value = None + assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None + mongo.update_one.assert_not_called() + + +def test_adaptive_state_without_frequency_alert_is_noop(model_metrics_activity, mongo): + mongo.find_one.return_value = _pipeline() + assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None + mongo.update_one.assert_not_called() + + +def test_adaptive_state_breach_escalates_and_bumps_updated_at(model_metrics_activity, mongo): + mongo.find_one.return_value = _pipeline(frequency_alert='5m') + + result = model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) + + assert result == {'escalated': True, 'healthy_streak': 0} + update = _set_of(mongo) + assert update['adaptive_state'] == {'escalated': True, 'healthy_streak': 0} + # BSON datetime, never a string: the orchestrator calls .replace(tzinfo=...) on it. + assert isinstance(update['updated_at'], datetime) + assert update['updated_at'].tzinfo is not None + + +def test_adaptive_state_healthy_run_increments_streak_without_updated_at( + model_metrics_activity, mongo +): + # 60 min window / 5m = N 12, so one healthy run only moves the streak. + mongo.find_one.return_value = _pipeline( + frequency_alert='5m', adaptive_state={'escalated': True, 'healthy_streak': 0} + ) + + result = model_metrics_activity.update_simple_metrics_adaptive_state(_HEALTHY_INPUT) + + assert result == {'escalated': True, 'healthy_streak': 1} + assert _set_of(mongo) == {'adaptive_state': {'escalated': True, 'healthy_streak': 1}} + + +def test_adaptive_state_recovers_after_window_renewed(model_metrics_activity, mongo): + # 60 min window / 30m = N 2: second consecutive healthy run recovers. + mongo.find_one.return_value = _pipeline( + frequency_alert='30m', adaptive_state={'escalated': True, 'healthy_streak': 1} + ) + + result = model_metrics_activity.update_simple_metrics_adaptive_state(_HEALTHY_INPUT) + + assert result == {'escalated': False, 'healthy_streak': 0} + update = _set_of(mongo) + assert update['adaptive_state'] == {'escalated': False, 'healthy_streak': 0} + assert isinstance(update['updated_at'], datetime) + + +def test_adaptive_state_empty_window_counts_as_healthy_run(model_metrics_activity, mongo): + # Spec decision 8: an empty/under-sampled window never escalates but counts toward recovery. + mongo.find_one.return_value = _pipeline( + frequency_alert='5m', adaptive_state={'escalated': True, 'healthy_streak': 3} + ) + + result = model_metrics_activity.update_simple_metrics_adaptive_state(_EMPTY_INPUT) + + assert result == {'escalated': True, 'healthy_streak': 4} + + +def test_adaptive_state_empty_window_never_escalates(model_metrics_activity, mongo): + mongo.find_one.return_value = _pipeline(frequency_alert='5m') + + result = model_metrics_activity.update_simple_metrics_adaptive_state(_EMPTY_INPUT) + + assert result == {'escalated': False, 'healthy_streak': 0} + mongo.update_one.assert_not_called() + + +def test_adaptive_state_unchanged_does_not_write(model_metrics_activity, mongo): + mongo.find_one.return_value = _pipeline( + frequency_alert='5m', adaptive_state={'escalated': False, 'healthy_streak': 0} + ) + + result = model_metrics_activity.update_simple_metrics_adaptive_state(_HEALTHY_INPUT) + + assert result == {'escalated': False, 'healthy_streak': 0} + mongo.update_one.assert_not_called() + + +def test_adaptive_state_invalid_frequency_alert_is_swallowed(model_metrics_activity, mongo): + mongo.find_one.return_value = _pipeline(frequency_alert='5min') + + assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None + mongo.update_one.assert_not_called() + model_metrics_activity.error.assert_called_once() + + +def test_adaptive_state_mongo_error_is_swallowed(model_metrics_activity, mongo): + mongo.find_one.side_effect = RuntimeError('mongo down') + + assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None + model_metrics_activity.error.assert_called_once() From e05ec1c0a93578634b76c8cc8d312f6ae71c868b Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 7 Oct 2026 15:54:44 -0300 Subject: [PATCH 17/29] feat(simple_metrics): update adaptive state after each run, empty windows included (SIENTIAPDE-2191) --- e2e/conftest.py | 1 + laborious/worker/worker.py | 1 + laborious/workflows/simple_metrics.py | 90 +++++++++++------- tests/laborious/worker/test_worker.py | 6 ++ .../workflows/test_simple_metrics.py | 94 ++++++++++++++++++- 5 files changed, 152 insertions(+), 40 deletions(-) diff --git a/e2e/conftest.py b/e2e/conftest.py index 807a5ce..a108127 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -445,6 +445,7 @@ def _simple_metrics_worker_activity_list(test_activities: Activities): test_activities.load_custom_query, test_activities.calculate_simple_metrics, test_activities.export_data_to_postgres, + test_activities.update_simple_metrics_adaptive_state, ] diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index b44d829..c33fe9b 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -217,6 +217,7 @@ async def main(): activities.load_custom_query, activities.calculate_simple_metrics, activities.export_data_to_postgres, + activities.update_simple_metrics_adaptive_state, ], logger=logger, runtime='core', diff --git a/laborious/workflows/simple_metrics.py b/laborious/workflows/simple_metrics.py index fad9a89..279d77e 100644 --- a/laborious/workflows/simple_metrics.py +++ b/laborious/workflows/simple_metrics.py @@ -71,48 +71,68 @@ class SimpleMetrics: start_to_close_timeout=timedelta(seconds=300), ) - if not target_data: - return - # Too few rows for a reliable value: metrics are still persisted, but a noisy short - # window must not alert (SIENTIAPDE-1995). - enough_samples = not min_samples or len(target_data) >= min_samples - if not enough_samples: + # window must neither alert nor escalate (SIENTIAPDE-1995). + enough_samples = bool(target_data) and (not min_samples or len(target_data) >= min_samples) + if target_data and not enough_samples: workflow.logger.info( f'{len(target_data)} rows < simple_metrics_min_samples={min_samples}: ' 'threshold alerting skipped' ) - simple_metrics = await workflow.execute_local_activity_method( - Activities.calculate_simple_metrics, - { - **metadata, - 'model_id': model_id, - 'target_data': target_data, - 'metrics': input_data.get('metrics', ['rmse', 'mse', 'mae', 'r2']), - 'interval_minutes': interval_minutes, - 'model_type': model_type, - 'thresholds': thresholds if enough_samples else None, - }, - retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=300), - ) + simple_metrics = None + if target_data: + simple_metrics = await workflow.execute_local_activity_method( + Activities.calculate_simple_metrics, + { + **metadata, + 'model_id': model_id, + 'target_data': target_data, + 'metrics': input_data.get('metrics', ['rmse', 'mse', 'mae', 'r2']), + 'interval_minutes': interval_minutes, + 'model_type': model_type, + 'thresholds': thresholds if enough_samples else None, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=300), + ) - if not simple_metrics: + if simple_metrics: + await workflow.execute_activity_method( + Activities.export_data_to_postgres, + { + **metadata, + 'data': simple_metrics, + 'schema': input_data['schema'], + 'table_name': input_data['target_table_name'], + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_WITH_TZ, + }, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=300), + ) + + if not thresholds: + # No thresholds = no breach possible, so skip the Mongo round trip. return - await workflow.execute_activity_method( - Activities.export_data_to_postgres, - { - **metadata, - 'data': simple_metrics, - 'schema': input_data['schema'], - 'table_name': input_data['target_table_name'], - 'timestamp_conversion': { - 'column': 'timestamp', - 'format': DATETIME_FORMAT_WITH_TZ, + try: + await workflow.execute_activity_method( + Activities.update_simple_metrics_adaptive_state, + { + **metadata, + # Only a valid breach escalates: an empty or under-sampled window is sent as + # no rows, which counts as a healthy run (spec decision 8). + 'simple_metrics': (simple_metrics or []) if enough_samples else [], + 'thresholds': thresholds, + 'interval_minutes': interval_minutes, }, - }, - retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=300), - ) + retry_policy=retry_policy, + start_to_close_timeout=timedelta(minutes=1), + ) + except Exception as e: + # Best-effort: metrics are already persisted. retry_policy has maximum_attempts=1, so + # a dispatch failure would otherwise fail the workflow (same as prediction_process). + workflow.logger.warning(f'Failed to update adaptive state, continuing anyway: {e}') diff --git a/tests/laborious/worker/test_worker.py b/tests/laborious/worker/test_worker.py index a4c1898..c9ded87 100644 --- a/tests/laborious/worker/test_worker.py +++ b/tests/laborious/worker/test_worker.py @@ -16,6 +16,7 @@ def _build_fake_activities(): inst.export_data_to_postgres = MagicMock() inst.load_custom_query = MagicMock() inst.calculate_simple_metrics = MagicMock() + inst.update_simple_metrics_adaptive_state = MagicMock() inst.get_reference_data = MagicMock() inst.calculate_drift = MagicMock() inst.request_predict = MagicMock() @@ -188,6 +189,11 @@ async def test_main_success_exit_zero(monkeypatch): # Drift and SimpleMetrics load no model code, so every runtime shares the `core` queues. assert prepare_calls[1].kwargs['runtime'] == 'core' assert prepare_calls[2].kwargs['runtime'] == 'core' + # prepare_calls[1] is the SimpleMetrics worker (order: MinimalRetrain, SimpleMetrics, Drift, PredictionsBatch). + assert ( + fake_activities.update_simple_metrics_adaptive_state + in prepare_calls[1].kwargs['activities'] + ) @mark.asyncio diff --git a/tests/laborious/workflows/test_simple_metrics.py b/tests/laborious/workflows/test_simple_metrics.py index bafd007..dc9b6bf 100644 --- a/tests/laborious/workflows/test_simple_metrics.py +++ b/tests/laborious/workflows/test_simple_metrics.py @@ -245,7 +245,7 @@ _METRICS = [{'metric': 'rmse', 'value': 9.0}] @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metrics, n_rows): workflow_mock.logger = MagicMock() - workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * n_rows, None]) + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * n_rows, None, None]) workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS) await simple_metrics.run( @@ -254,8 +254,8 @@ async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metr activity_input = workflow_mock.execute_local_activity_method.call_args.args[1] assert activity_input['thresholds'] is None - # Persist, don't alert. - assert workflow_mock.execute_activity_method.call_count == 2 + # Persist, don't alert: load, export, adaptive state. + assert workflow_mock.execute_activity_method.call_count == 3 export_call = workflow_mock.execute_activity_method.call_args_list[1] assert export_call.args[0] == Activities.export_data_to_postgres assert export_call.args[1]['data'] == _METRICS @@ -265,7 +265,7 @@ async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metr @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics): workflow_mock.logger = MagicMock() - workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 5, None]) + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 5, None, None]) workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS) await simple_metrics.run( @@ -274,6 +274,43 @@ async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics activity_input = workflow_mock.execute_local_activity_method.call_args.args[1] assert activity_input['thresholds'] == {'rmse_max': 1.0} + assert workflow_mock.execute_activity_method.call_count == 3 + + +_BREACH_ROWS = [{'metric': 'rmse', 'value': 9.0}] +_THRESHOLDS = {'rmse_max': 1.0} + + +@mark.asyncio +@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) +async def test_run_updates_adaptive_state_after_export(workflow_mock, simple_metrics): + workflow_mock.logger = MagicMock() + workflow_mock.execute_activity_method = AsyncMock( + side_effect=[[_ROW] * 3, None, {'escalated': True, 'healthy_streak': 0}] + ) + workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS) + + await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS)) + + calls = workflow_mock.execute_activity_method.call_args_list + assert calls[1].args[0] is Activities.export_data_to_postgres + assert calls[2].args[0] is Activities.update_simple_metrics_adaptive_state + assert calls[2].args[1] == { + **metadata, + 'simple_metrics': _BREACH_ROWS, + 'thresholds': _THRESHOLDS, + 'interval_minutes': 60, + } + + +@mark.asyncio +@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) +async def test_run_without_thresholds_skips_adaptive_state(workflow_mock, simple_metrics): + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 3, None]) + workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS) + + await simple_metrics.run(_input()) + assert workflow_mock.execute_activity_method.call_count == 2 @@ -284,7 +321,7 @@ async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) async def test_run_guard_off_keeps_thresholds(workflow_mock, simple_metrics, min_samples, warns): workflow_mock.logger = MagicMock() - workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None]) + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None, None]) workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS) await simple_metrics.run( @@ -294,3 +331,50 @@ async def test_run_guard_off_keeps_thresholds(workflow_mock, simple_metrics, min activity_input = workflow_mock.execute_local_activity_method.call_args.args[1] assert activity_input['thresholds'] == {'rmse_max': 1.0} assert workflow_mock.logger.warning.call_count == int(warns) + + +@mark.asyncio +@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) +async def test_run_empty_window_counts_as_healthy_run(workflow_mock, simple_metrics): + workflow_mock.logger = MagicMock() + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[], None]) + workflow_mock.execute_local_activity_method = AsyncMock() + + await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS)) + + workflow_mock.execute_local_activity_method.assert_not_called() + calls = workflow_mock.execute_activity_method.call_args_list + assert len(calls) == 2 # load + adaptive state, no export + assert calls[1].args[0] is Activities.update_simple_metrics_adaptive_state + assert calls[1].args[1]['simple_metrics'] == [] + + +@mark.asyncio +@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) +async def test_run_below_min_samples_sends_no_rows_to_adaptive_state(workflow_mock, simple_metrics): + workflow_mock.logger = MagicMock() + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None, None]) + workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS) + + await simple_metrics.run( + _input(simple_metrics_thresholds=_THRESHOLDS, simple_metrics_min_samples=5) + ) + + calls = workflow_mock.execute_activity_method.call_args_list + assert calls[1].args[0] is Activities.export_data_to_postgres # metrics still persisted + assert calls[2].args[0] is Activities.update_simple_metrics_adaptive_state + assert calls[2].args[1]['simple_metrics'] == [] # never a breach + + +@mark.asyncio +@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) +async def test_run_adaptive_state_failure_does_not_fail_workflow(workflow_mock, simple_metrics): + workflow_mock.logger = MagicMock() + workflow_mock.execute_activity_method = AsyncMock( + side_effect=[[_ROW] * 3, None, RuntimeError('dispatch failed')] + ) + workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS) + + await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS)) + + workflow_mock.logger.warning.assert_called_once() From 844a29ed27ed39289dbd523972275586afcc4476 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 7 Oct 2026 16:01:35 -0300 Subject: [PATCH 18/29] test(e2e): adaptive state is escalated in pipelines on threshold breach (SIENTIAPDE-2191) --- e2e/scenarios.md | 9 +++++++ e2e/test_simple_metrics.py | 52 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/e2e/scenarios.md b/e2e/scenarios.md index 78fc0ae..032917b 100644 --- a/e2e/scenarios.md +++ b/e2e/scenarios.md @@ -413,6 +413,15 @@ four metrics with closed-form expected values. #### S.1.2 Subset metrics **Summary**: Requesting `metrics=['rmse']` writes only the rmse row. +#### S.1.3 Threshold breach escalates the adaptive state +**Summary**: With `simple_metrics_thresholds` breached and a `pipelines` doc +(same `schedule_name`) carrying `frequency_alert`, the workflow sets +`adaptive_state={escalated: true, healthy_streak: 0}` and stamps `updated_at`. + +**Expected Outcome**: +- `adaptive_state` written on the `pipelines` doc. +- `updated_at` is a BSON datetime (the orchestrator calls `.replace(tzinfo=UTC)` on it). + ### 6.2 Edge cases #### S.2.1 Zero-variance target returns r2=0 diff --git a/e2e/test_simple_metrics.py b/e2e/test_simple_metrics.py index 900fd24..3edcfce 100644 --- a/e2e/test_simple_metrics.py +++ b/e2e/test_simple_metrics.py @@ -15,6 +15,7 @@ from decimal import Decimal import math import pytest +from pymongo import MongoClient from sqlalchemy import text from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker @@ -332,3 +333,54 @@ async def test_simple_metrics_no_overlapping_data_short_circuits( {'m': str(model_id)}, ).scalar() assert count == 0, 'Empty target data must short-circuit and skip persistence' + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_simple_metrics_breach_escalates_adaptive_state( + temporal_test_env: WorkflowEnvironment, + temporal_worker_simple_metrics: Worker, + test_activities: Activities, + postgres_engine, + mongo_container, +): + """ + Scenario S.1.3: a threshold breach on a schedule whose pipelines doc has + ``frequency_alert`` escalates ``adaptive_state`` and stamps ``updated_at`` + as a BSON datetime (the type the orchestrator reads). + """ + model_id = 541 + schedule_name = 'test-simple-metrics-adaptive' + _seed_predictions_and_targets( + postgres_engine, model_id=model_id, pairs=[(1.0, 2.0), (2.0, 4.0), (3.0, 5.0)] + ) + + mongo = MongoClient(f'mongodb://localhost:{mongo_container.get_exposed_port(27017)}') + pipelines = mongo['test_db']['pipelines'] + try: + pipelines.delete_many({'schedule_name': schedule_name}) + pipelines.insert_one( + {'schedule_name': schedule_name, 'frequency': '1h', 'frequency_alert': '5m'} + ) + + input_data = _simple_metrics_input( + model_id, + schedule_name=schedule_name, + model_config={ + 'target': 'sensor_target', + 'simple_metrics_thresholds': {'rmse_max': 0.01}, + }, + ) + await start_and_await_workflow( + temporal_test_env.client, + SimpleMetrics.run, + input_data, + make_workflow_id('test-simple-metrics-adaptive'), + ) + + doc = pipelines.find_one({'schedule_name': schedule_name}) + finally: + mongo.close() + + assert doc['adaptive_state'] == {'escalated': True, 'healthy_streak': 0} + assert isinstance(doc['updated_at'], datetime) From 44116f3f3f08dee3662537027675e27927707d45 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 7 Oct 2026 16:14:43 -0300 Subject: [PATCH 19/29] docs(simple_metrics): document min samples guard and adaptive state step (SIENTIAPDE-2191) --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 66f59be..2c3be54 100644 --- a/README.md +++ b/README.md @@ -489,14 +489,16 @@ The **SimpleMetrics** workflow calculates regression metrics (RMSE, MSE, MAE, R 1. **Data Loading**: Loads prediction vs target data via a JOIN query 2. **Metrics Calculation**: Calculates configured regression metrics; alerts on `simple_metrics_thresholds` breaches (one alert per model, the message names the window) unless the window has fewer rows than `simple_metrics_min_samples` 3. **Data Export**: Exports metrics to PostgreSQL +4. **Adaptive State** (only with thresholds): when the schedule's `pipelines` doc has `frequency_alert`, escalates `adaptive_state` on a breach and recovers after the window is renewed without one (`min(12, ceil(interval_minutes / frequency_alert))` runs; empty or under-sampled windows count as healthy). The orchestrator runs the schedule at `frequency_alert` while escalated #### Architecture Diagram ```mermaid flowchart LR - A[1. load_custom_query] --> B[2. calculate_simple_metrics] --> C[3. export_data_to_postgres] + A[1. load_custom_query] --> B[2. calculate_simple_metrics] --> C[3. export_data_to_postgres] --> D[4. update_simple_metrics_adaptive_state] A -.-> Database[(Database)] C -.-> PostgreSQL[(PostgreSQL)] + D -.-> MongoDB[(MongoDB pipelines)] ``` #### Input Parameters From ac30f4c464bd312e1ae91d65eb4a4ffee15ee5a0 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 7 Oct 2026 17:50:52 -0300 Subject: [PATCH 20/29] fix(simple_metrics): update adaptive state even without thresholds so removed thresholds recover (SIENTIAPDE-2191) --- README.md | 2 +- laborious/activities/model_metrics.py | 2 +- laborious/workflows/simple_metrics.py | 10 ++-- .../workflows/test_simple_metrics.py | 52 +++++++++++++++---- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 2c3be54..8e5c04d 100644 --- a/README.md +++ b/README.md @@ -489,7 +489,7 @@ The **SimpleMetrics** workflow calculates regression metrics (RMSE, MSE, MAE, R 1. **Data Loading**: Loads prediction vs target data via a JOIN query 2. **Metrics Calculation**: Calculates configured regression metrics; alerts on `simple_metrics_thresholds` breaches (one alert per model, the message names the window) unless the window has fewer rows than `simple_metrics_min_samples` 3. **Data Export**: Exports metrics to PostgreSQL -4. **Adaptive State** (only with thresholds): when the schedule's `pipelines` doc has `frequency_alert`, escalates `adaptive_state` on a breach and recovers after the window is renewed without one (`min(12, ceil(interval_minutes / frequency_alert))` runs; empty or under-sampled windows count as healthy). The orchestrator runs the schedule at `frequency_alert` while escalated +4. **Adaptive State**: when the schedule's `pipelines` doc has `frequency_alert`, escalates `adaptive_state` on a breach and recovers after the window is renewed without one (`min(12, ceil(interval_minutes / frequency_alert))` runs, at least 1; empty, under-sampled or threshold-less windows count as healthy). The orchestrator runs the schedule at `frequency_alert` while escalated #### Architecture Diagram ```mermaid diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index 2aa8fa6..074b6fc 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -537,7 +537,7 @@ class ModelMetrics(SientiaMonitoring): - metadata (dict): Workflow metadata; `schedule_name` identifies the doc. - simple_metrics (list[dict]): Rows from `calculate_simple_metrics`; empty for an empty or under-sampled window, which counts as a non-breach run. - - thresholds (dict): `simple_metrics_thresholds` from `model_config`. + - thresholds (dict): `simple_metrics_thresholds` from `model_config` (may be None: no breach). - interval_minutes (int): Window size, used to compute the recovery runs. Return: diff --git a/laborious/workflows/simple_metrics.py b/laborious/workflows/simple_metrics.py index 279d77e..142f59d 100644 --- a/laborious/workflows/simple_metrics.py +++ b/laborious/workflows/simple_metrics.py @@ -77,7 +77,7 @@ class SimpleMetrics: if target_data and not enough_samples: workflow.logger.info( f'{len(target_data)} rows < simple_metrics_min_samples={min_samples}: ' - 'threshold alerting skipped' + 'threshold alerting and escalation skipped' ) simple_metrics = None @@ -114,17 +114,15 @@ class SimpleMetrics: start_to_close_timeout=timedelta(seconds=300), ) - if not thresholds: - # No thresholds = no breach possible, so skip the Mongo round trip. - return - try: await workflow.execute_activity_method( Activities.update_simple_metrics_adaptive_state, { **metadata, # Only a valid breach escalates: an empty or under-sampled window is sent as - # no rows, which counts as a healthy run (spec decision 8). + # no rows, which counts as a healthy run (spec decision 8). Always called, even + # without thresholds (no breach), so an escalated doc recovers after they are + # removed. 'simple_metrics': (simple_metrics or []) if enough_samples else [], 'thresholds': thresholds, 'interval_minutes': interval_minutes, diff --git a/tests/laborious/workflows/test_simple_metrics.py b/tests/laborious/workflows/test_simple_metrics.py index dc9b6bf..dfb6833 100644 --- a/tests/laborious/workflows/test_simple_metrics.py +++ b/tests/laborious/workflows/test_simple_metrics.py @@ -42,12 +42,17 @@ async def test_run(workflow_mock: AsyncMock, simple_metrics: SimpleMetrics): target_data = {'data': 'test_target_data'} simple_metrics_data = {'metrics': 'test_simple_metrics_data'} - workflow_mock.execute_activity_method = AsyncMock(side_effect=[target_data, None]) + workflow_mock.logger = MagicMock() + workflow_mock.execute_activity_method = AsyncMock(side_effect=[target_data, None, None]) workflow_mock.execute_local_activity_method = AsyncMock(return_value=simple_metrics_data) # Act await simple_metrics.run(input_data) + # Assert - load, export and adaptive state (no thresholds still updates it) + assert workflow_mock.execute_activity_method.call_count == 3 + workflow_mock.logger.warning.assert_not_called() + # Assert - Check load_custom_query call # Query format matches psycopg2.sql output (identifiers with double quotes, literals with single quotes) expected_query = f""" @@ -130,13 +135,22 @@ async def test_run_empty_target_data(workflow_mock: AsyncMock, simple_metrics: S target_data = None + workflow_mock.logger = MagicMock() workflow_mock.execute_activity_method = AsyncMock(return_value=target_data) + workflow_mock.execute_local_activity_method = AsyncMock() # Act await simple_metrics.run(input_data) - # Assert - Should not call calculate_simple_metrics or export - assert workflow_mock.execute_activity_method.call_count == 1 + # Assert - No calculate or export; load + adaptive state (empty window = healthy run) + workflow_mock.execute_local_activity_method.assert_not_called() + calls = workflow_mock.execute_activity_method.call_args_list + assert [c.args[0] for c in calls] == [ + Activities.load_custom_query, + Activities.update_simple_metrics_adaptive_state, + ] + assert calls[1].args[1]['simple_metrics'] == [] + workflow_mock.logger.warning.assert_not_called() @mark.asyncio @@ -159,15 +173,22 @@ async def test_run_empty_simple_metrics(workflow_mock: AsyncMock, simple_metrics target_data = {'data': 'test_target_data'} simple_metrics_data = None + workflow_mock.logger = MagicMock() workflow_mock.execute_activity_method = AsyncMock(return_value=target_data) workflow_mock.execute_local_activity_method = AsyncMock(return_value=simple_metrics_data) # Act await simple_metrics.run(input_data) - # Assert - Should call calculate_simple_metrics but not export - workflow_mock.execute_activity_method.assert_called_once() + # Assert - calculate returns None: no export, adaptive state gets [] (the `or []` is live) workflow_mock.execute_local_activity_method.assert_called_once() + calls = workflow_mock.execute_activity_method.call_args_list + assert [c.args[0] for c in calls] == [ + Activities.load_custom_query, + Activities.update_simple_metrics_adaptive_state, + ] + assert calls[1].args[1]['simple_metrics'] == [] + workflow_mock.logger.warning.assert_not_called() @mark.asyncio @@ -190,12 +211,17 @@ async def test_run_default_metrics(workflow_mock: AsyncMock, simple_metrics: Sim target_data = {'data': 'test_target_data'} simple_metrics_data = {'metrics': 'test_simple_metrics_data'} - workflow_mock.execute_activity_method = AsyncMock(side_effect=[target_data, None]) + workflow_mock.logger = MagicMock() + workflow_mock.execute_activity_method = AsyncMock(side_effect=[target_data, None, None]) workflow_mock.execute_local_activity_method = AsyncMock(return_value=simple_metrics_data) # Act await simple_metrics.run(input_data) + # Assert - all three activities ran (a swallowed exhausted side_effect would warn) + assert workflow_mock.execute_activity_method.call_count == 3 + workflow_mock.logger.warning.assert_not_called() + # Assert - Check calculate_simple_metrics call with default metrics workflow_mock.execute_activity_method.assert_any_call( Activities.load_custom_query, @@ -305,13 +331,21 @@ async def test_run_updates_adaptive_state_after_export(workflow_mock, simple_met @mark.asyncio @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) -async def test_run_without_thresholds_skips_adaptive_state(workflow_mock, simple_metrics): - workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 3, None]) +async def test_run_without_thresholds_still_updates_adaptive_state(workflow_mock, simple_metrics): + # No thresholds = never a breach; still calling lets an escalated doc recover after the + # thresholds are removed. + workflow_mock.logger = MagicMock() + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 3, None, None]) workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS) await simple_metrics.run(_input()) - assert workflow_mock.execute_activity_method.call_count == 2 + calls = workflow_mock.execute_activity_method.call_args_list + assert len(calls) == 3 + assert calls[2].args[0] is Activities.update_simple_metrics_adaptive_state + assert calls[2].args[1]['thresholds'] is None + assert calls[2].args[1]['simple_metrics'] == _BREACH_ROWS + workflow_mock.logger.warning.assert_not_called() @mark.asyncio From 877a0c754390823c09b4ac4c889624990b257cc7 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Thu, 8 Oct 2026 08:48:08 -0300 Subject: [PATCH 21/29] fix(simple_metrics): warn when the pipelines doc is missing; document Mongo requirements (SIENTIAPDE-2191) --- README.md | 4 ++-- laborious/activities/model_metrics.py | 11 ++++++++++- tests/laborious/activities/test_model_metrics.py | 4 ++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8e5c04d..8c9d0dc 100644 --- a/README.md +++ b/README.md @@ -529,7 +529,7 @@ flowchart LR - PostgreSQL database - MLFlow server - MinIO object storage (for MLFlow artifacts) -- MongoDB server (for notifications) +- MongoDB server (notifications, prediction metadata, and read/write on the orchestrator's `pipelines` collection for the simple-metrics adaptive state) - OPC server(s) if using OPC export - PI Web API server if using PI Web API export @@ -905,7 +905,7 @@ See [OPC UA Communication](#opc-ua-communication) for semantics, concurrency, an | `MONGODB_URL` | MongoDB connection URI | `localhost:27018` | Yes | | `MONGODB_USERNAME` | MongoDB username | `root` | Yes | | `MONGODB_PASSWORD` | MongoDB password | `wKZDbMNU1c` | Yes | -| `MONGODB_DATABASE_NAME` | MongoDB database name | `sientia` | Yes | +| `MONGODB_DATABASE_NAME` | MongoDB database name; must be the same database as the orchestrator's `pipelines` | `sientia` | Yes | | `MONGODB_TTL_INDEX_HOURS` | MongoDB TTL index hours | `1` | No | | `LOG_LEVEL` | Application log level | `INFO` | No | | `PROJECT_NAME` | Project name for metrics | `laborious` | No | diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index 074b6fc..27e7087 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -551,7 +551,16 @@ class ModelMetrics(SientiaMonitoring): query = {'schedule_name': metadata['schedule_name']} try: pipeline = self.mongo_repository.find_one(self._PIPELINES_COLLECTION, query, metadata) - if not pipeline or not pipeline.get('frequency_alert'): + if not pipeline: + # Likely misconfig: MONGODB_DATABASE_NAME differs from the orchestrator's, or the + # schedule_name does not match its doc. + self.warning( + f'No pipelines doc for schedule {metadata["schedule_name"]}: ' + 'adaptive state skipped', + metadata, + ) + return None + if not pipeline.get('frequency_alert'): return None runs = recovery_runs(input_data['interval_minutes'], pipeline['frequency_alert']) diff --git a/tests/laborious/activities/test_model_metrics.py b/tests/laborious/activities/test_model_metrics.py index a6a2919..1c8cf8b 100644 --- a/tests/laborious/activities/test_model_metrics.py +++ b/tests/laborious/activities/test_model_metrics.py @@ -1340,12 +1340,16 @@ def test_adaptive_state_missing_pipeline_doc_is_noop(model_metrics_activity, mon mongo.find_one.return_value = None assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None mongo.update_one.assert_not_called() + # Likely misconfig (other MONGODB_DATABASE_NAME or schedule_name mismatch): not silent. + model_metrics_activity.warning.assert_called_once() + assert 'm1-simple_metrics-60m' in model_metrics_activity.warning.call_args.args[0] def test_adaptive_state_without_frequency_alert_is_noop(model_metrics_activity, mongo): mongo.find_one.return_value = _pipeline() assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None mongo.update_one.assert_not_called() + model_metrics_activity.warning.assert_not_called() def test_adaptive_state_breach_escalates_and_bumps_updated_at(model_metrics_activity, mongo): From d265e3328cd9a7dddda6eaa60356c1afdee07df5 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Thu, 8 Oct 2026 17:45:44 -0300 Subject: [PATCH 22/29] fix(simple_metrics): fail the run explicitly when the adaptive state update fails (SIENTIAPDE-2191) Drop the try/except around update_simple_metrics_adaptive_state in the workflow and the broad except in the activity, so Mongo failures fail the run and show up in the Temporal metrics (PR #59 review). Invalid frequency_alert stays a warning + skip (the orchestrator ignores it too) and a malformed adaptive_state is read as not escalated. --- laborious/activities/model_metrics.py | 84 +++++++++++-------- laborious/workflows/simple_metrics.py | 37 ++++---- .../activities/test_model_metrics.py | 31 +++++-- .../workflows/test_simple_metrics.py | 13 +-- 4 files changed, 98 insertions(+), 67 deletions(-) diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index 27e7087..1d070b3 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -524,7 +524,7 @@ class ModelMetrics(SientiaMonitoring): @activity.defn(name='update_simple_metrics_adaptive_state') def update_simple_metrics_adaptive_state(self, input_data: dict[str, Any]) -> dict | None: """ - Best-effort update of the adaptive frequency state on the schedule's `pipelines` doc. + Update the adaptive frequency state on the schedule's `pipelines` doc. The orchestrator derives the schedule frequency from `adaptive_state.escalated` and `frequency_alert`. State is read from Mongo, not from the workflow input, because the @@ -541,48 +541,62 @@ class ModelMetrics(SientiaMonitoring): - interval_minutes (int): Window size, used to compute the recovery runs. Return: - The new state, or `None` when adaptive mode is off, Mongo is not configured, or the - read/parse/write failed. + The new state, or `None` when adaptive mode is off, Mongo is not configured, the doc + is missing or `frequency_alert` is invalid. + + Raises: + Exception: Mongo read/write failures, so the workflow run fails explicitly. """ metadata = input_data['metadata'] if self.mongo_repository is None: return None + # Mongo errors propagate on purpose: the run must fail explicitly so the problem shows + # up in the Temporal metrics (PR #59 review). Metrics were already exported by then. query = {'schedule_name': metadata['schedule_name']} - try: - pipeline = self.mongo_repository.find_one(self._PIPELINES_COLLECTION, query, metadata) - if not pipeline: - # Likely misconfig: MONGODB_DATABASE_NAME differs from the orchestrator's, or the - # schedule_name does not match its doc. - self.warning( - f'No pipelines doc for schedule {metadata["schedule_name"]}: ' - 'adaptive state skipped', - metadata, - ) - return None - if not pipeline.get('frequency_alert'): - return None - - runs = recovery_runs(input_data['interval_minutes'], pipeline['frequency_alert']) - breached = bool( - threshold_breaches(input_data['simple_metrics'], input_data.get('thresholds')) + pipeline = self.mongo_repository.find_one(self._PIPELINES_COLLECTION, query, metadata) + if not pipeline: + # Likely misconfig: MONGODB_DATABASE_NAME differs from the orchestrator's, or the + # schedule_name does not match its doc. + self.warning( + f'No pipelines doc for schedule {metadata["schedule_name"]}: ' + 'adaptive state skipped', + metadata, ) - state = pipeline.get('adaptive_state') or {'escalated': False, 'healthy_streak': 0} - new_state = next_adaptive_state(state, breached, runs) - if new_state == state: - return new_state - - update: dict[str, Any] = {'adaptive_state': new_state} - if new_state['escalated'] != bool(state.get('escalated')): - # BSON datetime, never a string: the orchestrator normalizes `updated_at` with - # `.replace(tzinfo=UTC).strftime(...)` and a string would fail its whole tick. - update['updated_at'] = now() - self.mongo_repository.update_one( - self._PIPELINES_COLLECTION, query, {'$set': update}, metadata - ) - except Exception as e: - self.error(f'Failed to update simple metrics adaptive state: {e}', metadata) return None + frequency_alert = pipeline.get('frequency_alert') + if not frequency_alert: + return None + + try: + runs = recovery_runs(input_data['interval_minutes'], str(frequency_alert)) + except ValueError: + # Config error, not a failure: the orchestrator ignores this value too (keeps + # `frequency`) and already sends SIMPLE_METRICS_FREQUENCY_ALERT_BELOW_FLOOR for it. + self.warning( + f'Invalid frequency_alert={frequency_alert!r}: adaptive state skipped', metadata + ) + return None + + breached = bool( + threshold_breaches(input_data['simple_metrics'], input_data.get('thresholds')) + ) + state = pipeline.get('adaptive_state') + if not isinstance(state, dict): + # Missing or hand-edited garbage: not escalated, same as the orchestrator reads it. + state = {'escalated': False, 'healthy_streak': 0} + new_state = next_adaptive_state(state, breached, runs) + if new_state == state: + return new_state + + update: dict[str, Any] = {'adaptive_state': new_state} + if new_state['escalated'] != bool(state.get('escalated')): + # BSON datetime, never a string: the orchestrator normalizes `updated_at` with + # `.replace(tzinfo=UTC).strftime(...)` and a string would fail its whole tick. + update['updated_at'] = now() + self.mongo_repository.update_one( + self._PIPELINES_COLLECTION, query, {'$set': update}, metadata + ) self.info(f'Adaptive state of {query["schedule_name"]}: {state} -> {new_state}', metadata) return new_state diff --git a/laborious/workflows/simple_metrics.py b/laborious/workflows/simple_metrics.py index 142f59d..61531c1 100644 --- a/laborious/workflows/simple_metrics.py +++ b/laborious/workflows/simple_metrics.py @@ -114,23 +114,20 @@ class SimpleMetrics: start_to_close_timeout=timedelta(seconds=300), ) - try: - await workflow.execute_activity_method( - Activities.update_simple_metrics_adaptive_state, - { - **metadata, - # Only a valid breach escalates: an empty or under-sampled window is sent as - # no rows, which counts as a healthy run (spec decision 8). Always called, even - # without thresholds (no breach), so an escalated doc recovers after they are - # removed. - 'simple_metrics': (simple_metrics or []) if enough_samples else [], - 'thresholds': thresholds, - 'interval_minutes': interval_minutes, - }, - retry_policy=retry_policy, - start_to_close_timeout=timedelta(minutes=1), - ) - except Exception as e: - # Best-effort: metrics are already persisted. retry_policy has maximum_attempts=1, so - # a dispatch failure would otherwise fail the workflow (same as prediction_process). - workflow.logger.warning(f'Failed to update adaptive state, continuing anyway: {e}') + # No try/except: a failure here must fail the run so it shows up in the Temporal metrics + # (metrics are already persisted above). + await workflow.execute_activity_method( + Activities.update_simple_metrics_adaptive_state, + { + **metadata, + # Only a valid breach escalates: an empty or under-sampled window is sent as no + # rows, which counts as a healthy run (spec decision 8). Always called, even + # without thresholds (no breach), so an escalated doc recovers after they are + # removed. + 'simple_metrics': (simple_metrics or []) if enough_samples else [], + 'thresholds': thresholds, + 'interval_minutes': interval_minutes, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(minutes=1), + ) diff --git a/tests/laborious/activities/test_model_metrics.py b/tests/laborious/activities/test_model_metrics.py index 1c8cf8b..79128b0 100644 --- a/tests/laborious/activities/test_model_metrics.py +++ b/tests/laborious/activities/test_model_metrics.py @@ -1424,16 +1424,33 @@ def test_adaptive_state_unchanged_does_not_write(model_metrics_activity, mongo): mongo.update_one.assert_not_called() -def test_adaptive_state_invalid_frequency_alert_is_swallowed(model_metrics_activity, mongo): - mongo.find_one.return_value = _pipeline(frequency_alert='5min') +@mark.parametrize('frequency_alert', ['5min', '0m', 5]) +def test_adaptive_state_invalid_frequency_alert_is_skipped( + model_metrics_activity, mongo, frequency_alert +): + mongo.find_one.return_value = _pipeline(frequency_alert=frequency_alert) assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None mongo.update_one.assert_not_called() - model_metrics_activity.error.assert_called_once() + model_metrics_activity.warning.assert_called_once() -def test_adaptive_state_mongo_error_is_swallowed(model_metrics_activity, mongo): - mongo.find_one.side_effect = RuntimeError('mongo down') +@mark.parametrize('adaptive_state', ['garbage', ['escalated']]) +def test_adaptive_state_malformed_state_is_not_escalated( + model_metrics_activity, mongo, adaptive_state +): + mongo.find_one.return_value = _pipeline(frequency_alert='5m', adaptive_state=adaptive_state) - assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None - model_metrics_activity.error.assert_called_once() + result = model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) + + assert result == {'escalated': True, 'healthy_streak': 0} + mongo.update_one.assert_called_once() + + +@mark.parametrize('failing_call', ['find_one', 'update_one']) +def test_adaptive_state_mongo_error_propagates(model_metrics_activity, mongo, failing_call): + mongo.find_one.return_value = _pipeline(frequency_alert='5m') + getattr(mongo, failing_call).side_effect = RuntimeError('mongo down') + + with raises(RuntimeError, match='mongo down'): + model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) diff --git a/tests/laborious/workflows/test_simple_metrics.py b/tests/laborious/workflows/test_simple_metrics.py index dfb6833..065ccec 100644 --- a/tests/laborious/workflows/test_simple_metrics.py +++ b/tests/laborious/workflows/test_simple_metrics.py @@ -1,6 +1,6 @@ from unittest.mock import ANY, AsyncMock, MagicMock, call, patch -from pytest import fixture, mark +from pytest import fixture, mark, raises from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ from laborious.activities.activities import Activities @@ -218,7 +218,7 @@ async def test_run_default_metrics(workflow_mock: AsyncMock, simple_metrics: Sim # Act await simple_metrics.run(input_data) - # Assert - all three activities ran (a swallowed exhausted side_effect would warn) + # Assert - all three activities ran assert workflow_mock.execute_activity_method.call_count == 3 workflow_mock.logger.warning.assert_not_called() @@ -402,13 +402,16 @@ async def test_run_below_min_samples_sends_no_rows_to_adaptive_state(workflow_mo @mark.asyncio @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) -async def test_run_adaptive_state_failure_does_not_fail_workflow(workflow_mock, simple_metrics): +async def test_run_adaptive_state_failure_fails_workflow(workflow_mock, simple_metrics): workflow_mock.logger = MagicMock() workflow_mock.execute_activity_method = AsyncMock( side_effect=[[_ROW] * 3, None, RuntimeError('dispatch failed')] ) workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS) - await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS)) + with raises(RuntimeError, match='dispatch failed'): + await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS)) - workflow_mock.logger.warning.assert_called_once() + # Metrics were exported before the failing call + calls = workflow_mock.execute_activity_method.call_args_list + assert calls[1].args[0] is Activities.export_data_to_postgres From a812acd4e9b31188abe36887ae2f902b44b29300 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Fri, 9 Oct 2026 09:50:37 -0300 Subject: [PATCH 23/29] refactor(activities): move Mongo activities into MongoActivities (SIENTIAPDE-2191) Rename PredictionMetadata (prediction_metadata.py) to MongoActivities (mongo.py) and move update_simple_metrics_adaptive_state there from ModelMetrics, so both Mongo-backed activities share one class and repository (PR #59 review, agreed with the class author). write_prediction_metadata is unchanged; the pure hysteresis helpers stay in model_metrics.py. References in activities.py, docstrings and tests updated. --- laborious/activities/activities.py | 18 +- laborious/activities/gates.py | 2 +- laborious/activities/model_metrics.py | 89 +---- laborious/activities/mongo.py | 224 ++++++++++++ laborious/activities/prediction_metadata.py | 132 ------- laborious/activities/storage.py | 2 +- .../sub_workflows/prediction_process.py | 2 +- tests/laborious/activities/test_activities.py | 28 +- tests/laborious/activities/test_gates.py | 4 +- .../activities/test_model_metrics.py | 154 -------- tests/laborious/activities/test_mongo.py | 341 ++++++++++++++++++ .../activities/test_prediction_metadata.py | 179 --------- 12 files changed, 594 insertions(+), 581 deletions(-) create mode 100644 laborious/activities/mongo.py delete mode 100644 laborious/activities/prediction_metadata.py create mode 100644 tests/laborious/activities/test_mongo.py delete mode 100644 tests/laborious/activities/test_prediction_metadata.py diff --git a/laborious/activities/activities.py b/laborious/activities/activities.py index 0d8ef09..cd75673 100644 --- a/laborious/activities/activities.py +++ b/laborious/activities/activities.py @@ -16,22 +16,22 @@ with workflow.unsafe.imports_passed_through(): from laborious.activities.gates import Gates from laborious.activities.mlflow import MLFlow from laborious.activities.model_metrics import ModelMetrics + from laborious.activities.mongo import MongoActivities from laborious.activities.opc import OPC - from laborious.activities.prediction_metadata import PredictionMetadata from laborious.activities.storage import Storage from laborious.utils.connectors_config import build_mlflow_config -class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, PredictionMetadata): +class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, MongoActivities): """ Central orchestrator for all Temporal activities used by Laborious workflows. Composes Storage (Postgres + MinIO offload), MLFlow (wrapper-based inference and retrain via ``SientiaMLflowRepository``), Gates (data quality and ML response filters), OPC exports, - drift/simple metrics, PI Web API writes, and PredictionMetadata (the single write point for - gate metadata — see `laborious.activities.prediction_metadata`). The worker constructs one - ``Activities`` instance per process and registers its callables on multiple workers bound to - different task queues. + drift/simple metrics, PI Web API writes, and MongoActivities (the single write point for + gate metadata and the simple metrics adaptive state — see `laborious.activities.mongo`). + The worker constructs one ``Activities`` instance per process and registers its callables on + multiple workers bound to different task queues. MLflow connectivity: unless ``mlflow_repository`` is injected (tests only), this class builds ``SientiaMLflowRepository`` from ``build_mlflow_config()`` so tracking credentials and URL @@ -73,7 +73,7 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, PredictionMetad - metrics_controller: Optional shared controller; if ``None``, a new one is created. - mlflow_repository: Optional ``SientiaMLflowRepository`` for unit/e2e tests; in production leave unset so the repository is built from environment via ``build_mlflow_config()``. - - mongo_repository: Optional ``MongoDBRepository`` (prediction metadata, best-effort) for + - mongo_repository: Optional ``MongoDBRepository`` (prediction metadata, adaptive state) for unit/e2e tests; in production leave unset so the repository is built from environment via ``build_mongodb_config()``, same pattern as ``mlflow_repository`` above. @@ -153,7 +153,7 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, PredictionMetad metrics_controller=mc, ) - PredictionMetadata.__init__( + MongoActivities.__init__( self, mongo_repository=mongo_repository, logger=logger, @@ -202,4 +202,4 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, PredictionMetad OPC.close(self) ModelMetrics.close(self) API.close(self) - PredictionMetadata.close(self) + MongoActivities.close(self) diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index c6fece9..e07bde3 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -219,7 +219,7 @@ class Gates(SientiaMonitoring): 5. Handles errors gracefully with notification integration 6. Builds a metadata document describing which filter(s) triggered, with their policy and a counter, whenever one does — this activity only computes it, it never writes it - (see `PredictionMetadata.write_prediction_metadata` for the actual write) + (see `MongoActivities.write_prediction_metadata` for the actual write) Args: input_data: Configuration and data for input validation diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index 1d070b3..d2271b9 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -14,8 +14,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.logger import Logger from sientia_do.observability.metrics_controller import MetricsController from sientia_do.observability.sientia_monitoring import SientiaMonitoring - from sientia_do.repository.mongodb_repository_sync import MongoDBRepository - from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ from sientia_model.analytics.drift_analysis import DriftAnalysis, DriftInsufficientDataError from sientia_model.metrics.regression import RegressionMetrics @@ -104,12 +103,6 @@ class ModelMetrics(SientiaMonitoring): _MAX_DEBUG_DATAFRAME_ROWS = 100 - # ponytail: same declaration as PredictionMetadata; the Activities composite sets the - # instance value in PredictionMetadata.__init__. A standalone ModelMetrics (unit tests) keeps - # None and the adaptive-state activity becomes a no-op. - mongo_repository: MongoDBRepository | None = None - _PIPELINES_COLLECTION = 'pipelines' - def __init__( self, logger: Logger, @@ -520,83 +513,3 @@ class ModelMetrics(SientiaMonitoring): ) return data.to_dict(orient='records') - - @activity.defn(name='update_simple_metrics_adaptive_state') - def update_simple_metrics_adaptive_state(self, input_data: dict[str, Any]) -> dict | None: - """ - Update the adaptive frequency state on the schedule's `pipelines` doc. - - The orchestrator derives the schedule frequency from `adaptive_state.escalated` and - `frequency_alert`. State is read from Mongo, not from the workflow input, because the - input only refreshes on the orchestrator tick and would be stale while escalated. - `updated_at` is bumped only when `escalated` flips: that is what makes the orchestrator - rewrite the schedule, and a streak change alone does not change the frequency. - - Args: - input_data: Dict with: - - metadata (dict): Workflow metadata; `schedule_name` identifies the doc. - - simple_metrics (list[dict]): Rows from `calculate_simple_metrics`; empty for an - empty or under-sampled window, which counts as a non-breach run. - - thresholds (dict): `simple_metrics_thresholds` from `model_config` (may be None: no breach). - - interval_minutes (int): Window size, used to compute the recovery runs. - - Return: - The new state, or `None` when adaptive mode is off, Mongo is not configured, the doc - is missing or `frequency_alert` is invalid. - - Raises: - Exception: Mongo read/write failures, so the workflow run fails explicitly. - """ - metadata = input_data['metadata'] - if self.mongo_repository is None: - return None - - # Mongo errors propagate on purpose: the run must fail explicitly so the problem shows - # up in the Temporal metrics (PR #59 review). Metrics were already exported by then. - query = {'schedule_name': metadata['schedule_name']} - pipeline = self.mongo_repository.find_one(self._PIPELINES_COLLECTION, query, metadata) - if not pipeline: - # Likely misconfig: MONGODB_DATABASE_NAME differs from the orchestrator's, or the - # schedule_name does not match its doc. - self.warning( - f'No pipelines doc for schedule {metadata["schedule_name"]}: ' - 'adaptive state skipped', - metadata, - ) - return None - frequency_alert = pipeline.get('frequency_alert') - if not frequency_alert: - return None - - try: - runs = recovery_runs(input_data['interval_minutes'], str(frequency_alert)) - except ValueError: - # Config error, not a failure: the orchestrator ignores this value too (keeps - # `frequency`) and already sends SIMPLE_METRICS_FREQUENCY_ALERT_BELOW_FLOOR for it. - self.warning( - f'Invalid frequency_alert={frequency_alert!r}: adaptive state skipped', metadata - ) - return None - - breached = bool( - threshold_breaches(input_data['simple_metrics'], input_data.get('thresholds')) - ) - state = pipeline.get('adaptive_state') - if not isinstance(state, dict): - # Missing or hand-edited garbage: not escalated, same as the orchestrator reads it. - state = {'escalated': False, 'healthy_streak': 0} - new_state = next_adaptive_state(state, breached, runs) - if new_state == state: - return new_state - - update: dict[str, Any] = {'adaptive_state': new_state} - if new_state['escalated'] != bool(state.get('escalated')): - # BSON datetime, never a string: the orchestrator normalizes `updated_at` with - # `.replace(tzinfo=UTC).strftime(...)` and a string would fail its whole tick. - update['updated_at'] = now() - self.mongo_repository.update_one( - self._PIPELINES_COLLECTION, query, {'$set': update}, metadata - ) - - self.info(f'Adaptive state of {query["schedule_name"]}: {state} -> {new_state}', metadata) - return new_state diff --git a/laborious/activities/mongo.py b/laborious/activities/mongo.py new file mode 100644 index 0000000..2e060e8 --- /dev/null +++ b/laborious/activities/mongo.py @@ -0,0 +1,224 @@ +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + from typing import Any + + from bson import ObjectId + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.observability.logger import Logger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.repository.mongodb_repository_sync import MongoDBRepository + from sientia_do.temporal.constants import now + + from laborious.activities.model_metrics import ( + next_adaptive_state, + recovery_runs, + threshold_breaches, + ) + + +class MongoActivities(SientiaMonitoring): + """ + Activities that read from or write to MongoDB, sharing one `MongoDBRepository`. + + - `write_prediction_metadata`: centralized, best-effort writer for prediction gate metadata + (triggered filter, policy, counter). Gates only compute and return a metadata document; + they never write it themselves. Keeping the write here, instead of in each gate, puts the + one real write point in the subworkflow that's about to persist a `predictions` row — the + only place that reliably knows whether one is being written at all — and lets other + metadata sources reuse the same activity instead of duplicating write logic wherever they + originate. + - `update_simple_metrics_adaptive_state`: adaptive frequency state on the `pipelines` doc of + a simple metrics schedule (SIENTIAPDE-1995). + """ + + mongo_repository: MongoDBRepository | None = None + _PREDICTION_METADATA_COLLECTION = 'prediction_metadata' + _PIPELINES_COLLECTION = 'pipelines' + + def __init__( + self, + mongo_repository: MongoDBRepository | None = None, + logger: Logger | None = None, + notification_handler: NotificationHandler | None = None, + metrics_controller: MetricsController | None = None, + ): + """ + Initialize the Mongo activities. + + Args: + mongo_repository: MongoDB, where gate metadata (triggered filter, policy, counter) is + written best-effort and the simple metrics adaptive state is kept. `None` is a + valid state: both activities become no-ops. + + Raises: + Exception: If BaseActivity initialization fails + """ + self.mongo_repository = mongo_repository + SientiaMonitoring.__init__( + self, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + def close(self) -> None: + """ + Close the Mongo repository and clean up resources. + """ + if self.mongo_repository is not None: + try: + self.mongo_repository.close() + finally: + self.mongo_repository = None + SientiaMonitoring.shutdown(self) + + def __del__(self): + self.close() + + @activity.defn(name='write_prediction_metadata') + def write_prediction_metadata(self, input_data: dict[str, Any]) -> str | None: + """ + Best-effort write of one or more gate metadata documents to Mongo, combined into a single + `prediction_metadata` document. + + Implements both business rules for this feature in one place: + - The prediction write must never be blocked or failed by a metadata write. A failed + `insert(...)` is swallowed, never re-raised, logged here with call-site context (which + `mongo_repository.insert(...)` itself can't know) — matching how `storage.py`'s + `cleanup_minio_objects_expired` also logs again at the call site even though the + repository already logs internally and emits its own Prometheus metric + (`MONGODB_WRITE_ERROR_COUNT`); the generic library log is for the metric/trace, this + one is for whoever is reading logs for this specific call. + - Nothing is written when there's nothing relevant to report. An empty/missing + `documents` list (or one full of falsy/empty entries) skips Mongo entirely, so callers + can call this unconditionally and get `None` back for "no metadata this time" instead + of deciding upfront whether there's something worth saving. + + Takes a *list* of documents, not a single one: each caller contributes its own document + (shaped like `{'filters': [...]}`), and this flattens every `filters` entry across all of + them into one `prediction_metadata` document before the single insert. Today only one + gate ever contributes a document per prediction cycle (the gate chain short-circuits on + the first non-`None` `path_flag`), but the list shape means this activity won't need to + change again once more than one source contributes metadata for the same cycle. + + `insert(...)` doesn't hand back the inserted document's `_id`, so it is generated here, + client-side, before the insert, as a plain string (not a BSON ObjectId) — matching this + codebase's convention of string `_id`s (e.g. the `models` collection) and the 24-hex-char + size of the `predictions.metadata_id` Postgres column that will reference it. + + Args: + input_data: Dict with: + - documents (list[dict[str, Any]]): Metadata payloads to combine, each shaped + like `{'filters': [...]}`. Missing/empty, or entries with no `filters`, skip + the write entirely. + - metadata (dict): Workflow metadata for contextual logging. + + Return: + The generated id, or `None` when there's nothing to write, Mongo isn't configured, or + the insert failed. + """ + metadata = input_data['metadata'] + documents = input_data.get('documents') or [] + + filters = [ + entry for document in documents if document for entry in document.get('filters', []) + ] + + if not filters or self.mongo_repository is None: + return None + + generated_id = str(ObjectId()) + + try: + self.mongo_repository.insert( + self._PREDICTION_METADATA_COLLECTION, + {'filters': filters, '_id': generated_id}, + metadata, + ) + except Exception as e: + self.error(f'Failed to write prediction metadata: {e}', metadata) + return None + + return generated_id + + @activity.defn(name='update_simple_metrics_adaptive_state') + def update_simple_metrics_adaptive_state(self, input_data: dict[str, Any]) -> dict | None: + """ + Update the adaptive frequency state on the schedule's `pipelines` doc. + + The orchestrator derives the schedule frequency from `adaptive_state.escalated` and + `frequency_alert`. State is read from Mongo, not from the workflow input, because the + input only refreshes on the orchestrator tick and would be stale while escalated. + `updated_at` is bumped only when `escalated` flips: that is what makes the orchestrator + rewrite the schedule, and a streak change alone does not change the frequency. + + Args: + input_data: Dict with: + - metadata (dict): Workflow metadata; `schedule_name` identifies the doc. + - simple_metrics (list[dict]): Rows from `calculate_simple_metrics`; empty for an + empty or under-sampled window, which counts as a non-breach run. + - thresholds (dict): `simple_metrics_thresholds` from `model_config` (may be None: no breach). + - interval_minutes (int): Window size, used to compute the recovery runs. + + Return: + The new state, or `None` when adaptive mode is off, Mongo is not configured, the doc + is missing or `frequency_alert` is invalid. + + Raises: + Exception: Mongo read/write failures, so the workflow run fails explicitly. + """ + metadata = input_data['metadata'] + if self.mongo_repository is None: + return None + + # Mongo errors propagate on purpose: the run must fail explicitly so the problem shows + # up in the Temporal metrics (PR #59 review). Metrics were already exported by then. + query = {'schedule_name': metadata['schedule_name']} + pipeline = self.mongo_repository.find_one(self._PIPELINES_COLLECTION, query, metadata) + if not pipeline: + # Likely misconfig: MONGODB_DATABASE_NAME differs from the orchestrator's, or the + # schedule_name does not match its doc. + self.warning( + f'No pipelines doc for schedule {metadata["schedule_name"]}: ' + 'adaptive state skipped', + metadata, + ) + return None + frequency_alert = pipeline.get('frequency_alert') + if not frequency_alert: + return None + + try: + runs = recovery_runs(input_data['interval_minutes'], str(frequency_alert)) + except ValueError: + # Config error, not a failure: the orchestrator ignores this value too (keeps + # `frequency`) and already sends SIMPLE_METRICS_FREQUENCY_ALERT_BELOW_FLOOR for it. + self.warning( + f'Invalid frequency_alert={frequency_alert!r}: adaptive state skipped', metadata + ) + return None + + breached = bool( + threshold_breaches(input_data['simple_metrics'], input_data.get('thresholds')) + ) + state = pipeline.get('adaptive_state') + if not isinstance(state, dict): + # Missing or hand-edited garbage: not escalated, same as the orchestrator reads it. + state = {'escalated': False, 'healthy_streak': 0} + new_state = next_adaptive_state(state, breached, runs) + if new_state == state: + return new_state + + update: dict[str, Any] = {'adaptive_state': new_state} + if new_state['escalated'] != bool(state.get('escalated')): + # BSON datetime, never a string: the orchestrator normalizes `updated_at` with + # `.replace(tzinfo=UTC).strftime(...)` and a string would fail its whole tick. + update['updated_at'] = now() + self.mongo_repository.update_one( + self._PIPELINES_COLLECTION, query, {'$set': update}, metadata + ) + + self.info(f'Adaptive state of {query["schedule_name"]}: {state} -> {new_state}', metadata) + return new_state diff --git a/laborious/activities/prediction_metadata.py b/laborious/activities/prediction_metadata.py deleted file mode 100644 index 1fdb519..0000000 --- a/laborious/activities/prediction_metadata.py +++ /dev/null @@ -1,132 +0,0 @@ -from temporalio import activity, workflow - -with workflow.unsafe.imports_passed_through(): - from typing import Any - - from bson import ObjectId - from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler - from sientia_do.observability.logger import Logger - from sientia_do.observability.metrics_controller import MetricsController - from sientia_do.observability.sientia_monitoring import SientiaMonitoring - from sientia_do.repository.mongodb_repository_sync import MongoDBRepository - - -class PredictionMetadata(SientiaMonitoring): - """ - Centralized, best-effort writer for prediction gate metadata (triggered filter, policy, - counter) to MongoDB. - - Gates only compute and return a metadata document; they never write it themselves. Keeping - the write here, instead of in each gate, puts the one real write point in the subworkflow - that's about to persist a `predictions` row — the only place that reliably knows whether one - is being written at all — and lets other metadata sources reuse the same activity instead of - duplicating write logic wherever they originate. - """ - - mongo_repository: MongoDBRepository | None = None - _PREDICTION_METADATA_COLLECTION = 'prediction_metadata' - - def __init__( - self, - mongo_repository: MongoDBRepository | None = None, - logger: Logger | None = None, - notification_handler: NotificationHandler | None = None, - metrics_controller: MetricsController | None = None, - ): - """ - Initialize the prediction metadata writer. - - Args: - mongo_repository: MongoDB, where gate metadata (triggered filter, policy, counter) is - written best-effort. `None` is a valid state (see `write_prediction_metadata`). - - Raises: - Exception: If BaseActivity initialization fails - """ - self.mongo_repository = mongo_repository - SientiaMonitoring.__init__( - self, - logger=logger, - notification_handler=notification_handler, - metrics_controller=metrics_controller, - ) - - def close(self) -> None: - """ - Close the prediction metadata writer and clean up resources. - """ - if self.mongo_repository is not None: - try: - self.mongo_repository.close() - finally: - self.mongo_repository = None - SientiaMonitoring.shutdown(self) - - def __del__(self): - self.close() - - @activity.defn(name='write_prediction_metadata') - def write_prediction_metadata(self, input_data: dict[str, Any]) -> str | None: - """ - Best-effort write of one or more gate metadata documents to Mongo, combined into a single - `prediction_metadata` document. - - Implements both business rules for this feature in one place: - - The prediction write must never be blocked or failed by a metadata write. A failed - `insert(...)` is swallowed, never re-raised, logged here with call-site context (which - `mongo_repository.insert(...)` itself can't know) — matching how `storage.py`'s - `cleanup_minio_objects_expired` also logs again at the call site even though the - repository already logs internally and emits its own Prometheus metric - (`MONGODB_WRITE_ERROR_COUNT`); the generic library log is for the metric/trace, this - one is for whoever is reading logs for this specific call. - - Nothing is written when there's nothing relevant to report. An empty/missing - `documents` list (or one full of falsy/empty entries) skips Mongo entirely, so callers - can call this unconditionally and get `None` back for "no metadata this time" instead - of deciding upfront whether there's something worth saving. - - Takes a *list* of documents, not a single one: each caller contributes its own document - (shaped like `{'filters': [...]}`), and this flattens every `filters` entry across all of - them into one `prediction_metadata` document before the single insert. Today only one - gate ever contributes a document per prediction cycle (the gate chain short-circuits on - the first non-`None` `path_flag`), but the list shape means this activity won't need to - change again once more than one source contributes metadata for the same cycle. - - `insert(...)` doesn't hand back the inserted document's `_id`, so it is generated here, - client-side, before the insert, as a plain string (not a BSON ObjectId) — matching this - codebase's convention of string `_id`s (e.g. the `models` collection) and the 24-hex-char - size of the `predictions.metadata_id` Postgres column that will reference it. - - Args: - input_data: Dict with: - - documents (list[dict[str, Any]]): Metadata payloads to combine, each shaped - like `{'filters': [...]}`. Missing/empty, or entries with no `filters`, skip - the write entirely. - - metadata (dict): Workflow metadata for contextual logging. - - Return: - The generated id, or `None` when there's nothing to write, Mongo isn't configured, or - the insert failed. - """ - metadata = input_data['metadata'] - documents = input_data.get('documents') or [] - - filters = [ - entry for document in documents if document for entry in document.get('filters', []) - ] - - if not filters or self.mongo_repository is None: - return None - - generated_id = str(ObjectId()) - - try: - self.mongo_repository.insert( - self._PREDICTION_METADATA_COLLECTION, - {'filters': filters, '_id': generated_id}, - metadata, - ) - except Exception as e: - self.error(f'Failed to write prediction metadata: {e}', metadata) - return None - - return generated_id diff --git a/laborious/activities/storage.py b/laborious/activities/storage.py index b4ae983..b59ddd8 100644 --- a/laborious/activities/storage.py +++ b/laborious/activities/storage.py @@ -184,7 +184,7 @@ class Storage(Postgres, SientiaMonitoring): REPEAT persists a new row (unlike STOP, which persists nothing), reusing the previous row's `prediction`/`response_time`/`prediction_status`/`prediction_confidence` values with a fresh `timestamp`/`created_at`. `metadata_id` references the `prediction_metadata` - document for this cycle, if any (see `PredictionMetadata.write_prediction_metadata`); + document for this cycle, if any (see `MongoActivities.write_prediction_metadata`); `None` is a valid value — the column is nullable. Args (query_items): diff --git a/laborious/workflows/sub_workflows/prediction_process.py b/laborious/workflows/sub_workflows/prediction_process.py index 9c59240..543e953 100644 --- a/laborious/workflows/sub_workflows/prediction_process.py +++ b/laborious/workflows/sub_workflows/prediction_process.py @@ -267,7 +267,7 @@ class PredictionProcess: Return: The generated id, or `None` when there's nothing to write, including when the - activity call itself fails (see `PredictionMetadata.write_prediction_metadata`). + activity call itself fails (see `MongoActivities.write_prediction_metadata`). """ if not document: # Skip the remote activity call entirely rather than dispatching it just to have diff --git a/tests/laborious/activities/test_activities.py b/tests/laborious/activities/test_activities.py index f902b09..7dee4c4 100644 --- a/tests/laborious/activities/test_activities.py +++ b/tests/laborious/activities/test_activities.py @@ -5,8 +5,8 @@ from laborious.activities.api import API from laborious.activities.gates import Gates from laborious.activities.mlflow import MLFlow from laborious.activities.model_metrics import ModelMetrics +from laborious.activities.mongo import MongoActivities from laborious.activities.opc import OPC -from laborious.activities.prediction_metadata import PredictionMetadata from laborious.activities.storage import Storage @@ -14,7 +14,7 @@ from laborious.activities.storage import Storage @patch('laborious.activities.activities.MLFlow.__init__') @patch('laborious.activities.activities.OPC.__init__') @patch('laborious.activities.activities.Gates.__init__') -@patch('laborious.activities.activities.PredictionMetadata.__init__') +@patch('laborious.activities.activities.MongoActivities.__init__') @patch('laborious.activities.activities.ModelMetrics.__init__') @patch('laborious.activities.activities.API.__init__') @patch('laborious.activities.activities.MinioRepository') @@ -24,7 +24,7 @@ def test___init__( mock_minio_repository, mock_api_init, mock_model_metrics_init, - mock_prediction_metadata_init, + mock_mongo_activities_init, mock_gates_init, mock_opc_init, mock_mlflow_init, @@ -87,7 +87,7 @@ def test___init__( assert isinstance(activities, Gates) assert isinstance(activities, ModelMetrics) assert isinstance(activities, API) - assert isinstance(activities, PredictionMetadata) + assert isinstance(activities, MongoActivities) mock_storage_init.assert_called_once_with( ANY, @@ -131,7 +131,7 @@ def test___init__( metrics_controller=mock_metrics_controller.return_value, ) - mock_prediction_metadata_init.assert_called_once_with( + mock_mongo_activities_init.assert_called_once_with( ANY, mongo_repository=mongo_repository, logger=logger, @@ -172,7 +172,7 @@ def test___init__( @patch('laborious.activities.activities.MLFlow') @patch('laborious.activities.activities.OPC') @patch('laborious.activities.activities.Gates') -@patch('laborious.activities.activities.PredictionMetadata') +@patch('laborious.activities.activities.MongoActivities') @patch('laborious.activities.activities.ModelMetrics') @patch('laborious.activities.activities.API') @patch('laborious.activities.activities.MinioRepository') @@ -180,7 +180,7 @@ def test_shutdown( _mock_minio_repository, mock_api_init, mock_model_metrics_init, - mock_prediction_metadata_init, + mock_mongo_activities_init, mock_gates_init, mock_opc_init, mock_mlflow_init, @@ -243,7 +243,7 @@ def test_shutdown( mock_gates_init.close.assert_called_once() mock_model_metrics_init.close.assert_called_once() mock_api_init.close.assert_called_once() - mock_prediction_metadata_init.close.assert_called_once() + mock_mongo_activities_init.close.assert_called_once() @patch('laborious.activities.activities.SientiaMLflowRepository') @@ -252,7 +252,7 @@ def test_shutdown( @patch('laborious.activities.activities.MLFlow.__init__') @patch('laborious.activities.activities.OPC.__init__') @patch('laborious.activities.activities.Gates.__init__') -@patch('laborious.activities.activities.PredictionMetadata.__init__') +@patch('laborious.activities.activities.MongoActivities.__init__') @patch('laborious.activities.activities.ModelMetrics.__init__') @patch('laborious.activities.activities.API.__init__') @patch('laborious.activities.activities.MinioRepository') @@ -262,7 +262,7 @@ def test___init___builds_mlflow_repository_when_not_provided( mock_minio_repository, _mock_api_init, _mock_model_metrics_init, - _mock_prediction_metadata_init, + _mock_mongo_activities_init, _mock_gates_init, _mock_opc_init, _mock_mlflow_init, @@ -326,7 +326,7 @@ def test___init___builds_mlflow_repository_when_not_provided( @patch('laborious.activities.activities.MLFlow.__init__') @patch('laborious.activities.activities.OPC.__init__') @patch('laborious.activities.activities.Gates.__init__') -@patch('laborious.activities.activities.PredictionMetadata.__init__') +@patch('laborious.activities.activities.MongoActivities.__init__') @patch('laborious.activities.activities.ModelMetrics.__init__') @patch('laborious.activities.activities.API.__init__') @patch('laborious.activities.activities.MinioRepository') @@ -336,7 +336,7 @@ def test___init___builds_mongo_repository_when_not_provided( mock_minio_repository, _mock_api_init, _mock_model_metrics_init, - mock_prediction_metadata_init, + mock_mongo_activities_init, mock_gates_init, _mock_opc_init, _mock_mlflow_init, @@ -398,9 +398,9 @@ def test___init___builds_mongo_repository_when_not_provided( notification_handler=notification_handler, metrics_controller=mock_metrics_controller.return_value, ) - # The same repository built here is the one handed to PredictionMetadata, not a second + # The same repository built here is the one handed to MongoActivities, not a second # instance. - mock_prediction_metadata_init.assert_called_once_with( + mock_mongo_activities_init.assert_called_once_with( ANY, mongo_repository=mock_mongo_repository_cls.return_value, logger=logger, diff --git a/tests/laborious/activities/test_gates.py b/tests/laborious/activities/test_gates.py index 8f0eb80..ab178e6 100644 --- a/tests/laborious/activities/test_gates.py +++ b/tests/laborious/activities/test_gates.py @@ -147,8 +147,8 @@ def test_input_gate_with_filter(gates_activity): # Assert: the gate only computes the document now — it never writes anywhere, and never # checks path_flag to decide whether to build one (that decision moved to the caller, see - # `PredictionMetadata.write_prediction_metadata` and `prediction_process.py`), so even a STOP - # resolution still gets its document built. + # `MongoActivities.write_prediction_metadata` and `prediction_process.py`), so even a + # STOP resolution still gets its document built. assert (path_flag, confidence, comment) == ('STOP', -1, 'Input data with bad quality') assert document == {'filters': [{'name': 'EMPTY_DATA', 'policy': 'STOP', 'counter': 0}]} gates_activity.debug.assert_called() diff --git a/tests/laborious/activities/test_model_metrics.py b/tests/laborious/activities/test_model_metrics.py index 79128b0..c6490ad 100644 --- a/tests/laborious/activities/test_model_metrics.py +++ b/tests/laborious/activities/test_model_metrics.py @@ -1,4 +1,3 @@ -from datetime import datetime from unittest.mock import ANY, MagicMock, patch from pandas import DataFrame, Series, Timestamp @@ -1301,156 +1300,3 @@ def test_frequency_to_seconds_rejects_invalid(frequency): ) def test_recovery_runs(interval_minutes, frequency_alert, expected): assert recovery_runs(interval_minutes, frequency_alert) == expected - - -_SM_METADATA = {**metadata['metadata'], 'schedule_name': 'm1-simple_metrics-60m'} -_BREACH_INPUT = { - 'metadata': _SM_METADATA, - 'simple_metrics': [{'metric': 'rmse', 'value': 2.0}], - 'thresholds': {'rmse_max': 1.0}, - 'interval_minutes': 60, -} -_HEALTHY_INPUT = {**_BREACH_INPUT, 'simple_metrics': [{'metric': 'rmse', 'value': 0.5}]} -_EMPTY_INPUT = {**_BREACH_INPUT, 'simple_metrics': []} - - -@fixture -def mongo(model_metrics_activity): - repository = MagicMock() - model_metrics_activity.mongo_repository = repository - return repository - - -def _pipeline(**fields) -> dict: - return {'schedule_name': 'm1-simple_metrics-60m', 'frequency': '1h', **fields} - - -def _set_of(mongo: MagicMock) -> dict: - collection, query, update, _metadata = mongo.update_one.call_args.args - assert collection == 'pipelines' - assert query == {'schedule_name': 'm1-simple_metrics-60m'} - return update['$set'] - - -def test_adaptive_state_no_mongo_is_noop(model_metrics_activity): - assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None - - -def test_adaptive_state_missing_pipeline_doc_is_noop(model_metrics_activity, mongo): - mongo.find_one.return_value = None - assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None - mongo.update_one.assert_not_called() - # Likely misconfig (other MONGODB_DATABASE_NAME or schedule_name mismatch): not silent. - model_metrics_activity.warning.assert_called_once() - assert 'm1-simple_metrics-60m' in model_metrics_activity.warning.call_args.args[0] - - -def test_adaptive_state_without_frequency_alert_is_noop(model_metrics_activity, mongo): - mongo.find_one.return_value = _pipeline() - assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None - mongo.update_one.assert_not_called() - model_metrics_activity.warning.assert_not_called() - - -def test_adaptive_state_breach_escalates_and_bumps_updated_at(model_metrics_activity, mongo): - mongo.find_one.return_value = _pipeline(frequency_alert='5m') - - result = model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) - - assert result == {'escalated': True, 'healthy_streak': 0} - update = _set_of(mongo) - assert update['adaptive_state'] == {'escalated': True, 'healthy_streak': 0} - # BSON datetime, never a string: the orchestrator calls .replace(tzinfo=...) on it. - assert isinstance(update['updated_at'], datetime) - assert update['updated_at'].tzinfo is not None - - -def test_adaptive_state_healthy_run_increments_streak_without_updated_at( - model_metrics_activity, mongo -): - # 60 min window / 5m = N 12, so one healthy run only moves the streak. - mongo.find_one.return_value = _pipeline( - frequency_alert='5m', adaptive_state={'escalated': True, 'healthy_streak': 0} - ) - - result = model_metrics_activity.update_simple_metrics_adaptive_state(_HEALTHY_INPUT) - - assert result == {'escalated': True, 'healthy_streak': 1} - assert _set_of(mongo) == {'adaptive_state': {'escalated': True, 'healthy_streak': 1}} - - -def test_adaptive_state_recovers_after_window_renewed(model_metrics_activity, mongo): - # 60 min window / 30m = N 2: second consecutive healthy run recovers. - mongo.find_one.return_value = _pipeline( - frequency_alert='30m', adaptive_state={'escalated': True, 'healthy_streak': 1} - ) - - result = model_metrics_activity.update_simple_metrics_adaptive_state(_HEALTHY_INPUT) - - assert result == {'escalated': False, 'healthy_streak': 0} - update = _set_of(mongo) - assert update['adaptive_state'] == {'escalated': False, 'healthy_streak': 0} - assert isinstance(update['updated_at'], datetime) - - -def test_adaptive_state_empty_window_counts_as_healthy_run(model_metrics_activity, mongo): - # Spec decision 8: an empty/under-sampled window never escalates but counts toward recovery. - mongo.find_one.return_value = _pipeline( - frequency_alert='5m', adaptive_state={'escalated': True, 'healthy_streak': 3} - ) - - result = model_metrics_activity.update_simple_metrics_adaptive_state(_EMPTY_INPUT) - - assert result == {'escalated': True, 'healthy_streak': 4} - - -def test_adaptive_state_empty_window_never_escalates(model_metrics_activity, mongo): - mongo.find_one.return_value = _pipeline(frequency_alert='5m') - - result = model_metrics_activity.update_simple_metrics_adaptive_state(_EMPTY_INPUT) - - assert result == {'escalated': False, 'healthy_streak': 0} - mongo.update_one.assert_not_called() - - -def test_adaptive_state_unchanged_does_not_write(model_metrics_activity, mongo): - mongo.find_one.return_value = _pipeline( - frequency_alert='5m', adaptive_state={'escalated': False, 'healthy_streak': 0} - ) - - result = model_metrics_activity.update_simple_metrics_adaptive_state(_HEALTHY_INPUT) - - assert result == {'escalated': False, 'healthy_streak': 0} - mongo.update_one.assert_not_called() - - -@mark.parametrize('frequency_alert', ['5min', '0m', 5]) -def test_adaptive_state_invalid_frequency_alert_is_skipped( - model_metrics_activity, mongo, frequency_alert -): - mongo.find_one.return_value = _pipeline(frequency_alert=frequency_alert) - - assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None - mongo.update_one.assert_not_called() - model_metrics_activity.warning.assert_called_once() - - -@mark.parametrize('adaptive_state', ['garbage', ['escalated']]) -def test_adaptive_state_malformed_state_is_not_escalated( - model_metrics_activity, mongo, adaptive_state -): - mongo.find_one.return_value = _pipeline(frequency_alert='5m', adaptive_state=adaptive_state) - - result = model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) - - assert result == {'escalated': True, 'healthy_streak': 0} - mongo.update_one.assert_called_once() - - -@mark.parametrize('failing_call', ['find_one', 'update_one']) -def test_adaptive_state_mongo_error_propagates(model_metrics_activity, mongo, failing_call): - mongo.find_one.return_value = _pipeline(frequency_alert='5m') - getattr(mongo, failing_call).side_effect = RuntimeError('mongo down') - - with raises(RuntimeError, match='mongo down'): - model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) diff --git a/tests/laborious/activities/test_mongo.py b/tests/laborious/activities/test_mongo.py new file mode 100644 index 0000000..415cafd --- /dev/null +++ b/tests/laborious/activities/test_mongo.py @@ -0,0 +1,341 @@ +from datetime import datetime +from unittest.mock import MagicMock, call, patch + +from pytest import fixture, mark, raises +from sientia_do.observability.metrics_controller import MetricsController +from sientia_do.observability.sientia_monitoring import SientiaMonitoring + +from laborious.activities.mongo import MongoActivities + + +@fixture +def mongo() -> MagicMock: + return MagicMock() + + +@fixture +def prediction_metadata(mongo: MagicMock): + activity = MongoActivities( + mongo_repository=mongo, + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(spec=MetricsController), + ) + activity.error = MagicMock() + return activity + + +metadata = { + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', + }, +} + + +def test_write_prediction_metadata_no_documents_skips_mongo(prediction_metadata, mongo: MagicMock): + # Act + result = prediction_metadata.write_prediction_metadata({**metadata, 'documents': []}) + + # Assert + assert result is None + mongo.insert.assert_not_called() + + +def test_write_prediction_metadata_missing_documents_key_skips_mongo( + prediction_metadata, mongo: MagicMock +): + # Act + result = prediction_metadata.write_prediction_metadata({**metadata}) + + # Assert + assert result is None + mongo.insert.assert_not_called() + + +def test_write_prediction_metadata_documents_with_no_filters_skips_mongo( + prediction_metadata, mongo: MagicMock +): + # Arrange: a falsy document (None) and one with an empty `filters` list — neither contributes + # anything to write. + documents = [None, {'filters': []}] + + # Act + result = prediction_metadata.write_prediction_metadata({**metadata, 'documents': documents}) + + # Assert + assert result is None + mongo.insert.assert_not_called() + + +def test_write_prediction_metadata_not_configured_returns_none(): + # Arrange + activity = MongoActivities( + mongo_repository=None, + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(spec=MetricsController), + ) + activity.error = MagicMock() + document = {'filters': [{'name': 'EMPTY_DATA', 'policy': 'STOP', 'counter': 0}]} + + # Act + result = activity.write_prediction_metadata({**metadata, 'documents': [document]}) + + # Assert + assert result is None + activity.error.assert_not_called() + + +def test_write_prediction_metadata_success_generates_id_and_inserts( + prediction_metadata, mongo: MagicMock +): + # Arrange + document = {'filters': [{'name': 'EMPTY_DATA', 'policy': 'STOP', 'counter': 3}]} + + # Act + result = prediction_metadata.write_prediction_metadata({**metadata, 'documents': [document]}) + + # Assert: a plain 24-hex-char string id (matches predictions.metadata_id's varchar(24)), + # not a BSON ObjectId — `insert(...)` never hands one back, so it's generated client-side. + assert result is not None + assert len(result) == 24 + int(result, 16) # raises ValueError if not hex + + collection, inserted_document, inserted_metadata = mongo.insert.call_args.args + assert collection == MongoActivities._PREDICTION_METADATA_COLLECTION + assert inserted_document == {'filters': document['filters'], '_id': result} + assert inserted_metadata == metadata['metadata'] + + +def test_write_prediction_metadata_combines_multiple_documents( + prediction_metadata, mongo: MagicMock +): + # Arrange: two documents (e.g. from two different sources contributing metadata for the same + # prediction cycle) — their `filters` entries are flattened into one combined document. + documents = [ + {'filters': [{'name': 'EMPTY_DATA', 'policy': 'CONTINUE', 'counter': 0}]}, + { + 'filters': [ + {'name': 'SPECIFIC_VARIABLES_NULL_VALUES', 'policy': 'CONTINUE', 'counter': 2} + ] + }, + ] + + # Act + result = prediction_metadata.write_prediction_metadata({**metadata, 'documents': documents}) + + # Assert + assert result is not None + _, inserted_document, _ = mongo.insert.call_args.args + assert inserted_document['filters'] == [ + {'name': 'EMPTY_DATA', 'policy': 'CONTINUE', 'counter': 0}, + {'name': 'SPECIFIC_VARIABLES_NULL_VALUES', 'policy': 'CONTINUE', 'counter': 2}, + ] + + +def test_write_prediction_metadata_insert_failure_swallowed(prediction_metadata, mongo: MagicMock): + # Arrange + mongo.insert.side_effect = RuntimeError('connection reset') + document = {'filters': [{'name': 'EMPTY_DATA', 'policy': 'STOP', 'counter': 0}]} + + # Act + result = prediction_metadata.write_prediction_metadata({**metadata, 'documents': [document]}) + + # Assert: the prediction write must never be blocked or failed by this + assert result is None + prediction_metadata.error.assert_called_once() + + +@patch.object(SientiaMonitoring, 'shutdown') +def test_close_disposes_mongo_repository(mock_shutdown): + """``MongoActivities.close`` should close the Mongo client and clear the reference.""" + mongo = MagicMock() + activity = MongoActivities( + mongo_repository=mongo, + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(spec=MetricsController), + ) + activity.close() + mongo.close.assert_called_once() + assert activity.mongo_repository is None + # Other instances collected by GC during the test also hit the patched shutdown. + assert mock_shutdown.call_args_list.count(call(activity)) == 1 + + +@patch.object(SientiaMonitoring, 'shutdown') +def test_close_without_mongo_repository(mock_shutdown): + """When no Mongo repository is configured, ``close`` only shuts down monitoring.""" + activity = MongoActivities( + mongo_repository=None, + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(spec=MetricsController), + ) + activity.close() + # Other instances collected by GC during the test also hit the patched shutdown. + assert mock_shutdown.call_args_list.count(call(activity)) == 1 + + +_SM_METADATA = {**metadata['metadata'], 'schedule_name': 'm1-simple_metrics-60m'} +_BREACH_INPUT = { + 'metadata': _SM_METADATA, + 'simple_metrics': [{'metric': 'rmse', 'value': 2.0}], + 'thresholds': {'rmse_max': 1.0}, + 'interval_minutes': 60, +} +_HEALTHY_INPUT = {**_BREACH_INPUT, 'simple_metrics': [{'metric': 'rmse', 'value': 0.5}]} +_EMPTY_INPUT = {**_BREACH_INPUT, 'simple_metrics': []} + + +@fixture +def mongo_activities(mongo: MagicMock): + activity = MongoActivities( + mongo_repository=mongo, + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(spec=MetricsController), + ) + activity.info = MagicMock() + activity.warning = MagicMock() + return activity + + +def _pipeline(**fields) -> dict: + return {'schedule_name': 'm1-simple_metrics-60m', 'frequency': '1h', **fields} + + +def _set_of(mongo: MagicMock) -> dict: + collection, query, update, _metadata = mongo.update_one.call_args.args + assert collection == 'pipelines' + assert query == {'schedule_name': 'm1-simple_metrics-60m'} + return update['$set'] + + +def test_adaptive_state_no_mongo_is_noop(): + activity = MongoActivities( + mongo_repository=None, + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(spec=MetricsController), + ) + assert activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None + + +def test_adaptive_state_missing_pipeline_doc_is_noop(mongo_activities, mongo): + mongo.find_one.return_value = None + assert mongo_activities.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None + mongo.update_one.assert_not_called() + # Likely misconfig (other MONGODB_DATABASE_NAME or schedule_name mismatch): not silent. + mongo_activities.warning.assert_called_once() + assert 'm1-simple_metrics-60m' in mongo_activities.warning.call_args.args[0] + + +def test_adaptive_state_without_frequency_alert_is_noop(mongo_activities, mongo): + mongo.find_one.return_value = _pipeline() + assert mongo_activities.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None + mongo.update_one.assert_not_called() + mongo_activities.warning.assert_not_called() + + +def test_adaptive_state_breach_escalates_and_bumps_updated_at(mongo_activities, mongo): + mongo.find_one.return_value = _pipeline(frequency_alert='5m') + + result = mongo_activities.update_simple_metrics_adaptive_state(_BREACH_INPUT) + + assert result == {'escalated': True, 'healthy_streak': 0} + update = _set_of(mongo) + assert update['adaptive_state'] == {'escalated': True, 'healthy_streak': 0} + # BSON datetime, never a string: the orchestrator calls .replace(tzinfo=...) on it. + assert isinstance(update['updated_at'], datetime) + assert update['updated_at'].tzinfo is not None + + +def test_adaptive_state_healthy_run_increments_streak_without_updated_at(mongo_activities, mongo): + # 60 min window / 5m = N 12, so one healthy run only moves the streak. + mongo.find_one.return_value = _pipeline( + frequency_alert='5m', adaptive_state={'escalated': True, 'healthy_streak': 0} + ) + + result = mongo_activities.update_simple_metrics_adaptive_state(_HEALTHY_INPUT) + + assert result == {'escalated': True, 'healthy_streak': 1} + assert _set_of(mongo) == {'adaptive_state': {'escalated': True, 'healthy_streak': 1}} + + +def test_adaptive_state_recovers_after_window_renewed(mongo_activities, mongo): + # 60 min window / 30m = N 2: second consecutive healthy run recovers. + mongo.find_one.return_value = _pipeline( + frequency_alert='30m', adaptive_state={'escalated': True, 'healthy_streak': 1} + ) + + result = mongo_activities.update_simple_metrics_adaptive_state(_HEALTHY_INPUT) + + assert result == {'escalated': False, 'healthy_streak': 0} + update = _set_of(mongo) + assert update['adaptive_state'] == {'escalated': False, 'healthy_streak': 0} + assert isinstance(update['updated_at'], datetime) + + +def test_adaptive_state_empty_window_counts_as_healthy_run(mongo_activities, mongo): + # Spec decision 8: an empty/under-sampled window never escalates but counts toward recovery. + mongo.find_one.return_value = _pipeline( + frequency_alert='5m', adaptive_state={'escalated': True, 'healthy_streak': 3} + ) + + result = mongo_activities.update_simple_metrics_adaptive_state(_EMPTY_INPUT) + + assert result == {'escalated': True, 'healthy_streak': 4} + + +def test_adaptive_state_empty_window_never_escalates(mongo_activities, mongo): + mongo.find_one.return_value = _pipeline(frequency_alert='5m') + + result = mongo_activities.update_simple_metrics_adaptive_state(_EMPTY_INPUT) + + assert result == {'escalated': False, 'healthy_streak': 0} + mongo.update_one.assert_not_called() + + +def test_adaptive_state_unchanged_does_not_write(mongo_activities, mongo): + mongo.find_one.return_value = _pipeline( + frequency_alert='5m', adaptive_state={'escalated': False, 'healthy_streak': 0} + ) + + result = mongo_activities.update_simple_metrics_adaptive_state(_HEALTHY_INPUT) + + assert result == {'escalated': False, 'healthy_streak': 0} + mongo.update_one.assert_not_called() + + +@mark.parametrize('frequency_alert', ['5min', '0m', 5]) +def test_adaptive_state_invalid_frequency_alert_is_skipped( + mongo_activities, mongo, frequency_alert +): + mongo.find_one.return_value = _pipeline(frequency_alert=frequency_alert) + + assert mongo_activities.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None + mongo.update_one.assert_not_called() + mongo_activities.warning.assert_called_once() + + +@mark.parametrize('adaptive_state', ['garbage', ['escalated']]) +def test_adaptive_state_malformed_state_is_not_escalated(mongo_activities, mongo, adaptive_state): + mongo.find_one.return_value = _pipeline(frequency_alert='5m', adaptive_state=adaptive_state) + + result = mongo_activities.update_simple_metrics_adaptive_state(_BREACH_INPUT) + + assert result == {'escalated': True, 'healthy_streak': 0} + mongo.update_one.assert_called_once() + + +@mark.parametrize('failing_call', ['find_one', 'update_one']) +def test_adaptive_state_mongo_error_propagates(mongo_activities, mongo, failing_call): + mongo.find_one.return_value = _pipeline(frequency_alert='5m') + getattr(mongo, failing_call).side_effect = RuntimeError('mongo down') + + with raises(RuntimeError, match='mongo down'): + mongo_activities.update_simple_metrics_adaptive_state(_BREACH_INPUT) diff --git a/tests/laborious/activities/test_prediction_metadata.py b/tests/laborious/activities/test_prediction_metadata.py deleted file mode 100644 index b7c1143..0000000 --- a/tests/laborious/activities/test_prediction_metadata.py +++ /dev/null @@ -1,179 +0,0 @@ -from unittest.mock import MagicMock, call, patch - -from pytest import fixture -from sientia_do.observability.metrics_controller import MetricsController -from sientia_do.observability.sientia_monitoring import SientiaMonitoring - -from laborious.activities.prediction_metadata import PredictionMetadata - - -@fixture -def mongo() -> MagicMock: - return MagicMock() - - -@fixture -def prediction_metadata(mongo: MagicMock): - activity = PredictionMetadata( - mongo_repository=mongo, - logger=MagicMock(), - notification_handler=MagicMock(), - metrics_controller=MagicMock(spec=MetricsController), - ) - activity.error = MagicMock() - return activity - - -metadata = { - 'metadata': { - 'model_id': 'test_model', - 'model_name': 'test_model', - 'workflow_name': 'test_workflow', - 'schema_name': 'test_schedule', - }, -} - - -def test_write_prediction_metadata_no_documents_skips_mongo(prediction_metadata, mongo: MagicMock): - # Act - result = prediction_metadata.write_prediction_metadata({**metadata, 'documents': []}) - - # Assert - assert result is None - mongo.insert.assert_not_called() - - -def test_write_prediction_metadata_missing_documents_key_skips_mongo( - prediction_metadata, mongo: MagicMock -): - # Act - result = prediction_metadata.write_prediction_metadata({**metadata}) - - # Assert - assert result is None - mongo.insert.assert_not_called() - - -def test_write_prediction_metadata_documents_with_no_filters_skips_mongo( - prediction_metadata, mongo: MagicMock -): - # Arrange: a falsy document (None) and one with an empty `filters` list — neither contributes - # anything to write. - documents = [None, {'filters': []}] - - # Act - result = prediction_metadata.write_prediction_metadata({**metadata, 'documents': documents}) - - # Assert - assert result is None - mongo.insert.assert_not_called() - - -def test_write_prediction_metadata_not_configured_returns_none(): - # Arrange - activity = PredictionMetadata( - mongo_repository=None, - logger=MagicMock(), - notification_handler=MagicMock(), - metrics_controller=MagicMock(spec=MetricsController), - ) - activity.error = MagicMock() - document = {'filters': [{'name': 'EMPTY_DATA', 'policy': 'STOP', 'counter': 0}]} - - # Act - result = activity.write_prediction_metadata({**metadata, 'documents': [document]}) - - # Assert - assert result is None - activity.error.assert_not_called() - - -def test_write_prediction_metadata_success_generates_id_and_inserts( - prediction_metadata, mongo: MagicMock -): - # Arrange - document = {'filters': [{'name': 'EMPTY_DATA', 'policy': 'STOP', 'counter': 3}]} - - # Act - result = prediction_metadata.write_prediction_metadata({**metadata, 'documents': [document]}) - - # Assert: a plain 24-hex-char string id (matches predictions.metadata_id's varchar(24)), - # not a BSON ObjectId — `insert(...)` never hands one back, so it's generated client-side. - assert result is not None - assert len(result) == 24 - int(result, 16) # raises ValueError if not hex - - collection, inserted_document, inserted_metadata = mongo.insert.call_args.args - assert collection == PredictionMetadata._PREDICTION_METADATA_COLLECTION - assert inserted_document == {'filters': document['filters'], '_id': result} - assert inserted_metadata == metadata['metadata'] - - -def test_write_prediction_metadata_combines_multiple_documents( - prediction_metadata, mongo: MagicMock -): - # Arrange: two documents (e.g. from two different sources contributing metadata for the same - # prediction cycle) — their `filters` entries are flattened into one combined document. - documents = [ - {'filters': [{'name': 'EMPTY_DATA', 'policy': 'CONTINUE', 'counter': 0}]}, - { - 'filters': [ - {'name': 'SPECIFIC_VARIABLES_NULL_VALUES', 'policy': 'CONTINUE', 'counter': 2} - ] - }, - ] - - # Act - result = prediction_metadata.write_prediction_metadata({**metadata, 'documents': documents}) - - # Assert - assert result is not None - _, inserted_document, _ = mongo.insert.call_args.args - assert inserted_document['filters'] == [ - {'name': 'EMPTY_DATA', 'policy': 'CONTINUE', 'counter': 0}, - {'name': 'SPECIFIC_VARIABLES_NULL_VALUES', 'policy': 'CONTINUE', 'counter': 2}, - ] - - -def test_write_prediction_metadata_insert_failure_swallowed(prediction_metadata, mongo: MagicMock): - # Arrange - mongo.insert.side_effect = RuntimeError('connection reset') - document = {'filters': [{'name': 'EMPTY_DATA', 'policy': 'STOP', 'counter': 0}]} - - # Act - result = prediction_metadata.write_prediction_metadata({**metadata, 'documents': [document]}) - - # Assert: the prediction write must never be blocked or failed by this - assert result is None - prediction_metadata.error.assert_called_once() - - -@patch.object(SientiaMonitoring, 'shutdown') -def test_close_disposes_mongo_repository(mock_shutdown): - """``PredictionMetadata.close`` should close the Mongo client and clear the reference.""" - mongo = MagicMock() - activity = PredictionMetadata( - mongo_repository=mongo, - logger=MagicMock(), - notification_handler=MagicMock(), - metrics_controller=MagicMock(spec=MetricsController), - ) - activity.close() - mongo.close.assert_called_once() - assert activity.mongo_repository is None - # Other instances collected by GC during the test also hit the patched shutdown. - assert mock_shutdown.call_args_list.count(call(activity)) == 1 - - -@patch.object(SientiaMonitoring, 'shutdown') -def test_close_without_mongo_repository(mock_shutdown): - """When no Mongo repository is configured, ``close`` only shuts down monitoring.""" - activity = PredictionMetadata( - mongo_repository=None, - logger=MagicMock(), - notification_handler=MagicMock(), - metrics_controller=MagicMock(spec=MetricsController), - ) - activity.close() - # Other instances collected by GC during the test also hit the patched shutdown. - assert mock_shutdown.call_args_list.count(call(activity)) == 1 From e1b94b1da4b390d8fa2f3a42c8a2d0d23e1814d0 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Fri, 9 Oct 2026 10:06:05 -0300 Subject: [PATCH 24/29] fix(simple_metrics): skip invalid interval_minutes and restart a malformed healthy_streak (SIENTIAPDE-2191) - interval_minutes is untyped Mongo config: a string or null now warns and skips the adaptive state instead of failing every run (TypeError). - A hand-edited healthy_streak (null, text, inf) restarts the count and keeps escalated, instead of raising on every run. - README: step 4 runs on every schedule, so a MongoDB failure fails the run. --- README.md | 2 +- laborious/activities/model_metrics.py | 5 ++++- laborious/activities/mongo.py | 11 +++++++---- tests/laborious/activities/test_model_metrics.py | 3 +++ tests/laborious/activities/test_mongo.py | 16 ++++++++++++++++ 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8c9d0dc..6f3172c 100644 --- a/README.md +++ b/README.md @@ -489,7 +489,7 @@ The **SimpleMetrics** workflow calculates regression metrics (RMSE, MSE, MAE, R 1. **Data Loading**: Loads prediction vs target data via a JOIN query 2. **Metrics Calculation**: Calculates configured regression metrics; alerts on `simple_metrics_thresholds` breaches (one alert per model, the message names the window) unless the window has fewer rows than `simple_metrics_min_samples` 3. **Data Export**: Exports metrics to PostgreSQL -4. **Adaptive State**: when the schedule's `pipelines` doc has `frequency_alert`, escalates `adaptive_state` on a breach and recovers after the window is renewed without one (`min(12, ceil(interval_minutes / frequency_alert))` runs, at least 1; empty, under-sampled or threshold-less windows count as healthy). The orchestrator runs the schedule at `frequency_alert` while escalated +4. **Adaptive State**: when the schedule's `pipelines` doc has `frequency_alert`, escalates `adaptive_state` on a breach and recovers after the window is renewed without one (`min(12, ceil(interval_minutes / frequency_alert))` runs, at least 1; empty, under-sampled or threshold-less windows count as healthy). The orchestrator runs the schedule at `frequency_alert` while escalated. Runs on every schedule (it reads the `pipelines` doc to check for `frequency_alert`), so a MongoDB failure here fails the run; the metrics are already exported by then #### Architecture Diagram ```mermaid diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index d2271b9..650ed94 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -85,7 +85,10 @@ def next_adaptive_state(state: dict[str, Any], breached: bool, runs: int) -> dic return {'escalated': True, 'healthy_streak': 0} if not state.get('escalated'): return {'escalated': False, 'healthy_streak': 0} - streak = int(state.get('healthy_streak', 0)) + 1 + try: + streak = int(state.get('healthy_streak', 0)) + 1 + except (TypeError, ValueError, OverflowError): + streak = 1 # hand-edited streak (null, text, inf): restart the count, stay escalated if streak >= runs: return {'escalated': False, 'healthy_streak': 0} return {'escalated': True, 'healthy_streak': streak} diff --git a/laborious/activities/mongo.py b/laborious/activities/mongo.py index 2e060e8..fc4d92f 100644 --- a/laborious/activities/mongo.py +++ b/laborious/activities/mongo.py @@ -192,11 +192,14 @@ class MongoActivities(SientiaMonitoring): try: runs = recovery_runs(input_data['interval_minutes'], str(frequency_alert)) - except ValueError: - # Config error, not a failure: the orchestrator ignores this value too (keeps - # `frequency`) and already sends SIMPLE_METRICS_FREQUENCY_ALERT_BELOW_FLOOR for it. + except (TypeError, ValueError): + # Config error, not a failure: the orchestrator ignores an invalid `frequency_alert` + # too (keeps `frequency`) and already sends SIMPLE_METRICS_FREQUENCY_ALERT_BELOW_FLOOR + # for it. `interval_minutes` is untyped Mongo config as well (e.g. "60" or null). self.warning( - f'Invalid frequency_alert={frequency_alert!r}: adaptive state skipped', metadata + f'Invalid frequency_alert={frequency_alert!r} or ' + f'interval_minutes={input_data["interval_minutes"]!r}: adaptive state skipped', + metadata, ) return None diff --git a/tests/laborious/activities/test_model_metrics.py b/tests/laborious/activities/test_model_metrics.py index c6490ad..ed7ce56 100644 --- a/tests/laborious/activities/test_model_metrics.py +++ b/tests/laborious/activities/test_model_metrics.py @@ -1267,6 +1267,9 @@ def _state(escalated: bool, streak: int) -> dict: (_state(True, 2), False, 3, _state(False, 0)), # N-th healthy run recovers (_state(True, 2), True, 3, _state(True, 0)), # breach resets the streak (_state(True, 0), False, 1, _state(False, 0)), # N=1 recovers on first healthy run + (_state(True, None), False, 3, _state(True, 1)), # hand-edited streak restarts + (_state(True, 'x'), False, 3, _state(True, 1)), + (_state(True, float('inf')), False, 3, _state(True, 1)), ], ) def test_next_adaptive_state(state, breached, runs, expected): diff --git a/tests/laborious/activities/test_mongo.py b/tests/laborious/activities/test_mongo.py index 415cafd..837e410 100644 --- a/tests/laborious/activities/test_mongo.py +++ b/tests/laborious/activities/test_mongo.py @@ -322,6 +322,22 @@ def test_adaptive_state_invalid_frequency_alert_is_skipped( mongo_activities.warning.assert_called_once() +@mark.parametrize('interval_minutes', ['60', None]) +def test_adaptive_state_invalid_interval_minutes_is_skipped( + mongo_activities, mongo, interval_minutes +): + # Untyped Mongo config: "60" * 60 is a string, None * 60 a TypeError. + mongo.find_one.return_value = _pipeline(frequency_alert='5m') + + result = mongo_activities.update_simple_metrics_adaptive_state( + {**_BREACH_INPUT, 'interval_minutes': interval_minutes} + ) + + assert result is None + mongo.update_one.assert_not_called() + mongo_activities.warning.assert_called_once() + + @mark.parametrize('adaptive_state', ['garbage', ['escalated']]) def test_adaptive_state_malformed_state_is_not_escalated(mongo_activities, mongo, adaptive_state): mongo.find_one.return_value = _pipeline(frequency_alert='5m', adaptive_state=adaptive_state) From 6cc69b0f9311da3f6fe08ad5b199f7d8d51bd570 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Fri, 9 Oct 2026 10:36:32 -0300 Subject: [PATCH 25/29] test(simple_metrics): build the input outside pytest.raises (Sonar python:S5778) (SIENTIAPDE-2191) --- tests/laborious/workflows/test_simple_metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/laborious/workflows/test_simple_metrics.py b/tests/laborious/workflows/test_simple_metrics.py index 065ccec..a3c8632 100644 --- a/tests/laborious/workflows/test_simple_metrics.py +++ b/tests/laborious/workflows/test_simple_metrics.py @@ -408,9 +408,10 @@ async def test_run_adaptive_state_failure_fails_workflow(workflow_mock, simple_m side_effect=[[_ROW] * 3, None, RuntimeError('dispatch failed')] ) workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS) + input_data = _input(simple_metrics_thresholds=_THRESHOLDS) with raises(RuntimeError, match='dispatch failed'): - await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS)) + await simple_metrics.run(input_data) # Metrics were exported before the failing call calls = workflow_mock.execute_activity_method.call_args_list From 694915e94f709556d037312d326049da484b438b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 9 Oct 2026 17:26:21 +0000 Subject: [PATCH 26/29] chore: bump version to 0.9.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b808703..80df0da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "laborious" -version = "0.9.0-rc2" +version = "0.9.0" description = "Sientia DataOps Laborious - ML Model Orchestration System" readme = "README.md" requires-python = ">=3.11" From a9a2d7af80a1c1fbe067eb2f1f1dc6ab5113d2a4 Mon Sep 17 00:00:00 2001 From: "ci-dependencies-fetcher[bot]" <1275848+ci-dependencies-fetcher[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:26:58 +0000 Subject: [PATCH 27/29] Revert "chore: bump version to 0.9.0" This reverts commit 694915e94f709556d037312d326049da484b438b. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 80df0da..b808703 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "laborious" -version = "0.9.0" +version = "0.9.0-rc2" description = "Sientia DataOps Laborious - ML Model Orchestration System" readme = "README.md" requires-python = ">=3.11" From cb8cc0a383773ea86135cc393d01a37a31a29d1d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 9 Oct 2026 17:32:03 +0000 Subject: [PATCH 28/29] chore: bump version to 0.9.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b808703..80df0da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "laborious" -version = "0.9.0-rc2" +version = "0.9.0" description = "Sientia DataOps Laborious - ML Model Orchestration System" readme = "README.md" requires-python = ">=3.11" From 5d2bb1527f643530100ab6d9b12f1625d53ceccf Mon Sep 17 00:00:00 2001 From: "ci-dependencies-fetcher[bot]" <1275848+ci-dependencies-fetcher[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:32:57 +0000 Subject: [PATCH 29/29] Revert "chore: bump version to 0.9.0" This reverts commit cb8cc0a383773ea86135cc393d01a37a31a29d1d. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 80df0da..b808703 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "laborious" -version = "0.9.0" +version = "0.9.0-rc2" description = "Sientia DataOps Laborious - ML Model Orchestration System" readme = "README.md" requires-python = ">=3.11"