SIENTIAPDE-1273

Update version and enhance metrics calculation in Laborious system

- Updated image tag in values.yaml from 1.1.0 to 1.1.1.
- Modified GITHUB_BRANCH environment variable for consistency.
- Added a new method `calculate_simple_metrics` in model_metrics.py to compute various model performance metrics including RMSE, MSE, MAE, and R2.
- Integrated the new metrics calculation into the worker setup, allowing for concurrent processing of simple metrics.
- Updated tests to cover the new metrics calculation functionality, ensuring comprehensive validation of the implementation.

laborious/activities/model_metrics.py

@@ -13,6 +13,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia.ModelAnalysis import ModelAnalysis
     from laborious import metrics
     import time
+    import numpy as np
     from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
     import warnings
     import traceback
@@ -20,7 +21,6 @@ with workflow.unsafe.imports_passed_through():
 warnings.filterwarnings('ignore', category=RuntimeWarning, message='Degrees of freedom <= 0')
 warnings.filterwarnings('ignore', category=RuntimeWarning, message='invalid value encountered in scalar divide')
 
-
 class ModelMetrics(SientiaMonitoring):
     """
     Metrics activities for the Laborious system.
@@ -264,3 +264,84 @@ class ModelMetrics(SientiaMonitoring):
         
         return drift_df.to_dict()
 
+    async def calculate_simple_metrics(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
+        """
+        Calculate simple metrics for a model. Metrics available are:
+            - rmse
+            - mse
+            - mae
+            - r2
+            - accuracy
+            - precision
+            - recall
+            - f1
+        Args:
+            input_data (dict[str, Any]): Input data containing:
+                - metadata (dict): Workflow execution metadata
+                - model_id (str): ID of the MLFlow model
+                - target_data (pd.DataFrame): Target data for calculating metrics, containing target and prediction columns
+                - metrics (list[str]): List of metrics to calculate
+        Returns:
+            dict[Hashable, Any]: Dictionary containing the calculated metrics
+        """
+
+        metadata = input_data['metadata']
+        model_id = input_data['model_id']
+        target_data = DataFrame(input_data['target_data'])
+        metrics = input_data['metrics']
+        interval_minutes = input_data['interval_minutes']
+
+        data_size = len(target_data)
+
+        output_data = []
+
+        diff = target_data['target'] - target_data['prediction']
+        diff_squared = diff ** 2
+
+        self.info(f'Calculating simple metrics for model {model_id}: {metrics}', metadata)
+
+        for metric in metrics:
+            if metric == 'rmse':
+                output_data.append({
+                    'metric': 'rmse',
+                    'value': np.sqrt(np.mean(diff_squared))
+                })
+            elif metric == 'mse':
+                output_data.append({
+                    'metric': 'mse',
+                    'value': np.mean(diff_squared)
+                })
+            elif metric == 'mae':
+                output_data.append({
+                    'metric': 'mae',
+                    'value': np.mean(np.abs(diff))
+                })
+            elif metric == 'r2':
+                y_true = target_data['target']
+                y_mean = np.mean(y_true)
+                
+                ss_res = np.sum(diff_squared)
+                ss_tot = np.sum((y_true - y_mean) ** 2)
+                
+                # Evita divisão por zero
+                if ss_tot == 0:
+                    r2_score = 0.0
+                else:
+                    r2_score = 1 - (ss_res / ss_tot)
+                
+                output_data.append({
+                    'metric': 'r2',
+                    'value': r2_score
+                })
+        
+        data = DataFrame(output_data)
+        data['model_id'] = model_id
+        data['timestamp'] = target_data['timestamp'].max()
+        data['data_size'] = data_size
+        data['interval_minutes'] = interval_minutes
+
+        self.debug(f'Simple metrics dataframe: Size {data.shape} \n{data.head(5).to_string()}', metadata)
+
+        return data.to_dict()
+
+

laborious/worker/worker.py

@@ -51,6 +51,8 @@ with workflow.unsafe.imports_passed_through():
     from laborious.workflows.minimal_retrain import MinimalRetrain
     from laborious.workflows.predictions_batch import PredictionsBatch
     from laborious.workflows.drift import Drift
+    from laborious.workflows.simple_metrics import SientiaMetrics
+    
     from laborious.workflows.sub_workflows.format_and_export_prediction import (
         FormatAndExportPrediction,
     )
@@ -175,6 +177,22 @@ async def main():
             workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
             activity_task_poller_behavior=PollerBehaviorAutoscaling(),
         ),
+        Worker(
+            temporal_client,
+            task_queue='simple_metrics-queue',
+            workflows=[SimpleMetrics],
+            activities=[
+                activities.load_custom_query,
+                activities.calculate_simple_metrics,
+                activities.export_data_to_postgres,
+            ],
+            max_concurrent_workflow_tasks=50,
+            max_concurrent_activities=50,
+            max_concurrent_local_activities=50,
+            max_cached_workflows=2,
+            workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
+            activity_task_poller_behavior=PollerBehaviorAutoscaling(),
+        ),
         Worker(
             temporal_client,
             task_queue='predictions_batch-queue',

laborious/workflows/drift.py

@@ -80,7 +80,8 @@ class Drift:
                 'model_name': input_data['model_name'],
                 'model_id': input_data['model_id'],
                 'target_name': target_name,
-                'drift_metrics': input_data['drift_metrics'],
+                'drift_metrics': input_data.get('drift_metrics',
+                    ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']),
                 'chunk_period': input_data.get('chunk_period', 'min'),
             },
             retry_policy=retry_policy,

laborious/workflows/simple_metrics.py

@@ -0,0 +1,95 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from datetime import timedelta
+    from typing import Any
+
+    from sientia_do.temporal.policies import retry_policy
+
+    from laborious.activities.activities import Activities
+    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
+
+
+@workflow.defn(name='simple_metrics')
+class SimpleMetrics:
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]):
+        """
+        Execute the simple metrics workflow.
+        """
+        metadata = {
+            'metadata': {
+                'schedule_name': input_data['schedule_name'],
+                'model_name': input_data['model_name'],
+                'model_id': input_data['model_id'],
+                'workflow_name': 'simple_metrics',
+            }
+        }
+
+        model_id = input_data['model_id']
+        interval_minutes = input_data['interval_minutes']
+
+        model_config = input_data['model_config']
+        target_name = model_config['target']
+
+        query = f"""
+            select p."timestamp", p.prediction, ld.value as "target"
+            from {input_data['schema']}.{input_data['predictions_table_name']} p
+            inner join {input_data['schema']}.{input_data['data_table_name']} ld
+                on p."timestamp" = ld."timestamp"
+            where
+                p.model_id = {model_id} and
+                p.prediction is not null and
+                ld.variable = '{target_name}' and
+                ld.value is not null and
+                p."timestamp" >= NOW() - INTERVAL '{interval_minutes} minutes'
+            order by
+                p."timestamp" desc;
+        """
+
+        target_data = await workflow.execute_local_activity_method(
+            Activities.load_custom_query,
+            {
+                **metadata,
+                'query': query,
+                'datetime_columns': ['timestamp'],
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=300),
+        )
+
+        if not target_data:
+            return
+
+        simple_metrics = await workflow.execute_local_activity_method(
+            Activities.calculate_simple_metrics,
+            {
+                **metadata,
+                'model_id': model_id,
+                'target_data': target_data,
+                'metrics': input_data.get('metrics', ['rmse', 'mse', 'mae', 'r2']),
+                'interval_minutes': interval_minutes,
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=300),
+        )
+        
+
+        if not simple_metrics:
+            return
+
+        await workflow.execute_activity_method(
+            Activities.export_data_to_postgres,
+            {
+                **metadata,
+                'data': simple_metrics,
+                'schema': input_data['schema'],
+                'table_name': input_data['target_table_name'],
+                'timestamp_conversion': {
+                    'column': 'timestamp',
+                    'format': DATETIME_FORMAT_WITH_TZ,
+                },
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=300),
+        )