SIENTIAPDE-1171

Update dependencies and enhance ML model retraining functionality

- Updated sientia-dataops-library version in requirements.txt from 1.3.3 to 1.3.4.
- Incremented image tag in values.yaml from 0.2.4 to 0.2.5 and added a new environment variable MONGODB_TTL_INDEX_HOURS.
- Introduced new methods in MLFlowRepository for model retraining and production model updates, including error handling and logging.
- Added retrain_model and update_production_model activities in mlflow.py to support model management workflows.
- Modified MongoDB connection settings in connectors_config.py for improved security and configuration flexibility.

laborious/activities/mlflow.py

@@ -1,5 +1,3 @@
-import numpy as np
-from pandas import DataFrame
 from temporalio import activity, workflow
 
 
@@ -9,6 +7,9 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.temporal.utils.logger import Logger
     from laborious.utils.repository.model_repository import MLFlowRepository
     from typing import Any
+    import numpy as np
+    from pandas import DataFrame
+    import traceback
 
 
 class MLFlow(BaseActivity):
@@ -96,3 +97,107 @@ class MLFlow(BaseActivity):
         self.debug(response_data, metadata)
 
         return response_data
+
+    @activity.defn(name="retrain_model")
+    async def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
+        """
+        Retrain the model.
+        Args:
+            - input_data (dict): The input data. Contains:
+                - model_name (str): The name of the model.
+                - data (dict[str, Any]): The data to retrain the model.
+        """
+        metadata = input_data['metadata']
+        data = DataFrame(input_data['data'])
+        model_name = input_data['model_name']
+
+        self.info(f'Retraining model {model_name}...', metadata)
+
+        timestamp = data['timestamp'].max()
+        self.debug(f'Timestamp: {timestamp}', metadata)
+
+        data.drop(columns=['model_id'], inplace=True, errors='ignore')
+        data.drop(columns=['created_at'], inplace=True, errors='ignore')
+
+        data = data.pivot(index='timestamp', columns='variable',
+                          values='value')
+        data.sort_index(inplace=True)
+        data.reset_index(inplace=True)
+
+        data = data.dropna()
+        data.columns.name = None
+
+        try:
+            retrain_output, experiment = self.model_monitoring_repository.retrain_model(
+                data=data,
+                model_name=model_name
+            )
+
+            return {
+                'status': retrain_output,
+                'timestamp': timestamp,
+                'experiment': experiment
+            }
+        except Exception as e:
+            trace = traceback.format_exc()
+            self.send_notification(
+                metadata=metadata,
+                notification_id='RETRAIN_MODEL_ERROR',
+                message=f'Error retraining model {model_name}: {e}',
+                block='retrain_model',
+                attachment_content=trace
+            )
+            self.error(trace, metadata=metadata)
+            raise e
+
+    @activity.defn(name="update_production_model")
+    async def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]:
+        """
+        Update the production model.
+        Args:
+            - input_data (dict): The input data. Contains:
+                - model_name (str): The name of the model.
+                - experiment (str): The name of the experiment.
+                - model_id (str): The id of the model.
+                - timestamp (str): The timestamp of the model.
+                - status (str): The status of the model.
+        Returns:
+            dict[Any, Any]: The report of the model.
+        """
+        metadata = input_data['metadata']
+        model_name = input_data['model_name']
+        model_id = input_data['model_id']
+        experiment = input_data['experiment']
+        timestamp = input_data['timestamp']
+        status = input_data['status']
+
+        self.info(
+            f'Updating production model {model_name} from experiment {experiment}...', metadata)
+
+        try:
+            response = self.model_monitoring_repository.update_production_model(
+                experiment=experiment,
+                model_name=model_name
+            )
+
+            report = DataFrame([response])
+            report['model_id'] = model_id
+            report['model_name'] = model_name
+            report['timestamp'] = timestamp
+            report['status'] = status
+
+            self.info(
+                f'Production model {model_name} updated successfully', metadata)
+            return report.to_dict()
+
+        except Exception as e:
+            trace = traceback.format_exc()
+            self.send_notification(
+                metadata=metadata,
+                notification_id='UPDATE_PRODUCTION_MODEL_ERROR',
+                message=f'Error updating production model {model_name}: {e}',
+                block='update_production_model',
+                attachment_content=trace
+            )
+            self.error(trace, metadata=metadata)
+            raise e