SIENTIAPDE-1231

Enhance Gates and MLFlowRepository with new functionalities and improvements

- Added a new method `clean_tmp_files` in the Gates class to remove temporary files associated with model retraining.
- Updated MLFlowRepository methods to improve experiment handling, including dynamic parameter logging and model retrieval.
- Refactored model loading methods to streamline the process and enhance error handling.
- Improved logging for model operations and added support for model parameter retrieval.
- Adjusted minimal_retrain workflow to extend timeouts for activities and ensure proper model configuration handling.

laborious/activities/gates.py

@@ -17,6 +17,8 @@ with workflow.unsafe.imports_passed_through():
     )
     from pandas import DataFrame
     from laborious import metrics
+    from os import path
+    from shutil import rmtree
 
 # Input filter function mappings
 input_filter_functions = {
@@ -614,3 +616,19 @@ class Gates(BaseActivity):
 
         self.info(
             f"Metrics written for model {metadata['model_name']}", metadata)
+
+    @activity.defn(name="clean_tmp_files")
+    async def clean_tmp_files(self, input_data: dict[str, Any]):
+        """
+        Clean temporary files in the tmp directory.
+        """
+        model_name = input_data['model_name']
+        metadata = input_data['metadata']
+        self.info(f"Cleaning tmp files for model {model_name}...", metadata)
+
+        if path.exists(f"tmp/retrain_data/{model_name}"):
+            rmtree(f"tmp/retrain_data/{model_name}")
+        if path.exists(f"tmp/artifacts/{model_name}"):
+            rmtree(f"tmp/artifacts/{model_name}")
+
+        self.info("Tmp files cleaned", metadata)

laborious/activities/mlflow.py

@@ -304,7 +304,6 @@ class MLFlow(BaseActivity):
         data['timestamp'] = to_datetime(
             data['timestamp'], format=DATETIME_FORMAT)
 
-        # data = data.dropna()
         data.columns.name = None
 
         retrain_output = self.model_monitoring_repository.retrain_model(