SIENTIAPDE-1231

Update .gitignore, values.yaml, and model_repository.py for improved data handling and logging

- Added 'catboost_info/' to .gitignore to prevent tracking of additional temporary files.
- Updated GITHUB_BRANCH in values.yaml to reflect the current branch for model retraining.
- Enhanced model_repository.py to drop duplicate timestamps in treated data and streamline attribute logging during model retraining.

.gitignore

@@ -45,4 +45,5 @@ git_log
 
 .env
 
-tmp/
+tmp/
+catboost_info/

laborious/utils/repository/model_repository.py

@@ -709,6 +709,9 @@ class MLFlowRepository():
         treated_data = self.detect_and_parse_datetime_index(
             treated_data, metadata)
 
+        treated_data = treated_data.drop_duplicates(
+            subset=['timestamp'], keep='first')
+
         self.logger.custom_debug(
             f"Treated data index: {treated_data.index}", metadata)
 
@@ -820,24 +823,33 @@ class MLFlowRepository():
         data_model_atributes = vars(data_model)  # load class attributes
         experiment_description = f"Retrain model {model_name} with new data"
         current_run_name = self.get_next_run_name(experiment)
+
+        attributes = {}
+
+        for name_atribute, val_atribute in pred_model_atributes.items():
+            if name_atribute != "model":
+                attributes[name_atribute] = val_atribute
+        for name_atribute, val_atribute in data_model_atributes.items():
+            if name_atribute != "model":
+                attributes[name_atribute] = val_atribute
+
+        self.logger.custom_debug(
+            f"Attributes: {attributes}", metadata)
+
         with mlflow.start_run(
             run_name=current_run_name, description=experiment_description
         ) as _run:
             # update transfomation model
             # fixed parameters
-            for name_atribute, val_atribute in pred_model_atributes.items():
+            for name_atribute, val_atribute in attributes.items():
-                if name_atribute != "model":
+                mlflow.log_param(name_atribute, val_atribute)
-                    mlflow.log_param(name_atribute, val_atribute)
+
-            # update prediction model
-            for name_atribute, val_atribute in data_model_atributes.items():
-                if name_atribute != "model":
-                    mlflow.log_param(name_atribute, val_atribute)
             # dynamic parameters, including model itself
             mlflow.sklearn.log_model(data_model, "data_model")
 
-            makedirs("temp", exist_ok=True)
+            makedirs("tmp/retrain_data", exist_ok=True)
 
-            file_path = f"temp/raw_data_{model_name}.csv"
+            file_path = f"tmp/retrain_data/retrain_data_{model_name}.csv"
             data.to_csv(file_path, index=True)
 
             # log the data raw
@@ -1157,7 +1169,7 @@ class MLFlowRepository():
             self.logger.custom_info(
                 f"Model experiment created successfully: {experiment}", metadata)
 
-            self.logger.custom_info("Performing model retraining", metadata)
+            self.logger.custom_info("Saving model retrain", metadata)
             experiment = self.perform_model_retrain(
                 prediction_model, data_model, experiment, model_name, data, metadata)
             self.logger.custom_info(

values.yaml

@@ -151,7 +151,7 @@ env:
   - name: GITHUB_REPO_URL
     value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git"
   - name: GITHUB_BRANCH
-    value: main
+    value: SIENTIAPDE-1231-ajustar-o-retreino-do-courier-no-laborious
   - name: PYTHON_APP
     value: "laborious.worker.worker"