SIENTIAPDE-1712

Remove code validation script and refactor imports in activities and workflows

- Deleted the `validate.sh` script, which was responsible for running code quality checks.
- Cleaned up import statements in `activities.py`, `gates.py`, `mlflow.py`, and `storage.py` by removing unused imports and organizing them.
- Refactored initialization methods in `MinioManager` and `MLFlow` classes for improved readability.
- Updated various workflows to ensure compatibility with the new structure and removed unnecessary comments.
- Enhanced test cases to accommodate changes in the activities and workflows, ensuring proper mocking of dependencies.

laborious/utils/models/minio_dataframe_payload.py

@@ -12,16 +12,16 @@ Otherwise, it is inlined as a Temporal-friendly ``dict``.
 
 import pickle
 import re
-from dataclasses import dataclass, field
+from collections.abc import Hashable
+from dataclasses import dataclass
 from datetime import datetime
 from io import BytesIO
 from os import getenv
-from typing import Any, Hashable, Literal
+from typing import Any, Literal
 
 from pandas import DataFrame, read_parquet
-
-from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, DATETIME_FORMAT_WITH_TZ, now
 from sientia_do.repository.minio_repository import MinioRepository
+from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, DATETIME_FORMAT_WITH_TZ, now
 
 # Keys that are part of the serialized wire format (not arbitrary metadata).
 _SERIALIZED_FIELD_KEYS = frozenset({'data', 'bucket', 'object_key', 'object_prefix', 'uri'})
@@ -30,7 +30,9 @@ _OBJECT_TIMESTAMP_PATTERN = re.compile(
     r'-(?:initial|transform)-(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})\.parquet$'
 )
 
-OFFLOAD_THRESHOLD_BYTES = int(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_BYTES', '1.5')) * 1024 * 1024
+OFFLOAD_THRESHOLD_BYTES = int(
+    float(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_BYTES', '1.5')) * 1024 * 1024
+)
 
 # Relative prefix used for storing offloaded training datasets in MinIO.
 # It is also the root directory for retention cleanup listing.
@@ -79,7 +81,6 @@ class MinioDataFramePayload:
     object_prefix: str | None = None
     uri: str | None = None
 
-
     @staticmethod
     def estimate_size_bytes(df: DataFrame) -> int:
         """
@@ -117,21 +118,6 @@ class MinioDataFramePayload:
         except ValueError:
             return None
 
-    @staticmethod
-    def is_offloaded_dict(payload: dict[str, Any]) -> bool:
-        """
-        Return True if the dict represents a MinIO-backed payload without inline data.
-
-        Args:
-            payload: Flat dict possibly produced by to_dict() / from_dataframe_to_dict().
-
-        Return:
-            bool: True when object_key is set and inline data is absent.
-        """
-        if not payload.get('object_key'):
-            return False
-        return payload.get('data') is None
-
     @staticmethod
     def cleanup_prefix(self) -> str | None:
         """
@@ -141,6 +127,12 @@ class MinioDataFramePayload:
             return self.object_prefix
         return None
 
+    def has_data(self) -> bool:
+        """
+        Return True if the payload has some data internally or in MinIO.
+        """
+        return (self.data is not None and not self.data != {}) or self.object_key is not None
+
     @classmethod
     async def from_dataframe(
         cls,
@@ -172,7 +164,9 @@ class MinioDataFramePayload:
         """
 
         if not dataframe or dataframe.empty:
-            return cls(data=None, last_timestamp=now().strftime(DATETIME_FORMAT_WITH_TZ), status=status)
+            return cls(
+                data=None, last_timestamp=now().strftime(DATETIME_FORMAT_WITH_TZ), status=status
+            )
 
         last_timestamp = max(dataframe['timestamp'].values.tolist())
 
@@ -207,7 +201,9 @@ class MinioDataFramePayload:
             last_timestamp=last_timestamp,
         )
 
-    async def retrieve(self, minio_repo: MinioRepository, workflow_metadata: dict[str, Any] | None = None) -> DataFrame:
+    async def retrieve(
+        self, minio_repo: MinioRepository, workflow_metadata: dict[str, Any] | None = None
+    ) -> DataFrame:
         """
         Load parquet from MinIO when object_key is set and populate inline data.
 
@@ -221,10 +217,11 @@ class MinioDataFramePayload:
         if self.data is not None:
             return DataFrame(self.data)
 
-        if self.data is None and self.object_key is None:
+        if not self.has_data():
             return DataFrame()
 
         file_bytes = await minio_repo.download_file(
-            object_name=self.object_key, metadata=workflow_metadata)
+            object_name=self.object_key, metadata=workflow_metadata
+        )
         df = read_parquet(BytesIO(file_bytes))
         return df

laborious/utils/repository/minio_manager.py

@@ -1,14 +1,20 @@
+from sientia_do.notifications.handlers import NotificationHandler
 from sientia_do.observability.logger import Logger
 from sientia_do.observability.metrics_controller import MetricsController
-from sientia_do.notifications.handlers import NotificationHandler
-from sientia_do.repository.minio_repository import MinioRepository
 from sientia_do.observability.sientia_monitoring import SientiaMonitoring
-    
+from sientia_do.repository.minio_repository import MinioRepository
+
 
 class MinioManager(SientiaMonitoring):
     minio_repository: MinioRepository | None = None
 
-    def __init__(self, minio_repository: MinioRepository | None = None, logger: Logger | None = None, notification_handler: NotificationHandler | None = None, metrics_controller: MetricsController | None = None):
+    def __init__(
+        self,
+        minio_repository: MinioRepository | None = None,
+        logger: Logger | None = None,
+        notification_handler: NotificationHandler | None = None,
+        metrics_controller: MetricsController | None = None,
+    ):
         if self.minio_repository is None:
             self.minio_repository = minio_repository
         SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
@@ -23,4 +29,4 @@ class MinioManager(SientiaMonitoring):
             finally:
                 self.minio_repository = None
 
-        SientiaMonitoring.shutdown(self)
+        SientiaMonitoring.shutdown(self)