SIENTIAPDE-1712

Refactor MinioDataFramePayload usage across activities

- Updated instances of MinioDataFramePayload initialization in Gates, MLFlow, and Storage classes to use the new from_dict method for better data reconstruction from dictionaries.
- Enhanced the PredictionProcess workflow to utilize the updated payload handling.
- Added passthrough fixtures in tests to accommodate the new from_dict method for consistent testing behavior.

laborious/activities/gates.py

@@ -157,7 +157,7 @@ class Gates(MinioManager):
         self.info('Performing input gate...', metadata)
 
         filters = input_data['filters']
-        payload: MinioDataFramePayload = input_data['data']
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
         data = await payload.retrieve(self.minio_repository, metadata)
         path_priority = input_data['path_priority']
 
@@ -236,7 +236,7 @@ class Gates(MinioManager):
 
         filters = input_data['filters']
 
-        payload: MinioDataFramePayload = input_data['data']
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
         data = await payload.retrieve(self.minio_repository, metadata)
 
         gate_type = input_data['type']
@@ -327,7 +327,7 @@ class Gates(MinioManager):
 
         filters = input_data['filters']
 
-        payload: MinioDataFramePayload = input_data['data']
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
         data = await payload.retrieve(self.minio_repository, metadata)
 
         gate_type = input_data['type']
@@ -465,7 +465,7 @@ class Gates(MinioManager):
 
         self.info('Formatting transformed data...', metadata)
 
-        payload: MinioDataFramePayload = input_data['data']
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
         data = await payload.retrieve(self.minio_repository, metadata)
 
         data['timestamp'] = data.index

laborious/activities/mlflow.py

@@ -128,7 +128,7 @@ class MLFlow(MinioManager):
         metadata = input_data['metadata']
         self.info('Transforming data...', metadata)
 
-        payload: MinioDataFramePayload = input_data['data']
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
         data = await payload.retrieve(self.minio_repository, metadata)
 
         model_name = input_data['model_name']
@@ -222,7 +222,7 @@ class MLFlow(MinioManager):
         metadata = input_data['metadata']
         self.info('Predicting data...', metadata)
 
-        payload: MinioDataFramePayload = input_data['data']
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
         data = await payload.retrieve(self.minio_repository, metadata)
 
         model_name = input_data['model_name']
@@ -311,7 +311,7 @@ class MLFlow(MinioManager):
 
         try:
             # Payload-based retrain input (inline dict or MinIO offloaded).
-            payload: MinioDataFramePayload = input_data['data']
+            payload = MinioDataFramePayload.from_dict(input_data['data'])
             data = await payload.retrieve(self.minio_repository, metadata)
 
         except Exception as e:

laborious/activities/storage.py

@@ -8,7 +8,6 @@ with workflow.unsafe.imports_passed_through():
     # Extend the Temporal Postgres activities for convenient query -> MinIO export
     import traceback
     from datetime import timedelta
-    from io import BytesIO
     from typing import Any
 
     import pandas as pd
@@ -18,7 +17,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.observability.metrics_controller import MetricsController
     from sientia_do.repository.minio_repository import MinioRepository
     from sientia_do.temporal.activities.postgres import Postgres
-    from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, now
+    from sientia_do.temporal.constants import now
 
     from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
 
@@ -116,7 +115,7 @@ class Storage(Postgres, MinioManager):
         Export a payload to PostgreSQL.
         """
         metadata = input_data.get('metadata')
-        payload: MinioDataFramePayload = input_data['data']
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
         data = await payload.retrieve(self.minio_repository, metadata)
 
         return await self.export_data_to_postgres(
@@ -142,7 +141,8 @@ class Storage(Postgres, MinioManager):
             raise ValueError('Minio repository not initialized')
 
         metadata = input_data.get('metadata', {})
-        prefix = input_data['prefix']
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
+        prefix = payload.cleanup_prefix()
         base = now()
         cutoff = (base.replace(tzinfo=None) if base.tzinfo else base) - timedelta(
             hours=self.retention_hours
@@ -206,7 +206,6 @@ class Storage(Postgres, MinioManager):
 
         return report
 
-
     def close(self) -> None:
         """Close Storage resources (MinIO client and Postgres engine)."""
         Postgres.close(self)

laborious/utils/models/minio_dataframe_payload.py

@@ -81,6 +81,38 @@ class MinioDataFramePayload:
     object_prefix: str | None = None
     uri: str | None = None
 
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any] | 'MinioDataFramePayload') -> 'MinioDataFramePayload':
+        """
+        Reconstruct a MinioDataFramePayload from a plain dict produced by Temporal serialization.
+
+        Temporal converts dataclass return values into plain dicts when crossing
+        workflow/activity boundaries. This method rebuilds the typed instance so
+        that methods like ``retrieve``, ``cleanup_prefix`` and ``has_data`` are
+        available on the receiving side.
+
+        If the argument is already a MinioDataFramePayload, it is returned as-is.
+
+        Args:
+            raw: Dict with keys matching the dataclass fields
+                (last_timestamp, status, data, bucket, object_key, object_prefix, uri),
+                or an existing MinioDataFramePayload instance.
+
+        Return:
+            MinioDataFramePayload: Reconstructed (or original) instance.
+        """
+        if isinstance(raw, MinioDataFramePayload):
+            return raw
+        return cls(
+            last_timestamp=raw['last_timestamp'],
+            status=raw.get('status'),
+            data=raw.get('data'),
+            bucket=raw.get('bucket'),
+            object_key=raw.get('object_key'),
+            object_prefix=raw.get('object_prefix'),
+            uri=raw.get('uri'),
+        )
+
     @staticmethod
     def estimate_size_bytes(df: DataFrame) -> int:
         """
@@ -118,7 +150,6 @@ class MinioDataFramePayload:
         except ValueError:
             return None
 
-    @staticmethod
     def cleanup_prefix(self) -> str | None:
         """
         Return True if cleanup is enabled for this payload.

laborious/workflows/sub_workflows/prediction_process.py

@@ -38,8 +38,6 @@ class PredictionProcess:
         8. Export Delegation: Delegates to FormatAndExportPrediction workflow
     """
 
-    cleanup_prefixes: set[str] = set()
-
     @workflow.run
     async def run(self, input_data: dict[str, Any]):
         """
@@ -90,8 +88,6 @@ class PredictionProcess:
         model_config = input_data.get('model_config', {})
         save_transform = input_data.get('save_transform', True)
 
-        prefix = data.cleanup_prefix()
-
         try:
             await self._run_prediction_pipeline(
                 input_data,
@@ -103,13 +99,12 @@ class PredictionProcess:
                 save_transform,
             )
         finally:
-            if self.cleanup_prefixes:
-                await workflow.execute_activity_method(
-                    Activities.cleanup_minio_objects_expired,
-                    {**metadata, 'prefix': prefix},
-                    retry_policy=retry_policy,
-                    start_to_close_timeout=timedelta(minutes=5),
-                )
+            await workflow.execute_activity_method(
+                Activities.cleanup_minio_objects_expired,
+                {**metadata, 'data': data},
+                retry_policy=retry_policy,
+                start_to_close_timeout=timedelta(minutes=5),
+            )
 
     async def _run_prediction_pipeline(
         self,