SIENTIAPDE-1712

Implement MinIO Offload and Retention Features

- Added configuration options for MinIO retention hours and offload threshold in README.
- Introduced MinIO payload offloading for large DataFrame-derived payloads, storing them as parquet files.
- Updated activities to utilize MinIO for data loading and cleanup, including new methods for offloading and retention management.
- Refactored existing activities to integrate MinIO functionality, ensuring compatibility with previous workflows.
- Removed the legacy MinioRepository class, consolidating MinIO operations under a new manager structure.
- Updated requirements to use the latest version of the sientia-dataops-library.

laborious/activities/activities.py

@@ -6,6 +6,7 @@ with workflow.unsafe.imports_passed_through():
 
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.observability.logger import Logger
+    from sientia_do.repository.minio_repository import MinioRepository
 
     from laborious.activities.api import API
     from laborious.activities.gates import Gates
@@ -13,6 +14,7 @@ with workflow.unsafe.imports_passed_through():
     from laborious.activities.model_metrics import ModelMetrics
     from laborious.activities.opc import OPC
     from laborious.activities.storage import Storage
+    
 
 
 class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
@@ -73,6 +75,16 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
         """
         metrics_controller = MetricsController(logger=logger)
 
+        minio_repository = MinioRepository(
+            endpoint_url=minio_config['endpoint_url'],
+            access_key=minio_config['access_key'],
+            secret_key=minio_config['secret_key'],
+            bucket=minio_config['default_bucket'],
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
         # Initialize parent classes
         Storage.__init__(
             self,
@@ -83,7 +95,8 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
             dbname=postgres_config['dbname'],
             min_connections=postgres_config['min_connections'],
             max_connections=postgres_config['max_connections'],
-            minio_config=minio_config,
+            retention_hours=minio_config['retention_hours'],
+            minio_repository=minio_repository,
             logger=logger,
             notification_handler=notification_handler,
             metrics_controller=metrics_controller,
@@ -95,7 +108,7 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
             mlflow_port=mlflow_config['port'],
             mlflow_username=mlflow_config['username'],
             mlflow_password=mlflow_config['password'],
-            minio_config=minio_config,
+            minio_repository=minio_repository,
             logger=logger,
             notification_handler=notification_handler,
             metrics_controller=metrics_controller,
@@ -103,6 +116,7 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
 
         Gates.__init__(
             self,
+            minio_repository=minio_repository,
             logger=logger,
             notification_handler=notification_handler,
             metrics_controller=metrics_controller,

laborious/activities/gates.py

@@ -1,5 +1,8 @@
+from sientia_do.repository.minio_repository import MinioRepository
 from temporalio import activity, workflow
 
+from laborious.utils.repository.minio_manager import MinioManager
+
 with workflow.unsafe.imports_passed_through():
     import traceback
     from collections.abc import Callable, Mapping
@@ -15,6 +18,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.utils.formatters import create_sample_dict
 
     from laborious import metrics
+    from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
     from laborious.utils.filters.conditional_filters import (
         filter_empty_data,
         filter_specific_variables_null_values,
@@ -63,7 +67,7 @@ mlflow_content_path_confidence: Mapping[str, int] = {
 }
 
 
-class Gates(SientiaMonitoring):
+class Gates(MinioManager):
     """
     Data quality gates and filtering activities for the Laborious system.
 
@@ -83,11 +87,14 @@ class Gates(SientiaMonitoring):
         mlflow_content_filter_functions (dict): Mapping of MLFlow content filter names to functions
     """
 
+    minio_repository: MinioRepository | None = None
+
     def __init__(
         self,
-        logger: Logger,
-        notification_handler: NotificationHandler,
-        metrics_controller: MetricsController,
+        minio_repository: MinioRepository | None = None,
+        logger: Logger | None = None,
+        notification_handler: NotificationHandler | None = None,
+        metrics_controller: MetricsController | None = None,
     ):
         """
         Initialize data quality gates with logging and notification capabilities.
@@ -99,13 +106,14 @@ class Gates(SientiaMonitoring):
         Raises:
             Exception: If BaseActivity initialization fails
         """
-        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
+        MinioManager.__init__(self, minio_repository, logger, notification_handler, metrics_controller)
 
     def close(self) -> None:
         """
         Close the gates activity and clean up resources.
         """
-        SientiaMonitoring.shutdown(self)
+
+        MinioManager.close(self)
 
     def __del__(self):
         self.close()
@@ -149,7 +157,8 @@ class Gates(SientiaMonitoring):
         self.info('Performing input gate...', metadata)
 
         filters = input_data['filters']
-        data = DataFrame(input_data['data'])
+        payload: MinioDataFramePayload = input_data['data']
+        data = await payload.retrieve(self.minio_repository, metadata)
         path_priority = input_data['path_priority']
 
         filter_output = []
@@ -183,6 +192,9 @@ class Gates(SientiaMonitoring):
                 return path_flag, input_path_confidence[path_flag], 'Input data with bad quality'
 
         self.info('Nothing was filtered by the input gate', metadata)
+
+        del data
+
         return None, 0, ''
 
     @activity.defn(name='mlflow_response_gate')
@@ -223,7 +235,10 @@ class Gates(SientiaMonitoring):
         self.info('Performing mlflow response gate...', metadata)
 
         filters = input_data['filters']
-        data = input_data['data']
+
+        payload: MinioDataFramePayload = input_data['data']
+        data = await payload.retrieve(self.minio_repository, metadata)
+        
         gate_type = input_data['type']
         path_priority = input_data['path_priority']
 
@@ -233,13 +248,16 @@ class Gates(SientiaMonitoring):
         self.debug(f'Filters: {filters}', metadata)
 
         comments = []
+
+        status = payload.status or {}
+
         for fil, config in filters.items():
             if fil not in mlflow_response_filter_functions:
                 continue
             try:
-                if mlflow_response_filter_functions[fil](data, config):
+                if mlflow_response_filter_functions[fil](status, config):
                     filter_output.append(config['policy'])
-                    comments.append(data['content']['message'])
+                    comments.append(status['message'])
                     await self.send_notification_async(
                         metadata=metadata,
                         notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
@@ -265,6 +283,9 @@ class Gates(SientiaMonitoring):
                 return path_flag, mlflow_response_path_confidence[path_flag], ', '.join(comments)
 
         self.info('Nothing was filtered by the mlflow response gate', metadata)
+
+        del data
+
         return None, 0, ''
 
     @activity.defn(name='mlflow_content_gate')
@@ -305,7 +326,10 @@ class Gates(SientiaMonitoring):
         self.info('Performing mlflow content gate...', metadata)
 
         filters = input_data['filters']
-        data = DataFrame(input_data['data'])
+        
+        payload: MinioDataFramePayload = input_data['data']
+        data = await payload.retrieve(self.minio_repository, metadata)
+
         gate_type = input_data['type']
         path_priority = input_data['path_priority']
 
@@ -349,6 +373,9 @@ class Gates(SientiaMonitoring):
                 )
 
         self.info('Nothing was filtered by the mlflow content gate', metadata)
+
+        del data
+        
         return None, 0, ''
 
     def get_prediction_store_policy(
@@ -403,7 +430,7 @@ class Gates(SientiaMonitoring):
         return policy_type, int(policy_value)
 
     @activity.defn(name='format_transformed_data')
-    async def format_transformed_data(self, input_data: dict[str, Any]) -> dict:
+    async def format_transformed_data(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
         """
         Format transformed data for storage and export operations.
 
@@ -438,7 +465,8 @@ class Gates(SientiaMonitoring):
 
         self.info('Formatting transformed data...', metadata)
 
-        data = DataFrame(input_data['data'])
+        payload: MinioDataFramePayload = input_data['data']
+        data = await payload.retrieve(self.minio_repository, metadata)
 
         data['timestamp'] = data.index
         data = data.reset_index(drop=True)
@@ -446,7 +474,13 @@ class Gates(SientiaMonitoring):
         data = data.melt(id_vars='timestamp', var_name='variable', value_name='value')
         data['model_id'] = model_id
 
-        return data.to_dict()
+        return await MinioDataFramePayload.from_dataframe(
+            dataframe=data,
+            minio_repo=self.minio_repository,
+            model_name=input_data['model_name'],
+            operation='transform',
+            workflow_metadata=metadata
+        )
 
     @activity.defn(name='format_prediction')
     async def format_prediction(self, input_data: dict[str, Any]) -> dict:
@@ -477,7 +511,8 @@ class Gates(SientiaMonitoring):
         prediction_store_policy = input_data['prediction_store_policy']
         self.info('Formatting prediction...', metadata)
 
-        data = DataFrame(input_data['data'])
+        payload: MinioDataFramePayload = input_data['data']
+        data = await payload.retrieve(self.minio_repository, metadata)
 
         # Create timestamp column from index and reset index
         data['timestamp'] = data.index
@@ -566,6 +601,7 @@ class Gates(SientiaMonitoring):
         self.info(f'Default prediction formatted: {data.size} rows', metadata)
         return data.to_dict()
 
+
     @activity.defn(name='format_retrain_report')
     async def format_retrain_report(self, input_data: dict[str, Any]) -> dict:
         """
@@ -636,45 +672,6 @@ class Gates(SientiaMonitoring):
 
         return report.to_dict()
 
-    @activity.defn(name='get_last_timestamp')
-    async def get_last_timestamp(self, input_data: dict[str, Any]) -> str:
-        """
-        Extract the most recent timestamp from prediction data.
-
-        This method analyzes prediction data to find the latest timestamp,
-        enabling incremental processing and data continuity tracking.
-        It handles empty datasets gracefully by returning the current time
-        as a fallback timestamp.
-
-        The method is essential for:
-        1. Incremental data processing workflows
-        2. Data continuity validation
-        3. Timestamp-based data loading optimization
-        4. Workflow execution tracking
-
-        Args:
-            input_data (dict): Input data containing:
-                - data (dict[str, Any]): Prediction data to analyze
-
-        Returns:
-            str: Formatted timestamp string in UTC with timezone
-        """
-        metadata = input_data['metadata']
-
-        self.info('Getting last timestamp...', metadata)
-
-        data = DataFrame(input_data['data'])
-
-        self.debug(f'Input data: {data.head(5).to_string()}', metadata)
-
-        if data.empty:
-            return now().strftime(DATETIME_FORMAT_WITH_TZ)
-
-        max_timestamp = max(data['timestamp'].values.tolist())
-
-        self.info(f'Last timestamp: {max_timestamp}', metadata)
-
-        return max_timestamp
 
     @activity.defn(name='write_metrics')
     async def write_metrics(self, input_data: dict[str, Any]):

laborious/activities/mlflow.py

@@ -1,11 +1,15 @@
+from re import M
 from temporalio import activity, workflow
 
+from laborious.utils.repository.minio_manager import MinioManager
+
 with workflow.unsafe.imports_passed_through():
     import traceback
     from typing import Any
 
     import numpy as np
-    from pandas import DataFrame, to_datetime
+    from io import BytesIO
+    from pandas import DataFrame, read_parquet, to_datetime
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.notifications.models import NotificationLevel
     from sientia_do.observability.logger import Logger
@@ -19,11 +23,12 @@ with workflow.unsafe.imports_passed_through():
     )
     from sientia_do.utils.formatters import create_sample_dict
 
-    from laborious.utils.repository.minio_repository import MinioRepository
+    from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
+    from sientia_do.repository.minio_repository import MinioRepository
     from laborious.utils.repository.model_repository import MLFlowRepository
 
 
-class MLFlow(SientiaMonitoring):
+class MLFlow(MinioManager):
     """
     MLFlow integration activities for model inference operations.
 
@@ -47,11 +52,11 @@ class MLFlow(SientiaMonitoring):
         mlflow_host: str,
         mlflow_port: int,
         mlflow_username: str,
-        minio_config: dict[str, Any],
         mlflow_password: str,
-        logger: Logger,
-        notification_handler: NotificationHandler,
-        metrics_controller: MetricsController,
+        minio_repository: MinioRepository | None = None,
+        logger: Logger | None = None,
+        notification_handler: NotificationHandler | None = None,
+        metrics_controller: MetricsController | None = None,
     ):
         """
         Initialize MLFlow activities with server configuration.
@@ -67,7 +72,7 @@ class MLFlow(SientiaMonitoring):
         Raises:
             Exception: If MLFlowRepository initialization fails
         """
-        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
+        MinioManager.__init__(self, minio_repository, logger, notification_handler, metrics_controller)
         self.mlflow_host = mlflow_host
         self.mlflow_port = mlflow_port
         self.mlflow_username = mlflow_username
@@ -82,32 +87,17 @@ class MLFlow(SientiaMonitoring):
             metrics_controller,
         )
 
-        if not hasattr(self, 'minio_repository'):
-            self.minio_repository: MinioRepository | None = None
-
-        if self.minio_repository is None:
-            self.minio_repository = MinioRepository(
-                logger=logger,
-                notification_handler=notification_handler,
-                minio_endpoint_url=minio_config['endpoint_url'],
-                minio_access_key=minio_config['access_key'],
-                minio_secret_key=minio_config['secret_key'],
-                minio_region_name=minio_config['region_name'],
-                minio_default_bucket=minio_config['default_bucket'],
-                metrics_controller=metrics_controller,
-            )
-
     def close(self) -> None:
         """
         Close the MLFlow activity and clean up resources.
         """
-        SientiaMonitoring.shutdown(self)
+        MinioManager.close(self)
 
     def __del__(self):
         self.close()
 
     @activity.defn(name='request_transform')
-    async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]:
+    async def request_transform(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
         """
         Transform input data using MLFlow models.
 
@@ -138,7 +128,10 @@ class MLFlow(SientiaMonitoring):
         """
         metadata = input_data['metadata']
         self.info('Transforming data...', metadata)
-        data = DataFrame(input_data['data'])
+        
+        payload: MinioDataFramePayload = input_data['data']
+        data = await payload.retrieve(self.minio_repository, metadata)
+
         model_name = input_data['model_name']
         model_config = input_data.get('model_config', {})
 
@@ -176,10 +169,29 @@ class MLFlow(SientiaMonitoring):
 
         self.info('Data transformed successfully', metadata)
 
-        return response_data
+        if not response_data.get('success', False):
+            return await MinioDataFramePayload.from_dataframe(
+                dataframe=None,
+                minio_repo=self.minio_repository,
+                model_name=model_name,
+                operation='transform',
+                status=response_data,
+                workflow_metadata=metadata,
+            )
+
+        return await MinioDataFramePayload.from_dataframe(
+            dataframe=response_data['content'],
+            minio_repo=self.minio_repository,
+            model_name=model_name,
+            operation='transform',
+            workflow_metadata=metadata,
+            status={
+                'success': True,
+            },
+        )
 
     @activity.defn(name='request_predict')
-    async def request_predict(self, input_data: dict[str, Any]) -> dict[str, Any]:
+    async def request_predict(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
         """
         Execute predictions using MLFlow models.
 
@@ -210,7 +222,10 @@ class MLFlow(SientiaMonitoring):
         """
         metadata = input_data['metadata']
         self.info('Predicting data...', metadata)
-        data = DataFrame(input_data['data'])
+
+        payload: MinioDataFramePayload = input_data['data']
+        data = await payload.retrieve(self.minio_repository, metadata)
+
         model_name = input_data['model_name']
         model_config = input_data.get('model_config', {})
 
@@ -236,7 +251,28 @@ class MLFlow(SientiaMonitoring):
 
         self.info('Data predicted successfully', metadata)
 
-        return response_data
+
+        if not response_data.get('success', False):
+            return await MinioDataFramePayload.from_dataframe(
+                dataframe=None,
+                minio_repo=self.minio_repository,
+                model_name=model_name,
+                operation='predict',
+                status=response_data,
+                workflow_metadata=metadata,
+            )
+
+        return await MinioDataFramePayload.from_dataframe(
+            dataframe=response_data['content'],
+            minio_repo=self.minio_repository,
+            model_name=model_name,
+            operation='predict',
+            workflow_metadata=metadata,
+            status={
+                'success': True,
+            }
+        )
+
 
     @activity.defn(name='retrain_model')
     async def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
@@ -275,14 +311,24 @@ class MLFlow(SientiaMonitoring):
             raise ValueError('Minio repository not initialized')
 
         metadata = input_data['metadata']
-        object_key = input_data['object_key']
-
-        self.info(f'Loading retrain data from Key: {object_key}', metadata)
 
         try:
-            data = await self.minio_repository.get_parquet_as_dataframe(
-                object_key=object_key, metadata=metadata
-            )
+            if 'data' in input_data:
+                # New path: payload-based retrain input (inline or MinIO offloaded).
+                data = await MinioDataFramePayload.dataframe_from_wire(
+                    input_data['data'],
+                    self.minio_repository,
+                    metadata,
+                )
+            else:
+                # Backward compatibility: legacy query_to_minio contract.
+                object_key = input_data['object_key']
+                self.info(f'Loading retrain data from Key: {object_key}', metadata)
+                file_bytes = await self.minio_repository.download_file(
+                    object_name=object_key,
+                    metadata=metadata,
+                )
+                data = read_parquet(BytesIO(file_bytes))
         except Exception as e:
             trace = traceback.format_exc()
             await self.send_notification_async(

laborious/activities/storage.py

@@ -1,8 +1,15 @@
+import json
 from temporalio import activity, workflow
 
+from laborious.utils.repository.minio_manager import MinioManager
+
 with workflow.unsafe.imports_passed_through():
     # Extend the Temporal Postgres activities for convenient query -> MinIO export
+    import pickle
     import traceback
+    from datetime import timedelta
+    from io import BytesIO
+    from os import getenv
     from typing import Any
 
     import pandas as pd
@@ -13,15 +20,20 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.temporal.activities.postgres import Postgres
     from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, now
 
-    from laborious.utils.repository.minio_repository import MinioRepository
+    from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
+    from sientia_do.repository.minio_repository import MinioRepository
+
+_LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'})
 
 
-class Storage(Postgres):
+class Storage(Postgres, MinioManager):
     """
     Extensions for Postgres activities with a helper to export query results
     directly to MinIO as Parquet and return the object name.
     """
 
+    minio_repository: MinioRepository | None = None
+
     def __init__(
         self,
         host: str,
@@ -31,12 +43,15 @@ class Storage(Postgres):
         dbname: str,
         min_connections: int,
         max_connections: int,
-        minio_config: dict[str, Any],
-        logger: Logger,
-        notification_handler: NotificationHandler,
-        metrics_controller: MetricsController,
+        retention_hours: int = 24,
+        minio_repository: MinioRepository | None = None,
+        logger: Logger | None = None,
+        notification_handler: NotificationHandler | None = None,
+        metrics_controller: MetricsController | None = None,
     ):
-        super().__init__(
+        self.retention_hours = retention_hours
+        Postgres.__init__(
+            self,
             host=host,
             port=port,
             user=user,
@@ -49,20 +64,144 @@ class Storage(Postgres):
             metrics_controller=metrics_controller,
         )
 
-        if not hasattr(self, 'minio_repository'):
-            self.minio_repository: MinioRepository | None = None
+        MinioManager.__init__(self, minio_repository, logger, notification_handler, metrics_controller)
 
+    @activity.defn(name='load_query_with_minio_offload')
+    async def load_query_with_minio_offload(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
+        """
+        Run the custom SQL load, then return a MinIO-aware dataframe wire dict.
+
+        Args (input_data):
+            metadata (dict): Workflow metadata (same as load_custom_query).
+            query (str): SQL query.
+            datetime_columns (list[str], optional): Datetime column names.
+            model_name (str): Model name for object key basename.
+            key_prefix (str, optional): Directory prefix inside the bucket.
+            size_threshold_bytes (int, optional): Override env offload threshold.
+
+        Returns:
+            dict[str, Any]: Flat ``MinioDataFramePayload`` dict or ``success: False`` on failure.
+        """
         if self.minio_repository is None:
-            self.minio_repository = MinioRepository(
-                logger=logger,
-                notification_handler=notification_handler,
-                minio_endpoint_url=minio_config['endpoint_url'],
-                minio_access_key=minio_config['access_key'],
-                minio_secret_key=minio_config['secret_key'],
-                minio_region_name=minio_config['region_name'],
-                minio_default_bucket=minio_config['default_bucket'],
-                metrics_controller=metrics_controller,
+            raise ValueError('Minio repository not initialized')
+
+        metadata: dict = input_data.get('metadata', {})
+        model_name = input_data['model_name']
+
+        rows = await self.load_custom_query(
+            input_data,
+        )
+        if not rows:
+            self.error('load_query_with_minio_offload failed: No data returned from query', metadata)
+            dataframe = None
+        else:
+            dataframe = pd.DataFrame(rows)
+
+        return await MinioDataFramePayload.from_dataframe(
+            dataframe,
+            minio_repo=self.minio_repository,
+            workflow_metadata=metadata,
+            model_name=model_name,
+            operation='initial',
+        )
+
+    @activity.defn(name='export_payload_to_postgres')
+    async def export_payload_to_postgres(self, input_data: dict[str, Any]) -> dict:
+        """
+        Export a payload to PostgreSQL.
+        """
+        metadata = input_data.get('metadata')
+        payload: MinioDataFramePayload = input_data['data']
+        data = await payload.retrieve(self.minio_repository, metadata)
+
+        return await self.export_data_to_postgres(
+            {
+                **input_data,
+                'data': data,
+            }
+        )
+
+    @activity.defn(name='cleanup_minio_objects_expired')
+    async def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]:
+        """
+        Delete objects under the given prefixes that are older than the retention window.
+
+        Args (input_data):
+            metadata (dict): Workflow metadata for logging and metrics.
+            prefixes (list[str]): Key prefixes to scan (one level or subtree per prefix).
+
+        Returns:
+            dict[str, Any]: ``success``, ``deleted_count``, and optional ``message``.
+        """
+        if self.minio_repository is None:
+            raise ValueError('Minio repository not initialized')
+
+        metadata = input_data.get('metadata', {})
+        prefix = input_data['prefix']
+        base = now()
+        cutoff = (base.replace(tzinfo=None) if base.tzinfo else base) - timedelta(
+            hours=self.retention_hours
+        )
+
+        report: dict[str, Any] = {
+            'failed': {},
+            'deleted': {},
+            'failed_count': 0,
+            'deleted_count': 0,
+        }
+        try:
+            keys = await self.minio_repository.list_objects(
+                prefix=prefix,
+                recursive=True,
+                metadata=metadata,
             )
+            for key in keys:
+                try:
+                    ts = MinioDataFramePayload.parse_object_timestamp(key)
+                    if ts is None:
+                        continue
+                    if ts >= cutoff:
+                        continue
+                    await self.minio_repository.delete_file(
+                        object_name=key,
+                        metadata=metadata,
+                    )
+                except Exception as e:
+                    report['failed'][key] = {
+                        'success': False,
+                        'message': str(e),
+                    }
+                    report['failed_count'] += 1
+                    continue
+                report['deleted'][key] = {
+                    'success': True,
+                    'message': 'Deleted',
+                }
+                report['deleted_count'] += 1
+        except Exception as e:
+            trace = traceback.format_exc()
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='ERROR_CLEANUP_MINIO_OBJECTS_EXPIRED',
+                message=f'Error cleaning up MinIO objects: {e}',
+                block='cleanup_minio_objects_expired',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+            self.error(trace, metadata)
+        else:
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='CLEANUP_MINIO_OBJECTS_EXPIRED',
+                message='MinIO objects cleaned up successfully',
+                block='cleanup_minio_objects_expired',
+                level=NotificationLevel.INFO,
+                attachment_content=json.dumps(report),
+            )
+
+        return report
+            
+
 
     @activity.defn(name='query_to_minio')
     async def query_to_minio(self, input_data: dict[str, Any]) -> dict[str, Any]:
@@ -83,11 +222,18 @@ class Storage(Postgres):
             raise ValueError('Minio repository not initialized')
 
         metadata = input_data.get('metadata', {})
+        model_name = input_data.get('model_name') or metadata.get('model_name') or 'unknown'
         object_prefix = input_data.get('object_prefix', 'datasets/retrain')
 
         timestamp = now().strftime(DATETIME_FORMAT_FILENAME)
-        object_name = f'{object_prefix}_{timestamp}.parquet'
-        uri = f's3://{self.minio_repository.minio_bucket}/{object_name}'
+        # Keep a stable model-level layout for minimal_retrain:
+        #   training_datasets/<model_name>/<filename>
+        # Sanitize object_prefix to avoid extra subdirectories in the relative key.
+        safe_prefix = str(object_prefix).strip().strip('/').replace('/', '_')
+        filename = f'{safe_prefix}_{timestamp}.parquet'
+        relative_key = f'training_datasets/{model_name}/{filename}'
+        bucket = getattr(self.minio_repository, 'bucket', 'streamlit-connectors')
+        uri = f's3://{bucket}/{relative_key}'
 
         try:
             data = await self.load_custom_query(input_data)
@@ -98,12 +244,20 @@ class Storage(Postgres):
             # Ensure we have a DataFrame
             data = pd.DataFrame(data)
 
-            # Write parquet to memory and upload via persistent client
-            await self.minio_repository.store_dataframe_as_parquet(
-                dataframe=data, uri=uri, object_name=object_name, metadata=metadata
+            # Convert DataFrame -> parquet bytes, then upload using the new MinIO interface.
+            parquet_buffer = BytesIO()
+            data.to_parquet(parquet_buffer, engine='pyarrow', index=True)
+            file_bytes = parquet_buffer.getvalue()
+
+            upload_result = await self.minio_repository.upload_file(
+                file_bytes=file_bytes,
+                relative_key=relative_key,
+                metadata=metadata,
             )
 
-            return {'success': True, 'object_key': object_name, 'uri': uri}
+            object_key_full = upload_result.get('minio_object_name', relative_key)
+            uri = f's3://{bucket}/{object_key_full}'
+            return {'success': True, 'object_key': object_key_full, 'uri': uri}
         except Exception as e:
             trace = traceback.format_exc()
             await self.send_notification_async(
@@ -121,18 +275,8 @@ class Storage(Postgres):
 
     def close(self) -> None:
         """Close Storage resources (MinIO client and Postgres engine)."""
-        try:
-            if hasattr(self, 'minio_repository') and self.minio_repository is not None:
-                try:
-                    self.minio_repository.close()
-                finally:
-                    self.minio_repository = None
-        finally:
-            # Ensure Postgres resources are disposed as well
-            try:
-                super().close()
-            except Exception:
-                self.logger.error('Error closing Postgres resources')
+        Postgres.close(self)
+        MinioManager.close(self)
 
     def __del__(self):
         self.close()