SIENTIAPDE-1712
Implement MinIO Offload and Retention Features - Added configuration options for MinIO retention hours and offload threshold in README. - Introduced MinIO payload offloading for large DataFrame-derived payloads, storing them as parquet files. - Updated activities to utilize MinIO for data loading and cleanup, including new methods for offloading and retention management. - Refactored existing activities to integrate MinIO functionality, ensuring compatibility with previous workflows. - Removed the legacy MinioRepository class, consolidating MinIO operations under a new manager structure. - Updated requirements to use the latest version of the sientia-dataops-library.
This commit is contained in:
vitor-aignosi
26
laborious/utils/repository/minio_manager.py
Normal file
26
laborious/utils/repository/minio_manager.py
Normal file
@@ -0,0 +1,26 @@
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from sientia_do.repository.minio_repository import MinioRepository
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
|
||||
|
||||
class MinioManager(SientiaMonitoring):
|
||||
minio_repository: MinioRepository | None = None
|
||||
|
||||
def __init__(self, minio_repository: MinioRepository | None = None, logger: Logger | None = None, notification_handler: NotificationHandler | None = None, metrics_controller: MetricsController | None = None):
|
||||
if self.minio_repository is None:
|
||||
self.minio_repository = minio_repository
|
||||
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Close the MinioManager and clean up resources.
|
||||
"""
|
||||
if self.minio_repository is not None:
|
||||
try:
|
||||
self.minio_repository.close()
|
||||
finally:
|
||||
self.minio_repository = None
|
||||
|
||||
SientiaMonitoring.shutdown(self)
|
||||
@@ -1,215 +0,0 @@
|
||||
"""
|
||||
MinIO repository utilities.
|
||||
|
||||
This module provides a lightweight repository around a MinIO/S3-compatible
|
||||
object storage using boto3. It supports creating buckets on demand and
|
||||
storing/loading pandas DataFrames in Parquet format.
|
||||
"""
|
||||
|
||||
import time
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from pandas import DataFrame, read_parquet
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
|
||||
from laborious import metrics
|
||||
|
||||
|
||||
class MinioRepository(SientiaMonitoring):
|
||||
"""
|
||||
Repository for interacting with a MinIO (S3-compatible) object storage.
|
||||
|
||||
This class encapsulates a reusable `boto3` S3 client and convenience
|
||||
helpers to persist and retrieve pandas DataFrames as Parquet files.
|
||||
|
||||
Attributes:
|
||||
storage_options (dict): Options compatible with pandas s3fs usage.
|
||||
minio_bucket (str): Default bucket name used for operations.
|
||||
minio_endpoint_url (str): MinIO endpoint URL.
|
||||
minio_region_name (str): MinIO region name.
|
||||
s3_client (Any): Reusable S3 client from `boto3`.
|
||||
logger (Logger): Observability logger.
|
||||
notification_handler (NotificationHandler): Notifications handler.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
minio_endpoint_url: str,
|
||||
minio_access_key: str,
|
||||
minio_secret_key: str,
|
||||
minio_region_name: str,
|
||||
minio_default_bucket: str,
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
metrics_controller: MetricsController,
|
||||
):
|
||||
"""Initialize the repository and S3 client.
|
||||
|
||||
Args:
|
||||
minio_endpoint_url (str): MinIO endpoint URL.
|
||||
minio_access_key (str): Access key (AK).
|
||||
minio_secret_key (str): Secret key (SK).
|
||||
minio_region_name (str): Region name for the client.
|
||||
minio_default_bucket (str): Default bucket name to operate on.
|
||||
logger (Logger): Logger instance for structured logs.
|
||||
notification_handler (NotificationHandler): Notification handler.
|
||||
"""
|
||||
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||
# MinIO settings shared with pandas s3fs
|
||||
self.storage_options = {
|
||||
'key': minio_access_key,
|
||||
'secret': minio_secret_key,
|
||||
'client_kwargs': {'endpoint_url': minio_endpoint_url},
|
||||
}
|
||||
self.minio_bucket = minio_default_bucket
|
||||
self.minio_endpoint_url = minio_endpoint_url
|
||||
self.minio_region_name = minio_region_name
|
||||
|
||||
logger.info(
|
||||
f'Connecting to MinIO at {self.minio_endpoint_url}, default bucket: {self.minio_bucket}'
|
||||
)
|
||||
|
||||
# Reusable MinIO client
|
||||
self.s3_client: Any = boto3.client(
|
||||
's3',
|
||||
endpoint_url=self.minio_endpoint_url,
|
||||
aws_access_key_id=self.storage_options['key'],
|
||||
aws_secret_access_key=self.storage_options['secret'],
|
||||
region_name=self.minio_region_name,
|
||||
config=Config(
|
||||
signature_version='s3v4',
|
||||
s3={'addressing_style': 'path'},
|
||||
retries={'max_attempts': 5, 'mode': 'standard'},
|
||||
connect_timeout=5,
|
||||
read_timeout=120,
|
||||
),
|
||||
)
|
||||
|
||||
def close(self):
|
||||
"""Close the underlying S3 client."""
|
||||
self.s3_client.close()
|
||||
|
||||
async def create_bucket(self, metadata: dict[str, Any]) -> None:
|
||||
core_labels = {
|
||||
**self.get_core_labels(metadata, operation_type='create_bucket'),
|
||||
'bucket_name': self.minio_bucket,
|
||||
'object_name': '-',
|
||||
}
|
||||
self.info(f"Creating bucket '{self.minio_bucket}'", metadata)
|
||||
|
||||
start_time = time.time()
|
||||
try:
|
||||
self.s3_client.create_bucket(Bucket=self.minio_bucket)
|
||||
except Exception as e:
|
||||
await self.emit_metric(metric_object=metrics.MINIO_WRITE_ERROR_COUNT, tags=core_labels)
|
||||
raise e
|
||||
|
||||
await self.observe_lag(start_time, metrics.MINIO_WRITE_LAG, core_labels)
|
||||
await self.emit_metric(metric_object=metrics.MINIO_WRITE_COUNT, tags=core_labels)
|
||||
|
||||
async def ensure_bucket_exists(self, metadata: dict[str, Any]) -> None:
|
||||
"""Ensure the default bucket exists; create it if missing.
|
||||
|
||||
Args:
|
||||
metadata (dict[str, Any]): Metadata used for structured logging.
|
||||
"""
|
||||
self.info(f"Checking if bucket '{self.minio_bucket}' exists", metadata)
|
||||
core_labels = {
|
||||
**self.get_core_labels(metadata, operation_type='head_bucket'),
|
||||
'bucket_name': self.minio_bucket,
|
||||
'object_name': '-',
|
||||
}
|
||||
self.info(f"Checking if bucket '{self.minio_bucket}' exists", metadata)
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
self.s3_client.head_bucket(Bucket=self.minio_bucket)
|
||||
except ClientError:
|
||||
await self.create_bucket(metadata)
|
||||
|
||||
except Exception as e:
|
||||
await self.emit_metric(metric_object=metrics.MINIO_WRITE_ERROR_COUNT, tags=core_labels)
|
||||
raise e
|
||||
|
||||
else:
|
||||
await self.observe_lag(start_time, metrics.MINIO_READ_LAG, core_labels)
|
||||
await self.emit_metric(metric_object=metrics.MINIO_READ_COUNT, tags=core_labels)
|
||||
|
||||
async def store_dataframe_as_parquet(
|
||||
self, dataframe: DataFrame, uri: str, object_name: str, metadata: dict[str, Any]
|
||||
):
|
||||
"""Persist a DataFrame as a Parquet object in the default bucket.
|
||||
|
||||
Args:
|
||||
dataframe (DataFrame): DataFrame to persist.
|
||||
uri (str): Human-friendly URI used for logging context.
|
||||
object_name (str): Object key (path/key within the bucket).
|
||||
metadata (dict[str, Any]): Metadata used for structured logging.
|
||||
"""
|
||||
await self.ensure_bucket_exists(metadata)
|
||||
|
||||
self.info(f'Storing dataframe as parquet in {uri}', metadata)
|
||||
|
||||
buffer = BytesIO()
|
||||
dataframe.to_parquet(buffer, engine='pyarrow', index=True)
|
||||
buffer.seek(0)
|
||||
|
||||
core_labels = {
|
||||
**self.get_core_labels(metadata, operation_type='put_object'),
|
||||
'bucket_name': self.minio_bucket,
|
||||
'object_name': object_name,
|
||||
}
|
||||
start_time = time.time()
|
||||
try:
|
||||
self.s3_client.put_object(
|
||||
Bucket=self.minio_bucket, Key=object_name, Body=buffer.getvalue()
|
||||
)
|
||||
except Exception as e:
|
||||
await self.emit_metric(metric_object=metrics.MINIO_WRITE_ERROR_COUNT, tags=core_labels)
|
||||
raise e
|
||||
|
||||
await self.observe_lag(start_time, metrics.MINIO_WRITE_LAG, core_labels)
|
||||
await self.emit_metric(metric_object=metrics.MINIO_WRITE_COUNT, tags=core_labels)
|
||||
|
||||
self.info(f'Dataframe stored as parquet in {uri}', metadata)
|
||||
|
||||
async def get_parquet_as_dataframe(
|
||||
self, object_key: str, metadata: dict[str, Any]
|
||||
) -> DataFrame:
|
||||
"""Load a Parquet object from the default bucket into a DataFrame.
|
||||
|
||||
Args:
|
||||
object_key (str): Object key to retrieve from the bucket.
|
||||
metadata (dict[str, Any]): Metadata used for structured logging.
|
||||
|
||||
Returns:
|
||||
DataFrame: Loaded DataFrame.
|
||||
"""
|
||||
self.info(f'Getting parquet as dataframe from {object_key}', metadata)
|
||||
|
||||
core_labels = {
|
||||
**self.get_core_labels(metadata, operation_type='get_object'),
|
||||
'bucket_name': self.minio_bucket,
|
||||
'object_name': object_key,
|
||||
}
|
||||
start_time = time.time()
|
||||
try:
|
||||
response = self.s3_client.get_object(Bucket=self.minio_bucket, Key=object_key)
|
||||
except Exception as e:
|
||||
await self.emit_metric(metric_object=metrics.MINIO_READ_ERROR_COUNT, tags=core_labels)
|
||||
raise e
|
||||
|
||||
await self.observe_lag(start_time, metrics.MINIO_READ_LAG, core_labels)
|
||||
await self.emit_metric(metric_object=metrics.MINIO_READ_COUNT, tags=core_labels)
|
||||
|
||||
# Read the content into a BytesIO buffer to support seek operations
|
||||
buffer = BytesIO(response['Body'].read())
|
||||
return read_parquet(buffer)
|
||||
@@ -1134,7 +1134,7 @@ class MLFlowRepository(SientiaMonitoring):
|
||||
|
||||
async def transform(
|
||||
self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict
|
||||
):
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Transform data using a cached transformation model.
|
||||
|
||||
@@ -1196,7 +1196,7 @@ class MLFlowRepository(SientiaMonitoring):
|
||||
|
||||
transformed_data = self.detect_and_parse_datetime_index(transformed_data, metadata)
|
||||
|
||||
return {'success': True, 'content': transformed_data.to_dict()}
|
||||
return {'success': True, 'content': transformed_data}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
@@ -1290,7 +1290,7 @@ class MLFlowRepository(SientiaMonitoring):
|
||||
predict_data.index = input_index
|
||||
predict_data['response_time'] = (end_time - start_time).total_seconds()
|
||||
|
||||
return {'success': True, 'content': predict_data.to_dict()}
|
||||
return {'success': True, 'content': predict_data}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user