SIENTIAPDE-1712
Implement MinIO Offload and Retention Features - Added configuration options for MinIO retention hours and offload threshold in README. - Introduced MinIO payload offloading for large DataFrame-derived payloads, storing them as parquet files. - Updated activities to utilize MinIO for data loading and cleanup, including new methods for offloading and retention management. - Refactored existing activities to integrate MinIO functionality, ensuring compatibility with previous workflows. - Removed the legacy MinioRepository class, consolidating MinIO operations under a new manager structure. - Updated requirements to use the latest version of the sientia-dataops-library.
This commit is contained in:
vitor-aignosi
@@ -88,4 +88,5 @@ def build_minio_config() -> dict[str, Any]:
|
||||
'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'),
|
||||
'region_name': getenv('MINIO_REGION_NAME', 'us-east-1'),
|
||||
'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious'),
|
||||
'retention_hours': int(getenv('MINIO_RETENTION_HOURS', '24')),
|
||||
}
|
||||
|
||||
0
laborious/utils/models/__init__.py
Normal file
0
laborious/utils/models/__init__.py
Normal file
230
laborious/utils/models/minio_dataframe_payload.py
Normal file
230
laborious/utils/models/minio_dataframe_payload.py
Normal file
@@ -0,0 +1,230 @@
|
||||
"""
|
||||
MinIO-backed DataFrame payload for Temporal workflows.
|
||||
|
||||
Data is never stored as a pandas ``DataFrame`` field on the dataclass.
|
||||
Instead, the DataFrame is only provided as an input to:
|
||||
`from_dataframe` / `from_dataframe_to_dict`.
|
||||
|
||||
At build time, the DataFrame is evaluated for its serialized size; if it exceeds
|
||||
the configured threshold, it is serialized to parquet bytes and uploaded to MinIO.
|
||||
Otherwise, it is inlined as a Temporal-friendly ``dict``.
|
||||
"""
|
||||
|
||||
import pickle
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from os import getenv
|
||||
from typing import Any, Hashable, Literal
|
||||
|
||||
from pandas import DataFrame, read_parquet
|
||||
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, DATETIME_FORMAT_WITH_TZ, now
|
||||
from sientia_do.repository.minio_repository import MinioRepository
|
||||
|
||||
# Keys that are part of the serialized wire format (not arbitrary metadata).
|
||||
_SERIALIZED_FIELD_KEYS = frozenset({'data', 'bucket', 'object_key', 'object_prefix', 'uri'})
|
||||
|
||||
_OBJECT_TIMESTAMP_PATTERN = re.compile(
|
||||
r'-(?:initial|transform)-(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})\.parquet$'
|
||||
)
|
||||
|
||||
OFFLOAD_THRESHOLD_BYTES = int(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_BYTES', '1.5')) * 1024 * 1024
|
||||
|
||||
# Relative prefix used for storing offloaded training datasets in MinIO.
|
||||
# It is also the root directory for retention cleanup listing.
|
||||
TRAINING_DATASETS_PREFIX = 'training_datasets'
|
||||
|
||||
OperationKind = Literal['initial', 'transform', 'predict']
|
||||
|
||||
|
||||
def _build_object_key(
|
||||
model_name: str, operation: OperationKind, timestamp: str
|
||||
) -> tuple[str, str | None]:
|
||||
"""
|
||||
Build the MinIO object key and the directory prefix used for retention listing.
|
||||
|
||||
Args:
|
||||
model_name: Registered model name used in the pipeline.
|
||||
operation: Either initial (pre-transform load) or transform (post-MLFlow transform).
|
||||
timestamp: Filename timestamp segment from DATETIME_FORMAT_FILENAME.
|
||||
|
||||
Return:
|
||||
tuple[str, str | None]: Full object key and normalized prefix (or None if at bucket root).
|
||||
"""
|
||||
# Naming convention:
|
||||
# - Directory is always `training_datasets/<model_name>`
|
||||
# - Filename follows the retention-parsing pattern
|
||||
basename = f'{model_name}-{operation}-{timestamp}.parquet'
|
||||
model_dir = model_name.strip().strip('/')
|
||||
prefix = f'{TRAINING_DATASETS_PREFIX}/{model_dir}'
|
||||
return f'{prefix}/{basename}', prefix
|
||||
|
||||
|
||||
@dataclass
|
||||
class MinioDataFramePayload:
|
||||
"""
|
||||
Serializable payload after a DataFrame was evaluated: inline tabular dict and/or MinIO keys.
|
||||
|
||||
Build from a live DataFrame only via `from_dataframe` / `from_dataframe_to_dict`.
|
||||
Rehydrate from Temporal via `from_dict`. The DataFrame is not a field on this class.
|
||||
"""
|
||||
|
||||
last_timestamp: str
|
||||
status: dict[str, Any] | None = None
|
||||
data: dict[Hashable, Any] | None = None
|
||||
bucket: str | None = None
|
||||
object_key: str | None = None
|
||||
object_prefix: str | None = None
|
||||
uri: str | None = None
|
||||
|
||||
|
||||
@staticmethod
|
||||
def estimate_size_bytes(df: DataFrame) -> int:
|
||||
"""
|
||||
Approximate serialized size of the DataFrame as the default-orient dict.
|
||||
|
||||
Args:
|
||||
df: DataFrame whose tabular content size is estimated.
|
||||
|
||||
Return:
|
||||
int: Estimated size in bytes (pickle of dict representation).
|
||||
"""
|
||||
try:
|
||||
return len(pickle.dumps(df.to_dict()))
|
||||
except Exception:
|
||||
return len(pickle.dumps(df))
|
||||
|
||||
@staticmethod
|
||||
def parse_object_timestamp(object_key: str) -> datetime | None:
|
||||
"""
|
||||
Parse the timestamp embedded in the object key basename (before .parquet).
|
||||
|
||||
Args:
|
||||
object_key: S3/MinIO object key whose basename follows
|
||||
``{model}-{initial|transform}-{DATETIME_FORMAT_FILENAME}.parquet``.
|
||||
|
||||
Return:
|
||||
datetime | None: Parsed UTC-naive datetime from the key, or None if not matched.
|
||||
"""
|
||||
basename = object_key.rsplit('/', 1)[-1]
|
||||
match = _OBJECT_TIMESTAMP_PATTERN.search(basename)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(match.group(1), DATETIME_FORMAT_FILENAME)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def is_offloaded_dict(payload: dict[str, Any]) -> bool:
|
||||
"""
|
||||
Return True if the dict represents a MinIO-backed payload without inline data.
|
||||
|
||||
Args:
|
||||
payload: Flat dict possibly produced by to_dict() / from_dataframe_to_dict().
|
||||
|
||||
Return:
|
||||
bool: True when object_key is set and inline data is absent.
|
||||
"""
|
||||
if not payload.get('object_key'):
|
||||
return False
|
||||
return payload.get('data') is None
|
||||
|
||||
@staticmethod
|
||||
def cleanup_prefix(self) -> str | None:
|
||||
"""
|
||||
Return True if cleanup is enabled for this payload.
|
||||
"""
|
||||
if self.object_key is not None and self.data is None:
|
||||
return self.object_prefix
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def from_dataframe(
|
||||
cls,
|
||||
dataframe: DataFrame | None,
|
||||
minio_repo: MinioRepository,
|
||||
model_name: str,
|
||||
operation: OperationKind,
|
||||
status: dict[str, Any] | None = None,
|
||||
workflow_metadata: dict | None = None,
|
||||
) -> 'MinioDataFramePayload':
|
||||
"""
|
||||
Evaluate the DataFrame size, then either inline dict or upload parquet to MinIO.
|
||||
|
||||
The DataFrame is not stored on the returned instance.
|
||||
|
||||
Args:
|
||||
dataframe: Tabular data to evaluate and persist (inline or MinIO).
|
||||
metadata: Small metadata dict merged into the payload (e.g. success, message).
|
||||
minio_repo: sientia_do MinioRepository (or compatible) with `upload_file()`.
|
||||
workflow_metadata: Metadata passed to MinIO store for logging/metrics.
|
||||
model_name: Registered model name used in the object basename.
|
||||
operation: Either ``initial`` (query load) or ``transform`` (post-transform).
|
||||
key_prefix: Backward-compatible parameter (currently ignored for object naming).
|
||||
size_threshold_bytes: Byte limit before offload. When None, the module-level
|
||||
environment-derived default is used.
|
||||
|
||||
Return:
|
||||
MinioDataFramePayload: Instance with data and/or MinIO fields set.
|
||||
"""
|
||||
|
||||
if not dataframe or dataframe.empty:
|
||||
return cls(data=None, last_timestamp=now().strftime(DATETIME_FORMAT_WITH_TZ), status=status)
|
||||
|
||||
last_timestamp = max(dataframe['timestamp'].values.tolist())
|
||||
|
||||
if cls.estimate_size_bytes(dataframe) <= OFFLOAD_THRESHOLD_BYTES:
|
||||
return cls(data=dataframe.to_dict(), last_timestamp=last_timestamp)
|
||||
|
||||
timestamp = now().strftime(DATETIME_FORMAT_FILENAME)
|
||||
object_key, object_prefix = _build_object_key(model_name, operation, timestamp)
|
||||
|
||||
# Upload using the relative object key. The upstream repository will
|
||||
# prefix it internally under its MinIO namespace.
|
||||
parquet_buffer = BytesIO()
|
||||
dataframe.to_parquet(parquet_buffer, engine='pyarrow', index=True)
|
||||
file_bytes = parquet_buffer.getvalue()
|
||||
|
||||
upload_result = await minio_repo.upload_file(
|
||||
file_bytes=file_bytes,
|
||||
relative_key=object_key,
|
||||
metadata=workflow_metadata,
|
||||
)
|
||||
|
||||
bucket = minio_repo.bucket
|
||||
object_key_full = upload_result.get('minio_object_name', object_key)
|
||||
uri = f's3://{bucket}/{object_key_full}' if bucket else None
|
||||
|
||||
return cls(
|
||||
data=None,
|
||||
bucket=bucket,
|
||||
object_key=object_key_full,
|
||||
object_prefix=object_prefix,
|
||||
uri=uri,
|
||||
last_timestamp=last_timestamp,
|
||||
)
|
||||
|
||||
async def retrieve(self, minio_repo: MinioRepository, workflow_metadata: dict[str, Any] | None = None) -> DataFrame:
|
||||
"""
|
||||
Load parquet from MinIO when object_key is set and populate inline data.
|
||||
|
||||
Args:
|
||||
minio_repo: sientia_do MinioRepository (or compatible) with download_file().
|
||||
workflow_metadata: Metadata passed to MinIO read for logging/metrics.
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Flat dict with data filled (same keys as to_dict after load).
|
||||
"""
|
||||
if self.data is not None:
|
||||
return DataFrame(self.data)
|
||||
|
||||
if self.data is None and self.object_key is None:
|
||||
return DataFrame()
|
||||
|
||||
file_bytes = await minio_repo.download_file(
|
||||
object_name=self.object_key, metadata=workflow_metadata)
|
||||
df = read_parquet(BytesIO(file_bytes))
|
||||
return df
|
||||
26
laborious/utils/repository/minio_manager.py
Normal file
26
laborious/utils/repository/minio_manager.py
Normal file
@@ -0,0 +1,26 @@
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from sientia_do.repository.minio_repository import MinioRepository
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
|
||||
|
||||
class MinioManager(SientiaMonitoring):
|
||||
minio_repository: MinioRepository | None = None
|
||||
|
||||
def __init__(self, minio_repository: MinioRepository | None = None, logger: Logger | None = None, notification_handler: NotificationHandler | None = None, metrics_controller: MetricsController | None = None):
|
||||
if self.minio_repository is None:
|
||||
self.minio_repository = minio_repository
|
||||
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Close the MinioManager and clean up resources.
|
||||
"""
|
||||
if self.minio_repository is not None:
|
||||
try:
|
||||
self.minio_repository.close()
|
||||
finally:
|
||||
self.minio_repository = None
|
||||
|
||||
SientiaMonitoring.shutdown(self)
|
||||
@@ -1,215 +0,0 @@
|
||||
"""
|
||||
MinIO repository utilities.
|
||||
|
||||
This module provides a lightweight repository around a MinIO/S3-compatible
|
||||
object storage using boto3. It supports creating buckets on demand and
|
||||
storing/loading pandas DataFrames in Parquet format.
|
||||
"""
|
||||
|
||||
import time
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from pandas import DataFrame, read_parquet
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
|
||||
from laborious import metrics
|
||||
|
||||
|
||||
class MinioRepository(SientiaMonitoring):
|
||||
"""
|
||||
Repository for interacting with a MinIO (S3-compatible) object storage.
|
||||
|
||||
This class encapsulates a reusable `boto3` S3 client and convenience
|
||||
helpers to persist and retrieve pandas DataFrames as Parquet files.
|
||||
|
||||
Attributes:
|
||||
storage_options (dict): Options compatible with pandas s3fs usage.
|
||||
minio_bucket (str): Default bucket name used for operations.
|
||||
minio_endpoint_url (str): MinIO endpoint URL.
|
||||
minio_region_name (str): MinIO region name.
|
||||
s3_client (Any): Reusable S3 client from `boto3`.
|
||||
logger (Logger): Observability logger.
|
||||
notification_handler (NotificationHandler): Notifications handler.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
minio_endpoint_url: str,
|
||||
minio_access_key: str,
|
||||
minio_secret_key: str,
|
||||
minio_region_name: str,
|
||||
minio_default_bucket: str,
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
metrics_controller: MetricsController,
|
||||
):
|
||||
"""Initialize the repository and S3 client.
|
||||
|
||||
Args:
|
||||
minio_endpoint_url (str): MinIO endpoint URL.
|
||||
minio_access_key (str): Access key (AK).
|
||||
minio_secret_key (str): Secret key (SK).
|
||||
minio_region_name (str): Region name for the client.
|
||||
minio_default_bucket (str): Default bucket name to operate on.
|
||||
logger (Logger): Logger instance for structured logs.
|
||||
notification_handler (NotificationHandler): Notification handler.
|
||||
"""
|
||||
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||
# MinIO settings shared with pandas s3fs
|
||||
self.storage_options = {
|
||||
'key': minio_access_key,
|
||||
'secret': minio_secret_key,
|
||||
'client_kwargs': {'endpoint_url': minio_endpoint_url},
|
||||
}
|
||||
self.minio_bucket = minio_default_bucket
|
||||
self.minio_endpoint_url = minio_endpoint_url
|
||||
self.minio_region_name = minio_region_name
|
||||
|
||||
logger.info(
|
||||
f'Connecting to MinIO at {self.minio_endpoint_url}, default bucket: {self.minio_bucket}'
|
||||
)
|
||||
|
||||
# Reusable MinIO client
|
||||
self.s3_client: Any = boto3.client(
|
||||
's3',
|
||||
endpoint_url=self.minio_endpoint_url,
|
||||
aws_access_key_id=self.storage_options['key'],
|
||||
aws_secret_access_key=self.storage_options['secret'],
|
||||
region_name=self.minio_region_name,
|
||||
config=Config(
|
||||
signature_version='s3v4',
|
||||
s3={'addressing_style': 'path'},
|
||||
retries={'max_attempts': 5, 'mode': 'standard'},
|
||||
connect_timeout=5,
|
||||
read_timeout=120,
|
||||
),
|
||||
)
|
||||
|
||||
def close(self):
|
||||
"""Close the underlying S3 client."""
|
||||
self.s3_client.close()
|
||||
|
||||
async def create_bucket(self, metadata: dict[str, Any]) -> None:
|
||||
core_labels = {
|
||||
**self.get_core_labels(metadata, operation_type='create_bucket'),
|
||||
'bucket_name': self.minio_bucket,
|
||||
'object_name': '-',
|
||||
}
|
||||
self.info(f"Creating bucket '{self.minio_bucket}'", metadata)
|
||||
|
||||
start_time = time.time()
|
||||
try:
|
||||
self.s3_client.create_bucket(Bucket=self.minio_bucket)
|
||||
except Exception as e:
|
||||
await self.emit_metric(metric_object=metrics.MINIO_WRITE_ERROR_COUNT, tags=core_labels)
|
||||
raise e
|
||||
|
||||
await self.observe_lag(start_time, metrics.MINIO_WRITE_LAG, core_labels)
|
||||
await self.emit_metric(metric_object=metrics.MINIO_WRITE_COUNT, tags=core_labels)
|
||||
|
||||
async def ensure_bucket_exists(self, metadata: dict[str, Any]) -> None:
|
||||
"""Ensure the default bucket exists; create it if missing.
|
||||
|
||||
Args:
|
||||
metadata (dict[str, Any]): Metadata used for structured logging.
|
||||
"""
|
||||
self.info(f"Checking if bucket '{self.minio_bucket}' exists", metadata)
|
||||
core_labels = {
|
||||
**self.get_core_labels(metadata, operation_type='head_bucket'),
|
||||
'bucket_name': self.minio_bucket,
|
||||
'object_name': '-',
|
||||
}
|
||||
self.info(f"Checking if bucket '{self.minio_bucket}' exists", metadata)
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
self.s3_client.head_bucket(Bucket=self.minio_bucket)
|
||||
except ClientError:
|
||||
await self.create_bucket(metadata)
|
||||
|
||||
except Exception as e:
|
||||
await self.emit_metric(metric_object=metrics.MINIO_WRITE_ERROR_COUNT, tags=core_labels)
|
||||
raise e
|
||||
|
||||
else:
|
||||
await self.observe_lag(start_time, metrics.MINIO_READ_LAG, core_labels)
|
||||
await self.emit_metric(metric_object=metrics.MINIO_READ_COUNT, tags=core_labels)
|
||||
|
||||
async def store_dataframe_as_parquet(
|
||||
self, dataframe: DataFrame, uri: str, object_name: str, metadata: dict[str, Any]
|
||||
):
|
||||
"""Persist a DataFrame as a Parquet object in the default bucket.
|
||||
|
||||
Args:
|
||||
dataframe (DataFrame): DataFrame to persist.
|
||||
uri (str): Human-friendly URI used for logging context.
|
||||
object_name (str): Object key (path/key within the bucket).
|
||||
metadata (dict[str, Any]): Metadata used for structured logging.
|
||||
"""
|
||||
await self.ensure_bucket_exists(metadata)
|
||||
|
||||
self.info(f'Storing dataframe as parquet in {uri}', metadata)
|
||||
|
||||
buffer = BytesIO()
|
||||
dataframe.to_parquet(buffer, engine='pyarrow', index=True)
|
||||
buffer.seek(0)
|
||||
|
||||
core_labels = {
|
||||
**self.get_core_labels(metadata, operation_type='put_object'),
|
||||
'bucket_name': self.minio_bucket,
|
||||
'object_name': object_name,
|
||||
}
|
||||
start_time = time.time()
|
||||
try:
|
||||
self.s3_client.put_object(
|
||||
Bucket=self.minio_bucket, Key=object_name, Body=buffer.getvalue()
|
||||
)
|
||||
except Exception as e:
|
||||
await self.emit_metric(metric_object=metrics.MINIO_WRITE_ERROR_COUNT, tags=core_labels)
|
||||
raise e
|
||||
|
||||
await self.observe_lag(start_time, metrics.MINIO_WRITE_LAG, core_labels)
|
||||
await self.emit_metric(metric_object=metrics.MINIO_WRITE_COUNT, tags=core_labels)
|
||||
|
||||
self.info(f'Dataframe stored as parquet in {uri}', metadata)
|
||||
|
||||
async def get_parquet_as_dataframe(
|
||||
self, object_key: str, metadata: dict[str, Any]
|
||||
) -> DataFrame:
|
||||
"""Load a Parquet object from the default bucket into a DataFrame.
|
||||
|
||||
Args:
|
||||
object_key (str): Object key to retrieve from the bucket.
|
||||
metadata (dict[str, Any]): Metadata used for structured logging.
|
||||
|
||||
Returns:
|
||||
DataFrame: Loaded DataFrame.
|
||||
"""
|
||||
self.info(f'Getting parquet as dataframe from {object_key}', metadata)
|
||||
|
||||
core_labels = {
|
||||
**self.get_core_labels(metadata, operation_type='get_object'),
|
||||
'bucket_name': self.minio_bucket,
|
||||
'object_name': object_key,
|
||||
}
|
||||
start_time = time.time()
|
||||
try:
|
||||
response = self.s3_client.get_object(Bucket=self.minio_bucket, Key=object_key)
|
||||
except Exception as e:
|
||||
await self.emit_metric(metric_object=metrics.MINIO_READ_ERROR_COUNT, tags=core_labels)
|
||||
raise e
|
||||
|
||||
await self.observe_lag(start_time, metrics.MINIO_READ_LAG, core_labels)
|
||||
await self.emit_metric(metric_object=metrics.MINIO_READ_COUNT, tags=core_labels)
|
||||
|
||||
# Read the content into a BytesIO buffer to support seek operations
|
||||
buffer = BytesIO(response['Body'].read())
|
||||
return read_parquet(buffer)
|
||||
@@ -1134,7 +1134,7 @@ class MLFlowRepository(SientiaMonitoring):
|
||||
|
||||
async def transform(
|
||||
self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict
|
||||
):
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Transform data using a cached transformation model.
|
||||
|
||||
@@ -1196,7 +1196,7 @@ class MLFlowRepository(SientiaMonitoring):
|
||||
|
||||
transformed_data = self.detect_and_parse_datetime_index(transformed_data, metadata)
|
||||
|
||||
return {'success': True, 'content': transformed_data.to_dict()}
|
||||
return {'success': True, 'content': transformed_data}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
@@ -1290,7 +1290,7 @@ class MLFlowRepository(SientiaMonitoring):
|
||||
predict_data.index = input_index
|
||||
predict_data['response_time'] = (end_time - start_time).total_seconds()
|
||||
|
||||
return {'success': True, 'content': predict_data.to_dict()}
|
||||
return {'success': True, 'content': predict_data}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user