gitea_admin/sientia-dataops-laborious-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1231

Browse Source 
Enhance MinIO integration and update environment configurations

- Added MinIO configuration parameters to .env.example and values.yaml for improved storage management.
- Updated requirements.txt to include necessary libraries for MinIO support.
- Refactored Activities class to utilize Storage for MinIO interactions.
- Enhanced MLFlow class to integrate MinIO for data retrieval during model retraining.
- Introduced build_minio_config function to streamline MinIO configuration setup.
- Updated minimal_retrain workflow to support data storage in MinIO.
This commit is contained in:
Vitor Pimentel dos Santos
2025-10-07 16:56:16 -03:00
parent 16a3aa7022
commit 7512963e19
11 changed files with 454 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
laborious/activities/activities.py
View File

@@ -1,7 +1,7 @@
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
from sientia_do.temporal.activities.postgres import Postgres
from laborious.activities.storage import Storage
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import Logger
from laborious.activities.mlflow import MLFlow
@@ -10,7 +10,7 @@ with workflow.unsafe.imports_passed_through():
from typing import Any
class Activities(Postgres, MLFlow, Gates, OPC):
class Activities(Storage, MLFlow, Gates, OPC):
"""
Main activities orchestrator for the Laborious system.
@@ -35,6 +35,7 @@ class Activities(Postgres, MLFlow, Gates, OPC):
def __init__(self,
postgres_config: dict[str, Any],
mlflow_config: dict[str, Any],
minio_config: dict[str, Any],
opc_config: dict[str, Any],
logger: Logger,
notification_handler: NotificationHandler):
@@ -58,20 +59,22 @@ class Activities(Postgres, MLFlow, Gates, OPC):
Exception: If any parent class initialization fails
"""
# Initialize parent classes
Postgres.__init__(self, host=postgres_config['host'],
port=postgres_config['port'],
user=postgres_config['user'],
password=postgres_config['password'],
dbname=postgres_config['dbname'],
min_connections=postgres_config['min_connections'],
max_connections=postgres_config['max_connections'],
logger=logger,
notification_handler=notification_handler)
Storage.__init__(self, host=postgres_config['host'],
port=postgres_config['port'],
user=postgres_config['user'],
password=postgres_config['password'],
dbname=postgres_config['dbname'],
min_connections=postgres_config['min_connections'],
max_connections=postgres_config['max_connections'],
minio_config=minio_config,
logger=logger,
notification_handler=notification_handler)
MLFlow.__init__(self, mlflow_host=mlflow_config['host'],
mlflow_port=mlflow_config['port'],
mlflow_username=mlflow_config['username'],
mlflow_password=mlflow_config['password'],
minio_config=minio_config,
logger=logger,
notification_handler=notification_handler)
@@ -95,5 +98,5 @@ class Activities(Postgres, MLFlow, Gates, OPC):
The method should be called before the application terminates to ensure
proper resource cleanup and prevent resource leaks.
"""
Postgres.close(self)
Storage.close(self)
await OPC.shutdown(self)

58
laborious/activities/mlflow.py
View File

@@ -1,9 +1,9 @@
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ, now
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
from datetime import datetime
from pandas import Timestamp, to_datetime
from pandas import to_datetime
from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
@@ -15,6 +15,7 @@ with workflow.unsafe.imports_passed_through():
import numpy as np
from pandas import DataFrame
import traceback
from laborious.utils.repository.minio_repository import MinioRepository
class MLFlow(BaseActivity):
@@ -37,7 +38,8 @@ class MLFlow(BaseActivity):
"""
def __init__(self, mlflow_host: str, mlflow_port: int, mlflow_username: str,
mlflow_password: str, logger: Logger, notification_handler: NotificationHandler):
minio_config: dict[str, Any], mlflow_password: str,
logger: Logger, notification_handler: NotificationHandler):
"""
Initialize MLFlow activities with server configuration.
@@ -63,6 +65,26 @@ class MLFlow(BaseActivity):
f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger
)
if not hasattr(self, 'minio_repository'):
self.minio_repository = MinioRepository(
logger=logger,
notification_handler=notification_handler,
minio_endpoint_url=minio_config['endpoint_url'],
minio_access_key=minio_config['access_key'],
minio_secret_key=minio_config['secret_key'],
minio_region_name=minio_config['region_name'],
minio_default_bucket=minio_config['default_bucket'])
if self.minio_repository is None:
self.minio_repository = MinioRepository(
logger=logger,
notification_handler=notification_handler,
minio_endpoint_url=minio_config['endpoint_url'],
minio_access_key=minio_config['access_key'],
minio_secret_key=minio_config['secret_key'],
minio_region_name=minio_config['region_name'],
minio_default_bucket=minio_config['default_bucket'])
@activity.defn(name="request_transform")
async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
@@ -223,7 +245,35 @@ class MLFlow(BaseActivity):
Exception: If retraining fails or encounters critical errors
"""
metadata = input_data['metadata']
data = DataFrame(input_data['data'])
object_key = input_data['object_key']
self.info(f'Loading retrain data from Key: {object_key}', metadata)
try:
data = self.minio_repository.get_parquet_as_dataframe(
object_key=object_key, metadata=metadata)
except Exception as e:
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id='ERROR_LOADING_RETRAIN_DATA',
message=f'Error loading retrain data: {e}',
block='retrain_model',
level=NotificationLevel.ERROR,
attachment_content=trace
)
self.error(trace, metadata)
return {
'success': False,
'message': f'Error loading retrain data: {e}',
'traceback': trace,
'timestamp': now().strftime(DATETIME_FORMAT_MS_WITH_TZ)
}
self.debug(
f'Retrain data loaded successfully: shape {data.shape}', metadata)
model_name = input_data['model_name']
model_config = input_data.get('model_config', {})

142
laborious/activities/storage.py Normal file
View File

@@ -0,0 +1,142 @@
from temporalio import activity, workflow
from laborious.utils.repository.minio_repository import MinioRepository
with workflow.unsafe.imports_passed_through():
# Extend the Temporal Postgres activities for convenient query -> MinIO export
from sientia_do.temporal.activities.postgres import Postgres
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import Logger
from sientia_do.temporal.constants import now
from sientia_do.notifications.models import NotificationLevel
from typing import Any
import traceback
import pandas as pd
DATETIME_FILENAME_FORMAT = "%Y-%m-%d_%H-%M-%S"
class Storage(Postgres):
"""
Extensions for Postgres activities with a helper to export query results
directly to MinIO as Parquet and return the object name.
"""
def __init__(self,
host: str,
port: int,
user: str,
password: str,
dbname: str,
min_connections: int,
max_connections: int,
minio_config: dict[str, Any],
logger: Logger,
notification_handler: NotificationHandler):
super().__init__(host=host,
port=port,
user=user,
password=password,
dbname=dbname,
min_connections=min_connections,
max_connections=max_connections,
logger=logger,
notification_handler=notification_handler)
if not hasattr(self, 'minio_repository'):
self.minio_repository = MinioRepository(
logger=logger,
notification_handler=notification_handler,
minio_endpoint_url=minio_config['endpoint_url'],
minio_access_key=minio_config['access_key'],
minio_secret_key=minio_config['secret_key'],
minio_region_name=minio_config['region_name'],
minio_default_bucket=minio_config['default_bucket'])
if self.minio_repository is None:
self.minio_repository = MinioRepository(
logger=logger,
notification_handler=notification_handler,
minio_endpoint_url=minio_config['endpoint_url'],
minio_access_key=minio_config['access_key'],
minio_secret_key=minio_config['secret_key'],
minio_region_name=minio_config['region_name'],
minio_default_bucket=minio_config['default_bucket'])
@activity.defn(name='query_to_minio')
async def query_to_minio(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Execute SQL query, write result as Parquet to MinIO, and return object name.
Args (input_data):
metadata (dict): Workflow metadata
query (str): SQL query
model_name (str): Model name for object naming
object_prefix (str, optional): Prefix inside bucket (default: datasets/retrain)
Returns:
dict: { success: bool, object_name: str, uri: str }
"""
metadata = input_data.get('metadata', {})
object_prefix = input_data.get('object_prefix', 'datasets/retrain')
timestamp = now().strftime(DATETIME_FILENAME_FORMAT)
object_name = f"{object_prefix}_{timestamp}.parquet"
uri = f"s3://{self.minio_repository.minio_bucket}/{object_name}"
try:
data = await self.load_custom_query(input_data)
if not data:
self.error(
f"query_to_minio failed: No data returned from query", metadata)
return {"success": False, "message": "No data returned from query"}
# Ensure we have a DataFrame
data = pd.DataFrame(data)
# Write parquet to memory and upload via persistent client
self.minio_repository.store_dataframe_as_parquet(
dataframe=data,
uri=uri,
object_name=object_name,
metadata=metadata
)
return {"success": True, "object_key": object_name, "uri": uri}
except Exception as e:
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id="ERROR_LOADING_CUSTOM_QUERY",
message=f"Error fetching data from query: {e}",
block="load_custom_query",
level=NotificationLevel.ERROR,
attachment_content=trace
)
self.error(trace, metadata)
return {"success": False, "message": str(e)}
def close(self) -> None:
"""Close Storage resources (MinIO client and Postgres engine)."""
try:
if hasattr(self, 's3_client') and self.s3_client is not None:
try:
self.s3_client.close()
finally:
self.s3_client = None
finally:
# Ensure Postgres resources are disposed as well
try:
super().close()
except Exception:
pass
def __del__(self):
try:
self.close()
except Exception:
pass