gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1712

Browse Source 
Remove code validation script and refactor imports in activities and workflows

- Deleted the `validate.sh` script, which was responsible for running code quality checks.
- Cleaned up import statements in `activities.py`, `gates.py`, `mlflow.py`, and `storage.py` by removing unused imports and organizing them.
- Refactored initialization methods in `MinioManager` and `MLFlow` classes for improved readability.
- Updated various workflows to ensure compatibility with the new structure and removed unnecessary comments.
- Enhanced test cases to accommodate changes in the activities and workflows, ensuring proper mocking of dependencies.
This commit is contained in:
vitor-aignosi
2026-03-20 09:14:16 -03:00
parent 981ac700d4
commit 5d0d049082
25 changed files with 1224 additions and 705 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
laborious/activities/activities.py
View File

@@ -14,7 +14,6 @@ with workflow.unsafe.imports_passed_through():
from laborious.activities.model_metrics import ModelMetrics
from laborious.activities.opc import OPC
from laborious.activities.storage import Storage
class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):

18
laborious/activities/gates.py
View File

@@ -13,17 +13,15 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now
from sientia_do.utils.formatters import create_sample_dict
from laborious import metrics
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
from laborious.utils.filters.conditional_filters import (
filter_empty_data,
filter_specific_variables_null_values,
)
from laborious.utils.filters.mlflow_filters import api_error_filter, nan_values_filter
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
# Strongly-typed filter function signatures
InputFilterFunc = Callable[[DataFrame, dict[str, Any]], bool]
@@ -106,7 +104,9 @@ class Gates(MinioManager):
Raises:
Exception: If BaseActivity initialization fails
"""
MinioManager.__init__(self, minio_repository, logger, notification_handler, metrics_controller)
MinioManager.__init__(
self, minio_repository, logger, notification_handler, metrics_controller
)
def close(self) -> None:
"""
@@ -238,7 +238,7 @@ class Gates(MinioManager):
payload: MinioDataFramePayload = input_data['data']
data = await payload.retrieve(self.minio_repository, metadata)
gate_type = input_data['type']
path_priority = input_data['path_priority']
@@ -326,7 +326,7 @@ class Gates(MinioManager):
self.info('Performing mlflow content gate...', metadata)
filters = input_data['filters']
payload: MinioDataFramePayload = input_data['data']
data = await payload.retrieve(self.minio_repository, metadata)
@@ -375,7 +375,7 @@ class Gates(MinioManager):
self.info('Nothing was filtered by the mlflow content gate', metadata)
del data
return None, 0, ''
def get_prediction_store_policy(
@@ -479,7 +479,7 @@ class Gates(MinioManager):
minio_repo=self.minio_repository,
model_name=input_data['model_name'],
operation='transform',
workflow_metadata=metadata
workflow_metadata=metadata,
)
@activity.defn(name='format_prediction')
@@ -601,7 +601,6 @@ class Gates(MinioManager):
self.info(f'Default prediction formatted: {data.size} rows', metadata)
return data.to_dict()
@activity.defn(name='format_retrain_report')
async def format_retrain_report(self, input_data: dict[str, Any]) -> dict:
"""
@@ -672,7 +671,6 @@ class Gates(MinioManager):
return report.to_dict()
@activity.defn(name='write_metrics')
async def write_metrics(self, input_data: dict[str, Any]):
"""

37
laborious/activities/mlflow.py
View File

@@ -1,4 +1,3 @@
from re import M
from temporalio import activity, workflow
from laborious.utils.repository.minio_manager import MinioManager
@@ -8,13 +7,12 @@ with workflow.unsafe.imports_passed_through():
from typing import Any
import numpy as np
from io import BytesIO
from pandas import DataFrame, read_parquet, to_datetime
from pandas import to_datetime
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.temporal.constants import (
DATETIME_FORMAT,
DATETIME_FORMAT_MS_WITH_TZ,
@@ -24,7 +22,6 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.utils.formatters import create_sample_dict
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
from sientia_do.repository.minio_repository import MinioRepository
from laborious.utils.repository.model_repository import MLFlowRepository
@@ -72,7 +69,9 @@ class MLFlow(MinioManager):
Raises:
Exception: If MLFlowRepository initialization fails
"""
MinioManager.__init__(self, minio_repository, logger, notification_handler, metrics_controller)
MinioManager.__init__(
self, minio_repository, logger, notification_handler, metrics_controller
)
self.mlflow_host = mlflow_host
self.mlflow_port = mlflow_port
self.mlflow_username = mlflow_username
@@ -128,7 +127,7 @@ class MLFlow(MinioManager):
"""
metadata = input_data['metadata']
self.info('Transforming data...', metadata)
payload: MinioDataFramePayload = input_data['data']
data = await payload.retrieve(self.minio_repository, metadata)
@@ -251,7 +250,6 @@ class MLFlow(MinioManager):
self.info('Data predicted successfully', metadata)
if not response_data.get('success', False):
return await MinioDataFramePayload.from_dataframe(
dataframe=None,
@@ -270,10 +268,9 @@ class MLFlow(MinioManager):
workflow_metadata=metadata,
status={
'success': True,
}
},
)
@activity.defn(name='retrain_model')
async def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
@@ -313,22 +310,10 @@ class MLFlow(MinioManager):
metadata = input_data['metadata']
try:
if 'data' in input_data:
# New path: payload-based retrain input (inline or MinIO offloaded).
data = await MinioDataFramePayload.dataframe_from_wire(
input_data['data'],
self.minio_repository,
metadata,
)
else:
# Backward compatibility: legacy query_to_minio contract.
object_key = input_data['object_key']
self.info(f'Loading retrain data from Key: {object_key}', metadata)
file_bytes = await self.minio_repository.download_file(
object_name=object_key,
metadata=metadata,
)
data = read_parquet(BytesIO(file_bytes))
# Payload-based retrain input (inline dict or MinIO offloaded).
payload: MinioDataFramePayload = input_data['data']
data = await payload.retrieve(self.minio_repository, metadata)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(

19
laborious/activities/storage.py
View File

@@ -1,15 +1,14 @@
import json
from temporalio import activity, workflow
from laborious.utils.repository.minio_manager import MinioManager
with workflow.unsafe.imports_passed_through():
# Extend the Temporal Postgres activities for convenient query -> MinIO export
import pickle
import traceback
from datetime import timedelta
from io import BytesIO
from os import getenv
from typing import Any
import pandas as pd
@@ -17,11 +16,11 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.temporal.activities.postgres import Postgres
from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, now
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
from sientia_do.repository.minio_repository import MinioRepository
_LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'})
@@ -64,10 +63,14 @@ class Storage(Postgres, MinioManager):
metrics_controller=metrics_controller,
)
MinioManager.__init__(self, minio_repository, logger, notification_handler, metrics_controller)
MinioManager.__init__(
self, minio_repository, logger, notification_handler, metrics_controller
)
@activity.defn(name='load_query_with_minio_offload')
async def load_query_with_minio_offload(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
async def load_query_with_minio_offload(
self, input_data: dict[str, Any]
) -> MinioDataFramePayload:
"""
Run the custom SQL load, then return a MinIO-aware dataframe wire dict.
@@ -92,7 +95,9 @@ class Storage(Postgres, MinioManager):
input_data,
)
if not rows:
self.error('load_query_with_minio_offload failed: No data returned from query', metadata)
self.error(
'load_query_with_minio_offload failed: No data returned from query', metadata
)
dataframe = None
else:
dataframe = pd.DataFrame(rows)
@@ -200,8 +205,6 @@ class Storage(Postgres, MinioManager):
)
return report
@activity.defn(name='query_to_minio')
async def query_to_minio(self, input_data: dict[str, Any]) -> dict[str, Any]:

47
laborious/utils/models/minio_dataframe_payload.py
View File

@@ -12,16 +12,16 @@ Otherwise, it is inlined as a Temporal-friendly ``dict``.
import pickle
import re
from dataclasses import dataclass, field
from collections.abc import Hashable
from dataclasses import dataclass
from datetime import datetime
from io import BytesIO
from os import getenv
from typing import Any, Hashable, Literal
from typing import Any, Literal
from pandas import DataFrame, read_parquet
from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, DATETIME_FORMAT_WITH_TZ, now
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, DATETIME_FORMAT_WITH_TZ, now
# Keys that are part of the serialized wire format (not arbitrary metadata).
_SERIALIZED_FIELD_KEYS = frozenset({'data', 'bucket', 'object_key', 'object_prefix', 'uri'})
@@ -30,7 +30,9 @@ _OBJECT_TIMESTAMP_PATTERN = re.compile(
r'-(?:initial|transform)-(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})\.parquet$'
)
OFFLOAD_THRESHOLD_BYTES = int(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_BYTES', '1.5')) * 1024 * 1024
OFFLOAD_THRESHOLD_BYTES = int(
float(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_BYTES', '1.5')) * 1024 * 1024
)
# Relative prefix used for storing offloaded training datasets in MinIO.
# It is also the root directory for retention cleanup listing.
@@ -79,7 +81,6 @@ class MinioDataFramePayload:
object_prefix: str | None = None
uri: str | None = None
@staticmethod
def estimate_size_bytes(df: DataFrame) -> int:
"""
@@ -117,21 +118,6 @@ class MinioDataFramePayload:
except ValueError:
return None
@staticmethod
def is_offloaded_dict(payload: dict[str, Any]) -> bool:
"""
Return True if the dict represents a MinIO-backed payload without inline data.
Args:
payload: Flat dict possibly produced by to_dict() / from_dataframe_to_dict().
Return:
bool: True when object_key is set and inline data is absent.
"""
if not payload.get('object_key'):
return False
return payload.get('data') is None
@staticmethod
def cleanup_prefix(self) -> str | None:
"""
@@ -141,6 +127,12 @@ class MinioDataFramePayload:
return self.object_prefix
return None
def has_data(self) -> bool:
"""
Return True if the payload has some data internally or in MinIO.
"""
return (self.data is not None and not self.data != {}) or self.object_key is not None
@classmethod
async def from_dataframe(
cls,
@@ -172,7 +164,9 @@ class MinioDataFramePayload:
"""
if not dataframe or dataframe.empty:
return cls(data=None, last_timestamp=now().strftime(DATETIME_FORMAT_WITH_TZ), status=status)
return cls(
data=None, last_timestamp=now().strftime(DATETIME_FORMAT_WITH_TZ), status=status
)
last_timestamp = max(dataframe['timestamp'].values.tolist())
@@ -207,7 +201,9 @@ class MinioDataFramePayload:
last_timestamp=last_timestamp,
)
async def retrieve(self, minio_repo: MinioRepository, workflow_metadata: dict[str, Any] | None = None) -> DataFrame:
async def retrieve(
self, minio_repo: MinioRepository, workflow_metadata: dict[str, Any] | None = None
) -> DataFrame:
"""
Load parquet from MinIO when object_key is set and populate inline data.
@@ -221,10 +217,11 @@ class MinioDataFramePayload:
if self.data is not None:
return DataFrame(self.data)
if self.data is None and self.object_key is None:
if not self.has_data():
return DataFrame()
file_bytes = await minio_repo.download_file(
object_name=self.object_key, metadata=workflow_metadata)
object_name=self.object_key, metadata=workflow_metadata
)
df = read_parquet(BytesIO(file_bytes))
return df

16
laborious/utils/repository/minio_manager.py
View File

@@ -1,14 +1,20 @@
from sientia_do.notifications.handlers import NotificationHandler
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.notifications.handlers import NotificationHandler
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository import MinioRepository
class MinioManager(SientiaMonitoring):
minio_repository: MinioRepository | None = None
def __init__(self, minio_repository: MinioRepository | None = None, logger: Logger | None = None, notification_handler: NotificationHandler | None = None, metrics_controller: MetricsController | None = None):
def __init__(
self,
minio_repository: MinioRepository | None = None,
logger: Logger | None = None,
notification_handler: NotificationHandler | None = None,
metrics_controller: MetricsController | None = None,
):
if self.minio_repository is None:
self.minio_repository = minio_repository
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
@@ -23,4 +29,4 @@ class MinioManager(SientiaMonitoring):
finally:
self.minio_repository = None
SientiaMonitoring.shutdown(self)
SientiaMonitoring.shutdown(self)

2
laborious/workflows/drift.py
View File

@@ -44,7 +44,7 @@ class Drift:
model_id = '{input_data['model_id']}' AND
timestamp > NOW() - INTERVAL '{input_data['interval']} minutes'
ORDER BY timestamp ASC
"""
""" # nosec B608 - values come from internal Temporal workflow config, not user input
target_data_handler = workflow.start_local_activity_method(
Activities.load_custom_query,

4
laborious/workflows/minimal_retrain.py
View File

@@ -84,8 +84,8 @@ class MinimalRetrain:
start_to_close_timeout=timedelta(seconds=600),
)
if isinstance(storage_result, dict) and storage_result.get('success') is False:
return
if not storage_result.has_data():
raise ValueError('No data returned from query')
experiment_response = await workflow.execute_activity_method(
Activities.retrain_model,

2
laborious/workflows/simple_metrics.py
View File

@@ -45,7 +45,7 @@ class SimpleMetrics:
p."timestamp" >= NOW() - INTERVAL '{interval_minutes} minutes'
order by
p."timestamp" desc;
"""
""" # nosec B608 - values come from internal Temporal workflow config, not user input
target_data = await workflow.execute_local_activity_method(
Activities.load_custom_query,

11
laborious/workflows/sub_workflows/prediction_process.py
View File

@@ -2,7 +2,6 @@ from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from datetime import timedelta
from collections.abc import Callable
from typing import Any
from sientia_do.temporal.policies import retry_policy
@@ -120,9 +119,8 @@ class PredictionProcess:
model_id: Any,
model_name: str,
model_config: dict[str, Any],
save_transform: bool
save_transform: bool,
) -> None:
last_timestamp = data.last_timestamp
# Apply input data quality gates
@@ -149,12 +147,7 @@ class PredictionProcess:
# Request MLFlow model transformation
response_data = await workflow.execute_local_activity_method(
Activities.request_transform,
{
**metadata,
'data': data,
'model_name': model_name,
'model_config': model_config
},
{**metadata, 'data': data, 'model_name': model_name, 'model_config': model_config},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(minutes=5),
)