SIENTIAPDE-1712
Remove code validation script and refactor imports in activities and workflows - Deleted the `validate.sh` script, which was responsible for running code quality checks. - Cleaned up import statements in `activities.py`, `gates.py`, `mlflow.py`, and `storage.py` by removing unused imports and organizing them. - Refactored initialization methods in `MinioManager` and `MLFlow` classes for improved readability. - Updated various workflows to ensure compatibility with the new structure and removed unnecessary comments. - Enhanced test cases to accommodate changes in the activities and workflows, ensuring proper mocking of dependencies.
This commit is contained in:
@@ -1,15 +1,14 @@
|
||||
import json
|
||||
|
||||
from temporalio import activity, workflow
|
||||
|
||||
from laborious.utils.repository.minio_manager import MinioManager
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
# Extend the Temporal Postgres activities for convenient query -> MinIO export
|
||||
import pickle
|
||||
import traceback
|
||||
from datetime import timedelta
|
||||
from io import BytesIO
|
||||
from os import getenv
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
@@ -17,11 +16,11 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.repository.minio_repository import MinioRepository
|
||||
from sientia_do.temporal.activities.postgres import Postgres
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, now
|
||||
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
from sientia_do.repository.minio_repository import MinioRepository
|
||||
|
||||
_LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'})
|
||||
|
||||
@@ -64,10 +63,14 @@ class Storage(Postgres, MinioManager):
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
MinioManager.__init__(self, minio_repository, logger, notification_handler, metrics_controller)
|
||||
MinioManager.__init__(
|
||||
self, minio_repository, logger, notification_handler, metrics_controller
|
||||
)
|
||||
|
||||
@activity.defn(name='load_query_with_minio_offload')
|
||||
async def load_query_with_minio_offload(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
|
||||
async def load_query_with_minio_offload(
|
||||
self, input_data: dict[str, Any]
|
||||
) -> MinioDataFramePayload:
|
||||
"""
|
||||
Run the custom SQL load, then return a MinIO-aware dataframe wire dict.
|
||||
|
||||
@@ -92,7 +95,9 @@ class Storage(Postgres, MinioManager):
|
||||
input_data,
|
||||
)
|
||||
if not rows:
|
||||
self.error('load_query_with_minio_offload failed: No data returned from query', metadata)
|
||||
self.error(
|
||||
'load_query_with_minio_offload failed: No data returned from query', metadata
|
||||
)
|
||||
dataframe = None
|
||||
else:
|
||||
dataframe = pd.DataFrame(rows)
|
||||
@@ -200,8 +205,6 @@ class Storage(Postgres, MinioManager):
|
||||
)
|
||||
|
||||
return report
|
||||
|
||||
|
||||
|
||||
@activity.defn(name='query_to_minio')
|
||||
async def query_to_minio(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
Reference in New Issue
Block a user