SIENTIAPDE-1712
Refactor MinioDataFramePayload usage across activities - Updated instances of MinioDataFramePayload initialization in Gates, MLFlow, and Storage classes to use the new from_dict method for better data reconstruction from dictionaries. - Enhanced the PredictionProcess workflow to utilize the updated payload handling. - Added passthrough fixtures in tests to accommodate the new from_dict method for consistent testing behavior.
This commit is contained in:
@@ -8,7 +8,6 @@ with workflow.unsafe.imports_passed_through():
|
||||
# Extend the Temporal Postgres activities for convenient query -> MinIO export
|
||||
import traceback
|
||||
from datetime import timedelta
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
@@ -18,7 +17,7 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.repository.minio_repository import MinioRepository
|
||||
from sientia_do.temporal.activities.postgres import Postgres
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, now
|
||||
from sientia_do.temporal.constants import now
|
||||
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
|
||||
@@ -116,7 +115,7 @@ class Storage(Postgres, MinioManager):
|
||||
Export a payload to PostgreSQL.
|
||||
"""
|
||||
metadata = input_data.get('metadata')
|
||||
payload: MinioDataFramePayload = input_data['data']
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
data = await payload.retrieve(self.minio_repository, metadata)
|
||||
|
||||
return await self.export_data_to_postgres(
|
||||
@@ -142,7 +141,8 @@ class Storage(Postgres, MinioManager):
|
||||
raise ValueError('Minio repository not initialized')
|
||||
|
||||
metadata = input_data.get('metadata', {})
|
||||
prefix = input_data['prefix']
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
prefix = payload.cleanup_prefix()
|
||||
base = now()
|
||||
cutoff = (base.replace(tzinfo=None) if base.tzinfo else base) - timedelta(
|
||||
hours=self.retention_hours
|
||||
@@ -206,7 +206,6 @@ class Storage(Postgres, MinioManager):
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close Storage resources (MinIO client and Postgres engine)."""
|
||||
Postgres.close(self)
|
||||
|
||||
Reference in New Issue
Block a user