Code import - branch release/SIENTIAPDE-1646
This commit is contained in:
231
laborious/activities/storage.py
Normal file
231
laborious/activities/storage.py
Normal file
@@ -0,0 +1,231 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
# Extend the Temporal Postgres activities for convenient query -> MinIO export
|
||||
import traceback
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_do.repository.minio_repository_sync import MinioRepository
|
||||
from sientia_do.temporal.activities.postgres_sync import Postgres
|
||||
from sientia_do.temporal.constants import now
|
||||
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
|
||||
_LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'})
|
||||
|
||||
|
||||
class Storage(Postgres, SientiaMonitoring):
|
||||
"""
|
||||
Extensions for Postgres activities with a helper to export query results
|
||||
directly to MinIO as Parquet and return the object name.
|
||||
"""
|
||||
|
||||
minio_repository: MinioRepository | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
user: str,
|
||||
password: str,
|
||||
dbname: str,
|
||||
min_connections: int,
|
||||
max_connections: int,
|
||||
retention_hours: int = 24,
|
||||
minio_repository: MinioRepository | None = None,
|
||||
logger: Logger | None = None,
|
||||
notification_handler: NotificationHandler | None = None,
|
||||
metrics_controller: MetricsController | None = None,
|
||||
):
|
||||
self.retention_hours = retention_hours
|
||||
Postgres.__init__(
|
||||
self,
|
||||
host=host,
|
||||
port=port,
|
||||
user=user,
|
||||
password=password,
|
||||
dbname=dbname,
|
||||
min_connections=min_connections,
|
||||
max_connections=max_connections,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
self.minio_repository = minio_repository
|
||||
SientiaMonitoring.__init__(
|
||||
self,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
@activity.defn(name='load_query_with_minio_offload')
|
||||
def load_query_with_minio_offload(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
|
||||
"""
|
||||
Run the custom SQL load, then return a MinIO-aware dataframe wire dict.
|
||||
|
||||
Args (input_data):
|
||||
metadata (dict): Workflow metadata (same as load_custom_query).
|
||||
query (str): SQL query.
|
||||
datetime_columns (list[str], optional): Datetime column names.
|
||||
model_name (str): Model name for object key basename.
|
||||
key_prefix (str, optional): Directory prefix inside the bucket.
|
||||
size_threshold_bytes (int, optional): Override env offload threshold.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: Flat ``MinioDataFramePayload`` dict or ``success: False`` on failure.
|
||||
"""
|
||||
if self.minio_repository is None:
|
||||
raise ValueError('Minio repository not initialized')
|
||||
|
||||
metadata: dict = input_data.get('metadata', {})
|
||||
model_name = input_data['model_name']
|
||||
|
||||
rows = self.load_custom_query(
|
||||
input_data,
|
||||
)
|
||||
if not rows:
|
||||
self.error(
|
||||
'load_query_with_minio_offload failed: No data returned from query', metadata
|
||||
)
|
||||
dataframe = None
|
||||
else:
|
||||
dataframe = pd.DataFrame(rows)
|
||||
|
||||
return MinioDataFramePayload.from_dataframe(
|
||||
dataframe,
|
||||
minio_repo=self.minio_repository,
|
||||
workflow_metadata=metadata,
|
||||
model_name=model_name,
|
||||
operation='initial',
|
||||
logger=self.logger,
|
||||
)
|
||||
|
||||
@activity.defn(name='export_payload_to_postgres')
|
||||
def export_payload_to_postgres(self, input_data: dict[str, Any]) -> dict:
|
||||
"""
|
||||
Resolve a MinIO-aware payload into a DataFrame and persist it into PostgreSQL.
|
||||
|
||||
This activity accepts the serialized payload produced by previous steps
|
||||
(inline dict or MinIO object reference), reconstructs the tabular data,
|
||||
and delegates the final write to ``export_data_to_postgres`` using the
|
||||
same input contract expected by the Postgres activity mixin.
|
||||
|
||||
Args:
|
||||
- input_data (dict[str, Any]): Activity input containing ``data`` as a
|
||||
``MinioDataFramePayload``-compatible dict plus database write options
|
||||
(schema/table/on_conflict/metadata and related fields).
|
||||
|
||||
Return:
|
||||
dict: Result dictionary returned by ``export_data_to_postgres``, including
|
||||
success status and optional write diagnostics.
|
||||
"""
|
||||
metadata = input_data.get('metadata')
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
data = payload.retrieve(self.minio_repository, metadata)
|
||||
|
||||
return self.export_data_to_postgres(
|
||||
{
|
||||
**input_data,
|
||||
'data': data,
|
||||
}
|
||||
)
|
||||
|
||||
@activity.defn(name='cleanup_minio_objects_expired')
|
||||
def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Delete objects under the given prefixes that are older than the retention window.
|
||||
|
||||
Args (input_data):
|
||||
metadata (dict): Workflow metadata for logging and metrics.
|
||||
prefixes (list[str]): Key prefixes to scan (one level or subtree per prefix).
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: ``success``, ``deleted_count``, and optional ``message``.
|
||||
"""
|
||||
if self.minio_repository is None:
|
||||
raise ValueError('Minio repository not initialized')
|
||||
|
||||
metadata = input_data.get('metadata', {})
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
prefix = payload.cleanup_prefix()
|
||||
base = now()
|
||||
cutoff = (base.replace(tzinfo=None) if base.tzinfo else base) - timedelta(
|
||||
hours=self.retention_hours
|
||||
)
|
||||
|
||||
report: dict[str, Any] = {
|
||||
'failed': {},
|
||||
'deleted': {},
|
||||
'failed_count': 0,
|
||||
'deleted_count': 0,
|
||||
}
|
||||
try:
|
||||
keys = self.minio_repository.list_objects(
|
||||
prefix=prefix,
|
||||
recursive=True,
|
||||
metadata=metadata,
|
||||
)
|
||||
for key in keys:
|
||||
try:
|
||||
ts = MinioDataFramePayload.parse_object_timestamp(key)
|
||||
if ts is None:
|
||||
continue
|
||||
if ts >= cutoff:
|
||||
continue
|
||||
self.minio_repository.delete_file(
|
||||
object_name=key,
|
||||
metadata=metadata,
|
||||
)
|
||||
except Exception as e:
|
||||
report['failed'][key] = {
|
||||
'success': False,
|
||||
'message': str(e),
|
||||
}
|
||||
report['failed_count'] += 1
|
||||
continue
|
||||
report['deleted'][key] = {
|
||||
'success': True,
|
||||
'message': 'Deleted',
|
||||
}
|
||||
report['deleted_count'] += 1
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='ERROR_CLEANUP_MINIO_OBJECTS_EXPIRED',
|
||||
message=f'Error cleaning up MinIO objects: {e}',
|
||||
block='cleanup_minio_objects_expired',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace,
|
||||
)
|
||||
self.error(trace, metadata)
|
||||
else:
|
||||
# Cleanup success is expected in normal flow; avoid noisy INFO notifications
|
||||
# that do not impact behavior and can flood observability in test runs.
|
||||
self.info('MinIO objects cleaned up successfully', metadata)
|
||||
|
||||
return report
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Shutdown Storage resources in deterministic order.
|
||||
|
||||
The method first closes Postgres resources via ``Postgres.close`` (engine,
|
||||
sessions, and monitoring hooks), then closes the optional MinIO repository
|
||||
and clears the local reference to avoid accidental reuse after shutdown.
|
||||
"""
|
||||
Postgres.close(self)
|
||||
if self.minio_repository is not None:
|
||||
try:
|
||||
self.minio_repository.close()
|
||||
finally:
|
||||
self.minio_repository = None
|
||||
Reference in New Issue
Block a user