b203b7d22c
Implement workflows for fake data generation, scouter processing, and core scouter operations - Added `FakeData` workflow to generate random data and send it to a Kafka topic. - Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow. - Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL. - Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling. - Enhanced Redis and Postgres activities with robust testing for data handling and error notifications. - Introduced quality filters for data validation and implemented tests to verify their functionality.
89 lines
3.0 KiB
Python
89 lines
3.0 KiB
Python
from temporalio import workflow, activity
|
|
|
|
with workflow.unsafe.imports_passed_through():
|
|
from logging import Logger
|
|
from sientia_do.notifications.handlers import NotificationHandler
|
|
from scouter.activities.base import BaseActivity
|
|
import redis
|
|
import json
|
|
from typing import Any
|
|
from pandas import DataFrame
|
|
import numpy as np
|
|
from datetime import datetime
|
|
|
|
|
|
class Redis(BaseActivity):
|
|
def __init__(self, host: str, port: int,
|
|
logger: Logger, notification_handler: NotificationHandler):
|
|
self.host = host
|
|
self.port = port
|
|
|
|
self.redis_client = redis.Redis(
|
|
host=self.host,
|
|
port=self.port,
|
|
decode_responses=True
|
|
)
|
|
|
|
BaseActivity.__init__(self, logger, notification_handler)
|
|
|
|
def get(self, key: str):
|
|
history = self.redis_client.get(key)
|
|
return json.loads(history) if history else None
|
|
|
|
def set(self, key: str, data: dict, ttl=600):
|
|
self.redis_client.set(key, json.dumps(data), ex=ttl)
|
|
|
|
@activity.defn(name="group_and_hold_data")
|
|
async def group_and_hold_data(self, input_data: dict[str, Any]):
|
|
"""
|
|
Groups and holds data in redis. Keep a copy of the most recent
|
|
received data for a given pipeline and schedule. This activity updates
|
|
the data in redis and return the full keeped data.
|
|
|
|
Args:
|
|
input_data (dict[str, Any]): The data to group and hold.
|
|
workflow_name (str): The name of the workflow.
|
|
schedule_name (str): The name of the schedule.
|
|
data (dict[str, Any]): The data to group and hold.
|
|
retention_time (int): The retention time for data in redis in seconds.
|
|
"""
|
|
self.logger.debug("Grouping and holding data...")
|
|
data = DataFrame(input_data['data'])
|
|
retention_time = input_data['retention_time']
|
|
|
|
key = f"{input_data['workflow_name']}_{input_data['schedule_name']}"
|
|
|
|
data_hold = self.get(key)
|
|
|
|
if not data_hold:
|
|
data_hold = {}
|
|
if data.empty:
|
|
self.logger.warning("No data to export")
|
|
return data_hold
|
|
|
|
for _, row in data.iterrows():
|
|
value = row['value']
|
|
|
|
if value is None:
|
|
data_hold[row['name']] = np.nan
|
|
|
|
else:
|
|
data_hold[row['name']] = value
|
|
|
|
data_hold['timestamp'] = data['timestamp'].max() if not data.empty else \
|
|
datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
self.set(key, data_hold, ttl=retention_time)
|
|
|
|
data_hold_df = DataFrame(data_hold, index=[0])
|
|
data_hold_melted = data_hold_df.melt(
|
|
id_vars='timestamp', var_name='variable', value_name='value')
|
|
data_hold_melted['model_id'] = input_data['model_id']
|
|
|
|
data_hold_melted.reset_index(drop=True, inplace=True)
|
|
|
|
self.logger.debug(
|
|
f"Data grouped and held successfully:\n {data_hold_melted.to_string()}")
|
|
|
|
return data_hold_melted.to_dict()
|