SIENTIAPDE-1005
Implement workflows for fake data generation, scouter processing, and core scouter operations - Added `FakeData` workflow to generate random data and send it to a Kafka topic. - Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow. - Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL. - Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling. - Enhanced Redis and Postgres activities with robust testing for data handling and error notifications. - Introduced quality filters for data validation and implemented tests to verify their functionality.
This commit is contained in:
vitor-aignosi
@@ -2,11 +2,11 @@ from temporalio import workflow, activity
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from pandas import DataFrame
|
||||
from scouter.activities.base import BaseActivity
|
||||
from scouter.utils.quality.filters import null_values_filter, out_of_bounds_filter
|
||||
from typing import Any
|
||||
import traceback
|
||||
from pandas import DataFrame
|
||||
|
||||
quality_gate_filters = {
|
||||
'NULL_VALUES_FILTER': null_values_filter,
|
||||
@@ -15,6 +15,133 @@ quality_gate_filters = {
|
||||
|
||||
|
||||
class Gates(BaseActivity):
|
||||
|
||||
def apply_aggregation(self, group: DataFrame, aggr_function: str) -> float | None | str:
|
||||
"""
|
||||
Apply aggregation function to a group of data.
|
||||
|
||||
Args:
|
||||
group (DataFrame): The group of data to apply the aggregation function to.
|
||||
aggr_function (str): The aggregation function to apply.
|
||||
|
||||
Returns:
|
||||
float | None | str: The result of the aggregation function.
|
||||
"""
|
||||
if len(group) == 1:
|
||||
return group['value'].item()
|
||||
|
||||
# Apply aggregation function to value
|
||||
if aggr_function == 'lts':
|
||||
return group['value'].iloc[-1]
|
||||
else:
|
||||
group.dropna(inplace=True, subset=['value'])
|
||||
|
||||
if group.empty:
|
||||
return None
|
||||
|
||||
if aggr_function == 'avg':
|
||||
return group['value'].mean()
|
||||
elif aggr_function == 'mdn':
|
||||
return group['value'].median()
|
||||
elif aggr_function == 'max':
|
||||
return group['value'].max()
|
||||
elif aggr_function == 'min':
|
||||
return group['value'].min()
|
||||
else:
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id="AGGREGATION_ISSUES",
|
||||
message=f"Invalid aggregation function: {aggr_function}",
|
||||
block="aggregate_data",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=traceback.format_exc()
|
||||
)
|
||||
return 'continue'
|
||||
|
||||
@activity.defn(name="aggregate_data")
|
||||
async def aggregate_data(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Aggregates time series data by tag and name, applying specified
|
||||
aggregation functions and taking the latest timestamp.
|
||||
|
||||
Args:
|
||||
input_data (dict[str, Any]): The data to aggregate. Contains:
|
||||
data (dict[str, Any]): The time series data.
|
||||
model_tags (dict[str, Any]): The tags configuration
|
||||
containing aggregation functions.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: The aggregated data.
|
||||
"""
|
||||
|
||||
try:
|
||||
# Convert input data to DataFrame
|
||||
df = DataFrame(input_data['data'])
|
||||
|
||||
self.logger.debug(
|
||||
f"Aggregating time series data: {df.to_string()}")
|
||||
|
||||
# Initialize result dictionary
|
||||
result = {}
|
||||
|
||||
# Group by tag and name
|
||||
grouped = df.groupby(['tag', 'name'])
|
||||
|
||||
for (tag, name), group in grouped:
|
||||
# Get the aggregation function from model_tags
|
||||
aggr_function = input_data['model_tags'].get(
|
||||
name, {}).get('aggr_function', 'lts')
|
||||
|
||||
group.sort_values(by='timestamp', inplace=True)
|
||||
|
||||
# Get the latest timestamp
|
||||
latest_timestamp = group['timestamp'].max()
|
||||
|
||||
if group.empty:
|
||||
continue
|
||||
|
||||
aggr_value = self.apply_aggregation(group, aggr_function)
|
||||
|
||||
if aggr_value == 'continue':
|
||||
continue
|
||||
|
||||
self.logger.debug(
|
||||
f"Aggregated data: {aggr_value}")
|
||||
self.logger.debug(
|
||||
f"Latest timestamp: {latest_timestamp}")
|
||||
self.logger.debug(
|
||||
f"Groups: {group.to_string()}")
|
||||
self.logger.debug(
|
||||
f"group name: {name}")
|
||||
self.logger.debug(
|
||||
f"group tag: {tag}")
|
||||
|
||||
# Store the result
|
||||
result[f"{tag}_{name}"] = {
|
||||
'tag': tag,
|
||||
'name': name,
|
||||
'value': aggr_value,
|
||||
'timestamp': latest_timestamp,
|
||||
'aggregation_function': aggr_function
|
||||
}
|
||||
|
||||
result_df = DataFrame(list(result.values()))
|
||||
self.logger.debug(f"Aggregated data:\n {result_df.to_string()}")
|
||||
return result_df.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id="AGGREGATION_ISSUES",
|
||||
message=f"Error aggregating data: {e}",
|
||||
block="aggregate_data",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
|
||||
self.logger.error(trace)
|
||||
raise
|
||||
|
||||
@activity.defn(name="data_quality_gate")
|
||||
async def data_quality_gate(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
@@ -29,6 +156,8 @@ class Gates(BaseActivity):
|
||||
filter_name: The name of the filter.
|
||||
policy: The policy to apply. Can be "DISCARD" or "KEEP".
|
||||
data (dict[str, Any]): The data to validate.
|
||||
model_tags (dict[str, Any]): The tags of the model.
|
||||
And it's respective configuration.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: The data validated.
|
||||
@@ -36,6 +165,10 @@ class Gates(BaseActivity):
|
||||
|
||||
filters = input_data['filters']
|
||||
data = DataFrame(input_data['data'])
|
||||
model_tags = input_data['model_tags']
|
||||
|
||||
self.logger.debug(
|
||||
f"Applying quality gate to data: {data.to_string()}")
|
||||
|
||||
for filter_name, policy in filters.items():
|
||||
if filter_name not in quality_gate_filters:
|
||||
@@ -43,17 +176,21 @@ class Gates(BaseActivity):
|
||||
continue
|
||||
|
||||
try:
|
||||
filtered_data = quality_gate_filters[filter_name](data)
|
||||
filtered_data = quality_gate_filters[filter_name](
|
||||
data, model_tags)
|
||||
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id="DATA_QUALITY_GATE_ISSUES",
|
||||
message=f"Error applying filter {filter_name}: {e}",
|
||||
block="data_quality_gate",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=traceback.format_exc()
|
||||
attachment_content=trace
|
||||
)
|
||||
|
||||
self.logger.error(trace)
|
||||
|
||||
else:
|
||||
if filtered_data.empty:
|
||||
continue
|
||||
@@ -62,7 +199,7 @@ class Gates(BaseActivity):
|
||||
attachment = filtered_data.to_string()
|
||||
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id="DATA_QUALITY_GATE_ISSUES",
|
||||
notification_id=f"DATA_QUALITY_GATE_ISSUES__{filter_name}",
|
||||
message=message,
|
||||
block="data_quality_gate",
|
||||
level=NotificationLevel.WARNING,
|
||||
@@ -70,6 +207,8 @@ class Gates(BaseActivity):
|
||||
)
|
||||
|
||||
if policy == "DISCARD":
|
||||
data = data[not data.isin(filtered_data).all(axis=1)]
|
||||
data = data[~data.index.isin(filtered_data.index)]
|
||||
|
||||
self.logger.debug("Data quality gate applied")
|
||||
|
||||
return data.to_dict()
|
||||
|
||||
Reference in New Issue
Block a user