gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1316

Browse Source 
Update .gitignore and refactor metrics.py, activities.py, and gates.py for improved clarity and consistency. Added coverage.xml and cache directories to .gitignore. Standardized string formatting and parameter handling in metrics and activities classes, enhancing code readability. Removed the deprecated faker.py file and adjusted related tests accordingly.
This commit is contained in:
vitor-aignosi
2025-10-16 13:31:12 -03:00
parent 8bdbf049b8
commit 97eb5bc904
27 changed files with 1101 additions and 1336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
scouter/activities/gates.py
View File

@@ -1,20 +1,23 @@
from temporalio import workflow, activity
from collections.abc import Hashable
from scouter import metrics
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.notifications.handlers import NotificationHandler
from sientia_do.observability.logger import Logger
from scouter.utils.quality.filters import null_values_filter, out_of_bounds_filter
from typing import Any
import traceback
from typing import Any
from pandas import DataFrame
from sientia_do.notifications.handlers import NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.temporal.activities.base import BaseActivity
from scouter import metrics
from scouter.utils.quality.filters import null_values_filter, out_of_bounds_filter
quality_gate_filters = {
'NULL_VALUES_FILTER': null_values_filter,
'OUT_OF_BOUNDS_FILTER': out_of_bounds_filter
'OUT_OF_BOUNDS_FILTER': out_of_bounds_filter,
}
@@ -41,11 +44,11 @@ class Gates(BaseActivity):
logger (Logger): Logger instance for operation logging
notification_handler (NotificationHandler): Handler for system notifications
"""
BaseActivity.__init__(
self, logger, notification_handler, set_error_counter=True)
BaseActivity.__init__(self, logger, notification_handler, set_error_counter=True)
def apply_aggregation(self, values: DataFrame, aggr_function: str,
metadata: dict[str, Any]) -> float | None | str:
def apply_aggregation(
self, values: DataFrame, aggr_function: str, metadata: dict[str, Any]
) -> float | None | str:
"""
Apply aggregation function to a group of time-series data.
@@ -84,7 +87,7 @@ class Gates(BaseActivity):
'avg': lambda x: x.mean(),
'mdn': lambda x: x.median(),
'max': lambda x: x.max(),
'min': lambda x: x.min()
'min': lambda x: x.min(),
}
if aggr_function in aggregation_map:
@@ -92,16 +95,16 @@ class Gates(BaseActivity):
else:
self.send_notification(
metadata=metadata,
notification_id="AGGREGATION_ISSUES",
message=f"Invalid aggregation function: {aggr_function}",
block="aggregate_data",
notification_id='AGGREGATION_ISSUES',
message=f'Invalid aggregation function: {aggr_function}',
block='aggregate_data',
level=NotificationLevel.ERROR,
attachment_content=traceback.format_exc()
attachment_content=traceback.format_exc(),
)
return 'continue'
@activity.defn(name="aggregate_data")
async def aggregate_data(self, input_data: dict[str, Any]) -> dict[str, Any]:
@activity.defn(name='aggregate_data')
async def aggregate_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
"""
Aggregate time-series data by tag and name using specified functions.
@@ -116,7 +119,7 @@ class Gates(BaseActivity):
- model_tags (dict[str, Any]): Tag configuration with aggregation functions
Returns:
dict[str, Any]: Aggregated data organized by tag and name
dict[Hashable, Any]: Aggregated data organized by tag and name
Raises:
Exception: If aggregation operation fails
@@ -128,10 +131,7 @@ class Gates(BaseActivity):
# Convert input data to DataFrame
df = DataFrame(input_data['data'])
self.info(
f"Aggregating time series data for {len(df)} rows",
metadata=metadata
)
self.info(f'Aggregating time series data for {len(df)} rows', metadata=metadata)
# Sort once by timestamp for all data (more efficient than sorting each group)
df = df.sort_values(['tag', 'name', 'timestamp'])
@@ -147,49 +147,34 @@ class Gates(BaseActivity):
results = []
for (tag, name), group in grouped:
# Get the aggregation function from model_tags
aggr_function = model_tags.get(
name, {}).get('aggr_func', 'lts')
aggr_function = model_tags.get(name, {}).get('aggr_func', 'lts')
# Get the latest timestamp (last row since data is sorted)
latest_timestamp = group['timestamp'].iloc[-1]
aggr_value = self.apply_aggregation(
group, aggr_function, metadata)
aggr_value = self.apply_aggregation(group, aggr_function, metadata)
if aggr_value == 'continue':
continue
# Batch debug logging to reduce overhead
if self.logger.level <= 10: # DEBUG level
self.debug(
f"Processed {tag}_{name}: value={aggr_value}, "
f"timestamp={latest_timestamp}, func={aggr_function}",
metadata=metadata
)
# Store the result directly in list for better performance
results.append({
'tag': tag,
'name': name,
'value': aggr_value,
'timestamp': latest_timestamp,
'aggregation_function': aggr_function
})
results.append(
{
'tag': tag,
'name': name,
'value': aggr_value,
'timestamp': latest_timestamp,
'aggregation_function': aggr_function,
}
)
self.info(
f"Aggregated data has {len(results)} rows",
metadata=metadata
)
self.info(f'Aggregated data has {len(results)} rows', metadata=metadata)
# Convert to DataFrame only once at the end if we have results
if results:
result_df = DataFrame(results)
if self.logger.level <= 10: # DEBUG level
self.debug(
f"Final aggregated data:\n{result_df.to_string()}",
metadata=metadata
)
self.debug(f'Final aggregated data:\n{result_df.to_string()}', metadata=metadata)
return result_df.to_dict()
else:
@@ -201,18 +186,18 @@ class Gates(BaseActivity):
self.send_notification(
metadata=metadata,
notification_id="AGGREGATION_ISSUES",
message=f"Error aggregating data: {e}",
block="aggregate_data",
notification_id='AGGREGATION_ISSUES',
message=f'Error aggregating data: {e}',
block='aggregate_data',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
self.error(trace, metadata=metadata)
raise e
@activity.defn(name="data_quality_gate")
async def data_quality_gate(self, input_data: dict[str, Any]) -> dict[str, Any]:
@activity.defn(name='data_quality_gate')
async def data_quality_gate(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
"""
Apply data quality filters to incoming data.
@@ -228,7 +213,7 @@ class Gates(BaseActivity):
- model_tags (dict[str, Any]): Tag-specific validation rules
Returns:
dict[str, Any]: Filtered data that passes quality validation
dict[Hashable, Any]: Filtered data that passes quality validation
Raises:
Exception: If quality validation fails
@@ -240,10 +225,7 @@ class Gates(BaseActivity):
data = DataFrame(input_data['data'])
model_tags = input_data['model_tags']
self.info(
f"Applying quality gate to data to {len(data)} rows",
metadata=metadata
)
self.info(f'Applying quality gate to data to {len(data)} rows', metadata=metadata)
tags = list(model_tags.keys())
@@ -252,25 +234,21 @@ class Gates(BaseActivity):
for filter_name, config in filters.items():
policy = config['policy']
if filter_name not in quality_gate_filters:
self.warning(
f"Filter {filter_name} not found",
metadata=metadata
)
self.warning(f'Filter {filter_name} not found', metadata=metadata)
continue
try:
filtered_data = quality_gate_filters[filter_name](
data, model_tags)
filtered_data = quality_gate_filters[filter_name](data, model_tags)
except Exception as e:
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id="DATA_QUALITY_GATE_ISSUES",
message=f"Error applying filter {filter_name}: {e}",
block="data_quality_gate",
notification_id='DATA_QUALITY_GATE_ISSUES',
message=f'Error applying filter {filter_name}: {e}',
block='data_quality_gate',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
self.error(trace, metadata=metadata)
@@ -279,30 +257,27 @@ class Gates(BaseActivity):
if filtered_data.empty:
continue
message = f"{len(filtered_data)} rows has quality issues: {filter_name}: {policy}"
message = f'{len(filtered_data)} rows has quality issues: {filter_name}: {policy}'
attachment = filtered_data.to_string()
self.send_notification(
metadata=metadata,
notification_id=f"DATA_QUALITY_GATE_ISSUES__{filter_name}",
notification_id=f'DATA_QUALITY_GATE_ISSUES__{filter_name}',
message=message,
block="data_quality_gate",
block='data_quality_gate',
level=NotificationLevel.WARNING,
attachment_content=attachment
attachment_content=attachment,
)
if policy == "DISCARD":
if policy == 'DISCARD':
data = data[~data.index.isin(filtered_data.index)]
self.info(
f"Data quality gate applied, final data has {len(data)} rows",
metadata=metadata
)
self.info(f'Data quality gate applied, final data has {len(data)} rows', metadata=metadata)
return data.to_dict()
@activity.defn(name="write_metrics")
async def write_metrics(self, input_data: dict[str, Any]):
@activity.defn(name='write_metrics')
async def write_metrics(self, input_data: dict[str, Any]) -> None:
"""
Write metrics to the database.
input_data:
@@ -310,18 +285,12 @@ class Gates(BaseActivity):
"""
metadata = input_data['metadata']
self.info(
f"Writing metrics for {metadata['model_name']}",
metadata=metadata
)
self.info(f'Writing metrics for {metadata["model_name"]}', metadata=metadata)
metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name']
pipeline_name=metadata['workflow_name'],
).inc()
self.info(
f"Metrics written for {metadata['model_name']}",
metadata=metadata
)
self.info(f'Metrics written for {metadata["model_name"]}', metadata=metadata)