SIENTIAPDE-1005

Implement workflows for fake data generation, scouter processing, and core scouter operations

- Added `FakeData` workflow to generate random data and send it to a Kafka topic.
- Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow.
- Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL.
- Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling.
- Enhanced Redis and Postgres activities with robust testing for data handling and error notifications.
- Introduced quality filters for data validation and implemented tests to verify their functionality.

scouter/utils/policies.py

@@ -0,0 +1,9 @@
+from temporalio.common import RetryPolicy
+from datetime import timedelta
+
+retry_policy = RetryPolicy(
+    initial_interval=timedelta(seconds=1),
+    backoff_coefficient=2.0,
+    maximum_interval=timedelta(minutes=1),
+    maximum_attempts=1
+)

scouter/utils/quality/filters.py

@@ -1,8 +1,20 @@
 from pandas import DataFrame
+import numpy as np
+from typing import Any
 
 
-def check_data_range(value, val_range: list) -> bool:
-    if not value:
+def check_data_range(value: float | int | None, val_range: list) -> bool:
+    """
+    Check if a value is out of a given range.
+
+    Args:
+        value (float | int | None): The value to check.
+        val_range (list): The range to check against.
+
+    Returns:
+        bool: True if the value is out of the range, False otherwise.
+    """
+    if value is None or np.isnan(value):
         return True
 
     bottom = val_range[0]
@@ -11,11 +23,32 @@ def check_data_range(value, val_range: list) -> bool:
     return value < bottom or value > up
 
 
-def out_of_bounds_filter(df: DataFrame, nodes_data_range: dict):
+def out_of_bounds_filter(df: DataFrame, model_tags: dict[str, Any]):
+    """
+    Filter out rows where the value is out of the range.
+
+    Args:
+        df (DataFrame): The DataFrame to filter.
+        model_tags (dict[str, Any]): The model tags. Contains
+        the data_range for each tag. If the tag does not have a data_range,
+        it will be considered as (-inf, inf).
+
+    Returns:
+        DataFrame: The filtered DataFrame.
+    """
     return df[df.apply(lambda x: check_data_range(
-        x['value'], nodes_data_range[x['tag']]),
+        x['value'], model_tags[x['name']].get('data_range', (-np.inf, np.inf))),
         axis=1)]
 
 
-def null_values_filter(df: DataFrame):
+def null_values_filter(df: DataFrame, _model_tags: dict[str, Any]):
+    """
+    Filter out rows where the value is null.
+
+    Args:
+        df (DataFrame): The DataFrame to filter.
+
+    Returns:
+        DataFrame: The filtered DataFrame.
+    """
     return df[df['value'].isnull()]