SIENTIAPDE-1005

Implement workflows for fake data generation, scouter processing, and core scouter operations

- Added `FakeData` workflow to generate random data and send it to a Kafka topic.
- Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow.
- Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL.
- Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling.
- Enhanced Redis and Postgres activities with robust testing for data handling and error notifications.
- Introduced quality filters for data validation and implemented tests to verify their functionality.

scouter/workflow/sub_workflows/core_scouter.py

@@ -0,0 +1,84 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from scouter.activities.activities import Activities
+    from typing import Any
+    from datetime import timedelta
+    from scouter.utils.policies import retry_policy
+
+
+@workflow.defn(name="core_scouter")
+class CoreScouter:
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]):
+        """
+        Core scouter workflow. Passes data through data_quality_gate,
+        group_and_hold_data, and then asynchronously exports data to postgres
+        using export_data_to_postgres and in the future will trigger_laborious
+        if needed.
+
+        Args:
+            input_data (dict[str, Any]): The data to process. Contains:
+                workflow_name (str): The name of the workflow.
+                schedule_name (str): The name of the schedule.
+                model_name (str): The name of the model.
+                model_id (str): The id of the model.
+                data (dict[str, Any]): The data to process.
+                trigger_laborious (bool): Whether to trigger laborious.
+                filters (dict[str, str]): The filters to apply.
+                schema (str): The schema of the table to export data to.
+                table_name (str): The name of the table to export data to.
+                retention_time (int): The retention time for data in redis in seconds.
+        """
+
+        filtered_data = await workflow.execute_local_activity_method(
+            Activities.data_quality_gate,
+            {
+                'filters': input_data['filters'],
+                'data': input_data['data'],
+                'model_tags': input_data['model_tags']
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=60)
+        )
+
+        grouped_data = await workflow.execute_local_activity_method(
+            Activities.aggregate_data,
+            {
+                'data': filtered_data,
+                'model_tags': input_data['model_tags']
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=60)
+        )
+
+        held_data = await workflow.execute_local_activity_method(
+            Activities.group_and_hold_data,
+            {
+                'workflow_name': input_data['workflow_name'],
+                'schedule_name': input_data['schedule_name'],
+                'data': grouped_data,
+                'model_id': input_data['model_id'],
+                'retention_time': input_data['retention_time']
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=60)
+        )
+
+        if held_data == {}:
+            return
+
+        async_export = workflow.execute_activity_method(
+            Activities.export_data_to_postgres,
+            {
+                'schema': input_data['schema'],
+                'table_name': input_data['table_name'],
+                'data': held_data
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=60)
+        )
+
+        # TODO: Trigger laborious if needed
+
+        await async_export