SIENTIAPDE-1005

Implement workflows for fake data generation, scouter processing, and core scouter operations

- Added `FakeData` workflow to generate random data and send it to a Kafka topic.
- Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow.
- Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL.
- Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling.
- Enhanced Redis and Postgres activities with robust testing for data handling and error notifications.
- Introduced quality filters for data validation and implemented tests to verify their functionality.

scouter/activities/postgres.py

@@ -2,7 +2,9 @@ import traceback
 from temporalio import workflow, activity
 
 with workflow.unsafe.imports_passed_through():
-    from psycopg2.pool import ThreadedConnectionPool
+    from sqlalchemy import create_engine
+    from sqlalchemy.orm import sessionmaker
+    from sqlalchemy.pool import QueuePool
     from pandas import DataFrame
     from logging import Logger
     from sientia_do.notifications.handlers import NotificationHandler
@@ -22,19 +24,20 @@ class Postgres(BaseActivity):
         self.password = password
         self.dbname = dbname
 
-        self.pool = ThreadedConnectionPool(
-            minconn=min_connections,
-            maxconn=max_connections,
-            host=self.host,
-            port=self.port,
-            user=self.user,
-            password=self.password,
-            dbname=self.dbname)
+        # Create SQLAlchemy engine with connection pooling
+        self.engine = create_engine(
+            f'postgresql://{user}:{password}@{host}:{port}/{dbname}',
+            poolclass=QueuePool,
+            pool_size=min_connections,
+            max_overflow=max_connections - min_connections,
+            pool_pre_ping=True
+        )
+        self.session_factory = sessionmaker(bind=self.engine)
 
-        super().__init__(logger, notification_handler)
+        BaseActivity.__init__(self, logger, notification_handler)
 
     def close(self):
-        self.pool.closeall()
+        self.engine.dispose()
 
     def __del__(self):
         self.close()
@@ -51,28 +54,32 @@ class Postgres(BaseActivity):
                 data (DataFrame): The data to export.
         """
 
+        self.logger.debug(
+            f"Exporting data to postgres: {input_data['data']}")
+
         schema = input_data["schema"]
         table_name = input_data["table_name"]
         data = DataFrame(input_data["data"])
 
-        conn = self.pool.getconn()
+        with self.session_factory() as session:
+            try:
+                data.to_sql(table_name, self.engine, schema=schema,
+                            if_exists="append", index=False)
+                session.commit()
 
-        try:
-            data.to_sql(table_name, conn, schema=schema,
-                        if_exists="append", index=False)
-            conn.commit()
+            except Exception as e:
+                trace = traceback.format_exc()
+                self.notification_handler.build_and_send_notification(
+                    notification_id="ERROR_EXPORTING_DATA_TO_POSTGRES",
+                    message=f"Error exporting data to postgres: {e}",
+                    block="export_data_to_postgres",
+                    level=NotificationLevel.ERROR,
+                    attachment_content=trace
+                )
 
-        except Exception as e:
-            trace = traceback.format_exc()
-            self.notification_handler.build_and_send_notification(
-                notification_id="ERROR_EXPORTING_DATA_TO_POSTGRES",
-                message=f"Error exporting data to postgres: {e}",
-                block="export_data_to_postgres",
-                level=NotificationLevel.ERROR,
-                attachment_content=trace
-            )
+                self.logger.error(trace)
 
-            self.logger.error(trace)
-
-        finally:
-            self.pool.putconn(conn)
+            else:
+                self.logger.debug("Data exported to postgres")
+            finally:
+                session.close()