SIENTIAPDE-1005

Full coverage

Refactor code structure and improve logging configuration; update tests for activities and connectors

scouter/activities/gates.py

@@ -96,9 +96,6 @@ class Gates(BaseActivity):
                 # Get the latest timestamp
                 latest_timestamp = group['timestamp'].max()
 
-                if group.empty:
-                    continue
-
                 aggr_value = self.apply_aggregation(group, aggr_function)
 
                 if aggr_value == 'continue':

scouter/activities/redis.py

@@ -64,11 +64,7 @@ class Redis(BaseActivity):
         for _, row in data.iterrows():
             value = row['value']
 
-            if value is None:
-                data_hold[row['name']] = np.nan
-
-            else:
-                data_hold[row['name']] = value
+            data_hold[row['name']] = value
 
         data_hold['timestamp'] = data['timestamp'].max() if not data.empty else \
             datetime.now().strftime("%Y-%m-%d %H:%M:%S")

scouter/utils/connectors_config.py

@@ -0,0 +1,28 @@
+from os import getenv
+
+
+def build_postgres_config():
+    return {
+        'host': getenv('POSTGRES_HOST', 'localhost'),
+        'port': int(getenv('POSTGRES_PORT', '5432')),
+        'user': getenv('POSTGRES_USER', 'sientia'),
+        'password': getenv('POSTGRES_PASSWORD', 'sientia'),
+        'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
+        'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
+        'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20'))
+    }
+
+
+def build_kafka_config():
+    return {
+        'bootstrap_servers': getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092'),
+        'polling_time': int(getenv('KAFKA_POLLING_TIME', '1000')),
+        'group_id': 'scouter-group'
+    }
+
+
+def build_redis_config():
+    return {
+        'host': getenv('REDIS_HOST', 'localhost'),
+        'port': int(getenv('REDIS_PORT', '6379')),
+    }

scouter/utils/logger.py

@@ -0,0 +1,22 @@
+from os import getenv
+import logging
+import sys
+
+
+def get_logger(name: str):
+    log_level = getenv('LOG_LEVEL', 'INFO').upper()
+
+    logger = logging.getLogger(name)
+    logger.setLevel(log_level)
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setLevel(log_level)
+
+    stream_handler.setFormatter(
+        logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+    )
+
+    logger.addHandler(stream_handler)
+
+    return logger

scouter/worker/worker.py

@@ -3,7 +3,6 @@ from temporalio.worker import Worker
 
 with workflow.unsafe.imports_passed_through():
     import os
-    import logging
     from sientia_do.notifications.handlers import NotificationHandler
     from scouter.activities.activities import Activities
     from scouter.workflow.scouter import Scouter
@@ -11,53 +10,18 @@ with workflow.unsafe.imports_passed_through():
     from scouter.workflow.fake_data import FakeData
     from scouter.activities.faker import Faker
     import asyncio
-    import sys
-
-
-def build_postgres_config():
-    return {
-        'host': os.getenv('POSTGRES_HOST', 'localhost'),
-        'port': int(os.getenv('POSTGRES_PORT', '5432')),
-        'user': os.getenv('POSTGRES_USER', 'sientia'),
-        'password': os.getenv('POSTGRES_PASSWORD', 'sientia'),
-        'dbname': os.getenv('POSTGRES_DBNAME', 'sientia'),
-        'min_connections': int(os.getenv('POSTGRES_MIN_CONNECTIONS', '5')),
-        'max_connections': int(os.getenv('POSTGRES_MAX_CONNECTIONS', '20'))
-    }
-
-
-def build_kafka_config():
-    return {
-        'bootstrap_servers': os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092'),
-        'polling_time': int(os.getenv('KAFKA_POLLING_TIME', '1000')),
-        'group_id': 'scouter-group'
-    }
-
-
-def build_redis_config():
-    return {
-        'host': os.getenv('REDIS_HOST', 'localhost'),
-        'port': int(os.getenv('REDIS_PORT', '6379')),
-    }
+    from scouter.utils.logger import get_logger
+    from scouter.utils.connectors_config import (
+        build_postgres_config,
+        build_kafka_config,
+        build_redis_config
+    )
 
 
 async def main():
-
-    log_level = os.getenv('LOG_LEVEL', 'INFO').upper()
-
     host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
-    logger = logging.getLogger(__name__)
-    logger.setLevel(log_level)
-    stream_handler = logging.StreamHandler(sys.stdout)
-    stream_handler.setLevel(log_level)
 
-    stream_handler.setFormatter(
-        logging.Formatter(
-            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-        )
-    )
-
-    logger.addHandler(stream_handler)
+    logger = get_logger(__name__)
 
     logger.info('Starting Worker...')