SIENTIAPDE-1005

Enhance README and configuration files; add Redis authentication and update workflows

scouter/activities/redis.py

@@ -8,20 +8,24 @@ with workflow.unsafe.imports_passed_through():
     import json
     from typing import Any
     from pandas import DataFrame
-    import numpy as np
     from datetime import datetime
 
 
 class Redis(BaseActivity):
     def __init__(self, host: str, port: int,
+                 username: str, password: str,
                  logger: Logger, notification_handler: NotificationHandler):
         self.host = host
         self.port = port
+        self.username = username
+        self.password = password
 
         self.redis_client = redis.Redis(
             host=self.host,
             port=self.port,
-            decode_responses=True
+            decode_responses=True,
+            username=self.username,
+            password=self.password
         )
 
         BaseActivity.__init__(self, logger, notification_handler)

scouter/utils/connectors_config.py

@@ -25,4 +25,6 @@ def build_redis_config():
     return {
         'host': getenv('REDIS_HOST', 'localhost'),
         'port': int(getenv('REDIS_PORT', '6379')),
+        'username': getenv('REDIS_USERNAME', None),
+        'password': getenv('REDIS_PASSWORD', None)
     }

scouter/workflow/fake_data.py

@@ -17,8 +17,6 @@ class FakeData:
         Args:
             workflow_input (dict[str, Any]): The input data containing:
                 topic (str): The Kafka topic to send data to
-                num_messages (int, optional): Number of messages to generate. 
-                Defaults to random.randint(1, len(self.tags)).
         """
         await workflow.execute_activity_method(
             Faker.generate_and_send_data,

scouter/workflow/scouter.py

@@ -18,7 +18,6 @@ class Scouter:
         Args:
             input_data (dict[str, Any]): The data to process. Contains:
                 topic (str): The topic to load data from.
-                workflow_name (str): The name of the workflow.
                 schedule_name (str): The name of the schedule.
                 model_name (str): The name of the model.
                 model_id (str): The id of the model.
@@ -27,9 +26,22 @@ class Scouter:
                 schema (str): The schema of the table to export data to.
                 table_name (str): The name of the table to export data to.
                 retention_time (int): The retention time for data in redis in seconds.
-                model_tags (dict[str, Any]): The tags of the model. And it's respective configuration.
+                model_tags (dict[str, Any]): The tags of the model.
+                    And it's respective configuration.
         """
 
+        input_data['workflow_name'] = 'scouter'
+
+        await workflow.execute_local_activity_method(
+            Activities.prepare_activity,
+            {
+                'workflow_name': input_data['workflow_name'],
+                'schedule_name': input_data['schedule_name'],
+                'model_name': input_data['model_name'],
+                'model_id': input_data['model_id']
+            }
+        )
+
         data = await workflow.execute_activity_method(
             Activities.load_from_kafka,
             {