SIENTIAPDE-1222

Refactor orchestration and email handling

- Updated the orchestration queries to streamline model lookups and added conditions for active models.
- Changed the entry point in the local run script to use the worker module.
- Removed the deprecated samples.json file.
- Enhanced the Email activity to skip sending if the SMTP server is not configured.
- Added timestamp field handling in MongoDB activities for better data management.
- Updated connectors configuration to allow for a None SMTP server.
- Improved the common configuration function to include model configuration details.

orchestrator/activities/email.py

@@ -48,11 +48,12 @@ class Email(BaseActivity):
 
         logger.info(f"Initializing Email with {smtp_server}:{smtp_port}")
 
-        self.server = smtplib.SMTP(smtp_server, smtp_port, timeout=20)
+        if smtp_server is not None:
+            self.server = smtplib.SMTP(smtp_server, smtp_port, timeout=20)
 
-        if self.sender_password:
-            self.server.starttls()
-            self.server.login(self.sender_email, self.sender_password)
+            if self.sender_password:
+                self.server.starttls()
+                self.server.login(self.sender_email, self.sender_password)
 
         BaseActivity.__init__(self,
                               logger=logger,
@@ -200,6 +201,11 @@ class Email(BaseActivity):
         receiver_groups = input_data['receiver_groups']
         mail_type = input_data['mail_type']
 
+        if self.smtp_server is None:
+            self.info(f"Skipping email sending for {mail_type} mail type.",
+                      metadata=metadata)
+            return {}
+
         self.info(f"Sending email for {mail_type} mail type.",
                   metadata=metadata)
 

orchestrator/activities/formatters.py

@@ -1,19 +1,15 @@
-
-from pandas import DataFrame
 from temporalio import activity, workflow
 
-from orchestrator.utils.orchestrator_functions import minimal_retrain
-
-
 with workflow.unsafe.imports_passed_through():
     import json
     from typing import Any
     from logging import Logger
+    from pandas import DataFrame
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.temporal.activities.base import BaseActivity
     from sientia_do.notifications.models import NotificationLevel
     from orchestrator.utils.orchestrator_functions import (
-        scouter, predictions_batch, gather_read_tags, build_tag_config
+        scouter, predictions_batch, gather_read_tags, build_tag_config, minimal_retrain
     )
     from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ, now
     from math import ceil

orchestrator/activities/mongo_db.py

@@ -130,6 +130,7 @@ class MongoDB(BaseActivity):
 
         query = input_data.get("query", {})
         metadata = input_data.get("metadata", {})
+        timestamp_fields = input_data.get("timestamp_fields", [])
 
         collection_name = query.get("collection")
         if not collection_name:
@@ -146,6 +147,12 @@ class MongoDB(BaseActivity):
             self.info(
                 f"Loaded {len(documents)} documents from collection '{collection_name}'", metadata=metadata)
 
+            for document in documents:
+                for timestamp_field in timestamp_fields:
+                    if timestamp_field in document:
+                        document[timestamp_field] = document[timestamp_field].replace(tzinfo=timezone.utc).strftime(
+                            DATETIME_FORMAT_MS_WITH_TZ)
+
             self.debug(
                 f"Documents loaded: {documents}", metadata=metadata)
 
@@ -181,6 +188,7 @@ class MongoDB(BaseActivity):
 
         query = input_data.get("query", {})
         metadata = input_data.get("metadata", {})
+        timestamp_fields = input_data.get("timestamp_fields", [])
 
         collection_name = query.get("collection")
         if not collection_name:
@@ -204,6 +212,12 @@ class MongoDB(BaseActivity):
             self.info(
                 f"Aggregated {len(aggregated_documents)} documents from collection '{collection_name}'", metadata=metadata)
 
+            for document in aggregated_documents:
+                for timestamp_field in timestamp_fields:
+                    if timestamp_field in document:
+                        document[timestamp_field] = document[timestamp_field].replace(tzinfo=timezone.utc).strftime(
+                            DATETIME_FORMAT_MS_WITH_TZ)
+
             self.debug(
                 f"Aggregation result: {aggregated_documents}", metadata=metadata)
 

orchestrator/utils/connectors_config.py

@@ -92,6 +92,6 @@ def build_email_config():
     return {
         'sender_email': getenv('EMAIL_SENDER', 'sientia-alerts@aignosi.com'),
         'sender_password': getenv('EMAIL_SENDER_PASSWORD', 'sientia'),
-        'smtp_server': getenv('EMAIL_SMTP_SERVER', 'smtp.gmail.com'),
+        'smtp_server': getenv('EMAIL_SMTP_SERVER', None),
         'smtp_port': int(getenv('EMAIL_SMTP_PORT', '587'))
     }

orchestrator/utils/orchestrator_functions.py

@@ -17,6 +17,7 @@ def common_config(config: dict[str, Any]):
     Returns:
         dict[str, Any]: Common configuration dictionary with extracted parameters.
     """
+    model = config['model']
     return {
         "workflow_type": config['workflow_type'],
         "schedule_name": config['schedule_name'],
@@ -24,7 +25,8 @@ def common_config(config: dict[str, Any]):
         "max_retry_policy": config.get('max_retry_policy', 1),
 
         "model_id": config['model_id'],
-        "model_name": config['models']['name'],
+        "model_name": model['name'],
+        "model_config": model.get('model_config', {}),
     }
 
 

orchestrator/workflows/orchestrator.py

@@ -57,7 +57,8 @@ class Orchestrator:
             Activities.aggregate_documents_in_mongodb,
             {
                 **metadata,
-                'query': input_data['pipelines_query']
+                'query': input_data['pipelines_query'],
+                "timestamp_fields": ["updated_at"]
             },
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(seconds=60)
@@ -79,7 +80,8 @@ class Orchestrator:
                 **metadata,
                 'query': {
                     'collection': 'orchestrated_schedules'
-                }
+                },
+                "timestamp_fields": ["updated_at"]
             },
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(seconds=60)

orchestrator/workflows/subworkflows/process_notifications.py

@@ -69,6 +69,9 @@ class ProcessNotifications:
             retry_policy=retry_policy
         )
 
+        if not log_report:
+            return
+
         # Format the log report to a dataframe to be stored in the database
         log_report = await workflow.execute_local_activity_method(
             Activities.format_log_report,