Code import - branch feature/SIENTIAPDE-1646

tests/orchestrator/activities/test_activities.py

@@ -0,0 +1,148 @@
+from unittest.mock import ANY, MagicMock, patch
+
+from orchestrator.activities.activities import Activities
+from orchestrator.activities.formatters import Formatters
+from orchestrator.activities.mongo_db import MongoDB
+from orchestrator.activities.slot_manager import SlotManager
+from orchestrator.activities.temporal_manager import TemporalManager
+
+
+@patch('orchestrator.activities.mongo_db.MongoDB.__init__')
+@patch('orchestrator.activities.temporal_manager.TemporalManager.__init__')
+@patch('orchestrator.activities.slot_manager.SlotManager.__init__')
+@patch('orchestrator.activities.formatters.Formatters.__init__')
+@patch('orchestrator.activities.email.Email.__init__')
+@patch('sientia_do.temporal.activities.postgres_sync.Postgres.__init__')
+@patch('orchestrator.activities.activities.MetricsController')
+def test___init__(
+    mock_metrics_controller,
+    mock_postgres_init,
+    mock_email_init,
+    mock_formatters_init,
+    mock_slot_manager_init,
+    mock_temporal_manager_init,
+    mock_mongodb_init,
+):
+    mongo_db_config = {
+        'connection_string': 'mongodb://localhost:27017',
+        'database_name': 'test_db',
+        'ttl_index_seconds': 3600,
+    }
+
+    redis_config = {'host': 'localhost', 'port': 6379, 'username': 'admin', 'password': 'password'}
+
+    temporal_config = {
+        'temporal_host': 'localhost',
+        'temporal_scouter_namespace': 'scouter',
+        'temporal_laborious_namespace': 'laborious',
+    }
+
+    email_config = {
+        'sender_email': 'test@test.com',
+        'sender_password': 'test',
+        'smtp_server': 'test',
+        'smtp_port': 587,
+    }
+
+    postgres_config = {
+        'host': 'localhost',
+        'port': 5432,
+        'user': 'admin',
+        'password': 'password',
+        'dbname': 'test_db',
+        'min_connections': 1,
+        'max_connections': 10,
+    }
+
+    logger = MagicMock()
+    notification_handler = MagicMock()
+
+    activities = Activities(
+        temporal_config=temporal_config,
+        redis_config=redis_config,
+        mongodb_config=mongo_db_config,
+        email_config=email_config,
+        postgres_config=postgres_config,
+        logger=logger,
+        notification_handler=notification_handler,
+    )
+
+    assert isinstance(activities, Activities)
+    assert isinstance(activities, MongoDB)
+    assert isinstance(activities, TemporalManager)
+    assert isinstance(activities, SlotManager)
+    assert isinstance(activities, Formatters)
+
+    mock_slot_manager_init.assert_called_once_with(
+        ANY,
+        host='localhost',
+        port=6379,
+        username='admin',
+        password='password',
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+    mock_mongodb_init.assert_called_once_with(
+        ANY,
+        connection_string='mongodb://localhost:27017',
+        database_name='test_db',
+        ttl_index_seconds=3600,
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+    mock_temporal_manager_init.assert_called_once_with(
+        ANY,
+        host='localhost',
+        scouter_namespace='scouter',
+        laborious_namespace='laborious',
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+    mock_formatters_init.assert_called_once_with(
+        ANY,
+        scouter_namespace='scouter',
+        laborious_namespace='laborious',
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+
+@patch('orchestrator.activities.activities.MongoDB')
+@patch('orchestrator.activities.activities.TemporalManager')
+@patch('orchestrator.activities.activities.SlotManager')
+@patch('orchestrator.activities.activities.Formatters')
+@patch('orchestrator.activities.activities.Email')
+@patch('orchestrator.activities.activities.Postgres')
+def test_shutdown(
+    mock_mongodb,
+    mock_temporal_manager,
+    mock_slot_manager,
+    mock_formatters,
+    mock_email,
+    mock_postgres,
+):
+    activities = Activities(
+        temporal_config=MagicMock(),
+        redis_config=MagicMock(),
+        mongodb_config=MagicMock(),
+        email_config=MagicMock(),
+        postgres_config=MagicMock(),
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+    )
+
+    activities.shutdown()
+
+    mock_mongodb.close.assert_called()
+    mock_temporal_manager.close.assert_called()
+    mock_slot_manager.close.assert_called()
+    mock_formatters.close.assert_called()
+    mock_email.close.assert_called()
+    mock_postgres.close.assert_called()

tests/orchestrator/activities/test_email.py

@@ -0,0 +1,326 @@
+from smtplib import SMTPServerDisconnected
+from unittest.mock import AsyncMock, MagicMock, call, patch
+
+from pytest import fixture
+
+from orchestrator.activities.email import Email
+
+
+@fixture
+@patch('orchestrator.activities.email.EmailBuilder')
+@patch('orchestrator.activities.email.smtplib')
+def email(smtplib, email_builder):
+    email = Email(
+        sender_email='test@test.com',
+        sender_password='test',
+        smtp_server='test',
+        smtp_port=587,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+    email.send_notification = MagicMock()
+    email.emit_metric = AsyncMock()
+
+    return email
+
+
+@patch('orchestrator.activities.email.EmailBuilder')
+@patch('orchestrator.activities.email.smtplib')
+def test___init___with_password(smtplib, email_builder):
+    email = Email(
+        sender_email='test@test.com',
+        sender_password='test',
+        smtp_server='test',
+        smtp_port=587,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+
+    assert email.sender_email == 'test@test.com'
+    assert email.sender_password == 'test'
+    assert email.smtp_port == 587
+
+    smtplib.SMTP.assert_called_once_with('test', 587, timeout=20)
+    smtplib.SMTP.return_value.starttls.assert_called_once()
+    smtplib.SMTP.return_value.login.assert_called_once_with('test@test.com', 'test')
+
+    assert email.server == smtplib.SMTP.return_value
+
+
+@patch('orchestrator.activities.email.EmailBuilder')
+@patch('orchestrator.activities.email.smtplib')
+def test___init___without_password(smtplib, email_builder):
+    email = Email(
+        sender_email='test@test.com',
+        sender_password=None,
+        smtp_server='test',
+        smtp_port=587,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+
+    assert email.sender_email == 'test@test.com'
+    assert email.sender_password is None
+    assert email.smtp_port == 587
+
+    smtplib.SMTP.assert_called_once_with('test', 587, timeout=20)
+    assert email.server == smtplib.SMTP.return_value
+
+
+metadata = {
+    'metadata': {
+        'schedule_name': 'test',
+        'model_name': 'test',
+        'model_id': 'test',
+        'workflow_name': 'test',
+    }
+}
+
+
+@patch('orchestrator.activities.email.SientiaMonitoring')
+def test_close(sientia_monitoring_mock, email):
+    email.close()
+
+    email.server.quit.assert_called_once()
+    sientia_monitoring_mock.shutdown.assert_called_once()
+
+
+def test_build_email_html(email):
+    email.email_builder.build_email = MagicMock(return_value='test')
+    input_data = {
+        **metadata,
+        'receiver_groups': {
+            'group_1': {
+                'notifications': [
+                    {'type': 'test', 'subject': 'test', 'body': 'test'},
+                    {'type': 'test', 'subject': 'test', 'body': 'test'},
+                ]
+            }
+        },
+        'mail_type': 'test',
+    }
+
+    response = email.build_email_html(input_data)
+
+    assert response == {
+        'group_1': {
+            'notifications': [
+                {
+                    'type': 'test',
+                    'subject': 'test',
+                    'body': 'test',
+                },
+                {
+                    'type': 'test',
+                    'subject': 'test',
+                    'body': 'test',
+                },
+            ],
+            'html': 'test',
+        }
+    }
+
+    email.email_builder.build_email.assert_called_once_with(
+        input_data['receiver_groups']['group_1']['notifications'], input_data['mail_type']
+    )
+
+
+@patch('orchestrator.activities.email.MIMEBase')
+@patch('orchestrator.activities.email.encoders')
+def test_handle_attachments_success(encoders, mime_base, email):
+    message = MagicMock()
+
+    attachments = [
+        {'filename': 'file_1', 'attachment_content': 'test_content_1'},
+        {'filename': 'file_2', 'attachment_content': 'test_content_2'},
+        {'filename': 'file_3', 'attachment_content': 'test_content_3'},
+    ]
+
+    response = email.handle_attachments(attachments, message)
+
+    assert response == message
+
+    mime_base.assert_called_with('application', 'octet-stream')
+    assert mime_base.return_value.set_payload.call_count == 3
+
+    mime_base.return_value.set_payload.assert_has_calls(
+        [
+            call(b'test_content_1'),
+            call(b'test_content_2'),
+            call(b'test_content_3'),
+        ]
+    )
+
+    encoders.encode_base64.assert_called_with(mime_base.return_value)
+    assert encoders.encode_base64.call_count == 3
+
+    mime_base.return_value.add_header.assert_has_calls(
+        [
+            call('Content-Disposition', 'attachment; filename="file_1"'),
+            call('Content-Disposition', 'attachment; filename="file_2"'),
+            call('Content-Disposition', 'attachment; filename="file_3"'),
+        ]
+    )
+
+    message.attach.assert_called_with(mime_base.return_value)
+    assert message.attach.call_count == 3
+
+
+@patch('orchestrator.activities.email.MIMEBase')
+def test_handle_attachments_failure(mime_base, email):
+    message = MagicMock()
+
+    mime_base.side_effect = Exception('test')
+
+    attachments = [{'filename': 'file_1', 'attachment_content': 'test_content_1'}]
+
+    try:
+        email.handle_attachments(attachments, message)
+    except Exception as e:
+        assert str(e) == 'test'
+
+    assert message.attach.call_count == 0
+
+
+def test_try_send_email_success(email):
+    email.server.sendmail = MagicMock()
+
+    msg = MagicMock()
+
+    email.try_send_email(msg, 'test')
+
+    email.server.sendmail.assert_called_once_with(
+        'test@test.com', 'test', msg.as_string.return_value
+    )
+
+
+@patch('orchestrator.activities.email.smtplib.SMTP')
+def test_try_send_email_reconnect_quit_success(smtp, email):
+    email.server.sendmail = MagicMock(side_effect=SMTPServerDisconnected('test'))
+    email.server.quit = MagicMock()
+
+    msg = MagicMock()
+
+    email.try_send_email(msg, 'test')
+
+    smtp.assert_has_calls([call('test', 587, timeout=20)])
+
+    smtp.return_value.starttls.assert_called_once()
+    smtp.return_value.login.assert_called_once_with('test@test.com', 'test')
+
+    smtp.return_value.sendmail.assert_called_once_with(
+        'test@test.com', 'test', msg.as_string.return_value
+    )
+
+
+@patch('orchestrator.activities.email.smtplib.SMTP')
+def test_try_send_email_reconnect_quit_failure_disconnect(smtp, email):
+    email.server.sendmail = MagicMock(side_effect=SMTPServerDisconnected('test'))
+    email.server.quit = MagicMock(side_effect=SMTPServerDisconnected('test'))
+
+    msg = MagicMock()
+
+    email.try_send_email(msg, 'test')
+
+    smtp.assert_has_calls([call('test', 587, timeout=20)])
+    smtp.return_value.starttls.assert_called_once()
+    smtp.return_value.login.assert_called_once_with('test@test.com', 'test')
+
+    smtp.return_value.sendmail.assert_called_once_with(
+        'test@test.com', 'test', msg.as_string.return_value
+    )
+
+
+@patch('orchestrator.activities.email.smtplib.SMTP')
+def test_try_send_email_reconnect_quit_failure(smtp, email):
+    email.server.sendmail = MagicMock(side_effect=SMTPServerDisconnected('test'))
+    email.server.quit = MagicMock(side_effect=Exception('test'))
+
+    msg = MagicMock()
+
+    try:
+        email.try_send_email(msg, 'test')
+    except Exception as e:
+        assert str(e) == 'test'
+    else:
+        raise AssertionError('Expected exception')
+
+
+def test_send_email_without_smtp_server(email):
+    email.smtp_server = None
+    input_data = {**metadata, 'receiver_groups': {}, 'mail_type': 'test_TYPE'}
+    response = email.send_email(input_data)
+
+    assert response == {}
+
+
+@patch('orchestrator.activities.email.MIMEText')
+@patch('orchestrator.activities.email.MIMEMultipart')
+def test_send_email(mimemultipart, mimetext, email):
+    side_effect_1 = MagicMock()
+    side_effect_2 = MagicMock()
+    mimemultipart.side_effect = [side_effect_1, side_effect_2]
+
+    email.email_builder.send_notification = MagicMock()
+
+    email.try_send_email = MagicMock(side_effect=[None, Exception('test')])
+
+    input_data = {
+        **metadata,
+        'receiver_groups': {
+            'group_1': {
+                'members': ['test@test.com', 'test2@test.com'],
+                'notifications': [
+                    {
+                        'attachment_content': 'test_content_1',
+                        'trigger': 'test_trigger',
+                        'notification_id': 'test_notification_id',
+                    }
+                ],
+                'html': 'test_html1',
+            },
+            'group_2': {
+                'members': ['test3@test.com', 'test4@test.com'],
+                'notifications': [],
+                'html': 'test_html2',
+            },
+        },
+        'mail_type': 'test_TYPE',
+    }
+
+    response = email.send_email(input_data)
+
+    assert response['group_1']['status'] == 'sent'
+    assert response['group_2']['status'] == 'failed'
+
+    assert mimemultipart.call_count == 2
+
+    mimetext.assert_has_calls([call('test_html1', 'html'), call('test_html2', 'html')])
+
+    side_effect_1.__setitem__.assert_has_calls(
+        [
+            call('From', 'test@test.com'),
+            call('To', 'test@test.com, test2@test.com'),
+            call('Subject', 'SIENTIA™ test_TYPE'),
+        ]
+    )
+
+    side_effect_2.__setitem__.assert_has_calls(
+        [
+            call('From', 'test@test.com'),
+            call('To', 'test3@test.com, test4@test.com'),
+            call('Subject', 'SIENTIA™ test_TYPE'),
+        ]
+    )
+
+    email.try_send_email.assert_has_calls(
+        [
+            call(side_effect_1, 'test@test.com, test2@test.com'),
+            call(side_effect_2, 'test3@test.com, test4@test.com'),
+        ]
+    )
+
+    assert email.try_send_email.call_count == 2

tests/orchestrator/activities/test_formatters.py

@@ -0,0 +1,768 @@
+import json
+from unittest.mock import AsyncMock, MagicMock, call, patch
+
+from pandas import DataFrame
+from pytest import fixture
+from sientia_do.notifications.models import NotificationLevel
+
+from orchestrator.activities.formatters import Formatters
+
+
+@fixture
+def formatters():
+    formatters = Formatters(
+        scouter_namespace='scouter',
+        laborious_namespace='laborious',
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+
+    formatters.send_notification = MagicMock()
+    formatters.emit_metric = AsyncMock()
+    formatters.error = MagicMock()
+    formatters.info = MagicMock()
+    formatters.debug = MagicMock()
+    return formatters
+
+
+metadata = {
+    'metadata': {
+        'schedule_name': 'test_schedule_name',
+        'workflow_name': 'test_workflow_name',
+        'model_name': 'test_model_name',
+        'model_id': 'test_model_id',
+    }
+}
+
+
+def test_process_schedules(formatters):
+    mock_scouter = MagicMock(return_value={'test_scouter': 'test_scouter'})
+    mock_predictions_batch = MagicMock(
+        return_value={'test_predictions_batch': 'test_predictions_batch'}
+    )
+    mock_minimal_retrain = MagicMock(return_value={'test_minimal_retrain': 'test_minimal_retrain'})
+    mock_drift = MagicMock(return_value={'test_drift': 'test_drift'})
+    mock_simple_metrics = MagicMock(return_value={'test_simple_metrics': 'test_simple_metrics'})
+
+    mock_schedule_types = {
+        'scouter': {
+            'namespace': 'scouter',
+            'function': mock_scouter,
+        },
+        'pi_web_api_scouter': {
+            'namespace': 'scouter',
+            'function': mock_scouter,
+        },
+        'predictions_batch': {
+            'namespace': 'laborious',
+            'function': mock_predictions_batch,
+        },
+        'minimal_retrain': {
+            'namespace': 'laborious',
+            'function': mock_minimal_retrain,
+        },
+        'drift': {
+            'namespace': 'laborious',
+            'function': mock_drift,
+        },
+        'simple_metrics': {
+            'namespace': 'laborious',
+            'function': mock_simple_metrics,
+        },
+    }
+
+    input_data = {
+        'pipelines': [
+            {
+                'schedule_name': 'test_schedule_name',
+                'workflow_type': 'scouter',
+                'model_name': 'test_model_name',
+                'model_id': 'test_model_id',
+                'updated_at': '2021-01-01',
+            },
+            {
+                'schedule_name': 'test_schedule_name2',
+                'workflow_type': 'predictions_batch',
+                'model_name': 'test_model_name',
+                'model_id': 'test_model_id',
+                'updated_at': '2021-01-02',
+            },
+            {
+                'schedule_name': 'test_schedule_name3',
+                'workflow_type': 'minimal_retrain',
+                'model_name': 'test_model_name',
+                'model_id': 'test_model_id',
+                'updated_at': '2021-01-03',
+            },
+            {
+                'schedule_name': 'test_schedule_name4',
+                'workflow_type': 'drift',
+                'model_name': 'test_model_name',
+                'model_id': 'test_model_id',
+                'updated_at': '2021-01-04',
+            },
+            {
+                'schedule_name': 'test_schedule_name5',
+                'workflow_type': 'simple_metrics',
+                'model_name': 'test_model_name',
+                'model_id': 'test_model_id',
+                'updated_at': '2021-01-05',
+            },
+        ]
+    }
+
+    with patch('orchestrator.activities.formatters.schedule_types', mock_schedule_types):
+        result = formatters.process_schedules(input_data)
+
+    assert result == {
+        'scouter': {
+            'test_schedule_name': {'test_scouter': 'test_scouter', 'updated_at': '2021-01-01'}
+        },
+        'laborious': {
+            'test_schedule_name2': {
+                'test_predictions_batch': 'test_predictions_batch',
+                'updated_at': '2021-01-02',
+            },
+            'test_schedule_name3': {
+                'test_minimal_retrain': 'test_minimal_retrain',
+                'updated_at': '2021-01-03',
+            },
+            'test_schedule_name4': {
+                'test_drift': 'test_drift',
+                'updated_at': '2021-01-04',
+            },
+            'test_schedule_name5': {
+                'test_simple_metrics': 'test_simple_metrics',
+                'updated_at': '2021-01-05',
+            },
+        },
+    }
+
+    mock_scouter.assert_called_once_with(input_data['pipelines'][0])
+    mock_predictions_batch.assert_called_once_with(input_data['pipelines'][1])
+    mock_minimal_retrain.assert_called_once_with(input_data['pipelines'][2])
+    mock_drift.assert_called_once_with(input_data['pipelines'][3])
+    mock_simple_metrics.assert_called_once_with(input_data['pipelines'][4])
+
+
+def test_process_schedules_with_invalid_workflow_type(formatters):
+    """Test that process_schedules handles invalid workflow types correctly"""
+    mock_scouter = MagicMock(return_value={'test_scouter': 'test_scouter'})
+
+    mock_schedule_types = {
+        'scouter': {
+            'namespace': 'scouter',
+            'function': mock_scouter,
+        },
+    }
+
+    pipelines = [
+        {
+            'schedule_name': 'test_schedule_name_valid',
+            'workflow_type': 'scouter',
+            'model_name': 'test_model_name',
+            'model_id': 'test_model_id',
+            'updated_at': '2021-01-01',
+        },
+        {
+            'schedule_name': 'test_schedule_name_invalid',
+            'workflow_type': 'invalid_workflow_type',
+            'model_name': 'test_model_name',
+            'model_id': 'test_model_id',
+            'updated_at': '2021-01-02',
+        },
+        {
+            'schedule_name': 'test_schedule_name_valid2',
+            'workflow_type': 'scouter',
+            'model_name': 'test_model_name',
+            'model_id': 'test_model_id',
+            'updated_at': '2021-01-03',
+        },
+    ]
+
+    input_data = {
+        'pipelines': pipelines,
+        'metadata': {
+            'schedule_name': 'test_schedule',
+            'workflow_name': 'test_workflow',
+        },
+    }
+
+    with patch('orchestrator.activities.formatters.schedule_types', mock_schedule_types):
+        result = formatters.process_schedules(input_data)
+
+    # Assert that error was called for invalid workflow type
+    formatters.error.assert_called_once_with(
+        'Workflow type invalid_workflow_type not supported', metadata=input_data['metadata']
+    )
+
+    # Assert that only valid pipelines were processed
+    assert result == {
+        'scouter': {
+            'test_schedule_name_valid': {
+                'test_scouter': 'test_scouter',
+                'updated_at': '2021-01-01',
+            },
+            'test_schedule_name_valid2': {
+                'test_scouter': 'test_scouter',
+                'updated_at': '2021-01-03',
+            },
+        },
+        'laborious': {},
+    }
+
+    # Assert that the mock function was called only for valid pipelines
+    assert mock_scouter.call_count == 2
+    mock_scouter.assert_any_call(pipelines[0])
+    mock_scouter.assert_any_call(pipelines[2])
+
+
+@patch(
+    'orchestrator.activities.formatters.gather_read_tags',
+    return_value={
+        '1:test_tag_address': {
+            'server_id': '1',
+            'server_name': 'test_server_name',
+            'tag_address': 'test_tag_address',
+            'topics': ['raw_test_schedule'],
+            'frequency': 1000,
+        },
+        '2:test_tag_address2': {
+            'server_id': '2',
+            'server_name': 'test_server_name2',
+            'tag_address': 'test_tag_address2',
+            'topics': ['raw_test_schedule2'],
+            'frequency': 1000,
+        },
+        '2:test_tag_address3': {
+            'server_id': '2',
+            'server_name': 'test_server_name2',
+            'tag_address': 'test_tag_address3',
+            'topics': ['raw_test_schedule2'],
+            'frequency': 1000,
+        },
+    },
+)
+@patch('orchestrator.activities.formatters.build_tag_config')
+def test_process_slots(mock_build_tag_config, mock_gather_read_tags, formatters):
+    input_data = {
+        'opc_servers': [
+            {'id': '1', 'server_name': 'test_server_name', 'url': 'test_url', 'uri': 'test_uri'},
+            {'id': '2', 'server_name': 'test_server_name2', 'url': 'test_url2', 'uri': 'test_uri2'},
+        ],
+        'active_ingestors': ['test_active_ingestor1', 'test_active_ingestor2'],
+        'pipelines': 'test_gather_read_tags',
+        **metadata,
+    }
+
+    expected_opc_servers = {
+        '1': {
+            'id': '1',
+            'server_name': 'test_server_name',
+            'url': 'test_url',
+            'uri': 'test_uri',
+        },
+        '2': {
+            'id': '2',
+            'server_name': 'test_server_name2',
+            'url': 'test_url2',
+            'uri': 'test_uri2',
+        },
+    }
+
+    slot_mock = {
+        'test_server_name': {
+            'server_id': '1',
+            'name': 'test_server_name',
+            'url': 'test_url',
+            'server_uri': 'test_uri',
+            'cert_path': None,
+            'private_key_path': None,
+            'server_cert_path': None,
+        }
+    }
+
+    mock_build_tag_config.return_value = (slot_mock, ['2'])
+
+    result = formatters.process_slots(input_data)
+
+    tags = list(mock_gather_read_tags.return_value.values())
+
+    mock_gather_read_tags.assert_called_once_with(input_data['pipelines'])
+
+    mock_build_tag_config.assert_has_calls(
+        [
+            call(tags[:2], expected_opc_servers),
+            call(tags[2:], expected_opc_servers),
+        ]
+    )
+
+    formatters.send_notification.assert_has_calls(
+        [
+            call(
+                metadata=metadata['metadata'],
+                notification_id='ORCHESTRATOR_SERVER_NOT_FOUND_DURING_SLOT_CONFIGURATION',
+                message='Servers 2 not found in opc_servers',
+                block='orchestrator',
+                level=NotificationLevel.ERROR,
+            ),
+            call(
+                metadata=metadata['metadata'],
+                notification_id='ORCHESTRATOR_SERVER_NOT_FOUND_DURING_SLOT_CONFIGURATION',
+                message='Servers 2 not found in opc_servers',
+                block='orchestrator',
+                level=NotificationLevel.ERROR,
+            ),
+        ]
+    )
+
+    assert result == {
+        '1': slot_mock,
+        '2': slot_mock,
+    }
+
+
+def test_format_schedule_config(formatters):
+    input_data = {
+        'schedule_config': [
+            {'namespace': 'test_namespace1', 'schedule_name': 'test1', 'updated_at': '2021-01-01'},
+            {'namespace': 'test_namespace2', 'schedule_name': 'test2', 'updated_at': '2021-01-02'},
+        ],
+        **metadata,
+    }
+
+    result = formatters.format_schedule_config(input_data)
+
+    assert result == {
+        'test_namespace1': {'test1': '2021-01-01'},
+        'test_namespace2': {'test2': '2021-01-02'},
+    }
+
+
+def test_create_schedule_config(formatters):
+    input_data = {
+        'current_schedule_config': {
+            'scouter': {
+                'test_schedule_name_to_delete': '2021-01-01',
+                'test_schedule_name_to_update': '2021-01-02',
+            }
+        },
+        'schedule_config': {
+            'laborious': {
+                'test_schedule_name_to_create': {
+                    'frequency': 60,
+                    'data': {'test': 'test'},
+                    'updated_at': '2021-01-03',
+                }
+            },
+            'scouter': {
+                'test_schedule_name_to_update': {
+                    'frequency': 60,
+                    'data': {'test': 'test2'},
+                    'updated_at': '2021-01-04',
+                }
+            },
+        },
+    }
+
+    result = formatters.create_schedule_config(input_data)
+
+    assert result == {
+        'to_create': {
+            'laborious': {
+                'test_schedule_name_to_create': {
+                    'frequency': 60,
+                    'data': {'test': 'test'},
+                    'updated_at': '2021-01-03',
+                }
+            },
+            'scouter': {},
+        },
+        'to_update': {
+            'scouter': {
+                'test_schedule_name_to_update': {
+                    'frequency': 60,
+                    'data': {'test': 'test2'},
+                    'updated_at': '2021-01-04',
+                }
+            },
+            'laborious': {},
+        },
+        'to_delete': {'scouter': ['test_schedule_name_to_delete'], 'laborious': []},
+    }
+
+
+def test_create_slot_config(formatters):
+    input_data = {
+        'current_slot_config': {
+            '1': {'frequency': 60, 'data': {'test': 'test'}},
+            '2': {'frequency': 60, 'data': {'test': 'test'}},
+        },
+        'slot_config': {'1': {'frequency': 60, 'data': {'test': 'test2'}}},
+    }
+
+    result = formatters.create_slot_config(input_data)
+
+    assert result == {
+        'to_delete': ['2'],
+        'to_insert': {'1': {'frequency': 60, 'data': {'test': 'test2'}}},
+    }
+
+
+def test_send_success_report(formatters):
+    formatters.send_success_report(
+        metadata=metadata,
+        message='test_message',
+        notification_id='test_notification_id',
+        attachment={'test': 'test'},
+    )
+    formatters.send_notification.assert_called_once_with(
+        metadata=metadata,
+        notification_id='test_notification_id',
+        message='test_message',
+        block='report_orchestration',
+        level=NotificationLevel.INFO,
+        attachment_content=json.dumps({'test': 'test'}, indent=4, sort_keys=True),
+    )
+
+
+def test_send_error_report(formatters):
+    formatters.send_error_report(
+        metadata=metadata,
+        message='test_message',
+        notification_id='test_notification_id',
+        attachment='test_attachment',
+    )
+    formatters.send_notification.assert_called_once_with(
+        metadata=metadata,
+        notification_id='test_notification_id',
+        message='test_message',
+        block='report_orchestration',
+        level=NotificationLevel.ERROR,
+        attachment_content='test_attachment',
+    )
+
+
+def test_parse_report(formatters):
+    input_data = {
+        'test_key': {'success': True},
+        'test_key2': {'success': False, 'message': 'test_error'},
+    }
+
+    result = formatters.parse_report(input_data)
+
+    assert result == (['test_key'], ['test_key2'])
+
+
+def test_parse_report_schedule(formatters):
+    input_data = [
+        {
+            'namespace': 'test_namespace',
+            'schedule_name': 'test_schedule_name_to_create',
+            'success': True,
+        },
+        {
+            'namespace': 'test_namespace',
+            'schedule_name': 'test_schedule_name_to_create_error',
+            'success': False,
+            'message': 'test_error',
+            'attachment': 'test_attachment',
+        },
+    ]
+    result = formatters.parse_report_schedule(input_data)
+
+    assert result == (
+        ['test_namespace/test_schedule_name_to_create'],
+        {
+            'test_namespace/test_schedule_name_to_create_error': {
+                'message': 'test_error',
+                'attachment': 'test_attachment',
+            }
+        },
+    )
+
+
+def test_report_schedule_orchestration(formatters):
+    formatters.parse_report_schedule = MagicMock(side_effect=formatters.parse_report_schedule)
+    formatters.send_success_report = AsyncMock()
+    formatters.send_error_report = AsyncMock()
+
+    input_data = {
+        **metadata,
+        'created_schedules': [
+            {
+                'namespace': 'test_namespace',
+                'schedule_name': 'test_schedule_name_to_create',
+                'success': True,
+            },
+            {
+                'namespace': 'test_namespace',
+                'schedule_name': 'test_schedule_name_to_create_error',
+                'success': False,
+                'message': 'test_error1',
+                'attachment': 'test_attachment1',
+            },
+            {
+                'namespace': 'test_namespace',
+                'schedule_name': 'test_schedule_name_to_create_error2',
+                'success': False,
+                'message': 'test_error2',
+            },
+        ],
+        'updated_schedules': [
+            {
+                'namespace': 'test_namespace',
+                'schedule_name': 'test_schedule_name_to_update',
+                'success': True,
+            },
+            {
+                'namespace': 'test_namespace',
+                'schedule_name': 'test_schedule_name_to_update_error',
+                'success': False,
+                'message': 'test_error2',
+                'attachment': 'test_attachment2',
+            },
+            {
+                'namespace': 'test_namespace',
+                'schedule_name': 'test_schedule_name_to_update_error2',
+                'success': False,
+                'message': 'test_error3',
+                'attachment': 'test_attachment3',
+            },
+            {
+                'namespace': 'test_namespace',
+                'schedule_name': 'test_schedule_name_to_update_error3',
+                'success': False,
+                'message': 'test_error4',
+            },
+        ],
+        'deleted_schedules': [
+            {
+                'namespace': 'test_namespace',
+                'schedule_name': 'test_schedule_name_to_delete',
+                'success': True,
+            },
+            {
+                'namespace': 'test_namespace',
+                'schedule_name': 'test_schedule_name_to_delete_error',
+                'success': False,
+                'message': 'test_error4',
+                'attachment': 'test_attachment4',
+            },
+            {
+                'namespace': 'test_namespace',
+                'schedule_name': 'test_schedule_name_to_delete_error2',
+                'success': False,
+                'message': 'test_error5',
+            },
+        ],
+    }
+
+    formatters.report_schedule_orchestration(input_data)
+
+    formatters.parse_report_schedule.assert_has_calls(
+        [
+            call(input_data['created_schedules']),
+            call(input_data['updated_schedules']),
+            call(input_data['deleted_schedules']),
+        ]
+    )
+    formatters.send_success_report.assert_has_calls(
+        [
+            call(
+                metadata=metadata['metadata'],
+                message='Successfully created schedules: \n test_namespace/test_schedule_name_to_create',
+                notification_id='REPORT_ORCHESTRATION_CREATED_SCHEDULES',
+                attachment=input_data['created_schedules'],
+            ),
+            call(
+                metadata=metadata['metadata'],
+                message='Successfully updated schedules: \n test_namespace/test_schedule_name_to_update',
+                notification_id='REPORT_ORCHESTRATION_UPDATED_SCHEDULES',
+                attachment=input_data['updated_schedules'],
+            ),
+            call(
+                metadata=metadata['metadata'],
+                message='Successfully deleted schedules: \n test_namespace/test_schedule_name_to_delete',
+                notification_id='REPORT_ORCHESTRATION_DELETED_SCHEDULES',
+                attachment=input_data['deleted_schedules'],
+            ),
+        ]
+    )
+    formatters.send_error_report.assert_has_calls(
+        [
+            call(
+                metadata=metadata['metadata'],
+                message='Fails on created schedules: \n test_namespace/test_schedule_name_to_create_error, test_namespace/test_schedule_name_to_create_error2',
+                notification_id='REPORT_ORCHESTRATION_CREATED_SCHEDULES_ERROR',
+                attachment='test_namespace/test_schedule_name_to_create_error:\ntest_error1\ntest_attachment1\n ========== \ntest_namespace/test_schedule_name_to_create_error2:\ntest_error2',
+            ),
+            call(
+                metadata=metadata['metadata'],
+                message='Fails on updated schedules: \n test_namespace/test_schedule_name_to_update_error, test_namespace/test_schedule_name_to_update_error2, test_namespace/test_schedule_name_to_update_error3',
+                notification_id='REPORT_ORCHESTRATION_UPDATED_SCHEDULES_ERROR',
+                attachment='test_namespace/test_schedule_name_to_update_error:\ntest_error2\ntest_attachment2\n ========== \ntest_namespace/test_schedule_name_to_update_error2:\ntest_error3\ntest_attachment3\n ========== \ntest_namespace/test_schedule_name_to_update_error3:\ntest_error4',
+            ),
+            call(
+                metadata=metadata['metadata'],
+                message='Fails on deleted schedules: \n test_namespace/test_schedule_name_to_delete_error, test_namespace/test_schedule_name_to_delete_error2',
+                notification_id='REPORT_ORCHESTRATION_DELETED_SCHEDULES_ERROR',
+                attachment='test_namespace/test_schedule_name_to_delete_error:\ntest_error4\ntest_attachment4\n ========== \ntest_namespace/test_schedule_name_to_delete_error2:\ntest_error5',
+            ),
+        ]
+    )
+
+
+def test_report_slot_orchestration(formatters):
+    formatters.parse_report = MagicMock(side_effect=formatters.parse_report)
+    formatters.send_success_report = AsyncMock()
+    formatters.send_error_report = AsyncMock()
+
+    input_data = {
+        **metadata,
+        'inserted_slots': {
+            'test_slot_name_to_create': {'success': True},
+            'test_slot_name_to_create_error': {'success': False, 'error': 'test_error'},
+        },
+        'deleted_slots': {
+            'test_slot_name_to_delete': {'success': True},
+            'test_slot_name_to_delete_error': {'success': False, 'error': 'test_error'},
+        },
+    }
+
+    formatters.report_slot_orchestration(input_data)
+
+    formatters.parse_report.assert_has_calls(
+        [call(input_data['inserted_slots']), call(input_data['deleted_slots'])]
+    )
+    formatters.send_success_report.assert_has_calls(
+        [
+            call(
+                metadata=metadata['metadata'],
+                message='Inserted slots: \n test_slot_name_to_create',
+                notification_id='REPORT_ORCHESTRATION_INSERTED_SLOTS',
+            ),
+            call(
+                metadata=metadata['metadata'],
+                message='Deleted slots: \n test_slot_name_to_delete',
+                notification_id='REPORT_ORCHESTRATION_DELETED_SLOTS',
+            ),
+        ]
+    )
+    formatters.send_error_report.assert_has_calls(
+        [
+            call(
+                metadata=metadata['metadata'],
+                message='Failed to insert slots: \n test_slot_name_to_create_error',
+                notification_id='REPORT_ORCHESTRATION_INSERTED_SLOTS',
+                attachment=input_data['inserted_slots'],
+            ),
+            call(
+                metadata=metadata['metadata'],
+                message='Failed to delete slots: \n test_slot_name_to_delete_error',
+                notification_id='REPORT_ORCHESTRATION_DELETED_SLOTS',
+                attachment=input_data['deleted_slots'],
+            ),
+        ]
+    )
+
+
+def test_format_log_report(formatters):
+    input_data = {
+        **metadata,
+        'receiver_groups': {
+            'test_receiver_group': {
+                'notifications': [
+                    {
+                        'notification_id': 'test_notification_id',
+                        'trigger': 'test_trigger',
+                        'timestamp': '2021-01-01',
+                        'message': 'test_message',
+                        'level': 'test_level',
+                        'block': 'test_block',
+                        'pipeline': 'test_pipeline',
+                        'project': 'test_project',
+                        'model_name': 'test_model_name',
+                        'model_id': 'test_model_id',
+                    }
+                ],
+                'status': 'sent',
+            },
+            'test_receiver_group2': {
+                'notifications': [
+                    {
+                        'notification_id': 'test_notification_id',
+                        'trigger': 'test_trigger',
+                        'timestamp': '2021-01-01',
+                        'message': 'test_message',
+                        'level': 'test_level',
+                        'block': 'test_block',
+                        'pipeline': 'test_pipeline',
+                        'project': 'test_project',
+                        'model_name': 'test_model_name',
+                        'model_id': 'test_model_id',
+                    }
+                ],
+                'status': 'sent',
+            },
+        },
+        'mail_type': 'test_mail_type',
+    }
+
+    result = formatters.format_log_report(input_data)
+
+    expected_result = DataFrame(
+        [
+            {
+                'status': 'sent',
+                'timestamp': '2021-01-01',
+                'groups': ['test_receiver_group', 'test_receiver_group2'],
+                'message': 'test_message',
+                'level': 'test_level',
+                'notification_id': 'test_notification_id',
+                'block': 'test_block',
+                'schedule': 'test_trigger',
+                'pipeline': 'test_pipeline',
+                'project': 'test_project',
+                'model_name': 'test_model_name',
+                'model_id': 'test_model_id',
+                'mail_type': 'test_mail_type',
+            }
+        ]
+    )
+
+    assert DataFrame(result).equals(expected_result)
+
+
+def test_filter_notification_reports(formatters):
+    input_data = {
+        **metadata,
+        'notification_package': [
+            {'trigger': 'test_trigger_1', 'notification_id': 'test_notification_id_1'},
+            {'trigger': 'test_trigger_2', 'notification_id': 'test_notification_id_2'},
+            {'trigger': 'test_trigger_3', 'notification_id': 'test_notification_id_3'},
+        ],
+        'sending_configs': [
+            {
+                'group_name': 'test_group_1',
+                'contents': ['reports'],
+                'ignore': ['test_notification_id_1'],
+            },
+            {'group_name': 'test_group_2', 'contents': ['core_alerts']},
+        ],
+    }
+
+    response = formatters.filter_notification_reports(input_data)
+
+    assert response == {
+        'test_group_1': {
+            'group_name': 'test_group_1',
+            'contents': ['reports'],
+            'ignore': ['test_notification_id_1'],
+            'notifications': [
+                {'trigger': 'test_trigger_2', 'notification_id': 'test_notification_id_2'},
+                {'trigger': 'test_trigger_3', 'notification_id': 'test_notification_id_3'},
+            ],
+        }
+    }

tests/orchestrator/activities/test_mongo_db.py

@@ -0,0 +1,545 @@
+from datetime import datetime
+from unittest.mock import ANY, AsyncMock, MagicMock, patch
+
+from pytest import fixture
+from sientia_do.notifications.models import NotificationLevel
+from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
+
+from orchestrator.activities.mongo_db import MongoDB
+
+
+@fixture
+@patch('orchestrator.activities.mongo_db.MongoDBRepository')
+def mongo_db(mongo_mock):
+    mongo = MongoDB(
+        connection_string='mongodb://localhost:27017',
+        database_name='test_db',
+        ttl_index_seconds=3600,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+    mongo.send_notification = MagicMock()
+    mongo.emit_metric = AsyncMock()
+
+    return mongo
+
+
+@patch('orchestrator.activities.mongo_db.MongoDBRepository')
+def test___init__(mongo_mock):
+    logger = MagicMock()
+    notification_handler = MagicMock()
+    metrics_controller = AsyncMock()
+
+    MongoDB(
+        connection_string='mongodb://localhost:27017',
+        database_name='test_db',
+        ttl_index_seconds=3600,
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+
+    mongo_mock.assert_called_once_with(
+        connection_string='mongodb://localhost:27017',
+        database_name='test_db',
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+
+
+@patch('orchestrator.activities.mongo_db.SientiaMonitoring')
+def test_close(sientia_monitoring_mock, mongo_db):
+    mongo_db.close()
+    mongo_db.mongo_db_repository.close.assert_called_once()
+    sientia_monitoring_mock.shutdown.assert_called_once()
+
+
+def test___del__(mongo_db):
+    mongo_db.close = MagicMock()
+    mongo_db.__del__()
+    mongo_db.close.assert_called_once()
+
+
+def test_find_documents_in_mongodb_success(mongo_db):
+    input_data = {'collection': 'test_collection', 'filters': {'name': {'$exists': True}}}
+
+    mongo_db.mongo_db_repository.find = MagicMock(
+        return_value=[
+            {
+                'name': 'test1',
+                'timestamp': datetime.strptime(
+                    '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
+                ),
+            },
+            {
+                'name': 'test2',
+                'timestamp': datetime.strptime(
+                    '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
+                ),
+            },
+        ]
+    )
+
+    result = mongo_db.find_documents_in_mongodb(
+        {'query': input_data, 'timestamp_fields': ['timestamp']}
+    )
+
+    assert len(result) == 2
+    assert result[0] == {'name': 'test1', 'timestamp': '2023-01-01 12:00:00.000000+0000'}
+    assert result[1] == {'name': 'test2', 'timestamp': '2023-01-01 12:00:00.000000+0000'}
+    mongo_db.mongo_db_repository.find.assert_called_once_with(
+        'test_collection', {'name': {'$exists': True}}, {}
+    )
+
+
+metadata = {
+    'metadata': {
+        'schedule_name': 'test_schedule_name',
+        'workflow_name': 'test_workflow_name',
+        'model_name': 'test_model_name',
+        'model_id': 'test_model_id',
+    }
+}
+
+
+def test_find_documents_in_mongodb_failure(mongo_db):
+    input_data = {'collection': 'test_collection', 'filters': {'name': {'$exists': True}}}
+    mongo_db.mongo_db_repository.find = MagicMock(side_effect=Exception('Error'))
+
+    try:
+        mongo_db.find_documents_in_mongodb({'query': input_data, **metadata})
+    except Exception as e:
+        assert str(e) == 'Error'
+        mongo_db.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='MONGODB_QUERY_ERROR',
+            message='Failed to execute MongoDB query: Error',
+            level=NotificationLevel.ERROR,
+            block='load_query_from_mongodb',
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected an exception to be raised')
+
+
+def test_find_documents_in_mongodb_missing_collection(mongo_db):
+    input_data = {'query': {'filters': {}}}
+
+    try:
+        mongo_db.find_documents_in_mongodb(input_data)
+
+    except ValueError as e:
+        assert str(e) == 'Collection name must be provided in the query.'
+
+    else:
+        raise AssertionError('Expected a ValueError to be raised')
+
+
+def test_aggregate_documents_in_mongodb_success(mongo_db):
+    input_data = {
+        'collection': 'test_collection',
+        'aggregation': [{'$match': {'name': {'$exists': True}}}, {'$project': {'name': 1}}],
+    }
+    mongo_db.mongo_db_repository.aggregate = MagicMock(
+        return_value=[
+            {
+                'name': 'test1',
+                'timestamp': datetime.strptime(
+                    '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
+                ),
+            },
+            {
+                'name': 'test2',
+                'timestamp': datetime.strptime(
+                    '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
+                ),
+            },
+        ]
+    )
+
+    result = mongo_db.aggregate_documents_in_mongodb(
+        {'query': input_data, 'timestamp_fields': ['timestamp']}
+    )
+
+    assert len(result) == 2
+    assert result[0] == {'name': 'test1', 'timestamp': '2023-01-01 12:00:00.000000+0000'}
+    assert result[1] == {'name': 'test2', 'timestamp': '2023-01-01 12:00:00.000000+0000'}
+    expected_pipeline = input_data['aggregation']
+    expected_pipeline.append({'$project': {'_id': 0}})
+
+    mongo_db.mongo_db_repository.aggregate.assert_called_once_with(
+        'test_collection', expected_pipeline, {}
+    )
+
+
+def test_aggregate_documents_in_mongodb_failure(mongo_db):
+    input_data = {
+        'collection': 'test_collection',
+        'aggregation': [{'$match': {'name': {'$exists': True}}}, {'$project': {'name': 1}}],
+    }
+    mongo_db.mongo_db_repository.aggregate = MagicMock(side_effect=Exception('Error'))
+
+    try:
+        mongo_db.aggregate_documents_in_mongodb({'query': input_data, **metadata})
+    except Exception as e:
+        assert str(e) == 'Error'
+        mongo_db.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='MONGODB_AGGREGATION_ERROR',
+            message='Failed to execute MongoDB aggregation: Error',
+            block='aggregate_documents_in_mongodb',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected an exception to be raised')
+
+
+def test_aggregate_documents_in_mongodb_missing_collection(mongo_db):
+    input_data = {'query': {'aggregation': []}}
+
+    try:
+        mongo_db.aggregate_documents_in_mongodb(input_data)
+
+    except ValueError as e:
+        assert str(e) == 'Collection name must be provided in the query.'
+
+    else:
+        raise AssertionError('Expected a ValueError to be raised')
+
+
+def test_aggregate_documents_in_mongodb_missing_aggregation(mongo_db):
+    input_data = {'query': {'collection': 'test_collection'}}
+
+    try:
+        mongo_db.aggregate_documents_in_mongodb(input_data)
+
+    except ValueError as e:
+        assert str(e) == 'Aggregation must be provided.'
+
+    else:
+        raise AssertionError('Expected a ValueError to be raised')
+
+
+@patch('orchestrator.activities.mongo_db.now')
+def test_update_pipelines_timestamps_success(now_mock, mongo_db):
+    input_data = {
+        'updated_pipelines': [
+            {'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
+            {'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
+        ]
+    }
+    mongo_db.mongo_db_repository.update_many = MagicMock(return_value=MagicMock())
+    mongo_db.update_pipelines_timestamps(input_data)
+    mongo_db.mongo_db_repository.update_many.assert_called_once_with(
+        'orchestrated_schedules',
+        {
+            '$or': [
+                {'schedule_name': 'test1', 'namespace': 'test1'},
+                {'schedule_name': 'test2', 'namespace': 'test2'},
+            ]
+        },
+        {'$set': {'updated_at': now_mock.return_value}},
+        {},
+    )
+
+
+@patch('orchestrator.activities.mongo_db.now')
+def test_update_pipelines_timestamps_failure(now_mock, mongo_db):
+    input_data = {
+        'updated_pipelines': [
+            {'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
+            {'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
+        ],
+        **metadata,
+    }
+
+    mongo_db.mongo_db_repository.update_many.side_effect = Exception('Error')
+    try:
+        mongo_db.update_pipelines_timestamps(input_data)
+    except Exception as e:
+        assert str(e) == 'Error'
+        mongo_db.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='MONGODB_UPDATE_PIPELINES_ERROR',
+            message='Failed to update pipelines timestamps: Error',
+            block='update_pipelines_timestamps',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected an exception to be raised')
+
+
+@patch('orchestrator.activities.mongo_db.now')
+def test_create_pipelines_timestamps_success(now_mock, mongo_db):
+    input_data = {
+        'created_pipelines': [
+            {'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
+            {'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
+        ]
+    }
+    mongo_db.mongo_db_repository.insert_many = MagicMock(return_value=MagicMock())
+    mongo_db.create_pipelines_timestamps(input_data)
+    mongo_db.mongo_db_repository.insert_many.assert_called_once_with(
+        'orchestrated_schedules',
+        [
+            {'schedule_name': 'test1', 'namespace': 'test1', 'updated_at': now_mock.return_value},
+            {'schedule_name': 'test2', 'namespace': 'test2', 'updated_at': now_mock.return_value},
+        ],
+        {},
+    )
+
+
+@patch('orchestrator.activities.mongo_db.now')
+def test_create_pipelines_timestamps_failure(now_mock, mongo_db):
+    input_data = {
+        'created_pipelines': [
+            {'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
+            {'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
+        ],
+        **metadata,
+    }
+    mongo_db.mongo_db_repository.insert_many.side_effect = Exception('Error')
+    try:
+        mongo_db.create_pipelines_timestamps(input_data)
+    except Exception as e:
+        assert str(e) == 'Error'
+        mongo_db.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='MONGODB_CREATE_PIPELINES_ERROR',
+            message='Failed to create pipelines timestamps: Error',
+            block='create_pipelines_timestamps',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected an exception to be raised')
+
+
+@patch('orchestrator.activities.mongo_db.now')
+def test_delete_pipelines_timestamps_success(now_mock, mongo_db):
+    input_data = {
+        'deleted_pipelines': [
+            {'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
+            {'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
+        ]
+    }
+    mongo_db.mongo_db_repository.delete_many = MagicMock(return_value=MagicMock())
+    mongo_db.delete_pipelines_timestamps(input_data)
+    mongo_db.mongo_db_repository.delete_many.assert_called_once_with(
+        'orchestrated_schedules',
+        {
+            '$or': [
+                {'schedule_name': 'test1', 'namespace': 'test1'},
+                {'schedule_name': 'test2', 'namespace': 'test2'},
+            ]
+        },
+        {},
+    )
+
+
+@patch('orchestrator.activities.mongo_db.now')
+def test_delete_pipelines_timestamps_failure(now_mock, mongo_db):
+    input_data = {
+        'deleted_pipelines': [
+            {'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
+            {'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
+        ],
+        **metadata,
+    }
+    mongo_db.mongo_db_repository.delete_many.side_effect = Exception('Error')
+    try:
+        mongo_db.delete_pipelines_timestamps(input_data)
+    except Exception as e:
+        assert str(e) == 'Error'
+        mongo_db.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='MONGODB_DELETE_PIPELINES_ERROR',
+            message='Failed to delete pipelines timestamps: Error',
+            block='delete_pipelines_timestamps',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected an exception to be raised')
+
+
+def test_create_collection_with_ttl_index_success(mongo_db):
+    input_data = {
+        **metadata,
+        'pipelines': {
+            'scouter-pipeline': {'topic': 'raw_scouter_pipeline'},
+            'scouter-pipeline-2': {'topic': 'raw_scouter_pipeline_2'},
+            'scouter-pipeline-3': {'topic': 'raw_scouter_pipeline_3'},
+        },
+    }
+
+    mongo_db.mongo_db_repository.database.list_collection_names = MagicMock(
+        return_value=[
+            'raw_scouter_pipeline_2',
+            'raw_scouter_pipeline_3',
+        ]
+    )
+
+    collection_1 = MagicMock(
+        list_indexes=MagicMock(
+            return_value=[
+                {
+                    'key': 'asdad',
+                }
+            ]
+        )
+    )
+
+    collection_2 = MagicMock(
+        list_indexes=MagicMock(return_value=[{'key': 'inserted_at', 'expireAfterSeconds': None}])
+    )
+    collection_3 = MagicMock(
+        list_indexes=MagicMock(return_value=[{'key': 'inserted_at', 'expireAfterSeconds': 3600}])
+    )
+
+    mongo_db.mongo_db_repository.database.__getitem__ = MagicMock(
+        side_effect=[collection_1, collection_2, collection_3]
+    )
+
+    mongo_db.create_collection_with_ttl_index(input_data)
+
+    mongo_db.mongo_db_repository.database.list_collection_names.assert_called_once_with()
+
+    mongo_db.mongo_db_repository.database.create_collection.assert_called_once_with(
+        'raw_scouter_pipeline'
+    )
+
+    collection_1.list_indexes.assert_called_once()
+    collection_1.create_index.assert_called_once_with(
+        'inserted_at', expireAfterSeconds=3600, background=True
+    )
+
+    collection_2.list_indexes.assert_called_once()
+    collection_2.create_index.assert_called_once_with(
+        'inserted_at', expireAfterSeconds=3600, background=True
+    )
+
+    collection_3.list_indexes.assert_called_once()
+    collection_3.create_index.assert_not_called()
+
+
+def test_create_collection_with_ttl_index_failure(mongo_db):
+    input_data = {**metadata, 'pipelines': {'scouter-pipeline': {'topic': 'raw_scouter_pipeline'}}}
+
+    mongo_db.mongo_db_repository.database.list_collection_names.return_value = []
+
+    mongo_db.mongo_db_repository.database.create_collection.side_effect = Exception('Error')
+
+    try:
+        mongo_db.create_collection_with_ttl_index(input_data)
+    except Exception as e:
+        assert str(e) == 'Error'
+        mongo_db.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='MONGODB_CREATE_COLLECTION_ERROR',
+            message='Failed to create collection raw_scouter_pipeline with TTL index: Error',
+            block='create_collection_with_ttl_index',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+    else:
+        raise AssertionError('Expected an exception to be raised')
+
+
+def test_load_latest_data_none_last_data_timestamp(mongo_db):
+    """Test load_latest_data"""
+    mongo_db.mongo_db_repository.find = MagicMock(
+        return_value=[
+            {
+                'name': 'test1',
+                'value': 1,
+                'timestamp': '2023-01-01 12:00:00+0000',
+            }
+        ]
+    )
+
+    result = mongo_db.load_latest_data(
+        {
+            'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+            'collection_name': 'test_collection',
+            'last_data_timestamp': None,
+            'base_data_filter': {'level': 'ERROR'},
+        }
+    )
+
+    mongo_db.mongo_db_repository.find.assert_called_once_with(
+        'test_collection',
+        {'level': 'ERROR'},
+        {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+    )
+
+    assert result == [{'name': 'test1', 'value': 1, 'timestamp': '2023-01-01 12:00:00+0000'}]
+
+
+def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
+    """Test load_latest_data"""
+
+    mongo_db.mongo_db_repository.find = MagicMock(
+        return_value=[
+            {
+                'name': 'test1',
+                'value': 1,
+                'timestamp': '2023-01-01 12:00:00+0000',
+            }
+        ]
+    )
+
+    result = mongo_db.load_latest_data(
+        {
+            'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+            'collection_name': 'test_collection',
+            'last_data_timestamp': '2023-01-01 12:00:00+0000',
+            'base_data_filter': {'level': 'ERROR'},
+        }
+    )
+
+    mongo_db.mongo_db_repository.find.assert_called_once_with(
+        'test_collection',
+        {
+            'level': 'ERROR',
+            'timestamp': {'$gt': '2023-01-01 12:00:00+0000'},
+        },
+        {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+    )
+
+    assert result == [{'name': 'test1', 'value': 1, 'timestamp': '2023-01-01 12:00:00+0000'}]
+
+
+def test_load_latest_data_error(mongo_db):
+    """Test load_latest_data"""
+    mongo_db.mongo_db_repository.find.side_effect = Exception('test')
+
+    try:
+        mongo_db.load_latest_data(
+            {
+                'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+                'collection_name': 'test_collection',
+                'last_data_timestamp': '2023-01-01 12:00:00+0000',
+                'base_data_filter': {'level': 'ERROR'},
+            }
+        )
+    except Exception as e:
+        assert str(e) == 'test'
+
+        mongo_db.send_notification.assert_called_once_with(
+            metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+            notification_id='MONGO_LOAD_ERROR',
+            message='Error loading data from MongoDB: test',
+            block='load_latest_data',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )

tests/orchestrator/activities/test_slot_manager.py

@@ -0,0 +1,409 @@
+from datetime import timedelta
+from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
+
+from pandas import DataFrame
+from pytest import fixture
+from sientia_do.notifications.models import NotificationLevel
+from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ, now
+
+from orchestrator.activities.slot_manager import SlotManager
+
+metadata = {
+    'metadata': {
+        'schedule_name': 'test_schedule_name',
+        'workflow_name': 'test_workflow_name',
+        'model_name': 'test_model_name',
+        'model_id': 'test_model_id',
+    }
+}
+
+
+@fixture
+@patch('orchestrator.activities.slot_manager.RedisRepository')
+def slot_manager(_redis_mock):
+    slot_manager = SlotManager(
+        host='localhost',
+        port=6379,
+        username='admin',
+        password='password',
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+
+    slot_manager.redis_repository = MagicMock()
+    slot_manager.logger = MagicMock()
+    slot_manager.notification_handler = MagicMock()
+    slot_manager.send_notification = MagicMock()
+    slot_manager.emit_metric = AsyncMock()
+
+    return slot_manager
+
+
+def test_load_opc_slots_no_slot_keys(slot_manager):
+    slot_manager.redis_repository.keys = MagicMock(return_value=[])
+    assert slot_manager.load_opc_slots(metadata) == {}
+
+
+def test_load_opc_slots(slot_manager):
+    slot_manager.redis_repository.keys = MagicMock(
+        return_value=[
+            b'slot:opc_tags:1',
+            b'slot:opc_tags:2',
+            b'slot:opc_tags:3',
+        ]
+    )
+
+    slot_manager.redis_repository.get = MagicMock(side_effect=['value1', 'value2', None])
+
+    response = slot_manager.load_opc_slots(metadata)
+
+    assert response == {
+        'slot:opc_tags:1': 'value1',
+        'slot:opc_tags:2': 'value2',
+        'slot:opc_tags:3': None,
+    }
+
+
+def test_load_opc_slots_no_decode(slot_manager):
+    slot_manager.redis_repository.keys = MagicMock(
+        return_value=[
+            'slot:opc_tags:1',
+            'slot:opc_tags:2',
+            'slot:opc_tags:3',
+        ]
+    )
+
+    slot_manager.redis_repository.get = MagicMock(side_effect=['value1', 'value2', None])
+
+    response = slot_manager.load_opc_slots(metadata)
+
+    assert response == {
+        'slot:opc_tags:1': 'value1',
+        'slot:opc_tags:2': 'value2',
+        'slot:opc_tags:3': None,
+    }
+
+
+def test_load_opc_slots_error(slot_manager):
+    slot_manager.redis_repository.keys = MagicMock(
+        return_value=[
+            'slot:opc_tags:1',
+            'slot:opc_tags:2',
+            'slot:opc_tags:3',
+        ]
+    )
+
+    slot_manager.redis_repository.get = MagicMock(side_effect=Exception('Test exception'))
+
+    try:
+        slot_manager.load_opc_slots(metadata)
+    except Exception as e:
+        assert str(e) == 'Test exception'
+        slot_manager.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='REDIS_GET_ERROR',
+            message='Failed to load OPC slots: Test exception',
+            block='load_opc_slots',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected an exception to be raised')
+
+
+def test_load_active_ingestors(slot_manager):
+    slot_manager.redis_repository.keys = MagicMock(
+        return_value=[
+            b'heartbeat:ingestor:1',
+            b'heartbeat:ingestor:2',
+            'heartbeat:ingestor:3',
+        ]
+    )
+
+    response = slot_manager.load_active_ingestors(metadata)
+
+    assert response == ['heartbeat:ingestor:1', 'heartbeat:ingestor:2', 'heartbeat:ingestor:3']
+
+
+def test_load_active_ingestors_error(slot_manager):
+    slot_manager.redis_repository.keys = MagicMock(
+        return_value=[
+            'heartbeat:ingestor:1',
+            'heartbeat:ingestor:2',
+            'heartbeat:ingestor:3',
+        ]
+    )
+
+    slot_manager.redis_repository.keys = MagicMock(side_effect=Exception('Test exception'))
+
+    try:
+        slot_manager.load_active_ingestors(metadata)
+    except Exception as e:
+        assert str(e) == 'Test exception'
+        slot_manager.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='REDIS_GET_ERROR',
+            message='Failed to load active ingestors: Test exception',
+            block='load_active_ingestors',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected an exception to be raised')
+
+
+def test_update_slots(slot_manager):
+    slot_manager.redis_repository.set = MagicMock(side_effect=[None, Exception('Test exception')])
+
+    response = slot_manager.update_slots({'to_insert': {'1': 'value1', '2': 'value2'}})
+
+    slot_manager.redis_repository.set.assert_has_calls(
+        [call('slot:opc_tags:1', 'value1', ttl=None), call('slot:opc_tags:2', 'value2', ttl=None)]
+    )
+
+    assert response == {
+        '1': {'success': True, 'message': 'Slot updated successfully'},
+        '2': {'success': False, 'message': 'Test exception'},
+    }
+
+
+def test_delete_slots(slot_manager):
+    slot_manager.redis_repository.delete = MagicMock(
+        side_effect=[None, Exception('Test exception')]
+    )
+
+    response = slot_manager.delete_slots({'to_delete': ['1', '2']})
+
+    slot_manager.redis_repository.delete.assert_has_calls(
+        [call('slot:opc_tags:1'), call('slot:opc_tags:2')]
+    )
+
+    assert response == {
+        '1': {'success': True, 'message': 'Slot deleted successfully'},
+        '2': {'success': False, 'message': 'Test exception'},
+    }
+
+
+def test_get_last_data_timestamp_none(slot_manager):
+    """Test get_last_data_timestamp"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'mail_type': 'test_mail_type',
+    }
+
+    slot_manager.redis_repository.get = MagicMock(return_value=None)
+
+    result = slot_manager.get_last_data_timestamp(test_data)
+
+    assert result is None
+
+
+def test_get_last_data_timestamp_not_none(slot_manager):
+    """Test get_last_data_timestamp"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'mail_type': 'test_mail_type',
+    }
+
+    slot_manager.redis_repository.get = MagicMock(return_value='2023-01-01 12:00:00')
+
+    result = slot_manager.get_last_data_timestamp(test_data)
+
+    slot_manager.redis_repository.get.assert_called_once_with(
+        'notification_last_timestamp:test_mail_type'
+    )
+
+    assert result == '2023-01-01 12:00:00'
+
+
+def test_get_last_data_timestamp_error(slot_manager):
+    """Test get_last_data_timestamp error"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'mail_type': 'test_mail_type',
+    }
+
+    slot_manager.redis_repository.get = MagicMock(side_effect=Exception('test'))
+
+    try:
+        slot_manager.get_last_data_timestamp(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+        slot_manager.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='REDIS_GET_ERROR',
+            message='Error getting last data timestamp: test',
+            block='get_last_data_timestamp',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected exception')
+
+
+def test_put_last_data_timestamp_empty_dataframe(slot_manager):
+    """Test put_last_data_timestamp with empty dataframe"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
+        'mail_type': 'test_mail_type',
+    }
+
+    slot_manager.set = MagicMock()
+
+    result = slot_manager.put_last_data_timestamp(test_data)
+
+    assert result is None
+
+    slot_manager.set.assert_not_called()
+
+
+def test_put_last_data_timestamp_not_empty_dataframe(slot_manager):
+    """Test put_last_data_timestamp with not empty dataframe"""
+
+    data = DataFrame(
+        {
+            'name': ['sensor1', 'sensor2'],
+            'value': [25.5, 30.0],
+            'timestamp': ['2023-01-01 12:00:00', '2023-01-01 12:00:01'],
+        }
+    )
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'data': data.to_dict('records'),
+        'mail_type': 'test_mail_type',
+    }
+
+    slot_manager.redis_repository.set = MagicMock()
+
+    result = slot_manager.put_last_data_timestamp(test_data)
+
+    assert result == '2023-01-01 12:00:01'
+
+    slot_manager.redis_repository.set.assert_called_once_with(
+        'notification_last_timestamp:test_mail_type', '2023-01-01 12:00:01', ttl=18000
+    )
+
+
+def test_put_last_data_timestamp_error(slot_manager):
+    """Test put_last_data_timestamp error"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'data': DataFrame(
+            {
+                'name': ['sensor1', 'sensor2'],
+                'value': [25.5, 30.0],
+                'timestamp': ['2023-01-01 12:00:00'] * 2,
+            }
+        ).to_dict('records'),
+        'mail_type': 'test_mail_type',
+    }
+
+    slot_manager.redis_repository.set = MagicMock(side_effect=Exception('test'))
+
+    try:
+        slot_manager.put_last_data_timestamp(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+        slot_manager.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='REDIS_SET_ERROR',
+            message='Error setting last data timestamp: test',
+            block='put_last_data_timestamp',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected exception')
+
+
+def test_filter_notification_alerts(slot_manager):
+    slot_manager.redis_repository.get = MagicMock(
+        side_effect=[
+            None,
+            (now() - timedelta(seconds=600)).strftime(DATETIME_FORMAT_MS_WITH_TZ),
+            now().strftime(DATETIME_FORMAT_MS_WITH_TZ),
+            None,
+            (now() - timedelta(seconds=600)).strftime(DATETIME_FORMAT_MS_WITH_TZ),
+            now().strftime(DATETIME_FORMAT_MS_WITH_TZ),
+        ]
+    )
+
+    input_data = {
+        **metadata,
+        'notification_package': [
+            {'trigger': 'test_trigger_1', 'notification_id': 'test_notification_id_1'},
+            {'trigger': 'test_trigger_2', 'notification_id': 'test_notification_id_2'},
+            {'trigger': 'test_trigger_3', 'notification_id': 'test_notification_id_3'},
+        ],
+        'sending_configs': [
+            {'group_name': 'test_group_1', 'contents': ['core_alerts', 'persistent_alerts']},
+            {'group_name': 'test_group_2', 'contents': ['core_alerts']},
+        ],
+        'notification_ttl': 300,
+        'mail_type': 'test_mail_type',
+    }
+
+    response = slot_manager.filter_notification_alerts(input_data)
+
+    assert response == {
+        'test_group_1': {
+            'group_name': 'test_group_1',
+            'contents': ['core_alerts', 'persistent_alerts'],
+            'notifications': [
+                {'trigger': 'test_trigger_1', 'notification_id': 'test_notification_id_1'},
+                {'trigger': 'test_trigger_2', 'notification_id': 'test_notification_id_2'},
+            ],
+        },
+        'test_group_2': {
+            'group_name': 'test_group_2',
+            'contents': ['core_alerts'],
+            'notifications': [
+                {'trigger': 'test_trigger_1', 'notification_id': 'test_notification_id_1'}
+            ],
+        },
+    }
+
+
+def test_store_notification_cache(slot_manager):
+    """Test store_notification_cache"""
+    test_data = {
+        **metadata,
+        'log_report': DataFrame(
+            {
+                'status': ['sent', 'error'],
+                'schedule': ['test_schedule_1', 'test_schedule_2'],
+                'notification_id': ['test_notification_id_1', 'test_notification_id_2'],
+            }
+        ).to_dict(),
+        'sent_ttl': 600,
+    }
+
+    slot_manager.redis_repository.set = MagicMock()
+
+    slot_manager.store_notification_cache(test_data)
+
+    slot_manager.redis_repository.set.assert_called_once_with(
+        'test_schedule_1:test_notification_id_1', ANY, ttl=600
+    )

tests/orchestrator/activities/test_temporal_manager.py

@@ -0,0 +1,587 @@
+from datetime import timedelta
+from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
+
+from pytest import fixture, mark
+from sientia_do.notifications.models import NotificationLevel
+
+from orchestrator.activities.temporal_manager import TemporalManager
+from orchestrator.utils.converters import parse_frequency
+
+metadata = {
+    'metadata': {
+        'schedule_name': 'test_schedule_name',
+        'workflow_name': 'test_workflow_name',
+        'model_name': 'test_model_name',
+        'model_id': 'test_model_id',
+    }
+}
+
+
+@fixture
+def temporal_manager():
+    temporal_manager = TemporalManager(
+        host='localhost:7233',
+        scouter_namespace='scouter',
+        laborious_namespace='laborious',
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+
+    temporal_manager.temporal_clients['scouter'] = MagicMock()
+    temporal_manager.temporal_clients['laborious'] = MagicMock()
+    temporal_manager.send_notification_async = AsyncMock()
+    temporal_manager.emit_metric = AsyncMock()
+    temporal_manager.send_notification = MagicMock()
+
+    return temporal_manager
+
+
+@mark.asyncio
+@patch('orchestrator.activities.temporal_manager.Client.connect', new_callable=AsyncMock)
+async def test_connect_to_temporal(connect_mock, temporal_manager):
+    await temporal_manager.connect_to_temporal()
+    connect_mock.assert_has_calls(
+        [
+            call(target_host='localhost:7233', namespace='scouter'),
+            call(target_host='localhost:7233', namespace='laborious'),
+        ]
+    )
+
+
+async def async_iter():
+    yield MagicMock(id='test-schedule-id', search_attributes={'orchestrated': ['true']})
+    yield MagicMock(id='test-schedule-id-2', search_attributes={'Attr': ['false']})
+    yield MagicMock(id='test-schedule-id-3', search_attributes={'Attr': ['false']})
+
+
+@mark.asyncio
+async def test_normalize_schedules(temporal_manager):
+    input_data = {
+        'orchestrated_schedules': {
+            'scouter': {'test-scouter': '2021-01-01'},
+            'laborious': {'test-schedule-id1': '2021-01-01'},
+        },
+        **metadata,
+    }
+
+    temporal_manager.temporal_clients['scouter'].list_schedules = AsyncMock(
+        return_value=async_iter()
+    )
+    temporal_manager.temporal_clients['laborious'].list_schedules = AsyncMock(
+        return_value=async_iter()
+    )
+
+    temporal_manager.temporal_clients['scouter'].get_schedule_handle = MagicMock(
+        return_value=MagicMock(delete=AsyncMock())
+    )
+    temporal_manager.temporal_clients['laborious'].get_schedule_handle = MagicMock(
+        return_value=MagicMock(delete=AsyncMock())
+    )
+
+    await temporal_manager.normalize_schedules(input_data)
+
+    temporal_manager.temporal_clients['scouter'].get_schedule_handle.assert_has_calls(
+        [
+            call('test-schedule-id'),
+        ]
+    )
+
+    temporal_manager.temporal_clients[
+        'scouter'
+    ].get_schedule_handle.return_value.delete.assert_awaited_once()
+
+
+@mark.asyncio
+async def test_normalize_schedules_error(temporal_manager):
+    temporal_manager.temporal_clients['scouter'].list_schedules = AsyncMock(
+        side_effect=Exception('Test exception')
+    )
+
+    try:
+        await temporal_manager.normalize_schedules(metadata)
+    except Exception as e:
+        assert str(e) == 'Test exception'
+        temporal_manager.send_notification_async.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='SCHEDULER_NORMALIZE_SCHEDULES_ERROR',
+            message='Failed to normalize schedules: Test exception',
+            block='normalize_schedules',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected an exception to be raised')
+
+
+@mark.asyncio
+@patch('orchestrator.activities.temporal_manager.parse_frequency', side_effect=parse_frequency)
+@patch('orchestrator.activities.temporal_manager.Schedule')
+@patch('orchestrator.activities.temporal_manager.ScheduleActionStartWorkflow')
+@patch('orchestrator.activities.temporal_manager.ScheduleIntervalSpec')
+@patch('orchestrator.activities.temporal_manager.ScheduleSpec')
+@patch('orchestrator.activities.temporal_manager.TypedSearchAttributes')
+@patch('orchestrator.activities.temporal_manager.SearchAttributePair')
+async def test_create_schedule(
+    mock_search_attribute_pair,
+    mock_typed_search_attributes,
+    mock_schedule_spec,
+    mock_schedule_interval_spec,
+    mock_schedule_action_start_workflow,
+    mock_schedule,
+    mock_parse_frequency,
+    temporal_manager,
+):
+    input_data = {
+        'schedules': {
+            'scouter': {
+                'test-schedule': {
+                    'model_id': 1,
+                    'model_name': 'test-model-name',
+                    'workflow_type': 'test-workflow',
+                    'frequency': '1m',
+                    'data': {'test': 'test'},
+                    'execution_timeout_seconds': 100,
+                    'task_timeout_seconds': 100,
+                },
+                'test-schedule-invalid-frequency': {
+                    'model_id': 2,
+                    'model_name': 'test-model-name',
+                    'workflow_type': 'test-workflow',
+                    'frequency': '10y',
+                    'offset': '5m',
+                    'data': {'test': 'test'},
+                    'execution_timeout_seconds': 400,
+                    'task_timeout_seconds': 400,
+                },
+            },
+            'laborious': {
+                'test-schedule-laborious': {
+                    'model_id': 1,
+                    'model_name': 'test-model-name',
+                    'workflow_type': 'test-workflow',
+                    'frequency': '2m',
+                    'offset': '1h',
+                    'data': {'test': 'test'},
+                    'execution_timeout_seconds': 500,
+                    'task_timeout_seconds': 500,
+                }
+            },
+        }
+    }
+
+    temporal_manager.temporal_clients['scouter'].create_schedule = AsyncMock()
+    temporal_manager.temporal_clients['laborious'].create_schedule = AsyncMock()
+
+    report = await temporal_manager.create_schedules(input_data)
+
+    temporal_manager.temporal_clients['scouter'].create_schedule.assert_called_once_with(
+        'test-schedule',
+        mock_schedule.return_value,
+        search_attributes=mock_typed_search_attributes.return_value,
+    )
+    temporal_manager.temporal_clients['laborious'].create_schedule.assert_called_once_with(
+        'test-schedule-laborious',
+        mock_schedule.return_value,
+        search_attributes=mock_typed_search_attributes.return_value,
+    )
+
+    mock_schedule.assert_has_calls(
+        [
+            call(
+                action=mock_schedule_action_start_workflow.return_value,
+                spec=mock_schedule_spec.return_value,
+            ),
+            call(
+                action=mock_schedule_action_start_workflow.return_value,
+                spec=mock_schedule_spec.return_value,
+            ),
+        ]
+    )
+
+    mock_schedule_action_start_workflow.assert_has_calls(
+        [
+            call(
+                'test-workflow',
+                input_data['schedules']['scouter']['test-schedule'],
+                id='test-schedule',
+                task_queue='test-workflow-legacy-queue',
+                execution_timeout=timedelta(seconds=100),
+                run_timeout=timedelta(seconds=100),
+                task_timeout=timedelta(seconds=100),
+                typed_search_attributes=mock_typed_search_attributes.return_value,
+            ),
+            call(
+                'test-workflow',
+                input_data['schedules']['scouter']['test-schedule-invalid-frequency'],
+                id='test-schedule-invalid-frequency',
+                task_queue='test-workflow-legacy-queue',
+                execution_timeout=timedelta(seconds=400),
+                run_timeout=timedelta(seconds=400),
+                task_timeout=timedelta(seconds=400),
+                typed_search_attributes=mock_typed_search_attributes.return_value,
+            ),
+            call(
+                'test-workflow',
+                input_data['schedules']['laborious']['test-schedule-laborious'],
+                id='test-schedule-laborious',
+                task_queue='test-workflow-legacy-queue',
+                execution_timeout=timedelta(seconds=500),
+                run_timeout=timedelta(seconds=500),
+                task_timeout=timedelta(seconds=500),
+                typed_search_attributes=mock_typed_search_attributes.return_value,
+            ),
+        ]
+    )
+
+    mock_schedule_spec.assert_has_calls(
+        [
+            call(intervals=[mock_schedule_interval_spec.return_value]),
+            call(intervals=[mock_schedule_interval_spec.return_value]),
+        ]
+    )
+
+    mock_schedule_interval_spec.assert_has_calls(
+        [
+            call(every=timedelta(seconds=60), offset=timedelta(seconds=0)),
+            call(every=timedelta(seconds=120), offset=timedelta(seconds=3600)),
+        ]
+    )
+
+    mock_parse_frequency.assert_has_calls(
+        [call('1m'), call('0m'), call('10y'), call('2m'), call('1h')]
+    )
+
+    mock_typed_search_attributes.assert_has_calls(
+        [
+            call(
+                [
+                    mock_search_attribute_pair.return_value,
+                    mock_search_attribute_pair.return_value,
+                    mock_search_attribute_pair.return_value,
+                ]
+            ),
+            call(
+                [
+                    mock_search_attribute_pair.return_value,
+                    mock_search_attribute_pair.return_value,
+                    mock_search_attribute_pair.return_value,
+                ]
+            ),
+            call(
+                [
+                    mock_search_attribute_pair.return_value,
+                    mock_search_attribute_pair.return_value,
+                    mock_search_attribute_pair.return_value,
+                ]
+            ),
+        ]
+    )
+
+    mock_search_attribute_pair.assert_has_calls(
+        [
+            call(key=temporal_manager.model_id_id_key, value=1),
+            call(key=temporal_manager.model_name_id_key, value='test-model-name'),
+            call(key=temporal_manager.orchestrated_id_key, value='true'),
+            call(key=temporal_manager.model_id_id_key, value=2),
+            call(key=temporal_manager.model_name_id_key, value='test-model-name'),
+            call(key=temporal_manager.orchestrated_id_key, value='true'),
+        ]
+    )
+
+    assert report == [
+        {
+            'schedule_name': 'test-schedule',
+            'namespace': 'scouter',
+            'success': True,
+            'message': 'Schedule created successfully',
+        },
+        {
+            'schedule_name': 'test-schedule-invalid-frequency',
+            'namespace': 'scouter',
+            'success': False,
+            'message': 'Invalid frequency',
+        },
+        {
+            'schedule_name': 'test-schedule-laborious',
+            'namespace': 'laborious',
+            'success': True,
+            'message': 'Schedule created successfully',
+        },
+    ]
+
+
+@mark.asyncio
+async def test_create_schedules_with_no_client(temporal_manager):
+    temporal_manager.temporal_clients = {}
+    input_data = {
+        'schedules': {'abc': {'test-schedule': {'frequency': '1m', 'data': {'test': 'test'}}}}
+    }
+
+    try:
+        await temporal_manager.create_schedules(input_data)
+    except Exception as e:
+        assert (
+            str(e)
+            == f'Temporal client for abc not found, clients: {temporal_manager.temporal_clients}'
+        )
+
+
+@mark.asyncio
+@patch('orchestrator.activities.temporal_manager.parse_frequency', side_effect=parse_frequency)
+@patch('orchestrator.activities.temporal_manager.ScheduleIntervalSpec')
+async def test_update_schedules(
+    _mock_schedule_interval_spec, _mock_parse_frequency, temporal_manager
+):
+    input_mock = MagicMock(args=MagicMock())
+    temporal_manager.schedule_handles = {
+        'scouter': {
+            'test-schedule': MagicMock(
+                update=AsyncMock(update=AsyncMock(side_effect=lambda f: f(input_mock)))
+            )
+        },
+        'laborious': {
+            'test-schedule-laborious': MagicMock(
+                update=AsyncMock(update=AsyncMock(side_effect=lambda f: f(input_mock)))
+            )
+        },
+    }
+    input_data = {
+        'schedules': {
+            'scouter': {
+                'test-schedule': {'frequency': '1m', 'data': {'test': 'test'}},
+                'test-schedule_no_handler': {'frequency': '1m', 'data': {'test': 'test'}},
+            },
+            'laborious': {'test-schedule-laborious': {'frequency': '2m', 'data': {'test': 'test'}}},
+        }
+    }
+
+    handler_scouter = MagicMock(
+        update=AsyncMock(update=AsyncMock(side_effect=lambda f: f(input_mock)))
+    )
+
+    handler_laborious = MagicMock(
+        update=AsyncMock(update=AsyncMock(side_effect=lambda f: f(input_mock)))
+    )
+
+    temporal_manager.temporal_clients['scouter'].get_schedule_handle = MagicMock(
+        side_effect=[handler_scouter, None]
+    )
+
+    temporal_manager.temporal_clients['laborious'].get_schedule_handle = MagicMock(
+        side_effect=[handler_laborious]
+    )
+
+    report = await temporal_manager.update_schedules(input_data)
+
+    temporal_manager.temporal_clients['scouter'].get_schedule_handle.assert_has_calls(
+        [call('test-schedule'), call('test-schedule_no_handler')]
+    )
+    temporal_manager.temporal_clients['laborious'].get_schedule_handle.assert_has_calls(
+        [call('test-schedule-laborious')]
+    )
+
+    handler_scouter.update.assert_called_once()
+    handler_laborious.update.assert_called_once()
+
+    assert report == [
+        {
+            'schedule_name': 'test-schedule',
+            'namespace': 'scouter',
+            'success': True,
+            'message': 'Schedule updated successfully',
+        },
+        {
+            'schedule_name': 'test-schedule_no_handler',
+            'namespace': 'scouter',
+            'success': False,
+            'message': 'Schedule test-schedule_no_handler not found',
+        },
+        {
+            'schedule_name': 'test-schedule-laborious',
+            'namespace': 'laborious',
+            'success': True,
+            'message': 'Schedule updated successfully',
+        },
+    ]
+
+
+@mark.asyncio
+async def test_update_schedules_with_no_client(temporal_manager):
+    temporal_manager.temporal_clients = {}
+    input_data = {
+        'schedules': {'abc': {'test-schedule': {'frequency': '1m', 'data': {'test': 'test'}}}}
+    }
+
+    try:
+        await temporal_manager.update_schedules(input_data)
+    except Exception as e:
+        assert (
+            str(e)
+            == f'Temporal client for abc not found, clients: {temporal_manager.temporal_clients}'
+        )
+
+
+@mark.asyncio
+async def test_delete_schedules(temporal_manager):
+    temporal_manager.schedule_handles = {
+        'scouter': {'test-schedule': MagicMock(delete=AsyncMock())},
+        'laborious': {'test-schedule-laborious': MagicMock(delete=AsyncMock())},
+    }
+
+    input_data = {
+        'schedules': {
+            'scouter': ['test-schedule', 'test-schedule_no_handler'],
+            'laborious': ['test-schedule-laborious'],
+        }
+    }
+    handler_scouter = MagicMock(delete=AsyncMock())
+
+    handler_laborious = MagicMock(delete=AsyncMock())
+
+    temporal_manager.temporal_clients['scouter'].get_schedule_handle = MagicMock(
+        side_effect=[handler_scouter, None]
+    )
+
+    temporal_manager.temporal_clients['laborious'].get_schedule_handle = MagicMock(
+        side_effect=[handler_laborious]
+    )
+
+    report = await temporal_manager.delete_schedules(input_data)
+
+    handler_scouter.delete.assert_called_once()
+    handler_laborious.delete.assert_called_once()
+
+    assert report == [
+        {
+            'schedule_name': 'test-schedule',
+            'namespace': 'scouter',
+            'success': True,
+            'message': 'Schedule deleted successfully',
+        },
+        {
+            'schedule_name': 'test-schedule_no_handler',
+            'namespace': 'scouter',
+            'success': False,
+            'message': 'Schedule test-schedule_no_handler not found',
+            'attachment': ANY,
+        },
+        {
+            'schedule_name': 'test-schedule-laborious',
+            'namespace': 'laborious',
+            'success': True,
+            'message': 'Schedule deleted successfully',
+        },
+    ]
+
+
+@mark.asyncio
+async def test_delete_schedules_with_no_client(temporal_manager):
+    temporal_manager.temporal_clients = {}
+    input_data = {'schedules': {'abc': ['test-schedule']}}
+
+    try:
+        await temporal_manager.delete_schedules(input_data)
+    except Exception as e:
+        assert (
+            str(e)
+            == f'Temporal client for abc not found, clients: {temporal_manager.temporal_clients}'
+        )
+
+
+@mark.asyncio
+@patch('orchestrator.activities.temporal_manager.parse_frequency', side_effect=parse_frequency)
+@patch('orchestrator.activities.temporal_manager.Schedule')
+@patch('orchestrator.activities.temporal_manager.ScheduleActionStartWorkflow')
+@patch('orchestrator.activities.temporal_manager.ScheduleIntervalSpec')
+@patch('orchestrator.activities.temporal_manager.ScheduleSpec')
+@patch('orchestrator.activities.temporal_manager.TypedSearchAttributes')
+@patch('orchestrator.activities.temporal_manager.SearchAttributePair')
+async def test_create_schedules_default_runtime_legacy_queue(
+    _mock_search_attribute_pair,
+    _mock_typed_search_attributes,
+    _mock_schedule_spec,
+    _mock_schedule_interval_spec,
+    mock_schedule_action_start_workflow,
+    _mock_schedule,
+    _mock_parse_frequency,
+    temporal_manager,
+):
+    input_data = {
+        'schedules': {
+            'scouter': {
+                'test-schedule': {
+                    'model_id': 1,
+                    'model_name': 'test-model-name',
+                    'workflow_type': 'scouter',
+                    'frequency': '1m',
+                    'data': {'test': 'test'},
+                }
+            }
+        }
+    }
+
+    temporal_manager.temporal_clients['scouter'].create_schedule = AsyncMock()
+
+    await temporal_manager.create_schedules(input_data)
+
+    mock_schedule_action_start_workflow.assert_called_once_with(
+        'scouter',
+        input_data['schedules']['scouter']['test-schedule'],
+        id='test-schedule',
+        task_queue='scouter-legacy-queue',
+        execution_timeout=timedelta(seconds=300),
+        run_timeout=timedelta(seconds=300),
+        task_timeout=timedelta(seconds=300),
+        typed_search_attributes=_mock_typed_search_attributes.return_value,
+    )
+
+
+@mark.asyncio
+@patch('orchestrator.activities.temporal_manager.parse_frequency', side_effect=parse_frequency)
+@patch('orchestrator.activities.temporal_manager.Schedule')
+@patch('orchestrator.activities.temporal_manager.ScheduleActionStartWorkflow')
+@patch('orchestrator.activities.temporal_manager.ScheduleIntervalSpec')
+@patch('orchestrator.activities.temporal_manager.ScheduleSpec')
+@patch('orchestrator.activities.temporal_manager.TypedSearchAttributes')
+@patch('orchestrator.activities.temporal_manager.SearchAttributePair')
+async def test_create_schedules_tenant_runtime_queue(
+    _mock_search_attribute_pair,
+    _mock_typed_search_attributes,
+    _mock_schedule_spec,
+    _mock_schedule_interval_spec,
+    mock_schedule_action_start_workflow,
+    _mock_schedule,
+    _mock_parse_frequency,
+    temporal_manager,
+):
+    input_data = {
+        'schedules': {
+            'scouter': {
+                'test-schedule': {
+                    'model_id': 1,
+                    'model_name': 'test-model-name',
+                    'workflow_type': 'scouter',
+                    'frequency': '1m',
+                    'runtime': 'tenant-x',
+                    'data': {'test': 'test'},
+                }
+            }
+        }
+    }
+
+    temporal_manager.temporal_clients['scouter'].create_schedule = AsyncMock()
+
+    await temporal_manager.create_schedules(input_data)
+
+    mock_schedule_action_start_workflow.assert_called_once_with(
+        'scouter',
+        input_data['schedules']['scouter']['test-schedule'],
+        id='test-schedule',
+        task_queue='scouter-tenant-x-queue',
+        execution_timeout=timedelta(seconds=300),
+        run_timeout=timedelta(seconds=300),
+        task_timeout=timedelta(seconds=300),
+        typed_search_attributes=_mock_typed_search_attributes.return_value,
+    )

tests/orchestrator/utils/test_connectors_config.py

@@ -0,0 +1,174 @@
+from os import environ
+
+from orchestrator.utils.connectors_config import (
+    build_couchbase_config,
+    build_email_config,
+    build_mongodb_config,
+    build_postgres_config,
+    build_redis_config,
+    build_temporal_config,
+)
+
+
+def test_build_redis_config_with_env_vars():
+    environ['REDIS_HOST'] = 'localhost'
+    environ['REDIS_PORT'] = '6379'
+    environ['REDIS_USERNAME'] = 'sientia'
+    environ['REDIS_PASSWORD'] = 'sientia'
+    assert build_redis_config() == {
+        'host': 'localhost',
+        'port': 6379,
+        'username': 'sientia',
+        'password': 'sientia',
+    }
+
+
+def test_build_couchbase_config_with_env_vars():
+    environ['COUCHBASE_CONNECTION_STRING'] = 'couchbase://localhost'
+    environ['COUCHBASE_USERNAME'] = 'sientia'
+    environ['COUCHBASE_PASSWORD'] = 'sientia'
+    assert build_couchbase_config() == {
+        'connection_string': 'couchbase://localhost',
+        'username': 'sientia',
+        'password': 'sientia',
+    }
+
+
+def test_build_redis_config_with_defaults():
+    environ.pop('REDIS_HOST', None)
+    environ.pop('REDIS_PORT', None)
+    environ.pop('REDIS_USERNAME', None)
+    environ.pop('REDIS_PASSWORD', None)
+    assert build_redis_config() == {
+        'host': 'localhost',
+        'port': 6379,
+        'username': 'default',
+        'password': 'bdnZOpcyiL',
+    }
+
+
+def test_build_couchbase_config_with_defaults():
+    environ.pop('COUCHBASE_CONNECTION_STRING', None)
+    environ.pop('COUCHBASE_USERNAME', None)
+    environ.pop('COUCHBASE_PASSWORD', None)
+    assert build_couchbase_config() == {
+        'connection_string': 'couchbase://localhost',
+        'username': 'sientia',
+        'password': 'sientia',
+    }
+
+
+def test_build_mongo_db_config_with_env_vars():
+    environ['MONGODB_USERNAME'] = 'sientia1'
+    environ['MONGODB_PASSWORD'] = 'sientia1'
+    environ['MONGODB_URL'] = 'localhost:27018'
+    environ['MONGODB_DATABASE_NAME'] = 'test_db'
+    environ['MONGODB_TTL_INDEX_HOURS'] = '2'
+    assert build_mongodb_config() == {
+        'connection_string': 'mongodb://sientia1:sientia1@localhost:27018',
+        'database_name': 'test_db',
+        'ttl_index_seconds': 7200,
+    }
+
+
+def test_build_mongo_db_config_with_defaults():
+    environ.pop('MONGODB_USERNAME', None)
+    environ.pop('MONGODB_PASSWORD', None)
+    environ.pop('MONGODB_DATABASE_NAME', None)
+    environ.pop('MONGODB_URL', None)
+    environ.pop('MONGODB_TTL_INDEX_HOURS', None)
+    assert build_mongodb_config() == {
+        'connection_string': 'mongodb://root:wKZDbMNU1c@localhost:27018',
+        'database_name': 'sientia',
+        'ttl_index_seconds': 3600,
+    }
+
+
+def test_build_temporal_config_with_env_vars():
+    environ['TEMPORAL_HOST'] = 'localhost:7233'
+    environ['TEMPORAL_SCOUTER_NAMESPACE'] = 'scouter'
+    environ['TEMPORAL_LABORIOUS_NAMESPACE'] = 'laborious'
+    assert build_temporal_config() == {
+        'temporal_host': 'localhost:7233',
+        'temporal_namespace': 'default',
+        'temporal_scouter_namespace': 'scouter',
+        'temporal_laborious_namespace': 'laborious',
+    }
+
+
+def test_build_temporal_config_with_defaults():
+    environ.pop('TEMPORAL_HOST', None)
+    environ.pop('TEMPORAL_SCOUTER_NAMESPACE', None)
+    environ.pop('TEMPORAL_LABORIOUS_NAMESPACE', None)
+    assert build_temporal_config() == {
+        'temporal_host': 'localhost:7233',
+        'temporal_namespace': 'default',
+        'temporal_scouter_namespace': 'scouter',
+        'temporal_laborious_namespace': 'laborious',
+    }
+
+
+def test_build_email_config_with_env_vars():
+    environ['EMAIL_SENDER'] = 'test@test.com'
+    environ['EMAIL_SENDER_PASSWORD'] = 'test'
+    environ['EMAIL_SMTP_SERVER'] = 'test'
+    environ['EMAIL_SMTP_PORT'] = '587'
+    assert build_email_config() == {
+        'sender_email': 'test@test.com',
+        'sender_password': 'test',
+        'smtp_server': 'test',
+        'smtp_port': 587,
+    }
+
+
+def test_build_email_config_with_defaults():
+    environ.pop('EMAIL_SENDER', None)
+    environ.pop('EMAIL_SENDER_PASSWORD', None)
+    environ.pop('EMAIL_SMTP_SERVER', None)
+    environ.pop('EMAIL_SMTP_PORT', None)
+
+    assert build_email_config() == {
+        'sender_email': 'sientia-alerts@aignosi.com',
+        'sender_password': 'sientia',
+        'smtp_server': None,
+        'smtp_port': 587,
+    }
+
+
+def test_build_postgres_config_with_env_vars():
+    environ['POSTGRES_HOST'] = 'localhost'
+    environ['POSTGRES_PORT'] = '5432'
+    environ['POSTGRES_USER'] = 'sientia'
+    environ['POSTGRES_PASSWORD'] = 'sientia'
+    environ['POSTGRES_DBNAME'] = 'sientia'
+    environ['POSTGRES_MIN_CONNECTIONS'] = '5'
+    environ['POSTGRES_MAX_CONNECTIONS'] = '20'
+    assert build_postgres_config() == {
+        'host': 'localhost',
+        'port': 5432,
+        'user': 'sientia',
+        'password': 'sientia',
+        'dbname': 'sientia',
+        'min_connections': 5,
+        'max_connections': 20,
+    }
+
+
+def test_build_postgres_config_with_defaults():
+    environ.pop('POSTGRES_HOST', None)
+    environ.pop('POSTGRES_PORT', None)
+    environ.pop('POSTGRES_USER', None)
+    environ.pop('POSTGRES_PASSWORD', None)
+    environ.pop('POSTGRES_DBNAME', None)
+    environ.pop('POSTGRES_MIN_CONNECTIONS', None)
+    environ.pop('POSTGRES_MAX_CONNECTIONS', None)
+
+    assert build_postgres_config() == {
+        'host': 'localhost',
+        'port': 5432,
+        'user': 'sientia',
+        'password': 'sientia',
+        'dbname': 'sientia',
+        'min_connections': 5,
+        'max_connections': 20,
+    }

tests/orchestrator/utils/test_converters.py

@@ -0,0 +1,15 @@
+from orchestrator.utils.converters import parse_frequency
+
+
+def test_parse_frequency():
+    assert parse_frequency('1s') == 1
+    assert parse_frequency('1m') == 60
+    assert parse_frequency('1h') == 60 * 60
+    assert parse_frequency('1d') == 60 * 60 * 24
+
+    try:
+        parse_frequency('1')
+    except ValueError as e:
+        assert str(e) == 'Invalid frequency'
+    else:
+        raise AssertionError('Expected an exception to be raised')

tests/orchestrator/utils/test_email_builder.py

@@ -0,0 +1,198 @@
+from unittest.mock import MagicMock, patch
+
+from pytest import fixture
+
+from orchestrator.utils.email_builder import EmailBuilder
+
+
+@fixture
+@patch('orchestrator.utils.email_builder.open')
+def report_builder(open_mock):
+    return EmailBuilder(MagicMock())
+
+
+@patch('orchestrator.utils.email_builder.Template')
+def test_replace_parameters(template, report_builder):
+    output = report_builder.replace_parameters('template', {'key': 'value'})
+
+    assert output == template.return_value.render.return_value
+
+    template.assert_called_once_with('template')
+    template.return_value.render.assert_called_once_with({'key': 'value'})
+
+
+def test_parameters(report_builder):
+    report_builder.replace_parameters = MagicMock()
+    general_events = {
+        'ERROR': {
+            'models': [
+                {
+                    'model_name': 'model_name',
+                    'events': [{'notification_id': 'ID_1', 'level': 'ERROR', 'project': 'project'}],
+                },
+            ]
+        },
+        'WARNING': {
+            'models': [
+                {
+                    'model_name': 'model_name',
+                    'events': [
+                        {'notification_id': 'ID_2', 'level': 'WARNING', 'project': 'project'}
+                    ],
+                },
+            ]
+        },
+        'INFO': {
+            'models': [
+                {
+                    'model_name': 'model_name',
+                    'events': [{'notification_id': 'ID_3', 'level': 'INFO', 'project': 'project'}],
+                },
+            ]
+        },
+    }
+
+    output = report_builder.parameters(general_events, 'model_name')
+
+    assert output == {
+        'mail_type': 'model_name',
+        'error_events': report_builder.replace_parameters.return_value,
+        'warning_events': report_builder.replace_parameters.return_value,
+        'info_events': report_builder.replace_parameters.return_value,
+    }
+
+    report_builder.replace_parameters.assert_any_call(
+        report_builder.general_template,
+        {
+            'models': [
+                {
+                    'model_name': 'model_name',
+                    'events': [{'notification_id': 'ID_1', 'level': 'ERROR', 'project': 'project'}],
+                },
+            ]
+        },
+    )
+    report_builder.replace_parameters.assert_any_call(
+        report_builder.general_template,
+        {
+            'models': [
+                {
+                    'model_name': 'model_name',
+                    'events': [
+                        {'notification_id': 'ID_2', 'level': 'WARNING', 'project': 'project'}
+                    ],
+                },
+            ]
+        },
+    )
+    report_builder.replace_parameters.assert_any_call(
+        report_builder.general_template,
+        {
+            'models': [
+                {
+                    'model_name': 'model_name',
+                    'events': [{'notification_id': 'ID_3', 'level': 'INFO', 'project': 'project'}],
+                },
+            ]
+        },
+    )
+
+
+def test_build_email(report_builder):
+    report_builder.parameters = MagicMock()
+    report_builder.replace_parameters = MagicMock()
+
+    report_data = [
+        {
+            'notification_id': 'ID_1',
+            'level': 'ERROR',
+            'project': 'project',
+            'model_name': 'model_name',
+        },
+        {
+            'notification_id': 'ID_2',
+            'level': 'WARNING',
+            'project': 'project',
+            'model_name': 'model_name',
+        },
+        {
+            'notification_id': 'ID_2',
+            'level': 'INFO',
+            'project': 'project',
+            'model_name': 'model_name',
+        },
+        {
+            'notification_id': 'ID_3',
+            'level': 'ERROR',
+            'project': 'project',
+            'model_name': 'model_name',
+        },
+    ]
+
+    html = report_builder.build_email(report_data, 'type_1')
+
+    report_builder.replace_parameters.assert_called_once_with(
+        report_builder.report_template, report_builder.parameters.return_value
+    )
+
+    assert html == report_builder.replace_parameters.return_value
+
+    report_builder.parameters.assert_called_once_with(
+        {
+            'ERROR': {
+                'section_name': 'Errors detected:',
+                'models': [
+                    {
+                        'model_name': 'model_name',
+                        'events': [
+                            {
+                                'notification_id': 'ID_1',
+                                'level': 'ERROR',
+                                'project': 'project',
+                                'model_name': 'model_name',
+                            },
+                            {
+                                'notification_id': 'ID_3',
+                                'level': 'ERROR',
+                                'project': 'project',
+                                'model_name': 'model_name',
+                            },
+                        ],
+                    }
+                ],
+            },
+            'WARNING': {
+                'section_name': 'Warnings detected:',
+                'models': [
+                    {
+                        'model_name': 'model_name',
+                        'events': [
+                            {
+                                'notification_id': 'ID_2',
+                                'level': 'WARNING',
+                                'project': 'project',
+                                'model_name': 'model_name',
+                            }
+                        ],
+                    }
+                ],
+            },
+            'INFO': {
+                'section_name': 'Infos detected:',
+                'models': [
+                    {
+                        'model_name': 'model_name',
+                        'events': [
+                            {
+                                'notification_id': 'ID_2',
+                                'level': 'INFO',
+                                'project': 'project',
+                                'model_name': 'model_name',
+                            }
+                        ],
+                    }
+                ],
+            },
+        },
+        'type_1',
+    )

tests/orchestrator/utils/test_orchestrator_functions.py

@@ -0,0 +1,764 @@
+from unittest.mock import call, patch
+
+from orchestrator.utils.orchestrator_functions import (
+    base_scouter,
+    build_tag_config,
+    common_config,
+    drift,
+    gather_read_tags,
+    minimal_retrain,
+    overlap_filter_config,
+    pi_web_api_scouter,
+    predictions_batch,
+    process_path_priority,
+    scouter,
+    simple_metrics,
+)
+
+
+def test_common_config():
+    config = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'model_id': 'test_model_id',
+        'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
+    }
+    result = common_config(config)
+    expected = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'frequency': '1m',
+        'offset': '0m',
+        'max_retry_policy': 1,
+        'model_id': 'test_model_id',
+        'model_name': 'test_model_name',
+        'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
+        'execution_timeout_seconds': 300,
+        'task_timeout_seconds': 300,
+        'runtime': 'legacy',
+    }
+    assert result == expected
+
+
+def test_common_config_preserves_explicit_runtime():
+    config = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'model_id': 'test_model_id',
+        'model': {'name': 'test_model_name'},
+        'runtime': 'tenant-x',
+    }
+    assert common_config(config)['runtime'] == 'tenant-x'
+
+
+def test_drift():
+    config = {
+        'workflow_type': 'drift',
+        'schedule_name': 'test_schedule',
+        'model_id': 'test_model_id',
+        'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
+        'interval_minutes': 120,
+        'drift_metrics': ['kolmogorov_smirnov', 'jensen_shannon'],
+    }
+    result = drift(config)
+    expected = {
+        'workflow_type': 'drift',
+        'schedule_name': 'test_schedule',
+        'frequency': '1m',
+        'offset': '0m',
+        'max_retry_policy': 1,
+        'model_id': 'test_model_id',
+        'model_name': 'test_model_name',
+        'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
+        'schema': 'sientia_data',
+        'source_table_name': 'laborious_data',
+        'target_table_name': 'drift_metrics',
+        'interval': 120,
+        'drift_metrics': ['kolmogorov_smirnov', 'jensen_shannon'],
+        'execution_timeout_seconds': 300,
+        'task_timeout_seconds': 300,
+        'runtime': 'legacy',
+    }
+    assert result == expected
+
+
+def test_simple_metrics():
+    config = {
+        'workflow_type': 'simple_metrics',
+        'schedule_name': 'test_schedule',
+        'model_id': 'test_model_id',
+        'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
+        'interval_minutes': 120,
+        'metrics': ['rmse', 'mse'],
+    }
+    result = simple_metrics(config)
+    expected = {
+        'workflow_type': 'simple_metrics',
+        'schedule_name': 'test_schedule',
+        'frequency': '1m',
+        'offset': '0m',
+        'max_retry_policy': 1,
+        'model_id': 'test_model_id',
+        'model_name': 'test_model_name',
+        'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
+        'schema': 'sientia_data',
+        'predictions_table_name': 'predictions',
+        'data_table_name': 'laborious_data',
+        'target_table_name': 'simple_metrics',
+        'interval_minutes': 120,
+        'metrics': ['rmse', 'mse'],
+        'execution_timeout_seconds': 300,
+        'task_timeout_seconds': 300,
+        'runtime': 'legacy',
+    }
+    assert result == expected
+
+
+def test_minimal_retrain():
+    config = {
+        'workflow_type': 'minimal_retrain',
+        'schedule_name': 'test_schedule',
+        'model_id': 'test_model_id',
+        'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
+        'query': 'select * from sientia_data.laborious_data order by "timestamp" desc limit 30;',
+        'datetime_columns': ['timestamp'],
+    }
+    result = minimal_retrain(config)
+    expected = {
+        'workflow_type': 'minimal_retrain',
+        'schedule_name': 'test_schedule',
+        'frequency': '1m',
+        'offset': '0m',
+        'max_retry_policy': 1,
+        'model_id': 'test_model_id',
+        'model_name': 'test_model_name',
+        'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
+        'query': 'select * from sientia_data.laborious_data order by "timestamp" desc limit 30;',
+        'schema': 'sientia_data',
+        'table_name': 'log_retrain',
+        'datetime_columns': ['timestamp'],
+        'execution_timeout_seconds': 300,
+        'task_timeout_seconds': 300,
+        'runtime': 'legacy',
+    }
+    assert result == expected
+
+
+def test_scouter():
+    config = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'model_id': 'test_model_id',
+        'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
+        'filters': [{'filter_name': 'test_filter_name', 'policy': 'test_policy'}],
+        'read_tags': [
+            {'tag_name': 'test_tag_name', 'aggr_func': 'test_aggr_func', 'data_range': [1, 2]}
+        ],
+        'tag_retention_minutes': 10,
+        'execution_timeout_seconds': 300,
+        'task_timeout_seconds': 300,
+        'runtime': 'legacy',
+    }
+    result = scouter(config)
+    expected = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'frequency': '1m',
+        'offset': '0m',
+        'max_retry_policy': 1,
+        'model_id': 'test_model_id',
+        'model_name': 'test_model_name',
+        'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
+        'topic': 'raw_test_schedule',
+        'trigger_laborious': False,
+        'filters': {'test_filter_name': {'policy': 'test_policy'}},
+        'schema': 'sientia_data',
+        'table_name': 'laborious_data',
+        'retention_time': 10 * 60,
+        'model_tags': {'test_tag_name': {'aggr_func': 'test_aggr_func', 'data_range': [1, 2]}},
+        'debug_data_package': False,
+        'execution_timeout_seconds': 300,
+        'task_timeout_seconds': 300,
+        'fill_missing_tags': False,
+        'runtime': 'legacy',
+    }
+    assert result == expected
+
+
+def test_overlap_filter_config():
+    config = [
+        {'filter_name': 'test_filter_name', 'policy': 'test_policy'},
+        {'filter_name': 'test_filter_name2', 'policy': 'test_policy2'},
+    ]
+    result = overlap_filter_config({'test_filter_name': {'policy': 'test_policy'}}, config)
+    expected = {
+        'test_filter_name': {'policy': 'test_policy', 'config': {}},
+        'test_filter_name2': {'policy': 'test_policy2', 'config': {}},
+    }
+    assert result == expected
+
+
+def test_process_path_priority():
+    config = ['OTHER', 'STOP', 'CONTINUE']
+    result = process_path_priority(config)
+    expected = ['STOP', 'CONTINUE', 'REPEAT']
+    assert result == expected
+
+
+@patch(
+    'orchestrator.utils.orchestrator_functions.overlap_filter_config',
+    return_value={'test_filter_name': {'policy': 'test_policy', 'config': {}}},
+)
+@patch(
+    'orchestrator.utils.orchestrator_functions.process_path_priority',
+    return_value=['STOP', 'CONTINUE', 'REPEAT'],
+)
+def test_predictions_batch(mock_process_path_priority, mock_overlap_filter_config):
+    config = {
+        'schedule_name': 'test_schedule',
+        'workflow_type': 'predictions_batch',
+        'model_id': 'test_model_id',
+        'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
+        'query': 'test_query',
+        'write_tags': [
+            {'server_id': 'test_server_id', 'type': 'prediction', 'addr': 'test_addr'},
+            {'server_id': 'test_server_id', 'type': 'confidence', 'addr': 'test_addr'},
+        ],
+        'pi_web_api_output_config': {
+            'endpoint': 'test_endpoint',
+            'prediction_tags': {
+                'tag_1': 'webid_1',
+            },
+            'confidence_tags': {
+                'tag_2': 'webid_2',
+            },
+        },
+        'input_filters': [{'filter_name': 'test_filter_name', 'policy': 'test_policy'}],
+        'mlflow_transform_filters': [{'filter_name': 'test_filter_name', 'policy': 'test_policy'}],
+        'mlflow_predict_filters': [{'filter_name': 'test_filter_name', 'policy': 'test_policy'}],
+        'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
+        'datetime_columns': ['timestamp'],
+        'predictions_storage_policy': 'erl:1',
+    }
+
+    result = predictions_batch(config)
+
+    mock_overlap_filter_config.assert_has_calls(
+        [call({'EMPTY_DATA': {'policy': 'STOP', 'config': {}}}, config['input_filters'])]
+    )
+    mock_overlap_filter_config.assert_has_calls(
+        [
+            call(
+                {
+                    'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
+                    'API_ERROR': {'policy': 'STOP', 'config': {}},
+                },
+                config['mlflow_transform_filters'],
+            )
+        ]
+    )
+    mock_overlap_filter_config.assert_has_calls(
+        [call({'API_ERROR': {'policy': 'STOP', 'config': {}}}, config['mlflow_predict_filters'])]
+    )
+    mock_process_path_priority.assert_called_once_with(config['path_priority'])
+
+    expected = {
+        'workflow_type': 'predictions_batch',
+        'schedule_name': 'test_schedule',
+        'frequency': '1m',
+        'offset': '0m',
+        'max_retry_policy': 1,
+        'model_id': 'test_model_id',
+        'model_name': 'test_model_name',
+        'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
+        'query': 'test_query',
+        'schema': 'sientia_data',
+        'table_name': 'predictions',
+        'save_transform': True,
+        'transform_table_name': 'transformed_data',
+        'retention_time': 60 * 60,
+        'opc_output_config': {
+            'test_server_id': {
+                'prediction_tags': {'test_addr': {'data_type': 'float'}},
+                'confidence_tags': {'test_addr': {'data_type': 'float'}},
+            }
+        },
+        'pi_web_api_output_config': {
+            'endpoint': 'test_endpoint',
+            'prediction_tags': {
+                'tag_1': 'webid_1',
+            },
+            'confidence_tags': {
+                'tag_2': 'webid_2',
+            },
+        },
+        'input_filters': {'test_filter_name': {'policy': 'test_policy', 'config': {}}},
+        'mlflow_transform_filters': {'test_filter_name': {'policy': 'test_policy', 'config': {}}},
+        'mlflow_predict_filters': {'test_filter_name': {'policy': 'test_policy', 'config': {}}},
+        'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
+        'datetime_columns': ['timestamp'],
+        'predictions_storage_policy': 'erl:1',
+        'execution_timeout_seconds': 300,
+        'task_timeout_seconds': 300,
+        'runtime': 'legacy',
+    }
+    assert result == expected
+
+
+def test_gather_read_tags():
+    pipelines = [
+        {
+            'workflow_type': 'scouter',
+            'schedule_name': 'test_schedule',
+            'read_tags': [
+                {
+                    'server_id': '1',
+                    'server_name': 'test_server_name',
+                    'tag_address': 'test_tag_address',
+                }
+            ],
+        },
+        {
+            'workflow_type': 'scouter',
+            'schedule_name': 'test_schedule2',
+            'read_tags': [
+                {
+                    'server_id': '2',
+                    'server_name': 'test_server_name2',
+                    'tag_address': 'test_tag_address2',
+                },
+                {
+                    'server_id': '2',
+                    'server_name': 'test_server_name2',
+                    'tag_address': 'test_tag_address3',
+                },
+            ],
+        },
+    ]
+
+    result = gather_read_tags(pipelines)
+
+    expected = {
+        '1:test_tag_address': {
+            'server_id': '1',
+            'server_name': 'test_server_name',
+            'tag_address': 'test_tag_address',
+            'topics': ['raw_test_schedule'],
+        },
+        '2:test_tag_address2': {
+            'server_id': '2',
+            'server_name': 'test_server_name2',
+            'tag_address': 'test_tag_address2',
+            'topics': ['raw_test_schedule2'],
+        },
+        '2:test_tag_address3': {
+            'server_id': '2',
+            'server_name': 'test_server_name2',
+            'tag_address': 'test_tag_address3',
+            'topics': ['raw_test_schedule2'],
+        },
+    }
+
+    assert result == expected
+
+
+def test_gather_read_tags_filters_non_scouter_workflows():
+    """Test that gather_read_tags only processes scouter workflow types and ignores others"""
+    pipelines = [
+        {
+            'workflow_type': 'scouter',
+            'schedule_name': 'test_scouter_schedule',
+            'read_tags': [
+                {
+                    'server_id': '1',
+                    'server_name': 'test_server_name',
+                    'tag_address': 'test_tag_address',
+                }
+            ],
+        },
+        {
+            'workflow_type': 'predictions_batch',
+            'schedule_name': 'test_predictions_schedule',
+            'read_tags': [
+                {
+                    'server_id': '2',
+                    'server_name': 'test_server_name2',
+                    'tag_address': 'test_tag_address_predictions',
+                }
+            ],
+        },
+        {
+            'workflow_type': 'minimal_retrain',
+            'schedule_name': 'test_retrain_schedule',
+            'read_tags': [
+                {
+                    'server_id': '3',
+                    'server_name': 'test_server_name3',
+                    'tag_address': 'test_tag_address_retrain',
+                }
+            ],
+        },
+        {
+            'workflow_type': 'pi_web_api_scouter',
+            'schedule_name': 'test_pi_web_api_schedule',
+            'read_tags': [
+                {
+                    'server_id': '4',
+                    'server_name': 'test_server_name4',
+                    'tag_address': 'test_tag_address_pi_web_api',
+                }
+            ],
+        },
+        {
+            'workflow_type': 'drift',
+            'schedule_name': 'test_drift_schedule',
+            'read_tags': [
+                {
+                    'server_id': '5',
+                    'server_name': 'test_server_name5',
+                    'tag_address': 'test_tag_address_drift',
+                }
+            ],
+        },
+        {
+            'workflow_type': 'scouter',
+            'schedule_name': 'test_scouter_schedule2',
+            'read_tags': [
+                {
+                    'server_id': '1',
+                    'server_name': 'test_server_name',
+                    'tag_address': 'test_tag_address2',
+                }
+            ],
+        },
+    ]
+
+    result = gather_read_tags(pipelines)
+
+    # Only scouter workflow types should be included
+    expected = {
+        '1:test_tag_address': {
+            'server_id': '1',
+            'server_name': 'test_server_name',
+            'tag_address': 'test_tag_address',
+            'topics': ['raw_test_scouter_schedule'],
+        },
+        '1:test_tag_address2': {
+            'server_id': '1',
+            'server_name': 'test_server_name',
+            'tag_address': 'test_tag_address2',
+            'topics': ['raw_test_scouter_schedule2'],
+        },
+    }
+
+    assert result == expected
+    # Ensure non-scouter pipelines are not included
+    assert '2:test_tag_address_predictions' not in result
+    assert '3:test_tag_address_retrain' not in result
+    assert '4:test_tag_address_pi_web_api' not in result
+    assert '5:test_tag_address_drift' not in result
+
+
+def test_build_tag_config():
+    tags = [
+        {
+            'server_id': '1',
+            'server_name': 'test_server_name',
+            'tag_address': 'test_tag_address',
+            'frequency': 1000,
+        },
+        {
+            'server_id': '2',
+            'server_name': 'test_server_name2',
+            'tag_address': 'test_tag_address2',
+            'frequency': 1000,
+        },
+        {
+            'server_id': '2',
+            'server_name': 'test_server_name2',
+            'tag_address': 'test_tag_address3',
+            'frequency': 300,
+        },
+        {
+            'server_id': '3',
+            'server_name': 'test_server_name3',
+            'tag_address': 'test_tag_address4',
+            'frequency': 200,
+        },
+    ]
+
+    opc_servers = {
+        '1': {
+            'server_name': 'test_server_name',
+            'url': 'test_url',
+            'uri': 'test_uri',
+        },
+        '2': {
+            'server_name': 'test_server_name2',
+            'url': 'test_url2',
+            'uri': 'test_uri2',
+        },
+    }
+
+    result = build_tag_config(tags, opc_servers)
+
+    expected = {
+        'test_server_name': {
+            'server_id': '1',
+            'name': 'test_server_name',
+            'url': 'test_url',
+            'server_uri': 'test_uri',
+            'cert_path': None,
+            'private_key_path': None,
+            'server_cert_path': None,
+            'subscription_period_ms': 500,
+            'tags': {
+                'test_tag_address': {
+                    'server_id': '1',
+                    'server_name': 'test_server_name',
+                    'tag_address': 'test_tag_address',
+                    'frequency': 1000,
+                },
+            },
+        },
+        'test_server_name2': {
+            'server_id': '2',
+            'name': 'test_server_name2',
+            'url': 'test_url2',
+            'server_uri': 'test_uri2',
+            'cert_path': None,
+            'private_key_path': None,
+            'server_cert_path': None,
+            'subscription_period_ms': 150,
+            'tags': {
+                'test_tag_address2': {
+                    'server_id': '2',
+                    'server_name': 'test_server_name2',
+                    'tag_address': 'test_tag_address2',
+                    'frequency': 1000,
+                },
+                'test_tag_address3': {
+                    'server_id': '2',
+                    'server_name': 'test_server_name2',
+                    'tag_address': 'test_tag_address3',
+                    'frequency': 300,
+                },
+            },
+        },
+    }
+
+    assert result == (expected, ['3'])
+
+
+def test_base_scouter():
+    config = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'model_id': 'test_model_id',
+        'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
+        'filters': [
+            {'filter_name': 'test_filter_name', 'policy': 'test_policy'},
+            {'filter_name': 'test_filter_name2', 'policy': 'test_policy2'},
+        ],
+        'tag_retention_minutes': 30,
+        'debug_data_package': True,
+        'fill_missing_tags': True,
+    }
+    result = base_scouter(config)
+    expected = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'frequency': '1m',
+        'offset': '0m',
+        'max_retry_policy': 1,
+        'model_id': 'test_model_id',
+        'model_name': 'test_model_name',
+        'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
+        'trigger_laborious': False,
+        'filters': {
+            'test_filter_name': {'policy': 'test_policy'},
+            'test_filter_name2': {'policy': 'test_policy2'},
+        },
+        'schema': 'sientia_data',
+        'table_name': 'laborious_data',
+        'retention_time': 30 * 60,
+        'debug_data_package': True,
+        'execution_timeout_seconds': 300,
+        'task_timeout_seconds': 300,
+        'fill_missing_tags': True,
+        'runtime': 'legacy',
+    }
+    assert result == expected
+
+
+def test_pi_web_api_scouter():
+    config = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'model_id': 'test_model_id',
+        'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
+        'filters': [{'filter_name': 'test_filter_name', 'policy': 'test_policy'}],
+        'read_tags': {
+            'test_tag_name': {
+                'webid': 'test_webid',
+                'aggr_func': 'test_aggr_func',
+                'data_range': [1, 2],
+            }
+        },
+        'tag_retention_minutes': 10,
+        'pi_web_api_config': {
+            'endpoint': 'https://test-endpoint.com',
+            'period': '*-2d',
+            'max_count': 5,
+            'api_timeout': 30,
+        },
+        'frequency': '1m',
+    }
+    result = pi_web_api_scouter(config)
+    expected = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'frequency': '1m',
+        'offset': '0m',
+        'max_retry_policy': 1,
+        'model_id': 'test_model_id',
+        'model_name': 'test_model_name',
+        'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
+        'trigger_laborious': False,
+        'filters': {'test_filter_name': {'policy': 'test_policy'}},
+        'schema': 'sientia_data',
+        'table_name': 'laborious_data',
+        'retention_time': 10 * 60,
+        'model_tags': {
+            'test_tag_name': {
+                'webid': 'test_webid',
+                'aggr_func': 'test_aggr_func',
+                'data_range': [1, 2],
+            }
+        },
+        'debug_data_package': False,
+        'execution_timeout_seconds': 300,
+        'task_timeout_seconds': 300,
+        'fill_missing_tags': False,
+        'pi_web_api_query': {
+            'endpoint': 'https://test-endpoint.com',
+            'period': '*-2d',
+            'max_count': 5,
+            'api_timeout': 30,
+        },
+        'runtime': 'legacy',
+    }
+    assert result == expected
+
+
+def test_pi_web_api_scouter_with_timeout_greater_than_frequency():
+    config = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'model_id': 'test_model_id',
+        'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
+        'filters': [],
+        'read_tags': {
+            'test_tag_name': {
+                'webid': 'test_webid',
+            }
+        },
+        'tag_retention_minutes': 10,
+        'pi_web_api_config': {
+            'endpoint': 'https://test-endpoint.com',
+            'api_timeout': 120,
+        },
+        'frequency': '1m',
+    }
+    result = pi_web_api_scouter(config)
+    expected = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'frequency': '1m',
+        'offset': '0m',
+        'max_retry_policy': 1,
+        'model_id': 'test_model_id',
+        'model_name': 'test_model_name',
+        'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
+        'trigger_laborious': False,
+        'filters': {},
+        'schema': 'sientia_data',
+        'table_name': 'laborious_data',
+        'retention_time': 10 * 60,
+        'model_tags': {
+            'test_tag_name': {'webid': 'test_webid', 'aggr_func': 'lts', 'data_range': [-100, 100]}
+        },
+        'debug_data_package': False,
+        'execution_timeout_seconds': 300,
+        'task_timeout_seconds': 300,
+        'fill_missing_tags': False,
+        'pi_web_api_query': {
+            'endpoint': 'https://test-endpoint.com',
+            'period': '*-1d',
+            'max_count': 1,
+            'api_timeout': 60,
+        },
+        'runtime': 'legacy',
+    }
+    assert result == expected
+
+
+def test_pi_web_api_scouter_with_no_timeout():
+    config = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'model_id': 'test_model_id',
+        'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
+        'filters': [],
+        'read_tags': {
+            'test_tag_name': {
+                'webid': 'test_webid',
+            }
+        },
+        'tag_retention_minutes': 10,
+        'pi_web_api_config': {
+            'endpoint': 'https://test-endpoint.com',
+        },
+        'frequency': '30s',
+    }
+    result = pi_web_api_scouter(config)
+    expected = {
+        'workflow_type': 'scouter',
+        'schedule_name': 'test_schedule',
+        'frequency': '30s',
+        'offset': '0m',
+        'max_retry_policy': 1,
+        'model_id': 'test_model_id',
+        'model_name': 'test_model_name',
+        'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
+        'trigger_laborious': False,
+        'filters': {},
+        'schema': 'sientia_data',
+        'table_name': 'laborious_data',
+        'retention_time': 10 * 60,
+        'model_tags': {
+            'test_tag_name': {'webid': 'test_webid', 'aggr_func': 'lts', 'data_range': [-100, 100]}
+        },
+        'debug_data_package': False,
+        'execution_timeout_seconds': 300,
+        'task_timeout_seconds': 300,
+        'fill_missing_tags': False,
+        'pi_web_api_query': {
+            'endpoint': 'https://test-endpoint.com',
+            'period': '*-1d',
+            'max_count': 1,
+            'api_timeout': 30,
+        },
+        'runtime': 'legacy',
+    }
+    assert result == expected

tests/orchestrator/worker/test_worker.py

@@ -0,0 +1,49 @@
+import asyncio
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from pytest import mark
+
+from orchestrator.worker.worker import main
+from orchestrator.workflows.alerts import Alerts
+from orchestrator.workflows.orchestrator import Orchestrator
+from orchestrator.workflows.reports import Reports
+
+
+def test_main_is_coroutine():
+    assert asyncio.iscoroutinefunction(main)
+
+
+@mark.asyncio
+@patch('orchestrator.worker.worker.sys')
+@patch('orchestrator.worker.worker.start_http_server')
+@patch('orchestrator.worker.worker.NotificationHandler')
+@patch('orchestrator.worker.worker.Activities')
+@patch('orchestrator.worker.worker.prepare_worker')
+@patch('orchestrator.worker.worker.client.Client.connect', new_callable=AsyncMock)
+async def test_main_starts_three_workers_for_expected_workflows(
+    _connect_mock,
+    prepare_worker_mock,
+    activities_mock,
+    _notification_handler_mock,
+    _start_http_server,
+    _sys_mock,
+):
+    """
+    main() must spin up exactly three Temporal workers, one per main workflow
+    (Orchestrator, Alerts, Reports), and call run() on each.
+    """
+    activities_instance = activities_mock.return_value
+    activities_instance.connect_to_temporal = AsyncMock()
+
+    worker_mock = MagicMock()
+    worker_mock.run = AsyncMock()
+    prepare_worker_mock.return_value = worker_mock
+
+    await main()
+
+    assert prepare_worker_mock.call_count == 3
+
+    main_workflows = [call.kwargs['main_workflow'] for call in prepare_worker_mock.call_args_list]
+    assert main_workflows == [Orchestrator, Alerts, Reports]
+
+    assert worker_mock.run.call_count == 3

tests/orchestrator/workflows/subworkflows/test_load_notification_package.py

@@ -0,0 +1,171 @@
+from unittest.mock import ANY, AsyncMock, call, patch
+
+from pytest import fixture, mark
+
+from orchestrator.activities.activities import Activities
+from orchestrator.workflows.subworkflows.load_notification_package import LoadNotificationPackage
+
+
+@fixture
+def load_notification_package():
+    return LoadNotificationPackage()
+
+
+metadata = {
+    'metadata': {
+        'schedule_name': 'test-schedule-name',
+        'workflow_name': 'test-workflow',
+        'model_name': '-',
+        'model_id': '-',
+    }
+}
+
+
+@mark.asyncio
+@patch(
+    'orchestrator.workflows.subworkflows.load_notification_package.workflow', new_callable=AsyncMock
+)
+async def test_run(workflow_mock, load_notification_package):
+    input_data = {
+        'metadata': metadata,
+        'mail_type': 'test_mail_type',
+        'base_data_filter': {'level': 'ERROR'},
+    }
+
+    workflow_mock.start_local_activity_method.side_effect = [
+        '2023-01-01 12:00:00',
+        [
+            {
+                'id': '1',
+            }
+        ],
+        [
+            {
+                'id_r': '1',
+            }
+        ],
+    ]
+
+    output = await load_notification_package.run(input_data)
+
+    assert output == {
+        'last_timestamp': '2023-01-01 12:00:00',
+        'notification_package': [
+            {
+                'id': '1',
+            }
+        ],
+        'sending_configs': [
+            {
+                'id_r': '1',
+            }
+        ],
+    }
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.get_last_data_timestamp,
+                {**input_data['metadata'], 'mail_type': 'test_mail_type'},
+                start_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.find_documents_in_mongodb,
+                {
+                    **input_data['metadata'],
+                    'query': {'collection': 'receiver_groups', 'filters': {'active': True}},
+                },
+                start_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.load_latest_data,
+                {
+                    **input_data['metadata'],
+                    'collection_name': 'notification_queue',
+                    'last_data_timestamp': '2023-01-01 12:00:00',
+                    'base_data_filter': {'level': 'ERROR'},
+                },
+                start_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.put_last_data_timestamp,
+                {
+                    **input_data['metadata'],
+                    'data': [
+                        {
+                            'id': '1',
+                        }
+                    ],
+                    'mail_type': 'test_mail_type',
+                },
+                start_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+
+@mark.asyncio
+@patch(
+    'orchestrator.workflows.subworkflows.load_notification_package.workflow', new_callable=AsyncMock
+)
+async def test_run_no_data(workflow_mock, load_notification_package):
+    input_data = {
+        'metadata': metadata,
+        'mail_type': 'test_mail_type',
+        'base_data_filter': {'level': 'ERROR'},
+    }
+
+    workflow_mock.start_local_activity_method.side_effect = ['2023-01-01 12:00:00', [], []]
+
+    output = await load_notification_package.run(input_data)
+
+    assert output == {
+        'last_timestamp': '2023-01-01 12:00:00',
+        'notification_package': [],
+        'sending_configs': [],
+    }
+
+    workflow_mock.start_activity_method.assert_not_called()
+
+
+@mark.asyncio
+@patch(
+    'orchestrator.workflows.subworkflows.load_notification_package.workflow', new_callable=AsyncMock
+)
+async def test_run_has_data(workflow_mock, load_notification_package):
+    input_data = {
+        'metadata': metadata,
+        'base_data_filter': {'level': 'ERROR'},
+        'mail_type': 'test_mail_type',
+    }
+
+    workflow_mock.start_local_activity_method.side_effect = ['2023-01-01 12:00:00', ['data'], []]
+
+    output = await load_notification_package.run(input_data)
+
+    assert output == {
+        'last_timestamp': '2023-01-01 12:00:00',
+        'notification_package': ['data'],
+        'sending_configs': [],
+    }
+
+    workflow_mock.start_activity_method.assert_not_called()

tests/orchestrator/workflows/subworkflows/test_process_notifications.py

@@ -0,0 +1,151 @@
+from unittest.mock import ANY, AsyncMock, call, patch
+
+from pytest import fixture, mark
+from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
+
+from orchestrator.activities.activities import Activities
+from orchestrator.workflows.subworkflows.process_notifications import ProcessNotifications
+
+
+@fixture
+def process_notifications():
+    return ProcessNotifications()
+
+
+metadata = {
+    'metadata': {
+        'schedule_name': 'test-schedule-name',
+        'workflow_name': 'test-workflow',
+        'model_name': '-',
+        'model_id': '-',
+    }
+}
+
+
+@mark.asyncio
+@patch('orchestrator.workflows.subworkflows.process_notifications.workflow', new_callable=AsyncMock)
+async def test_run(workflow_mock, process_notifications):
+    input_data = {
+        'metadata': metadata,
+        'notification_package': ['content'],
+        'mail_type': 'test_mail_type',
+        'schema': 'test_schema',
+        'table_name': 'test_table_name',
+    }
+
+    response = await process_notifications.run(input_data)
+
+    assert response == workflow_mock.execute_local_activity_method.return_value
+
+    workflow_mock.execute_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.build_email_html,
+                {
+                    **metadata,
+                    'receiver_groups': input_data['notification_package'],
+                    'mail_type': input_data['mail_type'],
+                },
+                schedule_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+    workflow_mock.execute_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.format_log_report,
+                {
+                    **metadata,
+                    'receiver_groups': workflow_mock.execute_activity_method.return_value,
+                    'mail_type': input_data['mail_type'],
+                },
+                schedule_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.execute_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.send_email,
+                {
+                    **metadata,
+                    'receiver_groups': workflow_mock.execute_local_activity_method.return_value,
+                    'mail_type': input_data['mail_type'],
+                },
+                schedule_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.execute_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.export_data_to_postgres,
+                {
+                    **metadata,
+                    'schema': input_data['schema'],
+                    'table_name': input_data['table_name'],
+                    'data': workflow_mock.execute_local_activity_method.return_value,
+                    'timestamp_conversion': {
+                        'column': 'timestamp',
+                        'format': DATETIME_FORMAT_WITH_TZ,
+                    },
+                },
+                schedule_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+
+@mark.asyncio
+@patch('orchestrator.workflows.subworkflows.process_notifications.workflow', new_callable=AsyncMock)
+async def test_run_send_email_return_empty(workflow_mock, process_notifications):
+    input_data = {
+        'metadata': metadata,
+        'notification_package': ['content'],
+        'mail_type': 'test_mail_type',
+        'schema': 'test_schema',
+        'table_name': 'test_table_name',
+    }
+
+    workflow_mock.execute_activity_method.return_value = []
+
+    await process_notifications.run(input_data)
+
+    workflow_mock.execute_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.build_email_html,
+                {
+                    **metadata,
+                    'receiver_groups': input_data['notification_package'],
+                    'mail_type': input_data['mail_type'],
+                },
+                schedule_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.execute_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.send_email,
+                {
+                    **metadata,
+                    'receiver_groups': workflow_mock.execute_local_activity_method.return_value,
+                    'mail_type': input_data['mail_type'],
+                },
+                schedule_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+    assert workflow_mock.execute_activity_method.call_count == 1
+    assert workflow_mock.execute_local_activity_method.call_count == 1

tests/orchestrator/workflows/test_alerts.py

@@ -0,0 +1,137 @@
+from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
+
+from pytest import fixture, mark
+
+from orchestrator.activities.activities import Activities
+from orchestrator.workflows.alerts import Alerts
+
+
+@fixture
+def alerts():
+    return Alerts()
+
+
+metadata = {
+    'metadata': {
+        'schedule_name': 'test-schedule-name',
+        'workflow_name': 'alerts',
+        'model_name': '-',
+        'model_id': '-',
+    }
+}
+
+
+@mark.asyncio
+@patch('orchestrator.workflows.alerts.workflow', new_callable=AsyncMock)
+async def test_run_full_flow(workflow_mock, alerts):
+    input_data = {'schedule_name': 'test-schedule-name', 'notification_ttl': 300, 'sent_ttl': 600}
+
+    await alerts.run(input_data)
+
+    workflow_mock.execute_child_workflow.assert_has_calls(
+        [
+            call(
+                'subworkflow.load_notification_package',
+                {**input_data, 'metadata': metadata, 'base_data_filter': {'level': 'ERROR'}},
+            )
+        ]
+    )
+
+    workflow_mock.execute_child_workflow.assert_has_calls(
+        [
+            call(
+                'subworkflow.process_notifications',
+                {
+                    'metadata': metadata,
+                    'mail_type': 'Alerts',
+                    'notification_package': workflow_mock.execute_local_activity_method.return_value,
+                    'schema': 'sientia_data',
+                    'table_name': 'log_report',
+                },
+            )
+        ]
+    )
+
+    workflow_mock.execute_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.filter_notification_alerts,
+                {
+                    **metadata,
+                    'notification_package': workflow_mock.execute_child_workflow.return_value[
+                        'notification_package'
+                    ],
+                    'sending_configs': workflow_mock.execute_child_workflow.return_value[
+                        'sending_configs'
+                    ],
+                    'notification_ttl': input_data['notification_ttl'],
+                },
+                schedule_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.execute_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.store_notification_cache,
+                {
+                    **metadata,
+                    'log_report': workflow_mock.execute_child_workflow.return_value,
+                    'sent_ttl': input_data['sent_ttl'],
+                },
+                schedule_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+
+@mark.asyncio
+@patch('orchestrator.workflows.alerts.workflow', new_callable=AsyncMock)
+async def test_run_no_data(workflow_mock, alerts):
+    workflow_mock.execute_child_workflow.return_value = {
+        'last_timestamp': '2023-01-01 12:00:00.000000',
+        'notification_package': [],
+        'sending_configs': [],
+    }
+
+    input_data = {'schedule_name': 'test-schedule-name', 'notification_ttl': 300, 'sent_ttl': 600}
+
+    await alerts.run(input_data)
+
+    workflow_mock.execute_child_workflow.assert_has_calls(
+        [
+            call(
+                'subworkflow.load_notification_package',
+                {**input_data, 'metadata': metadata, 'base_data_filter': {'level': 'ERROR'}},
+            )
+        ]
+    )
+
+    workflow_mock.execute_local_activity_method.assert_not_called()
+
+
+@mark.asyncio
+@patch('orchestrator.workflows.alerts.workflow', new_callable=AsyncMock)
+async def test_run_no_receiver_groups(workflow_mock, alerts):
+    workflow_mock.execute_local_activity_method.return_value = {}
+
+    input_data = {'schedule_name': 'test-schedule-name', 'notification_ttl': 300, 'sent_ttl': 600}
+
+    await alerts.run(input_data)
+
+    assert workflow_mock.execute_child_workflow.call_count == 1
+
+
+@mark.asyncio
+@patch('orchestrator.workflows.alerts.workflow', new_callable=AsyncMock)
+async def test_run_no_log_report(workflow_mock, alerts):
+    workflow_mock.execute_child_workflow.side_effect = [MagicMock(), []]
+
+    input_data = {'schedule_name': 'test-schedule-name', 'notification_ttl': 300, 'sent_ttl': 600}
+
+    await alerts.run(input_data)
+
+    workflow_mock.execute_activity_method.assert_not_called()

tests/orchestrator/workflows/test_orchestrator.py

@@ -0,0 +1,348 @@
+from unittest.mock import ANY, AsyncMock, call, patch
+
+from pytest import fixture, mark
+
+from orchestrator.activities.activities import Activities
+from orchestrator.workflows.orchestrator import Orchestrator
+
+
+@fixture
+def orchestrator():
+    return Orchestrator()
+
+
+metadata = {
+    'metadata': {
+        'schedule_name': 'test-schedule-name',
+        'workflow_name': 'orchestrator',
+        'model_name': '-',
+        'model_id': '-',
+    }
+}
+
+
+@mark.asyncio
+@patch('orchestrator.workflows.orchestrator.workflow', new_callable=AsyncMock)
+async def test_run(workflow_mock, orchestrator):
+    input_data = {
+        'pipelines_query': 'SELECT * FROM bucket',
+        'opc_servers_query': 'SELECT * FROM servers',
+        'schedule_name': 'test-schedule-name',
+    }
+
+    await orchestrator.run(input_data)
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.aggregate_documents_in_mongodb,
+                {
+                    **metadata,
+                    'query': input_data['pipelines_query'],
+                    'timestamp_fields': ['updated_at'],
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.find_documents_in_mongodb,
+                {**metadata, 'query': input_data['opc_servers_query']},
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.find_documents_in_mongodb,
+                {
+                    **metadata,
+                    'query': {'collection': 'orchestrated_schedules'},
+                    'timestamp_fields': ['updated_at'],
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.load_opc_slots,
+                {**metadata},
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.load_active_ingestors,
+                {**metadata},
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.format_schedule_config,
+                {
+                    **metadata,
+                    'schedule_config': workflow_mock.start_local_activity_method.return_value,
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.process_schedules,
+                {**metadata, 'pipelines': workflow_mock.start_local_activity_method.return_value},
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.process_slots,
+                {
+                    **metadata,
+                    'opc_servers': workflow_mock.start_local_activity_method.return_value,
+                    'active_ingestors': workflow_mock.start_local_activity_method.return_value,
+                    'pipelines': workflow_mock.start_local_activity_method.return_value,
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.create_schedule_config,
+                {
+                    **metadata,
+                    'current_schedule_config': workflow_mock.start_local_activity_method.return_value,
+                    'schedule_config': workflow_mock.start_local_activity_method.return_value,
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.create_slot_config,
+                {
+                    **metadata,
+                    'current_slot_config': workflow_mock.start_local_activity_method.return_value,
+                    'slot_config': workflow_mock.start_local_activity_method.return_value,
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.normalize_schedules,
+                {
+                    **metadata,
+                    'orchestrated_schedules': workflow_mock.start_local_activity_method.return_value,
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.create_collection_with_ttl_index,
+                {
+                    **metadata,
+                    'pipelines': workflow_mock.start_local_activity_method.return_value['scouter'],
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.delete_slots,
+                {
+                    **metadata,
+                    'to_delete': workflow_mock.start_local_activity_method.return_value[
+                        'to_delete'
+                    ],
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.update_slots,
+                {
+                    **metadata,
+                    'to_insert': workflow_mock.start_local_activity_method.return_value[
+                        'to_insert'
+                    ],
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.delete_schedules,
+                {
+                    **metadata,
+                    'schedules': workflow_mock.start_local_activity_method.return_value[
+                        'to_delete'
+                    ],
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.create_schedules,
+                {
+                    **metadata,
+                    'schedules': workflow_mock.start_local_activity_method.return_value[
+                        'to_create'
+                    ],
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.update_schedules,
+                {
+                    **metadata,
+                    'schedules': workflow_mock.start_local_activity_method.return_value[
+                        'to_update'
+                    ],
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.report_schedule_orchestration,
+                {
+                    **metadata,
+                    'created_schedules': workflow_mock.start_activity_method.return_value,
+                    'updated_schedules': workflow_mock.start_activity_method.return_value,
+                    'deleted_schedules': workflow_mock.start_activity_method.return_value,
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.report_slot_orchestration,
+                {
+                    **metadata,
+                    'inserted_slots': workflow_mock.start_activity_method.return_value,
+                    'deleted_slots': workflow_mock.start_activity_method.return_value,
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.update_pipelines_timestamps,
+                {
+                    **metadata,
+                    'updated_pipelines': workflow_mock.start_activity_method.return_value,
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.delete_pipelines_timestamps,
+                {
+                    **metadata,
+                    'deleted_pipelines': workflow_mock.start_activity_method.return_value,
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )
+
+    workflow_mock.start_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.create_pipelines_timestamps,
+                {
+                    **metadata,
+                    'created_pipelines': workflow_mock.start_activity_method.return_value,
+                },
+                retry_policy=ANY,
+                start_to_close_timeout=ANY,
+            )
+        ]
+    )

tests/orchestrator/workflows/test_reports.py

@@ -0,0 +1,109 @@
+from unittest.mock import ANY, AsyncMock, call, patch
+
+from pytest import fixture, mark
+
+from orchestrator.activities.activities import Activities
+from orchestrator.workflows.reports import Reports
+
+
+@fixture
+def reports():
+    return Reports()
+
+
+metadata = {
+    'metadata': {
+        'schedule_name': 'test-schedule-name',
+        'workflow_name': 'reports',
+        'model_name': '-',
+        'model_id': '-',
+    }
+}
+
+
+@mark.asyncio
+@patch('orchestrator.workflows.reports.workflow', new_callable=AsyncMock)
+async def test_run_full_flow(workflow_mock, reports):
+    input_data = {'schedule_name': 'test-schedule-name', 'notification_ttl': 300, 'sent_ttl': 600}
+
+    await reports.run(input_data)
+
+    workflow_mock.execute_child_workflow.assert_has_calls(
+        [
+            call(
+                'subworkflow.load_notification_package',
+                {**input_data, 'metadata': metadata, 'base_data_filter': {}},
+            )
+        ]
+    )
+
+    workflow_mock.execute_child_workflow.assert_has_calls(
+        [
+            call(
+                'subworkflow.process_notifications',
+                {
+                    'metadata': metadata,
+                    'mail_type': 'Reports',
+                    'notification_package': workflow_mock.execute_local_activity_method.return_value,
+                    'schema': 'sientia_data',
+                    'table_name': 'log_report',
+                },
+            )
+        ]
+    )
+
+    workflow_mock.execute_local_activity_method.assert_has_calls(
+        [
+            call(
+                Activities.filter_notification_reports,
+                {
+                    **metadata,
+                    'notification_package': workflow_mock.execute_child_workflow.return_value[
+                        'notification_package'
+                    ],
+                    'sending_configs': workflow_mock.execute_child_workflow.return_value[
+                        'sending_configs'
+                    ],
+                },
+                schedule_to_close_timeout=ANY,
+                retry_policy=ANY,
+            )
+        ]
+    )
+
+
+@mark.asyncio
+@patch('orchestrator.workflows.reports.workflow', new_callable=AsyncMock)
+async def test_run_no_data(workflow_mock, reports):
+    workflow_mock.execute_child_workflow.return_value = {
+        'last_timestamp': '2023-01-01 12:00:00.000000',
+        'notification_package': [],
+        'sending_configs': [],
+    }
+
+    input_data = {'schedule_name': 'test-schedule-name', 'notification_ttl': 300, 'sent_ttl': 600}
+
+    await reports.run(input_data)
+
+    workflow_mock.execute_child_workflow.assert_has_calls(
+        [
+            call(
+                'subworkflow.load_notification_package',
+                {**input_data, 'metadata': metadata, 'base_data_filter': {}},
+            )
+        ]
+    )
+
+    workflow_mock.execute_local_activity_method.assert_not_called()
+
+
+@mark.asyncio
+@patch('orchestrator.workflows.reports.workflow', new_callable=AsyncMock)
+async def test_run_no_groups(workflow_mock, reports):
+    workflow_mock.execute_local_activity_method.return_value = []
+
+    input_data = {'schedule_name': 'test-schedule-name', 'notification_ttl': 300, 'sent_ttl': 600}
+
+    await reports.run(input_data)
+
+    workflow_mock.execute_child_workflow.assert_called_once()