gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1172

Browse Source 
refactor: streamline email generation and enhance notification handling

- Updated email HTML structure to improve clarity and presentation of notifications.
- Enhanced notification generator to better manage execution counts and output formatting.
- Refactored Email class to improve error handling and logging during email sending.
- Adjusted EmailBuilder to efficiently process error, warning, and info models.
- Improved test coverage for email and notification functionalities to ensure reliability.
This commit is contained in:
vitor-aignosi
2025-07-29 10:25:31 -03:00
parent c3ba2e4b5a
commit bc456cc8be
14 changed files with 503 additions and 401 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
tests/orchestrator/activities/test_activities.py
View File

@@ -12,7 +12,12 @@ from orchestrator.activities.formatters import Formatters
@patch('orchestrator.activities.temporal_manager.TemporalManager.__init__')
@patch('orchestrator.activities.slot_manager.SlotManager.__init__')
@patch('orchestrator.activities.formatters.Formatters.__init__')
def test___init__(mock_formatters_init, mock_slot_manager_init,
@patch('orchestrator.activities.email.Email.__init__')
@patch('sientia_do.temporal.activities.postgres.Postgres.__init__')
def test___init__(mock_postgres_init,
mock_email_init,
mock_formatters_init,
mock_slot_manager_init,
mock_temporal_manager_init,
mock_mongodb_init):
@@ -35,6 +40,23 @@ def test___init__(mock_formatters_init, mock_slot_manager_init,
'temporal_laborious_namespace': 'laborious'
}
email_config = {
'sender_email': 'test@test.com',
'sender_password': 'test',
'smtp_server': 'test',
'smtp_port': 587
}
postgres_config = {
'host': 'localhost',
'port': 5432,
'user': 'admin',
'password': 'password',
'dbname': 'test_db',
'min_connections': 1,
'max_connections': 10,
}
logger = MagicMock()
notification_handler = MagicMock()
@@ -42,6 +64,8 @@ def test___init__(mock_formatters_init, mock_slot_manager_init,
temporal_config=temporal_config,
redis_config=redis_config,
mongodb_config=mongo_db_config,
email_config=email_config,
postgres_config=postgres_config,
logger=logger,
notification_handler=notification_handler
)

28
tests/orchestrator/activities/test_email.py
View File

@@ -11,8 +11,8 @@ def email(smtplib, email_builder):
email = Email(
sender_email="test@test.com",
sender_password="test",
smpt_server="test",
port=587,
smtp_server="test",
smtp_port=587,
logger=MagicMock(),
notification_handler=MagicMock()
)
@@ -27,21 +27,22 @@ def test___init___with_password(smtplib, email_builder):
email = Email(
sender_email="test@test.com",
sender_password="test",
smpt_server="test",
port=587,
smtp_server="test",
smtp_port=587,
logger=MagicMock(),
notification_handler=MagicMock()
)
assert email.sender_email == "test@test.com"
assert email.sender_password == "test"
assert email.port == 587
assert email.smtp_port == 587
smtplib.SMTP_SSL.assert_called_once_with("test", 587)
smtplib.SMTP_SSL.return_value.login.assert_called_once_with(
smtplib.SMTP.assert_called_once_with("test", 587)
smtplib.SMTP.return_value.starttls.assert_called_once()
smtplib.SMTP.return_value.login.assert_called_once_with(
"test@test.com", "test")
assert email.server == smtplib.SMTP_SSL.return_value
assert email.server == smtplib.SMTP.return_value
@patch('orchestrator.activities.email.EmailBuilder')
@@ -50,15 +51,15 @@ def test___init___without_password(smtplib, email_builder):
email = Email(
sender_email="test@test.com",
sender_password=None,
smpt_server="test",
port=587,
smtp_server="test",
smtp_port=587,
logger=MagicMock(),
notification_handler=MagicMock()
)
assert email.sender_email == "test@test.com"
assert email.sender_password is None
assert email.port == 587
assert email.smtp_port == 587
smtplib.SMTP.assert_called_once_with("test", 587)
assert email.server == smtplib.SMTP.return_value
@@ -189,7 +190,10 @@ def test_handle_attachments_failure(mime_base, email):
}
]
email.handle_attachments(attachments, message)
try:
email.handle_attachments(attachments, message)
except Exception as e:
assert str(e) == "test"
assert message.attach.call_count == 0

32
tests/orchestrator/activities/test_mongo_db.py
View File

@@ -555,17 +555,11 @@ async def test_load_latest_data_none_last_data_timestamp(mongo_db):
{"_id": 0}
)
assert result == {
'name': {
0: 'test1'
},
'value': {
0: 1
},
'timestamp': {
0: '2023-01-01 12:00:00.000000'
}
}
assert result == [{
'name': 'test1',
'value': 1,
'timestamp': '2023-01-01 12:00:00.000000'
}]
@mark.asyncio
@@ -606,17 +600,11 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
{"_id": 0}
)
assert result == {
'name': {
0: 'test1'
},
'value': {
0: 1
},
'timestamp': {
0: '2023-01-01 12:00:00.000000'
}
}
assert result == [{
'name': 'test1',
'value': 1,
'timestamp': '2023-01-01 12:00:00.000000'
}]
@mark.asyncio

71
tests/orchestrator/utils/test_connectors_config.py
View File

@@ -1,7 +1,10 @@
from os import environ
from orchestrator.utils.connectors_config import (build_redis_config,
build_couchbase_config,
build_mongodb_config, build_temporal_config)
build_mongodb_config,
build_temporal_config,
build_email_config,
build_postgres_config)
def test_build_redis_config_with_env_vars():
@@ -100,3 +103,69 @@ def test_build_temporal_config_with_defaults():
'temporal_scouter_namespace': 'scouter',
'temporal_laborious_namespace': 'laborious'
}
def test_build_email_config_with_env_vars():
environ['EMAIL_SENDER'] = 'test@test.com'
environ['EMAIL_SENDER_PASSWORD'] = 'test'
environ['EMAIL_SMTP_SERVER'] = 'test'
environ['EMAIL_SMTP_PORT'] = '587'
assert build_email_config() == {
'sender_email': 'test@test.com',
'sender_password': 'test',
'smtp_server': 'test',
'smtp_port': 587
}
def test_build_email_config_with_defaults():
environ.pop('EMAIL_SENDER', None)
environ.pop('EMAIL_SENDER_PASSWORD', None)
environ.pop('EMAIL_SMTP_SERVER', None)
environ.pop('EMAIL_SMTP_PORT', None)
assert build_email_config() == {
'sender_email': 'sientia-alerts@aignosi.com',
'sender_password': 'sientia',
'smtp_server': 'smtp.gmail.com',
'smtp_port': 587
}
def test_build_postgres_config_with_env_vars():
environ['POSTGRES_HOST'] = 'localhost'
environ['POSTGRES_PORT'] = '5432'
environ['POSTGRES_USER'] = 'sientia'
environ['POSTGRES_PASSWORD'] = 'sientia'
environ['POSTGRES_DBNAME'] = 'sientia'
environ['POSTGRES_MIN_CONNECTIONS'] = '5'
environ['POSTGRES_MAX_CONNECTIONS'] = '20'
assert build_postgres_config() == {
'host': 'localhost',
'port': 5432,
'user': 'sientia',
'password': 'sientia',
'dbname': 'sientia',
'min_connections': 5,
'max_connections': 20
}
def test_build_postgres_config_with_defaults():
environ.pop('POSTGRES_HOST', None)
environ.pop('POSTGRES_PORT', None)
environ.pop('POSTGRES_USER', None)
environ.pop('POSTGRES_PASSWORD', None)
environ.pop('POSTGRES_DBNAME', None)
environ.pop('POSTGRES_MIN_CONNECTIONS', None)
environ.pop('POSTGRES_MAX_CONNECTIONS', None)
assert build_postgres_config() == {
'host': 'localhost',
'port': 5432,
'user': 'sientia',
'password': 'sientia',
'dbname': 'sientia',
'min_connections': 5,
'max_connections': 20
}

59
tests/orchestrator/workflows/subworkflows/test_load_notification_package.py
View File

@@ -23,7 +23,10 @@ metadata = {
@patch("orchestrator.workflows.subworkflows.load_notification_package.workflow", new_callable=AsyncMock)
async def test_run(workflow_mock, load_notification_package):
input_data = {
'metadata': metadata
'metadata': metadata,
'base_data_filter': {
'level': 'ERROR'
}
}
workflow_mock.start_local_activity_method.side_effect = [
@@ -113,3 +116,57 @@ async def test_run(workflow_mock, load_notification_package):
retry_policy=ANY
)
])
@mark.asyncio
@patch("orchestrator.workflows.subworkflows.load_notification_package.workflow", new_callable=AsyncMock)
async def test_run_no_data(workflow_mock, load_notification_package):
input_data = {
'metadata': metadata,
'base_data_filter': {
'level': 'ERROR'
}
}
workflow_mock.start_local_activity_method.side_effect = [
'2023-01-01 12:00:00',
[],
[]
]
output = await load_notification_package.run(input_data)
assert output == {
'last_timestamp': '2023-01-01 12:00:00',
'notification_package': [],
'sending_configs': []
}
workflow_mock.start_activity_method.assert_not_called()
@mark.asyncio
@patch("orchestrator.workflows.subworkflows.load_notification_package.workflow", new_callable=AsyncMock)
async def test_run_no_data(workflow_mock, load_notification_package):
input_data = {
'metadata': metadata,
'base_data_filter': {
'level': 'ERROR'
}
}
workflow_mock.start_local_activity_method.side_effect = [
'2023-01-01 12:00:00',
["data"],
[]
]
output = await load_notification_package.run(input_data)
assert output == {
'last_timestamp': '2023-01-01 12:00:00',
'notification_package': ["data"],
'sending_configs': []
}
workflow_mock.start_activity_method.assert_not_called()

9
tests/orchestrator/workflows/subworkflows/test_process_notifications.py
View File

@@ -39,11 +39,14 @@ async def test_run(workflow_mock, process_notifications):
Activities.build_email_html,
{
**metadata,
'receiver_groups': input_data['notification_package']
'receiver_groups': input_data['notification_package'],
'mail_type': input_data['mail_type']
},
schedule_to_close_timeout=ANY,
retry_policy=ANY
),
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
call(
Activities.format_log_report,
{
@@ -51,6 +54,8 @@ async def test_run(workflow_mock, process_notifications):
'receiver_groups': workflow_mock.execute_activity_method.return_value,
'mail_type': input_data['mail_type']
},
schedule_to_close_timeout=ANY,
retry_policy=ANY
)
])

39
tests/orchestrator/workflows/test_alerts.py
View File

@@ -64,7 +64,9 @@ async def test_run_full_flow(workflow_mock, alerts):
'notification_package': workflow_mock.execute_child_workflow.return_value['notification_package'],
'sending_configs': workflow_mock.execute_child_workflow.return_value['sending_configs'],
'notification_ttl': input_data['notification_ttl']
}
},
schedule_to_close_timeout=ANY,
retry_policy=ANY
)
])
@@ -75,6 +77,41 @@ async def test_run_full_flow(workflow_mock, alerts):
**metadata,
'log_report': workflow_mock.execute_child_workflow.return_value,
'sent_ttl': input_data['sent_ttl']
},
schedule_to_close_timeout=ANY,
retry_policy=ANY
)
])
@mark.asyncio
@patch("orchestrator.workflows.alerts.workflow", new_callable=AsyncMock)
async def test_run_no_data(workflow_mock, alerts):
workflow_mock.execute_child_workflow.return_value = {
'last_timestamp': '2023-01-01 12:00:00.000000',
'notification_package': [],
'sending_configs': []
}
input_data = {
'schedule_name': 'test-schedule-name',
'notification_ttl': 300,
'sent_ttl': 600
}
await alerts.run(input_data)
workflow_mock.execute_child_workflow.assert_has_calls([
call(
'load_notification_package',
{
**input_data,
'metadata': metadata,
'base_data_filter': {
'level': 'ERROR'
}
}
)
])
workflow_mock.execute_local_activity_method.assert_not_called()