diff --git a/email.html b/email.html index 4207ca5..c75e731 100644 --- a/email.html +++ b/email.html @@ -15,231 +15,11 @@

SIENTIA™ Alerts

-

Errors detected:

+

-

Model: do elit

- - - - - - - - - - - - - - - - - - - -
Notification IDBlockTimestampMessage
REPORT_PARTITION_MANAGERlorem_adipiscing2025-07-28 20:05:38.650661+00:00incididunt dolor lorem incididunt elit incididunt dolor consectetur ipsum ut
+

-

Model: dolor magna

- - - - - - - - - - - - - - - - - - - -
Notification IDBlockTimestampMessage
REPORT_PARTITION_MANAGERtempor_dolore_eiusmod2025-07-28 20:05:38.650910+00:00ipsum amet ipsum aliqua et eiusmod
- -

Model: labore sit lorem

- - - - - - - - - - - - - - - - - - - -
Notification IDBlockTimestampMessage
OPC_CONNECTION_RETRY__server_6dolor_labore_lorem2025-07-28 20:05:38.650949+00:00et adipiscing ipsum ut aliqua ut adipiscing aliqua amet
- -

Warnings detected:

- -

Model: incididunt adipiscing elit sit

- - - - - - - - - - - - - - - - - - - -
Notification IDBlockTimestampMessage
ADIPISCING_AMET_IPSUMeiusmod_aliqua_incididunt2025-07-28 20:05:38.650621+00:00magna incididunt dolore sed eiusmod sit incididunt ut
- -

Model: do adipiscing et

- - - - - - - - - - - - - - - - - - - -
Notification IDBlockTimestampMessage
TAG_node46:tag_73_LISTENNING_STOPPEDamet_dolor2025-07-28 20:05:38.650706+00:00dolore do et consectetur adipiscing do magna eiusmod eiusmod
- -

Model: lorem dolore amet

- - - - - - - - - - - - - - - - - - - -
Notification IDBlockTimestampMessage
TAG_node13:tag_59_LISTENNING_STOPPEDipsum_tempor_ipsum2025-07-28 20:05:38.650785+00:00et sed adipiscing tempor dolor magna do lorem
- -

Model: eiusmod sed

- - - - - - - - - - - - - - - - - - - -
Notification IDBlockTimestampMessage
OPC_SUBSCRIPTION__server_2do_ipsum2025-07-28 20:05:38.650826+00:00eiusmod et consectetur do dolore dolor sed amet
- -

Infos detected:

- -

Model: et sed

- - - - - - - - - - - - - - - - - - - -
Notification IDBlockTimestampMessage
OPC_LISTENNING_STOPPED__server_6labore_do_ipsum_ipsum2025-07-28 20:05:38.650570+00:00tempor dolore eiusmod sit sit sed elit
- -

Model: dolore sit

- - - - - - - - - - - - - - - - - - - -
Notification IDBlockTimestampMessage
REPORT_PARTITION_MANAGERet_lorem_incididunt_ipsum2025-07-28 20:05:38.650742+00:00adipiscing magna adipiscing adipiscing sit tempor
- -

Model: adipiscing do

- - - - - - - - - - - - - - - - - - - -
Notification IDBlockTimestampMessage
TAG_node76:tag_73_LISTENNING_STOPPEDtempor_elit2025-07-28 20:05:38.650875+00:00et do aliqua lorem incididunt
+

diff --git a/notification_generator.ipynb b/notification_generator.ipynb index 19a64c4..8d24f92 100644 --- a/notification_generator.ipynb +++ b/notification_generator.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 13, + "execution_count": 58, "id": "1786cbf0", "metadata": {}, "outputs": [ @@ -86,7 +86,144 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 59, + "id": "8a67c90d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'WARNING': {'section_name': 'Warnings detected:', 'models': [{'model_name': 'amet sed consectetur', 'events': [{'project': 'sientia-laborious', 'pipeline': 'elit_adipiscing_magna', 'trigger': 'eiusmod_do_incididunt_dolor_aliqua', 'model_name': 'amet sed consectetur', 'model_id': '8', 'block': 'do_amet_sit', 'level': 'WARNING', 'message': 'aliqua lorem elit magna eiusmod', 'attachment_content': 'sit ut consectetur dolore ut ipsum eiusmod tempor incididunt sit dolore elit aliqua amet amet aliqua magna dolore ut amet lorem aliqua dolore eiusmod', 'notification_id': 'OPC_CONNECTION_RETRY__server_9', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121908, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'sit dolor lorem', 'events': [{'project': 'sientia-laborious', 'pipeline': 'consectetur_adipiscing_incididunt', 'trigger': 'labore_ut_amet_dolor', 'model_name': 'sit dolor lorem', 'model_id': '44', 'block': 'adipiscing_amet_aliqua_eiusmod', 'level': 'WARNING', 'message': 'consectetur elit incididunt dolore magna ut sit sit adipiscing', 'attachment_content': 'lorem ipsum elit eiusmod do lorem et do do eiusmod eiusmod dolore eiusmod lorem consectetur amet ipsum eiusmod dolor dolore aliqua adipiscing dolore magna ut', 'notification_id': 'OPC_LISTENNING_BACK__server_5', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121982, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'ut labore ipsum ut', 'events': [{'project': 'sientia-laborious', 'pipeline': 'lorem_elit', 'trigger': 'magna_sed_tempor_labore', 'model_name': 'ut labore ipsum ut', 'model_id': '99', 'block': 'sed_dolor_do', 'level': 'WARNING', 'message': 'magna magna incididunt ut incididunt incididunt consectetur', 'attachment_content': 'sed lorem ut amet labore incididunt tempor incididunt do lorem aliqua dolore adipiscing labore labore tempor sit eiusmod tempor aliqua dolor ut sed incididunt adipiscing', 'notification_id': 'OPC_QUEUE_RETRIEVE__server_2', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 122025, tzinfo=datetime.timezone.utc)}]}]}, 'ERROR': {'section_name': 'Errors detected:', 'models': [{'model_name': 'ut elit et eiusmod', 'events': [{'project': 'sientia-laborious', 'pipeline': 'do_magna_aliqua_aliqua', 'trigger': 'dolore_sit_sed', 'model_name': 'ut elit et eiusmod', 'model_id': '20', 'block': 'do_magna', 'level': 'ERROR', 'message': 'dolor aliqua labore labore eiusmod do', 'attachment_content': 'eiusmod sed magna amet tempor labore magna adipiscing incididunt tempor ipsum ipsum aliqua magna tempor do tempor consectetur amet aliqua lorem sed lorem ut consectetur', 'notification_id': 'REPORT_PARTITION_MANAGER', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121924, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'dolore aliqua eiusmod eiusmod', 'events': [{'project': 'sientia-laborious', 'pipeline': 'aliqua_dolor_elit', 'trigger': 'consectetur_incididunt_adipiscing', 'model_name': 'dolore aliqua eiusmod eiusmod', 'model_id': '97', 'block': 'consectetur_magna_do', 'level': 'ERROR', 'message': 'lorem eiusmod et aliqua aliqua sit', 'attachment_content': 'amet lorem labore amet eiusmod do do sit adipiscing ut adipiscing sed sed do ipsum tempor incididunt dolor sed et ipsum amet incididunt sed', 'notification_id': 'REPORT_PARTITION_MANAGER', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121953, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'incididunt eiusmod amet eiusmod', 'events': [{'project': 'sientia-laborious', 'pipeline': 'ipsum_et', 'trigger': 'elit_amet_lorem_dolore_elit', 'model_name': 'incididunt eiusmod amet eiusmod', 'model_id': '31', 'block': 'amet_tempor_tempor', 'level': 'ERROR', 'message': 'ut dolor consectetur amet elit do elit', 'attachment_content': 'amet ut adipiscing lorem adipiscing ut et amet adipiscing incididunt elit magna et aliqua incididunt lorem incididunt lorem incididunt eiusmod sit amet tempor lorem', 'notification_id': 'ELIT_ALIQUA_TEMPOR', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 122040, tzinfo=datetime.timezone.utc)}]}]}, 'INFO': {'section_name': 'Infos detected:', 'models': [{'model_name': 'do magna', 'events': [{'project': 'sientia-laborious', 'pipeline': 'tempor_amet', 'trigger': 'incididunt_adipiscing_incididunt_eiusmod_incididunt_tempor', 'model_name': 'do magna', 'model_id': '29', 'block': 'incididunt_ipsum', 'level': 'INFO', 'message': 'sit labore consectetur labore magna labore eiusmod ipsum', 'attachment_content': 'ipsum amet et ipsum lorem consectetur incididunt dolore consectetur et elit eiusmod elit do eiusmod amet consectetur adipiscing adipiscing lorem eiusmod magna dolor eiusmod sit eiusmod lorem amet magna consectetur', 'notification_id': 'OPC_SUBSCRIPTION__server_4', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121940, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'amet adipiscing ut', 'events': [{'project': 'sientia-laborious', 'pipeline': 'aliqua_dolore_adipiscing_adipiscing', 'trigger': 'ut_incididunt_ut_et_tempor_incididunt', 'model_name': 'amet adipiscing ut', 'model_id': '28', 'block': 'do_sed', 'level': 'INFO', 'message': 'consectetur elit amet dolor incididunt dolor dolor labore adipiscing', 'attachment_content': 'do incididunt aliqua tempor dolor sed lorem elit consectetur et tempor labore elit sed elit dolor sed amet ipsum do eiusmod aliqua adipiscing et sit', 'notification_id': 'TAG_node56:tag_30_LISTENNING_STOPPED', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121968, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'sed lorem et lorem', 'events': [{'project': 'sientia-laborious', 'pipeline': 'do_ipsum_consectetur_ut', 'trigger': 'dolore_incididunt_consectetur_et', 'model_name': 'sed lorem et lorem', 'model_id': '82', 'block': 'tempor_incididunt', 'level': 'INFO', 'message': 'lorem tempor eiusmod dolore tempor aliqua amet sed', 'attachment_content': 'sed dolore amet ut eiusmod aliqua sed dolore incididunt consectetur adipiscing et elit magna consectetur ipsum ipsum ipsum dolor eiusmod do', 'notification_id': 'REPORT_PARTITION_MANAGER', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121994, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'eiusmod elit incididunt', 'events': [{'project': 'sientia-laborious', 'pipeline': 'sit_dolore', 'trigger': 'eiusmod_sed_amet_ut', 'model_name': 'eiusmod elit incididunt', 'model_id': '85', 'block': 'amet_incididunt', 'level': 'INFO', 'message': 'do consectetur incididunt magna dolore et et elit incididunt', 'attachment_content': 'tempor eiusmod sit elit consectetur ut consectetur eiusmod labore labore consectetur ut eiusmod sit dolor ipsum sit do et amet lorem lorem et do ut magna dolor', 'notification_id': 'REPORT_PARTITION_MANAGER', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 122010, tzinfo=datetime.timezone.utc)}]}]}}\n" + ] + } + ], + "source": [ + "from orchestrator.utils.email_builder import EmailBuilder\n", + "from unittest.mock import MagicMock\n", + "\n", + "\n", + "email_builder = EmailBuilder(logger=MagicMock())\n", + "\n", + "\n", + "general_events = {}\n", + "\n", + "for report in notifications:\n", + "\n", + " level = report['level']\n", + " model_name = report['model_name']\n", + "\n", + " if level not in general_events:\n", + " general_events[level] = {\n", + " 'section_name': f'{level.capitalize()}s detected:',\n", + " 'models': {}\n", + " }\n", + "\n", + " if model_name not in general_events[level]['models']:\n", + " general_events[level]['models'][model_name] = {\n", + " 'model_name': model_name,\n", + " 'events': []\n", + " }\n", + "\n", + " general_events[level]['models'][model_name]['events'].append(\n", + " report)\n", + "\n", + "for _type, content in general_events.items():\n", + " content['models'] = list(content['models'].values())\n", + "\n", + "print(general_events)" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "6944697a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'model_name': 'ut elit et eiusmod', 'events': [{'project': 'sientia-laborious', 'pipeline': 'do_magna_aliqua_aliqua', 'trigger': 'dolore_sit_sed', 'model_name': 'ut elit et eiusmod', 'model_id': '20', 'block': 'do_magna', 'level': 'ERROR', 'message': 'dolor aliqua labore labore eiusmod do', 'attachment_content': 'eiusmod sed magna amet tempor labore magna adipiscing incididunt tempor ipsum ipsum aliqua magna tempor do tempor consectetur amet aliqua lorem sed lorem ut consectetur', 'notification_id': 'REPORT_PARTITION_MANAGER', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121924, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'dolore aliqua eiusmod eiusmod', 'events': [{'project': 'sientia-laborious', 'pipeline': 'aliqua_dolor_elit', 'trigger': 'consectetur_incididunt_adipiscing', 'model_name': 'dolore aliqua eiusmod eiusmod', 'model_id': '97', 'block': 'consectetur_magna_do', 'level': 'ERROR', 'message': 'lorem eiusmod et aliqua aliqua sit', 'attachment_content': 'amet lorem labore amet eiusmod do do sit adipiscing ut adipiscing sed sed do ipsum tempor incididunt dolor sed et ipsum amet incididunt sed', 'notification_id': 'REPORT_PARTITION_MANAGER', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121953, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'incididunt eiusmod amet eiusmod', 'events': [{'project': 'sientia-laborious', 'pipeline': 'ipsum_et', 'trigger': 'elit_amet_lorem_dolore_elit', 'model_name': 'incididunt eiusmod amet eiusmod', 'model_id': '31', 'block': 'amet_tempor_tempor', 'level': 'ERROR', 'message': 'ut dolor consectetur amet elit do elit', 'attachment_content': 'amet ut adipiscing lorem adipiscing ut et amet adipiscing incididunt elit magna et aliqua incididunt lorem incididunt lorem incididunt eiusmod sit amet tempor lorem', 'notification_id': 'ELIT_ALIQUA_TEMPOR', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 122040, tzinfo=datetime.timezone.utc)}]}]\n", + "[{'model_name': 'amet sed consectetur', 'events': [{'project': 'sientia-laborious', 'pipeline': 'elit_adipiscing_magna', 'trigger': 'eiusmod_do_incididunt_dolor_aliqua', 'model_name': 'amet sed consectetur', 'model_id': '8', 'block': 'do_amet_sit', 'level': 'WARNING', 'message': 'aliqua lorem elit magna eiusmod', 'attachment_content': 'sit ut consectetur dolore ut ipsum eiusmod tempor incididunt sit dolore elit aliqua amet amet aliqua magna dolore ut amet lorem aliqua dolore eiusmod', 'notification_id': 'OPC_CONNECTION_RETRY__server_9', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121908, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'sit dolor lorem', 'events': [{'project': 'sientia-laborious', 'pipeline': 'consectetur_adipiscing_incididunt', 'trigger': 'labore_ut_amet_dolor', 'model_name': 'sit dolor lorem', 'model_id': '44', 'block': 'adipiscing_amet_aliqua_eiusmod', 'level': 'WARNING', 'message': 'consectetur elit incididunt dolore magna ut sit sit adipiscing', 'attachment_content': 'lorem ipsum elit eiusmod do lorem et do do eiusmod eiusmod dolore eiusmod lorem consectetur amet ipsum eiusmod dolor dolore aliqua adipiscing dolore magna ut', 'notification_id': 'OPC_LISTENNING_BACK__server_5', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121982, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'ut labore ipsum ut', 'events': [{'project': 'sientia-laborious', 'pipeline': 'lorem_elit', 'trigger': 'magna_sed_tempor_labore', 'model_name': 'ut labore ipsum ut', 'model_id': '99', 'block': 'sed_dolor_do', 'level': 'WARNING', 'message': 'magna magna incididunt ut incididunt incididunt consectetur', 'attachment_content': 'sed lorem ut amet labore incididunt tempor incididunt do lorem aliqua dolore adipiscing labore labore tempor sit eiusmod tempor aliqua dolor ut sed incididunt adipiscing', 'notification_id': 'OPC_QUEUE_RETRIEVE__server_2', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 122025, tzinfo=datetime.timezone.utc)}]}]\n", + "[{'model_name': 'do magna', 'events': [{'project': 'sientia-laborious', 'pipeline': 'tempor_amet', 'trigger': 'incididunt_adipiscing_incididunt_eiusmod_incididunt_tempor', 'model_name': 'do magna', 'model_id': '29', 'block': 'incididunt_ipsum', 'level': 'INFO', 'message': 'sit labore consectetur labore magna labore eiusmod ipsum', 'attachment_content': 'ipsum amet et ipsum lorem consectetur incididunt dolore consectetur et elit eiusmod elit do eiusmod amet consectetur adipiscing adipiscing lorem eiusmod magna dolor eiusmod sit eiusmod lorem amet magna consectetur', 'notification_id': 'OPC_SUBSCRIPTION__server_4', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121940, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'amet adipiscing ut', 'events': [{'project': 'sientia-laborious', 'pipeline': 'aliqua_dolore_adipiscing_adipiscing', 'trigger': 'ut_incididunt_ut_et_tempor_incididunt', 'model_name': 'amet adipiscing ut', 'model_id': '28', 'block': 'do_sed', 'level': 'INFO', 'message': 'consectetur elit amet dolor incididunt dolor dolor labore adipiscing', 'attachment_content': 'do incididunt aliqua tempor dolor sed lorem elit consectetur et tempor labore elit sed elit dolor sed amet ipsum do eiusmod aliqua adipiscing et sit', 'notification_id': 'TAG_node56:tag_30_LISTENNING_STOPPED', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121968, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'sed lorem et lorem', 'events': [{'project': 'sientia-laborious', 'pipeline': 'do_ipsum_consectetur_ut', 'trigger': 'dolore_incididunt_consectetur_et', 'model_name': 'sed lorem et lorem', 'model_id': '82', 'block': 'tempor_incididunt', 'level': 'INFO', 'message': 'lorem tempor eiusmod dolore tempor aliqua amet sed', 'attachment_content': 'sed dolore amet ut eiusmod aliqua sed dolore incididunt consectetur adipiscing et elit magna consectetur ipsum ipsum ipsum dolor eiusmod do', 'notification_id': 'REPORT_PARTITION_MANAGER', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121994, tzinfo=datetime.timezone.utc)}]}, {'model_name': 'eiusmod elit incididunt', 'events': [{'project': 'sientia-laborious', 'pipeline': 'sit_dolore', 'trigger': 'eiusmod_sed_amet_ut', 'model_name': 'eiusmod elit incididunt', 'model_id': '85', 'block': 'amet_incididunt', 'level': 'INFO', 'message': 'do consectetur incididunt magna dolore et et elit incididunt', 'attachment_content': 'tempor eiusmod sit elit consectetur ut consectetur eiusmod labore labore consectetur ut eiusmod sit dolor ipsum sit do et amet lorem lorem et do ut magna dolor', 'notification_id': 'REPORT_PARTITION_MANAGER', 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 122010, tzinfo=datetime.timezone.utc)}]}]\n" + ] + } + ], + "source": [ + "error_models = general_events.get('ERROR', {}).get('models', [])\n", + "warning_models = general_events.get('WARNING', {}).get('models', [])\n", + "info_models = general_events.get('INFO', {}).get('models', [])\n", + "mail_type = 'Alerts'\n", + "\n", + "print(error_models)\n", + "print(warning_models)\n", + "print(info_models)" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "02e7dabf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'

Errors detected:

\\n\\n

Model: ut elit et eiusmod

\\n\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
Notification IDScheduleBlockTimestampMessage
REPORT_PARTITION_MANAGERdolore_sit_seddo_magna2025-07-29 12:21:45.121924+00:00dolor aliqua labore labore eiusmod do
\\n\\n

Model: dolore aliqua eiusmod eiusmod

\\n\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
Notification IDScheduleBlockTimestampMessage
REPORT_PARTITION_MANAGERconsectetur_incididunt_adipiscingconsectetur_magna_do2025-07-29 12:21:45.121953+00:00lorem eiusmod et aliqua aliqua sit
\\n\\n

Model: incididunt eiusmod amet eiusmod

\\n\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
Notification IDScheduleBlockTimestampMessage
ELIT_ALIQUA_TEMPORelit_amet_lorem_dolore_elitamet_tempor_tempor2025-07-29 12:21:45.122040+00:00ut dolor consectetur amet elit do elit
\\n'" + ] + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "email_builder.replace_parameters(email_builder.general_template,\n", + " general_events.get('ERROR', {})) if general_events else ''" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "bebde0ea", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'mail_type': 'Alerts',\n", + " 'error_events': '

\\n',\n", + " 'warning_events': '

\\n',\n", + " 'info_events': '

\\n'}" + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "\n", + "\n", + "\n", + "d = {\n", + " 'mail_type': mail_type,\n", + " 'error_events': email_builder.replace_parameters(email_builder.general_template,\n", + " error_models) if error_models else '',\n", + " 'warning_events': email_builder.replace_parameters(email_builder.general_template,\n", + " warning_models) if warning_models else '',\n", + " 'info_events': email_builder.replace_parameters(email_builder.general_template,\n", + " info_models) if info_models else '',\n", + "}\n", + "\n", + "d" + ] + }, + { + "cell_type": "code", + "execution_count": 63, "id": "c1467b72", "metadata": {}, "outputs": [], @@ -105,7 +242,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 64, "id": "ffd7be77", "metadata": {}, "outputs": [ @@ -113,118 +250,118 @@ "data": { "text/plain": [ "[{'project': 'sientia-laborious',\n", - " 'pipeline': 'magna_incididunt',\n", - " 'trigger': 'dolor_dolor_aliqua_ipsum_do_magna',\n", - " 'model_name': 'et sed',\n", - " 'model_id': '53',\n", - " 'block': 'labore_do_ipsum_ipsum',\n", - " 'level': 'INFO',\n", - " 'message': 'tempor dolore eiusmod sit sit sed elit',\n", - " 'attachment_content': 'sit labore sed amet sit magna ut ut labore do ut adipiscing amet incididunt magna do sit eiusmod dolor do sit incididunt amet incididunt consectetur elit et ipsum elit amet',\n", - " 'notification_id': 'OPC_LISTENNING_STOPPED__server_6',\n", - " 'timestamp': datetime.datetime(2025, 7, 28, 20, 5, 38, 650570, tzinfo=datetime.timezone.utc)},\n", - " {'project': 'sientia-laborious',\n", - " 'pipeline': 'adipiscing_do',\n", - " 'trigger': 'sit_lorem_dolore',\n", - " 'model_name': 'incididunt adipiscing elit sit',\n", - " 'model_id': '64',\n", - " 'block': 'eiusmod_aliqua_incididunt',\n", + " 'pipeline': 'elit_adipiscing_magna',\n", + " 'trigger': 'eiusmod_do_incididunt_dolor_aliqua',\n", + " 'model_name': 'amet sed consectetur',\n", + " 'model_id': '8',\n", + " 'block': 'do_amet_sit',\n", " 'level': 'WARNING',\n", - " 'message': 'magna incididunt dolore sed eiusmod sit incididunt ut',\n", - " 'attachment_content': 'incididunt incididunt adipiscing do ipsum tempor ut eiusmod sit tempor tempor aliqua aliqua dolor labore incididunt amet do sed tempor dolor labore',\n", - " 'notification_id': 'ADIPISCING_AMET_IPSUM',\n", - " 'timestamp': datetime.datetime(2025, 7, 28, 20, 5, 38, 650621, tzinfo=datetime.timezone.utc)},\n", + " 'message': 'aliqua lorem elit magna eiusmod',\n", + " 'attachment_content': 'sit ut consectetur dolore ut ipsum eiusmod tempor incididunt sit dolore elit aliqua amet amet aliqua magna dolore ut amet lorem aliqua dolore eiusmod',\n", + " 'notification_id': 'OPC_CONNECTION_RETRY__server_9',\n", + " 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121908, tzinfo=datetime.timezone.utc)},\n", " {'project': 'sientia-laborious',\n", - " 'pipeline': 'sed_aliqua',\n", - " 'trigger': 'aliqua_adipiscing_adipiscing_sed_do_sit',\n", - " 'model_name': 'do elit',\n", - " 'model_id': '73',\n", - " 'block': 'lorem_adipiscing',\n", + " 'pipeline': 'do_magna_aliqua_aliqua',\n", + " 'trigger': 'dolore_sit_sed',\n", + " 'model_name': 'ut elit et eiusmod',\n", + " 'model_id': '20',\n", + " 'block': 'do_magna',\n", " 'level': 'ERROR',\n", - " 'message': 'incididunt dolor lorem incididunt elit incididunt dolor consectetur ipsum ut',\n", - " 'attachment_content': 'dolor ut do amet dolore dolor do eiusmod eiusmod incididunt aliqua et sed labore incididunt aliqua sed et tempor consectetur elit et consectetur do ipsum',\n", + " 'message': 'dolor aliqua labore labore eiusmod do',\n", + " 'attachment_content': 'eiusmod sed magna amet tempor labore magna adipiscing incididunt tempor ipsum ipsum aliqua magna tempor do tempor consectetur amet aliqua lorem sed lorem ut consectetur',\n", " 'notification_id': 'REPORT_PARTITION_MANAGER',\n", - " 'timestamp': datetime.datetime(2025, 7, 28, 20, 5, 38, 650661, tzinfo=datetime.timezone.utc)},\n", + " 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121924, tzinfo=datetime.timezone.utc)},\n", " {'project': 'sientia-laborious',\n", - " 'pipeline': 'dolore_et_aliqua_et',\n", - " 'trigger': 'adipiscing_amet_sit_do',\n", - " 'model_name': 'do adipiscing et',\n", - " 'model_id': '10',\n", - " 'block': 'amet_dolor',\n", - " 'level': 'WARNING',\n", - " 'message': 'dolore do et consectetur adipiscing do magna eiusmod eiusmod',\n", - " 'attachment_content': 'sit adipiscing adipiscing consectetur magna sed magna adipiscing dolor incididunt et adipiscing adipiscing elit et labore tempor tempor adipiscing sit dolore sed dolore dolor labore sed',\n", - " 'notification_id': 'TAG_node46:tag_73_LISTENNING_STOPPED',\n", - " 'timestamp': datetime.datetime(2025, 7, 28, 20, 5, 38, 650706, tzinfo=datetime.timezone.utc)},\n", - " {'project': 'sientia-laborious',\n", - " 'pipeline': 'ipsum_lorem_sit_sit',\n", - " 'trigger': 'et_ut_sit_dolore',\n", - " 'model_name': 'dolore sit',\n", - " 'model_id': '58',\n", - " 'block': 'et_lorem_incididunt_ipsum',\n", + " 'pipeline': 'tempor_amet',\n", + " 'trigger': 'incididunt_adipiscing_incididunt_eiusmod_incididunt_tempor',\n", + " 'model_name': 'do magna',\n", + " 'model_id': '29',\n", + " 'block': 'incididunt_ipsum',\n", " 'level': 'INFO',\n", - " 'message': 'adipiscing magna adipiscing adipiscing sit tempor',\n", - " 'attachment_content': 'incididunt amet dolor incididunt dolore et lorem amet ut magna tempor amet do consectetur dolore magna elit aliqua eiusmod ipsum labore',\n", + " 'message': 'sit labore consectetur labore magna labore eiusmod ipsum',\n", + " 'attachment_content': 'ipsum amet et ipsum lorem consectetur incididunt dolore consectetur et elit eiusmod elit do eiusmod amet consectetur adipiscing adipiscing lorem eiusmod magna dolor eiusmod sit eiusmod lorem amet magna consectetur',\n", + " 'notification_id': 'OPC_SUBSCRIPTION__server_4',\n", + " 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121940, tzinfo=datetime.timezone.utc)},\n", + " {'project': 'sientia-laborious',\n", + " 'pipeline': 'aliqua_dolor_elit',\n", + " 'trigger': 'consectetur_incididunt_adipiscing',\n", + " 'model_name': 'dolore aliqua eiusmod eiusmod',\n", + " 'model_id': '97',\n", + " 'block': 'consectetur_magna_do',\n", + " 'level': 'ERROR',\n", + " 'message': 'lorem eiusmod et aliqua aliqua sit',\n", + " 'attachment_content': 'amet lorem labore amet eiusmod do do sit adipiscing ut adipiscing sed sed do ipsum tempor incididunt dolor sed et ipsum amet incididunt sed',\n", " 'notification_id': 'REPORT_PARTITION_MANAGER',\n", - " 'timestamp': datetime.datetime(2025, 7, 28, 20, 5, 38, 650742, tzinfo=datetime.timezone.utc)},\n", + " 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121953, tzinfo=datetime.timezone.utc)},\n", " {'project': 'sientia-laborious',\n", - " 'pipeline': 'dolor_sit_aliqua',\n", - " 'trigger': 'lorem_incididunt_labore',\n", - " 'model_name': 'lorem dolore amet',\n", - " 'model_id': '13',\n", - " 'block': 'ipsum_tempor_ipsum',\n", - " 'level': 'WARNING',\n", - " 'message': 'et sed adipiscing tempor dolor magna do lorem',\n", - " 'attachment_content': 'do incididunt dolore eiusmod adipiscing amet amet amet eiusmod lorem dolor labore labore ipsum dolore amet labore ipsum labore do amet',\n", - " 'notification_id': 'TAG_node13:tag_59_LISTENNING_STOPPED',\n", - " 'timestamp': datetime.datetime(2025, 7, 28, 20, 5, 38, 650785, tzinfo=datetime.timezone.utc)},\n", - " {'project': 'sientia-laborious',\n", - " 'pipeline': 'lorem_do_consectetur',\n", - " 'trigger': 'sit_aliqua_sit',\n", - " 'model_name': 'eiusmod sed',\n", - " 'model_id': '9',\n", - " 'block': 'do_ipsum',\n", - " 'level': 'WARNING',\n", - " 'message': 'eiusmod et consectetur do dolore dolor sed amet',\n", - " 'attachment_content': 'sed dolore dolore et et dolore dolor eiusmod aliqua adipiscing incididunt dolor lorem tempor consectetur dolor amet elit et aliqua lorem ipsum consectetur tempor tempor elit labore labore dolore',\n", - " 'notification_id': 'OPC_SUBSCRIPTION__server_2',\n", - " 'timestamp': datetime.datetime(2025, 7, 28, 20, 5, 38, 650826, tzinfo=datetime.timezone.utc)},\n", - " {'project': 'sientia-laborious',\n", - " 'pipeline': 'labore_elit_adipiscing',\n", - " 'trigger': 'dolore_incididunt_tempor_lorem_dolore',\n", - " 'model_name': 'adipiscing do',\n", - " 'model_id': '63',\n", - " 'block': 'tempor_elit',\n", + " 'pipeline': 'aliqua_dolore_adipiscing_adipiscing',\n", + " 'trigger': 'ut_incididunt_ut_et_tempor_incididunt',\n", + " 'model_name': 'amet adipiscing ut',\n", + " 'model_id': '28',\n", + " 'block': 'do_sed',\n", " 'level': 'INFO',\n", - " 'message': 'et do aliqua lorem incididunt',\n", - " 'attachment_content': 'ut eiusmod consectetur sit labore ut tempor et sit sit do incididunt incididunt sed eiusmod dolor adipiscing dolor ut sed amet labore amet amet sed et elit sed ipsum',\n", - " 'notification_id': 'TAG_node76:tag_73_LISTENNING_STOPPED',\n", - " 'timestamp': datetime.datetime(2025, 7, 28, 20, 5, 38, 650875, tzinfo=datetime.timezone.utc)},\n", + " 'message': 'consectetur elit amet dolor incididunt dolor dolor labore adipiscing',\n", + " 'attachment_content': 'do incididunt aliqua tempor dolor sed lorem elit consectetur et tempor labore elit sed elit dolor sed amet ipsum do eiusmod aliqua adipiscing et sit',\n", + " 'notification_id': 'TAG_node56:tag_30_LISTENNING_STOPPED',\n", + " 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121968, tzinfo=datetime.timezone.utc)},\n", " {'project': 'sientia-laborious',\n", - " 'pipeline': 'ut_lorem_et',\n", - " 'trigger': 'labore_labore_ipsum_magna',\n", - " 'model_name': 'dolor magna',\n", - " 'model_id': '74',\n", - " 'block': 'tempor_dolore_eiusmod',\n", - " 'level': 'ERROR',\n", - " 'message': 'ipsum amet ipsum aliqua et eiusmod',\n", - " 'attachment_content': 'incididunt dolore do sit sed ut sed sit magna sed amet elit elit consectetur dolor labore magna consectetur tempor magna',\n", + " 'pipeline': 'consectetur_adipiscing_incididunt',\n", + " 'trigger': 'labore_ut_amet_dolor',\n", + " 'model_name': 'sit dolor lorem',\n", + " 'model_id': '44',\n", + " 'block': 'adipiscing_amet_aliqua_eiusmod',\n", + " 'level': 'WARNING',\n", + " 'message': 'consectetur elit incididunt dolore magna ut sit sit adipiscing',\n", + " 'attachment_content': 'lorem ipsum elit eiusmod do lorem et do do eiusmod eiusmod dolore eiusmod lorem consectetur amet ipsum eiusmod dolor dolore aliqua adipiscing dolore magna ut',\n", + " 'notification_id': 'OPC_LISTENNING_BACK__server_5',\n", + " 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121982, tzinfo=datetime.timezone.utc)},\n", + " {'project': 'sientia-laborious',\n", + " 'pipeline': 'do_ipsum_consectetur_ut',\n", + " 'trigger': 'dolore_incididunt_consectetur_et',\n", + " 'model_name': 'sed lorem et lorem',\n", + " 'model_id': '82',\n", + " 'block': 'tempor_incididunt',\n", + " 'level': 'INFO',\n", + " 'message': 'lorem tempor eiusmod dolore tempor aliqua amet sed',\n", + " 'attachment_content': 'sed dolore amet ut eiusmod aliqua sed dolore incididunt consectetur adipiscing et elit magna consectetur ipsum ipsum ipsum dolor eiusmod do',\n", " 'notification_id': 'REPORT_PARTITION_MANAGER',\n", - " 'timestamp': datetime.datetime(2025, 7, 28, 20, 5, 38, 650910, tzinfo=datetime.timezone.utc)},\n", + " 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 121994, tzinfo=datetime.timezone.utc)},\n", " {'project': 'sientia-laborious',\n", - " 'pipeline': 'magna_sit_amet_sed',\n", - " 'trigger': 'ut_dolor_sed',\n", - " 'model_name': 'labore sit lorem',\n", - " 'model_id': '13',\n", - " 'block': 'dolor_labore_lorem',\n", + " 'pipeline': 'sit_dolore',\n", + " 'trigger': 'eiusmod_sed_amet_ut',\n", + " 'model_name': 'eiusmod elit incididunt',\n", + " 'model_id': '85',\n", + " 'block': 'amet_incididunt',\n", + " 'level': 'INFO',\n", + " 'message': 'do consectetur incididunt magna dolore et et elit incididunt',\n", + " 'attachment_content': 'tempor eiusmod sit elit consectetur ut consectetur eiusmod labore labore consectetur ut eiusmod sit dolor ipsum sit do et amet lorem lorem et do ut magna dolor',\n", + " 'notification_id': 'REPORT_PARTITION_MANAGER',\n", + " 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 122010, tzinfo=datetime.timezone.utc)},\n", + " {'project': 'sientia-laborious',\n", + " 'pipeline': 'lorem_elit',\n", + " 'trigger': 'magna_sed_tempor_labore',\n", + " 'model_name': 'ut labore ipsum ut',\n", + " 'model_id': '99',\n", + " 'block': 'sed_dolor_do',\n", + " 'level': 'WARNING',\n", + " 'message': 'magna magna incididunt ut incididunt incididunt consectetur',\n", + " 'attachment_content': 'sed lorem ut amet labore incididunt tempor incididunt do lorem aliqua dolore adipiscing labore labore tempor sit eiusmod tempor aliqua dolor ut sed incididunt adipiscing',\n", + " 'notification_id': 'OPC_QUEUE_RETRIEVE__server_2',\n", + " 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 122025, tzinfo=datetime.timezone.utc)},\n", + " {'project': 'sientia-laborious',\n", + " 'pipeline': 'ipsum_et',\n", + " 'trigger': 'elit_amet_lorem_dolore_elit',\n", + " 'model_name': 'incididunt eiusmod amet eiusmod',\n", + " 'model_id': '31',\n", + " 'block': 'amet_tempor_tempor',\n", " 'level': 'ERROR',\n", - " 'message': 'et adipiscing ipsum ut aliqua ut adipiscing aliqua amet',\n", - " 'attachment_content': 'aliqua labore aliqua lorem labore aliqua sed sit magna ut do amet do eiusmod sit consectetur sed ipsum elit amet aliqua labore sit',\n", - " 'notification_id': 'OPC_CONNECTION_RETRY__server_6',\n", - " 'timestamp': datetime.datetime(2025, 7, 28, 20, 5, 38, 650949, tzinfo=datetime.timezone.utc)}]" + " 'message': 'ut dolor consectetur amet elit do elit',\n", + " 'attachment_content': 'amet ut adipiscing lorem adipiscing ut et amet adipiscing incididunt elit magna et aliqua incididunt lorem incididunt lorem incididunt eiusmod sit amet tempor lorem',\n", + " 'notification_id': 'ELIT_ALIQUA_TEMPOR',\n", + " 'timestamp': datetime.datetime(2025, 7, 29, 12, 21, 45, 122040, tzinfo=datetime.timezone.utc)}]" ] }, - "execution_count": 15, + "execution_count": 64, "metadata": {}, "output_type": "execute_result" } @@ -235,17 +372,17 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 65, "id": "55aed216", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "InsertManyResult([ObjectId('6887d812645b7515e8402983'), ObjectId('6887d812645b7515e8402984'), ObjectId('6887d812645b7515e8402985'), ObjectId('6887d812645b7515e8402986'), ObjectId('6887d812645b7515e8402987'), ObjectId('6887d812645b7515e8402988'), ObjectId('6887d812645b7515e8402989'), ObjectId('6887d812645b7515e840298a'), ObjectId('6887d812645b7515e840298b'), ObjectId('6887d812645b7515e840298c')], acknowledged=True)" + "InsertManyResult([ObjectId('6888bcd90ccc3300e4a6c2e5'), ObjectId('6888bcd90ccc3300e4a6c2e6'), ObjectId('6888bcd90ccc3300e4a6c2e7'), ObjectId('6888bcd90ccc3300e4a6c2e8'), ObjectId('6888bcd90ccc3300e4a6c2e9'), ObjectId('6888bcd90ccc3300e4a6c2ea'), ObjectId('6888bcd90ccc3300e4a6c2eb'), ObjectId('6888bcd90ccc3300e4a6c2ec'), ObjectId('6888bcd90ccc3300e4a6c2ed'), ObjectId('6888bcd90ccc3300e4a6c2ee')], acknowledged=True)" ] }, - "execution_count": 16, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" } diff --git a/orchestrator/activities/email.py b/orchestrator/activities/email.py index b21321d..e85ee30 100644 --- a/orchestrator/activities/email.py +++ b/orchestrator/activities/email.py @@ -95,6 +95,8 @@ class Email(BaseActivity): self.logger.error( f"Failed to attach content of {att_name}: {e}") + raise e + return msg @activity.defn(name="send_email") @@ -113,29 +115,29 @@ class Email(BaseActivity): metadata=metadata) for group_name, group_config in receiver_groups.items(): - - receivers = ", ".join(group_config['members']) - - self.info(f"Sending email to {group_name}: {receivers}", - metadata=metadata) - - msg = MIMEMultipart() - msg.attach(MIMEText(group_config['html'], 'html')) - msg['From'] = self.sender_email - msg['To'] = receivers - msg['Subject'] = f"SIENTIA™ {mail_type}" - - msg = self.handle_attachments( - [ - { - "filename": f"{notification['trigger']}_{notification['notification_id']}.txt", - "content": notification['attachment_content'] - } - for notification in group_config['notifications'] - if notification['attachment_content']], - msg) - try: + + receivers = ", ".join(group_config['members']) + + self.info(f"Sending email to {group_name}: {receivers}", + metadata=metadata) + + msg = MIMEMultipart() + msg.attach(MIMEText(group_config['html'], 'html')) + msg['From'] = self.sender_email + msg['To'] = receivers + msg['Subject'] = f"SIENTIA™ {mail_type}" + + msg = self.handle_attachments( + [ + { + "filename": f"{notification['trigger']}_{notification['notification_id']}.txt", + "attachment_content": notification['attachment_content'] + } + for notification in group_config['notifications'] + if notification.get('attachment_content') is not None], + msg) + self.server.sendmail( self.sender_email, receivers, msg.as_string()) except Exception as e: @@ -146,7 +148,7 @@ class Email(BaseActivity): else: group_config['status'] = 'sent' - self.info(f"Email sent to {group_name}.", + self.info(f"Email sent to {group_name}: {receivers}", metadata=metadata) self.info(f"Email sent for {mail_type} mail type.", diff --git a/orchestrator/activities/slot_manager.py b/orchestrator/activities/slot_manager.py index 40ef29a..17dbd07 100644 --- a/orchestrator/activities/slot_manager.py +++ b/orchestrator/activities/slot_manager.py @@ -335,3 +335,5 @@ class SlotManager(Redis): if status == 'sent': key = f"{row['schedule']}:{row['notification_id']}" self.set(key, now, ttl=sent_ttl) + + self.info("Notification cache stored...", metadata=metadata) diff --git a/orchestrator/utils/email_builder.py b/orchestrator/utils/email_builder.py index e01d580..88a765f 100644 --- a/orchestrator/utils/email_builder.py +++ b/orchestrator/utils/email_builder.py @@ -31,11 +31,14 @@ class EmailBuilder: return { 'mail_type': mail_type, 'error_events': self.replace_parameters(self.general_template, - error_models) if error_models else '', + general_events.get( + 'ERROR')) if error_models else '', 'warning_events': self.replace_parameters(self.general_template, - warning_models) if warning_models else '', + general_events.get( + 'WARNING')) if warning_models else '', 'info_events': self.replace_parameters(self.general_template, - info_models) if info_models else '', + general_events.get( + 'INFO')) if info_models else '', } def build_email(self, report_data: list[dict], mail_type: str) -> str: diff --git a/orchestrator/utils/templates/general_template.html b/orchestrator/utils/templates/general_template.html index 7736178..0ae39fa 100644 --- a/orchestrator/utils/templates/general_template.html +++ b/orchestrator/utils/templates/general_template.html @@ -5,6 +5,7 @@ Notification ID + Schedule Block Timestamp Message @@ -14,6 +15,7 @@ {% for event in model.events %} {{ event.notification_id }} + {{ event.trigger }} {{ event.block }} {{ event.timestamp }} {{ event.message }} diff --git a/orchestrator/workflows/subworkflows/load_notification_package.py b/orchestrator/workflows/subworkflows/load_notification_package.py index dd33887..83a1c0f 100644 --- a/orchestrator/workflows/subworkflows/load_notification_package.py +++ b/orchestrator/workflows/subworkflows/load_notification_package.py @@ -67,14 +67,6 @@ class LoadNotificationPackage: start_to_close_timeout=timedelta(seconds=60), retry_policy=retry_policy ) - - if not notification_package: - return { - 'last_timestamp': last_timestamp, - 'notification_package': [], - 'sending_configs': await sending_configs_handler - } - sending_configs = await sending_configs_handler if not sending_configs or not notification_package: @@ -103,5 +95,5 @@ class LoadNotificationPackage: return { 'last_timestamp': last_timestamp, 'notification_package': notification_package, - 'sending_configs': await sending_configs_handler + 'sending_configs': sending_configs } diff --git a/tests/orchestrator/activities/test_activities.py b/tests/orchestrator/activities/test_activities.py index 1d76bae..d2640d1 100644 --- a/tests/orchestrator/activities/test_activities.py +++ b/tests/orchestrator/activities/test_activities.py @@ -12,7 +12,12 @@ from orchestrator.activities.formatters import Formatters @patch('orchestrator.activities.temporal_manager.TemporalManager.__init__') @patch('orchestrator.activities.slot_manager.SlotManager.__init__') @patch('orchestrator.activities.formatters.Formatters.__init__') -def test___init__(mock_formatters_init, mock_slot_manager_init, +@patch('orchestrator.activities.email.Email.__init__') +@patch('sientia_do.temporal.activities.postgres.Postgres.__init__') +def test___init__(mock_postgres_init, + mock_email_init, + mock_formatters_init, + mock_slot_manager_init, mock_temporal_manager_init, mock_mongodb_init): @@ -35,6 +40,23 @@ def test___init__(mock_formatters_init, mock_slot_manager_init, 'temporal_laborious_namespace': 'laborious' } + email_config = { + 'sender_email': 'test@test.com', + 'sender_password': 'test', + 'smtp_server': 'test', + 'smtp_port': 587 + } + + postgres_config = { + 'host': 'localhost', + 'port': 5432, + 'user': 'admin', + 'password': 'password', + 'dbname': 'test_db', + 'min_connections': 1, + 'max_connections': 10, + } + logger = MagicMock() notification_handler = MagicMock() @@ -42,6 +64,8 @@ def test___init__(mock_formatters_init, mock_slot_manager_init, temporal_config=temporal_config, redis_config=redis_config, mongodb_config=mongo_db_config, + email_config=email_config, + postgres_config=postgres_config, logger=logger, notification_handler=notification_handler ) diff --git a/tests/orchestrator/activities/test_email.py b/tests/orchestrator/activities/test_email.py index e29c68e..e130c28 100644 --- a/tests/orchestrator/activities/test_email.py +++ b/tests/orchestrator/activities/test_email.py @@ -11,8 +11,8 @@ def email(smtplib, email_builder): email = Email( sender_email="test@test.com", sender_password="test", - smpt_server="test", - port=587, + smtp_server="test", + smtp_port=587, logger=MagicMock(), notification_handler=MagicMock() ) @@ -27,21 +27,22 @@ def test___init___with_password(smtplib, email_builder): email = Email( sender_email="test@test.com", sender_password="test", - smpt_server="test", - port=587, + smtp_server="test", + smtp_port=587, logger=MagicMock(), notification_handler=MagicMock() ) assert email.sender_email == "test@test.com" assert email.sender_password == "test" - assert email.port == 587 + assert email.smtp_port == 587 - smtplib.SMTP_SSL.assert_called_once_with("test", 587) - smtplib.SMTP_SSL.return_value.login.assert_called_once_with( + smtplib.SMTP.assert_called_once_with("test", 587) + smtplib.SMTP.return_value.starttls.assert_called_once() + smtplib.SMTP.return_value.login.assert_called_once_with( "test@test.com", "test") - assert email.server == smtplib.SMTP_SSL.return_value + assert email.server == smtplib.SMTP.return_value @patch('orchestrator.activities.email.EmailBuilder') @@ -50,15 +51,15 @@ def test___init___without_password(smtplib, email_builder): email = Email( sender_email="test@test.com", sender_password=None, - smpt_server="test", - port=587, + smtp_server="test", + smtp_port=587, logger=MagicMock(), notification_handler=MagicMock() ) assert email.sender_email == "test@test.com" assert email.sender_password is None - assert email.port == 587 + assert email.smtp_port == 587 smtplib.SMTP.assert_called_once_with("test", 587) assert email.server == smtplib.SMTP.return_value @@ -189,7 +190,10 @@ def test_handle_attachments_failure(mime_base, email): } ] - email.handle_attachments(attachments, message) + try: + email.handle_attachments(attachments, message) + except Exception as e: + assert str(e) == "test" assert message.attach.call_count == 0 diff --git a/tests/orchestrator/activities/test_mongo_db.py b/tests/orchestrator/activities/test_mongo_db.py index 4c4c956..e3fa686 100644 --- a/tests/orchestrator/activities/test_mongo_db.py +++ b/tests/orchestrator/activities/test_mongo_db.py @@ -555,17 +555,11 @@ async def test_load_latest_data_none_last_data_timestamp(mongo_db): {"_id": 0} ) - assert result == { - 'name': { - 0: 'test1' - }, - 'value': { - 0: 1 - }, - 'timestamp': { - 0: '2023-01-01 12:00:00.000000' - } - } + assert result == [{ + 'name': 'test1', + 'value': 1, + 'timestamp': '2023-01-01 12:00:00.000000' + }] @mark.asyncio @@ -606,17 +600,11 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongo_db): {"_id": 0} ) - assert result == { - 'name': { - 0: 'test1' - }, - 'value': { - 0: 1 - }, - 'timestamp': { - 0: '2023-01-01 12:00:00.000000' - } - } + assert result == [{ + 'name': 'test1', + 'value': 1, + 'timestamp': '2023-01-01 12:00:00.000000' + }] @mark.asyncio diff --git a/tests/orchestrator/utils/test_connectors_config.py b/tests/orchestrator/utils/test_connectors_config.py index a1c219e..f1a62b9 100644 --- a/tests/orchestrator/utils/test_connectors_config.py +++ b/tests/orchestrator/utils/test_connectors_config.py @@ -1,7 +1,10 @@ from os import environ from orchestrator.utils.connectors_config import (build_redis_config, build_couchbase_config, - build_mongodb_config, build_temporal_config) + build_mongodb_config, + build_temporal_config, + build_email_config, + build_postgres_config) def test_build_redis_config_with_env_vars(): @@ -100,3 +103,69 @@ def test_build_temporal_config_with_defaults(): 'temporal_scouter_namespace': 'scouter', 'temporal_laborious_namespace': 'laborious' } + + +def test_build_email_config_with_env_vars(): + environ['EMAIL_SENDER'] = 'test@test.com' + environ['EMAIL_SENDER_PASSWORD'] = 'test' + environ['EMAIL_SMTP_SERVER'] = 'test' + environ['EMAIL_SMTP_PORT'] = '587' + assert build_email_config() == { + 'sender_email': 'test@test.com', + 'sender_password': 'test', + 'smtp_server': 'test', + 'smtp_port': 587 + } + + +def test_build_email_config_with_defaults(): + environ.pop('EMAIL_SENDER', None) + environ.pop('EMAIL_SENDER_PASSWORD', None) + environ.pop('EMAIL_SMTP_SERVER', None) + environ.pop('EMAIL_SMTP_PORT', None) + + assert build_email_config() == { + 'sender_email': 'sientia-alerts@aignosi.com', + 'sender_password': 'sientia', + 'smtp_server': 'smtp.gmail.com', + 'smtp_port': 587 + } + + +def test_build_postgres_config_with_env_vars(): + environ['POSTGRES_HOST'] = 'localhost' + environ['POSTGRES_PORT'] = '5432' + environ['POSTGRES_USER'] = 'sientia' + environ['POSTGRES_PASSWORD'] = 'sientia' + environ['POSTGRES_DBNAME'] = 'sientia' + environ['POSTGRES_MIN_CONNECTIONS'] = '5' + environ['POSTGRES_MAX_CONNECTIONS'] = '20' + assert build_postgres_config() == { + 'host': 'localhost', + 'port': 5432, + 'user': 'sientia', + 'password': 'sientia', + 'dbname': 'sientia', + 'min_connections': 5, + 'max_connections': 20 + } + + +def test_build_postgres_config_with_defaults(): + environ.pop('POSTGRES_HOST', None) + environ.pop('POSTGRES_PORT', None) + environ.pop('POSTGRES_USER', None) + environ.pop('POSTGRES_PASSWORD', None) + environ.pop('POSTGRES_DBNAME', None) + environ.pop('POSTGRES_MIN_CONNECTIONS', None) + environ.pop('POSTGRES_MAX_CONNECTIONS', None) + + assert build_postgres_config() == { + 'host': 'localhost', + 'port': 5432, + 'user': 'sientia', + 'password': 'sientia', + 'dbname': 'sientia', + 'min_connections': 5, + 'max_connections': 20 + } diff --git a/tests/orchestrator/workflows/subworkflows/test_load_notification_package.py b/tests/orchestrator/workflows/subworkflows/test_load_notification_package.py index 952cd3f..3841728 100644 --- a/tests/orchestrator/workflows/subworkflows/test_load_notification_package.py +++ b/tests/orchestrator/workflows/subworkflows/test_load_notification_package.py @@ -23,7 +23,10 @@ metadata = { @patch("orchestrator.workflows.subworkflows.load_notification_package.workflow", new_callable=AsyncMock) async def test_run(workflow_mock, load_notification_package): input_data = { - 'metadata': metadata + 'metadata': metadata, + 'base_data_filter': { + 'level': 'ERROR' + } } workflow_mock.start_local_activity_method.side_effect = [ @@ -113,3 +116,57 @@ async def test_run(workflow_mock, load_notification_package): retry_policy=ANY ) ]) + + +@mark.asyncio +@patch("orchestrator.workflows.subworkflows.load_notification_package.workflow", new_callable=AsyncMock) +async def test_run_no_data(workflow_mock, load_notification_package): + input_data = { + 'metadata': metadata, + 'base_data_filter': { + 'level': 'ERROR' + } + } + + workflow_mock.start_local_activity_method.side_effect = [ + '2023-01-01 12:00:00', + [], + [] + ] + + output = await load_notification_package.run(input_data) + + assert output == { + 'last_timestamp': '2023-01-01 12:00:00', + 'notification_package': [], + 'sending_configs': [] + } + + workflow_mock.start_activity_method.assert_not_called() + + +@mark.asyncio +@patch("orchestrator.workflows.subworkflows.load_notification_package.workflow", new_callable=AsyncMock) +async def test_run_no_data(workflow_mock, load_notification_package): + input_data = { + 'metadata': metadata, + 'base_data_filter': { + 'level': 'ERROR' + } + } + + workflow_mock.start_local_activity_method.side_effect = [ + '2023-01-01 12:00:00', + ["data"], + [] + ] + + output = await load_notification_package.run(input_data) + + assert output == { + 'last_timestamp': '2023-01-01 12:00:00', + 'notification_package': ["data"], + 'sending_configs': [] + } + + workflow_mock.start_activity_method.assert_not_called() diff --git a/tests/orchestrator/workflows/subworkflows/test_process_notifications.py b/tests/orchestrator/workflows/subworkflows/test_process_notifications.py index 4dd6913..df25c84 100644 --- a/tests/orchestrator/workflows/subworkflows/test_process_notifications.py +++ b/tests/orchestrator/workflows/subworkflows/test_process_notifications.py @@ -39,11 +39,14 @@ async def test_run(workflow_mock, process_notifications): Activities.build_email_html, { **metadata, - 'receiver_groups': input_data['notification_package'] + 'receiver_groups': input_data['notification_package'], + 'mail_type': input_data['mail_type'] }, schedule_to_close_timeout=ANY, retry_policy=ANY - ), + ) + ]) + workflow_mock.execute_local_activity_method.assert_has_calls([ call( Activities.format_log_report, { @@ -51,6 +54,8 @@ async def test_run(workflow_mock, process_notifications): 'receiver_groups': workflow_mock.execute_activity_method.return_value, 'mail_type': input_data['mail_type'] }, + schedule_to_close_timeout=ANY, + retry_policy=ANY ) ]) diff --git a/tests/orchestrator/workflows/test_alerts.py b/tests/orchestrator/workflows/test_alerts.py index 788e06d..0dfffd3 100644 --- a/tests/orchestrator/workflows/test_alerts.py +++ b/tests/orchestrator/workflows/test_alerts.py @@ -64,7 +64,9 @@ async def test_run_full_flow(workflow_mock, alerts): 'notification_package': workflow_mock.execute_child_workflow.return_value['notification_package'], 'sending_configs': workflow_mock.execute_child_workflow.return_value['sending_configs'], 'notification_ttl': input_data['notification_ttl'] - } + }, + schedule_to_close_timeout=ANY, + retry_policy=ANY ) ]) @@ -75,6 +77,41 @@ async def test_run_full_flow(workflow_mock, alerts): **metadata, 'log_report': workflow_mock.execute_child_workflow.return_value, 'sent_ttl': input_data['sent_ttl'] + }, + schedule_to_close_timeout=ANY, + retry_policy=ANY + ) + ]) + + +@mark.asyncio +@patch("orchestrator.workflows.alerts.workflow", new_callable=AsyncMock) +async def test_run_no_data(workflow_mock, alerts): + workflow_mock.execute_child_workflow.return_value = { + 'last_timestamp': '2023-01-01 12:00:00.000000', + 'notification_package': [], + 'sending_configs': [] + } + + input_data = { + 'schedule_name': 'test-schedule-name', + 'notification_ttl': 300, + 'sent_ttl': 600 + } + + await alerts.run(input_data) + + workflow_mock.execute_child_workflow.assert_has_calls([ + call( + 'load_notification_package', + { + **input_data, + 'metadata': metadata, + 'base_data_filter': { + 'level': 'ERROR' + } } ) ]) + + workflow_mock.execute_local_activity_method.assert_not_called()