gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
f2cb0f057d0f2b8a43a47e6b405ad5fc8d89c733
sientia-dataops-orchestrato…/notification_generator.ipynb
vitor-aignosi 2a5534cf64 SIENTIAPDE-1172 
feat: enhance orchestrator activities with new email and Postgres integrations

- Added Email and Postgres classes to the Activities class for improved functionality.
- Introduced new methods in MongoDB and SlotManager for loading and managing data.
- Updated requirements.txt to include jinja2.
- Added new formatting activity for log reports in Formatters class.
- Enhanced test coverage for MongoDB and SlotManager activities.
2025-07-25 16:03:39 -03:00

288 lines
20 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"id": "1786cbf0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Generated 10 notifications\n"
]
}
],
"source": [
"from unittest.mock import MagicMock\n",
"from orchestrator.activities.mongo_db import MongoDB\n",
"from orchestrator.utils.connectors_config import build_mongodb_config\n",
"import random\n",
"from datetime import datetime, timezone\n",
"import json\n",
"\n",
"word_list = [\n",
" 'lorem', 'ipsum', 'dolor', 'sit', 'amet', 'consectetur',\n",
" 'adipiscing', 'elit', 'sed', 'do', 'eiusmod', 'tempor',\n",
" 'incididunt', 'ut', 'labore', 'et', 'dolore', 'magna', 'aliqua'\n",
" ]\n",
"\n",
"config = build_mongodb_config()\n",
"\n",
"mongo_db = MongoDB(\n",
" connection_string=config['connection_string'],\n",
" database_name=config['database_name'],\n",
" ttl_index_seconds=config['ttl_index_seconds'],\n",
" logger=MagicMock(),\n",
" notification_handler=MagicMock()\n",
")\n",
"\n",
"AMOUT = 10\n",
"\n",
"notification_ids = [\n",
" \"TAG_node<tag>:<name>_LISTENNING_STOPPED\",\n",
" \"OPC_<event>__<server>\",\n",
" \"REPORT_PARTITION_MANAGER\",\n",
" \"<random_id>\"\n",
"]\n",
"\n",
"notifications = []\n",
"\n",
"for _ in range(AMOUT):\n",
" # Choose random notification ID and fill in placeholders\n",
" notification_id = random.choice(notification_ids)\n",
" \n",
" if \"<tag>\" in notification_id:\n",
" notification_id = notification_id.replace(\"<tag>\", str(random.randint(1,100)))\n",
" if \"<name>\" in notification_id:\n",
" notification_id = notification_id.replace(\"<name>\", f\"tag_{random.randint(1,100)}\")\n",
" if \"<event>\" in notification_id:\n",
" events = [\"LISTENNING_STOPPED\", \"CONNECTION_RETRY\", \"LISTENNING_BACK\", \"SUBSCRIPTION\", \"QUEUE_RETRIEVE\"]\n",
" notification_id = notification_id.replace(\"<event>\", random.choice(events))\n",
" if \"<server>\" in notification_id:\n",
" notification_id = notification_id.replace(\"<server>\", f\"server_{random.randint(1,10)}\")\n",
" if \"<random_id>\" in notification_id:\n",
" notification_id = notification_id.replace(\"<random_id>\", \"_\".join(random.choices(word_list, k=random.randint(3,5))).upper())\n",
"\n",
" notification = {\n",
" \"project\": \"sientia-laborious\",\n",
" \"pipeline\": \"_\".join(random.choices(word_list, k=random.randint(2,4))), \n",
" \"trigger\": \"_\".join(random.choices(word_list, k=random.randint(3,6))),\n",
" \"model_name\": \" \".join(random.choices(word_list, k=random.randint(2,4))),\n",
" \"model_id\": str(random.randint(1,100)),\n",
" \"block\": \"_\".join(random.choices(word_list, k=random.randint(2,4))),\n",
" \"level\": random.choice([\"INFO\", \"WARNING\", \"ERROR\"]),\n",
" \"message\": \" \".join(random.choices(word_list, k=random.randint(5,10))),\n",
" \"attachment_content\": \" \".join(random.choices(word_list, k=random.randint(20,30))),\n",
" \"notification_id\": notification_id,\n",
" \"timestamp\": datetime.now(timezone.utc)\n",
" }\n",
" \n",
" notifications.append(notification)\n",
"\n",
"print(f\"Generated {len(notifications)} notifications\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "c1467b72",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'WARNING': {'section_name': 'Warnings detected:', 'models': {'amet eiusmod dolor ipsum': {'model_name': 'amet eiusmod dolor ipsum', 'events': [{'project': 'sientia-laborious', 'pipeline': 'ut_eiusmod_do', 'trigger': 'aliqua_magna_labore_magna_do_ut', 'model_name': 'amet eiusmod dolor ipsum', 'model_id': '26', 'block': 'adipiscing_amet_ut_et', 'level': 'WARNING', 'message': 'eiusmod et amet sit lorem incididunt', 'attachment_content': 'sed et ipsum aliqua et consectetur incididunt aliqua incididunt aliqua eiusmod adipiscing incididunt sed et ut ipsum aliqua elit ut', 'notification_id': 'REPORT_PARTITION_MANAGER', 'timestamp': datetime.datetime(2025, 7, 25, 16, 50, 48, 925368, tzinfo=datetime.timezone.utc)}]}, 'sit et labore': {'model_name': 'sit et labore', 'events': [{'project': 'sientia-laborious', 'pipeline': 'tempor_eiusmod_amet_amet', 'trigger': 'labore_elit_adipiscing_adipiscing', 'model_name': 'sit et labore', 'model_id': '8', 'block': 'adipiscing_elit_ut', 'level': 'WARNING', 'message': 'magna do eiusmod et labore elit', 'attachment_content': 'incididunt incididunt do adipiscing consectetur adipiscing ipsum incididunt et sed sed magna dolore ut ipsum amet dolor sed tempor lorem incididunt do labore elit aliqua magna eiusmod dolor', 'notification_id': 'SIT_CONSECTETUR_EIUSMOD', 'timestamp': datetime.datetime(2025, 7, 25, 16, 50, 48, 925392, tzinfo=datetime.timezone.utc)}]}, 'consectetur labore do consectetur': {'model_name': 'consectetur labore do consectetur', 'events': [{'project': 'sientia-laborious', 'pipeline': 'et_consectetur_ut', 'trigger': 'labore_aliqua_tempor_amet', 'model_name': 'consectetur labore do consectetur', 'model_id': '50', 'block': 'dolor_dolore_ut', 'level': 'WARNING', 'message': 'magna sed incididunt dolor tempor', 'attachment_content': 'lorem labore tempor amet adipiscing lorem dolor aliqua do et incididunt adipiscing labore aliqua dolor elit aliqua eiusmod sed aliqua aliqua consectetur incididunt sed ipsum eiusmod labore do', 'notification_id': 'SED_SIT_DOLOR_CONSECTETUR', 'timestamp': datetime.datetime(2025, 7, 25, 16, 50, 48, 925435, tzinfo=datetime.timezone.utc)}]}, 'et sed ut et': {'model_name': 'et sed ut et', 'events': [{'project': 'sientia-laborious', 'pipeline': 'eiusmod_consectetur_eiusmod_dolor', 'trigger': 'aliqua_sit_eiusmod', 'model_name': 'et sed ut et', 'model_id': '60', 'block': 'sit_do', 'level': 'WARNING', 'message': 'do sed sed elit do eiusmod', 'attachment_content': 'sit labore magna labore lorem eiusmod aliqua magna et eiusmod magna labore et amet adipiscing consectetur sit ut dolore tempor tempor amet ut', 'notification_id': 'UT_UT_LABORE', 'timestamp': datetime.datetime(2025, 7, 25, 16, 50, 48, 925492, tzinfo=datetime.timezone.utc)}]}}}, 'INFO': {'section_name': 'Infos detected:', 'models': {'eiusmod eiusmod': {'model_name': 'eiusmod eiusmod', 'events': [{'project': 'sientia-laborious', 'pipeline': 'do_et_et_dolor', 'trigger': 'adipiscing_dolor_dolore_adipiscing_tempor', 'model_name': 'eiusmod eiusmod', 'model_id': '70', 'block': 'labore_sit_do', 'level': 'INFO', 'message': 'magna et ipsum amet dolore adipiscing adipiscing do', 'attachment_content': 'aliqua amet lorem amet elit tempor amet aliqua do aliqua ut dolore sed eiusmod consectetur tempor dolor sit magna eiusmod consectetur ipsum', 'notification_id': 'OPC_CONNECTION_RETRY__server_2', 'timestamp': datetime.datetime(2025, 7, 25, 16, 50, 48, 925407, tzinfo=datetime.timezone.utc)}]}, 'adipiscing consectetur': {'model_name': 'adipiscing consectetur', 'events': [{'project': 'sientia-laborious', 'pipeline': 'sit_do_aliqua', 'trigger': 'aliqua_amet_magna_ipsum_lorem', 'model_name': 'adipiscing consectetur', 'model_id': '48', 'block': 'incididunt_aliqua_ipsum_ut', 'level': 'INFO', 'message': 'labore magna adipiscing tempor elit et et magna elit lorem', 'attachment_content': 'magna labore dolor ipsum dolor lorem sed magna eiusmod ipsum do aliqua magna labore et elit consectetur et adipiscing lorem incididunt lorem', 'notification_id': 'REPORT_PARTITION_MANAGER', 'timestamp': datetime.datetime(2025, 7, 25, 16, 50, 48, 925447, tzinfo=datetime.timezone.utc)}]}, 'et amet': {'model_name': 'et amet', 'events': [{'project': 'sientia-laborious', 'pipeline': 'consectetur_labore_magna', 'trigger': 'eiusmod_incididunt_consectetur_tempor', 'model_name': 'et amet', 'model_id': '31', 'block': 'sed_elit_ut_do', 'level': 'INFO', 'message': 'dolor et adipiscing adipiscing dolore ipsum ipsum aliqua et elit', 'attachment_content': 'tempor dolore ipsum ut ut et dolor eiusmod do magna magna magna et tempor labore labore dolor labore eiusmod do dolor incididunt lorem dolore ipsum ipsum aliqua sed incididunt', 'notification_id': 'OPC_LISTENNING_BACK__server_2', 'timestamp': datetime.datetime(2025, 7, 25, 16, 50, 48, 925462, tzinfo=datetime.timezone.utc)}]}}}, 'ERROR': {'section_name': 'Errors detected:', 'models': {'ut aliqua ut ipsum': {'model_name': 'ut aliqua ut ipsum', 'events': [{'project': 'sientia-laborious', 'pipeline': 'sit_amet', 'trigger': 'aliqua_sit_consectetur_lorem_consectetur', 'model_name': 'ut aliqua ut ipsum', 'model_id': '80', 'block': 'dolor_elit_aliqua', 'level': 'ERROR', 'message': 'sit elit ipsum labore incididunt magna do et sit incididunt', 'attachment_content': 'amet ut et ut sed ipsum aliqua elit incididunt sed dolore dolore ipsum sed elit sit adipiscing incididunt elit labore do magna et consectetur aliqua', 'notification_id': 'TAG_node5:tag_87_LISTENNING_STOPPED', 'timestamp': datetime.datetime(2025, 7, 25, 16, 50, 48, 925421, tzinfo=datetime.timezone.utc)}]}, 'elit dolore': {'model_name': 'elit dolore', 'events': [{'project': 'sientia-laborious', 'pipeline': 'sed_et_tempor_labore', 'trigger': 'aliqua_tempor_sit_lorem_amet_et', 'model_name': 'elit dolore', 'model_id': '20', 'block': 'magna_magna_et', 'level': 'ERROR', 'message': 'aliqua eiusmod ut eiusmod ut et ipsum', 'attachment_content': 'labore do dolore dolore adipiscing amet tempor eiusmod labore elit consectetur incididunt dolor et elit eiusmod ut ut aliqua incididunt', 'notification_id': 'OPC_LISTENNING_BACK__server_10', 'timestamp': datetime.datetime(2025, 7, 25, 16, 50, 48, 925478, tzinfo=datetime.timezone.utc)}]}, 'lorem lorem ipsum amet': {'model_name': 'lorem lorem ipsum amet', 'events': [{'project': 'sientia-laborious', 'pipeline': 'amet_eiusmod_eiusmod', 'trigger': 'magna_aliqua_dolore_incididunt_do_do', 'model_name': 'lorem lorem ipsum amet', 'model_id': '47', 'block': 'do_sit', 'level': 'ERROR', 'message': 'labore do adipiscing adipiscing incididunt eiusmod', 'attachment_content': 'sit amet dolor magna et sit adipiscing sed consectetur labore sit ut adipiscing magna amet dolor lorem dolore eiusmod sed dolor dolor sit consectetur sit aliqua dolore tempor', 'notification_id': 'TAG_node8:tag_11_LISTENNING_STOPPED', 'timestamp': datetime.datetime(2025, 7, 25, 16, 50, 48, 925506, tzinfo=datetime.timezone.utc)}]}}}}\n"
]
}
],
"source": [
"from orchestrator.utils.email_builder import EmailBuilder\n",
"from unittest.mock import MagicMock\n",
"\n",
"\n",
"email_builder = EmailBuilder(logger=MagicMock())\n",
"\n",
"html = email_builder.build_email(notifications, \"Alerts\")\n",
"\n",
"with open('email.html', 'w') as file:\n",
" file.write(html)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "ffd7be77",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'project': 'sientia-laborious',\n",
" 'pipeline': 'lorem_sed_ut',\n",
" 'trigger': 'dolore_elit_lorem',\n",
" 'model_name': 'lorem tempor',\n",
" 'model_id': '71',\n",
" 'block': 'aliqua_aliqua_et',\n",
" 'level': 'ERROR',\n",
" 'message': 'tempor eiusmod dolor eiusmod sed amet labore et magna',\n",
" 'attachment_content': 'sed amet incididunt et amet lorem lorem ut adipiscing magna incididunt sit aliqua ut dolore dolore ipsum aliqua amet labore dolor amet et elit dolore dolor consectetur',\n",
" 'notification_id': 'OPC_LISTENNING_BACK__server_1',\n",
" 'timestamp': datetime.datetime(2025, 7, 25, 15, 44, 24, 147390, tzinfo=datetime.timezone.utc)},\n",
" {'project': 'sientia-laborious',\n",
" 'pipeline': 'et_do_eiusmod_adipiscing',\n",
" 'trigger': 'consectetur_magna_elit',\n",
" 'model_name': 'elit incididunt elit',\n",
" 'model_id': '85',\n",
" 'block': 'amet_elit_adipiscing',\n",
" 'level': 'WARNING',\n",
" 'message': 'do labore consectetur lorem lorem sit do aliqua magna magna',\n",
" 'attachment_content': 'aliqua tempor aliqua adipiscing sit elit ut magna sit ipsum magna tempor do incididunt sit magna sit consectetur amet elit lorem ipsum incididunt ipsum',\n",
" 'notification_id': 'ADIPISCING_DO_DO_DOLOR',\n",
" 'timestamp': datetime.datetime(2025, 7, 25, 15, 44, 24, 147411, tzinfo=datetime.timezone.utc)},\n",
" {'project': 'sientia-laborious',\n",
" 'pipeline': 'magna_tempor',\n",
" 'trigger': 'et_elit_labore_ut_eiusmod',\n",
" 'model_name': 'sit ipsum et eiusmod',\n",
" 'model_id': '14',\n",
" 'block': 'incididunt_do',\n",
" 'level': 'INFO',\n",
" 'message': 'sit elit eiusmod consectetur magna labore',\n",
" 'attachment_content': 'amet sit et do dolore eiusmod incididunt sed dolor dolor ipsum tempor lorem dolore sed amet ipsum ut lorem aliqua',\n",
" 'notification_id': 'IPSUM_ADIPISCING_IPSUM_MAGNA_LABORE',\n",
" 'timestamp': datetime.datetime(2025, 7, 25, 15, 44, 24, 147426, tzinfo=datetime.timezone.utc)},\n",
" {'project': 'sientia-laborious',\n",
" 'pipeline': 'adipiscing_adipiscing_adipiscing',\n",
" 'trigger': 'elit_dolor_sit_et',\n",
" 'model_name': 'sit adipiscing tempor sed',\n",
" 'model_id': '46',\n",
" 'block': 'tempor_tempor_sit',\n",
" 'level': 'WARNING',\n",
" 'message': 'tempor adipiscing tempor labore dolore tempor elit',\n",
" 'attachment_content': 'elit sit dolore consectetur ut ipsum ipsum aliqua incididunt elit eiusmod labore sed eiusmod eiusmod aliqua eiusmod eiusmod labore magna incididunt consectetur ipsum et aliqua sit adipiscing labore do',\n",
" 'notification_id': 'TAG_node57:tag_97_LISTENNING_STOPPED',\n",
" 'timestamp': datetime.datetime(2025, 7, 25, 15, 44, 24, 147440, tzinfo=datetime.timezone.utc)},\n",
" {'project': 'sientia-laborious',\n",
" 'pipeline': 'eiusmod_amet_aliqua',\n",
" 'trigger': 'ipsum_lorem_et',\n",
" 'model_name': 'consectetur adipiscing',\n",
" 'model_id': '72',\n",
" 'block': 'tempor_et_adipiscing_sed',\n",
" 'level': 'ERROR',\n",
" 'message': 'lorem ipsum do amet elit eiusmod elit lorem',\n",
" 'attachment_content': 'ipsum ut dolor eiusmod dolore elit consectetur ipsum dolor do sed dolor aliqua sit consectetur aliqua sit sit consectetur et incididunt adipiscing lorem do adipiscing',\n",
" 'notification_id': 'REPORT_PARTITION_MANAGER',\n",
" 'timestamp': datetime.datetime(2025, 7, 25, 15, 44, 24, 147453, tzinfo=datetime.timezone.utc)},\n",
" {'project': 'sientia-laborious',\n",
" 'pipeline': 'eiusmod_et_incididunt',\n",
" 'trigger': 'dolore_do_lorem_aliqua',\n",
" 'model_name': 'aliqua sed tempor ipsum',\n",
" 'model_id': '2',\n",
" 'block': 'et_elit_amet',\n",
" 'level': 'WARNING',\n",
" 'message': 'consectetur aliqua dolore amet eiusmod aliqua',\n",
" 'attachment_content': 'sit amet amet aliqua incididunt aliqua amet ipsum magna do adipiscing aliqua tempor magna dolor dolor labore et lorem consectetur dolore do ut adipiscing dolore',\n",
" 'notification_id': 'AMET_ELIT_ELIT_MAGNA_ALIQUA',\n",
" 'timestamp': datetime.datetime(2025, 7, 25, 15, 44, 24, 147482, tzinfo=datetime.timezone.utc)},\n",
" {'project': 'sientia-laborious',\n",
" 'pipeline': 'amet_dolore',\n",
" 'trigger': 'labore_magna_labore',\n",
" 'model_name': 'dolor ut incididunt adipiscing',\n",
" 'model_id': '3',\n",
" 'block': 'dolor_sit',\n",
" 'level': 'WARNING',\n",
" 'message': 'ut ipsum sed consectetur eiusmod',\n",
" 'attachment_content': 'aliqua dolor magna magna sed et magna sit aliqua labore aliqua sed ipsum aliqua amet dolore ipsum aliqua consectetur adipiscing eiusmod',\n",
" 'notification_id': 'REPORT_PARTITION_MANAGER',\n",
" 'timestamp': datetime.datetime(2025, 7, 25, 15, 44, 24, 147494, tzinfo=datetime.timezone.utc)},\n",
" {'project': 'sientia-laborious',\n",
" 'pipeline': 'dolor_ipsum',\n",
" 'trigger': 'do_sed_et_dolore_elit',\n",
" 'model_name': 'adipiscing aliqua labore magna',\n",
" 'model_id': '74',\n",
" 'block': 'incididunt_dolore',\n",
" 'level': 'INFO',\n",
" 'message': 'amet do amet sed sit',\n",
" 'attachment_content': 'et sed labore adipiscing labore dolore amet tempor incididunt aliqua dolor labore tempor adipiscing incididunt consectetur magna tempor tempor aliqua ut sit magna dolore ipsum',\n",
" 'notification_id': 'REPORT_PARTITION_MANAGER',\n",
" 'timestamp': datetime.datetime(2025, 7, 25, 15, 44, 24, 147510, tzinfo=datetime.timezone.utc)},\n",
" {'project': 'sientia-laborious',\n",
" 'pipeline': 'magna_sed_ut_sit',\n",
" 'trigger': 'ut_dolore_consectetur_labore_incididunt',\n",
" 'model_name': 'aliqua et consectetur',\n",
" 'model_id': '15',\n",
" 'block': 'ipsum_et',\n",
" 'level': 'ERROR',\n",
" 'message': 'ut dolore ipsum magna tempor sed magna labore amet magna',\n",
" 'attachment_content': 'ipsum adipiscing et amet elit dolor ut aliqua consectetur ut do elit amet tempor incididunt eiusmod eiusmod dolore ipsum sit dolore adipiscing aliqua elit adipiscing labore sed eiusmod ut',\n",
" 'notification_id': 'LABORE_INCIDIDUNT_ADIPISCING_DO_LABORE',\n",
" 'timestamp': datetime.datetime(2025, 7, 25, 15, 44, 24, 147525, tzinfo=datetime.timezone.utc)},\n",
" {'project': 'sientia-laborious',\n",
" 'pipeline': 'lorem_adipiscing_lorem_incididunt',\n",
" 'trigger': 'adipiscing_magna_dolor_consectetur_elit',\n",
" 'model_name': 'dolor sit lorem',\n",
" 'model_id': '51',\n",
" 'block': 'dolor_ipsum',\n",
" 'level': 'INFO',\n",
" 'message': 'labore aliqua dolor eiusmod incididunt adipiscing tempor',\n",
" 'attachment_content': 'lorem magna dolore consectetur eiusmod sit dolor eiusmod ut do dolor labore eiusmod et dolore sit tempor consectetur ipsum adipiscing eiusmod dolore consectetur labore sit do',\n",
" 'notification_id': 'CONSECTETUR_MAGNA_AMET_LOREM_TEMPOR',\n",
" 'timestamp': datetime.datetime(2025, 7, 25, 15, 44, 24, 147539, tzinfo=datetime.timezone.utc)}]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"notifications"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "55aed216",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"InsertManyResult([ObjectId('68839c5572e2953521d60ada'), ObjectId('68839c5572e2953521d60adb'), ObjectId('68839c5572e2953521d60adc'), ObjectId('68839c5572e2953521d60add'), ObjectId('68839c5572e2953521d60ade'), ObjectId('68839c5572e2953521d60adf'), ObjectId('68839c5572e2953521d60ae0'), ObjectId('68839c5572e2953521d60ae1'), ObjectId('68839c5572e2953521d60ae2'), ObjectId('68839c5572e2953521d60ae3')], acknowledged=True)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mongo_db.database['notification_queue'].insert_many(notifications)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink