gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
Enhance orchestration configuration and documentation. Added `RUNTIME` variable to `.env.example`, updated `.gitignore` to exclude `openspec/` and `.cursor/`, and modified `README.md` to clarify queue naming conventions and runtime handling. Refactored activities to use synchronous database and email handling, improving performance and consistency. Updated test cases to reflect these changes and ensure compatibility with new activity definitions.
This commit is contained in:
vitor-aignosi
2026-05-22 15:02:16 -03:00
parent 4d5231ca16
commit 474c2ef42c
57 changed files with 2744 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/orchestrator/activities/test_activities.py
View File

@@ -12,7 +12,7 @@ from orchestrator.activities.temporal_manager import TemporalManager
@patch('orchestrator.activities.slot_manager.SlotManager.__init__')
@patch('orchestrator.activities.formatters.Formatters.__init__')
@patch('orchestrator.activities.email.Email.__init__')
@patch('sientia_do.temporal.activities.postgres.Postgres.__init__')
@patch('sientia_do.temporal.activities.postgres_sync.Postgres.__init__')
@patch('orchestrator.activities.activities.MetricsController')
def test___init__(
mock_metrics_controller,

20
tests/orchestrator/activities/test_email.py
View File

@@ -1,7 +1,7 @@
from smtplib import SMTPServerDisconnected
from unittest.mock import AsyncMock, MagicMock, call, patch
from pytest import fixture, mark
from pytest import fixture
from orchestrator.activities.email import Email
@@ -19,9 +19,6 @@ def email(smtplib, email_builder):
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
)
email_builder.send_notification_async = AsyncMock()
email_builder.send_notification = MagicMock()
email.send_notification_async = AsyncMock()
email.send_notification = MagicMock()
email.emit_metric = AsyncMock()
@@ -91,8 +88,7 @@ def test_close(sientia_monitoring_mock, email):
sientia_monitoring_mock.shutdown.assert_called_once()
@mark.asyncio
async def test_build_email_html(email):
def test_build_email_html(email):
email.email_builder.build_email = MagicMock(return_value='test')
input_data = {
**metadata,
@@ -107,7 +103,7 @@ async def test_build_email_html(email):
'mail_type': 'test',
}
response = await email.build_email_html(input_data)
response = email.build_email_html(input_data)
assert response == {
'group_1': {
@@ -253,19 +249,17 @@ def test_try_send_email_reconnect_quit_failure(smtp, email):
raise AssertionError('Expected exception')
@mark.asyncio
async def test_send_email_without_smtp_server(email):
def test_send_email_without_smtp_server(email):
email.smtp_server = None
input_data = {**metadata, 'receiver_groups': {}, 'mail_type': 'test_TYPE'}
response = await email.send_email(input_data)
response = email.send_email(input_data)
assert response == {}
@mark.asyncio
@patch('orchestrator.activities.email.MIMEText')
@patch('orchestrator.activities.email.MIMEMultipart')
async def test_send_email(mimemultipart, mimetext, email):
def test_send_email(mimemultipart, mimetext, email):
side_effect_1 = MagicMock()
side_effect_2 = MagicMock()
mimemultipart.side_effect = [side_effect_1, side_effect_2]
@@ -297,7 +291,7 @@ async def test_send_email(mimemultipart, mimetext, email):
'mail_type': 'test_TYPE',
}
response = await email.send_email(input_data)
response = email.send_email(input_data)
assert response['group_1']['status'] == 'sent'
assert response['group_2']['status'] == 'failed'

69
tests/orchestrator/activities/test_formatters.py
View File

@@ -2,7 +2,7 @@ import json
from unittest.mock import AsyncMock, MagicMock, call, patch
from pandas import DataFrame
from pytest import fixture, mark
from pytest import fixture
from sientia_do.notifications.models import NotificationLevel
from orchestrator.activities.formatters import Formatters
@@ -19,7 +19,6 @@ def formatters():
)
formatters.send_notification = MagicMock()
formatters.send_notification_async = AsyncMock()
formatters.emit_metric = AsyncMock()
formatters.error = MagicMock()
formatters.info = MagicMock()
@@ -37,8 +36,7 @@ metadata = {
}
@mark.asyncio
async def test_process_schedules(formatters):
def test_process_schedules(formatters):
mock_scouter = MagicMock(return_value={'test_scouter': 'test_scouter'})
mock_predictions_batch = MagicMock(
return_value={'test_predictions_batch': 'test_predictions_batch'}
@@ -115,7 +113,7 @@ async def test_process_schedules(formatters):
}
with patch('orchestrator.activities.formatters.schedule_types', mock_schedule_types):
result = await formatters.process_schedules(input_data)
result = formatters.process_schedules(input_data)
assert result == {
'scouter': {
@@ -148,8 +146,7 @@ async def test_process_schedules(formatters):
mock_simple_metrics.assert_called_once_with(input_data['pipelines'][4])
@mark.asyncio
async def test_process_schedules_with_invalid_workflow_type(formatters):
def test_process_schedules_with_invalid_workflow_type(formatters):
"""Test that process_schedules handles invalid workflow types correctly"""
mock_scouter = MagicMock(return_value={'test_scouter': 'test_scouter'})
@@ -193,7 +190,7 @@ async def test_process_schedules_with_invalid_workflow_type(formatters):
}
with patch('orchestrator.activities.formatters.schedule_types', mock_schedule_types):
result = await formatters.process_schedules(input_data)
result = formatters.process_schedules(input_data)
# Assert that error was called for invalid workflow type
formatters.error.assert_called_once_with(
@@ -221,7 +218,6 @@ async def test_process_schedules_with_invalid_workflow_type(formatters):
mock_scouter.assert_any_call(pipelines[2])
@mark.asyncio
@patch(
'orchestrator.activities.formatters.gather_read_tags',
return_value={
@@ -249,7 +245,7 @@ async def test_process_schedules_with_invalid_workflow_type(formatters):
},
)
@patch('orchestrator.activities.formatters.build_tag_config')
async def test_process_slots(mock_build_tag_config, mock_gather_read_tags, formatters):
def test_process_slots(mock_build_tag_config, mock_gather_read_tags, formatters):
input_data = {
'opc_servers': [
{'id': '1', 'server_name': 'test_server_name', 'url': 'test_url', 'uri': 'test_uri'},
@@ -289,7 +285,7 @@ async def test_process_slots(mock_build_tag_config, mock_gather_read_tags, forma
mock_build_tag_config.return_value = (slot_mock, ['2'])
result = await formatters.process_slots(input_data)
result = formatters.process_slots(input_data)
tags = list(mock_gather_read_tags.return_value.values())
@@ -302,7 +298,7 @@ async def test_process_slots(mock_build_tag_config, mock_gather_read_tags, forma
]
)
formatters.send_notification_async.assert_has_calls(
formatters.send_notification.assert_has_calls(
[
call(
metadata=metadata['metadata'],
@@ -327,8 +323,7 @@ async def test_process_slots(mock_build_tag_config, mock_gather_read_tags, forma
}
@mark.asyncio
async def test_format_schedule_config(formatters):
def test_format_schedule_config(formatters):
input_data = {
'schedule_config': [
{'namespace': 'test_namespace1', 'schedule_name': 'test1', 'updated_at': '2021-01-01'},
@@ -337,7 +332,7 @@ async def test_format_schedule_config(formatters):
**metadata,
}
result = await formatters.format_schedule_config(input_data)
result = formatters.format_schedule_config(input_data)
assert result == {
'test_namespace1': {'test1': '2021-01-01'},
@@ -345,8 +340,7 @@ async def test_format_schedule_config(formatters):
}
@mark.asyncio
async def test_create_schedule_config(formatters):
def test_create_schedule_config(formatters):
input_data = {
'current_schedule_config': {
'scouter': {
@@ -372,7 +366,7 @@ async def test_create_schedule_config(formatters):
},
}
result = await formatters.create_schedule_config(input_data)
result = formatters.create_schedule_config(input_data)
assert result == {
'to_create': {
@@ -399,8 +393,7 @@ async def test_create_schedule_config(formatters):
}
@mark.asyncio
async def test_create_slot_config(formatters):
def test_create_slot_config(formatters):
input_data = {
'current_slot_config': {
'1': {'frequency': 60, 'data': {'test': 'test'}},
@@ -409,7 +402,7 @@ async def test_create_slot_config(formatters):
'slot_config': {'1': {'frequency': 60, 'data': {'test': 'test2'}}},
}
result = await formatters.create_slot_config(input_data)
result = formatters.create_slot_config(input_data)
assert result == {
'to_delete': ['2'],
@@ -417,15 +410,14 @@ async def test_create_slot_config(formatters):
}
@mark.asyncio
async def test_send_success_report(formatters):
await formatters.send_success_report(
def test_send_success_report(formatters):
formatters.send_success_report(
metadata=metadata,
message='test_message',
notification_id='test_notification_id',
attachment={'test': 'test'},
)
formatters.send_notification_async.assert_called_once_with(
formatters.send_notification.assert_called_once_with(
metadata=metadata,
notification_id='test_notification_id',
message='test_message',
@@ -435,15 +427,14 @@ async def test_send_success_report(formatters):
)
@mark.asyncio
async def test_send_error_report(formatters):
await formatters.send_error_report(
def test_send_error_report(formatters):
formatters.send_error_report(
metadata=metadata,
message='test_message',
notification_id='test_notification_id',
attachment='test_attachment',
)
formatters.send_notification_async.assert_called_once_with(
formatters.send_notification.assert_called_once_with(
metadata=metadata,
notification_id='test_notification_id',
message='test_message',
@@ -492,8 +483,7 @@ def test_parse_report_schedule(formatters):
)
@mark.asyncio
async def test_report_schedule_orchestration(formatters):
def test_report_schedule_orchestration(formatters):
formatters.parse_report_schedule = MagicMock(side_effect=formatters.parse_report_schedule)
formatters.send_success_report = AsyncMock()
formatters.send_error_report = AsyncMock()
@@ -569,7 +559,7 @@ async def test_report_schedule_orchestration(formatters):
],
}
await formatters.report_schedule_orchestration(input_data)
formatters.report_schedule_orchestration(input_data)
formatters.parse_report_schedule.assert_has_calls(
[
@@ -624,8 +614,7 @@ async def test_report_schedule_orchestration(formatters):
)
@mark.asyncio
async def test_report_slot_orchestration(formatters):
def test_report_slot_orchestration(formatters):
formatters.parse_report = MagicMock(side_effect=formatters.parse_report)
formatters.send_success_report = AsyncMock()
formatters.send_error_report = AsyncMock()
@@ -642,7 +631,7 @@ async def test_report_slot_orchestration(formatters):
},
}
await formatters.report_slot_orchestration(input_data)
formatters.report_slot_orchestration(input_data)
formatters.parse_report.assert_has_calls(
[call(input_data['inserted_slots']), call(input_data['deleted_slots'])]
@@ -679,8 +668,7 @@ async def test_report_slot_orchestration(formatters):
)
@mark.asyncio
async def test_format_log_report(formatters):
def test_format_log_report(formatters):
input_data = {
**metadata,
'receiver_groups': {
@@ -722,7 +710,7 @@ async def test_format_log_report(formatters):
'mail_type': 'test_mail_type',
}
result = await formatters.format_log_report(input_data)
result = formatters.format_log_report(input_data)
expected_result = DataFrame(
[
@@ -747,8 +735,7 @@ async def test_format_log_report(formatters):
assert DataFrame(result).equals(expected_result)
@mark.asyncio
async def test_filter_notification_reports(formatters):
def test_filter_notification_reports(formatters):
input_data = {
**metadata,
'notification_package': [
@@ -766,7 +753,7 @@ async def test_filter_notification_reports(formatters):
],
}
response = await formatters.filter_notification_reports(input_data)
response = formatters.filter_notification_reports(input_data)
assert response == {
'test_group_1': {

131
tests/orchestrator/activities/test_mongo_db.py
View File

@@ -1,9 +1,9 @@
from datetime import datetime
from unittest.mock import ANY, AsyncMock, MagicMock, patch
from pytest import fixture, mark
from pytest import fixture
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ, DATETIME_FORMAT_WITH_TZ
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
from orchestrator.activities.mongo_db import MongoDB
@@ -20,7 +20,6 @@ def mongo_db(mongo_mock):
metrics_controller=AsyncMock(),
)
mongo.send_notification = MagicMock()
mongo.send_notification_async = AsyncMock()
mongo.emit_metric = AsyncMock()
return mongo
@@ -63,11 +62,10 @@ def test___del__(mongo_db):
mongo_db.close.assert_called_once()
@mark.asyncio
async def test_find_documents_in_mongodb_success(mongo_db):
def test_find_documents_in_mongodb_success(mongo_db):
input_data = {'collection': 'test_collection', 'filters': {'name': {'$exists': True}}}
mongo_db.mongo_db_repository.find = AsyncMock(
mongo_db.mongo_db_repository.find = MagicMock(
return_value=[
{
'name': 'test1',
@@ -84,7 +82,7 @@ async def test_find_documents_in_mongodb_success(mongo_db):
]
)
result = await mongo_db.find_documents_in_mongodb(
result = mongo_db.find_documents_in_mongodb(
{'query': input_data, 'timestamp_fields': ['timestamp']}
)
@@ -106,16 +104,15 @@ metadata = {
}
@mark.asyncio
async def test_find_documents_in_mongodb_failure(mongo_db):
def test_find_documents_in_mongodb_failure(mongo_db):
input_data = {'collection': 'test_collection', 'filters': {'name': {'$exists': True}}}
mongo_db.mongo_db_repository.find = AsyncMock(side_effect=Exception('Error'))
mongo_db.mongo_db_repository.find = MagicMock(side_effect=Exception('Error'))
try:
await mongo_db.find_documents_in_mongodb({'query': input_data, **metadata})
mongo_db.find_documents_in_mongodb({'query': input_data, **metadata})
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification_async.assert_called_once_with(
mongo_db.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_QUERY_ERROR',
message='Failed to execute MongoDB query: Error',
@@ -128,12 +125,11 @@ async def test_find_documents_in_mongodb_failure(mongo_db):
raise AssertionError('Expected an exception to be raised')
@mark.asyncio
async def test_find_documents_in_mongodb_missing_collection(mongo_db):
def test_find_documents_in_mongodb_missing_collection(mongo_db):
input_data = {'query': {'filters': {}}}
try:
await mongo_db.find_documents_in_mongodb(input_data)
mongo_db.find_documents_in_mongodb(input_data)
except ValueError as e:
assert str(e) == 'Collection name must be provided in the query.'
@@ -142,13 +138,12 @@ async def test_find_documents_in_mongodb_missing_collection(mongo_db):
raise AssertionError('Expected a ValueError to be raised')
@mark.asyncio
async def test_aggregate_documents_in_mongodb_success(mongo_db):
def test_aggregate_documents_in_mongodb_success(mongo_db):
input_data = {
'collection': 'test_collection',
'aggregation': [{'$match': {'name': {'$exists': True}}}, {'$project': {'name': 1}}],
}
mongo_db.mongo_db_repository.aggregate = AsyncMock(
mongo_db.mongo_db_repository.aggregate = MagicMock(
return_value=[
{
'name': 'test1',
@@ -165,7 +160,7 @@ async def test_aggregate_documents_in_mongodb_success(mongo_db):
]
)
result = await mongo_db.aggregate_documents_in_mongodb(
result = mongo_db.aggregate_documents_in_mongodb(
{'query': input_data, 'timestamp_fields': ['timestamp']}
)
@@ -180,19 +175,18 @@ async def test_aggregate_documents_in_mongodb_success(mongo_db):
)
@mark.asyncio
async def test_aggregate_documents_in_mongodb_failure(mongo_db):
def test_aggregate_documents_in_mongodb_failure(mongo_db):
input_data = {
'collection': 'test_collection',
'aggregation': [{'$match': {'name': {'$exists': True}}}, {'$project': {'name': 1}}],
}
mongo_db.mongo_db_repository.aggregate = AsyncMock(side_effect=Exception('Error'))
mongo_db.mongo_db_repository.aggregate = MagicMock(side_effect=Exception('Error'))
try:
await mongo_db.aggregate_documents_in_mongodb({'query': input_data, **metadata})
mongo_db.aggregate_documents_in_mongodb({'query': input_data, **metadata})
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification_async.assert_called_once_with(
mongo_db.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_AGGREGATION_ERROR',
message='Failed to execute MongoDB aggregation: Error',
@@ -205,12 +199,11 @@ async def test_aggregate_documents_in_mongodb_failure(mongo_db):
raise AssertionError('Expected an exception to be raised')
@mark.asyncio
async def test_aggregate_documents_in_mongodb_missing_collection(mongo_db):
def test_aggregate_documents_in_mongodb_missing_collection(mongo_db):
input_data = {'query': {'aggregation': []}}
try:
await mongo_db.aggregate_documents_in_mongodb(input_data)
mongo_db.aggregate_documents_in_mongodb(input_data)
except ValueError as e:
assert str(e) == 'Collection name must be provided in the query.'
@@ -219,12 +212,11 @@ async def test_aggregate_documents_in_mongodb_missing_collection(mongo_db):
raise AssertionError('Expected a ValueError to be raised')
@mark.asyncio
async def test_aggregate_documents_in_mongodb_missing_aggregation(mongo_db):
def test_aggregate_documents_in_mongodb_missing_aggregation(mongo_db):
input_data = {'query': {'collection': 'test_collection'}}
try:
await mongo_db.aggregate_documents_in_mongodb(input_data)
mongo_db.aggregate_documents_in_mongodb(input_data)
except ValueError as e:
assert str(e) == 'Aggregation must be provided.'
@@ -233,17 +225,16 @@ async def test_aggregate_documents_in_mongodb_missing_aggregation(mongo_db):
raise AssertionError('Expected a ValueError to be raised')
@mark.asyncio
@patch('orchestrator.activities.mongo_db.now')
async def test_update_pipelines_timestamps_success(now_mock, mongo_db):
def test_update_pipelines_timestamps_success(now_mock, mongo_db):
input_data = {
'updated_pipelines': [
{'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
{'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
]
}
mongo_db.mongo_db_repository.update_many = AsyncMock(return_value=MagicMock())
await mongo_db.update_pipelines_timestamps(input_data)
mongo_db.mongo_db_repository.update_many = MagicMock(return_value=MagicMock())
mongo_db.update_pipelines_timestamps(input_data)
mongo_db.mongo_db_repository.update_many.assert_called_once_with(
'orchestrated_schedules',
{
@@ -257,9 +248,8 @@ async def test_update_pipelines_timestamps_success(now_mock, mongo_db):
)
@mark.asyncio
@patch('orchestrator.activities.mongo_db.now')
async def test_update_pipelines_timestamps_failure(now_mock, mongo_db):
def test_update_pipelines_timestamps_failure(now_mock, mongo_db):
input_data = {
'updated_pipelines': [
{'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
@@ -270,10 +260,10 @@ async def test_update_pipelines_timestamps_failure(now_mock, mongo_db):
mongo_db.mongo_db_repository.update_many.side_effect = Exception('Error')
try:
await mongo_db.update_pipelines_timestamps(input_data)
mongo_db.update_pipelines_timestamps(input_data)
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification_async.assert_called_once_with(
mongo_db.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_UPDATE_PIPELINES_ERROR',
message='Failed to update pipelines timestamps: Error',
@@ -286,17 +276,16 @@ async def test_update_pipelines_timestamps_failure(now_mock, mongo_db):
raise AssertionError('Expected an exception to be raised')
@mark.asyncio
@patch('orchestrator.activities.mongo_db.now')
async def test_create_pipelines_timestamps_success(now_mock, mongo_db):
def test_create_pipelines_timestamps_success(now_mock, mongo_db):
input_data = {
'created_pipelines': [
{'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
{'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
]
}
mongo_db.mongo_db_repository.insert_many = AsyncMock(return_value=MagicMock())
await mongo_db.create_pipelines_timestamps(input_data)
mongo_db.mongo_db_repository.insert_many = MagicMock(return_value=MagicMock())
mongo_db.create_pipelines_timestamps(input_data)
mongo_db.mongo_db_repository.insert_many.assert_called_once_with(
'orchestrated_schedules',
[
@@ -307,9 +296,8 @@ async def test_create_pipelines_timestamps_success(now_mock, mongo_db):
)
@mark.asyncio
@patch('orchestrator.activities.mongo_db.now')
async def test_create_pipelines_timestamps_failure(now_mock, mongo_db):
def test_create_pipelines_timestamps_failure(now_mock, mongo_db):
input_data = {
'created_pipelines': [
{'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
@@ -319,10 +307,10 @@ async def test_create_pipelines_timestamps_failure(now_mock, mongo_db):
}
mongo_db.mongo_db_repository.insert_many.side_effect = Exception('Error')
try:
await mongo_db.create_pipelines_timestamps(input_data)
mongo_db.create_pipelines_timestamps(input_data)
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification_async.assert_called_once_with(
mongo_db.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_CREATE_PIPELINES_ERROR',
message='Failed to create pipelines timestamps: Error',
@@ -335,17 +323,16 @@ async def test_create_pipelines_timestamps_failure(now_mock, mongo_db):
raise AssertionError('Expected an exception to be raised')
@mark.asyncio
@patch('orchestrator.activities.mongo_db.now')
async def test_delete_pipelines_timestamps_success(now_mock, mongo_db):
def test_delete_pipelines_timestamps_success(now_mock, mongo_db):
input_data = {
'deleted_pipelines': [
{'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
{'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
]
}
mongo_db.mongo_db_repository.delete_many = AsyncMock(return_value=MagicMock())
await mongo_db.delete_pipelines_timestamps(input_data)
mongo_db.mongo_db_repository.delete_many = MagicMock(return_value=MagicMock())
mongo_db.delete_pipelines_timestamps(input_data)
mongo_db.mongo_db_repository.delete_many.assert_called_once_with(
'orchestrated_schedules',
{
@@ -358,9 +345,8 @@ async def test_delete_pipelines_timestamps_success(now_mock, mongo_db):
)
@mark.asyncio
@patch('orchestrator.activities.mongo_db.now')
async def test_delete_pipelines_timestamps_failure(now_mock, mongo_db):
def test_delete_pipelines_timestamps_failure(now_mock, mongo_db):
input_data = {
'deleted_pipelines': [
{'schedule_name': 'test1', 'namespace': 'test1', 'success': True},
@@ -370,10 +356,10 @@ async def test_delete_pipelines_timestamps_failure(now_mock, mongo_db):
}
mongo_db.mongo_db_repository.delete_many.side_effect = Exception('Error')
try:
await mongo_db.delete_pipelines_timestamps(input_data)
mongo_db.delete_pipelines_timestamps(input_data)
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification_async.assert_called_once_with(
mongo_db.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_DELETE_PIPELINES_ERROR',
message='Failed to delete pipelines timestamps: Error',
@@ -386,8 +372,7 @@ async def test_delete_pipelines_timestamps_failure(now_mock, mongo_db):
raise AssertionError('Expected an exception to be raised')
@mark.asyncio
async def test_create_collection_with_ttl_index_success(mongo_db):
def test_create_collection_with_ttl_index_success(mongo_db):
input_data = {
**metadata,
'pipelines': {
@@ -425,7 +410,7 @@ async def test_create_collection_with_ttl_index_success(mongo_db):
side_effect=[collection_1, collection_2, collection_3]
)
await mongo_db.create_collection_with_ttl_index(input_data)
mongo_db.create_collection_with_ttl_index(input_data)
mongo_db.mongo_db_repository.database.list_collection_names.assert_called_once_with()
@@ -447,8 +432,7 @@ async def test_create_collection_with_ttl_index_success(mongo_db):
collection_3.create_index.assert_not_called()
@mark.asyncio
async def test_create_collection_with_ttl_index_failure(mongo_db):
def test_create_collection_with_ttl_index_failure(mongo_db):
input_data = {**metadata, 'pipelines': {'scouter-pipeline': {'topic': 'raw_scouter_pipeline'}}}
mongo_db.mongo_db_repository.database.list_collection_names.return_value = []
@@ -456,10 +440,10 @@ async def test_create_collection_with_ttl_index_failure(mongo_db):
mongo_db.mongo_db_repository.database.create_collection.side_effect = Exception('Error')
try:
await mongo_db.create_collection_with_ttl_index(input_data)
mongo_db.create_collection_with_ttl_index(input_data)
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification_async.assert_called_once_with(
mongo_db.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_CREATE_COLLECTION_ERROR',
message='Failed to create collection raw_scouter_pipeline with TTL index: Error',
@@ -471,10 +455,9 @@ async def test_create_collection_with_ttl_index_failure(mongo_db):
raise AssertionError('Expected an exception to be raised')
@mark.asyncio
async def test_load_latest_data_none_last_data_timestamp(mongo_db):
def test_load_latest_data_none_last_data_timestamp(mongo_db):
"""Test load_latest_data"""
mongo_db.mongo_db_repository.find = AsyncMock(
mongo_db.mongo_db_repository.find = MagicMock(
return_value=[
{
'name': 'test1',
@@ -484,7 +467,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongo_db):
]
)
result = await mongo_db.load_latest_data(
result = mongo_db.load_latest_data(
{
'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
'collection_name': 'test_collection',
@@ -502,11 +485,10 @@ async def test_load_latest_data_none_last_data_timestamp(mongo_db):
assert result == [{'name': 'test1', 'value': 1, 'timestamp': '2023-01-01 12:00:00+0000'}]
@mark.asyncio
async def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
"""Test load_latest_data"""
mongo_db.mongo_db_repository.find = AsyncMock(
mongo_db.mongo_db_repository.find = MagicMock(
return_value=[
{
'name': 'test1',
@@ -516,7 +498,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
]
)
result = await mongo_db.load_latest_data(
result = mongo_db.load_latest_data(
{
'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
'collection_name': 'test_collection',
@@ -529,9 +511,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
'test_collection',
{
'level': 'ERROR',
'timestamp': {
'$gt': datetime.strptime('2023-01-01 12:00:00+0000', DATETIME_FORMAT_WITH_TZ)
},
'timestamp': {'$gt': '2023-01-01 12:00:00+0000'},
},
{'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
)
@@ -539,13 +519,12 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
assert result == [{'name': 'test1', 'value': 1, 'timestamp': '2023-01-01 12:00:00+0000'}]
@mark.asyncio
async def test_load_latest_data_error(mongo_db):
def test_load_latest_data_error(mongo_db):
"""Test load_latest_data"""
mongo_db.mongo_db_repository.find.side_effect = Exception('test')
try:
await mongo_db.load_latest_data(
mongo_db.load_latest_data(
{
'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
'collection_name': 'test_collection',
@@ -556,7 +535,7 @@ async def test_load_latest_data_error(mongo_db):
except Exception as e:
assert str(e) == 'test'
mongo_db.send_notification_async.assert_called_once_with(
mongo_db.send_notification.assert_called_once_with(
metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
notification_id='MONGO_LOAD_ERROR',
message='Error loading data from MongoDB: test',

131
tests/orchestrator/activities/test_slot_manager.py
View File

@@ -2,7 +2,7 @@ from datetime import timedelta
from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
from pandas import DataFrame
from pytest import fixture, mark
from pytest import fixture
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ, now
@@ -35,21 +35,18 @@ def slot_manager(_redis_mock):
slot_manager.logger = MagicMock()
slot_manager.notification_handler = MagicMock()
slot_manager.send_notification = MagicMock()
slot_manager.send_notification_async = AsyncMock()
slot_manager.emit_metric = AsyncMock()
return slot_manager
@mark.asyncio
async def test_load_opc_slots_no_slot_keys(slot_manager):
slot_manager.redis_repository.keys = AsyncMock(return_value=[])
assert await slot_manager.load_opc_slots(metadata) == {}
def test_load_opc_slots_no_slot_keys(slot_manager):
slot_manager.redis_repository.keys = MagicMock(return_value=[])
assert slot_manager.load_opc_slots(metadata) == {}
@mark.asyncio
async def test_load_opc_slots(slot_manager):
slot_manager.redis_repository.keys = AsyncMock(
def test_load_opc_slots(slot_manager):
slot_manager.redis_repository.keys = MagicMock(
return_value=[
b'slot:opc_tags:1',
b'slot:opc_tags:2',
@@ -57,9 +54,9 @@ async def test_load_opc_slots(slot_manager):
]
)
slot_manager.redis_repository.get = AsyncMock(side_effect=['value1', 'value2', None])
slot_manager.redis_repository.get = MagicMock(side_effect=['value1', 'value2', None])
response = await slot_manager.load_opc_slots(metadata)
response = slot_manager.load_opc_slots(metadata)
assert response == {
'slot:opc_tags:1': 'value1',
@@ -68,9 +65,8 @@ async def test_load_opc_slots(slot_manager):
}
@mark.asyncio
async def test_load_opc_slots_no_decode(slot_manager):
slot_manager.redis_repository.keys = AsyncMock(
def test_load_opc_slots_no_decode(slot_manager):
slot_manager.redis_repository.keys = MagicMock(
return_value=[
'slot:opc_tags:1',
'slot:opc_tags:2',
@@ -78,9 +74,9 @@ async def test_load_opc_slots_no_decode(slot_manager):
]
)
slot_manager.redis_repository.get = AsyncMock(side_effect=['value1', 'value2', None])
slot_manager.redis_repository.get = MagicMock(side_effect=['value1', 'value2', None])
response = await slot_manager.load_opc_slots(metadata)
response = slot_manager.load_opc_slots(metadata)
assert response == {
'slot:opc_tags:1': 'value1',
@@ -89,9 +85,8 @@ async def test_load_opc_slots_no_decode(slot_manager):
}
@mark.asyncio
async def test_load_opc_slots_error(slot_manager):
slot_manager.redis_repository.keys = AsyncMock(
def test_load_opc_slots_error(slot_manager):
slot_manager.redis_repository.keys = MagicMock(
return_value=[
'slot:opc_tags:1',
'slot:opc_tags:2',
@@ -99,13 +94,13 @@ async def test_load_opc_slots_error(slot_manager):
]
)
slot_manager.redis_repository.get = AsyncMock(side_effect=Exception('Test exception'))
slot_manager.redis_repository.get = MagicMock(side_effect=Exception('Test exception'))
try:
await slot_manager.load_opc_slots(metadata)
slot_manager.load_opc_slots(metadata)
except Exception as e:
assert str(e) == 'Test exception'
slot_manager.send_notification_async.assert_called_once_with(
slot_manager.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='REDIS_GET_ERROR',
message='Failed to load OPC slots: Test exception',
@@ -118,9 +113,8 @@ async def test_load_opc_slots_error(slot_manager):
raise AssertionError('Expected an exception to be raised')
@mark.asyncio
async def test_load_active_ingestors(slot_manager):
slot_manager.redis_repository.keys = AsyncMock(
def test_load_active_ingestors(slot_manager):
slot_manager.redis_repository.keys = MagicMock(
return_value=[
b'heartbeat:ingestor:1',
b'heartbeat:ingestor:2',
@@ -128,14 +122,13 @@ async def test_load_active_ingestors(slot_manager):
]
)
response = await slot_manager.load_active_ingestors(metadata)
response = slot_manager.load_active_ingestors(metadata)
assert response == ['heartbeat:ingestor:1', 'heartbeat:ingestor:2', 'heartbeat:ingestor:3']
@mark.asyncio
async def test_load_active_ingestors_error(slot_manager):
slot_manager.redis_repository.keys = AsyncMock(
def test_load_active_ingestors_error(slot_manager):
slot_manager.redis_repository.keys = MagicMock(
return_value=[
'heartbeat:ingestor:1',
'heartbeat:ingestor:2',
@@ -143,13 +136,13 @@ async def test_load_active_ingestors_error(slot_manager):
]
)
slot_manager.redis_repository.keys = AsyncMock(side_effect=Exception('Test exception'))
slot_manager.redis_repository.keys = MagicMock(side_effect=Exception('Test exception'))
try:
await slot_manager.load_active_ingestors(metadata)
slot_manager.load_active_ingestors(metadata)
except Exception as e:
assert str(e) == 'Test exception'
slot_manager.send_notification_async.assert_called_once_with(
slot_manager.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='REDIS_GET_ERROR',
message='Failed to load active ingestors: Test exception',
@@ -162,11 +155,10 @@ async def test_load_active_ingestors_error(slot_manager):
raise AssertionError('Expected an exception to be raised')
@mark.asyncio
async def test_update_slots(slot_manager):
slot_manager.redis_repository.set = AsyncMock(side_effect=[None, Exception('Test exception')])
def test_update_slots(slot_manager):
slot_manager.redis_repository.set = MagicMock(side_effect=[None, Exception('Test exception')])
response = await slot_manager.update_slots({'to_insert': {'1': 'value1', '2': 'value2'}})
response = slot_manager.update_slots({'to_insert': {'1': 'value1', '2': 'value2'}})
slot_manager.redis_repository.set.assert_has_calls(
[call('slot:opc_tags:1', 'value1', ttl=None), call('slot:opc_tags:2', 'value2', ttl=None)]
@@ -178,13 +170,12 @@ async def test_update_slots(slot_manager):
}
@mark.asyncio
async def test_delete_slots(slot_manager):
slot_manager.redis_repository.delete = AsyncMock(
def test_delete_slots(slot_manager):
slot_manager.redis_repository.delete = MagicMock(
side_effect=[None, Exception('Test exception')]
)
response = await slot_manager.delete_slots({'to_delete': ['1', '2']})
response = slot_manager.delete_slots({'to_delete': ['1', '2']})
slot_manager.redis_repository.delete.assert_has_calls(
[call('slot:opc_tags:1'), call('slot:opc_tags:2')]
@@ -196,8 +187,7 @@ async def test_delete_slots(slot_manager):
}
@mark.asyncio
async def test_get_last_data_timestamp_none(slot_manager):
def test_get_last_data_timestamp_none(slot_manager):
"""Test get_last_data_timestamp"""
test_data = {
**metadata,
@@ -206,15 +196,14 @@ async def test_get_last_data_timestamp_none(slot_manager):
'mail_type': 'test_mail_type',
}
slot_manager.redis_repository.get = AsyncMock(return_value=None)
slot_manager.redis_repository.get = MagicMock(return_value=None)
result = await slot_manager.get_last_data_timestamp(test_data)
result = slot_manager.get_last_data_timestamp(test_data)
assert result is None
@mark.asyncio
async def test_get_last_data_timestamp_not_none(slot_manager):
def test_get_last_data_timestamp_not_none(slot_manager):
"""Test get_last_data_timestamp"""
test_data = {
**metadata,
@@ -223,9 +212,9 @@ async def test_get_last_data_timestamp_not_none(slot_manager):
'mail_type': 'test_mail_type',
}
slot_manager.redis_repository.get = AsyncMock(return_value='2023-01-01 12:00:00')
slot_manager.redis_repository.get = MagicMock(return_value='2023-01-01 12:00:00')
result = await slot_manager.get_last_data_timestamp(test_data)
result = slot_manager.get_last_data_timestamp(test_data)
slot_manager.redis_repository.get.assert_called_once_with(
'notification_last_timestamp:test_mail_type'
@@ -234,8 +223,7 @@ async def test_get_last_data_timestamp_not_none(slot_manager):
assert result == '2023-01-01 12:00:00'
@mark.asyncio
async def test_get_last_data_timestamp_error(slot_manager):
def test_get_last_data_timestamp_error(slot_manager):
"""Test get_last_data_timestamp error"""
test_data = {
**metadata,
@@ -244,16 +232,15 @@ async def test_get_last_data_timestamp_error(slot_manager):
'mail_type': 'test_mail_type',
}
slot_manager.send_notification_async = AsyncMock()
slot_manager.redis_repository.get = AsyncMock(side_effect=Exception('test'))
slot_manager.redis_repository.get = MagicMock(side_effect=Exception('test'))
try:
await slot_manager.get_last_data_timestamp(test_data)
slot_manager.get_last_data_timestamp(test_data)
except Exception as e:
assert str(e) == 'test'
slot_manager.send_notification_async.assert_called_once_with(
slot_manager.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='REDIS_GET_ERROR',
message='Error getting last data timestamp: test',
@@ -266,8 +253,7 @@ async def test_get_last_data_timestamp_error(slot_manager):
raise AssertionError('Expected exception')
@mark.asyncio
async def test_put_last_data_timestamp_empty_dataframe(slot_manager):
def test_put_last_data_timestamp_empty_dataframe(slot_manager):
"""Test put_last_data_timestamp with empty dataframe"""
test_data = {
**metadata,
@@ -279,15 +265,14 @@ async def test_put_last_data_timestamp_empty_dataframe(slot_manager):
slot_manager.set = MagicMock()
result = await slot_manager.put_last_data_timestamp(test_data)
result = slot_manager.put_last_data_timestamp(test_data)
assert result is None
slot_manager.set.assert_not_called()
@mark.asyncio
async def test_put_last_data_timestamp_not_empty_dataframe(slot_manager):
def test_put_last_data_timestamp_not_empty_dataframe(slot_manager):
"""Test put_last_data_timestamp with not empty dataframe"""
data = DataFrame(
@@ -305,9 +290,9 @@ async def test_put_last_data_timestamp_not_empty_dataframe(slot_manager):
'mail_type': 'test_mail_type',
}
slot_manager.redis_repository.set = AsyncMock()
slot_manager.redis_repository.set = MagicMock()
result = await slot_manager.put_last_data_timestamp(test_data)
result = slot_manager.put_last_data_timestamp(test_data)
assert result == '2023-01-01 12:00:01'
@@ -316,8 +301,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(slot_manager):
)
@mark.asyncio
async def test_put_last_data_timestamp_error(slot_manager):
def test_put_last_data_timestamp_error(slot_manager):
"""Test put_last_data_timestamp error"""
test_data = {
**metadata,
@@ -333,16 +317,15 @@ async def test_put_last_data_timestamp_error(slot_manager):
'mail_type': 'test_mail_type',
}
slot_manager.send_notification_async = AsyncMock()
slot_manager.redis_repository.set = AsyncMock(side_effect=Exception('test'))
slot_manager.redis_repository.set = MagicMock(side_effect=Exception('test'))
try:
await slot_manager.put_last_data_timestamp(test_data)
slot_manager.put_last_data_timestamp(test_data)
except Exception as e:
assert str(e) == 'test'
slot_manager.send_notification_async.assert_called_once_with(
slot_manager.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='REDIS_SET_ERROR',
message='Error setting last data timestamp: test',
@@ -355,9 +338,8 @@ async def test_put_last_data_timestamp_error(slot_manager):
raise AssertionError('Expected exception')
@mark.asyncio
async def test_filter_notification_alerts(slot_manager):
slot_manager.redis_repository.get = AsyncMock(
def test_filter_notification_alerts(slot_manager):
slot_manager.redis_repository.get = MagicMock(
side_effect=[
None,
(now() - timedelta(seconds=600)).strftime(DATETIME_FORMAT_MS_WITH_TZ),
@@ -383,7 +365,7 @@ async def test_filter_notification_alerts(slot_manager):
'mail_type': 'test_mail_type',
}
response = await slot_manager.filter_notification_alerts(input_data)
response = slot_manager.filter_notification_alerts(input_data)
assert response == {
'test_group_1': {
@@ -404,8 +386,7 @@ async def test_filter_notification_alerts(slot_manager):
}
@mark.asyncio
async def test_store_notification_cache(slot_manager):
def test_store_notification_cache(slot_manager):
"""Test store_notification_cache"""
test_data = {
**metadata,
@@ -419,9 +400,9 @@ async def test_store_notification_cache(slot_manager):
'sent_ttl': 600,
}
slot_manager.redis_repository.set = AsyncMock()
slot_manager.redis_repository.set = MagicMock()
await slot_manager.store_notification_cache(test_data)
slot_manager.store_notification_cache(test_data)
slot_manager.redis_repository.set.assert_called_once_with(
'test_schedule_1:test_notification_id_1', ANY, ttl=600

103
tests/orchestrator/activities/test_temporal_manager.py
View File

@@ -206,7 +206,7 @@ async def test_create_schedule(
'test-workflow',
input_data['schedules']['scouter']['test-schedule'],
id='test-schedule',
task_queue='test-workflow-queue',
task_queue='test-workflow-legacy-queue',
execution_timeout=timedelta(seconds=100),
run_timeout=timedelta(seconds=100),
task_timeout=timedelta(seconds=100),
@@ -216,7 +216,7 @@ async def test_create_schedule(
'test-workflow',
input_data['schedules']['scouter']['test-schedule-invalid-frequency'],
id='test-schedule-invalid-frequency',
task_queue='test-workflow-queue',
task_queue='test-workflow-legacy-queue',
execution_timeout=timedelta(seconds=400),
run_timeout=timedelta(seconds=400),
task_timeout=timedelta(seconds=400),
@@ -226,7 +226,7 @@ async def test_create_schedule(
'test-workflow',
input_data['schedules']['laborious']['test-schedule-laborious'],
id='test-schedule-laborious',
task_queue='test-workflow-queue',
task_queue='test-workflow-legacy-queue',
execution_timeout=timedelta(seconds=500),
run_timeout=timedelta(seconds=500),
task_timeout=timedelta(seconds=500),
@@ -488,3 +488,100 @@ async def test_delete_schedules_with_no_client(temporal_manager):
str(e)
== f'Temporal client for abc not found, clients: {temporal_manager.temporal_clients}'
)
@mark.asyncio
@patch('orchestrator.activities.temporal_manager.parse_frequency', side_effect=parse_frequency)
@patch('orchestrator.activities.temporal_manager.Schedule')
@patch('orchestrator.activities.temporal_manager.ScheduleActionStartWorkflow')
@patch('orchestrator.activities.temporal_manager.ScheduleIntervalSpec')
@patch('orchestrator.activities.temporal_manager.ScheduleSpec')
@patch('orchestrator.activities.temporal_manager.TypedSearchAttributes')
@patch('orchestrator.activities.temporal_manager.SearchAttributePair')
async def test_create_schedules_default_runtime_legacy_queue(
_mock_search_attribute_pair,
_mock_typed_search_attributes,
_mock_schedule_spec,
_mock_schedule_interval_spec,
mock_schedule_action_start_workflow,
_mock_schedule,
_mock_parse_frequency,
temporal_manager,
):
input_data = {
'schedules': {
'scouter': {
'test-schedule': {
'model_id': 1,
'model_name': 'test-model-name',
'workflow_type': 'scouter',
'frequency': '1m',
'data': {'test': 'test'},
}
}
}
}
temporal_manager.temporal_clients['scouter'].create_schedule = AsyncMock()
await temporal_manager.create_schedules(input_data)
mock_schedule_action_start_workflow.assert_called_once_with(
'scouter',
input_data['schedules']['scouter']['test-schedule'],
id='test-schedule',
task_queue='scouter-legacy-queue',
execution_timeout=timedelta(seconds=300),
run_timeout=timedelta(seconds=300),
task_timeout=timedelta(seconds=300),
typed_search_attributes=_mock_typed_search_attributes.return_value,
)
@mark.asyncio
@patch('orchestrator.activities.temporal_manager.parse_frequency', side_effect=parse_frequency)
@patch('orchestrator.activities.temporal_manager.Schedule')
@patch('orchestrator.activities.temporal_manager.ScheduleActionStartWorkflow')
@patch('orchestrator.activities.temporal_manager.ScheduleIntervalSpec')
@patch('orchestrator.activities.temporal_manager.ScheduleSpec')
@patch('orchestrator.activities.temporal_manager.TypedSearchAttributes')
@patch('orchestrator.activities.temporal_manager.SearchAttributePair')
async def test_create_schedules_tenant_runtime_queue(
_mock_search_attribute_pair,
_mock_typed_search_attributes,
_mock_schedule_spec,
_mock_schedule_interval_spec,
mock_schedule_action_start_workflow,
_mock_schedule,
_mock_parse_frequency,
temporal_manager,
):
input_data = {
'schedules': {
'scouter': {
'test-schedule': {
'model_id': 1,
'model_name': 'test-model-name',
'workflow_type': 'scouter',
'frequency': '1m',
'runtime': 'tenant-x',
'data': {'test': 'test'},
}
}
}
}
temporal_manager.temporal_clients['scouter'].create_schedule = AsyncMock()
await temporal_manager.create_schedules(input_data)
mock_schedule_action_start_workflow.assert_called_once_with(
'scouter',
input_data['schedules']['scouter']['test-schedule'],
id='test-schedule',
task_queue='scouter-tenant-x-queue',
execution_timeout=timedelta(seconds=300),
run_timeout=timedelta(seconds=300),
task_timeout=timedelta(seconds=300),
typed_search_attributes=_mock_typed_search_attributes.return_value,
)

22
tests/orchestrator/utils/test_orchestrator_functions.py
View File

@@ -36,10 +36,22 @@ def test_common_config():
'on_conflict': 'error',
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'runtime': 'legacy',
}
assert result == expected
def test_common_config_preserves_explicit_runtime():
config = {
'workflow_type': 'scouter',
'schedule_name': 'test_schedule',
'model_id': 'test_model_id',
'model': {'name': 'test_model_name'},
'runtime': 'tenant-x',
}
assert common_config(config)['runtime'] == 'tenant-x'
def test_drift():
config = {
'workflow_type': 'drift',
@@ -67,6 +79,7 @@ def test_drift():
'drift_metrics': ['kolmogorov_smirnov', 'jensen_shannon'],
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'runtime': 'legacy',
}
assert result == expected
@@ -99,6 +112,7 @@ def test_simple_metrics():
'metrics': ['rmse', 'mse'],
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'runtime': 'legacy',
}
assert result == expected
@@ -129,6 +143,7 @@ def test_minimal_retrain():
'datetime_columns': ['timestamp'],
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'runtime': 'legacy',
}
assert result == expected
@@ -146,6 +161,7 @@ def test_scouter():
'tag_retention_minutes': 10,
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'runtime': 'legacy',
}
result = scouter(config)
expected = {
@@ -169,6 +185,7 @@ def test_scouter():
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'fill_missing_tags': False,
'runtime': 'legacy',
}
assert result == expected
@@ -289,6 +306,7 @@ def test_predictions_batch(mock_process_path_priority, mock_overlap_filter_confi
'predictions_storage_policy': 'erl:1',
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'runtime': 'legacy',
}
assert result == expected
@@ -575,6 +593,7 @@ def test_base_scouter():
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'fill_missing_tags': True,
'runtime': 'legacy',
}
assert result == expected
@@ -635,6 +654,7 @@ def test_pi_web_api_scouter():
'max_count': 5,
'api_timeout': 30,
},
'runtime': 'legacy',
}
assert result == expected
@@ -687,6 +707,7 @@ def test_pi_web_api_scouter_with_timeout_greater_than_frequency():
'max_count': 1,
'api_timeout': 60,
},
'runtime': 'legacy',
}
assert result == expected
@@ -738,5 +759,6 @@ def test_pi_web_api_scouter_with_no_timeout():
'max_count': 1,
'api_timeout': 30,
},
'runtime': 'legacy',
}
assert result == expected

49
tests/orchestrator/worker/test_worker.py Normal file
View File

@@ -0,0 +1,49 @@
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
from pytest import mark
from orchestrator.worker.worker import main
from orchestrator.workflows.alerts import Alerts
from orchestrator.workflows.orchestrator import Orchestrator
from orchestrator.workflows.reports import Reports
def test_main_is_coroutine():
assert asyncio.iscoroutinefunction(main)
@mark.asyncio
@patch('orchestrator.worker.worker.sys')
@patch('orchestrator.worker.worker.start_http_server')
@patch('orchestrator.worker.worker.NotificationHandler')
@patch('orchestrator.worker.worker.Activities')
@patch('orchestrator.worker.worker.prepare_worker')
@patch('orchestrator.worker.worker.client.Client.connect', new_callable=AsyncMock)
async def test_main_starts_three_workers_for_expected_workflows(
_connect_mock,
prepare_worker_mock,
activities_mock,
_notification_handler_mock,
_start_http_server,
_sys_mock,
):
"""
main() must spin up exactly three Temporal workers, one per main workflow
(Orchestrator, Alerts, Reports), and call run() on each.
"""
activities_instance = activities_mock.return_value
activities_instance.connect_to_temporal = AsyncMock()
worker_mock = MagicMock()
worker_mock.run = AsyncMock()
prepare_worker_mock.return_value = worker_mock
await main()
assert prepare_worker_mock.call_count == 3
main_workflows = [call.kwargs['main_workflow'] for call in prepare_worker_mock.call_args_list]
assert main_workflows == [Orchestrator, Alerts, Reports]
assert worker_mock.run.call_count == 3

4
tests/orchestrator/workflows/subworkflows/test_process_notifications.py
View File

@@ -1,7 +1,7 @@
from unittest.mock import ANY, AsyncMock, call, patch
from pytest import fixture, mark
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from orchestrator.activities.activities import Activities
from orchestrator.workflows.subworkflows.process_notifications import ProcessNotifications
@@ -92,7 +92,7 @@ async def test_run(workflow_mock, process_notifications):
'data': workflow_mock.execute_local_activity_method.return_value,
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_MS_WITH_TZ,
'format': DATETIME_FORMAT_WITH_TZ,
},
},
schedule_to_close_timeout=ANY,