gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1325

Browse Source 
Remove unused Couchbase configurations and related code. Refactor Activities class to eliminate Couchbase dependency, updating initialization and shutdown methods. Update README and tests to reflect these changes. Upgrade sientia-dataops-library dependency version in requirements.txt.
This commit is contained in:
vitor-aignosi
2025-11-06 10:45:23 -03:00
parent f099366bed
commit acde9475ec
20 changed files with 458 additions and 614 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
tests/orchestrator/activities/test_activities.py
View File

@@ -13,7 +13,9 @@ from orchestrator.activities.temporal_manager import TemporalManager
@patch('orchestrator.activities.formatters.Formatters.__init__')
@patch('orchestrator.activities.email.Email.__init__')
@patch('sientia_do.temporal.activities.postgres.Postgres.__init__')
@patch('orchestrator.activities.activities.MetricsController')
def test___init__(
mock_metrics_controller,
mock_postgres_init,
mock_email_init,
mock_formatters_init,
@@ -79,6 +81,7 @@ def test___init__(
password='password',
logger=logger,
notification_handler=notification_handler,
metrics_controller=mock_metrics_controller.return_value,
)
mock_mongodb_init.assert_called_once_with(
@@ -88,6 +91,7 @@ def test___init__(
ttl_index_seconds=3600,
logger=logger,
notification_handler=notification_handler,
metrics_controller=mock_metrics_controller.return_value,
)
mock_temporal_manager_init.assert_called_once_with(
@@ -97,6 +101,7 @@ def test___init__(
laborious_namespace='laborious',
logger=logger,
notification_handler=notification_handler,
metrics_controller=mock_metrics_controller.return_value,
)
mock_formatters_init.assert_called_once_with(
@@ -105,28 +110,23 @@ def test___init__(
laborious_namespace='laborious',
logger=logger,
notification_handler=notification_handler,
metrics_controller=mock_metrics_controller.return_value,
)
@patch('orchestrator.activities.mongo_db.MongoDB.__init__')
@patch('orchestrator.activities.temporal_manager.TemporalManager.__init__')
@patch('orchestrator.activities.slot_manager.SlotManager.__init__')
@patch('orchestrator.activities.formatters.Formatters.__init__')
@patch('orchestrator.activities.email.Email.__init__')
@patch('sientia_do.temporal.activities.postgres.Postgres.__init__')
@patch('orchestrator.activities.email.Email.shutdown')
@patch('sientia_do.temporal.activities.postgres.Postgres.close')
@patch('orchestrator.activities.mongo_db.MongoDB.shutdown')
@patch('orchestrator.activities.activities.MongoDB')
@patch('orchestrator.activities.activities.TemporalManager')
@patch('orchestrator.activities.activities.SlotManager')
@patch('orchestrator.activities.activities.Formatters')
@patch('orchestrator.activities.activities.Email')
@patch('orchestrator.activities.activities.Postgres')
def test_shutdown(
mock_mongodb_close,
mock_postgres_shutdown,
mock_email_close,
mock_postgres_init,
mock_email_init,
mock_formatters_init,
mock_slot_manager_init,
mock_temporal_manager_init,
mock_mongodb_init,
mock_mongodb,
mock_temporal_manager,
mock_slot_manager,
mock_formatters,
mock_email,
mock_postgres,
):
activities = Activities(
temporal_config=MagicMock(),
@@ -140,6 +140,9 @@ def test_shutdown(
activities.shutdown()
mock_mongodb_close.assert_called()
mock_postgres_shutdown.assert_called()
mock_email_close.assert_called()
mock_mongodb.close.assert_called()
mock_temporal_manager.close.assert_called()
mock_slot_manager.close.assert_called()
mock_formatters.close.assert_called()
mock_email.close.assert_called()
mock_postgres.close.assert_called()

75
tests/orchestrator/activities/test_couchbase.py
View File

@@ -1,75 +0,0 @@
from unittest.mock import ANY, MagicMock, patch
from pytest import fixture, mark, raises
from sientia_do.notifications.models import NotificationLevel
from orchestrator.activities.couchbase import Couchbase
@fixture
@patch('orchestrator.activities.couchbase.Cluster')
def couchbase(_cluster_mock):
return Couchbase(
connection_string='couchbase://localhost',
username='admin',
password='password',
logger=MagicMock(),
notification_handler=MagicMock(),
)
def test_shutdown_success(couchbase):
couchbase.shutdown()
couchbase.cluster.close.assert_called_once()
def test_shutdown_failure(couchbase):
couchbase.cluster.close.side_effect = Exception('Test error')
couchbase.shutdown()
couchbase.logger.error.assert_called_once_with(
f'Failed to close Couchbase connection: {couchbase.cluster.close.side_effect}'
)
@mark.asyncio
async def test_load_query_from_couchbase_success(couchbase):
couchbase.cluster.query.return_value.rows.return_value = [
{'id': '1', 'name': 'test'},
{'id': '2', 'name': 'test2'},
]
query = 'SELECT * FROM bucket'
result = await couchbase.load_query_from_couchbase(
{
'query': query,
}
)
assert result == [
{'id': '1', 'name': 'test'},
{'id': '2', 'name': 'test2'},
]
couchbase.cluster.query.assert_called_once_with(query)
couchbase.notification_handler.build_and_send_notification.assert_not_called()
@mark.asyncio
async def test_load_query_from_couchbase_failure(couchbase):
couchbase.cluster.query.side_effect = ValueError('Test error')
query = 'SELECT * FROM bucket'
with raises(ValueError):
await couchbase.load_query_from_couchbase(
{
'query': query,
}
)
couchbase.cluster.query.assert_called_once_with(query)
couchbase.notification_handler.build_and_send_notification.assert_called_once_with(
notification_id='COUCHBASE_LOAD_QUERY_ERROR',
message='Failed to execute couchbase query: Test error',
block='load_query_from_couchbase',
level=NotificationLevel.ERROR,
attachment_content=ANY,
)

15
tests/orchestrator/activities/test_email.py
View File

@@ -1,5 +1,5 @@
from smtplib import SMTPServerDisconnected
from unittest.mock import MagicMock, call, patch
from unittest.mock import AsyncMock, MagicMock, call, patch
from pytest import fixture, mark
@@ -17,8 +17,13 @@ def email(smtplib, email_builder):
smtp_port=587,
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
)
email_builder.send_notification_async = AsyncMock()
email_builder.send_notification = MagicMock()
email.send_notification_async = AsyncMock()
email.send_notification = MagicMock()
email.emit_metric = AsyncMock()
return email
@@ -33,6 +38,7 @@ def test___init___with_password(smtplib, email_builder):
smtp_port=587,
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
)
assert email.sender_email == 'test@test.com'
@@ -56,6 +62,7 @@ def test___init___without_password(smtplib, email_builder):
smtp_port=587,
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
)
assert email.sender_email == 'test@test.com'
@@ -76,10 +83,12 @@ metadata = {
}
def test_shutdown(email):
email.shutdown()
@patch('orchestrator.activities.email.SientiaMonitoring')
def test_close(sientia_monitoring_mock, email):
email.close()
email.server.quit.assert_called_once()
sientia_monitoring_mock.shutdown.assert_called_once()
@mark.asyncio

29
tests/orchestrator/activities/test_formatters.py
View File

@@ -1,5 +1,5 @@
import json
from unittest.mock import MagicMock, call, patch
from unittest.mock import AsyncMock, MagicMock, call, patch
from pandas import DataFrame
from pytest import fixture, mark
@@ -15,9 +15,12 @@ def formatters():
laborious_namespace='laborious',
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
)
formatters.send_notification = MagicMock()
formatters.send_notification_async = AsyncMock()
formatters.emit_metric = AsyncMock()
return formatters
@@ -173,7 +176,7 @@ async def test_process_slots(mock_build_tag_config, mock_gather_read_tags, forma
]
)
formatters.send_notification.assert_has_calls(
formatters.send_notification_async.assert_has_calls(
[
call(
metadata=metadata['metadata'],
@@ -288,14 +291,15 @@ async def test_create_slot_config(formatters):
}
def test_send_success_report(formatters):
formatters.send_success_report(
@mark.asyncio
async def test_send_success_report(formatters):
await formatters.send_success_report(
metadata=metadata,
message='test_message',
notification_id='test_notification_id',
attachment={'test': 'test'},
)
formatters.send_notification.assert_called_once_with(
formatters.send_notification_async.assert_called_once_with(
metadata=metadata,
notification_id='test_notification_id',
message='test_message',
@@ -305,14 +309,15 @@ def test_send_success_report(formatters):
)
def test_send_error_report(formatters):
formatters.send_error_report(
@mark.asyncio
async def test_send_error_report(formatters):
await formatters.send_error_report(
metadata=metadata,
message='test_message',
notification_id='test_notification_id',
attachment='test_attachment',
)
formatters.send_notification.assert_called_once_with(
formatters.send_notification_async.assert_called_once_with(
metadata=metadata,
notification_id='test_notification_id',
message='test_message',
@@ -364,8 +369,8 @@ def test_parse_report_schedule(formatters):
@mark.asyncio
async def test_report_schedule_orchestration(formatters):
formatters.parse_report_schedule = MagicMock(side_effect=formatters.parse_report_schedule)
formatters.send_success_report = MagicMock()
formatters.send_error_report = MagicMock()
formatters.send_success_report = AsyncMock()
formatters.send_error_report = AsyncMock()
input_data = {
**metadata,
@@ -496,8 +501,8 @@ async def test_report_schedule_orchestration(formatters):
@mark.asyncio
async def test_report_slot_orchestration(formatters):
formatters.parse_report = MagicMock(side_effect=formatters.parse_report)
formatters.send_success_report = MagicMock()
formatters.send_error_report = MagicMock()
formatters.send_success_report = AsyncMock()
formatters.send_error_report = AsyncMock()
input_data = {
**metadata,

307
tests/orchestrator/activities/test_mongo_db.py
View File

@@ -1,47 +1,15 @@
from datetime import datetime
from unittest.mock import ANY, MagicMock, patch
from unittest.mock import ANY, AsyncMock, MagicMock, patch
from pytest import fixture, mark
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
from orchestrator.activities.mongo_db import MongoDB, clear_mongo_id
def test_clear_mongo_id():
input_data = [
[
{
'name': 'test',
'_id': '12345',
}
],
{
'name': 'test',
'_id': '12345',
'nested': {
'_id': '67890',
'value': [1, 2, 3],
'list': [{'_id': 'abcde', 'item': 'value'}],
},
'nested_list': [{'_id': 'fghij', 'item': 'value1'}, {'_id': 'klmno', 'item': 'value2'}],
},
]
output = clear_mongo_id(input_data)
assert output == [
[{'name': 'test'}],
{
'name': 'test',
'nested': {'value': [1, 2, 3], 'list': [{'item': 'value'}]},
'nested_list': [{'item': 'value1'}, {'item': 'value2'}],
},
]
from orchestrator.activities.mongo_db import MongoDB
@fixture
@patch('orchestrator.activities.mongo_db.MongoClient')
@patch('orchestrator.activities.mongo_db.MongoDBRepository')
def mongo_db(mongo_mock):
mongo = MongoDB(
connection_string='mongodb://localhost:27017',
@@ -49,63 +17,72 @@ def mongo_db(mongo_mock):
ttl_index_seconds=3600,
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
)
mongo.send_notification = MagicMock()
mongo.send_notification_async = AsyncMock()
mongo.emit_metric = AsyncMock()
return mongo
@patch('orchestrator.activities.mongo_db.MongoClient')
@patch('orchestrator.activities.mongo_db.MongoDBRepository')
def test___init__(mongo_mock):
mongo_db = MongoDB(
logger = MagicMock()
notification_handler = MagicMock()
metrics_controller = AsyncMock()
MongoDB(
connection_string='mongodb://localhost:27017',
database_name='test_db',
ttl_index_seconds=3600,
logger=MagicMock(),
notification_handler=MagicMock(),
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
assert mongo_db.connection_string == 'mongodb://localhost:27017'
assert mongo_db.database_name == 'test_db'
mongo_mock.assert_called_once_with('mongodb://localhost:27017', serverSelectionTimeoutMS=5000)
mongo_db.client.server_info.assert_called_once()
mongo_db.client.__getitem__.assert_called_once_with('test_db')
def test_shutdown_success(mongo_db):
mongo_db.shutdown()
mongo_db.client.close.assert_called_once()
mongo_db.logger.info.assert_any_call('Closing MongoDB connection...')
mongo_db.logger.info.assert_any_call('MongoDB connection closed successfully')
def test_shutdown_failure(mongo_db):
mongo_db.client.close.side_effect = Exception('Close failed')
mongo_db.shutdown()
mongo_db.logger.error.assert_called_once_with(
'Failed to close MongoDB connection: Close failed'
mongo_mock.assert_called_once_with(
connection_string='mongodb://localhost:27017',
database_name='test_db',
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
@patch('orchestrator.activities.mongo_db.SientiaMonitoring')
def test_close(sientia_monitoring_mock, mongo_db):
mongo_db.close()
mongo_db.mongo_db_repository.close.assert_called_once()
sientia_monitoring_mock.shutdown.assert_called_once()
def test___del__(mongo_db):
mongo_db.close = MagicMock()
mongo_db.__del__()
mongo_db.close.assert_called_once()
@mark.asyncio
async def test_find_documents_in_mongodb_success(mongo_db):
input_data = {'collection': 'test_collection', 'filters': {'name': {'$exists': True}}}
mock_collection = MagicMock()
mock_collection.find.return_value = [
{
'_id': '12345',
'name': 'test1',
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
},
{
'_id': '67890',
'name': 'test2',
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
},
]
mongo_db.database.__getitem__.return_value = mock_collection
mongo_db.mongo_db_repository.find = AsyncMock(
return_value=[
{
'name': 'test1',
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
},
{
'name': 'test2',
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
},
]
)
result = await mongo_db.find_documents_in_mongodb(
{'query': input_data, 'timestamp_fields': ['timestamp']}
@@ -114,7 +91,9 @@ async def test_find_documents_in_mongodb_success(mongo_db):
assert len(result) == 2
assert result[0] == {'name': 'test1', 'timestamp': '2023-01-01 12:00:00.000000+0000'}
assert result[1] == {'name': 'test2', 'timestamp': '2023-01-01 12:00:00.000000+0000'}
mock_collection.find.assert_called_once_with({'name': {'$exists': True}}, {'_id': 0})
mongo_db.mongo_db_repository.find.assert_called_once_with(
'test_collection', {'name': {'$exists': True}}, {}
)
metadata = {
@@ -130,15 +109,13 @@ metadata = {
@mark.asyncio
async def test_find_documents_in_mongodb_failure(mongo_db):
input_data = {'collection': 'test_collection', 'filters': {'name': {'$exists': True}}}
mongo_db.database.__getitem__.return_value = MagicMock(
find=MagicMock(side_effect=Exception('Error'))
)
mongo_db.mongo_db_repository.find = AsyncMock(side_effect=Exception('Error'))
try:
await mongo_db.find_documents_in_mongodb({'query': input_data, **metadata})
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification.assert_called_once_with(
mongo_db.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_QUERY_ERROR',
message='Failed to execute MongoDB query: Error',
@@ -171,24 +148,22 @@ async def test_aggregate_documents_in_mongodb_success(mongo_db):
'collection': 'test_collection',
'aggregation': [{'$match': {'name': {'$exists': True}}}, {'$project': {'name': 1}}],
}
mock_collection = MagicMock()
mock_collection.aggregate.return_value = [
{
'_id': 'asdad',
'name': 'test1',
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
},
{
'_id': 'adzx',
'name': 'test2',
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
},
]
mongo_db.database.__getitem__.return_value = mock_collection
mongo_db.mongo_db_repository.aggregate = AsyncMock(
return_value=[
{
'name': 'test1',
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
},
{
'name': 'test2',
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
},
]
)
result = await mongo_db.aggregate_documents_in_mongodb(
{'query': input_data, 'timestamp_fields': ['timestamp']}
@@ -200,7 +175,9 @@ async def test_aggregate_documents_in_mongodb_success(mongo_db):
expected_pipeline = input_data['aggregation']
expected_pipeline.append({'$project': {'_id': 0}})
mock_collection.aggregate.assert_called_once_with(expected_pipeline)
mongo_db.mongo_db_repository.aggregate.assert_called_once_with(
'test_collection', expected_pipeline, {}
)
@mark.asyncio
@@ -209,15 +186,13 @@ async def test_aggregate_documents_in_mongodb_failure(mongo_db):
'collection': 'test_collection',
'aggregation': [{'$match': {'name': {'$exists': True}}}, {'$project': {'name': 1}}],
}
mongo_db.database.__getitem__.return_value = MagicMock(
aggregate=MagicMock(side_effect=Exception('Error'))
)
mongo_db.mongo_db_repository.aggregate = AsyncMock(side_effect=Exception('Error'))
try:
await mongo_db.aggregate_documents_in_mongodb({'query': input_data, **metadata})
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification.assert_called_once_with(
mongo_db.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_AGGREGATION_ERROR',
message='Failed to execute MongoDB aggregation: Error',
@@ -267,9 +242,10 @@ async def test_update_pipelines_timestamps_success(now_mock, mongo_db):
{'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
]
}
mongo_db.database['pipelines'].update_many.return_value = MagicMock()
mongo_db.mongo_db_repository.update_many = AsyncMock(return_value=MagicMock())
await mongo_db.update_pipelines_timestamps(input_data)
mongo_db.database['pipelines'].update_many.assert_called_once_with(
mongo_db.mongo_db_repository.update_many.assert_called_once_with(
'orchestrated_schedules',
{
'$or': [
{'schedule_name': 'test1', 'namespace': 'test1'},
@@ -277,6 +253,7 @@ async def test_update_pipelines_timestamps_success(now_mock, mongo_db):
]
},
{'$set': {'updated_at': now_mock.return_value}},
{},
)
@@ -291,12 +268,12 @@ async def test_update_pipelines_timestamps_failure(now_mock, mongo_db):
**metadata,
}
mongo_db.database['pipelines'].update_many.side_effect = Exception('Error')
mongo_db.mongo_db_repository.update_many.side_effect = Exception('Error')
try:
await mongo_db.update_pipelines_timestamps(input_data)
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification.assert_called_once_with(
mongo_db.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_UPDATE_PIPELINES_ERROR',
message='Failed to update pipelines timestamps: Error',
@@ -318,13 +295,15 @@ async def test_create_pipelines_timestamps_success(now_mock, mongo_db):
{'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
]
}
mongo_db.database['pipelines'].insert_many.return_value = MagicMock()
mongo_db.mongo_db_repository.insert_many = AsyncMock(return_value=MagicMock())
await mongo_db.create_pipelines_timestamps(input_data)
mongo_db.database['pipelines'].insert_many.assert_called_once_with(
mongo_db.mongo_db_repository.insert_many.assert_called_once_with(
'orchestrated_schedules',
[
{'schedule_name': 'test1', 'namespace': 'test1', 'updated_at': now_mock.return_value},
{'schedule_name': 'test2', 'namespace': 'test2', 'updated_at': now_mock.return_value},
]
],
{},
)
@@ -338,12 +317,12 @@ async def test_create_pipelines_timestamps_failure(now_mock, mongo_db):
],
**metadata,
}
mongo_db.database['pipelines'].insert_many.side_effect = Exception('Error')
mongo_db.mongo_db_repository.insert_many.side_effect = Exception('Error')
try:
await mongo_db.create_pipelines_timestamps(input_data)
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification.assert_called_once_with(
mongo_db.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_CREATE_PIPELINES_ERROR',
message='Failed to create pipelines timestamps: Error',
@@ -365,15 +344,17 @@ async def test_delete_pipelines_timestamps_success(now_mock, mongo_db):
{'schedule_name': 'test2', 'namespace': 'test2', 'success': True},
]
}
mongo_db.database['pipelines'].delete_many.return_value = MagicMock()
mongo_db.mongo_db_repository.delete_many = AsyncMock(return_value=MagicMock())
await mongo_db.delete_pipelines_timestamps(input_data)
mongo_db.database['pipelines'].delete_many.assert_called_once_with(
mongo_db.mongo_db_repository.delete_many.assert_called_once_with(
'orchestrated_schedules',
{
'$or': [
{'schedule_name': 'test1', 'namespace': 'test1'},
{'schedule_name': 'test2', 'namespace': 'test2'},
]
}
},
{},
)
@@ -387,12 +368,12 @@ async def test_delete_pipelines_timestamps_failure(now_mock, mongo_db):
],
**metadata,
}
mongo_db.database['pipelines'].delete_many.side_effect = Exception('Error')
mongo_db.mongo_db_repository.delete_many.side_effect = Exception('Error')
try:
await mongo_db.delete_pipelines_timestamps(input_data)
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification.assert_called_once_with(
mongo_db.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_DELETE_PIPELINES_ERROR',
message='Failed to delete pipelines timestamps: Error',
@@ -416,10 +397,12 @@ async def test_create_collection_with_ttl_index_success(mongo_db):
},
}
mongo_db.database.list_collection_names.return_value = [
'raw_scouter_pipeline_2',
'raw_scouter_pipeline_3',
]
mongo_db.mongo_db_repository.database.list_collection_names = MagicMock(
return_value=[
'raw_scouter_pipeline_2',
'raw_scouter_pipeline_3',
]
)
collection_1 = MagicMock(
list_indexes=MagicMock(
@@ -438,15 +421,17 @@ async def test_create_collection_with_ttl_index_success(mongo_db):
list_indexes=MagicMock(return_value=[{'key': 'inserted_at', 'expireAfterSeconds': 3600}])
)
mongo_db.database.__getitem__ = MagicMock(
mongo_db.mongo_db_repository.database.__getitem__ = MagicMock(
side_effect=[collection_1, collection_2, collection_3]
)
await mongo_db.create_collection_with_ttl_index(input_data)
mongo_db.database.list_collection_names.assert_called_once_with()
mongo_db.mongo_db_repository.database.list_collection_names.assert_called_once_with()
mongo_db.database.create_collection.assert_called_once_with('raw_scouter_pipeline')
mongo_db.mongo_db_repository.database.create_collection.assert_called_once_with(
'raw_scouter_pipeline'
)
collection_1.list_indexes.assert_called_once()
collection_1.create_index.assert_called_once_with(
@@ -466,15 +451,15 @@ async def test_create_collection_with_ttl_index_success(mongo_db):
async def test_create_collection_with_ttl_index_failure(mongo_db):
input_data = {**metadata, 'pipelines': {'scouter-pipeline': {'topic': 'raw_scouter_pipeline'}}}
mongo_db.database.list_collection_names.return_value = []
mongo_db.mongo_db_repository.database.list_collection_names.return_value = []
mongo_db.database.create_collection.side_effect = Exception('Error')
mongo_db.mongo_db_repository.database.create_collection.side_effect = Exception('Error')
try:
await mongo_db.create_collection_with_ttl_index(input_data)
except Exception as e:
assert str(e) == 'Error'
mongo_db.send_notification.assert_called_once_with(
mongo_db.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_CREATE_COLLECTION_ERROR',
message='Failed to create collection raw_scouter_pipeline with TTL index: Error',
@@ -489,18 +474,17 @@ async def test_create_collection_with_ttl_index_failure(mongo_db):
@mark.asyncio
async def test_load_latest_data_none_last_data_timestamp(mongo_db):
"""Test load_latest_data"""
collection = MagicMock()
mongo_db.database.__getitem__.return_value = collection
collection.find.return_value = [
{
'name': 'test1',
'value': 1,
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
}
]
mongo_db.mongo_db_repository.find = AsyncMock(
return_value=[
{
'name': 'test1',
'value': 1,
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
}
]
)
result = await mongo_db.load_latest_data(
{
@@ -511,9 +495,11 @@ async def test_load_latest_data_none_last_data_timestamp(mongo_db):
}
)
mongo_db.database.__getitem__.assert_called_once_with('test_collection')
collection.find.assert_called_once_with({'level': 'ERROR'}, {'_id': 0})
mongo_db.mongo_db_repository.find.assert_called_once_with(
'test_collection',
{'level': 'ERROR'},
{'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
)
assert result == [{'name': 'test1', 'value': 1, 'timestamp': '2023-01-01 12:00:00.000000+0000'}]
@@ -521,18 +507,18 @@ async def test_load_latest_data_none_last_data_timestamp(mongo_db):
@mark.asyncio
async def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
"""Test load_latest_data"""
collection = MagicMock()
mongo_db.database.__getitem__.return_value = collection
collection.find.return_value = [
{
'name': 'test1',
'value': 1,
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
}
]
mongo_db.mongo_db_repository.find = AsyncMock(
return_value=[
{
'name': 'test1',
'value': 1,
'timestamp': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
}
]
)
result = await mongo_db.load_latest_data(
{
@@ -543,9 +529,8 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
}
)
mongo_db.database.__getitem__.assert_called_once_with('test_collection')
collection.find.assert_called_once_with(
mongo_db.mongo_db_repository.find.assert_called_once_with(
'test_collection',
{
'level': 'ERROR',
'timestamp': {
@@ -554,7 +539,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
)
},
},
{'_id': 0},
{'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
)
assert result == [{'name': 'test1', 'value': 1, 'timestamp': '2023-01-01 12:00:00.000000+0000'}]
@@ -563,11 +548,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
@mark.asyncio
async def test_load_latest_data_error(mongo_db):
"""Test load_latest_data"""
collection = MagicMock()
mongo_db.send_notification = MagicMock()
mongo_db.database.__getitem__.return_value = collection
collection.find.side_effect = Exception('test')
mongo_db.mongo_db_repository.find.side_effect = Exception('test')
try:
await mongo_db.load_latest_data(
@@ -581,7 +562,7 @@ async def test_load_latest_data_error(mongo_db):
except Exception as e:
assert str(e) == 'test'
mongo_db.send_notification.assert_called_once_with(
mongo_db.send_notification_async.assert_called_once_with(
metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
notification_id='MONGO_LOAD_ERROR',
message='Error loading data from MongoDB: test',

125
tests/orchestrator/activities/test_slot_manager.py
View File

@@ -1,5 +1,5 @@
from datetime import timedelta
from unittest.mock import ANY, MagicMock, call, patch
from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
from pandas import DataFrame
from pytest import fixture, mark
@@ -19,7 +19,7 @@ metadata = {
@fixture
@patch('orchestrator.activities.slot_manager.Redis.__init__')
@patch('orchestrator.activities.slot_manager.RedisRepository')
def slot_manager(_redis_mock):
slot_manager = SlotManager(
host='localhost',
@@ -28,31 +28,36 @@ def slot_manager(_redis_mock):
password='password',
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
)
slot_manager.redis_client = MagicMock()
slot_manager.redis_repository = MagicMock()
slot_manager.logger = MagicMock()
slot_manager.notification_handler = MagicMock()
slot_manager.send_notification = MagicMock()
slot_manager.send_notification_async = AsyncMock()
slot_manager.emit_metric = AsyncMock()
return slot_manager
@mark.asyncio
async def test_load_opc_slots_no_slot_keys(slot_manager):
slot_manager.redis_client.keys.return_value = []
slot_manager.redis_repository.keys = AsyncMock(return_value=[])
assert await slot_manager.load_opc_slots(metadata) == {}
@mark.asyncio
async def test_load_opc_slots(slot_manager):
slot_manager.redis_client.keys.return_value = [
b'slot:opc_tags:1',
b'slot:opc_tags:2',
b'slot:opc_tags:3',
]
slot_manager.redis_repository.keys = AsyncMock(
return_value=[
b'slot:opc_tags:1',
b'slot:opc_tags:2',
b'slot:opc_tags:3',
]
)
slot_manager.get = MagicMock(side_effect=['value1', 'value2', None])
slot_manager.redis_repository.get = AsyncMock(side_effect=['value1', 'value2', None])
response = await slot_manager.load_opc_slots(metadata)
@@ -65,13 +70,15 @@ async def test_load_opc_slots(slot_manager):
@mark.asyncio
async def test_load_opc_slots_no_decode(slot_manager):
slot_manager.redis_client.keys.return_value = [
'slot:opc_tags:1',
'slot:opc_tags:2',
'slot:opc_tags:3',
]
slot_manager.redis_repository.keys = AsyncMock(
return_value=[
'slot:opc_tags:1',
'slot:opc_tags:2',
'slot:opc_tags:3',
]
)
slot_manager.get = MagicMock(side_effect=['value1', 'value2', None])
slot_manager.redis_repository.get = AsyncMock(side_effect=['value1', 'value2', None])
response = await slot_manager.load_opc_slots(metadata)
@@ -84,19 +91,21 @@ async def test_load_opc_slots_no_decode(slot_manager):
@mark.asyncio
async def test_load_opc_slots_error(slot_manager):
slot_manager.redis_client.keys.return_value = [
'slot:opc_tags:1',
'slot:opc_tags:2',
'slot:opc_tags:3',
]
slot_manager.redis_repository.keys = AsyncMock(
return_value=[
'slot:opc_tags:1',
'slot:opc_tags:2',
'slot:opc_tags:3',
]
)
slot_manager.get = MagicMock(side_effect=Exception('Test exception'))
slot_manager.redis_repository.get = AsyncMock(side_effect=Exception('Test exception'))
try:
await slot_manager.load_opc_slots(metadata)
except Exception as e:
assert str(e) == 'Test exception'
slot_manager.send_notification.assert_called_once_with(
slot_manager.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='REDIS_GET_ERROR',
message='Failed to load OPC slots: Test exception',
@@ -111,11 +120,13 @@ async def test_load_opc_slots_error(slot_manager):
@mark.asyncio
async def test_load_active_ingestors(slot_manager):
slot_manager.redis_client.keys.return_value = [
b'heartbeat:ingestor:1',
b'heartbeat:ingestor:2',
'heartbeat:ingestor:3',
]
slot_manager.redis_repository.keys = AsyncMock(
return_value=[
b'heartbeat:ingestor:1',
b'heartbeat:ingestor:2',
'heartbeat:ingestor:3',
]
)
response = await slot_manager.load_active_ingestors(metadata)
@@ -124,19 +135,21 @@ async def test_load_active_ingestors(slot_manager):
@mark.asyncio
async def test_load_active_ingestors_error(slot_manager):
slot_manager.redis_client.keys.return_value = [
'heartbeat:ingestor:1',
'heartbeat:ingestor:2',
'heartbeat:ingestor:3',
]
slot_manager.redis_repository.keys = AsyncMock(
return_value=[
'heartbeat:ingestor:1',
'heartbeat:ingestor:2',
'heartbeat:ingestor:3',
]
)
slot_manager.redis_client.keys.side_effect = Exception('Test exception')
slot_manager.redis_repository.keys = AsyncMock(side_effect=Exception('Test exception'))
try:
await slot_manager.load_active_ingestors(metadata)
except Exception as e:
assert str(e) == 'Test exception'
slot_manager.send_notification.assert_called_once_with(
slot_manager.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='REDIS_GET_ERROR',
message='Failed to load active ingestors: Test exception',
@@ -151,11 +164,11 @@ async def test_load_active_ingestors_error(slot_manager):
@mark.asyncio
async def test_update_slots(slot_manager):
slot_manager.set = MagicMock(side_effect=[None, Exception('Test exception')])
slot_manager.redis_repository.set = AsyncMock(side_effect=[None, Exception('Test exception')])
response = await slot_manager.update_slots({'to_insert': {'1': 'value1', '2': 'value2'}})
slot_manager.set.assert_has_calls(
slot_manager.redis_repository.set.assert_has_calls(
[call('slot:opc_tags:1', 'value1', ttl=None), call('slot:opc_tags:2', 'value2', ttl=None)]
)
@@ -167,11 +180,13 @@ async def test_update_slots(slot_manager):
@mark.asyncio
async def test_delete_slots(slot_manager):
slot_manager.redis_client.delete = MagicMock(side_effect=[None, Exception('Test exception')])
slot_manager.redis_repository.delete = AsyncMock(
side_effect=[None, Exception('Test exception')]
)
response = await slot_manager.delete_slots({'to_delete': ['1', '2']})
slot_manager.redis_client.delete.assert_has_calls(
slot_manager.redis_repository.delete.assert_has_calls(
[call('slot:opc_tags:1'), call('slot:opc_tags:2')]
)
@@ -191,7 +206,7 @@ async def test_get_last_data_timestamp_none(slot_manager):
'mail_type': 'test_mail_type',
}
slot_manager.get = MagicMock(return_value=None)
slot_manager.redis_repository.get = AsyncMock(return_value=None)
result = await slot_manager.get_last_data_timestamp(test_data)
@@ -208,11 +223,13 @@ async def test_get_last_data_timestamp_not_none(slot_manager):
'mail_type': 'test_mail_type',
}
slot_manager.get = MagicMock(return_value='2023-01-01 12:00:00')
slot_manager.redis_repository.get = AsyncMock(return_value='2023-01-01 12:00:00')
result = await slot_manager.get_last_data_timestamp(test_data)
slot_manager.get.assert_called_once_with('notification_last_timestamp:test_mail_type')
slot_manager.redis_repository.get.assert_called_once_with(
'notification_last_timestamp:test_mail_type'
)
assert result == '2023-01-01 12:00:00'
@@ -227,8 +244,8 @@ async def test_get_last_data_timestamp_error(slot_manager):
'mail_type': 'test_mail_type',
}
slot_manager.send_notification = MagicMock()
slot_manager.get = MagicMock(side_effect=Exception('test'))
slot_manager.send_notification_async = AsyncMock()
slot_manager.redis_repository.get = AsyncMock(side_effect=Exception('test'))
try:
await slot_manager.get_last_data_timestamp(test_data)
@@ -236,7 +253,7 @@ async def test_get_last_data_timestamp_error(slot_manager):
except Exception as e:
assert str(e) == 'test'
slot_manager.send_notification.assert_called_once_with(
slot_manager.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='REDIS_GET_ERROR',
message='Error getting last data timestamp: test',
@@ -288,13 +305,13 @@ async def test_put_last_data_timestamp_not_empty_dataframe(slot_manager):
'mail_type': 'test_mail_type',
}
slot_manager.set = MagicMock()
slot_manager.redis_repository.set = AsyncMock()
result = await slot_manager.put_last_data_timestamp(test_data)
assert result == '2023-01-01 12:00:01'
slot_manager.set.assert_called_once_with(
slot_manager.redis_repository.set.assert_called_once_with(
'notification_last_timestamp:test_mail_type', '2023-01-01 12:00:01', ttl=18000
)
@@ -316,8 +333,8 @@ async def test_put_last_data_timestamp_error(slot_manager):
'mail_type': 'test_mail_type',
}
slot_manager.send_notification = MagicMock()
slot_manager.set = MagicMock(side_effect=Exception('test'))
slot_manager.send_notification_async = AsyncMock()
slot_manager.redis_repository.set = AsyncMock(side_effect=Exception('test'))
try:
await slot_manager.put_last_data_timestamp(test_data)
@@ -325,7 +342,7 @@ async def test_put_last_data_timestamp_error(slot_manager):
except Exception as e:
assert str(e) == 'test'
slot_manager.send_notification.assert_called_once_with(
slot_manager.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='REDIS_SET_ERROR',
message='Error setting last data timestamp: test',
@@ -340,7 +357,7 @@ async def test_put_last_data_timestamp_error(slot_manager):
@mark.asyncio
async def test_filter_notification_alerts(slot_manager):
slot_manager.get = MagicMock(
slot_manager.redis_repository.get = AsyncMock(
side_effect=[
None,
(now() - timedelta(seconds=600)).strftime(DATETIME_FORMAT_MS_WITH_TZ),
@@ -402,8 +419,10 @@ async def test_store_notification_cache(slot_manager):
'sent_ttl': 600,
}
slot_manager.set = MagicMock()
slot_manager.redis_repository.set = AsyncMock()
await slot_manager.store_notification_cache(test_data)
slot_manager.set.assert_called_once_with('test_schedule_1:test_notification_id_1', ANY, ttl=600)
slot_manager.redis_repository.set.assert_called_once_with(
'test_schedule_1:test_notification_id_1', ANY, ttl=600
)

5
tests/orchestrator/activities/test_temporal_manager.py
View File

@@ -26,10 +26,13 @@ def temporal_manager(connect_mock):
laborious_namespace='laborious',
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
)
temporal_manager.temporal_clients['scouter'] = MagicMock()
temporal_manager.temporal_clients['laborious'] = MagicMock()
temporal_manager.send_notification_async = AsyncMock()
temporal_manager.emit_metric = AsyncMock()
temporal_manager.send_notification = MagicMock()
return temporal_manager
@@ -100,7 +103,7 @@ async def test_normalize_schedules_error(temporal_manager):
await temporal_manager.normalize_schedules(metadata)
except Exception as e:
assert str(e) == 'Test exception'
temporal_manager.send_notification.assert_called_once_with(
temporal_manager.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='SCHEDULER_NORMALIZE_SCHEDULES_ERROR',
message='Failed to normalize schedules: Test exception',