gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1150

Browse Source 
refactor: update logging in Couchbase activity to use f-strings for improved readability; modify orchestration workflow to conditionally execute report handlers and replace method calls for consistency
This commit is contained in:
vitor-aignosi
2025-07-15 10:03:04 -03:00
parent 499eae44d9
commit 2533ea882f
5 changed files with 114 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
tests/orchestrator/activities/test_formatters.py
View File

@@ -18,9 +18,9 @@ def formatters():
@mark.asyncio
@patch("orchestrator.activities.formatters.scouter",
return_value="test_scouter")
return_value={"test_scouter": "test_scouter"})
@patch("orchestrator.activities.formatters.predictions_batch",
return_value="test_predictions_batch")
return_value={"test_predictions_batch": "test_predictions_batch"})
async def test_process_schedules(mock_predictions_batch, mock_scouter, formatters):
input_data = {
"pipelines": [
@@ -28,13 +28,15 @@ async def test_process_schedules(mock_predictions_batch, mock_scouter, formatter
"schedule_name": "test_schedule_name",
"workflow_type": "scouter",
"model_name": "test_model_name",
"model_id": "test_model_id"
"model_id": "test_model_id",
"updated_at": "2021-01-01"
},
{
"schedule_name": "test_schedule_name2",
"workflow_type": "predictions_batch",
"model_name": "test_model_name",
"model_id": "test_model_id"
"model_id": "test_model_id",
"updated_at": "2021-01-02"
}
]
}
@@ -43,10 +45,16 @@ async def test_process_schedules(mock_predictions_batch, mock_scouter, formatter
assert result == {
"scouter": {
"test_schedule_name": "test_scouter"
"test_schedule_name": {
"test_scouter": "test_scouter",
"updated_at": "2021-01-01"
}
},
"laborious": {
"test_schedule_name2": "test_predictions_batch"
"test_schedule_name2": {
"test_predictions_batch": "test_predictions_batch",
"updated_at": "2021-01-02"
}
}
}

6
tests/orchestrator/utils/test_connectors_config.py
View File

@@ -36,8 +36,8 @@ def test_build_redis_config_with_defaults():
assert build_redis_config() == {
'host': 'localhost',
'port': 6379,
'username': None,
'password': None
'username': 'default',
'password': 'bdnZOpcyiL'
}
@@ -71,7 +71,7 @@ def test_build_mongo_db_config_with_defaults():
environ.pop('MONGODB_URL', None)
assert build_mongodb_config() == {
'connection_string': 'mongodb://sientia:sientia@localhost:27017',
'connection_string': 'mongodb://root:wKZDbMNU1c@localhost:27018',
'database_name': 'sientia'
}

120
tests/orchestrator/workflows/test_orchestrator.py
View File

@@ -20,7 +20,7 @@ async def test_run(workflow_mock, orchestrator):
await orchestrator.run(input_data)
workflow_mock.execute_local_activity_method.assert_has_calls([
workflow_mock.start_local_activity_method.assert_has_calls([
call(
Activities.aggregate_documents_in_mongodb,
{
@@ -31,7 +31,7 @@ async def test_run(workflow_mock, orchestrator):
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
workflow_mock.start_local_activity_method.assert_has_calls([
call(
Activities.find_documents_in_mongodb,
{
@@ -42,7 +42,7 @@ async def test_run(workflow_mock, orchestrator):
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
workflow_mock.start_local_activity_method.assert_has_calls([
call(
Activities.find_documents_in_mongodb,
{
@@ -55,7 +55,7 @@ async def test_run(workflow_mock, orchestrator):
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
workflow_mock.start_local_activity_method.assert_has_calls([
call(
Activities.load_opc_slots,
retry_policy=ANY,
@@ -63,7 +63,7 @@ async def test_run(workflow_mock, orchestrator):
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
workflow_mock.start_local_activity_method.assert_has_calls([
call(
Activities.load_active_ingestors,
retry_policy=ANY,
@@ -71,130 +71,188 @@ async def test_run(workflow_mock, orchestrator):
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
workflow_mock.start_local_activity_method.assert_has_calls([
call(
Activities.format_schedule_config,
{
'schedule_config': workflow_mock.execute_local_activity_method.return_value
'schedule_config': workflow_mock.start_local_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
workflow_mock.start_local_activity_method.assert_has_calls([
call(
Activities.process_schedules,
{
'pipelines': workflow_mock.execute_local_activity_method.return_value
'pipelines': workflow_mock.start_local_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
workflow_mock.start_local_activity_method.assert_has_calls([
call(
Activities.process_slots,
{
'opc_servers': workflow_mock.execute_local_activity_method.return_value,
'active_ingestors': workflow_mock.execute_local_activity_method.return_value,
'pipelines': workflow_mock.execute_local_activity_method.return_value,
'opc_servers': workflow_mock.start_local_activity_method.return_value,
'active_ingestors': workflow_mock.start_local_activity_method.return_value,
'pipelines': workflow_mock.start_local_activity_method.return_value,
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
workflow_mock.start_local_activity_method.assert_has_calls([
call(
Activities.create_schedule_config,
{
'current_schedule_config': workflow_mock.execute_local_activity_method.return_value,
'schedule_config': workflow_mock.execute_local_activity_method.return_value
'current_schedule_config': workflow_mock.start_local_activity_method.return_value,
'schedule_config': workflow_mock.start_local_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
workflow_mock.start_local_activity_method.assert_has_calls([
call(
Activities.create_slot_config,
{
'current_slot_config': workflow_mock.execute_local_activity_method.return_value,
'slot_config': workflow_mock.execute_local_activity_method.return_value
'current_slot_config': workflow_mock.start_local_activity_method.return_value,
'slot_config': workflow_mock.start_local_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
workflow_mock.start_local_activity_method.assert_has_calls([
call(
Activities.normalize_schedules,
{
'orchestrated_schedules': workflow_mock.execute_local_activity_method.return_value
'orchestrated_schedules': workflow_mock.start_local_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_activity_method.assert_has_calls([
workflow_mock.start_activity_method.assert_has_calls([
call(
Activities.delete_slots,
{
'to_delete':
workflow_mock.execute_local_activity_method.return_value['to_delete']
workflow_mock.start_local_activity_method.return_value['to_delete']
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_activity_method.assert_has_calls([
workflow_mock.start_activity_method.assert_has_calls([
call(
Activities.update_slots,
{
'to_insert':
workflow_mock.execute_local_activity_method.return_value['to_insert']
workflow_mock.start_local_activity_method.return_value['to_insert']
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_activity_method.assert_has_calls([
workflow_mock.start_activity_method.assert_has_calls([
call(
Activities.delete_schedules,
{
'schedules':
workflow_mock.execute_local_activity_method.return_value['to_delete']
workflow_mock.start_local_activity_method.return_value['to_delete']
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_activity_method.assert_has_calls([
workflow_mock.start_activity_method.assert_has_calls([
call(
Activities.create_schedules,
{
'schedules':
workflow_mock.execute_local_activity_method.return_value['to_create']
workflow_mock.start_local_activity_method.return_value['to_create']
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_activity_method.assert_has_calls([
workflow_mock.start_activity_method.assert_has_calls([
call(
Activities.update_schedules,
{
'schedules':
workflow_mock.execute_local_activity_method.return_value['to_update']
workflow_mock.start_local_activity_method.return_value['to_update']
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.start_activity_method.assert_has_calls([
call(
Activities.report_schedule_orchestration,
{
'created_schedules': workflow_mock.start_activity_method.return_value,
'updated_schedules': workflow_mock.start_activity_method.return_value,
'deleted_schedules': workflow_mock.start_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.start_activity_method.assert_has_calls([
call(
Activities.report_slot_orchestration,
{
'inserted_slots': workflow_mock.start_activity_method.return_value,
'deleted_slots': workflow_mock.start_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.start_activity_method.assert_has_calls([
call(
Activities.update_pipelines_timestamps,
{
'updated_pipelines': workflow_mock.start_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.start_activity_method.assert_has_calls([
call(
Activities.delete_pipelines_timestamps,
{
'deleted_pipelines': workflow_mock.start_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.start_activity_method.assert_has_calls([
call(
Activities.create_pipelines_timestamps,
{
'created_pipelines': workflow_mock.start_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY