gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1150

Browse Source 
fix: implement schedule normalization and timestamp management in MongoDB activities, enhance orchestration workflow with new formatting and normalization methods
This commit is contained in:
vitor-aignosi
2025-07-14 12:59:52 -03:00
parent e3a88eb1d5
commit 43206b578d
10 changed files with 648 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
tests/orchestrator/activities/test_formatters.py
View File

@@ -247,32 +247,51 @@ async def test_process_slots(mock_build_tag_config, mock_gather_read_tags, forma
}
@mark.asyncio
async def test_format_schedule_config(formatters):
input_data = {
"schedule_config": [
{"namespace": "test_namespace1", "schedule_name": "test1",
"updated_at": "2021-01-01"},
{"namespace": "test_namespace2", "schedule_name": "test2",
"updated_at": "2021-01-02"}
]
}
result = await formatters.format_schedule_config(input_data)
assert result == {
"test_namespace1": {
"test1": "2021-01-01"
},
"test_namespace2": {
"test2": "2021-01-02"
}
}
@mark.asyncio
async def test_create_schedule_config(formatters):
input_data = {
"current_schedule_config": {
"scouter": {
"test_schedule_name_to_delete": {
"frequency": 60,
"data": {"test": "test"}
},
"test_schedule_name_to_update": {
"frequency": 60,
"data": {"test": "test"}
}
"test_schedule_name_to_delete": '2021-01-01',
"test_schedule_name_to_update": '2021-01-02'
}
},
"schedule_config": {
"laborious": {
"test_schedule_name_to_create": {
"frequency": 60,
"data": {"test": "test"}
"data": {"test": "test"},
"updated_at": "2021-01-03"
}
},
"scouter": {
"test_schedule_name_to_update": {
"frequency": 60,
"data": {"test": "test2"}
"data": {"test": "test2"},
"updated_at": "2021-01-04"
}
}
}
@@ -285,7 +304,8 @@ async def test_create_schedule_config(formatters):
"laborious": {
"test_schedule_name_to_create": {
"frequency": 60,
"data": {"test": "test"}
"data": {"test": "test"},
"updated_at": "2021-01-03"
}
},
"scouter": {}
@@ -294,7 +314,8 @@ async def test_create_schedule_config(formatters):
"scouter": {
"test_schedule_name_to_update": {
"frequency": 60,
"data": {"test": "test2"}
"data": {"test": "test2"},
"updated_at": "2021-01-04"
}
},
"laborious": {}
@@ -369,95 +390,129 @@ def test_send_error_report(formatters):
def test_parse_report(formatters):
input_data = {
"test_schedule_name_to_create": {
"test_key": {
"success": True
},
"test_schedule_name_to_create_error": {
"test_key2": {
"success": False,
"error": "test_error"
"message": "test_error"
}
}
result = formatters.parse_report(input_data)
assert result == (
["test_schedule_name_to_create"],
["test_schedule_name_to_create_error"]
["test_key"],
["test_key2"]
)
def test_parse_report_schedule(formatters):
input_data = [
{
"namespace": "test_namespace",
"schedule_name": "test_schedule_name_to_create",
"success": True
},
{
"namespace": "test_namespace",
"schedule_name": "test_schedule_name_to_create_error",
"success": False,
"message": "test_error"
}
]
result = formatters.parse_report_schedule(input_data)
assert result == (
["test_namespace/test_schedule_name_to_create"],
["test_namespace/test_schedule_name_to_create_error: test_error"]
)
@mark.asyncio
async def test_report_schedule_orchestration(formatters):
formatters.parse_report = MagicMock(
side_effect=formatters.parse_report
formatters.parse_report_schedule = MagicMock(
side_effect=formatters.parse_report_schedule
)
formatters.send_success_report = MagicMock()
formatters.send_error_report = MagicMock()
input_data = {
"created_schedules": {
"test_schedule_name_to_create": {
"created_schedules": [
{
"namespace": "test_namespace",
"schedule_name": "test_schedule_name_to_create",
"success": True
},
"test_schedule_name_to_create_error": {
{
"namespace": "test_namespace",
"schedule_name": "test_schedule_name_to_create_error",
"success": False,
"error": "test_error"
"message": "test_error"
}
},
"updated_schedules": {
"test_schedule_name_to_update": {
],
"updated_schedules": [
{
"namespace": "test_namespace",
"schedule_name": "test_schedule_name_to_update",
"success": True
},
"test_schedule_name_to_update_error": {
{
"namespace": "test_namespace",
"schedule_name": "test_schedule_name_to_update_error",
"success": False,
"error": "test_error"
"message": "test_error"
}
},
"deleted_schedules": {
"test_schedule_name_to_delete": {
],
"deleted_schedules": [
{
"namespace": "test_namespace",
"schedule_name": "test_schedule_name_to_delete",
"success": True
},
"test_schedule_name_to_delete_error": {
{
"namespace": "test_namespace",
"schedule_name": "test_schedule_name_to_delete_error",
"success": False,
"error": "test_error"
"message": "test_error"
}
}
]
}
await formatters.report_schedule_orchestration(input_data)
formatters.parse_report.assert_has_calls([
formatters.parse_report_schedule.assert_has_calls([
call(input_data['created_schedules']),
call(input_data['updated_schedules']),
call(input_data['deleted_schedules'])
])
formatters.send_success_report.assert_has_calls([
call(
"Created schedules: \n test_schedule_name_to_create",
"Created schedules: \n test_namespace/test_schedule_name_to_create",
"REPORT_ORCHESTRATION_CREATED_SCHEDULES"
),
call(
"Updated schedules: \n test_schedule_name_to_update",
"Updated schedules: \n test_namespace/test_schedule_name_to_update",
"REPORT_ORCHESTRATION_UPDATED_SCHEDULES"
),
call(
"Deleted schedules: \n test_schedule_name_to_delete",
"Deleted schedules: \n test_namespace/test_schedule_name_to_delete",
"REPORT_ORCHESTRATION_DELETED_SCHEDULES"
)
])
formatters.send_error_report.assert_has_calls([
call(
"Failed to create schedules: \n test_schedule_name_to_create_error",
"Failed to create schedules: \n test_namespace/test_schedule_name_to_create_error: test_error",
"REPORT_ORCHESTRATION_CREATED_SCHEDULES",
input_data['created_schedules']
),
call(
"Failed to update schedules: \n test_schedule_name_to_update_error",
"Failed to update schedules: \n test_namespace/test_schedule_name_to_update_error: test_error",
"REPORT_ORCHESTRATION_UPDATED_SCHEDULES",
input_data['updated_schedules']
),
call(
"Failed to delete schedules: \n test_schedule_name_to_delete_error",
"Failed to delete schedules: \n test_namespace/test_schedule_name_to_delete_error: test_error",
"REPORT_ORCHESTRATION_DELETED_SCHEDULES",
input_data['deleted_schedules']
)

55
tests/orchestrator/activities/test_mongo_db.py
View File

@@ -244,3 +244,58 @@ async def test_aggregate_documents_in_mongodb_missing_aggregation(mongo_db):
else:
assert False, "Expected a ValueError to be raised"
@mark.asyncio
@patch("orchestrator.activities.mongo_db.datetime")
async def test_update_pipelines_timestamps_success(datetime_mock, mongo_db):
input_data = {"updated_pipelines": [
{"schedule_name": "test1", "namespace": "test1"},
{"schedule_name": "test2", "namespace": "test2"}
]}
mongo_db.database["pipelines"].update_many.return_value = MagicMock()
await mongo_db.update_pipelines_timestamps(input_data)
mongo_db.database["pipelines"].update_many.assert_called_once_with(
{"$or": [
{"schedule_name": "test1", "namespace": "test1"},
{"schedule_name": "test2", "namespace": "test2"}
]},
{"$set": {
"updated_at": datetime_mock.now.return_value.strftime.return_value}}
)
@mark.asyncio
@patch("orchestrator.activities.mongo_db.datetime")
async def test_create_pipelines_timestamps_success(datetime_mock, mongo_db):
input_data = {"created_pipelines": [
{"schedule_name": "test1", "namespace": "test1"},
{"schedule_name": "test2", "namespace": "test2"}
]}
mongo_db.database["pipelines"].insert_many.return_value = MagicMock()
await mongo_db.create_pipelines_timestamps(input_data)
mongo_db.database["pipelines"].insert_many.assert_called_once_with(
[
{"schedule_name": "test1", "namespace": "test1",
"updated_at": datetime_mock.now.return_value.strftime.return_value},
{"schedule_name": "test2", "namespace": "test2",
"updated_at": datetime_mock.now.return_value.strftime.return_value}
]
)
@mark.asyncio
@patch("orchestrator.activities.mongo_db.datetime")
async def test_delete_pipelines_timestamps_success(datetime_mock, mongo_db):
input_data = {"deleted_pipelines": [
{"schedule_name": "test1", "namespace": "test1"},
{"schedule_name": "test2", "namespace": "test2"}
]}
mongo_db.database["pipelines"].delete_many.return_value = MagicMock()
await mongo_db.delete_pipelines_timestamps(input_data)
mongo_db.database["pipelines"].delete_many.assert_called_once_with(
{"$or": [
{"schedule_name": "test1", "namespace": "test1"},
{"schedule_name": "test2", "namespace": "test2"}
]}
)

199
tests/orchestrator/activities/test_temporal_manager.py
View File

@@ -41,30 +41,32 @@ async def test_connect_to_temporal(connect_mock, temporal_manager):
])
async def async_iter():
yield MagicMock(
id="test-schedule-id",
search_attributes={
"orchestrated": ["true"]
}
)
yield MagicMock(
id="test-schedule-id-2",
search_attributes={
"Attr": ["false"]
}
)
yield MagicMock(
id="test-schedule-id-3",
search_attributes={
"Attr": ["false"]
}
)
@mark.asyncio
@patch("orchestrator.activities.temporal_manager.MessageToDict",
return_value={"data": base64.b64encode(json.dumps({"test": "test"}).encode('utf-8'))})
async def test_load_schedule(_mock_message_to_dict, temporal_manager):
# Create async iterator mock
async def async_iter():
yield MagicMock(
id="test-schedule-id",
search_attributes={
"orchestrated": ["true"]
}
)
yield MagicMock(
id="test-schedule-id-2",
search_attributes={
"Attr": ["false"]
}
)
yield MagicMock(
id="test-schedule-id-3",
search_attributes={
"Attr": ["false"]
}
)
handle = MagicMock(
describe=AsyncMock(
@@ -134,6 +136,43 @@ async def test_load_schedule(_mock_message_to_dict, temporal_manager):
}
@mark.asyncio
async def test_normalize_schedules(temporal_manager):
input_data = {
"orchestrated_schedules": {
"scouter": {"test-scouter": "2021-01-01"},
"laborious": {"test-schedule-id1": "2021-01-01"}
}
}
temporal_manager.temporal_clients['scouter'].list_schedules = AsyncMock(
return_value=async_iter()
)
temporal_manager.temporal_clients['laborious'].list_schedules = AsyncMock(
return_value=async_iter()
)
temporal_manager.temporal_clients['scouter'].get_schedule_handle = MagicMock(
return_value=MagicMock(
delete=AsyncMock()
)
)
temporal_manager.temporal_clients['laborious'].get_schedule_handle = MagicMock(
return_value=MagicMock(
delete=AsyncMock()
)
)
await temporal_manager.normalize_schedules(input_data)
temporal_manager.temporal_clients['scouter'].get_schedule_handle.assert_has_calls([
call("test-schedule-id"),
])
temporal_manager.temporal_clients['scouter'].get_schedule_handle.return_value.delete.assert_awaited_once(
)
@mark.asyncio
@patch("orchestrator.activities.temporal_manager.parse_frequency",
side_effect=parse_frequency)
@@ -308,20 +347,26 @@ async def test_create_schedule(
)
])
assert report == {
"test-schedule": {
assert report == [
{
"schedule_name": "test-schedule",
"namespace": "scouter",
"success": True,
"message": "Schedule created successfully"
},
"test-schedule-invalid-frequency": {
{
"schedule_name": "test-schedule-invalid-frequency",
"namespace": "scouter",
"success": False,
"message": "Invalid frequency"
},
"test-schedule-laborious": {
{
"schedule_name": "test-schedule-laborious",
"namespace": "laborious",
"success": True,
"message": "Schedule created successfully"
}
}
]
@mark.asyncio
@@ -397,29 +442,72 @@ async def test_update_schedules(
}
}
handler_scouter = MagicMock(
update=AsyncMock(
update=AsyncMock(
side_effect=lambda f: f(input_mock)
)
)
)
handler_laborious = MagicMock(
update=AsyncMock(
update=AsyncMock(
side_effect=lambda f: f(input_mock)
)
)
)
temporal_manager.temporal_clients['scouter'].get_schedule_handle = MagicMock(
side_effect=[
handler_scouter,
None
]
)
temporal_manager.temporal_clients['laborious'].get_schedule_handle = MagicMock(
side_effect=[
handler_laborious
]
)
report = await temporal_manager.update_schedules(input_data)
temporal_manager.schedule_handles['scouter']['test-schedule'].update.assert_called_once()
temporal_manager.schedule_handles['laborious']['test-schedule-laborious'].update.assert_called_once()
temporal_manager.temporal_clients['scouter'].get_schedule_handle.assert_has_calls([
call("test-schedule"),
call("test-schedule_no_handler")
])
temporal_manager.temporal_clients['laborious'].get_schedule_handle.assert_has_calls([
call("test-schedule-laborious")
])
assert report == {
"test-schedule": {
handler_scouter.update.assert_called_once()
handler_laborious.update.assert_called_once()
assert report == [
{
"schedule_name": "test-schedule",
"namespace": "scouter",
"success": True,
"message": "Schedule updated successfully"
},
"test-schedule_no_handler": {
{
"schedule_name": "test-schedule_no_handler",
"namespace": "scouter",
"success": False,
"message": "Schedule test-schedule_no_handler not found"
},
"test-schedule-laborious": {
{
"schedule_name": "test-schedule-laborious",
"namespace": "laborious",
"success": True,
"message": "Schedule updated successfully"
}
}
]
@mark.asyncio
async def test_update_schedules_with_no_handle(temporal_manager):
async def test_update_schedules_with_no_client(temporal_manager):
temporal_manager.temporal_clients = {}
input_data = {
"schedules": {
@@ -436,7 +524,7 @@ async def test_update_schedules_with_no_handle(temporal_manager):
await temporal_manager.update_schedules(input_data)
except Exception as e:
assert str(
e) == f"Schedule handles for abc not found, handles: {temporal_manager.schedule_handles}"
e) == f"Temporal client for abc not found, clients: {temporal_manager.temporal_clients}"
@mark.asyncio
@@ -464,28 +552,57 @@ async def test_delete_schedules(temporal_manager):
]
}
}
handler_scouter = MagicMock(
delete=AsyncMock()
)
handler_laborious = MagicMock(
delete=AsyncMock()
)
temporal_manager.temporal_clients['scouter'].get_schedule_handle = MagicMock(
side_effect=[
handler_scouter,
None
]
)
temporal_manager.temporal_clients['laborious'].get_schedule_handle = MagicMock(
side_effect=[
handler_laborious
]
)
report = await temporal_manager.delete_schedules(input_data)
assert report == {
"test-schedule": {
handler_scouter.delete.assert_called_once()
handler_laborious.delete.assert_called_once()
assert report == [
{
"schedule_name": "test-schedule",
"namespace": "scouter",
"success": True,
"message": "Schedule deleted successfully"
},
"test-schedule_no_handler": {
{
"schedule_name": "test-schedule_no_handler",
"namespace": "scouter",
"success": False,
"message": "Schedule test-schedule_no_handler not found"
},
"test-schedule-laborious": {
{
"schedule_name": "test-schedule-laborious",
"namespace": "laborious",
"success": True,
"message": "Schedule deleted successfully"
}
}
]
@mark.asyncio
async def test_delete_schedules_with_no_handle(temporal_manager):
temporal_manager.schedule_handles = {}
async def test_delete_schedules_with_no_client(temporal_manager):
temporal_manager.temporal_clients = {}
input_data = {
"schedules": {
"abc": [
@@ -498,4 +615,4 @@ async def test_delete_schedules_with_no_handle(temporal_manager):
await temporal_manager.delete_schedules(input_data)
except Exception as e:
assert str(
e) == f"Schedule handles for abc not found, handles: {temporal_manager.schedule_handles}"
e) == f"Temporal client for abc not found, clients: {temporal_manager.temporal_clients}"

29
tests/orchestrator/workflows/test_orchestrator.py
View File

@@ -44,7 +44,12 @@ async def test_run(workflow_mock, orchestrator):
workflow_mock.execute_local_activity_method.assert_has_calls([
call(
Activities.load_schedule,
Activities.find_documents_in_mongodb,
{
"query": {
"collection": "orchestrated_schedules"
}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
@@ -66,6 +71,17 @@ async def test_run(workflow_mock, orchestrator):
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
call(
Activities.format_schedule_config,
{
'schedule_config': workflow_mock.execute_local_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
call(
Activities.process_schedules,
@@ -114,6 +130,17 @@ async def test_run(workflow_mock, orchestrator):
)
])
workflow_mock.execute_local_activity_method.assert_has_calls([
call(
Activities.normalize_schedules,
{
'orchestrated_schedules': workflow_mock.execute_local_activity_method.return_value
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_activity_method.assert_has_calls([
call(
Activities.delete_slots,