From 0d3ff73557affa96ba52df619d1f95d84cca5f83 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Wed, 7 Oct 2026 15:18:38 -0300 Subject: [PATCH 1/3] fix(orchestrator): do not stamp or untrack every schedule when all updates/deletes fail (SIENTIAPDE-2192) --- orchestrator/activities/mongo_db.py | 15 +++++++-- .../orchestrator/activities/test_mongo_db.py | 31 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/orchestrator/activities/mongo_db.py b/orchestrator/activities/mongo_db.py index d101658..d0256d0 100644 --- a/orchestrator/activities/mongo_db.py +++ b/orchestrator/activities/mongo_db.py @@ -242,7 +242,12 @@ class MongoDB(SientiaMonitoring): for pipeline in updated_pipelines if pipeline['success'] ] - data_filter = {'$or': argument} if argument else {} + if not argument: + # Every update failed: stamp nothing so they are retried next tick (same guard as + # create_pipelines_timestamps). An empty filter would match every tracked schedule. + self.info('No successful pipeline updates to timestamp', metadata=metadata) + return + data_filter = {'$or': argument} try: self.mongo_db_repository.update_many( @@ -343,7 +348,13 @@ class MongoDB(SientiaMonitoring): for pipeline in deleted_pipelines if pipeline['success'] ] - data_filter = {'$or': argument} if argument else {} + if not argument: + # Every delete failed: remove nothing (same guard as create_pipelines_timestamps). + # An empty filter would wipe orchestrated_schedules and make the next tick delete + # and recreate every orchestrated schedule. + self.info('No successful pipeline deletions to untrack', metadata=metadata) + return + data_filter = {'$or': argument} try: self.mongo_db_repository.delete_many('orchestrated_schedules', data_filter, metadata) diff --git a/tests/orchestrator/activities/test_mongo_db.py b/tests/orchestrator/activities/test_mongo_db.py index e1496b1..8f82bb0 100644 --- a/tests/orchestrator/activities/test_mongo_db.py +++ b/tests/orchestrator/activities/test_mongo_db.py @@ -265,6 +265,22 @@ def test_update_pipelines_timestamps_success(now_mock, mongo_db): ) +@patch('orchestrator.activities.mongo_db.now') +def test_update_pipelines_timestamps_all_failed_stamps_nothing(now_mock, mongo_db): + input_data = { + 'updated_pipelines': [ + {'schedule_name': 'test1', 'namespace': 'test1', 'success': False}, + ] + } + mongo_db.mongo_db_repository.update_many = MagicMock() + + mongo_db.update_pipelines_timestamps(input_data) + + # An empty filter would match and stamp every tracked schedule, so the failed one + # would never be retried. + mongo_db.mongo_db_repository.update_many.assert_not_called() + + @patch('orchestrator.activities.mongo_db.now') def test_update_pipelines_timestamps_failure(now_mock, mongo_db): input_data = { @@ -362,6 +378,21 @@ def test_delete_pipelines_timestamps_success(now_mock, mongo_db): ) +def test_delete_pipelines_timestamps_all_failed_deletes_nothing(mongo_db): + input_data = { + 'deleted_pipelines': [ + {'schedule_name': 'test1', 'namespace': 'test1', 'success': False}, + ] + } + mongo_db.mongo_db_repository.delete_many = MagicMock() + + mongo_db.delete_pipelines_timestamps(input_data) + + # An empty filter would wipe orchestrated_schedules; the next tick's normalize_schedules + # would then delete and recreate every orchestrated Temporal schedule. + mongo_db.mongo_db_repository.delete_many.assert_not_called() + + @patch('orchestrator.activities.mongo_db.now') def test_delete_pipelines_timestamps_failure(now_mock, mongo_db): input_data = { From d963cd3e8a74886f1c843dbc36af97d65b1e6598 Mon Sep 17 00:00:00 2001 From: PedroHMCosme Date: Thu, 8 Oct 2026 08:46:05 -0300 Subject: [PATCH 2/3] test(orchestrator): cover mixed and empty timestamp updates/deletes (SIENTIAPDE-2192) --- .../orchestrator/activities/test_mongo_db.py | 62 +++++++++++++++---- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/tests/orchestrator/activities/test_mongo_db.py b/tests/orchestrator/activities/test_mongo_db.py index 8f82bb0..6feda41 100644 --- a/tests/orchestrator/activities/test_mongo_db.py +++ b/tests/orchestrator/activities/test_mongo_db.py @@ -1,7 +1,7 @@ from datetime import UTC, datetime from unittest.mock import ANY, AsyncMock, MagicMock, patch -from pytest import fixture +from pytest import fixture, mark from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ @@ -266,12 +266,11 @@ def test_update_pipelines_timestamps_success(now_mock, mongo_db): @patch('orchestrator.activities.mongo_db.now') -def test_update_pipelines_timestamps_all_failed_stamps_nothing(now_mock, mongo_db): - input_data = { - 'updated_pipelines': [ - {'schedule_name': 'test1', 'namespace': 'test1', 'success': False}, - ] - } +@mark.parametrize( + 'pipelines', [[], [{'schedule_name': 'test1', 'namespace': 'test1', 'success': False}]] +) +def test_update_pipelines_timestamps_all_failed_stamps_nothing(now_mock, mongo_db, pipelines): + input_data = {'updated_pipelines': pipelines} mongo_db.mongo_db_repository.update_many = MagicMock() mongo_db.update_pipelines_timestamps(input_data) @@ -281,6 +280,26 @@ def test_update_pipelines_timestamps_all_failed_stamps_nothing(now_mock, mongo_d mongo_db.mongo_db_repository.update_many.assert_not_called() +@patch('orchestrator.activities.mongo_db.now') +def test_update_pipelines_timestamps_mixed_stamps_only_successes(now_mock, mongo_db): + input_data = { + 'updated_pipelines': [ + {'schedule_name': 'test1', 'namespace': 'test1', 'success': True}, + {'schedule_name': 'test2', 'namespace': 'test2', 'success': False}, + ] + } + mongo_db.mongo_db_repository.update_many = MagicMock() + + mongo_db.update_pipelines_timestamps(input_data) + + mongo_db.mongo_db_repository.update_many.assert_called_once_with( + 'orchestrated_schedules', + {'$or': [{'schedule_name': 'test1', 'namespace': 'test1'}]}, + {'$set': {'updated_at': now_mock.return_value}}, + {}, + ) + + @patch('orchestrator.activities.mongo_db.now') def test_update_pipelines_timestamps_failure(now_mock, mongo_db): input_data = { @@ -378,12 +397,11 @@ def test_delete_pipelines_timestamps_success(now_mock, mongo_db): ) -def test_delete_pipelines_timestamps_all_failed_deletes_nothing(mongo_db): - input_data = { - 'deleted_pipelines': [ - {'schedule_name': 'test1', 'namespace': 'test1', 'success': False}, - ] - } +@mark.parametrize( + 'pipelines', [[], [{'schedule_name': 'test1', 'namespace': 'test1', 'success': False}]] +) +def test_delete_pipelines_timestamps_all_failed_deletes_nothing(mongo_db, pipelines): + input_data = {'deleted_pipelines': pipelines} mongo_db.mongo_db_repository.delete_many = MagicMock() mongo_db.delete_pipelines_timestamps(input_data) @@ -393,6 +411,24 @@ def test_delete_pipelines_timestamps_all_failed_deletes_nothing(mongo_db): mongo_db.mongo_db_repository.delete_many.assert_not_called() +def test_delete_pipelines_timestamps_mixed_deletes_only_successes(mongo_db): + input_data = { + 'deleted_pipelines': [ + {'schedule_name': 'test1', 'namespace': 'test1', 'success': True}, + {'schedule_name': 'test2', 'namespace': 'test2', 'success': False}, + ] + } + mongo_db.mongo_db_repository.delete_many = MagicMock() + + mongo_db.delete_pipelines_timestamps(input_data) + + mongo_db.mongo_db_repository.delete_many.assert_called_once_with( + 'orchestrated_schedules', + {'$or': [{'schedule_name': 'test1', 'namespace': 'test1'}]}, + {}, + ) + + @patch('orchestrator.activities.mongo_db.now') def test_delete_pipelines_timestamps_failure(now_mock, mongo_db): input_data = { From da27e2ba91c1f7b8b936783166b2b256e6fdcf4a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 8 Oct 2026 13:52:43 +0000 Subject: [PATCH 3/3] chore: bump version to 0.6.8 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 42fd82e..026f790 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "orchestrator" -version = "0.6.7" +version = "0.6.8" description = "Sientia DataOps Orchestrator - ML Model Orchestration System" readme = "README.md" requires-python = ">=3.11"