From 0aa58ba1da261c7577f985d8a5c87b3baa930a4f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 7 Jul 2025 11:03:48 -0300 Subject: [PATCH 1/5] SIENTIAPDE-1110 SIENTIAPDE-1110 add debug_data_package configuration option to scouter function --- orchestrator/utils/orchestrator_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/orchestrator/utils/orchestrator_functions.py b/orchestrator/utils/orchestrator_functions.py index 5375e14..f2bd63e 100644 --- a/orchestrator/utils/orchestrator_functions.py +++ b/orchestrator/utils/orchestrator_functions.py @@ -36,7 +36,8 @@ def scouter(config: dict[str, Any]): "schema": "sientia_data", "table_name": "laborious_data", "retention_time": config.get('tag_retention_minutes', 60) * 60, - "model_tags": tags + "model_tags": tags, + "debug_data_package": config.get('debug_data_package', False) } From f7f95d3148844ed6fd2193f5a3544ad8ca665526 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 7 Jul 2025 15:50:19 -0300 Subject: [PATCH 2/5] SIENTIAPDE-1110 Update GITHUB_BRANCH and helm version in values.yaml for SIENTIAPDE-1110 --- orchestrator/utils/orchestrator_functions.py | 2 +- values.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/orchestrator/utils/orchestrator_functions.py b/orchestrator/utils/orchestrator_functions.py index f2bd63e..a40046c 100644 --- a/orchestrator/utils/orchestrator_functions.py +++ b/orchestrator/utils/orchestrator_functions.py @@ -65,7 +65,7 @@ def process_path_priority(path_priority: list[str]): def predictions_batch(config: dict[str, Any]): tags = {} - for tag in config['write_tags']: + for tag in config.get('write_tags', []): if tag['server_id'] not in tags: tags[tag['server_id']] = {} diff --git a/values.yaml b/values.yaml index d06746b..e346cfd 100644 --- a/values.yaml +++ b/values.yaml @@ -123,7 +123,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-orchestrator_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1107-testar-o-uso-do-mongo-db-no-lugar-do-couchbase" + value: "SIENTIAPDE-1110-criar-testes-e-2-e" - name: PYTHON_APP value: "orchestrator.worker.worker" @@ -180,7 +180,7 @@ ssh: # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp -# helm upgrade --install sientia-orchestrator-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.2.0-uat +# helm upgrade --install sientia-orchestrator-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.4.0-uat # kubectl create secret generic git-ssh-key-sientia-orchestrator-worker \ # --namespace sientia \ From a848f20c6bc5363dea9e7388b69e386115166d34 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 7 Jul 2025 15:56:33 -0300 Subject: [PATCH 3/5] SIENTIAPDE-1110 fix: handle missing input filters in predictions_batch function --- orchestrator/utils/orchestrator_functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/orchestrator/utils/orchestrator_functions.py b/orchestrator/utils/orchestrator_functions.py index a40046c..270273a 100644 --- a/orchestrator/utils/orchestrator_functions.py +++ b/orchestrator/utils/orchestrator_functions.py @@ -97,19 +97,19 @@ def predictions_batch(config: dict[str, Any]): "policy": "STOP", "config": {} } - }, config['input_filters']), + }, config.get('input_filters', [])), "mlflow_transform_filters": overlap_filter_config({ "API_ERROR": { "policy": "STOP", "config": {} } - }, config['mlflow_transform_filters']), + }, config.get('mlflow_transform_filters', [])), "mlflow_predict_filters": overlap_filter_config({ "API_ERROR": { "policy": "STOP", "config": {} } - }, config['mlflow_predict_filters']), + }, config.get('mlflow_predict_filters', [])), "path_priority": path_priority } From 5d4faab557963829786856b5b4db5fcf1d45f648 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 8 Jul 2025 16:34:47 -0300 Subject: [PATCH 4/5] SIENTIAPDE-1110 chore: update sientia-dataops-library version to 1.2.1 and add debug_data_package option in test_scouter --- requirements.txt | 2 +- tests/orchestrator/utils/test_orchestrator_functions.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 25595c6..8c759a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ sqlalchemy redis couchbase pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.1.14 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.2.1 diff --git a/tests/orchestrator/utils/test_orchestrator_functions.py b/tests/orchestrator/utils/test_orchestrator_functions.py index 67a05d5..09c541d 100644 --- a/tests/orchestrator/utils/test_orchestrator_functions.py +++ b/tests/orchestrator/utils/test_orchestrator_functions.py @@ -77,7 +77,8 @@ def test_scouter(): "aggr_func": "test_aggr_func", "data_range": [1, 2] } - } + }, + "debug_data_package": False } assert result == expected From e1f1d3e3f4b9e419debcf8a6316a2ed5d0e8053e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 8 Jul 2025 16:39:35 -0300 Subject: [PATCH 5/5] SIENTIAPDE-1110 refactor: remove unused test_prepare_activity function from test_activities.py --- .../activities/test_activities.py | 53 ------------------- 1 file changed, 53 deletions(-) diff --git a/tests/orchestrator/activities/test_activities.py b/tests/orchestrator/activities/test_activities.py index 7c0fbb3..4ebbc18 100644 --- a/tests/orchestrator/activities/test_activities.py +++ b/tests/orchestrator/activities/test_activities.py @@ -95,56 +95,3 @@ def test___init__(mock_formatters_init, mock_slot_manager_init, logger=logger, notification_handler=notification_handler ) - - -@mark.asyncio -@patch('orchestrator.activities.mongo_db.MongoClient') -async def test_prepare_activity(_mock_cluster): - couchbase_config = { - 'connection_string': 'couchbase://localhost', - 'username': 'admin', - 'password': 'password' - } - - mongo_db_config = { - 'connection_string': 'mongodb://localhost:27017', - 'database_name': 'test_db' - } - - redis_config = { - 'host': 'localhost', - 'port': 6379, - 'username': 'admin', - 'password': 'password' - } - - temporal_client = MagicMock() - logger = MagicMock() - notification_handler = MagicMock() - - activities = Activities( - temporal_client=temporal_client, - # couchbase_config=couchbase_config, - redis_config=redis_config, - mongodb_config=mongo_db_config, - logger=logger, - notification_handler=notification_handler - ) - - input_data = { - 'workflow_name': 'test-workflow-name', - 'schedule_name': 'test-schedule-name', - 'model_name': 'test-model-name', - 'model_id': 'test-model-id' - } - - await activities.prepare_activity(input_data) - - assert activities.notification_handler.base_notification.pipeline == input_data[ - 'workflow_name'] - assert activities.notification_handler.base_notification.trigger == input_data[ - 'schedule_name'] - assert activities.notification_handler.base_notification.model_name == input_data[ - 'model_name'] - assert activities.notification_handler.base_notification.model_id == input_data[ - 'model_id']