diff --git a/orchestrator/utils/orchestrator_functions.py b/orchestrator/utils/orchestrator_functions.py index 5375e14..270273a 100644 --- a/orchestrator/utils/orchestrator_functions.py +++ b/orchestrator/utils/orchestrator_functions.py @@ -36,7 +36,8 @@ def scouter(config: dict[str, Any]): "schema": "sientia_data", "table_name": "laborious_data", "retention_time": config.get('tag_retention_minutes', 60) * 60, - "model_tags": tags + "model_tags": tags, + "debug_data_package": config.get('debug_data_package', False) } @@ -64,7 +65,7 @@ def process_path_priority(path_priority: list[str]): def predictions_batch(config: dict[str, Any]): tags = {} - for tag in config['write_tags']: + for tag in config.get('write_tags', []): if tag['server_id'] not in tags: tags[tag['server_id']] = {} @@ -96,19 +97,19 @@ def predictions_batch(config: dict[str, Any]): "policy": "STOP", "config": {} } - }, config['input_filters']), + }, config.get('input_filters', [])), "mlflow_transform_filters": overlap_filter_config({ "API_ERROR": { "policy": "STOP", "config": {} } - }, config['mlflow_transform_filters']), + }, config.get('mlflow_transform_filters', [])), "mlflow_predict_filters": overlap_filter_config({ "API_ERROR": { "policy": "STOP", "config": {} } - }, config['mlflow_predict_filters']), + }, config.get('mlflow_predict_filters', [])), "path_priority": path_priority } diff --git a/requirements.txt b/requirements.txt index 25595c6..8c759a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ sqlalchemy redis couchbase pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.1.14 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.2.1 diff --git a/tests/orchestrator/activities/test_activities.py b/tests/orchestrator/activities/test_activities.py index 7c0fbb3..4ebbc18 100644 --- a/tests/orchestrator/activities/test_activities.py +++ b/tests/orchestrator/activities/test_activities.py @@ -95,56 +95,3 @@ def test___init__(mock_formatters_init, mock_slot_manager_init, logger=logger, notification_handler=notification_handler ) - - -@mark.asyncio -@patch('orchestrator.activities.mongo_db.MongoClient') -async def test_prepare_activity(_mock_cluster): - couchbase_config = { - 'connection_string': 'couchbase://localhost', - 'username': 'admin', - 'password': 'password' - } - - mongo_db_config = { - 'connection_string': 'mongodb://localhost:27017', - 'database_name': 'test_db' - } - - redis_config = { - 'host': 'localhost', - 'port': 6379, - 'username': 'admin', - 'password': 'password' - } - - temporal_client = MagicMock() - logger = MagicMock() - notification_handler = MagicMock() - - activities = Activities( - temporal_client=temporal_client, - # couchbase_config=couchbase_config, - redis_config=redis_config, - mongodb_config=mongo_db_config, - logger=logger, - notification_handler=notification_handler - ) - - input_data = { - 'workflow_name': 'test-workflow-name', - 'schedule_name': 'test-schedule-name', - 'model_name': 'test-model-name', - 'model_id': 'test-model-id' - } - - await activities.prepare_activity(input_data) - - assert activities.notification_handler.base_notification.pipeline == input_data[ - 'workflow_name'] - assert activities.notification_handler.base_notification.trigger == input_data[ - 'schedule_name'] - assert activities.notification_handler.base_notification.model_name == input_data[ - 'model_name'] - assert activities.notification_handler.base_notification.model_id == input_data[ - 'model_id'] diff --git a/tests/orchestrator/utils/test_orchestrator_functions.py b/tests/orchestrator/utils/test_orchestrator_functions.py index 67a05d5..09c541d 100644 --- a/tests/orchestrator/utils/test_orchestrator_functions.py +++ b/tests/orchestrator/utils/test_orchestrator_functions.py @@ -77,7 +77,8 @@ def test_scouter(): "aggr_func": "test_aggr_func", "data_range": [1, 2] } - } + }, + "debug_data_package": False } assert result == expected diff --git a/values.yaml b/values.yaml index d06746b..e346cfd 100644 --- a/values.yaml +++ b/values.yaml @@ -123,7 +123,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-orchestrator_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1107-testar-o-uso-do-mongo-db-no-lugar-do-couchbase" + value: "SIENTIAPDE-1110-criar-testes-e-2-e" - name: PYTHON_APP value: "orchestrator.worker.worker" @@ -180,7 +180,7 @@ ssh: # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp -# helm upgrade --install sientia-orchestrator-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.2.0-uat +# helm upgrade --install sientia-orchestrator-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.4.0-uat # kubectl create secret generic git-ssh-key-sientia-orchestrator-worker \ # --namespace sientia \