From c7f44a24234033aa35e4e07c60bffff1dc2a3f85 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Thu, 6 Nov 2025 15:34:03 -0300 Subject: [PATCH] SIENTIAPDE-1309: Update README with Helm instructions and refactor experiment status messages. Also, update values.yaml with new image and configurations. --- README.md | 46 ++++---- .../utils/models/experiment_status.py | 12 +-- model_manager/workflows/train_model.py | 8 +- scripts/run_training_test.py | 2 +- tests/utils/models/test_experiment_status.py | 36 ++++--- tests/utils/models/test_init.py | 2 +- tests/workflows/test_train_model.py | 20 ++-- todo-list.txt | 2 + values.yaml | 100 ++++++++++++------ 9 files changed, 133 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 9ff060a..8289dc1 100644 --- a/README.md +++ b/README.md @@ -1037,13 +1037,13 @@ For support and questions: ### Create image -```shell +```bash $ docker build --ssh default --no-cache --progress=plain -t aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0 . ``` ### Create container -```shell +```bash $ docker run --env-file .env --network="host" --name sientia-dataops-model-manager -d aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0 $ docker logs -f sientia-dataops-model-manager @@ -1051,13 +1051,13 @@ $ docker logs -f sientia-dataops-model-manager ### Login using access token -```shell +```bash $ docker login -u bruno-aignosi -p LD/IyZ4vtDI7khRYnH4HzfdTx3toorg6hlCetJM54n+ACRDim3xO aignosi.azurecr.io ``` ### Push image to repository -```shell +```bash $ docker push aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0 ``` @@ -1065,40 +1065,42 @@ $ docker push aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0 ### Reference -https://www.baeldung.com/ops/kubernetes-helm +https://aignosi-wiki.atlassian.net/wiki/spaces/IT1/pages/274563074/Como+utilizar+o+Helm+Repo+Privado -### Create Helm Chart folder +### Add Helm Chart repository -```shell -# inside project root folder -$ helm create helm-chart -``` +```bash +$ helm repo add sientia \ + https://raw.githubusercontent.com/Aignosi/sientia-dataops-helm-repo/refs/heads/main/ \ + --username $GITHUB_USER \ + --password $GITHUB_PASS -### Helm Lint +# Update repository +$ helm repo update -```shell -# Firstly, this is a simple command that takes the path to a chart and runs a battery of tests to ensure that the chart is well-formed: -$ helm lint ./helm-chart -``` +# List repositories +$ helm repo list -### Helm Template +# List versions of a specific chart +$ helm search repo sientia --versions -```shell -# Also, we've this command to render the template locally for quick feedback: -$ helm template ./helm-chart +# List all charts available +$ helm search repo sientia + +# List chart details +$ helm show all sientia/sientia-module ``` ### Helm Install ```shell -# Once we've verified the chart to be fine, finally, we can run this command to install the chart into the Kubernetes cluster: -$ helm upgrade --install mlops-bff ./helm-chart -n sientia-core +$ helm upgrade --install sientia-dataops-model-manager sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.5.0 ``` ### Uninstall Helm Chart ```shell -$ helm uninstall mlops-bff -n sientia-core +$ helm uninstall sientia-dataops-model-manager -n sientia ``` --- diff --git a/model_manager/utils/models/experiment_status.py b/model_manager/utils/models/experiment_status.py index b7975eb..64c6e4d 100644 --- a/model_manager/utils/models/experiment_status.py +++ b/model_manager/utils/models/experiment_status.py @@ -15,20 +15,20 @@ class ExperimentStatus(str, Enum): Attributes: ORCHESTRATOR_VALIDATION_ERROR: Error in the parameters validation. - MAGE_WAITING_PROC: Initial status indicating experiment is registered and waiting for processing. + ORCHESTRATOR_WAITING_PROC: Initial status indicating experiment is registered and waiting for processing. TRAINING_SUCCESS: Training completed successfully with model and metrics calculated. TRAINING_ERROR: Training failed due to data issues, model errors, or other exceptions. - MLFLOW_SENT: Model successfully saved to MLFlow. - MLFLOW_SEND_ERROR: Model saving to MLFlow failed due to connection or serialization errors. + TRACKING_SENT: Model successfully saved to MLFlow. + TRACKING_SEND_ERROR: Model saving to MLFlow failed due to connection or serialization errors. FILE_DELETED: Cleanup completed successfully with all artifacts removed. FILE_DELETE_ERROR: Cleanup failed due to file system or MinIO errors. """ ORCHESTRATOR_VALIDATION_ERROR = 'ORCHESTRATOR_VALIDATION_ERROR' - MAGE_WAITING_PROC = 'MAGE_WAITING_PROC' + ORCHESTRATOR_WAITING_PROC = 'ORCHESTRATOR_WAITING_PROC' TRAINING_SUCCESS = 'TRAINING_SUCCESS' TRAINING_ERROR = 'TRAINING_ERROR' - MLFLOW_SENT = 'MLFLOW_SENT' - MLFLOW_SEND_ERROR = 'MLFLOW_SEND_ERROR' + TRACKING_SENT = 'TRACKING_SENT' + TRACKING_SEND_ERROR = 'TRACKING_SEND_ERROR' FILE_DELETED = 'FILE_DELETED' FILE_DELETE_ERROR = 'FILE_DELETE_ERROR' diff --git a/model_manager/workflows/train_model.py b/model_manager/workflows/train_model.py index 6fa5eac..da1df5f 100644 --- a/model_manager/workflows/train_model.py +++ b/model_manager/workflows/train_model.py @@ -163,7 +163,7 @@ class TrainModel: Validate and convert training parameters from dict to TrainModelParams. This method calls the validate_train_params activity to convert and validate - the input parameters. On success, updates DB status to MAGE_WAITING_PROC. + the input parameters. On success, updates DB status to ORCHESTRATOR_WAITING_PROC. On error, updates DB status to ORCHESTRATOR_VALIDATION_ERROR. Args: @@ -192,7 +192,7 @@ class TrainModel: metadata=metadata, experiment_run_id=experiment_run_id, update_type=UpdateType.STATUS, - status=ExperimentStatus.MAGE_WAITING_PROC, + status=ExperimentStatus.ORCHESTRATOR_WAITING_PROC, ) return train_params @@ -246,7 +246,7 @@ class TrainModel: metadata=metadata, experiment_run_id=experiment_run_id, update_type=UpdateType.MODEL_SAVED, - status=ExperimentStatus.MLFLOW_SENT, + status=ExperimentStatus.TRACKING_SENT, run_name=train_result.get('run_name'), ) @@ -260,7 +260,7 @@ class TrainModel: status = ExperimentStatus.TRAINING_ERROR if isinstance(e, ModelTrainingError) and (e.model_trained and not e.model_saved): - status = ExperimentStatus.MLFLOW_SEND_ERROR + status = ExperimentStatus.TRACKING_SEND_ERROR await self._update_experiment_run( metadata=metadata, diff --git a/scripts/run_training_test.py b/scripts/run_training_test.py index a3f06a0..dbb641f 100644 --- a/scripts/run_training_test.py +++ b/scripts/run_training_test.py @@ -120,7 +120,7 @@ def insert_experiment_run(file_name: str, request_data: dict) -> int: ( request_data['experimentName'], request_data['username'], - 'MAGE_REQUEST_SENT', + 'ORCHESTRATOR_REQUEST_SENT', now, now, MINIO_BUCKET, diff --git a/tests/utils/models/test_experiment_status.py b/tests/utils/models/test_experiment_status.py index 4ec5ae4..44f494a 100644 --- a/tests/utils/models/test_experiment_status.py +++ b/tests/utils/models/test_experiment_status.py @@ -5,11 +5,11 @@ from model_manager.utils.models.experiment_status import ExperimentStatus def test_experiment_status_values(): """Test that all expected status values exist.""" - assert ExperimentStatus.MAGE_WAITING_PROC == 'MAGE_WAITING_PROC' + assert ExperimentStatus.ORCHESTRATOR_WAITING_PROC == 'ORCHESTRATOR_WAITING_PROC' assert ExperimentStatus.TRAINING_SUCCESS == 'TRAINING_SUCCESS' assert ExperimentStatus.TRAINING_ERROR == 'TRAINING_ERROR' - assert ExperimentStatus.MLFLOW_SENT == 'MLFLOW_SENT' - assert ExperimentStatus.MLFLOW_SEND_ERROR == 'MLFLOW_SEND_ERROR' + assert ExperimentStatus.TRACKING_SENT == 'TRACKING_SENT' + assert ExperimentStatus.TRACKING_SEND_ERROR == 'TRACKING_SEND_ERROR' assert ExperimentStatus.FILE_DELETED == 'FILE_DELETED' assert ExperimentStatus.FILE_DELETE_ERROR == 'FILE_DELETE_ERROR' @@ -28,11 +28,11 @@ def test_experiment_status_is_string(): def test_experiment_status_membership(): """Test membership checks for status values.""" - assert 'MAGE_WAITING_PROC' in [s.value for s in ExperimentStatus] + assert 'ORCHESTRATOR_WAITING_PROC' in [s.value for s in ExperimentStatus] assert 'TRAINING_SUCCESS' in [s.value for s in ExperimentStatus] assert 'TRAINING_ERROR' in [s.value for s in ExperimentStatus] - assert 'MLFLOW_SENT' in [s.value for s in ExperimentStatus] - assert 'MLFLOW_SEND_ERROR' in [s.value for s in ExperimentStatus] + assert 'TRACKING_SENT' in [s.value for s in ExperimentStatus] + assert 'TRACKING_SEND_ERROR' in [s.value for s in ExperimentStatus] assert 'FILE_DELETED' in [s.value for s in ExperimentStatus] assert 'FILE_DELETE_ERROR' in [s.value for s in ExperimentStatus] @@ -41,39 +41,43 @@ def test_experiment_status_iteration(): """Test that enum can be iterated.""" statuses = list(ExperimentStatus) assert len(statuses) == 8 - assert ExperimentStatus.MAGE_WAITING_PROC in statuses + assert ExperimentStatus.ORCHESTRATOR_WAITING_PROC in statuses assert ExperimentStatus.TRAINING_SUCCESS in statuses assert ExperimentStatus.TRAINING_ERROR in statuses - assert ExperimentStatus.MLFLOW_SENT in statuses - assert ExperimentStatus.MLFLOW_SEND_ERROR in statuses + assert ExperimentStatus.TRACKING_SENT in statuses + assert ExperimentStatus.TRACKING_SEND_ERROR in statuses assert ExperimentStatus.FILE_DELETED in statuses assert ExperimentStatus.FILE_DELETE_ERROR in statuses def test_experiment_status_comparison(): """Test that enum values can be compared with strings.""" - assert ExperimentStatus.MAGE_WAITING_PROC == 'MAGE_WAITING_PROC' + assert ExperimentStatus.ORCHESTRATOR_WAITING_PROC == 'ORCHESTRATOR_WAITING_PROC' assert ExperimentStatus.TRAINING_SUCCESS == 'TRAINING_SUCCESS' assert ExperimentStatus.TRAINING_ERROR != 'TRAINING_SUCCESS' def test_experiment_status_access_by_name(): """Test accessing enum members by name.""" - assert ExperimentStatus['MAGE_WAITING_PROC'] == ExperimentStatus.MAGE_WAITING_PROC + assert ( + ExperimentStatus['ORCHESTRATOR_WAITING_PROC'] == ExperimentStatus.ORCHESTRATOR_WAITING_PROC + ) assert ExperimentStatus['TRAINING_SUCCESS'] == ExperimentStatus.TRAINING_SUCCESS assert ExperimentStatus['TRAINING_ERROR'] == ExperimentStatus.TRAINING_ERROR - assert ExperimentStatus['MLFLOW_SENT'] == ExperimentStatus.MLFLOW_SENT - assert ExperimentStatus['MLFLOW_SEND_ERROR'] == ExperimentStatus.MLFLOW_SEND_ERROR + assert ExperimentStatus['TRACKING_SENT'] == ExperimentStatus.TRACKING_SENT + assert ExperimentStatus['TRACKING_SEND_ERROR'] == ExperimentStatus.TRACKING_SEND_ERROR assert ExperimentStatus['FILE_DELETED'] == ExperimentStatus.FILE_DELETED assert ExperimentStatus['FILE_DELETE_ERROR'] == ExperimentStatus.FILE_DELETE_ERROR def test_experiment_status_access_by_value(): """Test accessing enum members by value.""" - assert ExperimentStatus('MAGE_WAITING_PROC') == ExperimentStatus.MAGE_WAITING_PROC + assert ( + ExperimentStatus('ORCHESTRATOR_WAITING_PROC') == ExperimentStatus.ORCHESTRATOR_WAITING_PROC + ) assert ExperimentStatus('TRAINING_SUCCESS') == ExperimentStatus.TRAINING_SUCCESS assert ExperimentStatus('TRAINING_ERROR') == ExperimentStatus.TRAINING_ERROR - assert ExperimentStatus('MLFLOW_SENT') == ExperimentStatus.MLFLOW_SENT - assert ExperimentStatus('MLFLOW_SEND_ERROR') == ExperimentStatus.MLFLOW_SEND_ERROR + assert ExperimentStatus('TRACKING_SENT') == ExperimentStatus.TRACKING_SENT + assert ExperimentStatus('TRACKING_SEND_ERROR') == ExperimentStatus.TRACKING_SEND_ERROR assert ExperimentStatus('FILE_DELETED') == ExperimentStatus.FILE_DELETED assert ExperimentStatus('FILE_DELETE_ERROR') == ExperimentStatus.FILE_DELETE_ERROR diff --git a/tests/utils/models/test_init.py b/tests/utils/models/test_init.py index aa7cff1..b211248 100644 --- a/tests/utils/models/test_init.py +++ b/tests/utils/models/test_init.py @@ -10,7 +10,7 @@ from model_manager.utils.models import ( def test_experiment_status_import(): """Test that ExperimentStatus can be imported from models package.""" assert ExperimentStatus is not None - assert hasattr(ExperimentStatus, 'MAGE_WAITING_PROC') + assert hasattr(ExperimentStatus, 'ORCHESTRATOR_WAITING_PROC') assert hasattr(ExperimentStatus, 'TRAINING_SUCCESS') diff --git a/tests/workflows/test_train_model.py b/tests/workflows/test_train_model.py index c2426f6..2157bb2 100644 --- a/tests/workflows/test_train_model.py +++ b/tests/workflows/test_train_model.py @@ -286,9 +286,9 @@ async def test_train_model_mlflow_error(mock_workflow_module, mock_train_params) with pytest.raises(ModelTrainingError): await workflow_instance._train_model(mock_train_params, 123, metadata) - # Verify MLFLOW_SEND_ERROR status was set + # Verify TRACKING_SEND_ERROR status was set call_args = mock_workflow_module.execute_activity_method.call_args_list[1] - assert call_args[0][1]['status'] == ExperimentStatus.MLFLOW_SEND_ERROR + assert call_args[0][1]['status'] == ExperimentStatus.TRACKING_SEND_ERROR @pytest.mark.asyncio @@ -359,14 +359,14 @@ async def test_update_experiment_run_status_only(mock_workflow_module): metadata=metadata, experiment_run_id=123, update_type=UpdateType.STATUS, - status=ExperimentStatus.MAGE_WAITING_PROC, + status=ExperimentStatus.ORCHESTRATOR_WAITING_PROC, ) # Verify activity was called with correct parameters call_args = mock_workflow_module.execute_activity_method.call_args[0] assert call_args[1]['experiment_run_id'] == 123 assert call_args[1]['update_type'] == UpdateType.STATUS - assert call_args[1]['status'] == ExperimentStatus.MAGE_WAITING_PROC + assert call_args[1]['status'] == ExperimentStatus.ORCHESTRATOR_WAITING_PROC assert 'error_message' not in call_args[1] or call_args[1].get('error_message') is None @@ -411,7 +411,7 @@ async def test_update_experiment_run_with_run_name(mock_workflow_module): metadata=metadata, experiment_run_id=123, update_type=UpdateType.MODEL_SAVED, - status=ExperimentStatus.MLFLOW_SENT, + status=ExperimentStatus.TRACKING_SENT, run_name='test-run-123', ) @@ -438,9 +438,9 @@ async def test_run_complete_workflow_success( mock_workflow_module.execute_activity_method = AsyncMock( side_effect=[ mock_train_params, # validate_train_params - None, # update status (MAGE_WAITING_PROC) + None, # update status (ORCHESTRATOR_WAITING_PROC) train_result, # train_model - None, # update status (MLFLOW_SENT) + None, # update status (TRACKING_SENT) None, # cleanup_resources None, # update status (FILE_DELETED) ] @@ -490,7 +490,7 @@ async def test_run_workflow_training_error( mock_workflow_module.execute_activity_method = AsyncMock( side_effect=[ mock_train_params, # validate_train_params - None, # update status (MAGE_WAITING_PROC) + None, # update status (ORCHESTRATOR_WAITING_PROC) RuntimeError('Training failed'), # train_model fails None, # update status (TRAINING_ERROR) ] @@ -521,9 +521,9 @@ async def test_run_workflow_cleanup_error( mock_workflow_module.execute_activity_method = AsyncMock( side_effect=[ mock_train_params, # validate_train_params - None, # update status (MAGE_WAITING_PROC) + None, # update status (ORCHESTRATOR_WAITING_PROC) train_result, # train_model - None, # update status (MLFLOW_SENT) + None, # update status (TRACKING_SENT) RuntimeError('Cleanup failed'), # cleanup_resources fails None, # update status (FILE_DELETE_ERROR) ] diff --git a/todo-list.txt b/todo-list.txt index f9fee32..7e5e956 100644 --- a/todo-list.txt +++ b/todo-list.txt @@ -1,3 +1,5 @@ +- adiconar volume mounts por causa dos reports. + - Criar o dashboard do grafana. - Atualizar o .github/workflows/quality-gate.yml para usar os pipelines genéricos do github; diff --git a/values.yaml b/values.yaml index d8d6f2b..2e55112 100644 --- a/values.yaml +++ b/values.yaml @@ -7,18 +7,19 @@ replicaCount: 1 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: - repository: aignosi.azurecr.io/sientia-module-courier + repository: aignosi.azurecr.io/sientia-dataops-model-manager # This sets the pull policy for images. - pullPolicy: Always + pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "0.0.2" + tag: "0.0.0" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: - name: docker-hub-secret + # This is to override the chart name. -nameOverride: "sientia-model-manager-worker" -fullnameOverride: "sientia-model-manager-worker" +nameOverride: "sientia-dataops-model-manager" +fullnameOverride: "sientia-dataops-model-manager" namespace: sientia # This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ @@ -31,7 +32,7 @@ serviceAccount: annotations: {} # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template - name: "sientia-model-manager-worker" + name: "sientia-dataops-model-manager" # This is for setting Kubernetes Annotations to a Pod. # For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -68,22 +69,21 @@ resources: {} livenessProbe: exec: command: - - sh + - python3 - -c - - pgrep -f "model_manager.worker.worker" + - "import requests; requests.get('http://localhost:9090/metrics')" initialDelaySeconds: 20 periodSeconds: 30 readinessProbe: exec: command: - - sh + - python3 - -c - - pgrep -f "model_manager.worker.worker" + - "import requests; requests.get('http://localhost:9090/metrics')" initialDelaySeconds: 10 periodSeconds: 15 - # This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ autoscaling: enabled: false @@ -105,6 +105,17 @@ volumeMounts: [] # mountPath: "/etc/foo" # readOnly: true +# Deployment strategy configuration +# More information: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: Recreate + # rollingUpdate: + # maxSurge: 0 + # maxUnavailable: 1 + +# Number of old ReplicaSets to retain +revisionHistoryLimit: 2 + nodeSelector: {} tolerations: [] @@ -118,7 +129,6 @@ services: port: 9091 targetPort: 9091 name: sdk-metrics - metrics: enabled: true type: ClusterIP @@ -141,48 +151,32 @@ serviceMonitor: path: /metrics interval: 30s relabelings: [] - additionalLabels: release: kube-prometheus-stack - env: - # Entrypoint variables - - name: GITHUB_REPO_URL - value: "git@github.com:Aignosi/sientia-dataops-model-manager.git" - - name: GITHUB_BRANCH - value: SIENTIAPDE-1222-ajustar-a-library-para-fazer-o-download-do-courier - - name: PYTHON_APP - value: "model_manager.worker.worker" - - # Application variables - name: POSTGRES_HOST value: "paradedb-rw.paradedb.svc.cluster.local" - name: POSTGRES_PORT value: "5432" - name: POSTGRES_USER - value: "sientia" + value: "postgres" - name: POSTGRES_PASSWORD - value: "sientia" + value: "nFqc81y6kwmr2zuAIx43DhiOosFCVPpeEfTtTWZflkNjB2j1KtEeIANkhFR9mAX3" - name: POSTGRES_DBNAME - value: "sientia" + value: "sientia-core-mlops-bff" - name: POSTGRES_MIN_CONNECTIONS value: "10" - name: POSTGRES_MAX_CONNECTIONS value: "30" - - name: MLFLOW_HOST - value: "http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local" - - name: MLFLOW_PORT - value: "80" + - name: MLFLOW_URL + value: "http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local:80" - name: MLFLOW_USERNAME value: "aignosi" - name: MLFLOW_PASSWORD value: "1L0FP50j3ncp123" - - name: KAFKA_BOOTSTRAP_SERVERS - value: "kafka.kafka.svc.cluster.local:9092" - - name: LOG_LEVEL value: "DEBUG" - name: HTTP_METRICS_PORT @@ -208,15 +202,51 @@ env: - name: MONGODB_TTL_INDEX_HOURS value: "1" + - name: MINIO_ENDPOINT_URL + value: "http://minio.minio.svc.cluster.local:9000" + - name: MINIO_ACCESS_KEY + value: "model-training-user" + - name: MINIO_SECRET_KEY + value: "modelTrainingUser123" + - name: MINIO_REGION + value: "us-east-1" + - name: MINIO_USE_SSL + value: "false" + - name: MINIO_MAX_RETRY_ATTEMPTS + value: "3" + - name: MINIO_RETRY_MODE + value: "adaptive" + - name: MINIO_CONNECT_TIMEOUT + value: "10" + - name: MINIO_READ_TIMEOUT + value: "60" + + - name: TIMEOUT_VALIDATE_PARAMS + value: "30" + - name: TIMEOUT_TRAIN_MODEL + value: "2700" + - name: TIMEOUT_DELETE_FILE + value: "120" + - name: TIMEOUT_UPDATE_DATABASE + value: "30" + + - name: EXTRA_PIP_REQUIREMENTS + value: "git+https://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@github.com/Aignosi/sientia-mlops-library.git" + + - name: POD_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + ssh: - enabled: true + enabled: false secretName: git-ssh-key-sientia-model-manager-worker sshPath: /mnt/.ssh knownHostsPath: /mnt/known_hosts # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp -# helm upgrade --install sientia-model-manager-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.5.0 +# helm upgrade --install sientia-dataops-model-manager sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.5.0 # kubectl create secret generic git-ssh-key-sientia-model-manager-worker \ # --namespace sientia \