feat: update values.yaml and refactor cleanup paths

- Changed project name in values.yaml from "sientia-dataops-model-manager" to "sientia-model-manager".
- Added new environment variables for GitHub repository and branch configuration.
- Refactored cleanup paths to use a centralized REPORTS_TEMP_DIR constant for consistency.
- Updated runtime configurations and adjusted volume mounts for better resource management.
- Enabled SSH access for the model manager and disabled Grafana dashboard creation.
- Updated tests to reflect changes in directory paths and environment variable usage.

model_manager/activities/cleanup.py

@@ -23,6 +23,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.observability.sientia_monitoring import SientiaMonitoring
 
     from model_manager.metrics import ACTIVITY_EXECUTION_TOTAL, WORKFLOW_EXECUTION_TOTAL
+    from model_manager.runtime_paths import REPORTS_TEMP_DIR
 
     RETENTION_HOURS = int(os.getenv('CLEANUP_RETENTION_HOURS', '24'))
     DRY_RUN = os.getenv('CLEANUP_DRY_RUN', 'false').lower() == 'true'
@@ -82,7 +83,7 @@ class Cleanup(SientiaMonitoring):
             Exception: If cleanup fails (after sending notification)
         """
         metadata = input_data.get('metadata', {})
-        temp_path = input_data.get('temp_path', 'model_manager/reports/temp')
+        temp_path = input_data.get('temp_path', REPORTS_TEMP_DIR)
         metrics_status = 'success'
 
         cutoff_time = datetime.now() - timedelta(hours=self.retention_hours)

model_manager/runtime_paths.py

@@ -0,0 +1,26 @@
+"""Filesystem layout for worker runtime data outside the application package tree."""
+
+from os import makedirs
+from os.path import join
+
+# Root for all mutable runtime data (not under /app; avoids clashing with git clone under /app).
+RUNTIME_DATA_ROOT = '/var/lib/model-manager'
+
+# Training reports (HTML, CSV exports, etc.) and related outputs.
+REPORTS_ROOT = join(RUNTIME_DATA_ROOT, 'reports')
+
+# Per-training run folders (name + timestamp); cleanup cron deletes stale entries here.
+REPORTS_TEMP_DIR = join(REPORTS_ROOT, 'temp')
+
+# Worker log files when file logging is wired; stdout remains primary until then.
+LOGS_DIR = join(RUNTIME_DATA_ROOT, 'logs')
+
+
+def ensure_runtime_directories() -> None:
+    """Create runtime directories expected by the worker process."""
+    # REPORTS_ROOT: base directory for report artifacts; remove if all outputs move elsewhere.
+    makedirs(REPORTS_ROOT, exist_ok=True)
+    # REPORTS_TEMP_DIR: transient run subdirs; remove after retention/cleanup is centralized.
+    makedirs(REPORTS_TEMP_DIR, exist_ok=True)
+    # LOGS_DIR: on-disk logs; remove if logging stays stdout-only forever.
+    makedirs(LOGS_DIR, exist_ok=True)

model_manager/utils/repository/data_manager_repository.py

@@ -27,6 +27,7 @@ from sientia_do.observability.sientia_monitoring import SientiaMonitoring
 from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
 from sientia_model.wrappers.sientia_model import SientiaModel
 
+from model_manager.runtime_paths import REPORTS_ROOT
 from model_manager.sientia.metrics import mae, mse, r2
 from model_manager.sientia.reports import Reports  # type: ignore[import-untyped]
 from model_manager.utils.models.train_model_params import TrainModelParams
@@ -394,14 +395,9 @@ class DataManagerRepository(SientiaMonitoring):
         Get the absolute path to the reports directory.
 
         Returns:
-            str: Absolute path to model_manager/reports directory.
+            str: Absolute path to the runtime reports root.
         """
-        # Get the directory where this file is located (model_manager/utils/repository/)
+        return REPORTS_ROOT
-        current_file_dir = path.dirname(path.abspath(__file__))
-        # Navigate up to model_manager/ and then to reports/
-        model_manager_dir = path.dirname(path.dirname(current_file_dir))
-        reports_dir = path.join(model_manager_dir, 'reports')
-        return reports_dir
 
     def _create_run_directory(
         self, base_path: str, run_name: str, metadata: dict[str, Any] | None = None

model_manager/worker/worker.py

@@ -43,6 +43,7 @@ with workflow.unsafe.imports_passed_through():
 
     from model_manager import metrics
     from model_manager.activities.activities import Activities
+    from model_manager.runtime_paths import ensure_runtime_directories
     from model_manager.schedules.cleanup_schedule import create_cleanup_schedule
     from model_manager.utils.connectors_config import (
         build_minio_config,
@@ -97,6 +98,8 @@ async def main():
 
     runtime = _get_runtime(RUNTIME)
 
+    ensure_runtime_directories()
+
     host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
     use_tls = os.getenv('TEMPORAL_USE_TLS', 'false').lower() == 'true'
     logger = get_logger(__name__)

model_manager/workflows/cleanup_files.py

@@ -13,6 +13,7 @@ with workflow.unsafe.imports_passed_through():
     from typing import Any
 
     from model_manager.activities.activities import Activities
+    from model_manager.runtime_paths import REPORTS_TEMP_DIR
     from model_manager.workflows.train_model import no_retry_policy
 
     TIMEOUT_CLEANUP_LOCAL = int(os.getenv('TIMEOUT_CLEANUP_LOCAL', '120'))
@@ -39,8 +40,7 @@ class CleanupFiles:
         in sequence. No exception handling is needed as activities handle their
         own errors and notifications.
         """
-        # Default temp path for local cleanup
+        temp_path = REPORTS_TEMP_DIR
-        temp_path = 'model_manager/reports/temp'
 
         # Metadata for tracking
         metadata = {

tests/test_runtime_paths.py

@@ -0,0 +1,18 @@
+"""Tests for runtime filesystem layout constants."""
+
+from unittest.mock import patch
+
+
+def test_ensure_runtime_directories_creates_expected_paths():
+    from model_manager.runtime_paths import (
+        LOGS_DIR,
+        REPORTS_ROOT,
+        REPORTS_TEMP_DIR,
+        ensure_runtime_directories,
+    )
+
+    with patch('model_manager.runtime_paths.makedirs') as makedirs_mock:
+        ensure_runtime_directories()
+
+    created = {call.args[0] for call in makedirs_mock.call_args_list}
+    assert created == {REPORTS_ROOT, REPORTS_TEMP_DIR, LOGS_DIR}

tests/utils/repository/test_data_manager_repository.py

@@ -10,6 +10,7 @@ import numpy as np
 import pandas as pd
 import pytest
 
+from model_manager.runtime_paths import REPORTS_ROOT
 from model_manager.utils.models.train_model_params import TrainModelParams
 from model_manager.utils.models.train_model_result import TrainModelResult
 from model_manager.utils.repository import data_manager_repository as dmr
@@ -475,5 +476,4 @@ def test_extract_model_equation_more_features_than_coefficients():
 def test_get_reports_directory_path():
     repo = dmr.DataManagerRepository(MagicMock())
     reports_dir = repo._get_reports_directory()
-    assert reports_dir.endswith('reports')
+    assert reports_dir == REPORTS_ROOT
-    assert 'model_manager' in reports_dir

tests/worker/test_worker.py

@@ -191,7 +191,9 @@ def test_start_prometheus_server_failure(
 @patch('model_manager.worker.worker.get_logger')
 @patch('model_manager.worker.worker.start_prometheus_server')
 @patch('model_manager.worker.worker.metrics')
+@patch('model_manager.worker.worker.ensure_runtime_directories')
 async def test_main_successful_startup(
+    mock_ensure_runtime_directories,
     mock_metrics,
     mock_start_prometheus,
     mock_get_logger,
@@ -301,7 +303,9 @@ async def test_main_successful_startup(
 @patch('model_manager.worker.worker.get_logger')
 @patch('model_manager.worker.worker.start_prometheus_server')
 @patch('model_manager.worker.worker.metrics')
+@patch('model_manager.worker.worker.ensure_runtime_directories')
 async def test_main_handles_exception(
+    mock_ensure_runtime_directories,
     mock_metrics,
     mock_start_prometheus,
     mock_get_logger,
@@ -406,7 +410,9 @@ async def test_main_handles_exception(
 @patch('model_manager.worker.worker.get_logger')
 @patch('model_manager.worker.worker.start_prometheus_server')
 @patch('model_manager.worker.worker.metrics')
+@patch('model_manager.worker.worker.ensure_runtime_directories')
 async def test_main_temporal_client_configuration(
+    mock_ensure_runtime_directories,
     mock_metrics,
     mock_start_prometheus,
     mock_get_logger,
@@ -522,7 +528,9 @@ async def test_main_temporal_client_configuration(
 @patch('model_manager.worker.worker.get_logger')
 @patch('model_manager.worker.worker.start_prometheus_server')
 @patch('model_manager.worker.worker.metrics')
+@patch('model_manager.worker.worker.ensure_runtime_directories')
 async def test_main_worker_configuration(
+    mock_ensure_runtime_directories,
     mock_metrics,
     mock_start_prometheus,
     mock_get_logger,
@@ -645,7 +653,9 @@ async def test_main_worker_configuration(
 @patch('model_manager.worker.worker.get_logger')
 @patch('model_manager.worker.worker.start_prometheus_server')
 @patch('model_manager.worker.worker.metrics')
+@patch('model_manager.worker.worker.ensure_runtime_directories')
 async def test_main_schedule_creation_failure_does_not_stop_worker(
+    mock_ensure_runtime_directories,
     mock_metrics,
     mock_start_prometheus,
     mock_get_logger,
@@ -754,7 +764,9 @@ async def test_main_schedule_creation_failure_does_not_stop_worker(
 @patch('model_manager.worker.worker.get_logger')
 @patch('model_manager.worker.worker.start_prometheus_server')
 @patch('model_manager.worker.worker.metrics')
+@patch('model_manager.worker.worker.ensure_runtime_directories')
 async def test_main_missing_runtime_uses_single_fallback(
+    mock_ensure_runtime_directories,
     mock_metrics,
     mock_start_prometheus,
     mock_get_logger,

tests/workflows/test_cleanup_files.py

@@ -10,6 +10,7 @@ import pytest
 @patch('model_manager.workflows.cleanup_files.workflow')
 async def test_cleanup_files_workflow(mock_workflow_module):
     """Test the CleanupFiles workflow."""
+    from model_manager.runtime_paths import REPORTS_TEMP_DIR
     from model_manager.workflows.cleanup_files import CleanupFiles
 
     # Mock execute_activity_method
@@ -26,7 +27,7 @@ async def test_cleanup_files_workflow(mock_workflow_module):
 
     # Check cleanup_temp_directories call
     local_call_args = calls[0][0][1]
-    assert local_call_args['temp_path'] == 'model_manager/reports/temp'
+    assert local_call_args['temp_path'] == REPORTS_TEMP_DIR
     assert local_call_args['metadata'] == {
         'pod_id': 'temporal-pod',
         'workflow_name': 'cleanup_files',

values.yaml

@@ -1,10 +1,10 @@
 #
-# Default values for sientia-dataops-model-manager using the sientia-module chart (0.6.x).
+# Default values for sientia-model-manager using the sientia-module chart (0.6.x).
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 #
 
-projectName: &projectName "sientia-dataops-model-manager"
+projectName: &projectName "sientia-model-manager"
 
 # -----------------------------------------------------------------------------
 # Image configuration (chart-level)
@@ -72,6 +72,16 @@ global:
 
   # Environment variables shared by all runtimes.
   env:
+    # Entrypoint variables 
+    - name: GITHUB_REPO_URL
+      value: "git@github.com:Aignosi/sientia-dataops-model-manager.git"
+    - name: GITHUB_BRANCH
+      value: "release/SIENTIAPDE-1645"
+    - name: PYTHON_APP
+      value: "model_manager.worker.worker"
+    - name: PYPI_SERVER
+      value: "http://library-distribution-server.library.svc.cluster.local:5000"
+
     - name: POSTGRES_HOST
       value: "paradedb-rw.paradedb.svc.cluster.local"
     - name: POSTGRES_PORT
@@ -195,30 +205,17 @@ global:
     - name: CLEANUP_EXECUTION_TIMEOUT_HOURS
       value: "1"
 
-    - name: PYPI_SERVER
-      value: "http://library-distribution-server.library.svc.cluster.local:5000"
-    - name: PYPI_USERNAME
-      valueFrom:
-        secretKeyRef:
-          name: sientia-plugin-store-credentials
-          key: pypi_username
-          optional: true
-    - name: PYPI_PASSWORD
-      valueFrom:
-        secretKeyRef:
-          name: sientia-plugin-store-credentials
-          key: pypi_password
-          optional: true
-
 # -----------------------------------------------------------------------------
 # Runtimes configuration
 # -----------------------------------------------------------------------------
 # Each runtime inherits settings from `global` (resources, env, probes, autoscaling)
 # unless overridden here.
 runtimes:
-  - name: "model-manager-worker"
+  - name: "basic"
     # Replicas for this runtime. Replaces the old replicaCount.
     replicas: 1
+  - name: "xgboost"
+    replicas: 1
 
 # -----------------------------------------------------------------------------
 # Chart-level configuration (applies to all runtimes)
@@ -264,14 +261,14 @@ securityContext: {}
 
 # Additional volumes on the output Deployment definition.
 volumes:
-  - name: reports-volume
+  - name: model-manager-runtime
     emptyDir:
       sizeLimit: 1Gi
 
 # Additional volumeMounts on the output Deployment definition.
 volumeMounts:
-  - name: reports-volume
+  - name: model-manager-runtime
-    mountPath: "/app/model_manager/reports/temp"
+    mountPath: "/var/lib/model-manager"
 
 # Deployment strategy configuration
 # More information: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy
@@ -323,7 +320,7 @@ serviceMonitor:
     release: kube-prometheus-stack
 
 ssh:
-  enabled: false
+  enabled: true
   secretName: git-ssh-key-sientia-model-manager-worker
   sshPath: /mnt/.ssh
   knownHostsPath: /mnt/known_hosts
@@ -331,7 +328,7 @@ ssh:
 # Configuração para dashboards do Grafana
 grafanaDashboard:
   # Habilita a criação de ConfigMaps para dashboards
-  enabled: true
+  enabled: false
   # Namespace onde o Grafana está instalado (ajuste conforme seu ambiente)
   namespace: monitoring
   # Labels para que o sidecar do Grafana encontre os dashboards
@@ -371,7 +368,7 @@ grafanaDatasource:
 # -----------------------------------------------------------------------------
 # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=<pwd>
 #
-# helm upgrade --install sientia-dataops-model-manager sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.6.1
+# helm upgrade --install sientia-model-manager sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.6.1
 #
 # Global/runtimes layout note:
 # - Shared configuration lives under `global` (env, probes, autoscaling, namespace).
@@ -383,4 +380,9 @@ grafanaDatasource:
 #   --from-file=ssh-privatekey=git_key \
 #   --type=kubernetes.io/ssh-auth
 
+# kubectl create secret generic sientia-plugin-store-credentials \
+#   --namespace sientia \
+#   --from-literal=username=<username> \
+#   --from-literal=password=<password>
+