diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 257dcbf..3eaa27c 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -197,7 +197,7 @@ jobs: - name: ๐Ÿงช Run Tests with Pytest run: | - pytest tests --junitxml=pytest.xml --cov=model-manager --cov-report=xml --cov-report=term + pytest tests --junitxml=pytest.xml --cov=model_manager --cov-report=xml --cov-report=term - name: Run SonarQube Analysis uses: SonarSource/sonarqube-scan-action@v5 diff --git a/README.md b/README.md index 5f05ecc..42f49c2 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa ### Key Components -#### **Worker (`laborious/worker/worker.py`)** +#### **Worker (`model_manager/worker/worker.py`)** - **Purpose**: Main application orchestrator managing Temporal workers and task queues - **Responsibilities**: - Temporal client initialization and connection management @@ -68,7 +68,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa - Multi-instance deployment support - Two dedicated task queues: `predictions_batch-queue` and `minimal_retrain-queue` -#### **Workflows (`laborious/workflows/`)** +#### **Workflows (`model_manager/workflows/`)** - **PredictionsBatch**: Main entry point for batch prediction pipelines - **PredictionProcess**: Core prediction pipeline with MLFlow integration - **FormatAndExportPrediction**: Data formatting and export operations @@ -79,7 +79,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa - Comprehensive error handling and recovery - Configurable timeout and retry strategies -#### **Activities (`laborious/activities/`)** +#### **Activities (`model_manager/activities/`)** - **Activities**: Main activity orchestrator combining all functionality through multiple inheritance - **Gates**: Data quality validation and filtering mechanisms - **MLFlow**: Model transformation and prediction operations @@ -92,7 +92,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa - Comprehensive error handling and notification integration - Support for multiple OPC servers with independent configurations -#### **Data Services (`laborious/utils/`)** +#### **Data Services (`model_manager/utils/`)** - **Connectors Config**: Environment variable-based configuration management - **Repository**: Data access layer for MLFlow and OPC operations - `model_repository.py`: MLFlow model operations and retraining @@ -475,7 +475,7 @@ source ./venv/bin/activate pytest # Run with coverage -pytest --cov=laborious --cov-report=html +pytest --cov=model_manager --cov-report=html # Run specific test categories pytest tests/activities/ @@ -496,7 +496,7 @@ if [ -f .env ]; then fi # Start the laborious worker -python -m laborious.worker.worker +python -m model_manager.worker.worker ``` ## ๐Ÿงช Testing @@ -516,7 +516,7 @@ tests/ pip install pytest pytest-cov pytest-asyncio # Run tests with coverage -pytest --cov=laborious --cov-report=html +pytest --cov=model_manager --cov-report=html # Run specific test modules pytest tests/activities/test_gates.py @@ -722,7 +722,7 @@ This is the configuration created by the Orchestrator in Temporal. ### Project Structure ``` -laborious/ +model_manager/ โ”œโ”€โ”€ activities/ # Temporal activity implementations โ”‚ โ”œโ”€โ”€ activities.py # Main activities orchestrator โ”‚ โ”œโ”€โ”€ gates.py # Data quality gates and filtering diff --git a/model-manager/__init__.py b/model_manager/__init__.py similarity index 100% rename from model-manager/__init__.py rename to model_manager/__init__.py diff --git a/model-manager/activities/__init__.py b/model_manager/activities/__init__.py similarity index 100% rename from model-manager/activities/__init__.py rename to model_manager/activities/__init__.py diff --git a/model-manager/activities/activities.py b/model_manager/activities/activities.py similarity index 96% rename from model-manager/activities/activities.py rename to model_manager/activities/activities.py index ec5ae46..1fe2d6f 100644 --- a/model-manager/activities/activities.py +++ b/model_manager/activities/activities.py @@ -4,9 +4,9 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.temporal.activities.postgres import Postgres from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import Logger - from laborious.activities.mlflow import MLFlow - from laborious.activities.gates import Gates - from laborious.activities.opc import OPC + from model_manager.activities.mlflow import MLFlow + from model_manager.activities.gates import Gates + from model_manager.activities.opc import OPC from typing import Any diff --git a/model-manager/activities/gates.py b/model_manager/activities/gates.py similarity index 99% rename from model-manager/activities/gates.py rename to model_manager/activities/gates.py index 4a71455..ec5e806 100644 --- a/model-manager/activities/gates.py +++ b/model_manager/activities/gates.py @@ -9,14 +9,14 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.logger import Logger from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now from sientia_do.formatters import create_sample_dict - from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter - from typing import Any - from laborious.utils.filters.conditional_filters import ( + from model_manager.utils.filters.mlflow_filters import nan_values_filter, api_error_filter + + from model_manager.utils.filters.conditional_filters import ( filter_empty_data, filter_specific_variables_null_values ) from pandas import DataFrame - from laborious import metrics + from model_manager import metrics # Input filter function mappings input_filter_functions = { diff --git a/model-manager/activities/mlflow.py b/model_manager/activities/mlflow.py similarity index 99% rename from model-manager/activities/mlflow.py rename to model_manager/activities/mlflow.py index bd06283..e55a55e 100644 --- a/model-manager/activities/mlflow.py +++ b/model_manager/activities/mlflow.py @@ -10,7 +10,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger from sientia_do.formatters import create_sample_dict - from laborious.utils.repository.model_repository import MLFlowRepository + from model_manager.utils.repository.model_repository import MLFlowRepository from typing import Any import numpy as np from pandas import DataFrame diff --git a/model-manager/activities/opc.py b/model_manager/activities/opc.py similarity index 99% rename from model-manager/activities/opc.py rename to model_manager/activities/opc.py index 0b4ec9f..2222eed 100644 --- a/model-manager/activities/opc.py +++ b/model_manager/activities/opc.py @@ -6,7 +6,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.base import BaseActivity from sientia_do.observability.logger import Logger - from laborious.utils.repository.opc_repository import OpcRepository + from model_manager.utils.repository.opc_repository import OpcRepository from typing import Any import traceback from pandas import DataFrame diff --git a/model-manager/metrics.py b/model_manager/metrics.py similarity index 100% rename from model-manager/metrics.py rename to model_manager/metrics.py diff --git a/model-manager/utils/__init__.py b/model_manager/utils/__init__.py similarity index 100% rename from model-manager/utils/__init__.py rename to model_manager/utils/__init__.py diff --git a/model-manager/utils/connectors_config.py b/model_manager/utils/connectors_config.py similarity index 100% rename from model-manager/utils/connectors_config.py rename to model_manager/utils/connectors_config.py diff --git a/model-manager/utils/filters/__init__.py b/model_manager/utils/filters/__init__.py similarity index 100% rename from model-manager/utils/filters/__init__.py rename to model_manager/utils/filters/__init__.py diff --git a/model-manager/utils/filters/conditional_filters.py b/model_manager/utils/filters/conditional_filters.py similarity index 100% rename from model-manager/utils/filters/conditional_filters.py rename to model_manager/utils/filters/conditional_filters.py diff --git a/model-manager/utils/filters/mlflow_filters.py b/model_manager/utils/filters/mlflow_filters.py similarity index 100% rename from model-manager/utils/filters/mlflow_filters.py rename to model_manager/utils/filters/mlflow_filters.py diff --git a/model-manager/utils/repository/model_repository.py b/model_manager/utils/repository/model_repository.py similarity index 100% rename from model-manager/utils/repository/model_repository.py rename to model_manager/utils/repository/model_repository.py diff --git a/model-manager/utils/repository/opc_repository.py b/model_manager/utils/repository/opc_repository.py similarity index 99% rename from model-manager/utils/repository/opc_repository.py rename to model_manager/utils/repository/opc_repository.py index 96ecfef..94b6c46 100644 --- a/model-manager/utils/repository/opc_repository.py +++ b/model_manager/utils/repository/opc_repository.py @@ -11,7 +11,7 @@ from regex import F from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger -from laborious import metrics +from model_manager import metrics data_type_map = { 'float': { diff --git a/model-manager/worker/__init__.py b/model_manager/worker/__init__.py similarity index 100% rename from model-manager/worker/__init__.py rename to model_manager/worker/__init__.py diff --git a/model-manager/worker/worker.py b/model_manager/worker/worker.py similarity index 94% rename from model-manager/worker/worker.py rename to model_manager/worker/worker.py index 567a257..a10c4cb 100644 --- a/model-manager/worker/worker.py +++ b/model_manager/worker/worker.py @@ -33,13 +33,13 @@ with workflow.unsafe.imports_passed_through(): import os import sys import asyncio - from laborious.workflows.minimal_retrain import MinimalRetrain - from laborious.workflows.predictions_batch import PredictionsBatch - from laborious.workflows.sub_workflows.prediction_process import PredictionProcess - from laborious.workflows.sub_workflows.format_and_export_prediction import \ + from model_manager.workflows.minimal_retrain import MinimalRetrain + from model_manager.workflows.predictions_batch import PredictionsBatch + from model_manager.workflows.sub_workflows.prediction_process import PredictionProcess + from model_manager.workflows.sub_workflows.format_and_export_prediction import \ FormatAndExportPrediction - from laborious.activities.activities import Activities - from laborious.utils.connectors_config import ( + from model_manager.activities.activities import Activities + from model_manager.utils.connectors_config import ( build_postgres_config, build_mlflow_config, build_opc_config, @@ -47,7 +47,7 @@ with workflow.unsafe.imports_passed_through(): ) from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import get_logger - from laborious import metrics + from model_manager import metrics from prometheus_client import start_http_server POD_ID = os.getenv('POD_ID') diff --git a/model-manager/workflows/__init__.py b/model_manager/workflows/__init__.py similarity index 100% rename from model-manager/workflows/__init__.py rename to model_manager/workflows/__init__.py diff --git a/model-manager/workflows/minimal_retrain.py b/model_manager/workflows/minimal_retrain.py similarity index 98% rename from model-manager/workflows/minimal_retrain.py rename to model_manager/workflows/minimal_retrain.py index 1893e25..4701c91 100644 --- a/model-manager/workflows/minimal_retrain.py +++ b/model_manager/workflows/minimal_retrain.py @@ -1,7 +1,7 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities + from model_manager.activities.activities import Activities from typing import Any from sientia_do.temporal.policies import retry_policy from datetime import timedelta diff --git a/model-manager/workflows/predictions_batch.py b/model_manager/workflows/predictions_batch.py similarity index 98% rename from model-manager/workflows/predictions_batch.py rename to model_manager/workflows/predictions_batch.py index e522eaa..330493d 100644 --- a/model-manager/workflows/predictions_batch.py +++ b/model_manager/workflows/predictions_batch.py @@ -1,7 +1,7 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities + from model_manager.activities.activities import Activities from typing import Any from sientia_do.temporal.policies import retry_policy from datetime import timedelta diff --git a/model-manager/workflows/sub_workflows/__init__.py b/model_manager/workflows/sub_workflows/__init__.py similarity index 100% rename from model-manager/workflows/sub_workflows/__init__.py rename to model_manager/workflows/sub_workflows/__init__.py diff --git a/model-manager/workflows/sub_workflows/format_and_export_prediction.py b/model_manager/workflows/sub_workflows/format_and_export_prediction.py similarity index 98% rename from model-manager/workflows/sub_workflows/format_and_export_prediction.py rename to model_manager/workflows/sub_workflows/format_and_export_prediction.py index 8e7df07..ef195c4 100644 --- a/model-manager/workflows/sub_workflows/format_and_export_prediction.py +++ b/model_manager/workflows/sub_workflows/format_and_export_prediction.py @@ -1,7 +1,7 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities + from model_manager.activities.activities import Activities from typing import Any from datetime import timedelta from sientia_do.temporal.policies import retry_policy diff --git a/model-manager/workflows/sub_workflows/prediction_process.py b/model_manager/workflows/sub_workflows/prediction_process.py similarity index 99% rename from model-manager/workflows/sub_workflows/prediction_process.py rename to model_manager/workflows/sub_workflows/prediction_process.py index 777fa1c..4addd75 100644 --- a/model-manager/workflows/sub_workflows/prediction_process.py +++ b/model_manager/workflows/sub_workflows/prediction_process.py @@ -1,7 +1,7 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities + from model_manager.activities.activities import Activities from typing import Any from sientia_do.temporal.policies import retry_policy from datetime import timedelta diff --git a/run_coverage.sh b/run_coverage.sh index f9af4cb..7d4f6f4 100755 --- a/run_coverage.sh +++ b/run_coverage.sh @@ -6,6 +6,6 @@ set -e echo "Activating virtual environment..." source ./venv/bin/activate -pytest --cov=laborious --cov-report=html +pytest --cov=model_manager --cov-report=html xdg-open htmlcov/index.html \ No newline at end of file diff --git a/run_local.sh b/run_local.sh index 2bbd5c2..680ff15 100755 --- a/run_local.sh +++ b/run_local.sh @@ -15,4 +15,4 @@ else fi echo "Starting ingestor application..." -python -m laborious.worker.worker +python -m model_manager.worker.worker diff --git a/sonar-project.properties b/sonar-project.properties index 0edb732..e8ae7ff 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,6 +1,6 @@ sonar.projectKey=Aignosi_sientia-dataops-model-manager_dc6e051c-995a-4b57-9cde-f511018184c7 sonar.projectName=sientia-dataops-model-manager -sonar.sources=model-manager +sonar.sources=model_manager sonar.tests=tests sonar.qualitygate.wait=true sonar.qualitygate.timeout=300 diff --git a/tests/laborious/activities/test_activities.py b/tests/laborious/activities/test_activities.py index de418be..106ccf7 100644 --- a/tests/laborious/activities/test_activities.py +++ b/tests/laborious/activities/test_activities.py @@ -1,16 +1,16 @@ from pytest import mark from unittest.mock import patch, MagicMock, ANY from sientia_do.temporal.activities.postgres import Postgres -from laborious.activities.activities import Activities -from laborious.activities.mlflow import MLFlow -from laborious.activities.gates import Gates -from laborious.activities.opc import OPC +from model_manager.activities.activities import Activities +from model_manager.activities.mlflow import MLFlow +from model_manager.activities.gates import Gates +from model_manager.activities.opc import OPC -@patch('laborious.activities.activities.Postgres.__init__') -@patch('laborious.activities.activities.MLFlow.__init__') -@patch('laborious.activities.activities.OPC.__init__') -@patch('laborious.activities.activities.Gates.__init__') +@patch('model_manager.activities.activities.Postgres.__init__') +@patch('model_manager.activities.activities.MLFlow.__init__') +@patch('model_manager.activities.activities.OPC.__init__') +@patch('model_manager.activities.activities.Gates.__init__') def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgres_init): postgres_config = { @@ -91,9 +91,9 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre @mark.asyncio -@patch('laborious.activities.activities.Postgres', return_value=MagicMock()) -@patch('laborious.activities.activities.MLFlow', return_value=MagicMock()) -@patch('laborious.activities.activities.OPC', return_value=MagicMock()) +@patch('model_manager.activities.activities.Postgres', return_value=MagicMock()) +@patch('model_manager.activities.activities.MLFlow', return_value=MagicMock()) +@patch('model_manager.activities.activities.OPC', return_value=MagicMock()) async def test_shutdown(mock_opc_init, _mock_mlflow_init, mock_postgres_init): postgres_config = { diff --git a/tests/laborious/activities/test_gates.py b/tests/laborious/activities/test_gates.py index a65c6d4..b587b93 100644 --- a/tests/laborious/activities/test_gates.py +++ b/tests/laborious/activities/test_gates.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock, ANY, patch from pytest import fixture, mark from sientia_do.notifications.models import NotificationLevel -from laborious.activities.gates import Gates +from model_manager.activities.gates import Gates @fixture @@ -52,7 +52,7 @@ async def test_input_gate_invalid_filter(gates_activity): @mark.asyncio -@patch('laborious.activities.gates.input_filter_functions') +@patch('model_manager.activities.gates.input_filter_functions') async def test_input_gate_filter_exception(mock_input_filter_functions, gates_activity): # Arrange mock_input_filter_functions.__contains__.return_value = True @@ -141,7 +141,7 @@ async def test_mlflow_response_gate_invalid_filter(gates_activity): @mark.asyncio -@patch('laborious.activities.gates.mlflow_response_filter_functions') +@patch('model_manager.activities.gates.mlflow_response_filter_functions') async def test_mlflow_response_gate_filter_exception(mock_mlflow_response_filter_functions, gates_activity): # Arrange @@ -241,7 +241,7 @@ async def test_mlflow_content_gate_invalid_filter(gates_activity): @mark.asyncio -@patch('laborious.activities.gates.mlflow_content_filter_functions') +@patch('model_manager.activities.gates.mlflow_content_filter_functions') async def test_mlflow_content_gate_filter_exception(mock_mlflow_content_filter_functions, gates_activity): # Arrange @@ -559,7 +559,7 @@ async def test_get_last_timestamp_no_data(gates_activity): @mark.asyncio -@patch('laborious.activities.gates.metrics') +@patch('model_manager.activities.gates.metrics') async def test_write_metrics(mock_metrics, gates_activity): """Test write_metrics method.""" input_data = { diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index 85cb7f1..243982f 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -5,11 +5,11 @@ import numpy as np from pandas import DataFrame, Timestamp from pytest import fixture, mark, raises from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ -from laborious.activities.mlflow import MLFlow +from model_manager.activities.mlflow import MLFlow from sientia_do.notifications.models import NotificationLevel -@patch("laborious.activities.mlflow.MLFlowRepository") +@patch("model_manager.activities.mlflow.MLFlowRepository") def test___init__(mock_mlflow_repository): mlflow = MLFlow( mlflow_host="http://localhost", @@ -31,7 +31,7 @@ def test___init__(mock_mlflow_repository): @fixture -@patch("laborious.activities.mlflow.MLFlowRepository") +@patch("model_manager.activities.mlflow.MLFlowRepository") def mlflow(mock_mlflow_repository): mlflow = MLFlow( mlflow_host="http://localhost:5000", @@ -58,8 +58,8 @@ metadata = { @mark.asyncio -@patch("laborious.activities.mlflow.DataFrame") -@patch("laborious.activities.mlflow.max") +@patch("model_manager.activities.mlflow.DataFrame") +@patch("model_manager.activities.mlflow.max") async def test_request_transform_success(mock_max, mock_dataframe, mlflow): mock_max.return_value = '2024-01-02' # Mock input data @@ -114,9 +114,9 @@ async def test_request_transform_success(mock_max, mock_dataframe, mlflow): @mark.asyncio -@patch("laborious.activities.mlflow.DataFrame") -@patch("laborious.activities.mlflow.to_datetime") -@patch("laborious.activities.mlflow.max") +@patch("model_manager.activities.mlflow.DataFrame") +@patch("model_manager.activities.mlflow.to_datetime") +@patch("model_manager.activities.mlflow.max") async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflow): mock_max.return_value = '2024-01-02' # Mock input data diff --git a/tests/laborious/activities/test_opc.py b/tests/laborious/activities/test_opc.py index 07f00b8..dcff019 100644 --- a/tests/laborious/activities/test_opc.py +++ b/tests/laborious/activities/test_opc.py @@ -4,7 +4,7 @@ from pytest import fixture, mark import pytest_asyncio from sientia_do.notifications.models import NotificationLevel -from laborious.activities.opc import OPC +from model_manager.activities.opc import OPC metadata = { "metadata": { @@ -31,8 +31,8 @@ def test__init__(): @mark.asyncio -@patch("laborious.activities.opc.OpcRepository") -@patch("laborious.activities.opc.OPC.send_notification") +@patch("model_manager.activities.opc.OpcRepository") +@patch("model_manager.activities.opc.OPC.send_notification") async def test_init_opc(mock_send_notification, mock_opc_repository): mock_logger = MagicMock() server1 = MagicMock( @@ -147,7 +147,7 @@ async def test_init_opc(mock_send_notification, mock_opc_repository): @pytest_asyncio.fixture -@patch("laborious.activities.opc.OpcRepository") +@patch("model_manager.activities.opc.OpcRepository") async def opc(mock_opc_repository): servers = { 'server1': { diff --git a/tests/laborious/utils/filters/test_conditional_filters.py b/tests/laborious/utils/filters/test_conditional_filters.py index 405bc9b..a7f3efd 100644 --- a/tests/laborious/utils/filters/test_conditional_filters.py +++ b/tests/laborious/utils/filters/test_conditional_filters.py @@ -1,6 +1,6 @@ from pandas import DataFrame -from laborious.utils.filters.conditional_filters import ( +from model_manager.utils.filters.conditional_filters import ( filter_specific_variables_null_values, filter_empty_data ) diff --git a/tests/laborious/utils/filters/test_mlflow_filters.py b/tests/laborious/utils/filters/test_mlflow_filters.py index f9c61e9..6fddf51 100644 --- a/tests/laborious/utils/filters/test_mlflow_filters.py +++ b/tests/laborious/utils/filters/test_mlflow_filters.py @@ -1,5 +1,5 @@ from pandas import DataFrame -from laborious.utils.filters.mlflow_filters import api_error_filter, nan_values_filter +from model_manager.utils.filters.mlflow_filters import api_error_filter, nan_values_filter def test_api_error_filter_invalid_response(): diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index cdc59b4..ab65913 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -4,12 +4,12 @@ from pandas import DataFrame import pytest from datetime import datetime, timezone from pandas import Timestamp -from laborious.utils.repository.model_repository import MLFlowRepository +from model_manager.utils.repository.model_repository import MLFlowRepository @pytest.fixture def mlflow_repository(): - with patch('laborious.utils.repository.model_repository.ModelServing', + with patch('model_manager.utils.repository.model_repository.ModelServing', autospec=True) as mock_model_serving: mock_instance = mock_model_serving.return_value mock_instance.get_transformed_data = MagicMock() @@ -217,7 +217,7 @@ def test_predict_error(mlflow_repository): } -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_experiment_by_run_id(mlflow, mlflow_repository): mlflow.get_run.return_value = MagicMock( info=MagicMock( @@ -233,7 +233,7 @@ def test_get_experiment_by_run_id(mlflow, mlflow_repository): mlflow.get_experiment.assert_called_once_with('0') -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_next_run_name(mlflow, mlflow_repository): mlflow.search_runs.return_value = [1, 2, 3] output = mlflow_repository.get_next_run_name('run') @@ -244,7 +244,7 @@ def test_get_next_run_name(mlflow, mlflow_repository): ) -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_experiment_success(mlflow, mlflow_repository): mlflow.get_experiment_by_name.return_value = MagicMock( experiment_id='0') @@ -254,7 +254,7 @@ def test_get_experiment_success(mlflow, mlflow_repository): assert output == 0 -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_experiment_error(mlflow, mlflow_repository): mlflow.get_experiment_by_name.return_value = None @@ -266,7 +266,7 @@ def test_get_experiment_error(mlflow, mlflow_repository): assert False -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_experiment_last_run(mlflow, mlflow_repository): mlflow.search_runs.return_value = DataFrame({ 'params.retrain': ['True', 'False', 'True', 'False'], @@ -285,7 +285,7 @@ def test_get_experiment_last_run(mlflow, mlflow_repository): assert output == '2' -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_experiment_last_run_error(mlflow, mlflow_repository): mlflow.search_runs.return_value = [] @@ -297,8 +297,8 @@ def test_get_experiment_last_run_error(mlflow, mlflow_repository): assert False -@patch('laborious.utils.repository.model_repository.mlflow.sklearn') -@patch('laborious.utils.repository.model_repository.mlflow.set_experiment') +@patch('model_manager.utils.repository.model_repository.mlflow.sklearn') +@patch('model_manager.utils.repository.model_repository.mlflow.set_experiment') def test_create_model_experiment(set_experiment, sklearn, mlflow_repository): mlflow_repository.model_serving.get_model_run_id = MagicMock( @@ -360,10 +360,10 @@ def test_create_model_experiment(set_experiment, sklearn, mlflow_repository): mlflow_repository.get_experiment_by_run_id.return_value) -@patch('laborious.utils.repository.model_repository.mlflow.start_run') -@patch('laborious.utils.repository.model_repository.mlflow.log_param') -@patch('laborious.utils.repository.model_repository.mlflow.sklearn.log_model') -@patch('laborious.utils.repository.model_repository.mlflow.log_artifact') +@patch('model_manager.utils.repository.model_repository.mlflow.start_run') +@patch('model_manager.utils.repository.model_repository.mlflow.log_param') +@patch('model_manager.utils.repository.model_repository.mlflow.sklearn.log_model') +@patch('model_manager.utils.repository.model_repository.mlflow.log_artifact') def test_perform_model_retrain(log_artifact, log_model, log_param, start_run, mlflow_repository): prediction_model_mock = MagicMock() @@ -423,7 +423,7 @@ def test_retrain_model(mlflow_repository): assert output == 'Model retrained successfully' -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_update_production_model_by_run_id(mlflow, mlflow_repository): client_mock = MagicMock() mlflow.tracking.MlflowClient.return_value = client_mock @@ -458,7 +458,7 @@ def test_update_production_model_by_run_id(mlflow, mlflow_repository): } -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_update_production_model_by_run_id_error(mlflow, mlflow_repository): mlflow.tracking.MlflowClient.return_value = MagicMock( get_registered_model=MagicMock( diff --git a/tests/laborious/utils/repository/test_opc_repository.py b/tests/laborious/utils/repository/test_opc_repository.py index bc84db8..9795a10 100644 --- a/tests/laborious/utils/repository/test_opc_repository.py +++ b/tests/laborious/utils/repository/test_opc_repository.py @@ -1,7 +1,7 @@ import pytest from unittest.mock import AsyncMock, Mock, patch, MagicMock, ANY, call from asyncua.crypto.security_policies import SecurityPolicyBasic256 -from laborious.utils.repository.opc_repository import OpcRepository +from model_manager.utils.repository.opc_repository import OpcRepository from sientia_do.notifications.models import NotificationLevel from datetime import datetime @@ -28,7 +28,7 @@ def opc_repository(mock_logger): @pytest.fixture def mock_client(): - with patch('laborious.utils.repository.opc_repository.Client') as mock: + with patch('model_manager.utils.repository.opc_repository.Client') as mock: client_instance = AsyncMock() mock.return_value = client_instance yield client_instance @@ -212,7 +212,7 @@ async def test_validate_connection_error_validate_connection_error(opc_repositor @pytest.mark.asyncio -@patch('laborious.utils.repository.opc_repository.datetime') +@patch('model_manager.utils.repository.opc_repository.datetime') async def test_validate_connection_lost_not_time_to_reconnect(_mock_datetime, opc_repository): _mock_datetime.now = MagicMock( return_value=datetime(2025, 1, 1, 0, 0, 0)) @@ -233,7 +233,7 @@ async def test_validate_connection_lost_not_time_to_reconnect(_mock_datetime, op @pytest.mark.asyncio -@patch('laborious.utils.repository.opc_repository.datetime') +@patch('model_manager.utils.repository.opc_repository.datetime') async def test_validate_connection_lost_time_to_reconnect(mock_datetime, opc_repository): mock_datetime.now = MagicMock( return_value=datetime(2025, 1, 1, 1, 0, 0)) @@ -333,7 +333,7 @@ async def test_write_data_invalid_data_type(opc_repository, mock_client): @pytest.mark.asyncio -@patch('laborious.utils.repository.opc_repository.metrics') +@patch('model_manager.utils.repository.opc_repository.metrics') async def test_write_data(mock_metrics, opc_repository, mock_client): opc_repository.validate_connection = AsyncMock(return_value=(True, {})) opc_repository.client = mock_client diff --git a/tests/laborious/utils/test_connectors_config.py b/tests/laborious/utils/test_connectors_config.py index 910439c..43a2ec9 100644 --- a/tests/laborious/utils/test_connectors_config.py +++ b/tests/laborious/utils/test_connectors_config.py @@ -1,5 +1,5 @@ from os import environ -from laborious.utils.connectors_config import (build_mlflow_config, +from model_manager.utils.connectors_config import (build_mlflow_config, build_opc_config, build_postgres_config, build_mongodb_config) diff --git a/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py b/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py index a8e6e20..356c32b 100644 --- a/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py +++ b/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py @@ -1,8 +1,8 @@ from unittest.mock import call, patch, AsyncMock, ANY from pytest import mark, fixture -from laborious.activities.activities import Activities -from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction +from model_manager.activities.activities import Activities +from model_manager.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ @@ -22,7 +22,7 @@ metadata = { @mark.asyncio -@patch("laborious.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock) async def test_run_none_path_flag(workflow_mock, format_and_export_prediction): input_data = { @@ -91,7 +91,7 @@ async def test_run_none_path_flag(workflow_mock, format_and_export_prediction): @mark.asyncio -@patch("laborious.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock) async def test_run_default_path_flag(workflow_mock, format_and_export_prediction): input_data = { diff --git a/tests/laborious/workflows/subworkflows/test_prediction_process.py b/tests/laborious/workflows/subworkflows/test_prediction_process.py index df60ada..46b1565 100644 --- a/tests/laborious/workflows/subworkflows/test_prediction_process.py +++ b/tests/laborious/workflows/subworkflows/test_prediction_process.py @@ -1,7 +1,7 @@ from unittest.mock import AsyncMock, patch, call, ANY from pytest import fixture, mark -from laborious.activities.activities import Activities -from laborious.workflows.sub_workflows.prediction_process import PredictionProcess +from model_manager.activities.activities import Activities +from model_manager.workflows.sub_workflows.prediction_process import PredictionProcess @fixture @@ -20,7 +20,7 @@ metadata = { @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_run(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock(return_value=False) # Arrange @@ -135,7 +135,7 @@ async def test_run(workflow_mock, prediction_process): @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_run_stop_at_input_gate(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock(return_value=True) # Arrange @@ -183,7 +183,7 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process): @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock(side_effect=[False, True]) # Arrange @@ -253,7 +253,7 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock( side_effect=[False, False, True]) @@ -333,7 +333,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock( side_effect=[False, False, False, True]) @@ -429,7 +429,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_path_flag_handler_stop(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} @@ -464,7 +464,7 @@ async def test_path_flag_handler_stop(workflow_mock, prediction_process): @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_path_flag_handler_repeat(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} @@ -510,7 +510,7 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process): @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_path_flag_handler_continue(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} @@ -565,7 +565,7 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_path_flag_handler_unknown(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} diff --git a/tests/laborious/workflows/test_minimal_retrain.py b/tests/laborious/workflows/test_minimal_retrain.py index b3b03b5..1d8e6d7 100644 --- a/tests/laborious/workflows/test_minimal_retrain.py +++ b/tests/laborious/workflows/test_minimal_retrain.py @@ -1,7 +1,7 @@ from unittest.mock import AsyncMock, MagicMock, call, patch, ANY from pytest import fixture, mark -from laborious.activities.activities import Activities -from laborious.workflows.minimal_retrain import MinimalRetrain +from model_manager.activities.activities import Activities +from model_manager.workflows.minimal_retrain import MinimalRetrain @fixture @@ -20,7 +20,7 @@ metadata = { @mark.asyncio -@patch('laborious.workflows.minimal_retrain.workflow', new_callable=AsyncMock) +@patch('model_manager.workflows.minimal_retrain.workflow', new_callable=AsyncMock) async def test_run(workflow_mock: AsyncMock, minimal_retrain: MinimalRetrain): input_data = { "model_id": "test_model_id", diff --git a/tests/laborious/workflows/test_predictions_batch.py b/tests/laborious/workflows/test_predictions_batch.py index 90d7d21..0d7d477 100644 --- a/tests/laborious/workflows/test_predictions_batch.py +++ b/tests/laborious/workflows/test_predictions_batch.py @@ -1,7 +1,7 @@ from unittest.mock import AsyncMock, call, patch, ANY from pytest import fixture, mark -from laborious.activities.activities import Activities -from laborious.workflows.predictions_batch import PredictionsBatch +from model_manager.activities.activities import Activities +from model_manager.workflows.predictions_batch import PredictionsBatch @fixture @@ -20,7 +20,7 @@ metadata = { @mark.asyncio -@patch('laborious.workflows.predictions_batch.workflow', new_callable=AsyncMock) +@patch('model_manager.workflows.predictions_batch.workflow', new_callable=AsyncMock) async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch): workflow_mock.execute_local_activity_method.return_value = { 'data': 'test_data' diff --git a/values.yaml b/values.yaml index 4eb83d7..3af8b24 100644 --- a/values.yaml +++ b/values.yaml @@ -70,7 +70,7 @@ livenessProbe: command: - sh - -c - - pgrep -f "laborious.worker.worker" + - pgrep -f "model_manager.worker.worker" initialDelaySeconds: 20 periodSeconds: 30 @@ -79,7 +79,7 @@ readinessProbe: command: - sh - -c - - pgrep -f "laborious.worker.worker" + - pgrep -f "model_manager.worker.worker" initialDelaySeconds: 10 periodSeconds: 15 @@ -153,7 +153,7 @@ env: - name: GITHUB_BRANCH value: SIENTIAPDE-1222-ajustar-a-library-para-fazer-o-download-do-courier - name: PYTHON_APP - value: "laborious.worker.worker" + value: "model_manager.worker.worker" # Application variables - name: POSTGRES_HOST