From aba4a3f1a5460a265d12f4fba6d31d06819c414d Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Wed, 1 Oct 2025 14:29:24 -0300 Subject: [PATCH] SIENTIAPDE-1243: Refactor: Rename 'laborious' package to 'model_manager' This commit renames the 'laborious' package to 'model_manager' across the entire project. This includes renaming directories, modules, references in code, configuration files, and documentation to reflect the new package name. This change improves clarity and consistency within the project. --- .github/workflows/quality-gate.yml | 2 +- README.md | 16 +++++----- {model-manager => model_manager}/__init__.py | 0 .../activities/__init__.py | 0 .../activities/activities.py | 6 ++-- .../activities/gates.py | 8 ++--- .../activities/mlflow.py | 2 +- .../activities/opc.py | 2 +- {model-manager => model_manager}/metrics.py | 0 .../utils/__init__.py | 0 .../utils/connectors_config.py | 0 .../utils/filters/__init__.py | 0 .../utils/filters/conditional_filters.py | 0 .../utils/filters/mlflow_filters.py | 0 .../utils/repository/model_repository.py | 0 .../utils/repository/opc_repository.py | 2 +- .../worker/__init__.py | 0 .../worker/worker.py | 14 ++++---- .../workflows/__init__.py | 0 .../workflows/minimal_retrain.py | 2 +- .../workflows/predictions_batch.py | 2 +- .../workflows/sub_workflows/__init__.py | 0 .../format_and_export_prediction.py | 2 +- .../sub_workflows/prediction_process.py | 2 +- run_coverage.sh | 2 +- run_local.sh | 2 +- sonar-project.properties | 2 +- tests/laborious/activities/test_activities.py | 22 ++++++------- tests/laborious/activities/test_gates.py | 10 +++--- tests/laborious/activities/test_mlflow.py | 16 +++++----- tests/laborious/activities/test_opc.py | 8 ++--- .../utils/filters/test_conditional_filters.py | 2 +- .../utils/filters/test_mlflow_filters.py | 2 +- .../utils/repository/test_model_repository.py | 32 +++++++++---------- .../utils/repository/test_opc_repository.py | 10 +++--- .../laborious/utils/test_connectors_config.py | 2 +- .../test_format_and_export_prediction.py | 8 ++--- .../subworkflows/test_prediction_process.py | 22 ++++++------- .../workflows/test_minimal_retrain.py | 6 ++-- .../workflows/test_predictions_batch.py | 6 ++-- values.yaml | 6 ++-- 41 files changed, 109 insertions(+), 109 deletions(-) rename {model-manager => model_manager}/__init__.py (100%) rename {model-manager => model_manager}/activities/__init__.py (100%) rename {model-manager => model_manager}/activities/activities.py (96%) rename {model-manager => model_manager}/activities/gates.py (99%) rename {model-manager => model_manager}/activities/mlflow.py (99%) rename {model-manager => model_manager}/activities/opc.py (99%) rename {model-manager => model_manager}/metrics.py (100%) rename {model-manager => model_manager}/utils/__init__.py (100%) rename {model-manager => model_manager}/utils/connectors_config.py (100%) rename {model-manager => model_manager}/utils/filters/__init__.py (100%) rename {model-manager => model_manager}/utils/filters/conditional_filters.py (100%) rename {model-manager => model_manager}/utils/filters/mlflow_filters.py (100%) rename {model-manager => model_manager}/utils/repository/model_repository.py (100%) rename {model-manager => model_manager}/utils/repository/opc_repository.py (99%) rename {model-manager => model_manager}/worker/__init__.py (100%) rename {model-manager => model_manager}/worker/worker.py (94%) rename {model-manager => model_manager}/workflows/__init__.py (100%) rename {model-manager => model_manager}/workflows/minimal_retrain.py (98%) rename {model-manager => model_manager}/workflows/predictions_batch.py (98%) rename {model-manager => model_manager}/workflows/sub_workflows/__init__.py (100%) rename {model-manager => model_manager}/workflows/sub_workflows/format_and_export_prediction.py (98%) rename {model-manager => model_manager}/workflows/sub_workflows/prediction_process.py (99%) diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 257dcbf..3eaa27c 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -197,7 +197,7 @@ jobs: - name: ๐Ÿงช Run Tests with Pytest run: | - pytest tests --junitxml=pytest.xml --cov=model-manager --cov-report=xml --cov-report=term + pytest tests --junitxml=pytest.xml --cov=model_manager --cov-report=xml --cov-report=term - name: Run SonarQube Analysis uses: SonarSource/sonarqube-scan-action@v5 diff --git a/README.md b/README.md index 5f05ecc..42f49c2 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa ### Key Components -#### **Worker (`laborious/worker/worker.py`)** +#### **Worker (`model_manager/worker/worker.py`)** - **Purpose**: Main application orchestrator managing Temporal workers and task queues - **Responsibilities**: - Temporal client initialization and connection management @@ -68,7 +68,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa - Multi-instance deployment support - Two dedicated task queues: `predictions_batch-queue` and `minimal_retrain-queue` -#### **Workflows (`laborious/workflows/`)** +#### **Workflows (`model_manager/workflows/`)** - **PredictionsBatch**: Main entry point for batch prediction pipelines - **PredictionProcess**: Core prediction pipeline with MLFlow integration - **FormatAndExportPrediction**: Data formatting and export operations @@ -79,7 +79,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa - Comprehensive error handling and recovery - Configurable timeout and retry strategies -#### **Activities (`laborious/activities/`)** +#### **Activities (`model_manager/activities/`)** - **Activities**: Main activity orchestrator combining all functionality through multiple inheritance - **Gates**: Data quality validation and filtering mechanisms - **MLFlow**: Model transformation and prediction operations @@ -92,7 +92,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa - Comprehensive error handling and notification integration - Support for multiple OPC servers with independent configurations -#### **Data Services (`laborious/utils/`)** +#### **Data Services (`model_manager/utils/`)** - **Connectors Config**: Environment variable-based configuration management - **Repository**: Data access layer for MLFlow and OPC operations - `model_repository.py`: MLFlow model operations and retraining @@ -475,7 +475,7 @@ source ./venv/bin/activate pytest # Run with coverage -pytest --cov=laborious --cov-report=html +pytest --cov=model_manager --cov-report=html # Run specific test categories pytest tests/activities/ @@ -496,7 +496,7 @@ if [ -f .env ]; then fi # Start the laborious worker -python -m laborious.worker.worker +python -m model_manager.worker.worker ``` ## ๐Ÿงช Testing @@ -516,7 +516,7 @@ tests/ pip install pytest pytest-cov pytest-asyncio # Run tests with coverage -pytest --cov=laborious --cov-report=html +pytest --cov=model_manager --cov-report=html # Run specific test modules pytest tests/activities/test_gates.py @@ -722,7 +722,7 @@ This is the configuration created by the Orchestrator in Temporal. ### Project Structure ``` -laborious/ +model_manager/ โ”œโ”€โ”€ activities/ # Temporal activity implementations โ”‚ โ”œโ”€โ”€ activities.py # Main activities orchestrator โ”‚ โ”œโ”€โ”€ gates.py # Data quality gates and filtering diff --git a/model-manager/__init__.py b/model_manager/__init__.py similarity index 100% rename from model-manager/__init__.py rename to model_manager/__init__.py diff --git a/model-manager/activities/__init__.py b/model_manager/activities/__init__.py similarity index 100% rename from model-manager/activities/__init__.py rename to model_manager/activities/__init__.py diff --git a/model-manager/activities/activities.py b/model_manager/activities/activities.py similarity index 96% rename from model-manager/activities/activities.py rename to model_manager/activities/activities.py index ec5ae46..1fe2d6f 100644 --- a/model-manager/activities/activities.py +++ b/model_manager/activities/activities.py @@ -4,9 +4,9 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.temporal.activities.postgres import Postgres from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import Logger - from laborious.activities.mlflow import MLFlow - from laborious.activities.gates import Gates - from laborious.activities.opc import OPC + from model_manager.activities.mlflow import MLFlow + from model_manager.activities.gates import Gates + from model_manager.activities.opc import OPC from typing import Any diff --git a/model-manager/activities/gates.py b/model_manager/activities/gates.py similarity index 99% rename from model-manager/activities/gates.py rename to model_manager/activities/gates.py index 4a71455..ec5e806 100644 --- a/model-manager/activities/gates.py +++ b/model_manager/activities/gates.py @@ -9,14 +9,14 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.logger import Logger from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now from sientia_do.formatters import create_sample_dict - from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter - from typing import Any - from laborious.utils.filters.conditional_filters import ( + from model_manager.utils.filters.mlflow_filters import nan_values_filter, api_error_filter + + from model_manager.utils.filters.conditional_filters import ( filter_empty_data, filter_specific_variables_null_values ) from pandas import DataFrame - from laborious import metrics + from model_manager import metrics # Input filter function mappings input_filter_functions = { diff --git a/model-manager/activities/mlflow.py b/model_manager/activities/mlflow.py similarity index 99% rename from model-manager/activities/mlflow.py rename to model_manager/activities/mlflow.py index bd06283..e55a55e 100644 --- a/model-manager/activities/mlflow.py +++ b/model_manager/activities/mlflow.py @@ -10,7 +10,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger from sientia_do.formatters import create_sample_dict - from laborious.utils.repository.model_repository import MLFlowRepository + from model_manager.utils.repository.model_repository import MLFlowRepository from typing import Any import numpy as np from pandas import DataFrame diff --git a/model-manager/activities/opc.py b/model_manager/activities/opc.py similarity index 99% rename from model-manager/activities/opc.py rename to model_manager/activities/opc.py index 0b4ec9f..2222eed 100644 --- a/model-manager/activities/opc.py +++ b/model_manager/activities/opc.py @@ -6,7 +6,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.base import BaseActivity from sientia_do.observability.logger import Logger - from laborious.utils.repository.opc_repository import OpcRepository + from model_manager.utils.repository.opc_repository import OpcRepository from typing import Any import traceback from pandas import DataFrame diff --git a/model-manager/metrics.py b/model_manager/metrics.py similarity index 100% rename from model-manager/metrics.py rename to model_manager/metrics.py diff --git a/model-manager/utils/__init__.py b/model_manager/utils/__init__.py similarity index 100% rename from model-manager/utils/__init__.py rename to model_manager/utils/__init__.py diff --git a/model-manager/utils/connectors_config.py b/model_manager/utils/connectors_config.py similarity index 100% rename from model-manager/utils/connectors_config.py rename to model_manager/utils/connectors_config.py diff --git a/model-manager/utils/filters/__init__.py b/model_manager/utils/filters/__init__.py similarity index 100% rename from model-manager/utils/filters/__init__.py rename to model_manager/utils/filters/__init__.py diff --git a/model-manager/utils/filters/conditional_filters.py b/model_manager/utils/filters/conditional_filters.py similarity index 100% rename from model-manager/utils/filters/conditional_filters.py rename to model_manager/utils/filters/conditional_filters.py diff --git a/model-manager/utils/filters/mlflow_filters.py b/model_manager/utils/filters/mlflow_filters.py similarity index 100% rename from model-manager/utils/filters/mlflow_filters.py rename to model_manager/utils/filters/mlflow_filters.py diff --git a/model-manager/utils/repository/model_repository.py b/model_manager/utils/repository/model_repository.py similarity index 100% rename from model-manager/utils/repository/model_repository.py rename to model_manager/utils/repository/model_repository.py diff --git a/model-manager/utils/repository/opc_repository.py b/model_manager/utils/repository/opc_repository.py similarity index 99% rename from model-manager/utils/repository/opc_repository.py rename to model_manager/utils/repository/opc_repository.py index 96ecfef..94b6c46 100644 --- a/model-manager/utils/repository/opc_repository.py +++ b/model_manager/utils/repository/opc_repository.py @@ -11,7 +11,7 @@ from regex import F from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger -from laborious import metrics +from model_manager import metrics data_type_map = { 'float': { diff --git a/model-manager/worker/__init__.py b/model_manager/worker/__init__.py similarity index 100% rename from model-manager/worker/__init__.py rename to model_manager/worker/__init__.py diff --git a/model-manager/worker/worker.py b/model_manager/worker/worker.py similarity index 94% rename from model-manager/worker/worker.py rename to model_manager/worker/worker.py index 567a257..a10c4cb 100644 --- a/model-manager/worker/worker.py +++ b/model_manager/worker/worker.py @@ -33,13 +33,13 @@ with workflow.unsafe.imports_passed_through(): import os import sys import asyncio - from laborious.workflows.minimal_retrain import MinimalRetrain - from laborious.workflows.predictions_batch import PredictionsBatch - from laborious.workflows.sub_workflows.prediction_process import PredictionProcess - from laborious.workflows.sub_workflows.format_and_export_prediction import \ + from model_manager.workflows.minimal_retrain import MinimalRetrain + from model_manager.workflows.predictions_batch import PredictionsBatch + from model_manager.workflows.sub_workflows.prediction_process import PredictionProcess + from model_manager.workflows.sub_workflows.format_and_export_prediction import \ FormatAndExportPrediction - from laborious.activities.activities import Activities - from laborious.utils.connectors_config import ( + from model_manager.activities.activities import Activities + from model_manager.utils.connectors_config import ( build_postgres_config, build_mlflow_config, build_opc_config, @@ -47,7 +47,7 @@ with workflow.unsafe.imports_passed_through(): ) from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import get_logger - from laborious import metrics + from model_manager import metrics from prometheus_client import start_http_server POD_ID = os.getenv('POD_ID') diff --git a/model-manager/workflows/__init__.py b/model_manager/workflows/__init__.py similarity index 100% rename from model-manager/workflows/__init__.py rename to model_manager/workflows/__init__.py diff --git a/model-manager/workflows/minimal_retrain.py b/model_manager/workflows/minimal_retrain.py similarity index 98% rename from model-manager/workflows/minimal_retrain.py rename to model_manager/workflows/minimal_retrain.py index 1893e25..4701c91 100644 --- a/model-manager/workflows/minimal_retrain.py +++ b/model_manager/workflows/minimal_retrain.py @@ -1,7 +1,7 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities + from model_manager.activities.activities import Activities from typing import Any from sientia_do.temporal.policies import retry_policy from datetime import timedelta diff --git a/model-manager/workflows/predictions_batch.py b/model_manager/workflows/predictions_batch.py similarity index 98% rename from model-manager/workflows/predictions_batch.py rename to model_manager/workflows/predictions_batch.py index e522eaa..330493d 100644 --- a/model-manager/workflows/predictions_batch.py +++ b/model_manager/workflows/predictions_batch.py @@ -1,7 +1,7 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities + from model_manager.activities.activities import Activities from typing import Any from sientia_do.temporal.policies import retry_policy from datetime import timedelta diff --git a/model-manager/workflows/sub_workflows/__init__.py b/model_manager/workflows/sub_workflows/__init__.py similarity index 100% rename from model-manager/workflows/sub_workflows/__init__.py rename to model_manager/workflows/sub_workflows/__init__.py diff --git a/model-manager/workflows/sub_workflows/format_and_export_prediction.py b/model_manager/workflows/sub_workflows/format_and_export_prediction.py similarity index 98% rename from model-manager/workflows/sub_workflows/format_and_export_prediction.py rename to model_manager/workflows/sub_workflows/format_and_export_prediction.py index 8e7df07..ef195c4 100644 --- a/model-manager/workflows/sub_workflows/format_and_export_prediction.py +++ b/model_manager/workflows/sub_workflows/format_and_export_prediction.py @@ -1,7 +1,7 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities + from model_manager.activities.activities import Activities from typing import Any from datetime import timedelta from sientia_do.temporal.policies import retry_policy diff --git a/model-manager/workflows/sub_workflows/prediction_process.py b/model_manager/workflows/sub_workflows/prediction_process.py similarity index 99% rename from model-manager/workflows/sub_workflows/prediction_process.py rename to model_manager/workflows/sub_workflows/prediction_process.py index 777fa1c..4addd75 100644 --- a/model-manager/workflows/sub_workflows/prediction_process.py +++ b/model_manager/workflows/sub_workflows/prediction_process.py @@ -1,7 +1,7 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities + from model_manager.activities.activities import Activities from typing import Any from sientia_do.temporal.policies import retry_policy from datetime import timedelta diff --git a/run_coverage.sh b/run_coverage.sh index f9af4cb..7d4f6f4 100755 --- a/run_coverage.sh +++ b/run_coverage.sh @@ -6,6 +6,6 @@ set -e echo "Activating virtual environment..." source ./venv/bin/activate -pytest --cov=laborious --cov-report=html +pytest --cov=model_manager --cov-report=html xdg-open htmlcov/index.html \ No newline at end of file diff --git a/run_local.sh b/run_local.sh index 2bbd5c2..680ff15 100755 --- a/run_local.sh +++ b/run_local.sh @@ -15,4 +15,4 @@ else fi echo "Starting ingestor application..." -python -m laborious.worker.worker +python -m model_manager.worker.worker diff --git a/sonar-project.properties b/sonar-project.properties index 0edb732..e8ae7ff 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,6 +1,6 @@ sonar.projectKey=Aignosi_sientia-dataops-model-manager_dc6e051c-995a-4b57-9cde-f511018184c7 sonar.projectName=sientia-dataops-model-manager -sonar.sources=model-manager +sonar.sources=model_manager sonar.tests=tests sonar.qualitygate.wait=true sonar.qualitygate.timeout=300 diff --git a/tests/laborious/activities/test_activities.py b/tests/laborious/activities/test_activities.py index de418be..106ccf7 100644 --- a/tests/laborious/activities/test_activities.py +++ b/tests/laborious/activities/test_activities.py @@ -1,16 +1,16 @@ from pytest import mark from unittest.mock import patch, MagicMock, ANY from sientia_do.temporal.activities.postgres import Postgres -from laborious.activities.activities import Activities -from laborious.activities.mlflow import MLFlow -from laborious.activities.gates import Gates -from laborious.activities.opc import OPC +from model_manager.activities.activities import Activities +from model_manager.activities.mlflow import MLFlow +from model_manager.activities.gates import Gates +from model_manager.activities.opc import OPC -@patch('laborious.activities.activities.Postgres.__init__') -@patch('laborious.activities.activities.MLFlow.__init__') -@patch('laborious.activities.activities.OPC.__init__') -@patch('laborious.activities.activities.Gates.__init__') +@patch('model_manager.activities.activities.Postgres.__init__') +@patch('model_manager.activities.activities.MLFlow.__init__') +@patch('model_manager.activities.activities.OPC.__init__') +@patch('model_manager.activities.activities.Gates.__init__') def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgres_init): postgres_config = { @@ -91,9 +91,9 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre @mark.asyncio -@patch('laborious.activities.activities.Postgres', return_value=MagicMock()) -@patch('laborious.activities.activities.MLFlow', return_value=MagicMock()) -@patch('laborious.activities.activities.OPC', return_value=MagicMock()) +@patch('model_manager.activities.activities.Postgres', return_value=MagicMock()) +@patch('model_manager.activities.activities.MLFlow', return_value=MagicMock()) +@patch('model_manager.activities.activities.OPC', return_value=MagicMock()) async def test_shutdown(mock_opc_init, _mock_mlflow_init, mock_postgres_init): postgres_config = { diff --git a/tests/laborious/activities/test_gates.py b/tests/laborious/activities/test_gates.py index a65c6d4..b587b93 100644 --- a/tests/laborious/activities/test_gates.py +++ b/tests/laborious/activities/test_gates.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock, ANY, patch from pytest import fixture, mark from sientia_do.notifications.models import NotificationLevel -from laborious.activities.gates import Gates +from model_manager.activities.gates import Gates @fixture @@ -52,7 +52,7 @@ async def test_input_gate_invalid_filter(gates_activity): @mark.asyncio -@patch('laborious.activities.gates.input_filter_functions') +@patch('model_manager.activities.gates.input_filter_functions') async def test_input_gate_filter_exception(mock_input_filter_functions, gates_activity): # Arrange mock_input_filter_functions.__contains__.return_value = True @@ -141,7 +141,7 @@ async def test_mlflow_response_gate_invalid_filter(gates_activity): @mark.asyncio -@patch('laborious.activities.gates.mlflow_response_filter_functions') +@patch('model_manager.activities.gates.mlflow_response_filter_functions') async def test_mlflow_response_gate_filter_exception(mock_mlflow_response_filter_functions, gates_activity): # Arrange @@ -241,7 +241,7 @@ async def test_mlflow_content_gate_invalid_filter(gates_activity): @mark.asyncio -@patch('laborious.activities.gates.mlflow_content_filter_functions') +@patch('model_manager.activities.gates.mlflow_content_filter_functions') async def test_mlflow_content_gate_filter_exception(mock_mlflow_content_filter_functions, gates_activity): # Arrange @@ -559,7 +559,7 @@ async def test_get_last_timestamp_no_data(gates_activity): @mark.asyncio -@patch('laborious.activities.gates.metrics') +@patch('model_manager.activities.gates.metrics') async def test_write_metrics(mock_metrics, gates_activity): """Test write_metrics method.""" input_data = { diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index 85cb7f1..243982f 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -5,11 +5,11 @@ import numpy as np from pandas import DataFrame, Timestamp from pytest import fixture, mark, raises from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ -from laborious.activities.mlflow import MLFlow +from model_manager.activities.mlflow import MLFlow from sientia_do.notifications.models import NotificationLevel -@patch("laborious.activities.mlflow.MLFlowRepository") +@patch("model_manager.activities.mlflow.MLFlowRepository") def test___init__(mock_mlflow_repository): mlflow = MLFlow( mlflow_host="http://localhost", @@ -31,7 +31,7 @@ def test___init__(mock_mlflow_repository): @fixture -@patch("laborious.activities.mlflow.MLFlowRepository") +@patch("model_manager.activities.mlflow.MLFlowRepository") def mlflow(mock_mlflow_repository): mlflow = MLFlow( mlflow_host="http://localhost:5000", @@ -58,8 +58,8 @@ metadata = { @mark.asyncio -@patch("laborious.activities.mlflow.DataFrame") -@patch("laborious.activities.mlflow.max") +@patch("model_manager.activities.mlflow.DataFrame") +@patch("model_manager.activities.mlflow.max") async def test_request_transform_success(mock_max, mock_dataframe, mlflow): mock_max.return_value = '2024-01-02' # Mock input data @@ -114,9 +114,9 @@ async def test_request_transform_success(mock_max, mock_dataframe, mlflow): @mark.asyncio -@patch("laborious.activities.mlflow.DataFrame") -@patch("laborious.activities.mlflow.to_datetime") -@patch("laborious.activities.mlflow.max") +@patch("model_manager.activities.mlflow.DataFrame") +@patch("model_manager.activities.mlflow.to_datetime") +@patch("model_manager.activities.mlflow.max") async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflow): mock_max.return_value = '2024-01-02' # Mock input data diff --git a/tests/laborious/activities/test_opc.py b/tests/laborious/activities/test_opc.py index 07f00b8..dcff019 100644 --- a/tests/laborious/activities/test_opc.py +++ b/tests/laborious/activities/test_opc.py @@ -4,7 +4,7 @@ from pytest import fixture, mark import pytest_asyncio from sientia_do.notifications.models import NotificationLevel -from laborious.activities.opc import OPC +from model_manager.activities.opc import OPC metadata = { "metadata": { @@ -31,8 +31,8 @@ def test__init__(): @mark.asyncio -@patch("laborious.activities.opc.OpcRepository") -@patch("laborious.activities.opc.OPC.send_notification") +@patch("model_manager.activities.opc.OpcRepository") +@patch("model_manager.activities.opc.OPC.send_notification") async def test_init_opc(mock_send_notification, mock_opc_repository): mock_logger = MagicMock() server1 = MagicMock( @@ -147,7 +147,7 @@ async def test_init_opc(mock_send_notification, mock_opc_repository): @pytest_asyncio.fixture -@patch("laborious.activities.opc.OpcRepository") +@patch("model_manager.activities.opc.OpcRepository") async def opc(mock_opc_repository): servers = { 'server1': { diff --git a/tests/laborious/utils/filters/test_conditional_filters.py b/tests/laborious/utils/filters/test_conditional_filters.py index 405bc9b..a7f3efd 100644 --- a/tests/laborious/utils/filters/test_conditional_filters.py +++ b/tests/laborious/utils/filters/test_conditional_filters.py @@ -1,6 +1,6 @@ from pandas import DataFrame -from laborious.utils.filters.conditional_filters import ( +from model_manager.utils.filters.conditional_filters import ( filter_specific_variables_null_values, filter_empty_data ) diff --git a/tests/laborious/utils/filters/test_mlflow_filters.py b/tests/laborious/utils/filters/test_mlflow_filters.py index f9c61e9..6fddf51 100644 --- a/tests/laborious/utils/filters/test_mlflow_filters.py +++ b/tests/laborious/utils/filters/test_mlflow_filters.py @@ -1,5 +1,5 @@ from pandas import DataFrame -from laborious.utils.filters.mlflow_filters import api_error_filter, nan_values_filter +from model_manager.utils.filters.mlflow_filters import api_error_filter, nan_values_filter def test_api_error_filter_invalid_response(): diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index cdc59b4..ab65913 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -4,12 +4,12 @@ from pandas import DataFrame import pytest from datetime import datetime, timezone from pandas import Timestamp -from laborious.utils.repository.model_repository import MLFlowRepository +from model_manager.utils.repository.model_repository import MLFlowRepository @pytest.fixture def mlflow_repository(): - with patch('laborious.utils.repository.model_repository.ModelServing', + with patch('model_manager.utils.repository.model_repository.ModelServing', autospec=True) as mock_model_serving: mock_instance = mock_model_serving.return_value mock_instance.get_transformed_data = MagicMock() @@ -217,7 +217,7 @@ def test_predict_error(mlflow_repository): } -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_experiment_by_run_id(mlflow, mlflow_repository): mlflow.get_run.return_value = MagicMock( info=MagicMock( @@ -233,7 +233,7 @@ def test_get_experiment_by_run_id(mlflow, mlflow_repository): mlflow.get_experiment.assert_called_once_with('0') -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_next_run_name(mlflow, mlflow_repository): mlflow.search_runs.return_value = [1, 2, 3] output = mlflow_repository.get_next_run_name('run') @@ -244,7 +244,7 @@ def test_get_next_run_name(mlflow, mlflow_repository): ) -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_experiment_success(mlflow, mlflow_repository): mlflow.get_experiment_by_name.return_value = MagicMock( experiment_id='0') @@ -254,7 +254,7 @@ def test_get_experiment_success(mlflow, mlflow_repository): assert output == 0 -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_experiment_error(mlflow, mlflow_repository): mlflow.get_experiment_by_name.return_value = None @@ -266,7 +266,7 @@ def test_get_experiment_error(mlflow, mlflow_repository): assert False -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_experiment_last_run(mlflow, mlflow_repository): mlflow.search_runs.return_value = DataFrame({ 'params.retrain': ['True', 'False', 'True', 'False'], @@ -285,7 +285,7 @@ def test_get_experiment_last_run(mlflow, mlflow_repository): assert output == '2' -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_get_experiment_last_run_error(mlflow, mlflow_repository): mlflow.search_runs.return_value = [] @@ -297,8 +297,8 @@ def test_get_experiment_last_run_error(mlflow, mlflow_repository): assert False -@patch('laborious.utils.repository.model_repository.mlflow.sklearn') -@patch('laborious.utils.repository.model_repository.mlflow.set_experiment') +@patch('model_manager.utils.repository.model_repository.mlflow.sklearn') +@patch('model_manager.utils.repository.model_repository.mlflow.set_experiment') def test_create_model_experiment(set_experiment, sklearn, mlflow_repository): mlflow_repository.model_serving.get_model_run_id = MagicMock( @@ -360,10 +360,10 @@ def test_create_model_experiment(set_experiment, sklearn, mlflow_repository): mlflow_repository.get_experiment_by_run_id.return_value) -@patch('laborious.utils.repository.model_repository.mlflow.start_run') -@patch('laborious.utils.repository.model_repository.mlflow.log_param') -@patch('laborious.utils.repository.model_repository.mlflow.sklearn.log_model') -@patch('laborious.utils.repository.model_repository.mlflow.log_artifact') +@patch('model_manager.utils.repository.model_repository.mlflow.start_run') +@patch('model_manager.utils.repository.model_repository.mlflow.log_param') +@patch('model_manager.utils.repository.model_repository.mlflow.sklearn.log_model') +@patch('model_manager.utils.repository.model_repository.mlflow.log_artifact') def test_perform_model_retrain(log_artifact, log_model, log_param, start_run, mlflow_repository): prediction_model_mock = MagicMock() @@ -423,7 +423,7 @@ def test_retrain_model(mlflow_repository): assert output == 'Model retrained successfully' -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_update_production_model_by_run_id(mlflow, mlflow_repository): client_mock = MagicMock() mlflow.tracking.MlflowClient.return_value = client_mock @@ -458,7 +458,7 @@ def test_update_production_model_by_run_id(mlflow, mlflow_repository): } -@patch('laborious.utils.repository.model_repository.mlflow') +@patch('model_manager.utils.repository.model_repository.mlflow') def test_update_production_model_by_run_id_error(mlflow, mlflow_repository): mlflow.tracking.MlflowClient.return_value = MagicMock( get_registered_model=MagicMock( diff --git a/tests/laborious/utils/repository/test_opc_repository.py b/tests/laborious/utils/repository/test_opc_repository.py index bc84db8..9795a10 100644 --- a/tests/laborious/utils/repository/test_opc_repository.py +++ b/tests/laborious/utils/repository/test_opc_repository.py @@ -1,7 +1,7 @@ import pytest from unittest.mock import AsyncMock, Mock, patch, MagicMock, ANY, call from asyncua.crypto.security_policies import SecurityPolicyBasic256 -from laborious.utils.repository.opc_repository import OpcRepository +from model_manager.utils.repository.opc_repository import OpcRepository from sientia_do.notifications.models import NotificationLevel from datetime import datetime @@ -28,7 +28,7 @@ def opc_repository(mock_logger): @pytest.fixture def mock_client(): - with patch('laborious.utils.repository.opc_repository.Client') as mock: + with patch('model_manager.utils.repository.opc_repository.Client') as mock: client_instance = AsyncMock() mock.return_value = client_instance yield client_instance @@ -212,7 +212,7 @@ async def test_validate_connection_error_validate_connection_error(opc_repositor @pytest.mark.asyncio -@patch('laborious.utils.repository.opc_repository.datetime') +@patch('model_manager.utils.repository.opc_repository.datetime') async def test_validate_connection_lost_not_time_to_reconnect(_mock_datetime, opc_repository): _mock_datetime.now = MagicMock( return_value=datetime(2025, 1, 1, 0, 0, 0)) @@ -233,7 +233,7 @@ async def test_validate_connection_lost_not_time_to_reconnect(_mock_datetime, op @pytest.mark.asyncio -@patch('laborious.utils.repository.opc_repository.datetime') +@patch('model_manager.utils.repository.opc_repository.datetime') async def test_validate_connection_lost_time_to_reconnect(mock_datetime, opc_repository): mock_datetime.now = MagicMock( return_value=datetime(2025, 1, 1, 1, 0, 0)) @@ -333,7 +333,7 @@ async def test_write_data_invalid_data_type(opc_repository, mock_client): @pytest.mark.asyncio -@patch('laborious.utils.repository.opc_repository.metrics') +@patch('model_manager.utils.repository.opc_repository.metrics') async def test_write_data(mock_metrics, opc_repository, mock_client): opc_repository.validate_connection = AsyncMock(return_value=(True, {})) opc_repository.client = mock_client diff --git a/tests/laborious/utils/test_connectors_config.py b/tests/laborious/utils/test_connectors_config.py index 910439c..43a2ec9 100644 --- a/tests/laborious/utils/test_connectors_config.py +++ b/tests/laborious/utils/test_connectors_config.py @@ -1,5 +1,5 @@ from os import environ -from laborious.utils.connectors_config import (build_mlflow_config, +from model_manager.utils.connectors_config import (build_mlflow_config, build_opc_config, build_postgres_config, build_mongodb_config) diff --git a/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py b/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py index a8e6e20..356c32b 100644 --- a/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py +++ b/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py @@ -1,8 +1,8 @@ from unittest.mock import call, patch, AsyncMock, ANY from pytest import mark, fixture -from laborious.activities.activities import Activities -from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction +from model_manager.activities.activities import Activities +from model_manager.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ @@ -22,7 +22,7 @@ metadata = { @mark.asyncio -@patch("laborious.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock) async def test_run_none_path_flag(workflow_mock, format_and_export_prediction): input_data = { @@ -91,7 +91,7 @@ async def test_run_none_path_flag(workflow_mock, format_and_export_prediction): @mark.asyncio -@patch("laborious.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock) async def test_run_default_path_flag(workflow_mock, format_and_export_prediction): input_data = { diff --git a/tests/laborious/workflows/subworkflows/test_prediction_process.py b/tests/laborious/workflows/subworkflows/test_prediction_process.py index df60ada..46b1565 100644 --- a/tests/laborious/workflows/subworkflows/test_prediction_process.py +++ b/tests/laborious/workflows/subworkflows/test_prediction_process.py @@ -1,7 +1,7 @@ from unittest.mock import AsyncMock, patch, call, ANY from pytest import fixture, mark -from laborious.activities.activities import Activities -from laborious.workflows.sub_workflows.prediction_process import PredictionProcess +from model_manager.activities.activities import Activities +from model_manager.workflows.sub_workflows.prediction_process import PredictionProcess @fixture @@ -20,7 +20,7 @@ metadata = { @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_run(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock(return_value=False) # Arrange @@ -135,7 +135,7 @@ async def test_run(workflow_mock, prediction_process): @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_run_stop_at_input_gate(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock(return_value=True) # Arrange @@ -183,7 +183,7 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process): @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock(side_effect=[False, True]) # Arrange @@ -253,7 +253,7 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock( side_effect=[False, False, True]) @@ -333,7 +333,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock( side_effect=[False, False, False, True]) @@ -429,7 +429,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_path_flag_handler_stop(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} @@ -464,7 +464,7 @@ async def test_path_flag_handler_stop(workflow_mock, prediction_process): @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_path_flag_handler_repeat(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} @@ -510,7 +510,7 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process): @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_path_flag_handler_continue(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} @@ -565,7 +565,7 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) async def test_path_flag_handler_unknown(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} diff --git a/tests/laborious/workflows/test_minimal_retrain.py b/tests/laborious/workflows/test_minimal_retrain.py index b3b03b5..1d8e6d7 100644 --- a/tests/laborious/workflows/test_minimal_retrain.py +++ b/tests/laborious/workflows/test_minimal_retrain.py @@ -1,7 +1,7 @@ from unittest.mock import AsyncMock, MagicMock, call, patch, ANY from pytest import fixture, mark -from laborious.activities.activities import Activities -from laborious.workflows.minimal_retrain import MinimalRetrain +from model_manager.activities.activities import Activities +from model_manager.workflows.minimal_retrain import MinimalRetrain @fixture @@ -20,7 +20,7 @@ metadata = { @mark.asyncio -@patch('laborious.workflows.minimal_retrain.workflow', new_callable=AsyncMock) +@patch('model_manager.workflows.minimal_retrain.workflow', new_callable=AsyncMock) async def test_run(workflow_mock: AsyncMock, minimal_retrain: MinimalRetrain): input_data = { "model_id": "test_model_id", diff --git a/tests/laborious/workflows/test_predictions_batch.py b/tests/laborious/workflows/test_predictions_batch.py index 90d7d21..0d7d477 100644 --- a/tests/laborious/workflows/test_predictions_batch.py +++ b/tests/laborious/workflows/test_predictions_batch.py @@ -1,7 +1,7 @@ from unittest.mock import AsyncMock, call, patch, ANY from pytest import fixture, mark -from laborious.activities.activities import Activities -from laborious.workflows.predictions_batch import PredictionsBatch +from model_manager.activities.activities import Activities +from model_manager.workflows.predictions_batch import PredictionsBatch @fixture @@ -20,7 +20,7 @@ metadata = { @mark.asyncio -@patch('laborious.workflows.predictions_batch.workflow', new_callable=AsyncMock) +@patch('model_manager.workflows.predictions_batch.workflow', new_callable=AsyncMock) async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch): workflow_mock.execute_local_activity_method.return_value = { 'data': 'test_data' diff --git a/values.yaml b/values.yaml index 4eb83d7..3af8b24 100644 --- a/values.yaml +++ b/values.yaml @@ -70,7 +70,7 @@ livenessProbe: command: - sh - -c - - pgrep -f "laborious.worker.worker" + - pgrep -f "model_manager.worker.worker" initialDelaySeconds: 20 periodSeconds: 30 @@ -79,7 +79,7 @@ readinessProbe: command: - sh - -c - - pgrep -f "laborious.worker.worker" + - pgrep -f "model_manager.worker.worker" initialDelaySeconds: 10 periodSeconds: 15 @@ -153,7 +153,7 @@ env: - name: GITHUB_BRANCH value: SIENTIAPDE-1222-ajustar-a-library-para-fazer-o-download-do-courier - name: PYTHON_APP - value: "laborious.worker.worker" + value: "model_manager.worker.worker" # Application variables - name: POSTGRES_HOST