SIENTIAPDE-1243: Refactor: Rename 'laborious' package to 'model_manager'
This commit renames the 'laborious' package to 'model_manager' across the entire project. This includes renaming directories, modules, references in code, configuration files, and documentation to reflect the new package name. This change improves clarity and consistency within the project.
This commit is contained in:
2
.github/workflows/quality-gate.yml
vendored
2
.github/workflows/quality-gate.yml
vendored
@@ -197,7 +197,7 @@ jobs:
|
||||
|
||||
- name: 🧪 Run Tests with Pytest
|
||||
run: |
|
||||
pytest tests --junitxml=pytest.xml --cov=model-manager --cov-report=xml --cov-report=term
|
||||
pytest tests --junitxml=pytest.xml --cov=model_manager --cov-report=xml --cov-report=term
|
||||
|
||||
- name: Run SonarQube Analysis
|
||||
uses: SonarSource/sonarqube-scan-action@v5
|
||||
|
||||
16
README.md
16
README.md
@@ -52,7 +52,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa
|
||||
|
||||
### Key Components
|
||||
|
||||
#### **Worker (`laborious/worker/worker.py`)**
|
||||
#### **Worker (`model_manager/worker/worker.py`)**
|
||||
- **Purpose**: Main application orchestrator managing Temporal workers and task queues
|
||||
- **Responsibilities**:
|
||||
- Temporal client initialization and connection management
|
||||
@@ -68,7 +68,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa
|
||||
- Multi-instance deployment support
|
||||
- Two dedicated task queues: `predictions_batch-queue` and `minimal_retrain-queue`
|
||||
|
||||
#### **Workflows (`laborious/workflows/`)**
|
||||
#### **Workflows (`model_manager/workflows/`)**
|
||||
- **PredictionsBatch**: Main entry point for batch prediction pipelines
|
||||
- **PredictionProcess**: Core prediction pipeline with MLFlow integration
|
||||
- **FormatAndExportPrediction**: Data formatting and export operations
|
||||
@@ -79,7 +79,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa
|
||||
- Comprehensive error handling and recovery
|
||||
- Configurable timeout and retry strategies
|
||||
|
||||
#### **Activities (`laborious/activities/`)**
|
||||
#### **Activities (`model_manager/activities/`)**
|
||||
- **Activities**: Main activity orchestrator combining all functionality through multiple inheritance
|
||||
- **Gates**: Data quality validation and filtering mechanisms
|
||||
- **MLFlow**: Model transformation and prediction operations
|
||||
@@ -92,7 +92,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa
|
||||
- Comprehensive error handling and notification integration
|
||||
- Support for multiple OPC servers with independent configurations
|
||||
|
||||
#### **Data Services (`laborious/utils/`)**
|
||||
#### **Data Services (`model_manager/utils/`)**
|
||||
- **Connectors Config**: Environment variable-based configuration management
|
||||
- **Repository**: Data access layer for MLFlow and OPC operations
|
||||
- `model_repository.py`: MLFlow model operations and retraining
|
||||
@@ -475,7 +475,7 @@ source ./venv/bin/activate
|
||||
pytest
|
||||
|
||||
# Run with coverage
|
||||
pytest --cov=laborious --cov-report=html
|
||||
pytest --cov=model_manager --cov-report=html
|
||||
|
||||
# Run specific test categories
|
||||
pytest tests/activities/
|
||||
@@ -496,7 +496,7 @@ if [ -f .env ]; then
|
||||
fi
|
||||
|
||||
# Start the laborious worker
|
||||
python -m laborious.worker.worker
|
||||
python -m model_manager.worker.worker
|
||||
```
|
||||
|
||||
## 🧪 Testing
|
||||
@@ -516,7 +516,7 @@ tests/
|
||||
pip install pytest pytest-cov pytest-asyncio
|
||||
|
||||
# Run tests with coverage
|
||||
pytest --cov=laborious --cov-report=html
|
||||
pytest --cov=model_manager --cov-report=html
|
||||
|
||||
# Run specific test modules
|
||||
pytest tests/activities/test_gates.py
|
||||
@@ -722,7 +722,7 @@ This is the configuration created by the Orchestrator in Temporal.
|
||||
|
||||
### Project Structure
|
||||
```
|
||||
laborious/
|
||||
model_manager/
|
||||
├── activities/ # Temporal activity implementations
|
||||
│ ├── activities.py # Main activities orchestrator
|
||||
│ ├── gates.py # Data quality gates and filtering
|
||||
|
||||
@@ -4,9 +4,9 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.temporal.activities.postgres import Postgres
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.observability.logger import Logger
|
||||
from laborious.activities.mlflow import MLFlow
|
||||
from laborious.activities.gates import Gates
|
||||
from laborious.activities.opc import OPC
|
||||
from model_manager.activities.mlflow import MLFlow
|
||||
from model_manager.activities.gates import Gates
|
||||
from model_manager.activities.opc import OPC
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -9,14 +9,14 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now
|
||||
from sientia_do.formatters import create_sample_dict
|
||||
from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter
|
||||
from typing import Any
|
||||
from laborious.utils.filters.conditional_filters import (
|
||||
from model_manager.utils.filters.mlflow_filters import nan_values_filter, api_error_filter
|
||||
|
||||
from model_manager.utils.filters.conditional_filters import (
|
||||
filter_empty_data,
|
||||
filter_specific_variables_null_values
|
||||
)
|
||||
from pandas import DataFrame
|
||||
from laborious import metrics
|
||||
from model_manager import metrics
|
||||
|
||||
# Input filter function mappings
|
||||
input_filter_functions = {
|
||||
@@ -10,7 +10,7 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.formatters import create_sample_dict
|
||||
from laborious.utils.repository.model_repository import MLFlowRepository
|
||||
from model_manager.utils.repository.model_repository import MLFlowRepository
|
||||
from typing import Any
|
||||
import numpy as np
|
||||
from pandas import DataFrame
|
||||
@@ -6,7 +6,7 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from sientia_do.observability.logger import Logger
|
||||
from laborious.utils.repository.opc_repository import OpcRepository
|
||||
from model_manager.utils.repository.opc_repository import OpcRepository
|
||||
from typing import Any
|
||||
import traceback
|
||||
from pandas import DataFrame
|
||||
@@ -11,7 +11,7 @@ from regex import F
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from laborious import metrics
|
||||
from model_manager import metrics
|
||||
|
||||
data_type_map = {
|
||||
'float': {
|
||||
@@ -33,13 +33,13 @@ with workflow.unsafe.imports_passed_through():
|
||||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
from laborious.workflows.minimal_retrain import MinimalRetrain
|
||||
from laborious.workflows.predictions_batch import PredictionsBatch
|
||||
from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
|
||||
from laborious.workflows.sub_workflows.format_and_export_prediction import \
|
||||
from model_manager.workflows.minimal_retrain import MinimalRetrain
|
||||
from model_manager.workflows.predictions_batch import PredictionsBatch
|
||||
from model_manager.workflows.sub_workflows.prediction_process import PredictionProcess
|
||||
from model_manager.workflows.sub_workflows.format_and_export_prediction import \
|
||||
FormatAndExportPrediction
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.utils.connectors_config import (
|
||||
from model_manager.activities.activities import Activities
|
||||
from model_manager.utils.connectors_config import (
|
||||
build_postgres_config,
|
||||
build_mlflow_config,
|
||||
build_opc_config,
|
||||
@@ -47,7 +47,7 @@ with workflow.unsafe.imports_passed_through():
|
||||
)
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.observability.logger import get_logger
|
||||
from laborious import metrics
|
||||
from model_manager import metrics
|
||||
from prometheus_client import start_http_server
|
||||
|
||||
POD_ID = os.getenv('POD_ID')
|
||||
@@ -1,7 +1,7 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from laborious.activities.activities import Activities
|
||||
from model_manager.activities.activities import Activities
|
||||
from typing import Any
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from datetime import timedelta
|
||||
@@ -1,7 +1,7 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from laborious.activities.activities import Activities
|
||||
from model_manager.activities.activities import Activities
|
||||
from typing import Any
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from datetime import timedelta
|
||||
@@ -1,7 +1,7 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from laborious.activities.activities import Activities
|
||||
from model_manager.activities.activities import Activities
|
||||
from typing import Any
|
||||
from datetime import timedelta
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
@@ -1,7 +1,7 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from laborious.activities.activities import Activities
|
||||
from model_manager.activities.activities import Activities
|
||||
from typing import Any
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from datetime import timedelta
|
||||
@@ -6,6 +6,6 @@ set -e
|
||||
echo "Activating virtual environment..."
|
||||
source ./venv/bin/activate
|
||||
|
||||
pytest --cov=laborious --cov-report=html
|
||||
pytest --cov=model_manager --cov-report=html
|
||||
|
||||
xdg-open htmlcov/index.html
|
||||
@@ -15,4 +15,4 @@ else
|
||||
fi
|
||||
|
||||
echo "Starting ingestor application..."
|
||||
python -m laborious.worker.worker
|
||||
python -m model_manager.worker.worker
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
sonar.projectKey=Aignosi_sientia-dataops-model-manager_dc6e051c-995a-4b57-9cde-f511018184c7
|
||||
sonar.projectName=sientia-dataops-model-manager
|
||||
sonar.sources=model-manager
|
||||
sonar.sources=model_manager
|
||||
sonar.tests=tests
|
||||
sonar.qualitygate.wait=true
|
||||
sonar.qualitygate.timeout=300
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
from pytest import mark
|
||||
from unittest.mock import patch, MagicMock, ANY
|
||||
from sientia_do.temporal.activities.postgres import Postgres
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.activities.mlflow import MLFlow
|
||||
from laborious.activities.gates import Gates
|
||||
from laborious.activities.opc import OPC
|
||||
from model_manager.activities.activities import Activities
|
||||
from model_manager.activities.mlflow import MLFlow
|
||||
from model_manager.activities.gates import Gates
|
||||
from model_manager.activities.opc import OPC
|
||||
|
||||
|
||||
@patch('laborious.activities.activities.Postgres.__init__')
|
||||
@patch('laborious.activities.activities.MLFlow.__init__')
|
||||
@patch('laborious.activities.activities.OPC.__init__')
|
||||
@patch('laborious.activities.activities.Gates.__init__')
|
||||
@patch('model_manager.activities.activities.Postgres.__init__')
|
||||
@patch('model_manager.activities.activities.MLFlow.__init__')
|
||||
@patch('model_manager.activities.activities.OPC.__init__')
|
||||
@patch('model_manager.activities.activities.Gates.__init__')
|
||||
def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgres_init):
|
||||
|
||||
postgres_config = {
|
||||
@@ -91,9 +91,9 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('laborious.activities.activities.Postgres', return_value=MagicMock())
|
||||
@patch('laborious.activities.activities.MLFlow', return_value=MagicMock())
|
||||
@patch('laborious.activities.activities.OPC', return_value=MagicMock())
|
||||
@patch('model_manager.activities.activities.Postgres', return_value=MagicMock())
|
||||
@patch('model_manager.activities.activities.MLFlow', return_value=MagicMock())
|
||||
@patch('model_manager.activities.activities.OPC', return_value=MagicMock())
|
||||
async def test_shutdown(mock_opc_init,
|
||||
_mock_mlflow_init, mock_postgres_init):
|
||||
postgres_config = {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from unittest.mock import MagicMock, ANY, patch
|
||||
from pytest import fixture, mark
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from laborious.activities.gates import Gates
|
||||
from model_manager.activities.gates import Gates
|
||||
|
||||
|
||||
@fixture
|
||||
@@ -52,7 +52,7 @@ async def test_input_gate_invalid_filter(gates_activity):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('laborious.activities.gates.input_filter_functions')
|
||||
@patch('model_manager.activities.gates.input_filter_functions')
|
||||
async def test_input_gate_filter_exception(mock_input_filter_functions, gates_activity):
|
||||
# Arrange
|
||||
mock_input_filter_functions.__contains__.return_value = True
|
||||
@@ -141,7 +141,7 @@ async def test_mlflow_response_gate_invalid_filter(gates_activity):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('laborious.activities.gates.mlflow_response_filter_functions')
|
||||
@patch('model_manager.activities.gates.mlflow_response_filter_functions')
|
||||
async def test_mlflow_response_gate_filter_exception(mock_mlflow_response_filter_functions,
|
||||
gates_activity):
|
||||
# Arrange
|
||||
@@ -241,7 +241,7 @@ async def test_mlflow_content_gate_invalid_filter(gates_activity):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('laborious.activities.gates.mlflow_content_filter_functions')
|
||||
@patch('model_manager.activities.gates.mlflow_content_filter_functions')
|
||||
async def test_mlflow_content_gate_filter_exception(mock_mlflow_content_filter_functions,
|
||||
gates_activity):
|
||||
# Arrange
|
||||
@@ -559,7 +559,7 @@ async def test_get_last_timestamp_no_data(gates_activity):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('laborious.activities.gates.metrics')
|
||||
@patch('model_manager.activities.gates.metrics')
|
||||
async def test_write_metrics(mock_metrics, gates_activity):
|
||||
"""Test write_metrics method."""
|
||||
input_data = {
|
||||
|
||||
@@ -5,11 +5,11 @@ import numpy as np
|
||||
from pandas import DataFrame, Timestamp
|
||||
from pytest import fixture, mark, raises
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
|
||||
from laborious.activities.mlflow import MLFlow
|
||||
from model_manager.activities.mlflow import MLFlow
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
|
||||
|
||||
@patch("laborious.activities.mlflow.MLFlowRepository")
|
||||
@patch("model_manager.activities.mlflow.MLFlowRepository")
|
||||
def test___init__(mock_mlflow_repository):
|
||||
mlflow = MLFlow(
|
||||
mlflow_host="http://localhost",
|
||||
@@ -31,7 +31,7 @@ def test___init__(mock_mlflow_repository):
|
||||
|
||||
|
||||
@fixture
|
||||
@patch("laborious.activities.mlflow.MLFlowRepository")
|
||||
@patch("model_manager.activities.mlflow.MLFlowRepository")
|
||||
def mlflow(mock_mlflow_repository):
|
||||
mlflow = MLFlow(
|
||||
mlflow_host="http://localhost:5000",
|
||||
@@ -58,8 +58,8 @@ metadata = {
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.activities.mlflow.DataFrame")
|
||||
@patch("laborious.activities.mlflow.max")
|
||||
@patch("model_manager.activities.mlflow.DataFrame")
|
||||
@patch("model_manager.activities.mlflow.max")
|
||||
async def test_request_transform_success(mock_max, mock_dataframe, mlflow):
|
||||
mock_max.return_value = '2024-01-02'
|
||||
# Mock input data
|
||||
@@ -114,9 +114,9 @@ async def test_request_transform_success(mock_max, mock_dataframe, mlflow):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.activities.mlflow.DataFrame")
|
||||
@patch("laborious.activities.mlflow.to_datetime")
|
||||
@patch("laborious.activities.mlflow.max")
|
||||
@patch("model_manager.activities.mlflow.DataFrame")
|
||||
@patch("model_manager.activities.mlflow.to_datetime")
|
||||
@patch("model_manager.activities.mlflow.max")
|
||||
async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflow):
|
||||
mock_max.return_value = '2024-01-02'
|
||||
# Mock input data
|
||||
|
||||
@@ -4,7 +4,7 @@ from pytest import fixture, mark
|
||||
import pytest_asyncio
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
|
||||
from laborious.activities.opc import OPC
|
||||
from model_manager.activities.opc import OPC
|
||||
|
||||
metadata = {
|
||||
"metadata": {
|
||||
@@ -31,8 +31,8 @@ def test__init__():
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.activities.opc.OpcRepository")
|
||||
@patch("laborious.activities.opc.OPC.send_notification")
|
||||
@patch("model_manager.activities.opc.OpcRepository")
|
||||
@patch("model_manager.activities.opc.OPC.send_notification")
|
||||
async def test_init_opc(mock_send_notification, mock_opc_repository):
|
||||
mock_logger = MagicMock()
|
||||
server1 = MagicMock(
|
||||
@@ -147,7 +147,7 @@ async def test_init_opc(mock_send_notification, mock_opc_repository):
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@patch("laborious.activities.opc.OpcRepository")
|
||||
@patch("model_manager.activities.opc.OpcRepository")
|
||||
async def opc(mock_opc_repository):
|
||||
servers = {
|
||||
'server1': {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from pandas import DataFrame
|
||||
|
||||
from laborious.utils.filters.conditional_filters import (
|
||||
from model_manager.utils.filters.conditional_filters import (
|
||||
filter_specific_variables_null_values,
|
||||
filter_empty_data
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from pandas import DataFrame
|
||||
from laborious.utils.filters.mlflow_filters import api_error_filter, nan_values_filter
|
||||
from model_manager.utils.filters.mlflow_filters import api_error_filter, nan_values_filter
|
||||
|
||||
|
||||
def test_api_error_filter_invalid_response():
|
||||
|
||||
@@ -4,12 +4,12 @@ from pandas import DataFrame
|
||||
import pytest
|
||||
from datetime import datetime, timezone
|
||||
from pandas import Timestamp
|
||||
from laborious.utils.repository.model_repository import MLFlowRepository
|
||||
from model_manager.utils.repository.model_repository import MLFlowRepository
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mlflow_repository():
|
||||
with patch('laborious.utils.repository.model_repository.ModelServing',
|
||||
with patch('model_manager.utils.repository.model_repository.ModelServing',
|
||||
autospec=True) as mock_model_serving:
|
||||
mock_instance = mock_model_serving.return_value
|
||||
mock_instance.get_transformed_data = MagicMock()
|
||||
@@ -217,7 +217,7 @@ def test_predict_error(mlflow_repository):
|
||||
}
|
||||
|
||||
|
||||
@patch('laborious.utils.repository.model_repository.mlflow')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow')
|
||||
def test_get_experiment_by_run_id(mlflow, mlflow_repository):
|
||||
mlflow.get_run.return_value = MagicMock(
|
||||
info=MagicMock(
|
||||
@@ -233,7 +233,7 @@ def test_get_experiment_by_run_id(mlflow, mlflow_repository):
|
||||
mlflow.get_experiment.assert_called_once_with('0')
|
||||
|
||||
|
||||
@patch('laborious.utils.repository.model_repository.mlflow')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow')
|
||||
def test_get_next_run_name(mlflow, mlflow_repository):
|
||||
mlflow.search_runs.return_value = [1, 2, 3]
|
||||
output = mlflow_repository.get_next_run_name('run')
|
||||
@@ -244,7 +244,7 @@ def test_get_next_run_name(mlflow, mlflow_repository):
|
||||
)
|
||||
|
||||
|
||||
@patch('laborious.utils.repository.model_repository.mlflow')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow')
|
||||
def test_get_experiment_success(mlflow, mlflow_repository):
|
||||
mlflow.get_experiment_by_name.return_value = MagicMock(
|
||||
experiment_id='0')
|
||||
@@ -254,7 +254,7 @@ def test_get_experiment_success(mlflow, mlflow_repository):
|
||||
assert output == 0
|
||||
|
||||
|
||||
@patch('laborious.utils.repository.model_repository.mlflow')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow')
|
||||
def test_get_experiment_error(mlflow, mlflow_repository):
|
||||
mlflow.get_experiment_by_name.return_value = None
|
||||
|
||||
@@ -266,7 +266,7 @@ def test_get_experiment_error(mlflow, mlflow_repository):
|
||||
assert False
|
||||
|
||||
|
||||
@patch('laborious.utils.repository.model_repository.mlflow')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow')
|
||||
def test_get_experiment_last_run(mlflow, mlflow_repository):
|
||||
mlflow.search_runs.return_value = DataFrame({
|
||||
'params.retrain': ['True', 'False', 'True', 'False'],
|
||||
@@ -285,7 +285,7 @@ def test_get_experiment_last_run(mlflow, mlflow_repository):
|
||||
assert output == '2'
|
||||
|
||||
|
||||
@patch('laborious.utils.repository.model_repository.mlflow')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow')
|
||||
def test_get_experiment_last_run_error(mlflow, mlflow_repository):
|
||||
mlflow.search_runs.return_value = []
|
||||
|
||||
@@ -297,8 +297,8 @@ def test_get_experiment_last_run_error(mlflow, mlflow_repository):
|
||||
assert False
|
||||
|
||||
|
||||
@patch('laborious.utils.repository.model_repository.mlflow.sklearn')
|
||||
@patch('laborious.utils.repository.model_repository.mlflow.set_experiment')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.sklearn')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.set_experiment')
|
||||
def test_create_model_experiment(set_experiment, sklearn, mlflow_repository):
|
||||
|
||||
mlflow_repository.model_serving.get_model_run_id = MagicMock(
|
||||
@@ -360,10 +360,10 @@ def test_create_model_experiment(set_experiment, sklearn, mlflow_repository):
|
||||
mlflow_repository.get_experiment_by_run_id.return_value)
|
||||
|
||||
|
||||
@patch('laborious.utils.repository.model_repository.mlflow.start_run')
|
||||
@patch('laborious.utils.repository.model_repository.mlflow.log_param')
|
||||
@patch('laborious.utils.repository.model_repository.mlflow.sklearn.log_model')
|
||||
@patch('laborious.utils.repository.model_repository.mlflow.log_artifact')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.start_run')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.log_param')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.sklearn.log_model')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.log_artifact')
|
||||
def test_perform_model_retrain(log_artifact, log_model, log_param, start_run, mlflow_repository):
|
||||
|
||||
prediction_model_mock = MagicMock()
|
||||
@@ -423,7 +423,7 @@ def test_retrain_model(mlflow_repository):
|
||||
assert output == 'Model retrained successfully'
|
||||
|
||||
|
||||
@patch('laborious.utils.repository.model_repository.mlflow')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow')
|
||||
def test_update_production_model_by_run_id(mlflow, mlflow_repository):
|
||||
client_mock = MagicMock()
|
||||
mlflow.tracking.MlflowClient.return_value = client_mock
|
||||
@@ -458,7 +458,7 @@ def test_update_production_model_by_run_id(mlflow, mlflow_repository):
|
||||
}
|
||||
|
||||
|
||||
@patch('laborious.utils.repository.model_repository.mlflow')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow')
|
||||
def test_update_production_model_by_run_id_error(mlflow, mlflow_repository):
|
||||
mlflow.tracking.MlflowClient.return_value = MagicMock(
|
||||
get_registered_model=MagicMock(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, Mock, patch, MagicMock, ANY, call
|
||||
from asyncua.crypto.security_policies import SecurityPolicyBasic256
|
||||
from laborious.utils.repository.opc_repository import OpcRepository
|
||||
from model_manager.utils.repository.opc_repository import OpcRepository
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from datetime import datetime
|
||||
|
||||
@@ -28,7 +28,7 @@ def opc_repository(mock_logger):
|
||||
|
||||
@pytest.fixture
|
||||
def mock_client():
|
||||
with patch('laborious.utils.repository.opc_repository.Client') as mock:
|
||||
with patch('model_manager.utils.repository.opc_repository.Client') as mock:
|
||||
client_instance = AsyncMock()
|
||||
mock.return_value = client_instance
|
||||
yield client_instance
|
||||
@@ -212,7 +212,7 @@ async def test_validate_connection_error_validate_connection_error(opc_repositor
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch('laborious.utils.repository.opc_repository.datetime')
|
||||
@patch('model_manager.utils.repository.opc_repository.datetime')
|
||||
async def test_validate_connection_lost_not_time_to_reconnect(_mock_datetime, opc_repository):
|
||||
_mock_datetime.now = MagicMock(
|
||||
return_value=datetime(2025, 1, 1, 0, 0, 0))
|
||||
@@ -233,7 +233,7 @@ async def test_validate_connection_lost_not_time_to_reconnect(_mock_datetime, op
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch('laborious.utils.repository.opc_repository.datetime')
|
||||
@patch('model_manager.utils.repository.opc_repository.datetime')
|
||||
async def test_validate_connection_lost_time_to_reconnect(mock_datetime, opc_repository):
|
||||
mock_datetime.now = MagicMock(
|
||||
return_value=datetime(2025, 1, 1, 1, 0, 0))
|
||||
@@ -333,7 +333,7 @@ async def test_write_data_invalid_data_type(opc_repository, mock_client):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch('laborious.utils.repository.opc_repository.metrics')
|
||||
@patch('model_manager.utils.repository.opc_repository.metrics')
|
||||
async def test_write_data(mock_metrics, opc_repository, mock_client):
|
||||
opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
|
||||
opc_repository.client = mock_client
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from os import environ
|
||||
from laborious.utils.connectors_config import (build_mlflow_config,
|
||||
from model_manager.utils.connectors_config import (build_mlflow_config,
|
||||
build_opc_config,
|
||||
build_postgres_config,
|
||||
build_mongodb_config)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
from unittest.mock import call, patch, AsyncMock, ANY
|
||||
from pytest import mark, fixture
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction
|
||||
from model_manager.activities.activities import Activities
|
||||
from model_manager.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ metadata = {
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock)
|
||||
@patch("model_manager.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock)
|
||||
async def test_run_none_path_flag(workflow_mock, format_and_export_prediction):
|
||||
|
||||
input_data = {
|
||||
@@ -91,7 +91,7 @@ async def test_run_none_path_flag(workflow_mock, format_and_export_prediction):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock)
|
||||
@patch("model_manager.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock)
|
||||
async def test_run_default_path_flag(workflow_mock, format_and_export_prediction):
|
||||
|
||||
input_data = {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from unittest.mock import AsyncMock, patch, call, ANY
|
||||
from pytest import fixture, mark
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
|
||||
from model_manager.activities.activities import Activities
|
||||
from model_manager.workflows.sub_workflows.prediction_process import PredictionProcess
|
||||
|
||||
|
||||
@fixture
|
||||
@@ -20,7 +20,7 @@ metadata = {
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
async def test_run(workflow_mock, prediction_process):
|
||||
prediction_process.path_flag_handler = AsyncMock(return_value=False)
|
||||
# Arrange
|
||||
@@ -135,7 +135,7 @@ async def test_run(workflow_mock, prediction_process):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
async def test_run_stop_at_input_gate(workflow_mock, prediction_process):
|
||||
prediction_process.path_flag_handler = AsyncMock(return_value=True)
|
||||
# Arrange
|
||||
@@ -183,7 +183,7 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_process):
|
||||
prediction_process.path_flag_handler = AsyncMock(side_effect=[False, True])
|
||||
# Arrange
|
||||
@@ -253,7 +253,7 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process):
|
||||
prediction_process.path_flag_handler = AsyncMock(
|
||||
side_effect=[False, False, True])
|
||||
@@ -333,7 +333,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_process):
|
||||
prediction_process.path_flag_handler = AsyncMock(
|
||||
side_effect=[False, False, False, True])
|
||||
@@ -429,7 +429,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
async def test_path_flag_handler_stop(workflow_mock, prediction_process):
|
||||
# Arrange
|
||||
data = {'test': 'data'}
|
||||
@@ -464,7 +464,7 @@ async def test_path_flag_handler_stop(workflow_mock, prediction_process):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
async def test_path_flag_handler_repeat(workflow_mock, prediction_process):
|
||||
# Arrange
|
||||
data = {'test': 'data'}
|
||||
@@ -510,7 +510,7 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
async def test_path_flag_handler_continue(workflow_mock, prediction_process):
|
||||
# Arrange
|
||||
data = {'test': 'data'}
|
||||
@@ -565,7 +565,7 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
@patch("model_manager.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock)
|
||||
async def test_path_flag_handler_unknown(workflow_mock, prediction_process):
|
||||
# Arrange
|
||||
data = {'test': 'data'}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from unittest.mock import AsyncMock, MagicMock, call, patch, ANY
|
||||
from pytest import fixture, mark
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.workflows.minimal_retrain import MinimalRetrain
|
||||
from model_manager.activities.activities import Activities
|
||||
from model_manager.workflows.minimal_retrain import MinimalRetrain
|
||||
|
||||
|
||||
@fixture
|
||||
@@ -20,7 +20,7 @@ metadata = {
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('laborious.workflows.minimal_retrain.workflow', new_callable=AsyncMock)
|
||||
@patch('model_manager.workflows.minimal_retrain.workflow', new_callable=AsyncMock)
|
||||
async def test_run(workflow_mock: AsyncMock, minimal_retrain: MinimalRetrain):
|
||||
input_data = {
|
||||
"model_id": "test_model_id",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from unittest.mock import AsyncMock, call, patch, ANY
|
||||
from pytest import fixture, mark
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.workflows.predictions_batch import PredictionsBatch
|
||||
from model_manager.activities.activities import Activities
|
||||
from model_manager.workflows.predictions_batch import PredictionsBatch
|
||||
|
||||
|
||||
@fixture
|
||||
@@ -20,7 +20,7 @@ metadata = {
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('laborious.workflows.predictions_batch.workflow', new_callable=AsyncMock)
|
||||
@patch('model_manager.workflows.predictions_batch.workflow', new_callable=AsyncMock)
|
||||
async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch):
|
||||
workflow_mock.execute_local_activity_method.return_value = {
|
||||
'data': 'test_data'
|
||||
|
||||
@@ -70,7 +70,7 @@ livenessProbe:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- pgrep -f "laborious.worker.worker"
|
||||
- pgrep -f "model_manager.worker.worker"
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 30
|
||||
|
||||
@@ -79,7 +79,7 @@ readinessProbe:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- pgrep -f "laborious.worker.worker"
|
||||
- pgrep -f "model_manager.worker.worker"
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 15
|
||||
|
||||
@@ -153,7 +153,7 @@ env:
|
||||
- name: GITHUB_BRANCH
|
||||
value: SIENTIAPDE-1222-ajustar-a-library-para-fazer-o-download-do-courier
|
||||
- name: PYTHON_APP
|
||||
value: "laborious.worker.worker"
|
||||
value: "model_manager.worker.worker"
|
||||
|
||||
# Application variables
|
||||
- name: POSTGRES_HOST
|
||||
|
||||
Reference in New Issue
Block a user