SIENTIAPDE-994

Refactor activity methods and update requirements.txt to enhance functionality and remove deprecated filters. Added detailed docstrings for clarity and improved error handling in data processing workflows.

tests/laborious/utils/filters/repository/test_model_repository.py

@@ -0,0 +1,278 @@
+from unittest.mock import ANY, MagicMock, patch
+import numpy as np
+from pandas import DataFrame
+import pytest
+from laborious.utils.repository.model_repository import MLFlowRepository
+
+
+@pytest.fixture
+def mlflow_repository():
+    with patch('laborious.utils.repository.model_repository.ModelServing', autospec=True) as MockModelServing:
+        mock_instance = MockModelServing.return_value
+        mock_instance.get_transformed_data = MagicMock()
+
+        repo = MLFlowRepository(
+            host='http://localhost:5000',
+            username='admin',
+            password='admin'
+        )
+        return repo
+
+
+def test_get_current_data_df(mlflow_repository):
+    current_data = {
+        'prediction': [1, 3],
+        'target': [1, 1],
+    }
+    mlflow_repository.model_serving.get_transformed_data.return_value = {
+        'var1': [1, 2],
+        'var2': [2, np.nan],
+    }
+    expected = DataFrame({
+        'var1': [1],
+        'var2': [2],
+        'prediction': [1],
+        'target': [1],
+    })
+    output = mlflow_repository.get_current_data_df(current_data,
+                                                   'model', 'target')
+
+    mlflow_repository.model_serving.get_transformed_data.assert_called_once_with(
+        'model', current_data, by='model')
+
+    diff = output.compare(expected)
+    assert diff.empty
+
+
+def test_get_artifact(mlflow_repository):
+    mlflow_repository.get_artifact(
+        'destination', 'search_by', 'run_id', 'model', 'artifact'
+    )
+    mlflow_repository.model_serving.get_artifact.assert_called_once_with(
+        destination='destination',
+        search_by='search_by',
+        run_id='run_id',
+        model_name='model',
+        artifact_name='artifact'
+    )
+
+
+def test_calculate_model_metrics(mlflow_repository):
+    mlflow_repository.model_serving.get_model_metrics.return_value = 'data'
+    real_data = 'real_data'
+    predictions = 'predictions'
+    flag = 'flag'
+    output = mlflow_repository.calculate_model_metrics(
+        real_data, predictions, flag
+    )
+    mlflow_repository.model_serving.get_model_metrics.assert_called_once_with(
+        reference_data=None,
+        real_data=real_data,
+        predictions=predictions,
+        type_flag=flag
+    )
+    assert output == 'data'
+
+
+@patch('laborious.utils.repository.model_repository.mlflow')
+def test_get_experiment_by_run_id(mlflow, mlflow_repository):
+    mlflow.get_run.return_value = MagicMock(
+        info=MagicMock(
+            experiment_id='0',
+        )
+    )
+    mlflow.get_experiment.return_value = MagicMock()
+    mlflow.get_experiment.return_value.name = 'test'
+
+    output = mlflow_repository.get_experiment_by_run_id('0')
+    assert output == 'test'
+    mlflow.get_run.assert_called_once_with('0')
+    mlflow.get_experiment.assert_called_once_with('0')
+
+
+@patch('laborious.utils.repository.model_repository.mlflow')
+def test_get_next_run_name(mlflow, mlflow_repository):
+    mlflow.search_runs.return_value = [1, 2, 3]
+    output = mlflow_repository.get_next_run_name('run')
+    assert output == 'run-4'
+    mlflow.search_runs.assert_called_once_with(
+        experiment_names=['run'],
+        order_by=['start_time desc'],
+    )
+
+
+@patch('laborious.utils.repository.model_repository.mlflow')
+def test_get_experiment_success(mlflow, mlflow_repository):
+    mlflow.get_experiment_by_name.return_value = MagicMock(
+        experiment_id='0')
+
+    output = mlflow_repository.get_experiment('test')
+
+    assert output == 0
+
+
+@patch('laborious.utils.repository.model_repository.mlflow')
+def test_get_experiment_error(mlflow, mlflow_repository):
+    mlflow.get_experiment_by_name.return_value = None
+
+    try:
+        mlflow_repository.get_experiment('test')
+    except ValueError as e:
+        assert str(e) == 'Experiment test not found'
+    else:
+        assert False
+
+
+@patch('laborious.utils.repository.model_repository.mlflow')
+def test_get_experiment_last_run(mlflow, mlflow_repository):
+    mlflow.search_runs.return_value = DataFrame({
+        'params.retrain': ['True', 'False', 'True', 'False'],
+        'end_time': ['2021-01-01', '2021-01-02', '2021-01-03', '2021-01-04'],
+        'run_id': ['0', '1', '2', '3'],
+    })
+
+    output = mlflow_repository.get_experiment_last_run(0)
+
+    mlflow.search_runs.assert_called_once_with(
+        experiment_ids=[0],
+        filter_string="",
+        output_format="pandas",
+    )
+
+    assert output == '2'
+
+
+@patch('laborious.utils.repository.model_repository.mlflow')
+def test_update_production_model_by_run_id(mlflow, mlflow_repository):
+    client_mock = MagicMock()
+    mlflow.tracking.MlflowClient.return_value = client_mock
+
+    client_mock.get_registered_model.return_value = MagicMock(
+        latest_versions=[
+            MagicMock(version='1'),
+            MagicMock(version='2'),
+            MagicMock(version='3'),
+        ]
+    )
+    output = mlflow_repository.update_production_model_by_run_id('0', 'test')
+
+    mlflow.register_model.assert_called_once_with(
+        "runs:/0/prediction_model",
+        'test',
+    )
+
+    mlflow.tracking.MlflowClient.assert_called_once()
+    client_mock.get_registered_model.assert_called_once_with('test')
+    client_mock.transition_model_version_stage.assert_called_once_with(
+        name='test',
+        version='3',
+        stage='Production',
+        archive_existing_versions=True,
+    )
+
+    assert output == {
+        'model_name': 'test',
+        'version': '3',
+        'mlflow_run_id': '0',
+    }
+
+
+def test_update_production_model(mlflow_repository):
+    connector = mlflow_repository
+
+    with patch.object(connector, 'get_experiment',
+                      return_value='0') as get_experiment:
+        with patch.object(connector, 'get_experiment_last_run',
+                          return_value='2') as get_experiment_last_run:
+            with patch.object(connector, 'update_production_model_by_run_id',
+                              return_value={'model_name': 'test', 'version': '3',
+                                            'mlflow_run_id': '0'}) as update_production_model_by_run_id:
+
+                output = connector.update_production_model('0', 'test')
+
+                get_experiment.assert_called_once_with('0')
+                get_experiment_last_run.assert_called_once_with('0')
+                update_production_model_by_run_id.assert_called_once_with(
+                    '2', 'test')
+
+                assert output == {
+                    'model_name': 'test',
+                    'version': '3',
+                    'mlflow_run_id': '0',
+                    'mlflow_experiment_id': '0',
+                }
+
+
+def test_transform_success(mlflow_repository):
+    data = 'data'
+    model_name = 'model'
+
+    output = mlflow_repository.transform(model_name, data, 1)
+
+    mlflow_repository.model_serving.get_cached_transform.assert_called_once_with(
+        model_name, data, 1)
+
+    assert output == {
+        'success': True,
+        'content': mlflow_repository.model_serving.get_cached_transform.return_value.to_dict.return_value
+    }
+
+
+def test_transform_error(mlflow_repository):
+    data = 'data'
+    model_name = 'model'
+
+    mlflow_repository.model_serving.get_cached_transform.side_effect = Exception(
+        'error')
+
+    output = mlflow_repository.transform(model_name, data, 1)
+
+    mlflow_repository.model_serving.get_cached_transform.assert_called_once_with(
+        model_name, data, 1)
+
+    assert output == {
+        'success': False,
+        'content': {
+            'message': 'error',
+            'traceback': ANY
+        }
+    }
+
+
+def test_predict_success(mlflow_repository):
+    data = 'data'
+    model_name = 'model'
+    mlflow_repository.model_serving.get_cached_predict.return_value = np.array(
+        [2, 3]
+    )
+
+    output = mlflow_repository.predict(model_name, data, 1)
+
+    mlflow_repository.model_serving.get_cached_predict.assert_called_once_with(
+        model_name, data, 1)
+
+    assert output['success'] == True
+    assert output['content'] == {'prediction': {
+        0: 2, 1: 3}, 'response_time': ANY}
+
+
+def test_predict_error(mlflow_repository):
+    data = 'data'
+    model_name = 'model'
+
+    mlflow_repository.model_serving.get_cached_predict = MagicMock(
+        side_effect=Exception('error')
+    )
+
+    output = mlflow_repository.predict(model_name, data, 1)
+
+    mlflow_repository.model_serving.get_cached_predict.assert_called_once_with(
+        model_name, data, 1)
+
+    assert output == {
+        'success': False,
+        'content': {
+            'message': 'error',
+            'traceback': ANY
+        }
+    }

tests/laborious/utils/filters/repository/test_opc_repository.py

@@ -0,0 +1,106 @@
+from unittest.mock import Mock, patch, MagicMock
+from pathlib import Path
+from asyncua.sync import Client
+from asyncua.crypto.security_policies import SecurityPolicyBasic256
+from asyncua.ua import DataValue, Variant, VariantType
+from pytest import fixture
+from laborious.utils.repository.opc_repository import OpcRepository
+
+
+@fixture
+def mock_logger():
+    return Mock()
+
+
+@fixture
+def opc_repository(mock_logger):
+    return OpcRepository(
+        name="test_repo",
+        url="opc.tcp://localhost:4840",
+        logger=mock_logger,
+        server_uri="urn:test:server",
+        cert_path="/path/to/cert.pem",
+        private_key_path="/path/to/key.pem",
+        server_cert_path="/path/to/server_cert.pem"
+    )
+
+
+@fixture
+def mock_client():
+    with patch('laborious.utils.repository.opc_repository.Client') as mock:
+        client_instance = MagicMock()
+        mock.return_value = client_instance
+        yield client_instance
+
+
+def test_init(opc_repository):
+    assert opc_repository.name == "test_repo"
+    assert opc_repository.url == "opc.tcp://localhost:4840"
+    assert opc_repository.server_uri == "urn:test:server"
+    assert opc_repository.cert_path == "/path/to/cert.pem"
+    assert opc_repository.private_key_path == "/path/to/key.pem"
+    assert opc_repository.server_cert_path == "/path/to/server_cert.pem"
+    assert opc_repository.non_receive_count == 0
+    assert opc_repository.client is None
+
+
+def test_set_security(opc_repository, mock_client):
+    opc_repository.client = mock_client
+    opc_repository.set_security()
+
+    mock_client.application_uri = "urn:test:server"
+    mock_client.set_security.assert_called_once_with(
+        SecurityPolicyBasic256,
+        certificate="/path/to/cert.pem",
+        private_key="/path/to/key.pem",
+        server_certificate="/path/to/server_cert.pem"
+    )
+    assert mock_client.secure_channel_timeout == 10000000
+    assert mock_client.session_timeout == 10000000
+
+
+def test_set_security_missing_certificates(opc_repository):
+    opc_repository.cert_path = None
+    opc_repository.private_key_path = None
+
+    try:
+        opc_repository.set_security()
+    except ValueError as e:
+        assert str(
+            e) == "Certificate and private key paths must be provided for secure connection."
+
+
+def test_connect_with_security(opc_repository, mock_client):
+    opc_repository.connect()
+
+    mock_client.connect.assert_called_once()
+    assert opc_repository.client == mock_client
+
+
+def test_connect_without_security(opc_repository, mock_client):
+    opc_repository.cert_path = None
+    opc_repository.connect()
+
+    mock_client.connect.assert_called_once()
+    assert opc_repository.client == mock_client
+
+
+def test_disconnect(opc_repository, mock_client):
+    opc_repository.client = mock_client
+    opc_repository.disconnect()
+
+    mock_client.disconnect.assert_called_once()
+    assert opc_repository.client is None
+
+
+def test_write_data(opc_repository, mock_client, mock_logger):
+    opc_repository.client = mock_client
+    mock_node = MagicMock()
+    mock_client.get_node.return_value = mock_node
+
+    opc_repository.write_data("ns=2;s=TestNode", 42.0, "float", mock_logger)
+
+    mock_client.get_node.assert_called_once_with("ns=2;s=TestNode")
+    mock_node.write_value.assert_called_once()
+    mock_logger.info.assert_called_once_with(
+        "Writing 42.0 - <class 'float'> to " + str(mock_node))

tests/laborious/utils/filters/test_conditional_filters.py

@@ -7,20 +7,21 @@ def test_filter_specific_variables_null_values():
     assert filter_specific_variables_null_values(
         DataFrame(
             {'variable': ['variable1', 'variable2'], 'value': [1, 2]}),
-        variables=['variable2']) == True
+        config={'VARIABLES': ['variable2']}) == True
 
 
 def test_filter_specific_variables_null_values_with_null_values():
     assert filter_specific_variables_null_values(
         DataFrame(
             {'variable': ['variable1', 'variable2'], 'value': [1, None]}),
-        variables=['variable2']) == False
+        config={'VARIABLES': ['variable2']}) == False
 
 
 def test_filter_empty_data():
-    assert filter_empty_data(DataFrame()) == True
+    assert filter_empty_data(DataFrame(), {}) == True
 
 
 def test_filter_empty_data_with_data():
     assert filter_empty_data(
-        DataFrame({'variable': ['variable1', 'variable2'], 'value': [1, 2]})) == False
+        DataFrame({'variable': ['variable1', 'variable2'], 'value': [1, 2]}),
+        {}) == False

tests/laborious/utils/filters/test_mlflow_filters.py

@@ -0,0 +1,22 @@
+from pandas import DataFrame
+from laborious.utils.filters.mlflow_filters import api_error_filter, nan_values_filter
+
+
+def test_api_error_filter_invalid_response():
+    assert api_error_filter(None, {}) == True  # NOSONAR
+
+
+def test_api_error_filter_valid_response_fail():
+    assert api_error_filter({'success': False}, {}) == True
+
+
+def test_api_error_filter_valid_response_success():
+    assert api_error_filter({'success': True}, {}) == False
+
+
+def test_nan_values_filter_all_nan_values():
+    assert nan_values_filter(DataFrame({'variable': [None, None]}), {}) == True
+
+
+def test_nan_values_filter_no_nan_values():
+    assert nan_values_filter(DataFrame({'variable': [1, 2]}), {}) == False