SIENTIAPDE-1231

Enhance validation script and refactor code in various modules

- Updated the validation script to include automatic code formatting and linting fixes using Ruff.
- Removed the `clean_tmp_files` method from the Gates class to streamline functionality.
- Simplified conditional checks in the OpcRepository for better clarity and error handling.
- Added model ID to the minimal retrain workflow for improved tracking.
- Introduced new test cases for error handling in MLFlow and storage operations, ensuring robustness in repository interactions.

tests/laborious/utils/filters/test_conditional_filters.py

@@ -16,6 +16,13 @@ def test_filter_specific_variables_null_values():
     )
 
 
+def test_filter_specific_variables_null_values_with_empty_data():
+    assert (
+        filter_specific_variables_null_values(DataFrame(), config={'variables': ['variable2']})
+        is False
+    )
+
+
 def test_filter_specific_variables_null_values_with_null_values():
     assert (
         filter_specific_variables_null_values(

tests/laborious/utils/repository/test_minio_repository.py

@@ -61,6 +61,11 @@ def minio_repository(mock_boto3, mock_config):
     )
 
 
+def test_close(minio_repository):
+    minio_repository.close()
+    minio_repository.s3_client.close.assert_called_once()
+
+
 def test_ensure_bucket_exists_bucket_exists(minio_repository):
     assert minio_repository.ensure_bucket_exists({}) is True
 

tests/laborious/utils/repository/test_model_repository.py

@@ -173,6 +173,11 @@ def test_get_experiment_none_not_create(mlflow, mlflow_repository):
         assert str(e) == 'Experiment test not found'
 
 
+def test_get_model_params(mlflow, mlflow_repository):
+    output = mlflow_repository.get_model_params('test')
+    assert output == mlflow.get_run.return_value.data.params
+
+
 @patch('laborious.utils.repository.model_repository.path')
 @patch('laborious.utils.repository.model_repository.rmtree')
 @patch('laborious.utils.repository.model_repository.makedirs')
@@ -202,6 +207,35 @@ def test_download_artifacts_success(makedirs, rmtree, path, mlflow_repository):
     assert output == mlflow_repository.client.download_artifacts.return_value
 
 
+@patch('laborious.utils.repository.model_repository.path')
+@patch('laborious.utils.repository.model_repository.rmtree')
+@patch('laborious.utils.repository.model_repository.makedirs')
+def test_download_artifacts_success_path_false(makedirs, rmtree, path, mlflow_repository):
+    mlflow_repository.get_model_run_id = MagicMock(return_value='test')
+
+    path.exists.return_value = False
+
+    output = mlflow_repository.dowload_artifacts('test', 'path')
+
+    mlflow_repository.get_model_run_id.assert_called_once_with(
+        model_name='test', stage='Production'
+    )
+
+    path.join.assert_called_once_with('./tmp/artifacts/test', 'path')
+
+    path.exists.assert_called_once_with(path.join.return_value)
+
+    rmtree.assert_not_called()
+
+    makedirs.assert_called_once_with('./tmp/artifacts/test', exist_ok=True)
+
+    mlflow_repository.client.download_artifacts.assert_called_once_with(
+        mlflow_repository.get_model_run_id.return_value, 'path', './tmp/artifacts/test'
+    )
+
+    assert output == mlflow_repository.client.download_artifacts.return_value
+
+
 def test_get_experiment_error(mlflow, mlflow_repository):
     mlflow.get_experiment_by_name.return_value = None
 
@@ -383,6 +417,15 @@ valid_cases = [
         },
         ['2025-01-01 12:00:00+0000', '2025-01-02 12:00:00+0000'],
     ),
+    (
+        {
+            'value': {
+                datetime(2025, 1, 1, 12, 0, 0, tzinfo=None): 1,
+                datetime(2025, 1, 2, 12, 0, 0, tzinfo=None): 2,
+            }
+        },
+        ['2025-01-01 12:00:00+0000', '2025-01-02 12:00:00+0000'],
+    ),
     (
         {
             'value': {
@@ -987,6 +1030,20 @@ def test_retrain_model(mlflow_repository):
     }
 
 
+def test_retrain_model_error(mlflow_repository):
+    data = MagicMock()
+    model_name = 'test'
+    model_config = {'target': 'target', 'transform_flavor': 'sklearn', 'predict_flavor': 'pyfunc'}
+    mlflow_repository.get_model_run_id = MagicMock(side_effect=Exception('error'))
+    output = mlflow_repository.retrain_model(data, model_name, model_config, metadata['metadata'])
+    assert output == {
+        'success': False,
+        'experiment': None,
+        'message': 'Error retraining model test: error',
+        'traceback': ANY,
+    }
+
+
 def test_update_production_model(mlflow_repository):
     experiment = {'run_id': '0', 'experiment_id': '0'}
     model_name = 'test'

tests/laborious/utils/repository/test_opc_repository.py

@@ -86,6 +86,15 @@ async def test_set_security_missing_certificates(opc_repository):
         assert str(e) == 'Certificate and private key paths must be provided for secure connection.'
 
 
+@pytest.mark.asyncio
+async def test_set_security_missing_client(opc_repository):
+    opc_repository.client = None
+    try:
+        await opc_repository.set_security()
+    except ValueError as e:
+        assert str(e) == 'Client must be initialized before setting security'
+
+
 @pytest.mark.asyncio
 async def test_connect_with_security(opc_repository, mock_client):
     opc_repository.try_connect = AsyncMock(return_value=(True, {}))
@@ -137,6 +146,21 @@ async def test_try_connect_fail(opc_repository):
     assert error_data['attachment_content'] is not None
 
 
+@pytest.mark.asyncio
+async def test_try_connect_no_client(opc_repository):
+    opc_repository.client = None
+    result = await opc_repository.try_connect()
+    assert result == (
+        False,
+        {
+            'notification_id': f'OPC_CONNECTION_ERROR_{opc_repository.id}',
+            'message': 'Client is not initialized',
+            'block': 'opc_repository',
+            'level': NotificationLevel.ERROR,
+        },
+    )
+
+
 @pytest.mark.asyncio
 async def test_disconnect(opc_repository, mock_client):
     opc_repository.client = mock_client

tests/laborious/utils/test_connectors_config.py

@@ -1,6 +1,7 @@
 from os import environ
 
 from laborious.utils.connectors_config import (
+    build_minio_config,
     build_mlflow_config,
     build_mongodb_config,
     build_opc_config,
@@ -162,3 +163,33 @@ def test_build_mongo_db_config_with_defaults():
         'database_name': 'sientia',
         'ttl_index_seconds': 3600,
     }
+
+
+def test_build_minio_config_with_env_vars():
+    environ['MINIO_ENDPOINT_URL'] = 'http://test-host'
+    environ['MINIO_ACCESS_KEY'] = 'test-key'
+    environ['MINIO_SECRET_KEY'] = 'test-secret'
+    environ['MINIO_REGION_NAME'] = 'test-region'
+    environ['MINIO_DEFAULT_BUCKET'] = 'test-bucket'
+    assert build_minio_config() == {
+        'endpoint_url': 'http://test-host',
+        'access_key': 'test-key',
+        'secret_key': 'test-secret',
+        'region_name': 'test-region',
+        'default_bucket': 'test-bucket',
+    }
+
+
+def test_build_minio_config_with_defaults():
+    environ.pop('MINIO_ENDPOINT_URL', None)
+    environ.pop('MINIO_ACCESS_KEY', None)
+    environ.pop('MINIO_SECRET_KEY', None)
+    environ.pop('MINIO_REGION_NAME', None)
+    environ.pop('MINIO_DEFAULT_BUCKET', None)
+    assert build_minio_config() == {
+        'endpoint_url': 'http://localhost:9000',
+        'access_key': 'minioadmin',
+        'secret_key': 'minioadmin',
+        'region_name': 'us-east-1',
+        'default_bucket': 'laborious',
+    }