feat: enhance test scenarios and configuration for regression models

- Updated `pyproject.toml` to include new linting rules for end-to-end tests.
- Modified `requirements-dev.txt` to add dependencies for E2E testing with `testcontainers` and `requests`.
- Refactored multiple JSON test scenario files to standardize structure, including new fields for `experiment_run_id`, `bucket_name`, and `file_name`.
- Improved model training parameters in `train_model_params.py` to use `experiment_name` directly.
- Adjusted `data_manager_repository.py` to utilize the updated `experiment_name` for logging.

These changes improve the organization and clarity of regression model tests and enhance the overall testing framework.

e2e/test_cleanup_files_workflow.py

@@ -0,0 +1,109 @@
+"""
+End-to-end tests for CleanupFiles workflow.
+
+Covers scenarios 3.x: cleanup of temporary local directories.
+"""
+
+import os
+import shutil
+import tempfile
+
+import pytest
+import pytest_asyncio
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import make_workflow_id, start_and_await_workflow
+from model_manager.workflows.cleanup_files import CleanupFiles
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_1_cleanup_with_no_temp_dirs(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    tmp_path,
+):
+    """Scenario 3.1.1 – Cleanup when the temp directory is empty.
+
+    The cleanup_temp_directories activity should complete without error
+    and the workflow should finish successfully.
+    """
+    # Use an empty temp directory as the reports path
+    empty_dir = tmp_path / 'reports_temp'
+    empty_dir.mkdir()
+
+    result = await start_and_await_workflow(
+        temporal_test_env.client,
+        CleanupFiles.run,
+        {'temp_path': str(empty_dir)},
+        make_workflow_id('test-s3-1-1'),
+    )
+
+    # Workflow returns None on success
+    assert result is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_2_cleanup_removes_old_temp_dirs(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    tmp_path,
+):
+    """Scenario 3.1.2 – Cleanup removes stale subdirectories from the temp dir.
+
+    Creates two subdirectories with timestamp suffixes inside the reports
+    temp directory and verifies the activity removes them.
+    """
+    reports_dir = tmp_path / 'reports_temp'
+    reports_dir.mkdir()
+
+    # Create two stale run directories
+    stale1 = reports_dir / 'run-1234567890'
+    stale2 = reports_dir / 'run-9876543210'
+    stale1.mkdir()
+    stale2.mkdir()
+    (stale1 / 'model.pkl').write_bytes(b'fake-model-data')
+    (stale2 / 'report.json').write_bytes(b'{"status": "old"}')
+
+    result = await start_and_await_workflow(
+        temporal_test_env.client,
+        CleanupFiles.run,
+        {'temp_path': str(reports_dir)},
+        make_workflow_id('test-s3-1-2'),
+    )
+
+    assert result is None
+
+    # The activity should have cleaned up the stale directories
+    remaining = list(reports_dir.iterdir())
+    assert len(remaining) == 0, (
+        f'Expected all stale dirs to be removed, but found: {remaining}'
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_3_cleanup_nonexistent_temp_path(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    tmp_path,
+):
+    """Scenario 3.1.3 – Cleanup with a temp_path that does not exist.
+
+    The activity must handle a missing directory gracefully without
+    raising an unhandled exception, since the directory may have already
+    been cleaned by a previous run.
+    """
+    nonexistent = str(tmp_path / 'does_not_exist' / 'reports')
+
+    # Should not raise — the activity is expected to handle a missing path
+    result = await start_and_await_workflow(
+        temporal_test_env.client,
+        CleanupFiles.run,
+        {'temp_path': nonexistent},
+        make_workflow_id('test-s3-1-3'),
+    )
+
+    assert result is None