diff --git a/requirements-local.txt b/requirements-local.txt index 80eea27..ea6a188 100644 --- a/requirements-local.txt +++ b/requirements-local.txt @@ -7,4 +7,5 @@ git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.0 git+ssh://git@github.com/Aignosi/sientia-model-library.git@0.8.3 prometheus-client==0.23.1 beautifulsoup4==4.12.3 -evidently==0.6.7 \ No newline at end of file +evidently==0.6.7 +jsonschema==4.26.0 \ No newline at end of file diff --git a/tests/activities/test_training.py b/tests/activities/test_training.py index 9c2395e..5f53ca7 100644 --- a/tests/activities/test_training.py +++ b/tests/activities/test_training.py @@ -369,7 +369,10 @@ def test_prepare_data_increments_error_counter_and_still_observes_lag_on_failure training.observe_lag_sync.assert_called_once() training.emit_metric_sync.assert_called_once() call_args = training.emit_metric_sync.call_args - assert call_args.kwargs['metric_object'] is mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL + assert ( + call_args.kwargs['metric_object'] + is mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL + ) def test_fit_model_observes_lag_on_success(training): @@ -419,12 +422,16 @@ def test_fit_model_increments_error_counter_on_failure(training): training.observe_lag_sync.assert_called_once() training.emit_metric_sync.assert_called_once() call_args = training.emit_metric_sync.call_args - assert call_args.kwargs['metric_object'] is mm_metrics.SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL + assert ( + call_args.kwargs['metric_object'] is mm_metrics.SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL + ) @patch('model_manager.activities.training.mm_metrics') @patch('model_manager.activities.training.mlflow') -def test_train_model_sets_quality_gauges_after_compute_metrics(mock_mlflow, mock_mm_metrics, training): +def test_train_model_sets_quality_gauges_after_compute_metrics( + mock_mlflow, mock_mm_metrics, training +): tp = TrainModelParams.from_dict( {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} ) @@ -473,9 +480,15 @@ def test_train_model_sets_quality_gauges_after_compute_metrics(mock_mlflow, mock training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) - mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels.return_value.set.assert_called_once_with(0.5) - mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels.return_value.set.assert_called_once_with(0.3) - mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels.return_value.set.assert_called_once_with(-0.1) + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels.return_value.set.assert_called_once_with( + 0.5 + ) + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels.return_value.set.assert_called_once_with( + 0.3 + ) + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels.return_value.set.assert_called_once_with( + -0.1 + ) @patch('model_manager.activities.training.mm_metrics') diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 2060902..a1734dd 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -237,7 +237,10 @@ def test_sientia_training_data_preparation_error_count_total_is_counter(): from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL, Counter) - assert 'sientia_training_data_preparation_error_count' in SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL._name + assert ( + 'sientia_training_data_preparation_error_count' + in SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL._name + ) _assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL) @@ -257,7 +260,10 @@ def test_sientia_training_model_fit_error_count_total_is_counter(): from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL assert isinstance(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL, Counter) - assert 'sientia_training_model_fit_error_count' in SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL._name + assert ( + 'sientia_training_model_fit_error_count' + in SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL._name + ) _assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL) @@ -340,9 +346,15 @@ def test_sientia_training_info_is_gauge(): assert SIENTIA_TRAINING_INFO._name == 'sientia_training_info' expected_labels = { - 'pod_id', 'model_name', 'model_type', - 'dataset_train_rows', 'dataset_val_rows', 'feature_count', - 'mse', 'mae', 'r2', + 'pod_id', + 'model_name', + 'model_type', + 'dataset_train_rows', + 'dataset_val_rows', + 'feature_count', + 'mse', + 'mae', + 'r2', } assert expected_labels == set(SIENTIA_TRAINING_INFO._labelnames) diff --git a/validate.sh b/validate.sh deleted file mode 100755 index 96beac1..0000000 --- a/validate.sh +++ /dev/null @@ -1,195 +0,0 @@ -#!/bin/bash -# Model Manager Code Validation Script -# This script runs all code quality checks before committing or deploying -# -# Usage: -# ./validate.sh # Run all checks including tests (default) -# ./validate.sh --no-tests # Skip unit tests -# ./validate.sh --skip-tests # Skip unit tests (alias) -# ./validate.sh --only-tests # Run only unit tests -# ./validate.sh --fix # Auto-fix formatting and linting, then run validations (no tests) - -set -e # Exit on any error - -# Parse command line arguments -RUN_TESTS=true -ONLY_TESTS=false -FIX_MODE=false - -for arg in "$@"; do - case $arg in - --no-tests|--skip-tests) - RUN_TESTS=false - shift - ;; - --only-tests) - ONLY_TESTS=true - shift - ;; - --fix) - FIX_MODE=true - RUN_TESTS=false - shift - ;; - --help|-h) - echo "Usage: $0 [OPTIONS]" - echo "" - echo "Options:" - echo " --no-tests, --skip-tests Skip unit tests (default: run tests)" - echo " --only-tests Run only unit tests" - echo " --fix Auto-fix formatting and linting, then run validations (no tests)" - echo " --help, -h Show this help message" - echo "" - exit 0 - ;; - *) - echo "Unknown option: $arg" - echo "Use --help for usage information" - exit 1 - ;; - esac -done - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}" -echo -e "${BLUE}║ Model Manager - Code Validation Suite ║${NC}" -echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}" -echo "" - -# Handle --fix mode -if [ "$FIX_MODE" = true ]; then - echo -e "${BLUE}🔧 Running auto-fix mode...${NC}" - echo "" - - echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - echo -e "${BLUE}▶ Auto-fixing code formatting (Ruff)${NC}" - echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - ruff format model_manager/ tests/ - echo -e "${GREEN}✅ Code formatting applied${NC}" - echo "" - - echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - echo -e "${BLUE}▶ Auto-fixing linting issues (Ruff)${NC}" - echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - ruff check --fix model_manager/ tests/ - echo -e "${GREEN}✅ Linting fixes applied${NC}" - echo "" - - echo -e "${YELLOW}ℹ️ Now running validations (without tests)...${NC}" - echo "" -fi - -# Handle --only-tests mode -if [ "$ONLY_TESTS" = true ]; then - echo -e "${BLUE}🧪 Running only unit tests...${NC}" - echo "" -fi - -if [ "$RUN_TESTS" = false ] && [ "$ONLY_TESTS" = false ]; then - echo -e "${YELLOW}ℹ️ Unit tests will be skipped${NC}" - echo "" -fi - -# Check if virtual environment is activated -if [[ -z "${VIRTUAL_ENV}" ]] && [[ -z "${CONDA_DEFAULT_ENV}" ]]; then - echo -e "${YELLOW}⚠️ Warning: No virtual environment detected${NC}" - echo -e "${YELLOW} Consider activating your venv/conda environment${NC}" - echo "" -fi - -# Function to run a validation step -run_step() { - local step_name=$1 - local step_command=$2 - - echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - echo -e "${BLUE}▶ ${step_name}${NC}" - echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - - if eval "$step_command"; then - echo -e "${GREEN}✅ ${step_name} - PASSED${NC}" - echo "" - return 0 - else - echo -e "${RED}❌ ${step_name} - FAILED${NC}" - echo "" - return 1 - fi -} - -# Track failures -FAILED_STEPS=() - -# Handle --only-tests mode -if [ "$ONLY_TESTS" = true ]; then - # Step 5: Unit Tests (pytest) - if ! run_step "Unit Tests (pytest)" "pytest tests/ --cov=model_manager --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=80 -q"; then - FAILED_STEPS+=("Unit Tests") - fi -else - # Step 1: Code Formatting Check (Ruff) - if ! run_step "1. Code Formatting (Ruff)" "ruff format --check model_manager/ tests/"; then - FAILED_STEPS+=("Code Formatting") - fi - - # Step 2: Linting (Ruff) - if ! run_step "2. Code Linting (Ruff)" "ruff check model_manager/ tests/"; then - FAILED_STEPS+=("Linting") - fi - - # Step 3: Type Checking (mypy) - if ! run_step "3. Type Checking (mypy)" "mypy model_manager/"; then - FAILED_STEPS+=("Type Checking") - fi - - # Step 4: Security Analysis (Bandit) - if ! run_step "4. Security Analysis (Bandit)" "bandit -r model_manager/ -ll -q"; then - FAILED_STEPS+=("Security Analysis") - fi - - # Step 5: Unit Tests (pytest) - if [ "$RUN_TESTS" = true ]; then - if ! run_step "5. Unit Tests (pytest)" "pytest tests/ --cov=model_manager --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=80 -q"; then - FAILED_STEPS+=("Unit Tests") - fi - else - echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - echo -e "${BLUE}▶ 5. Unit Tests (pytest)${NC}" - echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - echo -e "${YELLOW}⏭️ Unit Tests - SKIPPED${NC}" - echo "" - fi -fi - -# Summary -echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}" -echo -e "${BLUE}║ Validation Summary ║${NC}" -echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}" -echo "" - -if [ ${#FAILED_STEPS[@]} -eq 0 ]; then - echo -e "${GREEN}✅ All validation checks passed!${NC}" - echo -e "${GREEN} Your code is ready for commit/deployment.${NC}" - echo "" - exit 0 -else - echo -e "${RED}❌ Validation failed for the following steps:${NC}" - for step in "${FAILED_STEPS[@]}"; do - echo -e "${RED} • ${step}${NC}" - done - echo "" - echo -e "${YELLOW}💡 Tips:${NC}" - echo -e "${YELLOW} • Run 'ruff format model_manager/ tests/' to auto-fix formatting${NC}" - echo -e "${YELLOW} • Run 'ruff check --fix model_manager/ tests/' to auto-fix linting issues${NC}" - echo -e "${YELLOW} • Review mypy errors and add type hints where needed${NC}" - echo -e "${YELLOW} • Check bandit warnings for security issues${NC}" - echo -e "${YELLOW} • Fix failing tests or improve test coverage${NC}" - echo "" - exit 1 -fi