"""Unit tests for model_manager.metrics module. This module tests the Prometheus metrics configuration used for monitoring and observability in the Sientia DataOps Model Manager. """ def test_app_up_metric_exists(): """Test that APP_UP metric is properly defined.""" from model_manager.metrics import APP_UP assert APP_UP is not None assert APP_UP._name == 'app_up' assert ( APP_UP._documentation == 'Indicates if the application is running (1) or shutting down (0)' ) def test_app_up_metric_has_pod_id_label(): """Test that APP_UP metric has pod_id label.""" from model_manager.metrics import APP_UP assert 'pod_id' in APP_UP._labelnames def test_app_up_metric_is_gauge(): """Test that APP_UP is a Gauge metric.""" from prometheus_client import Gauge from model_manager.metrics import APP_UP assert isinstance(APP_UP, Gauge) def test_app_up_metric_can_be_set_to_one(): """Test that APP_UP metric can be set to 1 (running).""" from model_manager.metrics import APP_UP # Set metric to 1 for a specific pod APP_UP.labels(pod_id='test-pod-1').set(1) # Verify the metric value metric_value = APP_UP.labels(pod_id='test-pod-1')._value._value assert metric_value == 1 def test_app_up_metric_can_be_set_to_zero(): """Test that APP_UP metric can be set to 0 (shutting down).""" from model_manager.metrics import APP_UP # Set metric to 0 for a specific pod APP_UP.labels(pod_id='test-pod-2').set(0) # Verify the metric value metric_value = APP_UP.labels(pod_id='test-pod-2')._value._value assert metric_value == 0 def test_app_up_metric_multiple_pods(): """Test that APP_UP metric can track multiple pods independently.""" from model_manager.metrics import APP_UP # Set different values for different pods APP_UP.labels(pod_id='pod-1').set(1) APP_UP.labels(pod_id='pod-2').set(0) APP_UP.labels(pod_id='pod-3').set(1) # Verify each pod has correct value assert APP_UP.labels(pod_id='pod-1')._value._value == 1 assert APP_UP.labels(pod_id='pod-2')._value._value == 0 assert APP_UP.labels(pod_id='pod-3')._value._value == 1 def test_app_up_metric_default_value(): """Test that APP_UP metric starts with no value set.""" # Create a new label that hasn't been used yet import uuid from model_manager.metrics import APP_UP unique_pod = f'test-pod-{uuid.uuid4()}' # The metric should exist but not have a value until set metric = APP_UP.labels(pod_id=unique_pod) assert metric is not None def test_metrics_module_imports(): """Test that metrics module can be imported successfully.""" import model_manager.metrics assert hasattr(model_manager.metrics, 'APP_UP') assert hasattr(model_manager.metrics, 'Gauge') def test_metrics_module_docstring(): """Test that metrics module has proper documentation.""" import model_manager.metrics assert model_manager.metrics.__doc__ is not None assert 'Prometheus' in model_manager.metrics.__doc__ assert 'metrics' in model_manager.metrics.__doc__ def test_app_up_metric_can_increment(): """Test that APP_UP metric value can be incremented.""" from model_manager.metrics import APP_UP pod_id = 'test-pod-increment' APP_UP.labels(pod_id=pod_id).set(0) # Increment the metric APP_UP.labels(pod_id=pod_id).inc() metric_value = APP_UP.labels(pod_id=pod_id)._value._value assert metric_value == 1 def test_app_up_metric_can_decrement(): """Test that APP_UP metric value can be decremented.""" from model_manager.metrics import APP_UP pod_id = 'test-pod-decrement' APP_UP.labels(pod_id=pod_id).set(1) # Decrement the metric APP_UP.labels(pod_id=pod_id).dec() metric_value = APP_UP.labels(pod_id=pod_id)._value._value assert metric_value == 0 def test_app_up_metric_set_to_timestamp(): """Test that APP_UP metric can be set to current timestamp.""" import time from model_manager.metrics import APP_UP pod_id = 'test-pod-timestamp' current_time = time.time() # Set to timestamp APP_UP.labels(pod_id=pod_id).set_to_current_time() metric_value = APP_UP.labels(pod_id=pod_id)._value._value # Should be close to current time assert abs(metric_value - current_time) < 2 # Within 2 seconds def test_app_up_metric_label_validation(): """Test that APP_UP metric validates label names.""" from model_manager.metrics import APP_UP # Should work with valid label APP_UP.labels(pod_id='valid-pod-name').set(1) # Should work with empty string (though not recommended) APP_UP.labels(pod_id='').set(1) # Should work with special characters APP_UP.labels(pod_id='pod-123_test.example').set(1) def test_module_exports(): """Test that metrics module exports expected symbols.""" import model_manager.metrics as metrics_module # Check that module has the expected exports module_contents = dir(metrics_module) assert 'APP_UP' in module_contents assert 'Gauge' in module_contents def test_app_up_metric_thread_safety(): """Test that APP_UP metric is thread-safe.""" import threading from model_manager.metrics import APP_UP pod_id = 'test-pod-threading' APP_UP.labels(pod_id=pod_id).set(0) def increment_metric(): for _ in range(100): APP_UP.labels(pod_id=pod_id).inc() # Create multiple threads that increment the metric threads = [threading.Thread(target=increment_metric) for _ in range(5)] for thread in threads: thread.start() for thread in threads: thread.join() # Should have incremented 500 times total metric_value = APP_UP.labels(pod_id=pod_id)._value._value assert metric_value == 500 def test_prometheus_client_gauge_import(): """Test that Gauge is properly imported from prometheus_client.""" from prometheus_client import Gauge as PrometheusGauge from model_manager.metrics import Gauge assert Gauge is PrometheusGauge # --------------------------------------------------------------------------- # Training metrics — existence, type, and labels # --------------------------------------------------------------------------- _TRAINING_LABEL_NAMES = ('pod_id', 'model_name', 'model_type') def _assert_training_labels(metric): for label in _TRAINING_LABEL_NAMES: assert label in metric._labelnames def test_sientia_training_data_preparation_lag_is_histogram(): from prometheus_client import Histogram from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_LAG assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_LAG, Histogram) assert SIENTIA_TRAINING_DATA_PREPARATION_LAG._name == 'sientia_training_data_preparation_lag' _assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_LAG) def test_sientia_training_data_preparation_error_count_total_is_counter(): from prometheus_client import Counter from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL, Counter) assert ( 'sientia_training_data_preparation_error_count' in SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL._name ) _assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL) def test_sientia_training_model_fit_lag_is_histogram(): from prometheus_client import Histogram from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_LAG assert isinstance(SIENTIA_TRAINING_MODEL_FIT_LAG, Histogram) assert SIENTIA_TRAINING_MODEL_FIT_LAG._name == 'sientia_training_model_fit_lag' _assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_LAG) def test_sientia_training_model_fit_error_count_total_is_counter(): from prometheus_client import Counter from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL assert isinstance(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL, Counter) assert ( 'sientia_training_model_fit_error_count' in SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL._name ) _assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL) def test_sientia_training_model_quality_mse_is_gauge(): from prometheus_client import Gauge from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MSE assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MSE, Gauge) assert SIENTIA_TRAINING_MODEL_QUALITY_MSE._name == 'sientia_training_model_quality_mse' _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MSE) def test_sientia_training_model_quality_mae_is_gauge(): from prometheus_client import Gauge from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MAE assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MAE, Gauge) assert SIENTIA_TRAINING_MODEL_QUALITY_MAE._name == 'sientia_training_model_quality_mae' _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MAE) def test_sientia_training_model_quality_r2_is_gauge(): from prometheus_client import Gauge from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_R2 assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_R2, Gauge) assert SIENTIA_TRAINING_MODEL_QUALITY_R2._name == 'sientia_training_model_quality_r2' _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_R2) def test_sientia_training_dataset_train_rows_is_gauge(): from prometheus_client import Gauge from model_manager.metrics import SIENTIA_TRAINING_DATASET_TRAIN_ROWS assert isinstance(SIENTIA_TRAINING_DATASET_TRAIN_ROWS, Gauge) assert SIENTIA_TRAINING_DATASET_TRAIN_ROWS._name == 'sientia_training_dataset_train_rows' _assert_training_labels(SIENTIA_TRAINING_DATASET_TRAIN_ROWS) def test_sientia_training_dataset_val_rows_is_gauge(): from prometheus_client import Gauge from model_manager.metrics import SIENTIA_TRAINING_DATASET_VAL_ROWS assert isinstance(SIENTIA_TRAINING_DATASET_VAL_ROWS, Gauge) assert SIENTIA_TRAINING_DATASET_VAL_ROWS._name == 'sientia_training_dataset_val_rows' _assert_training_labels(SIENTIA_TRAINING_DATASET_VAL_ROWS) def test_sientia_training_model_trained_total_is_counter(): from prometheus_client import Counter from model_manager.metrics import SIENTIA_TRAINING_MODEL_TRAINED_TOTAL assert isinstance(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL, Counter) assert 'sientia_training_model_trained' in SIENTIA_TRAINING_MODEL_TRAINED_TOTAL._name _assert_training_labels(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL) def test_sientia_training_feature_count_is_gauge(): from prometheus_client import Gauge from model_manager.metrics import SIENTIA_TRAINING_FEATURE_COUNT assert isinstance(SIENTIA_TRAINING_FEATURE_COUNT, Gauge) assert SIENTIA_TRAINING_FEATURE_COUNT._name == 'sientia_training_feature_count' _assert_training_labels(SIENTIA_TRAINING_FEATURE_COUNT) def test_sientia_training_info_is_gauge(): from prometheus_client import Gauge from model_manager.metrics import SIENTIA_TRAINING_INFO assert isinstance(SIENTIA_TRAINING_INFO, Gauge) assert SIENTIA_TRAINING_INFO._name == 'sientia_training_info' expected_labels = { 'pod_id', 'model_name', 'model_type', 'dataset_train_rows', 'dataset_val_rows', 'feature_count', 'mse', 'mae', 'r2', } assert expected_labels == set(SIENTIA_TRAINING_INFO._labelnames) def test_sientia_training_info_set_value(): import time from model_manager.metrics import SIENTIA_TRAINING_INFO ts = time.time() * 1000 SIENTIA_TRAINING_INFO.labels( pod_id='test-pod', model_name='my_model', model_type='linear', dataset_train_rows='1000', dataset_val_rows='200', feature_count='5', mse='0.01', mae='0.08', r2='0.95', ).set(ts) value = SIENTIA_TRAINING_INFO.labels( pod_id='test-pod', model_name='my_model', model_type='linear', dataset_train_rows='1000', dataset_val_rows='200', feature_count='5', mse='0.01', mae='0.08', r2='0.95', )._value._value assert abs(value - ts) < 2000