392 lines
12 KiB
Python
392 lines
12 KiB
Python
"""Unit tests for model_manager.metrics module.
|
|
|
|
This module tests the Prometheus metrics configuration used for
|
|
monitoring and observability in the Sientia DataOps Model Manager.
|
|
"""
|
|
|
|
|
|
def test_app_up_metric_exists():
|
|
"""Test that APP_UP metric is properly defined."""
|
|
from model_manager.metrics import APP_UP
|
|
|
|
assert APP_UP is not None
|
|
assert APP_UP._name == 'app_up'
|
|
assert (
|
|
APP_UP._documentation == 'Indicates if the application is running (1) or shutting down (0)'
|
|
)
|
|
|
|
|
|
def test_app_up_metric_has_pod_id_label():
|
|
"""Test that APP_UP metric has pod_id label."""
|
|
from model_manager.metrics import APP_UP
|
|
|
|
assert 'pod_id' in APP_UP._labelnames
|
|
|
|
|
|
def test_app_up_metric_is_gauge():
|
|
"""Test that APP_UP is a Gauge metric."""
|
|
from prometheus_client import Gauge
|
|
|
|
from model_manager.metrics import APP_UP
|
|
|
|
assert isinstance(APP_UP, Gauge)
|
|
|
|
|
|
def test_app_up_metric_can_be_set_to_one():
|
|
"""Test that APP_UP metric can be set to 1 (running)."""
|
|
from model_manager.metrics import APP_UP
|
|
|
|
# Set metric to 1 for a specific pod
|
|
APP_UP.labels(pod_id='test-pod-1').set(1)
|
|
|
|
# Verify the metric value
|
|
metric_value = APP_UP.labels(pod_id='test-pod-1')._value._value
|
|
assert metric_value == 1
|
|
|
|
|
|
def test_app_up_metric_can_be_set_to_zero():
|
|
"""Test that APP_UP metric can be set to 0 (shutting down)."""
|
|
from model_manager.metrics import APP_UP
|
|
|
|
# Set metric to 0 for a specific pod
|
|
APP_UP.labels(pod_id='test-pod-2').set(0)
|
|
|
|
# Verify the metric value
|
|
metric_value = APP_UP.labels(pod_id='test-pod-2')._value._value
|
|
assert metric_value == 0
|
|
|
|
|
|
def test_app_up_metric_multiple_pods():
|
|
"""Test that APP_UP metric can track multiple pods independently."""
|
|
from model_manager.metrics import APP_UP
|
|
|
|
# Set different values for different pods
|
|
APP_UP.labels(pod_id='pod-1').set(1)
|
|
APP_UP.labels(pod_id='pod-2').set(0)
|
|
APP_UP.labels(pod_id='pod-3').set(1)
|
|
|
|
# Verify each pod has correct value
|
|
assert APP_UP.labels(pod_id='pod-1')._value._value == 1
|
|
assert APP_UP.labels(pod_id='pod-2')._value._value == 0
|
|
assert APP_UP.labels(pod_id='pod-3')._value._value == 1
|
|
|
|
|
|
def test_app_up_metric_default_value():
|
|
"""Test that APP_UP metric starts with no value set."""
|
|
# Create a new label that hasn't been used yet
|
|
import uuid
|
|
|
|
from model_manager.metrics import APP_UP
|
|
|
|
unique_pod = f'test-pod-{uuid.uuid4()}'
|
|
|
|
# The metric should exist but not have a value until set
|
|
metric = APP_UP.labels(pod_id=unique_pod)
|
|
assert metric is not None
|
|
|
|
|
|
def test_metrics_module_imports():
|
|
"""Test that metrics module can be imported successfully."""
|
|
import model_manager.metrics
|
|
|
|
assert hasattr(model_manager.metrics, 'APP_UP')
|
|
assert hasattr(model_manager.metrics, 'Gauge')
|
|
|
|
|
|
def test_metrics_module_docstring():
|
|
"""Test that metrics module has proper documentation."""
|
|
import model_manager.metrics
|
|
|
|
assert model_manager.metrics.__doc__ is not None
|
|
assert 'Prometheus' in model_manager.metrics.__doc__
|
|
assert 'metrics' in model_manager.metrics.__doc__
|
|
|
|
|
|
def test_app_up_metric_can_increment():
|
|
"""Test that APP_UP metric value can be incremented."""
|
|
from model_manager.metrics import APP_UP
|
|
|
|
pod_id = 'test-pod-increment'
|
|
APP_UP.labels(pod_id=pod_id).set(0)
|
|
|
|
# Increment the metric
|
|
APP_UP.labels(pod_id=pod_id).inc()
|
|
|
|
metric_value = APP_UP.labels(pod_id=pod_id)._value._value
|
|
assert metric_value == 1
|
|
|
|
|
|
def test_app_up_metric_can_decrement():
|
|
"""Test that APP_UP metric value can be decremented."""
|
|
from model_manager.metrics import APP_UP
|
|
|
|
pod_id = 'test-pod-decrement'
|
|
APP_UP.labels(pod_id=pod_id).set(1)
|
|
|
|
# Decrement the metric
|
|
APP_UP.labels(pod_id=pod_id).dec()
|
|
|
|
metric_value = APP_UP.labels(pod_id=pod_id)._value._value
|
|
assert metric_value == 0
|
|
|
|
|
|
def test_app_up_metric_set_to_timestamp():
|
|
"""Test that APP_UP metric can be set to current timestamp."""
|
|
import time
|
|
|
|
from model_manager.metrics import APP_UP
|
|
|
|
pod_id = 'test-pod-timestamp'
|
|
current_time = time.time()
|
|
|
|
# Set to timestamp
|
|
APP_UP.labels(pod_id=pod_id).set_to_current_time()
|
|
|
|
metric_value = APP_UP.labels(pod_id=pod_id)._value._value
|
|
|
|
# Should be close to current time
|
|
assert abs(metric_value - current_time) < 2 # Within 2 seconds
|
|
|
|
|
|
def test_app_up_metric_label_validation():
|
|
"""Test that APP_UP metric validates label names."""
|
|
from model_manager.metrics import APP_UP
|
|
|
|
# Should work with valid label
|
|
APP_UP.labels(pod_id='valid-pod-name').set(1)
|
|
|
|
# Should work with empty string (though not recommended)
|
|
APP_UP.labels(pod_id='').set(1)
|
|
|
|
# Should work with special characters
|
|
APP_UP.labels(pod_id='pod-123_test.example').set(1)
|
|
|
|
|
|
def test_module_exports():
|
|
"""Test that metrics module exports expected symbols."""
|
|
import model_manager.metrics as metrics_module
|
|
|
|
# Check that module has the expected exports
|
|
module_contents = dir(metrics_module)
|
|
|
|
assert 'APP_UP' in module_contents
|
|
assert 'Gauge' in module_contents
|
|
|
|
|
|
def test_app_up_metric_thread_safety():
|
|
"""Test that APP_UP metric is thread-safe."""
|
|
import threading
|
|
|
|
from model_manager.metrics import APP_UP
|
|
|
|
pod_id = 'test-pod-threading'
|
|
APP_UP.labels(pod_id=pod_id).set(0)
|
|
|
|
def increment_metric():
|
|
for _ in range(100):
|
|
APP_UP.labels(pod_id=pod_id).inc()
|
|
|
|
# Create multiple threads that increment the metric
|
|
threads = [threading.Thread(target=increment_metric) for _ in range(5)]
|
|
|
|
for thread in threads:
|
|
thread.start()
|
|
|
|
for thread in threads:
|
|
thread.join()
|
|
|
|
# Should have incremented 500 times total
|
|
metric_value = APP_UP.labels(pod_id=pod_id)._value._value
|
|
assert metric_value == 500
|
|
|
|
|
|
def test_prometheus_client_gauge_import():
|
|
"""Test that Gauge is properly imported from prometheus_client."""
|
|
from prometheus_client import Gauge as PrometheusGauge
|
|
|
|
from model_manager.metrics import Gauge
|
|
|
|
assert Gauge is PrometheusGauge
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Training metrics — existence, type, and labels
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_TRAINING_LABEL_NAMES = ('pod_id', 'model_name', 'model_type')
|
|
|
|
|
|
def _assert_training_labels(metric):
|
|
for label in _TRAINING_LABEL_NAMES:
|
|
assert label in metric._labelnames
|
|
|
|
|
|
def test_sientia_training_data_preparation_lag_is_histogram():
|
|
from prometheus_client import Histogram
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_LAG
|
|
|
|
assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_LAG, Histogram)
|
|
assert SIENTIA_TRAINING_DATA_PREPARATION_LAG._name == 'sientia_training_data_preparation_lag'
|
|
_assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_LAG)
|
|
|
|
|
|
def test_sientia_training_data_preparation_error_count_total_is_counter():
|
|
from prometheus_client import Counter
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL
|
|
|
|
assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL, Counter)
|
|
assert (
|
|
'sientia_training_data_preparation_error_count'
|
|
in SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL._name
|
|
)
|
|
_assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL)
|
|
|
|
|
|
def test_sientia_training_model_fit_lag_is_histogram():
|
|
from prometheus_client import Histogram
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_LAG
|
|
|
|
assert isinstance(SIENTIA_TRAINING_MODEL_FIT_LAG, Histogram)
|
|
assert SIENTIA_TRAINING_MODEL_FIT_LAG._name == 'sientia_training_model_fit_lag'
|
|
_assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_LAG)
|
|
|
|
|
|
def test_sientia_training_model_fit_error_count_total_is_counter():
|
|
from prometheus_client import Counter
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL
|
|
|
|
assert isinstance(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL, Counter)
|
|
assert (
|
|
'sientia_training_model_fit_error_count'
|
|
in SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL._name
|
|
)
|
|
_assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL)
|
|
|
|
|
|
def test_sientia_training_model_quality_mse_is_gauge():
|
|
from prometheus_client import Gauge
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MSE
|
|
|
|
assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MSE, Gauge)
|
|
assert SIENTIA_TRAINING_MODEL_QUALITY_MSE._name == 'sientia_training_model_quality_mse'
|
|
_assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MSE)
|
|
|
|
|
|
def test_sientia_training_model_quality_mae_is_gauge():
|
|
from prometheus_client import Gauge
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MAE
|
|
|
|
assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MAE, Gauge)
|
|
assert SIENTIA_TRAINING_MODEL_QUALITY_MAE._name == 'sientia_training_model_quality_mae'
|
|
_assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MAE)
|
|
|
|
|
|
def test_sientia_training_model_quality_r2_is_gauge():
|
|
from prometheus_client import Gauge
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_R2
|
|
|
|
assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_R2, Gauge)
|
|
assert SIENTIA_TRAINING_MODEL_QUALITY_R2._name == 'sientia_training_model_quality_r2'
|
|
_assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_R2)
|
|
|
|
|
|
def test_sientia_training_dataset_train_rows_is_gauge():
|
|
from prometheus_client import Gauge
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_DATASET_TRAIN_ROWS
|
|
|
|
assert isinstance(SIENTIA_TRAINING_DATASET_TRAIN_ROWS, Gauge)
|
|
assert SIENTIA_TRAINING_DATASET_TRAIN_ROWS._name == 'sientia_training_dataset_train_rows'
|
|
_assert_training_labels(SIENTIA_TRAINING_DATASET_TRAIN_ROWS)
|
|
|
|
|
|
def test_sientia_training_dataset_val_rows_is_gauge():
|
|
from prometheus_client import Gauge
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_DATASET_VAL_ROWS
|
|
|
|
assert isinstance(SIENTIA_TRAINING_DATASET_VAL_ROWS, Gauge)
|
|
assert SIENTIA_TRAINING_DATASET_VAL_ROWS._name == 'sientia_training_dataset_val_rows'
|
|
_assert_training_labels(SIENTIA_TRAINING_DATASET_VAL_ROWS)
|
|
|
|
|
|
def test_sientia_training_model_trained_total_is_counter():
|
|
from prometheus_client import Counter
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_MODEL_TRAINED_TOTAL
|
|
|
|
assert isinstance(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL, Counter)
|
|
assert 'sientia_training_model_trained' in SIENTIA_TRAINING_MODEL_TRAINED_TOTAL._name
|
|
_assert_training_labels(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL)
|
|
|
|
|
|
def test_sientia_training_feature_count_is_gauge():
|
|
from prometheus_client import Gauge
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_FEATURE_COUNT
|
|
|
|
assert isinstance(SIENTIA_TRAINING_FEATURE_COUNT, Gauge)
|
|
assert SIENTIA_TRAINING_FEATURE_COUNT._name == 'sientia_training_feature_count'
|
|
_assert_training_labels(SIENTIA_TRAINING_FEATURE_COUNT)
|
|
|
|
|
|
def test_sientia_training_info_is_gauge():
|
|
from prometheus_client import Gauge
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_INFO
|
|
|
|
assert isinstance(SIENTIA_TRAINING_INFO, Gauge)
|
|
assert SIENTIA_TRAINING_INFO._name == 'sientia_training_info'
|
|
|
|
expected_labels = {
|
|
'pod_id',
|
|
'model_name',
|
|
'model_type',
|
|
'dataset_train_rows',
|
|
'dataset_val_rows',
|
|
'feature_count',
|
|
'mse',
|
|
'mae',
|
|
'r2',
|
|
}
|
|
assert expected_labels == set(SIENTIA_TRAINING_INFO._labelnames)
|
|
|
|
|
|
def test_sientia_training_info_set_value():
|
|
import time
|
|
|
|
from model_manager.metrics import SIENTIA_TRAINING_INFO
|
|
|
|
ts = time.time() * 1000
|
|
SIENTIA_TRAINING_INFO.labels(
|
|
pod_id='test-pod',
|
|
model_name='my_model',
|
|
model_type='linear',
|
|
dataset_train_rows='1000',
|
|
dataset_val_rows='200',
|
|
feature_count='5',
|
|
mse='0.01',
|
|
mae='0.08',
|
|
r2='0.95',
|
|
).set(ts)
|
|
|
|
value = SIENTIA_TRAINING_INFO.labels(
|
|
pod_id='test-pod',
|
|
model_name='my_model',
|
|
model_type='linear',
|
|
dataset_train_rows='1000',
|
|
dataset_val_rows='200',
|
|
feature_count='5',
|
|
mse='0.01',
|
|
mae='0.08',
|
|
r2='0.95',
|
|
)._value._value
|
|
assert abs(value - ts) < 2000
|