gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1645: Add comprehensive model training observability metrics and Grafana dashboard.

Browse Source
This commit is contained in:
vitor-aignosi
2026-06-17 10:34:56 -03:00
parent 445fe643fe
commit 76f926a8ab
12 changed files with 866 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
tests/activities/test_cleanup.py
View File

@@ -125,13 +125,11 @@ def test_cleanup_temp_directories_nonexistent_path(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
cleanup.warning = MagicMock()
cleanup.cleanup_temp_directories({'temp_path': '/nonexistent/path', 'metadata': {}})
cleanup.warning.assert_called_once()
cleanup._emit_metrics.assert_called_once()
@patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'})
@@ -152,8 +150,6 @@ def test_cleanup_temp_directories_success_with_deletions(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
old_dir = os.path.join(temp_dir, f'old_dir_{old_time}')
os.makedirs(old_dir)
@@ -166,7 +162,6 @@ def test_cleanup_temp_directories_success_with_deletions(
assert not os.path.exists(old_dir)
assert os.path.exists(recent_dir)
cleanup._emit_metrics.assert_called_once()
@patch.dict(os.environ, {'CLEANUP_DRY_RUN': 'true'})
@@ -187,8 +182,6 @@ def test_cleanup_temp_directories_dry_run(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
old_dir = os.path.join(temp_dir, f'old_dir_{old_time}')
os.makedirs(old_dir)
@@ -196,7 +189,6 @@ def test_cleanup_temp_directories_dry_run(
cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
assert os.path.exists(old_dir)
cleanup._emit_metrics.assert_called_once()
@patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'})
@@ -217,7 +209,6 @@ def test_cleanup_temp_directories_delete_error(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
cleanup.error = MagicMock()
old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
@@ -228,35 +219,9 @@ def test_cleanup_temp_directories_delete_error(
cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
cleanup.error.assert_called_once()
cleanup._emit_metrics.assert_called_once()
# --- Metrics and Utility Tests ---
def test_emit_metrics(
mock_logger,
mock_notification_handler,
mock_metrics_controller,
):
"""Test that _emit_metrics calls the public emit_metric method."""
from model_manager.activities.cleanup import Cleanup
cleanup = Cleanup(
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup.emit_metric_sync = MagicMock()
cleanup._emit_metrics(
metadata={'pod_id': 'p1', 'workflow_name': 'wf1'},
metrics_status='success',
activity_name='test_activity',
emit_workflow_metric=True,
)
assert cleanup.emit_metric_sync.call_count == 2
# --- Utility Tests ---
def test_cleanup_temp_directories_with_files_and_unmatched_dirs(
@@ -276,7 +241,6 @@ def test_cleanup_temp_directories_with_files_and_unmatched_dirs(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
cleanup.debug = MagicMock()
# Create a file and a directory with a non-matching name
@@ -290,7 +254,6 @@ def test_cleanup_temp_directories_with_files_and_unmatched_dirs(
cleanup.debug.assert_called_with(
'Skipping directory without timestamp pattern: a_directory_with_no_timestamp', {}
)
cleanup._emit_metrics.assert_called_once()
def test_cleanup_temp_directories_invalid_timestamp_format(
@@ -310,7 +273,6 @@ def test_cleanup_temp_directories_invalid_timestamp_format(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
cleanup.error = MagicMock()
# Create a directory with a malformed timestamp that matches the regex but fails parsing
@@ -320,7 +282,6 @@ def test_cleanup_temp_directories_invalid_timestamp_format(
cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
cleanup.error.assert_called_once()
cleanup._emit_metrics.assert_called_once()
def test_cleanup_temp_directories_generic_exception(
@@ -340,7 +301,6 @@ def test_cleanup_temp_directories_generic_exception(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
cleanup.send_notification = MagicMock()
with patch('os.listdir', side_effect=Exception('Unexpected OS Error')):
@@ -348,29 +308,3 @@ def test_cleanup_temp_directories_generic_exception(
cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
cleanup.send_notification.assert_called_once()
cleanup._emit_metrics.assert_called_once()
def test_emit_metrics_activity_only(
mock_logger,
mock_notification_handler,
mock_metrics_controller,
):
"""Test that _emit_metrics can emit only the activity metric."""
from model_manager.activities.cleanup import Cleanup
cleanup = Cleanup(
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup.emit_metric_sync = MagicMock()
cleanup._emit_metrics(
metadata={'pod_id': 'p1', 'workflow_name': 'wf1'},
metrics_status='success',
activity_name='test_activity',
emit_workflow_metric=False,
)
cleanup.emit_metric_sync.assert_called_once()