SIENTIAPDE-1325

Refactor monitoring and metrics integration across various components

- Removed coverage options from `pyproject.toml`.
- Updated prediction metrics in `README.md` to replace `pipeline_name` with `workflow_name`.
- Upgraded `sientia-dataops-library` dependency version in `requirements-light.txt` and `requirements.txt`.
- Enhanced metrics handling in `laborious` activities, including `Activities`, `Gates`, `MLFlow`, and `OPC`, to utilize a new `MetricsController`.
- Refactored metric emission methods to improve clarity and consistency across the codebase.
- Updated tests to reflect changes in metrics handling and ensure proper functionality.

laborious/metrics.py

@@ -19,11 +19,12 @@ Key Metric Categories:
 Metric Labels:
 - pod_id: Kubernetes pod identifier for multi-instance deployments
 - model_name: Name of the ML model being used
-- pipeline_name: Name of the prediction pipeline
+- workflow_name: Name of the prediction pipeline
 - opc_server_id: Identifier for OPC server operations
 """
 
 from prometheus_client import Counter, Gauge, Histogram
+from sientia_do.observability.metrics import CORE_LABELS as SIENTIA_CORE_LABELS
 
 # Application health metric
 APP_UP = Gauge(
@@ -33,7 +34,7 @@ APP_UP = Gauge(
 )
 
 # Core labels used across multiple metrics
-CORE_LABELS = ['pod_id', 'model_name', 'pipeline_name']
+CORE_LABELS = ['pod_id', 'model_name', 'workflow_name']
 
 # Prediction operation metrics
 PREDICTIONS_WRITTEN_COUNT = Counter(
@@ -49,7 +50,7 @@ PREDICTION_CONFIDENCE_MONITOR = Gauge(
     CORE_LABELS,
 )
 
-# Performance monitoring metrics
+# Prediction total response time
 PREDICTION_RESPONSE_TIME_MONITOR = Histogram(
     'laborious_prediction_response_time_monitor',
     'Current response time of each prediction',
@@ -57,7 +58,49 @@ PREDICTION_RESPONSE_TIME_MONITOR = Histogram(
     buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
 )
 
-# OPC export metrics
+# ================== MinIO metrics ==================
+
+MINIO_READ_LAG = Histogram(
+    'laborious_minio_read_lag',
+    'Lag between the last write to MinIO and the last read from MinIO',
+    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
+    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
+)
+
+MINIO_WRITE_LAG = Histogram(
+    'laborious_minio_write_lag',
+    'Lag between the last write to MinIO and the last read from MinIO',
+    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
+    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
+)
+
+MINIO_READ_COUNT = Counter(
+    'laborious_minio_read_count',
+    'Number of reads from MinIO',
+    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
+)
+
+MINIO_WRITE_COUNT = Counter(
+    'laborious_minio_write_count',
+    'Number of writes to MinIO',
+    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
+)
+
+MINIO_READ_ERROR_COUNT = Counter(
+    'laborious_minio_read_error_count',
+    'Number of errors reading from MinIO',
+    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
+)
+
+MINIO_WRITE_ERROR_COUNT = Counter(
+    'laborious_minio_write_error_count',
+    'Number of errors writing to MinIO',
+    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
+)
+
+# ================== OPC metrics ==================
+
+
 PREDICTION_OPC_WRITING_COUNT = Counter(
     'laborious_prediction_opc_writing_count',
     'Number of predictions written to the OPC server',
@@ -70,3 +113,49 @@ PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR = Histogram(
     [*CORE_LABELS, 'opc_server_id', 'tag'],
     buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
 )
+
+OPC_CONNECTION_STATUS = Gauge(
+    'laborious_opc_connection_status',
+    'Connection status with the OPC server (1=connected, 0=disconnected)',
+    ['pod_id', 'opc_server_id'],
+)
+
+# ================== Model metrics ==================
+
+MODEL_READ_LAG = Histogram(
+    'laborious_model_read_lag',
+    'Lag between the start and read of read operations',
+    SIENTIA_CORE_LABELS,
+    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
+)
+
+MODEL_WRITE_LAG = Histogram(
+    'laborious_model_write_lag',
+    'Lag between the start and end of write operations',
+    SIENTIA_CORE_LABELS,
+    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
+)
+
+MODEL_READ_COUNT = Counter(
+    'laborious_model_read_count',
+    'Number of reads from the model',
+    SIENTIA_CORE_LABELS,
+)
+
+MODEL_WRITE_COUNT = Counter(
+    'laborious_model_write_count',
+    'Number of writes to the model',
+    SIENTIA_CORE_LABELS,
+)
+
+MODEL_READ_ERROR_COUNT = Counter(
+    'laborious_model_read_error_count',
+    'Number of errors reading from the model',
+    SIENTIA_CORE_LABELS,
+)
+
+MODEL_WRITE_ERROR_COUNT = Counter(
+    'laborious_model_write_error_count',
+    'Number of errors writing to the model',
+    SIENTIA_CORE_LABELS,
+)