SIENTIAPDE-1325
Refactor monitoring and metrics integration across various components - Removed coverage options from `pyproject.toml`. - Updated prediction metrics in `README.md` to replace `pipeline_name` with `workflow_name`. - Upgraded `sientia-dataops-library` dependency version in `requirements-light.txt` and `requirements.txt`. - Enhanced metrics handling in `laborious` activities, including `Activities`, `Gates`, `MLFlow`, and `OPC`, to utilize a new `MetricsController`. - Refactored metric emission methods to improve clarity and consistency across the codebase. - Updated tests to reflect changes in metrics handling and ensure proper functionality.
This commit is contained in:
vitor-aignosi
@@ -10,7 +10,8 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now
|
||||
|
||||
from laborious import metrics
|
||||
@@ -62,7 +63,7 @@ mlflow_content_path_confidence: Mapping[str, int] = {
|
||||
}
|
||||
|
||||
|
||||
class Gates(BaseActivity):
|
||||
class Gates(SientiaMonitoring):
|
||||
"""
|
||||
Data quality gates and filtering activities for the Laborious system.
|
||||
|
||||
@@ -82,7 +83,12 @@ class Gates(BaseActivity):
|
||||
mlflow_content_filter_functions (dict): Mapping of MLFlow content filter names to functions
|
||||
"""
|
||||
|
||||
def __init__(self, logger: Logger, notification_handler: NotificationHandler):
|
||||
def __init__(
|
||||
self,
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
metrics_controller: MetricsController,
|
||||
):
|
||||
"""
|
||||
Initialize data quality gates with logging and notification capabilities.
|
||||
|
||||
@@ -93,7 +99,16 @@ class Gates(BaseActivity):
|
||||
Raises:
|
||||
Exception: If BaseActivity initialization fails
|
||||
"""
|
||||
BaseActivity.__init__(self, logger, notification_handler, set_error_counter=True)
|
||||
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Close the gates activity and clean up resources.
|
||||
"""
|
||||
SientiaMonitoring.shutdown(self)
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
@activity.defn(name='input_gate')
|
||||
async def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
@@ -153,7 +168,7 @@ class Gates(BaseActivity):
|
||||
filter_output.append(config['policy'])
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
await self.send_notification_async(
|
||||
metadata=metadata,
|
||||
notification_id=f'INTPUT_GATE_ERROR__{fil}',
|
||||
message=f'Error in filter {fil}:{config}: \n {e}',
|
||||
@@ -225,7 +240,7 @@ class Gates(BaseActivity):
|
||||
if mlflow_response_filter_functions[fil](data, config):
|
||||
filter_output.append(config['policy'])
|
||||
comments.append(data['content']['message'])
|
||||
self.send_notification(
|
||||
await self.send_notification_async(
|
||||
metadata=metadata,
|
||||
notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
|
||||
message=data['content']['message'],
|
||||
@@ -235,7 +250,7 @@ class Gates(BaseActivity):
|
||||
)
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
await self.send_notification_async(
|
||||
metadata=metadata,
|
||||
notification_id=f'MLFLOW_GATE_RESPONSE_FILTER__{fil}',
|
||||
message=f'Error in filter {fil}:{config}: \n {e}',
|
||||
@@ -305,7 +320,7 @@ class Gates(BaseActivity):
|
||||
try:
|
||||
if mlflow_content_filter_functions[fil](data, config):
|
||||
filter_output.append(config['policy'])
|
||||
self.send_notification(
|
||||
await self.send_notification_async(
|
||||
metadata=metadata,
|
||||
notification_id=f'{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}',
|
||||
message=f'Data not passed the content filter {fil}:{config}',
|
||||
@@ -315,7 +330,7 @@ class Gates(BaseActivity):
|
||||
)
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
await self.send_notification_async(
|
||||
metadata=metadata,
|
||||
notification_id=f'MLFLOW_GATE_CONTENT_FILTER__{fil}',
|
||||
message=f'Error in filter {fil}:{config}: \n {e}',
|
||||
@@ -608,39 +623,61 @@ class Gates(BaseActivity):
|
||||
|
||||
self.info(f'Writing metrics for model {metadata["model_name"]}', metadata)
|
||||
|
||||
metrics.PREDICTIONS_WRITTEN_COUNT.labels(
|
||||
pod_id=self.pod_id,
|
||||
model_name=metadata['model_name'],
|
||||
pipeline_name=metadata['workflow_name'],
|
||||
).inc()
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.PREDICTIONS_WRITTEN_COUNT,
|
||||
tags={
|
||||
'pod_id': self.pod_id,
|
||||
'model_name': metadata['model_name'],
|
||||
'workflow_name': metadata['workflow_name'],
|
||||
},
|
||||
)
|
||||
|
||||
metrics.PREDICTION_CONFIDENCE_MONITOR.labels(
|
||||
pod_id=self.pod_id,
|
||||
model_name=metadata['model_name'],
|
||||
pipeline_name=metadata['workflow_name'],
|
||||
).set(prediction_confidence)
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.PREDICTION_CONFIDENCE_MONITOR,
|
||||
method='set',
|
||||
tags={
|
||||
'pod_id': self.pod_id,
|
||||
'model_name': metadata['model_name'],
|
||||
'workflow_name': metadata['workflow_name'],
|
||||
},
|
||||
value=prediction_confidence,
|
||||
)
|
||||
|
||||
metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels(
|
||||
pod_id=self.pod_id,
|
||||
model_name=metadata['model_name'],
|
||||
pipeline_name=metadata['workflow_name'],
|
||||
).observe(response_time)
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.PREDICTION_RESPONSE_TIME_MONITOR,
|
||||
method='observe',
|
||||
tags={
|
||||
'pod_id': self.pod_id,
|
||||
'model_name': metadata['model_name'],
|
||||
'workflow_name': metadata['workflow_name'],
|
||||
},
|
||||
value=response_time,
|
||||
)
|
||||
|
||||
for server_id, tags in opc_metrics.items():
|
||||
for tag, response_time in tags.items():
|
||||
metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR.labels(
|
||||
pod_id=self.pod_id,
|
||||
model_name=metadata['model_name'],
|
||||
pipeline_name=metadata['workflow_name'],
|
||||
opc_server_id=server_id,
|
||||
tag=tag,
|
||||
).observe(response_time)
|
||||
metrics.PREDICTION_OPC_WRITING_COUNT.labels(
|
||||
pod_id=self.pod_id,
|
||||
model_name=metadata['model_name'],
|
||||
pipeline_name=metadata['workflow_name'],
|
||||
opc_server_id=server_id,
|
||||
tag=tag,
|
||||
).inc()
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR,
|
||||
method='observe',
|
||||
tags={
|
||||
'pod_id': self.pod_id,
|
||||
'model_name': metadata['model_name'],
|
||||
'workflow_name': metadata['workflow_name'],
|
||||
'opc_server_id': server_id,
|
||||
'tag': tag,
|
||||
},
|
||||
value=response_time,
|
||||
)
|
||||
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.PREDICTION_OPC_WRITING_COUNT,
|
||||
tags={
|
||||
'pod_id': self.pod_id,
|
||||
'model_name': metadata['model_name'],
|
||||
'workflow_name': metadata['workflow_name'],
|
||||
'opc_server_id': server_id,
|
||||
'tag': tag,
|
||||
},
|
||||
)
|
||||
|
||||
self.info(f'Metrics written for model {metadata["model_name"]}', metadata)
|
||||
|
||||
Reference in New Issue
Block a user