gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1314

Browse Source 
Enhance OPC Metrics Handling and Refactor Write Operations

- Updated the OPC class to return response times for write operations, improving metrics tracking.
- Refactored the Gates activity to incorporate OPC metrics into the metrics writing process.
- Adjusted the manage_output_tags method in OpcRepository to return response times for each tag written.
- Modified tests to validate the new metrics structure and ensure correct behavior of the updated methods.
This commit is contained in:
vitor-aignosi
2025-10-23 17:48:43 -03:00
parent 32a76b35ea
commit 0324e2e143
10 changed files with 177 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
laborious/activities/gates.py
View File

@@ -604,6 +604,7 @@ class Gates(BaseActivity):
prediction = DataFrame(input_data['prediction'])
prediction_confidence = prediction['prediction_confidence'].values[0]
response_time = prediction['response_time'].values[0]
opc_metrics = input_data['opc_metrics']
self.info(f'Writing metrics for model {metadata["model_name"]}', metadata)
@@ -625,4 +626,21 @@ class Gates(BaseActivity):
pipeline_name=metadata['workflow_name'],
).observe(response_time)
for server_id, tags in opc_metrics.items():
for tag, response_time in tags.items():
metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name'],
opc_server_id=server_id,
tag=tag,
).observe(response_time)
metrics.PREDICTION_OPC_WRITING_COUNT.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name'],
opc_server_id=server_id,
tag=tag,
).inc()
self.info(f'Metrics written for model {metadata["model_name"]}', metadata)

52
laborious/activities/opc.py
View File

@@ -112,7 +112,7 @@ class OPC(BaseActivity):
data_type: str,
tag_type: str,
metadata: dict[str, Any],
) -> bool:
) -> float | None:
"""
Write data to a specific OPC server tag with comprehensive error handling.
@@ -133,20 +133,20 @@ class OPC(BaseActivity):
"""
try:
is_success, error_data = await self.opc_repository[server_id].write_data(
is_success, info_data = await self.opc_repository[server_id].write_data(
tag, data, data_type, self.logger, metadata
)
if not is_success:
self.send_notification(
metadata=metadata,
notification_id=error_data['notification_id'],
message=error_data['message'],
block=error_data['block'],
level=error_data.get('level', NotificationLevel.ERROR),
attachment_content=error_data.get('attachment_content', None),
notification_id=info_data['notification_id'],
message=info_data['message'],
block=info_data['block'],
level=info_data.get('level', NotificationLevel.ERROR),
attachment_content=info_data.get('attachment_content', None),
)
return False
return True
return None
return info_data['response_time']
except Exception as e:
trace = traceback.format_exc()
self.send_notification(
@@ -200,7 +200,7 @@ class OPC(BaseActivity):
data: DataFrame,
metadata: dict[str, Any],
success: bool,
) -> tuple[bool, int]:
) -> tuple[bool, dict[str, float | None]]:
"""
Manage the writing of prediction and confidence data to OPC server tags.
@@ -228,10 +228,11 @@ class OPC(BaseActivity):
- total_tags_written: Count of successfully written tags
"""
count = 0
response_times: dict[str, float | None] = {}
if 'prediction_tags' in config:
for tag, tag_config in config['prediction_tags'].items():
local_success = await self.write_data(
response_time = await self.write_data(
server_id=server_id,
tag=tag,
data=data.head(1)['prediction'].values[0],
@@ -239,17 +240,16 @@ class OPC(BaseActivity):
tag_type='prediction',
metadata=metadata,
)
if local_success:
if response_time is not None:
self.info(
f'Prediction data written to OPC server {server_id} for tag {tag}.',
metadata,
)
count += 1
success = success and local_success
response_times[tag] = response_time
if 'confidence_tags' in config:
for tag, tag_config in config['confidence_tags'].items():
local_success = await self.write_data(
response_time = await self.write_data(
server_id=server_id,
tag=tag,
data=data.head(1)['prediction_confidence'].values[0],
@@ -257,15 +257,16 @@ class OPC(BaseActivity):
tag_type='confidence',
metadata=metadata,
)
if local_success:
if response_time is not None:
self.info(
f'Confidence data written to OPC server {server_id} for tag {tag}.',
metadata,
)
count += 1
success = success and local_success
response_times[tag] = response_time
return success, count
success = None not in response_times.values()
return success, response_times
@activity.defn(name='write_opc_data')
async def write_opc_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
@@ -294,14 +295,18 @@ class OPC(BaseActivity):
success = True
metrics: dict[str, dict[str, float | None]] = {}
for server_id, config in opc_output_config.items():
if not self.validate_server(server_id, metadata):
success = False
continue
local_success, local_count = await self.manage_output_tags(
local_success, local_response_times = await self.manage_output_tags(
server_id, config, data, metadata, success
)
metrics[server_id] = local_response_times
local_count = len(local_response_times)
success = success and local_success
self.info(
@@ -309,7 +314,10 @@ class OPC(BaseActivity):
metadata,
)
return self.process_confidence(data, success, metadata)
return {
'data': self.process_confidence(data, success, metadata),
'metrics': metrics,
}
def process_confidence(
self, data: DataFrame, success: bool, metadata: dict[str, Any]

4
laborious/metrics.py
View File

@@ -61,12 +61,12 @@ PREDICTION_RESPONSE_TIME_MONITOR = Histogram(
PREDICTION_OPC_WRITING_COUNT = Counter(
'laborious_prediction_opc_writing_count',
'Number of predictions written to the OPC server',
[*CORE_LABELS, 'opc_server_id'],
[*CORE_LABELS, 'opc_server_id', 'tag'],
)
PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR = Histogram(
'laborious_prediction_opc_writing_response_time_monitor',
'Current response time of each prediction written to the OPC server',
[*CORE_LABELS, 'opc_server_id'],
[*CORE_LABELS, 'opc_server_id', 'tag'],
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
)

19
laborious/utils/repository/opc_repository.py
View File

@@ -14,8 +14,6 @@ from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.temporal.activities.base import BaseActivity
from laborious import metrics
data_type_map = {
'float': {
'converter': float,
@@ -395,21 +393,8 @@ class OpcRepository(BaseActivity):
try:
await node_obj.write_value(ua_data)
metrics.PREDICTION_OPC_WRITING_COUNT.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name'],
opc_server_id=self.id,
).inc()
end_time = time.time()
response_time = end_time - start_time
metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name'],
opc_server_id=self.id,
).observe(response_time)
except Exception as e:
trace = traceback.format_exc()
@@ -424,4 +409,6 @@ class OpcRepository(BaseActivity):
}
self.error_count = 0
return True, {}
return True, {
'response_time': response_time,
}

14
laborious/workflows/sub_workflows/format_and_export_prediction.py
View File

@@ -103,9 +103,13 @@ class FormatAndExportPrediction:
)
# write to opc
prediction = await workflow.execute_activity_method(
prediction, opc_metrics = await workflow.execute_activity_method(
Activities.write_opc_data,
{**metadata, 'opc_output_config': input_data['opc_output_config'], 'data': prediction},
{
'opc_output_config': input_data['opc_output_config'],
'data': prediction,
**metadata,
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60),
)
@@ -126,7 +130,11 @@ class FormatAndExportPrediction:
await workflow.execute_activity_method(
Activities.write_metrics,
{**metadata, 'prediction': prediction},
{
**metadata,
'prediction': prediction,
'opc_metrics': opc_metrics,
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60),
)