gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1199

Browse Source 
Refactor logging in Gates and MLFlow activities to use info level for key operations

- Updated logging statements in the Gates class to replace debug logs with info logs for input and output gate operations, enhancing visibility.
- Modified MLFlow class to use info logs for data transformation and prediction processes, improving clarity in the logging output.
- Adjusted OPC class to return the count of successfully written tags, providing better insight into data writing operations.
This commit is contained in:
vitor-aignosi
2025-08-20 10:38:08 -03:00
parent 89384d7783
commit 8b9bb8d720
4 changed files with 67 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
laborious/activities/gates.py
View File

@@ -68,7 +68,7 @@ class Gates(BaseActivity):
"""
metadata = input_data['metadata']
self.debug("Performing input gate...", metadata)
self.info("Performing input gate...", metadata)
self.debug(f"Input data: {input_data}", metadata)
@@ -103,11 +103,11 @@ class Gates(BaseActivity):
for path_flag in path_priority:
if path_flag in filter_output:
self.debug(f"Input gate result: {path_flag}", metadata)
self.info(f"Input gate result: {path_flag}", metadata)
return path_flag, input_filter_functions['path_confidence'][path_flag], \
"Input data with bad quality"
self.debug("Nothing was filtered by the input gate", metadata)
self.info("Nothing was filtered by the input gate", metadata)
return None, 0, ""
@activity.defn(name="mlflow_response_gate")
@@ -128,7 +128,7 @@ class Gates(BaseActivity):
"""
metadata = input_data['metadata']
self.debug("Performing mlflow response gate...", metadata)
self.info("Performing mlflow response gate...", metadata)
filters = input_data['filters']
data = input_data['data']
@@ -169,12 +169,12 @@ class Gates(BaseActivity):
for path_flag in path_priority:
if path_flag in filter_output:
self.debug(
self.info(
f"Mlflow response gate result: {path_flag}", metadata)
return path_flag, mlflow_response_filter_functions['path_confidence'][path_flag], \
", ".join(comments)
self.debug("Nothing was filtered by the mlflow response gate", metadata)
self.info("Nothing was filtered by the mlflow response gate", metadata)
return None, 0, ""
@activity.defn(name="mlflow_content_gate")
@@ -195,7 +195,7 @@ class Gates(BaseActivity):
"""
metadata = input_data['metadata']
self.debug("Performing mlflow content gate...", metadata)
self.info("Performing mlflow content gate...", metadata)
filters = input_data['filters']
data = DataFrame(input_data['data'])
@@ -234,12 +234,12 @@ class Gates(BaseActivity):
for path_flag in path_priority:
if path_flag in filter_output:
self.debug(
self.info(
f"Mlflow content gate result: {path_flag}", metadata)
return path_flag, mlflow_content_filter_functions['path_confidence'][path_flag], \
"Transformed data not passed the content filter"
self.debug("Nothing was filtered by the mlflow content gate", metadata)
self.info("Nothing was filtered by the mlflow content gate", metadata)
return None, 0, ""
@activity.defn(name="format_prediction")
@@ -256,7 +256,7 @@ class Gates(BaseActivity):
dict: The formatted data.
"""
metadata = input_data['metadata']
self.debug("Formatting prediction...", metadata)
self.info("Formatting prediction...", metadata)
data = DataFrame(input_data['data'])
data['timestamp'] = input_data['timestamp']
@@ -266,6 +266,8 @@ class Gates(BaseActivity):
data['comments'] = ""
data = data.sort_values(by='timestamp')
self.info(f"Prediction formatted: {data.size} rows", metadata)
return data.to_dict()
@activity.defn(name="format_default_prediction")
@@ -287,7 +289,7 @@ class Gates(BaseActivity):
metadata = input_data['metadata']
self.debug("Formatting default prediction...", metadata)
return DataFrame({
data = DataFrame({
'prediction': [0],
'response_time': [0],
'timestamp': [input_data['timestamp']],
@@ -295,7 +297,10 @@ class Gates(BaseActivity):
'prediction_confidence': [input_data['prediction_confidence']],
'prediction_status': ['Bad'],
'comments': [input_data['comment']]
}).to_dict()
})
self.info(f"Default prediction formatted: {data.size} rows", metadata)
return data.to_dict()
@activity.defn(name="get_last_timestamp")
async def get_last_timestamp(self, input_data: dict[str, Any]) -> str:
@@ -307,9 +312,18 @@ class Gates(BaseActivity):
Returns:
str: The last timestamp of the data.
"""
metadata = input_data['metadata']
self.info("Getting last timestamp...", metadata)
data = DataFrame(input_data['data'])
if data.empty:
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.info(
f"Last timestamp: {max(data['timestamp'].values.tolist())}", metadata)
return max(data['timestamp'].values.tolist())
@activity.defn(name="write_metrics")
@@ -325,6 +339,9 @@ class Gates(BaseActivity):
prediction_confidence = prediction['prediction_confidence'].values[0]
response_time = prediction['response_time'].values[0]
self.info(
f"Writing metrics for model {metadata['model_name']}", metadata)
metrics.PREDICTIONS_WRITTEN_COUNT.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
@@ -342,3 +359,6 @@ class Gates(BaseActivity):
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name']
).observe(response_time)
self.info(
f"Metrics written for model {metadata['model_name']}", metadata)