SIENTIAPDE-1325

Enhance metrics handling in Gates and MLFlow classes

- Added checks for `None` response times before emitting OPC writing metrics in the Gates class to prevent unnecessary metric emissions.
- Updated the MLFlow class to conditionally sort and drop duplicates based on the presence of the 'created_at' column, ensuring robustness in data processing.
- Adjusted corresponding tests to validate the new behavior in both classes.

laborious/activities/gates.py

@@ -656,28 +656,29 @@ class Gates(SientiaMonitoring):
 
         for server_id, tags in opc_metrics.items():
             for tag, response_time in tags.items():
-                await self.emit_metric(
-                    metric_object=metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR,
-                    method='observe',
-                    tags={
-                        'pod_id': self.pod_id,
-                        'model_name': metadata['model_name'],
-                        'workflow_name': metadata['workflow_name'],
-                        'opc_server_id': server_id,
-                        'tag': tag,
-                    },
-                    value=response_time,
-                )
+                if response_time is not None:
+                    await self.emit_metric(
+                        metric_object=metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR,
+                        method='observe',
+                        tags={
+                            'pod_id': self.pod_id,
+                            'model_name': metadata['model_name'],
+                            'workflow_name': metadata['workflow_name'],
+                            'opc_server_id': server_id,
+                            'tag': tag,
+                        },
+                        value=response_time,
+                    )
 
-                await self.emit_metric(
-                    metric_object=metrics.PREDICTION_OPC_WRITING_COUNT,
-                    tags={
-                        'pod_id': self.pod_id,
-                        'model_name': metadata['model_name'],
-                        'workflow_name': metadata['workflow_name'],
-                        'opc_server_id': server_id,
-                        'tag': tag,
-                    },
-                )
+                    await self.emit_metric(
+                        metric_object=metrics.PREDICTION_OPC_WRITING_COUNT,
+                        tags={
+                            'pod_id': self.pod_id,
+                            'model_name': metadata['model_name'],
+                            'workflow_name': metadata['workflow_name'],
+                            'opc_server_id': server_id,
+                            'tag': tag,
+                        },
+                    )
 
         self.info(f'Metrics written for model {metadata["model_name"]}', metadata)

laborious/activities/mlflow.py

@@ -312,9 +312,12 @@ class MLFlow(SientiaMonitoring):
         self.debug(f'Timestamp: {timestamp}', metadata)
 
         # Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair
-        data = data.sort_values('created_at', ascending=False).drop_duplicates(
-            subset=['variable', 'timestamp'], keep='first'
-        )
+        if 'created_at' in data.columns:
+            data = data.sort_values('created_at', ascending=False).drop_duplicates(
+                subset=['variable', 'timestamp'], keep='first'
+            )
+        else:
+            data = data.drop_duplicates(subset=['variable', 'timestamp'], keep='first')
 
         data.drop(columns=['model_id'], inplace=True, errors='ignore')
         data.drop(columns=['created_at'], inplace=True, errors='ignore')