SIENTIAPDE-1478

Enhance end-to-end tests for PredictionsBatch workflow scenarios

- Introduced mock repositories for PI Web API and OPC operations to improve test coverage.
- Updated test scenarios to handle partial write errors for PI Web API and OPC.
- Refactored existing tests to assert correct behavior under various error conditions.
- Enhanced logging and error handling in API and OPC activities to provide clearer feedback on failures.
- Removed outdated integration test file to streamline test suite.

laborious/activities/api.py

@@ -81,7 +81,7 @@ class API(SientiaMonitoring):
         tags: dict[str, str],
         core_labels: dict[str, str],
         metadata: dict[str, Any],
-    ) -> int:
+    ) -> tuple[int, str]:
         """
         Process the response data from PI Web API write operation.
 
@@ -106,6 +106,8 @@ class API(SientiaMonitoring):
 
         confidence = 0
 
+        message = ''
+
         # Evaluate response for each tag
         written_tags = []
         response_items = response_data.get('Items', [])
@@ -144,10 +146,10 @@ class API(SientiaMonitoring):
                 written_tags.append(tag_name)
 
         if len(written_tags) != len(tag_names):
-            self.error(
-                f'The number of written tags does not match the number of tag names: Expected {tag_names} tags, but {written_tags} tags were written',
-                metadata,
-            )
+            message = f'The number of written tags does not match the number of tag names: Expected {tag_names} tags, but {written_tags} tags were written.'
+
+            self.error(f"{message}\nResponse:\n {json.dumps(response_data, indent=4)}\nTags:\n {json.dumps(tags, indent=4)}", metadata)
+
             await self.send_notification_async(
                 metadata=metadata,
                 notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
@@ -157,7 +159,7 @@ class API(SientiaMonitoring):
             )
             confidence = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
 
-        return confidence
+        return confidence, message
 
     @activity.defn(name='write_pi_web_api_data')
     async def write_pi_web_api_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
@@ -185,7 +187,7 @@ class API(SientiaMonitoring):
         data = DataFrame(input_data['data'])
         pi_web_api_output_config = input_data['pi_web_api_output_config']
 
-        self.info('Writing data to PI Web API...', metadata)
+        self.info(f'Writing data to PI Web API... config: {pi_web_api_output_config}', metadata)
 
         endpoint = pi_web_api_output_config['endpoint']
 
@@ -213,7 +215,7 @@ class API(SientiaMonitoring):
                 metadata=metadata,
             )
 
-            confidence = await self.process_pi_web_api_response(
+            confidence, message = await self.process_pi_web_api_response(
                 response_data=prediction_response,
                 tags=raw_prediction_tags,
                 core_labels=core_labels,
@@ -221,6 +223,7 @@ class API(SientiaMonitoring):
             )
 
             data['prediction_confidence'] = confidence
+            data['comments'] = message
 
         except Exception as e:
             trace = traceback.format_exc()
@@ -234,6 +237,7 @@ class API(SientiaMonitoring):
             )
 
             data['prediction_confidence'] = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
+            data['comments'] = str(e)
 
             return data.to_dict()
 

laborious/activities/opc.py

@@ -352,10 +352,13 @@ class OPC(SientiaMonitoring):
             This allows downstream systems to handle data quality appropriately.
         """
 
+        message = 'Some data could not be written to OPC servers'
+
         if not success:
             data['prediction_confidence'] = OPC_WRITTING_ERROR_CONFIDENCE
+            data['comments'] = message
             self.debug(
-                f'Some data could not be written to OPC servers, setting confidence to  {OPC_WRITTING_ERROR_CONFIDENCE}.',
+                f'{message}, setting confidence to  {OPC_WRITTING_ERROR_CONFIDENCE}.',
                 metadata,
             )