SIENTIAPDE-1445

Enhance PI Web API integration and update test configurations

- Adjusted execution counts in tests.ipynb for better notebook state management.
- Introduced a new pi_web_api_query structure in tests.ipynb for improved API configuration.
- Updated API class to format timestamps using DATETIME_FORMAT_WITH_TZ.
- Refactored PIWebAPIClient to ensure correct web ID retrieval and emit metrics accurately.
- Streamlined workflow parameters in pi_web_api_scouter.py for clarity and flexibility.

scouter/activities/api.py

@@ -11,7 +11,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.observability.sientia_monitoring import SientiaMonitoring
 
     from scouter.utils.clients.pi_web_api_client import PIWebAPIClient
-
+    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
 
 class API(SientiaMonitoring):
     """
@@ -127,6 +127,8 @@ class API(SientiaMonitoring):
             )
             raise e
 
+        latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
+
         self.debug(f'Latest values: {latest_values}', metadata=metadata)
         self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata)
 

scouter/utils/clients/pi_web_api_client.py

@@ -212,10 +212,7 @@ class PIWebAPIClient(SientiaMonitoring):
                 metric_object=metrics.GENERIC_REST_CLIENT_LAG,
                 tags=core_labels,
             )
-            await self.emit_metric(
-                metric_object=metrics.GENERIC_REST_READ_COUNT,
-                tags=core_labels,
-            )
+
 
             status_code = c.getinfo(pycurl.RESPONSE_CODE)
             body = buffer.getvalue().decode('utf-8', errors='replace')
@@ -227,6 +224,10 @@ class PIWebAPIClient(SientiaMonitoring):
                 )
                 raise PIMSRequestError(f"HTTP {status_code} calling '{full_url}': {body[:200]}")
 
+            await self.emit_metric(
+                metric_object=metrics.GENERIC_REST_READ_COUNT,
+                tags=core_labels,
+            )
             try:
                 return json.loads(body)
             except json.JSONDecodeError as e:
@@ -315,7 +316,7 @@ class PIWebAPIClient(SientiaMonitoring):
                 if isinstance(it, dict) and 'Timestamp' in it and 'Value' in it:
                     ts = it.get('Timestamp')
                     val = it.get('Value')
-                    web_id = web_ids.get(tag_name)
+                    web_id = web_ids[tag_name]['webid']
                     if ts is not None:
                         records.append(
                             {

scouter/workflow/pi_web_api_scouter.py

@@ -78,15 +78,17 @@ class PIWebAPIScouter:
             }
         }
 
+        pi_web_api_query = input_data['pi_web_api_query']
+
         data = await workflow.execute_local_activity_method(
             Activities.get_tag_values,
             {
                 **metadata,
-                'endpoint': input_data['endpoint'],
+                'endpoint': pi_web_api_query['endpoint'],
                 'web_ids': input_data['model_tags'],
-                'period': input_data['period'],
-                'api_timeout': input_data['api_timeout'],
-                'max_count': input_data.get('max_count', 1),
+                'period': pi_web_api_query['period'],
+                'max_count': pi_web_api_query.get('max_count', 1),
+                'api_timeout': pi_web_api_query['api_timeout'],
             },
             start_to_close_timeout=timedelta(seconds=60),
             retry_policy=retry_policy,