SIENTIAPDE-1445

Enhance PI Web API integration and update test configurations

- Adjusted execution counts in tests.ipynb for better notebook state management.
- Introduced a new pi_web_api_query structure in tests.ipynb for improved API configuration.
- Updated API class to format timestamps using DATETIME_FORMAT_WITH_TZ.
- Refactored PIWebAPIClient to ensure correct web ID retrieval and emit metrics accurately.
- Streamlined workflow parameters in pi_web_api_scouter.py for clarity and flexibility.

scouter/activities/api.py

@@ -11,7 +11,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.observability.sientia_monitoring import SientiaMonitoring
 
     from scouter.utils.clients.pi_web_api_client import PIWebAPIClient
-
+    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
 
 class API(SientiaMonitoring):
     """
@@ -127,6 +127,8 @@ class API(SientiaMonitoring):
             )
             raise e
 
+        latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
+
         self.debug(f'Latest values: {latest_values}', metadata=metadata)
         self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata)
 

scouter/utils/clients/pi_web_api_client.py

@@ -212,10 +212,7 @@ class PIWebAPIClient(SientiaMonitoring):
                 metric_object=metrics.GENERIC_REST_CLIENT_LAG,
                 tags=core_labels,
             )
-            await self.emit_metric(
+
-                metric_object=metrics.GENERIC_REST_READ_COUNT,
-                tags=core_labels,
-            )
 
             status_code = c.getinfo(pycurl.RESPONSE_CODE)
             body = buffer.getvalue().decode('utf-8', errors='replace')
@@ -227,6 +224,10 @@ class PIWebAPIClient(SientiaMonitoring):
                 )
                 raise PIMSRequestError(f"HTTP {status_code} calling '{full_url}': {body[:200]}")
 
+            await self.emit_metric(
+                metric_object=metrics.GENERIC_REST_READ_COUNT,
+                tags=core_labels,
+            )
             try:
                 return json.loads(body)
             except json.JSONDecodeError as e:
@@ -315,7 +316,7 @@ class PIWebAPIClient(SientiaMonitoring):
                 if isinstance(it, dict) and 'Timestamp' in it and 'Value' in it:
                     ts = it.get('Timestamp')
                     val = it.get('Value')
-                    web_id = web_ids.get(tag_name)
+                    web_id = web_ids[tag_name]['webid']
                     if ts is not None:
                         records.append(
                             {

scouter/workflow/pi_web_api_scouter.py

@@ -78,15 +78,17 @@ class PIWebAPIScouter:
             }
         }
 
+        pi_web_api_query = input_data['pi_web_api_query']
+
         data = await workflow.execute_local_activity_method(
             Activities.get_tag_values,
             {
                 **metadata,
-                'endpoint': input_data['endpoint'],
+                'endpoint': pi_web_api_query['endpoint'],
                 'web_ids': input_data['model_tags'],
-                'period': input_data['period'],
+                'period': pi_web_api_query['period'],
-                'api_timeout': input_data['api_timeout'],
+                'max_count': pi_web_api_query.get('max_count', 1),
-                'max_count': input_data.get('max_count', 1),
+                'api_timeout': pi_web_api_query['api_timeout'],
             },
             start_to_close_timeout=timedelta(seconds=60),
             retry_policy=retry_policy,

tests.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "id": "9d16b24a",
    "metadata": {},
    "outputs": [],
@@ -38,7 +38,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 14,
    "id": "5e344fb0",
    "metadata": {},
    "outputs": [],
@@ -69,20 +69,26 @@
     "    \"CI-W3W03S1\": {\n",
     "      \"aggr_func\": \"avg\",\n",
     "      \"data_range\": [\n",
-    "        -100,\n",
+    "        -100000,\n",
-    "        100\n",
+    "        100000\n",
     "      ],\n",
     "      \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujATFUAAAUElIQVZDXENJLVczVzAzUzE\"\n",
     "    },\n",
     "    \"CI-W3A05F1\": {\n",
     "      \"aggr_func\": \"lts\",\n",
     "      \"data_range\": [\n",
-    "        -100,\n",
+    "        -100000,\n",
-    "        100\n",
+    "        100000\n",
     "      ],\n",
     "      \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE\"\n",
     "    }\n",
     "  },\n",
+    "  \"pi_web_api_query\": {\n",
+    "    \"endpoint\": \"/streamsets/recorded\",\n",
+    "    \"period\": \"*-1d\",\n",
+    "    \"max_count\": 1,\n",
+    "    \"api_timeout\": 5\n",
+    "  },\n",
     "  \"offset\": \"0m\",\n",
     "  \"retention_time\": 3600,\n",
     "  \"schedule_name\": \"pi-web-api-scouter-test\",\n",
@@ -97,7 +103,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "9350bff3",
    "metadata": {},
    "outputs": [],
@@ -112,7 +118,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 18,
    "id": "45712d7a",
    "metadata": {},
    "outputs": [],