From 23e6890692d8cc07727de5d6e0e7d2282b2bbca8 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 18 Dec 2025 09:06:11 -0300 Subject: [PATCH] SIENTIAPDE-1445 Enhance PI Web API integration and update test configurations - Adjusted execution counts in tests.ipynb for better notebook state management. - Introduced a new pi_web_api_query structure in tests.ipynb for improved API configuration. - Updated API class to format timestamps using DATETIME_FORMAT_WITH_TZ. - Refactored PIWebAPIClient to ensure correct web ID retrieval and emit metrics accurately. - Streamlined workflow parameters in pi_web_api_scouter.py for clarity and flexibility. --- scouter/activities/api.py | 4 +++- scouter/utils/clients/pi_web_api_client.py | 11 ++++++----- scouter/workflow/pi_web_api_scouter.py | 10 ++++++---- tests.ipynb | 22 ++++++++++++++-------- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/scouter/activities/api.py b/scouter/activities/api.py index f588736..eb00b2e 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -11,7 +11,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.sientia_monitoring import SientiaMonitoring from scouter.utils.clients.pi_web_api_client import PIWebAPIClient - + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ class API(SientiaMonitoring): """ @@ -127,6 +127,8 @@ class API(SientiaMonitoring): ) raise e + latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ) + self.debug(f'Latest values: {latest_values}', metadata=metadata) self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata) diff --git a/scouter/utils/clients/pi_web_api_client.py b/scouter/utils/clients/pi_web_api_client.py index 0453a52..d4bbf12 100644 --- a/scouter/utils/clients/pi_web_api_client.py +++ b/scouter/utils/clients/pi_web_api_client.py @@ -212,10 +212,7 @@ class PIWebAPIClient(SientiaMonitoring): metric_object=metrics.GENERIC_REST_CLIENT_LAG, tags=core_labels, ) - await self.emit_metric( - metric_object=metrics.GENERIC_REST_READ_COUNT, - tags=core_labels, - ) + status_code = c.getinfo(pycurl.RESPONSE_CODE) body = buffer.getvalue().decode('utf-8', errors='replace') @@ -227,6 +224,10 @@ class PIWebAPIClient(SientiaMonitoring): ) raise PIMSRequestError(f"HTTP {status_code} calling '{full_url}': {body[:200]}") + await self.emit_metric( + metric_object=metrics.GENERIC_REST_READ_COUNT, + tags=core_labels, + ) try: return json.loads(body) except json.JSONDecodeError as e: @@ -315,7 +316,7 @@ class PIWebAPIClient(SientiaMonitoring): if isinstance(it, dict) and 'Timestamp' in it and 'Value' in it: ts = it.get('Timestamp') val = it.get('Value') - web_id = web_ids.get(tag_name) + web_id = web_ids[tag_name]['webid'] if ts is not None: records.append( { diff --git a/scouter/workflow/pi_web_api_scouter.py b/scouter/workflow/pi_web_api_scouter.py index 313f42c..6582e4e 100644 --- a/scouter/workflow/pi_web_api_scouter.py +++ b/scouter/workflow/pi_web_api_scouter.py @@ -78,15 +78,17 @@ class PIWebAPIScouter: } } + pi_web_api_query = input_data['pi_web_api_query'] + data = await workflow.execute_local_activity_method( Activities.get_tag_values, { **metadata, - 'endpoint': input_data['endpoint'], + 'endpoint': pi_web_api_query['endpoint'], 'web_ids': input_data['model_tags'], - 'period': input_data['period'], - 'api_timeout': input_data['api_timeout'], - 'max_count': input_data.get('max_count', 1), + 'period': pi_web_api_query['period'], + 'max_count': pi_web_api_query.get('max_count', 1), + 'api_timeout': pi_web_api_query['api_timeout'], }, start_to_close_timeout=timedelta(seconds=60), retry_policy=retry_policy, diff --git a/tests.ipynb b/tests.ipynb index 82cd427..ceed274 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "9d16b24a", "metadata": {}, "outputs": [], @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 14, "id": "5e344fb0", "metadata": {}, "outputs": [], @@ -69,20 +69,26 @@ " \"CI-W3W03S1\": {\n", " \"aggr_func\": \"avg\",\n", " \"data_range\": [\n", - " -100,\n", - " 100\n", + " -100000,\n", + " 100000\n", " ],\n", " \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujATFUAAAUElIQVZDXENJLVczVzAzUzE\"\n", " },\n", " \"CI-W3A05F1\": {\n", " \"aggr_func\": \"lts\",\n", " \"data_range\": [\n", - " -100,\n", - " 100\n", + " -100000,\n", + " 100000\n", " ],\n", " \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE\"\n", " }\n", " },\n", + " \"pi_web_api_query\": {\n", + " \"endpoint\": \"/streamsets/recorded\",\n", + " \"period\": \"*-1d\",\n", + " \"max_count\": 1,\n", + " \"api_timeout\": 5\n", + " },\n", " \"offset\": \"0m\",\n", " \"retention_time\": 3600,\n", " \"schedule_name\": \"pi-web-api-scouter-test\",\n", @@ -97,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "9350bff3", "metadata": {}, "outputs": [], @@ -112,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 18, "id": "45712d7a", "metadata": {}, "outputs": [],