SIENTIAPDE-1445
Enhance Activities and API Integration - Updated Activities class to include API operations for external data ingestion. - Added API configuration builder to connectors_config.py for environment variable management. - Integrated API configuration into worker setup. - Expanded unit tests to cover new API functionality and configuration handling. - Updated requirements.txt to include pycurl and prometheus-client for enhanced metrics support.
This commit is contained in:
vitor-aignosi
335
scouter/utils/clients/pi_web_api_client.py
Normal file
335
scouter/utils/clients/pi_web_api_client.py
Normal file
@@ -0,0 +1,335 @@
|
||||
import io
|
||||
import json
|
||||
import time
|
||||
import warnings
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import pandas as pd
|
||||
import pycurl
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
|
||||
from scouter import metrics
|
||||
|
||||
warnings.simplefilter('ignore') # Ignore warnings such as 'verify=False'
|
||||
|
||||
|
||||
class PIMSRequestError(Exception):
|
||||
"""Generic error for failed requests to PI Web API using pycurl."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class PIWebAPIClient(SientiaMonitoring):
|
||||
"""
|
||||
Client for interacting with the PI Web API.
|
||||
|
||||
This class provides a robust interface for querying historical and real-time
|
||||
data from OSIsoft PI systems through the PI Web API. It implements:
|
||||
- Asynchronous HTTP requests using pycurl
|
||||
- Authentication support (Basic and Bearer)
|
||||
- Automatic data normalization and timestamp handling
|
||||
- Comprehensive error handling and monitoring
|
||||
- Metrics collection for observability
|
||||
|
||||
The client is designed for high-performance data retrieval with proper
|
||||
connection management and error recovery mechanisms.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
auth_config: dict[str, Any],
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
metrics_controller: MetricsController,
|
||||
headers_config: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the PI Web API client with connection parameters.
|
||||
|
||||
Args:
|
||||
base_url (str): Base URL of the PI Web API server
|
||||
auth_config (dict[str, Any]): Authentication configuration.
|
||||
Required fields:
|
||||
- type (str): Authentication type ('basic' or 'bearer')
|
||||
- token (str): Authentication token
|
||||
logger (Logger): Logger instance for operation logging
|
||||
notification_handler (NotificationHandler): Handler for system notifications
|
||||
metrics_controller (MetricsController): Controller for metrics collection
|
||||
headers_config (dict[str, Any], optional): HTTP headers configuration.
|
||||
Default headers include content-type, accept, and x-requested-with
|
||||
max_concurrency (int, optional): Maximum number of concurrent requests. Defaults to 8
|
||||
"""
|
||||
if headers_config is None:
|
||||
headers_config = {
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json',
|
||||
'x-requested-with': 'XMLHttpRequest',
|
||||
}
|
||||
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||
|
||||
self.base_url = base_url.rstrip('/')
|
||||
|
||||
# auth_config spec:
|
||||
# 'type': 'basic' or 'bearer',
|
||||
# 'token': 'token',
|
||||
self.auth_config = auth_config
|
||||
self.auth_config['type'] = self.auth_config['type'].lower()
|
||||
self.headers: dict[str, str] = headers_config
|
||||
|
||||
self.authenticate()
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Close the client and shutdown monitoring services.
|
||||
"""
|
||||
SientiaMonitoring.shutdown(self)
|
||||
|
||||
def _to_clean_timestamp(self, series: pd.Series) -> pd.Series:
|
||||
"""
|
||||
Convert a Series of timestamps to datetime, UTC, and round to the nearest second.
|
||||
|
||||
Args:
|
||||
series (pd.Series): Series containing timestamp values
|
||||
|
||||
Returns:
|
||||
pd.Series: Cleaned timestamp series in UTC, floored to seconds
|
||||
"""
|
||||
series = pd.to_datetime(series, utc=True, errors='coerce')
|
||||
return series.dt.floor('s')
|
||||
|
||||
def _extract_numeric(self, value: Any) -> float | None:
|
||||
"""
|
||||
Normalize a value (potentially nested) to float.
|
||||
|
||||
This method handles PI Web API response values that may be nested
|
||||
in dictionaries or other structures, extracting the numeric value.
|
||||
|
||||
Args:
|
||||
value (Any): Value to extract and normalize
|
||||
|
||||
Returns:
|
||||
float | None: Numeric value as float, or None if conversion fails
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
value = value.get('Value', value)
|
||||
return pd.to_numeric(value, errors='coerce')
|
||||
|
||||
def authenticate(self):
|
||||
"""
|
||||
Configure authentication headers based on auth_config.
|
||||
|
||||
This method sets up the Authorization header using either Basic or Bearer
|
||||
authentication based on the configured authentication type.
|
||||
|
||||
Raises:
|
||||
ValueError: If authentication type is not 'basic' or 'bearer'
|
||||
"""
|
||||
self.logger.info(f'Authenticating with {self.auth_config["type"]} authentication')
|
||||
|
||||
if self.auth_config['type'] == 'basic':
|
||||
self.headers['Authorization'] = f'Basic {self.auth_config["token"]}'
|
||||
elif self.auth_config['type'] == 'bearer':
|
||||
self.headers['Authorization'] = f'Bearer {self.auth_config["token"]}'
|
||||
else:
|
||||
raise ValueError(f'Invalid authentication type: {self.auth_config["type"]}')
|
||||
|
||||
async def _curl_get_json(
|
||||
self,
|
||||
url: str,
|
||||
params: list[tuple[str, str]] | None = None,
|
||||
timeout: int = 30,
|
||||
verify: bool = True,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Perform a GET request using pycurl and return the decoded JSON response.
|
||||
|
||||
This method executes an asynchronous HTTP GET request with proper error handling,
|
||||
metrics collection, and timeout management. It automatically tracks request
|
||||
latency and emits monitoring metrics.
|
||||
|
||||
Args:
|
||||
url (str): Target URL for the GET request
|
||||
params (list[tuple[str, str]], optional): Query parameters as list of tuples.
|
||||
Each tuple contains (parameter_name, parameter_value)
|
||||
timeout (int, optional): Request timeout in seconds. Defaults to 30
|
||||
verify (bool, optional): Verify SSL certificates. Defaults to True
|
||||
metadata (dict[str, Any], optional): Workflow execution metadata for tracking
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: Parsed JSON response body
|
||||
|
||||
Raises:
|
||||
PIMSRequestError: If HTTP error, connection error, or JSON parsing error occurs
|
||||
"""
|
||||
if metadata is None:
|
||||
metadata = {}
|
||||
|
||||
buffer = io.BytesIO()
|
||||
c = pycurl.Curl()
|
||||
|
||||
core_labels = self.get_core_labels(metadata=metadata, operation_type='get_json')
|
||||
|
||||
try:
|
||||
if params:
|
||||
query_string = urlencode(params, doseq=True)
|
||||
full_url = f'{url}?{query_string}'
|
||||
else:
|
||||
full_url = url
|
||||
|
||||
c.setopt(pycurl.URL, full_url.encode('utf-8'))
|
||||
c.setopt(pycurl.WRITEDATA, buffer)
|
||||
|
||||
# Configure HTTP headers
|
||||
header_list = [f'{k}: {v}' for k, v in self.headers.items()]
|
||||
if header_list:
|
||||
c.setopt(pycurl.HTTPHEADER, header_list)
|
||||
|
||||
# Set request timeout
|
||||
c.setopt(pycurl.TIMEOUT, timeout)
|
||||
|
||||
# Configure SSL verification
|
||||
if not verify:
|
||||
c.setopt(pycurl.SSL_VERIFYPEER, 0)
|
||||
c.setopt(pycurl.SSL_VERIFYHOST, 0)
|
||||
|
||||
start_time = time.time()
|
||||
try:
|
||||
c.perform()
|
||||
except Exception as e:
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.GENERIC_REST_READ_ERROR_COUNT,
|
||||
tags=core_labels,
|
||||
)
|
||||
raise e
|
||||
await self.observe_lag(
|
||||
start_time=start_time,
|
||||
metric_object=metrics.GENERIC_REST_CLIENT_LAG,
|
||||
tags=core_labels,
|
||||
)
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.GENERIC_REST_READ_COUNT,
|
||||
tags=core_labels,
|
||||
)
|
||||
|
||||
status_code = c.getinfo(pycurl.RESPONSE_CODE)
|
||||
body = buffer.getvalue().decode('utf-8', errors='replace')
|
||||
|
||||
if status_code >= 400:
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.GENERIC_REST_READ_ERROR_COUNT,
|
||||
tags=core_labels,
|
||||
)
|
||||
raise PIMSRequestError(f"HTTP {status_code} calling '{full_url}': {body[:200]}")
|
||||
|
||||
try:
|
||||
return json.loads(body)
|
||||
except json.JSONDecodeError as e:
|
||||
raise PIMSRequestError(
|
||||
f"Error decoding JSON response from '{full_url}': {e}; body: {body[:200]}"
|
||||
) from e
|
||||
|
||||
except pycurl.error as e:
|
||||
raise PIMSRequestError(f"Connection error calling '{url}': {e}") from e
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
async def get_latest_values_df(
|
||||
self,
|
||||
web_ids: dict[str, dict[str, str]],
|
||||
endpoint: str,
|
||||
timeout: int = 30,
|
||||
start_time: str = '*-1d',
|
||||
end_time: str = '*',
|
||||
max_count: int | None = 1,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Retrieve historical values for multiple WebIds using PI Web API streamsets.
|
||||
|
||||
This method queries the PI Web API's /streamsets/recorded endpoint to fetch
|
||||
historical data for multiple tags simultaneously. It returns a normalized
|
||||
DataFrame with timestamps, tag names, values, and WebIds.
|
||||
|
||||
Args:
|
||||
web_ids (dict[str, str | None]): Dictionary mapping tag names to their WebIds.
|
||||
None values are filtered out before querying
|
||||
endpoint (str): PI Web API endpoint path (e.g., '/streamsets/recorded')
|
||||
timeout (int, optional): Request timeout in seconds. Defaults to 30
|
||||
start_time (str, optional): Start time in PI Web API format (e.g., "*-50d").
|
||||
Defaults to "*-1d" (1 day ago)
|
||||
end_time (str, optional): End time in PI Web API format (e.g., "*").
|
||||
Defaults to "*" (current time)
|
||||
max_count (int, optional): Maximum number of data points per series.
|
||||
Defaults to 1. If None, maxCount parameter is not sent
|
||||
metadata (dict[str, Any], optional): Workflow execution metadata for tracking
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: DataFrame with columns:
|
||||
- timestamp: Cleaned timestamp (UTC, floored to seconds)
|
||||
- name: Tag name
|
||||
- value: Numeric value (normalized)
|
||||
- tag: WebId of the tag
|
||||
Returns empty DataFrame if no data is found
|
||||
|
||||
Raises:
|
||||
PIMSRequestError: If API request fails or returns invalid data
|
||||
"""
|
||||
if metadata is None:
|
||||
metadata = {}
|
||||
|
||||
url = f'{self.base_url}{endpoint}'
|
||||
|
||||
# Build query parameters with WebIds (filtering out None values)
|
||||
params: list[tuple[str, str]] = [('webid', web_id['webid']) for web_id in web_ids.values()]
|
||||
params.extend(
|
||||
[
|
||||
('startTime', start_time),
|
||||
('endtime', end_time),
|
||||
('selectedFields', 'Items.Name;Items.Items.Timestamp;Items.Items.Value'),
|
||||
]
|
||||
)
|
||||
|
||||
params.append(('maxCount', str(max_count)))
|
||||
|
||||
data = await self._curl_get_json(
|
||||
url=url,
|
||||
params=params,
|
||||
timeout=timeout,
|
||||
verify=False,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
raw_data = data.get('Items', [])
|
||||
|
||||
records = []
|
||||
for entry in raw_data:
|
||||
tag_name = entry.get('Name')
|
||||
series_items = entry.get('Items', [])
|
||||
for it in series_items:
|
||||
if isinstance(it, dict) and 'Timestamp' in it and 'Value' in it:
|
||||
ts = it.get('Timestamp')
|
||||
val = it.get('Value')
|
||||
web_id = web_ids.get(tag_name)
|
||||
if ts is not None:
|
||||
records.append(
|
||||
{
|
||||
'timestamp': ts,
|
||||
'name': tag_name,
|
||||
'value': self._extract_numeric(val),
|
||||
'tag': web_id,
|
||||
}
|
||||
)
|
||||
|
||||
if not records:
|
||||
return pd.DataFrame()
|
||||
|
||||
df = pd.DataFrame.from_records(records)
|
||||
df['timestamp'] = self._to_clean_timestamp(df['timestamp'])
|
||||
|
||||
return df
|
||||
@@ -83,6 +83,23 @@ def build_mongodb_config() -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def build_api_config() -> dict[str, Any]:
|
||||
"""
|
||||
Build API connection configuration from environment variables.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: API configuration dictionary with keys:
|
||||
- base_url: API base URL (default: https://pi.example.com)
|
||||
- auth_type: API authentication type (default: basic)
|
||||
- auth_token: API authentication token (default: None)
|
||||
"""
|
||||
return {
|
||||
'base_url': getenv('API_BASE_URL', 'https://pi.example.com'),
|
||||
'auth_type': getenv('API_AUTH_TYPE', 'basic'),
|
||||
'auth_token': getenv('API_AUTH_TOKEN', None),
|
||||
}
|
||||
|
||||
|
||||
def build_druid_config() -> dict[str, Any]:
|
||||
"""
|
||||
Build Apache Druid connection configuration from environment variables.
|
||||
|
||||
Reference in New Issue
Block a user