SIENTIAPDE-1445

Enhance Activities and API Integration

- Updated Activities class to include API operations for external data ingestion.
- Added API configuration builder to connectors_config.py for environment variable management.
- Integrated API configuration into worker setup.
- Expanded unit tests to cover new API functionality and configuration handling.
- Updated requirements.txt to include pycurl and prometheus-client for enhanced metrics support.
This commit is contained in:
vitor-aignosi
2025-12-17 15:16:00 -03:00
parent ab99695702
commit e87c5d2da6
14 changed files with 1709 additions and 4 deletions

View File

View File

@@ -0,0 +1,560 @@
import json
from unittest.mock import AsyncMock, MagicMock, patch
import pandas as pd
import pycurl
import pytest
from scouter.utils.clients.pi_web_api_client import PIMSRequestError, PIWebAPIClient
@pytest.fixture
def mock_logger():
return MagicMock()
@pytest.fixture
def mock_notification_handler():
return AsyncMock()
@pytest.fixture
def mock_metrics_controller():
return AsyncMock()
@pytest.fixture
def auth_config_basic():
return {'type': 'basic', 'token': 'test_token_123'}
@pytest.fixture
def auth_config_bearer():
return {'type': 'bearer', 'token': 'bearer_token_456'}
@pytest.fixture
def pi_client(mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic):
return PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=auth_config_basic,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
def test_init_with_basic_auth(
mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic
):
"""Test initialization with basic authentication"""
client = PIWebAPIClient(
base_url='https://pi.example.com/',
auth_config=auth_config_basic,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
assert client.base_url == 'https://pi.example.com'
assert client.auth_config['type'] == 'basic'
assert client.headers['Authorization'] == 'Basic test_token_123'
assert client.headers['Content-Type'] == 'application/json'
assert client.headers['Accept'] == 'application/json'
mock_logger.info.assert_called_with('Authenticating with basic authentication')
def test_init_with_bearer_auth(
mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_bearer
):
"""Test initialization with bearer authentication"""
client = PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=auth_config_bearer,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
assert client.base_url == 'https://pi.example.com'
assert client.auth_config['type'] == 'bearer'
assert client.headers['Authorization'] == 'Bearer bearer_token_456'
mock_logger.info.assert_called_with('Authenticating with bearer authentication')
def test_init_with_custom_headers(
mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic
):
"""Test initialization with custom headers"""
custom_headers = {
'Content-Type': 'application/xml',
'Custom-Header': 'custom_value',
}
client = PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=auth_config_basic,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
headers_config=custom_headers,
)
assert client.headers['Content-Type'] == 'application/xml'
assert client.headers['Custom-Header'] == 'custom_value'
assert client.headers['Authorization'] == 'Basic test_token_123'
def test_authenticate_invalid_type(mock_logger, mock_notification_handler, mock_metrics_controller):
"""Test that invalid authentication type raises ValueError"""
invalid_auth_config = {'type': 'invalid', 'token': 'test_token'}
with pytest.raises(ValueError) as exc_info:
PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=invalid_auth_config,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
assert 'Invalid authentication type: invalid' in str(exc_info.value)
@patch('scouter.utils.clients.pi_web_api_client.SientiaMonitoring.shutdown')
def test_close(mock_shutdown, pi_client):
"""Test close method calls shutdown"""
pi_client.close()
mock_shutdown.assert_called_once()
def test_to_clean_timestamp(pi_client):
"""Test timestamp cleaning and normalization"""
timestamps = pd.Series(
[
'2025-01-15T10:30:45.123456Z',
'2025-01-15T10:30:46.789012Z',
'2025-01-15T10:30:47.999999Z',
]
)
result = pi_client._to_clean_timestamp(timestamps)
assert isinstance(result, pd.Series)
assert result.dtype == 'datetime64[ns, UTC]'
# Verify microseconds are floored to seconds
assert result[0] == pd.Timestamp('2025-01-15T10:30:45Z')
assert result[1] == pd.Timestamp('2025-01-15T10:30:46Z')
assert result[2] == pd.Timestamp('2025-01-15T10:30:47Z')
def test_to_clean_timestamp_with_invalid_values(pi_client):
"""Test timestamp cleaning with invalid values returns NaT"""
timestamps = pd.Series(['invalid', 'not_a_date', '2025-01-15T10:30:45Z'])
result = pi_client._to_clean_timestamp(timestamps)
assert pd.isna(result[0])
assert pd.isna(result[1])
assert result[2] == pd.Timestamp('2025-01-15T10:30:45Z')
def test_extract_numeric_with_float(pi_client):
"""Test extracting numeric value from float"""
result = pi_client._extract_numeric(42.5)
assert result == 42.5
def test_extract_numeric_with_int(pi_client):
"""Test extracting numeric value from int"""
result = pi_client._extract_numeric(42)
assert result == 42.0
def test_extract_numeric_with_string(pi_client):
"""Test extracting numeric value from string"""
result = pi_client._extract_numeric('123.45')
assert result == 123.45
def test_extract_numeric_with_dict(pi_client):
"""Test extracting numeric value from dictionary"""
result = pi_client._extract_numeric({'Value': 99.9})
assert result == 99.9
def test_extract_numeric_with_invalid_value(pi_client):
"""Test extracting numeric value from invalid value returns None/NaN"""
result = pi_client._extract_numeric('invalid_number')
assert pd.isna(result)
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_success(mock_curl_class, pi_client):
"""Test successful GET request with JSON response"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'status': 'success', 'data': [1, 2, 3]}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
result = await pi_client._curl_get_json('https://pi.example.com/api/test')
assert result == response_data
mock_curl.setopt.assert_any_call(pycurl.TIMEOUT, 30)
mock_curl.perform.assert_called_once()
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_with_params(mock_curl_class, pi_client):
"""Test GET request with query parameters"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'result': 'ok'}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
params = [('key1', 'value1'), ('key2', 'value2')]
result = await pi_client._curl_get_json('https://pi.example.com/api', params=params)
assert result == response_data
# Verify URL includes query parameters
set_url_call = [call for call in mock_curl.setopt.call_args_list if call[0][0] == pycurl.URL][0]
assert b'key1=value1' in set_url_call[0][1]
assert b'key2=value2' in set_url_call[0][1]
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_http_error(mock_curl_class, pi_client):
"""Test GET request with HTTP error response"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
error_response = b'{"error": "Not found"}'
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(error_response)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 404
with pytest.raises(PIMSRequestError) as exc_info:
await pi_client._curl_get_json('https://pi.example.com/api/notfound')
assert 'HTTP 404' in str(exc_info.value)
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_connection_error(mock_curl_class, pi_client):
"""Test GET request with connection error"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
mock_curl.perform.side_effect = pycurl.error('Connection failed')
with pytest.raises(PIMSRequestError) as exc_info:
await pi_client._curl_get_json('https://pi.example.com/api/test')
assert 'Connection error' in str(exc_info.value)
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_invalid_json(mock_curl_class, pi_client):
"""Test GET request with invalid JSON response"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
invalid_json = b'This is not valid JSON'
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(invalid_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
with pytest.raises(PIMSRequestError) as exc_info:
await pi_client._curl_get_json('https://pi.example.com/api/test')
assert 'Error decoding JSON response' in str(exc_info.value)
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_with_custom_timeout(mock_curl_class, pi_client):
"""Test GET request with custom timeout"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'status': 'ok'}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
await pi_client._curl_get_json('https://pi.example.com/api/test', timeout=60)
mock_curl.setopt.assert_any_call(pycurl.TIMEOUT, 60)
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_without_ssl_verify(mock_curl_class, pi_client):
"""Test GET request with SSL verification disabled"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'status': 'ok'}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
await pi_client._curl_get_json('https://pi.example.com/api/test', verify=False)
mock_curl.setopt.assert_any_call(pycurl.SSL_VERIFYPEER, 0)
mock_curl.setopt.assert_any_call(pycurl.SSL_VERIFYHOST, 0)
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_success(mock_curl_get_json, pi_client):
"""Test successful retrieval of latest values"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5},
{'Timestamp': '2025-01-15T10:31:00Z', 'Value': 43.0},
],
},
{
'Name': 'tag2',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 100.0},
],
},
]
}
web_ids = {
'tag1': {'webid': 'webid1'},
'tag2': {'webid': 'webid2'},
}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
start_time='*-1d',
end_time='*',
max_count=10,
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 3
assert list(result.columns) == ['timestamp', 'name', 'value', 'tag']
assert result['name'].tolist() == ['tag1', 'tag1', 'tag2']
assert result['value'].tolist() == [42.5, 43.0, 100.0]
mock_curl_get_json.assert_called_once()
call_args = mock_curl_get_json.call_args
assert call_args[1]['url'] == 'https://pi.example.com/streamsets/recorded'
assert call_args[1]['timeout'] == 30
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_with_custom_params(mock_curl_get_json, pi_client):
"""Test get_latest_values_df with custom parameters"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5},
],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
metadata = {'model_id': 'test_model'}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
start_time='*-7d',
end_time='*-1d',
max_count=100,
timeout=60,
metadata=metadata,
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 1
mock_curl_get_json.assert_called_once()
call_args = mock_curl_get_json.call_args
params = call_args[1]['params']
# Verify parameters
assert ('startTime', '*-7d') in params
assert ('endtime', '*-1d') in params
assert ('maxCount', '100') in params
assert call_args[1]['timeout'] == 60
assert call_args[1]['metadata'] == metadata
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_empty_response(mock_curl_get_json, pi_client):
"""Test get_latest_values_df with empty response"""
mock_curl_get_json.return_value = {'Items': []}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 0
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_no_items_in_tag(mock_curl_get_json, pi_client):
"""Test get_latest_values_df when tag has no items"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 0
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_with_missing_timestamp(mock_curl_get_json, pi_client):
"""Test get_latest_values_df filters out items with missing timestamp"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5},
{'Value': 43.0}, # Missing Timestamp
{'Timestamp': None, 'Value': 44.0}, # None Timestamp
],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 1 # Only the first item should be included
assert result['value'].tolist() == [42.5]
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_with_nested_value(mock_curl_get_json, pi_client):
"""Test get_latest_values_df with nested value extraction"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': {'Value': 42.5}},
],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 1
assert result['value'].tolist() == [42.5]
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_default_max_count(mock_curl_get_json, pi_client):
"""Test get_latest_values_df uses default max_count of 1"""
mock_curl_get_json.return_value = {'Items': []}
web_ids = {'tag1': {'webid': 'webid1'}}
await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
call_args = mock_curl_get_json.call_args
params = call_args[1]['params']
assert ('maxCount', '1') in params

View File

@@ -4,6 +4,7 @@ from unittest.mock import patch
import pytest
from scouter.utils.connectors_config import (
build_api_config,
build_druid_config,
build_kafka_config,
build_mongodb_config,
@@ -154,6 +155,38 @@ def test_build_mongodb_config_with_env_vars():
}
@pytest.mark.usefixtures('mock_env_vars')
def test_build_api_config_defaults():
"""Test that build_api_config returns default values when no env vars are set"""
config = build_api_config()
assert config == {
'base_url': 'https://pi.example.com',
'auth_type': 'basic',
'auth_token': None,
}
@pytest.mark.usefixtures('mock_env_vars')
def test_build_api_config_with_env_vars():
"""Test that build_api_config uses env vars when set"""
with patch.dict(
os.environ,
{
'API_BASE_URL': 'https://api.production.com',
'API_AUTH_TYPE': 'bearer',
'API_AUTH_TOKEN': 'secret_token_123',
},
):
config = build_api_config()
assert config == {
'base_url': 'https://api.production.com',
'auth_type': 'bearer',
'auth_token': 'secret_token_123',
}
def test_build_druid_config_defaults():
"""Test that build_druid_config returns default values when no env vars are set"""
config = build_druid_config()