gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1445

Browse Source 
Enhance Activities and API Integration

- Updated Activities class to include API operations for external data ingestion.
- Added API configuration builder to connectors_config.py for environment variable management.
- Integrated API configuration into worker setup.
- Expanded unit tests to cover new API functionality and configuration handling.
- Updated requirements.txt to include pycurl and prometheus-client for enhanced metrics support.
This commit is contained in:
vitor-aignosi
2025-12-17 15:16:00 -03:00
parent ab99695702
commit e87c5d2da6
14 changed files with 1709 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
tests/activities/test_activities.py
View File

@@ -3,6 +3,7 @@ from unittest.mock import ANY, MagicMock, patch
from sientia_do.temporal.activities.postgres import Postgres
from scouter.activities.activities import Activities
from scouter.activities.api import API
from scouter.activities.gates import Gates
from scouter.activities.mongodb import MongoDB
from scouter.activities.redis import Redis
@@ -12,9 +13,15 @@ from scouter.activities.redis import Redis
@patch('scouter.activities.activities.Postgres.__init__')
@patch('scouter.activities.activities.Redis.__init__')
@patch('scouter.activities.activities.Gates.__init__')
@patch('scouter.activities.activities.API.__init__')
@patch('scouter.activities.activities.MetricsController')
def test___init__(
mock_metrics_controller, mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init
mock_metrics_controller,
mock_api_init,
mock_gates_init,
mock_redis_init,
mock_postgres_init,
mock_mongodb_init,
):
postgres_config = {
'host': 'localhost',
@@ -33,6 +40,12 @@ def test___init__(
'database_name': 'test_database',
}
api_config = {
'base_url': 'https://api.example.com',
'auth_type': 'bearer',
'auth_token': 'test_token',
}
logger = MagicMock()
notification_handler = MagicMock()
@@ -40,6 +53,7 @@ def test___init__(
postgres_config=postgres_config,
redis_config=redis_config,
mongodb_config=mongodb_config,
api_config=api_config,
logger=logger,
notification_handler=notification_handler,
)
@@ -49,6 +63,7 @@ def test___init__(
assert isinstance(activities, Redis)
assert isinstance(activities, MongoDB)
assert isinstance(activities, Gates)
assert isinstance(activities, API)
mock_postgres_init.assert_called_once_with(
ANY,
@@ -91,20 +106,34 @@ def test___init__(
metrics_controller=mock_metrics_controller.return_value,
)
mock_api_init.assert_called_once_with(
ANY,
base_url=api_config['base_url'],
auth_type=api_config['auth_type'],
auth_token=api_config['auth_token'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=mock_metrics_controller.return_value,
)
@patch('scouter.activities.activities.Postgres.__init__')
@patch('scouter.activities.activities.Redis.__init__')
@patch('scouter.activities.activities.Gates.__init__')
@patch('scouter.activities.activities.MongoDB.__init__')
@patch('scouter.activities.activities.API.__init__')
@patch('scouter.activities.activities.Postgres.close')
@patch('scouter.activities.activities.MongoDB.close')
@patch('scouter.activities.activities.Redis.close')
@patch('scouter.activities.activities.Gates.close')
@patch('scouter.activities.activities.API.close')
def test_shutdown(
mock_api_close,
mock_gates_close,
mock_redis_close,
mock_mongodb_close,
mock_postgres_close,
_mock_api_init,
_mock_mongodb_init,
_mock_gates_init,
_mock_redis_init,
@@ -127,6 +156,12 @@ def test_shutdown(
'database_name': 'test_database',
}
api_config = {
'base_url': 'https://api.example.com',
'auth_type': 'bearer',
'auth_token': 'test_token',
}
logger = MagicMock()
notification_handler = MagicMock()
@@ -134,6 +169,7 @@ def test_shutdown(
postgres_config=postgres_config,
redis_config=redis_config,
mongodb_config=mongodb_config,
api_config=api_config,
logger=logger,
notification_handler=notification_handler,
)
@@ -144,3 +180,4 @@ def test_shutdown(
mock_mongodb_close.assert_called()
mock_redis_close.assert_called()
mock_gates_close.assert_called()
mock_api_close.assert_called()

322
tests/activities/test_api.py Normal file
View File

@@ -0,0 +1,322 @@
from unittest.mock import ANY, AsyncMock, MagicMock, patch
import pandas as pd
import pytest
from sientia_do.notifications.models import NotificationLevel
from scouter.activities.api import API
@pytest.fixture
@patch('scouter.activities.api.PIWebAPIClient')
def api_activity(mock_pi_web_api_client):
"""Fixture to create an API activity instance with mocked dependencies."""
logger = MagicMock()
notification_handler = MagicMock()
metrics_controller = MagicMock()
activity = API(
base_url='https://pi.example.com',
auth_type='basic',
auth_token='test_token',
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
activity.logger = logger
activity.notification_handler = notification_handler
activity.metrics_controller = metrics_controller
activity.pod_id = 'test_pod_id'
return activity
metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'pi_web_api_scouter',
}
}
@patch('scouter.activities.api.PIWebAPIClient')
def test_api_initialization(mock_pi_web_api_client):
"""Test API activity initialization."""
logger = MagicMock()
notification_handler = MagicMock()
metrics_controller = MagicMock()
activity = API(
base_url='https://pi.example.com',
auth_type='basic',
auth_token='test_token',
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
mock_pi_web_api_client.assert_called_once_with(
base_url='https://pi.example.com',
auth_config={
'type': 'basic',
'token': 'test_token',
},
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
assert activity.pi_web_api_client is not None
@patch('scouter.activities.api.SientiaMonitoring')
def test_close(mock_sientia_monitoring, api_activity):
"""Test close method."""
api_activity.close()
api_activity.pi_web_api_client.close.assert_called_once()
mock_sientia_monitoring.shutdown.assert_called_once()
@pytest.mark.asyncio
async def test_get_tag_values_success(api_activity):
"""Test get_tag_values with successful data retrieval."""
# Setup test data
test_data = {
**metadata,
'endpoint': '/streamsets/recorded',
'web_ids': {
'tag1': {
'webid': 'webid1',
'aggr_function': 'avg',
'data_range': [0, 100],
},
'tag2': {
'webid': 'webid2',
'aggr_function': 'avg',
'data_range': [0, 100],
},
'tag3': {
'webid': 'webid3',
'aggr_function': 'avg',
'data_range': [0, 100],
},
},
'period': '*-1d',
'max_count': 10,
'api_timeout': 30,
}
# Mock DataFrame response
mock_df = pd.DataFrame(
{
'timestamp': ['2023-01-01 12:00:00', '2023-01-01 12:01:00', '2023-01-01 12:02:00'],
'name': ['tag1', 'tag2', 'tag3'],
'value': [10.5, 20.3, 30.7],
'tag': ['webid1', 'webid2', 'webid3'],
}
)
api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df)
# Execute
result = await api_activity.get_tag_values(test_data)
# Verify
api_activity.pi_web_api_client.get_latest_values_df.assert_called_once_with(
endpoint='/streamsets/recorded',
web_ids={
'tag1': {'webid': 'webid1', 'aggr_function': 'avg', 'data_range': [0, 100]},
'tag2': {'webid': 'webid2', 'aggr_function': 'avg', 'data_range': [0, 100]},
'tag3': {'webid': 'webid3', 'aggr_function': 'avg', 'data_range': [0, 100]},
},
start_time='*-1d',
max_count=10,
metadata=metadata['metadata'],
timeout=30,
)
assert len(result) == 3
assert result[0]['name'] == 'tag1'
assert result[0]['value'] == 10.5
assert result[1]['name'] == 'tag2'
assert result[2]['name'] == 'tag3'
@pytest.mark.asyncio
async def test_get_tag_values_with_default_max_count(api_activity):
"""Test get_tag_values with default max_count value."""
# Setup test data without max_count
test_data = {
**metadata,
'endpoint': '/streamsets/recorded',
'web_ids': {
'tag1': {
'webid': 'webid1',
'aggr_function': 'avg',
'data_range': [0, 100],
}
},
'period': '*-1h',
'api_timeout': 15,
}
mock_df = pd.DataFrame(
{
'timestamp': ['2023-01-01 12:00:00'],
'name': ['tag1'],
'value': [42.0],
'tag': ['webid1'],
}
)
api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df)
# Execute
result = await api_activity.get_tag_values(test_data)
# Verify default max_count is 1
api_activity.pi_web_api_client.get_latest_values_df.assert_called_once_with(
endpoint='/streamsets/recorded',
web_ids={'tag1': {'webid': 'webid1', 'aggr_function': 'avg', 'data_range': [0, 100]}},
start_time='*-1h',
max_count=1,
metadata=metadata['metadata'],
timeout=15,
)
assert len(result) == 1
@pytest.mark.asyncio
async def test_get_tag_values_with_none_webids(api_activity):
"""Test get_tag_values with some None WebIds."""
# Setup test data with None values
test_data = {
**metadata,
'endpoint': '/streamsets/recorded',
'web_ids': {
'tag1': {
'webid': 'webid1',
'aggr_function': 'avg',
'data_range': [0, 100],
},
'tag2': None,
'tag3': {
'webid': 'webid3',
'aggr_function': 'max',
'data_range': [0, 200],
},
},
'period': '*-1h',
'max_count': 5,
'api_timeout': 20,
}
mock_df = pd.DataFrame(
{
'timestamp': ['2023-01-01 12:00:00', '2023-01-01 12:01:00'],
'name': ['tag1', 'tag3'],
'value': [10.5, 30.7],
'tag': ['webid1', 'webid3'],
}
)
api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df)
# Execute
result = await api_activity.get_tag_values(test_data)
# Verify - should only query non-None WebIds
assert len(result) == 2
assert all(r['name'] in ['tag1', 'tag3'] for r in result)
@pytest.mark.asyncio
async def test_get_tag_values_api_error(api_activity):
"""Test get_tag_values when PI Web API client raises an error and sends notification."""
# Setup test data
test_data = {
**metadata,
'endpoint': '/streamsets/recorded',
'web_ids': {
'tag1': {
'webid': 'webid1',
'aggr_function': 'avg',
'data_range': [0, 100],
}
},
'period': '*-1d',
'max_count': 1,
'api_timeout': 30,
}
# Mock API error
api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(
side_effect=Exception('PI Web API connection error')
)
api_activity.send_notification_async = AsyncMock()
# Execute and verify exception is raised
with pytest.raises(Exception) as exc_info:
await api_activity.get_tag_values(test_data)
assert str(exc_info.value) == 'PI Web API connection error'
# Verify notification was sent
api_activity.send_notification_async.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='PI_WEB_API_REQUEST_ERROR',
message='Error getting tag values from PI Web API: PI Web API connection error',
block='get_tag_values',
level=NotificationLevel.ERROR,
attachment_content=ANY,
)
@pytest.mark.asyncio
async def test_get_tag_values_with_nan_values(api_activity):
"""Test get_tag_values handling NaN values in the DataFrame."""
# Setup test data
test_data = {
**metadata,
'endpoint': '/streamsets/recorded',
'web_ids': {
'tag1': {
'webid': 'webid1',
'aggr_function': 'avg',
'data_range': [0, 100],
},
'tag2': {
'webid': 'webid2',
'aggr_function': 'avg',
'data_range': [0, 100],
},
},
'period': '*-1d',
'max_count': 1,
'api_timeout': 30,
}
# Mock DataFrame with NaN values
mock_df = pd.DataFrame(
{
'timestamp': ['2023-01-01 12:00:00', '2023-01-01 12:00:00'],
'name': ['tag1', 'tag2'],
'value': [10.0, float('nan')],
'tag': ['webid1', 'webid2'],
}
)
api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df)
# Execute
result = await api_activity.get_tag_values(test_data)
# Verify
assert len(result) == 2
assert result[0]['value'] == 10.0
# NaN should be preserved in the result
assert pd.isna(result[1]['value'])

0
tests/utils/clients/__init__.py Normal file
View File

560
tests/utils/clients/test_pi_web_api_client.py Normal file
View File

@@ -0,0 +1,560 @@
import json
from unittest.mock import AsyncMock, MagicMock, patch
import pandas as pd
import pycurl
import pytest
from scouter.utils.clients.pi_web_api_client import PIMSRequestError, PIWebAPIClient
@pytest.fixture
def mock_logger():
return MagicMock()
@pytest.fixture
def mock_notification_handler():
return AsyncMock()
@pytest.fixture
def mock_metrics_controller():
return AsyncMock()
@pytest.fixture
def auth_config_basic():
return {'type': 'basic', 'token': 'test_token_123'}
@pytest.fixture
def auth_config_bearer():
return {'type': 'bearer', 'token': 'bearer_token_456'}
@pytest.fixture
def pi_client(mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic):
return PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=auth_config_basic,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
def test_init_with_basic_auth(
mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic
):
"""Test initialization with basic authentication"""
client = PIWebAPIClient(
base_url='https://pi.example.com/',
auth_config=auth_config_basic,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
assert client.base_url == 'https://pi.example.com'
assert client.auth_config['type'] == 'basic'
assert client.headers['Authorization'] == 'Basic test_token_123'
assert client.headers['Content-Type'] == 'application/json'
assert client.headers['Accept'] == 'application/json'
mock_logger.info.assert_called_with('Authenticating with basic authentication')
def test_init_with_bearer_auth(
mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_bearer
):
"""Test initialization with bearer authentication"""
client = PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=auth_config_bearer,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
assert client.base_url == 'https://pi.example.com'
assert client.auth_config['type'] == 'bearer'
assert client.headers['Authorization'] == 'Bearer bearer_token_456'
mock_logger.info.assert_called_with('Authenticating with bearer authentication')
def test_init_with_custom_headers(
mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic
):
"""Test initialization with custom headers"""
custom_headers = {
'Content-Type': 'application/xml',
'Custom-Header': 'custom_value',
}
client = PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=auth_config_basic,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
headers_config=custom_headers,
)
assert client.headers['Content-Type'] == 'application/xml'
assert client.headers['Custom-Header'] == 'custom_value'
assert client.headers['Authorization'] == 'Basic test_token_123'
def test_authenticate_invalid_type(mock_logger, mock_notification_handler, mock_metrics_controller):
"""Test that invalid authentication type raises ValueError"""
invalid_auth_config = {'type': 'invalid', 'token': 'test_token'}
with pytest.raises(ValueError) as exc_info:
PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=invalid_auth_config,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
assert 'Invalid authentication type: invalid' in str(exc_info.value)
@patch('scouter.utils.clients.pi_web_api_client.SientiaMonitoring.shutdown')
def test_close(mock_shutdown, pi_client):
"""Test close method calls shutdown"""
pi_client.close()
mock_shutdown.assert_called_once()
def test_to_clean_timestamp(pi_client):
"""Test timestamp cleaning and normalization"""
timestamps = pd.Series(
[
'2025-01-15T10:30:45.123456Z',
'2025-01-15T10:30:46.789012Z',
'2025-01-15T10:30:47.999999Z',
]
)
result = pi_client._to_clean_timestamp(timestamps)
assert isinstance(result, pd.Series)
assert result.dtype == 'datetime64[ns, UTC]'
# Verify microseconds are floored to seconds
assert result[0] == pd.Timestamp('2025-01-15T10:30:45Z')
assert result[1] == pd.Timestamp('2025-01-15T10:30:46Z')
assert result[2] == pd.Timestamp('2025-01-15T10:30:47Z')
def test_to_clean_timestamp_with_invalid_values(pi_client):
"""Test timestamp cleaning with invalid values returns NaT"""
timestamps = pd.Series(['invalid', 'not_a_date', '2025-01-15T10:30:45Z'])
result = pi_client._to_clean_timestamp(timestamps)
assert pd.isna(result[0])
assert pd.isna(result[1])
assert result[2] == pd.Timestamp('2025-01-15T10:30:45Z')
def test_extract_numeric_with_float(pi_client):
"""Test extracting numeric value from float"""
result = pi_client._extract_numeric(42.5)
assert result == 42.5
def test_extract_numeric_with_int(pi_client):
"""Test extracting numeric value from int"""
result = pi_client._extract_numeric(42)
assert result == 42.0
def test_extract_numeric_with_string(pi_client):
"""Test extracting numeric value from string"""
result = pi_client._extract_numeric('123.45')
assert result == 123.45
def test_extract_numeric_with_dict(pi_client):
"""Test extracting numeric value from dictionary"""
result = pi_client._extract_numeric({'Value': 99.9})
assert result == 99.9
def test_extract_numeric_with_invalid_value(pi_client):
"""Test extracting numeric value from invalid value returns None/NaN"""
result = pi_client._extract_numeric('invalid_number')
assert pd.isna(result)
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_success(mock_curl_class, pi_client):
"""Test successful GET request with JSON response"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'status': 'success', 'data': [1, 2, 3]}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
result = await pi_client._curl_get_json('https://pi.example.com/api/test')
assert result == response_data
mock_curl.setopt.assert_any_call(pycurl.TIMEOUT, 30)
mock_curl.perform.assert_called_once()
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_with_params(mock_curl_class, pi_client):
"""Test GET request with query parameters"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'result': 'ok'}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
params = [('key1', 'value1'), ('key2', 'value2')]
result = await pi_client._curl_get_json('https://pi.example.com/api', params=params)
assert result == response_data
# Verify URL includes query parameters
set_url_call = [call for call in mock_curl.setopt.call_args_list if call[0][0] == pycurl.URL][0]
assert b'key1=value1' in set_url_call[0][1]
assert b'key2=value2' in set_url_call[0][1]
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_http_error(mock_curl_class, pi_client):
"""Test GET request with HTTP error response"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
error_response = b'{"error": "Not found"}'
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(error_response)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 404
with pytest.raises(PIMSRequestError) as exc_info:
await pi_client._curl_get_json('https://pi.example.com/api/notfound')
assert 'HTTP 404' in str(exc_info.value)
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_connection_error(mock_curl_class, pi_client):
"""Test GET request with connection error"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
mock_curl.perform.side_effect = pycurl.error('Connection failed')
with pytest.raises(PIMSRequestError) as exc_info:
await pi_client._curl_get_json('https://pi.example.com/api/test')
assert 'Connection error' in str(exc_info.value)
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_invalid_json(mock_curl_class, pi_client):
"""Test GET request with invalid JSON response"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
invalid_json = b'This is not valid JSON'
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(invalid_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
with pytest.raises(PIMSRequestError) as exc_info:
await pi_client._curl_get_json('https://pi.example.com/api/test')
assert 'Error decoding JSON response' in str(exc_info.value)
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_with_custom_timeout(mock_curl_class, pi_client):
"""Test GET request with custom timeout"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'status': 'ok'}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
await pi_client._curl_get_json('https://pi.example.com/api/test', timeout=60)
mock_curl.setopt.assert_any_call(pycurl.TIMEOUT, 60)
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_without_ssl_verify(mock_curl_class, pi_client):
"""Test GET request with SSL verification disabled"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'status': 'ok'}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
await pi_client._curl_get_json('https://pi.example.com/api/test', verify=False)
mock_curl.setopt.assert_any_call(pycurl.SSL_VERIFYPEER, 0)
mock_curl.setopt.assert_any_call(pycurl.SSL_VERIFYHOST, 0)
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_success(mock_curl_get_json, pi_client):
"""Test successful retrieval of latest values"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5},
{'Timestamp': '2025-01-15T10:31:00Z', 'Value': 43.0},
],
},
{
'Name': 'tag2',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 100.0},
],
},
]
}
web_ids = {
'tag1': {'webid': 'webid1'},
'tag2': {'webid': 'webid2'},
}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
start_time='*-1d',
end_time='*',
max_count=10,
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 3
assert list(result.columns) == ['timestamp', 'name', 'value', 'tag']
assert result['name'].tolist() == ['tag1', 'tag1', 'tag2']
assert result['value'].tolist() == [42.5, 43.0, 100.0]
mock_curl_get_json.assert_called_once()
call_args = mock_curl_get_json.call_args
assert call_args[1]['url'] == 'https://pi.example.com/streamsets/recorded'
assert call_args[1]['timeout'] == 30
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_with_custom_params(mock_curl_get_json, pi_client):
"""Test get_latest_values_df with custom parameters"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5},
],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
metadata = {'model_id': 'test_model'}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
start_time='*-7d',
end_time='*-1d',
max_count=100,
timeout=60,
metadata=metadata,
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 1
mock_curl_get_json.assert_called_once()
call_args = mock_curl_get_json.call_args
params = call_args[1]['params']
# Verify parameters
assert ('startTime', '*-7d') in params
assert ('endtime', '*-1d') in params
assert ('maxCount', '100') in params
assert call_args[1]['timeout'] == 60
assert call_args[1]['metadata'] == metadata
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_empty_response(mock_curl_get_json, pi_client):
"""Test get_latest_values_df with empty response"""
mock_curl_get_json.return_value = {'Items': []}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 0
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_no_items_in_tag(mock_curl_get_json, pi_client):
"""Test get_latest_values_df when tag has no items"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 0
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_with_missing_timestamp(mock_curl_get_json, pi_client):
"""Test get_latest_values_df filters out items with missing timestamp"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5},
{'Value': 43.0}, # Missing Timestamp
{'Timestamp': None, 'Value': 44.0}, # None Timestamp
],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 1 # Only the first item should be included
assert result['value'].tolist() == [42.5]
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_with_nested_value(mock_curl_get_json, pi_client):
"""Test get_latest_values_df with nested value extraction"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': {'Value': 42.5}},
],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 1
assert result['value'].tolist() == [42.5]
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_default_max_count(mock_curl_get_json, pi_client):
"""Test get_latest_values_df uses default max_count of 1"""
mock_curl_get_json.return_value = {'Items': []}
web_ids = {'tag1': {'webid': 'webid1'}}
await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
call_args = mock_curl_get_json.call_args
params = call_args[1]['params']
assert ('maxCount', '1') in params

33
tests/utils/test_connectors_config.py
View File

@@ -4,6 +4,7 @@ from unittest.mock import patch
import pytest
from scouter.utils.connectors_config import (
build_api_config,
build_druid_config,
build_kafka_config,
build_mongodb_config,
@@ -154,6 +155,38 @@ def test_build_mongodb_config_with_env_vars():
}
@pytest.mark.usefixtures('mock_env_vars')
def test_build_api_config_defaults():
"""Test that build_api_config returns default values when no env vars are set"""
config = build_api_config()
assert config == {
'base_url': 'https://pi.example.com',
'auth_type': 'basic',
'auth_token': None,
}
@pytest.mark.usefixtures('mock_env_vars')
def test_build_api_config_with_env_vars():
"""Test that build_api_config uses env vars when set"""
with patch.dict(
os.environ,
{
'API_BASE_URL': 'https://api.production.com',
'API_AUTH_TYPE': 'bearer',
'API_AUTH_TOKEN': 'secret_token_123',
},
):
config = build_api_config()
assert config == {
'base_url': 'https://api.production.com',
'auth_type': 'bearer',
'auth_token': 'secret_token_123',
}
def test_build_druid_config_defaults():
"""Test that build_druid_config returns default values when no env vars are set"""
config = build_druid_config()

126
tests/workflow/test_pi_web_api_scouter.py Normal file
View File

@@ -0,0 +1,126 @@
from unittest.mock import ANY, AsyncMock, patch
from pytest import fixture, mark
from scouter.activities.activities import Activities
from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter
@fixture
def pi_web_api_scouter():
return PIWebAPIScouter()
@mark.asyncio
@patch('scouter.workflow.pi_web_api_scouter.workflow', new_callable=AsyncMock)
async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter):
mock_workflow.execute_local_activity_method.return_value = 'test_data'
await pi_web_api_scouter.run(
input_data={
'model_name': 'test_model',
'model_id': 'test_model_id',
'schedule_name': 'test_schedule',
'endpoint': '/streamsets/recorded',
'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
'period': {'start_time': '2025-01-01T00:00:00Z'},
'api_timeout': 30,
'max_count': 10,
'trigger_laborious': True,
'filters': {'quality': 'good'},
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter',
}
}
mock_workflow.execute_local_activity_method.assert_called_once_with(
Activities.get_tag_values,
{
**expected_metadata,
'endpoint': '/streamsets/recorded',
'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'},
'period': {'start_time': '2025-01-01T00:00:00Z'},
'api_timeout': 30,
'max_count': 10,
},
start_to_close_timeout=ANY,
retry_policy=ANY,
)
mock_workflow.execute_child_workflow.assert_called_once_with(
'subworkflow.core_scouter',
{
'model_name': 'test_model',
'model_id': 'test_model_id',
'schedule_name': 'test_schedule',
'endpoint': '/streamsets/recorded',
'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
'period': {'start_time': '2025-01-01T00:00:00Z'},
'api_timeout': 30,
'max_count': 10,
'trigger_laborious': True,
'filters': {'quality': 'good'},
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
'workflow_name': 'scouter',
'data': 'test_data',
'metadata': expected_metadata,
},
)
@mark.asyncio
@patch('scouter.workflow.pi_web_api_scouter.workflow', new_callable=AsyncMock)
async def test_pi_web_api_scouter_workflow_empty(mock_workflow, pi_web_api_scouter):
mock_workflow.execute_local_activity_method.return_value = []
await pi_web_api_scouter.run(
input_data={
'model_name': 'test_model',
'model_id': 'test_model_id',
'schedule_name': 'test_schedule',
'endpoint': '/streamsets/recorded',
'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
'period': {'start_time': '2025-01-01T00:00:00Z'},
'api_timeout': 30,
'trigger_laborious': True,
'filters': {'quality': 'good'},
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter',
}
}
mock_workflow.execute_local_activity_method.assert_called_once_with(
Activities.get_tag_values,
{
**expected_metadata,
'endpoint': '/streamsets/recorded',
'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'},
'period': {'start_time': '2025-01-01T00:00:00Z'},
'api_timeout': 30,
'max_count': 1,
},
start_to_close_timeout=ANY,
retry_policy=ANY,
)
mock_workflow.execute_child_workflow.assert_not_called()