gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1478

Browse Source 
Update pytest_asyncio fixture scopes in conftest.py for improved test isolation and add asyncio_default_fixture_loop_scope in pyproject.toml. Remove outdated scenarios from scenarios.md and delete unused test files for cleaner codebase.
This commit is contained in:
vitor-aignosi
2026-01-08 16:13:59 -03:00
parent cf7f9fb63c
commit 7958afbadf
9 changed files with 14 additions and 1085 deletions
Download Patch File Download Diff File Expand all files Collapse all files

588
tests/utils/clients/test_pi_web_api_client.py
View File

@@ -1,588 +0,0 @@
import json
from unittest.mock import AsyncMock, MagicMock, patch
import pandas as pd
import pycurl
import pytest
from scouter.utils.clients.pi_web_api_client import PIMSRequestError, PIWebAPIClient
@pytest.fixture
def mock_logger():
return MagicMock()
@pytest.fixture
def mock_notification_handler():
return AsyncMock()
@pytest.fixture
def mock_metrics_controller():
return AsyncMock()
@pytest.fixture
def auth_config_basic():
return {'type': 'basic', 'token': 'test_token_123'}
@pytest.fixture
def auth_config_bearer():
return {'type': 'bearer', 'token': 'bearer_token_456'}
@pytest.fixture
def pi_client(mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic):
return PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=auth_config_basic,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
def test_init_with_basic_auth(
mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic
):
"""Test initialization with basic authentication"""
client = PIWebAPIClient(
base_url='https://pi.example.com/',
auth_config=auth_config_basic,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
assert client.base_url == 'https://pi.example.com'
assert client.auth_config['type'] == 'basic'
assert client.headers['Authorization'] == 'Basic test_token_123'
assert client.headers['Content-Type'] == 'application/json'
assert client.headers['Accept'] == 'application/json'
mock_logger.info.assert_called_with('Authenticating with basic authentication')
def test_init_with_bearer_auth(
mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_bearer
):
"""Test initialization with bearer authentication"""
client = PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=auth_config_bearer,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
assert client.base_url == 'https://pi.example.com'
assert client.auth_config['type'] == 'bearer'
assert client.headers['Authorization'] == 'Bearer bearer_token_456'
mock_logger.info.assert_called_with('Authenticating with bearer authentication')
def test_init_with_custom_headers(
mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic
):
"""Test initialization with custom headers"""
custom_headers = {
'Content-Type': 'application/xml',
'Custom-Header': 'custom_value',
}
client = PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=auth_config_basic,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
headers_config=custom_headers,
)
assert client.headers['Content-Type'] == 'application/xml'
assert client.headers['Custom-Header'] == 'custom_value'
assert client.headers['Authorization'] == 'Basic test_token_123'
def test_authenticate_invalid_type(mock_logger, mock_notification_handler, mock_metrics_controller):
"""Test that invalid authentication type raises ValueError"""
invalid_auth_config = {'type': 'invalid', 'token': 'test_token'}
with pytest.raises(ValueError) as exc_info:
PIWebAPIClient(
base_url='https://pi.example.com',
auth_config=invalid_auth_config,
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
assert 'Invalid authentication type: invalid' in str(exc_info.value)
@patch('scouter.utils.clients.pi_web_api_client.SientiaMonitoring.shutdown')
def test_close(mock_shutdown, pi_client):
"""Test close method calls shutdown"""
pi_client.close()
mock_shutdown.assert_called_once()
def test_to_clean_timestamp(pi_client):
"""Test timestamp cleaning and normalization"""
timestamps = pd.Series(
[
'2025-01-15T10:30:45.123456Z',
'2025-01-15T10:30:46.789012Z',
'2025-01-15T10:30:47.999999Z',
]
)
result = pi_client._to_clean_timestamp(timestamps)
assert isinstance(result, pd.Series)
assert result.dtype == 'datetime64[ns, UTC]'
# Verify microseconds are floored to seconds
assert result[0] == pd.Timestamp('2025-01-15T10:30:45Z')
assert result[1] == pd.Timestamp('2025-01-15T10:30:46Z')
assert result[2] == pd.Timestamp('2025-01-15T10:30:47Z')
def test_to_clean_timestamp_with_invalid_values(pi_client):
"""Test timestamp cleaning with invalid values returns NaT"""
timestamps = pd.Series(['invalid', 'not_a_date', '2025-01-15T10:30:45Z'])
result = pi_client._to_clean_timestamp(timestamps)
assert pd.isna(result[0])
assert pd.isna(result[1])
assert result[2] == pd.Timestamp('2025-01-15T10:30:45Z')
def test_extract_numeric_with_float(pi_client):
"""Test extracting numeric value from float"""
result = pi_client._extract_numeric(42.5)
assert result == pytest.approx(42.5)
def test_extract_numeric_with_int(pi_client):
"""Test extracting numeric value from int"""
result = pi_client._extract_numeric(42)
assert result == pytest.approx(42.0)
def test_extract_numeric_with_string(pi_client):
"""Test extracting numeric value from string"""
result = pi_client._extract_numeric('123.45')
assert result == pytest.approx(123.45)
def test_extract_numeric_with_dict(pi_client):
"""Test extracting numeric value from dictionary"""
result = pi_client._extract_numeric({'Value': 99.9})
assert result == pytest.approx(99.9)
def test_extract_numeric_with_invalid_value(pi_client):
"""Test extracting numeric value from invalid value returns None/NaN"""
result = pi_client._extract_numeric('invalid_number')
assert pd.isna(result)
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_success(mock_curl_class, pi_client):
"""Test successful GET request with JSON response"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'status': 'success', 'data': [1, 2, 3]}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
result = await pi_client._curl_get_json('https://pi.example.com/api/test')
assert result == response_data
mock_curl.setopt.assert_any_call(pycurl.TIMEOUT, 30)
mock_curl.perform.assert_called_once()
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_with_params(mock_curl_class, pi_client):
"""Test GET request with query parameters"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'result': 'ok'}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
params = [('key1', 'value1'), ('key2', 'value2')]
result = await pi_client._curl_get_json('https://pi.example.com/api', params=params)
assert result == response_data
# Verify URL includes query parameters
set_url_call = [call for call in mock_curl.setopt.call_args_list if call[0][0] == pycurl.URL][0]
assert b'key1=value1' in set_url_call[0][1]
assert b'key2=value2' in set_url_call[0][1]
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_http_error(mock_curl_class, pi_client):
"""Test GET request with HTTP error response"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
error_response = b'{"error": "Not found"}'
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(error_response)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 404
with pytest.raises(PIMSRequestError) as exc_info:
await pi_client._curl_get_json('https://pi.example.com/api/notfound')
assert 'HTTP 404' in str(exc_info.value)
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_connection_error(mock_curl_class, pi_client):
"""Test GET request with connection error"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
mock_curl.perform.side_effect = pycurl.error('Connection failed')
with pytest.raises(PIMSRequestError) as exc_info:
await pi_client._curl_get_json('https://pi.example.com/api/test')
assert 'Connection error' in str(exc_info.value)
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_invalid_json(mock_curl_class, pi_client):
"""Test GET request with invalid JSON response"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
invalid_json = b'This is not valid JSON'
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(invalid_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
with pytest.raises(PIMSRequestError) as exc_info:
await pi_client._curl_get_json('https://pi.example.com/api/test')
assert 'Error decoding JSON response' in str(exc_info.value)
mock_curl.close.assert_called_once()
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_with_custom_timeout(mock_curl_class, pi_client):
"""Test GET request with custom timeout"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'status': 'ok'}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
await pi_client._curl_get_json('https://pi.example.com/api/test', timeout=60)
mock_curl.setopt.assert_any_call(pycurl.TIMEOUT, 60)
@pytest.mark.asyncio
@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl')
async def test_curl_get_json_without_ssl_verify(mock_curl_class, pi_client):
"""Test GET request with SSL verification disabled"""
mock_curl = MagicMock()
mock_curl_class.return_value = mock_curl
response_data = {'status': 'ok'}
response_json = json.dumps(response_data).encode('utf-8')
def mock_perform():
buffer = mock_curl.setopt.call_args_list[1][0][1]
buffer.write(response_json)
mock_curl.perform.side_effect = mock_perform
mock_curl.getinfo.return_value = 200
await pi_client._curl_get_json('https://pi.example.com/api/test', verify=False)
mock_curl.setopt.assert_any_call(pycurl.SSL_VERIFYPEER, 0)
mock_curl.setopt.assert_any_call(pycurl.SSL_VERIFYHOST, 0)
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_success(mock_curl_get_json, pi_client):
"""Test successful retrieval of latest values"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5},
{'Timestamp': '2025-01-15T10:31:00Z', 'Value': 43.0},
],
},
{
'Name': 'tag2',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 100.0},
],
},
]
}
web_ids = {
'tag1': {'webid': 'webid1'},
'tag2': {'webid': 'webid2'},
}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
start_time='*-1d',
end_time='*',
max_count=10,
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 3
assert list(result.columns) == ['timestamp', 'name', 'value', 'tag']
assert result['name'].tolist() == ['tag1', 'tag1', 'tag2']
assert result['value'].tolist() == [42.5, 43.0, 100.0]
mock_curl_get_json.assert_called_once()
call_args = mock_curl_get_json.call_args
assert call_args[1]['url'] == 'https://pi.example.com/streamsets/recorded'
assert call_args[1]['timeout'] == 30
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_with_custom_params(mock_curl_get_json, pi_client):
"""Test get_latest_values_df with custom parameters"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5},
],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
metadata = {'model_id': 'test_model'}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
start_time='*-7d',
end_time='*-1d',
max_count=100,
timeout=60,
metadata=metadata,
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 1
mock_curl_get_json.assert_called_once()
call_args = mock_curl_get_json.call_args
params = call_args[1]['params']
# Verify parameters (inverted: startTime uses end_time, endTime uses start_time)
assert ('startTime', '*-1d') in params
assert ('endtime', '*-7d') in params
assert ('maxCount', '100') in params
assert call_args[1]['timeout'] == 60
assert call_args[1]['metadata'] == metadata
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_empty_response(mock_curl_get_json, pi_client):
"""Test get_latest_values_df with empty response"""
mock_curl_get_json.return_value = {'Items': []}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 0
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_no_items_in_tag(mock_curl_get_json, pi_client):
"""Test get_latest_values_df when tag has no items"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 0
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_with_missing_timestamp(mock_curl_get_json, pi_client):
"""Test get_latest_values_df filters out items with missing timestamp"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5},
{'Value': 43.0}, # Missing Timestamp
{'Timestamp': None, 'Value': 44.0}, # None Timestamp
],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 1 # Only the first item should be included
assert result['value'].tolist() == [42.5]
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_with_nested_value(mock_curl_get_json, pi_client):
"""Test get_latest_values_df with nested value extraction"""
mock_curl_get_json.return_value = {
'Items': [
{
'Name': 'tag1',
'Items': [
{'Timestamp': '2025-01-15T10:30:00Z', 'Value': {'Value': 42.5}},
],
}
]
}
web_ids = {'tag1': {'webid': 'webid1'}}
result = await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
assert isinstance(result, pd.DataFrame)
assert len(result) == 1
assert result['value'].tolist() == [42.5]
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_default_max_count(mock_curl_get_json, pi_client):
"""Test get_latest_values_df uses default max_count of 1"""
mock_curl_get_json.return_value = {'Items': []}
web_ids = {'tag1': {'webid': 'webid1'}}
await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
)
call_args = mock_curl_get_json.call_args
params = call_args[1]['params']
assert ('maxCount', '1') in params
# Verify default time parameters are inverted (startTime uses end_time default, endTime uses start_time default)
assert ('startTime', '*') in params # Default end_time
assert ('endtime', '*-1d') in params # Default start_time
@pytest.mark.asyncio
@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock)
async def test_get_latest_values_df_with_none_max_count(mock_curl_get_json, pi_client):
"""Test get_latest_values_df does not send maxCount parameter when max_count is None"""
mock_curl_get_json.return_value = {'Items': []}
web_ids = {'tag1': {'webid': 'webid1'}}
await pi_client.get_latest_values_df(
web_ids=web_ids,
endpoint='/streamsets/recorded',
max_count=None,
)
call_args = mock_curl_get_json.call_args
params = call_args[1]['params']
# Verify maxCount parameter is not present when max_count is None
assert ('maxCount', '1') not in params
assert ('maxCount', None) not in params
# Verify time parameters are still present
assert ('startTime', '*') in params
assert ('endtime', '*-1d') in params

160
tests/utils/test_connectors_config.py
View File

@@ -4,12 +4,7 @@ from unittest.mock import patch
import pytest
from scouter.utils.connectors_config import (
build_api_config,
build_druid_config,
build_kafka_config,
build_mongodb_config,
build_postgres_config,
build_redis_config,
)
@@ -19,50 +14,6 @@ def mock_env_vars():
yield
@pytest.mark.usefixtures('mock_env_vars')
def test_build_postgres_config_defaults():
"""Test that build_postgres_config returns default values when no env vars are set"""
config = build_postgres_config()
assert config == {
'host': 'localhost',
'port': 5432,
'user': 'sientia',
'password': 'sientia',
'dbname': 'sientia',
'min_connections': 5,
'max_connections': 20,
}
@pytest.mark.usefixtures('mock_env_vars')
def test_build_postgres_config_with_env_vars():
"""Test that build_postgres_config uses env vars when set"""
with patch.dict(
os.environ,
{
'POSTGRES_HOST': 'db.example.com',
'POSTGRES_PORT': '5433',
'POSTGRES_USER': 'admin',
'POSTGRES_PASSWORD': 'secret',
'POSTGRES_DBNAME': 'test_db',
'POSTGRES_MIN_CONNECTIONS': '3',
'POSTGRES_MAX_CONNECTIONS': '15',
},
):
config = build_postgres_config()
assert config == {
'host': 'db.example.com',
'port': 5433,
'user': 'admin',
'password': 'secret',
'dbname': 'test_db',
'min_connections': 3,
'max_connections': 15,
}
@pytest.mark.usefixtures('mock_env_vars')
def test_build_kafka_config_defaults():
"""Test that build_kafka_config returns default values when no env vars are set"""
@@ -89,114 +40,3 @@ def test_build_kafka_config_with_env_vars():
'polling_time': 5000,
'group_id': 'scouter-group',
}
@pytest.mark.usefixtures('mock_env_vars')
def test_build_redis_config_defaults():
"""Test that build_redis_config returns default values when no env vars are set"""
config = build_redis_config()
assert config == {'host': 'localhost', 'port': 6379, 'username': None, 'password': None}
@pytest.mark.usefixtures('mock_env_vars')
def test_build_redis_config_with_env_vars():
"""Test that build_redis_config uses env vars when set"""
with patch.dict(
os.environ,
{
'REDIS_HOST': 'redis.example.com',
'REDIS_PORT': '6380',
'REDIS_USERNAME': 'test',
'REDIS_PASSWORD': 'test',
},
):
config = build_redis_config()
assert config == {
'host': 'redis.example.com',
'port': 6380,
'username': 'test',
'password': 'test',
}
def test_build_mongodb_config_defaults():
"""Test that build_mongodb_config returns default values when no env vars are set"""
os.environ['MONGODB_URL'] = 'localhost:27017'
os.environ['MONGODB_DATABASE_NAME'] = 'sientia'
os.environ['MONGODB_USERNAME'] = 'sientia'
os.environ['MONGODB_PASSWORD'] = 'sientia'
config = build_mongodb_config()
assert config == {
'connection_string': 'mongodb://sientia:sientia@localhost:27017', # NOSONAR
'database_name': 'sientia',
}
def test_build_mongodb_config_with_env_vars():
"""Test that build_mongodb_config uses env vars when set"""
with patch.dict(
os.environ,
{
'MONGODB_URL': 'mongodb.example.com:27017',
'MONGODB_DATABASE_NAME': 'test_db',
'MONGODB_USERNAME': 'test',
'MONGODB_PASSWORD': 'test',
},
):
config = build_mongodb_config()
assert config == {
'connection_string': 'mongodb://test:test@mongodb.example.com:27017',
'database_name': 'test_db',
}
@pytest.mark.usefixtures('mock_env_vars')
def test_build_api_config_defaults():
"""Test that build_api_config returns default values when no env vars are set"""
config = build_api_config()
assert config == {
'base_url': 'https://pi.example.com',
'auth_type': 'basic',
'auth_token': None,
}
@pytest.mark.usefixtures('mock_env_vars')
def test_build_api_config_with_env_vars():
"""Test that build_api_config uses env vars when set"""
with patch.dict(
os.environ,
{
'PI_WEB_API_BASE_URL': 'https://api.production.com',
'PI_WEB_API_AUTH_TYPE': 'bearer',
'PI_WEB_API_AUTH_TOKEN': 'secret_token_123',
},
):
config = build_api_config()
assert config == {
'base_url': 'https://api.production.com',
'auth_type': 'bearer',
'auth_token': 'secret_token_123',
}
def test_build_druid_config_defaults():
"""Test that build_druid_config returns default values when no env vars are set"""
config = build_druid_config()
assert config == {'host': 'localhost', 'port': 8082}
def test_build_druid_config_with_env_vars():
"""Test that build_druid_config uses env vars when set"""
with patch.dict(os.environ, {'DRUID_HOST': 'druid.example.com', 'DRUID_PORT': '8083'}):
config = build_druid_config()
assert config == {'host': 'druid.example.com', 'port': 8083}