Enhance Activities and API Integration - Updated Activities class to include API operations for external data ingestion. - Added API configuration builder to connectors_config.py for environment variable management. - Integrated API configuration into worker setup. - Expanded unit tests to cover new API functionality and configuration handling. - Updated requirements.txt to include pycurl and prometheus-client for enhanced metrics support.
203 lines
5.7 KiB
Python
203 lines
5.7 KiB
Python
import os
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from scouter.utils.connectors_config import (
|
|
build_api_config,
|
|
build_druid_config,
|
|
build_kafka_config,
|
|
build_mongodb_config,
|
|
build_postgres_config,
|
|
build_redis_config,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_env_vars():
|
|
with patch.dict(os.environ, {}, clear=True):
|
|
yield
|
|
|
|
|
|
@pytest.mark.usefixtures('mock_env_vars')
|
|
def test_build_postgres_config_defaults():
|
|
"""Test that build_postgres_config returns default values when no env vars are set"""
|
|
config = build_postgres_config()
|
|
|
|
assert config == {
|
|
'host': 'localhost',
|
|
'port': 5432,
|
|
'user': 'sientia',
|
|
'password': 'sientia',
|
|
'dbname': 'sientia',
|
|
'min_connections': 5,
|
|
'max_connections': 20,
|
|
}
|
|
|
|
|
|
@pytest.mark.usefixtures('mock_env_vars')
|
|
def test_build_postgres_config_with_env_vars():
|
|
"""Test that build_postgres_config uses env vars when set"""
|
|
with patch.dict(
|
|
os.environ,
|
|
{
|
|
'POSTGRES_HOST': 'db.example.com',
|
|
'POSTGRES_PORT': '5433',
|
|
'POSTGRES_USER': 'admin',
|
|
'POSTGRES_PASSWORD': 'secret',
|
|
'POSTGRES_DBNAME': 'test_db',
|
|
'POSTGRES_MIN_CONNECTIONS': '3',
|
|
'POSTGRES_MAX_CONNECTIONS': '15',
|
|
},
|
|
):
|
|
config = build_postgres_config()
|
|
|
|
assert config == {
|
|
'host': 'db.example.com',
|
|
'port': 5433,
|
|
'user': 'admin',
|
|
'password': 'secret',
|
|
'dbname': 'test_db',
|
|
'min_connections': 3,
|
|
'max_connections': 15,
|
|
}
|
|
|
|
|
|
@pytest.mark.usefixtures('mock_env_vars')
|
|
def test_build_kafka_config_defaults():
|
|
"""Test that build_kafka_config returns default values when no env vars are set"""
|
|
config = build_kafka_config()
|
|
|
|
assert config == {
|
|
'bootstrap_servers': 'localhost:9092',
|
|
'polling_time': 1000,
|
|
'group_id': 'scouter-group',
|
|
}
|
|
|
|
|
|
@pytest.mark.usefixtures('mock_env_vars')
|
|
def test_build_kafka_config_with_env_vars():
|
|
"""Test that build_kafka_config uses env vars when set"""
|
|
with patch.dict(
|
|
os.environ,
|
|
{'KAFKA_BOOTSTRAP_SERVERS': 'kafka.example.com:9092', 'KAFKA_POLLING_TIME': '5000'},
|
|
):
|
|
config = build_kafka_config()
|
|
|
|
assert config == {
|
|
'bootstrap_servers': 'kafka.example.com:9092',
|
|
'polling_time': 5000,
|
|
'group_id': 'scouter-group',
|
|
}
|
|
|
|
|
|
@pytest.mark.usefixtures('mock_env_vars')
|
|
def test_build_redis_config_defaults():
|
|
"""Test that build_redis_config returns default values when no env vars are set"""
|
|
config = build_redis_config()
|
|
|
|
assert config == {'host': 'localhost', 'port': 6379, 'username': None, 'password': None}
|
|
|
|
|
|
@pytest.mark.usefixtures('mock_env_vars')
|
|
def test_build_redis_config_with_env_vars():
|
|
"""Test that build_redis_config uses env vars when set"""
|
|
with patch.dict(
|
|
os.environ,
|
|
{
|
|
'REDIS_HOST': 'redis.example.com',
|
|
'REDIS_PORT': '6380',
|
|
'REDIS_USERNAME': 'test',
|
|
'REDIS_PASSWORD': 'test',
|
|
},
|
|
):
|
|
config = build_redis_config()
|
|
|
|
assert config == {
|
|
'host': 'redis.example.com',
|
|
'port': 6380,
|
|
'username': 'test',
|
|
'password': 'test',
|
|
}
|
|
|
|
|
|
def test_build_mongodb_config_defaults():
|
|
"""Test that build_mongodb_config returns default values when no env vars are set"""
|
|
os.environ['MONGODB_URL'] = 'localhost:27017'
|
|
os.environ['MONGODB_DATABASE_NAME'] = 'sientia'
|
|
os.environ['MONGODB_USERNAME'] = 'sientia'
|
|
os.environ['MONGODB_PASSWORD'] = 'sientia'
|
|
|
|
config = build_mongodb_config()
|
|
|
|
assert config == {
|
|
'connection_string': 'mongodb://sientia:sientia@localhost:27017', # NOSONAR
|
|
'database_name': 'sientia',
|
|
}
|
|
|
|
|
|
def test_build_mongodb_config_with_env_vars():
|
|
"""Test that build_mongodb_config uses env vars when set"""
|
|
with patch.dict(
|
|
os.environ,
|
|
{
|
|
'MONGODB_URL': 'mongodb.example.com:27017',
|
|
'MONGODB_DATABASE_NAME': 'test_db',
|
|
'MONGODB_USERNAME': 'test',
|
|
'MONGODB_PASSWORD': 'test',
|
|
},
|
|
):
|
|
config = build_mongodb_config()
|
|
|
|
assert config == {
|
|
'connection_string': 'mongodb://test:test@mongodb.example.com:27017',
|
|
'database_name': 'test_db',
|
|
}
|
|
|
|
|
|
@pytest.mark.usefixtures('mock_env_vars')
|
|
def test_build_api_config_defaults():
|
|
"""Test that build_api_config returns default values when no env vars are set"""
|
|
config = build_api_config()
|
|
|
|
assert config == {
|
|
'base_url': 'https://pi.example.com',
|
|
'auth_type': 'basic',
|
|
'auth_token': None,
|
|
}
|
|
|
|
|
|
@pytest.mark.usefixtures('mock_env_vars')
|
|
def test_build_api_config_with_env_vars():
|
|
"""Test that build_api_config uses env vars when set"""
|
|
with patch.dict(
|
|
os.environ,
|
|
{
|
|
'API_BASE_URL': 'https://api.production.com',
|
|
'API_AUTH_TYPE': 'bearer',
|
|
'API_AUTH_TOKEN': 'secret_token_123',
|
|
},
|
|
):
|
|
config = build_api_config()
|
|
|
|
assert config == {
|
|
'base_url': 'https://api.production.com',
|
|
'auth_type': 'bearer',
|
|
'auth_token': 'secret_token_123',
|
|
}
|
|
|
|
|
|
def test_build_druid_config_defaults():
|
|
"""Test that build_druid_config returns default values when no env vars are set"""
|
|
config = build_druid_config()
|
|
|
|
assert config == {'host': 'localhost', 'port': 8082}
|
|
|
|
|
|
def test_build_druid_config_with_env_vars():
|
|
"""Test that build_druid_config uses env vars when set"""
|
|
with patch.dict(os.environ, {'DRUID_HOST': 'druid.example.com', 'DRUID_PORT': '8083'}):
|
|
config = build_druid_config()
|
|
|
|
assert config == {'host': 'druid.example.com', 'port': 8083}
|