e87c5d2da6
Enhance Activities and API Integration - Updated Activities class to include API operations for external data ingestion. - Added API configuration builder to connectors_config.py for environment variable management. - Integrated API configuration into worker setup. - Expanded unit tests to cover new API functionality and configuration handling. - Updated requirements.txt to include pycurl and prometheus-client for enhanced metrics support.
116 lines
4.2 KiB
Python
116 lines
4.2 KiB
Python
from os import getenv
|
|
from typing import Any
|
|
|
|
|
|
def build_postgres_config() -> dict[str, Any]:
|
|
"""
|
|
Build PostgreSQL connection configuration from environment variables.
|
|
|
|
Returns:
|
|
dict[str, Any]: PostgreSQL configuration dictionary with keys:
|
|
- host: Database hostname (default: localhost)
|
|
- port: Database port (default: 5432)
|
|
- user: Database username (default: sientia)
|
|
- password: Database password (default: sientia)
|
|
- dbname: Database name (default: sientia)
|
|
- min_connections: Minimum connection pool size (default: 5)
|
|
- max_connections: Maximum connection pool size (default: 20)
|
|
"""
|
|
return {
|
|
'host': getenv('POSTGRES_HOST', 'localhost'),
|
|
'port': int(getenv('POSTGRES_PORT', '5432')),
|
|
'user': getenv('POSTGRES_USER', 'sientia'),
|
|
'password': getenv('POSTGRES_PASSWORD', 'sientia'),
|
|
'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
|
|
'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
|
|
'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20')),
|
|
}
|
|
|
|
|
|
def build_kafka_config() -> dict[str, Any]:
|
|
"""
|
|
Build Kafka configuration from environment variables.
|
|
|
|
Returns:
|
|
dict[str, Any]: Kafka configuration dictionary with keys:
|
|
- bootstrap_servers: Kafka broker addresses (default: localhost:9092)
|
|
- polling_time: Consumer polling interval in milliseconds (default: 1000)
|
|
- group_id: Consumer group identifier (default: scouter-group)
|
|
"""
|
|
return {
|
|
'bootstrap_servers': getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092'),
|
|
'polling_time': int(getenv('KAFKA_POLLING_TIME', '1000')),
|
|
'group_id': 'scouter-group',
|
|
}
|
|
|
|
|
|
def build_redis_config() -> dict[str, Any]:
|
|
"""
|
|
Build Redis connection configuration from environment variables.
|
|
|
|
Returns:
|
|
dict[str, Any]: Redis configuration dictionary with keys:
|
|
- host: Redis server hostname (default: localhost)
|
|
- port: Redis server port (default: 6379)
|
|
- username: Redis authentication username (default: None)
|
|
- password: Redis authentication password (default: None)
|
|
"""
|
|
return {
|
|
'host': getenv('REDIS_HOST', 'localhost'),
|
|
'port': int(getenv('REDIS_PORT', '6379')),
|
|
'username': getenv('REDIS_USERNAME', None),
|
|
'password': getenv('REDIS_PASSWORD', None),
|
|
}
|
|
|
|
|
|
def build_mongodb_config() -> dict[str, Any]:
|
|
"""
|
|
Build MongoDB connection configuration from environment variables.
|
|
|
|
Returns:
|
|
dict[str, Any]: MongoDB configuration dictionary with keys:
|
|
- connection_string: Complete MongoDB connection URI
|
|
- database_name: Target database name (default: sientia)
|
|
"""
|
|
username = getenv('MONGODB_USERNAME', 'sientia')
|
|
password = getenv('MONGODB_PASSWORD', 'sientia')
|
|
uri = getenv('MONGODB_URL', 'localhost:27017')
|
|
|
|
connection_string = f'mongodb://{username}:{password}@{uri}'
|
|
return {
|
|
'connection_string': connection_string,
|
|
'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
|
|
}
|
|
|
|
|
|
def build_api_config() -> dict[str, Any]:
|
|
"""
|
|
Build API connection configuration from environment variables.
|
|
|
|
Returns:
|
|
dict[str, Any]: API configuration dictionary with keys:
|
|
- base_url: API base URL (default: https://pi.example.com)
|
|
- auth_type: API authentication type (default: basic)
|
|
- auth_token: API authentication token (default: None)
|
|
"""
|
|
return {
|
|
'base_url': getenv('API_BASE_URL', 'https://pi.example.com'),
|
|
'auth_type': getenv('API_AUTH_TYPE', 'basic'),
|
|
'auth_token': getenv('API_AUTH_TOKEN', None),
|
|
}
|
|
|
|
|
|
def build_druid_config() -> dict[str, Any]:
|
|
"""
|
|
Build Apache Druid connection configuration from environment variables.
|
|
|
|
Returns:
|
|
dict[str, Any]: Druid configuration dictionary with keys:
|
|
- host: Druid server hostname (default: localhost)
|
|
- port: Druid server port (default: 8082)
|
|
"""
|
|
return {
|
|
'host': getenv('DRUID_HOST', 'localhost'),
|
|
'port': int(getenv('DRUID_PORT', '8082')),
|
|
}
|