dfc190c818
This commit includes several changes: - Reorganized imports and class inheritance in activities.py, gates.py and mlflow.py for better readability and maintainability. - Improved error handling and logging in gates.py and mlflow.py. - Added input validation and filtering in gates.py to ensure data quality. - Enhanced prediction formatting and storage policy management in gates.py. - Updated metrics.py to use consistent naming conventions and labels. - Refactored connectors_config.py to use type hints and improve code clarity. - Updated conditional and MLFlow filters for better data quality checks. - Improved model repository logic for retraining and updating models. - Enhanced worker.py to include SDK metrics and improved error handling. - Refactored workflows for better modularity and error handling. - Updated tests to reflect the changes and improve test coverage.
90 lines
3.5 KiB
Python
90 lines
3.5 KiB
Python
from os import getenv
|
|
from typing import Any
|
|
|
|
|
|
def build_postgres_config() -> dict[str, Any]:
|
|
"""
|
|
Build PostgreSQL database configuration from environment variables.
|
|
|
|
This function constructs a PostgreSQL configuration dictionary from
|
|
environment variables with sensible defaults for local development.
|
|
It handles connection pool configuration and security parameters.
|
|
|
|
Environment Variables:
|
|
POSTGRES_HOST: Database hostname (default: localhost)
|
|
POSTGRES_PORT: Database port (default: 5432)
|
|
POSTGRES_USER: Database username (default: sientia)
|
|
POSTGRES_PASSWORD: Database password (default: sientia)
|
|
POSTGRES_DBNAME: Database name (default: sientia)
|
|
POSTGRES_MIN_CONNECTIONS: Minimum connection pool size (default: 5)
|
|
POSTGRES_MAX_CONNECTIONS: Maximum connection pool size (default: 20)
|
|
|
|
Returns:
|
|
dict: PostgreSQL configuration dictionary with all required parameters
|
|
"""
|
|
return {
|
|
'host': getenv('POSTGRES_HOST', 'localhost'),
|
|
'port': int(getenv('POSTGRES_PORT', '5432')),
|
|
'user': getenv('POSTGRES_USER', 'sientia'),
|
|
'password': getenv('POSTGRES_PASSWORD', 'sientia'),
|
|
'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
|
|
'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
|
|
'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20')),
|
|
}
|
|
|
|
|
|
def build_mlflow_config() -> dict[str, Any]:
|
|
"""
|
|
Build MLFlow server configuration from environment variables.
|
|
|
|
This function constructs an MLFlow configuration dictionary from
|
|
environment variables with sensible defaults for local development.
|
|
It handles server connection and authentication parameters.
|
|
|
|
Environment Variables:
|
|
MLFLOW_HOST: MLFlow server hostname (default: http://localhost)
|
|
MLFLOW_PORT: MLFlow server port (default: 5080)
|
|
MLFLOW_USERNAME: MLFlow username (default: aignosi)
|
|
MLFLOW_PASSWORD: MLFlow password (default: aignosi)
|
|
|
|
Returns:
|
|
dict: MLFlow configuration dictionary with all required parameters
|
|
"""
|
|
return {
|
|
'host': getenv('MLFLOW_HOST', 'http://localhost'),
|
|
'port': int(getenv('MLFLOW_PORT', '5080')),
|
|
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
|
|
'password': getenv('MLFLOW_PASSWORD', 'aignosi'),
|
|
}
|
|
|
|
|
|
def build_mongodb_config() -> dict[str, Any]:
|
|
"""
|
|
Build MongoDB configuration from environment variables.
|
|
|
|
This function constructs a MongoDB configuration dictionary from
|
|
environment variables with sensible defaults for local development.
|
|
It handles connection string and database name configuration.
|
|
|
|
Environment Variables:
|
|
MONGODB_USERNAME: MongoDB username (default: root)
|
|
MONGODB_PASSWORD: MongoDB password (default: wKZDbMNU1c)
|
|
MONGODB_URL: MongoDB connection URI (default: localhost:27018)
|
|
MONGODB_DATABASE_NAME: MongoDB database name (default: sientia)
|
|
MONGODB_TTL_INDEX_HOURS: TTL index duration in hours (default: 1)
|
|
|
|
Returns:
|
|
dict: MongoDB configuration dictionary with connection parameters
|
|
"""
|
|
username = getenv('MONGODB_USERNAME', 'root')
|
|
password = getenv('MONGODB_PASSWORD', 'wKZDbMNU1c')
|
|
uri = getenv('MONGODB_URL', 'localhost:27018')
|
|
|
|
connection_string = f'mongodb://{username}:{password}@{uri}'
|
|
|
|
return {
|
|
'connection_string': connection_string,
|
|
'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
|
|
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600,
|
|
}
|