vitor-aignosi
7512963e19
Enhance MinIO integration and update environment configurations - Added MinIO configuration parameters to .env.example and values.yaml for improved storage management. - Updated requirements.txt to include necessary libraries for MinIO support. - Refactored Activities class to utilize Storage for MinIO interactions. - Enhanced MLFlow class to integrate MinIO for data retrieval during model retraining. - Introduced build_minio_config function to streamline MinIO configuration setup. - Updated minimal_retrain workflow to support data storage in MinIO.
153 lines
6.1 KiB
Python
153 lines
6.1 KiB
Python
from os import getenv
|
|
import json
|
|
from typing import Dict, Any
|
|
|
|
|
|
def build_postgres_config() -> Dict[str, Any]:
|
|
"""
|
|
Build PostgreSQL database configuration from environment variables.
|
|
|
|
This function constructs a PostgreSQL configuration dictionary from
|
|
environment variables with sensible defaults for local development.
|
|
It handles connection pool configuration and security parameters.
|
|
|
|
Environment Variables:
|
|
POSTGRES_HOST: Database hostname (default: localhost)
|
|
POSTGRES_PORT: Database port (default: 5432)
|
|
POSTGRES_USER: Database username (default: sientia)
|
|
POSTGRES_PASSWORD: Database password (default: sientia)
|
|
POSTGRES_DBNAME: Database name (default: sientia)
|
|
POSTGRES_MIN_CONNECTIONS: Minimum connection pool size (default: 5)
|
|
POSTGRES_MAX_CONNECTIONS: Maximum connection pool size (default: 20)
|
|
|
|
Returns:
|
|
dict: PostgreSQL configuration dictionary with all required parameters
|
|
"""
|
|
return {
|
|
'host': getenv('POSTGRES_HOST', 'localhost'),
|
|
'port': int(getenv('POSTGRES_PORT', '5432')),
|
|
'user': getenv('POSTGRES_USER', 'sientia'),
|
|
'password': getenv('POSTGRES_PASSWORD', 'sientia'),
|
|
'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
|
|
'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
|
|
'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20'))
|
|
}
|
|
|
|
|
|
def build_mlflow_config() -> Dict[str, Any]:
|
|
"""
|
|
Build MLFlow server configuration from environment variables.
|
|
|
|
This function constructs an MLFlow configuration dictionary from
|
|
environment variables with sensible defaults for local development.
|
|
It handles server connection and authentication parameters.
|
|
|
|
Environment Variables:
|
|
MLFLOW_HOST: MLFlow server hostname (default: http://localhost)
|
|
MLFLOW_PORT: MLFlow server port (default: 5080)
|
|
MLFLOW_USERNAME: MLFlow username (default: aignosi)
|
|
MLFLOW_PASSWORD: MLFlow password (default: aignosi)
|
|
|
|
Returns:
|
|
dict: MLFlow configuration dictionary with all required parameters
|
|
"""
|
|
return {
|
|
'host': getenv('MLFLOW_HOST', 'http://localhost'),
|
|
'port': int(getenv('MLFLOW_PORT', '5080')),
|
|
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
|
|
'password': getenv('MLFLOW_PASSWORD', 'aignosi')
|
|
}
|
|
|
|
|
|
def build_opc_config() -> Dict[str, Any]:
|
|
"""
|
|
Build OPC server configuration from environment variables.
|
|
|
|
This function constructs an OPC server configuration dictionary from
|
|
environment variables. It supports both single server and multi-server
|
|
configurations with flexible parameter handling.
|
|
|
|
Environment Variables:
|
|
OPC_CONFIG: JSON string containing multiple OPC server configurations
|
|
OPC_ID: OPC server ID (fallback, default: 1)
|
|
OPC_URL: Single OPC server URL (fallback, default: opc.tcp://localhost:4840)
|
|
OPC_SERVER_URI: Single OPC server URI (fallback, default: opc.tcp://localhost:4840)
|
|
OPC_CERT_PATH: Client certificate path (fallback, default: None)
|
|
OPC_PRIVATE_KEY_PATH: Client private key path (fallback, default: None)
|
|
OPC_SERVER_CERT_PATH: Server certificate path (fallback, default: None)
|
|
OPC_RECONNECTION_INTERVAL: Reconnection interval in seconds (fallback, default: 120)
|
|
|
|
Returns:
|
|
dict: OPC server configuration dictionary
|
|
"""
|
|
opc_raw = getenv('OPC_CONFIG', None)
|
|
|
|
if opc_raw:
|
|
return json.loads(opc_raw)
|
|
|
|
return {
|
|
getenv('OPC_ID', '1'): {
|
|
'id': getenv('OPC_ID', '1'),
|
|
'url': getenv('OPC_URL', 'opc.tcp://localhost:4840'),
|
|
'server_uri': getenv('OPC_SERVER_URI', 'opc.tcp://localhost:4840'),
|
|
'cert_path': getenv('OPC_CERT_PATH', None),
|
|
'private_key_path': getenv('OPC_PRIVATE_KEY_PATH', None),
|
|
'server_cert_path': getenv('OPC_SERVER_CERT_PATH', None),
|
|
'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120'))
|
|
}
|
|
}
|
|
|
|
|
|
def build_mongodb_config() -> Dict[str, Any]:
|
|
"""
|
|
Build MongoDB configuration from environment variables.
|
|
|
|
This function constructs a MongoDB configuration dictionary from
|
|
environment variables with sensible defaults for local development.
|
|
It handles connection string and database name configuration.
|
|
|
|
Environment Variables:
|
|
MONGODB_USERNAME: MongoDB username (default: root)
|
|
MONGODB_PASSWORD: MongoDB password (default: wKZDbMNU1c)
|
|
MONGODB_URL: MongoDB connection URI (default: localhost:27018)
|
|
MONGODB_DATABASE_NAME: MongoDB database name (default: sientia)
|
|
MONGODB_TTL_INDEX_HOURS: TTL index duration in hours (default: 1)
|
|
|
|
Returns:
|
|
dict: MongoDB configuration dictionary with connection parameters
|
|
"""
|
|
username = getenv('MONGODB_USERNAME', 'root')
|
|
password = getenv('MONGODB_PASSWORD', 'wKZDbMNU1c')
|
|
uri = getenv('MONGODB_URL', 'localhost:27018')
|
|
|
|
connection_string = f'mongodb://{username}:{password}@{uri}'
|
|
|
|
return {
|
|
'connection_string': connection_string,
|
|
'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
|
|
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600
|
|
}
|
|
|
|
|
|
def build_minio_config() -> Dict[str, Any]:
|
|
"""
|
|
Build MinIO (S3-compatible) configuration from environment variables.
|
|
|
|
Environment Variables:
|
|
MINIO_ENDPOINT: MinIO endpoint including scheme (default: http://localhost:9000)
|
|
MINIO_ACCESS_KEY: Access key (default: minioadmin)
|
|
MINIO_SECRET_KEY: Secret key (default: minioadmin)
|
|
MINIO_REGION: Region name for S3 client (default: us-east-1)
|
|
MINIO_BUCKET_DEFAULT: Default bucket for uploads (default: laborious)
|
|
|
|
Returns:
|
|
dict: MinIO configuration dictionary
|
|
"""
|
|
return {
|
|
'endpoint_url': getenv('MINIO_ENDPOINT_URL', 'http://localhost:9000'),
|
|
'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'),
|
|
'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'),
|
|
'region_name': getenv('MINIO_REGION_NAME', 'us-east-1'),
|
|
'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious')
|
|
}
|