172 lines
8.2 KiB
Python
172 lines
8.2 KiB
Python
import json
|
|
import tempfile
|
|
from os import getenv, path
|
|
from typing import Any
|
|
|
|
|
|
def build_mlflow_config() -> dict[str, Any]:
|
|
"""
|
|
Build MLFlow server configuration from environment variables.
|
|
|
|
This function constructs an MLFlow configuration dictionary from
|
|
environment variables with sensible defaults for local development.
|
|
It handles server connection and authentication parameters.
|
|
|
|
Environment Variables:
|
|
MLFLOW_HOST: MLFlow server hostname (default: http://localhost)
|
|
MLFLOW_PORT: MLFlow server port (default: 5080)
|
|
MLFLOW_USERNAME: MLFlow username (default: aignosi)
|
|
MLFLOW_PASSWORD: MLFlow password (default: aignosi)
|
|
|
|
Returns:
|
|
dict: MLFlow configuration dictionary with all required parameters
|
|
"""
|
|
return {
|
|
'host': getenv('MLFLOW_HOST', 'http://localhost'),
|
|
'port': int(getenv('MLFLOW_PORT', '5080')),
|
|
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
|
|
'password': getenv('MLFLOW_PASSWORD', 'aignosi'),
|
|
}
|
|
|
|
|
|
def build_opc_config() -> dict[str, Any]:
|
|
"""
|
|
Build OPC server configuration from environment variables.
|
|
|
|
This function constructs an OPC server configuration dictionary from
|
|
environment variables. It supports both single server and multi-server
|
|
configurations with flexible parameter handling.
|
|
|
|
Environment Variables:
|
|
OPC_CONFIG: JSON string containing multiple OPC server configurations
|
|
OPC_ID: OPC server ID (fallback, default: 1)
|
|
OPC_URL: Single OPC server URL (fallback, default: opc.tcp://localhost:4840)
|
|
OPC_SERVER_URI: Single OPC server URI (fallback, default: opc.tcp://localhost:4840)
|
|
OPC_CERT_PATH: Client certificate path (fallback, default: None)
|
|
OPC_PRIVATE_KEY_PATH: Client private key path (fallback, default: None)
|
|
OPC_SERVER_CERT_PATH: Server certificate path (fallback, default: None)
|
|
OPC_RECONNECTION_INTERVAL: Reconnection interval in seconds (fallback, default: 120)
|
|
|
|
Returns:
|
|
dict: OPC server configuration dictionary
|
|
"""
|
|
opc_raw = getenv('OPC_CONFIG', None)
|
|
|
|
if opc_raw:
|
|
return json.loads(opc_raw)
|
|
|
|
return {
|
|
getenv('OPC_ID', '1'): {
|
|
'id': getenv('OPC_ID', '1'),
|
|
'server_name': getenv('OPC_SERVER_NAME', 'default_server'),
|
|
'url': getenv('OPC_URL', 'opc.tcp://localhost:4840'),
|
|
'server_uri': getenv('OPC_SERVER_URI', 'opc.tcp://localhost:4840'),
|
|
'cert_path': getenv('OPC_CERT_PATH', None),
|
|
'private_key_path': getenv('OPC_PRIVATE_KEY_PATH', None),
|
|
'server_cert_path': getenv('OPC_SERVER_CERT_PATH', None),
|
|
'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120')),
|
|
}
|
|
}
|
|
|
|
|
|
def build_minio_config() -> dict[str, Any]:
|
|
"""
|
|
Build MinIO (S3-compatible) configuration from environment variables.
|
|
|
|
Environment Variables:
|
|
MINIO_ENDPOINT: MinIO endpoint including scheme (default: http://localhost:9000)
|
|
MINIO_ACCESS_KEY: Access key (default: minioadmin)
|
|
MINIO_SECRET_KEY: Secret key (default: minioadmin)
|
|
MINIO_REGION: Region name for S3 client (default: us-east-1)
|
|
MINIO_BUCKET_DEFAULT: Default bucket for uploads (default: laborious)
|
|
MINIO_SECURE: Whether to use HTTPS (default: false)
|
|
Returns:
|
|
dict: MinIO configuration dictionary
|
|
"""
|
|
return {
|
|
'endpoint_url': getenv('MINIO_ENDPOINT_URL', 'http://localhost:9000'),
|
|
'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'),
|
|
'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'),
|
|
'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious'),
|
|
'retention_hours': int(getenv('MINIO_RETENTION_HOURS', '24')),
|
|
'secure': getenv('MINIO_SECURE', 'false') == 'true',
|
|
}
|
|
|
|
|
|
def build_import_config() -> dict[str, Any]:
|
|
"""
|
|
Build the `.sientia` model-import configuration from environment variables.
|
|
|
|
The password key is read as the raw base64 string and **not** validated here: the worker must
|
|
start whether or not `IMPORT_MODEL_ENABLED` is set, and a missing or malformed key is an import
|
|
failure at the decryption step (`laborious.utils.import_password.load_envelope_key`), not a
|
|
boot failure.
|
|
|
|
Environment Variables:
|
|
IMPORT_PASSWORD_KEY: base64 of the 32-byte AES-256-GCM key that wraps bundle passwords
|
|
IMPORT_BUNDLE_BUCKET: bucket holding the uploaded bundles (default: sientia)
|
|
IMPORT_BUNDLE_PREFIX: key prefix the uploads must sit under (default: imported_models/)
|
|
IMPORT_WORK_DIR: worker scratch directory (default: <tempdir>/sientia-import)
|
|
IMPORT_MAX_OBJECT_BYTES: gate 1 object size ceiling (default: 1 GiB)
|
|
IMPORT_MAX_ARCHIVE_ENTRIES: gate 4 entry count ceiling (default: 5000)
|
|
IMPORT_MAX_UNCOMPRESSED_BYTES: gate 4 uncompressed size ceiling (default: 4 GiB)
|
|
IMPORT_MAX_COMPRESSION_RATIO: gate 4 compression ratio ceiling (default: 200)
|
|
IMPORT_MODELS_COLLECTION: MongoDB collection holding the model listing (default: models)
|
|
IMPORT_BUNDLE_RETENTION_DAYS: lifecycle expiry for the uploaded bundle (default: 7)
|
|
|
|
Returns:
|
|
dict: import configuration dictionary
|
|
"""
|
|
prefix = getenv('IMPORT_BUNDLE_PREFIX', 'imported_models/')
|
|
return {
|
|
'password_key': getenv('IMPORT_PASSWORD_KEY'),
|
|
'bucket': getenv('IMPORT_BUNDLE_BUCKET', 'sientia'),
|
|
'prefix': prefix if prefix.endswith('/') else f'{prefix}/',
|
|
'work_dir': getenv('IMPORT_WORK_DIR', path.join(tempfile.gettempdir(), 'sientia-import')),
|
|
'max_object_bytes': int(getenv('IMPORT_MAX_OBJECT_BYTES', str(1024 * 1024 * 1024))),
|
|
'max_entries': int(getenv('IMPORT_MAX_ARCHIVE_ENTRIES', '5000')),
|
|
'max_uncompressed_bytes': int(
|
|
getenv('IMPORT_MAX_UNCOMPRESSED_BYTES', str(4 * 1024 * 1024 * 1024))
|
|
),
|
|
'max_compression_ratio': float(getenv('IMPORT_MAX_COMPRESSION_RATIO', '200')),
|
|
'models_collection': getenv('IMPORT_MODELS_COLLECTION', 'models'),
|
|
'retention_days': int(getenv('IMPORT_BUNDLE_RETENTION_DAYS', '7')),
|
|
}
|
|
|
|
|
|
def build_import_status_config() -> dict[str, Any]:
|
|
"""
|
|
Build the connection to the import log database — the POSTGRES_* server, another database.
|
|
|
|
The import log is `public.experiment_run` in the database owned by the Spring Boot BFF
|
|
(`sientia-core-mlops-bff`), which sits on the same Postgres server as the `sientia` database
|
|
Laborious itself uses. Only the database name differs, so only the database name has a variable
|
|
of its own; host, port, user and password come from POSTGRES_*. The table and schema are
|
|
constants of the code, never inputs.
|
|
|
|
The four IMPORT_STATUS_DB_* connection variables remain as overrides, for a deployment where the
|
|
log genuinely lives on another server — the e2e suite is exactly that, two containers.
|
|
|
|
Environment Variables:
|
|
IMPORT_STATUS_DB_NAME: database (default: sientia-core-mlops-bff)
|
|
POSTGRES_HOST: server host, unless IMPORT_STATUS_DB_HOST overrides it (default: localhost)
|
|
POSTGRES_PORT: server port, unless IMPORT_STATUS_DB_PORT overrides it (default: 5432)
|
|
POSTGRES_USER: user, unless IMPORT_STATUS_DB_USER overrides it (default: sientia)
|
|
POSTGRES_PASSWORD: password, unless IMPORT_STATUS_DB_PASSWORD overrides it (default: empty)
|
|
|
|
Returns:
|
|
dict: import status database configuration dictionary
|
|
"""
|
|
# Written as `override or fallback or default` rather than
|
|
# `getenv(override) or getenv(fallback, default)`: inside an `or`, the two-argument `getenv`
|
|
# widens to `str | None` for mypy, so the `int(...)` below fails to type-check. Putting the
|
|
# default last as its own term keeps every value a plain `str` and reads in precedence order.
|
|
return {
|
|
'host': getenv('IMPORT_STATUS_DB_HOST') or getenv('POSTGRES_HOST') or 'localhost',
|
|
'port': int(getenv('IMPORT_STATUS_DB_PORT') or getenv('POSTGRES_PORT') or '5432'),
|
|
'dbname': getenv('IMPORT_STATUS_DB_NAME') or 'sientia-core-mlops-bff',
|
|
'user': getenv('IMPORT_STATUS_DB_USER') or getenv('POSTGRES_USER') or 'sientia',
|
|
'password': getenv('IMPORT_STATUS_DB_PASSWORD') or getenv('POSTGRES_PASSWORD') or '',
|
|
'schema': 'public',
|
|
}
|