SIENTIAPDE-1773
Enhance environment configuration and update dependencies - Added new environment variables for PluginStore and MLflow configuration in `.env.example`, including `RUNTIME`, `STORE_BASE_URL`, `STORE_OWNER`, `STORE_REPO`, `STORE_BRANCH`, `STORE_USERNAME`, `STORE_PASSWORD`, `STORE_CACHE_TTL_SECONDS`, `PYPI_SERVER`, `PYPI_USERNAME`, and `PYPI_PASSWORD`. - Updated `git-requirements-mapping.txt` to reflect changes in repository names. - Modified `requirements-light.txt` and `requirements.txt` to upgrade `sientia-dataops-library` to version 1.12.0 and `sientia-mlops-library` to version 0.8.1. - Updated `values.yaml` to include new environment variables for worker runtime and PluginStore configuration. - Refactored E2E tests to utilize new MLflow repository stubs and PluginStore mocks for improved testing accuracy.
This commit is contained in:
@@ -3,31 +3,88 @@ from os import getenv
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _build_mlflow_tracking_url() -> str:
|
||||
"""
|
||||
Compose a single tracking URI for ``SientiaMLflowRepository`` from host and port env vars.
|
||||
|
||||
If ``MLFLOW_HOST`` already contains a port in the authority (e.g. ``http://tracker:80``),
|
||||
it is returned unchanged so operators can override port logic explicitly.
|
||||
|
||||
Return:
|
||||
str: Full tracking URL (scheme + host [+ port]).
|
||||
"""
|
||||
|
||||
mlflow_host = getenv('MLFLOW_HOST', 'http://localhost').rstrip('/')
|
||||
mlflow_port = getenv('MLFLOW_PORT', '5080')
|
||||
|
||||
# Host already includes an explicit port (e.g. http://tracker:80)
|
||||
host_after_scheme = mlflow_host.split('://', 1)[-1]
|
||||
if ':' in host_after_scheme:
|
||||
return mlflow_host
|
||||
|
||||
return f'{mlflow_host}:{mlflow_port}'
|
||||
|
||||
|
||||
def build_mlflow_config() -> dict[str, Any]:
|
||||
"""
|
||||
Build MLFlow server configuration from environment variables.
|
||||
Read MLflow tracking and registry credentials from the environment.
|
||||
|
||||
This function constructs an MLFlow configuration dictionary from
|
||||
environment variables with sensible defaults for local development.
|
||||
It handles server connection and authentication parameters.
|
||||
Used by ``Activities`` when constructing ``SientiaMLflowRepository``. The ``url`` value is the
|
||||
same string workers and notebooks should use for ``MLFLOW_TRACKING_URI``-style clients.
|
||||
|
||||
Environment Variables:
|
||||
MLFLOW_HOST: MLFlow server hostname (default: http://localhost)
|
||||
MLFLOW_PORT: MLFlow server port (default: 5080)
|
||||
MLFLOW_USERNAME: MLFlow username (default: aignosi)
|
||||
MLFLOW_PASSWORD: MLFlow password (default: aignosi)
|
||||
MLFLOW_HOST: Host with scheme; port optional if MLFLOW_PORT is set (default: http://localhost)
|
||||
MLFLOW_PORT: Appended when host has no explicit port (default: 5080)
|
||||
MLFLOW_USERNAME: Basic-auth or service user (default: aignosi)
|
||||
MLFLOW_PASSWORD: Password or token (default: aignosi)
|
||||
|
||||
Returns:
|
||||
dict: MLFlow configuration dictionary with all required parameters
|
||||
Return:
|
||||
dict[str, Any]: ``url``, ``username``, ``password``.
|
||||
"""
|
||||
return {
|
||||
'host': getenv('MLFLOW_HOST', 'http://localhost'),
|
||||
'port': int(getenv('MLFLOW_PORT', '5080')),
|
||||
'url': _build_mlflow_tracking_url(),
|
||||
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
|
||||
'password': getenv('MLFLOW_PASSWORD', 'aignosi'),
|
||||
}
|
||||
|
||||
|
||||
def build_plugin_store_config() -> dict[str, Any]:
|
||||
"""
|
||||
Collect settings for ``PluginStore`` (Git-backed catalog + runtime install via pip).
|
||||
|
||||
Mirrors the model-manager service: the worker passes these kwargs into ``PluginStore`` after
|
||||
``install_runtime`` resolves wheels from the configured PyPI index. Missing optional env vars
|
||||
become ``None`` so the store can run without auth in local dev.
|
||||
|
||||
Environment Variables:
|
||||
STORE_BASE_URL: Git HTTP(S) server (e.g. Gitea) base URL (default: http://localhost:3000)
|
||||
STORE_OWNER: Namespace or org owning the store repo (default: sientia)
|
||||
STORE_REPO: Repository name (default: model-library-store)
|
||||
STORE_BRANCH: Checkout branch; unset lets the client use default
|
||||
STORE_USERNAME / STORE_PASSWORD: HTTP basic credentials for Git fetch
|
||||
STORE_CACHE_TTL_SECONDS: Optional integer seconds for metadata cache TTL
|
||||
PYPI_SERVER: Index URL for ``pip install`` during runtime install (default: http://localhost:5000)
|
||||
PYPI_USERNAME / PYPI_PASSWORD: Optional index authentication
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Keys aligned with ``PluginStore`` constructor parameter names.
|
||||
"""
|
||||
|
||||
cache_ttl_seconds = getenv('STORE_CACHE_TTL_SECONDS')
|
||||
return {
|
||||
'base_url': getenv('STORE_BASE_URL', 'http://localhost:3000'),
|
||||
'owner': getenv('STORE_OWNER', 'sientia'),
|
||||
'repo': getenv('STORE_REPO', 'model-library-store'),
|
||||
'username': getenv('STORE_USERNAME'),
|
||||
'password': getenv('STORE_PASSWORD'),
|
||||
'branch': getenv('STORE_BRANCH'),
|
||||
'cache_ttl_seconds': int(cache_ttl_seconds) if cache_ttl_seconds else None,
|
||||
'pypi_index_url': getenv('PYPI_SERVER', 'http://localhost:5000'),
|
||||
'pypi_username': getenv('PYPI_USERNAME'),
|
||||
'pypi_password': getenv('PYPI_PASSWORD'),
|
||||
}
|
||||
|
||||
|
||||
def build_opc_config() -> dict[str, Any]:
|
||||
"""
|
||||
Build OPC server configuration from environment variables.
|
||||
@@ -73,14 +130,15 @@ def build_minio_config() -> dict[str, Any]:
|
||||
Build MinIO (S3-compatible) configuration from environment variables.
|
||||
|
||||
Environment Variables:
|
||||
MINIO_ENDPOINT: MinIO endpoint including scheme (default: http://localhost:9000)
|
||||
MINIO_ENDPOINT_URL: Host:port or URL for the S3 API (default: http://localhost:9000)
|
||||
MINIO_ACCESS_KEY: Access key (default: minioadmin)
|
||||
MINIO_SECRET_KEY: Secret key (default: minioadmin)
|
||||
MINIO_REGION: Region name for S3 client (default: us-east-1)
|
||||
MINIO_BUCKET_DEFAULT: Default bucket for uploads (default: laborious)
|
||||
MINIO_SECURE: Whether to use HTTPS (default: false)
|
||||
Returns:
|
||||
dict: MinIO configuration dictionary
|
||||
MINIO_DEFAULT_BUCKET: Default bucket for Laborious payloads (default: laborious)
|
||||
MINIO_RETENTION_HOURS: Offloaded object retention window (default: 24)
|
||||
MINIO_SECURE: If ``true``, use HTTPS (default: false)
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Keys consumed by ``Activities`` / ``MinioRepository``.
|
||||
"""
|
||||
return {
|
||||
'endpoint_url': getenv('MINIO_ENDPOINT_URL', 'http://localhost:9000'),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user