vitor-aignosi
dac4f1718f
QTZPOC-20: Implement model import workflow for .sientia bundles Introduces the import_model workflow and its associated activities to enable importing .sientia bundles into MLflow and MongoDB. This includes: * New core workflow and activity logic for handling bundle download, decryption, validation, MLflow provisioning (experiments, runs, registered versions), and MongoDB model document creation. * Comprehensive E2E and unit test suites, including a dedicated test harness and a manual run script for development. * Updated .env.example and README.md with new configurations and usage instructions. * Ensures correct MLflow model source resolution for legacy loader compatibility and robust MongoDB document management. * Refactors existing utility scripts into a scripts/ directory.
75 lines
2.9 KiB
Plaintext
75 lines
2.9 KiB
Plaintext
POSTGRES_HOST="paradedb-rw.paradedb.svc.cluster.local"
|
|
POSTGRES_PORT="5432"
|
|
POSTGRES_USER="sientia"
|
|
POSTGRES_PASSWORD="password"
|
|
POSTGRES_DBNAME="sientia"
|
|
POSTGRES_MIN_CONNECTIONS="10"
|
|
POSTGRES_MAX_CONNECTIONS="30"
|
|
|
|
MLFLOW_HOST="http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local"
|
|
MLFLOW_PORT="80"
|
|
MLFLOW_USERNAME="aignosi"
|
|
MLFLOW_PASSWORD="mlflow_password"
|
|
|
|
OPC_ID="1"
|
|
OPC_URL="opc.tcp://sientia-opc-simulator-opc.sientia.svc.cluster.local:4840"
|
|
|
|
LOG_LEVEL="DEBUG"
|
|
HTTP_METRICS_PORT="9090"
|
|
HTTP_SDK_METRICS_PORT="9091"
|
|
PROJECT_NAME="sientia-laborious"
|
|
|
|
TEMPORAL_HOST="temporal-frontend.temporal.svc.cluster.local:7233"
|
|
TEMPORAL_NAMESPACE="laborious"
|
|
|
|
MONGODB_USERNAME="mongo_user"
|
|
MONGODB_PASSWORD="mongo_db_password"
|
|
MONGODB_URL="my-release-mongodb.mongodb.svc.cluster.local:27017"
|
|
MONGODB_DATABASE="sientia"
|
|
MONGODB_TTL_INDEX_HOURS="1"
|
|
|
|
MINIO_ENDPOINT_URL="http://localhost:9000"
|
|
MINIO_ACCESS_KEY="sientia"
|
|
MINIO_SECRET_KEY="sientia"
|
|
MINIO_REGION_NAME="sa-east-1"
|
|
MINIO_DEFAULT_BUCKET="sientia"
|
|
|
|
# Suffix for every Temporal task queue name; required, non-empty.
|
|
# The import worker listens on "import_model-${RUNTIME}-queue".
|
|
RUNTIME="local"
|
|
|
|
# --- Model import (.sientia bundles) -----------------------------------------
|
|
# The import worker is registered only when IMPORT_MODEL_ENABLED is truthy
|
|
# (1/true/yes/on). Do not turn it on before the V12 migration is applied to
|
|
# public.experiment_run in the BFF database.
|
|
IMPORT_MODEL_ENABLED="false"
|
|
|
|
# base64 of the 32-byte AES-256-GCM key that wraps bundle passwords.
|
|
# Generate a throwaway one with: openssl rand -base64 32
|
|
IMPORT_PASSWORD_KEY="replace-with-base64-of-32-random-bytes"
|
|
|
|
# Where the frontend uploads the bundle. The object key must sit under the prefix.
|
|
IMPORT_BUNDLE_BUCKET="sientia"
|
|
IMPORT_BUNDLE_PREFIX="imported_models/"
|
|
|
|
# MongoDB collection holding the model listing the import writes its document into.
|
|
IMPORT_MODELS_COLLECTION="models"
|
|
|
|
# The import log lives in the Spring Boot BFF database, a second Postgres
|
|
# connection independent of POSTGRES_* above. The schema (public) and the table
|
|
# (experiment_run) are constants of the code, not configuration.
|
|
IMPORT_STATUS_DB_HOST="localhost"
|
|
IMPORT_STATUS_DB_PORT="5432"
|
|
IMPORT_STATUS_DB_NAME="sientia-core-mlops-bff"
|
|
IMPORT_STATUS_DB_USER="mlops_bff_user"
|
|
IMPORT_STATUS_DB_PASSWORD="bff_password"
|
|
|
|
# --- scripts/import_model_manual_run.py only ---------------------------------
|
|
# Inputs for the manual run that stands in for the frontend. The script writes
|
|
# one object and one row to whatever the variables above name.
|
|
IMPORT_SCRIPT_BUNDLE_PATH="tests/fixtures/bundle/golden_model_v3.sientia"
|
|
IMPORT_SCRIPT_BUNDLE_PASSWORD="the-password-the-bundle-was-written-with"
|
|
IMPORT_SCRIPT_USERNAME="manual.run@example.com"
|
|
# Cell 2's guard: restate IMPORT_STATUS_DB_HOST here (or in the cell) after reading what cell 1
|
|
# printed. Left empty on purpose — the script refuses to write until it matches.
|
|
IMPORT_SCRIPT_CONFIRM_TARGET="" |