commit 40f26bcf1769ff9874b4bf2b847f01b26e331228 Author: vitor-aignosi Date: Thu Jul 16 13:29:23 2026 -0300 SIENTIAPDE-1645: code snapshot (part 1) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7f786a5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,93 @@ +# ============================================================================ +# WHITELIST APPROACH: Block everything by default, then allow only what's needed +# ============================================================================ + +# Block everything first +* + +# ============================================================================ +# ALLOW: Application source code (model_manager package) +# ============================================================================ + +# Allow the main package directory and all Python files +!model_manager/ +!model_manager/**/*.py +!model_manager/**/__init__.py + +# Allow subdirectories structure +!model_manager/activities/ +!model_manager/activities/** +!model_manager/schedules/ +!model_manager/schedules/** +!model_manager/sientia/ +!model_manager/sientia/** +!model_manager/utils/ +!model_manager/utils/** +!model_manager/utils/models/ +!model_manager/utils/models/** +!model_manager/utils/repository/ +!model_manager/utils/repository/** +!model_manager/worker/ +!model_manager/worker/** +!model_manager/workflows/ +!model_manager/workflows/** + +# Allow reports directory with header.html +!model_manager/reports/ +!model_manager/reports/header.html + +# Allow temp directory structure (but not its contents) +!model_manager/reports/temp/ + +# ============================================================================ +# ALLOW: Dependencies file (needed for pip install in Dockerfile) +# ============================================================================ +!requirements.txt + +# ============================================================================ +# BLOCK: Explicitly block unwanted files even if they match above patterns +# ============================================================================ + +# Python cache and compiled files +**/__pycache__/ +**/*.pyc +**/*.pyo +**/*.pyd +**/.Python +**/*.so +**/*.egg +**/*.egg-info/ + +# Tests (not needed in production) +model_manager/**/test_*.py +model_manager/**/*_test.py + +# IDE and editor files +**/.vscode/ +**/.idea/ +**/*.swp +**/*.swo +**/*~ + +# OS files +**/.DS_Store +**/Thumbs.db + +# Logs and temporary files +**/*.log +**/*.tmp +**/*.temp + +# Local configuration +**/.env +**/.env.local +**/*.local + +# Documentation inside code +**/*.md +**/README* + +# Backup files +**/*.bak +**/*.backup +**/*.old diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..22fea0c --- /dev/null +++ b/.env.example @@ -0,0 +1,57 @@ +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +POSTGRES_USER=postgres +POSTGRES_PASSWORD=changeme +POSTGRES_DBNAME=sientia +POSTGRES_MIN_CONNECTIONS=10 +POSTGRES_MAX_CONNECTIONS=30 + +MLFLOW_URL=http://localhost:5080 +MLFLOW_USERNAME=aignosi +MLFLOW_PASSWORD=changeme + +LOG_LEVEL=DEBUG +HTTP_METRICS_PORT=9090 +HTTP_SDK_METRICS_PORT=9091 +PROJECT_NAME=sientia-model-manager + +TEMPORAL_HOST=localhost:7233 +TEMPORAL_NAMESPACE=model-manager +TEMPORAL_USE_TLS=false +RUNTIME=single + +MONGODB_USERNAME=root +MONGODB_PASSWORD=changeme +MONGODB_URL=localhost:27017 +MONGODB_DATABASE=sientia +MONGODB_TTL_INDEX_HOURS=1 + +MINIO_ENDPOINT_URL=http://localhost:9000 +MINIO_ACCESS_KEY=minioadmin +MINIO_SECRET_KEY=minioadmin +MINIO_DEFAULT_BUCKET=model-training +MINIO_SECURE=false + +STORE_BASE_URL=http://localhost:3000 +STORE_OWNER=aignosi +STORE_REPO=suse-model-store +STORE_USERNAME= +STORE_PASSWORD= +STORE_CACHE_TTL_SECONDS=3600 +PYPI_SERVER=http://localhost:5000 +PYPI_USERNAME= +PYPI_PASSWORD= + +TIMEOUT_VALIDATE_PARAMS=30 +TIMEOUT_TRAIN_MODEL=2700 +TIMEOUT_DELETE_FILE=120 +TIMEOUT_UPDATE_DATABASE=30 + +CLEANUP_RETENTION_HOURS=24 +CLEANUP_DRY_RUN=false +TIMEOUT_CLEANUP_LOCAL=120 + +CLEANUP_SCHEDULE_ID=cleanup-files-daily +CLEANUP_CRON="0 0 * * *" +CLEANUP_TIMEZONE=UTC +CLEANUP_EXECUTION_TIMEOUT_HOURS=1 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..8ff00b2 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,33 @@ +name: Deploy Python Application + +on: + pull_request: + types: + - closed + branches: + - main + - 'release/**' + - 'feature/**' + +jobs: + deploy: + name: Deploy + if: github.event.pull_request.merged == true + permissions: write-all + uses: Aignosi/github_workflow_templates/.github/workflows/reusable-deploy.yml@main + with: + branch_name: ${{ github.event.pull_request.head.ref }} + project_type: 'python' + image_name: 'sientia-dataops-model-manager' + helm_chart_path: 'sientia-module' + helm_release_name: 'sientia-dataops-model-manager' + helm_namespace: 'sientia' + update_version_in: '["pyproject", "values"]' + app_owner: 'Aignosi' + private_repos: 'sientia-dataops-library' + helm_repo_name: 'sientia' + helm_repo_url: 'https://raw.githubusercontent.com/Aignosi/sientia-dataops-helm-repo/refs/heads/main/' + helm_chart_version: '0.6.0' + helm_values_file: './values.yaml' + use_vpn: true + secrets: inherit diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml new file mode 100644 index 0000000..10ca492 --- /dev/null +++ b/.github/workflows/quality-gate.yml @@ -0,0 +1,18 @@ +name: Quality gate + +on: + pull_request: + branches: + - main + - 'release/**' + - 'feature/**' + types: [ opened, synchronize, reopened ] + +jobs: + quality-gate: + uses: Aignosi/github_workflow_templates/.github/workflows/python-quality-gate.yml@main + permissions: write-all + with: + project_name: 'model_manager' + repositories: 'sientia-dataops-library,sientia-model-library' + secrets: inherit \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..683505a --- /dev/null +++ b/.gitignore @@ -0,0 +1,258 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Code quality tools cache +.bandit/ +validate.txt + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock +#poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +#pdm.lock +#pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +#pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +venv_311/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +.vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Cursor +# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to +# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data +# refer to https://docs.cursor.com/context/ignore-files +.cursorignore +.cursorindexingignore + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Ignore Docker volumes +docker-compose.override.yml +**/db_data/ +**/kafka-volume/ +**/zookeeper-volume/ +**/mage_data/ +**/minio_data/ +**/venv/ +**/certs/*.pem +**/certs/*.der +**/certs/*.csr +**/deploy/*.yaml +scouter/.file_versions/ +scouter/pipelines/**/triggers.yaml +**/postgres_data/** + +# Ignore Python cache files +*.pyc +*.pyo +*.pyd + +# Ignore temporary files +*.swp + +# Ignore test run reports in model_manager/reports/temp (but keep temp folder and .gitkeep) +model_manager/reports/temp/* +!model_manager/reports/temp/.gitkeep + +# Miscellaneous +git_key* +git_log +tmp/ +sientia-module/ +.secrets +.event.json + +models/ +.cursor +openspec + +# Training smoke test: track JSON inputs, ignore local sample CSVs +input_dataset.csv +scripts/**/*.csv +!scripts/inputs/ +.claude/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..641b3b0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,78 @@ +# Multi-stage build for optimized Python application +FROM python:3.11-slim AS builder + +# Set build-time environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# Install build dependencies only +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + && rm -rf /var/lib/apt/lists/* && \ + apt-get clean + +# Configure SSH to trust GitHub host key +RUN mkdir -p ~/.ssh && \ + ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts && \ + chmod 600 ~/.ssh/known_hosts + +# Create virtual environment +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +# Upgrade pip and wheel for better caching +RUN pip install --upgrade pip setuptools wheel + +# Copy requirements files for better Docker layer caching +COPY requirements.txt ./ + +# Install only production dependencies with no cache +RUN --mount=type=ssh echo "=== Installing dependencies ===" && \ + pip install --no-cache-dir -r requirements.txt && \ + echo "=== Dependencies installed successfully ===" && \ + pip list | wc -l && \ + echo "=== Cleaning cache files ===" && \ + find /opt/venv -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \ + find /opt/venv -name "*.pyc" -delete 2>/dev/null || true && \ + rm -rf /root/.cache/pip/* && \ + echo "=== Cleaning venv site-packages ===" && \ + find /opt/venv/lib/python3.11/site-packages/ -type f -name "*.md" -delete 2>/dev/null || true && \ + echo "=== Stripping .so files ===" && \ + find /opt/venv -name "*.so" -exec strip {} + 2>/dev/null || true + +# Production stage using python-slim for better functionality +FROM python:3.11-slim AS production + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH="/opt/venv/bin:$PATH" \ + POD_ID=unknown \ + HOME="/app" + +# Copy virtual environment from builder stage +COPY --from=builder /opt/venv /opt/venv + +# Set working directory +WORKDIR /app + +# Copy application code +COPY . . + +# Create necessary directories for runtime file creation +RUN mkdir -p /app/model_manager/reports /app/logs /app/temp /app/models /app/data && \ + chmod 755 /app/model_manager/reports /app/logs /app/temp /app/models /app/data + +# Create non-root user +RUN groupadd -r appuser && useradd -r -g appuser appuser && \ + chown -R appuser:appuser /app + +# Switch to non-root user +USER appuser + +# Set entrypoint for proper signal handling and PID 1 +ENTRYPOINT ["/opt/venv/bin/python", "-m", "model_manager.worker.worker"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3782ce9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 aignosi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PIPELINE_PARAMS_CHANGELOG.md b/PIPELINE_PARAMS_CHANGELOG.md new file mode 100644 index 0000000..35fc794 --- /dev/null +++ b/PIPELINE_PARAMS_CHANGELOG.md @@ -0,0 +1,227 @@ +# Changelog de Parâmetros do Pipeline de Treinamento + +Este documento descreve as alterações nos parâmetros de entrada do pipeline Temporal para treinamento de modelos. + +## Resumo das Alterações + +### Parâmetros ALTERADOS (Breaking Changes) + +| Parâmetro | Tipo Anterior | Tipo Novo | Descrição | +|-----------|---------------|-----------|-----------| +| `lag_train` | `int` | `dict[str, int]` | Agora é um dicionário com lag por variável. Ex: `{"var1": 2, "var2": 3}` | +| `lag_val` | `int` | `dict[str, int]` | Agora é um dicionário com lag por variável. Ex: `{"var1": 1, "var2": 1}` | + +### Parâmetros NOVOS (Obrigatórios) + +| Parâmetro | Tipo | Descrição | Valores Válidos | +|-----------|------|-----------|-----------------| +| `model_name` | `str` | Nome do tipo de modelo | `"Linear Regression"`, `"Polynomial Regression"` | +| `degree` | `int` | Grau do polinômio (1 = linear) | `>= 1` | +| `interaction_only` | `bool` | Apenas termos de interação para polinomial | `true`, `false` | +| `nan_treatment` | `str` | Tratamento de valores NaN | `"drop"`, `"linear interpolation"`, `"fill linear"` | +| `scaler_name` | `str` | Nome do scaler a usar | `"Standard Scaler"`, `"None"` | + +### Parâmetros NOVOS (Opcionais) + +| Parâmetro | Tipo | Descrição | Default | +|-----------|------|-----------|---------| +| `start_date` | `str \| null` | Data inicial para filtrar dados (formato: `"YYYY-MM-DD HH:MM:SS"`) | `null` | +| `end_date` | `str \| null` | Data final para filtrar dados (formato: `"YYYY-MM-DD HH:MM:SS"`) | `null` | +| `support_filters` | `dict \| null` | Filtros customizados por variável | `{}` | +| `static_threshold` | `int \| null` | Threshold para remoção de janelas estáticas (1-1000). Só usado quando `rem_static_win` é `true`. | `1` | + +--- + +## Exemplo de Input Completo + +### Formato ANTERIOR (não funciona mais): + +```json +{ + "experiment_run_id": 123, + "target_variable": "temperatura", + "variable_columns": ["pressao", "umidade", "velocidade"], + "lag_train": 2, + "lag_val": 1, + "rem_static_win": true, + "low_lim": {"pressao": 0, "umidade": 0, "velocidade": 0}, + "upp_lim": {"pressao": 100, "umidade": 100, "velocidade": 50}, + "window": 0, + "use_scaler": true, + "include_ar": false, + "bucket_name": "training-data", + "file_name": "dataset.csv", + "line_separator": ";", + "decimal_separator": ",", + "train_size": 80, + "shuffle": false, + "experiment_name": "modelo-temperatura", + "removed_intervals": [] +} +``` + +### Formato NOVO (obrigatório): + +```json +{ + "experiment_run_id": 123, + "target_variable": "temperatura", + "variable_columns": ["pressao", "umidade", "velocidade"], + + "lag_train": { + "pressao": 2, + "umidade": 2, + "velocidade": 2 + }, + "lag_val": { + "pressao": 1, + "umidade": 1, + "velocidade": 1 + }, + + "rem_static_win": true, + "static_threshold": null, + "low_lim": {"pressao": 0, "umidade": 0, "velocidade": 0}, + "upp_lim": {"pressao": 100, "umidade": 100, "velocidade": 50}, + "window": 0, + "use_scaler": true, + "include_ar": false, + "bucket_name": "training-data", + "file_name": "dataset.csv", + "line_separator": ";", + "decimal_separator": ",", + "train_size": 80, + "shuffle": false, + "experiment_name": "modelo-temperatura", + "removed_intervals": [], + + "model_name": "Linear Regression", + "degree": 1, + "interaction_only": false, + "nan_treatment": "drop", + "scaler_name": "Standard Scaler", + + "start_date": null, + "end_date": null, + "support_filters": {} +} +``` + +--- + +## Exemplo para Polynomial Regression + +```json +{ + "experiment_run_id": 124, + "target_variable": "temperatura", + "variable_columns": ["pressao", "umidade"], + + "lag_train": { + "pressao": 0, + "umidade": 0 + }, + "lag_val": { + "pressao": 0, + "umidade": 0 + }, + + "rem_static_win": false, + "low_lim": {"pressao": 0, "umidade": 0}, + "upp_lim": {"pressao": 100, "umidade": 100}, + "window": 0, + "use_scaler": true, + "include_ar": false, + "bucket_name": "training-data", + "file_name": "dataset.csv", + "line_separator": ";", + "decimal_separator": ",", + "train_size": 80, + "shuffle": false, + "experiment_name": "modelo-polinomial", + "removed_intervals": [], + + "model_name": "Polynomial Regression", + "degree": 2, + "interaction_only": false, + "nan_treatment": "linear interpolation", + "scaler_name": "Standard Scaler", + + "start_date": "2024-01-01 00:00:00", + "end_date": "2024-12-31 23:59:59", + "support_filters": {} +} +``` + +--- + +## Exemplo com Intervalos Removidos + +```json +{ + "removed_intervals": [ + ["2024-03-01 00:00:00", "2024-03-15 23:59:59"], + ["2024-06-01 00:00:00", "2024-06-30 23:59:59"] + ] +} +``` + +--- + +## Parâmetros Logados no MLflow + +Os seguintes parâmetros são agora logados no MLflow: + +| Parâmetro MLflow | Descrição | +|------------------|-----------| +| `model_name` | Nome do modelo (`Linear Regression` ou `Polynomial Regression`) | +| `models_params` | `{"degree": int, "interaction_only": bool}` | +| `target_variable` | Variável alvo | +| `input_variables` | Lista de variáveis de entrada | +| `nan_treatment` | Tratamento de NaN | +| `lag_train` | Dicionário de lags para treino | +| `lag_transform` | Dicionário de lags para transformação | +| `static_threshold` | Threshold para janelas estáticas (1 ou null) | +| `lower_limits` | Limites inferiores por variável | +| `upper_limits` | Limites superiores por variável | +| `scaler_name` | Nome do scaler | +| `scaler_params` | Parâmetros do scaler (mean, variance) | +| `include_ar` | Se inclui variável autoregressiva | +| `train_size` | Proporção de treino (0.0 - 1.0) | +| `test_size` | Proporção de teste (0.0 - 1.0) | +| `start_date` | Data inicial (ou null) | +| `end_date` | Data final (ou null) | +| `removed_intervals` | Lista de intervalos removidos | +| `retrain` | Sempre `false` para novos modelos | +| `support_filters` | Filtros customizados | + +--- + +## Validações de Negócio + +O sistema valida automaticamente: + +1. **`train_size`**: Deve estar entre 10 e 100 +2. **`variable_columns`**: Não pode estar vazio +3. **`lag_train` / `lag_val`**: Todos os valores devem ser >= 0 +4. **`window`**: Deve ser >= 0 +5. **`degree`**: Deve ser >= 1 +6. **`nan_treatment`**: Deve ser `"drop"`, `"linear interpolation"` ou `"fill linear"` +7. **`scaler_name`**: Deve ser `"Standard Scaler"` ou `"None"` +8. **`model_name`**: Deve ser `"Linear Regression"` ou `"Polynomial Regression"` +9. **`low_lim` / `upp_lim`**: Devem ter as mesmas chaves, e `low_lim[var] < upp_lim[var]` +10. **`bucket_name` / `file_name` / `experiment_name`**: Não podem estar vazios +11. **`degree` vs `model_name`**: Se `model_name` = "Polynomial Regression", `degree` deve ser >= 2; se "Linear Regression", `degree` deve ser = 1 +12. **`removed_intervals`**: Cada elemento deve ser lista/tupla com pelo menos 2 elementos (start, end) +13. **`target_variable`**: Não pode estar vazio +14. **`static_threshold`**: Se `rem_static_win` = `true` e `static_threshold` tiver valor, deve estar entre 1 e 1000 (inclusive). Se `null`, assume valor `1`. + +--- + +## Arquivos Modificados + +- `model_manager/sientia/models.py` - `DataPreprocessor` e `LinearRegressionModel` +- `model_manager/sientia/metrics.py` - Funções RCE adicionadas +- `model_manager/utils/models/train_model_params.py` - Novos parâmetros +- `model_manager/utils/repository/training_repository.py` - Uso dos novos parâmetros +- `model_manager/utils/repository/model_repository.py` - Logging no MLflow diff --git a/README.md b/README.md new file mode 100644 index 0000000..1a1b13b --- /dev/null +++ b/README.md @@ -0,0 +1,1554 @@ +# Sientia DataOps Model Manager + +An enterprise-grade ML model training orchestration platform built on Temporal. Provides robust, scalable workflows for training machine learning models with comprehensive validation, experiment tracking, and automated resource management. Integrates seamlessly with MLFlow for model persistence and PostgreSQL for experiment tracking. + +## 📑 Table of Contents + +- [Features](#features) + - [Core Functionality](#core-functionality) + - [Advanced Capabilities](#advanced-capabilities) + - [Development & Quality Assurance](#development--quality-assurance) +- [Architecture](#architecture) + - [Architecture Principles](#architecture-principles) + - [Key Components](#key-components) + - [Data Flow Architecture](#data-flow-architecture) + - [Security Architecture](#security-architecture) +- [Workflows](#workflows) + - [Train Model Workflow](#train-model-workflow-train_modelpy) + - [Train model workflow input (sample)](#train-model-workflow-input-sample) + - [Cleanup Files Workflow](#cleanup-files-workflow-cleanup_filespy) +- [Installation & Setup](#installation--setup) + - [Prerequisites](#prerequisites) + - [Environment Setup](#environment-setup) + - [Temporal Namespace Setup](#temporal-namespace-setup) + - [Local Development Setup](#local-development-setup) + - [Port Forward Setup Script](#port-forward-setup-script) +- [How to Run](#how-to-run) + - [Running the Model Manager Application](#running-the-model-manager-application) + - [Running Tests and Coverage](#running-tests-and-coverage) + - [Manual Test Execution](#manual-test-execution) + - [Manual Application Execution](#manual-application-execution) +- [Code Quality & Validation](#code-quality--validation) + - [Overview](#overview) + - [Validation Tools](#validation-tools) + - [Tools Installation](#tools-installation) + - [Complete Validation](#complete-validation) + - [Automatic Fixes](#automatic-fixes) + - [Configuration](#configuration) + - [CI/CD Integration](#cicd-integration) + - [Best Practices](#best-practices) +- [Testing](#testing) + - [Test Structure](#test-structure) + - [Test Execution](#test-execution) + - [Integration Tests](#integration-tests) + - [Running Integration Tests](#running-integration-tests) + - [Test Scenarios](#test-scenarios) + - [Scenario File Structure](#scenario-file-structure) + - [Creating New Scenarios](#creating-new-scenarios) + - [Important Validations](#important-validations) +- [Monitoring and Metrics](#monitoring-and-metrics) + - [Application Health Metrics](#application-health-metrics) + - [Training Metrics](#training-metrics) +- [Configuration](#configuration-1) + - [Environment Variables](#environment-variables) + - [Workflow Configuration](#workflow-configuration) +- [Development](#development) + - [Code Quality & Testing](#code-quality--testing) + - [Project Structure](#project-structure) + - [Adding New Features](#adding-new-features) + - [Test Coverage Guidelines](#test-coverage-guidelines) +- [Troubleshooting](#troubleshooting) + - [Common Issues](#common-issues) + - [Debug Mode](#debug-mode) +- [Performance Tuning](#performance-tuning) + - [Key Parameters](#key-parameters) + - [Scaling Considerations](#scaling-considerations) +- [Contributing](#contributing) + - [Code Quality Standards](#code-quality-standards) +- [License](#license) +- [Support](#support) +- [Local GitHub Actions Testing (act)](#local-github-actions-testing-act) + - [What is act?](#what-is-act) + - [Installation](#installation) + - [Configuration](#configuration-2) + - [Usage](#usage) + - [Command Reference](#command-reference) +- [Docker](#docker) +- [Helm Chart](#helm-chart) + +## Features + +### Core Functionality +- **Model Training Orchestration**: Complete training lifecycle management from validation to MLFlow deployment +- **Model Agnostic Pipeline**: Support for multiple model types via dynamic runtime and wrapper installation +- **Temporal Workflow Management**: Robust orchestration with fault tolerance and granular retry policies +- **Multi-Level Validation**: Defense-in-depth parameter validation with type checking and business rules +- **Experiment Tracking**: Integrated status tracking and metadata persistence in PostgreSQL +- **Interactive ML Reporting**: Automated generation of rich HTML reports (Data Drift, Quality, Performance) using **Evidently** +- **Automated Resource Management**: Efficient handling of temporary local storage and artifact persistence +- **Prometheus Monitoring**: Comprehensive observability with real-time metrics and operational logging +- **Scheduled Maintenance**: Automated lifecycle jobs for filesystem hygiene and stale file cleanup + +### Advanced Capabilities +- **Dynamic Runtime Provisioning**: Automated installation of required model runtimes from the Plugin Store +- **Granular Retry Policies**: Tailored strategies for network, MLFlow, database, and filesystem operations +- **Scalable Infrastructure**: Kubernetes-ready design with support for horizontal scaling and poller autoscaling +- **Secure Configuration**: Environment-driven connection management with fallback to sensible defaults +- **Notification Framework**: Multi-channel alerting and event notification via MongoDB integration +- **High-Performance Data Loading**: Optimized MinIO connectivity supporting large training datasets (up to 200MB) +- **Extensible Architecture**: Plugin-based system for easy integration of new models and preprocessing logic + +### Development & Quality Assurance +- **Code Quality Tools**: Ruff (linting/formatting), mypy (type checking), Bandit (security analysis) +- **Automated Validation**: Pre-commit validation script (`validate.sh`) and CI/CD integration +- **Comprehensive Testing**: pytest with async support and **100% code coverage** 🎯 +- **Type Safety**: Static type checking with mypy for improved code reliability +- **Coverage Visualization**: Integration with Coverage Gutters for real-time coverage feedback +- **Automated Versioning**: Semantic versioning based on branch patterns (release/*, feature/*, fix/*, rc/*) +- **Integration Test Scenarios**: JSON-based test scenarios with batch execution support + +## Architecture + +The Model Manager system uses a Temporal-based workflow architecture with clear separation of concerns and robust error handling. The architecture is designed for high availability, scalability, and operational excellence in production ML environments. + + +### Architecture Principles + +#### 1. **Separation of Concerns** +- **Worker Layer**: Manages Temporal workers, task queues, and application lifecycle +- **Workflow Layer**: Orchestrates business logic and process coordination +- **Activity Layer**: Implements specific operations and external system interactions +- **Data Layer**: Handles data persistence, caching, and external service connections + +#### 2. **Fault Tolerance & Resilience** +- **Automatic Retry Policies**: Configurable retry strategies for transient failures +- **Circuit Breaker Pattern**: Prevents cascading failures in external service calls +- **Graceful Degradation**: System continues operating with reduced functionality +- **Comprehensive Error Handling**: Detailed error reporting and notification integration + +#### 3. **Scalability & Performance** +- **Horizontal Scaling**: Multiple worker instances for load distribution +- **Task Queue Isolation**: Separate queues for different workflow types +- **Connection Pooling**: Optimized database and external service connections +- **Asynchronous Processing**: Non-blocking operations for improved throughput + +#### 4. **Observability & Monitoring** +- **Prometheus Metrics**: Comprehensive system and business metrics +- **Structured Logging**: Consistent log format with correlation IDs +- **Health Checks**: Endpoint health monitoring and alerting +- **Performance Tracing**: Request flow tracking and bottleneck identification + +### Key Components + +#### **Worker (`model_manager/worker/worker.py`)** +- **Purpose**: Main application orchestrator managing Temporal workers and task queues +- **Responsibilities**: + - Temporal client initialization and connection management + - Worker lifecycle management and graceful shutdown + - Task queue configuration and load balancing + - Prometheus metrics server initialization + - Notification handler setup and configuration + - **Runtime Installation**: Automatically installs the required model runtime from the Plugin Store +- **Key Features**: + - Automatic scaling with `PollerBehaviorAutoscaling` + - Health check endpoints for Kubernetes liveness/readiness probes + - Graceful shutdown with cleanup procedures + - Multi-instance deployment support + - Task queues are derived from `RUNTIME` (default `single`): `train_model--queue` and `cleanup_files--queue` (see `prepare_worker.build_queue_name`) + - Automated cleanup schedule management + +#### **Workflows (`model_manager/workflows/`)** +- **TrainModel**: Complete ML model training pipeline from validation to deployment +- **CleanupFiles**: Automated cleanup of stale files from MinIO and local filesystem +- **Key Features**: + - Temporal workflow definitions with granular retry policies + - Parameter validation with business rules + - Comprehensive error handling and status tracking + - Configurable timeouts for different operation types + - Automatic resource cleanup and management + - Scheduled cleanup jobs with cron expressions + +#### **Activities (`model_manager/activities/`)** +- **Activities**: Main activity orchestrator combining all functionality through multiple inheritance +- **ExperimentTracking**: ML experiment lifecycle tracking and database operations + - Unified `update_experiment_run()` method for all experiment status updates + - Support for three update types: STATUS, STATUS_WITH_ERROR, MODEL_SAVED + - Automatic error message truncation (1024 chars) + - Connection pooling and retry logic +- **Training**: ML model training operations with MLFlow and MinIO integration + - Unified `train_model()` method for complete training pipeline + - Receives pre-downloaded files (BytesIO) to avoid memory leaks + - Returns success/failure status with TrainModelResult or error message + - No exception raising on failure - allows workflow to handle errors gracefully + - Integration with `DataManagerRepository` for data processing and report generation + - MLFlow model saving and artifact management + - Calculates training predictions and performance metrics for reporting +- **Cleanup**: Local directory cleanup operations + - `cleanup_temp_directories()`: Cleans local temporary directories + - Configurable retention period (default: 24 hours) + - Dry-run mode for testing + - No MinIO cleanup (files are managed by external processes) +- **Key Features**: + - Multiple inheritance pattern for unified activity interface + - Parameter validation with business rules + - MLFlow integration for model persistence + - Comprehensive error handling and notification integration + - Experiment tracking with automatic status management + - Timestamp-based file cleanup with regex pattern matching + +#### **Data Services (`model_manager/utils/`)** +- **Connectors Config**: Environment variable-based configuration management +- **Repository**: Data access layer for training and artifact operations + - `data_manager_repository.py`: Core data loading, feature preparation, metrics calculation, and report generation. +- **Models**: Data models and schemas + - `train_model_params.py`: Training parameters model with comprehensive validation and 7 business rules. + - `train_model_result.py`: Training result model containing processed data, metrics, and artifact paths. + - `experiment_status.py`: Experiment status enum for tracking workflow progress. +- **Key Features**: + - Environment variable-based configuration with sensible defaults + - Connection pool management and optimization + - Security credential management + - Configuration validation and error handling + - Type-safe data models with validation + +### Data Flow Architecture + +#### **Model Training Pipeline** +``` +Training Request → Parameter Validation → Model Training → +MLFlow Model Save → Resource Cleanup → Status Update +``` + +**Key Stages:** +1. **Validation**: Experiment run ID and training parameters validation +2. **Training**: Execute ML model training with validated parameters (data provided in request) +3. **Persistence**: Save trained model and artifacts to MLFlow +4. **Cleanup**: Remove temporary local directories and update experiment status + +### Security Architecture + +#### **Authentication & Authorization** + +- **MLFlow API Authentication**: Username/password with secure transmission +- **Database Connection Security**: Encrypted connections with credential management +- **Kubernetes Secrets Integration**: Secure credential storage and access + +#### **Network Security** +- **TLS/SSL Encryption**: Secure communication channels +- **Network Isolation**: Kubernetes network policies and service mesh +- **Firewall Rules**: Controlled access to external services +- **VPN Integration**: Secure remote access and management + +#### **Data Security** +- **Data Encryption**: At-rest and in-transit encryption +- **Access Control**: Role-based access control (RBAC) +- **Audit Logging**: Comprehensive access and operation logging +- **Data Retention**: Configurable data lifecycle management + +## Workflows + +### Train Model Workflow (`train_model.py`) + +The **TrainModel** workflow orchestrates the complete ML model training pipeline from parameter validation through model saving and cleanup. + +#### Purpose +- **Model Training**: Complete ML model training pipeline +- **Parameter Validation**: Defense-in-depth validation with business rules +- **Resource Management**: Automatic cleanup of temporary resources +- **Status Tracking**: Comprehensive experiment tracking in database +- **Error Handling**: Robust error handling with detailed context logging + +#### Execution Flow +1. **Validate Experiment Run ID**: Critical validation before any DB updates +2. **Load Model Metadata**: Fetch model schemas and metadata from the Plugin Store +3. **Validate Training Parameters**: Type checking + business rules validation +4. **Train Model**: Execute ML model training with validated parameters and data +5. **Save to MLFlow**: Save trained model and artifacts to MLFlow +6. **Cleanup Resources**: Delete temporary local directories + +#### Key Features +- **Granular Retry Policies**: Different strategies for network, training, MLFlow, database, and filesystem operations +- **Configurable Timeouts**: Environment variable-based timeouts supporting files up to 200MB +- **Idempotent Cleanup**: Safe replay with Temporal workflow replay mechanism +- **Structured Logging**: Rich context in error messages for debugging +- **Business Validation**: 7 business rules including range checks, consistency validation, and dynamic schema-based validation + +#### Train model workflow input (sample) + +The workflow receives **one argument**: a JSON-serializable object whose keys match `TrainModelParams` (`model_manager/utils/models/train_model_params.py`). All fields are passed at the **top level** (not nested under `train_params`). + +A **minimal valid example** (only keys required by `TrainModelParams.from_dict`, plus a minimal `model_metadata` for `validate_business_rules`) is in **`input-sample.json`**. See also **`input-sample.md`** for SQL/MinIO notes. `date_column` is required. `date_format` may be omitted (default `yyyy-MM-dd HH:mm:ss`). Optional inputs include `random_state` (defaults to `42`), `val_file_name`, and `model_id`. + +When starting the workflow from a Temporal client, use the same **task queue** as the worker: `train_model--queue` (for example `train_model-single-queue` when `RUNTIME=single`). + +#### Architecture Diagram +```mermaid +flowchart TD + A[1. validate_experiment_run_id] --> B[2. load_model_metadata] + B --> C[3. validate_train_params] + C --> D[4. train_model] + D --> E[5. cleanup_resources] + + C -.-> DB[(PostgreSQL)] + D -.-> Training[ML Training] + D -.-> MLFlow[MLFlow] + E -.-> FS[Filesystem] +``` + +#### Retry Strategies + +The workflow implements 5 different retry policies optimized for each operation type: + +| Operation Type | Initial Interval | Max Interval | Backoff | Max Attempts | Use Case | +|---------------|------------------|--------------|---------|--------------|----------| +| **Network** | 1s | 10s | 2.0x | 5 | Network operations (transient errors) | +| **No Retry** | - | - | - | 1 | Training/Validation (permanent data errors) | +| **Database** | 2s | 20s | 2.0x | 5 | PostgreSQL updates (lock contention) | + +#### Business validation rules + +`TrainModelParams.validate_business_rules()` runs after type coercion. Notable checks: + +1. **train_size**: Must be between 10 and 100 (percent). +2. **variable_columns**: Must be a non-empty list. +3. **model_metadata**: Required (must be loaded before validation) to provide schemas for keyword arguments. +4. **Dynamic Kwargs Validation**: `data_model_kwargs`, `model_kwargs`, and `opt_params` are validated against JSON Schemas provided in `model_metadata` (if present) using `Draft202012Validator`. +5. **Required Strings**: `target_variable`, `bucket_name`, `file_name`, and `model_name` cannot be empty or whitespace. +6. **date_format**: Optional; if omitted or blank, defaults to `yyyy-MM-dd HH:mm:ss`. If set, must be one of the allowed frontend formats. +7. **experiment_run_id**: Must be a valid integer or numeric string. + +Model-specific rules live in the training stack and integration scenarios; see `docs/test-scenarios/` and `scripts/run_training_test.py` for scenario-based examples. + +### Cleanup Files Workflow (`cleanup_files.py`) + +The **CleanupFiles** workflow provides automated cleanup of stale local temporary directories. It runs on a scheduled basis (default: daily at midnight UTC) to maintain storage hygiene. + +#### Purpose +- **Storage Management**: Automatic removal of old temporary directories from local filesystem +- **Retention Policy**: Configurable retention period (default: 24 hours) +- **Scheduled Execution**: Cron-based scheduling for automated cleanup +- **Resource Optimization**: Prevents storage bloat and reduces disk usage + +#### Execution Flow +1. **Cleanup Local Directories**: Remove temporary directories older than retention period + +#### Key Features +- **Timestamp-Based Cleanup**: Uses directory timestamps for age determination +- **Pattern Matching**: Regex pattern for directories (`name_YYYYMMDD_HHMMSS_microseconds`) +- **Configurable Retention**: Environment variable-based retention period +- **Dry-Run Mode**: Test cleanup operations without actual deletion +- **Idempotent**: Safe to run multiple times +- **Error Handling**: Continues cleanup even if individual operations fail + +#### Input Parameters +```json +{ + "temp_path": "model_manager/reports/temp" // Optional, defaults to 'model_manager/reports/temp' +} +``` + +#### Schedule Configuration + +The cleanup schedule is automatically created when the worker starts: + +| Configuration | Environment Variable | Default | Description | +|--------------|---------------------|---------|-------------| +| **Schedule ID** | (derived) | `cleanup-files--daily` | Built from `RUNTIME` in `cleanup_schedule.build_cleanup_schedule_id` | +| **Cron Expression** | `CLEANUP_CRON` | `0 0 * * *` | Daily at midnight UTC | +| **Timezone** | `CLEANUP_TIMEZONE` | `UTC` | Timezone for cron execution | +| **Task Queue** | (derived) | `cleanup_files--queue` | Must match the cleanup worker queue (`build_queue_name('CleanupFiles', runtime)`) | +| **Execution Timeout** | `CLEANUP_EXECUTION_TIMEOUT_HOURS` | `1` | Maximum execution time (hours) | +| **Retention Period** | `CLEANUP_RETENTION_HOURS` | `24` | Files older than this are deleted | +| **Dry Run** | `CLEANUP_DRY_RUN` | `false` | Test mode without actual deletion | + +#### Architecture Diagram +```mermaid +flowchart TD + A[Scheduled Trigger] --> B[cleanup_temp_directories] + + B -.-> FS[Local Filesystem] +``` + +#### Retry Strategies + +| Operation Type | Initial Interval | Max Interval | Backoff | Max Attempts | Use Case | +|---------------|------------------|--------------|---------|--------------|----------| +| **No Retry** | - | - | - | 1 | Local filesystem operations (permanent errors) | + +#### Cleanup Patterns + +**Local Directories:** +- Pattern: `{name}_{YYYYMMDD}_{HHMMSS}_{microseconds}` +- Example: `temp_20231201_143052_123456` +- Retention: Directories older than `CLEANUP_RETENTION_HOURS` are deleted +- Location: `model_manager/reports/temp/` by default + +## Installation & Setup + +### Prerequisites + +- Python 3.11+ +- Temporal server/cluster +- PostgreSQL database +- MLFlow server +- MinIO object storage (for MLFlow artifacts) +- MongoDB server (for notifications) + +**Note**: External dependencies must be available either through: +- Kubernetes cluster deployment +- Docker Compose setup +- Cloud-managed services +- Local installations + +#### MinIO Setup + +MinIO is required for MLFlow artifact storage. For detailed installation and configuration instructions, refer to: + +📚 **[Install MinIO via Helm Chart on K8s](https://aignosi-wiki.atlassian.net/wiki/spaces/IT1/pages/225214465/Install+Minio+via+Helm+Chart+on+K8s)** + +This guide covers: +- Helm chart installation on Kubernetes +- Storage configuration and persistence +- Access credentials setup +- Integration with MLFlow + +### Environment Setup + +1. **Clone the repository**: + ```bash + git clone + cd sientia-dataops-model-manager + ``` + +2. **Create virtual environment**: + ```bash + python -m venv venv + source venv/bin/activate # On Windows: venv\Scripts\activate + ``` + +3. **Install dependencies**: + ```bash + pip install -r requirements.txt + ``` + +4. **Configure environment variables** (see [Configuration](#-configuration) section) + +5. **Run validation script**: + ```bash + ./validate.sh + ``` + +### Temporal Namespace Setup + +The Model Manager requires a dedicated Temporal namespace to isolate workflows and maintain proper execution history. The namespace must be created **before** starting the application. + +#### Why Create a Namespace? + +- **Isolation**: Separates Model Manager workflows from other applications +- **Retention Control**: Configures workflow history retention (default: 7 days) +- **Multi-tenancy**: Enables multiple environments (dev, staging, prod) on same cluster +- **Security**: Allows namespace-level access control and permissions + +#### When to Create? + +- ✅ **Before first deployment** in any environment +- ✅ **Once per environment** (dev, staging, production) +- ✅ **After Temporal cluster setup** or upgrade + +#### How to Create the Namespace + +**Option 1: Using Temporal Admin Tools Pod (Recommended for Kubernetes)** + +```bash +# 1. List Temporal pods +kubectl get pods -n temporal + +# 2. Connect to admin tools pod +kubectl exec -it -n temporal -- bash + +# 3. Create namespace +tctl --namespace model-manager namespace register \ + --retention 7 \ + --description "Model Manager - ML Model Orchestration Namespace" + +# 4. Verify creation +tctl --namespace model-manager namespace describe + +# 5. Exit pod +exit +``` + +**Option 2: Using Port Forward (Local Development)** + +```bash +# 1. Port forward Temporal frontend +kubectl port-forward -n temporal svc/temporal-frontend 7233:7233 + +# 2. In another terminal, create namespace +tctl --address localhost:7233 \ + --namespace model-manager \ + namespace register \ + --retention 7 \ + --description "Model Manager - ML Model Orchestration Namespace" + +# 3. Verify +tctl --address localhost:7233 --namespace model-manager namespace describe +``` + +**Option 3: Direct kubectl exec (One-liner)** + +```bash +kubectl exec -n temporal -- \ + tctl --namespace model-manager namespace register \ + --retention 7 \ + --description "Model Manager - ML Model Orchestration Namespace" +``` + +#### Namespace Configuration + +| Parameter | Value | Description | +|-----------|-------|-------------| +| **Name** | `model-manager` | Namespace identifier (configurable via `TEMPORAL_NAMESPACE` env var) | +| **Retention** | `7 days` | Workflow history retention period | +| **Description** | `Model Manager - ML Model Orchestration Namespace` | Human-readable description | + +#### Verification + +To verify the namespace was created successfully: + +```bash +# List all namespaces +kubectl exec -n temporal -- tctl namespace list + +# Describe specific namespace +kubectl exec -n temporal -- \ + tctl --namespace model-manager namespace describe +``` + +#### Troubleshooting + +**Error: "namespace already exists"** +- ✅ This is fine! The namespace is already configured +- No action needed, proceed with application deployment + +**Error: "connection refused"** +- ❌ Temporal server is not accessible +- Verify Temporal cluster is running: `kubectl get pods -n temporal` +- Check network connectivity and port forwarding + +**Error: "permission denied"** +- ❌ Insufficient permissions to create namespace +- Contact cluster administrator for namespace creation +- Or request elevated permissions for your service account + +### Local Development Setup + +1. **Clone the repository** + ```bash + git clone + cd sientia-dataops-model-manager + ``` + +2. **Create virtual environment** + ```bash + conda create -p ./venv python=3.11 + conda activate ./venv + ``` + +3. **Install dependencies** + + 1. **Install github cli** + ```bash + sudo apt update + sudo apt install gh -y + ``` + + 2. **Authenticate with github** + ```bash + gh auth login + ``` + + 3. **Install Python dependencies** + ```bash + python -m pip install --upgrade pip + # Install production dependencies + pip install -r requirements.txt + # Install development and testing tools + pip install -r requirements-dev.txt + ``` + +4. **Create environment configuration file** + ```bash + cp .env.example .env + # Edit .env with your connection details + ``` + +5. **Configure external dependencies** + + The Model Manager requires connections to several external services. For local development, you can use the provided port-forward script to establish connections to services running in your Kubernetes cluster. + +**Services and Ports:** + +| Local Port | Service | Description | Namespace | +|------------|---------|-------------|-----------| +| `5432` | `paradedb-rw` | PostgreSQL Database | `paradedb` | +| `45249` | `sientia-tracker-mlflow-tracking` | MLflow Tracking Server | `sientia-tracker` | +| `37463` | `temporal-frontend` | Temporal gRPC API | `temporal` | +| `8080` | `temporal-web` | Temporal Web UI | `temporal` | +| `42297` | `my-release-mongodb` | MongoDB Database | `mongodb` | +| `36577` | `minio` | MinIO Object Storage | `minio` | + +**Managing Port Forwards:** + +```bash +# View active port forwards +jobs -l + +# Stop all port forwards +jobs -p | xargs kill + +``` + +**Manual Port Forwarding:** + +If you prefer manual control or need different ports: + +```bash +# PostgreSQL +kubectl -n paradedb port-forward svc/paradedb-rw 5432:5432 & + +# MLflow +kubectl -n sientia-tracker port-forward svc/sientia-tracker-mlflow-tracking 45249:80 & + +# Temporal +kubectl -n temporal port-forward svc/temporal-frontend 37463:7233 & + +# Temporal UI +kubectl -n temporal port-forward svc/temporal-web 8080:8080 & + +# MongoDB +kubectl -n mongodb port-forward svc/my-release-mongodb 42297:27017 & + +# MinIO +kubectl -n minio port-forward svc/minio 36577:9000 & +``` + +## How to Run + +### Running the Model Manager Application + +Use the provided script to run the application locally: + +```bash +# Make script executable (first time only) +chmod +x run_local.sh + +# Run the application +./run_local.sh +``` + +The script will: +- Load environment variables from `.env` +- Start the model-manager worker application + +**Note**: Activate your virtual environment before running the script: +```bash +source ./venv/bin/activate # or: conda activate ./venv +./run_local.sh +``` + +### Running Tests and Coverage + +Run tests with coverage using pytest directly: + +```bash +# Run all tests with coverage +pytest tests/ --cov=model_manager --cov-report=term-missing --cov-report=html + +# Open the coverage report in your browser +open htmlcov/index.html # macOS +xdg-open htmlcov/index.html # Linux +``` + +### Manual Test Execution + +You can also run tests manually: + +```bash +# Activate virtual environment +source ./venv/bin/activate + +# Run all tests +pytest + +# Run with coverage +pytest --cov=model_manager --cov-report=html + +# Run specific test categories +pytest tests/activities/ +pytest tests/workflow/ +``` + +### Manual Application Execution + +For manual execution without scripts: + +```bash +# Activate virtual environment +source ./venv/bin/activate + +# Load environment variables (if using .env file) +if [ -f .env ]; then + export $(cat .env | grep -v '^#' | xargs) +fi + +# Start the model-manager worker +python -m model_manager.worker.worker +``` + +To start a **`train_model`** run from your own Temporal client, use the payload shape in **`input-sample.json`** (task queue `train_model--queue`, matching `RUNTIME` on the worker). For scripted tests that use the JSON scenarios under `docs/test-scenarios/`, see **`scripts/run_training_test.py`**. + +## Code Quality & Validation + +### Overview + +Since Python is not a compiled language, we use a robust set of tools to validate code quality, security, and correctness before execution. These tools detect errors, style issues, security vulnerabilities, and ensure code consistency. + +### Validation Tools + +#### 1. **Ruff** - Linting and Formatting ⚡ +Modern and extremely fast tool (written in Rust) that replaces multiple tools: +- **Linting**: Detects code errors, style issues (PEP 8), common bugs +- **Formatting**: Automatically formats code consistently +- **Speed**: 10-100x faster than Flake8/Black + +#### 2. **mypy** - Type Checking 🏷️ +Static type checker that analyzes type hints: +- Detects type errors before execution +- Improves code documentation +- Prevents bugs related to incorrect types + +#### 3. **Bandit** - Security Analysis 🔒 +Security vulnerability scanner: +- Detects insecure code patterns +- Identifies hardcoded passwords, SQL injection, etc. +- Ensures compliance with security practices + +#### 4. **pytest** - Automated Testing 🧪 +Testing framework with code coverage: +- Executes unit and integration tests +- Measures code coverage +- Supports asynchronous tests + +### Tools Installation + +```bash +# Install development dependencies +pip install -r requirements-dev.txt +``` + +### Complete Validation + +#### Option 1: Automated Script (Recommended) +```bash +# Run all validations at once +./validate.sh +``` + +The `validate.sh` script automatically executes: +1. ✅ Format checking (Ruff) +2. ✅ Code linting (Ruff) +3. ✅ Type checking (mypy) +4. ✅ Security analysis (Bandit) +5. ✅ Unit tests with coverage (pytest) + +#### Option 2: Individual Commands +```bash +# 1. Check formatting +ruff format --check model_manager/ tests/ + +# 2. Check linting +ruff check model_manager/ tests/ + +# 3. Check types +mypy model_manager/ + +# 4. Security analysis +bandit -r model_manager/ -ll + +# 5. Run tests +pytest tests/ --cov=model_manager --cov-report=term-missing +``` + +### Automatic Fixes + +Some tools can automatically fix issues: + +```bash +# Format code automatically +ruff format model_manager/ tests/ + +# Fix linting issues automatically +ruff check --fix model_manager/ tests/ +``` + +### Configuration + +All tools are configured in the `pyproject.toml` file: +- **Ruff**: Linting rules, formatting, complexity +- **mypy**: Type checking settings +- **pytest**: Test and coverage options +- **Bandit**: Security rules + +### CI/CD Integration + +The project uses **reusable workflows** from `Aignosi/github_workflow_templates` for CI/CD: + +#### Quality Gate (`.github/workflows/quality-gate.yml`) +Runs automatically on each Pull Request to `main`: +- ✅ Formatting and linting block merge if they fail +- ⚠️ Type checking and security generate warnings but don't block +- ✅ Tests must pass with minimum 80% coverage +- ✅ SonarQube analysis for code quality metrics +- ✅ Automatic version calculation based on branch pattern + +#### Deploy (`.github/workflows/deploy.yml`) +Runs automatically when a PR is merged to `main`: +- ✅ Builds and pushes Docker image to Azure Container Registry +- ✅ Creates GitHub release with calculated version +- ✅ Deploys to Kubernetes using Helm + +### Best Practices + +1. **Before Commit**: Run `./validate.sh` to ensure quality +2. **During Development**: Use `ruff check --watch` for real-time feedback +3. **Type Hints**: Add type hints to new functions for better validation +4. **Tests**: Maintain coverage above 80% +5. **Security**: Review and fix all Bandit warnings + +## Testing + +### Test Structure +``` +tests/ +├── activities/ # Activity implementation tests +├── workflows/ # Workflow orchestration tests +├── utils/ # Utility function tests +├── worker/ # Worker tests +├── schedules/ # Schedule configuration tests +└── sientia/ # Sientia module tests +``` + +### Test Execution +```bash +# Install test dependencies +pip install pytest pytest-cov pytest-asyncio + +# Run tests with coverage +pytest --cov=model_manager --cov-report=html + +# Run specific test modules +pytest tests/activities/test_training.py +pytest tests/workflows/test_train_model.py +``` + +### Integration Tests + +The project includes integration tests that validate the complete training workflow against a running Temporal cluster. These tests use JSON-based scenario files for easy configuration and maintenance. + +#### Running Training Smoke Tests + +1. Open `scripts/run_training_test.py` in your IDE. +2. Use the **"Run Cell"** or **"Run Below"** functionality (requires the Python/Jupyter extension). +3. The script will: + - Load configuration from `.env` + - Optionally clean up previous test runs in the database + - Insert a new `experiment_run` record + - Upload a sample dataset to MinIO + - Start the `train_model` workflow and wait for completion + +```bash +# Configuration for local testing is managed via the .env file +# Run the cells in scripts/run_training_test.py for end-to-end validation +``` + +#### Manual Cleanup Test + +For manual verification of the file cleanup logic, use the provided utility script: + +```bash +# Run once to clean up stale local directories +python scripts/run_cleanup_test.py +``` + +#### Test Scenarios + +Test scenarios are defined as JSON files in `docs/test-scenarios/`. Payloads use **snake_case** keys aligned with `TrainModelParams` / Temporal `train_model` workflow input (same shape as `input-sample.json`). `date_column` is required; if `date_format` is omitted or blank, the server uses the default `yyyy-MM-dd HH:mm:ss` (see `DEFAULT_TRAIN_DATE_FORMAT` in `train_model_params.py`). + +Automated coverage: pytest E2E under `e2e/` runs every scenario listed below (see [`e2e/scenarios.md`](e2e/scenarios.md)). + +| Scenario | Description | Key Features | +|----------|-------------|--------------| +| `01-linear-regression-basic` | Basic linear regression | No scaler, no lags | +| `02-linear-regression-with-scaler` | Linear regression with normalization | `model_kwargs.scaler_name`: `"Standard Scaler"` | +| `03-polynomial-regression-degree2` | Polynomial regression (degree 2) | Scaler recommended / required for stability | +| `04-polynomial-regression-degree3` | Polynomial regression (degree 3) | Scaler recommended / required for stability | +| `05-linear-regression-with-lags` | Linear regression with lag features | `data_model_kwargs.lag_train` / `lag_val` | +| `06-linear-regression-nan-interpolation` | Linear regression with NaN handling | `data_model_kwargs.nan_treatment`: `"linear interpolation"` | +| `07-linear-regression-static-window-removal` | Linear regression with static window removal | `data_model_kwargs.rem_static_win`: `true` | +| `08-linear-regression-with-limits` | Linear regression with variable limits | `data_model_kwargs.support_filters` (`min`/`max`) | +| `09-polynomial-degree2-with-scaler-and-lags` | Complete polynomial scenario | Scaler + lags + degree 2 | +| `10-linear-regression-with-ar` | Autoregressive placeholder | `opt_params.include_ar`: `true` (wrapper-specific) | +| `11-linear-regression-static-threshold-custom` | Linear regression with custom static threshold | `data_model_kwargs.static_threshold`: `100` | +| `12-angular-test-date-format` | Alternate date column / format | `date_column` `DATA`, `date_format` `dd/MM/yyyy HH:mm:ss`, file `training_data_dd_mm_yyyy.csv` in E2E | +| `13-angular-test-double-date-column` | Alternate CSV + date window | Same MinIO object as 12; bounded `start_date` / `end_date` | +| `14-angular-test-polynomial-support-filters` | Polynomial + line support filters | `support_filters` with `upper_line` / `lower_line` | +| `15-linear-regression-custom-target-column` | Custom target column name | `target_variable` not named `target`; MinIO `training_data_custom_target.csv` | +| `16-linear-regression-naive-timestamp-header` | Naive `Timestamp` column header | `date_column` `Timestamp`, `training_data_timestamp_naive.csv` | +| `17-linear-regression-blank-timestamp-row` | Missing timestamp on one row | Row dropped; `training_data_blank_timestamp_row.csv` | + +#### Scenario File Structure + +```json +{ + "experiment_run_id": 1001, + "variable_columns": ["feature_a", "feature_b"], + "target_variable": "target", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "train_size": 80, + "shuffle": true, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": {"feature_a": 0, "feature_b": 0}, + "lag_val": {"feature_a": 0, "feature_b": 0}, + "nan_treatment": "drop" + }, + "model_kwargs": { + "degree": 1, + "scaler_name": "Standard Scaler" + }, + "opt_params": {} +} +``` + +#### Creating New Scenarios + +1. Copy an existing scenario file as a template +2. Modify parameters according to your test case +3. Save with a descriptive name: `XX-description.json` +4. Add or extend a test in `e2e/test_train_model_workflow.py` (and update `e2e/scenarios.md`) so the scenario stays executable +5. For ad-hoc manual runs against a real Temporal/MinIO/Postgres stack, adapt the cells in `scripts/run_training_test.py` to load your JSON payload + +#### Important validations + +- **Workflow payload** (`input-sample.json`, Temporal `execute_workflow`, `docs/test-scenarios/*.json`): snake_case fields validated by `TrainModelParams` (see [Business validation rules](#business-validation-rules) above). + +## Monitoring and Metrics + +The Model Manager system exposes comprehensive Prometheus metrics for operational visibility and performance monitoring: + +### Application Health Metrics +- `app_up`: Application health status (1=healthy, 0=unhealthy) + - Labels: `pod_id` + +### Workflow Execution Metrics +- `workflow_execution_total`: Total workflow executions + - Labels: `workflow_name`, `status` (success/failure) +- `activity_execution_total`: Total activity executions + - Labels: `activity_name`, `status` (success/failure) + +### Training Metrics +- Training success/failure rates through notification system +- Model save performance metrics +- Experiment status tracking +- **RCE Drift Metrics**: Reduced Coulomb Energy (RCE) for drift detection + - `silverman_radius`: Optimal bandwidth for kernel density estimation + - `rce_reference`: RCE value for reference data + - `rce_current`: RCE value for current data + - `rce_drift`: Drift score between reference and current distributions + +### Cleanup Metrics +- Cleanup execution success/failure rates +- Number of directories cleaned from local filesystem +- Cleanup duration and performance + +## Configuration + +### Environment Variables + +Values are read in `model_manager/utils/connectors_config.py` and `model_manager/worker/worker.py`. Defaults below match the code. + +| Variable | Description | Default | Required | +|----------|-------------|---------|----------| +| `TEMPORAL_HOST` | Temporal server address (`host:port`) | `localhost:7233` | Yes | +| `TEMPORAL_NAMESPACE` | Temporal namespace | `model-manager` | No | +| `TEMPORAL_USE_TLS` | Use TLS for Temporal gRPC (`true`/`false`). Set `true` when the endpoint serves TLS or you get HTTP redirects (for example 308) to HTTPS | `false` | No | +| `RUNTIME` | Suffix for worker task queues (`train_model--queue`, `cleanup_files--queue`) | `single` (via `_get_runtime`) | No | +| `TRAIN_TASK_QUEUE` | Used by **clients** (for example `scripts/run_training_test.py`), not by the worker process | unset | No | +| `CLEANUP_TASK_QUEUE` | Used by **clients** (for example `scripts/run_cleanup_test.py`), not by the worker | unset | No | +| `POSTGRES_HOST` | PostgreSQL hostname | `localhost` | Yes | +| `POSTGRES_PORT` | PostgreSQL port | `5432` | Yes | +| `POSTGRES_USER` | PostgreSQL username | `sientia` | Yes | +| `POSTGRES_PASSWORD` | PostgreSQL password | `sientia` | Yes | +| `POSTGRES_DBNAME` | PostgreSQL database | `sientia` | Yes | +| `POSTGRES_MIN_CONNECTIONS` | Minimum pool size | `5` | No | +| `POSTGRES_MAX_CONNECTIONS` | Maximum pool size | `20` | No | +| `MLFLOW_URL` | MLflow tracking URL (scheme, host, and port) | `http://localhost:5080` | Yes | +| `MLFLOW_USERNAME` | MLflow basic auth username | `aignosi` | Yes | +| `MLFLOW_PASSWORD` | MLflow basic auth password | `aignosi` | Yes | +| `MINIO_ENDPOINT_URL` | MinIO / S3 endpoint URL | `http://localhost:9000` | Yes | +| `MINIO_ACCESS_KEY` | MinIO access key | `minioadmin` | Yes | +| `MINIO_SECRET_KEY` | MinIO secret key | `minioadmin` | Yes | +| `MINIO_REGION` | MinIO region | `us-east-1` | No | +| `MINIO_SECURE` | Use TLS for MinIO client (`true`/`false`) | `false` | No | +| `MINIO_DEFAULT_BUCKET` | Default bucket for `MinioRepository` | `model-training` | No | +| `MINIO_MAX_RETRY_ATTEMPTS` | S3 retry attempts | `3` | No | +| `MINIO_RETRY_MODE` | Retry mode | `adaptive` | No | +| `MINIO_CONNECT_TIMEOUT` | Connection timeout (seconds) | `10` | No | +| `MINIO_READ_TIMEOUT` | Read timeout (seconds) | `60` | No | +| `MONGODB_URL` | MongoDB host:port (no scheme; used inside connection string) | `localhost:27018` | Yes | +| `MONGODB_USERNAME` | MongoDB username | `root` | Yes | +| `MONGODB_PASSWORD` | MongoDB password | `wKZDbMNU1c` | Yes | +| `MONGODB_DATABASE` | MongoDB database name | `sientia` | Yes | +| `MONGODB_TTL_INDEX_HOURS` | TTL index duration (hours) | `1` | No | +| `STORE_BASE_URL` | Plugin store Git server base URL | `http://localhost:3000` | No | +| `STORE_OWNER` | Git owner/org | `sientia` | No | +| `STORE_REPO` | Git repository | `model-library-store` | No | +| `STORE_BRANCH` | Optional branch | unset | No | +| `STORE_USERNAME` / `STORE_PASSWORD` | Git HTTP credentials | unset | No | +| `STORE_CACHE_TTL_SECONDS` | Plugin index cache TTL | unset | No | +| `PYPI_SERVER` | Custom PyPI index URL | `http://localhost:5000` | No | +| `PYPI_USERNAME` / `PYPI_PASSWORD` | PyPI credentials | unset | No | +| `CLEANUP_CRON` | Cleanup schedule cron | `0 0 * * *` | No | +| `CLEANUP_TIMEZONE` | Cleanup schedule timezone | `UTC` | No | +| `CLEANUP_EXECUTION_TIMEOUT_HOURS` | Cleanup workflow timeout (hours) | `1` | No | +| `CLEANUP_RETENTION_HOURS` | Local temp retention (hours) | `24` | No | +| `CLEANUP_DRY_RUN` | Cleanup dry-run | `false` | No | +| `LOG_LEVEL` | Log level | `INFO` | No | +| `PROJECT_NAME` | Project name for notifications/metrics | `model-manager` | No | +| `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No | +| `HTTP_SDK_METRICS_PORT` | Temporal SDK metrics port | `9091` | No | +| `POD_ID` | Pod label for metrics | unset | No | +| `EXTRA_PIP_REQUIREMENTS` | Extra `pip` packages for runtime installs | unset | No | +| `TIMEOUT_VALIDATE_PARAMS` | Activity timeout (seconds) | `30` | No | +| `TIMEOUT_TRAIN_MODEL` | Training activity timeout (seconds) | `2700` | No | +| `TIMEOUT_DELETE_FILE` | Delete/cleanup activity timeout (seconds) | `120` | No | +| `TIMEOUT_UPDATE_DATABASE` | DB update activity timeout (seconds) | `30` | No | +| `TIMEOUT_CLEANUP_LOCAL` | Cleanup workflow activity timeout (seconds) | `120` | No | + +#### Workflow Activity Timeouts + +These timeouts control how long each activity in workflows can run before timing out. All values are in seconds. + +**Training Workflow Timeouts:** + +| Variable | Description | Default | Calculation Basis | +|----------|-------------|---------|-------------------| +| `TIMEOUT_VALIDATE_PARAMS` | Parameter validation timeout | `30` | Fast operation, no I/O | +| `TIMEOUT_TRAIN_MODEL` | Model training timeout | `2700` | Large dataset processing (45 min) | +| `TIMEOUT_DELETE_FILE` | File delete / related I/O timeout | `120` | Network storage | +| `TIMEOUT_UPDATE_DATABASE` | Database update timeout | `30` | PostgreSQL update query (30 sec) | + +**Cleanup Workflow Timeouts:** + +| Variable | Description | Default | Calculation Basis | +|----------|-------------|---------|-------------------| +| `TIMEOUT_CLEANUP_LOCAL` | Local cleanup timeout | `120` | Scan and delete directories (2 min) | + +**Note**: These timeouts can be adjusted based on your infrastructure performance and file sizes. If you're processing files larger than 200MB or have slower network/compute resources, increase these values accordingly. + +## Development + +### Code Quality & Testing + +The project maintains **100% code coverage** with comprehensive unit and integration tests. + +#### Running Tests + +```bash +# Run all tests with coverage +pytest tests/ --cov=model_manager --cov-report=term-missing --cov-report=xml --cov-report=html + +# Run specific test file +pytest tests/activities/test_training.py -v + +# Run with coverage visualization +pytest tests/ --cov=model_manager --cov-report=xml +# Then open htmlcov/index.html in browser +``` + +#### Validation Script + +The `validate.sh` script runs all quality checks before commit: + +```bash +./validate.sh +``` + +This script performs: +1. ✅ **Code Formatting** (Ruff) +2. ✅ **Code Linting** (Ruff) +3. ✅ **Type Checking** (mypy) +4. ✅ **Security Analysis** (Bandit) +5. ✅ **Unit Tests** (pytest with 80%+ coverage requirement) + +#### Coverage Visualization + +For real-time coverage feedback in VS Code/Windsurf: + +1. **Install Coverage Gutters extension** +2. **Configure `.vscode/settings.json`**: + ```json + { + "coverage-gutters.coverageBaseDir": "${workspaceFolder}", + "coverage-gutters.coverageFileNames": ["coverage.xml"], + "coverage-gutters.showLineCoverage": true, + "coverage-gutters.showRulerCoverage": true + } + ``` +3. **Run tests to generate coverage**: + ```bash + pytest tests/ --cov=model_manager --cov-report=xml + ``` +4. **Activate Coverage Gutters**: Press `Ctrl+Shift+7` (or `Cmd+Shift+7` on Mac) + +#### Automated Versioning + +The project uses semantic versioning based on branch patterns: + +| Branch Pattern | Version Change | Example | +|---------------|----------------|---------| +| `release/*` | Major version bump | `2.0.0` | +| `feature/*` | Minor version bump | `1.2.0` | +| `fix/*` | Patch version bump | `1.1.3` | +| `rc/*` | Release candidate | `1.1.2-rc2` | + +Version is calculated automatically in the CI/CD pipeline and passed to SonarQube. + +### Project Structure +``` +sientia-dataops-model-manager/ +├── model_manager/ # Main application package +│ ├── activities/ # Temporal activity implementations +│ │ ├── __init__.py +│ │ ├── activities.py # Main activities orchestrator (combines all activities) +│ │ ├── experiment_tracking.py # Experiment status tracking and database operations +│ │ ├── training.py # ML model training operations (includes MLFlow & MinIO) +│ │ └── cleanup.py # File and directory cleanup operations +│ ├── workflows/ # Temporal workflow definitions +│ │ ├── __init__.py +│ │ ├── train_model.py # Complete ML model training workflow +│ │ └── cleanup_files.py # Automated file cleanup workflow +│ ├── schedules/ # Temporal schedule configurations +│ │ ├── __init__.py +│ │ └── cleanup_schedule.py # Cleanup schedule creation and management +│ ├── worker/ # Worker implementation +│ │ ├── __init__.py +│ │ └── worker.py # Main worker orchestrator (Temporal client setup) +│ ├── utils/ # Utility functions and helpers +│ │ ├── __init__.py +│ │ ├── connectors_config.py # Environment-based configuration builders +│ │ ├── exceptions.py # Custom exception definitions +│ │ ├── logger_helper.py # Logger initialization utilities +│ │ ├── models/ # Data models and schemas +│ │ │ ├── __init__.py +│ │ │ ├── train_model_params.py # Training parameters model +│ │ │ ├── train_model_result.py # Training result model +│ │ │ └── experiment_status.py # Experiment status enum +│ │ └── repository/ # Data access layer +│ │ └── data_manager_repository.py # Core data logic & report generation +│ ├── sientia/ # Sientia-specific implementations +│ │ ├── __init__.py +│ │ ├── exceptions.py # Custom exceptions +│ │ ├── metrics.py # Business metrics (includes RCE drift detection) +│ │ ├── reports.py # Report generation logic +│ ├── reports/ # Report templates and temporary files +│ │ └── temp/ # Temporary report files (cleaned up automatically) +│ ├── metrics.py # Prometheus metrics definitions +│ └── runtime_paths.py # Runtime directory management +├── scripts/ # Test and utility scripts +│ ├── run_cleanup_test.py # Manual cleanup workflow test +│ └── run_training_test.py # Training test with scenario support (--all for batch) +├── tests/ # Test suite +│ ├── activities/ # Activity tests +│ ├── workflows/ # Workflow tests +│ ├── utils/ # Utility tests +│ ├── worker/ # Worker tests +│ ├── schedules/ # Schedule tests +│ └── sientia/ # Sientia module tests +├── docs/ # Documentation and test data +│ ├── test-scenarios/ # JSON test scenario files for integration tests +│ └── test-model-data.csv # Sample CSV data for testing +├── .github/workflows/ # CI/CD workflows +│ ├── quality-gate.yml # PR quality checks +│ └── deploy.yml # Deployment workflow +├── Dockerfile # Container image definition +├── values.yaml # Helm chart values +├── pyproject.toml # Project configuration +├── requirements.txt # Production dependencies +├── requirements-dev.txt # Development dependencies +├── validate.sh # Code quality validation script +├── run_local.sh # Local execution script +├── input-sample.json # Example payload for the train_model workflow +└── README.md # This file +``` + +### Adding New Features + +1. **Follow Temporal patterns** for new workflows and activities +2. **Add comprehensive docstrings** for all public methods +3. **Include Prometheus metrics** for monitoring +4. **Add unit tests** for new functionality (maintain 80%+ coverage) +5. **Run validation script** (`./validate.sh`) before committing +6. **Update this README** with new features and configuration + +### Test Coverage Guidelines + +- **Minimum coverage**: 80% (enforced by CI/CD) +- **Current coverage**: 100% 🎯 +- **Test all branches**: Use Coverage Gutters to identify uncovered lines +- **Mock external dependencies**: Use `unittest.mock` for external services +- **Async testing**: Use `pytest-asyncio` for async activities and workflows +- **Test structure**: + ``` + tests/ + ├── activities/ # Activity tests + ├── workflows/ # Workflow tests + ├── utils/ # Utility tests + ├── worker/ # Worker tests + ├── schedules/ # Schedule tests + └── sientia/ # Sientia module tests + ``` + +## Troubleshooting + +### Common Issues + +1. **Temporal connection failures** + - Verify Temporal server is running and reachable at `TEMPORAL_HOST` + - Check namespace configuration and permissions + - Review server logs for connection issues + - If you see **308 Permanent Redirect** or **invalid compression flag** on connect, the endpoint likely expects **TLS** while `TEMPORAL_USE_TLS` is `false`. Set `TEMPORAL_USE_TLS=true` and point `TEMPORAL_HOST` at the correct TLS gRPC address. + +2. **MLFlow connection issues** + - Verify MLFlow server is running and accessible + - Check authentication credentials and permissions + - Ensure model names and versions exist + +3. **Database connection issues** + - Verify PostgreSQL service is running + - Check connection credentials and network access + - Ensure proper connection pool configuration + +4. **Workflow execution failures** + - Review activity error logs and notifications + - Check training parameter validation errors + - Verify input data format and required fields + +### Debug Mode + +Enable debug logging by setting the log level: +```bash +export LOG_LEVEL=DEBUG +``` + +## Performance Tuning + +### Key Parameters + +- **Worker Concurrency**: Adjust `max_concurrent_workflow_tasks` and `max_concurrent_activities` + - Training workflows: 10 concurrent tasks/activities + - Cleanup workflows: 20 concurrent tasks/activities +- **Connection Pools**: Optimize database connection pool sizes +- **Model Retention**: Configure MLFlow model retention based on requirements +- **Batch Sizes**: Adjust data processing batch sizes for optimal throughput + +### Scaling Considerations + +- **Horizontal Scaling**: Deploy multiple worker instances +- **Task queue distribution**: Workers register `train_model--queue` and `cleanup_files--queue` (see `RUNTIME`) +- **Database Performance**: Optimize indexes and connection pooling +- **MLFlow Performance**: Configure appropriate model serving resources +- **Storage Management**: Adjust cleanup retention period based on storage capacity and costs + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes with comprehensive testing +4. Update documentation and docstrings +5. Submit a pull request + +### Code Quality Standards + +- Follow PEP 8 style guidelines +- Include comprehensive docstrings for all public methods +- Maintain test coverage above 80% +- Use type hints where appropriate +- Follow Temporal.io best practices + +## License + +This project is licensed under the terms specified in the LICENSE file. + +## Support + +For support and questions: +- Check the troubleshooting section above +- Review the metrics and logs for error patterns +- Open an issue in the project repository +- Contact the development team + +## Local GitHub Actions Testing (act) + +### What is act? + +[act](https://nektosact.com/) is a tool that allows you to run GitHub Actions locally using Docker. This is useful for: + +- **Testing workflows** before pushing to the repository +- **Debugging issues** in workflows without creating commits +- **Speeding up development** by avoiding push/wait/check cycles +- **Saving GitHub Actions minutes** during development + +### Installation + +#### Prerequisites + +- Docker installed and running +- Go (for installation via `go install`) + +#### Installation Steps + +```bash +# 1. Update packages +sudo apt-get update + +# 2. Install Go (if not already installed) +sudo apt-get install golang + +# 3. Install act +go install github.com/nektos/act@latest + +# 4. Add Go bin to PATH +echo 'export PATH="$PATH:$HOME/go/bin"' >> ~/.bashrc +source ~/.bashrc + +# 5. Verify installation +act --version +``` + +On first run, `act` will ask which Docker image to use: +- **Large** (~17GB): Full image, compatible with almost all actions +- **Medium** (~500MB): Balanced image, compatible with most actions ✅ Recommended +- **Micro** (<200MB): Minimal image, Node.js only + +### Configuration + +#### `.secrets` File + +Create a `.secrets` file in the project root to store tokens and credentials: + +```bash +# .secrets +GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxx +SONAR_TOKEN=sqp_xxxxxxxxxxxxxxxxxxxx +SONAR_HOST_URL=https://sonarqube.example.com +CI_DEPS_APP_ID=123456 +CI_DEPS_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" +``` + +> ⚠️ **Important**: The `.secrets` file is already in `.gitignore`. Never commit this file! + +#### `.event.json` File + +Create a `.event.json` file to simulate GitHub events (e.g., pull request): + +```json +{ + "pull_request": { + "head": { + "ref": "feature/my-feature" + }, + "number": 1 + } +} +``` + +> ⚠️ **Important**: The `.event.json` file is already in `.gitignore`. Never commit this file! + +### Usage + +#### List Available Jobs + +```bash +act -l +``` + +This command lists all available workflows and jobs in the repository. + +#### Run Quality Gate Locally + +```bash +act pull_request -j quality-gate \ + --secret-file .secrets \ + --env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true" \ + --eventpath .event.json +``` + +#### Run Deploy Workflow with Local Repository + +If you have workflows that reference external repositories (e.g., reusable workflows), you can map them locally: + +```bash +act pull_request \ + -e .event.json \ + --secret-file .secrets \ + -W .github/workflows/deploy.yml \ + --local-repository Aignosi/github_workflow_templates=/path/to/local/github_workflow_templates \ + --container-daemon-socket /var/run/docker.sock \ + --container-options "--user $(id -u):$(id -g)" +``` + +#### Docker Socket Permissions + +If you encounter permission issues with Docker socket: + +```bash +# Grant temporary access to Docker socket +sudo chmod 666 /var/run/docker.sock + +# Fix file ownership after running act (if needed) +sudo chown -R $USER:$USER /path/to/project +``` + +### Command Reference + +#### Command Parameters + +| Parameter | Description | +|-----------|-------------| +| `pull_request` | Event type to simulate (can be `push`, `pull_request`, `workflow_dispatch`, etc.) | +| `-j quality-gate` | Specific job name to execute (use `act -l` to see available jobs) | +| `--secret-file .secrets` | File containing secrets (tokens, credentials) | +| `--env VAR=value` | Sets environment variables for execution | +| `--eventpath .event.json` | JSON file with the simulated event payload | + +#### Special Parameter: `SONAR_SCANNER_OPTS` + +```bash +--env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true" +``` + +This parameter is required because SonarQube tries to automatically detect the CI environment. When running locally with `act`, the complete GitHub Actions context is not available, causing errors. The `-Dsonar.ci.autoconfig.disabled=true` flag disables this automatic detection. + +#### Other Useful Commands + +```bash +# List all jobs +act -l + +# Run with verbose output +act pull_request -j quality-gate --secret-file .secrets -v + +# Run a push event +act push -j build --secret-file .secrets + +# Use a specific Docker image +act -P ubuntu-latest=catthehacker/ubuntu:act-latest + +# Dry-run (doesn't execute, only shows what would be done) +act -n +``` + +#### Troubleshooting + +| Problem | Solution | +|---------|----------| +| `SyntaxError: Unexpected end of JSON input` | Check if `.event.json` is properly formatted | +| `NullPointerException` in SonarQube | Add `--env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true"` | +| `Not Found` when accessing GitHub API | Check if `GITHUB_TOKEN` in `.secrets` is valid | +| Job not found | Use `act -l` to see the correct job names | + +## Docker + +### Create image + +```bash +$ docker build --ssh default --no-cache --progress=plain -t aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0 . +``` + +### Create container + +```bash +$ docker run --env-file .env --network="host" --name sientia-dataops-model-manager -d aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0 + +$ docker logs -f sientia-dataops-model-manager +``` + +### Login using access token + +```bash +$ docker login -u -p aignosi.azurecr.io +``` + +### Push image to repository + +```bash +$ docker push aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0 +``` + +## Helm Chart + +### Reference + +https://aignosi-wiki.atlassian.net/wiki/spaces/IT1/pages/274563074/Como+utilizar+o+Helm+Repo+Privado + +### Add Helm Chart repository + +```bash +$ helm repo add sientia \ + https://raw.githubusercontent.com/Aignosi/sientia-dataops-helm-repo/refs/heads/main/ \ + --username $GITHUB_USER \ + --password $GITHUB_PASS + +# Update repository +$ helm repo update + +# List repositories +$ helm repo list + +# List versions of a specific chart +$ helm search repo sientia --versions + +# List all charts available +$ helm search repo sientia + +# List chart details +$ helm show all sientia/sientia-module + +# Download chart to current directory +$ helm pull sientia/sientia-module --version 0.6.0 --untar + +# Remove chart directory +$ rm -rf sientia-module +``` + +### Helm Install + +```shell +$ helm upgrade --install sientia-dataops-model-manager sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.6.0 +``` + +### Uninstall Helm Chart + +```shell +$ helm uninstall sientia-dataops-model-manager -n sientia +``` + +--- + +**Note**: The Model Manager system is designed for production use in industrial ML environments. Ensure proper security configuration and network isolation for production deployments. diff --git a/dashboards/sientia-dataops-model-manager.json b/dashboards/sientia-dataops-model-manager.json new file mode 100644 index 0000000..0b1ab53 --- /dev/null +++ b/dashboards/sientia-dataops-model-manager.json @@ -0,0 +1,197 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "id": 1, + "title": "Data Preparation Latency (p50 / p95)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(sientia_training_data_preparation_lag_bucket[5m]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(sientia_training_data_preparation_lag_bucket[5m]))", + "legendFormat": "p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { "unit": "s", "color": { "mode": "palette-classic" } } + }, + "options": { "tooltip": { "mode": "multi" } } + }, + { + "id": 2, + "title": "Model Fit Latency (p50 / p95)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(sientia_training_model_fit_lag_bucket[5m]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(sientia_training_model_fit_lag_bucket[5m]))", + "legendFormat": "p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { "unit": "s", "color": { "mode": "palette-classic" } } + }, + "options": { "tooltip": { "mode": "multi" } } + }, + { + "id": 3, + "title": "Data Preparation Errors / s", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "targets": [ + { + "expr": "rate(sientia_training_data_preparation_error_count_total[5m])", + "legendFormat": "{{model_name}} / {{model_type}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "ops", "color": { "mode": "palette-classic" } } + } + }, + { + "id": 4, + "title": "Model Fit Errors / s", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "rate(sientia_training_model_fit_error_count_total[5m])", + "legendFormat": "{{model_name}} / {{model_type}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "ops", "color": { "mode": "palette-classic" } } + } + }, + { + "id": 5, + "title": "Model Quality — MSE", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 16 }, + "targets": [ + { + "expr": "sientia_training_model_quality_mse", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "decimals": 4 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" } + }, + { + "id": 6, + "title": "Model Quality — MAE", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 16 }, + "targets": [ + { + "expr": "sientia_training_model_quality_mae", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "decimals": 4 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" } + }, + { + "id": 7, + "title": "Model Quality — R²", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 16 }, + "targets": [ + { + "expr": "sientia_training_model_quality_r2", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "percentunit", "decimals": 3, "min": -1, "max": 1 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "background" } + }, + { + "id": 8, + "title": "Dataset — Train Rows", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 20 }, + "targets": [ + { + "expr": "sientia_training_dataset_train_rows", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "short", "decimals": 0 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" } + }, + { + "id": 9, + "title": "Dataset — Val Rows", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 20 }, + "targets": [ + { + "expr": "sientia_training_dataset_val_rows", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "short", "decimals": 0 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" } + }, + { + "id": 10, + "title": "Dataset — Feature Count", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 20 }, + "targets": [ + { + "expr": "sientia_training_feature_count", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "short", "decimals": 0 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" } + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["sientia", "model-manager", "training"], + "templating": { "list": [] }, + "time": { "from": "now-3h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Sientia DataOps Model Manager", + "uid": "sientia-dataops-model-manager", + "version": 1 +} diff --git a/docs/DB_CV022_WIT230 _double_date_column.csv b/docs/DB_CV022_WIT230 _double_date_column.csv new file mode 100644 index 0000000..452543b --- /dev/null +++ b/docs/DB_CV022_WIT230 _double_date_column.csv @@ -0,0 +1,33 @@ +DATA,DATE2,03CV022/CORRENTE_N_M1_PV(Value),303-WIT-230(Value) +01/05/2022 00:00:00,07-01-2022 01:00:00,170,33 +01/05/2022 00:00:10,07-01-2022 01:00:10,169,605 +01/05/2022 00:00:20,07-01-2022 01:00:20,169,178 +01/05/2022 00:00:30,07-01-2022 01:00:30,166,468 +01/05/2022 00:00:40,07-01-2022 01:00:40,162,136 +01/05/2022 00:00:50,07-01-2022 01:00:50,157,804 +01/05/2022 00:01:00,07-01-2022 01:01:00,155,883 +01/05/2022 00:01:10,07-01-2022 01:01:10,155,684 +01/05/2022 00:01:20,07-01-2022 01:01:20,155,484 +01/05/2022 00:01:30,07-01-2022 01:01:30,155,284 +01/05/2022 00:01:40,07-01-2022 01:01:40,155,85 +01/05/2022 00:01:50,07-01-2022 01:01:50,154,885 +01/05/2022 00:02:00,09-01-2022 02:02:00,154,685 +01/05/2022 00:02:10,09-01-2022 02:02:10,154,486 +01/05/2022 00:02:20,09-01-2022 02:02:20,154,286 +01/05/2022 00:02:30,09-01-2022 02:02:30,154,86 +01/05/2022 00:02:40,09-01-2022 02:02:40,152,866 +01/05/2022 00:02:50,09-01-2022 02:02:50,150,87 +01/05/2022 00:03:00,09-01-2022 02:03:00,148,874 +01/05/2022 00:03:10,09-01-2022 02:03:10,148,2134 +01/05/2022 00:03:20,09-01-2022 02:03:20,148,2068 +01/05/2022 00:03:30,09-01-2022 02:03:30,148,2022 +01/05/2022 00:03:40,09-01-2022 02:03:40,148,1976 +01/05/2022 00:03:50,09-01-2022 02:03:50,154,139 +01/05/2022 00:04:00,11-01-2022 03:04:00,165,112 +01/05/2022 00:04:10,11-01-2022 03:04:10,170,42 +01/05/2022 00:04:20,11-01-2022 03:04:20,170,116 +01/05/2022 00:04:30,11-01-2022 03:04:30,170,191 +01/05/2022 00:04:40,11-01-2022 03:04:40,170,266 +01/05/2022 00:04:50,11-01-2022 03:04:50,170,341 +01/05/2022 00:05:00,11-01-2022 03:05:00,170,416 +01/05/2022 00:05:10,11-01-2022 03:05:10,170,491 \ No newline at end of file diff --git a/docs/DB_CV022_WIT230.csv b/docs/DB_CV022_WIT230.csv new file mode 100644 index 0000000..b8a886e --- /dev/null +++ b/docs/DB_CV022_WIT230.csv @@ -0,0 +1,647 @@ +DATA,03CV022/CORRENTE_N_M1_PV(Value),303-WIT-230(Value) +01/05/2022 00:00:00,170,33 +01/05/2022 00:00:10,169,605 +01/05/2022 00:00:20,169,178 +01/05/2022 00:00:30,166,468 +01/05/2022 00:00:40,162,136 +01/05/2022 00:00:50,157,804 +01/05/2022 00:01:00,155,883 +01/05/2022 00:01:10,155,684 +01/05/2022 00:01:20,155,484 +01/05/2022 00:01:30,155,284 +01/05/2022 00:01:40,155,85 +01/05/2022 00:01:50,154,885 +01/05/2022 00:02:00,154,685 +01/05/2022 00:02:10,154,486 +01/05/2022 00:02:20,154,286 +01/05/2022 00:02:30,154,86 +01/05/2022 00:02:40,152,866 +01/05/2022 00:02:50,150,87 +01/05/2022 00:03:00,148,874 +01/05/2022 00:03:10,148,2134 +01/05/2022 00:03:20,148,2068 +01/05/2022 00:03:30,148,2022 +01/05/2022 00:03:40,148,1976 +01/05/2022 00:03:50,154,139 +01/05/2022 00:04:00,165,112 +01/05/2022 00:04:10,170,42 +01/05/2022 00:04:20,170,116 +01/05/2022 00:04:30,170,191 +01/05/2022 00:04:40,170,266 +01/05/2022 00:04:50,170,341 +01/05/2022 00:05:00,170,416 +01/05/2022 00:05:10,170,491 +01/05/2022 00:05:20,170,566 +01/05/2022 00:05:30,170,641 +01/05/2022 00:05:40,170,716 +01/05/2022 00:05:50,170,79 +01/05/2022 00:06:00,170,865 +01/05/2022 00:06:10,170,94 +01/05/2022 00:06:20,171,15 +01/05/2022 00:06:30,171,9 +01/05/2022 00:06:40,171,165 +01/05/2022 00:06:50,171,24 +01/05/2022 00:07:00,171,315 +01/05/2022 00:07:10,171,39 +01/05/2022 00:07:20,171,465 +01/05/2022 00:07:30,171,539 +01/05/2022 00:07:40,171,614 +01/05/2022 00:07:50,171,689 +01/05/2022 00:08:00,171,764 +01/05/2022 00:08:10,171,839 +01/05/2022 00:08:20,171,914 +01/05/2022 00:08:30,171,989 +01/05/2022 00:08:40,172,64 +01/05/2022 00:08:50,172,139 +01/05/2022 00:09:00,172,214 +01/05/2022 00:09:10,172,288 +01/05/2022 00:09:20,172,363 +01/05/2022 00:09:30,172,438 +01/05/2022 00:09:40,172,513 +01/05/2022 00:09:50,172,588 +01/05/2022 00:10:00,172,663 +01/05/2022 00:10:10,172,738 +01/05/2022 00:10:20,172,813 +01/05/2022 00:10:30,172,888 +01/05/2022 00:10:40,172,962 +01/05/2022 00:10:50,172,981 +01/05/2022 00:11:00,172,943 +01/05/2022 00:11:10,172,905 +01/05/2022 00:11:20,172,867 +01/05/2022 00:11:30,172,829 +01/05/2022 00:11:40,172,79 +01/05/2022 00:11:50,172,752 +01/05/2022 00:12:00,172,714 +01/05/2022 00:12:10,172,676 +01/05/2022 00:12:20,172,638 +01/05/2022 00:12:30,172,6 +01/05/2022 00:12:40,172,562 +01/05/2022 00:12:50,172,524 +01/05/2022 00:13:00,172,485 +01/05/2022 00:13:10,172,447 +01/05/2022 00:13:20,172,409 +01/05/2022 00:13:30,172,371 +01/05/2022 00:13:40,172,333 +01/05/2022 00:13:50,172,295 +01/05/2022 00:14:00,172,257 +01/05/2022 00:14:10,172,219 +01/05/2022 00:14:20,172,18 +01/05/2022 00:14:30,172,142 +01/05/2022 00:14:40,172,104 +01/05/2022 00:14:50,172,66 +01/05/2022 00:15:00,172,28 +01/05/2022 00:15:10,171,99 +01/05/2022 00:15:20,171,952 +01/05/2022 00:15:30,171,914 +01/05/2022 00:15:40,171,876 +01/05/2022 00:15:50,171,837 +01/05/2022 00:16:00,171,799 +01/05/2022 00:16:10,171,761 +01/05/2022 00:16:20,171,723 +01/05/2022 00:16:30,171,685 +01/05/2022 00:16:40,171,647 +01/05/2022 00:16:50,171,609 +01/05/2022 00:17:00,171,571 +01/05/2022 00:17:10,171,533 +01/05/2022 00:17:20,171,494 +01/05/2022 00:17:30,171,456 +01/05/2022 00:17:40,171,418 +01/05/2022 00:17:50,171,38 +01/05/2022 00:18:00,171,342 +01/05/2022 00:18:10,171,304 +01/05/2022 00:18:20,171,266 +01/05/2022 00:18:30,171,228 +01/05/2022 00:18:40,171,189 +01/05/2022 00:18:50,171,151 +01/05/2022 00:19:00,171,113 +01/05/2022 00:19:10,171,75 +01/05/2022 00:19:20,171,37 +01/05/2022 00:19:30,170,999 +01/05/2022 00:19:40,170,961 +01/05/2022 00:19:50,170,923 +01/05/2022 00:20:00,170,884 +01/05/2022 00:20:10,170,846 +01/05/2022 00:20:20,170,808 +01/05/2022 00:20:30,170,77 +01/05/2022 00:20:40,170,732 +01/05/2022 00:20:50,170,694 +01/05/2022 00:21:00,170,656 +01/05/2022 00:21:10,170,618 +01/05/2022 00:21:20,170,58 +01/05/2022 00:21:30,170,541 +01/05/2022 00:21:40,170,503 +01/05/2022 00:21:50,170,465 +01/05/2022 00:22:00,170,427 +01/05/2022 00:22:10,170,389 +01/05/2022 00:22:20,170,351 +01/05/2022 00:22:30,170,313 +01/05/2022 00:22:40,170,275 +01/05/2022 00:22:50,170,236 +01/05/2022 00:23:00,170,198 +01/05/2022 00:23:10,170,16 +01/05/2022 00:23:20,170,122 +01/05/2022 00:23:30,170,84 +01/05/2022 00:23:40,170,46 +01/05/2022 00:23:50,170,8 +01/05/2022 00:24:00,169,97 +01/05/2022 00:24:10,169,932 +01/05/2022 00:24:20,169,893 +01/05/2022 00:24:30,169,855 +01/05/2022 00:24:40,169,817 +01/05/2022 00:24:50,169,779 +01/05/2022 00:25:00,169,741 +01/05/2022 00:25:10,169,703 +01/05/2022 00:25:20,169,665 +01/05/2022 00:25:30,169,627 +01/05/2022 00:25:40,169,588 +01/05/2022 00:25:50,169,55 +01/05/2022 00:26:00,169,512 +01/05/2022 00:26:10,169,474 +01/05/2022 00:26:20,169,436 +01/05/2022 00:26:30,169,398 +01/05/2022 00:26:40,169,36 +01/05/2022 00:26:50,169,322 +01/05/2022 00:27:00,169,284 +01/05/2022 00:27:10,169,245 +01/05/2022 00:27:20,169,207 +01/05/2022 00:27:30,169,169 +01/05/2022 00:27:40,169,131 +01/05/2022 00:27:50,169,93 +01/05/2022 00:28:00,169,55 +01/05/2022 00:28:10,169,17 +01/05/2022 00:28:20,168,979 +01/05/2022 00:28:30,168,94 +01/05/2022 00:28:40,168,902 +01/05/2022 00:28:50,168,864 +01/05/2022 00:29:00,168,826 +01/05/2022 00:29:10,168,788 +01/05/2022 00:29:20,168,75 +01/05/2022 00:29:30,168,712 +01/05/2022 00:29:40,168,674 +01/05/2022 00:29:50,168,636 +01/05/2022 00:30:00,168,597 +01/05/2022 00:30:10,168,559 +01/05/2022 00:30:20,168,521 +01/05/2022 00:30:30,168,483 +01/05/2022 00:30:40,168,445 +01/05/2022 00:30:50,168,407 +01/05/2022 00:31:00,168,369 +01/05/2022 00:31:10,168,331 +01/05/2022 00:31:20,168,292 +01/05/2022 00:31:30,168,254 +01/05/2022 00:31:40,168,216 +01/05/2022 00:31:50,168,178 +01/05/2022 00:32:00,168,14 +01/05/2022 00:32:10,168,102 +01/05/2022 00:32:20,168,64 +01/05/2022 00:32:30,168,26 +01/05/2022 00:32:40,168,26 +01/05/2022 00:32:50,168,105 +01/05/2022 00:33:00,168,183 +01/05/2022 00:33:10,168,262 +01/05/2022 00:33:20,168,341 +01/05/2022 00:33:30,168,42 +01/05/2022 00:33:40,168,499 +01/05/2022 00:33:50,168,578 +01/05/2022 00:34:00,168,657 +01/05/2022 00:34:10,168,735 +01/05/2022 00:34:20,168,814 +01/05/2022 00:34:30,168,893 +01/05/2022 00:34:40,168,972 +01/05/2022 00:34:50,169,51 +01/05/2022 00:35:00,169,13 +01/05/2022 00:35:10,169,209 +01/05/2022 00:35:20,169,287 +01/05/2022 00:35:30,169,366 +01/05/2022 00:35:40,169,445 +01/05/2022 00:35:50,169,524 +01/05/2022 00:36:00,169,603 +01/05/2022 00:36:10,169,682 +01/05/2022 00:36:20,169,761 +01/05/2022 00:36:30,169,839 +01/05/2022 00:36:40,169,918 +01/05/2022 00:36:50,169,997 +01/05/2022 00:37:00,170,76 +01/05/2022 00:37:10,170,155 +01/05/2022 00:37:20,170,234 +01/05/2022 00:37:30,170,312 +01/05/2022 00:37:40,170,391 +01/05/2022 00:37:50,170,47 +01/05/2022 00:38:00,170,549 +01/05/2022 00:38:10,170,628 +01/05/2022 00:38:20,170,707 +01/05/2022 00:38:30,170,786 +01/05/2022 00:38:40,170,864 +01/05/2022 00:38:50,170,943 +01/05/2022 00:39:00,170,993 +01/05/2022 00:39:10,170,967 +01/05/2022 00:39:20,170,942 +01/05/2022 00:39:30,170,917 +01/05/2022 00:39:40,170,891 +01/05/2022 00:39:50,170,866 +01/05/2022 00:40:00,170,841 +01/05/2022 00:40:10,170,815 +01/05/2022 00:40:20,170,79 +01/05/2022 00:40:30,170,764 +01/05/2022 00:40:40,170,739 +01/05/2022 00:40:50,170,714 +01/05/2022 00:41:00,170,688 +01/05/2022 00:41:10,170,663 +01/05/2022 00:41:20,170,637 +01/05/2022 00:41:30,170,612 +01/05/2022 00:41:40,170,587 +01/05/2022 00:41:50,170,561 +01/05/2022 00:42:00,170,536 +01/05/2022 00:42:10,170,51 +01/05/2022 00:42:20,170,485 +01/05/2022 00:42:30,170,46 +01/05/2022 00:42:40,170,434 +01/05/2022 00:42:50,170,409 +01/05/2022 00:43:00,170,384 +01/05/2022 00:43:10,170,358 +01/05/2022 00:43:20,170,333 +01/05/2022 00:43:30,170,307 +01/05/2022 00:43:40,170,282 +01/05/2022 00:43:50,170,257 +01/05/2022 00:44:00,170,231 +01/05/2022 00:44:10,170,206 +01/05/2022 00:44:20,170,18 +01/05/2022 00:44:30,170,155 +01/05/2022 00:44:40,170,13 +01/05/2022 00:44:50,170,104 +01/05/2022 00:45:00,170,79 +01/05/2022 00:45:10,170,54 +01/05/2022 00:45:20,170,28 +01/05/2022 00:45:30,170,3 +01/05/2022 00:45:40,169,977 +01/05/2022 00:45:50,169,952 +01/05/2022 00:46:00,169,927 +01/05/2022 00:46:10,169,901 +01/05/2022 00:46:20,169,876 +01/05/2022 00:46:30,169,85 +01/05/2022 00:46:40,169,825 +01/05/2022 00:46:50,169,8 +01/05/2022 00:47:00,169,774 +01/05/2022 00:47:10,169,749 +01/05/2022 00:47:20,169,723 +01/05/2022 00:47:30,169,698 +01/05/2022 00:47:40,169,673 +01/05/2022 00:47:50,169,647 +01/05/2022 00:48:00,169,622 +01/05/2022 00:48:10,169,597 +01/05/2022 00:48:20,169,571 +01/05/2022 00:48:30,169,546 +01/05/2022 00:48:40,169,52 +01/05/2022 00:48:50,169,495 +01/05/2022 00:49:00,169,47 +01/05/2022 00:49:10,169,444 +01/05/2022 00:49:20,169,419 +01/05/2022 00:49:30,169,393 +01/05/2022 00:49:40,169,368 +01/05/2022 00:49:50,169,343 +01/05/2022 00:50:00,169,317 +01/05/2022 00:50:10,169,292 +01/05/2022 00:50:20,169,266 +01/05/2022 00:50:30,169,241 +01/05/2022 00:50:40,169,216 +01/05/2022 00:50:50,169,19 +01/05/2022 00:51:00,169,165 +01/05/2022 00:51:10,169,14 +01/05/2022 00:51:20,169,114 +01/05/2022 00:51:30,169,89 +01/05/2022 00:51:40,169,63 +01/05/2022 00:51:50,169,38 +01/05/2022 00:52:00,169,13 +01/05/2022 00:52:10,168,987 +01/05/2022 00:52:20,168,962 +01/05/2022 00:52:30,168,936 +01/05/2022 00:52:40,168,911 +01/05/2022 00:52:50,168,886 +01/05/2022 00:53:00,168,86 +01/05/2022 00:53:10,168,835 +01/05/2022 00:53:20,168,809 +01/05/2022 00:53:30,168,784 +01/05/2022 00:53:40,168,759 +01/05/2022 00:53:50,168,733 +01/05/2022 00:54:00,168,708 +01/05/2022 00:54:10,168,683 +01/05/2022 00:54:20,168,657 +01/05/2022 00:54:30,168,632 +01/05/2022 00:54:40,168,606 +01/05/2022 00:54:50,168,581 +01/05/2022 00:55:00,168,556 +01/05/2022 00:55:10,168,53 +01/05/2022 00:55:20,168,505 +01/05/2022 00:55:30,168,479 +01/05/2022 00:55:40,168,454 +01/05/2022 00:55:50,168,429 +01/05/2022 00:56:00,168,403 +01/05/2022 00:56:10,168,378 +01/05/2022 00:56:20,168,352 +01/05/2022 00:56:30,168,327 +01/05/2022 00:56:40,168,302 +01/05/2022 00:56:50,168,276 +01/05/2022 00:57:00,168,251 +01/05/2022 00:57:10,168,226 +01/05/2022 00:57:20,168,2 +01/05/2022 00:57:30,168,175 +01/05/2022 00:57:40,168,149 +01/05/2022 00:57:50,168,124 +01/05/2022 00:58:00,168,99 +01/05/2022 00:58:10,168,73 +01/05/2022 00:58:20,168,48 +01/05/2022 00:58:30,168,22 +01/05/2022 00:58:40,167,76 +01/05/2022 00:58:50,160,151 +01/05/2022 00:59:00,161,484 +01/05/2022 00:59:10,162,817 +01/05/2022 00:59:20,164,281 +01/05/2022 00:59:30,166,777 +01/05/2022 00:59:40,167,148 +01/05/2022 00:59:50,148,45 +01/05/2022 01:00:00,117,525 +01/05/2022 01:00:10,105,3 +01/05/2022 01:00:20,105,315 +01/05/2022 01:00:30,105,6 +01/05/2022 01:00:40,105,886 +01/05/2022 01:00:50,106,171 +01/05/2022 01:01:00,106,456 +01/05/2022 01:01:10,106,742 +01/05/2022 01:01:20,107,698 +01/05/2022 01:01:30,115,81 +01/05/2022 01:01:40,122,464 +01/05/2022 01:01:50,129,847 +01/05/2022 01:02:00,137,231 +01/05/2022 01:02:10,144,138 +01/05/2022 01:02:20,145,801 +01/05/2022 01:02:30,147,464 +01/05/2022 01:02:40,149,835 +01/05/2022 01:02:50,160,808 +01/05/2022 01:03:00,171,2 +01/05/2022 01:03:10,171,36 +01/05/2022 01:03:20,171,7 +01/05/2022 01:03:30,171,103 +01/05/2022 01:03:40,171,137 +01/05/2022 01:03:50,171,171 +01/05/2022 01:04:00,171,204 +01/05/2022 01:04:10,171,238 +01/05/2022 01:04:20,171,272 +01/05/2022 01:04:30,171,305 +01/05/2022 01:04:40,171,339 +01/05/2022 01:04:50,171,372 +01/05/2022 01:05:00,171,406 +01/05/2022 01:05:10,171,44 +01/05/2022 01:05:20,171,473 +01/05/2022 01:05:30,171,507 +01/05/2022 01:05:40,171,541 +01/05/2022 01:05:50,171,574 +01/05/2022 01:06:00,171,608 +01/05/2022 01:06:10,171,642 +01/05/2022 01:06:20,171,675 +01/05/2022 01:06:30,171,709 +01/05/2022 01:06:40,171,743 +01/05/2022 01:06:50,171,776 +01/05/2022 01:07:00,171,81 +01/05/2022 01:07:10,171,844 +01/05/2022 01:07:20,171,877 +01/05/2022 01:07:30,171,911 +01/05/2022 01:07:40,171,944 +01/05/2022 01:07:50,171,978 +01/05/2022 01:08:00,172,12 +01/05/2022 01:08:10,172,45 +01/05/2022 01:08:20,172,79 +01/05/2022 01:08:30,172,113 +01/05/2022 01:08:40,172,146 +01/05/2022 01:08:50,172,18 +01/05/2022 01:09:00,172,214 +01/05/2022 01:09:10,172,247 +01/05/2022 01:09:20,172,281 +01/05/2022 01:09:30,172,315 +01/05/2022 01:09:40,172,348 +01/05/2022 01:09:50,172,382 +01/05/2022 01:10:00,172,415 +01/05/2022 01:10:10,172,449 +01/05/2022 01:10:20,172,483 +01/05/2022 01:10:30,172,516 +01/05/2022 01:10:40,172,55 +01/05/2022 01:10:50,172,584 +01/05/2022 01:11:00,172,617 +01/05/2022 01:11:10,172,651 +01/05/2022 01:11:20,172,685 +01/05/2022 01:11:30,172,718 +01/05/2022 01:11:40,172,752 +01/05/2022 01:11:50,172,786 +01/05/2022 01:12:00,172,819 +01/05/2022 01:12:10,172,853 +01/05/2022 01:12:20,172,887 +01/05/2022 01:12:30,172,92 +01/05/2022 01:12:40,172,954 +01/05/2022 01:12:50,172,987 +01/05/2022 01:13:00,173,21 +01/05/2022 01:13:10,173,55 +01/05/2022 01:13:20,173,88 +01/05/2022 01:13:30,173,122 +01/05/2022 01:13:40,173,156 +01/05/2022 01:13:50,173,189 +01/05/2022 01:14:00,173,223 +01/05/2022 01:14:10,173,257 +01/05/2022 01:14:20,173,29 +01/05/2022 01:14:30,173,324 +01/05/2022 01:14:40,173,358 +01/05/2022 01:14:50,173,391 +01/05/2022 01:15:00,173,425 +01/05/2022 01:15:10,173,458 +01/05/2022 01:15:20,173,492 +01/05/2022 01:15:30,173,526 +01/05/2022 01:15:40,173,559 +01/05/2022 01:15:50,173,593 +01/05/2022 01:16:00,173,627 +01/05/2022 01:16:10,173,66 +01/05/2022 01:16:20,173,694 +01/05/2022 01:16:30,173,728 +01/05/2022 01:16:40,173,761 +01/05/2022 01:16:50,173,795 +01/05/2022 01:17:00,173,829 +01/05/2022 01:17:10,173,862 +01/05/2022 01:17:20,173,896 +01/05/2022 01:17:30,173,93 +01/05/2022 01:17:40,173,963 +01/05/2022 01:17:50,173,997 +01/05/2022 01:18:00,174,103 +01/05/2022 01:18:10,174,218 +01/05/2022 01:18:20,174,332 +01/05/2022 01:18:30,174,446 +01/05/2022 01:18:40,174,56 +01/05/2022 01:18:50,174,674 +01/05/2022 01:19:00,174,788 +01/05/2022 01:19:10,174,902 +01/05/2022 01:19:20,175,17 +01/05/2022 01:19:30,175,131 +01/05/2022 01:19:40,175,245 +01/05/2022 01:19:50,175,359 +01/05/2022 01:20:00,175,473 +01/05/2022 01:20:10,175,587 +01/05/2022 01:20:20,175,701 +01/05/2022 01:20:30,175,816 +01/05/2022 01:20:40,175,93 +01/05/2022 01:20:50,176,44 +01/05/2022 01:21:00,176,158 +01/05/2022 01:21:10,176,272 +01/05/2022 01:21:20,176,386 +01/05/2022 01:21:30,176,501 +01/05/2022 01:21:40,176,615 +01/05/2022 01:21:50,176,729 +01/05/2022 01:22:00,176,843 +01/05/2022 01:22:10,176,957 +01/05/2022 01:22:20,177,71 +01/05/2022 01:22:30,177,185 +01/05/2022 01:22:40,177,3 +01/05/2022 01:22:50,177,414 +01/05/2022 01:23:00,177,528 +01/05/2022 01:23:10,177,642 +01/05/2022 01:23:20,177,756 +01/05/2022 01:23:30,177,87 +01/05/2022 01:23:40,177,984 +01/05/2022 01:23:50,178,7 +01/05/2022 01:24:00,178,15 +01/05/2022 01:24:10,178,24 +01/05/2022 01:24:20,178,32 +01/05/2022 01:24:30,178,4 +01/05/2022 01:24:40,178,48 +01/05/2022 01:24:50,178,57 +01/05/2022 01:25:00,178,65 +01/05/2022 01:25:10,178,73 +01/05/2022 01:25:20,178,81 +01/05/2022 01:25:30,178,9 +01/05/2022 01:25:40,178,98 +01/05/2022 01:25:50,178,106 +01/05/2022 01:26:00,178,114 +01/05/2022 01:26:10,178,123 +01/05/2022 01:26:20,178,131 +01/05/2022 01:26:30,178,139 +01/05/2022 01:26:40,178,147 +01/05/2022 01:26:50,178,156 +01/05/2022 01:27:00,178,164 +01/05/2022 01:27:10,178,172 +01/05/2022 01:27:20,178,18 +01/05/2022 01:27:30,178,189 +01/05/2022 01:27:40,178,197 +01/05/2022 01:27:50,178,205 +01/05/2022 01:28:00,178,213 +01/05/2022 01:28:10,178,222 +01/05/2022 01:28:20,178,23 +01/05/2022 01:28:30,178,238 +01/05/2022 01:28:40,178,246 +01/05/2022 01:28:50,178,255 +01/05/2022 01:29:00,178,263 +01/05/2022 01:29:10,178,271 +01/05/2022 01:29:20,178,279 +01/05/2022 01:29:30,178,288 +01/05/2022 01:29:40,178,296 +01/05/2022 01:29:50,178,304 +01/05/2022 01:30:00,178,312 +01/05/2022 01:30:10,178,321 +01/05/2022 01:30:20,178,329 +01/05/2022 01:30:30,178,337 +01/05/2022 01:30:40,178,345 +01/05/2022 01:30:50,178,354 +01/05/2022 01:31:00,178,362 +01/05/2022 01:31:10,178,37 +01/05/2022 01:31:20,178,378 +01/05/2022 01:31:30,178,387 +01/05/2022 01:31:40,178,395 +01/05/2022 01:31:50,178,403 +01/05/2022 01:32:00,178,411 +01/05/2022 01:32:10,178,42 +01/05/2022 01:32:20,178,428 +01/05/2022 01:32:30,178,436 +01/05/2022 01:32:40,178,444 +01/05/2022 01:32:50,178,453 +01/05/2022 01:33:00,178,461 +01/05/2022 01:33:10,178,469 +01/05/2022 01:33:20,178,477 +01/05/2022 01:33:30,178,486 +01/05/2022 01:33:40,178,494 +01/05/2022 01:33:50,178,502 +01/05/2022 01:34:00,178,51 +01/05/2022 01:34:10,178,519 +01/05/2022 01:34:20,178,527 +01/05/2022 01:34:30,178,535 +01/05/2022 01:34:40,178,543 +01/05/2022 01:34:50,178,552 +01/05/2022 01:35:00,178,56 +01/05/2022 01:35:10,178,568 +01/05/2022 01:35:20,178,576 +01/05/2022 01:35:30,178,585 +01/05/2022 01:35:40,178,593 +01/05/2022 01:35:50,178,601 +01/05/2022 01:36:00,178,609 +01/05/2022 01:36:10,178,618 +01/05/2022 01:36:20,178,626 +01/05/2022 01:36:30,178,634 +01/05/2022 01:36:40,178,642 +01/05/2022 01:36:50,178,651 +01/05/2022 01:37:00,178,659 +01/05/2022 01:37:10,178,667 +01/05/2022 01:37:20,178,675 +01/05/2022 01:37:30,178,684 +01/05/2022 01:37:40,178,692 +01/05/2022 01:37:50,178,7 +01/05/2022 01:38:00,178,708 +01/05/2022 01:38:10,178,717 +01/05/2022 01:38:20,178,725 +01/05/2022 01:38:30,178,733 +01/05/2022 01:38:40,178,741 +01/05/2022 01:38:50,178,75 +01/05/2022 01:39:00,178,758 +01/05/2022 01:39:10,178,766 +01/05/2022 01:39:20,178,774 +01/05/2022 01:39:30,178,783 +01/05/2022 01:39:40,178,791 +01/05/2022 01:39:50,178,799 +01/05/2022 01:40:00,178,807 +01/05/2022 01:40:10,178,816 +01/05/2022 01:40:20,178,824 +01/05/2022 01:40:30,178,832 +01/05/2022 01:40:40,178,84 +01/05/2022 01:40:50,178,849 +01/05/2022 01:41:00,178,857 +01/05/2022 01:41:10,178,865 +01/05/2022 01:41:20,178,873 +01/05/2022 01:41:30,178,882 +01/05/2022 01:41:40,178,89 +01/05/2022 01:41:50,178,898 +01/05/2022 01:42:00,178,906 +01/05/2022 01:42:10,178,915 +01/05/2022 01:42:20,178,923 +01/05/2022 01:42:30,178,931 +01/05/2022 01:42:40,178,939 +01/05/2022 01:42:50,178,948 +01/05/2022 01:43:00,178,956 +01/05/2022 01:43:10,178,964 +01/05/2022 01:43:20,178,972 +01/05/2022 01:43:30,178,981 +01/05/2022 01:43:40,178,989 +01/05/2022 01:43:50,178,997 +01/05/2022 01:44:00,169,147 +01/05/2022 01:44:10,148,934 +01/05/2022 01:44:20,119,458 +01/05/2022 01:44:30,108,326 +01/05/2022 01:44:40,108,825 +01/05/2022 01:44:50,109,324 +01/05/2022 01:45:00,109,824 +01/05/2022 01:45:10,110,323 +01/05/2022 01:45:20,110,822 +01/05/2022 01:45:30,123,3 +01/05/2022 01:45:40,141,628 +01/05/2022 01:45:50,160,252 +01/05/2022 01:46:00,169,863 +01/05/2022 01:46:10,174,353 +01/05/2022 01:46:20,176,8 +01/05/2022 01:46:30,176,19 +01/05/2022 01:46:40,176,31 +01/05/2022 01:46:50,176,43 +01/05/2022 01:47:00,176,55 +01/05/2022 01:47:10,176,67 +01/05/2022 01:47:20,176,79 +01/05/2022 01:47:30,176,91 \ No newline at end of file diff --git a/docs/scenarios.md b/docs/scenarios.md new file mode 100644 index 0000000..1251097 --- /dev/null +++ b/docs/scenarios.md @@ -0,0 +1,66 @@ +# E2E Test Scenarios + +This document maps the workflow scenarios tested in the E2E suite to their corresponding JSON input files and expected behaviors. + +## Infrastructure (second-pass review) + +- **Containers:** PostgreSQL, MinIO, MongoDB, and Gitea via testcontainers; real clients and `Activities` code paths. +- **MLflow:** `file://` tracking URI (real SDK, no remote server). +- **Temporal:** `WorkflowEnvironment.start_time_skipping()` — official temporalio test runtime; workflows and activities are not stubbed. +- **Logging/metrics:** `get_logger` + `MetricsController` (sientia_do); no `unittest.mock` for observability in `e2e/conftest.py`. +- **Unit tests** under `tests/` may still use mocks where appropriate; that policy is separate from this E2E suite. + +## 1. TrainModel Workflow (`test_train_model_workflow.py`) + +### 1.1 Happy Paths (Successful execution) + +| Test Function | Input JSON | Expected Status | Description | +|---|---|---|---| +| `test_scenario_1_1_1_linear_regression_basic` | `01-linear-regression-basic.json` | `TRAINING_SUCCESS` | Basic linear regression without scaler. Verifies end-to-end pipeline. | +| `test_scenario_1_1_2_linear_regression_with_scaler` | `02-linear-regression-with-scaler.json` | `TRAINING_SUCCESS` | Linear regression with `Standard Scaler`. | +| `test_scenario_1_1_3_polynomial_regression_degree2_with_scaler` | `03-polynomial-regression-degree2.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 2) with Standard Scaler. | +| `test_scenario_1_1_4_polynomial_regression_degree3_with_scaler` | `04-polynomial-regression-degree3.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 3) with Standard Scaler. | +| `test_scenario_1_1_5_linear_regression_with_lags` | `05-linear-regression-with-lags.json` | `TRAINING_SUCCESS` | Linear regression with `lag_train`/`lag_val` per variable. | +| `test_scenario_1_1_6_linear_regression_nan_interpolation` | `06-linear-regression-nan-interpolation.json` | `TRAINING_SUCCESS` | Linear regression with `nan_treatment='linear interpolation'`. | +| `test_scenario_1_1_7_linear_regression_static_window_removal` | `07-linear-regression-static-window-removal.json` | `TRAINING_SUCCESS` | `rem_static_win=true` with default `static_threshold`. | +| `test_scenario_1_1_8_linear_regression_with_limits` | `08-linear-regression-with-limits.json` | `TRAINING_SUCCESS` | `support_filters` with `min`/`max` per variable. | +| `test_scenario_1_1_9_polynomial_degree2_scaler_and_lags` | `09-polynomial-degree2-with-scaler-and-lags.json` | `TRAINING_SUCCESS` | Polynomial (degree 2), Standard Scaler, and lags. | +| `test_scenario_1_1_10_linear_regression_with_ar_opt_params` | `10-linear-regression-with-ar.json` | `TRAINING_SUCCESS` | `opt_params.include_ar=true` (placeholder for future AR behavior). | +| `test_scenario_1_1_11_linear_regression_static_threshold_custom` | `11-linear-regression-static-threshold-custom.json` | `TRAINING_SUCCESS` | `rem_static_win=true` with custom `static_threshold`. | +| `test_scenario_1_1_12_alternate_date_format_dd_mm_yyyy` | `12-angular-test-date-format.json` | `TRAINING_SUCCESS` | `date_column=DATA`, `dd/MM/yyyy` format, object `training_data_dd_mm_yyyy.csv`. | +| `test_scenario_1_1_13_alternate_csv_narrow_date_window` | `13-angular-test-double-date-column.json` | `TRAINING_SUCCESS` | Same alternate CSV with a bounded `start_date`/`end_date` window. | +| `test_scenario_1_1_14_polynomial_with_support_filters` | `14-angular-test-polynomial-support-filters.json` | `TRAINING_SUCCESS` | Polynomial (degree 4), scaler, `upper_line`/`lower_line` support filters. | +| `test_scenario_1_1_15_linear_regression_custom_target_column_name` | `15-linear-regression-custom-target-column.json` | `TRAINING_SUCCESS` | Custom `target_variable` column name (not literal ``target``); Evidently/report columns must match. | +| `test_scenario_1_1_16_naive_timestamp_header_column` | `16-linear-regression-naive-timestamp-header.json` | `TRAINING_SUCCESS` | `date_column`=`Timestamp`, naive CSV `training_data_timestamp_naive.csv`. | +| `test_scenario_1_1_17_linear_regression_blank_timestamp_row_dropped` | `17-linear-regression-blank-timestamp-row.json` | `TRAINING_SUCCESS` | One empty timestamp cell; row dropped before index. | + +### 1.2 Error Paths + +| Test Function | Input JSON | Expected Status | Description | +|---|---|---|---| +| `test_scenario_1_2_1_minio_file_not_found` | `01-linear-regression-basic.json` | `TRAINING_ERROR` | MinIO file does not exist. Workflow fails during file download. | +| `test_scenario_1_2_2_experiment_run_id_not_in_db` | `01-linear-regression-basic.json` | N/A (raises Exception) | `experiment_run_id` does not exist in DB. Workflow fails immediately on status update attempt. | + +## 2. Parameter Validation (`test_train_model_validation.py`) + +These scenarios test the business rule validations inside `validate_train_params`. All are expected to terminate with `ORCHESTRATOR_VALIDATION_ERROR`. + +| Test Function | Modification | Expected Error Substring | +|---|---|---| +| `test_scenario_2_1_1_train_size_out_of_range` | `train_size = 5` | `'train_size'` | +| `test_scenario_2_1_2_empty_variable_columns` | `variable_columns = []` | `'variable_columns'` | +| `test_scenario_2_1_3_invalid_date_format` | `date_format = 'INVALID'` | `'date_format'` | +| `test_scenario_2_1_4_whitespace_only_model_name` | `model_name = ' '` | `'model_name'` | +| `test_scenario_2_1_5_unknown_model_type` | `model_type = 'totally_unknown_model'` | `'totally_unknown_model'` | +| `test_scenario_2_1_6_missing_target_variable` | `target_variable = ''` | `'target_variable'` | +| `test_scenario_2_1_7_missing_experiment_run_id` | Missing `experiment_run_id` | N/A (raises ValueError immediately) | +| `test_scenario_2_1_8_missing_date_column` | Missing `date_column` | N/A (raises ValueError immediately) | +| `test_scenario_2_1_9_whitespace_date_column` | `date_column = ' '` | `'date_column'` | + +## 3. CleanupFiles Workflow (`test_cleanup_files_workflow.py`) + +| Test Function | Description | +|---|---| +| `test_scenario_3_1_1_cleanup_with_no_temp_dirs` | Temp directory is empty. Activity completes without error. | +| `test_scenario_3_1_2_cleanup_removes_old_temp_dirs` | Two stale directories matching `name_YYYYMMDD_HHMMSS_microseconds` are removed when older than retention. | +| `test_scenario_3_1_3_cleanup_nonexistent_temp_path` | Target path does not exist. Handled gracefully without error. | diff --git a/docs/test-scenarios/01-linear-regression-basic.json b/docs/test-scenarios/01-linear-regression-basic.json new file mode 100644 index 0000000..28ff007 --- /dev/null +++ b/docs/test-scenarios/01-linear-regression-basic.json @@ -0,0 +1,40 @@ +{ + "_description": "Cenário básico de regressão linear sem scaler", + "experiment_run_id": 1001, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/02-linear-regression-with-scaler.json b/docs/test-scenarios/02-linear-regression-with-scaler.json new file mode 100644 index 0000000..1c02fb5 --- /dev/null +++ b/docs/test-scenarios/02-linear-regression-with-scaler.json @@ -0,0 +1,40 @@ +{ + "_description": "Regressão linear com Standard Scaler habilitado", + "experiment_run_id": 1002, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "Standard Scaler" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/03-polynomial-regression-degree2.json b/docs/test-scenarios/03-polynomial-regression-degree2.json new file mode 100644 index 0000000..050c44f --- /dev/null +++ b/docs/test-scenarios/03-polynomial-regression-degree2.json @@ -0,0 +1,40 @@ +{ + "_description": "Regressão polinomial de grau 2 com scaler (obrigatório para evitar overflow)", + "experiment_run_id": 1003, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Polynomial Regression", + "model_type": "polynomial_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 2, + "interaction_only": false, + "scaler_name": "Standard Scaler" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/04-polynomial-regression-degree3.json b/docs/test-scenarios/04-polynomial-regression-degree3.json new file mode 100644 index 0000000..5e64c10 --- /dev/null +++ b/docs/test-scenarios/04-polynomial-regression-degree3.json @@ -0,0 +1,40 @@ +{ + "_description": "Regressão polinomial de grau 3 com scaler", + "experiment_run_id": 1004, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Polynomial Regression", + "model_type": "polynomial_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 3, + "interaction_only": false, + "scaler_name": "Standard Scaler" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/05-linear-regression-with-lags.json b/docs/test-scenarios/05-linear-regression-with-lags.json new file mode 100644 index 0000000..f72cd81 --- /dev/null +++ b/docs/test-scenarios/05-linear-regression-with-lags.json @@ -0,0 +1,40 @@ +{ + "_description": "Regressão linear com lags de treino e validação", + "experiment_run_id": 1005, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 5 + }, + "lag_val": { + "303-WIT-200(Value)": 3 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/06-linear-regression-nan-interpolation.json b/docs/test-scenarios/06-linear-regression-nan-interpolation.json new file mode 100644 index 0000000..8d906b0 --- /dev/null +++ b/docs/test-scenarios/06-linear-regression-nan-interpolation.json @@ -0,0 +1,40 @@ +{ + "_description": "Regressão linear com tratamento de NaN por interpolação linear", + "experiment_run_id": 1006, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "linear interpolation", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/07-linear-regression-static-window-removal.json b/docs/test-scenarios/07-linear-regression-static-window-removal.json new file mode 100644 index 0000000..26721be --- /dev/null +++ b/docs/test-scenarios/07-linear-regression-static-window-removal.json @@ -0,0 +1,40 @@ +{ + "_description": "Regressão linear com remoção de janelas estáticas", + "experiment_run_id": 1007, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": true, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/08-linear-regression-with-limits.json b/docs/test-scenarios/08-linear-regression-with-limits.json new file mode 100644 index 0000000..f7784e8 --- /dev/null +++ b/docs/test-scenarios/08-linear-regression-with-limits.json @@ -0,0 +1,45 @@ +{ + "_description": "Regressão linear com limites inferior e superior para variáveis", + "experiment_run_id": 1008, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": { + "303-WIT-200(Value)": { + "min": 0.0, + "max": 1000.0 + } + }, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/09-polynomial-degree2-with-scaler-and-lags.json b/docs/test-scenarios/09-polynomial-degree2-with-scaler-and-lags.json new file mode 100644 index 0000000..e3ef6c0 --- /dev/null +++ b/docs/test-scenarios/09-polynomial-degree2-with-scaler-and-lags.json @@ -0,0 +1,40 @@ +{ + "_description": "Cenário completo: regressão polinomial grau 2 com scaler e lags", + "experiment_run_id": 1009, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Polynomial Regression", + "model_type": "polynomial_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 3 + }, + "lag_val": { + "303-WIT-200(Value)": 2 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 2, + "interaction_only": false, + "scaler_name": "Standard Scaler" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/10-linear-regression-with-ar.json b/docs/test-scenarios/10-linear-regression-with-ar.json new file mode 100644 index 0000000..dcf98e8 --- /dev/null +++ b/docs/test-scenarios/10-linear-regression-with-ar.json @@ -0,0 +1,42 @@ +{ + "_description": "Linear regression placeholder for autoregressive features; include_ar is reserved for future wrapper support (see opt_params).", + "experiment_run_id": 1010, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": { + "include_ar": true + } +} \ No newline at end of file diff --git a/docs/test-scenarios/11-linear-regression-static-threshold-custom.json b/docs/test-scenarios/11-linear-regression-static-threshold-custom.json new file mode 100644 index 0000000..879f699 --- /dev/null +++ b/docs/test-scenarios/11-linear-regression-static-threshold-custom.json @@ -0,0 +1,40 @@ +{ + "_description": "Regressão linear com remoção de janelas estáticas e static_threshold customizado", + "experiment_run_id": 1011, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": true, + "static_threshold": 100, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/12-angular-test-date-format.json b/docs/test-scenarios/12-angular-test-date-format.json new file mode 100644 index 0000000..ec89dbc --- /dev/null +++ b/docs/test-scenarios/12-angular-test-date-format.json @@ -0,0 +1,40 @@ +{ + "_description": "Alternate date column (DATA) and dd/MM/yyyy HH:mm:ss format; uses MinIO object training_data_dd_mm_yyyy.csv from E2E fixtures.", + "experiment_run_id": 1012, + "variable_columns": [ + "303-WIT-230(Value)" + ], + "target_variable": "03CV022/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data_dd_mm_yyyy.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "DATA", + "date_format": "dd/MM/yyyy HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-230(Value)": 3 + }, + "lag_val": { + "303-WIT-230(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": "01/05/2022", + "end_date": "31/07/2022", + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/13-angular-test-double-date-column.json b/docs/test-scenarios/13-angular-test-double-date-column.json new file mode 100644 index 0000000..290f051 --- /dev/null +++ b/docs/test-scenarios/13-angular-test-double-date-column.json @@ -0,0 +1,40 @@ +{ + "_description": "Same alternate CSV as scenario 12 (DATA + dd/MM/yyyy); narrow date window for regression coverage. Not a multi-date-column dataset.", + "experiment_run_id": 1013, + "variable_columns": [ + "303-WIT-230(Value)" + ], + "target_variable": "03CV022/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data_dd_mm_yyyy.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "DATA", + "date_format": "dd/MM/yyyy HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-230(Value)": 0 + }, + "lag_val": { + "303-WIT-230(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": "01/05/2022 00:00:00", + "end_date": "31/05/2022 23:59:59", + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/14-angular-test-polynomial-support-filters.json b/docs/test-scenarios/14-angular-test-polynomial-support-filters.json new file mode 100644 index 0000000..bfc9c57 --- /dev/null +++ b/docs/test-scenarios/14-angular-test-polynomial-support-filters.json @@ -0,0 +1,51 @@ +{ + "_description": "Cenário angular-test-01: regressão polinomial degree 4, scaler, support filters em 303-WIT-200", + "experiment_run_id": 1014, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Polynomial Regression", + "model_type": "polynomial_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": "2025-06-02 00:00:00", + "end_date": "2025-06-08 23:59:59", + "support_filters": { + "303-WIT-200(Value)": { + "upper_line": { + "intercept": 40.400002, + "angle": 0 + }, + "lower_line": { + "intercept": 30.5, + "angle": 0 + } + } + }, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 4, + "interaction_only": false, + "scaler_name": "Standard Scaler" + }, + "opt_params": {} +} \ No newline at end of file diff --git a/docs/test-scenarios/15-linear-regression-custom-target-column.json b/docs/test-scenarios/15-linear-regression-custom-target-column.json new file mode 100644 index 0000000..17467a0 --- /dev/null +++ b/docs/test-scenarios/15-linear-regression-custom-target-column.json @@ -0,0 +1,40 @@ +{ + "_description": "Target column name is not ``target``; report/Evidently sections must use params.target_variable.", + "experiment_run_id": 1015, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "MY_CUSTOM_TARGET_COLUMN", + "bucket_name": "model-training", + "file_name": "training_data_custom_target.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} diff --git a/docs/test-scenarios/16-linear-regression-naive-timestamp-header.json b/docs/test-scenarios/16-linear-regression-naive-timestamp-header.json new file mode 100644 index 0000000..d74b229 --- /dev/null +++ b/docs/test-scenarios/16-linear-regression-naive-timestamp-header.json @@ -0,0 +1,40 @@ +{ + "_description": "Naive Timestamp column header; snake_case date_column/date_format and training_data_timestamp_naive.csv.", + "experiment_run_id": 1016, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data_timestamp_naive.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "Timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} diff --git a/docs/test-scenarios/17-linear-regression-blank-timestamp-row.json b/docs/test-scenarios/17-linear-regression-blank-timestamp-row.json new file mode 100644 index 0000000..c8056ac --- /dev/null +++ b/docs/test-scenarios/17-linear-regression-blank-timestamp-row.json @@ -0,0 +1,40 @@ +{ + "_description": "One CSV row has an empty timestamp; pipeline should drop it and continue training.", + "experiment_run_id": 1017, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data_blank_timestamp_row.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} diff --git a/docs/train-model-workflow-io-diff-main-vs-current-branch.md b/docs/train-model-workflow-io-diff-main-vs-current-branch.md new file mode 100644 index 0000000..675d6cd --- /dev/null +++ b/docs/train-model-workflow-io-diff-main-vs-current-branch.md @@ -0,0 +1,417 @@ +# Train Model Workflow IO Diff (`main` vs current branch) + +Base comparison: `git diff main...HEAD` +Workflow analyzed: `train_model` + +## 1) Executive overview + +This branch introduces a structural refactor of the training stack and a contract update for workflow input/output. + +Main impacts: + +- The old in-house training stack (`TrainingRepository`, `ModelRepository`, `StorageRepository`, `model_manager.sientia.models`) was replaced by: + - `DataManagerRepository` (data prep + metrics + report generation) + - `SientiaModel` wrapper from plugin store (`sientia_model`) + - `SientiaMLflowRepository` (MLflow integration) + - `MinioRepository` (storage integration) +- Input contract moved from many fixed legacy ML params to a plugin/wrapper-oriented schema (`model_type`, `*_kwargs`, `model_metadata`, optional `val_file_name`). +- Workflow return changed from `None` to a serializable result object (`dict[str, Any] | None`) containing training execution metadata. +- Queue naming and worker bootstrap architecture now depend on runtime (`train_model--queue`). + +--- + +## 2) Input contract diff (before vs now) + +### 2.1 Previous contract (`main`) + +`TrainModelParams` in `main` required a large set of explicit fields for the old preprocessing/model pipeline, focused only in linear regression model: + +- Core: + - `experiment_run_id`, `variable_columns`, `target_variable` + - `bucket_name`, `file_name`, `line_separator`, `decimal_separator` + - `train_size`, `shuffle` +- Legacy preprocessing/model fields focused only in linear regression model (required in `from_dict`): + - `lag_train`, `lag_val` + - `rem_static_win`, `low_lim`, `upp_lim`, `window` + - `use_scaler`, `include_ar`, `scaler_name` + - `removed_intervals`, `start_date`, `end_date`, `nan_treatment` + - `degree`, `interaction_only` + - `experiment_name`, `model_name` + - `support_filters` (optional dict), `static_threshold` (optional int) + +Validation was strongly tied to this structure (lag ranges, limits consistency, polynomial/scaler constraints, etc.). + +### 2.2 Current contract (this branch) + +`TrainModelParams` now supports a plugin-driven schema and wrapper kwargs: + +- Kept/mandatory core fields: + - `experiment_run_id` (now accepts numeric string too; coerced to int) + - `variable_columns`, `target_variable` + - `bucket_name`, `file_name`, `line_separator`, `decimal_separator` + - `train_size`, `shuffle` + - `model_name` + - `model_type` + - `data_model_kwargs`, `model_kwargs`, `opt_params` (required as dict by current `from_dict`) +- New/updated fields: + - `random_state` (default `42`) + - `val_file_name` (optional explicit validation file) + - `model_id` (currently optional, but needs discussion, since the model metadata in MongoDB should be created before the model training) +- Removed from required input contract: + - `lag_train`, `lag_val`, `rem_static_win`, `low_lim`, `upp_lim`, `window` + - `use_scaler`, `include_ar` + - `degree`, `interaction_only`, `nan_treatment` + - `start_date`, `end_date`, `scaler_name` + - `removed_intervals`, `support_filters`, `static_threshold` +- Parameters internally derived: + - `model_metadata` model type info from plugin store. + - `run_name` is internally derived from experiment name and datetime. + - `experiment_name` is internally derived from `model_name`. + +### 2.3 Validation behavior changes + +Before: +- Validation was mostly hardcoded business checks tied to legacy linear/polynomial stack. + +Now: +- Validation still checks core constraints (`train_size`, non-empty strings, etc.), but model-specific validation moved to JSON Schema driven checks, using OpenAPI/JSON Schema definitions from plugin store: + - `model_metadata.schemas.components.schemas.data_model` + - `model_metadata.schemas.components.schemas.model` + - `model_metadata.schemas.components.schemas.opt_params` +- `model_metadata` is now a required semantic dependency for `validate_business_rules()`. +- Date format validation remains, but allowed formats are defined locally in `train_model_params.py`. + +### 2.4 Input loading pipeline changes in workflow + +Before: +- `validate_train_params` directly consumed workflow input. + +Now: +1. `load_model_metadata` runs first (fetches model index/schema from plugin store and injects `model_metadata`). +2. `validate_train_params` runs with enriched payload. + +This means IO preprocessing now depends on plugin-store metadata resolution before final validation. + +--- + +## 3) Output contract diff (before vs now) + +### 3.1 Workflow return (`train_model.run`) + +Before (`main`): +- Return type: `None` +- Workflow side effects were persisted mainly via DB status updates and MLflow artifacts. + +Now: +- Return type: `dict[str, Any] | None` +- Workflow returns the training activity summary when successful. + +### 3.2 Activity-level training result payload + +Before (from `Training.train_model` in `main` path): +- Returned minimal dict: + - `run_name` + - `run_dir` + +Now: +- Returns extended dict: + - `run_name` + - `experiment_name` + - `run_id` + - `run_dir` + +### 3.3 Persistence map by destination (DB, MLflow, MinIO, local filesystem) + +This section maps where each artifact/metadata goes, in which format, and how that changed from `main`. + +#### 3.3.1 PostgreSQL (`experiment_run` table) + +## Before (`main`) + +- Update path: `update_experiment_run` activity with `UpdateType.MODEL_SAVED`. +- Persisted on success: + - `status` transition to `TRAINING_SUCCESS` + - `run_name` (MLflow run identifier used by current implementation) +- Persisted on failures: + - `status` transition to validation/training error statuses + - `error_message` + +## Now (current branch) + +- Same update path and status/error behavior. +- Even though train activity now returns more metadata (`run_id`, `experiment_name`), current workflow update for `MODEL_SAVED` still forwards mainly `run_name`. +- Practical effect: + - DB remains status-centric and run-name-centric + - richer identifiers exist in workflow return payload, not fully mirrored to DB columns in current flow + +#### 3.3.2 MLflow (tracking server/artifact store) + +## Before (`main`) + +- Persistence orchestration lived in `ModelRepository.save_model()` + `_save_run()`. +- Typical persisted content: + - model params (many legacy params such as lags, limits, scaler config, removed intervals) + - regression metrics (`MSE`, `R2`, `MAE`) + - model objects: + - `data_model` + - `prediction_model` + - artifacts: + - `report.html` + - `train_data.csv` + - `test_data.csv` + - optional `model_equation.json` +- Run naming: + - computed by querying existing runs and appending sequence (`-` style) + +## Now (current branch) + +- Persistence orchestrated in `Training._persist_training_artifacts()` and MLflow run context is opened by `SientiaMLflowRepository.start_run(...)`. +- Persisted content now: + - model wrapper itself via `wrapper.store_model(name=train_params.model_name)` + - regression metrics also logged as MLflow params via `mlflow.log_param(...)`: + - `mse_val` + - `mae_val` + - `r2_val` + - artifacts explicitly logged with `mlflow.log_artifact(...)`: + - `report.html` + - `train_data.csv` + - `test_data.csv` + - metrics are computed before save (`mse_val`, `mae_val`, `r2_val`) and persisted in the run as params +- Run identifiers now exposed back to workflow: + - `experiment_name` + - `run_name` + - `run_id` +- Notable behavioral change: + - `wrapper._input_example` is cleared (`None`) before storing model. + +#### 3.3.3 MinIO object storage + +## Before (`main`) + +- Read path: + - single source object downloaded via `StorageRepository.fetch_file(bucket_name, file_name)` +- Write path: + - training workflow did not write generated outputs to MinIO in this code path + - generated artifacts were persisted to MLflow, not uploaded back to MinIO +- Location: + - source data in input bucket/key provided by workflow input (`bucket_name` + `file_name`) + +## Now (current branch) + +- Read path migrated to `MinioRepository.download_file(...)`. +- Supports two input objects: + - mandatory training object: `bucket_name` + `file_name` + - optional validation object: same `bucket_name` + `val_file_name` +- Write path: + - still no artifact upload to MinIO in this workflow path + - report/CSV outputs continue to flow to MLflow artifacts +- Location details: + - bucket resolved from payload (`bucket_name`) + - object key exactly from payload (`file_name`, optional `val_file_name`) + - default bucket in env/config is `MINIO_DEFAULT_BUCKET`, but runtime payload can override via `bucket_name` + +#### 3.3.4 Local filesystem (ephemeral runtime workspace) + +## Before (`main`) + +- Temporary run dir created under reports root using run name + timestamp suffix. +- Artifacts generated locally in that directory: + - `report.html` + - `train_data.csv` + - `test_data.csv` + - optional `model_equation.json` +- After MLflow logging, cleanup activity removed temp directory. + +## Now (current branch) + +- Temporary run dir managed by `DataManagerRepository` under runtime reports root (`.../reports/temp/`). +- Same artifact family generated locally: + - `report.html` + - `train_data.csv` + - `test_data.csv` + - optional `model_equation.json` (for `linear_regression`) +- Cleanup behavior is now tolerant: + - cleanup runs in guarded `finally` + - training success is not reverted if cleanup later fails + +#### 3.3.5 Quick matrix (before vs now) + +- **Postgres** + - before: status + run_name + errors + - now: same persisted shape; workflow return contains extra IDs +- **MLflow** + - before: legacy model objects + params/metrics + report/data artifacts + - now: wrapper-based model persistence + `mse_val`/`mae_val`/`r2_val` as params + report/data artifacts + run_id exposed +- **MinIO** + - before: reads 1 CSV input object + - now: reads 1 or 2 CSV input objects (train + optional validation), still no output upload +- **Local temp** + - before: generated artifacts, then cleanup + - now: generated artifacts, then best-effort cleanup (non-blocking for success result) + +### 3.4 Cleanup behavior impact on output semantics + +Before: +- Cleanup was called directly after training result; failures propagated straightforwardly. + +Now: +- Cleanup is in a guarded `finally`. +- If training succeeded but cleanup fails, workflow warns and does not rollback success semantics. +- Effective output semantics: successful training result can be returned even if temp cleanup fails. + +--- + +## 4) Detailed field mapping (old -> new) + +## Kept (or equivalent role) + +- `experiment_run_id` -> kept (broader accepted types: int or numeric string) +- `variable_columns` -> kept +- `target_variable` -> kept +- `bucket_name` -> kept +- `file_name` -> kept +- `line_separator` -> kept +- `decimal_separator` -> kept +- `date_column` -> required (snake_case key; must exist in CSV) +- `date_format` -> optional in payload; omitted/null/blank resolves to default `yyyy-MM-dd HH:mm:ss` +- `train_size` -> kept +- `shuffle` -> kept +- `model_name` -> kept (now less coupled to legacy model enum) + +## Added + +- `model_type` (primary selector for plugin wrapper/index lookup) +- `data_model_kwargs` +- `model_kwargs` +- `opt_params` +- `val_file_name` (optional second dataset input) +- `model_id` (optional metadata) +- `model_metadata` (loaded/required for schema validation) +- `random_state` (explicit split reproducibility control) + +## Removed from new required contract + +- `lag_train`, `lag_val` +- `rem_static_win`, `static_threshold` +- `low_lim`, `upp_lim` +- `window` +- `use_scaler`, `include_ar` +- `degree`, `interaction_only` +- `nan_treatment` +- `start_date`, `end_date` +- `scaler_name` +- `removed_intervals` +- `support_filters` +- `experiment_name` (no longer required as top-level client input) + +--- + +## 5) Internal architecture update notes + +### 5.1 Repository layer redesign + +Removed: +- `model_manager/utils/repository/model_repository.py` +- `model_manager/utils/repository/training_repository.py` +- `model_manager/utils/repository/storage_repository.py` + +Added: +- `model_manager/utils/repository/data_manager_repository.py` + +Interpretation: +- Data preprocessing/report/metrics responsibilities were consolidated into `DataManagerRepository`. +- Training/model persistence shifted to wrapper + plugin store + MLflow repository integrations. + +### 5.2 Model engine abstraction migration + +Before: +- Strong coupling to local classes in `model_manager.sientia.models` and custom preprocessing/model objects in `TrainModelResult`. + +Now: +- Training uses `SientiaModel` wrapper dynamically obtained by `plugin_store.get_model(model_type=...)`. +- Contract is wrapper-driven (`train`, `transform`, `predict`, `store_model`). +- The codebase removed `model_manager/sientia/models.py`, `model_serving.py`, and `utils.py`, indicating full migration to externalized model runtime abstraction. + +### 5.3 Worker/runtime architecture changes + +- New `prepare_worker.py` centralizes worker setup and autoscaling parameters. +- Queue names are now runtime-derived: + - `train_model--queue` + - `cleanup_files--queue` +- `worker.py` now installs runtime via plugin store (`plugin_store.install_runtime(runtime_name=...)`) before starting workers. +- This introduces environment/runtime-aware deployment and model packaging behavior. + +### 5.4 Synchronous activity and tracking adjustments + +- `experiment_tracking` migrated from async postgres helper to sync postgres client path (`postgres_sync`). +- Several activities switched to sync method signatures. +- Error handling in workflow and DB status update paths is more defensive (secondary failures while persisting error status are logged and do not mask primary failure cause). + +### 5.5 `TrainModelResult` shape update + +Before: +- Stored classic split artifacts (`x_train`, `x_test`, `y_train`, `y_test`) + concrete preprocessing/model objects (`process_data`, `regr`, `scaler_dict`). + +Now: +- Stores `train_data`, `val_data` and prediction DataFrames, plus tracking identifiers (`experiment_name`, `run_id`). +- Result object is less tied to internal estimator classes and more aligned with serializable workflow/model-store integration. + +--- + +## 6) Net IO compatibility assessment + +## Input compatibility + +Not backward compatible with old payloads without adaptation. + +Key reasons: +- Legacy required fields removed/ignored by new path. +- New required fields introduced (`model_type`, `*_kwargs` dicts, runtime metadata flow dependency). +- Validation pipeline now expects model metadata semantics. + +## Output compatibility + +Behavior changed: +- Workflow now returns a result object (previously `None`). +- Training summary includes `experiment_name` and `run_id` in addition to `run_name` and `run_dir`. +- DB update still centered on `run_name`; callers relying only on DB may not see all new output info unless workflow return is consumed. + +--- + +## 7) Practical migration guidance (client side) + +To call `train_model` in this branch: + +1. Send snake_case payload aligned to new `TrainModelParams`. +2. Always provide: + - `model_name` slugified model name (ex.: `test_model_name or test-model-name`) + - `model_type` + - `data_model_kwargs` (dict) + - `model_kwargs` (dict) + - `opt_params` (dict) +3. Keep `experiment_run_id` numeric (int or numeric string). +4. Use runtime queue naming consistent with worker runtime: + - `train_model--queue` +5. If you need explicit validation split file, send `val_file_name`; otherwise split uses `train_size`/`shuffle`/`random_state`. + +--- + +## 8) Source references used for this document + +Primary diffs: +- `model_manager/workflows/train_model.py` +- `model_manager/utils/models/train_model_params.py` +- `model_manager/utils/models/train_model_result.py` +- `model_manager/activities/training.py` +- `model_manager/activities/activities.py` +- `model_manager/activities/experiment_tracking.py` +- `model_manager/utils/repository/data_manager_repository.py` +- `model_manager/utils/repository/model_repository.py` (removed) +- `model_manager/utils/repository/training_repository.py` (removed) +- `model_manager/utils/repository/storage_repository.py` (removed) +- `model_manager/worker/worker.py` +- `model_manager/worker/prepare_worker.py` +- `README.md` +- `input-sample.md` +- `scripts/run_training_test.py` + diff --git a/e2e/__init__.py b/e2e/__init__.py new file mode 100644 index 0000000..d02a73a --- /dev/null +++ b/e2e/__init__.py @@ -0,0 +1,3 @@ +""" +End-to-end tests for the Model Manager Temporal workflows. +""" diff --git a/e2e/conftest.py b/e2e/conftest.py new file mode 100644 index 0000000..638a216 --- /dev/null +++ b/e2e/conftest.py @@ -0,0 +1,697 @@ +""" +Pytest configuration and fixtures for E2E tests. + +External dependencies use testcontainers or real SDK integrations (no mocks of +model_manager or other first-party code): + +- PostgreSQL, MinIO, MongoDB, Gitea: testcontainers. +- MLflow: real client with ``file://`` tracking URI (no MLflow server process). +- Temporal: ``WorkflowEnvironment.start_time_skipping()`` — official in-process + test runtime from temporalio; exercises real workflows and activity code, not + stubs of business logic. +- Observability: ``Logger`` (``get_logger`` from ``model_manager.utils.logger_helper``) + and ``MetricsController`` from sientia_do, same stack as production. +""" + +from concurrent.futures import ThreadPoolExecutor +import base64 +import csv +import io +import os +import shutil +import tempfile +import time + +import mlflow +import pytest +import pytest_asyncio +import requests +from minio import Minio +from sqlalchemy import create_engine, text +from testcontainers.core.container import DockerContainer # type: ignore[import,import-untyped] +from testcontainers.minio import MinioContainer # type: ignore[import-untyped] +from testcontainers.mongodb import MongoDbContainer # type: ignore[import-untyped] +from testcontainers.postgres import PostgresContainer # type: ignore[import,import-untyped] +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + +from model_manager.activities.activities import Activities +from model_manager.utils.logger_helper import get_logger +from model_manager.workflows.cleanup_files import CleanupFiles +from model_manager.workflows.train_model import TrainModel +from sientia_do.notifications.handlers import CoreNotificationHandler +from sientia_do.observability.metrics_controller import MetricsController +from sientia_model.model_repository.plugin_store import PluginStore + +# --------------------------------------------------------------------------- +# CSV training data: columns must match the variable_columns and target_variable +# used across all test scenarios. +_TRAIN_CSV_COLUMNS = [ + 'timestamp', + '303-WIT-200(Value)', + '03CV020/CORRENTE_N_M1_PV(Value)', + '303-WIT-230(Value)', + '03CV022/CORRENTE_N_M1_PV(Value)', +] +_MINIO_BUCKET = 'model-training' +_MINIO_OBJECT = 'training_data.csv' + +# --------------------------------------------------------------------------- +# Helpers – CSV generation +# --------------------------------------------------------------------------- + +def _build_training_csv() -> bytes: + """ + Generate a 150-row CSV with all columns needed by test scenarios. + + The numeric values cycle deterministically so lags and static-window + removal always find enough rows in both train and validation splits. + """ + output = io.StringIO() + writer = csv.writer(output) + writer.writerow(_TRAIN_CSV_COLUMNS) + for i in range(150): + ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00' + wit200 = round(30.0 + (i % 20) * 0.5, 2) + cv020 = round(100.0 + (i % 15) * 0.3, 2) + wit230 = round(25.0 + (i % 18) * 0.4, 2) + cv022 = round(90.0 + (i % 12) * 0.25, 2) + writer.writerow([ts, wit200, cv020, wit230, cv022]) + return output.getvalue().encode('utf-8') + + +def _build_training_csv_dd_mm_yyyy() -> bytes: + """ + Generate a 150-row CSV with dd/MM/yyyy HH:mm:ss timestamps and + a DATA column header, for scenarios 12/13 that use a different date format. + """ + output = io.StringIO() + writer = csv.writer(output) + writer.writerow([ + 'DATA', + '303-WIT-230(Value)', + '03CV022/CORRENTE_N_M1_PV(Value)', + ]) + for i in range(150): + day = (i % 30) + 1 + ts = f'{day:02d}/05/2022 {i % 24:02d}:00:00' + wit230 = round(25.0 + (i % 18) * 0.4, 2) + cv022 = round(90.0 + (i % 12) * 0.25, 2) + writer.writerow([ts, wit230, cv022]) + return output.getvalue().encode('utf-8') + + +def _build_training_csv_custom_target_column() -> bytes: + """ + Same layout as the standard CSV but the target column has a non-default name + (not ``target``) to exercise report and metrics paths. + """ + output = io.StringIO() + writer = csv.writer(output) + writer.writerow([ + 'timestamp', + '303-WIT-200(Value)', + 'MY_CUSTOM_TARGET_COLUMN', + ]) + for i in range(150): + ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00' + wit200 = round(30.0 + (i % 20) * 0.5, 2) + target_val = round(100.0 + (i % 15) * 0.3, 2) + writer.writerow([ts, wit200, target_val]) + return output.getvalue().encode('utf-8') + + +def _build_training_csv_timestamp_header_naive() -> bytes: + """ + Naive datetimes under column ``Timestamp`` (common UI export) for scenario 16. + """ + output = io.StringIO() + writer = csv.writer(output) + writer.writerow([ + 'Timestamp', + '303-WIT-200(Value)', + '03CV020/CORRENTE_N_M1_PV(Value)', + ]) + for i in range(150): + ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00' + wit200 = round(30.0 + (i % 20) * 0.5, 2) + cv020 = round(100.0 + (i % 15) * 0.3, 2) + writer.writerow([ts, wit200, cv020]) + return output.getvalue().encode('utf-8') + + +def _build_training_csv_blank_timestamp_row() -> bytes: + """Standard columns with one row where ``timestamp`` is empty (NaN after parse).""" + output = io.StringIO() + writer = csv.writer(output) + writer.writerow(_TRAIN_CSV_COLUMNS) + for i in range(150): + wit200 = round(30.0 + (i % 20) * 0.5, 2) + cv020 = round(100.0 + (i % 15) * 0.3, 2) + wit230 = round(25.0 + (i % 18) * 0.4, 2) + cv022 = round(90.0 + (i % 12) * 0.25, 2) + if i == 17: + writer.writerow(['', wit200, cv020, wit230, cv022]) + else: + ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00' + writer.writerow([ts, wit200, cv020, wit230, cv022]) + return output.getvalue().encode('utf-8') + + +# --------------------------------------------------------------------------- +# Helpers – Gitea seed +# --------------------------------------------------------------------------- + +def _wait_for_gitea(base_url: str, timeout: int = 120) -> None: + """Poll Gitea until it responds to HTTP requests.""" + deadline = time.time() + timeout + last_err = None + while time.time() < deadline: + try: + resp = requests.get(f'{base_url}/', timeout=3) + if resp.status_code in (200, 404, 302): + return + except Exception as e: + last_err = e + time.sleep(2) + raise TimeoutError(f'Gitea did not start within {timeout}s at {base_url}. Last error: {last_err}') + + +def _gitea_api(method: str, url: str, auth: tuple, **kwargs) -> requests.Response: + resp = requests.request(method, url, auth=auth, timeout=30, **kwargs) + try: + resp.raise_for_status() + except requests.exceptions.HTTPError as e: + raise RuntimeError(f"Gitea API error {resp.status_code}: {resp.text}") from e + return resp + + +def _seed_gitea(base_url: str, admin_user: str, admin_pass: str) -> None: + """ + Create a fictitious model-store repository with dummy models. + """ + auth = (admin_user, admin_pass) + api = f'{base_url}/api/v1' + + # Create repository + _gitea_api( + 'POST', f'{api}/user/repos', auth, + json={'name': 'model-store', 'private': False, 'auto_init': False}, + ) + + # Root index.yaml + root_index = """ +store_name: "E2E Test Store" +version: 1 +models: + - name: "linear_regression" + version: 1 + runtime: "basic" + - name: "polynomial_regression" + version: 1 + runtime: "basic" +runtimes: + basic: + version: "1.0.0" + libraries: + - name: "pandas" + - name: "numpy" +""" + + # Model index.yaml (shared for all dummies) + model_index = """ +name: "{model_name}" +version: 1 +runtime: "basic" +path: "wrapper.py" +class: "DummyWrapper" +model: + class: "DummyModel" + path: "model_logic.py" + external: false +data_model: + class: "DummyTransformer" + path: "model_logic.py" + external: false +""" + + # schemas.yaml + schemas_yaml = """ +model: + type: object + properties: {} +data_model: + type: object + properties: {} +opt_params: + type: object + properties: {} +""" + + # wrapper.py + wrapper_py = """ +from sientia_model.wrappers.sientia_model import SientiaModel +import pandas as pd +import numpy as np +from typing import Any + +class DummyWrapper(SientiaModel): + def _predict(self, data: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, Any]]: + self._log("info", f"Predicting dummy model for {self.model_type}") + # Return a simple prediction (mean or 0.5) to allow metrics computation + preds = pd.DataFrame({self.target: [0.5] * len(data)}, index=data.index) + return preds, {} + + def _transform(self, data: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, Any]]: + return data, {} + + def _train_transformer(self, train_data: pd.DataFrame, val_data: pd.DataFrame) -> None: + pass + + def _train_model(self, x: pd.DataFrame, y: pd.DataFrame, x_val: pd.DataFrame | None = None, y_val: pd.DataFrame | None = None) -> None: + self.target = y.columns[0] + + def _retrain_transformer(self, data: pd.DataFrame) -> None: + pass + + def _retrain_model(self, x: pd.DataFrame, y: pd.DataFrame | None) -> None: + pass +""" + + # model_logic.py + model_logic_py = """ +class DummyModel: + def __init__(self, **kwargs): + pass + +class DummyTransformer: + def __init__(self, **kwargs): + pass +""" + + # requirements.txt required by current model packaging path in training activity. + requirements_txt = """ +pandas +numpy +""" + + def push_file(path: str, content: str): + encoded = base64.b64encode(content.encode()).decode() + _gitea_api( + 'POST', + f'{api}/repos/{admin_user}/model-store/contents/{path}', + auth, + json={'message': f'seed: {path}', 'content': encoded}, + ) + + # Push root index + push_file('index.yaml', root_index) + + # Push files for both models used in tests + for model_name in ['linear_regression', 'polynomial_regression']: + prefix = f'models/{model_name}' + push_file(f'{prefix}/index.yaml', model_index.format(model_name=model_name)) + push_file(f'{prefix}/schemas.yaml', schemas_yaml) + push_file(f'{prefix}/wrapper.py', wrapper_py) + push_file(f'{prefix}/model_logic.py', model_logic_py) + push_file(f'{prefix}/requirements.txt', requirements_txt.strip() + '\n') + push_file(f'{prefix}/__init__.py', "") + + # Push runtime + push_file('runtime/basic.yaml', 'name: basic\nversion: "1.0.0"\nlibraries: []') + + +# --------------------------------------------------------------------------- +# Session-scoped containers +# --------------------------------------------------------------------------- + +@pytest.fixture(scope='session') +def postgres_container(): + """PostgreSQL 15 container for experiment_run table.""" + container = PostgresContainer('postgres:15') + container.start() + yield container + container.stop() + + +@pytest.fixture(scope='session') +def minio_container(): + """MinIO container for training CSV storage.""" + container = MinioContainer() + container.start() + yield container + container.stop() + + +@pytest.fixture(scope='session') +def mongodb_container(): + """MongoDB container for CoreNotificationHandler.""" + container = MongoDbContainer('mongo:7') + container.start() + yield container + container.stop() + + +@pytest.fixture(scope='session') +def gitea_container(): + """ + Gitea container with a ``model-store`` repo seeded via REST API + (``index.yaml``, dummy model files, ``_seed_gitea``). + + The container starts with INSTALL_LOCK so no setup wizard is needed. + An admin user is created via Gitea's CLI before the HTTP API is used. + """ + admin_user = 'gitea_admin' + admin_pass = 'gitea_admin_pass' # noqa: S105 + + container = ( + DockerContainer('gitea/gitea:latest') + .with_env('GITEA__security__INSTALL_LOCK', 'true') + .with_env('GITEA__server__HTTP_PORT', '3000') + .with_env('GITEA__log__LEVEL', 'Warn') + .with_exposed_ports(3000) + ) + container.start() + + port = container.get_exposed_port(3000) + base_url = f'http://localhost:{port}' + + _wait_for_gitea(base_url) + time.sleep(5) # Wait a bit for DB to fully initialize after HTTP is up + + # Create admin user via Gitea CLI inside the container + # Must run after Gitea is fully initialized + gitea_cmd = ( + f'gitea admin user create ' + f'--username {admin_user} ' + f'--password {admin_pass} ' + f'--email admin@test.local ' + f'--admin ' + f'--must-change-password=false' + ) + exec_result = container.exec(f"su git -c '{gitea_cmd}'") + if exec_result.exit_code != 0: + raise RuntimeError(f"Failed to create Gitea admin user: {exec_result.output.decode('utf-8')}") + + _seed_gitea(base_url, admin_user, admin_pass) + + yield { + 'container': container, + 'base_url': base_url, + 'admin_user': admin_user, + 'admin_pass': admin_pass, + } + + container.stop() + + +@pytest.fixture(scope='session') +def mlflow_tracking_dir(): + """Local MLflow filesystem tracking directory (no network needed).""" + tmpdir = tempfile.mkdtemp(prefix='mlflow-e2e-') + mlflow.set_tracking_uri(f'file://{tmpdir}') + yield tmpdir + shutil.rmtree(tmpdir, ignore_errors=True) + + +@pytest.fixture(scope='session', autouse=True) +def e2e_runtime_reports_dir(): + """ + Route runtime report artifacts to a writable temp directory during E2E. + + Production defaults point to /var/lib/model-manager; in local CI/dev runs this + path may be unavailable. This fixture keeps the same code paths while avoiding + host permission issues. + """ + import model_manager.runtime_paths as runtime_paths + import model_manager.utils.repository.data_manager_repository as data_repo_module + + base_dir = tempfile.mkdtemp(prefix='model-manager-e2e-runtime-') + reports_root = f'{base_dir}/reports' + reports_temp_dir = f'{reports_root}/temp' + project_base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'model_manager')) + + old_runtime_reports_root = runtime_paths.REPORTS_ROOT + old_runtime_reports_temp = runtime_paths.REPORTS_TEMP_DIR + old_runtime_project_base = runtime_paths.PROJECT_BASE_PATH + old_repo_reports_root = data_repo_module.REPORTS_ROOT + old_repo_project_base = data_repo_module.PROJECT_BASE_PATH + + runtime_paths.REPORTS_ROOT = reports_root + runtime_paths.REPORTS_TEMP_DIR = reports_temp_dir + runtime_paths.PROJECT_BASE_PATH = project_base_path + data_repo_module.REPORTS_ROOT = reports_root + data_repo_module.PROJECT_BASE_PATH = project_base_path + + try: + yield reports_root + finally: + runtime_paths.REPORTS_ROOT = old_runtime_reports_root + runtime_paths.REPORTS_TEMP_DIR = old_runtime_reports_temp + runtime_paths.PROJECT_BASE_PATH = old_runtime_project_base + data_repo_module.REPORTS_ROOT = old_repo_reports_root + data_repo_module.PROJECT_BASE_PATH = old_repo_project_base + shutil.rmtree(base_dir, ignore_errors=True) + + +# --------------------------------------------------------------------------- +# Session-scoped: seed MinIO with training CSV +# --------------------------------------------------------------------------- + +@pytest.fixture(scope='session', autouse=True) +def upload_training_csv(minio_container, mlflow_tracking_dir): # noqa: ARG001 + """ + Upload training CSV files to the MinIO container before any test runs. + Depends on mlflow_tracking_dir to ensure the MLflow URI is set at session start. + """ + port = minio_container.get_exposed_port(9000) + client = Minio( + f'localhost:{port}', + access_key='minioadmin', + secret_key='minioadmin', + secure=False, + ) + + if not client.bucket_exists(_MINIO_BUCKET): + client.make_bucket(_MINIO_BUCKET) + + # Standard training CSV + csv_bytes = _build_training_csv() + client.put_object( + _MINIO_BUCKET, + _MINIO_OBJECT, + io.BytesIO(csv_bytes), + length=len(csv_bytes), + content_type='text/csv', + ) + + # dd/MM/yyyy format CSV for scenarios 12/13 + alt_csv_bytes = _build_training_csv_dd_mm_yyyy() + client.put_object( + _MINIO_BUCKET, + 'training_data_dd_mm_yyyy.csv', + io.BytesIO(alt_csv_bytes), + length=len(alt_csv_bytes), + content_type='text/csv', + ) + + custom_target = _build_training_csv_custom_target_column() + client.put_object( + _MINIO_BUCKET, + 'training_data_custom_target.csv', + io.BytesIO(custom_target), + length=len(custom_target), + content_type='text/csv', + ) + + ts_header = _build_training_csv_timestamp_header_naive() + client.put_object( + _MINIO_BUCKET, + 'training_data_timestamp_naive.csv', + io.BytesIO(ts_header), + length=len(ts_header), + content_type='text/csv', + ) + + blank_ts = _build_training_csv_blank_timestamp_row() + client.put_object( + _MINIO_BUCKET, + 'training_data_blank_timestamp_row.csv', + io.BytesIO(blank_ts), + length=len(blank_ts), + content_type='text/csv', + ) + + +# --------------------------------------------------------------------------- +# Function-scoped: database engine + schema setup +# --------------------------------------------------------------------------- + +@pytest.fixture +def postgres_engine(postgres_container): + """SQLAlchemy engine connected to the test PostgreSQL container.""" + engine = create_engine(postgres_container.get_connection_url()) + yield engine + engine.dispose() + + +@pytest.fixture(autouse=True) +def setup_experiment_run_table(postgres_engine): + """ + Create the experiment_run table before each test and drop it afterwards + to guarantee full isolation between tests. + """ + with postgres_engine.begin() as conn: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS public.experiment_run ( + id INT PRIMARY KEY, + experiment_name TEXT NOT NULL, + run_name TEXT, + username TEXT, + status TEXT NOT NULL DEFAULT 'ORCHESTRATOR_WAITING_PROC', + error_message TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + bucket_name TEXT, + file_name TEXT + ) + """)) + yield + with postgres_engine.begin() as conn: + conn.execute(text('DROP TABLE IF EXISTS public.experiment_run')) + + +# --------------------------------------------------------------------------- +# Observability (real sientia_do implementations) +# --------------------------------------------------------------------------- + +@pytest.fixture(scope='session') +def e2e_logger(): + """Shared production-style Logger for the whole E2E session.""" + return get_logger('model-manager-e2e') + + +@pytest.fixture +def metrics_controller(e2e_logger): + """MetricsController bound to the E2E logger (fresh instance per test).""" + return MetricsController(logger=e2e_logger) + + +# --------------------------------------------------------------------------- +# Application fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture +def notification_handler(mongodb_container, e2e_logger): + """ + Real CoreNotificationHandler connected to the MongoDB testcontainer. + """ + connection_url = mongodb_container.get_connection_url() + handler = CoreNotificationHandler( + connection_string=connection_url, + database='test_notifications', + logger=e2e_logger, + project_name='model-manager-e2e', + ) + yield handler + handler.shutdown() + + +@pytest.fixture +def plugin_store(gitea_container, e2e_logger, metrics_controller, notification_handler): + """ + Real PluginStore pointed at the Gitea testcontainer. + cache_ttl_seconds=0 forces a fresh download every test. + """ + store = PluginStore( + base_url=gitea_container['base_url'], + owner=gitea_container['admin_user'], + repo='model-store', + username=gitea_container['admin_user'], + password=gitea_container['admin_pass'], + cache_ttl_seconds=0, + logger=e2e_logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + yield store + + +@pytest.fixture +def test_activities( + postgres_container, + minio_container, + mlflow_tracking_dir, # noqa: ARG001 – ensures MLflow URI is set + plugin_store, + e2e_logger, + notification_handler, + metrics_controller, +): + """ + Real Activities instance wired to all testcontainers. + """ + pg_port = postgres_container.get_exposed_port(5432) + minio_port = minio_container.get_exposed_port(9000) + + activities = Activities( + postgres_config={ + 'host': 'localhost', + 'port': int(pg_port), + 'user': 'test', + 'password': 'test', + 'dbname': 'test', + 'min_connections': 1, + 'max_connections': 5, + }, + mlflow_config={ + 'url': mlflow.get_tracking_uri(), + 'username': None, + 'password': None, + }, + minio_config={ + 'endpoint_url': f'http://localhost:{minio_port}', + 'access_key': 'minioadmin', + 'secret_key': 'minioadmin', + 'use_ssl': False, + 'default_bucket': _MINIO_BUCKET, + }, + plugin_store=plugin_store, + logger=e2e_logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + yield activities + activities.shutdown() + + +def _activity_list(activities: Activities) -> list: + return [ + activities.update_experiment_run, + activities.load_model_metadata, + activities.validate_train_params, + activities.train_model, + activities.cleanup_resources, + activities.cleanup_temp_directories, + ] + + +@pytest_asyncio.fixture(scope='function') +async def temporal_test_env(): + """Temporal SDK test environment (time-skipping); runs real workflow/activity code.""" + env = await WorkflowEnvironment.start_time_skipping() + async with env: + yield env + + +@pytest_asyncio.fixture(scope='function') +async def temporal_worker(temporal_test_env, test_activities): + """Temporal worker registered with all workflows and activities.""" + with ThreadPoolExecutor() as executor: + async with Worker( + temporal_test_env.client, + task_queue='test-queue', + workflows=[TrainModel, CleanupFiles], + activities=_activity_list(test_activities), + activity_executor=executor, + ) as worker: + yield worker diff --git a/e2e/helpers.py b/e2e/helpers.py new file mode 100644 index 0000000..ef04e71 --- /dev/null +++ b/e2e/helpers.py @@ -0,0 +1,195 @@ +""" +Shared helpers for E2E tests (Temporal workflows + PostgreSQL). +""" + +import asyncio +import json +from datetime import datetime +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy import text +from sqlalchemy.engine import Engine + + +async def start_and_await_workflow( + client, + workflow_run, + input_data: dict, + workflow_id: str, + timeout: float = 600.0, +): + """ + Start a Temporal workflow and wait for its result. + + Args: + client: Temporal client from WorkflowEnvironment. + workflow_run: Workflow run method (e.g. TrainModel.run). + input_data: Workflow input payload. + workflow_id: Unique workflow id. + timeout: Max seconds to wait for completion (default allows cold testcontainer startup). + + Returns: + Workflow result value. + """ + handle = await client.start_workflow( + workflow_run, + input_data, + id=workflow_id, + task_queue='test-queue', + ) + return await asyncio.wait_for(handle.result(), timeout=timeout) + + +def make_workflow_id(prefix: str) -> str: + """Build a unique workflow id using a prefix and current timestamp.""" + return f'{prefix}-{datetime.now().timestamp()}' + + +def insert_experiment_run( + engine: Engine, + experiment_run_id: int, + experiment_name: str = 'test_experiment', + status: str = 'ORCHESTRATOR_WAITING_PROC', + bucket_name: str = 'model-training', + file_name: str = 'training_data.csv', +) -> None: + """ + Insert a minimal experiment_run row to satisfy foreign-key-style lookups. + + Args: + engine: SQLAlchemy engine connected to the test database. + experiment_run_id: Primary key for the row. + experiment_name: Human-readable experiment name. + status: Initial status string. + bucket_name: MinIO bucket name. + file_name: Training file name inside the bucket. + """ + with engine.begin() as conn: + conn.execute( + text(""" + INSERT INTO public.experiment_run + (id, experiment_name, status, bucket_name, file_name) + VALUES + (:id, :experiment_name, :status, :bucket_name, :file_name) + ON CONFLICT (id) DO NOTHING + """), + { + 'id': experiment_run_id, + 'experiment_name': experiment_name, + 'status': status, + 'bucket_name': bucket_name, + 'file_name': file_name, + }, + ) + + +def assert_experiment_status( + engine: Engine, + experiment_run_id: int, + expected_status: str, +) -> None: + """ + Assert the final status of an experiment_run row. + + Args: + engine: SQLAlchemy engine. + experiment_run_id: Row primary key. + expected_status: Expected status string. + """ + with engine.connect() as conn: + row = conn.execute( + text('SELECT status FROM public.experiment_run WHERE id = :id'), + {'id': experiment_run_id}, + ).fetchone() + + assert row is not None, ( + f'No experiment_run row found for id={experiment_run_id}' + ) + assert row[0] == expected_status, ( + f'Expected status={expected_status!r}, got {row[0]!r} ' + f'for experiment_run id={experiment_run_id}' + ) + + +def assert_experiment_run_name_set( + engine: Engine, + experiment_run_id: int, +) -> None: + """Assert that run_name is not null/empty after a successful training.""" + with engine.connect() as conn: + row = conn.execute( + text('SELECT run_name FROM public.experiment_run WHERE id = :id'), + {'id': experiment_run_id}, + ).fetchone() + + assert row is not None, ( + f'No experiment_run row found for id={experiment_run_id}' + ) + assert row[0] is not None and row[0].strip() != '', ( + f'Expected run_name to be set for experiment_run id={experiment_run_id}, got {row[0]!r}' + ) + + +def assert_experiment_error( + engine: Engine, + experiment_run_id: int, + expected_status: str, + error_substr: str, +) -> None: + """ + Assert status and that error_message contains a given substring. + + Args: + engine: SQLAlchemy engine. + experiment_run_id: Row primary key. + expected_status: Expected status string. + error_substr: Substring that must appear in error_message. + """ + with engine.connect() as conn: + row = conn.execute( + text( + 'SELECT status, error_message FROM public.experiment_run WHERE id = :id' + ), + {'id': experiment_run_id}, + ).fetchone() + + assert row is not None, ( + f'No experiment_run row found for id={experiment_run_id}' + ) + assert row[0] == expected_status, ( + f'Expected status={expected_status!r}, got {row[0]!r}' + ) + assert row[1] is not None and error_substr.lower() in row[1].lower(), ( + f'Expected error_message to contain {error_substr!r}, got {row[1]!r}' + ) + + +def assert_no_experiment_row(engine: Engine, experiment_run_id: int) -> None: + """Assert that no experiment_run row exists for the given id.""" + with engine.connect() as conn: + count = conn.execute( + text('SELECT COUNT(*) FROM public.experiment_run WHERE id = :id'), + {'id': experiment_run_id}, + ).scalar() + assert count == 0, ( + f'Expected no experiment_run row for id={experiment_run_id}, found {count}' + ) + + +def load_scenario(scenario_filename: str) -> dict[str, Any]: + """ + Load a test scenario JSON file from docs/test-scenarios/. + + Args: + scenario_filename: Filename without path (e.g. '01-linear-regression-basic.json'). + + Returns: + dict: Parsed scenario payload. + """ + scenario_path = ( + Path(__file__).parent.parent / 'docs' / 'test-scenarios' / scenario_filename + ) + with open(scenario_path) as f: + return json.load(f) diff --git a/e2e/test_cleanup_files_workflow.py b/e2e/test_cleanup_files_workflow.py new file mode 100644 index 0000000..be1b26e --- /dev/null +++ b/e2e/test_cleanup_files_workflow.py @@ -0,0 +1,108 @@ +""" +End-to-end tests for CleanupFiles workflow. + +Covers scenarios 3.x: cleanup of temporary local directories. +""" + +import pytest +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + +from e2e.helpers import make_workflow_id, start_and_await_workflow +from model_manager.workflows.cleanup_files import CleanupFiles + +# Matches Cleanup.dir_timestamp_pattern: name_YYYYMMDD_HHMMSS_microseconds +_STALE_DIR_OLD = 'stale_run_20200102_030405_000001' +_STALE_DIR_OLDER = 'stale_run_20191231_235959_999999' + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_3_1_1_cleanup_with_no_temp_dirs( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + tmp_path, +): + """Scenario 3.1.1 – Cleanup when the temp directory is empty. + + The cleanup_temp_directories activity should complete without error + and the workflow should finish successfully. + """ + # Use an empty temp directory as the reports path + empty_dir = tmp_path / 'reports_temp' + empty_dir.mkdir() + + result = await start_and_await_workflow( + temporal_test_env.client, + CleanupFiles.run, + {'temp_path': str(empty_dir)}, + make_workflow_id('test-s3-1-1'), + ) + + # Workflow returns None on success + assert result is None + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_3_1_2_cleanup_removes_old_temp_dirs( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + tmp_path, +): + """Scenario 3.1.2 – Cleanup removes stale subdirectories from the temp dir. + + Creates two subdirectories with timestamp suffixes inside the reports + temp directory and verifies the activity removes them. + """ + reports_dir = tmp_path / 'reports_temp' + reports_dir.mkdir() + + # Create two stale run directories (names must match cleanup activity regex) + stale1 = reports_dir / _STALE_DIR_OLD + stale2 = reports_dir / _STALE_DIR_OLDER + stale1.mkdir() + stale2.mkdir() + (stale1 / 'model.pkl').write_bytes(b'fake-model-data') + (stale2 / 'report.json').write_bytes(b'{"status": "old"}') + + result = await start_and_await_workflow( + temporal_test_env.client, + CleanupFiles.run, + {'temp_path': str(reports_dir)}, + make_workflow_id('test-s3-1-2'), + ) + + assert result is None + + # The activity should have cleaned up the stale directories + remaining = list(reports_dir.iterdir()) + assert len(remaining) == 0, ( + f'Expected all stale dirs to be removed, but found: {remaining}' + ) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_3_1_3_cleanup_nonexistent_temp_path( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + tmp_path, +): + """Scenario 3.1.3 – Cleanup with a temp_path that does not exist. + + The activity must handle a missing directory gracefully without + raising an unhandled exception, since the directory may have already + been cleaned by a previous run. + """ + nonexistent = str(tmp_path / 'does_not_exist' / 'reports') + + # Should not raise — the activity is expected to handle a missing path + result = await start_and_await_workflow( + temporal_test_env.client, + CleanupFiles.run, + {'temp_path': nonexistent}, + make_workflow_id('test-s3-1-3'), + ) + + assert result is None diff --git a/e2e/test_train_model_validation.py b/e2e/test_train_model_validation.py new file mode 100644 index 0000000..20e0031 --- /dev/null +++ b/e2e/test_train_model_validation.py @@ -0,0 +1,298 @@ +""" +End-to-end tests for TrainModel parameter validation paths. + +Covers scenarios 2.1.x: workflows that must terminate with +ORCHESTRATOR_VALIDATION_ERROR due to invalid parameter values. +""" + +import pytest +from temporalio.client import WorkflowFailureError +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + +from e2e.helpers import ( + assert_experiment_error, + insert_experiment_run, + load_scenario, + make_workflow_id, + start_and_await_workflow, +) +from model_manager.workflows.train_model import TrainModel + +# Base experiment_run ids for validation test scenarios (offset to avoid collision) +_VALIDATION_ID_BASE = 3000 + + +def _exception_chain_text(exc: BaseException) -> str: + """Concatenate messages from an exception __cause__/__context__ chain.""" + parts: list[str] = [] + cur: BaseException | None = exc + seen: set[int] = set() + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + text = str(cur).strip() + if text: + parts.append(text) + cur = cur.__cause__ or getattr(cur, '__context__', None) + return ' | '.join(parts).lower() + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_2_1_1_train_size_out_of_range( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 2.1.1 – train_size=5 violates the 10–100 business rule. + + Expected: workflow updates status → ORCHESTRATOR_VALIDATION_ERROR + and error_message references 'train_size'. + """ + experiment_run_id = _VALIDATION_ID_BASE + 1 + scenario = load_scenario('01-linear-regression-basic.json') + scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'train_size': 5} + insert_experiment_run(postgres_engine, experiment_run_id) + + with pytest.raises(Exception): + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s2-1-1'), + ) + + assert_experiment_error( + postgres_engine, + experiment_run_id, + expected_status='ORCHESTRATOR_VALIDATION_ERROR', + error_substr='train_size', + ) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_2_1_2_empty_variable_columns( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 2.1.2 – variable_columns=[] → ORCHESTRATOR_VALIDATION_ERROR.""" + experiment_run_id = _VALIDATION_ID_BASE + 2 + scenario = load_scenario('01-linear-regression-basic.json') + scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'variable_columns': []} + insert_experiment_run(postgres_engine, experiment_run_id) + + with pytest.raises(Exception): + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s2-1-2'), + ) + + assert_experiment_error( + postgres_engine, + experiment_run_id, + expected_status='ORCHESTRATOR_VALIDATION_ERROR', + error_substr='variable_columns', + ) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_2_1_3_invalid_date_format( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 2.1.3 – date_format='INVALID' is not in the allowed list.""" + experiment_run_id = _VALIDATION_ID_BASE + 3 + scenario = load_scenario('01-linear-regression-basic.json') + scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'date_format': 'INVALID'} + insert_experiment_run(postgres_engine, experiment_run_id) + + with pytest.raises(Exception): + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s2-1-3'), + ) + + assert_experiment_error( + postgres_engine, + experiment_run_id, + expected_status='ORCHESTRATOR_VALIDATION_ERROR', + error_substr='date_format', + ) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_2_1_4_whitespace_only_model_name( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 2.1.4 – model_name=' ' (whitespace) → ORCHESTRATOR_VALIDATION_ERROR.""" + experiment_run_id = _VALIDATION_ID_BASE + 4 + scenario = load_scenario('01-linear-regression-basic.json') + scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'model_name': ' '} + insert_experiment_run(postgres_engine, experiment_run_id) + + with pytest.raises(Exception): + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s2-1-4'), + ) + + assert_experiment_error( + postgres_engine, + experiment_run_id, + expected_status='ORCHESTRATOR_VALIDATION_ERROR', + error_substr='model_name', + ) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_2_1_5_unknown_model_type( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 2.1.5 – model_type='totally_unknown' → ORCHESTRATOR_VALIDATION_ERROR. + + The PluginStore will not find this model in the Gitea repo, causing + load_model_metadata to fail before validate_train_params is even called. + """ + experiment_run_id = _VALIDATION_ID_BASE + 5 + scenario = load_scenario('01-linear-regression-basic.json') + scenario = { + **scenario, + 'experiment_run_id': experiment_run_id, + 'model_type': 'totally_unknown_model', + } + insert_experiment_run(postgres_engine, experiment_run_id) + + with pytest.raises(Exception): + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s2-1-5'), + ) + + assert_experiment_error( + postgres_engine, + experiment_run_id, + expected_status='ORCHESTRATOR_VALIDATION_ERROR', + error_substr='totally_unknown_model', + ) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_2_1_6_missing_target_variable( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 2.1.6 – target_variable='' (empty string) → ORCHESTRATOR_VALIDATION_ERROR.""" + experiment_run_id = _VALIDATION_ID_BASE + 6 + scenario = load_scenario('01-linear-regression-basic.json') + scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'target_variable': ''} + insert_experiment_run(postgres_engine, experiment_run_id) + + with pytest.raises(Exception): + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s2-1-6'), + ) + + assert_experiment_error( + postgres_engine, + experiment_run_id, + expected_status='ORCHESTRATOR_VALIDATION_ERROR', + error_substr='target_variable', + ) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_2_1_7_missing_experiment_run_id( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, +): + """Scenario 2.1.7 – experiment_run_id missing → workflow raises ValueError immediately. + + No DB row is inserted because experiment_run_id is mandatory to even + know which row to update. The workflow should raise before any DB call. + """ + scenario = load_scenario('01-linear-regression-basic.json') + scenario = {k: v for k, v in scenario.items() if k != 'experiment_run_id'} + + with pytest.raises(WorkflowFailureError) as excinfo: + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s2-1-7'), + ) + combined = _exception_chain_text(excinfo.value) + assert 'experiment_run_id' in combined + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_2_1_8_missing_date_column( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, +): + """Scenario 2.1.8 – date_column missing in payload raises before workflow business validation.""" + scenario = load_scenario('01-linear-regression-basic.json') + scenario = {k: v for k, v in scenario.items() if k != 'date_column'} + + with pytest.raises(WorkflowFailureError) as excinfo: + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s2-1-8'), + ) + combined = _exception_chain_text(excinfo.value) + assert 'date_column' in combined + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_2_1_9_whitespace_date_column( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 2.1.9 – date_column=' ' must produce ORCHESTRATOR_VALIDATION_ERROR.""" + experiment_run_id = _VALIDATION_ID_BASE + 9 + scenario = load_scenario('01-linear-regression-basic.json') + scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'date_column': ' '} + insert_experiment_run(postgres_engine, experiment_run_id) + + with pytest.raises(Exception): + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s2-1-9'), + ) + + assert_experiment_error( + postgres_engine, + experiment_run_id, + expected_status='ORCHESTRATOR_VALIDATION_ERROR', + error_substr='date_column', + ) diff --git a/e2e/test_train_model_workflow.py b/e2e/test_train_model_workflow.py new file mode 100644 index 0000000..e11fc22 --- /dev/null +++ b/e2e/test_train_model_workflow.py @@ -0,0 +1,493 @@ +""" +End-to-end tests for TrainModel workflow – main workflow scenarios. + +Covers: + 1.1.x – Happy-path training (various scenarios from docs/test-scenarios/) + 1.2.x – Error paths (MinIO failure, missing DB row) +""" + +import pytest +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + +from e2e.helpers import ( + assert_experiment_error, + assert_experiment_run_name_set, + assert_experiment_status, + assert_no_experiment_row, + insert_experiment_run, + load_scenario, + make_workflow_id, + start_and_await_workflow, +) +from model_manager.workflows.train_model import TrainModel + + +# --------------------------------------------------------------------------- +# 1.1 – Happy paths +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_1_linear_regression_basic( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.1 – Linear Regression Basic (cenário 01). + + Validates the complete training pipeline end-to-end: + load_model_metadata → validate_train_params → train_model → + update_experiment_run (TRAINING_SUCCESS). + """ + scenario = load_scenario('01-linear-regression-basic.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-1'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_2_linear_regression_with_scaler( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.2 – Linear regression with Standard Scaler (cenário 02).""" + scenario = load_scenario('02-linear-regression-with-scaler.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-2'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_3_polynomial_regression_degree2_with_scaler( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.3 – Polynomial Regression Degree 2 with Standard Scaler (cenário 03).""" + scenario = load_scenario('03-polynomial-regression-degree2.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-3'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_4_polynomial_regression_degree3_with_scaler( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.4 – Polynomial regression degree 3 with Standard Scaler (cenário 04).""" + scenario = load_scenario('04-polynomial-regression-degree3.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-4'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_5_linear_regression_with_lags( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.5 – Linear Regression with lag_train/lag_val per variable (cenário 05).""" + scenario = load_scenario('05-linear-regression-with-lags.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-5'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_6_linear_regression_nan_interpolation( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.6 – nan_treatment='linear interpolation' (cenário 06).""" + scenario = load_scenario('06-linear-regression-nan-interpolation.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-6'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_7_linear_regression_static_window_removal( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.7 – rem_static_win=true with default static_threshold (cenário 07).""" + scenario = load_scenario('07-linear-regression-static-window-removal.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-7'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_8_linear_regression_with_limits( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.8 – support_filters with min/max limits per variable (cenário 08).""" + scenario = load_scenario('08-linear-regression-with-limits.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-8'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_9_polynomial_degree2_scaler_and_lags( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.9 – Polynomial degree 2, Standard Scaler and lags (cenário 09).""" + scenario = load_scenario('09-polynomial-degree2-with-scaler-and-lags.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-9'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_10_linear_regression_with_ar_opt_params( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.10 – Linear regression with include_ar in opt_params (cenário 10).""" + scenario = load_scenario('10-linear-regression-with-ar.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-10'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_11_linear_regression_static_threshold_custom( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.11 – rem_static_win=true with custom static_threshold (cenário 11).""" + scenario = load_scenario('11-linear-regression-static-threshold-custom.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-11'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_12_alternate_date_format_dd_mm_yyyy( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.12 – DATA column and dd/MM/yyyy format CSV (cenário 12).""" + scenario = load_scenario('12-angular-test-date-format.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run( + postgres_engine, + experiment_run_id, + file_name='training_data_dd_mm_yyyy.csv', + ) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-12'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_13_alternate_csv_narrow_date_window( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.13 – Same alternate CSV as 12 with date window (cenário 13).""" + scenario = load_scenario('13-angular-test-double-date-column.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run( + postgres_engine, + experiment_run_id, + file_name='training_data_dd_mm_yyyy.csv', + ) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-13'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_14_polynomial_with_support_filters( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.14 – Polynomial degree 4, Standard Scaler, line support filters (cenário 14).""" + scenario = load_scenario('14-angular-test-polynomial-support-filters.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-14'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_15_linear_regression_custom_target_column_name( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.15 – Target column is not named ``target``; report path uses target_variable.""" + scenario = load_scenario('15-linear-regression-custom-target-column.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run( + postgres_engine, + experiment_run_id, + file_name='training_data_custom_target.csv', + ) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-15'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_16_naive_timestamp_header_column( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.16 – CSV uses ``Timestamp`` header; snake_case date_column/date_format.""" + scenario = load_scenario('16-linear-regression-naive-timestamp-header.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run( + postgres_engine, + experiment_run_id, + file_name='training_data_timestamp_naive.csv', + ) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-16'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_17_linear_regression_blank_timestamp_row_dropped( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.17 – CSV has one empty timestamp cell; row is dropped and training succeeds.""" + scenario = load_scenario('17-linear-regression-blank-timestamp-row.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run( + postgres_engine, + experiment_run_id, + file_name='training_data_blank_timestamp_row.csv', + ) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-17'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +# --------------------------------------------------------------------------- +# 1.2 – Error paths +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_2_1_minio_file_not_found( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.2.1 – Training file does not exist in MinIO → TRAINING_ERROR.""" + scenario = load_scenario('01-linear-regression-basic.json') + scenario = {**scenario, 'experiment_run_id': 2001, 'file_name': 'does_not_exist.csv'} + experiment_run_id = 2001 + insert_experiment_run(postgres_engine, experiment_run_id) + + with pytest.raises(Exception): + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-2-1'), + ) + + assert_experiment_error( + postgres_engine, + experiment_run_id, + expected_status='TRAINING_ERROR', + error_substr='does_not_exist', + ) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_2_2_experiment_run_id_not_in_db( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.2.2 – experiment_run_id row absent → update_experiment_run raises.""" + scenario = load_scenario('01-linear-regression-basic.json') + scenario = {**scenario, 'experiment_run_id': 9999} + # Intentionally NOT inserting the row + + with pytest.raises(Exception): + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-2-2'), + ) + + assert_no_experiment_row(postgres_engine, 9999) diff --git a/git_requirements_mapping.txt b/git_requirements_mapping.txt new file mode 100644 index 0000000..3fb32e4 --- /dev/null +++ b/git_requirements_mapping.txt @@ -0,0 +1,2 @@ +sientia_do: git+ssh://git@github.com/Aignosi/sientia-dataops-library.git +sientia_model: git+ssh://git@github.com/Aignosi/sientia-model-library.git \ No newline at end of file diff --git a/input-sample.json b/input-sample.json new file mode 100644 index 0000000..2d40f1e --- /dev/null +++ b/input-sample.json @@ -0,0 +1,38 @@ +{ + "experiment_run_id": 1001, + "variable_columns": ["feature_a", "feature_b"], + "target_variable": "target", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "feature_a": 0, + "feature_b": 0 + }, + "lag_val": { + "feature_a": 0, + "feature_b": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "Standard Scaler" + }, + "opt_params": {} +} diff --git a/input-sample.md b/input-sample.md new file mode 100644 index 0000000..a1d1d3c --- /dev/null +++ b/input-sample.md @@ -0,0 +1,76 @@ +##### Insert a new experiment run + +```sql +-- Optional: remove a previous run with the same id +DELETE FROM public.experiment_run WHERE experiment_run_id = 1001; +``` + +```sql +INSERT INTO public.experiment_run +(experiment_name, run_name, username, status, error_message, created_at, +updated_at, bucket_name, file_name, request_data, orchestrator_response_data) +VALUES( + 'test-experiment-name', + 'test-run-name', + 'test-username', + 'ORCHESTRATOR_WAITING_PROC', + null, + now(), + now(), + 'model-training', + 'training_data.csv', + '{"experiment_run_id":1001,"variable_columns":["feature_a","feature_b"],"target_variable":"target","bucket_name":"model-training","file_name":"training_data.csv","line_separator":",","decimal_separator":".","train_size":80,"shuffle":true,"model_name":"Linear Regression","model_type":"linear_regression","data_model_kwargs":{"lag_train":{"feature_a":0,"feature_b":0},"lag_val":{"feature_a":0,"feature_b":0},"nan_treatment":"drop"},"model_kwargs":{"degree":1,"scaler_name":"Standard Scaler"},"opt_params":{}}', + null +); +``` + +##### Upload the input dataset to MinIO + +```bash +mc cp input_dataset.csv suse/model-training/training-sample-dataset-1001.csv +``` + +##### Temporal input payload sample + +Keys match `TrainModelParams.from_dict` in `model_manager/utils/models/train_model_params.py`: every field passed to `_check_none` must be present, including **`date_column`**; `model_metadata` must be non-empty for `validate_business_rules()`. You may omit **`date_format`** (defaults to `yyyy-MM-dd HH:mm:ss`). Omit optional keys (`random_state`, `val_file_name`, `model_id`) when defaults or `None` apply. + +```json +{ + "experiment_run_id": 1001, + "variable_columns": ["feature_a", "feature_b"], + "target_variable": "target", + "bucket_name": "model-training", + "file_name": "training-sample-dataset-1001.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "test-runtime-linear-regression-model", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "feature_a": 0, + "feature_b": 0 + }, + "lag_val": { + "feature_a": 0, + "feature_b": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "Standard Scaler" + }, + "opt_params": {} +} +``` diff --git a/input_dataset.csv b/input_dataset.csv new file mode 100644 index 0000000..ccaa4ae --- /dev/null +++ b/input_dataset.csv @@ -0,0 +1,11 @@ +timestamp,Counter,Rollout,Square +2026-01-01 00:00:00,1.0,2.0,5.1 +2026-01-01 00:01:00,2.0,2.0,5.9 +2026-01-01 00:02:00,2.0,3.0,8.3 +2026-01-01 00:03:00,3.0,3.0,9.2 +2026-01-01 00:04:00,3.0,4.0,10.8 +2026-01-01 00:05:00,4.0,4.0,11.7 +2026-01-01 00:06:00,4.0,5.0,14.1 +2026-01-01 00:07:00,5.0,5.0,15.15 +2026-01-01 00:08:00,5.0,6.0,17.2 +2026-01-01 00:09:00,6.0,6.0,17.9 \ No newline at end of file diff --git a/model-manager-plugin-store-migration-plan.md b/model-manager-plugin-store-migration-plan.md new file mode 100644 index 0000000..63456f0 --- /dev/null +++ b/model-manager-plugin-store-migration-plan.md @@ -0,0 +1,385 @@ +--- +tags: + - engineering + - sientia + - runtime-system + - model-manager + - plugin-store + - migration-plan +created: 2026-03-02 +modified: 2026-03-02 +created_by: Vitor Pimentel +modified_by: Vitor Pimentel +status: draft +--- + +# Sientia Model Manager — PluginStore Migration Plan + +> Implementation plan for migrating `sientia-dataops-model-manager` to use the Sientia PluginStore for runtime installation and model retrieval, aligned with the runtime architecture described in `analytics.md`. + +## Summary + +1. [[#Objectives and Scope|Objectives and Scope]] — What this migration must achieve +2. [[#Existing State Overview (model-manager)|Existing State Overview]] — Current responsibilities and coupling points +3. [[#Requirements Mapping|Requirements Mapping]] — Functional and non-functional requirements +4. [[#Target Architecture|Target Architecture]] — Desired runtime and model-loading architecture +5. [[#Implementation Plan|Implementation Plan]] — Phased, detailed changes to apply +6. [[#Testing Strategy|Testing Strategy]] — How to validate the new behavior +7. [[#Rollout and Migration Strategy|Rollout and Migration Strategy]] — How to safely roll out and deprecate old paths +8. [[#Potential Model Library Changes|Potential Model Library Changes]] — Expected impact on `sientia-model-library` +9. [[#Related Documents|Related Documents]] — Cross-links to supporting documents + +--- + +## Objectives and Scope + +The goal of this work is to evolve `sientia-dataops-model-manager` so that: + +- It **detects the runtime** before the worker starts, using the `RUNTIME` environment variable. +- It **installs the selected runtime** using the PluginStore runtime interface from `sientia_model.model_repository.plugin_store`: + - `PluginStore.install_runtime(runtime_name)`. +- It **uses PluginStore to obtain models from the store**, instead of constructing them from the local ML template / mlops library: + - Pipelines call `PluginStore.get_model(...)` to obtain `SientiaModel` instances. + - The previous “in-repo model implementation plus mlops library” path is removed. + +Out of scope: + +- Changing Temporal workflow semantics (queues, retry policies, etc.). +- Replacing the existing MLflow-based tracking and reporting; these remain the responsibility of `ModelRepository` and the reporting utilities. + +--- + +## Existing State Overview (model-manager) + +The `sientia-dataops-model-manager` application currently: + +- **Worker orchestration** (`model_manager/worker/worker.py`) + - Reads Temporal configuration from env (`TEMPORAL_HOST`, `TEMPORAL_NAMESPACE`, task queues). + - Sets up observability (Prometheus, metrics, Sientia logger). + - Builds connector configs for Postgres, MLflow, MinIO, MongoDB. + - Instantiates `Activities` and schedules, then starts Temporal workers. + - Does not validate or install any “runtime” concept before worker startup. + +- **Training pipeline** + - `Training` activity (`model_manager/activities/training.py`) coordinates: + - Parameter validation (`validate_train_params`). + - Training execution via `TrainingRepository`. + - Saving trained models and artifacts via `ModelRepository` to MLflow. + - `TrainingRepository` (`model_manager/utils/repository/training_repository.py`): + - Loads and preprocesses CSV data (via `DataPreprocessor`). + - Trains a local `LinearRegressionModel` defined in `model_manager.sientia.models`. + - Computes metrics and builds a `TrainModelResult` for downstream steps. + - `ModelRepository` (`model_manager/utils/repository/model_repository.py`): + - Uses `ModelServing` and `Reports` to: + - Generate reports and CSV artifacts. + - Log parameters, metrics and models into MLflow. + - Today, production models are identified using **stages** (for example `Production`) in the Model Registry; aliases like `@production` are not yet used. + - MLflow-related logic here overlaps conceptually with the MLflow interactions implemented in `sientia-dataops-laborious_temporal`, which motivates extracting a **shared MLflow repository** into `sientia-dataops-library` (see `mlflow-shared-repository-migration-plan`). + +- **Coupling to models and runtimes** + - Training is tightly coupled to internal classes (`DataPreprocessor`, `LinearRegressionModel`). + - No runtime installation; env assumed ready. + - The system does not yet use PluginStore’s `get_model` or `install_runtime` capabilities. + +**MLflow:** All MLflow operations (lookup, promotion, etc.) → [[mlflow-shared-repository-migration-plan|shared repository]]. Model Manager uses the interface; it does not implement these concepts. + +### Current vs Target — High-level Flow + +```mermaid +flowchart LR + subgraph currentState [Current State — Model Manager] + direction TB + WorkerMM["Temporal Worker\n(model_manager/worker.py)"] + ActivitiesMM["Activities\n(training, cleanup, etc.)"] + TrainRepo["TrainingRepository\n(local DataPreprocessor + LinearRegressionModel)"] + ModelRepoMM["ModelRepository\n(MLflow + reports)"] + + WorkerMM -->|"Temporal tasks"| ActivitiesMM + ActivitiesMM -->|"train_model activity"| TrainRepo + TrainRepo -->|"TrainModelResult"| ModelRepoMM + ModelRepoMM -->|"experiments, runs, artifacts"| MLflowMM["MLflow Server"] + end + + subgraph targetState [Target State — Model Manager] + direction TB + WorkerMM2["Temporal Worker\n+ Runtime bootstrap\n(read RUNTIME, install runtime)"] + ActivitiesMM2["Activities\n(+ ModelProvider)"] + PluginStoreNode["PluginStore\n(Gitea model store)"] + SientiaWrapper["SientiaModel wrapper\n(from store.get_model)"] + ModelRepoMM2["ModelRepository\n(MLflow + reports)"] + + WorkerMM2 -->|"install_runtime(RUNTIME)"| PluginStoreNode + WorkerMM2 -->|"Temporal tasks"| ActivitiesMM2 + ActivitiesMM2 -->|"get_training_model"| PluginStoreNode + PluginStoreNode -->|"SientiaModel instance"| SientiaWrapper + ActivitiesMM2 -->|"call train(...) on wrapper"| SientiaWrapper + SientiaWrapper -->|"TrainModelResult-compatible data"| ModelRepoMM2 + ModelRepoMM2 -->|"experiments, runs, artifacts"| MLflowMM2["MLflow Server"] + end +``` + +### Current vs Target — Training Hot Path (Code Sketch) + +Current hot path in `TrainingRepository.train` (simplified): + +```python +data = load_data(uploaded_file, params.line_separator, params.decimal_separator) +data = _ensure_date_column_parsed(data, params) +data = self._configure_datetime_index(data, params) + +process_data = self._init_data_preprocessor(params) +process_data.fit(data) +data_view = process_data.transform(data) + +x_train, x_test, y_train, y_test = split_train_test(...) + +regr = LinearRegressionModel( + target_variable=params.target_variable, + variable_columns=params.variable_columns, + degree=params.degree, + interaction_only=params.interaction_only, +) +regr.fit(pd.concat([x_train, y_train], axis=1)) +``` + +Target hot path with PluginStore + `SientiaModel` (conceptual): + +```python +data = load_and_preprocess(uploaded_file, params) # keep existing preprocessing +train_df, val_df = build_train_val_splits(data, params) # explicit train/val + +wrapper = model_provider.get_training_model( + model_name=params.model_name, + runtime=os.environ["RUNTIME"], + opt_params={"env": params.environment}, + model_kwargs={}, + data_model_kwargs={}, +) + +wrapper.train( + train_data=train_df, + val_data=val_df, + target=params.target_variable, +) + +# From here, metrics and artifacts are computed based on wrapper outputs +``` + +--- + +## Requirements Mapping + +### Functional Requirements + +| ID | Requirement | Description | Impacted Areas | +|-------|----------------------------------------------|------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------| +| FR-01 | Runtime detection via env var | Before starting workers, read `RUNTIME` env var and fail fast if missing or empty | `model_manager/worker/worker.py` | +| FR-02 | Runtime installation via PluginStore | Install the runtime defined in `RUNTIME` using `PluginStore.install_runtime(runtime_name)` | `worker.py`, `sientia_model.model_repository.plugin_store.PluginStore` | +| FR-03 | PluginStore-based model retrieval | Use `PluginStore.get_model(...)` to obtain `SientiaModel` instances for training | `TrainingRepository` (or new adapter), `Training` activity | +| FR-04 | Remove mlops-library-based construction | Do not construct models directly from `model_manager.sientia.models`; remove the mlops path | `TrainingRepository`, `model_manager.sientia.models`, dependency graph | +| FR-05 | Use shared MLflow repository | Delegate all MLflow ops (runs, metrics, artifacts, production lookup) to `SientiaMLflowRepository`; keep reports and `TrainModelResult` in Model Manager | `ModelRepository`, [[mlflow-shared-repository-migration-plan]] | +| FR-06 | Centralized PluginStore configuration | Configure PluginStore (Gitea URL, repo, auth, PyPI mirror) via env/config | Config helper, `worker.py` | + +### Non-Functional Requirements + +| ID | Requirement | Description | Impacted Areas | +|--------|------------------------------------------|-------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------| +| NFR-01 | Robust runtime bootstrap | If runtime installation fails, the worker must not start; errors must be explicit in logs/metrics | `worker.py`, PluginStore error handling | +| NFR-02 | Single path | Use PluginStore + SientiaModel path only; no toggling or parallel old path | `worker.py`, Training activities | +| NFR-03 | Observability and debuggability | Logs and metrics must cover runtime detection, install attempts and PluginStore interactions | Logging around runtime and PluginStore | +| NFR-04 | Testability | Unit and integration tests must cover runtime detection, install, and model retrieval | `tests/worker`, `tests/activities`, new PluginStore tests | +| NFR-05 | Security | No credentials hard-coded in source; rely on env/secret management | Config helpers, deployment manifests | + +--- + +## Target Architecture + +### Runtime bootstrap + +- **New required env var:** `RUNTIME` (name of the runtime; must match a runtime in the store index). +- **New PluginStore configuration env vars** (names to be finalized): + - `STORE_BASE_URL`, `STORE_OWNER`, `STORE_REPO` + - Optional: `STORE_BRANCH`, `STORE_USERNAME`, `STORE_PASSWORD`, `PYPI_INDEX_URL`, `PYPI_USERNAME`, `PYPI_PASSWORD` +- **Worker startup sequence** in `main()`: + 1. Initialize logger and basic metadata as today. + 2. Read `RUNTIME` from env; if missing/empty → log critical error and exit. + 3. Build `PluginStore` instance using configuration env vars. + 4. Call `store.install_runtime(runtime_name=RUNTIME)`; on failure → log error, set `APP_UP` metric to 0 and exit. + 5. Only after successful runtime installation → build connector configs, instantiate Activities, connect to Temporal. + +### PluginStore usage in training + +- **ModelProvider abstraction** (e.g. `model_manager/utils/model_provider.py`): + - Holds a `PluginStore` instance and logger. + - Exposes `get_training_model(model_name, runtime, opt_params, model_kwargs, data_model_kwargs) -> SientiaModel` via `store.get_model(...)`. +- **TrainingRepository integration:** + - Prepare input data using existing preprocessing logic. + - Ask ModelProvider for a `SientiaModel` instance (`model_name` from params, `runtime` from `RUNTIME`). + - Call the public `train(...)` method of the wrapper; collect outputs and metadata only through the public API. + - Build `TrainModelResult` from train/test splits, predictions, and artifacts required by `ModelRepository`. + - Model behavior lives in `SientiaModel`; manager focuses on orchestration and reports. + +--- + +## Implementation Plan + +### Phase 0 — Design Alignment + +- **P0-01**: Confirm with stakeholders: + - The expected values and semantics for `RUNTIME` (naming convention, mapping to store runtimes). + - Whether the manager will ever need to support more than one runtime per process (current assumption: no). +- **P0-02**: Decide the config strategy: + - Pure environment variables vs a config file + env overrides. +- **P0-03**: Validate how `model_name` will be passed into the training workflow: + - Confirm or extend `TrainModelParams` to carry `model_name` and runtime-related fields as needed. + +### Phase 1 — Runtime Detection and Installation + +- **P1-01**: Extend `worker.py` docs and configuration: + - Add `RUNTIME` to the environment variables list in the module docstring. + - Document failure behavior when `RUNTIME` is missing or empty. +- **P1-02**: Implement `build_plugin_store_from_env` helper: + - New function that: + - Reads `STORE_*` and `PYPI_*` env vars. + - Creates and returns a `PluginStore` instance with appropriate logger. + - Place it either in `worker.py` or in a dedicated utility module (e.g. `model_manager/utils/plugin_store_config.py`). +- **P1-03**: Integrate runtime installation into `main()`: + - Before any Temporal client initialization: + - Read `runtime_name = os.getenv("RUNTIME")`. + - Build PluginStore via the helper. + - Call `install_runtime(runtime_name)`. + - Log: + - Start and end of runtime installation. + - List of installed requirements (name and version). + - On failure: + - Emit a clear message (including runtime name and store repo). + - Mark the app as DOWN in metrics. + - Exit with non-zero status. + +### Phase 2 — ModelProvider and Training Integration + +- **P2-01**: Introduce `ModelProvider` abstraction: + - Implement `ModelProvider` with: + - A reference to the shared `PluginStore`. + - Methods for retrieving models for training: + - `get_training_model(model_name, runtime, opt_params, model_kwargs, data_model_kwargs)`. + - Ensure it logs: + - Model and runtime names. + - Cache hits/misses when appropriate. +- **P2-02**: Wire ModelProvider into Activities: + - Update the construction of `Activities` in `worker.py` to accept: + - A `ModelProvider` or a `PluginStore` that can be wrapped inside the `Training` activity. + - Update `Training.__init__` signature to accept this new dependency and store it as an attribute. +- **P2-03**: Adjust `TrainingRepository` to use `SientiaModel`: + - In `train`: + 1. Keep or refine current CSV loading and preprocessing pipeline (DataPreprocessor, split into train/test). + 2. Use `ModelProvider` to retrieve the model: + - `model_name` from `TrainModelParams`. + - `runtime` from `RUNTIME`. + - `opt_params` mirroring how `test_plugin_store.py` interacts with models. + 3. Call the model’s `train` method and produce the same core artifacts the current code expects: + - Train/test splits. + - Metrics and predictions for `TrainModelResult`. + - In `after_train_calculation`: + - Either re-use metrics from the SientiaModel, or continue calculating MSE/MAE/R² as a verification step. + +### Phase 3 — Removing mlops Dependencies + +- **P3-01**: Identify all usage points of: + - `LinearRegressionModel`. + - `DataPreprocessor` where behavior overlaps with what SientiaModel already does. +- **P3-02**: Remove mlops-based code: + - Remove unused mlops-library-based training hooks. + - Simplify `TrainingRepository` to delegate as much as possible to SientiaModel logic. + - Use PluginStore + `SientiaModel` path only (no migration flag or parallel old path). + +### Phase 4 — ModelRepository, Shared MLflow Repository and Reporting Alignment + +- **P4-01**: Ensure `TrainModelResult` is correctly populated: + - Confirm that: + - `x_train`, `x_test`, `y_train`, `y_test`, `y_pred`, `y_train_pred` are provided by the new flow. + - Any fields required by `_generate_report` and `_save_run` remain available. +- **P4-02**: Delegate all MLflow operations to `SientiaMLflowRepository` (see [[mlflow-shared-repository-migration-plan]]); keep in Model Manager only reporting orchestration and `TrainModelResult` construction. +- **P4-03**: Validate that MLflow reports remain consistent: + - Run a side-by-side comparison between: + - A run from the previous (mlops-based) pipeline. + - A PluginStore-based run for the same dataset/experiment using the shared repository. + - Compare: + - Logged parameters. + - Metrics. + - Artifacts (reports, CSVs, equation JSON). + +--- + +## Testing Strategy + +- **T1 — Unit tests** + - Worker: Test behavior when `RUNTIME` is missing or empty; test that `install_runtime` is called with the correct runtime name (mock PluginStore). + - ModelProvider: Test that it calls `PluginStore.get_model` with the expected arguments. + - TrainingRepository: Test that it uses the model returned by PluginStore instead of `LinearRegressionModel`. +- **T2 — Integration tests** + - Set up a test store with a minimal model and runtime. + - Run a full training workflow: verify runtime installation first; confirm model is retrieved and training completes; validate metrics and artifacts via MLflow. +- **T3 — Regression tests** + - Run the same experiment once with the old pipeline and once with the PluginStore-based pipeline. + - Compare key results (metrics and artifacts) to ensure differences are understood and acceptable. + +--- + +## Rollout and Migration Strategy + +- **R1 — Criar modelos com a nova arquitetura** + - Garantir que o pipeline de modelagem (Factory/Warehouse/Store) consiga produzir modelos: + - Encapsulados em wrappers que estendem `SientiaModel`. + - Com interface estável para `train`, `retrain` (quando existir), `predict`/`transform` e `store_model`. + - Publicar um conjunto inicial de modelos “pilot” no store que será consumido pelo Model Manager. + +- **R2 — Subir um ou mais runtimes para esses modelos** + - Configurar e instalar runtimes específicos para os novos modelos, alinhados ao `RUNTIME` esperado pelo Model Manager: + - Verificar que cada runtime contém todas as dependências necessárias (via PluginStore / runtime installer). + - Validar que, em um ambiente de teste, o runtime consegue: + - Instalar bibliotecas. + - Carregar o wrapper via PluginStore e executar pelo menos um ciclo de treino de ponta a ponta. + +- **R3 — Colocar os novos modelos para rodar no Model Manager** + - Integrar o uso de `PluginStore.get_model(...)` na pipeline de treino: + - Direcionar um subconjunto de fluxos de treinamento para os modelos “pilot” oriundos do store. + - Monitorar: + - Estabilidade dos workers. + - Tempo de treino e consumo de recursos. + - Artefatos e métricas geradas no MLflow. + - Quando estáveis, coordenar com as equipes de produto/negócio para considerar esses modelos como candidatos a produção e, quando já estiverem usando MLflow 3+ com wrappers, promover esses modelos para produção usando aliases (`@production`). + +- **R4 — Migrar progressivamente os demais modelos** + - Definir uma ordem de migração por domínio/família de modelo (por exemplo: modelos de predição de série temporal, modelos de classificação, etc.): + - Para cada modelo legado: + - Criar/ajustar o wrapper `SientiaModel` correspondente no Factory/Warehouse. + - Garantir que o modelo passe a ser entregue pelo store e consumido via PluginStore. + - Executar o ciclo de testes (T1–T3) descrito na seção anterior. + - Após migrar todas as famílias de modelo: + - Remover o caminho de código que instancia diretamente `LinearRegressionModel` e demais classes locais. + - Limpar variáveis de configuração relacionadas à pipeline antiga (mlops library local). + - Assumir como padrão único: Model Manager treinando apenas modelos vindos do store, via wrappers `SientiaModel` e, quando aplicável, com resolução de produção por aliases em MLflow 3+. + +--- + +## Potential Model Library Changes + +- **Clarifying the SientiaModel training interface:** + - Ensure that `SientiaModel` provides a stable way to train (including validation split handling). + - A clear contract for returning predictions and metrics. + - Access to any internal state needed for `TrainModelResult` construction. +- **Improving PluginStore ergonomics:** + - Optionally add a helper to build `PluginStore` from environment variables (reusable across applications). + - More structured exceptions for missing runtime definitions, missing models, network and authentication issues. +- These changes should be coordinated so that model manager and any other consumers can rely on a consistent, documented behavior. + +--- + +## Related Documents + +- [[mlflow-shared-repository-migration-plan|MLflow Shared Repository Migration Plan]] — Concepts implemented in the common interface (production lookup, wrapper loading, promotion, etc.) +- [[analytics-implementation-plan|Runtime Analytics Helm Implementation Plan]] +- [[analytics|Runtime Analytics Architecture and Analysis]] +- [[../model-plugin-system/06-end-to-end-flow|Model Plugin System — End-to-End Flow]] + diff --git a/model_manager/__init__.py b/model_manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_manager/activities/__init__.py b/model_manager/activities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_manager/activities/activities.py b/model_manager/activities/activities.py new file mode 100644 index 0000000..f81212a --- /dev/null +++ b/model_manager/activities/activities.py @@ -0,0 +1,166 @@ +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from typing import Any + + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.observability.logger import Logger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.repository.minio_repository_sync import MinioRepository + from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository + from sientia_model.model_repository.plugin_store import PluginStore + + from model_manager.activities.cleanup import Cleanup + from model_manager.activities.experiment_tracking import ExperimentTracking + from model_manager.activities.training import Training + + +class Activities(ExperimentTracking, Training, Cleanup): + """ + Main activities orchestrator for the Model Manager system. + + This class combines functionality from multiple activity classes to provide + a unified interface for all workflow operations. It manages database connections, + MLFlow model interactions, MinIO storage operations, and cleanup operations. + + The class implements multiple inheritance to combine specialized functionality: + - ExperimentTracking: ML experiment lifecycle tracking and database operations + - Training: ML model training operations with MLFlow and MinIO integration + - Cleanup: File and directory cleanup operations for MinIO and local filesystem + + Attributes: + postgres_config (dict): PostgreSQL connection configuration + mlflow_config (dict): MLFlow server configuration + minio_config (dict): MinIO storage configuration + logger (Logger): Logging and observability instance + notification_handler (NotificationHandler): Notification management instance + """ + + def __init__( + self, + postgres_config: dict[str, Any], + mlflow_config: dict[str, Any], + minio_config: dict[str, Any], + plugin_store: PluginStore, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ): + """ + Initialize the Activities orchestrator with all required configurations. + + This constructor initializes all parent classes with their respective + configurations and sets up the foundation for all activity operations. + + Args: + postgres_config: PostgreSQL connection configuration dictionary + Required keys: host, port, user, password, dbname, min_connections, max_connections + mlflow_config: MLFlow server configuration dictionary + Required keys: host, port, username, password + minio_config: MinIO storage configuration dictionary + Required keys: endpoint_url, access_key, secret_key, region, use_ssl, default_bucket + logger: Logger instance for observability and debugging + notification_handler: Notification handler for alerts and monitoring + + Raises: + Exception: If any parent class initialization fails + """ + + ExperimentTracking.__init__( + self, + host=postgres_config['host'], + port=postgres_config['port'], + user=postgres_config['user'], + password=postgres_config['password'], + dbname=postgres_config['dbname'], + min_connections=postgres_config['min_connections'], + max_connections=postgres_config['max_connections'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + self.mlflow_repository = SientiaMLflowRepository( + host=mlflow_config['url'], + username=mlflow_config['username'], + password=mlflow_config['password'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + # MinIO repository used for all object storage operations + endpoint_url = minio_config['endpoint_url'] + # MinioRepository expects the endpoint without scheme + if endpoint_url.startswith('http://'): + endpoint = endpoint_url.removeprefix('http://') + elif endpoint_url.startswith('https://'): + endpoint = endpoint_url.removeprefix('https://') + else: + endpoint = endpoint_url + + self.minio_repository = MinioRepository( + endpoint=endpoint, + access_key=minio_config['access_key'], + secret_key=minio_config['secret_key'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + secure=minio_config['use_ssl'], + bucket=minio_config['default_bucket'], + ) + + Training.__init__( + self, + mlflow_repository=self.mlflow_repository, + plugin_store=plugin_store, + minio_repository=self.minio_repository, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + Cleanup.__init__( + self, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + def __del__(self): + """ + Destructor to safely handle cleanup during garbage collection. + + This prevents AttributeError when the parent Postgres.__del__ tries to access + self.engine in objects with multiple inheritance. Only attempts cleanup if + the engine attribute exists. + """ + # Only call parent __del__ if engine attribute exists + # This prevents AttributeError in multiple inheritance scenarios + if hasattr(self, 'engine'): + try: + # Call parent class __del__ if it exists + if hasattr(super(), '__del__'): # pragma: no cover + super().__del__() # pragma: no cover + except Exception: # noqa: S110, BLE001 # pragma: no cover + # Silently ignore errors during garbage collection + # Logging here could cause issues if logger is already destroyed + pass + + def shutdown(self): + """ + Gracefully shutdown all activities and clean up resources. + + This method ensures proper cleanup of all resources including: + - PostgreSQL connection pools (via ExperimentTracking) + - Any other resources that need explicit cleanup + + The method should be called before the application terminates to ensure + proper resource cleanup and prevent resource leaks. + + Prefer calling this method explicitly rather than relying on __del__. + """ + ExperimentTracking.close(self) + self.info('Postgres client closed') + SientiaMonitoring.shutdown(self) diff --git a/model_manager/activities/cleanup.py b/model_manager/activities/cleanup.py new file mode 100644 index 0000000..8b53012 --- /dev/null +++ b/model_manager/activities/cleanup.py @@ -0,0 +1,177 @@ +""" +Cleanup activities for removing stale files from local filesystem. + +This module provides activities for cleaning up temporary files and directories +that are older than the configured retention period. It operates independently +of the database, using timestamps embedded in filenames. +""" + +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + import os + import re + import shutil + import traceback + from datetime import datetime, timedelta + from typing import Any + + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + + from model_manager.runtime_paths import REPORTS_TEMP_DIR + + RETENTION_HOURS = int(os.getenv('CLEANUP_RETENTION_HOURS', '24')) + DRY_RUN = os.getenv('CLEANUP_DRY_RUN', 'false').lower() == 'true' + + +class Cleanup(SientiaMonitoring): + """ + Activity for cleaning up stale files and directories. + + This activity extends SientiaMonitoring and handles cleanup of: + - Local temporary directories with timestamp suffixes + """ + + def __init__( + self, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ): + """ + Initialize Cleanup activity. + + Args: + logger: Logger instance for observability + notification_handler: Handler for sending notifications + metrics_controller: Controller for metrics emission + """ + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) + + # Configuration from environment variables + self.retention_hours = RETENTION_HOURS + self.dry_run = DRY_RUN + + # Regex patterns for timestamp extraction + self.dir_timestamp_pattern = re.compile( + r'^(.+)_(\d{8}_\d{6}_\d{6})$' + ) # name_YYYYMMDD_HHMMSS_microseconds + + @activity.defn(name='cleanup_temp_directories') + def cleanup_temp_directories(self, input_data: dict[str, Any]) -> None: + """ + Clean up stale temporary directories based on timestamp in directory name. + + This activity scans the reports/temp directory for subdirectories following + the pattern '{name}_{timestamp}' where timestamp is in YYYYMMDD_HHMMSS_microseconds format. + Directories older than the retention period are deleted. + + Args: + input_data: Cleanup configuration containing: + - metadata (dict): Workflow execution metadata + - temp_path (str): Path to temp directory (optional, defaults to reports/temp) + + Returns: + None: Results are logged and tracked via metrics + + Raises: + Exception: If cleanup fails (after sending notification) + """ + metadata = input_data.get('metadata', {}) + temp_path = input_data.get('temp_path', REPORTS_TEMP_DIR) + + cutoff_time = datetime.now() - timedelta(hours=self.retention_hours) + + try: + self.info( + f'Starting local directory cleanup - Path: {temp_path}, ' + f'Retention: {self.retention_hours}h, Dry run: {self.dry_run}', + metadata, + ) + + if not os.path.exists(temp_path): + self.warning(f'Temp directory does not exist: {temp_path}', metadata) + return + + directories_scanned = 0 + directories_deleted = 0 + errors = [] + + for item_name in os.listdir(temp_path): + item_path = os.path.join(temp_path, item_name) + + if not os.path.isdir(item_path): + continue + + directories_scanned += 1 + + # Extract timestamp from directory name + match = self.dir_timestamp_pattern.match(item_name) + if not match: + self.debug( + f'Skipping directory without timestamp pattern: {item_name}', metadata + ) + continue + + timestamp_str = match.group(2) + try: + # Parse YYYYMMDD_HHMMSS_microseconds + dir_time = datetime.strptime(timestamp_str, '%Y%m%d_%H%M%S_%f') + + if dir_time < cutoff_time: + age_hours = (datetime.now() - dir_time).total_seconds() / 3600 + + if self.dry_run: + self.info( + f'[DRY RUN] Would delete directory: {item_name} (age: {age_hours:.1f}h)', + metadata, + ) + directories_deleted += 1 + else: + try: + shutil.rmtree(item_path) + self.info( + f'Deleted stale directory: {item_name} (age: {age_hours:.1f}h)', + metadata, + ) + directories_deleted += 1 + except OSError as e: + error_msg = f'Failed to delete directory {item_name}: {str(e)}' + errors.append(error_msg) + self.error(error_msg, metadata) + else: + age_hours = (datetime.now() - dir_time).total_seconds() / 3600 + self.debug( + f'Keeping recent directory: {item_name} (age: {age_hours:.1f}h)', + metadata, + ) + + except ValueError as e: + error_msg = f'Invalid timestamp format in directory {item_name}: {str(e)}' + errors.append(error_msg) + self.error(error_msg, metadata) + + self.info( + f'Directory cleanup completed - Scanned: {directories_scanned}, ' + f'Deleted: {directories_deleted}, Errors: {len(errors)}', + metadata, + ) + + except Exception as e: + error_msg = f'Error in directory cleanup: {str(e)}' + trace = traceback.format_exc() + + self.send_notification( + metadata=metadata, + notification_id='CLEANUP_DIRECTORIES_ERROR', + message=error_msg, + block='cleanup_temp_directories', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + + raise diff --git a/model_manager/activities/experiment_tracking.py b/model_manager/activities/experiment_tracking.py new file mode 100644 index 0000000..baa2fa3 --- /dev/null +++ b/model_manager/activities/experiment_tracking.py @@ -0,0 +1,280 @@ +""" +Experiment tracking activities for managing ML experiment lifecycle. + +This module provides activities for tracking and updating experiment run status +in the PostgreSQL database, extending the synchronous Postgres client with specialized +methods for experiment management. +""" + +import enum + +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + import traceback + from collections.abc import Mapping + from datetime import UTC, datetime + from typing import Any + + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.temporal.activities.postgres_sync import Postgres + from sqlalchemy import text + + +class UpdateType(enum.StrEnum): + """Types of experiment run updates.""" + + STATUS = 'status' + STATUS_WITH_ERROR = 'status_with_error' + MODEL_SAVED = 'model_saved' + + +class ExperimentTracking(Postgres): + """ + Activity for tracking ML experiment lifecycle and status updates. + + This activity extends the Postgres activity to provide specialized methods + for managing experiment runs, including status updates, error tracking, and + model registration. It maintains the experiment lifecycle from initialization + through training, model saving, and cleanup. + + The activity uses a SQLAlchemy engine / connection pool and adds experiment-specific + operations with proper error handling and notifications. + """ + + def __init__( + self, + host: str, + port: int, + user: str, + password: str, + dbname: str, + min_connections: int, + max_connections: int, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ): + """ + Initialize ExperimentTracking activity with database configuration. + + Args: + host: PostgreSQL server hostname + port: PostgreSQL server port + user: Database user + password: Database password + dbname: Database name + min_connections: Minimum connections in pool + max_connections: Maximum connections in pool + logger: Logger instance for observability + notification_handler: Notification handler for alerts + metrics_controller: Metrics controller for observability + + Raises: + ConnectionError: If database connection cannot be established + """ + Postgres.__init__( + self, + host=host, + port=port, + user=user, + password=password, + dbname=dbname, + min_connections=min_connections, + max_connections=max_connections, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + self.info(f'Postgres client initialized at {host}:{port}') + + def __del__(self): + """ + Destructor to safely handle cleanup during garbage collection. + + This prevents AttributeError when used in multiple inheritance scenarios + where the parent Postgres.__del__ might be called on objects without + the engine attribute. + """ + # Only call parent __del__ if engine attribute exists + if hasattr(self, 'engine'): + try: + if hasattr(super(), '__del__'): + super().__del__() + except Exception: # noqa: S110, BLE001 + # Silently ignore errors during garbage collection + pass + + def _execute_update(self, query: str, params: Mapping[str, Any]) -> dict[str, Any]: + """ + Execute an UPDATE SQL statement. + + Args: + query: Parameterized SQL string to execute. + params: Mapping of parameters for the SQL query. + + Returns: + dict: A dictionary containing the affected row count: {'rowcount': int}. + """ + + with self.engine.begin() as connection: + result = connection.execute(text(query), params) + return {'rowcount': result.rowcount} + + def _build_status_update_query( + self, status: str | None, experiment_run_id: int + ) -> tuple[str, dict[str, Any]]: + """Build SQL query for simple status update.""" + if not isinstance(status, str) or not status: + raise ValueError('status is required for STATUS update type') + + sql_query = """ + UPDATE experiment_run + SET status = :status, updated_at = :updated_at + WHERE id = :experiment_run_id + """ + + query_params = { + 'status': status, + 'updated_at': datetime.now(UTC), + 'experiment_run_id': experiment_run_id, + } + + return sql_query, query_params + + def _build_status_with_error_query( + self, status: str | None, error_message: str | None, experiment_run_id: int + ) -> tuple[str, dict[str, Any]]: + """Build SQL query for status update with error message.""" + if not isinstance(status, str) or not status: + raise ValueError('status is required for STATUS_WITH_ERROR update type') + + if not isinstance(error_message, str) or not error_message: + raise ValueError('error_message is required for STATUS_WITH_ERROR update type') + + # Truncate error message if too long + truncated_error = error_message[:1024] if len(error_message) > 1024 else error_message + + sql_query = """ + UPDATE experiment_run + SET status = :status, error_message = :error_message, updated_at = :updated_at + WHERE id = :experiment_run_id + """ + + query_params = { + 'status': status, + 'error_message': truncated_error, + 'updated_at': datetime.now(UTC), + 'experiment_run_id': experiment_run_id, + } + + return sql_query, query_params + + def _build_model_saved_query( + self, run_name: str | None, status: str | None, experiment_run_id: int + ) -> tuple[str, dict[str, Any]]: + """Build SQL query for model saved update.""" + if not isinstance(run_name, str) or not run_name: + raise ValueError('run_name is required for MODEL_SAVED update type') + + if not isinstance(status, str) or not status: + raise ValueError('status is required for MODEL_SAVED update type') + + sql_query = """ + UPDATE experiment_run + SET run_name = :run_name, status = :status, updated_at = :updated_at + WHERE id = :experiment_run_id + """ + + query_params = { + 'run_name': run_name, + 'status': status, + 'updated_at': datetime.now(UTC), + 'experiment_run_id': experiment_run_id, + } + + return sql_query, query_params + + def _get_update_query_and_params( + self, update_type: str, experiment_run_id: int, input_data: dict[str, Any] + ) -> tuple[str, dict[str, Any]]: + """Get SQL query and parameters based on update type.""" + status = input_data.get('status') + error_message = input_data.get('error_message') + run_name = input_data.get('run_name') + + if update_type == UpdateType.STATUS: + return self._build_status_update_query(status, experiment_run_id) + + if update_type == UpdateType.STATUS_WITH_ERROR: + return self._build_status_with_error_query(status, error_message, experiment_run_id) + + if update_type == UpdateType.MODEL_SAVED: + return self._build_model_saved_query(run_name, status, experiment_run_id) + + raise ValueError(f'Invalid update_type: {update_type}') + + @activity.defn(name='update_experiment_run') + def update_experiment_run(self, input_data: dict[str, Any]) -> None: + """ + Update experiment run with status, errors, or model information. + + This activity provides a unified interface for all experiment run updates, + supporting different update types through a single method. It automatically + selects the appropriate SQL query based on the update type and parameters. + + Args: + input_data: Configuration for experiment run update operation + Required keys: + - metadata (dict): Workflow execution metadata + - experiment_run_id (int): Unique identifier for the experiment run + - update_type (str): Type of update (status, status_with_error, model_saved) + Optional keys: + - status (str): New status for the experiment run + - error_message (str): Error message if update failed + - run_name (str): MLFlow run name if model was saved + + Raises: + ValueError: If required parameters are missing for the update type + RuntimeError: If update operation fails + """ + metadata = input_data.get('metadata') + experiment_run_id = input_data['experiment_run_id'] + update_type = input_data['update_type'] + status = input_data.get('status') + + try: + sql_query, query_params = self._get_update_query_and_params( + update_type, experiment_run_id, input_data + ) + + result = self._execute_update(sql_query, query_params) + + if result.get('rowcount', 0) == 0: + error_msg = ( + f'No experiment_run row updated for id={experiment_run_id} ' + f'(row missing or id mismatch). update_type={update_type!r}, status={status!r}.' + ) + raise ValueError(error_msg) + + self.info( + f'Successfully updated experiment run {experiment_run_id} with status {status}', + metadata, + ) + except Exception as e: # noqa: BLE001 + error_msg = f'Error updating experiment run - ID: {experiment_run_id}, Status: {status}, Error: {str(e)}' + trace = traceback.format_exc() + + self.send_notification( + metadata=metadata or {}, + notification_id='UPDATE_EXPERIMENT_RUN_ERROR', + message=error_msg, + block='update_experiment_run', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + raise RuntimeError(error_msg) from e diff --git a/model_manager/activities/training.py b/model_manager/activities/training.py new file mode 100644 index 0000000..eb50a4f --- /dev/null +++ b/model_manager/activities/training.py @@ -0,0 +1,497 @@ +""" +Training activities for ML model training operations. + +This module provides activities for training machine learning models. +The activity extends BaseActivity and receives pre-downloaded files +and raises `ModelTrainingError` when training fails. +""" + +from sientia_model.wrappers.sientia_model import SientiaModel +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + import os + import time + import traceback + from typing import Any + + import mlflow + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.repository.minio_repository_sync import MinioRepository + from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository + from sientia_model.model_repository.plugin_store import PluginStore + + from model_manager import metrics as mm_metrics + from model_manager.utils.models.train_model_params import TrainModelParams + from model_manager.utils.models.train_model_result import TrainModelResult + from model_manager.utils.repository.data_manager_repository import DataManagerRepository + + +class Training(SientiaMonitoring): + """ + Activity for ML model training operations. + + This activity extends SientiaMonitoring and handles machine learning model + training with comprehensive error handling. It receives pre-downloaded + files from the workflow and raises `ModelTrainingError` on failure so the + workflow can map the correct experiment status. + """ + + def __init__( + self, + mlflow_repository: SientiaMLflowRepository, + plugin_store: PluginStore, + minio_repository: MinioRepository, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ): + """ + Initialize Training activity. + + Args: + logger: Logger instance for observability + notification_handler: Handler for sending notifications + """ + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) + self.data_manager_repository = DataManagerRepository(logger) + self.mlflow_repository = mlflow_repository + self.plugin_store = plugin_store + self.minio_repository = minio_repository + + @activity.defn(name='load_model_metadata') + def load_model_metadata(self, input_data: dict[str, Any]) -> dict[str, Any]: + """ + Load model metadata/schemas from the model store. + + This activity is responsible for fetching model metadata/schemas from the + model store index and extracting a serializable `model_metadata` dict that + `TrainModelParams.validate_business_rules()` depends on. + + Args: + input_data: Workflow input at the same level as `validate_train_params`, + including at least `model_name` and the fields required by + `TrainModelParams.from_dict` to build wrapper kwargs. + + Return: + dict[str, Any]: Updated `input_data` containing `input_data['model_metadata']`. + """ + metadata = input_data.get('metadata', {}) + + self.info(f'Loading model metadata for {input_data}', metadata) + + try: + train_params = TrainModelParams.from_dict(input_data) + model_metadata = self.plugin_store.get_model_index( + model_type=train_params.model_type, + metadata=metadata, + ) + + train_params.model_metadata = model_metadata + self.info(f'Model metadata loaded successfully for {input_data}', metadata) + self.debug(f'Model metadata: {model_metadata}', metadata) + return train_params.to_dict() + except Exception as exc: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id='LOAD_MODEL_METADATA_ERROR', + message=f'Error loading model metadata: {str(exc)}', + block='load_model_metadata', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + raise + + @activity.defn(name='validate_train_params') + def validate_train_params(self, input_data: dict[str, Any]) -> dict[str, Any]: + """ + Validate and convert training parameters from dict to TrainModelParams. + + This activity validates the input training parameters and converts them + to a TrainModelParams object. + + Args: + input_data: Training parameters and metadata at the same level + Required keys: + - metadata (dict): Workflow execution metadata + - All TrainModelParams fields (experiment_run_id, target_variable, etc.) + + Returns: + dict[str, Any]: Validated and converted training parameters as dictionary + + Raises: + Exception: If validation fails (after sending notification) + """ + metadata = input_data.get('metadata', {}) + + self.info(f'Validating training parameters for {input_data}', metadata) + + try: + train_params = TrainModelParams.from_dict(input_data) + + train_params.validate_business_rules() + + self.info( + f'Training parameters validated successfully - ' + f'Target: {train_params.target_variable}, ' + f'Experiment: {train_params.experiment_name}', + metadata, + ) + + self.debug( + f'Training parameters validated successfully: {train_params.to_dict()}', metadata + ) + + return train_params.to_dict() + except Exception as e: + error_msg = f'Error validating training parameters: {str(e)}' + trace = traceback.format_exc() + + self.send_notification( + metadata=metadata, + notification_id='VALIDATE_TRAIN_PARAMS_ERROR', + message=error_msg, + block='validate_train_params', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + raise + + @activity.defn(name='train_model') + def train_model(self, input_data: dict[str, Any]) -> dict[str, Any]: + """ + Train a machine learning model. + + This activity orchestrates the ML training pipeline: + 1. Validate input parameters. + 2. Prepare data via DataManagerRepository. + 3. Train the model and compute metrics. + + Args: + input_data: Training configuration containing: + - metadata (dict): Workflow execution metadata. + - uploaded_file (BytesIO): Training data already downloaded from MinIO. + - train_params (dict): Training parameters. + + Returns: + dict[str, Any]: Serializable summary (run identifiers, run_dir for cleanup, regression metrics). + + Raises: + ValueError: If input validation fails. + Exception: If training fails (after sending notification). + """ + metadata = input_data.get('metadata') + train_params = TrainModelParams.from_dict(input_data['train_params']) + labels = self._get_training_labels(train_params) + + self.info('Starting train_model process', metadata) + + try: + # Download training file bytes from MinIO + self.info( + f'Downloading training file from MinIO for {train_params.file_name}', metadata + ) + train_bytes = self.minio_repository.download_file( + object_name=train_params.file_name, + bucket=train_params.bucket_name, + metadata=metadata, + ) + + # Download optional validation file bytes from the same bucket + val_bytes: bytes | None = None + validation_name = train_params.val_file_name + if validation_name is not None: + self.info(f'Downloading validation file from MinIO for {validation_name}', metadata) + val_bytes = self.minio_repository.download_file( + object_name=validation_name, + bucket=train_params.bucket_name, + metadata=metadata, + ) + + self.info(f'Preparing training data for {train_params.file_name}', metadata) + train_result = self._prepare_data(train_bytes, val_bytes, train_params, metadata) + + mm_metrics.SIENTIA_TRAINING_DATASET_TRAIN_ROWS.labels(**labels).set( + len(train_result.train_data) + ) + mm_metrics.SIENTIA_TRAINING_DATASET_VAL_ROWS.labels(**labels).set( + len(train_result.val_data) + ) + mm_metrics.SIENTIA_TRAINING_FEATURE_COUNT.labels(**labels).set( + len(train_params.variable_columns) + ) + + self.info(f'Getting model wrapper for {train_params.model_type}', metadata) + wrapper = self.plugin_store.get_model( + model_type=train_params.model_type, + force_download=False, + opt_params=train_params.opt_params or {}, + model_kwargs=train_params.model_kwargs or {}, + data_model_kwargs=train_params.data_model_kwargs or {}, + metadata=metadata, + ) + + if self.logger is not None: + wrapper.logger = self.logger.base_logger + + self.info(f'Training model for {train_params.model_type}', metadata) + train_result = self._fit_model(wrapper, train_result, train_params, metadata) + + self.info(f'Computing regression metrics for {train_params.model_type}', metadata) + train_result = self.data_manager_repository.compute_regression_metrics( + train_result, + wrapper, + metadata=metadata, + ) + + if train_result.mse_val is not None: + mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels(**labels).set( + train_result.mse_val + ) + if train_result.mae_val is not None: + mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels(**labels).set( + train_result.mae_val + ) + if train_result.r2_val is not None: + mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels(**labels).set( + train_result.r2_val + ) + + mm_metrics.SIENTIA_TRAINING_INFO.labels( + pod_id=labels['pod_id'], + model_name=train_params.model_name, + model_type=train_params.model_type, + dataset_train_rows=str(len(train_result.train_data)), + dataset_val_rows=str(len(train_result.val_data)), + feature_count=str(len(train_params.variable_columns)), + mse=str(train_result.mse_val) if train_result.mse_val is not None else '', + mae=str(train_result.mae_val) if train_result.mae_val is not None else '', + r2=str(train_result.r2_val) if train_result.r2_val is not None else '', + ).set(time.time() * 1000) + + self.info(f'Starting MLflow run for {train_params.model_type}', metadata) + with self.mlflow_repository.start_run( + model_name=train_params.model_name, + run_name=train_result.run_name, + experiment_name=train_result.experiment_name, + tags=None, + metadata=metadata, + ) as run_info: + train_result.run_id = run_info.run_id + self._persist_training_artifacts(train_result, train_params, wrapper, metadata) + + self.emit_metric_sync( + metric_object=mm_metrics.SIENTIA_TRAINING_MODEL_TRAINED_TOTAL, + tags=labels, + ) + + return { + 'run_name': train_result.run_name, + 'experiment_name': train_result.experiment_name, + 'run_id': train_result.run_id, + 'run_dir': train_result.run_dir, + } + except Exception as e: # noqa: BLE001 + error_msg = f'Error training model - error: {str(e)}' + + trace = traceback.format_exc() + + self.send_notification( + metadata=metadata or {}, + notification_id='TRAIN_MODEL_ERROR', + message=error_msg, + block='train_model', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + + raise e + + def _get_training_labels(self, train_params: TrainModelParams) -> dict: + return { + 'pod_id': os.getenv('HOSTNAME', 'localhost'), + 'model_name': train_params.model_name, + 'model_type': train_params.model_type, + } + + def _prepare_data( + self, + train_bytes: bytes, + val_bytes: bytes | None, + train_params: TrainModelParams, + metadata: dict | None, + ) -> TrainModelResult: + labels = self._get_training_labels(train_params) + start_time = time.time() + try: + return self.data_manager_repository.prepare_training_data( + train_file_bytes=train_bytes, + validation_file_bytes=val_bytes, + params=train_params, + metadata=metadata, + ) + except Exception: + self.emit_metric_sync( + metric_object=mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL, + tags=labels, + ) + raise + finally: + self.observe_lag_sync( + start_time, + mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_LAG, + labels, + ) + + def _fit_model( + self, + wrapper: Any, + train_result: TrainModelResult, + train_params: TrainModelParams, + metadata: dict | None, + ) -> TrainModelResult: + labels = self._get_training_labels(train_params) + train_data = train_result.train_data + val_data = train_result.val_data + + self.debug( + f'train_model prepared data (head 10):\ntrain:\n{train_data.head(10).to_string()}' + f'\nval:\n{val_data.head(10).to_string()}', + metadata, + ) + + start_time = time.time() + try: + wrapper.train( + train_data=train_data, + val_data=val_data, + target=train_params.target_variable, + ) + + self.info( + f'Generating predictions using the trained wrapper for {train_params.model_type}', + metadata, + ) + transformed_train, _ = wrapper.transform(train_data) + transformed_val, _ = wrapper.transform(val_data) + + self.debug( + f'train_model transform (head 10):\ntrain:\n{transformed_train.head(10).to_string()}' + f'\nval:\n{transformed_val.head(10).to_string()}', + metadata, + ) + + y_train_pred_df, _ = wrapper.predict({}, transformed_train) + y_val_pred_df, _ = wrapper.predict({}, transformed_val) + + self.debug( + f'train_model predict (head 10):\ntrain:\n{y_train_pred_df.head(10).to_string()}' + f'\nval:\n{y_val_pred_df.head(10).to_string()}', + metadata, + ) + + y_train_pred_df.sort_index(inplace=True, ascending=False) + y_val_pred_df.sort_index(inplace=True, ascending=False) + + train_result.y_train_pred = y_train_pred_df + train_result.y_pred = y_val_pred_df + return train_result + except Exception: + self.emit_metric_sync( + metric_object=mm_metrics.SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL, + tags=labels, + ) + raise + finally: + self.observe_lag_sync( + start_time, + mm_metrics.SIENTIA_TRAINING_MODEL_FIT_LAG, + labels, + ) + + def _persist_training_artifacts( + self, + train_result: TrainModelResult, + train_params: TrainModelParams, + wrapper: SientiaModel, + metadata: dict[str, Any] | None, + ) -> None: + self.info(f'Generating report for {train_params.model_type}', metadata) + train_result = self.data_manager_repository.generate_report( + train_result, + metadata=metadata, + ) + + if ( + train_result.report_path is None + or train_result.train_data_path is None + or train_result.test_data_path is None + ): + raise ValueError('Report path, train data path, or test data path is not set') + + self.info(f'Storing model for {train_params.model_type}', metadata) + wrapper._input_example = None + wrapper.store_model(name=train_params.model_name) + self._log_regression_metrics_as_params(train_result) + self.info(f'Logging artifacts for {train_params.model_type}', metadata) + mlflow.log_artifact(train_result.report_path) + mlflow.log_artifact(train_result.train_data_path) + mlflow.log_artifact(train_result.test_data_path) + if train_result.equation_path is not None: + mlflow.log_artifact(train_result.equation_path) + + def _log_regression_metrics_as_params(self, train_result: TrainModelResult) -> None: + """ + Persist computed regression metrics as MLflow params. + + Args: + train_result: Training output containing computed regression metrics. + """ + metric_params = { + 'mse_val': train_result.mse_val, + 'mae_val': train_result.mae_val, + 'r2_val': train_result.r2_val, + } + + for key, value in metric_params.items(): + if value is not None: + mlflow.log_param(key, value) + + @activity.defn(name='cleanup_resources') + def cleanup_resources(self, input_data: dict[str, Any]) -> None: + """ + Cleanup temporary resources created during training. + + Args: + input_data: Cleanup configuration containing: + - metadata (dict): Workflow execution metadata. + - run_dir (str): Temporary directory to remove. + + Raises: + Exception: If cleanup fails (after sending notification). + """ + metadata = input_data.get('metadata', {}) + run_dir = input_data.get('run_dir', '') + + try: + self.data_manager_repository.cleanup_run_directory(run_dir, metadata) + except Exception as e: # noqa: BLE001 + error_msg = f'Error cleaning up resources - Run directory: {run_dir}, Error: {str(e)}' + + trace = traceback.format_exc() + + self.send_notification( + metadata=metadata, + notification_id='CLEANUP_RESOURCES_ERROR', + message=error_msg, + block='cleanup_resources', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + + raise diff --git a/model_manager/metrics.py b/model_manager/metrics.py new file mode 100644 index 0000000..ed11437 --- /dev/null +++ b/model_manager/metrics.py @@ -0,0 +1,110 @@ +""" +Model Manager Metrics Module + +This module defines all Prometheus metrics used by the Sientia DataOps Model Manager system +for monitoring and observability. The metrics provide insights into system performance, +training operations, and operational health. + +The metrics are designed to be scraped by Prometheus and can be visualized in +Grafana or other monitoring dashboards to provide real-time visibility into +the system's operation. + +Key Metric Categories: +- Application Health: Overall system status and availability + +Metric Labels: +- pod_id: Kubernetes pod identifier for multi-instance deployments +""" + +from prometheus_client import Counter, Gauge, Histogram + +# Application health metric +APP_UP = Gauge( + 'app_up', + 'Indicates if the application is running (1) or shutting down (0)', + ['pod_id'], +) + +_TRAINING_LABELS = ['pod_id', 'model_name', 'model_type'] + +SIENTIA_TRAINING_INFO = Gauge( + 'sientia_training_info', + 'Metadata and execution timestamp (ms) of the last successful model training run', + [ + 'pod_id', + 'model_name', + 'model_type', + 'dataset_train_rows', + 'dataset_val_rows', + 'feature_count', + 'mse', + 'mae', + 'r2', + ], +) + +SIENTIA_TRAINING_MODEL_TRAINED_TOTAL = Counter( + 'sientia_training_model_trained_total', + 'Number of successfully completed model training runs', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_DATA_PREPARATION_LAG = Histogram( + 'sientia_training_data_preparation_lag', + 'Latency of prepare_training_data() in seconds', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL = Counter( + 'sientia_training_data_preparation_error_count_total', + 'Number of failures in prepare_training_data()', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_MODEL_FIT_LAG = Histogram( + 'sientia_training_model_fit_lag', + 'Latency of wrapper.train() (model fitting) in seconds', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL = Counter( + 'sientia_training_model_fit_error_count_total', + 'Number of failures in wrapper.train() (model fitting)', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_MODEL_QUALITY_MSE = Gauge( + 'sientia_training_model_quality_mse', + 'Mean Squared Error of the last successful model training', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_MODEL_QUALITY_MAE = Gauge( + 'sientia_training_model_quality_mae', + 'Mean Absolute Error of the last successful model training', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_MODEL_QUALITY_R2 = Gauge( + 'sientia_training_model_quality_r2', + 'R-squared of the last successful model training', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_DATASET_TRAIN_ROWS = Gauge( + 'sientia_training_dataset_train_rows', + 'Number of rows in the training dataset after preparation', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_DATASET_VAL_ROWS = Gauge( + 'sientia_training_dataset_val_rows', + 'Number of rows in the validation dataset after preparation', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_FEATURE_COUNT = Gauge( + 'sientia_training_feature_count', + 'Number of input feature columns used for training', + _TRAINING_LABELS, +) diff --git a/model_manager/reports/header.html b/model_manager/reports/header.html new file mode 100644 index 0000000..cd73a20 --- /dev/null +++ b/model_manager/reports/header.html @@ -0,0 +1,167 @@ + + + + + + + + + Report + + + + + + +
+
+ +

Report

+ +
+ info +

+ Note that "current"
+ is related to the test
+ set while "reference"
+ refers to the training
+ set +

+
+
+
+
+
+
+
+
+
+
+
+
+ + + + diff --git a/model_manager/reports/temp/.gitkeep b/model_manager/reports/temp/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/model_manager/runtime_paths.py b/model_manager/runtime_paths.py new file mode 100644 index 0000000..e1d20ed --- /dev/null +++ b/model_manager/runtime_paths.py @@ -0,0 +1,29 @@ +"""Filesystem layout for worker runtime data outside the application package tree.""" + +from os import makedirs +from os.path import join + +# Root for all mutable runtime data (not under /app; avoids clashing with git clone under /app). +RUNTIME_DATA_ROOT = '/var/lib/model-manager' + +# Training reports (HTML, CSV exports, etc.) and related outputs. +REPORTS_ROOT = join(RUNTIME_DATA_ROOT, 'reports') + +# project base path +PROJECT_BASE_PATH = '/app/model_manager' + +# Per-training run folders (name + timestamp); cleanup cron deletes stale entries here. +REPORTS_TEMP_DIR = join(REPORTS_ROOT, 'temp') + +# Worker log files when file logging is wired; stdout remains primary until then. +LOGS_DIR = join(RUNTIME_DATA_ROOT, 'logs') + + +def ensure_runtime_directories() -> None: + """Create runtime directories expected by the worker process.""" + # REPORTS_ROOT: base directory for report artifacts; remove if all outputs move elsewhere. + makedirs(REPORTS_ROOT, exist_ok=True) + # REPORTS_TEMP_DIR: transient run subdirs; remove after retention/cleanup is centralized. + makedirs(REPORTS_TEMP_DIR, exist_ok=True) + # LOGS_DIR: on-disk logs; remove if logging stays stdout-only forever. + makedirs(LOGS_DIR, exist_ok=True) diff --git a/model_manager/schedules/__init__.py b/model_manager/schedules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_manager/schedules/cleanup_schedule.py b/model_manager/schedules/cleanup_schedule.py new file mode 100644 index 0000000..336fda5 --- /dev/null +++ b/model_manager/schedules/cleanup_schedule.py @@ -0,0 +1,160 @@ +"""Schedule configuration for cleanup workflow.""" + +import os +from datetime import timedelta +from typing import Any + +from sientia_do.observability.logger import Logger as SientiaLogger +from temporalio.client import ( + Client, + Schedule, + ScheduleActionStartWorkflow, + ScheduleSpec, +) + +from model_manager.worker.prepare_worker import build_queue_name + +# Schedule configuration from environment variables +CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 0 * * *') # Default: midnight UTC +CLEANUP_TIMEZONE = os.getenv('CLEANUP_TIMEZONE', 'UTC') +CLEANUP_EXECUTION_TIMEOUT_HOURS = int(os.getenv('CLEANUP_EXECUTION_TIMEOUT_HOURS', '1')) + + +def build_cleanup_schedule_id(runtime: str | None) -> str: + """ + Build cleanup schedule ID using runtime-derived naming. + + Args: + runtime: Runtime suffix used by workers + + Return: + str: Cleanup schedule ID + """ + normalized_runtime = runtime.strip() if runtime else '' + return f'cleanup-files-{normalized_runtime or "single"}-daily' + + +async def _needs_schedule_reconcile( + schedule_handle: Any, + cleanup_task_queue: str, + logger: SientiaLogger, + metadata: dict[str, str | None], +) -> bool: + """ + Compare configured cleanup schedule against current expected values. + + Args: + schedule_handle: Temporal schedule handle for current schedule ID + cleanup_task_queue: Expected cleanup task queue + logger: Logger instance for errors + metadata: Metadata dictionary for logging context + + Return: + bool: True when schedule should be recreated to apply current config + """ + try: + schedule_description = await schedule_handle.describe() + schedule = getattr(schedule_description, 'schedule', None) + action = getattr(schedule, 'action', None) + spec = getattr(schedule, 'spec', None) + + current_task_queue = getattr(action, 'task_queue', None) + current_execution_timeout = getattr(action, 'execution_timeout', None) + current_cron = getattr(spec, 'cron_expressions', None) + current_timezone = getattr(spec, 'time_zone_name', None) + + return ( + current_task_queue != cleanup_task_queue + or current_execution_timeout != timedelta(hours=CLEANUP_EXECUTION_TIMEOUT_HOURS) + or current_cron != [CLEANUP_CRON] + or current_timezone != CLEANUP_TIMEZONE + ) + except Exception as e: # noqa: BLE001 + logger.custom_error(f'Error describing cleanup schedule for reconcile: {e}', metadata) + return True + + +async def schedule_exists( + client: Client, schedule_id: str, logger: SientiaLogger, metadata: dict[str, str | None] +) -> bool: + """ + Check if a schedule already exists. + + Args: + client: Temporal client instance + schedule_id: ID of the schedule to check + logger: Logger instance for error logging + metadata: Metadata dictionary for logging context + + Returns: + True if schedule exists, False otherwise + """ + try: + async for schedule in await client.list_schedules(): + if schedule.id == schedule_id: + return True + return False + except Exception as e: # noqa: BLE001 + logger.custom_error(f'Error checking if schedule exists: {e}', metadata) + return False + + +async def create_cleanup_schedule( + client: Client, logger: SientiaLogger, metadata: dict[str, str | None] +) -> None: + """ + Create or update the cleanup files schedule. + + This function is idempotent and can be called multiple times safely. + It will only create the schedule if it doesn't already exist. + + Args: + client: Temporal client instance + logger: Logger instance for logging schedule operations + metadata: Metadata dictionary for logging context + """ + runtime = (os.getenv('RUNTIME') or 'single').strip() + cleanup_task_queue = build_queue_name('CleanupFiles', runtime or 'single') + schedule_id = build_cleanup_schedule_id(runtime) + updated = False + + if await schedule_exists(client, schedule_id, logger, metadata): + handle = client.get_schedule_handle(schedule_id) + if await _needs_schedule_reconcile(handle, cleanup_task_queue, logger, metadata): + await handle.delete() + updated = True + else: + logger.custom_info( + f"Schedule '{schedule_id}' is already up to date, no-op reconcile", + metadata, + ) + return + + await client.create_schedule( + schedule_id, + Schedule( + action=ScheduleActionStartWorkflow( + 'cleanup_files', + {}, # Empty input, will use default bucket from environment + id=f'cleanup-files-scheduled-{schedule_id}', + task_queue=cleanup_task_queue, + execution_timeout=timedelta(hours=CLEANUP_EXECUTION_TIMEOUT_HOURS), + ), + spec=ScheduleSpec( + cron_expressions=[CLEANUP_CRON], + time_zone_name=CLEANUP_TIMEZONE, + ), + ), + ) + if updated: + logger.custom_info( + f"Schedule '{schedule_id}' reconciled successfully. " + f'Cleanup will run at: {CLEANUP_CRON} ({CLEANUP_TIMEZONE})', + metadata, + ) + else: + logger.custom_info( + f"Schedule '{schedule_id}' created successfully. " + f'Cleanup will run at: {CLEANUP_CRON} ({CLEANUP_TIMEZONE})', + metadata, + ) diff --git a/model_manager/sientia/__init__.py b/model_manager/sientia/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_manager/sientia/exceptions.py b/model_manager/sientia/exceptions.py new file mode 100644 index 0000000..8a8cca4 --- /dev/null +++ b/model_manager/sientia/exceptions.py @@ -0,0 +1,3 @@ +from mlflow.exceptions import MlflowException + +SientiaMlException = MlflowException diff --git a/model_manager/sientia/metrics.py b/model_manager/sientia/metrics.py new file mode 100644 index 0000000..a3a3265 --- /dev/null +++ b/model_manager/sientia/metrics.py @@ -0,0 +1,148 @@ +import numpy as np +import pandas as pd +from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score + + +def mse(real_data: pd.Series, predictions: pd.Series) -> float: + """ + Calculates the mean squared error between the real data and the predictions. + """ + return round( + mean_squared_error(real_data.astype(np.float64), predictions.astype(np.float64)), 2 + ) + + +def mae(real_data: pd.Series, predictions: pd.Series) -> float: + """ + Calculates the mean absolute error between the real data and the predictions. + """ + return round( + mean_absolute_error(real_data.astype(np.float64), predictions.astype(np.float64)), 2 + ) + + +def r2(real_data: pd.Series, predictions: pd.Series) -> float: + """ + Calculates the R2 score between the real data and the predictions. + """ + return round(r2_score(real_data.astype(np.float64), predictions.astype(np.float64)), 2) + + +def silverman_radius(data: np.ndarray) -> float: + """ + Calculate the Silverman bandwidth (radius) for a given dataset. + + Args: + data (np.ndarray): Input data (1D array) + + Returns: + float: Silverman bandwidth (radius) + """ + n = len(data) + sigma = np.std(data) + iqr = np.percentile(data, 75) - np.percentile(data, 25) + radius = 0.9 * min(sigma, iqr / 1.34) * n ** (-1 / 5) + return radius + + +def rce_train(training_set: pd.DataFrame, radius: float | None = None) -> pd.DataFrame: + """ + Get the Reduced Coulomb Energy (RCE) prototypes. + + Args: + training_set (pd.DataFrame): The training set + radius (float | None): The radius of the RCE prototypes. If None, computed using Silverman's rule. + + Returns: + pd.DataFrame: The RCE prototypes + """ + train_vectors = training_set.values + + # Vectorized distance computation for the radius calculation + diff_vectors = train_vectors[:, np.newaxis] - train_vectors[np.newaxis, :] + distances = np.linalg.norm(diff_vectors, axis=-1) + + # Non-parametric radius: Silverman Radius (compute if not provided) + effective_radius = radius if radius is not None else silverman_radius(distances.flatten()) + + # Initialize prototypes with the first vector + prototypes = [train_vectors[0]] + + for vector in train_vectors[1:]: + # Vectorized distance check between current vector and all prototypes + distances_to_prototypes = np.linalg.norm(prototypes - vector, axis=1) + + # If no prototype is close, add the current vector as a new prototype + if np.all(distances_to_prototypes > effective_radius): + prototypes.append(vector) + + return pd.DataFrame(prototypes) + + +def rce_test(test_set: pd.DataFrame, prototypes: pd.DataFrame) -> pd.Series: + """ + Get the signed Reduced Coulomb Energy (RCE) predictions. + + Args: + test_set (pd.DataFrame): The test set + prototypes (pd.DataFrame): The RCE prototypes + + Returns: + pd.Series: The signed distances to the closest prototype for each test vector + """ + test_vectors = test_set.values + prototype_vectors = prototypes.values + + # Vectorized computation of distances between test vectors and all prototypes + diff_vectors = test_vectors[:, np.newaxis] - prototype_vectors[np.newaxis, :] + distances = np.linalg.norm(diff_vectors, axis=-1) + + # Find the closest prototype for each test vector + min_distances = np.min(distances, axis=1) + closest_prototypes = prototype_vectors[np.argmin(distances, axis=1)] + + # Compute the signed distance for each test vector + signed_distances = np.sqrt(min_distances**2) * np.sign( + np.mean(test_vectors - closest_prototypes, axis=1) + ) + + return pd.Series(signed_distances) + + +def rce_drift(reference_data: pd.DataFrame, real_data: pd.DataFrame, column: str) -> pd.Series: + """ + Detect drift using the Reduced Coulomb Energy (RCE) method. + + Args: + reference_data (pd.DataFrame): The reference data + real_data (pd.DataFrame): The real data + column (str): The target column to be analyzed. 'target' or 'prediction' + + Returns: + pd.Series: Normalized drift distances + """ + common_columns = list(set(reference_data.columns).intersection(real_data.columns)) + reference_data = reference_data[common_columns] + real_data = real_data[common_columns] + + # Get prototypes + if column == 'target': + prototypes = rce_train(reference_data.drop(columns=['prediction']), 0.1) + else: + prototypes = rce_train(reference_data.drop(columns=['target']), 0.1) + + # Distances to prototypes + if column == 'target': + distances_train = rce_test(reference_data.drop(columns=['prediction']), prototypes) + distances_test = rce_test(real_data.drop(columns=['prediction']), prototypes) + else: + distances_train = rce_test(reference_data.drop(columns=['target']), prototypes) + distances_test = rce_test(real_data.drop(columns=['target']), prototypes) + + # Find the maximum absolute distance in the training set + max_abs_distance = max(abs(distances_train.max()), abs(distances_train.min())) + + # Normalize while preserving sign + distances = distances_test / max_abs_distance + + return distances diff --git a/model_manager/sientia/reports.py b/model_manager/sientia/reports.py new file mode 100644 index 0000000..591c77d --- /dev/null +++ b/model_manager/sientia/reports.py @@ -0,0 +1,292 @@ +import os +from collections.abc import Sequence +from typing import Any + +from bs4 import BeautifulSoup, Tag +from evidently.metric_preset import DataDriftPreset +from evidently.metrics import ( + ColumnSummaryMetric, + ConflictTargetMetric, + DatasetCorrelationsMetric, + DatasetSummaryMetric, + RegressionAbsPercentageErrorPlot, + RegressionDummyMetric, + RegressionErrorDistribution, + RegressionErrorPlot, + RegressionPerformanceMetrics, + RegressionPredictedVsActualPlot, + RegressionPredictedVsActualScatter, +) +from evidently.metrics.base_metric import generate_column_metrics +from evidently.options import ColorOptions +from evidently.pipeline.column_mapping import ColumnMapping +from evidently.report import Report + +COLOR_DISCRETE_SEQUENCE = ( + '#ed0400', + '#0a5f38', + '#6c3461', + '#71aa34', + '#d8dcd6', + '#6b8ba4', +) + + +def load_html_from_file(file_path): + with open(file_path, encoding='utf-8') as file: + return file.read() + + +def inject_content(main_html, section_id, content): + soup = BeautifulSoup(main_html, 'html.parser') + section = soup.find(id=section_id) + + # Verifica se a seção foi encontrada E se ela é uma Tag (não uma string) + if section and isinstance(section, Tag): + section.clear() + # Converte o conteúdo para um fragmento de BeautifulSoup e anexa + new_content = BeautifulSoup(content, 'html.parser') + section.append(new_content) + + return str(soup) + + +class Reports: + """ + Report generator using Evidently library. + + Thread-safety: This class is NOT thread-safe. Multiple threads should not + call add_*_section() methods on the same instance simultaneously as they + modify shared state (self.metrics, self.sections, self.options). + + For multi-threaded environments: + - Create separate Reports instances per thread + - Or synchronize access using locks + - After generation, instances are safe for read-only operations + + I/O Note: This class relies on Evidently's report.save_html() method + for file operations. Ensure Evidently properly manages file handles. + """ + + def __init__( + self, + reference_data: Any, + current_data: Any, + target_name: str, + base_path: str | None = None, + template_path: str | None = None, + ) -> None: + """ + Initializes an instance of the AigReport class. + + Args: + reference_data: The reference data for the report. + current_data: The current data for the report. + base_path: The base path for the report. + """ + self.metrics: list[Any] = [] + self.options: list[Any] | None = None + self.sections: dict[str, Any] = {} + self.report: Any = None + self.ref_data = reference_data + self.cur_data = current_data + self.target_name = target_name + self.set_color_options(primary_color='#0F4C81', secondary_color='#001E60') + self.base_path = base_path + self.template_path = template_path + + def add_data_quality_section(self, columns: list[str] | None = None, run: bool = True) -> None: + """ + Adds a data quality section to the report. + + Args: + columns: The list of columns to include in the data quality section. If None, all columns will be included. + run: Indicates whether to run the report immediately after adding the section. + """ + metrics = [ + DatasetSummaryMetric(), + generate_column_metrics(ColumnSummaryMetric, columns=columns, skip_id_column=True), + ConflictTargetMetric(), + DatasetCorrelationsMetric(), + ] + self.metrics.extend(metrics) + if run: + mapping = ColumnMapping() + mapping.target = self.target_name + report = Report(metrics=metrics, options=self.options) + report.run( + reference_data=self.ref_data, + current_data=self.cur_data, + column_mapping=mapping, + ) + self.sections['data_quality'] = report.as_dict() + if self.base_path: + # Note: Relies on Evidently's save_html() to properly manage file I/O + report.save_html(os.path.join(self.base_path, 'data_quality.html')) + + def add_data_drift_section(self, columns: list[str] | None = None, run: bool = True) -> None: + """ + Adds a data drift section to the report. + + Args: + columns: The list of columns to include in the data drift section. If None, all columns will be included. + run: Indicates whether to run the report immediately after adding the section. + """ + self.metrics.append(DataDriftPreset(columns=columns)) + if run: + mapping = ColumnMapping() + mapping.target = self.target_name + report = Report(metrics=[DataDriftPreset(columns=columns)], options=self.options) + report.run( + reference_data=self.ref_data, + current_data=self.cur_data, + column_mapping=mapping, + ) + self.sections['data_drift'] = report.as_dict() + if self.base_path: + # Note: Relies on Evidently's save_html() to properly manage file I/O + report.save_html(os.path.join(self.base_path, 'data_drift.html')) + + def add_regression_section(self, run: bool = True) -> None: + """ + Adds a regression section to the report. + + Args: + run: Indicates whether to run the report immediately after adding the section. + """ + metrics = [ + RegressionPerformanceMetrics(), + RegressionDummyMetric(), + RegressionPredictedVsActualScatter(), + RegressionPredictedVsActualPlot(), + RegressionErrorPlot(), + RegressionAbsPercentageErrorPlot(), + RegressionErrorDistribution(), + ] + self.metrics.extend(metrics) + if run: + mapping = ColumnMapping() + + mapping.target = self.target_name + mapping.prediction = 'prediction' + + report = Report(metrics=metrics, options=self.options) + report.run( + reference_data=self.ref_data, + current_data=self.cur_data, + column_mapping=mapping, + ) + self.sections['regression'] = report.as_dict() + if self.base_path: + # Note: Relies on Evidently's save_html() to properly manage file I/O + report.save_html(os.path.join(self.base_path, 'regression.html')) + + def set_color_options( + self, + primary_color: str = '#0F4C81', + secondary_color: str = '#001E60', + current_data_color: str | None = None, + reference_data_color: str | None = None, + additional_data_color: str = '#0a5f38', + color_sequence: Sequence[str] = COLOR_DISCRETE_SEQUENCE, + fill_color: str = 'LightGreen', + zero_line_color: str = 'green', + non_visible_color: str = 'white', + underestimation_color: str = '#6574f7', + overestimation_color: str = '#ee5540', + majority_color: str = '#1acc98', + vertical_lines: str = 'green', + heatmap: str = 'RdBu_r', + ) -> None: + """ + Sets the color options for the report. + + Args: + primary_color: The primary color for the report. + secondary_color: The secondary color for the report. + current_data_color: The color for the current data. + reference_data_color: The color for the reference data. + additional_data_color: The color for additional data. + color_sequence: The color sequence for discrete values. + fill_color: The fill color for visualizations. + zero_line_color: The color for the zero line. + non_visible_color: The color for non-visible elements. + underestimation_color: The color for underestimation. + overestimation_color: The color for overestimation. + majority_color: The color for majority elements. + vertical_lines: The color for vertical lines. + heatmap: The color map for heatmaps. + """ + color_scheme = ColorOptions( + primary_color=primary_color, + secondary_color=secondary_color, + current_data_color=current_data_color, + reference_data_color=reference_data_color, + additional_data_color=additional_data_color, + color_sequence=color_sequence, + fill_color=fill_color, + zero_line_color=zero_line_color, + non_visible_color=non_visible_color, + underestimation_color=underestimation_color, + overestimation_color=overestimation_color, + majority_color=majority_color, + vertical_lines=vertical_lines, + heatmap=heatmap, + ) + + if self.options is None: + self.options = [color_scheme] + else: + self.options.append(color_scheme) + + def save_all_sections_html(self, report_path): + """ + Saves the report with all sections as HTML. + + Args: + report_path: The path to save the report HTML file. + + Raises: + ValueError: If base_path is not set + OSError: If directory creation or file writing fails + + Note: + This method uses context manager (with open) to ensure file is properly closed. + Creates parent directories if they don't exist. + """ + if not self.base_path: + raise ValueError('base_path is required to save all sections HTML') + + if not self.template_path: + raise ValueError('template_path is required to save all sections HTML') + + # Ensure output directory exists + output_dir = os.path.dirname(report_path) + if output_dir and not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + print(f'Output directory: {output_dir}') + print(f'Report path: {report_path}') + print(f'Base path: {self.base_path}') + print(f'Template path: {self.template_path}') + + # Load main HTML template + main_html_path = os.path.join(self.template_path, 'header.html') + main_html = load_html_from_file(main_html_path) + + # Load content from data_drift.html, data_quality.html, and regression.html + data_drift_content = load_html_from_file(os.path.join(self.base_path, 'data_drift.html')) + data_quality_content = load_html_from_file( + os.path.join(self.base_path, 'data_quality.html') + ) + regression_content = load_html_from_file(os.path.join(self.base_path, 'regression.html')) + + # Inject content into the main HTML template + main_html = inject_content(main_html, 'data_drift', data_drift_content) + main_html = inject_content(main_html, 'data_quality', data_quality_content) + main_html = inject_content(main_html, 'regression', regression_content) + + # Save the final HTML to a new file (report.html) + # Context manager ensures file is properly closed even if an error occurs + with open(report_path, 'w', encoding='utf-8') as report_file: + report_file.write(main_html) diff --git a/model_manager/utils/__init__.py b/model_manager/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_manager/utils/connectors_config.py b/model_manager/utils/connectors_config.py new file mode 100644 index 0000000..8b8d994 --- /dev/null +++ b/model_manager/utils/connectors_config.py @@ -0,0 +1,166 @@ +from os import getenv +from typing import Any + + +def build_postgres_config() -> dict[str, Any]: + """ + Build PostgreSQL database configuration from environment variables. + + This function constructs a PostgreSQL configuration dictionary from + environment variables with sensible defaults for local development. + It handles connection pool configuration and security parameters. + + Environment Variables: + POSTGRES_HOST: Database hostname (default: localhost) + POSTGRES_PORT: Database port (default: 5432) + POSTGRES_USER: Database username (default: sientia) + POSTGRES_PASSWORD: Database password (default: sientia) + POSTGRES_DBNAME: Database name (default: sientia) + POSTGRES_MIN_CONNECTIONS: Minimum connection pool size (default: 5) + POSTGRES_MAX_CONNECTIONS: Maximum connection pool size (default: 20) + + Returns: + dict: PostgreSQL configuration dictionary with all required parameters + """ + return { + 'host': getenv('POSTGRES_HOST', 'localhost'), + 'port': int(getenv('POSTGRES_PORT', '5432')), + 'user': getenv('POSTGRES_USER', 'sientia'), + 'password': getenv('POSTGRES_PASSWORD', 'sientia'), + 'dbname': getenv('POSTGRES_DBNAME', 'sientia'), + 'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')), + 'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20')), + } + + +def build_mlflow_config() -> dict[str, Any]: + """ + Build MLFlow server configuration from environment variables. + + This function constructs an MLFlow configuration dictionary from + environment variables with sensible defaults for local development. + It handles server connection and authentication parameters. + + Environment Variables: + MLFLOW_URL: Full MLflow tracking URL including scheme, host, and port + (default: http://localhost:5080) + MLFLOW_USERNAME: MLFlow username (default: aignosi) + MLFLOW_PASSWORD: MLFlow password (default: aignosi) + + Returns: + dict: MLFlow configuration dictionary with all required parameters + """ + return { + 'url': getenv('MLFLOW_URL', 'http://localhost:5080'), + 'username': getenv('MLFLOW_USERNAME', 'aignosi'), + 'password': getenv('MLFLOW_PASSWORD', 'aignosi'), + } + + +def build_mongodb_config() -> dict[str, Any]: + """ + Build MongoDB configuration from environment variables. + + This function constructs a MongoDB configuration dictionary from + environment variables with sensible defaults for local development. + It handles connection string and database name configuration. + + Environment Variables: + MONGODB_USERNAME: MongoDB username (default: root) + MONGODB_PASSWORD: MongoDB password (default: wKZDbMNU1c) + MONGODB_URL: MongoDB connection URI (default: localhost:27018) + MONGODB_DATABASE: MongoDB database name (default: sientia) + MONGODB_TTL_INDEX_HOURS: TTL index duration in hours (default: 1) + + Returns: + dict: MongoDB configuration dictionary with connection parameters + """ + username = getenv('MONGODB_USERNAME', 'root') + password = getenv('MONGODB_PASSWORD', 'wKZDbMNU1c') + uri = getenv('MONGODB_URL', 'localhost:27018') + + connection_string = f'mongodb://{username}:{password}@{uri}' + + return { + 'connection_string': connection_string, + 'database_name': getenv('MONGODB_DATABASE', 'sientia'), + 'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600, + 'uri': uri, + } + + +def build_minio_config() -> dict[str, Any]: + """ + Build MinIO (S3-compatible) configuration from environment variables. + + This function constructs a MinIO configuration dictionary from + environment variables with sensible defaults for local development. + It handles endpoint URL, authentication, connection parameters, and retry policies. + + Environment Variables: + MINIO_ENDPOINT_URL: MinIO server endpoint URL (default: http://localhost:9000) + MINIO_ACCESS_KEY: MinIO access key ID (default: minioadmin) + MINIO_SECRET_KEY: MinIO secret access key (default: minioadmin) + MINIO_REGION: MinIO region name (default: us-east-1) + MINIO_SECURE: Whether to use SSL/TLS (default: false) + MINIO_MAX_RETRY_ATTEMPTS: Maximum number of retry attempts (default: 3) + MINIO_RETRY_MODE: Retry mode - standard, legacy, or adaptive (default: adaptive) + MINIO_CONNECT_TIMEOUT: Connection timeout in seconds (default: 10) + MINIO_READ_TIMEOUT: Read timeout in seconds (default: 60) + MINIO_DEFAULT_BUCKET: Default S3 bucket for MinioRepository (default: model-training) + + Returns: + dict: MinIO configuration dictionary with all required parameters + """ + return { + 'endpoint_url': getenv('MINIO_ENDPOINT_URL', 'http://localhost:9000'), + 'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'), + 'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'), + 'region': getenv('MINIO_REGION', 'us-east-1'), + 'use_ssl': getenv('MINIO_SECURE', 'false').lower() == 'true', + 'max_retry_attempts': int(getenv('MINIO_MAX_RETRY_ATTEMPTS', '3')), + 'retry_mode': getenv('MINIO_RETRY_MODE', 'adaptive'), + 'connect_timeout': int(getenv('MINIO_CONNECT_TIMEOUT', '10')), + 'read_timeout': int(getenv('MINIO_READ_TIMEOUT', '60')), + 'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'model-training'), + } + + +def build_plugin_store_config() -> dict[str, Any]: + """ + Build PluginStore configuration from environment variables. + + This function constructs a configuration dictionary for the PluginStore + client using environment variables with sensible defaults for local + development. + + Environment Variables: + STORE_BASE_URL: Base URL of the PluginStore backing Git server + (default: http://localhost:3000) + STORE_OWNER: Repository owner/organization (default: sientia) + STORE_REPO: Repository name (default: model-library-store) + STORE_BRANCH: Optional branch name + STORE_USERNAME: Optional username for Git HTTP authentication + STORE_PASSWORD: Optional password/token for Git HTTP authentication + PYPI_SERVER: Optional custom PyPI index URL for runtime installation + (default: http://localhost:5000) + PYPI_USERNAME: Optional username for PyPI authentication + PYPI_PASSWORD: Optional password/token for PyPI authentication + + Returns: + dict: PluginStore configuration dictionary with all connector parameters + """ + + cache_ttl_seconds = getenv('STORE_CACHE_TTL_SECONDS') + return { + 'base_url': getenv('STORE_BASE_URL', 'http://localhost:3000'), + 'owner': getenv('STORE_OWNER', 'sientia'), + 'repo': getenv('STORE_REPO', 'model-library-store'), + 'username': getenv('STORE_USERNAME'), + 'password': getenv('STORE_PASSWORD'), + 'branch': getenv('STORE_BRANCH'), + 'cache_ttl_seconds': int(cache_ttl_seconds) if cache_ttl_seconds else None, + 'pypi_index_url': getenv('PYPI_SERVER', 'http://localhost:5000'), + 'pypi_username': getenv('PYPI_USERNAME'), + 'pypi_password': getenv('PYPI_PASSWORD'), + } diff --git a/model_manager/utils/logger_helper.py b/model_manager/utils/logger_helper.py new file mode 100644 index 0000000..b5fd47c --- /dev/null +++ b/model_manager/utils/logger_helper.py @@ -0,0 +1,27 @@ +""" +Logger helper to prevent duplicate logs caused by propagation. + +This module provides a wrapper around sientia_do Logger to disable +log propagation and prevent duplicate log entries in the Model Manager. +""" + +from sientia_do.observability.logger import Logger as SientiaLogger + + +def get_logger(name: str) -> SientiaLogger: + """ + Create a Logger instance with propagation disabled. + + This prevents duplicate logs caused by hierarchical propagation + in Python's logging system. + + Args: + name: Logger name (typically __name__ of the calling module). + + Returns: + Logger: Configured logger instance with propagation disabled. + """ + logger = SientiaLogger(name) + # Disable propagation to prevent duplicate logs + logger.base_logger.propagate = False + return logger diff --git a/model_manager/utils/models/__init__.py b/model_manager/utils/models/__init__.py new file mode 100644 index 0000000..6a50ae3 --- /dev/null +++ b/model_manager/utils/models/__init__.py @@ -0,0 +1,16 @@ +""" +Models and DTOs for the Model Manager system. + +This module contains data transfer objects (DTOs) and model classes used +throughout the Model Manager workflows and activities. +""" + +from model_manager.utils.models.experiment_status import ExperimentStatus +from model_manager.utils.models.train_model_params import TrainModelParams +from model_manager.utils.models.train_model_result import TrainModelResult + +__all__ = [ + 'ExperimentStatus', + 'TrainModelParams', + 'TrainModelResult', +] diff --git a/model_manager/utils/models/experiment_status.py b/model_manager/utils/models/experiment_status.py new file mode 100644 index 0000000..1fb70fa --- /dev/null +++ b/model_manager/utils/models/experiment_status.py @@ -0,0 +1,26 @@ +from enum import StrEnum + + +class ExperimentStatus(StrEnum): + """ + Status values for experiment run lifecycle. + + This enum defines all possible status values that an experiment run can have + throughout its lifecycle, from initialization through training, model saving, + and cleanup. These statuses are used to track progress and identify failures + in the training pipeline. + + The status values follow the naming convention from the original Mage pipeline + to maintain compatibility with existing database records and monitoring systems. + + Attributes: + ORCHESTRATOR_WAITING_PROC: Initial status indicating experiment is registered and waiting for processing. + ORCHESTRATOR_VALIDATION_ERROR: Error in the parameters validation. + TRAINING_SUCCESS: Training completed successfully with model and metrics calculated. + TRAINING_ERROR: Training failed due to data issues, model errors, or other exceptions. + """ + + ORCHESTRATOR_VALIDATION_ERROR = 'ORCHESTRATOR_VALIDATION_ERROR' + ORCHESTRATOR_WAITING_PROC = 'ORCHESTRATOR_WAITING_PROC' + TRAINING_SUCCESS = 'TRAINING_SUCCESS' + TRAINING_ERROR = 'TRAINING_ERROR' diff --git a/model_manager/utils/models/train_model_params.py b/model_manager/utils/models/train_model_params.py new file mode 100644 index 0000000..697cfad --- /dev/null +++ b/model_manager/utils/models/train_model_params.py @@ -0,0 +1,363 @@ +from dataclasses import dataclass +from typing import Any + +from jsonschema import Draft202012Validator, ValidationError # type: ignore[import-untyped] + +# Allowed frontend date formats and their strftime equivalents (single source of truth) +FRONTEND_DATE_FORMAT_TO_STRFTIME = { + 'dd/MM/yyyy HH:mm:ss': '%d/%m/%Y %H:%M:%S', + 'MM/dd/yyyy HH:mm:ss': '%m/%d/%Y %H:%M:%S', + 'yyyy/MM/dd HH:mm:ss': '%Y/%m/%d %H:%M:%S', + 'dd-MM-yyyy HH:mm:ss': '%d-%m-%Y %H:%M:%S', + 'MM-dd-yyyy HH:mm:ss': '%m-%d-%Y %H:%M:%S', + 'yyyy-MM-dd HH:mm:ss': '%Y-%m-%d %H:%M:%S', +} +ALLOWED_FRONTEND_DATE_FORMATS = frozenset(FRONTEND_DATE_FORMAT_TO_STRFTIME.keys()) + +# When the client omits date_format (or sends null/blank), parsing uses this frontend format. +DEFAULT_TRAIN_DATE_FORMAT = 'yyyy-MM-dd HH:mm:ss' + + +def validate_frontend_date_format(fmt: str | None) -> None: + """Raise ValueError if fmt is set and not one of the allowed frontend date formats.""" + if not fmt or not fmt.strip(): + return + if fmt not in ALLOWED_FRONTEND_DATE_FORMATS: + allowed = ', '.join(sorted(ALLOWED_FRONTEND_DATE_FORMATS)) + raise ValueError(f'Invalid date_format "{fmt}". Allowed formats: {allowed}') + + +# Model name constants +MODEL_LINEAR_REGRESSION = 'Linear Regression' +MODEL_POLYNOMIAL_REGRESSION = 'Polynomial Regression' + + +@dataclass +class TrainModelParams: + """ + Parameters for machine learning model training. + + This class encapsulates all configuration parameters required for the training + pipeline, including data processing settings, model configuration, and experiment + tracking information. All parameters are validated upon initialization to ensure + data integrity and prevent runtime errors. + + Use the `from_dict()` class method to create instances from dictionaries with + automatic validation of all fields. + + Attributes: + variable_columns (list[str]): List of variable column names to use as features. + target_variable (str): Name of the target variable to predict. + bucket_name (str): Name of the MinIO bucket containing training data. + file_name (str): Name of the file in the MinIO bucket. + line_separator (str): Line separator used in the CSV file. + decimal_separator (str): Decimal separator used in the CSV file. + date_column (str): Name of the date/time column in the dataset (required). + date_format (str): Format of the date column (allowed frontend strings). If omitted or blank + in the input dict, defaults to DEFAULT_TRAIN_DATE_FORMAT. + train_size (int): Percentage of data to use for training (0-100). + shuffle (bool): Whether to shuffle the data during train/test split. + experiment_run_id (int): Unique identifier for the experiment run. + model_name (str): Name of the model type ('Linear Regression' or 'Polynomial Regression'). + val_file_name (str | None): Name of the validation file in the MinIO bucket. + data_model_kwargs (dict | None): Keyword arguments for the data model. + model_kwargs (dict | None): Keyword arguments for the model. + opt_params (dict | None): Keyword optimazation arguments for the wrapper. + model_type (str): Type of the model to use (ex.: 'Linear Regression', 'XGBoost'). + """ + + # Old Parameters (keep) + + variable_columns: list[str] + target_variable: str + bucket_name: str + file_name: str + line_separator: str + decimal_separator: str + date_column: str + date_format: str + train_size: int + shuffle: bool + random_state: int + experiment_run_id: int + model_name: str + experiment_name: str + + # New Parameters + val_file_name: str | None + data_model_kwargs: dict | None # Removed params used in DataPreprocessor here + model_kwargs: dict | None # Removed params used in Linear Regression Model here + opt_params: dict | None + model_type: str + model_id: str | None + + # Context Parameters + model_metadata: dict | None + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> 'TrainModelParams': + """ + Create TrainModelParams from dictionary with validation. + + This factory method creates a TrainModelParams instance from a dictionary, + applying validation to ensure all required fields are present and have + the correct types. This is the recommended way to create instances from + workflow input data. + + Args: + data: Dictionary containing training parameters with keys matching the + attribute names (e.g. variable_columns, date_column, data_model_kwargs, model_kwargs, opt_params). + Unknown keys are ignored by from_dict; missing required snake_case keys raise. + model_metadata may be omitted or None until load_model_metadata fills it. + experiment_run_id may be an int or numeric string. + + Returns: + TrainModelParams: Validated instance with all fields populated + + Raises: + ValueError: If any required field is missing or None + TypeError: If any field has an incorrect type + KeyError: If any required key is missing from the dictionary + """ + # `from_dict()` should only build the "raw" object from the input dict. + # Semantic validation and defaults must be handled by `validate_business_rules()` + # (using `model_metadata` JSON Schemas). + + model_name = cls._check_none(data.get('model_name'), str, 'model_name') + + return cls( + variable_columns=cls._check_none( + data.get('variable_columns'), list, 'variable_columns' + ), + target_variable=cls._check_none(data.get('target_variable'), str, 'target_variable'), + bucket_name=cls._check_none(data.get('bucket_name'), str, 'bucket_name'), + file_name=cls._check_none(data.get('file_name'), str, 'file_name'), + line_separator=cls._check_none(data.get('line_separator'), str, 'line_separator'), + decimal_separator=cls._check_none( + data.get('decimal_separator'), str, 'decimal_separator' + ), + date_column=cls._check_none(data.get('date_column'), str, 'date_column'), + date_format=cls._resolve_date_format(data.get('date_format')), + train_size=cls._check_none(data.get('train_size'), int, 'train_size'), + shuffle=cls._check_none(data.get('shuffle'), bool, 'shuffle'), + random_state=cls._check_none(data.get('random_state', 42), int, 'random_state'), + experiment_run_id=cls._coerce_experiment_run_id(data.get('experiment_run_id')), + model_name=model_name, + experiment_name=model_name, + val_file_name=data.get('val_file_name'), + data_model_kwargs=cls._check_none( + data.get('data_model_kwargs'), dict, 'data_model_kwargs' + ), + model_kwargs=cls._check_none(data.get('model_kwargs'), dict, 'model_kwargs'), + opt_params=cls._check_none(data.get('opt_params'), dict, 'opt_params'), + model_type=cls._check_none(data.get('model_type'), str, 'model_type'), + model_id=data.get('model_id'), + model_metadata=cls._parse_optional_model_metadata(data.get('model_metadata')), + ) + + @staticmethod + def _resolve_date_format(raw: Any) -> str: + """ + Resolve date_format from workflow input. + + Omitted, null, or blank values use DEFAULT_TRAIN_DATE_FORMAT. Non-string types raise. + + Args: + raw: Raw date_format from the payload, or None if absent. + + Return: + str: Canonical frontend date format string. + """ + if raw is None: + return DEFAULT_TRAIN_DATE_FORMAT + if isinstance(raw, str) and not raw.strip(): + return DEFAULT_TRAIN_DATE_FORMAT + if not isinstance(raw, str): + raise TypeError( + f'date_format must be a string or omitted, but got {type(raw).__name__}.' + ) + return raw.strip() + + def to_dict(self) -> dict[str, Any]: + """ + Convert TrainModelParams to a dictionary. + """ + return self.__dict__ + + @staticmethod + def _check_none(value: Any | None, expected_type: type, field_name: str) -> Any: + """ + Validate that a value is not None and check its type. + + This method ensures that required parameters are provided and have the + correct type, raising descriptive errors if validation fails. + + Args: + value (Any | None): The value to validate. + expected_type (type): The expected type of the value. + field_name (str): The name of the field being validated (for error messages). + + Returns: + Any: The validated value if it is not None and matches the expected type. + + Raises: + ValueError: If the value is None. + TypeError: If the value is not of the expected type. + """ + if value is None: + error = f'{field_name} is required and cannot be None.' + raise ValueError(error) + + return TrainModelParams._check_type(value, expected_type, field_name) + + @staticmethod + def _check_type(value: Any | None, expected_type: type, field_name: str) -> Any: + """ + Validate that a value matches the expected type. + + This method checks type compatibility and raises a descriptive error + if the value does not match the expected type. + + Args: + value (Any | None): The value to validate. + expected_type (type): The expected type of the value. + field_name (str): The name of the field being validated (for error messages). + + Returns: + Any: The validated value if it matches the expected type. + + Raises: + TypeError: If the value is not of the expected type. + """ + if value is not None and not isinstance(value, expected_type): + error = f'{field_name} must be of type {expected_type.__name__}, but got {type(value).__name__}.' + raise TypeError(error) + + return value + + @staticmethod + def _coerce_experiment_run_id(value: Any) -> int: + """ + Coerce experiment_run_id to int. + + Workflow clients may send numeric strings; this keeps from_dict aligned with + workflow validation. + + Args: + value: Raw experiment_run_id from the payload. + + Returns: + int: Parsed experiment run id. + + Raises: + ValueError: If the value is None. + TypeError: If the value cannot be coerced to a non-boolean integer. + """ + if value is None: + raise ValueError('experiment_run_id is required and cannot be None.') + if isinstance(value, bool): + raise TypeError('experiment_run_id must be an integer, got bool.') + if isinstance(value, int): + return value + if isinstance(value, str) and value.strip().isdigit(): + return int(value.strip()) + if isinstance(value, float) and value.is_integer(): + return int(value) + raise TypeError( + f'experiment_run_id must be an integer or numeric string, but got {type(value).__name__}.' + ) + + @staticmethod + def _parse_optional_model_metadata(value: Any) -> dict | None: + """ + Parse model_metadata for from_dict before load_model_metadata fills the index. + + Args: + value: model_metadata from the payload, or None if not sent yet. + + Returns: + dict | None: Dict when provided; None when absent (filled later by load_model_metadata). + + Raises: + TypeError: If value is neither None nor a dict. + """ + if value is None: + return None + if isinstance(value, dict): + return value + raise TypeError(f'model_metadata must be a dict or None, but got {type(value).__name__}.') + + def validate_business_rules(self) -> None: + """ + Validate business rules and constraints for training parameters. + + This method performs additional validation beyond type checking to ensure + that parameter values are within acceptable ranges and logically consistent. + It implements defense-in-depth validation to catch configuration errors + early in the workflow. + + Raises: + ValueError: If any business rule is violated + """ + self._validate_numeric_ranges() + self._validate_model_params() + self._validate_required_strings() + self._validate_date_format() + + def _validate_numeric_ranges(self) -> None: + """Validate numeric parameters are within acceptable ranges.""" + if not 10 <= self.train_size <= 100: + raise ValueError(f'train_size must be between 10 and 100, got {self.train_size}') + + if not self.variable_columns: + raise ValueError('variable_columns cannot be empty') + + def _validate_model_params(self) -> None: + """Validate model-related parameters.""" + if not self.model_metadata: + raise ValueError('model_metadata is required') + + schemas = self.model_metadata.get('schemas', {}).get('components', {}).get('schemas') + + if not schemas: + return + + data_model_schema = schemas.get('data_model') + model_schema = schemas.get('model') + opt_params_schema = schemas.get('opt_params') + + if data_model_schema: + self._validate_model_param(data_model_schema, self.data_model_kwargs) + if model_schema: + self._validate_model_param(model_schema, self.model_kwargs) + if opt_params_schema: + self._validate_model_param(opt_params_schema, self.opt_params) + + def _validate_model_param(self, schema: dict[str, Any], value: Any) -> None: + """Validate model parameter against schema.""" + try: + validator = Draft202012Validator(schema) + validator.validate(value) + except ValidationError as e: + raise ValueError(f'Model parameters validation failed: {e.message}') from e + + def _validate_required_strings(self) -> None: + """Validate required string fields are not empty.""" + if not self.target_variable.strip(): + raise ValueError('target_variable cannot be empty or whitespace') + + if not self.bucket_name.strip(): + raise ValueError('bucket_name cannot be empty or whitespace') + + if not self.file_name.strip(): + raise ValueError('file_name cannot be empty or whitespace') + + if not self.model_name.strip(): + raise ValueError('model_name cannot be empty or whitespace') + + if not self.date_column.strip(): + raise ValueError('date_column cannot be empty or whitespace') + + def _validate_date_format(self) -> None: + """Validate date_format is one of the allowed frontend formats.""" + validate_frontend_date_format(self.date_format) diff --git a/model_manager/utils/models/train_model_result.py b/model_manager/utils/models/train_model_result.py new file mode 100644 index 0000000..e89183f --- /dev/null +++ b/model_manager/utils/models/train_model_result.py @@ -0,0 +1,53 @@ +from dataclasses import dataclass + +import pandas as pd + +from model_manager.utils.models.train_model_params import TrainModelParams + + +@dataclass +class TrainModelResult: + """ + A data container for storing the results of a machine learning training process. + + This dataclass encapsulates all outputs from the training pipeline, including + the prepared datasets, evaluation metrics, and paths to generated artifacts. + It is used to pass results between activities in the training workflow. + + Attributes: + params (TrainModelParams): The parameters used to train the model. + x_train (pd.DataFrame): The training dataset features. + x_test (pd.DataFrame): The testing dataset features. + y_train (pd.DataFrame): The training dataset target values. + y_test (pd.DataFrame): The testing dataset target values. + y_pred (pd.Series | None): The predicted target values for the testing dataset. Default is None. + y_train_pred (pd.Series | None): The predicted target values for the training dataset. Default is None. + mse_val (float | None): The Mean Squared Error (MSE) of the predictions. Default is None. + mae_val (float | None): The Mean Absolute Error (MAE) of the predictions. Default is None. + r2_val (float | None): The R-squared (R²) value of the predictions. Default is None. + equation (dict | None): The equation of the model. Default is None. + equation_path (str | None): The path to the equation file. Default is None. + run_name (str | None): The name of the MLFlow run. Default is None. + report_path (str | None): The path to the generated HTML report file. Default is None. + train_data_path (str | None): The path to the training dataset CSV file. Default is None. + test_data_path (str | None): The path to the testing dataset CSV file. Default is None. + """ + + params: TrainModelParams + train_data: pd.DataFrame + val_data: pd.DataFrame + y_pred: pd.DataFrame | None = None + y_train_pred: pd.DataFrame | None = None + mse_val: float | None = None + mae_val: float | None = None + r2_val: float | None = None + equation: dict | None = None + equation_path: str | None = None + run_name: str | None = None + experiment_name: str | None = None + run_id: str | None = None + report_path: str | None = None + train_data_path: str | None = None + test_data_path: str | None = None + + run_dir: str | None = None diff --git a/model_manager/utils/repository/data_manager_repository.py b/model_manager/utils/repository/data_manager_repository.py new file mode 100644 index 0000000..da80787 --- /dev/null +++ b/model_manager/utils/repository/data_manager_repository.py @@ -0,0 +1,636 @@ +""" +Data management repository for the training pipeline. + +This module provides the core data loading and preprocessing logic for the +training pipeline, including: +- CSV loading from in-memory bytes +- datetime parsing and index configuration +- optional support filters +- train/test split management (when no explicit validation dataset is provided) + +It is intentionally decoupled from any specific model implementation or MLflow +integration. Models are trained elsewhere (e.g., via SientiaModel wrappers), +and this repository focuses solely on preparing data structures for them. +""" + +import json +from datetime import datetime +from io import BytesIO +from os import makedirs, path +from shutil import rmtree +from typing import Any + +import numpy as np +import pandas as pd +from sientia_do.observability.logger import Logger +from sientia_do.observability.sientia_monitoring import SientiaMonitoring +from sientia_model.wrappers.sientia_model import SientiaModel + +from model_manager.runtime_paths import PROJECT_BASE_PATH, REPORTS_ROOT +from model_manager.sientia.metrics import mae, mse, r2 +from model_manager.sientia.reports import Reports # type: ignore[import-untyped] +from model_manager.utils.models.train_model_params import ( + FRONTEND_DATE_FORMAT_TO_STRFTIME, + TrainModelParams, +) +from model_manager.utils.models.train_model_result import TrainModelResult + + +def train_test_split( + data: pd.DataFrame, + train_size: float, + random_state: int | None = None, + shuffle: bool = True, +) -> tuple[pd.DataFrame, pd.DataFrame]: + # 1. Definir a semente (seed) para reprodutibilidade + if random_state is not None: + np.random.seed(random_state) + + # 2. Gerar índices e embaralhar se necessário + indices = np.arange(len(data)) + + if shuffle: + np.random.shuffle(indices) + + # 3. Calcular o ponto de corte (split point) + # Cálculo: N_treino = tamanho_total * proporcao_treino + n_train = int(len(data) * train_size) + + # 4. Dividir os índices + train_indices = indices[:n_train] + test_indices = indices[n_train:] + + # 5. Retornar os dados fatiados + return data.iloc[train_indices], data.iloc[test_indices] + + +def _ensure_date_column_parsed(data: pd.DataFrame, params: TrainModelParams) -> pd.DataFrame: + """ + Parse params.date_column using the frontend date_format mapping only. + + date_column must exist in ``data`` (callers validate before prepare). No broad + pandas inference or alternate timezone formats here—clients must send a supported + date_format or rely on the TrainModelParams default. + """ + if params.date_column not in data.columns: + raise ValueError( + f'date_column "{params.date_column}" not found in dataset columns: {list(data.columns)}' + ) + data = data.copy() + col = data[params.date_column] + if params.date_format not in FRONTEND_DATE_FORMAT_TO_STRFTIME: + raise ValueError( + f'date_format "{params.date_format}" is not mapped to a strftime pattern ' + '(must be one of the allowed frontend formats).' + ) + strf = FRONTEND_DATE_FORMAT_TO_STRFTIME[params.date_format] + try: + parsed = pd.to_datetime(col, format=strf, errors='raise') + data[params.date_column] = parsed + except Exception as e: + raise ValueError( + f'Failed to parse date column "{params.date_column}" with format "{params.date_format}": {e}' + ) from e + return data + + +class DataManagerRepository(SientiaMonitoring): + """ + Repository for data preparation in the training pipeline. + + This class encapsulates the core logic for preparing ML training data: + loading CSV bytes, applying date/index configuration, support filters, and + constructing train/test splits (or using an explicit validation dataset). + + Attributes: + logger (Logger): Logger instance for observability and debugging + """ + + def __init__(self, logger: Logger): + """ + Initialize DataManagerRepository with logger. + + Args: + logger: Logger instance for observability + """ + SientiaMonitoring.__init__( + self, + logger=logger, + notification_handler=None, + metrics_controller=None, + ) + + def _drop_rows_with_missing_timestamp( + self, + df: pd.DataFrame, + params: TrainModelParams, + metadata: dict[str, Any] | None, + ) -> pd.DataFrame: + """ + Remove rows where the configured date_column is missing (NaN/NaT/blank string). + + Empty timestamp cells cannot be placed on a DatetimeIndex and break + downstream joins and metrics. + """ + if params.date_column not in df.columns: + return df + series = df[params.date_column] + mask = series.notna() + if series.dtype == object: + stripped = series.astype(str).str.strip() + mask &= stripped.ne('') + mask &= stripped.str.lower().ne('nan') + n_drop = int((~mask).sum()) + if n_drop: + self.info( + f'Dropping {n_drop} row(s) with missing or blank timestamp column ' + f'"{params.date_column}"', + metadata, + ) + return df.loc[mask].copy() + + def _coerce_non_timestamp_columns_to_numeric( + self, + df: pd.DataFrame, + params: TrainModelParams, + metadata: dict[str, Any] | None, + ) -> pd.DataFrame: + """ + Coerce all non-timestamp columns to numeric dtype. + + The timestamp column defined by params.date_column is excluded from coercion. + Non-numeric values are coerced to NaN. + """ + out = df.copy() + for col in out.columns: + if col == params.date_column: + continue + original_na = int(out[col].isna().sum()) + out[col] = pd.to_numeric(out[col], errors='coerce') + new_na = int(out[col].isna().sum()) + introduced_na = new_na - original_na + if introduced_na > 0: + self.warning( + f'Column "{col}" had {introduced_na} non-numeric value(s) coerced to NaN', + metadata, + ) + return out + + def prepare_training_data( + self, + train_file_bytes: bytes, + validation_file_bytes: bytes | None, + params: TrainModelParams, + metadata: dict[str, Any] | None = None, + ) -> TrainModelResult: + """ + Build TrainModelResult from raw CSV bytes for train (and optional validation) data. + + This method orchestrates the data pipeline: + 1. Load training data from in-memory bytes + 2. Optionally load validation data from in-memory bytes + 3. Parse and configure datetime index + 4. Apply optional support filters + 5. Split into train/test sets when no explicit validation dataset is provided + + Args: + train_file_bytes: Raw bytes of the training CSV. + validation_file_bytes: Raw bytes of the validation CSV, or None when + validation should be derived via train/test split. + params: Training parameters (TrainModelParams). + + Returns: + TrainModelResult: Object containing processed data, train/test splits, + and scaler dictionary. + + Raises: + ValueError: If transformed data is empty. + Exception: If data loading or preprocessing fails. + """ + try: + train_df = pd.read_csv( + BytesIO(train_file_bytes), + sep=params.line_separator, + decimal=params.decimal_separator, + ) + except Exception as exc: # noqa: BLE001 + raise ValueError( + 'Failed to load training CSV data from MinIO object. ' + 'Check file encoding, line separator and decimal separator.' + ) from exc + + train_df = self._drop_rows_with_missing_timestamp(train_df, params, metadata) + train_df = _ensure_date_column_parsed(train_df, params) + train_df = self._configure_datetime_index(train_df, params, metadata) + train_df = self._set_timezone_on_index(train_df, metadata) + train_df = self._coerce_non_timestamp_columns_to_numeric(train_df, params, metadata) + + if len(train_df) <= 0: + raise ValueError('Training data view is empty after transformation') + + # Explicit validation dataset path + train_data = pd.DataFrame(train_df[params.variable_columns + [params.target_variable]]) + if validation_file_bytes is not None: + try: + val_df = pd.read_csv( + BytesIO(validation_file_bytes), + sep=params.line_separator, + decimal=params.decimal_separator, + ) + except Exception as exc: # noqa: BLE001 + raise ValueError( + 'Failed to load validation CSV data from MinIO object. ' + 'Check file encoding, line separator and decimal separator.' + ) from exc + + val_df = self._drop_rows_with_missing_timestamp(val_df, params, metadata) + val_df = _ensure_date_column_parsed(val_df, params) + val_df = self._configure_datetime_index(val_df, params, metadata) + val_df = self._set_timezone_on_index(val_df, metadata) + val_df = self._coerce_non_timestamp_columns_to_numeric(val_df, params, metadata) + + if len(val_df) <= 0: + raise ValueError('Validation data view is empty after transformation') + + val_data = pd.DataFrame(val_df[params.variable_columns + [params.target_variable]]) + else: + # Fallback path: derive validation via train/test split from a single dataset. + train_data, val_data = train_test_split( + train_data, + train_size=params.train_size / 100, + shuffle=params.shuffle, + random_state=params.random_state, + ) + + self.info( + f'Data preprocessed and split successfully - experiment run id: {params.experiment_run_id}', + metadata, + ) + + experiment_name = f'{params.experiment_name}' + run_name = f'{experiment_name}_{datetime.now().strftime("%Y%m%d_%H%M%S")}' + + return TrainModelResult( + params=params, + train_data=train_data, + val_data=val_data, + run_name=run_name, + experiment_name=experiment_name, + ) + + def _as_series(self, pred: pd.DataFrame | pd.Series) -> pd.Series: + if isinstance(pred, pd.Series): + return pred + # If the wrapper returns a single-column DataFrame, take its first column. + if pred.shape[1] == 1: + return pred.iloc[:, 0] + raise ValueError('y_pred/y_train_pred must be a Series or single-column DataFrame') + + def _extract_model_equation(self, regr: Any, params: TrainModelParams) -> dict: + """ + Extract the linear regression equation coefficients and create equation metadata. + + This method extracts the coefficients and intercept from the trained model + and creates a structured dictionary containing the equation information + for serialization as JSON artifact. + + Args: + regr: Trained LinearRegressionModel object + params: Training parameters containing variable information + + Returns: + dict: Equation metadata containing: + - target_variable: Name of the target variable + - coefficients: Dictionary mapping variable names to coefficients + - intercept: Model intercept value + - equation_string: Human-readable equation string + - latex_equation: LaTeX formatted equation + """ + coefficients = regr.regr.coef_ + intercept = regr.regr.intercept_ + + # Get feature names - for polynomial models, use poly_feature_names + model_kwargs = params.model_kwargs or {} + degree = model_kwargs.get('degree', 1) + poly_feature_names = model_kwargs.get('poly_feature_names', None) + + if degree > 1 and poly_feature_names: + feature_names = poly_feature_names + else: + feature_names = params.variable_columns + + # Create coefficients dictionary + coefficients_dict = {} + for i, var in enumerate(feature_names): + if i < len(coefficients): + coefficients_dict[var] = float(coefficients[i]) + + # Create equation string + equation_parts = [f'{coef:.6f} * {var}' for var, coef in coefficients_dict.items()] + equation_string = f'{params.target_variable} = {intercept:.6f} + ' + ' + '.join( + equation_parts + ) + + # Create LaTeX equation + latex_parts = [f'{coef:.6f} \\cdot {var}' for var, coef in coefficients_dict.items()] + latex_equation = f'{params.target_variable} = {intercept:.6f} + ' + ' + '.join(latex_parts) + + return { + 'target_variable': params.target_variable, + 'coefficients': coefficients_dict, + 'intercept': float(intercept), + 'equation_string': equation_string, + 'latex_equation': latex_equation, + 'model_type': params.model_name, + 'degree': degree, + 'interaction_only': model_kwargs.get('interaction_only', False), + 'original_features': feature_names, + } + + def compute_regression_metrics( + self, + tmr: TrainModelResult, + wrapper: SientiaModel, + metadata: dict[str, Any] | None = None, + ) -> TrainModelResult: + """ + Compute regression metrics for training results. + + This helper mirrors the previous TrainingRepository.after_train_calculation + behavior, assuming that predictions (y_pred/y_train_pred) are already on the + correct scale for metric calculation (any scaling is handled inside the + model wrapper). + + Args: + tmr: Training result containing: + - train_data/val_data DataFrames with a target column + - y_train_pred/y_pred populated (model predictions for train/val) + wrapper: Trained model wrapper (used for linear equation extraction). + metadata: Optional workflow metadata for debug logging. + + Return: + TrainModelResult: Same object with mse_val, mae_val and r2_val set. + """ + if tmr.y_pred is None: + raise ValueError('y_pred must be set before computing regression metrics') + + params = tmr.params + target = params.target_variable + + # True values are expected to come from val_data. + y_true_val = tmr.val_data[target] + + y_pred_val = self._as_series(tmr.y_pred).sort_index() + y_true_val = y_true_val.sort_index() + + # Align by index to avoid metric calculation errors if ordering differs. + common_index = y_true_val.index.intersection(y_pred_val.index) + head = min(5, len(y_true_val), len(y_pred_val)) + self.debug( + 'compute_regression_metrics index alignment: ' + f'val_n={len(y_true_val)} pred_n={len(y_pred_val)} common_n={len(common_index)}; ' + f'val_index_dtype={y_true_val.index.dtype} ' + f'pred_index_dtype={y_pred_val.index.dtype}; ' + f'val_index_sample={list(y_true_val.index[:head])} ' + f'pred_index_sample={list(y_pred_val.index[:head])}', + metadata, + ) + + if len(common_index) == 0: + raise ValueError( + 'No overlapping indices between val_data and y_pred. ' + f'val_n={len(y_true_val)} pred_n={len(y_pred_val)} ' + f'val_index_sample={list(y_true_val.index[:head])} ' + f'pred_index_sample={list(y_pred_val.index[:head])}' + ) + + y_true_val = y_true_val.loc[common_index] + y_pred_val = y_pred_val.loc[common_index] + + # Metrics helpers already round to 2 decimals. + tmr.mse_val = mse(y_true_val, y_pred_val) + tmr.mae_val = mae(y_true_val, y_pred_val) + tmr.r2_val = r2(y_true_val, y_pred_val) + + if params.model_type == 'linear_regression': + inner = getattr(wrapper, 'model', None) + regr = getattr(inner, 'regr', None) if inner is not None else None + if regr is not None and hasattr(regr, 'coef_') and hasattr(regr, 'intercept_'): + tmr.equation = self._extract_model_equation(inner, params) + + return tmr + + def _configure_datetime_index( + self, + data: pd.DataFrame | None, + params: TrainModelParams, + metadata: dict[str, Any] | None = None, + ) -> pd.DataFrame: + """ + Configure datetime index for the DataFrame. + + Guards against None to avoid 'NoneType' object has no attribute 'index' downstream. + Uses only params.date_column and assumes it was already parsed exactly once by + _ensure_date_column_parsed. + Args: + data: The DataFrame to configure the datetime index for. + params: The training parameters. + metadata: The metadata for the training run. + + Returns: + The DataFrame with the datetime index configured. + """ + if data is None: + raise ValueError( + 'Data is None after load_data. ' + 'Check file format, line separator and decimal separator.' + ) + + if params.date_column not in data.columns: + raise ValueError( + f'date_column "{params.date_column}" not found in dataset columns: {list(data.columns)}' + ) + + if not pd.api.types.is_datetime64_any_dtype(data[params.date_column]): + raise ValueError( + f'date_column "{params.date_column}" must be datetime before index configuration' + ) + + data = data.set_index(params.date_column) + data = data.sort_index() + self.info(f'Configured datetime index from column: {params.date_column}', metadata) + return data + + def _set_timezone_on_index( + self, data: pd.DataFrame, metadata: dict[str, Any] | None = None + ) -> pd.DataFrame: + """ + Check if the index has a timezone and if not, set it to UTC timezone. + + Args: + data: The DataFrame to set the timezone on. + metadata: The metadata for the training run. + + Returns: + The DataFrame with the timezone set. + """ + + if isinstance(data.index, pd.DatetimeIndex): + if data.index.tz is None: + data.index = data.index.tz_localize('UTC') + else: + data.index = data.index.tz_convert('UTC') + else: + raise ValueError('Index is not a DatetimeIndex') + + return data + + def _get_reports_directory(self) -> str: + """ + Get the absolute path to the reports directory. + + Returns: + str: Absolute path to the runtime reports root. + """ + return REPORTS_ROOT + + def _create_run_directory( + self, base_path: str, run_name: str, metadata: dict[str, Any] | None = None + ) -> str: + """ + Creates a directory inside the 'reports' folder with the run name and a timestamp. + + Uses microsecond precision in timestamp to minimize collision probability + in high-concurrency scenarios. + + Args: + base_path (str): The path to the 'reports' folder. + run_name (str): The name of the run. + + Returns: + str: The path to the created directory. + + Raises: + PermissionError: If there are insufficient permissions to create the directory. + OSError: If directory creation fails for any other reason. + """ + # Use microsecond precision to reduce collision probability + run_dir = path.join(base_path, 'temp', f'{run_name}') + + try: + makedirs(run_dir, exist_ok=True) + return run_dir + except PermissionError as e: + error_msg = f'Permission denied when creating directory: {run_dir}' + self.error(error_msg, metadata) + raise PermissionError(error_msg) from e + except OSError as e: + error_msg = f'Failed to create directory {run_dir}: {str(e)}' + self.error(error_msg, metadata) + raise OSError(error_msg) from e + + def generate_report( + self, data: TrainModelResult, metadata: dict[str, Any] | None = None + ) -> TrainModelResult: + """ + Generates a comprehensive report summarizing data quality, data drift, and regression analysis. + + Args: + reference_data (pd.DataFrame): The training dataset with predictions added. + current_data (pd.DataFrame): The testing dataset with predictions added. + data: The training model result containing the datasets, model, and parameters. + + Returns: + The updated result object with paths to the generated report and data files. + + Raises: + ValueError: If data conversion to float64 fails or DataFrames are invalid. + PermissionError: If there are insufficient permissions to write files. + OSError: If file writing fails for any other reason. + """ + + if data.run_name is None: + raise ValueError('run_name is not set, cannot generate report') + + if data.y_train_pred is None or data.y_pred is None: + raise ValueError('y_train_pred or y_pred is not set, cannot generate report') + + y_train_pred = data.y_train_pred.rename(columns={data.params.target_variable: 'prediction'}) + y_val_pred = data.y_pred.rename(columns={data.params.target_variable: 'prediction'}) + + # Join the predictions to the data + reference_data = y_train_pred[['prediction']].join(data.train_data, how='inner') + reference_data_float = reference_data.astype(np.float64) + + current_data = y_val_pred[['prediction']].join(data.val_data, how='inner') + current_data_float = current_data.astype(np.float64) + + # Evidently's ConflictTargetMetric expects a literal `target` column name. + # Keep the original target column and provide this alias for report metrics. + target_col = data.params.target_variable + + reference_data_float['target'] = reference_data_float[target_col] + current_data_float['target'] = current_data_float[target_col] + + # Initialize report generator + base_path = self._get_reports_directory() + data.run_dir = self._create_run_directory(base_path, data.run_name) + + # Template path is the code path of the model_manager package + template_path = path.join(PROJECT_BASE_PATH, 'reports') + + report = Reports( + reference_data=reference_data_float, + current_data=current_data_float, + base_path=data.run_dir, + template_path=template_path, + target_name=data.params.target_variable, + ) + + # Generate report sections + feature_and_target_cols = data.params.variable_columns + [target_col] + report.add_data_quality_section(columns=feature_and_target_cols) + report.add_data_drift_section(columns=feature_and_target_cols) + report.add_regression_section() + + # Save HTML report + data.report_path = path.join(data.run_dir, 'report.html') + report.save_all_sections_html(data.report_path) + + # Save training / validation CSVs using the same frames as the report (includes + # literal `target` alias for Evidently, plus predictions and float-cast features). + data.train_data_path = path.join(data.run_dir, 'train_data.csv') + reference_data_float.to_csv(data.train_data_path, index=False) + + # Save test data CSV + data.test_data_path = path.join(data.run_dir, 'test_data.csv') + current_data_float.to_csv(data.test_data_path, index=False) + + # Save equation as JSON + if data.equation is not None and data.params.model_type == 'linear_regression': + data.equation_path = path.join(data.run_dir, 'model_equation.json') + with open(data.equation_path, 'w', encoding='utf-8') as f: + json.dump(data.equation, f, indent=2, ensure_ascii=False) + + return data + + def cleanup_run_directory(self, run_dir: str, metadata: dict[str, Any] | None = None) -> None: + """ + Clean up temporary run directory after model training. + + This activity deletes the temporary directory created during model training + and artifact generation. It implements idempotent cleanup to handle cases + where the directory may have already been deleted. + + Args: + run_dir (str): Path to the run directory to delete + """ + if not run_dir: + self.info('No run directory specified, skipping cleanup') + return + + if path.exists(run_dir): + rmtree(run_dir) + self.info(f'Run directory deleted successfully: {run_dir}') + else: + self.info(f'Run directory already deleted: {run_dir}') diff --git a/model_manager/worker/__init__.py b/model_manager/worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_manager/worker/prepare_worker.py b/model_manager/worker/prepare_worker.py new file mode 100644 index 0000000..2dd4cbf --- /dev/null +++ b/model_manager/worker/prepare_worker.py @@ -0,0 +1,119 @@ +import os +import re +from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor +from typing import Any + +from sientia_do.observability.logger import Logger +from temporalio.client import Client +from temporalio.worker import PollerBehaviorAutoscaling, Worker + +# Worker configuration parameters with default values. +parameters = [ + ('MAX_CONCURRENT_WORKFLOW_TASKS', '200'), + ('MAX_CONCURRENT_ACTIVITIES', '200'), + ('MAX_CONCURRENT_LOCAL_ACTIVITIES', '200'), + ('MAX_CACHED_WORKFLOWS', '200'), + ('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'), + ('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'), + ('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'), + ('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'), + ('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'), + ('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'), + ('ACTIVITY_EXECUTOR_MAX_WORKERS', '200'), +] + + +def camel_to_snake(text: str) -> str: + """ + Convert a CamelCase or camelCase string into snake_case. + + Args: + - text: str, original string in CamelCase or camelCase format + + Return: + str: converted string in snake_case format + """ + text = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', text) + text = re.sub('([a-z0-9])([A-Z])', r'\1_\2', text) + return text.lower() + + +def build_queue_name(workflow_name: str, runtime: str | None = None) -> str: + """ + Build Temporal queue name from workflow name and runtime. + + Args: + - workflow_name: str, workflow class name in CamelCase format + - runtime: str | None, runtime suffix for environment-specific queues + + Return: + str: queue name in the format --queue or -queue + """ + snake_workflow_name = camel_to_snake(workflow_name) + if runtime: + return f'{snake_workflow_name}-{runtime}-queue' + return f'{snake_workflow_name}-queue' + + +def prepare_worker( + main_workflow: type, + other_workflows: Sequence[type], + activities: Sequence[Any], + temporal_client: Client, + logger: Logger, + runtime: str | None = None, +) -> Worker: + """ + Build and configure a Temporal worker for the given workflow and activities. + + Args: + - main_workflow: type, main workflow class used as worker entry point + - other_workflows: Sequence[type], additional workflows in the same worker + - activities: Sequence[Any], activity callables registered in this worker + - temporal_client: Client, Temporal client used by the worker + - logger: Logger, logger instance used during worker preparation + - runtime: str | None, runtime suffix appended to queue name when present + + Return: + Worker: fully configured Temporal worker instance ready to run + """ + main_workflow_name = main_workflow.__name__.upper() + queue_name = build_queue_name(main_workflow.__name__, runtime) + + local_workflow_parameters: dict[str, int] = {} + for parameter_name, default_value in parameters: + local_workflow_parameters[parameter_name] = int( + os.getenv(f'{main_workflow_name}_{parameter_name}', default_value) + ) + + logger.info(f'Preparing worker for {main_workflow_name} with queue {queue_name}') + + activity_executor = ThreadPoolExecutor( + max_workers=local_workflow_parameters['ACTIVITY_EXECUTOR_MAX_WORKERS'], + thread_name_prefix=f'{queue_name}-activity', + ) + + return Worker( + temporal_client, + task_queue=queue_name, + workflows=[main_workflow, *other_workflows], + activities=[*activities], + activity_executor=activity_executor, + max_concurrent_workflow_tasks=local_workflow_parameters['MAX_CONCURRENT_WORKFLOW_TASKS'], + max_concurrent_activities=local_workflow_parameters['MAX_CONCURRENT_ACTIVITIES'], + max_concurrent_local_activities=local_workflow_parameters[ + 'MAX_CONCURRENT_LOCAL_ACTIVITIES' + ], + max_cached_workflows=local_workflow_parameters['MAX_CACHED_WORKFLOWS'], + workflow_task_poller_behavior=PollerBehaviorAutoscaling( + minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MINIMUM'], + initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_INITIAL'], + maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM'], + ), + activity_task_poller_behavior=PollerBehaviorAutoscaling( + minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MINIMUM'], + initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_INITIAL'], + maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM'], + ), + ) diff --git a/model_manager/worker/worker.py b/model_manager/worker/worker.py new file mode 100644 index 0000000..df2ff5e --- /dev/null +++ b/model_manager/worker/worker.py @@ -0,0 +1,259 @@ +"""Model Manager Worker Module + +This module provides the main worker implementation for the Sientia DataOps Model Manager system. +It orchestrates Temporal workers, manages task queues, and handles the lifecycle of +model training and cleanup workflows. + +The worker supports two task queues: +- train_model--queue: For ML model training workflows +- cleanup_files--queue: For file cleanup workflows + +Key Features: +- Automatic scaling with PollerBehaviorAutoscaling +- Prometheus metrics integration +- Comprehensive error handling and logging +- Graceful shutdown with cleanup +- ML model training pipeline orchestration +- Automated cleanup schedule management + +Environment Variables: +- TEMPORAL_HOST: Temporal server address (default: localhost:7233) +- TEMPORAL_NAMESPACE: Temporal namespace (default: model-manager) +- TEMPORAL_USE_TLS: Enable TLS for Temporal connection (default: false) +- RUNTIME: Runtime identifier used in queue naming (default: single) +- POD_ID: Kubernetes pod identifier for metrics +- HTTP_METRICS_PORT: Prometheus metrics server port (default: 9090) +- HTTP_SDK_METRICS_PORT: Temporal SDK metrics port (default: 9091) +- PROJECT_NAME: Project name for notifications (default: model-manager) +""" + +from temporalio import client, workflow +from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig + +with workflow.unsafe.imports_passed_through(): + import asyncio + import os + import sys + + from prometheus_client import start_http_server + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.observability.logger import Logger as SientiaLogger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_model.model_repository.plugin_store import PluginStore + + from model_manager import metrics + from model_manager.activities.activities import Activities + from model_manager.runtime_paths import ensure_runtime_directories + from model_manager.schedules.cleanup_schedule import create_cleanup_schedule + from model_manager.utils.connectors_config import ( + build_minio_config, + build_mlflow_config, + build_mongodb_config, + build_plugin_store_config, + build_postgres_config, + ) + from model_manager.utils.logger_helper import get_logger + from model_manager.worker.prepare_worker import prepare_worker + from model_manager.workflows.cleanup_files import CleanupFiles + from model_manager.workflows.train_model import TrainModel + +POD_ID = os.getenv('POD_ID') +RUNTIME = os.getenv('RUNTIME') +SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091')) + + +def _get_runtime(runtime: str | None) -> str: + """ + Resolve runtime using fallback when missing. + + Args: + - runtime: str | None, runtime value from environment + + Return: + str: normalized runtime value + """ + normalized_runtime = runtime.strip() if runtime else '' + return normalized_runtime or 'single' + + +async def main(): + """ + Main entry point for the Model Manager worker application. + + This function initializes and starts all components of the worker: + 1. Sets up logging and metadata + 2. Starts Prometheus metrics server + 3. Initializes notification handler + 4. Creates and configures activities + 5. Starts Temporal client and workers + 6. Manages worker lifecycle and graceful shutdown + + The function runs indefinitely until interrupted or an error occurs. + On error, it performs cleanup and exits with a non-zero status code. + + Raises: + Exception: Any unhandled exception during worker execution + SystemExit: On graceful shutdown or error conditions + """ + + runtime = _get_runtime(RUNTIME) + + ensure_runtime_directories() + + host = os.getenv('TEMPORAL_HOST', 'localhost:7233') + use_tls = os.getenv('TEMPORAL_USE_TLS', 'false').lower() == 'true' + logger = get_logger(__name__) + + metadata = { + 'pod_id': POD_ID, + 'runtime': runtime, + } + + start_prometheus_server(logger, metadata) + mongo_config = build_mongodb_config() + + notification_handler = NotificationHandler( + connection_string=mongo_config['connection_string'], + database=mongo_config['database_name'], + logger=logger, + project_name=os.getenv('PROJECT_NAME', 'model-manager'), + ) + + logger.custom_info(f'MongoDB client initialized at {mongo_config["uri"]}', metadata) + + logger.custom_info('Initializing metrics controller', metadata) + + metrics_controller = MetricsController(logger=logger) + + logger.custom_info(f'Installing runtime {runtime}', metadata) + + plugin_store_parameters = build_plugin_store_config() + plugin_store = PluginStore( + base_url=plugin_store_parameters['base_url'], + owner=plugin_store_parameters['owner'], + repo=plugin_store_parameters['repo'], + username=plugin_store_parameters['username'], + password=plugin_store_parameters['password'], + branch=plugin_store_parameters['branch'], + cache_ttl_seconds=plugin_store_parameters['cache_ttl_seconds'], + pypi_index_url=plugin_store_parameters['pypi_index_url'], + pypi_username=plugin_store_parameters['pypi_username'], + pypi_password=plugin_store_parameters['pypi_password'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + await plugin_store.install_runtime(runtime_name=runtime) + + activities = Activities( + postgres_config=build_postgres_config(), + mlflow_config=build_mlflow_config(), + minio_config=build_minio_config(), + plugin_store=plugin_store, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + new_runtime = Runtime( + telemetry=TelemetryConfig( + metrics=PrometheusConfig(bind_address=f'0.0.0.0:{SDK_METRICS_PORT}') + ) + ) + + logger.custom_info(f'SDK metrics server initialized on port {SDK_METRICS_PORT}', metadata) + + namespace = os.getenv('TEMPORAL_NAMESPACE', 'model-manager') + temporal_client = await client.Client.connect( + target_host=host, + namespace=namespace, + runtime=new_runtime, + tls=use_tls, + ) + + logger.custom_info(f'Temporal client initialized at {host}/{namespace}', metadata) + + # Create cleanup schedule (idempotent - only creates if doesn't exist) + try: + await create_cleanup_schedule(temporal_client, logger, metadata) + except Exception as e: # noqa: BLE001 + logger.custom_error(f'Failed to configure cleanup schedule: {e}', metadata) + # Don't fail the worker startup if schedule creation fails + # The schedule can be created manually if needed + + workers = [ + prepare_worker( + main_workflow=TrainModel, + other_workflows=[], + activities=[ + activities.update_experiment_run, + activities.load_model_metadata, + activities.validate_train_params, + activities.train_model, + activities.cleanup_resources, + ], + temporal_client=temporal_client, + logger=logger, + runtime=runtime, + ), + prepare_worker( + main_workflow=CleanupFiles, + other_workflows=[], + activities=[ + activities.cleanup_temp_directories, + ], + temporal_client=temporal_client, + logger=logger, + runtime=runtime, + ), + ] + + handlers = [w.run() for w in workers] + + logger.custom_info('Model manager workers initialized', metadata) + + try: + # This will run the workers and wait for them to complete. + # If an exception occurs in any of the worker handlers, it will be propagated here. + await asyncio.gather(*handlers) + except BaseException as e: # noqa: BLE001 + logger.custom_error(f'An unhandled exception occurred: {e}', metadata) + finally: + notification_handler.shutdown() + logger.custom_info('MongoDB client closed', metadata) + activities.shutdown() + # Exit with a non-zero status code to indicate failure to Kubernetes + metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN + sys.exit(1) + + +def start_prometheus_server(logger: SientiaLogger, metadata: dict[str, str | None]): + """ + Starts the Prometheus metrics server for monitoring and observability. + + This function initializes the Prometheus HTTP server on the configured port + and sets the application health metric to indicate the service is running. + + The server exposes metrics that can be scraped by Prometheus for monitoring + the health and performance of the Model Manager worker. + + Environment Variables: + HTTP_METRICS_PORT: Port for the metrics server (default: 9090) + POD_ID: Pod identifier for metrics labeling + + Raises: + SystemExit: If the metrics server fails to start + """ + try: + port = int(os.getenv('HTTP_METRICS_PORT', 9090)) + start_http_server(port) + logger.custom_info(f'Prometheus server initialized on port {port}.', metadata) + metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP + except Exception as e: # noqa: BLE001 + logger.custom_critical(f'Failed to start Prometheus server: {e}', metadata) + os._exit(1) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/model_manager/workflows/__init__.py b/model_manager/workflows/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_manager/workflows/cleanup_files.py b/model_manager/workflows/cleanup_files.py new file mode 100644 index 0000000..3683f3c --- /dev/null +++ b/model_manager/workflows/cleanup_files.py @@ -0,0 +1,64 @@ +""" +Cleanup workflow for removing local filesystem. + +This module provides a Temporal cron workflow that runs daily to clean up +temporary files and directories older than the configured retention period. +""" + +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + import os + from datetime import timedelta + from typing import Any + + from model_manager.activities.activities import Activities + from model_manager.runtime_paths import REPORTS_TEMP_DIR + from model_manager.workflows.train_model import no_retry_policy + + TIMEOUT_CLEANUP_LOCAL = int(os.getenv('TIMEOUT_CLEANUP_LOCAL', '120')) + POD_ID = os.getenv('POD_ID') + + +@workflow.defn(name='cleanup_files') +class CleanupFiles: + """ + Cleanup workflow for removing stale files. + + This workflow cleans up: + - Local temporary directories with timestamp suffixes + + The workflow is designed to be simple and robust, with error handling + delegated to the individual activities. + """ + + @workflow.run + async def run(self, input_data: dict[str, Any] | None = None) -> None: + """ + Execute the cleanup workflow. + + This method orchestrates the cleanup of local directories + in sequence. No exception handling is needed as activities handle their + own errors and notifications. + """ + payload = input_data or {} + temp_path = payload.get('temp_path') or REPORTS_TEMP_DIR + + # Metadata for tracking + metadata = { + 'metadata': { + 'pod_id': POD_ID, + 'workflow_name': 'cleanup_files', + } + } + + # Execute local directory cleanup + await workflow.execute_activity_method( + Activities.cleanup_temp_directories, + { + **metadata, + 'temp_path': temp_path, + }, + retry_policy=no_retry_policy, + start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_LOCAL), + ) diff --git a/model_manager/workflows/train_model.py b/model_manager/workflows/train_model.py new file mode 100644 index 0000000..00557e7 --- /dev/null +++ b/model_manager/workflows/train_model.py @@ -0,0 +1,400 @@ +""" +Train Model Workflow for ML model training pipeline. + +This workflow orchestrates the complete ML model training process, including: +- Parameter validation and conversion +- Data download from MinIO +- Model training +- Model saving to MLFlow +- Experiment tracking and status updates +""" + +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + import os + from datetime import timedelta + from typing import Any + + from temporalio.common import RetryPolicy + from temporalio.exceptions import ApplicationError + + from model_manager.activities.activities import Activities + from model_manager.activities.experiment_tracking import UpdateType + from model_manager.utils.models.experiment_status import ExperimentStatus + + # Activity timeouts (seconds). Tune per environment (large uploads, long training). + # Training uses no_retry_policy: extend TIMEOUT_TRAIN_MODEL instead of adding retries + # to avoid duplicate MLflow side effects. Cleanup/delete uses network_retry_policy. + TIMEOUT_VALIDATE_PARAMS = int(os.getenv('TIMEOUT_VALIDATE_PARAMS', '30')) + TIMEOUT_TRAIN_MODEL = int(os.getenv('TIMEOUT_TRAIN_MODEL', '2700')) + TIMEOUT_DELETE_FILE = int(os.getenv('TIMEOUT_DELETE_FILE', '120')) + TIMEOUT_UPDATE_DATABASE = int(os.getenv('TIMEOUT_UPDATE_DATABASE', '30')) + + # Retry Policies - Granular strategies for different operation types + # Fast retry for transient network errors (MinIO operations) + network_retry_policy = RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=10), + backoff_coefficient=2.0, + maximum_attempts=5, + ) + + # No retry for training - data errors are permanent + no_retry_policy = RetryPolicy( + maximum_attempts=1, + ) + + # Database retry with exponential backoff + database_retry_policy = RetryPolicy( + initial_interval=timedelta(seconds=2), + maximum_interval=timedelta(seconds=20), + backoff_coefficient=2.0, + maximum_attempts=5, + ) + + +@workflow.defn(name='train_model') +class TrainModel: + """ + Complete ML model training workflow. + + This workflow implements the full training pipeline from parameter validation + through model training and saving to MLFlow. It provides comprehensive error + handling with database status updates at each stage. + + The workflow ensures: + - Proper parameter validation before training starts + - Status tracking in database for monitoring + - Error handling with detailed error messages + - Cleanup and proper resource management + """ + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any] | None: + """ + Execute the complete model training workflow. + + This method orchestrates all steps of the training pipeline: + 1. Validate and convert training parameters + 2. Download training data from MinIO + 3. Train the model + 4. Save model to MLFlow + 5. Update experiment tracking status + + Args: + input_data: Complete configuration for the training workflow + All TrainModelParams fields at the same level: + - experiment_run_id (int): Unique identifier for the experiment run (REQUIRED) + - target_variable (str): Target variable to predict + - variable_columns (list[str]): Feature columns + - train_size (int): Training data percentage + - ... (all other TrainModelParams fields) + + Raises: + ValueError: If experiment_run_id is missing or invalid + """ + workflow.logger.info(f'Starting train_model workflow for {input_data}') + + try: + experiment_run_id = self._validate_experiment_run_id(input_data) + except ValueError as exc: + # Prevent workflow-task retries on deterministic input contract violations. + raise ApplicationError(str(exc), non_retryable=True) from exc + input_data = {**input_data, 'experiment_run_id': experiment_run_id} + + model_name = input_data.get('model_name') + model_id = input_data.get('model_id') + + metadata = { + 'metadata': { + 'experiment_run_id': experiment_run_id, + 'workflow_name': 'train_model', + 'model_name': model_name, + 'model_id': model_id, + } + } + + train_params = await self._validate_training_parameters( + input_data, experiment_run_id, metadata + ) + + training_succeeded = False + train_result: dict[str, Any] | None = None + + try: + train_result = await self._train_model( + train_params=train_params, + experiment_run_id=experiment_run_id, + metadata=metadata, + ) + training_succeeded = True + finally: + try: + if train_result is not None: + await self._cleanup_resources( + run_dir=train_result.get('run_dir'), + metadata=metadata, + ) + else: + pass + except Exception: # noqa: BLE001 + # If cleanup fails after training failed, there is nothing extra to log (DB not committed). + if training_succeeded: # pragma: no branch + workflow.logger.warning( + 'cleanup_resources failed after successful training; model and DB status ' + 'are already committed. Temp files may remain until scheduled cleanup.', + ) + return train_result + + def _validate_experiment_run_id(self, input_data: dict[str, Any]) -> int: + """ + Validate experiment_run_id from input data. + + This method ensures that experiment_run_id is present and valid. + Without a valid experiment_run_id, we cannot update database status, + so this validation must happen before any other operation. + + Args: + input_data: Input data dictionary containing experiment_run_id + + Returns: + int: Validated experiment_run_id + + Raises: + ValueError: If experiment_run_id is missing or not an integer + """ + experiment_run_id = input_data.get('experiment_run_id') + + if experiment_run_id is None: + raise ValueError('experiment_run_id is required but was not provided') + + if isinstance(experiment_run_id, int): + return experiment_run_id + + if isinstance(experiment_run_id, str) and experiment_run_id.strip().isdigit(): + return int(experiment_run_id.strip()) + + raise ValueError( + f'experiment_run_id must be an integer or numeric string, got {type(experiment_run_id).__name__}' + ) + + async def _validate_training_parameters( + self, + input_data: dict[str, Any], + experiment_run_id: int, + metadata: dict[str, Any], + ) -> dict[str, Any]: + """ + Validate and convert training parameters from dict to TrainModelParams. + + This method calls the validate_train_params activity to convert and validate + the input parameters. On success, updates DB status to ORCHESTRATOR_WAITING_PROC. + On error, updates DB status to ORCHESTRATOR_VALIDATION_ERROR. + + Args: + input_data: Input data dictionary containing all training parameters + experiment_run_id: Validated experiment run ID + metadata: Workflow execution metadata + + Returns: + dict[str, Any]: Validated training parameters + + Raises: + Exception: If validation fails (after updating DB status) + """ + try: + input_data = await workflow.execute_activity_method( + Activities.load_model_metadata, + { + **input_data, + **metadata, + }, + retry_policy=no_retry_policy, + start_to_close_timeout=timedelta(seconds=TIMEOUT_VALIDATE_PARAMS), + ) + + train_params = await workflow.execute_activity_method( + Activities.validate_train_params, + { + **input_data, + **metadata, + }, + retry_policy=no_retry_policy, + start_to_close_timeout=timedelta(seconds=TIMEOUT_VALIDATE_PARAMS), + ) + + await self._update_experiment_run( + metadata=metadata, + experiment_run_id=experiment_run_id, + update_type=UpdateType.STATUS, + status=ExperimentStatus.ORCHESTRATOR_WAITING_PROC, + ) + + return train_params + except Exception as e: + try: + await self._update_experiment_run( + metadata=metadata, + experiment_run_id=experiment_run_id, + update_type=UpdateType.STATUS_WITH_ERROR, + status=ExperimentStatus.ORCHESTRATOR_VALIDATION_ERROR, + error_message=self._extract_error_message(e), + ) + except Exception as secondary: # noqa: BLE001 + workflow.logger.warning( + 'Failed to persist ORCHESTRATOR_VALIDATION_ERROR to experiment_run: %s', + secondary, + ) + raise + + async def _train_model( + self, + train_params: dict[str, Any], + experiment_run_id: int, + metadata: dict[str, Any], + ) -> dict[str, Any]: + """ + Download file from MinIO and train model. + + This method orchestrates the download and training steps using proper + resource management with try/catch/finally. On success, updates DB status + to TRAINING_SUCCESS. On error, updates DB status to TRAINING_ERROR. + + Args: + train_params: TrainModelParams object with training configuration + experiment_run_id: Validated experiment run ID + metadata: Workflow execution metadata + + Returns: + dict[str, Any]: Serializable training summary from the train_model activity + + Raises: + Exception: If download or training fails (after updating DB status) + """ + try: + train_result = await workflow.execute_activity_method( + Activities.train_model, + { + **metadata, + 'train_params': train_params, + }, + retry_policy=no_retry_policy, + start_to_close_timeout=timedelta(seconds=TIMEOUT_TRAIN_MODEL), + ) + + await self._update_experiment_run( + metadata=metadata, + experiment_run_id=experiment_run_id, + update_type=UpdateType.MODEL_SAVED, + status=ExperimentStatus.TRAINING_SUCCESS, + run_name=train_result.get('run_name'), + ) + + return train_result + except Exception as e: + try: + await self._update_experiment_run( + metadata=metadata, + experiment_run_id=experiment_run_id, + update_type=UpdateType.STATUS_WITH_ERROR, + status=ExperimentStatus.TRAINING_ERROR, + error_message=self._extract_error_message(e), + ) + except Exception as secondary: # noqa: BLE001 + workflow.logger.warning( + 'Failed to persist TRAINING_ERROR status to experiment_run: %s', + secondary, + ) + raise + + async def _cleanup_resources( + self, + run_dir: str | None, + metadata: dict[str, Any], + ) -> None: + """ + Cleanup resources. + + This method removes the temporary run directory via activity. + + Args: + run_dir: Temporary directory to remove + metadata: Workflow execution metadata + """ + if run_dir is None: + return + + await workflow.execute_activity_method( + Activities.cleanup_resources, + { + **metadata, + 'run_dir': run_dir, + }, + retry_policy=network_retry_policy, + start_to_close_timeout=timedelta(seconds=TIMEOUT_DELETE_FILE), + ) + + async def _update_experiment_run( + self, + metadata: dict[str, Any], + experiment_run_id: int, + update_type: UpdateType, + status: ExperimentStatus, + error_message: str | None = None, + run_name: str | None = None, + ) -> None: + """ + Update experiment run status in the database. + + This is a helper method to simplify calls to the update_experiment_run activity. + It handles both success and error status updates. + + Args: + metadata: Workflow execution metadata + experiment_run_id: Unique identifier for the experiment run + update_type: Type of update (STATUS, STATUS_WITH_ERROR, or MODEL_SAVED) + status: Status to set in the database + error_message: Error message (required if update_type is STATUS_WITH_ERROR) + run_name: MLFlow run name (required if update_type is MODEL_SAVED) + """ + update_input = { + **metadata, + 'experiment_run_id': experiment_run_id, + 'update_type': update_type, + 'status': status, + } + + if error_message is not None: + update_input['error_message'] = error_message + + if run_name is not None: + update_input['run_name'] = run_name + + await workflow.execute_activity_method( + Activities.update_experiment_run, + update_input, + retry_policy=database_retry_policy, + start_to_close_timeout=timedelta(seconds=TIMEOUT_UPDATE_DATABASE), + ) + + def _extract_error_message(self, exc: Exception) -> str: + message_parts: list[str] = [] + seen: set[int] = set() + current: Exception | None = exc + + while current and id(current) not in seen: + seen.add(id(current)) + text = str(current).strip() + + if text and text not in message_parts: + message_parts.append(text) + + cause = getattr(current, '__cause__', None) + context = getattr(current, '__context__', None) + current = cause if isinstance(cause, Exception) else context + + if not message_parts: + return repr(exc) + + return ' | '.join(message_parts) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8eae7e4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,203 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "model-manager" +version = "1.2.0" +description = "Sientia DataOps Model Manager - ML Model Orchestration System" +readme = "README.md" +requires-python = ">=3.11" +authors = [ + {name = "Aignosi", email = "dev@aignosi.com"} +] + +[tool.ruff] +line-length = 100 +target-version = "py311" +exclude = [ + ".git", + ".venv", + "venv", + "__pycache__", + "*.pyc", + ".pytest_cache", + "htmlcov", +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade + "N", # pep8-naming + "YTT", # flake8-2020 + "S", # flake8-bandit + "BLE", # flake8-blind-except + "A", # flake8-builtins + "C90", # mccabe complexity +] + +ignore = [ + "E501", # line too long (handled by formatter) + "S101", # use of assert (needed for tests) + "S105", # possible hardcoded password (false positives) + "S106", # possible hardcoded password (false positives) + "N802", # function name should be lowercase (temporal decorators) + "N806", # variable in function should be lowercase +] + +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = [ + "S101", # assert allowed in tests + "S105", # hardcoded passwords ok in tests + "S106", # hardcoded passwords ok in tests + "S108", # temp paths are expected in tests +] +"e2e/**/*.py" = [ + "S101", # assert allowed in tests + "S105", # hardcoded passwords ok in tests + "S106", # hardcoded passwords ok in tests + "S108", # temp paths are expected in tests + "ARG001", # unused function args in fixtures +] + +[tool.ruff.lint.mccabe] +max-complexity = 15 + +[tool.ruff.format] +quote-style = "single" +indent-style = "space" +line-ending = "auto" + +[tool.mypy] +python_version = "3.11" +warn_return_any = false +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = false +warn_no_return = true +strict_equality = true +ignore_missing_imports = false + +# Ignore missing imports for external packages +[[tool.mypy.overrides]] +module = "temporalio.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sientia_do.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mlflow.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "prometheus_client.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pandas.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "bs4.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "evidently.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sklearn.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sientia_model.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "yaml" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = [ + "model_manager.utils.repository.model_repository", + "model_manager.utils.repository.data_manager_repository", +] +ignore_errors = true + +[tool.pytest.ini_options] +testpaths = ["tests", "e2e"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-v", + "--strict-markers", + "--cov=model_manager", + "--cov-report=term-missing", + "--cov-report=html", + "--cov-report=xml", +] +markers = [ + "asyncio: marks tests as async", + "integration: marks tests as integration tests", + "unit: marks tests as unit tests", +] + +[tool.pyright] +reportMissingTypeStubs = false + +[tool.coverage.run] +source = ["model_manager"] +omit = [ + "*/tests/*", + "*/venv/*", + "*/__pycache__/*", + "*/site-packages/*", +] +branch = true + +[tool.coverage.report] +precision = 2 +show_missing = true +skip_covered = false +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "def __str__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", + "class .*\\bProtocol\\):", + "@(abc\\.)?abstractmethod", +] + +[tool.coverage.html] +directory = "htmlcov" + +[tool.bandit] +exclude_dirs = ["tests", "venv", ".venv"] +skips = ["B101", "B601"] # Skip assert and shell injection in controlled environments + +[tool.deptry] +known_first_party = [ + "model_manager" +] +requirements_files = [ + "requirements.txt" +] +requirements_files_dev = [ + "requirements-dev.txt" +] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..e98e671 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,23 @@ +# Development and Testing Dependencies +# These packages are only needed for development, testing, and code quality checks +# Install with: pip install -r requirements-dev.txt + +# Code Quality & Linting +ruff>=0.1.0 # Fast Python linter and formatter (replaces flake8, black, isort) +mypy>=1.7.0 # Static type checker +bandit>=1.7.5 # Security vulnerability scanner +pandas-stubs>=2.0.0 # Type stubs for pandas +types-psycopg2>=2.9.0 # Type stubs for psycopg2 +types-requests>=2.31.0 # Type stubs for requests + +# Testing +pytest>=7.4.0 # Testing framework +pytest-cov>=4.1.0 # Coverage plugin for pytest +pytest-asyncio>=0.21.0 # Async test support +testcontainers[postgres,minio,mongodb]>=4.0.0 # Real containers for E2E tests +requests>=2.31.0 # HTTP client for Gitea REST API seeding (E2E) + +# Development Tools +ipython>=8.12.0 # Enhanced Python shell +ipdb>=0.13.13 # IPython debugger +ipykernel<=6.26.0 # IPython kernel ( other versions cause problems with interactive mode) diff --git a/requirements-local.txt b/requirements-local.txt new file mode 100644 index 0000000..80eea27 --- /dev/null +++ b/requirements-local.txt @@ -0,0 +1,10 @@ +temporalio==1.23.0 +psycopg2-binary==2.9.11 +sqlalchemy==2.0.49 +boto3==1.42.70 +botocore==1.42.70 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.0 +git+ssh://git@github.com/Aignosi/sientia-model-library.git@0.8.3 +prometheus-client==0.23.1 +beautifulsoup4==4.12.3 +evidently==0.6.7 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..26f7036 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +temporalio==1.23.0 +psycopg2-binary==2.9.11 +sqlalchemy==2.0.50 +boto3==1.42.70 +botocore==1.42.70 +sientia_do>=1.11.0 +sientia_model>=0.8.1 +prometheus-client==0.23.1 +beautifulsoup4==4.12.3 +evidently==0.6.7 +jsonschema==4.26.0 diff --git a/run_local.sh b/run_local.sh new file mode 100755 index 0000000..4fa3fba --- /dev/null +++ b/run_local.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Exit on any error +set -e + +if [ -f .env ]; then + set -a + source <(cat .env | grep -v '^#' | grep -v '^$') + set +a + echo "Environment variables loaded from .env" +else + echo "Warning: .env file not found. Continuing without environment variables." +fi + +python -m model_manager.worker.worker diff --git a/scripts/inputs/linear_regression.json b/scripts/inputs/linear_regression.json new file mode 100644 index 0000000..b0abdf7 --- /dev/null +++ b/scripts/inputs/linear_regression.json @@ -0,0 +1,48 @@ +{ + "experiment": { + "experiment_run_id": 1001, + "experiment_name": "test-experiment-name", + "run_name": "test-run-name", + "username": "vitor.santos@aignosi.com.br", + "status": "ORCHESTRATOR_WAITING_PROC" + }, + "minio": { + "mc_alias": "suse", + "bucket_name": "model-training", + "file_name": "training-sample-dataset-1001.csv", + "local_csv": "input_dataset.csv" + }, + "temporal": { + "task_queue": "train_model-basic-queue", + "workflow_name": "train_model", + "execution_timeout_minutes": 5, + "run_timeout_minutes": 5, + "task_timeout_minutes": 5 + }, + "payload": { + "experiment_run_id": 1001, + "variable_columns": ["Counter", "Rollout"], + "target_variable": "Square", + "line_separator": ",", + "decimal_separator": ".", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "smoke-linreg", + "model_type": "linear_regression", + "model_id": 1001, + "data_model_kwargs": {}, + "model_kwargs": {}, + "opt_params": {}, + "date_column": "timestamp" + }, + "db_only": { + "model_metadata": { + "schemas": { + "components": { + "schemas": {} + } + } + } + } +} diff --git a/scripts/inputs/sin-approx.json b/scripts/inputs/sin-approx.json new file mode 100644 index 0000000..46a2659 --- /dev/null +++ b/scripts/inputs/sin-approx.json @@ -0,0 +1,49 @@ +{ + "experiment": { + "experiment_run_id": 2000, + "experiment_name": "experiment-sin-approx", + "run_name": "run-sin-approx", + "username": "vitor.santos@aignosi.com.br", + "status": "ORCHESTRATOR_WAITING_PROC" + }, + "minio": { + "mc_alias": "open-suse", + "bucket_name": "model-training", + "file_name": "training-sin-approx-dataset-1001.csv", + "local_csv": "data-1780946658143-pivot.csv" + }, + "temporal": { + "task_queue": "train_model-basic-queue", + "workflow_name": "train_model", + "execution_timeout_minutes": 5, + "run_timeout_minutes": 5, + "task_timeout_minutes": 5 + }, + "payload": { + "experiment_run_id": 2000, + "variable_columns": ["SourceTri"], + "target_variable": "TargetSin", + "line_separator": ",", + "decimal_separator": ".", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "sin-approx", + "model_type": "linear_regression", + "model_id": 1, + "data_model_kwargs": {}, + "model_kwargs": {}, + "opt_params": {}, + "date_column": "timestamp" + }, + "db_only": { + "model_metadata": { + "schemas": { + "components": { + "schemas": {} + } + } + } + } + } + \ No newline at end of file diff --git a/scripts/inputs/xgboost.json b/scripts/inputs/xgboost.json new file mode 100644 index 0000000..c1bf718 --- /dev/null +++ b/scripts/inputs/xgboost.json @@ -0,0 +1,63 @@ +{ + "experiment": { + "experiment_run_id": 1002, + "experiment_name": "test-experiment-xgboost", + "run_name": "test-run-xgboost", + "username": "vitor.santos@aignosi.com.br", + "status": "ORCHESTRATOR_WAITING_PROC" + }, + "minio": { + "mc_alias": "suse", + "bucket_name": "model-training", + "file_name": "training-sample-dataset-1002.csv", + "local_csv": "input_dataset.csv" + }, + "temporal": { + "task_queue": "train_model-basic-queue", + "workflow_name": "train_model", + "execution_timeout_minutes": 5, + "run_timeout_minutes": 5, + "task_timeout_minutes": 5 + }, + "payload": { + "experiment_run_id": 1002, + "variable_columns": ["Counter", "Rollout"], + "target_variable": "Square", + "line_separator": ",", + "decimal_separator": ".", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "smoke-xgb", + "model_type": "xgboost", + "model_id": 1002, + "data_model_kwargs": { + "scaler_method": "MinMax", + "window_size": 3, + "use_filtering": false, + "transform_mode": "all" + }, + "model_kwargs": {}, + "opt_params": { + "tree_method": "hist", + "device": "cuda", + "learning_rate": 0.3, + "n_estimators": 100, + "max_depth": 32, + "subsample": 0.8, + "colsample_bytree": 0.8, + "min_child_weight": 5, + "random_state": 42 + }, + "date_column": "timestamp" + }, + "db_only": { + "model_metadata": { + "schemas": { + "components": { + "schemas": {} + } + } + } + } +} diff --git a/scripts/run_cleanup_test.py b/scripts/run_cleanup_test.py new file mode 100644 index 0000000..612eb6b --- /dev/null +++ b/scripts/run_cleanup_test.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Run cleanup_files workflow once for manual testing. + +This script starts the Temporal workflow `cleanup_files` a single time, +using the same Temporal namespace and task queue as the main worker. + +It is intended only for local/manual testing; scheduling (cron) must be +configured separately in Temporal. +""" + +from __future__ import annotations + +import asyncio +import os +import sys +from datetime import timedelta +from pathlib import Path +from typing import Any + +from dotenv import load_dotenv +from temporalio.client import Client + +# Ensure project root is on PYTHONPATH when running directly (must run before model_manager import) +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if ROOT_DIR not in sys.path: + sys.path.insert(0, ROOT_DIR) + +from model_manager.workflows.cleanup_files import CleanupFiles # noqa: E402 + +# Carrega variáveis de ambiente do arquivo .env na raiz do projeto +PROJECT_ROOT = Path(__file__).resolve().parent.parent +ENV_PATH = PROJECT_ROOT / '.env' +if ENV_PATH.exists(): + load_dotenv(dotenv_path=ENV_PATH) + + +async def main(argv: list[str]) -> None: + """Entry point for manual cleanup workflow execution. + + Args: + argv: Command-line arguments (excluding program name). + """ + + # Config from environment / defaults + temporal_host = os.getenv('TEMPORAL_HOST') + temporal_namespace = os.getenv('TEMPORAL_NAMESPACE') + task_queue = os.getenv('CLEANUP_TASK_QUEUE') + use_tls = os.getenv('TEMPORAL_USE_TLS', 'false').lower() == 'true' + + print(f'Connecting to Temporal at {temporal_host} (namespace={temporal_namespace})...') + client = await Client.connect( + target_host=temporal_host, + namespace=temporal_namespace, + tls=use_tls, + ) + + input_data: dict[str, Any] = { + } + + workflow_id = f'cleanup-files-manual-{int(asyncio.get_event_loop().time())}' + + print( + f'Starting cleanup_files workflow once...\n' + f' workflow_id = {workflow_id}\n' + f' task_queue = {task_queue}' + ) + + handle = await client.start_workflow( + CleanupFiles.run, + input_data, + id=workflow_id, + task_queue=task_queue, + run_timeout=timedelta(minutes=10), + ) + + print('Workflow started, waiting for completion...') + await handle.result() + print('cleanup_files workflow completed successfully.') + + +if __name__ == '__main__': # pragma: no cover - manual utility script + asyncio.run(main(sys.argv[1:])) diff --git a/scripts/run_training_test.py b/scripts/run_training_test.py new file mode 100644 index 0000000..71c4c66 --- /dev/null +++ b/scripts/run_training_test.py @@ -0,0 +1,310 @@ +# --- +# jupyter: +# jupytext: +# formats: py:percent +# text_representation: +# extension: .py +# format_name: percent +# kernelspec: +# display_name: Python 3 +# language: python +# name: python3 +# --- + +# %% [markdown] +# # Training smoke test (`input-sample.md`) +# +# Run cells top to bottom in VS Code / Cursor (**Run Cell** on each `# %%` block). +# +# Steps mirror `input-sample.md`: optional DB delete + insert, `mc cp` to MinIO, Temporal `train_model`. +# Set `POSTGRES_*`, `TEMPORAL_*`, and configure the `mc` alias. Add/remove entries in +# `TRAINING_TEST_INPUTS` below to choose which experiments run. + +# %% +from __future__ import annotations + +import csv +import json +import os +import subprocess +import uuid +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any + +import psycopg2 +from dotenv import load_dotenv +from psycopg2.extras import Json +from temporalio import client + +TRAINING_TEST_INPUTS: list[str] = [ + 'scripts/inputs/linear_regression.json', + 'scripts/inputs/xgboost.json', +] + +TRAINING_TEST_INPUTS: list[str] = [ + 'scripts/inputs/sin-approx.json', +] + +_REQUIRED_INPUT_KEYS = ('experiment', 'minio', 'temporal', 'payload', 'db_only') + + +def _load_input(path: Path) -> dict[str, Any]: + """ + Load a training smoke-test input JSON and validate required top-level keys. + + Args: + - path: Path to the input JSON file + + Return: + Parsed input dict with experiment, minio, temporal, payload, and db_only sections + """ + data: dict[str, Any] = json.loads(path.read_text(encoding='utf-8')) + missing = [key for key in _REQUIRED_INPUT_KEYS if key not in data] + if missing: + raise KeyError(f'Input JSON missing required top-level keys: {", ".join(missing)}') + return data + + +def _postgres_connect_kwargs() -> dict[str, str | int]: + """ + Build psycopg2.connect keyword arguments from POSTGRES_* environment variables. + + Return: + host, port, user, password, and dbname suitable for psycopg2.connect + """ + host = os.getenv('POSTGRES_HOST') + user = os.getenv('POSTGRES_USER') + password = os.getenv('POSTGRES_PASSWORD') + dbname = os.getenv('POSTGRES_DBNAME') + if not host or not user or not password or not dbname: + raise RuntimeError( + 'Set POSTGRES_HOST, POSTGRES_USER, POSTGRES_PASSWORD, and POSTGRES_DBNAME' + ) + return { + 'host': host, + 'port': int(os.getenv('POSTGRES_PORT', '5432')), + 'user': user, + 'password': password, + 'dbname': dbname, + } + + +def _resolve_input_paths(project_root: Path, entries: list[str]) -> list[Path]: + """ + Resolve and validate the hardcoded TRAINING_TEST_INPUTS list against the filesystem. + + Args: + - project_root: Repository root used to resolve relative entries + - entries: Input file paths (absolute or relative to project_root) + + Return: + Absolute paths to the input JSON files, in declaration order + """ + if not entries: + raise RuntimeError('TRAINING_TEST_INPUTS is empty; add at least one input JSON path') + resolved: list[Path] = [] + for entry in entries: + candidate = Path(entry) + if not candidate.is_absolute(): + candidate = project_root / candidate + if not candidate.is_file(): + raise FileNotFoundError(f'Training test input file not found: {candidate}') + resolved.append(candidate.resolve()) + return resolved + + +def _validate_csv_columns( + local_csv: Path, + payload: dict[str, Any], + input_path: Path, +) -> None: + """ + Fail fast when the local CSV header does not match payload column names. + + Args: + - local_csv: Resolved path to the CSV on disk + - payload: Training payload with variable_columns, target_variable, and separators + - input_path: Input JSON path (for error messages) + + Return: + None; raises ValueError when required columns are missing from the CSV header + """ + line_separator = str(payload.get('line_separator', ',')) + with local_csv.open(newline='', encoding='utf-8') as handle: + header = next(csv.reader(handle, delimiter=line_separator)) + required = list(payload['variable_columns']) + [str(payload['target_variable'])] + date_column = payload.get('date_column') + if date_column: + required.append(str(date_column)) + missing = [column for column in required if column not in header] + if missing: + raise ValueError( + f'{input_path}: CSV {local_csv} header {header!r} is missing columns ' + f'{missing!r} (check payload variable_columns, target_variable, date_column)' + ) + + +def _resolve_local_csv(project_root: Path, local_csv: str) -> Path: + """ + Resolve a minio.local_csv entry against the project root when relative. + + Args: + - project_root: Repository root + - local_csv: Path declared in the input JSON (absolute or relative) + + Return: + Absolute path to the local CSV file + """ + candidate = Path(local_csv) + if not candidate.is_absolute(): + candidate = project_root / candidate + return candidate + + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +load_dotenv(PROJECT_ROOT / '.env') + +experiments: list[dict[str, Any]] = [] +for _input_path in _resolve_input_paths(PROJECT_ROOT, TRAINING_TEST_INPUTS): + _input = _load_input(_input_path) + _local_csv = _resolve_local_csv(PROJECT_ROOT, _input['minio']['local_csv']) + _validate_csv_columns(_local_csv, _input['payload'], _input_path) + experiments.append({'path': _input_path, **_input}) + _payload = _input['payload'] + _experiment = _input['experiment'] + print( + f'input={_input_path} ' + f'model_type={_payload["model_type"]} ' + f'model_name={_payload["model_name"]} ' + f'experiment_run_id={_experiment["experiment_run_id"]}' + ) + +# %% +# --- configuration (`.env` at repo root; per-run values from input JSON) --- + +PG = _postgres_connect_kwargs() + +TEMPORAL_HOST = os.getenv('TEMPORAL_HOST') +TEMPORAL_NAMESPACE = os.getenv('TEMPORAL_NAMESPACE') +TEMPORAL_TLS = os.getenv('TEMPORAL_USE_TLS', 'false').lower() in ('1', 'true', 'yes') + +print(PG) +print(TEMPORAL_HOST, TEMPORAL_NAMESPACE, TEMPORAL_TLS) +for exp in experiments: + exp_minio = exp['minio'] + exp_temporal = exp['temporal'] + print( + exp_minio['mc_alias'], + exp_minio['bucket_name'], + exp_minio['file_name'], + _resolve_local_csv(PROJECT_ROOT, exp_minio['local_csv']), + exp_temporal['task_queue'], + ) + +# %% +# --- 1) database: delete previous row (same id), then insert `experiment_run` for each experiment --- +# Primary key column is `id` (see `experiment_tracking` updates). + +now = datetime.utcnow() + +with psycopg2.connect( + host=PG['host'], + port=PG['port'], + user=PG['user'], + password=PG['password'], + dbname=PG['dbname'], +) as conn: + with conn.cursor() as cur: + for exp in experiments: + exp_experiment = exp['experiment'] + exp_minio = exp['minio'] + exp_payload = exp['payload'] + exp_db_only = exp['db_only'] + request_data = { + **exp_payload, + 'bucket_name': exp_minio['bucket_name'], + 'file_name': exp_minio['file_name'], + 'model_metadata': exp_db_only['model_metadata'], + } + cur.execute( + 'DELETE FROM public.experiment_run WHERE id = %s', + (exp_experiment['experiment_run_id'],), + ) + cur.execute( + """ + INSERT INTO public.experiment_run ( + id, experiment_name, run_name, username, status, error_message, + created_at, updated_at, bucket_name, file_name, request_data, orchestrator_response_data + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + exp_experiment['experiment_run_id'], + exp_experiment['experiment_name'], + exp_experiment['run_name'], + exp_experiment['username'], + exp_experiment['status'], + None, + now, + now, + exp_minio['bucket_name'], + exp_minio['file_name'], + Json(request_data), + None, + ), + ) + +# %% +# --- 2) MinIO: upload local CSV for each experiment (requires `mc` CLI and alias configured) --- +for exp in experiments: + exp_minio = exp['minio'] + local_csv = _resolve_local_csv(PROJECT_ROOT, exp_minio['local_csv']) + subprocess.run( + [ + 'mc', + 'cp', + '--insecure', + str(local_csv), + f'{exp_minio["mc_alias"]}/{exp_minio["bucket_name"]}/{exp_minio["file_name"]}', + ], + check=True, + ) + +# %% +# --- 3) Temporal: connect once, then start `train_model` per experiment --- + +if not TEMPORAL_HOST or not TEMPORAL_NAMESPACE: + raise RuntimeError('Set TEMPORAL_HOST and TEMPORAL_NAMESPACE in the environment') + +c = await client.Client.connect( # type: ignore[top-level-await] + target_host=TEMPORAL_HOST, + namespace=TEMPORAL_NAMESPACE, + tls=TEMPORAL_TLS, +) + +# %% + +for exp in experiments: + exp_minio = exp['minio'] + exp_temporal = exp['temporal'] + exp_payload = exp['payload'] + workflow_input = { + **exp_payload, + 'bucket_name': exp_minio['bucket_name'], + 'file_name': exp_minio['file_name'], + } + wid = f'train-model-test-{uuid.uuid4()}' + result = await c.execute_workflow( # type: ignore[top-level-await, call-overload] + exp_temporal['workflow_name'], + workflow_input, + id=wid, + task_queue=exp_temporal['task_queue'], + execution_timeout=timedelta(minutes=exp_temporal['execution_timeout_minutes']), + run_timeout=timedelta(minutes=exp_temporal['run_timeout_minutes']), + task_timeout=timedelta(minutes=exp_temporal['task_timeout_minutes']), + ) + print(wid) + print(result) + +# %% diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..e8ae7ff --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,9 @@ +sonar.projectKey=Aignosi_sientia-dataops-model-manager_dc6e051c-995a-4b57-9cde-f511018184c7 +sonar.projectName=sientia-dataops-model-manager +sonar.sources=model_manager +sonar.tests=tests +sonar.qualitygate.wait=true +sonar.qualitygate.timeout=300 +sonar.python.coverage.reportPaths=coverage.xml +sonar.python.xunit.reportPath=pytest.xml +sonar.python.version=3.11 diff --git a/t.py b/t.py new file mode 100644 index 0000000..5ca0f77 --- /dev/null +++ b/t.py @@ -0,0 +1,24 @@ +# %% +from pandas import read_csv, to_datetime + +df = read_csv('data-1780946658143.csv') + +# %% +df.head() + +# %% +# normalize timestamp to naive "yyyy-MM-dd HH:mm:ss" (drop the "+00" UTC offset) +df['timestamp'] = to_datetime(df['timestamp'], utc=True).dt.tz_localize(None) +df['timestamp'] = df['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S') + +# %% +# pivot the dataframe +df = df.pivot(index='timestamp', columns='variable', values='value') + +# %% +df.head() + + +# %% +df.to_csv('data-1780946658143-pivot.csv') +# %% diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/activities/__init__.py b/tests/activities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/activities/test_activities.py b/tests/activities/test_activities.py new file mode 100644 index 0000000..ace21b3 --- /dev/null +++ b/tests/activities/test_activities.py @@ -0,0 +1,144 @@ +"""Unit tests for Activities orchestrator (constructor, shutdown, destructor).""" + +from unittest.mock import MagicMock, Mock, patch + +import pytest +from sientia_do.observability.sientia_monitoring import SientiaMonitoring + +from model_manager.activities.activities import Activities + + +def _postgres(): + return { + 'host': 'h', + 'port': 5432, + 'user': 'u', + 'password': 'p', + 'dbname': 'db', + 'min_connections': 1, + 'max_connections': 2, + } + + +def _mlflow(): + return {'url': 'http://mlflow:5000', 'username': 'u', 'password': 'p'} + + +def _minio(endpoint_url: str): + return { + 'endpoint_url': endpoint_url, + 'access_key': 'a', + 'secret_key': 's', + 'region': 'r', + 'use_ssl': True, + 'default_bucket': 'test-bucket', + } + + +@pytest.mark.parametrize( + 'endpoint,expected_endpoint', + [ + ('http://minio:9000', 'minio:9000'), + ('https://minio:9000', 'minio:9000'), + ('minio:9000', 'minio:9000'), + ], +) +def test_activities_strips_minio_endpoint_scheme(endpoint, expected_endpoint): + with ( + patch( + 'model_manager.activities.activities.ExperimentTracking.__init__', + Mock(return_value=None), + ), + patch('model_manager.activities.activities.Training.__init__', Mock(return_value=None)), + patch('model_manager.activities.activities.Cleanup.__init__', Mock(return_value=None)), + patch('model_manager.activities.activities.SientiaMLflowRepository') as m_mlflow, + patch('model_manager.activities.activities.MinioRepository') as m_minio, + ): + Activities( + postgres_config=_postgres(), + mlflow_config=_mlflow(), + minio_config=_minio(endpoint), + plugin_store=MagicMock(), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(), + ) + m_minio.assert_called_once() + assert m_minio.call_args.kwargs['endpoint'] == expected_endpoint + assert m_minio.call_args.kwargs['bucket'] == 'test-bucket' + m_mlflow.assert_called_once() + + +def test_activities_shutdown_calls_parents(): + with ( + patch( + 'model_manager.activities.activities.ExperimentTracking.__init__', + Mock(return_value=None), + ), + patch('model_manager.activities.activities.Training.__init__', Mock(return_value=None)), + patch('model_manager.activities.activities.Cleanup.__init__', Mock(return_value=None)), + patch('model_manager.activities.activities.SientiaMLflowRepository'), + patch('model_manager.activities.activities.MinioRepository'), + patch('model_manager.activities.activities.ExperimentTracking.close') as m_close, + patch('model_manager.activities.activities.SientiaMonitoring.shutdown') as m_mon, + ): + a = Activities( + postgres_config=_postgres(), + mlflow_config=_mlflow(), + minio_config=_minio('http://x:9000'), + plugin_store=MagicMock(), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(), + ) + with patch.object(SientiaMonitoring, 'info', Mock()): + a.shutdown() + m_close.assert_called_once() + m_mon.assert_called_once() + + +def test_activities_del_with_engine_runs_without_error(): + with ( + patch( + 'model_manager.activities.activities.ExperimentTracking.__init__', + Mock(return_value=None), + ), + patch('model_manager.activities.activities.Training.__init__', Mock(return_value=None)), + patch('model_manager.activities.activities.Cleanup.__init__', Mock(return_value=None)), + patch('model_manager.activities.activities.SientiaMLflowRepository'), + patch('model_manager.activities.activities.MinioRepository'), + ): + a = Activities( + postgres_config=_postgres(), + mlflow_config=_mlflow(), + minio_config=_minio('http://x:9000'), + plugin_store=MagicMock(), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(), + ) + a.engine = MagicMock() + Activities.__del__(a) + + +def test_activities_del_without_engine_runs_without_error(): + with ( + patch( + 'model_manager.activities.activities.ExperimentTracking.__init__', + Mock(return_value=None), + ), + patch('model_manager.activities.activities.Training.__init__', Mock(return_value=None)), + patch('model_manager.activities.activities.Cleanup.__init__', Mock(return_value=None)), + patch('model_manager.activities.activities.SientiaMLflowRepository'), + patch('model_manager.activities.activities.MinioRepository'), + ): + a = Activities( + postgres_config=_postgres(), + mlflow_config=_mlflow(), + minio_config=_minio('http://x:9000'), + plugin_store=MagicMock(), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(), + ) + Activities.__del__(a) diff --git a/tests/activities/test_cleanup.py b/tests/activities/test_cleanup.py new file mode 100644 index 0000000..370f1a0 --- /dev/null +++ b/tests/activities/test_cleanup.py @@ -0,0 +1,310 @@ +"""Unit tests for the Cleanup activity, ensuring 100% code coverage.""" + +import os +import shutil +import tempfile +from datetime import datetime, timedelta +from importlib import reload +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +# Define mocks at the top level to be accessible by all tests + + +@pytest.fixture +def mock_logger(): + """Fixture for a mock logger.""" + return MagicMock() + + +@pytest.fixture +def mock_notification_handler(): + """Fixture for a mock notification handler.""" + return MagicMock() + + +@pytest.fixture +def mock_metrics_controller(): + """Fixture for a mock metrics controller with async methods.""" + controller = MagicMock() + controller.shutdown = AsyncMock() + controller.emit = AsyncMock() + return controller + + +@pytest.fixture +def temp_dir(): + """Fixture to create and clean up a temporary directory.""" + path = tempfile.mkdtemp() + yield path + shutil.rmtree(path) + + +# --- Initialization Tests --- + + +@patch.dict( + 'model_manager.activities.cleanup.os.environ', + { + 'CLEANUP_RETENTION_HOURS': '24', + 'CLEANUP_DRY_RUN': 'false', + }, +) +def test_cleanup_init_default_values( + mock_logger, + mock_notification_handler, + mock_metrics_controller, +): + """Test Cleanup initialization uses default environment values.""" + import model_manager.activities.cleanup + + reload(model_manager.activities.cleanup) + from model_manager.activities.cleanup import Cleanup + + cleanup = Cleanup( + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + assert cleanup.retention_hours == 24 + assert cleanup.dry_run is False + + +def test_cleanup_init_custom_env_values( + mock_logger, + mock_notification_handler, + mock_metrics_controller, +): + """Test Cleanup initialization with custom environment values.""" + with patch.dict( + os.environ, + { + 'CLEANUP_RETENTION_HOURS': '48', + 'CLEANUP_DRY_RUN': 'true', + }, + ): + import model_manager.activities.cleanup + + reload(model_manager.activities.cleanup) + from model_manager.activities.cleanup import Cleanup + + cleanup = Cleanup( + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + assert cleanup.retention_hours == 48 + assert cleanup.dry_run is True + + +@patch.dict(os.environ, {'CLEANUP_RETENTION_HOURS': 'invalid'}) +def test_cleanup_init_invalid_env_value_raises_error(): + """Test Cleanup module raises ValueError for invalid environment variables on import.""" + import model_manager.activities.cleanup + + with pytest.raises(ValueError): + reload(model_manager.activities.cleanup) + + +# --- Temp Directory Cleanup Tests --- + + +def test_cleanup_temp_directories_nonexistent_path( + mock_logger, + mock_notification_handler, + mock_metrics_controller, +): + """Test temp directory cleanup with a non-existent path.""" + from model_manager.activities.cleanup import Cleanup + + cleanup = Cleanup( + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + cleanup.warning = MagicMock() + + cleanup.cleanup_temp_directories({'temp_path': '/nonexistent/path', 'metadata': {}}) + + cleanup.warning.assert_called_once() + + +@patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'}) +def test_cleanup_temp_directories_success_with_deletions( + temp_dir, + mock_logger, + mock_notification_handler, + mock_metrics_controller, +): + """Test successful deletion of old temporary directories.""" + import model_manager.activities.cleanup + + reload(model_manager.activities.cleanup) + from model_manager.activities.cleanup import Cleanup + + cleanup = Cleanup( + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000') + old_dir = os.path.join(temp_dir, f'old_dir_{old_time}') + os.makedirs(old_dir) + + recent_time = (datetime.now() - timedelta(hours=1)).strftime('%Y%m%d_%H%M%S_000000') + recent_dir = os.path.join(temp_dir, f'recent_dir_{recent_time}') + os.makedirs(recent_dir) + + cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}) + + assert not os.path.exists(old_dir) + assert os.path.exists(recent_dir) + + +@patch.dict(os.environ, {'CLEANUP_DRY_RUN': 'true'}) +def test_cleanup_temp_directories_dry_run( + temp_dir, + mock_logger, + mock_notification_handler, + mock_metrics_controller, +): + """Test temp directory cleanup in dry_run mode does not delete.""" + import model_manager.activities.cleanup + + reload(model_manager.activities.cleanup) + from model_manager.activities.cleanup import Cleanup + + cleanup = Cleanup( + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000') + old_dir = os.path.join(temp_dir, f'old_dir_{old_time}') + os.makedirs(old_dir) + + cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}) + + assert os.path.exists(old_dir) + + +@patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'}) +def test_cleanup_temp_directories_delete_error( + temp_dir, + mock_logger, + mock_notification_handler, + mock_metrics_controller, +): + """Test graceful handling of errors during directory deletion.""" + import model_manager.activities.cleanup + + reload(model_manager.activities.cleanup) + from model_manager.activities.cleanup import Cleanup + + cleanup = Cleanup( + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + cleanup.error = MagicMock() + + old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000') + old_dir = os.path.join(temp_dir, f'old_dir_{old_time}') + os.makedirs(old_dir) + + with patch('shutil.rmtree', side_effect=OSError('Permission Denied')): + cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}) + + cleanup.error.assert_called_once() + + +# --- Utility Tests --- + + +def test_cleanup_temp_directories_with_files_and_unmatched_dirs( + temp_dir, + mock_logger, + mock_notification_handler, + mock_metrics_controller, +): + """Test that files and directories with non-matching names are skipped.""" + import model_manager.activities.cleanup + + reload(model_manager.activities.cleanup) + from model_manager.activities.cleanup import Cleanup + + cleanup = Cleanup( + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + cleanup.debug = MagicMock() + + # Create a file and a directory with a non-matching name + with open(os.path.join(temp_dir, 'a_file.txt'), 'w') as f: + f.write('hello') + os.makedirs(os.path.join(temp_dir, 'a_directory_with_no_timestamp')) + + cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}) + + # Ensure the debug message for skipping was called for the unmatched directory + cleanup.debug.assert_called_with( + 'Skipping directory without timestamp pattern: a_directory_with_no_timestamp', {} + ) + + +def test_cleanup_temp_directories_invalid_timestamp_format( + temp_dir, + mock_logger, + mock_notification_handler, + mock_metrics_controller, +): + """Test that a directory with an invalid timestamp format is handled correctly.""" + import model_manager.activities.cleanup + + reload(model_manager.activities.cleanup) + from model_manager.activities.cleanup import Cleanup + + cleanup = Cleanup( + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + cleanup.error = MagicMock() + + # Create a directory with a malformed timestamp that matches the regex but fails parsing + malformed_dir_name = 'dir_20239999_999999_999999' + os.makedirs(os.path.join(temp_dir, malformed_dir_name)) + + cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}) + + cleanup.error.assert_called_once() + + +def test_cleanup_temp_directories_generic_exception( + temp_dir, + mock_logger, + mock_notification_handler, + mock_metrics_controller, +): + """Test that a generic exception during directory cleanup is handled.""" + import model_manager.activities.cleanup + + reload(model_manager.activities.cleanup) + from model_manager.activities.cleanup import Cleanup + + cleanup = Cleanup( + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + cleanup.send_notification = MagicMock() + + with patch('os.listdir', side_effect=Exception('Unexpected OS Error')): + with pytest.raises(Exception, match='Unexpected OS Error'): + cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}) + + cleanup.send_notification.assert_called_once() diff --git a/tests/activities/test_experiment_tracking.py b/tests/activities/test_experiment_tracking.py new file mode 100644 index 0000000..2851560 --- /dev/null +++ b/tests/activities/test_experiment_tracking.py @@ -0,0 +1,643 @@ +"""Unit tests for ExperimentTracking class with 100% coverage.""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + + +@pytest.fixture +def mock_logger(): + """Create a mock logger.""" + return MagicMock() + + +@pytest.fixture +def mock_notification_handler(): + """Create a mock notification handler.""" + return MagicMock() + + +@pytest.fixture +def mock_metrics_controller(): + """Create a mock metrics controller.""" + controller = MagicMock() + controller.shutdown = AsyncMock() + controller.emit = AsyncMock() + return controller + + +@pytest.fixture +def db_config(): + """Create a valid database configuration.""" + return { + 'host': 'localhost', + 'port': 5432, + 'user': 'testuser', + 'password': 'testpass', + 'dbname': 'testdb', + 'min_connections': 1, + 'max_connections': 10, + } + + +def test_experiment_tracking_init( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test ExperimentTracking initialization.""" + from model_manager.activities.experiment_tracking import ExperimentTracking + + ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + +def test_experiment_tracking_del_without_engine( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test __del__ when engine attribute does not exist.""" + from model_manager.activities.experiment_tracking import ExperimentTracking + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + if hasattr(et, 'engine'): + delattr(et, 'engine') + + et.__del__() + + +def test_experiment_tracking_del_with_engine( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test __del__ when engine exists.""" + from model_manager.activities.experiment_tracking import ExperimentTracking + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + et.engine = MagicMock() + + class MockSuper: + def __del__(self): + pass + + with patch('builtins.super', return_value=MockSuper()): + et.__del__() + + +def test_experiment_tracking_del_with_engine_exception( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test __del__ catches exceptions.""" + from model_manager.activities.experiment_tracking import ExperimentTracking + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + et.engine = MagicMock() + + class MockSuperWithError: + _should_raise: bool + + def __init__(self) -> None: + self._should_raise = False + + def __del__(self): + # Only raise error if not being cleaned up by garbage collector + # This prevents the PytestUnraisableExceptionWarning + if hasattr(self, '_should_raise') and self._should_raise: + raise RuntimeError('Test error') + + # Suppress the PytestUnraisableExceptionWarning for this specific test + import warnings + + warnings.filterwarnings('ignore', category=pytest.PytestUnraisableExceptionWarning) + + mock_super = MockSuperWithError() + mock_super._should_raise = True + try: + with patch('builtins.super', return_value=mock_super): + et.__del__() + finally: + # Prevent the exception from being raised during garbage collection + mock_super._should_raise = False + + +def test_execute_update_success( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test _execute_update executes query successfully.""" + from model_manager.activities.experiment_tracking import ExperimentTracking + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + mock_connection = MagicMock() + mock_result = MagicMock() + mock_result.rowcount = 1 + mock_connection.execute.return_value = mock_result + mock_engine = MagicMock() + mock_engine.begin.return_value.__enter__.return_value = mock_connection + et.engine = mock_engine + + result = et._execute_update('UPDATE test SET x = :x', {'x': 1}) + + assert result == {'rowcount': 1} + mock_connection.execute.assert_called_once() + + +def test_update_experiment_run_status_success( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test update_experiment_run with STATUS update type.""" + from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + mock_execute = MagicMock() + + def mock_execute_update(*args, **kwargs): + mock_execute(*args, **kwargs) + return {'rowcount': 1} + + et._execute_update = mock_execute_update # type: ignore[method-assign] + et.info = MagicMock() + + input_data = { + 'metadata': {'workflow_id': 'test-123'}, + 'experiment_run_id': 1, + 'update_type': UpdateType.STATUS, + 'status': 'running', + } + + et.update_experiment_run(input_data) + + mock_execute.assert_called_once() + call_args = mock_execute.call_args + assert 'status' in call_args[0][1] + assert call_args[0][1]['status'] == 'running' + assert call_args[0][1]['experiment_run_id'] == 1 + et.info.assert_called_once() + + +def test_update_experiment_run_status_missing_status( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test update_experiment_run with STATUS but missing status parameter.""" + from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + et.send_notification = MagicMock() + + input_data = { + 'metadata': {'workflow_id': 'test-123'}, + 'experiment_run_id': 1, + 'update_type': UpdateType.STATUS, + } + + with pytest.raises(RuntimeError): + et.update_experiment_run(input_data) + + et.send_notification.assert_called_once() + + +def test_update_experiment_run_status_with_error_success( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test update_experiment_run with STATUS_WITH_ERROR update type.""" + from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + mock_execute = MagicMock() + + def mock_execute_update(*args, **kwargs): + mock_execute(*args, **kwargs) + return {'rowcount': 1} + + et._execute_update = mock_execute_update # type: ignore[method-assign] + et.info = MagicMock() + + input_data = { + 'metadata': {'workflow_id': 'test-123'}, + 'experiment_run_id': 1, + 'update_type': UpdateType.STATUS_WITH_ERROR, + 'status': 'failed', + 'error_message': 'Test error', + } + + et.update_experiment_run(input_data) + + mock_execute.assert_called_once() + call_args = mock_execute.call_args + assert 'status' in call_args[0][1] + assert call_args[0][1]['status'] == 'failed' + assert call_args[0][1]['error_message'] == 'Test error' + et.info.assert_called_once() + + +def test_update_experiment_run_status_with_error_truncate_message( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test update_experiment_run truncates error message if too long.""" + from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + mock_execute = MagicMock() + + def mock_execute_update(*args, **kwargs): + mock_execute(*args, **kwargs) + return {'rowcount': 1} + + et._execute_update = mock_execute_update # type: ignore[method-assign] + et.info = MagicMock() + + long_error = 'x' * 2000 + input_data = { + 'metadata': {'workflow_id': 'test-123'}, + 'experiment_run_id': 1, + 'update_type': UpdateType.STATUS_WITH_ERROR, + 'status': 'failed', + 'error_message': long_error, + } + + et.update_experiment_run(input_data) + + call_args = mock_execute.call_args + assert len(call_args[0][1]['error_message']) == 1024 + + +def test_update_experiment_run_status_with_error_missing_error_message( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test update_experiment_run with STATUS_WITH_ERROR but missing error_message.""" + from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + et.send_notification = MagicMock() + + input_data = { + 'metadata': {'workflow_id': 'test-123'}, + 'experiment_run_id': 1, + 'update_type': UpdateType.STATUS_WITH_ERROR, + 'status': 'failed', + } + + with pytest.raises(RuntimeError): + et.update_experiment_run(input_data) + + et.send_notification.assert_called_once() + + +def test_update_experiment_run_model_saved_success( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test update_experiment_run with MODEL_SAVED update type.""" + from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + mock_execute = MagicMock() + + def mock_execute_update(*args, **kwargs): + mock_execute(*args, **kwargs) + return {'rowcount': 1} + + et._execute_update = mock_execute_update # type: ignore[method-assign] + et.info = MagicMock() + + input_data = { + 'metadata': {'workflow_id': 'test-123'}, + 'experiment_run_id': 1, + 'update_type': UpdateType.MODEL_SAVED, + 'status': 'completed', + 'run_name': 'run_001', + } + + et.update_experiment_run(input_data) + + mock_execute.assert_called_once() + call_args = mock_execute.call_args + assert 'run_name' in call_args[0][1] + assert call_args[0][1]['run_name'] == 'run_001' + assert call_args[0][1]['status'] == 'completed' + et.info.assert_called_once() + + +def test_update_experiment_run_model_saved_missing_run_name( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test update_experiment_run with MODEL_SAVED but missing run_name.""" + from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + et.send_notification = MagicMock() + + input_data = { + 'metadata': {'workflow_id': 'test-123'}, + 'experiment_run_id': 1, + 'update_type': UpdateType.MODEL_SAVED, + 'status': 'completed', + } + + with pytest.raises(RuntimeError): + et.update_experiment_run(input_data) + + et.send_notification.assert_called_once() + + +def test_update_experiment_run_invalid_update_type( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test update_experiment_run with invalid update_type.""" + from model_manager.activities.experiment_tracking import ExperimentTracking + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + et.send_notification = MagicMock() + + input_data = { + 'metadata': {'workflow_id': 'test-123'}, + 'experiment_run_id': 1, + 'update_type': 'invalid_type', + } + + with pytest.raises(RuntimeError): + et.update_experiment_run(input_data) + + et.send_notification.assert_called_once() + + +def test_update_experiment_run_no_rows_updated( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test update_experiment_run raises error when no rows are updated.""" + from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + def mock_execute_update(*args, **kwargs): + return {'rowcount': 0} + + et._execute_update = mock_execute_update # type: ignore[method-assign] + et.send_notification = MagicMock() + + input_data = { + 'metadata': {'workflow_id': 'test-123'}, + 'experiment_run_id': 999, + 'update_type': UpdateType.STATUS, + 'status': 'running', + } + + with pytest.raises(RuntimeError): + et.update_experiment_run(input_data) + + et.send_notification.assert_called_once() + + +def test_update_experiment_run_status_with_error_missing_status( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test update_experiment_run with STATUS_WITH_ERROR but missing status - covers line 179.""" + from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + et.send_notification = MagicMock() + + input_data = { + 'metadata': {'workflow_id': 'test-123'}, + 'experiment_run_id': 1, + 'update_type': UpdateType.STATUS_WITH_ERROR, + 'error_message': 'Some error', + } + + with pytest.raises(RuntimeError): + et.update_experiment_run(input_data) + + et.send_notification.assert_called_once() + + +def test_update_experiment_run_model_saved_missing_status( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test update_experiment_run with MODEL_SAVED but missing status - covers line 204.""" + from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + et.send_notification = MagicMock() + + input_data = { + 'metadata': {'workflow_id': 'test-123'}, + 'experiment_run_id': 1, + 'update_type': UpdateType.MODEL_SAVED, + 'run_name': 'run_001', + } + + with pytest.raises(RuntimeError): + et.update_experiment_run(input_data) + + et.send_notification.assert_called_once() + + +def test_experiment_tracking_del_with_engine_no_super_del( + db_config, mock_logger, mock_notification_handler, mock_metrics_controller +): + """Test __del__ when engine exists but super has no __del__ - covers line 103.""" + from model_manager.activities.experiment_tracking import ExperimentTracking + + et = ExperimentTracking( + host=db_config['host'], + port=db_config['port'], + user=db_config['user'], + password=db_config['password'], + dbname=db_config['dbname'], + min_connections=db_config['min_connections'], + max_connections=db_config['max_connections'], + logger=mock_logger, + notification_handler=mock_notification_handler, + metrics_controller=mock_metrics_controller, + ) + + et.engine = MagicMock() + + class MockSuperNoDel: + pass + + with patch('builtins.super', return_value=MockSuperNoDel()): + et.__del__() diff --git a/tests/activities/test_training.py b/tests/activities/test_training.py new file mode 100644 index 0000000..9c2395e --- /dev/null +++ b/tests/activities/test_training.py @@ -0,0 +1,630 @@ +"""Unit tests for Training activities.""" + +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from model_manager.utils.models.train_model_params import TrainModelParams +from model_manager.utils.models.train_model_result import TrainModelResult + + +def _minimal_params_dict(): + return { + 'variable_columns': ['a'], + 'target_variable': 't', + 'bucket_name': 'b', + 'file_name': 'f.csv', + 'line_separator': '\n', + 'decimal_separator': '.', + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', + 'train_size': 80, + 'shuffle': True, + 'random_state': 42, + 'experiment_run_id': 1, + 'model_name': 'Linear Regression', + 'val_file_name': None, + 'data_model_kwargs': {}, + 'model_kwargs': {}, + 'opt_params': {}, + 'model_type': 'linear_regression', + 'model_id': None, + 'model_metadata': None, + } + + +@pytest.fixture +def training(): + from model_manager.activities.training import Training + + return Training( + mlflow_repository=MagicMock(), + plugin_store=MagicMock(), + minio_repository=MagicMock(), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(), + ) + + +def test_load_model_metadata_success(training): + training.plugin_store.get_model_index = MagicMock( + return_value={'schemas': {'components': {'schemas': {}}}} + ) + inp = {**_minimal_params_dict(), 'metadata': {'w': '1'}} + out = training.load_model_metadata(inp) + assert 'model_metadata' in out + assert out['model_metadata']['schemas'] + + +def test_load_model_metadata_notifies_on_error(training): + training.plugin_store.get_model_index = MagicMock(side_effect=RuntimeError('idx')) + training.send_notification = MagicMock() + inp = {**_minimal_params_dict(), 'metadata': {}} + with pytest.raises(RuntimeError, match='idx'): + training.load_model_metadata(inp) + training.send_notification.assert_called_once() + + +def test_validate_train_params_success(training): + pdict = _minimal_params_dict() + pdict['model_metadata'] = {'schemas': {'components': {'schemas': {}}}} + inp = {**pdict, 'metadata': {}} + out = training.validate_train_params(inp) + assert isinstance(out, dict) + assert out['target_variable'] == 't' + + +def test_validate_train_params_notifies(training): + training.send_notification = MagicMock() + inp = {'metadata': {}, 'experiment_run_id': 1} + with pytest.raises((KeyError, ValueError, TypeError)): + training.validate_train_params(inp) + training.send_notification.assert_called_once() + + +def test_train_model_download_fails_notifies(training): + """train_model notifies and re-raises when MinIO download fails.""" + tp = TrainModelParams.from_dict( + { + **_minimal_params_dict(), + 'model_metadata': {'schemas': {'components': {'schemas': {}}}}, + } + ) + training.minio_repository.download_file = MagicMock(side_effect=OSError('minio')) + training.send_notification = MagicMock() + with pytest.raises(OSError, match='minio'): + training.train_model({'metadata': {'pod': 'x'}, 'train_params': tp.to_dict()}) + training.send_notification.assert_called_once() + + +def test_cleanup_resources(training): + training.data_manager_repository.cleanup_run_directory = MagicMock() + training.cleanup_resources({'metadata': {}, 'run_dir': '/tmp/x'}) + training.data_manager_repository.cleanup_run_directory.assert_called_once_with('/tmp/x', {}) + + +def test_cleanup_resources_notifies_on_error(training): + training.data_manager_repository.cleanup_run_directory = MagicMock( + side_effect=RuntimeError('rm') + ) + training.send_notification = MagicMock() + with pytest.raises(RuntimeError, match='rm'): + training.cleanup_resources({'metadata': {'pod': 'p'}, 'run_dir': '/tmp/x'}) + training.send_notification.assert_called_once() + + +@patch('model_manager.activities.training.mlflow') +def test_train_model_success_serializes_result(mock_mlflow, training): + """Exercise train_model happy path with mocks (MinIO, plugin wrapper, MLflow).""" + tp = TrainModelParams.from_dict( + { + **_minimal_params_dict(), + 'model_metadata': {'schemas': {'components': {'schemas': {}}}}, + } + ) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + training.minio_repository.download_file = MagicMock(return_value=b'csv') + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + + def _set_metrics(x, _w, **_kw): + x.mse_val = 0.1 + x.mae_val = 0.2 + x.r2_val = 0.9 + return x + + training.data_manager_repository.compute_regression_metrics = MagicMock( + side_effect=_set_metrics + ) + + def _fill_report(x, **_kw): + x.report_path = '/tmp/report.html' + x.train_data_path = '/tmp/train.csv' + x.test_data_path = '/tmp/test.csv' + x.run_dir = '/tmp/run' + return x + + training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report) + + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + pred_train = pd.DataFrame({'p': [1.0, 2.0]}) + pred_val = pd.DataFrame({'p': [1.0]}) + wrapper.predict = MagicMock(side_effect=[(pred_train, None), (pred_val, None)]) + wrapper.store_model = MagicMock() + training.plugin_store.get_model = MagicMock(return_value=wrapper) + + @contextmanager + def _run_ctx(*_a, **_k): + info = MagicMock() + info.run_name = 'run-n' + info.run_id = 'run-i' + yield info + + training.mlflow_repository.start_run = _run_ctx + + out = training.train_model({'metadata': {'pod': 'p'}, 'train_params': tp.to_dict()}) + assert out['run_name'] is None + assert out['run_id'] == 'run-i' + assert out['run_dir'] == '/tmp/run' + mock_mlflow.log_param.assert_any_call('mse_val', 0.1) + mock_mlflow.log_param.assert_any_call('mae_val', 0.2) + mock_mlflow.log_param.assert_any_call('r2_val', 0.9) + mock_mlflow.log_artifact.assert_called() + + +@patch('model_manager.activities.training.mlflow') +def test_train_model_without_logger_does_not_set_wrapper_logger(_mock_mlflow, training): + """Covers branch where activity logger is None.""" + training.logger = None + tp = TrainModelParams.from_dict( + { + **_minimal_params_dict(), + 'model_metadata': {'schemas': {'components': {'schemas': {}}}}, + } + ) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + training.minio_repository.download_file = MagicMock(return_value=b'csv') + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + training.data_manager_repository.compute_regression_metrics = MagicMock( + side_effect=lambda x, _w, **_kw: x + ) + + def _fill_report(x, **_kw): + x.report_path = '/tmp/report.html' + x.train_data_path = '/tmp/train.csv' + x.test_data_path = '/tmp/test.csv' + x.equation_path = '/tmp/eq.json' + x.run_dir = '/tmp/run' + return x + + training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report) + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + wrapper.store_model = MagicMock() + training.plugin_store.get_model = MagicMock(return_value=wrapper) + + @contextmanager + def _run_ctx(*_a, **_k): + info = MagicMock() + info.run_name = 'n' + info.run_id = 'i' + yield info + + training.mlflow_repository.start_run = _run_ctx + + training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) + + +@patch('model_manager.activities.training.mlflow') +def test_train_model_train_params_as_dict(mock_mlflow, training): + """train_params may arrive as dict and is coerced via TrainModelParams.from_dict.""" + d = { + **_minimal_params_dict(), + 'model_metadata': {'schemas': {'components': {'schemas': {}}}}, + } + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tp = TrainModelParams.from_dict(d) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + training.minio_repository.download_file = MagicMock(return_value=b'csv') + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + training.data_manager_repository.compute_regression_metrics = MagicMock( + side_effect=lambda x, _w, **_kw: x + ) + + def _fill_report2(x, **_kw): + x.report_path = '/tmp/report.html' + x.train_data_path = '/tmp/train.csv' + x.test_data_path = '/tmp/test.csv' + x.run_dir = '/tmp/run' + return x + + training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report2) + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + wrapper.store_model = MagicMock() + training.plugin_store.get_model = MagicMock(return_value=wrapper) + + @contextmanager + def _run_ctx(*_a, **_k): + info = MagicMock() + info.run_name = 'n' + info.run_id = 'i' + yield info + + training.mlflow_repository.start_run = _run_ctx + + training.train_model({'metadata': {}, 'train_params': d}) + mock_mlflow.log_artifact.assert_called() + + +@patch('model_manager.activities.training.mlflow') +def test_train_model_downloads_validation_file_when_set(mock_mlflow, training): + """Second MinIO download when val_file_name is set (covers val_bytes branch).""" + d = { + **_minimal_params_dict(), + 'val_file_name': 'val.csv', + 'model_metadata': {'schemas': {'components': {'schemas': {}}}}, + } + tp = TrainModelParams.from_dict(d) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + def _dl(object_name, **_kwargs): + if object_name == tp.file_name: + return b'train' + if object_name == 'val.csv': + return b'val' + raise AssertionError(object_name) + + training.minio_repository.download_file = MagicMock(side_effect=_dl) + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + training.data_manager_repository.compute_regression_metrics = MagicMock( + side_effect=lambda x, _w, **_kw: x + ) + + def _fill(x, **_kw): + x.report_path = '/tmp/report.html' + x.train_data_path = '/tmp/train.csv' + x.test_data_path = '/tmp/test.csv' + x.run_dir = '/tmp/run' + return x + + training.data_manager_repository.generate_report = MagicMock(side_effect=_fill) + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + wrapper.store_model = MagicMock() + training.plugin_store.get_model = MagicMock(return_value=wrapper) + + @contextmanager + def _run_ctx(*_a, **_k): + info = MagicMock() + info.run_name = 'n' + info.run_id = 'i' + yield info + + training.mlflow_repository.start_run = _run_ctx + + training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) + assert training.minio_repository.download_file.call_count == 2 + mock_mlflow.log_artifact.assert_called() + + +def test_prepare_data_observes_lag_on_success(training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + from model_manager import metrics as mm_metrics + + training._prepare_data(b'csv', None, tp, {}) + + training.observe_lag_sync.assert_called_once() + call_args = training.observe_lag_sync.call_args + assert call_args.args[1] is mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_LAG + training.emit_metric_sync.assert_not_called() + + +def test_prepare_data_increments_error_counter_and_still_observes_lag_on_failure(training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + training.data_manager_repository.prepare_training_data = MagicMock( + side_effect=RuntimeError('prep-fail') + ) + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + from model_manager import metrics as mm_metrics + + with pytest.raises(RuntimeError, match='prep-fail'): + training._prepare_data(b'csv', None, tp, {}) + + training.observe_lag_sync.assert_called_once() + training.emit_metric_sync.assert_called_once() + call_args = training.emit_metric_sync.call_args + assert call_args.kwargs['metric_object'] is mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL + + +def test_fit_model_observes_lag_on_success(training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + from model_manager import metrics as mm_metrics + + training._fit_model(wrapper, tmr, tp, {}) + + training.observe_lag_sync.assert_called_once() + call_args = training.observe_lag_sync.call_args + assert call_args.args[1] is mm_metrics.SIENTIA_TRAINING_MODEL_FIT_LAG + training.emit_metric_sync.assert_not_called() + + +def test_fit_model_increments_error_counter_on_failure(training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + wrapper = MagicMock() + wrapper.train = MagicMock(side_effect=RuntimeError('fit-fail')) + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + from model_manager import metrics as mm_metrics + + with pytest.raises(RuntimeError, match='fit-fail'): + training._fit_model(wrapper, tmr, tp, {}) + + training.observe_lag_sync.assert_called_once() + training.emit_metric_sync.assert_called_once() + call_args = training.emit_metric_sync.call_args + assert call_args.kwargs['metric_object'] is mm_metrics.SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL + + +@patch('model_manager.activities.training.mm_metrics') +@patch('model_manager.activities.training.mlflow') +def test_train_model_sets_quality_gauges_after_compute_metrics(mock_mlflow, mock_mm_metrics, training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + training.minio_repository.download_file = MagicMock(return_value=b'csv') + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + + def _set_metrics(x, _w, **_kw): + x.mse_val = 0.5 + x.mae_val = 0.3 + x.r2_val = -0.1 + return x + + training.data_manager_repository.compute_regression_metrics = MagicMock( + side_effect=_set_metrics + ) + + def _fill_report(x, **_kw): + x.report_path = '/tmp/r.html' + x.train_data_path = '/tmp/tr.csv' + x.test_data_path = '/tmp/te.csv' + x.run_dir = '/tmp/run' + return x + + training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report) + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + wrapper.store_model = MagicMock() + training.plugin_store.get_model = MagicMock(return_value=wrapper) + + @contextmanager + def _run_ctx(*_a, **_k): + info = MagicMock() + info.run_id = 'rid' + yield info + + training.mlflow_repository.start_run = _run_ctx + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) + + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels.return_value.set.assert_called_once_with(0.5) + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels.return_value.set.assert_called_once_with(0.3) + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels.return_value.set.assert_called_once_with(-0.1) + + +@patch('model_manager.activities.training.mm_metrics') +@patch('model_manager.activities.training.mlflow') +def test_train_model_skips_quality_gauges_when_none(_mock_mlflow, mock_mm_metrics, training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + training.minio_repository.download_file = MagicMock(return_value=b'csv') + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + training.data_manager_repository.compute_regression_metrics = MagicMock( + side_effect=lambda x, _w, **_kw: x + ) + + def _fill_report(x, **_kw): + x.report_path = '/tmp/r.html' + x.train_data_path = '/tmp/tr.csv' + x.test_data_path = '/tmp/te.csv' + x.run_dir = '/tmp/run' + return x + + training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report) + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + wrapper.store_model = MagicMock() + training.plugin_store.get_model = MagicMock(return_value=wrapper) + + @contextmanager + def _run_ctx(*_a, **_k): + info = MagicMock() + info.run_id = 'rid' + yield info + + training.mlflow_repository.start_run = _run_ctx + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) + + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels.return_value.set.assert_not_called() + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels.return_value.set.assert_not_called() + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels.return_value.set.assert_not_called() + + +@patch('model_manager.activities.training.mm_metrics') +@patch('model_manager.activities.training.mlflow') +def test_train_model_increments_trained_total_on_success(_mock_mlflow, mock_mm_metrics, training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + training.minio_repository.download_file = MagicMock(return_value=b'csv') + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + training.data_manager_repository.compute_regression_metrics = MagicMock( + side_effect=lambda x, _w, **_kw: x + ) + + def _fill_report(x, **_kw): + x.report_path = '/tmp/r.html' + x.train_data_path = '/tmp/tr.csv' + x.test_data_path = '/tmp/te.csv' + x.run_dir = '/tmp/run' + return x + + training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report) + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + wrapper.store_model = MagicMock() + training.plugin_store.get_model = MagicMock(return_value=wrapper) + + @contextmanager + def _run_ctx(*_a, **_k): + info = MagicMock() + info.run_id = 'rid' + yield info + + training.mlflow_repository.start_run = _run_ctx + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) + + training.emit_metric_sync.assert_called_once_with( + metric_object=mock_mm_metrics.SIENTIA_TRAINING_MODEL_TRAINED_TOTAL, + tags=training._get_training_labels(tp), + ) + + +def test_train_model_does_not_increment_trained_total_on_failure(training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + training.minio_repository.download_file = MagicMock(side_effect=RuntimeError('dl-fail')) + training.send_notification = MagicMock() + training.emit_metric_sync = MagicMock() + + with pytest.raises(RuntimeError): + training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) + + training.emit_metric_sync.assert_not_called() + + +def test_train_model_value_error_when_paths_missing_after_report(training): + """Raises ValueError when report paths are not populated after generate_report.""" + tp = TrainModelParams.from_dict( + { + **_minimal_params_dict(), + 'model_metadata': {'schemas': {'components': {'schemas': {}}}}, + } + ) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + training.minio_repository.download_file = MagicMock(return_value=b'x') + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + training.data_manager_repository.compute_regression_metrics = MagicMock( + side_effect=lambda x, _w, **_kw: x + ) + training.data_manager_repository.generate_report = MagicMock(return_value=tmr) + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + training.plugin_store.get_model = MagicMock(return_value=wrapper) + + @contextmanager + def _run_ctx(*_a, **_k): + info = MagicMock() + info.run_name = 'n' + info.run_id = 'i' + yield info + + training.mlflow_repository.start_run = _run_ctx + training.send_notification = MagicMock() + + with pytest.raises(ValueError, match='Report path'): + training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..cdeaec1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,103 @@ +""" +Test bootstrap: stub optional `sientia_do` submodules not shipped in minimal installs. + +Must run before importing `model_manager.sientia.models` (pulled in via TrainModelParams). +Stubs Evidently submodules so `model_manager.sientia.reports` imports (via DataManagerRepository). +""" + +from __future__ import annotations + +import sys +from types import ModuleType + + +def _make_dummy(name: str) -> type: + return type(name, (), {}) + + +def _stub_evidently() -> None: + """Minimal Evidently API surface required to import `model_manager.sientia.reports`.""" + ev = ModuleType('evidently') + sys.modules['evidently'] = ev + + mp = ModuleType('evidently.metric_preset') + mp.DataDriftPreset = _make_dummy('DataDriftPreset') # type: ignore[attr-defined] + sys.modules['evidently.metric_preset'] = mp + + metrics = ModuleType('evidently.metrics') + _metric_names = ( + 'ColumnSummaryMetric', + 'ConflictTargetMetric', + 'DatasetCorrelationsMetric', + 'DatasetSummaryMetric', + 'RegressionAbsPercentageErrorPlot', + 'RegressionDummyMetric', + 'RegressionErrorDistribution', + 'RegressionErrorPlot', + 'RegressionPerformanceMetrics', + 'RegressionPredictedVsActualPlot', + 'RegressionPredictedVsActualScatter', + ) + for n in _metric_names: + setattr(metrics, n, _make_dummy(n)) + sys.modules['evidently.metrics'] = metrics + + base = ModuleType('evidently.metrics.base_metric') + + def generate_column_metrics(*_a, **_k): + return [] + + base.generate_column_metrics = generate_column_metrics # type: ignore[attr-defined] + sys.modules['evidently.metrics.base_metric'] = base + + opt = ModuleType('evidently.options') + opt.ColorOptions = _make_dummy('ColorOptions') # type: ignore[attr-defined] + sys.modules['evidently.options'] = opt + + pipeline = ModuleType('evidently.pipeline') + sys.modules['evidently.pipeline'] = pipeline + + colmap = ModuleType('evidently.pipeline.column_mapping') + colmap.ColumnMapping = _make_dummy('ColumnMapping') # type: ignore[attr-defined] + sys.modules['evidently.pipeline.column_mapping'] = colmap + + rep = ModuleType('evidently.report') + rep.Report = _make_dummy('Report') # type: ignore[attr-defined] + sys.modules['evidently.report'] = rep + + +def pytest_configure(config) -> None: # noqa: ARG001 + """Register stub modules so imports used by production code resolve in CI/dev venvs.""" + _stub_evidently() + + if 'sientia_do.operations.df_preprocessor' not in sys.modules: + df_pre = ModuleType('sientia_do.operations.df_preprocessor') + + def create_features(input_data, *_a, **_k): + return input_data + + def limit_dataset(input_data, low_lim, upp_lim, *_a, **_k): + return input_data, low_lim, upp_lim + + def treat_nan(input_data, *_a, **_k): + return input_data + + df_pre.create_features = create_features # type: ignore[attr-defined] + df_pre.limit_dataset = limit_dataset # type: ignore[attr-defined] + df_pre.treat_nan = treat_nan # type: ignore[attr-defined] + sys.modules['sientia_do.operations.df_preprocessor'] = df_pre + + sys.modules.setdefault('sientia_do.operations', ModuleType('sientia_do.operations')) + + if 'sientia_do.timeseries.analyzer' not in sys.modules: + ts_an = ModuleType('sientia_do.timeseries.analyzer') + + class TimeSeriesDiscontinuityAnalyzer: # noqa: D401 + """Stub for tests.""" + + pass + + ts_an.TimeSeriesDiscontinuityAnalyzer = TimeSeriesDiscontinuityAnalyzer # type: ignore[attr-defined] + sys.modules['sientia_do.timeseries.analyzer'] = ts_an + + sys.modules.setdefault('sientia_do.timeseries', ModuleType('sientia_do.timeseries')) diff --git a/tests/schedules/__init__.py b/tests/schedules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/schedules/test_cleanup_schedule.py b/tests/schedules/test_cleanup_schedule.py new file mode 100644 index 0000000..86f4d6a --- /dev/null +++ b/tests/schedules/test_cleanup_schedule.py @@ -0,0 +1,469 @@ +"""Tests for cleanup schedule management.""" + +import os +from datetime import timedelta +from importlib import reload +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + + +@pytest.fixture +def mock_temporal_client(): + """Fixture for a mock Temporal client.""" + client = AsyncMock() + client.list_schedules = AsyncMock() + client.create_schedule = AsyncMock() + handle = AsyncMock() + handle.delete = AsyncMock() + schedule = MagicMock() + schedule.action.task_queue = 'cleanup_files-model-manager-worker-queue' + schedule.action.execution_timeout = timedelta(hours=1) + schedule.spec.cron_expressions = ['0 0 * * *'] + schedule.spec.time_zone_name = 'UTC' + handle.describe = AsyncMock(return_value=MagicMock(schedule=schedule)) + client.get_schedule_handle = MagicMock(return_value=handle) + return client + + +@pytest.fixture +def mock_logger(): + """Fixture for a mock Sientia logger.""" + logger = MagicMock() + logger.custom_info = MagicMock() + logger.custom_error = MagicMock() + return logger + + +@pytest.fixture +def metadata(): + """Fixture for metadata dict.""" + return {'pod_id': 'test-pod', 'project_name': 'test-project'} + + +# --- schedule_exists Tests --- + + +@pytest.mark.asyncio +async def test_schedule_exists_returns_true_when_schedule_found( + mock_temporal_client, mock_logger, metadata +): + """Test that schedule_exists returns True when schedule is found.""" + from model_manager.schedules.cleanup_schedule import schedule_exists + + # Mock schedule list with matching schedule + mock_schedule = MagicMock() + mock_schedule.id = 'test-schedule-id' + + async def mock_list_schedules(): + yield mock_schedule + + mock_temporal_client.list_schedules.return_value = mock_list_schedules() + + result = await schedule_exists(mock_temporal_client, 'test-schedule-id', mock_logger, metadata) + + assert result is True + mock_temporal_client.list_schedules.assert_called_once() + + +@pytest.mark.asyncio +async def test_schedule_exists_returns_false_when_schedule_not_found( + mock_temporal_client, mock_logger, metadata +): + """Test that schedule_exists returns False when schedule is not found.""" + from model_manager.schedules.cleanup_schedule import schedule_exists + + # Mock empty schedule list + async def mock_list_schedules(): + return + yield # Make it an async generator + + mock_temporal_client.list_schedules.return_value = mock_list_schedules() + + result = await schedule_exists( + mock_temporal_client, 'nonexistent-schedule', mock_logger, metadata + ) + + assert result is False + mock_temporal_client.list_schedules.assert_called_once() + + +@pytest.mark.asyncio +async def test_schedule_exists_returns_false_when_different_schedule_found( + mock_temporal_client, mock_logger, metadata +): + """Test that schedule_exists returns False when only different schedules exist.""" + from model_manager.schedules.cleanup_schedule import schedule_exists + + # Mock schedule list with non-matching schedule + mock_schedule = MagicMock() + mock_schedule.id = 'different-schedule-id' + + async def mock_list_schedules(): + yield mock_schedule + + mock_temporal_client.list_schedules.return_value = mock_list_schedules() + + result = await schedule_exists(mock_temporal_client, 'test-schedule-id', mock_logger, metadata) + + assert result is False + mock_temporal_client.list_schedules.assert_called_once() + + +@pytest.mark.asyncio +async def test_schedule_exists_handles_exception(mock_temporal_client, mock_logger, metadata): + """Test that schedule_exists handles exceptions gracefully.""" + from model_manager.schedules.cleanup_schedule import schedule_exists + + # Mock list_schedules to raise an exception + mock_temporal_client.list_schedules.side_effect = Exception('Connection error') + + result = await schedule_exists(mock_temporal_client, 'test-schedule-id', mock_logger, metadata) + + assert result is False + mock_logger.custom_error.assert_called_once() + assert 'Error checking if schedule exists' in mock_logger.custom_error.call_args[0][0] + + +@pytest.mark.asyncio +async def test_needs_schedule_reconcile_handles_describe_exception(mock_logger, metadata): + """Test _needs_schedule_reconcile returns True and logs when describe fails.""" + from model_manager.schedules.cleanup_schedule import _needs_schedule_reconcile + + handle = AsyncMock() + handle.describe = AsyncMock(side_effect=RuntimeError('describe failed')) + + needs_reconcile = await _needs_schedule_reconcile( + schedule_handle=handle, + cleanup_task_queue='cleanup_files-model-manager-worker-queue', + logger=mock_logger, + metadata=metadata, + ) + + assert needs_reconcile is True + mock_logger.custom_error.assert_called_once() + assert ( + 'Error describing cleanup schedule for reconcile' + in mock_logger.custom_error.call_args[0][0] + ) + + +# --- create_cleanup_schedule Tests --- + + +@pytest.mark.asyncio +@patch.dict( + 'model_manager.schedules.cleanup_schedule.os.environ', + { + 'RUNTIME': 'model-manager-worker', + }, +) +async def test_create_cleanup_schedule_reconciles_when_exists( + mock_temporal_client, mock_logger, metadata +): + """Test that create_cleanup_schedule recreates schedule when it already exists.""" + import model_manager.schedules.cleanup_schedule + + reload(model_manager.schedules.cleanup_schedule) + from model_manager.schedules.cleanup_schedule import create_cleanup_schedule + + # Mock schedule already exists + mock_schedule = MagicMock() + mock_schedule.id = 'cleanup-files-model-manager-worker-daily' + + async def mock_list_schedules(): + yield mock_schedule + + mock_temporal_client.list_schedules.return_value = mock_list_schedules() + + # Force reconcile by diverging task queue + mock_temporal_client.get_schedule_handle.return_value.describe.return_value.schedule.action.task_queue = 'different-queue' + + await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata) + + # Verify schedule was reconciled via delete + create + mock_temporal_client.get_schedule_handle.assert_called_once_with( + 'cleanup-files-model-manager-worker-daily' + ) + mock_temporal_client.get_schedule_handle.return_value.delete.assert_called_once() + mock_temporal_client.create_schedule.assert_called_once() + + mock_logger.custom_info.assert_called_once() + assert 'reconciled successfully' in mock_logger.custom_info.call_args[0][0] + + +@pytest.mark.asyncio +@patch.dict( + 'model_manager.schedules.cleanup_schedule.os.environ', + { + 'RUNTIME': 'model-manager-worker', + }, +) +async def test_create_cleanup_schedule_noop_when_schedule_is_up_to_date( + mock_temporal_client, mock_logger, metadata +): + """Test no-op reconcile when existing schedule already matches current config.""" + import model_manager.schedules.cleanup_schedule + + reload(model_manager.schedules.cleanup_schedule) + from model_manager.schedules.cleanup_schedule import create_cleanup_schedule + + mock_schedule = MagicMock() + mock_schedule.id = 'cleanup-files-model-manager-worker-daily' + + async def mock_list_schedules(): + yield mock_schedule + + mock_temporal_client.list_schedules.return_value = mock_list_schedules() + + await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata) + + mock_temporal_client.get_schedule_handle.assert_called_once_with( + 'cleanup-files-model-manager-worker-daily' + ) + mock_temporal_client.get_schedule_handle.return_value.delete.assert_not_called() + mock_temporal_client.create_schedule.assert_not_called() + mock_logger.custom_info.assert_called_once() + assert 'no-op reconcile' in mock_logger.custom_info.call_args[0][0] + + +@pytest.mark.asyncio +@patch.dict( + 'model_manager.schedules.cleanup_schedule.os.environ', + { + 'RUNTIME': 'model-manager-worker', + 'CLEANUP_CRON': '0 2 * * *', + 'CLEANUP_TIMEZONE': 'America/Sao_Paulo', + 'CLEANUP_EXECUTION_TIMEOUT_HOURS': '2', + }, +) +async def test_create_cleanup_schedule_creates_with_custom_config( + mock_temporal_client, mock_logger, metadata +): + """Test that create_cleanup_schedule creates schedule with custom configuration.""" + import model_manager.schedules.cleanup_schedule + + reload(model_manager.schedules.cleanup_schedule) + from model_manager.schedules.cleanup_schedule import create_cleanup_schedule + + # Mock schedule does not exist (empty list) + async def mock_list_schedules(): + return + yield # Make it an async generator + + mock_temporal_client.list_schedules.return_value = mock_list_schedules() + + await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata) + + # Verify schedule creation was called + mock_temporal_client.create_schedule.assert_called_once() + + # Verify schedule parameters + call_args = mock_temporal_client.create_schedule.call_args + schedule_id = call_args[0][0] + schedule_obj = call_args[0][1] + + assert schedule_id == 'cleanup-files-model-manager-worker-daily' + assert schedule_obj.action.workflow == 'cleanup_files' + assert schedule_obj.action.task_queue == 'cleanup_files-model-manager-worker-queue' + assert schedule_obj.action.execution_timeout == timedelta(hours=2) + assert schedule_obj.spec.cron_expressions == ['0 2 * * *'] + assert schedule_obj.spec.time_zone_name == 'America/Sao_Paulo' + + # Verify success log was called + assert mock_logger.custom_info.call_count == 1 + assert 'created successfully' in mock_logger.custom_info.call_args[0][0] + + +@pytest.mark.asyncio +@patch.dict( + 'model_manager.schedules.cleanup_schedule.os.environ', + { + 'RUNTIME': 'model-manager-worker', + }, +) +async def test_create_cleanup_schedule_uses_defaults(mock_temporal_client, mock_logger, metadata): + """Test that create_cleanup_schedule uses default values when env vars not set.""" + import model_manager.schedules.cleanup_schedule + + # Remove optional env vars to test defaults + for key in [ + 'CLEANUP_CRON', + 'CLEANUP_TIMEZONE', + 'CLEANUP_EXECUTION_TIMEOUT_HOURS', + ]: + os.environ.pop(key, None) + + reload(model_manager.schedules.cleanup_schedule) + from model_manager.schedules.cleanup_schedule import create_cleanup_schedule + + # Mock schedule does not exist (empty list) + async def mock_list_schedules(): + return + yield # Make it an async generator + + mock_temporal_client.list_schedules.return_value = mock_list_schedules() + + await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata) + + # Verify schedule creation was called + mock_temporal_client.create_schedule.assert_called_once() + + # Verify default parameters + call_args = mock_temporal_client.create_schedule.call_args + schedule_obj = call_args[0][1] + + assert schedule_obj.spec.cron_expressions == ['0 0 * * *'] # Default midnight + assert schedule_obj.spec.time_zone_name == 'UTC' # Default UTC + assert schedule_obj.action.task_queue == 'cleanup_files-model-manager-worker-queue' + assert schedule_obj.action.execution_timeout == timedelta(hours=1) # Default 1 hour + + +@pytest.mark.asyncio +@patch.dict( + 'model_manager.schedules.cleanup_schedule.os.environ', + { + 'RUNTIME': 'model-manager-worker', + }, +) +async def test_create_cleanup_schedule_workflow_id_format( + mock_temporal_client, mock_logger, metadata +): + """Test that workflow ID is correctly formatted with schedule ID.""" + import model_manager.schedules.cleanup_schedule + + reload(model_manager.schedules.cleanup_schedule) + from model_manager.schedules.cleanup_schedule import create_cleanup_schedule + + # Mock schedule does not exist (empty list) + async def mock_list_schedules(): + return + yield # Make it an async generator + + mock_temporal_client.list_schedules.return_value = mock_list_schedules() + + await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata) + + # Verify workflow ID format + call_args = mock_temporal_client.create_schedule.call_args + schedule_obj = call_args[0][1] + + expected_workflow_id = 'cleanup-files-scheduled-cleanup-files-model-manager-worker-daily' + assert schedule_obj.action.id == expected_workflow_id + + +@pytest.mark.asyncio +@patch.dict( + 'model_manager.schedules.cleanup_schedule.os.environ', + { + 'RUNTIME': 'model-manager-worker', + }, +) +async def test_create_cleanup_schedule_empty_workflow_args( + mock_temporal_client, mock_logger, metadata +): + """Test that workflow is created with empty args (uses env defaults).""" + import model_manager.schedules.cleanup_schedule + + reload(model_manager.schedules.cleanup_schedule) + from model_manager.schedules.cleanup_schedule import create_cleanup_schedule + + # Mock schedule does not exist (empty list) + async def mock_list_schedules(): + return + yield # Make it an async generator + + mock_temporal_client.list_schedules.return_value = mock_list_schedules() + + await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata) + + # Verify workflow args are empty (it's a list with one empty dict) + call_args = mock_temporal_client.create_schedule.call_args + schedule_obj = call_args[0][1] + + # The args are passed as positional args, so it's a list with one element + assert schedule_obj.action.args == [{}] + + +# --- Environment Variable Configuration Tests --- + + +@patch.dict( + 'model_manager.schedules.cleanup_schedule.os.environ', + { + 'RUNTIME': 'model-manager-worker', + 'CLEANUP_CRON': '30 3 * * 1', + 'CLEANUP_TIMEZONE': 'Europe/London', + 'CLEANUP_EXECUTION_TIMEOUT_HOURS': '3', + }, +) +def test_environment_variables_loaded_correctly(): + """Test that environment variables are loaded correctly.""" + import model_manager.schedules.cleanup_schedule + + reload(model_manager.schedules.cleanup_schedule) + from model_manager.schedules.cleanup_schedule import ( + CLEANUP_CRON, + CLEANUP_EXECUTION_TIMEOUT_HOURS, + CLEANUP_TIMEZONE, + build_cleanup_schedule_id, + ) + + assert ( + build_cleanup_schedule_id('model-manager-worker') + == 'cleanup-files-model-manager-worker-daily' + ) + assert CLEANUP_CRON == '30 3 * * 1' + assert CLEANUP_TIMEZONE == 'Europe/London' + assert CLEANUP_EXECUTION_TIMEOUT_HOURS == 3 + + +def test_environment_variables_use_defaults_when_not_set(): + """Test that default values are used when environment variables are not set.""" + import model_manager.schedules.cleanup_schedule + + # Remove all env vars + for key in [ + 'RUNTIME', + 'CLEANUP_CRON', + 'CLEANUP_TIMEZONE', + 'CLEANUP_EXECUTION_TIMEOUT_HOURS', + ]: + os.environ.pop(key, None) + + reload(model_manager.schedules.cleanup_schedule) + from model_manager.schedules.cleanup_schedule import ( + CLEANUP_CRON, + CLEANUP_EXECUTION_TIMEOUT_HOURS, + CLEANUP_TIMEZONE, + build_cleanup_schedule_id, + ) + + assert build_cleanup_schedule_id(None) == 'cleanup-files-single-daily' + assert CLEANUP_CRON == '0 0 * * *' + assert CLEANUP_TIMEZONE == 'UTC' + assert CLEANUP_EXECUTION_TIMEOUT_HOURS == 1 + + +@pytest.mark.asyncio +async def test_create_cleanup_schedule_uses_single_runtime_when_runtime_missing( + mock_temporal_client, mock_logger, metadata +): + """Test create_cleanup_schedule uses single runtime fallback.""" + import model_manager.schedules.cleanup_schedule + + os.environ.pop('RUNTIME', None) + reload(model_manager.schedules.cleanup_schedule) + from model_manager.schedules.cleanup_schedule import create_cleanup_schedule + + async def mock_list_schedules(): + return + yield + + mock_temporal_client.list_schedules.return_value = mock_list_schedules() + + await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata) + + call_args = mock_temporal_client.create_schedule.call_args + schedule_obj = call_args[0][1] + assert schedule_obj.action.task_queue == 'cleanup_files-single-queue' diff --git a/tests/sientia/__init__.py b/tests/sientia/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/sientia/test_exceptions.py b/tests/sientia/test_exceptions.py new file mode 100644 index 0000000..c1d25e9 --- /dev/null +++ b/tests/sientia/test_exceptions.py @@ -0,0 +1,9 @@ +"""Unit tests for custom exception aliases.""" + +from mlflow.exceptions import MlflowException + +from model_manager.sientia.exceptions import SientiaMlException + + +def test_sientia_ml_exception_is_mlflow_exception_alias(): + assert SientiaMlException is MlflowException diff --git a/tests/sientia/test_metrics.py b/tests/sientia/test_metrics.py new file mode 100644 index 0000000..015a8e3 --- /dev/null +++ b/tests/sientia/test_metrics.py @@ -0,0 +1,481 @@ +"""Unit tests for sientia metrics module.""" + +import numpy as np +import pandas as pd + +from model_manager.sientia.metrics import ( + mae, + mse, + r2, + rce_drift, + rce_test, + rce_train, + silverman_radius, +) + + +def test_mse_perfect_predictions(): + """Test MSE with perfect predictions returns 0.0.""" + real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + predictions = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + + result = mse(real_data, predictions) + + assert result == 0.0 + + +def test_mse_with_errors(): + """Test MSE calculation with prediction errors.""" + real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + predictions = pd.Series([1.5, 2.5, 3.5, 4.5, 5.5]) + + result = mse(real_data, predictions) + + # MSE = mean((0.5^2, 0.5^2, 0.5^2, 0.5^2, 0.5^2)) = 0.25 + assert result == 0.25 + + +def test_mse_with_integer_input(): + """Test MSE handles integer input and converts to float64.""" + real_data = pd.Series([1, 2, 3, 4, 5]) + predictions = pd.Series([2, 3, 4, 5, 6]) + + result = mse(real_data, predictions) + + # MSE = mean((1^2, 1^2, 1^2, 1^2, 1^2)) = 1.0 + assert result == 1.0 + + +def test_mse_with_large_errors(): + """Test MSE with large prediction errors.""" + real_data = pd.Series([10.0, 20.0, 30.0]) + predictions = pd.Series([5.0, 15.0, 25.0]) + + result = mse(real_data, predictions) + + # MSE = mean((25, 25, 25)) = 25.0 + assert result == 25.0 + + +def test_mse_rounds_to_two_decimals(): + """Test MSE rounds result to 2 decimal places.""" + real_data = pd.Series([1.111, 2.222, 3.333]) + predictions = pd.Series([1.222, 2.333, 3.444]) + + result = mse(real_data, predictions) + + # Result should be rounded to 2 decimals + assert isinstance(result, float) + assert len(str(result).split('.')[-1]) <= 2 + + +def test_mae_perfect_predictions(): + """Test MAE with perfect predictions returns 0.0.""" + real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + predictions = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + + result = mae(real_data, predictions) + + assert result == 0.0 + + +def test_mae_with_errors(): + """Test MAE calculation with prediction errors.""" + real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + predictions = pd.Series([1.5, 2.5, 3.5, 4.5, 5.5]) + + result = mae(real_data, predictions) + + # MAE = mean(|0.5|, |0.5|, |0.5|, |0.5|, |0.5|) = 0.5 + assert result == 0.5 + + +def test_mae_with_integer_input(): + """Test MAE handles integer input and converts to float64.""" + real_data = pd.Series([1, 2, 3, 4, 5]) + predictions = pd.Series([2, 3, 4, 5, 6]) + + result = mae(real_data, predictions) + + # MAE = mean(|1|, |1|, |1|, |1|, |1|) = 1.0 + assert result == 1.0 + + +def test_mae_with_negative_errors(): + """Test MAE with negative prediction errors (absolute value).""" + real_data = pd.Series([10.0, 20.0, 30.0]) + predictions = pd.Series([15.0, 25.0, 35.0]) + + result = mae(real_data, predictions) + + # MAE = mean(|5|, |5|, |5|) = 5.0 + assert result == 5.0 + + +def test_mae_rounds_to_two_decimals(): + """Test MAE rounds result to 2 decimal places.""" + real_data = pd.Series([1.111, 2.222, 3.333]) + predictions = pd.Series([1.222, 2.333, 3.444]) + + result = mae(real_data, predictions) + + # Result should be rounded to 2 decimals + assert isinstance(result, float) + assert len(str(result).split('.')[-1]) <= 2 + + +def test_r2_perfect_predictions(): + """Test R2 with perfect predictions returns 1.0.""" + real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + predictions = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + + result = r2(real_data, predictions) + + assert result == 1.0 + + +def test_r2_with_good_predictions(): + """Test R2 calculation with good predictions.""" + real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + predictions = pd.Series([1.1, 2.1, 2.9, 4.1, 4.9]) + + result = r2(real_data, predictions) + + # R2 should be close to 1.0 for good predictions + assert result > 0.9 + assert result <= 1.0 + + +def test_r2_with_integer_input(): + """Test R2 handles integer input and converts to float64.""" + real_data = pd.Series([1, 2, 3, 4, 5]) + predictions = pd.Series([1, 2, 3, 4, 5]) + + result = r2(real_data, predictions) + + assert result == 1.0 + + +def test_r2_with_poor_predictions(): + """Test R2 with poor predictions returns low score.""" + real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) + predictions = pd.Series([5.0, 4.0, 3.0, 2.0, 1.0]) + + result = r2(real_data, predictions) + + # R2 should be negative for predictions worse than mean + assert result < 0 + + +def test_r2_rounds_to_two_decimals(): + """Test R2 rounds result to 2 decimal places.""" + real_data = pd.Series([1.111, 2.222, 3.333, 4.444, 5.555]) + predictions = pd.Series([1.222, 2.333, 3.444, 4.555, 5.666]) + + result = r2(real_data, predictions) + + # Result should be rounded to 2 decimals + assert isinstance(result, float) + assert len(str(result).split('.')[-1]) <= 2 + + +def test_mse_with_mixed_positive_negative(): + """Test MSE with mixed positive and negative values.""" + real_data = pd.Series([-5.0, -2.0, 0.0, 3.0, 7.0]) + predictions = pd.Series([-4.0, -1.0, 1.0, 4.0, 8.0]) + + result = mse(real_data, predictions) + + # MSE = mean((1^2, 1^2, 1^2, 1^2, 1^2)) = 1.0 + assert result == 1.0 + + +def test_mae_with_mixed_positive_negative(): + """Test MAE with mixed positive and negative values.""" + real_data = pd.Series([-5.0, -2.0, 0.0, 3.0, 7.0]) + predictions = pd.Series([-4.0, -1.0, 1.0, 4.0, 8.0]) + + result = mae(real_data, predictions) + + # MAE = mean(|1|, |1|, |1|, |1|, |1|) = 1.0 + assert result == 1.0 + + +def test_r2_with_mixed_positive_negative(): + """Test R2 with mixed positive and negative values.""" + real_data = pd.Series([-5.0, -2.0, 0.0, 3.0, 7.0]) + predictions = pd.Series([-5.0, -2.0, 0.0, 3.0, 7.0]) + + result = r2(real_data, predictions) + + assert result == 1.0 + + +# ============================================================================ +# Tests for silverman_radius +# ============================================================================ + + +def test_silverman_radius_basic(): + """Test silverman_radius returns a positive float.""" + data = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]) + + result = silverman_radius(data) + + assert isinstance(result, float) + assert result > 0 + + +def test_silverman_radius_uniform_data(): + """Test silverman_radius with uniformly distributed data.""" + data = np.linspace(0, 100, 50) + + result = silverman_radius(data) + + assert result > 0 + assert np.isfinite(result) + + +def test_silverman_radius_normal_distribution(): + """Test silverman_radius with normally distributed data.""" + np.random.seed(42) + data = np.random.normal(loc=50, scale=10, size=100) + + result = silverman_radius(data) + + assert result > 0 + assert np.isfinite(result) + + +def test_silverman_radius_small_dataset(): + """Test silverman_radius with small dataset.""" + data = np.array([1.0, 2.0, 3.0]) + + result = silverman_radius(data) + + assert result > 0 + + +# ============================================================================ +# Tests for rce_train +# ============================================================================ + + +def test_rce_train_returns_dataframe(): + """Test rce_train returns a DataFrame.""" + training_set = pd.DataFrame({'a': [1.0, 2.0, 3.0, 4.0, 5.0], 'b': [2.0, 3.0, 4.0, 5.0, 6.0]}) + + result = rce_train(training_set, 0.1) + + assert isinstance(result, pd.DataFrame) + + +def test_rce_train_includes_first_vector(): + """Test rce_train always includes the first vector as a prototype.""" + training_set = pd.DataFrame({'a': [1.0, 2.0, 3.0], 'b': [1.0, 2.0, 3.0]}) + + result = rce_train(training_set, 0.1) + + assert len(result) >= 1 + assert result.iloc[0].tolist() == [1.0, 1.0] + + +def test_rce_train_with_identical_vectors(): + """Test rce_train with identical vectors returns single prototype.""" + training_set = pd.DataFrame({'a': [1.0, 1.0, 1.0], 'b': [2.0, 2.0, 2.0]}) + + result = rce_train(training_set, 0.1) + + # All vectors are identical, so only one prototype should be created + assert len(result) == 1 + + +def test_rce_train_with_distant_vectors(): + """Test rce_train with very distant vectors creates multiple prototypes.""" + training_set = pd.DataFrame({'a': [0.0, 100.0, 200.0], 'b': [0.0, 100.0, 200.0]}) + + result = rce_train(training_set, 0.1) + + # Distant vectors should create multiple prototypes + assert len(result) >= 1 + + +# ============================================================================ +# Tests for rce_test +# ============================================================================ + + +def test_rce_test_returns_series(): + """Test rce_test returns a pandas Series.""" + test_set = pd.DataFrame({'a': [1.5, 2.5], 'b': [1.5, 2.5]}) + prototypes = pd.DataFrame({'a': [1.0, 3.0], 'b': [1.0, 3.0]}) + + result = rce_test(test_set, prototypes) + + assert isinstance(result, pd.Series) + assert len(result) == len(test_set) + + +def test_rce_test_with_exact_match(): + """Test rce_test with test vector matching a prototype.""" + test_set = pd.DataFrame({'a': [1.0], 'b': [2.0]}) + prototypes = pd.DataFrame({'a': [1.0], 'b': [2.0]}) + + result = rce_test(test_set, prototypes) + + # Distance should be 0 for exact match + assert result.iloc[0] == 0.0 + + +def test_rce_test_multiple_prototypes(): + """Test rce_test finds closest prototype.""" + test_set = pd.DataFrame({'a': [1.1], 'b': [1.1]}) + prototypes = pd.DataFrame({'a': [1.0, 10.0], 'b': [1.0, 10.0]}) + + result = rce_test(test_set, prototypes) + + # Should find the closest prototype (1.0, 1.0) + assert len(result) == 1 + assert np.isfinite(result.iloc[0]) + + +def test_rce_test_signed_distances(): + """Test rce_test returns signed distances.""" + test_set = pd.DataFrame({'a': [0.0, 5.0], 'b': [0.0, 5.0]}) + prototypes = pd.DataFrame({'a': [2.0], 'b': [2.0]}) + + result = rce_test(test_set, prototypes) + + assert len(result) == 2 + # First test vector (0,0) is less than prototype (2,2) - should be negative + # Second test vector (5,5) is greater than prototype (2,2) - should be positive + assert result.iloc[0] < 0 + assert result.iloc[1] > 0 + + +# ============================================================================ +# Tests for rce_drift +# ============================================================================ + + +def test_rce_drift_returns_series(): + """Test rce_drift returns a pandas Series.""" + reference_data = pd.DataFrame( + { + 'feature1': [1.0, 2.0, 3.0, 4.0, 5.0], + 'feature2': [2.0, 3.0, 4.0, 5.0, 6.0], + 'target': [10.0, 20.0, 30.0, 40.0, 50.0], + 'prediction': [11.0, 21.0, 31.0, 41.0, 51.0], + } + ) + real_data = pd.DataFrame( + { + 'feature1': [1.5, 2.5], + 'feature2': [2.5, 3.5], + 'target': [15.0, 25.0], + 'prediction': [16.0, 26.0], + } + ) + + result = rce_drift(reference_data, real_data, 'target') + + assert isinstance(result, pd.Series) + assert len(result) == len(real_data) + + +def test_rce_drift_with_target_column(): + """Test rce_drift using target column (drops prediction).""" + reference_data = pd.DataFrame( + { + 'feature1': [1.0, 2.0, 3.0], + 'target': [10.0, 20.0, 30.0], + 'prediction': [11.0, 21.0, 31.0], + } + ) + real_data = pd.DataFrame( + { + 'feature1': [1.5], + 'target': [15.0], + 'prediction': [16.0], + } + ) + + result = rce_drift(reference_data, real_data, 'target') + + assert isinstance(result, pd.Series) + assert len(result) == 1 + + +def test_rce_drift_with_prediction_column(): + """Test rce_drift using prediction column (drops target).""" + reference_data = pd.DataFrame( + { + 'feature1': [1.0, 2.0, 3.0], + 'target': [10.0, 20.0, 30.0], + 'prediction': [11.0, 21.0, 31.0], + } + ) + real_data = pd.DataFrame( + { + 'feature1': [1.5], + 'target': [15.0], + 'prediction': [16.0], + } + ) + + result = rce_drift(reference_data, real_data, 'prediction') + + assert isinstance(result, pd.Series) + assert len(result) == 1 + + +def test_rce_drift_normalized_output(): + """Test rce_drift returns normalized distances.""" + reference_data = pd.DataFrame( + { + 'feature1': [1.0, 2.0, 3.0, 4.0, 5.0], + 'target': [10.0, 20.0, 30.0, 40.0, 50.0], + 'prediction': [10.0, 20.0, 30.0, 40.0, 50.0], + } + ) + real_data = pd.DataFrame( + { + 'feature1': [2.5, 3.5], + 'target': [25.0, 35.0], + 'prediction': [25.0, 35.0], + } + ) + + result = rce_drift(reference_data, real_data, 'target') + + # Result should be a Series with same length as real_data + assert isinstance(result, pd.Series) + assert len(result) == len(real_data) + + +def test_rce_drift_handles_common_columns(): + """Test rce_drift correctly handles common columns between datasets.""" + reference_data = pd.DataFrame( + { + 'feature1': [1.0, 2.0, 3.0], + 'feature2': [2.0, 3.0, 4.0], + 'extra_ref': [100.0, 200.0, 300.0], + 'target': [10.0, 20.0, 30.0], + 'prediction': [11.0, 21.0, 31.0], + } + ) + real_data = pd.DataFrame( + { + 'feature1': [1.5], + 'feature2': [2.5], + 'extra_real': [150.0], + 'target': [15.0], + 'prediction': [16.0], + } + ) + + result = rce_drift(reference_data, real_data, 'target') + + # Should work with only common columns + assert isinstance(result, pd.Series) + assert len(result) == 1 diff --git a/tests/sientia/test_reports.py b/tests/sientia/test_reports.py new file mode 100644 index 0000000..27bd264 --- /dev/null +++ b/tests/sientia/test_reports.py @@ -0,0 +1,437 @@ +import os +from unittest.mock import MagicMock + +import pytest +from bs4 import BeautifulSoup + +try: + from model_manager.sientia import reports +except ImportError as exc: + pytest.skip( + f'reports requires Evidently API matching production pin: {exc}', + allow_module_level=True, + ) + + +@pytest.fixture +def stub_color_options(monkeypatch): + def fake_color_options(**kwargs): + return dict(kwargs) + + monkeypatch.setattr(reports, 'ColorOptions', fake_color_options) + + +def test_load_html_from_file_success(tmp_path): + sample_file = tmp_path / 'sample.html' + sample_file.write_text('

Hello

', encoding='utf-8') + + content = reports.load_html_from_file(str(sample_file)) + + assert content == '

Hello

' + + +def test_load_html_from_file_missing_file(): + with pytest.raises(FileNotFoundError): + reports.load_html_from_file('non-existent.html') + + +def test_load_html_from_file_os_error(monkeypatch): + def fake_open(*_args, **_kwargs): + raise OSError('boom') + + monkeypatch.setattr('builtins.open', fake_open) + + with pytest.raises(OSError, match='boom'): + reports.load_html_from_file('path.html') + + +def test_inject_content_replaces_section(): + main_html = "
old
" + content = 'new' + + result = reports.inject_content(main_html, 'target', content) + + soup = BeautifulSoup(result, 'html.parser') + section = soup.find(id='target') + assert section is not None + assert section.find('span').text == 'new' + + +def test_inject_content_missing_section(): + main_html = "
keep
" + + result = reports.inject_content(main_html, 'missing', '

ignored

') + + # Content should be unchanged when section is missing + soup = BeautifulSoup(result, 'html.parser') + assert soup.find(id='other') is not None + assert soup.find(id='other').text == 'keep' + + +def test_reports_init_sets_defaults(stub_color_options): + report = reports.Reports(reference_data='ref', current_data='cur', target_name='target') + + assert report.metrics == [] + assert isinstance(report.options, list) and len(report.options) == 1 + assert report.sections == {} + assert report.base_path is None + + +def test_add_data_quality_section_without_run(monkeypatch, stub_color_options): + monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: 'summary') + monkeypatch.setattr( + reports, + 'generate_column_metrics', + lambda *args, **kwargs: ('columns', kwargs), + ) + monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: 'conflict') + monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: 'correlations') + + report = reports.Reports(reference_data='ref', current_data='cur', target_name='target') + report.add_data_quality_section(columns=['col'], run=False) + + assert report.metrics[-4:] == [ + 'summary', + ('columns', {'columns': ['col'], 'skip_id_column': True}), + 'conflict', + 'correlations', + ] + assert 'data_quality' not in report.sections + + +def test_add_data_quality_section_with_run(monkeypatch, tmp_path, stub_color_options): + summary = object() + column_metrics = object() + conflict = object() + correlations = object() + monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: summary) + + def fake_generate_column_metrics(*_args, **kwargs): + return column_metrics + + monkeypatch.setattr(reports, 'generate_column_metrics', fake_generate_column_metrics) + monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: conflict) + monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: correlations) + + report_instance = MagicMock() + report_instance.as_dict.return_value = {'result': 'data_quality'} + ReportMock = MagicMock(return_value=report_instance) + monkeypatch.setattr(reports, 'Report', ReportMock) + + report = reports.Reports( + reference_data='ref', current_data='cur', target_name='target', base_path=str(tmp_path) + ) + report.add_data_quality_section(columns=['c1'], run=True) + + assert report.metrics[-4:] == [summary, column_metrics, conflict, correlations] + assert report.sections['data_quality'] == {'result': 'data_quality'} + ReportMock.assert_called_once_with( + metrics=[summary, column_metrics, conflict, correlations], options=report.options + ) + run_kwargs = report_instance.run.call_args.kwargs + assert run_kwargs['reference_data'] == 'ref' + assert run_kwargs['current_data'] == 'cur' + assert run_kwargs['column_mapping'].target == 'target' + report_instance.save_html.assert_called_once_with( + os.path.join(str(tmp_path), 'data_quality.html') + ) + + +def test_add_data_quality_section_run_without_base_path(monkeypatch, stub_color_options): + summary = object() + column_metrics = object() + conflict = object() + correlations = object() + monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: summary) + monkeypatch.setattr( + reports, + 'generate_column_metrics', + lambda *args, **kwargs: column_metrics, + ) + monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: conflict) + monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: correlations) + + report_instance = MagicMock() + report_instance.as_dict.return_value = {'result': 'quality'} + ReportMock = MagicMock(return_value=report_instance) + monkeypatch.setattr(reports, 'Report', ReportMock) + + report = reports.Reports(reference_data='ref', current_data='cur', target_name='target') + report.add_data_quality_section(run=True) + + assert report.sections['data_quality'] == {'result': 'quality'} + report_instance.save_html.assert_not_called() + + +def test_add_data_quality_section_non_default_target_keeps_conflict_metric( + monkeypatch, stub_color_options +): + summary = object() + column_metrics = object() + conflict = object() + correlations = object() + + monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: summary) + monkeypatch.setattr( + reports, + 'generate_column_metrics', + lambda *args, **kwargs: column_metrics, + ) + monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: conflict) + monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: correlations) + + report = reports.Reports(reference_data='ref', current_data='cur', target_name='sales') + report.add_data_quality_section(columns=['c1'], run=False) + + assert report.metrics[-4:] == [ + summary, + column_metrics, + conflict, + correlations, + ] + + +def test_add_data_drift_section_paths(monkeypatch, tmp_path, stub_color_options): + drift_instances = [object(), object(), object()] + DataDriftPresetMock = MagicMock(side_effect=drift_instances) + monkeypatch.setattr(reports, 'DataDriftPreset', DataDriftPresetMock) + + report_instance = MagicMock() + report_instance.as_dict.return_value = {'result': 'data_drift'} + ReportMock = MagicMock(return_value=report_instance) + monkeypatch.setattr(reports, 'Report', ReportMock) + + report = reports.Reports( + reference_data='ref', current_data='cur', target_name='target', base_path=str(tmp_path) + ) + report.add_data_drift_section(columns=['c1'], run=False) + assert report.metrics[-1] == drift_instances[0] + assert 'data_drift' not in report.sections + + report.add_data_drift_section(columns=['c1'], run=True) + assert report.sections['data_drift'] == {'result': 'data_drift'} + ReportMock.assert_called_with(metrics=[drift_instances[2]], options=report.options) + run_kwargs = report_instance.run.call_args.kwargs + assert run_kwargs['reference_data'] == 'ref' + assert run_kwargs['current_data'] == 'cur' + assert run_kwargs['column_mapping'].target == 'target' + report_instance.save_html.assert_called_with(os.path.join(str(tmp_path), 'data_drift.html')) + + +def test_add_data_drift_section_run_without_base_path(monkeypatch, stub_color_options): + drift_instances = [object(), object(), object()] + DataDriftPresetMock = MagicMock(side_effect=drift_instances) + monkeypatch.setattr(reports, 'DataDriftPreset', DataDriftPresetMock) + + report_instance = MagicMock() + report_instance.as_dict.return_value = {'result': 'drift'} + ReportMock = MagicMock(return_value=report_instance) + monkeypatch.setattr(reports, 'Report', ReportMock) + + report = reports.Reports(reference_data='ref', current_data='cur', target_name='target') + report.add_data_drift_section(run=True) + + assert report.sections['data_drift'] == {'result': 'drift'} + report_instance.save_html.assert_not_called() + + +def test_add_regression_section(monkeypatch, tmp_path, stub_color_options): + regression_metrics = [object() for _ in range(7)] + monkeypatch.setattr(reports, 'RegressionPerformanceMetrics', lambda: regression_metrics[0]) + monkeypatch.setattr(reports, 'RegressionDummyMetric', lambda: regression_metrics[1]) + monkeypatch.setattr( + reports, 'RegressionPredictedVsActualScatter', lambda: regression_metrics[2] + ) + monkeypatch.setattr(reports, 'RegressionPredictedVsActualPlot', lambda: regression_metrics[3]) + monkeypatch.setattr(reports, 'RegressionErrorPlot', lambda: regression_metrics[4]) + monkeypatch.setattr(reports, 'RegressionAbsPercentageErrorPlot', lambda: regression_metrics[5]) + monkeypatch.setattr(reports, 'RegressionErrorDistribution', lambda: regression_metrics[6]) + + report_instance = MagicMock() + report_instance.as_dict.return_value = {'result': 'regression'} + ReportMock = MagicMock(return_value=report_instance) + monkeypatch.setattr(reports, 'Report', ReportMock) + + report = reports.Reports( + reference_data='ref', current_data='cur', target_name='target', base_path=str(tmp_path) + ) + + report.add_regression_section(run=False) + assert report.metrics[-7:] == regression_metrics + assert 'regression' not in report.sections + + report.add_regression_section(run=True) + assert report.sections['regression'] == {'result': 'regression'} + ReportMock.assert_called_with(metrics=regression_metrics, options=report.options) + report_instance.run.assert_called_with( + reference_data='ref', + current_data='cur', + column_mapping=report_instance.run.call_args.kwargs['column_mapping'], + ) + report_instance.save_html.assert_called_with(os.path.join(str(tmp_path), 'regression.html')) + + +def test_add_regression_section_run_without_base_path(monkeypatch, stub_color_options): + regression_metrics = [object() for _ in range(7)] + monkeypatch.setattr(reports, 'RegressionPerformanceMetrics', lambda: regression_metrics[0]) + monkeypatch.setattr(reports, 'RegressionDummyMetric', lambda: regression_metrics[1]) + monkeypatch.setattr( + reports, 'RegressionPredictedVsActualScatter', lambda: regression_metrics[2] + ) + monkeypatch.setattr(reports, 'RegressionPredictedVsActualPlot', lambda: regression_metrics[3]) + monkeypatch.setattr(reports, 'RegressionErrorPlot', lambda: regression_metrics[4]) + monkeypatch.setattr(reports, 'RegressionAbsPercentageErrorPlot', lambda: regression_metrics[5]) + monkeypatch.setattr(reports, 'RegressionErrorDistribution', lambda: regression_metrics[6]) + + report_instance = MagicMock() + report_instance.as_dict.return_value = {'result': 'reg'} + ReportMock = MagicMock(return_value=report_instance) + monkeypatch.setattr(reports, 'Report', ReportMock) + + report = reports.Reports(reference_data='ref', current_data='cur', target_name='target') + report.add_regression_section(run=True) + + assert report.sections['regression'] == {'result': 'reg'} + report_instance.save_html.assert_not_called() + + +def test_set_color_options_appends(monkeypatch): + calls = [] + + def color_options_mock(**kwargs): + calls.append(kwargs) + return kwargs + + monkeypatch.setattr(reports, 'ColorOptions', color_options_mock) + + report = reports.Reports(reference_data='ref', current_data='cur', target_name='target') + report.set_color_options(primary_color='#111', secondary_color='#222') + + options = report.options + assert options is not None + assert len(options) == 2 + assert calls[0]['primary_color'] == '#0F4C81' + assert calls[1]['primary_color'] == '#111' + assert options[1]['secondary_color'] == '#222' + + +def test_save_all_sections_html_requires_base_path(stub_color_options): + report = reports.Reports(reference_data='ref', current_data='cur', target_name='target') + + with pytest.raises(ValueError): + report.save_all_sections_html('output/report.html') + + +def test_save_all_sections_html_requires_template_path(stub_color_options, tmp_path): + report = reports.Reports( + reference_data='ref', + current_data='cur', + target_name='target', + base_path=str(tmp_path), + ) + + with pytest.raises(ValueError, match='template_path is required'): + report.save_all_sections_html('output/report.html') + + +def test_save_all_sections_html_writes_output(tmp_path, stub_color_options): + base_dir = tmp_path / 'templates' + base_dir.mkdir() + (base_dir / 'header.html').write_text( + "
", + encoding='utf-8', + ) + (base_dir / 'data_drift.html').write_text('

Drift

', encoding='utf-8') + (base_dir / 'data_quality.html').write_text('

Quality

', encoding='utf-8') + (base_dir / 'regression.html').write_text('

Regression

', encoding='utf-8') + + report = reports.Reports( + reference_data='ref', + current_data='cur', + target_name='target', + base_path=str(base_dir), + template_path=str(base_dir), + ) + output_path = tmp_path / 'reports' / 'combined.html' + + report.save_all_sections_html(str(output_path)) + + assert output_path.exists() + content = output_path.read_text(encoding='utf-8') + assert '

Drift

' in content + assert '

Quality

' in content + assert '

Regression

' in content + + +def test_save_all_sections_html_creates_directory(monkeypatch, tmp_path, stub_color_options): + base_dir = tmp_path / 'templates' + base_dir.mkdir() + (base_dir / 'header.html').write_text( + "
", + encoding='utf-8', + ) + (base_dir / 'data_drift.html').write_text('

Drift

', encoding='utf-8') + (base_dir / 'data_quality.html').write_text('

Quality

', encoding='utf-8') + (base_dir / 'regression.html').write_text('

Regression

', encoding='utf-8') + + make_dirs_called = [] + report = reports.Reports( + reference_data='ref', + current_data='cur', + target_name='target', + base_path=str(base_dir), + template_path=str(base_dir), + ) + output_path = tmp_path / 'nested' / 'report.html' + output_dir = str(output_path.parent) + + original_exists = os.path.exists + original_makedirs = os.makedirs + + def fake_exists(path): + if path == output_dir: + return False + return original_exists(path) + + def fake_makedirs(path, exist_ok=False): + make_dirs_called.append((path, exist_ok)) + return original_makedirs(path, exist_ok=exist_ok) + + monkeypatch.setattr(os.path, 'exists', fake_exists) + monkeypatch.setattr(os, 'makedirs', fake_makedirs) + + report.save_all_sections_html(str(output_path)) + + assert make_dirs_called == [(str(output_path.parent), True)] + + +def test_save_all_sections_html_no_directory_needed(monkeypatch, tmp_path, stub_color_options): + base_dir = tmp_path / 'templates' + base_dir.mkdir() + (base_dir / 'header.html').write_text( + "
", + encoding='utf-8', + ) + (base_dir / 'data_drift.html').write_text('

Drift

', encoding='utf-8') + (base_dir / 'data_quality.html').write_text('

Quality

', encoding='utf-8') + (base_dir / 'regression.html').write_text('

Regression

', encoding='utf-8') + + mk_calls = [] + + def fake_makedirs(path, exist_ok=False): + mk_calls.append((path, exist_ok)) + + monkeypatch.setattr(os, 'makedirs', fake_makedirs) + monkeypatch.chdir(tmp_path) + + report = reports.Reports( + reference_data='ref', + current_data='cur', + target_name='target', + base_path=str(base_dir), + template_path=str(base_dir), + ) + report.save_all_sections_html('report.html') + + assert mk_calls == [] + assert (tmp_path / 'report.html').exists() diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..2060902 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,379 @@ +"""Unit tests for model_manager.metrics module. + +This module tests the Prometheus metrics configuration used for +monitoring and observability in the Sientia DataOps Model Manager. +""" + + +def test_app_up_metric_exists(): + """Test that APP_UP metric is properly defined.""" + from model_manager.metrics import APP_UP + + assert APP_UP is not None + assert APP_UP._name == 'app_up' + assert ( + APP_UP._documentation == 'Indicates if the application is running (1) or shutting down (0)' + ) + + +def test_app_up_metric_has_pod_id_label(): + """Test that APP_UP metric has pod_id label.""" + from model_manager.metrics import APP_UP + + assert 'pod_id' in APP_UP._labelnames + + +def test_app_up_metric_is_gauge(): + """Test that APP_UP is a Gauge metric.""" + from prometheus_client import Gauge + + from model_manager.metrics import APP_UP + + assert isinstance(APP_UP, Gauge) + + +def test_app_up_metric_can_be_set_to_one(): + """Test that APP_UP metric can be set to 1 (running).""" + from model_manager.metrics import APP_UP + + # Set metric to 1 for a specific pod + APP_UP.labels(pod_id='test-pod-1').set(1) + + # Verify the metric value + metric_value = APP_UP.labels(pod_id='test-pod-1')._value._value + assert metric_value == 1 + + +def test_app_up_metric_can_be_set_to_zero(): + """Test that APP_UP metric can be set to 0 (shutting down).""" + from model_manager.metrics import APP_UP + + # Set metric to 0 for a specific pod + APP_UP.labels(pod_id='test-pod-2').set(0) + + # Verify the metric value + metric_value = APP_UP.labels(pod_id='test-pod-2')._value._value + assert metric_value == 0 + + +def test_app_up_metric_multiple_pods(): + """Test that APP_UP metric can track multiple pods independently.""" + from model_manager.metrics import APP_UP + + # Set different values for different pods + APP_UP.labels(pod_id='pod-1').set(1) + APP_UP.labels(pod_id='pod-2').set(0) + APP_UP.labels(pod_id='pod-3').set(1) + + # Verify each pod has correct value + assert APP_UP.labels(pod_id='pod-1')._value._value == 1 + assert APP_UP.labels(pod_id='pod-2')._value._value == 0 + assert APP_UP.labels(pod_id='pod-3')._value._value == 1 + + +def test_app_up_metric_default_value(): + """Test that APP_UP metric starts with no value set.""" + # Create a new label that hasn't been used yet + import uuid + + from model_manager.metrics import APP_UP + + unique_pod = f'test-pod-{uuid.uuid4()}' + + # The metric should exist but not have a value until set + metric = APP_UP.labels(pod_id=unique_pod) + assert metric is not None + + +def test_metrics_module_imports(): + """Test that metrics module can be imported successfully.""" + import model_manager.metrics + + assert hasattr(model_manager.metrics, 'APP_UP') + assert hasattr(model_manager.metrics, 'Gauge') + + +def test_metrics_module_docstring(): + """Test that metrics module has proper documentation.""" + import model_manager.metrics + + assert model_manager.metrics.__doc__ is not None + assert 'Prometheus' in model_manager.metrics.__doc__ + assert 'metrics' in model_manager.metrics.__doc__ + + +def test_app_up_metric_can_increment(): + """Test that APP_UP metric value can be incremented.""" + from model_manager.metrics import APP_UP + + pod_id = 'test-pod-increment' + APP_UP.labels(pod_id=pod_id).set(0) + + # Increment the metric + APP_UP.labels(pod_id=pod_id).inc() + + metric_value = APP_UP.labels(pod_id=pod_id)._value._value + assert metric_value == 1 + + +def test_app_up_metric_can_decrement(): + """Test that APP_UP metric value can be decremented.""" + from model_manager.metrics import APP_UP + + pod_id = 'test-pod-decrement' + APP_UP.labels(pod_id=pod_id).set(1) + + # Decrement the metric + APP_UP.labels(pod_id=pod_id).dec() + + metric_value = APP_UP.labels(pod_id=pod_id)._value._value + assert metric_value == 0 + + +def test_app_up_metric_set_to_timestamp(): + """Test that APP_UP metric can be set to current timestamp.""" + import time + + from model_manager.metrics import APP_UP + + pod_id = 'test-pod-timestamp' + current_time = time.time() + + # Set to timestamp + APP_UP.labels(pod_id=pod_id).set_to_current_time() + + metric_value = APP_UP.labels(pod_id=pod_id)._value._value + + # Should be close to current time + assert abs(metric_value - current_time) < 2 # Within 2 seconds + + +def test_app_up_metric_label_validation(): + """Test that APP_UP metric validates label names.""" + from model_manager.metrics import APP_UP + + # Should work with valid label + APP_UP.labels(pod_id='valid-pod-name').set(1) + + # Should work with empty string (though not recommended) + APP_UP.labels(pod_id='').set(1) + + # Should work with special characters + APP_UP.labels(pod_id='pod-123_test.example').set(1) + + +def test_module_exports(): + """Test that metrics module exports expected symbols.""" + import model_manager.metrics as metrics_module + + # Check that module has the expected exports + module_contents = dir(metrics_module) + + assert 'APP_UP' in module_contents + assert 'Gauge' in module_contents + + +def test_app_up_metric_thread_safety(): + """Test that APP_UP metric is thread-safe.""" + import threading + + from model_manager.metrics import APP_UP + + pod_id = 'test-pod-threading' + APP_UP.labels(pod_id=pod_id).set(0) + + def increment_metric(): + for _ in range(100): + APP_UP.labels(pod_id=pod_id).inc() + + # Create multiple threads that increment the metric + threads = [threading.Thread(target=increment_metric) for _ in range(5)] + + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + + # Should have incremented 500 times total + metric_value = APP_UP.labels(pod_id=pod_id)._value._value + assert metric_value == 500 + + +def test_prometheus_client_gauge_import(): + """Test that Gauge is properly imported from prometheus_client.""" + from prometheus_client import Gauge as PrometheusGauge + + from model_manager.metrics import Gauge + + assert Gauge is PrometheusGauge + + +# --------------------------------------------------------------------------- +# Training metrics — existence, type, and labels +# --------------------------------------------------------------------------- + +_TRAINING_LABEL_NAMES = ('pod_id', 'model_name', 'model_type') + + +def _assert_training_labels(metric): + for label in _TRAINING_LABEL_NAMES: + assert label in metric._labelnames + + +def test_sientia_training_data_preparation_lag_is_histogram(): + from prometheus_client import Histogram + + from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_LAG + + assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_LAG, Histogram) + assert SIENTIA_TRAINING_DATA_PREPARATION_LAG._name == 'sientia_training_data_preparation_lag' + _assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_LAG) + + +def test_sientia_training_data_preparation_error_count_total_is_counter(): + from prometheus_client import Counter + + from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL + + assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL, Counter) + assert 'sientia_training_data_preparation_error_count' in SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL._name + _assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL) + + +def test_sientia_training_model_fit_lag_is_histogram(): + from prometheus_client import Histogram + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_LAG + + assert isinstance(SIENTIA_TRAINING_MODEL_FIT_LAG, Histogram) + assert SIENTIA_TRAINING_MODEL_FIT_LAG._name == 'sientia_training_model_fit_lag' + _assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_LAG) + + +def test_sientia_training_model_fit_error_count_total_is_counter(): + from prometheus_client import Counter + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL + + assert isinstance(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL, Counter) + assert 'sientia_training_model_fit_error_count' in SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL._name + _assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL) + + +def test_sientia_training_model_quality_mse_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MSE + + assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MSE, Gauge) + assert SIENTIA_TRAINING_MODEL_QUALITY_MSE._name == 'sientia_training_model_quality_mse' + _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MSE) + + +def test_sientia_training_model_quality_mae_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MAE + + assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MAE, Gauge) + assert SIENTIA_TRAINING_MODEL_QUALITY_MAE._name == 'sientia_training_model_quality_mae' + _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MAE) + + +def test_sientia_training_model_quality_r2_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_R2 + + assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_R2, Gauge) + assert SIENTIA_TRAINING_MODEL_QUALITY_R2._name == 'sientia_training_model_quality_r2' + _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_R2) + + +def test_sientia_training_dataset_train_rows_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_DATASET_TRAIN_ROWS + + assert isinstance(SIENTIA_TRAINING_DATASET_TRAIN_ROWS, Gauge) + assert SIENTIA_TRAINING_DATASET_TRAIN_ROWS._name == 'sientia_training_dataset_train_rows' + _assert_training_labels(SIENTIA_TRAINING_DATASET_TRAIN_ROWS) + + +def test_sientia_training_dataset_val_rows_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_DATASET_VAL_ROWS + + assert isinstance(SIENTIA_TRAINING_DATASET_VAL_ROWS, Gauge) + assert SIENTIA_TRAINING_DATASET_VAL_ROWS._name == 'sientia_training_dataset_val_rows' + _assert_training_labels(SIENTIA_TRAINING_DATASET_VAL_ROWS) + + +def test_sientia_training_model_trained_total_is_counter(): + from prometheus_client import Counter + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_TRAINED_TOTAL + + assert isinstance(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL, Counter) + assert 'sientia_training_model_trained' in SIENTIA_TRAINING_MODEL_TRAINED_TOTAL._name + _assert_training_labels(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL) + + +def test_sientia_training_feature_count_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_FEATURE_COUNT + + assert isinstance(SIENTIA_TRAINING_FEATURE_COUNT, Gauge) + assert SIENTIA_TRAINING_FEATURE_COUNT._name == 'sientia_training_feature_count' + _assert_training_labels(SIENTIA_TRAINING_FEATURE_COUNT) + + +def test_sientia_training_info_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_INFO + + assert isinstance(SIENTIA_TRAINING_INFO, Gauge) + assert SIENTIA_TRAINING_INFO._name == 'sientia_training_info' + + expected_labels = { + 'pod_id', 'model_name', 'model_type', + 'dataset_train_rows', 'dataset_val_rows', 'feature_count', + 'mse', 'mae', 'r2', + } + assert expected_labels == set(SIENTIA_TRAINING_INFO._labelnames) + + +def test_sientia_training_info_set_value(): + import time + + from model_manager.metrics import SIENTIA_TRAINING_INFO + + ts = time.time() * 1000 + SIENTIA_TRAINING_INFO.labels( + pod_id='test-pod', + model_name='my_model', + model_type='linear', + dataset_train_rows='1000', + dataset_val_rows='200', + feature_count='5', + mse='0.01', + mae='0.08', + r2='0.95', + ).set(ts) + + value = SIENTIA_TRAINING_INFO.labels( + pod_id='test-pod', + model_name='my_model', + model_type='linear', + dataset_train_rows='1000', + dataset_val_rows='200', + feature_count='5', + mse='0.01', + mae='0.08', + r2='0.95', + )._value._value + assert abs(value - ts) < 2000 diff --git a/tests/test_runtime_paths.py b/tests/test_runtime_paths.py new file mode 100644 index 0000000..3e13b99 --- /dev/null +++ b/tests/test_runtime_paths.py @@ -0,0 +1,18 @@ +"""Tests for runtime filesystem layout constants.""" + +from unittest.mock import patch + + +def test_ensure_runtime_directories_creates_expected_paths(): + from model_manager.runtime_paths import ( + LOGS_DIR, + REPORTS_ROOT, + REPORTS_TEMP_DIR, + ensure_runtime_directories, + ) + + with patch('model_manager.runtime_paths.makedirs') as makedirs_mock: + ensure_runtime_directories() + + created = {call.args[0] for call in makedirs_mock.call_args_list} + assert created == {REPORTS_ROOT, REPORTS_TEMP_DIR, LOGS_DIR} diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/utils/models/__init__.py b/tests/utils/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/utils/models/test_experiment_status.py b/tests/utils/models/test_experiment_status.py new file mode 100644 index 0000000..da4d7cc --- /dev/null +++ b/tests/utils/models/test_experiment_status.py @@ -0,0 +1,75 @@ +"""Unit tests for ExperimentStatus enum.""" + +from model_manager.utils.models.experiment_status import ExperimentStatus + + +def test_experiment_status_values(): + """Test that all expected status values exist.""" + assert ExperimentStatus.ORCHESTRATOR_VALIDATION_ERROR == 'ORCHESTRATOR_VALIDATION_ERROR' + assert ExperimentStatus.ORCHESTRATOR_WAITING_PROC == 'ORCHESTRATOR_WAITING_PROC' + assert ExperimentStatus.TRAINING_SUCCESS == 'TRAINING_SUCCESS' + assert ExperimentStatus.TRAINING_ERROR == 'TRAINING_ERROR' + + +def test_experiment_status_count(): + """Test that enum has exactly 4 status values.""" + assert len(ExperimentStatus) == 4 + + +def test_experiment_status_is_string(): + """Test that enum values are strings.""" + for status in ExperimentStatus: + assert isinstance(status.value, str) + assert isinstance(status, str) + + +def test_experiment_status_membership(): + """Test membership checks for status values.""" + assert 'ORCHESTRATOR_VALIDATION_ERROR' in [s.value for s in ExperimentStatus] + assert 'ORCHESTRATOR_WAITING_PROC' in [s.value for s in ExperimentStatus] + assert 'TRAINING_SUCCESS' in [s.value for s in ExperimentStatus] + assert 'TRAINING_ERROR' in [s.value for s in ExperimentStatus] + + +def test_experiment_status_iteration(): + """Test that enum can be iterated.""" + statuses = list(ExperimentStatus) + assert len(statuses) == 4 + assert ExperimentStatus.ORCHESTRATOR_VALIDATION_ERROR in statuses + assert ExperimentStatus.ORCHESTRATOR_WAITING_PROC in statuses + assert ExperimentStatus.TRAINING_SUCCESS in statuses + assert ExperimentStatus.TRAINING_ERROR in statuses + + +def test_experiment_status_comparison(): + """Test that enum values can be compared with strings.""" + assert ExperimentStatus.ORCHESTRATOR_VALIDATION_ERROR == 'ORCHESTRATOR_VALIDATION_ERROR' + assert ExperimentStatus.ORCHESTRATOR_WAITING_PROC == 'ORCHESTRATOR_WAITING_PROC' + assert ExperimentStatus.TRAINING_SUCCESS == 'TRAINING_SUCCESS' + assert str(ExperimentStatus.TRAINING_ERROR) != 'TRAINING_SUCCESS' + + +def test_experiment_status_access_by_name(): + """Test accessing enum members by name.""" + assert ( + ExperimentStatus['ORCHESTRATOR_VALIDATION_ERROR'] + == ExperimentStatus.ORCHESTRATOR_VALIDATION_ERROR + ) + assert ( + ExperimentStatus['ORCHESTRATOR_WAITING_PROC'] == ExperimentStatus.ORCHESTRATOR_WAITING_PROC + ) + assert ExperimentStatus['TRAINING_SUCCESS'] == ExperimentStatus.TRAINING_SUCCESS + assert ExperimentStatus['TRAINING_ERROR'] == ExperimentStatus.TRAINING_ERROR + + +def test_experiment_status_access_by_value(): + """Test accessing enum members by value.""" + assert ( + ExperimentStatus('ORCHESTRATOR_VALIDATION_ERROR') + == ExperimentStatus.ORCHESTRATOR_VALIDATION_ERROR + ) + assert ( + ExperimentStatus('ORCHESTRATOR_WAITING_PROC') == ExperimentStatus.ORCHESTRATOR_WAITING_PROC + ) + assert ExperimentStatus('TRAINING_SUCCESS') == ExperimentStatus.TRAINING_SUCCESS + assert ExperimentStatus('TRAINING_ERROR') == ExperimentStatus.TRAINING_ERROR diff --git a/tests/utils/models/test_init.py b/tests/utils/models/test_init.py new file mode 100644 index 0000000..b211248 --- /dev/null +++ b/tests/utils/models/test_init.py @@ -0,0 +1,51 @@ +"""Unit tests for models __init__.py module.""" + +from model_manager.utils.models import ( + ExperimentStatus, + TrainModelParams, + TrainModelResult, +) + + +def test_experiment_status_import(): + """Test that ExperimentStatus can be imported from models package.""" + assert ExperimentStatus is not None + assert hasattr(ExperimentStatus, 'ORCHESTRATOR_WAITING_PROC') + assert hasattr(ExperimentStatus, 'TRAINING_SUCCESS') + + +def test_train_model_params_import(): + """Test that TrainModelParams can be imported from models package.""" + assert TrainModelParams is not None + assert callable(TrainModelParams) + + +def test_train_model_result_import(): + """Test that TrainModelResult can be imported from models package.""" + assert TrainModelResult is not None + # Dataclasses have __dataclass_fields__ + assert hasattr(TrainModelResult, '__dataclass_fields__') + + +def test_all_exports(): + """Test that __all__ contains all expected exports.""" + from model_manager.utils.models import __all__ + + assert 'ExperimentStatus' in __all__ + assert 'TrainModelParams' in __all__ + assert 'TrainModelResult' in __all__ + assert len(__all__) == 3 + + +def test_no_extra_exports(): + """Test that only expected items are exported.""" + import model_manager.utils.models as models_module + + # Get all public attributes (not starting with _) + public_attrs = [attr for attr in dir(models_module) if not attr.startswith('_')] + + # Should only have the 3 main classes + expected_public = {'ExperimentStatus', 'TrainModelParams', 'TrainModelResult'} + + # Check that our expected classes are present + assert expected_public.issubset(set(public_attrs)) diff --git a/tests/utils/models/test_train_model_params.py b/tests/utils/models/test_train_model_params.py new file mode 100644 index 0000000..03055fb --- /dev/null +++ b/tests/utils/models/test_train_model_params.py @@ -0,0 +1,321 @@ +"""Unit tests for TrainModelParams (current schema).""" + +import copy +from unittest.mock import patch + +import pytest + +from model_manager.utils.models.train_model_params import ( + DEFAULT_TRAIN_DATE_FORMAT, + TrainModelParams, + validate_frontend_date_format, +) + + +@pytest.fixture +def minimal_model_metadata() -> dict: + """Minimal truthy metadata so validate_business_rules passes schema lookup.""" + return {'schemas': {'components': {'schemas': {}}}} + + +@pytest.fixture +def valid_train_params_dict(minimal_model_metadata) -> dict: + """Valid dictionary for TrainModelParams.from_dict.""" + return { + 'variable_columns': ['var1', 'var2'], + 'target_variable': 'target', + 'bucket_name': 'test-bucket', + 'file_name': 'test-file.csv', + 'line_separator': ',', + 'decimal_separator': '.', + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', + 'train_size': 80, + 'shuffle': True, + 'random_state': 42, + 'experiment_run_id': 1, + 'model_name': 'Linear Regression', + 'val_file_name': None, + 'data_model_kwargs': {}, + 'model_kwargs': {}, + 'opt_params': {}, + 'model_type': 'linear_regression', + 'model_id': None, + 'model_metadata': minimal_model_metadata, + } + + +def test_from_dict_success(valid_train_params_dict): + """from_dict builds params and experiment_name from model_name.""" + params = TrainModelParams.from_dict(valid_train_params_dict) + + assert params.variable_columns == ['var1', 'var2'] + assert params.target_variable == 'target' + assert params.bucket_name == 'test-bucket' + assert params.experiment_run_id == 1 + assert params.experiment_name == 'Linear Regression' + assert params.model_metadata is valid_train_params_dict['model_metadata'] + + +def test_from_dict_date_format_omitted_uses_default(valid_train_params_dict): + """Missing date_format defaults to DEFAULT_TRAIN_DATE_FORMAT.""" + d = copy.deepcopy(valid_train_params_dict) + del d['date_format'] + params = TrainModelParams.from_dict(d) + assert params.date_format == DEFAULT_TRAIN_DATE_FORMAT + + +def test_from_dict_date_format_blank_uses_default(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['date_format'] = ' ' + params = TrainModelParams.from_dict(d) + assert params.date_format == DEFAULT_TRAIN_DATE_FORMAT + + +def test_from_dict_superfluous_date_column_camel_key_is_ignored(valid_train_params_dict): + """Only snake_case keys are read; dateColumn does not populate date_column.""" + d = copy.deepcopy(valid_train_params_dict) + d['dateColumn'] = 'wrong_name' + params = TrainModelParams.from_dict(d) + assert params.date_column == 'timestamp' + + +def test_from_dict_missing_date_column_raises(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + del d['date_column'] + with pytest.raises(ValueError, match='date_column is required'): + TrainModelParams.from_dict(d) + + +def test_from_dict_date_format_non_string_raises(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['date_format'] = 12345 + with pytest.raises(TypeError, match='date_format must be a string'): + TrainModelParams.from_dict(d) + + +def test_from_dict_coerces_experiment_run_id_string(valid_train_params_dict): + """Numeric string experiment_run_id is coerced to int.""" + d = copy.deepcopy(valid_train_params_dict) + d['experiment_run_id'] = '42' + params = TrainModelParams.from_dict(d) + assert params.experiment_run_id == 42 + + +def test_from_dict_model_metadata_none(valid_train_params_dict): + """model_metadata may be None before load_model_metadata activity.""" + d = copy.deepcopy(valid_train_params_dict) + d['model_metadata'] = None + params = TrainModelParams.from_dict(d) + assert params.model_metadata is None + + +def test_coerce_experiment_run_id_rejects_bool(): + """Boolean must not be accepted as experiment_run_id.""" + with pytest.raises(TypeError, match='experiment_run_id must be an integer'): + TrainModelParams._coerce_experiment_run_id(True) + + +def test_parse_optional_model_metadata_rejects_list(): + """model_metadata must be dict or None.""" + with pytest.raises(TypeError, match='model_metadata must be a dict or None'): + TrainModelParams._parse_optional_model_metadata([]) + + +def test_check_none_raises_value_error(): + with pytest.raises(ValueError, match='test_field is required'): + TrainModelParams._check_none(None, str, 'test_field') + + +def test_check_none_raises_type_error(): + with pytest.raises(TypeError, match='test_field must be of type str'): + TrainModelParams._check_none(123, str, 'test_field') + + +def test_validate_business_rules_success(valid_train_params_dict): + params = TrainModelParams.from_dict(valid_train_params_dict) + params.validate_business_rules() + + +def test_validate_business_rules_missing_model_metadata(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['model_metadata'] = None + params = TrainModelParams.from_dict(d) + with pytest.raises(ValueError, match='model_metadata is required'): + params.validate_business_rules() + + +def test_validate_business_rules_train_size_out_of_range(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['train_size'] = 5 + params = TrainModelParams.from_dict(d) + with pytest.raises(ValueError, match='train_size must be between'): + params.validate_business_rules() + + +def test_validate_business_rules_empty_variable_columns(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['variable_columns'] = [] + params = TrainModelParams.from_dict(d) + with pytest.raises(ValueError, match='variable_columns cannot be empty'): + params.validate_business_rules() + + +def test_validate_business_rules_empty_target(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['target_variable'] = ' ' + params = TrainModelParams.from_dict(d) + with pytest.raises(ValueError, match='target_variable cannot be empty'): + params.validate_business_rules() + + +def test_validate_business_rules_whitespace_date_column(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['date_column'] = ' ' + params = TrainModelParams.from_dict(d) + with pytest.raises(ValueError, match='date_column cannot be empty'): + params.validate_business_rules() + + +def test_from_dict_missing_required_key(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + del d['bucket_name'] + with pytest.raises(ValueError, match='bucket_name is required'): + TrainModelParams.from_dict(d) + + +def test_to_dict_roundtrip_keys(valid_train_params_dict): + params = TrainModelParams.from_dict(valid_train_params_dict) + d = params.to_dict() + assert 'variable_columns' in d + assert d['experiment_run_id'] == 1 + + +def test_coerce_experiment_run_id_float(): + assert TrainModelParams._coerce_experiment_run_id(2.0) == 2 + + +def test_coerce_experiment_run_id_none_raises(): + with pytest.raises(ValueError, match='experiment_run_id is required'): + TrainModelParams._coerce_experiment_run_id(None) + + +def test_coerce_experiment_run_id_invalid_type(): + with pytest.raises(TypeError, match='integer or numeric string'): + TrainModelParams._coerce_experiment_run_id([1]) + + +def test_validate_model_param_schema_validation_error(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['model_metadata'] = { + 'schemas': { + 'components': { + 'schemas': { + 'data_model': { + 'type': 'object', + 'properties': {'x': {'type': 'integer'}}, + 'required': ['x'], + }, + } + } + } + } + p = TrainModelParams.from_dict(d) + p.data_model_kwargs = {} + with pytest.raises(ValueError, match='Model parameters validation failed'): + p.validate_business_rules() + + +def test_validate_model_param_unexpected_validator_error(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['model_metadata'] = { + 'schemas': { + 'components': { + 'schemas': { + 'data_model': {'type': 'object'}, + } + } + } + } + p = TrainModelParams.from_dict(d) + with patch('model_manager.utils.models.train_model_params.Draft202012Validator') as m: + m.return_value.validate.side_effect = RuntimeError('boom') + with pytest.raises(RuntimeError, match='boom'): + p.validate_business_rules() + + +def test_validate_business_rules_date_format_invalid(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['date_format'] = 'not-an-allowed-format' + p = TrainModelParams.from_dict(d) + with pytest.raises(ValueError, match='Invalid date_format'): + p.validate_business_rules() + + +def test_validate_required_strings_whitespace_bucket_file_model(valid_train_params_dict): + for field, msg in [ + ('bucket_name', 'bucket_name cannot be empty'), + ('file_name', 'file_name cannot be empty'), + ('model_name', 'model_name cannot be empty'), + ]: + d = copy.deepcopy(valid_train_params_dict) + d[field] = ' ' + p = TrainModelParams.from_dict(d) + with pytest.raises(ValueError, match=msg): + p.validate_business_rules() + + +def test_validate_model_param_only_data_model_schema(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['model_metadata'] = { + 'schemas': {'components': {'schemas': {'data_model': {'type': 'object'}}}} + } + p = TrainModelParams.from_dict(d) + p.data_model_kwargs = {} + p.validate_business_rules() + + +def test_validate_model_param_only_model_schema(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['model_metadata'] = {'schemas': {'components': {'schemas': {'model': {'type': 'object'}}}}} + p = TrainModelParams.from_dict(d) + p.model_kwargs = {} + p.validate_business_rules() + + +def test_validate_model_param_only_opt_params_schema(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['model_metadata'] = { + 'schemas': {'components': {'schemas': {'opt_params': {'type': 'object'}}}} + } + p = TrainModelParams.from_dict(d) + p.opt_params = {} + p.validate_business_rules() + + +def test_validate_model_param_all_schema_branches(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['model_metadata'] = { + 'schemas': { + 'components': { + 'schemas': { + 'data_model': {'type': 'object'}, + 'model': {'type': 'object'}, + 'opt_params': {'type': 'object'}, + } + } + } + } + p = TrainModelParams.from_dict(d) + p.data_model_kwargs = {} + p.model_kwargs = {} + p.opt_params = {} + p.validate_business_rules() + + +def test_validate_frontend_date_format_whitespace_returns(): + validate_frontend_date_format(' ') + + +def test_validate_frontend_date_format_valid_returns(): + validate_frontend_date_format('dd/MM/yyyy HH:mm:ss') diff --git a/tests/utils/models/test_train_model_result.py b/tests/utils/models/test_train_model_result.py new file mode 100644 index 0000000..a67c06e --- /dev/null +++ b/tests/utils/models/test_train_model_result.py @@ -0,0 +1,71 @@ +"""Unit tests for TrainModelResult dataclass.""" + +import pandas as pd +import pytest + +from model_manager.utils.models.train_model_params import TrainModelParams +from model_manager.utils.models.train_model_result import TrainModelResult + + +@pytest.fixture +def sample_params() -> TrainModelParams: + """Minimal TrainModelParams for TrainModelResult tests.""" + return TrainModelParams.from_dict( + { + 'variable_columns': ['a'], + 'target_variable': 't', + 'bucket_name': 'b', + 'file_name': 'f.csv', + 'line_separator': '\n', + 'decimal_separator': '.', + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', + 'train_size': 80, + 'shuffle': True, + 'random_state': 42, + 'experiment_run_id': 1, + 'model_name': 'Linear Regression', + 'val_file_name': None, + 'data_model_kwargs': {}, + 'model_kwargs': {}, + 'opt_params': {}, + 'model_type': 'linear_regression', + 'model_id': None, + 'model_metadata': {'schemas': {'components': {'schemas': {}}}}, + } + ) + + +@pytest.fixture +def sample_frames(): + train = pd.DataFrame({'a': [1, 2], 't': [1.0, 2.0]}) + val = pd.DataFrame({'a': [3], 't': [3.0]}) + return train, val + + +def test_train_model_result_creation(sample_params, sample_frames): + train, val = sample_frames + result = TrainModelResult(params=sample_params, train_data=train, val_data=val) + assert result.params is sample_params + assert result.train_data.equals(train) + assert result.val_data.equals(val) + assert result.run_name is None + + +def test_train_model_result_optional_paths(sample_params, sample_frames): + train, val = sample_frames + result = TrainModelResult( + params=sample_params, + train_data=train, + val_data=val, + run_name='run-1', + run_id='rid', + run_dir='/tmp/x', + mse_val=0.1, + mae_val=0.2, + r2_val=0.99, + ) + assert result.run_name == 'run-1' + assert result.run_id == 'rid' + assert result.run_dir == '/tmp/x' + assert result.mse_val == 0.1 diff --git a/tests/utils/repository/test_data_manager_repository.py b/tests/utils/repository/test_data_manager_repository.py new file mode 100644 index 0000000..a7c478c --- /dev/null +++ b/tests/utils/repository/test_data_manager_repository.py @@ -0,0 +1,630 @@ +"""Unit tests for DataManagerRepository and module helpers.""" + +from __future__ import annotations + +import json +from typing import Any +from unittest.mock import MagicMock, Mock, patch + +import numpy as np +import pandas as pd +import pytest + +from model_manager.runtime_paths import REPORTS_ROOT +from model_manager.utils.models.train_model_params import TrainModelParams +from model_manager.utils.models.train_model_result import TrainModelResult +from model_manager.utils.repository import data_manager_repository as dmr + + +def test_train_test_split_dataframe_shuffle(): + df = pd.DataFrame({'a': range(10)}) + tr, te = dmr.train_test_split(df, train_size=0.7, random_state=0, shuffle=True) + assert len(tr) == 7 and len(te) == 3 + + +def test_train_test_split_dataframe_no_shuffle(): + df = pd.DataFrame({'a': range(10)}) + tr, te = dmr.train_test_split(df, train_size=0.5, shuffle=False) + assert list(tr['a']) == [0, 1, 2, 3, 4] + + +def test_train_test_split_dataframe_returns_dataframes(): + df = pd.DataFrame(np.arange(20).reshape(10, 2), columns=['a', 'b']) + tr, te = dmr.train_test_split(df, train_size=0.5, shuffle=False, random_state=None) + assert isinstance(tr, pd.DataFrame) + assert isinstance(te, pd.DataFrame) + assert tr.shape[0] == 5 and te.shape[0] == 5 + + +def _params(**kwargs) -> TrainModelParams: + base: dict[str, Any] = { + 'variable_columns': ['v1'], + 'target_variable': 't', + 'bucket_name': 'b', + 'file_name': 'f.csv', + 'line_separator': ',', + 'decimal_separator': '.', + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', + 'train_size': 80, + 'shuffle': True, + 'random_state': 42, + 'experiment_run_id': 1, + 'model_name': 'Linear Regression', + 'val_file_name': None, + 'data_model_kwargs': {}, + 'model_kwargs': {}, + 'opt_params': {}, + 'model_type': 'linear_regression', + 'model_id': None, + 'model_metadata': {'schemas': {'components': {'schemas': {}}}}, + } + base.update(kwargs) + return TrainModelParams.from_dict(base) + + +def test_ensure_date_column_parsed_missing_column_raises(): + df = pd.DataFrame({'a': [1]}) + p = _params(date_column='missing') + with pytest.raises(ValueError, match='not found in dataset columns'): + dmr._ensure_date_column_parsed(df, p) + + +def test_ensure_date_column_parsed_success(): + df = pd.DataFrame({'a': range(3), 'ts': ['2024-01-01 10:00:00'] * 3}) + p = _params(date_column='ts') + out = dmr._ensure_date_column_parsed(df, p) + assert pd.api.types.is_datetime64_any_dtype(out['ts']) + + +def test_ensure_date_column_parsed_naive_with_frontend_format(): + """CSV timestamps without timezone use params.date_format strftime mapping.""" + df = pd.DataFrame({'ts': ['2025-06-02 00:00:00', '2025-06-02 01:00:00']}) + p = _params(date_column='ts', date_format='yyyy-MM-dd HH:mm:ss') + out = dmr._ensure_date_column_parsed(df, p) + assert pd.api.types.is_datetime64_any_dtype(out['ts']) + + +def test_ensure_date_column_parsed_invalid_raises(): + df = pd.DataFrame({'a': range(3), 'ts': ['not-a-date'] * 3}) + p = _params(date_column='ts') + with pytest.raises(ValueError, match='Failed to parse date column'): + dmr._ensure_date_column_parsed(df, p) + + +def test_prepare_training_data_csv_load_failure(): + repo = dmr.DataManagerRepository(MagicMock()) + p = _params() + with patch( + 'model_manager.utils.repository.data_manager_repository.pd.read_csv', + side_effect=pd.errors.ParserError('bad'), + ): + with pytest.raises(ValueError, match='Failed to load training CSV'): + repo.prepare_training_data(b'x', None, p, {}) + + +def test_prepare_training_data_empty_after_load(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + # Headers only; timestamp column present but no data rows + csv_bytes = b'timestamp,v1,t\n' + with pytest.raises(ValueError, match='Training data view is empty after transformation'): + repo.prepare_training_data(csv_bytes, None, p, {}) + + +def test_prepare_training_data_empty_after_transformation(monkeypatch): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + monkeypatch.setattr( + repo, + '_configure_datetime_index', + lambda *_args, **_kwargs: pd.DataFrame(columns=['v1', 't']), + ) + monkeypatch.setattr(repo, '_set_timezone_on_index', lambda data, *_args, **_kwargs: data) + with pytest.raises(ValueError, match='Training data view is empty after transformation'): + repo.prepare_training_data(b'timestamp,v1,t\n', None, p, {}) + + +def _minimal_dict_for_prepare(): + return { + 'variable_columns': ['v1'], + 'target_variable': 't', + 'bucket_name': 'b', + 'file_name': 'f.csv', + 'line_separator': ',', + 'decimal_separator': '.', + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', + 'train_size': 80, + 'shuffle': True, + 'random_state': 42, + 'experiment_run_id': 1, + 'model_name': 'Linear Regression', + 'val_file_name': None, + 'data_model_kwargs': {}, + 'model_kwargs': {}, + 'opt_params': {}, + 'model_type': 'linear_regression', + 'model_id': None, + 'model_metadata': {'schemas': {'components': {'schemas': {}}}}, + } + + +def _csv_bytes_with_ts(n_rows: int = 20) -> bytes: + """CSV with leading timestamp column (naive, matches default date_format).""" + lines = ['timestamp,v1,t'] + for i in range(n_rows): + lines.append(f'2024-01-{i + 1:02d} 00:00:00,{i},{i + 1}') + return '\n'.join(lines).encode() + + +def test_prepare_training_data_validation_csv_invalid(): + from io import BytesIO + + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + train_csv = _csv_bytes_with_ts(5) + train_df = pd.read_csv(BytesIO(train_csv), sep=',', decimal='.') + with patch.object( + dmr.pd, + 'read_csv', + side_effect=[train_df, pd.errors.ParserError('bad val')], + ): + with pytest.raises(ValueError, match='Failed to load validation CSV'): + repo.prepare_training_data(train_csv, b'broken', p, {}) + + +def test_prepare_training_data_validation_empty_val(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + train_csv = _csv_bytes_with_ts(5) + val_csv = b'timestamp,v1,t\n' + with pytest.raises(ValueError, match='Validation data view is empty'): + repo.prepare_training_data(train_csv, val_csv, p, {}) + + +def test_prepare_training_data_drops_row_with_blank_timestamp(): + """Rows with empty date_column values are removed before datetime parsing.""" + repo = dmr.DataManagerRepository(MagicMock()) + d = _minimal_dict_for_prepare() + d['date_column'] = 'timestamp' + d['date_format'] = 'yyyy-MM-dd HH:mm:ss' + p = TrainModelParams.from_dict(d) + lines = ['timestamp,v1,t'] + for i in range(10): + if i == 3: + lines.append(',1.0,2.0') + else: + lines.append(f'2025-06-01 {i:02d}:00:00,1.0,2.0') + csv = '\n'.join(lines).encode() + res = repo.prepare_training_data(csv, None, p, {}) + assert len(res.train_data) + len(res.val_data) == 9 + + +def test_prepare_training_data_split_path(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + train_csv = _csv_bytes_with_ts(20) + res = repo.prepare_training_data(train_csv, None, p, {}) + assert res.train_data is not None and res.val_data is not None + + +def test_prepare_training_data_explicit_validation_success(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + train_csv = _csv_bytes_with_ts(10) + val_csv = _csv_bytes_with_ts(5) + res = repo.prepare_training_data(train_csv, val_csv, p, {}) + assert len(res.val_data) == 5 + + +def test_coerce_non_timestamp_columns_to_numeric_success(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + idx = pd.date_range('2024-01-01', periods=2, freq='h', tz='UTC') + df = pd.DataFrame( + {'v1': ['1.25', '2.75'], 't': ['10', '11']}, + index=idx, + ) + out = repo._coerce_non_timestamp_columns_to_numeric(df, p, {}) + assert pd.api.types.is_numeric_dtype(out['v1']) + assert pd.api.types.is_numeric_dtype(out['t']) + assert float(out['v1'].iloc[0]) == 1.25 + assert float(out['t'].iloc[1]) == 11.0 + + +def test_coerce_non_timestamp_columns_to_numeric_invalid_values_to_nan(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + idx = pd.date_range('2024-01-01', periods=2, freq='h', tz='UTC') + df = pd.DataFrame( + {'v1': ['1.25', 'oops'], 't': ['10', 'bad']}, + index=idx, + ) + out = repo._coerce_non_timestamp_columns_to_numeric(df, p, {}) + assert np.isnan(out['v1'].iloc[1]) + assert np.isnan(out['t'].iloc[1]) + + +def test_prepare_training_data_coerces_non_timestamp_columns_to_numeric(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + lines = ['timestamp,v1,t'] + for i in range(10): + v1 = 'bad' if i == 4 else f'{i + 0.5}' + t = 'bad' if i == 7 else f'{i + 1.0}' + lines.append(f'2025-06-01 {i:02d}:00:00,{v1},{t}') + csv = '\n'.join(lines).encode() + res = repo.prepare_training_data(csv, None, p, {}) + joined = pd.concat([res.train_data, res.val_data], axis=0).sort_index() + assert pd.api.types.is_numeric_dtype(joined['v1']) + assert pd.api.types.is_numeric_dtype(joined['t']) + assert joined['v1'].isna().sum() == 1 + assert joined['t'].isna().sum() == 1 + + +def test_as_series_series(): + repo = dmr.DataManagerRepository(MagicMock()) + s = pd.Series([1.0, 2.0]) + assert repo._as_series(s).equals(s) + + +def test_as_series_one_column_df(): + repo = dmr.DataManagerRepository(MagicMock()) + df = pd.DataFrame({'x': [1.0, 2.0]}) + out = repo._as_series(df) + assert isinstance(out, pd.Series) + + +def test_as_series_multi_column_raises(): + repo = dmr.DataManagerRepository(MagicMock()) + df = pd.DataFrame({'a': [1.0], 'b': [2.0]}) + with pytest.raises(ValueError, match='single-column'): + repo._as_series(df) + + +def test_compute_regression_metrics_requires_y_pred(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + tmr = TrainModelResult( + params=p, + train_data=pd.DataFrame({'t': [1.0]}), + val_data=pd.DataFrame({'t': [1.0]}), + y_pred=None, + ) + with pytest.raises(ValueError, match='y_pred must be set'): + repo.compute_regression_metrics(tmr, MagicMock()) + + +def test_compute_regression_metrics_no_overlap(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + tmr = TrainModelResult( + params=p, + train_data=pd.DataFrame({'t': [1.0]}), + val_data=pd.DataFrame({'t': [1.0]}, index=[10]), + y_pred=pd.DataFrame({'p': [1.0]}, index=[20]), + ) + with pytest.raises(ValueError, match='No overlapping indices'): + repo.compute_regression_metrics(tmr, MagicMock()) + + +def test_compute_regression_metrics_linear_equation(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + p.model_type = 'linear_regression' + idx = pd.Index([0, 1]) + tmr = TrainModelResult( + params=p, + train_data=pd.DataFrame({'t': [1.0, 2.0]}, index=idx), + val_data=pd.DataFrame({'t': [1.0, 2.0]}, index=idx), + y_pred=pd.DataFrame({'p': [1.0, 2.0]}, index=idx), + ) + regr = MagicMock() + regr.coef_ = np.array([0.5]) + regr.intercept_ = 1.0 + wrapper = MagicMock() + wrapper.model = MagicMock() + wrapper.model.regr = regr + out = repo.compute_regression_metrics(tmr, wrapper) + assert out.mse_val is not None and out.equation is not None + + +def test_compute_regression_metrics_linear_skips_equation_without_sklearn_regr(): + """E2E dummy wrappers expose model without sklearn .regr; metrics still compute.""" + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + p.model_type = 'linear_regression' + idx = pd.Index([0, 1]) + tmr = TrainModelResult( + params=p, + train_data=pd.DataFrame({'t': [1.0, 2.0]}, index=idx), + val_data=pd.DataFrame({'t': [1.0, 2.0]}, index=idx), + y_pred=pd.DataFrame({'p': [1.0, 2.0]}, index=idx), + ) + wrapper = MagicMock() + wrapper.model = object() + out = repo.compute_regression_metrics(tmr, wrapper) + assert out.mse_val is not None and out.equation is None + + +def test_compute_regression_metrics_non_linear_skips_equation(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + p.model_type = 'xgboost' + idx = pd.Index([0, 1]) + tmr = TrainModelResult( + params=p, + train_data=pd.DataFrame({'t': [1.0, 2.0]}, index=idx), + val_data=pd.DataFrame({'t': [1.0, 2.0]}, index=idx), + y_pred=pd.DataFrame({'p': [1.0, 2.0]}, index=idx), + ) + out = repo.compute_regression_metrics(tmr, MagicMock()) + assert out.mse_val is not None and out.equation is None + + +def test_configure_datetime_index_none_raises(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + with pytest.raises(ValueError, match='Data is None'): + repo._configure_datetime_index(None, p, {}) + + +def test_configure_datetime_index_already_datetime_index(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + existing_idx = pd.date_range('2024-01-01', periods=3, freq='h') + df = pd.DataFrame( + { + 'timestamp': pd.to_datetime( + ['2024-01-03 00:00:00', '2024-01-01 00:00:00', '2024-01-02 00:00:00'] + ), + 'v1': [1, 2, 3], + 't': [1, 2, 3], + }, + index=existing_idx, + ) + out = repo._configure_datetime_index(df, p, {}) + assert isinstance(out.index, pd.DatetimeIndex) + assert out.index.equals(pd.DatetimeIndex(pd.to_datetime(sorted(df['timestamp'].tolist())))) + + +def test_configure_datetime_index_from_date_column(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict({**_minimal_dict_for_prepare(), 'date_column': 'mydate'}) + df = pd.DataFrame( + { + 'mydate': pd.date_range('2024-01-01', periods=3, freq='D'), + 'v1': [1, 2, 3], + 't': [1, 2, 3], + } + ) + out = repo._configure_datetime_index(df, p, {}) + assert isinstance(out.index, pd.DatetimeIndex) + + +def test_configure_datetime_index_missing_date_column_raises(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict({**_minimal_dict_for_prepare(), 'date_column': 'mydate'}) + df = pd.DataFrame( + { + 'timestamp': pd.date_range('2024-01-01', periods=3, freq='D'), + 'v1': [1, 2, 3], + 't': [1, 2, 3], + } + ) + with pytest.raises(ValueError, match='date_column "mydate" not found'): + repo._configure_datetime_index(df, p, {}) + + +def test_configure_datetime_index_non_datetime_date_column_raises(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict({**_minimal_dict_for_prepare(), 'date_column': 'mydate'}) + df = pd.DataFrame( + { + 'mydate': ['2024-01-01', '2024-01-02', '2024-01-03'], + 'v1': [1.0, 2.0, 3.0], + 't': [1.0, 2.0, 3.0], + } + ) + with pytest.raises(ValueError, match='must be datetime before index configuration'): + repo._configure_datetime_index(df, p, {}) + + +def test_create_run_directory_permission_error(): + repo = dmr.DataManagerRepository(MagicMock()) + with patch( + 'model_manager.utils.repository.data_manager_repository.makedirs', + side_effect=PermissionError('no'), + ): + with pytest.raises(PermissionError, match='Permission denied'): + repo._create_run_directory('/tmp', 'run', {}) + + +def test_create_run_directory_os_error(): + repo = dmr.DataManagerRepository(MagicMock()) + with patch( + 'model_manager.utils.repository.data_manager_repository.makedirs', + side_effect=OSError('disk'), + ): + with pytest.raises(OSError, match='Failed to create directory'): + repo._create_run_directory('/tmp', 'run', {}) + + +def test_generate_report_success(tmp_path): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + tmr = TrainModelResult( + params=p, + train_data=pd.DataFrame({'v1': [1.0, 2.0], 't': [1.0, 2.0]}), + val_data=pd.DataFrame({'v1': [1.0, 2.0], 't': [1.0, 2.0]}), + y_train_pred=pd.DataFrame({'t': [1.0, 2.0]}), + y_pred=pd.DataFrame({'t': [1.0, 2.0]}), + run_name='testrun', + ) + tmr.equation = {'target_variable': 't'} + with ( + patch.object(repo, '_get_reports_directory', return_value=str(tmp_path)), + patch('model_manager.utils.repository.data_manager_repository.Reports') as mrep, + ): + instance = mrep.return_value + instance.save_all_sections_html = Mock() + out = repo.generate_report(tmr, {}) + assert out.report_path and out.train_data_path and out.test_data_path + if out.equation_path: + with open(out.equation_path, encoding='utf-8') as f: + json.load(f) + + +def test_generate_report_adds_target_alias_for_reports(tmp_path): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + tmr = TrainModelResult( + params=p, + train_data=pd.DataFrame({'v1': [1.0, 2.0], 't': [1.0, 2.0]}), + val_data=pd.DataFrame({'v1': [1.0, 2.0], 't': [1.0, 2.0]}), + y_train_pred=pd.DataFrame({'t': [1.0, 2.0]}), + y_pred=pd.DataFrame({'t': [1.0, 2.0]}), + run_name='testrun', + ) + + with ( + patch.object(repo, '_get_reports_directory', return_value=str(tmp_path)), + patch('model_manager.utils.repository.data_manager_repository.Reports') as mrep, + ): + instance = mrep.return_value + instance.save_all_sections_html = Mock() + out = repo.generate_report(tmr, {}) + + kwargs = mrep.call_args.kwargs + reference_data = kwargs['reference_data'] + current_data = kwargs['current_data'] + assert 'target' in reference_data.columns + assert 'target' in current_data.columns + assert reference_data['target'].equals(reference_data['t']) + assert current_data['target'].equals(current_data['t']) + + assert out.train_data_path is not None + assert out.test_data_path is not None + train_csv = pd.read_csv(out.train_data_path) + test_csv = pd.read_csv(out.test_data_path) + assert 'target' in train_csv.columns + assert 'target' in test_csv.columns + assert train_csv['target'].equals(train_csv['t']) + assert test_csv['target'].equals(test_csv['t']) + + +def test_generate_report_skips_equation_file_when_not_linear(tmp_path): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + p.model_type = 'other' + tmr = TrainModelResult( + params=p, + train_data=pd.DataFrame({'v1': [1.0, 2.0], 't': [1.0, 2.0]}), + val_data=pd.DataFrame({'v1': [1.0, 2.0], 't': [1.0, 2.0]}), + y_train_pred=pd.DataFrame({'t': [1.0, 2.0]}), + y_pred=pd.DataFrame({'t': [1.0, 2.0]}), + run_name='testrun', + equation={'k': 'v'}, + ) + with ( + patch.object(repo, '_get_reports_directory', return_value=str(tmp_path)), + patch('model_manager.utils.repository.data_manager_repository.Reports'), + ): + out = repo.generate_report(tmr, {}) + assert out.equation_path is None + + +def test_generate_report_run_name_missing(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + tmr = TrainModelResult( + params=p, + train_data=pd.DataFrame({'t': [1.0]}), + val_data=pd.DataFrame({'t': [1.0]}), + run_name=None, + ) + with pytest.raises(ValueError, match='run_name is not set'): + repo.generate_report(tmr, {}) + + +def test_generate_report_requires_predictions(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + tmr = TrainModelResult( + params=p, + train_data=pd.DataFrame({'t': [1.0]}), + val_data=pd.DataFrame({'t': [1.0]}), + run_name='testrun', + y_train_pred=None, + y_pred=None, + ) + with pytest.raises(ValueError, match='y_train_pred or y_pred is not set'): + repo.generate_report(tmr, {}) + + +def test_cleanup_run_directory_empty(): + repo = dmr.DataManagerRepository(MagicMock()) + repo.cleanup_run_directory('', {}) + + +def test_cleanup_run_directory_exists(tmp_path): + repo = dmr.DataManagerRepository(MagicMock()) + d = tmp_path / 'subdir' + d.mkdir() + repo.cleanup_run_directory(str(d), {}) + assert not d.exists() + + +def test_cleanup_run_directory_missing(tmp_path): + repo = dmr.DataManagerRepository(MagicMock()) + repo.cleanup_run_directory(str(tmp_path / 'nope'), {}) + + +def test_extract_model_equation_polynomial_poly_names(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + p.model_kwargs = {'degree': 2, 'poly_feature_names': ['f1', 'f2']} + regr = MagicMock() + regr.coef_ = np.array([1.0, 2.0]) + regr.intercept_ = 3.0 + wrapper = MagicMock() + wrapper.model = MagicMock() + wrapper.model.regr = regr + eq = repo._extract_model_equation(wrapper.model, p) + assert 'equation_string' in eq and eq['degree'] == 2 + + +def test_extract_model_equation_extra_coefficients_ignored(): + """More coefficients than feature names: only the first len(names) are used.""" + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + regr = MagicMock() + regr.coef_ = np.array([1.0, 2.0, 3.0]) + regr.intercept_ = 0.0 + wrapper = MagicMock() + wrapper.model = MagicMock() + wrapper.model.regr = regr + eq = repo._extract_model_equation(wrapper.model, p) + assert len(eq['coefficients']) == len(p.variable_columns) + + +def test_extract_model_equation_more_features_than_coefficients(): + """Polynomial feature names longer than coef array: extra names get no coefficient entry.""" + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + p.model_kwargs = {'degree': 2, 'poly_feature_names': ['a', 'b', 'c']} + regr = MagicMock() + regr.coef_ = np.array([1.0, 2.0]) + regr.intercept_ = 0.0 + wrapper = MagicMock() + wrapper.model = MagicMock() + wrapper.model.regr = regr + eq = repo._extract_model_equation(wrapper.model, p) + assert list(eq['coefficients'].keys()) == ['a', 'b'] + + +def test_get_reports_directory_path(): + repo = dmr.DataManagerRepository(MagicMock()) + reports_dir = repo._get_reports_directory() + assert reports_dir == REPORTS_ROOT diff --git a/tests/utils/test_connectors_config.py b/tests/utils/test_connectors_config.py new file mode 100644 index 0000000..6411fb3 --- /dev/null +++ b/tests/utils/test_connectors_config.py @@ -0,0 +1,163 @@ +from os import environ +from unittest.mock import patch + +from model_manager.utils.connectors_config import ( + build_minio_config, + build_mlflow_config, + build_mongodb_config, + build_plugin_store_config, + build_postgres_config, +) + + +def test_build_mlflow_config_with_env_vars(): + environ.pop('MLFLOW_URL', None) + environ['MLFLOW_URL'] = 'http://test-host:8080' + environ['MLFLOW_USERNAME'] = 'test-user' + environ['MLFLOW_PASSWORD'] = 'test-pass' + + config = build_mlflow_config() + + assert config['url'] == 'http://test-host:8080' + assert config['username'] == 'test-user' + assert config['password'] == 'test-pass' + + +def test_build_mlflow_config_with_defaults(): + environ.pop('MLFLOW_URL', None) + environ.pop('MLFLOW_USERNAME', None) + environ.pop('MLFLOW_PASSWORD', None) + + config = build_mlflow_config() + + assert config['url'] == 'http://localhost:5080' + assert config['username'] == 'aignosi' + assert config['password'] == 'aignosi' + + +def test_build_postgres_config_with_env_vars(): + environ['POSTGRES_HOST'] = 'test-host' + environ['POSTGRES_PORT'] = '5433' + environ['POSTGRES_USER'] = 'test-user' + environ['POSTGRES_PASSWORD'] = 'test-pass' + environ['POSTGRES_DBNAME'] = 'test-db' + environ['POSTGRES_MIN_CONNECTIONS'] = '10' + environ['POSTGRES_MAX_CONNECTIONS'] = '30' + + config = build_postgres_config() + + assert config['host'] == 'test-host' + assert config['port'] == 5433 + assert config['user'] == 'test-user' + assert config['password'] == 'test-pass' + assert config['dbname'] == 'test-db' + assert config['min_connections'] == 10 + assert config['max_connections'] == 30 + + +def test_build_postgres_config_with_defaults(): + environ.pop('POSTGRES_HOST', None) + environ.pop('POSTGRES_PORT', None) + environ.pop('POSTGRES_USER', None) + environ.pop('POSTGRES_PASSWORD', None) + environ.pop('POSTGRES_DBNAME', None) + environ.pop('POSTGRES_MIN_CONNECTIONS', None) + environ.pop('POSTGRES_MAX_CONNECTIONS', None) + + config = build_postgres_config() + + assert config['host'] == 'localhost' + assert config['port'] == 5432 + assert config['user'] == 'sientia' + assert config['password'] == 'sientia' + assert config['dbname'] == 'sientia' + assert config['min_connections'] == 5 + assert config['max_connections'] == 20 + + +def test_build_mongo_db_config_with_env_vars(): + environ['MONGODB_USERNAME'] = 'sientia1' + environ['MONGODB_PASSWORD'] = 'sientia1' + environ['MONGODB_URL'] = 'localhost:27018' + environ['MONGODB_DATABASE'] = 'test_db' + environ['MONGODB_TTL_INDEX_HOURS'] = '1' + + assert build_mongodb_config() == { + 'connection_string': 'mongodb://sientia1:sientia1@localhost:27018', + 'database_name': 'test_db', + 'ttl_index_seconds': 3600, + 'uri': 'localhost:27018', + } + + +def test_build_mongo_db_config_with_defaults(): + environ.pop('MONGODB_USERNAME', None) + environ.pop('MONGODB_PASSWORD', None) + environ.pop('MONGODB_DATABASE', None) + environ.pop('MONGODB_URL', None) + environ.pop('MONGODB_TTL_INDEX_HOURS', None) + assert build_mongodb_config() == { + 'connection_string': 'mongodb://root:wKZDbMNU1c@localhost:27018', + 'database_name': 'sientia', + 'ttl_index_seconds': 3600, + 'uri': 'localhost:27018', + } + + +def test_build_minio_config_with_env_vars(): + environ['MINIO_ENDPOINT_URL'] = 'http://test-minio:9000' + environ['MINIO_ACCESS_KEY'] = 'test-access-key' + environ['MINIO_SECRET_KEY'] = 'test-secret-key' + environ['MINIO_REGION'] = 'eu-west-1' + environ['MINIO_SECURE'] = 'true' + environ['MINIO_MAX_RETRY_ATTEMPTS'] = '5' + environ['MINIO_RETRY_MODE'] = 'standard' + environ['MINIO_CONNECT_TIMEOUT'] = '20' + environ['MINIO_READ_TIMEOUT'] = '120' + environ['MINIO_DEFAULT_BUCKET'] = 'my-bucket' + + config = build_minio_config() + + assert config['endpoint_url'] == 'http://test-minio:9000' + assert config['access_key'] == 'test-access-key' + assert config['secret_key'] == 'test-secret-key' + assert config['region'] == 'eu-west-1' + assert config['use_ssl'] is True + assert config['max_retry_attempts'] == 5 + assert config['retry_mode'] == 'standard' + assert config['connect_timeout'] == 20 + assert config['read_timeout'] == 120 + assert config['default_bucket'] == 'my-bucket' + + +def test_build_plugin_store_config_cache_ttl_seconds(): + """STORE_CACHE_TTL_SECONDS is parsed to int when set.""" + with patch.dict(environ, {'STORE_CACHE_TTL_SECONDS': '7200'}, clear=False): + cfg = build_plugin_store_config() + assert cfg['cache_ttl_seconds'] == 7200 + + +def test_build_minio_config_with_defaults(): + environ.pop('MINIO_ENDPOINT_URL', None) + environ.pop('MINIO_ACCESS_KEY', None) + environ.pop('MINIO_SECRET_KEY', None) + environ.pop('MINIO_REGION', None) + environ.pop('MINIO_SECURE', None) + environ.pop('MINIO_MAX_RETRY_ATTEMPTS', None) + environ.pop('MINIO_RETRY_MODE', None) + environ.pop('MINIO_CONNECT_TIMEOUT', None) + environ.pop('MINIO_READ_TIMEOUT', None) + environ.pop('MINIO_DEFAULT_BUCKET', None) + + config = build_minio_config() + + assert config['endpoint_url'] == 'http://localhost:9000' + assert config['access_key'] == 'minioadmin' + assert config['secret_key'] == 'minioadmin' + assert config['region'] == 'us-east-1' + assert config['use_ssl'] is False + assert config['max_retry_attempts'] == 3 + assert config['retry_mode'] == 'adaptive' + assert config['connect_timeout'] == 10 + assert config['read_timeout'] == 60 + assert config['default_bucket'] == 'model-training' diff --git a/tests/utils/test_logger_helper.py b/tests/utils/test_logger_helper.py new file mode 100644 index 0000000..51d815c --- /dev/null +++ b/tests/utils/test_logger_helper.py @@ -0,0 +1,90 @@ +"""Unit tests for logger_helper module with 100% coverage.""" + +from unittest.mock import MagicMock, patch + + +@patch('model_manager.utils.logger_helper.SientiaLogger') +def test_get_logger_creates_logger_instance(mock_sientia_logger): + """Test get_logger creates a SientiaLogger instance with the given name.""" + from model_manager.utils.logger_helper import get_logger + + mock_logger_instance = MagicMock() + mock_logger_instance.base_logger = MagicMock() + mock_sientia_logger.return_value = mock_logger_instance + + result = get_logger('test_module') + + mock_sientia_logger.assert_called_once_with('test_module') + assert result is mock_logger_instance + + +@patch('model_manager.utils.logger_helper.SientiaLogger') +def test_get_logger_disables_propagation(mock_sientia_logger): + """Test get_logger disables log propagation.""" + from model_manager.utils.logger_helper import get_logger + + mock_logger_instance = MagicMock() + mock_base_logger = MagicMock() + mock_base_logger.propagate = True + mock_logger_instance.base_logger = mock_base_logger + mock_sientia_logger.return_value = mock_logger_instance + + get_logger('test_module') + + assert mock_base_logger.propagate is False + + +@patch('model_manager.utils.logger_helper.SientiaLogger') +def test_get_logger_with_different_names(mock_sientia_logger): + """Test get_logger works with different logger names.""" + from model_manager.utils.logger_helper import get_logger + + mock_logger_instance = MagicMock() + mock_logger_instance.base_logger = MagicMock() + mock_sientia_logger.return_value = mock_logger_instance + + logger1 = get_logger('module1') + logger2 = get_logger('module2') + logger3 = get_logger('my.nested.module') + + assert mock_sientia_logger.call_count == 3 + mock_sientia_logger.assert_any_call('module1') + mock_sientia_logger.assert_any_call('module2') + mock_sientia_logger.assert_any_call('my.nested.module') + assert logger1 is mock_logger_instance + assert logger2 is mock_logger_instance + assert logger3 is mock_logger_instance + + +@patch('model_manager.utils.logger_helper.SientiaLogger') +def test_get_logger_with_empty_name(mock_sientia_logger): + """Test get_logger with empty string name.""" + from model_manager.utils.logger_helper import get_logger + + mock_logger_instance = MagicMock() + mock_logger_instance.base_logger = MagicMock() + mock_sientia_logger.return_value = mock_logger_instance + + result = get_logger('') + + mock_sientia_logger.assert_called_once_with('') + assert result is mock_logger_instance + assert result.base_logger.propagate is False + + +@patch('model_manager.utils.logger_helper.SientiaLogger') +def test_get_logger_returns_configured_logger(mock_sientia_logger): + """Test get_logger returns the configured logger instance.""" + from model_manager.utils.logger_helper import get_logger + + mock_logger_instance = MagicMock() + mock_logger_instance.base_logger = MagicMock() + mock_logger_instance.base_logger.propagate = True + mock_sientia_logger.return_value = mock_logger_instance + + result = get_logger('test_logger') + + # Verify the logger is returned after configuration + assert result is mock_logger_instance + # Verify propagation was disabled + assert mock_logger_instance.base_logger.propagate is False diff --git a/tests/worker/test_prepare_worker.py b/tests/worker/test_prepare_worker.py new file mode 100644 index 0000000..c275c43 --- /dev/null +++ b/tests/worker/test_prepare_worker.py @@ -0,0 +1,84 @@ +"""Unit tests for local worker factory.""" + +from unittest.mock import MagicMock, patch + + +def test_build_queue_name_without_runtime_uses_default_suffix(): + from model_manager.worker.prepare_worker import build_queue_name + + assert build_queue_name('TrainModel') == 'train_model-queue' + + +def test_prepare_worker_train_queue_uses_train_limits(): + from model_manager.worker.prepare_worker import prepare_worker + from model_manager.workflows.train_model import TrainModel + + fake_worker = MagicMock() + fake_client = MagicMock() + fake_logger = MagicMock() + + with patch( + 'model_manager.worker.prepare_worker.Worker', return_value=fake_worker + ) as worker_class: + with patch.dict( + 'os.environ', + { + 'TRAINMODEL_ACTIVITY_EXECUTOR_MAX_WORKERS': '3', + 'TRAINMODEL_MAX_CONCURRENT_ACTIVITIES': '6', + 'TRAINMODEL_MAX_CONCURRENT_WORKFLOW_TASKS': '10', + }, + clear=False, + ): + worker = prepare_worker( + main_workflow=TrainModel, + other_workflows=[], + activities=[], + temporal_client=fake_client, + logger=fake_logger, + runtime='model-manager-worker', + ) + + assert worker is fake_worker + worker_class.assert_called_once() + kwargs = worker_class.call_args.kwargs + assert kwargs['task_queue'] == 'train_model-model-manager-worker-queue' + assert kwargs['max_concurrent_activities'] == 6 + assert kwargs['max_concurrent_workflow_tasks'] == 10 + assert kwargs['activity_executor']._max_workers == 3 + kwargs['activity_executor'].shutdown(wait=True, cancel_futures=True) + + +def test_prepare_worker_cleanup_queue_uses_cleanup_limits(): + from model_manager.worker.prepare_worker import prepare_worker + from model_manager.workflows.cleanup_files import CleanupFiles + + fake_worker = MagicMock() + fake_client = MagicMock() + fake_logger = MagicMock() + + with patch( + 'model_manager.worker.prepare_worker.Worker', return_value=fake_worker + ) as worker_class: + with patch.dict( + 'os.environ', + { + 'CLEANUPFILES_ACTIVITY_EXECUTOR_MAX_WORKERS': '5', + 'CLEANUPFILES_MAX_CONCURRENT_ACTIVITIES': '7', + }, + clear=False, + ): + worker = prepare_worker( + main_workflow=CleanupFiles, + other_workflows=[], + activities=[], + temporal_client=fake_client, + logger=fake_logger, + runtime='model-manager-worker', + ) + + assert worker is fake_worker + kwargs = worker_class.call_args.kwargs + assert kwargs['task_queue'] == 'cleanup_files-model-manager-worker-queue' + assert kwargs['max_concurrent_activities'] == 7 + assert kwargs['activity_executor']._max_workers == 5 + kwargs['activity_executor'].shutdown(wait=True, cancel_futures=True) diff --git a/tests/worker/test_worker.py b/tests/worker/test_worker.py new file mode 100644 index 0000000..b127853 --- /dev/null +++ b/tests/worker/test_worker.py @@ -0,0 +1,907 @@ +"""Unit tests for worker module.""" + +import asyncio +import os +import sys +from unittest.mock import AsyncMock, Mock, patch + +import pytest + + +@pytest.fixture +def mock_env_vars(): + """Set up test environment variables.""" + env_vars = { + 'POD_ID': 'test-pod-123', + 'HTTP_METRICS_PORT': '9090', + 'HTTP_SDK_METRICS_PORT': '9091', + 'TEMPORAL_HOST': 'localhost:7233', + 'TEMPORAL_NAMESPACE': 'test-namespace', + 'PROJECT_NAME': 'test-project', + 'TRAIN_TASK_QUEUE': 'train_model-local_queue', + 'CLEANUP_TASK_QUEUE': 'cleanup-local_queue', + 'RUNTIME': 'model-manager-worker', + 'STORE_BASE_URL': 'http://sientia-plugin-store.svc.cluster.local', + 'STORE_OWNER': 'sientia', + 'STORE_REPO': 'model-library-store', + } + + with patch.dict(os.environ, env_vars, clear=False): + yield env_vars + + +@pytest.fixture +def mock_logger(): + """Create a mock logger.""" + logger = Mock() + logger.custom_info = Mock() + logger.custom_error = Mock() + return logger + + +@pytest.fixture +def mock_temporal_client(): + """Create a mock Temporal client.""" + client_mock = AsyncMock() + client_mock.connect = AsyncMock() + return client_mock + + +@pytest.fixture +def mock_worker(): + """Create a mock Temporal worker.""" + worker_mock = Mock() + worker_mock.run = AsyncMock(return_value=None) + return worker_mock + + +@pytest.fixture +def mock_notification_handler(): + """Create a mock notification handler.""" + handler = Mock() + handler.shutdown = Mock() + return handler + + +@pytest.fixture +def mock_activities(): + """Create a mock Activities instance.""" + activities = AsyncMock() + activities.update_experiment_run = Mock() + activities.load_model_metadata = Mock() + activities.validate_train_params = Mock() + activities.train_model = Mock() + activities.cleanup_resources = Mock() + activities.shutdown = Mock() + return activities + + +def test_pod_id_from_env(): + """Test that POD_ID is correctly read from environment.""" + with patch.dict(os.environ, {'POD_ID': 'pod-test-123'}): + # Re-import to get new env value + import importlib + + import model_manager.worker.worker as worker_module + + importlib.reload(worker_module) + + assert worker_module.POD_ID == 'pod-test-123' + + +def test_sdk_metrics_port_default(): + """Test that SDK_METRICS_PORT uses default value.""" + with patch.dict(os.environ, {}, clear=True): + import importlib + + import model_manager.worker.worker as worker_module + + importlib.reload(worker_module) + + assert worker_module.SDK_METRICS_PORT == 9091 + + +def test_sdk_metrics_port_from_env(): + """Test that SDK_METRICS_PORT is read from environment.""" + with patch.dict(os.environ, {'HTTP_SDK_METRICS_PORT': '8888'}): + import importlib + + import model_manager.worker.worker as worker_module + + importlib.reload(worker_module) + + assert worker_module.SDK_METRICS_PORT == 8888 + + +@patch('model_manager.worker.worker.POD_ID', 'test-pod-123') +@patch('model_manager.worker.worker.start_http_server') +@patch('model_manager.worker.worker.metrics') +def test_start_prometheus_server_success( + mock_metrics, mock_start_http_server, mock_env_vars, mock_logger +): + """Test successful Prometheus server startup.""" + from model_manager.worker.worker import start_prometheus_server + + mock_app_up = Mock() + mock_metrics.APP_UP.labels.return_value = mock_app_up + + metadata: dict[str, str | None] = {'pod_id': 'test-pod-123', 'workflow_name': 'train_model'} + + start_prometheus_server(mock_logger, metadata) + + # Verify HTTP server started + mock_start_http_server.assert_called_once_with(9090) + + # Verify APP_UP metric was set to 1 + mock_metrics.APP_UP.labels.assert_called_once_with(pod_id='test-pod-123') + mock_app_up.set.assert_called_once_with(1) + mock_logger.custom_info.assert_called_once() + + +@patch('model_manager.worker.worker.start_http_server') +@patch('model_manager.worker.worker.metrics') +def test_start_prometheus_server_custom_port(mock_metrics, mock_start_http_server, mock_logger): + """Test Prometheus server startup with custom port.""" + from model_manager.worker.worker import start_prometheus_server + + with patch.dict(os.environ, {'HTTP_METRICS_PORT': '8080', 'POD_ID': 'custom-pod'}): + mock_app_up = Mock() + mock_metrics.APP_UP.labels.return_value = mock_app_up + + metadata: dict[str, str | None] = {'pod_id': 'custom-pod', 'workflow_name': 'train_model'} + + start_prometheus_server(mock_logger, metadata) + + mock_start_http_server.assert_called_once_with(8080) + + +@patch('model_manager.worker.worker.start_http_server') +@patch('model_manager.worker.worker.metrics') +@patch('model_manager.worker.worker.os._exit') +def test_start_prometheus_server_failure( + mock_exit, mock_metrics, mock_start_http_server, mock_env_vars, mock_logger +): + """Test Prometheus server startup failure.""" + from model_manager.worker.worker import start_prometheus_server + + mock_start_http_server.side_effect = OSError('Port already in use') + + metadata: dict[str, str | None] = {'pod_id': 'test-pod-123', 'workflow_name': 'train_model'} + + start_prometheus_server(mock_logger, metadata) + + # Verify exit was called with code 1 e log crítico emitido + mock_exit.assert_called_once_with(1) + mock_logger.custom_critical.assert_called_once() + + +@pytest.mark.asyncio +@patch('model_manager.worker.worker.RUNTIME', 'model-manager-worker') +@patch('model_manager.worker.worker.prepare_worker') +@patch('model_manager.worker.worker.client.Client') +@patch('model_manager.worker.worker.Runtime') +@patch('model_manager.worker.worker.Activities') +@patch('model_manager.worker.worker.NotificationHandler') +@patch('model_manager.worker.worker.PluginStore') +@patch('model_manager.worker.worker.build_plugin_store_config') +@patch('model_manager.worker.worker.build_mongodb_config') +@patch('model_manager.worker.worker.build_postgres_config') +@patch('model_manager.worker.worker.build_mlflow_config') +@patch('model_manager.worker.worker.build_minio_config') +@patch('model_manager.worker.worker.get_logger') +@patch('model_manager.worker.worker.start_prometheus_server') +@patch('model_manager.worker.worker.metrics') +@patch('model_manager.worker.worker.ensure_runtime_directories') +async def test_main_successful_startup( + mock_ensure_runtime_directories, + mock_metrics, + mock_start_prometheus, + mock_get_logger, + mock_build_minio, + mock_build_mlflow, + mock_build_postgres, + mock_build_mongodb, + mock_build_plugin_store_config, + mock_plugin_store_class, + mock_notification_handler_class, + mock_activities_class, + mock_runtime_class, + mock_client_class, + mock_prepare_worker, + mock_env_vars, + mock_logger, + mock_temporal_client, + mock_worker, + mock_notification_handler, + mock_activities, +): + """Test successful main() execution until workers start.""" + from model_manager.worker.worker import main + + # Setup mocks + mock_get_logger.return_value = mock_logger + mock_build_mongodb.return_value = { + 'connection_string': 'mongodb://test', + 'database_name': 'test_db', + 'uri': 'localhost:27018', + } + mock_build_postgres.return_value = {} + mock_build_mlflow.return_value = {} + mock_build_minio.return_value = {} + + mock_notification_handler_class.return_value = mock_notification_handler + mock_activities_class.return_value = mock_activities + + mock_plugin_store_instance = AsyncMock() + mock_plugin_store_instance.install_runtime = AsyncMock( + return_value={'runtime': 'model-manager-worker', 'installed': []}, + ) + mock_plugin_store_class.return_value = mock_plugin_store_instance + mock_build_plugin_store_config.return_value = { + 'base_url': 'http://sientia-plugin-store.svc.cluster.local', + 'owner': 'sientia', + 'repo': 'model-library-store', + 'branch': 'main', + 'username': 'gitea-user', + 'password': 'gitea-password', + 'pypi_index_url': 'http://library-distribution-server.library.svc.cluster.local:5000', + 'pypi_username': None, + 'pypi_password': None, + 'cache_ttl_seconds': None, + } + + mock_runtime = Mock() + mock_runtime_class.return_value = mock_runtime + + mock_client_instance = AsyncMock() + mock_client_instance.config = Mock(return_value={'plugins': [], 'interceptors': []}) + mock_client_class.connect = AsyncMock(return_value=mock_client_instance) + + mock_worker_instance = Mock() + mock_worker_instance.run = AsyncMock( + side_effect=asyncio.CancelledError() + ) # Simulate interruption + mock_prepare_worker.return_value = mock_worker_instance + + mock_app_up = Mock() + mock_metrics.APP_UP.labels.return_value = mock_app_up + + # Run main() and expect it to exit due to CancelledError + with pytest.raises(SystemExit) as exc_info: + await main() + + assert exc_info.value.code == 1 + + # Verify all initialization steps were called + mock_get_logger.assert_called_once() + mock_start_prometheus.assert_called_once() + mock_notification_handler_class.assert_called_once() + mock_activities_class.assert_called_once() + mock_client_class.connect.assert_called_once() + # Agora são criados dois Workers: um para train_model-queue e outro para cleanup-queue + assert mock_prepare_worker.call_count == 2 + + # Verify cleanup was performed + mock_notification_handler.shutdown.assert_called_once() + mock_activities.shutdown.assert_called_once() + mock_app_up.set.assert_called_with(0) + + +@pytest.mark.asyncio +@patch('model_manager.worker.worker.RUNTIME', 'model-manager-worker') +@patch('model_manager.worker.worker.prepare_worker') +@patch('model_manager.worker.worker.client.Client') +@patch('model_manager.worker.worker.Runtime') +@patch('model_manager.worker.worker.Activities') +@patch('model_manager.worker.worker.NotificationHandler') +@patch('model_manager.worker.worker.PluginStore') +@patch('model_manager.worker.worker.build_plugin_store_config') +@patch('model_manager.worker.worker.build_mongodb_config') +@patch('model_manager.worker.worker.build_postgres_config') +@patch('model_manager.worker.worker.build_mlflow_config') +@patch('model_manager.worker.worker.build_minio_config') +@patch('model_manager.worker.worker.get_logger') +@patch('model_manager.worker.worker.start_prometheus_server') +@patch('model_manager.worker.worker.metrics') +@patch('model_manager.worker.worker.ensure_runtime_directories') +async def test_main_handles_exception( + mock_ensure_runtime_directories, + mock_metrics, + mock_start_prometheus, + mock_get_logger, + mock_build_minio, + mock_build_mlflow, + mock_build_postgres, + mock_build_mongodb, + mock_build_plugin_store_config, + mock_plugin_store_class, + mock_notification_handler_class, + mock_activities_class, + mock_runtime_class, + mock_client_class, + mock_prepare_worker, + mock_env_vars, + mock_logger, +): + """Test main() handles exceptions and performs cleanup.""" + from model_manager.worker.worker import main + + # Setup mocks + mock_get_logger.return_value = mock_logger + mock_build_mongodb.return_value = { + 'connection_string': 'mongodb://test', + 'database_name': 'test_db', + 'uri': 'localhost:27018', + } + mock_build_postgres.return_value = {} + mock_build_mlflow.return_value = {} + mock_build_minio.return_value = {} + + mock_notification_handler = Mock() + mock_notification_handler.shutdown = Mock() + mock_notification_handler_class.return_value = mock_notification_handler + + mock_activities = AsyncMock() + mock_activities.shutdown = Mock() + mock_activities_class.return_value = mock_activities + + mock_runtime = Mock() + mock_runtime_class.return_value = mock_runtime + + mock_client_instance = AsyncMock() + mock_client_instance.config = Mock(return_value={'plugins': [], 'interceptors': []}) + mock_client_class.connect = AsyncMock(return_value=mock_client_instance) + + mock_worker_instance = Mock() + mock_worker_instance.run = AsyncMock(side_effect=RuntimeError('Worker failed')) + mock_prepare_worker.return_value = mock_worker_instance + + mock_app_up = Mock() + mock_metrics.APP_UP.labels.return_value = mock_app_up + + mock_plugin_store_instance = AsyncMock() + mock_plugin_store_instance.install_runtime = AsyncMock( + return_value={'runtime': 'model-manager-worker', 'installed': []}, + ) + mock_plugin_store_class.return_value = mock_plugin_store_instance + mock_build_plugin_store_config.return_value = { + 'base_url': 'http://sientia-plugin-store.svc.cluster.local', + 'owner': 'sientia', + 'repo': 'model-library-store', + 'branch': 'main', + 'username': 'gitea-user', + 'password': 'gitea-password', + 'pypi_index_url': 'http://library-distribution-server.library.svc.cluster.local:5000', + 'pypi_username': None, + 'pypi_password': None, + 'cache_ttl_seconds': None, + } + + # Run main() and expect SystemExit + with pytest.raises(SystemExit) as exc_info: + await main() + + assert exc_info.value.code == 1 + + # Verify error was logged + mock_logger.custom_error.assert_called_once() + assert 'Worker failed' in str(mock_logger.custom_error.call_args) + + # Verify cleanup was performed + mock_notification_handler.shutdown.assert_called_once() + mock_activities.shutdown.assert_called_once() + mock_app_up.set.assert_called_with(0) + + +@pytest.mark.asyncio +@patch('model_manager.worker.worker.RUNTIME', 'model-manager-worker') +@patch('model_manager.worker.worker.create_cleanup_schedule') +@patch('model_manager.worker.worker.prepare_worker') +@patch('model_manager.worker.worker.client.Client') +@patch('model_manager.worker.worker.Runtime') +@patch('model_manager.worker.worker.Activities') +@patch('model_manager.worker.worker.NotificationHandler') +@patch('model_manager.worker.worker.PluginStore') +@patch('model_manager.worker.worker.build_plugin_store_config') +@patch('model_manager.worker.worker.build_mongodb_config') +@patch('model_manager.worker.worker.build_postgres_config') +@patch('model_manager.worker.worker.build_mlflow_config') +@patch('model_manager.worker.worker.build_minio_config') +@patch('model_manager.worker.worker.get_logger') +@patch('model_manager.worker.worker.start_prometheus_server') +@patch('model_manager.worker.worker.metrics') +@patch('model_manager.worker.worker.ensure_runtime_directories') +async def test_main_temporal_client_configuration( + mock_ensure_runtime_directories, + mock_metrics, + mock_start_prometheus, + mock_get_logger, + mock_build_minio, + mock_build_mlflow, + mock_build_postgres, + mock_build_mongodb, + mock_build_plugin_store_config, + mock_plugin_store_class, + mock_notification_handler_class, + mock_activities_class, + mock_runtime_class, + mock_client_class, + mock_prepare_worker, + mock_create_cleanup_schedule, + mock_logger, +): + """Test that Temporal client is configured correctly.""" + from model_manager.worker.worker import main + + mock_create_cleanup_schedule.return_value = AsyncMock() + + with patch.dict( + os.environ, + { + 'TEMPORAL_HOST': 'temporal.example.com:7233', + 'TEMPORAL_NAMESPACE': 'production', + 'TEMPORAL_USE_TLS': 'true', + 'RUNTIME': 'model-manager-worker', + 'STORE_BASE_URL': 'http://sientia-plugin-store.svc.cluster.local', + 'STORE_OWNER': 'sientia', + 'STORE_REPO': 'model-library-store', + }, + ): + # Setup mocks + mock_get_logger.return_value = mock_logger + mock_build_mongodb.return_value = { + 'connection_string': 'mongodb://test', + 'database_name': 'test_db', + 'uri': 'localhost:27018', + } + mock_build_postgres.return_value = {} + mock_build_mlflow.return_value = {} + mock_build_minio.return_value = {} + + mock_plugin_store_instance = AsyncMock() + mock_plugin_store_instance.install_runtime = AsyncMock( + return_value={'runtime': 'model-manager-worker', 'installed': []}, + ) + mock_plugin_store_class.return_value = mock_plugin_store_instance + mock_build_plugin_store_config.return_value = { + 'base_url': 'http://sientia-plugin-store.svc.cluster.local', + 'owner': 'sientia', + 'repo': 'model-library-store', + 'branch': 'main', + 'username': 'gitea-user', + 'password': 'gitea-password', + 'pypi_index_url': 'http://library-distribution-server.library.svc.cluster.local:5000', + 'pypi_username': None, + 'pypi_password': None, + 'cache_ttl_seconds': None, + } + + mock_notification_handler = Mock() + mock_notification_handler.shutdown = Mock() + mock_notification_handler_class.return_value = mock_notification_handler + + mock_activities = AsyncMock() + mock_activities.shutdown = Mock() + mock_activities_class.return_value = mock_activities + + mock_runtime = Mock() + mock_runtime_class.return_value = mock_runtime + + mock_client_instance = AsyncMock() + mock_client_instance.config = Mock(return_value={'plugins': [], 'interceptors': []}) + mock_client_class.connect = AsyncMock(return_value=mock_client_instance) + + mock_worker_instance = Mock() + mock_worker_instance.run = AsyncMock(side_effect=asyncio.CancelledError()) + mock_prepare_worker.return_value = mock_worker_instance + + mock_app_up = Mock() + mock_metrics.APP_UP.labels.return_value = mock_app_up + + # Run main() + with pytest.raises(SystemExit): + await main() + + # Verify Temporal client was configured with correct parameters + mock_client_class.connect.assert_called_once_with( + target_host='temporal.example.com:7233', + namespace='production', + runtime=mock_runtime, + tls=True, + ) + + +@pytest.mark.asyncio +@patch('model_manager.worker.worker.RUNTIME', 'model-manager-worker') +@patch('model_manager.worker.worker.create_cleanup_schedule') +@patch('model_manager.worker.worker.prepare_worker') +@patch('model_manager.worker.worker.client.Client') +@patch('model_manager.worker.worker.Runtime') +@patch('model_manager.worker.worker.Activities') +@patch('model_manager.worker.worker.NotificationHandler') +@patch('model_manager.worker.worker.PluginStore') +@patch('model_manager.worker.worker.build_plugin_store_config') +@patch('model_manager.worker.worker.build_mongodb_config') +@patch('model_manager.worker.worker.build_postgres_config') +@patch('model_manager.worker.worker.build_mlflow_config') +@patch('model_manager.worker.worker.build_minio_config') +@patch('model_manager.worker.worker.get_logger') +@patch('model_manager.worker.worker.start_prometheus_server') +@patch('model_manager.worker.worker.metrics') +@patch('model_manager.worker.worker.ensure_runtime_directories') +async def test_main_worker_configuration( + mock_ensure_runtime_directories, + mock_metrics, + mock_start_prometheus, + mock_get_logger, + mock_build_minio, + mock_build_mlflow, + mock_build_postgres, + mock_build_mongodb, + mock_build_plugin_store_config, + mock_plugin_store_class, + mock_notification_handler_class, + mock_activities_class, + mock_runtime_class, + mock_client_class, + mock_prepare_worker, + mock_create_cleanup_schedule, + mock_env_vars, + mock_logger, +): + """Test that prepare_worker is configured with correct workflows and activities.""" + from model_manager.worker.worker import main + from model_manager.workflows.cleanup_files import CleanupFiles + from model_manager.workflows.train_model import TrainModel + + mock_create_cleanup_schedule.return_value = AsyncMock() + + mock_get_logger.return_value = mock_logger + mock_build_mongodb.return_value = { + 'connection_string': 'mongodb://test', + 'database_name': 'test_db', + 'uri': 'localhost:27018', + } + mock_build_postgres.return_value = {} + mock_build_mlflow.return_value = {} + mock_build_minio.return_value = {} + + mock_notification_handler = Mock() + mock_notification_handler_class.return_value = mock_notification_handler + + mock_activities = AsyncMock() + mock_activities.update_experiment_run = Mock() + mock_activities.load_model_metadata = Mock() + mock_activities.validate_train_params = Mock() + mock_activities.train_model = Mock() + mock_activities.cleanup_resources = Mock() + mock_activities.cleanup_temp_directories = Mock() + mock_activities.shutdown = Mock() + mock_activities_class.return_value = mock_activities + + mock_runtime = Mock() + mock_runtime_class.return_value = mock_runtime + + mock_client_instance = AsyncMock() + mock_client_instance.config = Mock(return_value={'plugins': [], 'interceptors': []}) + mock_client_class.connect = AsyncMock(return_value=mock_client_instance) + + mock_worker_instance = Mock() + mock_worker_instance.run = AsyncMock(side_effect=asyncio.CancelledError()) + mock_prepare_worker.return_value = mock_worker_instance + + mock_app_up = Mock() + mock_metrics.APP_UP.labels.return_value = mock_app_up + + mock_plugin_store_instance = AsyncMock() + mock_plugin_store_instance.install_runtime = AsyncMock( + return_value={'runtime': 'model-manager-worker', 'installed': []}, + ) + mock_plugin_store_class.return_value = mock_plugin_store_instance + mock_build_plugin_store_config.return_value = { + 'base_url': 'http://sientia-plugin-store.svc.cluster.local', + 'owner': 'sientia', + 'repo': 'model-library-store', + 'branch': 'main', + 'username': 'gitea-user', + 'password': 'gitea-password', + 'pypi_index_url': 'http://library-distribution-server.library.svc.cluster.local:5000', + 'pypi_username': None, + 'pypi_password': None, + 'cache_ttl_seconds': None, + } + + with pytest.raises(SystemExit): + await main() + + assert mock_prepare_worker.call_count == 2 + + train_call = mock_prepare_worker.call_args_list[0] + assert train_call.kwargs['temporal_client'] is mock_client_instance + assert train_call.kwargs['logger'] is mock_logger + assert train_call.kwargs['main_workflow'] is TrainModel + assert train_call.kwargs['other_workflows'] == [] + train_activities_list = train_call.kwargs['activities'] + assert mock_activities.update_experiment_run in train_activities_list + assert mock_activities.load_model_metadata in train_activities_list + assert mock_activities.validate_train_params in train_activities_list + assert mock_activities.train_model in train_activities_list + assert mock_activities.cleanup_resources in train_activities_list + + cleanup_call = mock_prepare_worker.call_args_list[1] + assert cleanup_call.kwargs['temporal_client'] is mock_client_instance + assert cleanup_call.kwargs['logger'] is mock_logger + assert cleanup_call.kwargs['main_workflow'] is CleanupFiles + assert cleanup_call.kwargs['other_workflows'] == [] + assert cleanup_call.kwargs['activities'] == [mock_activities.cleanup_temp_directories] + + +@pytest.mark.asyncio +@patch('model_manager.worker.worker.RUNTIME', 'model-manager-worker') +@patch('model_manager.worker.worker.create_cleanup_schedule') +@patch('model_manager.worker.worker.prepare_worker') +@patch('model_manager.worker.worker.client.Client') +@patch('model_manager.worker.worker.Runtime') +@patch('model_manager.worker.worker.Activities') +@patch('model_manager.worker.worker.NotificationHandler') +@patch('model_manager.worker.worker.PluginStore') +@patch('model_manager.worker.worker.build_plugin_store_config') +@patch('model_manager.worker.worker.build_mongodb_config') +@patch('model_manager.worker.worker.build_postgres_config') +@patch('model_manager.worker.worker.build_mlflow_config') +@patch('model_manager.worker.worker.build_minio_config') +@patch('model_manager.worker.worker.get_logger') +@patch('model_manager.worker.worker.start_prometheus_server') +@patch('model_manager.worker.worker.metrics') +@patch('model_manager.worker.worker.ensure_runtime_directories') +async def test_main_schedule_creation_failure_does_not_stop_worker( + mock_ensure_runtime_directories, + mock_metrics, + mock_start_prometheus, + mock_get_logger, + mock_build_minio, + mock_build_mlflow, + mock_build_postgres, + mock_build_mongodb, + mock_build_plugin_store_config, + mock_plugin_store_class, + mock_notification_handler_class, + mock_activities_class, + mock_runtime_class, + mock_client_class, + mock_prepare_worker, + mock_create_cleanup_schedule, + mock_logger, +): + """Test that schedule creation failure does not prevent worker startup.""" + from model_manager.worker.worker import main + + # Mock schedule creation to raise an exception (as coroutine) + async def mock_schedule_error(*args, **kwargs): + raise Exception('Schedule creation failed') + + mock_create_cleanup_schedule.side_effect = mock_schedule_error + + # Setup mocks + mock_get_logger.return_value = mock_logger + mock_build_mongodb.return_value = { + 'connection_string': 'mongodb://test', + 'database_name': 'test_db', + 'uri': 'localhost:27018', + } + mock_build_postgres.return_value = {} + mock_build_mlflow.return_value = {} + mock_build_minio.return_value = {} + + mock_notification_handler = Mock() + mock_notification_handler.shutdown = Mock() + mock_notification_handler_class.return_value = mock_notification_handler + + mock_activities = AsyncMock() + mock_activities.shutdown = Mock() + mock_activities_class.return_value = mock_activities + + mock_runtime = Mock() + mock_runtime_class.return_value = mock_runtime + + mock_client_instance = AsyncMock() + mock_client_instance.config = Mock(return_value={'plugins': [], 'interceptors': []}) + mock_client_class.connect = AsyncMock(return_value=mock_client_instance) + + mock_worker_instance = Mock() + mock_worker_instance.run = AsyncMock(side_effect=asyncio.CancelledError()) + mock_prepare_worker.return_value = mock_worker_instance + + mock_app_up = Mock() + mock_metrics.APP_UP.labels.return_value = mock_app_up + + mock_plugin_store_instance = AsyncMock() + mock_plugin_store_instance.install_runtime = AsyncMock( + return_value={'runtime': 'model-manager-worker', 'installed': []}, + ) + mock_plugin_store_class.return_value = mock_plugin_store_instance + mock_build_plugin_store_config.return_value = { + 'base_url': 'http://sientia-plugin-store.svc.cluster.local', + 'owner': 'sientia', + 'repo': 'model-library-store', + 'branch': 'main', + 'username': 'gitea-user', + 'password': 'gitea-password', + 'pypi_index_url': 'http://library-distribution-server.library.svc.cluster.local:5000', + 'pypi_username': None, + 'pypi_password': None, + 'cache_ttl_seconds': None, + } + + with pytest.raises(SystemExit): + await main() + + mock_create_cleanup_schedule.assert_called_once() + + schedule_error_logged = False + for call in mock_logger.custom_error.call_args_list: + if call[0] and 'Failed to configure cleanup schedule' in call[0][0]: + schedule_error_logged = True + break + assert schedule_error_logged, 'Schedule creation error should be logged' + + assert mock_prepare_worker.call_count == 2 + + +@pytest.mark.asyncio +@patch('model_manager.worker.worker.RUNTIME', None) +@patch('model_manager.worker.worker.prepare_worker') +@patch('model_manager.worker.worker.client.Client') +@patch('model_manager.worker.worker.Runtime') +@patch('model_manager.worker.worker.Activities') +@patch('model_manager.worker.worker.NotificationHandler') +@patch('model_manager.worker.worker.PluginStore') +@patch('model_manager.worker.worker.build_plugin_store_config') +@patch('model_manager.worker.worker.build_mongodb_config') +@patch('model_manager.worker.worker.build_postgres_config') +@patch('model_manager.worker.worker.build_mlflow_config') +@patch('model_manager.worker.worker.build_minio_config') +@patch('model_manager.worker.worker.get_logger') +@patch('model_manager.worker.worker.start_prometheus_server') +@patch('model_manager.worker.worker.metrics') +@patch('model_manager.worker.worker.ensure_runtime_directories') +async def test_main_missing_runtime_uses_single_fallback( + mock_ensure_runtime_directories, + mock_metrics, + mock_start_prometheus, + mock_get_logger, + mock_build_minio, + mock_build_mlflow, + mock_build_postgres, + mock_build_mongodb, + mock_build_plugin_store_config, + mock_plugin_store_class, + mock_notification_handler_class, + mock_activities_class, + mock_runtime_class, + mock_client_class, + mock_prepare_worker, + mock_logger, +): + """Test that main() uses single runtime fallback when RUNTIME is missing.""" + from model_manager.worker.worker import main + + mock_get_logger.return_value = mock_logger + mock_build_mongodb.return_value = { + 'connection_string': 'mongodb://test', + 'database_name': 'test_db', + 'uri': 'localhost:27018', + } + mock_build_postgres.return_value = {} + mock_build_mlflow.return_value = {} + mock_build_minio.return_value = {} + + mock_notification_handler = Mock() + mock_notification_handler.shutdown = Mock() + mock_notification_handler_class.return_value = mock_notification_handler + + mock_activities = AsyncMock() + mock_activities.shutdown = Mock() + mock_activities_class.return_value = mock_activities + + mock_runtime = Mock() + mock_runtime_class.return_value = mock_runtime + + mock_client_instance = AsyncMock() + mock_client_instance.config = Mock(return_value={'plugins': [], 'interceptors': []}) + mock_client_class.connect = AsyncMock(return_value=mock_client_instance) + + mock_worker_instance = Mock() + mock_worker_instance.run = AsyncMock(side_effect=asyncio.CancelledError()) + mock_prepare_worker.return_value = mock_worker_instance + + mock_plugin_store_instance = AsyncMock() + mock_plugin_store_instance.install_runtime = AsyncMock( + return_value={'runtime': 'single', 'installed': []}, + ) + mock_plugin_store_class.return_value = mock_plugin_store_instance + mock_build_plugin_store_config.return_value = { + 'base_url': 'http://sientia-plugin-store.svc.cluster.local', + 'owner': 'sientia', + 'repo': 'model-library-store', + 'branch': 'main', + 'username': 'gitea-user', + 'password': 'gitea-password', + 'pypi_index_url': 'http://library-distribution-server.library.svc.cluster.local:5000', + 'pypi_username': None, + 'pypi_password': None, + 'cache_ttl_seconds': None, + } + + mock_app_up = Mock() + mock_metrics.APP_UP.labels.return_value = mock_app_up + + with pytest.raises(SystemExit): + await main() + + assert mock_prepare_worker.call_count == 2 + assert mock_prepare_worker.call_args_list[0].kwargs['runtime'] == 'single' + assert mock_prepare_worker.call_args_list[1].kwargs['runtime'] == 'single' + + +@patch('model_manager.worker.worker.asyncio.run') +def test_main_entrypoint(mock_asyncio_run): + """Test the __main__ entrypoint.""" + # Import and execute the main block + with patch.object(sys, 'argv', ['worker.py']): + import model_manager.worker.worker as worker_module + + # Simulate running the module + worker_module.main = AsyncMock() + + # This would normally be called by asyncio.run(main()) + # We just verify the pattern is correct + assert callable(worker_module.main) + + +def test_worker_module_docstring(): + """Test that worker module has comprehensive documentation.""" + import model_manager.worker.worker as worker_module + + assert worker_module.__doc__ is not None + assert 'Temporal' in worker_module.__doc__ + assert 'worker' in worker_module.__doc__ + + +@patch('model_manager.worker.worker.start_http_server') +@patch('model_manager.worker.worker.metrics') +def test_start_prometheus_server_prints_success( + mock_metrics, mock_start_http_server, capsys, mock_env_vars, mock_logger +): + """Test that start_prometheus_server prints success message.""" + from model_manager.worker.worker import start_prometheus_server + + mock_app_up = Mock() + mock_metrics.APP_UP.labels.return_value = mock_app_up + + metadata: dict[str, str | None] = {'pod_id': 'test-pod-123', 'workflow_name': 'train_model'} + + start_prometheus_server(mock_logger, metadata) + + # Agora a mensagem é enviada via logger + mock_logger.custom_info.assert_called_once() + + +@patch('model_manager.worker.worker.start_http_server') +@patch('model_manager.worker.worker.metrics') +@patch('model_manager.worker.worker.os._exit') +def test_start_prometheus_server_prints_failure( + mock_exit, mock_metrics, mock_start_http_server, capsys, mock_env_vars, mock_logger +): + """Test that start_prometheus_server prints failure message.""" + from model_manager.worker.worker import start_prometheus_server + + mock_start_http_server.side_effect = Exception('Test error') + + metadata: dict[str, str | None] = {'pod_id': 'test-pod-123', 'workflow_name': 'train_model'} + + start_prometheus_server(mock_logger, metadata) + + # Agora o erro é logado via logger crítico + mock_logger.custom_critical.assert_called_once() + mock_exit.assert_called_once_with(1) diff --git a/tests/workflows/test_cleanup_files.py b/tests/workflows/test_cleanup_files.py new file mode 100644 index 0000000..7e84bcd --- /dev/null +++ b/tests/workflows/test_cleanup_files.py @@ -0,0 +1,30 @@ +"""Unit tests for the CleanupFiles workflow.""" + +from unittest.mock import AsyncMock, patch + +import pytest + + +@pytest.mark.asyncio +@patch('model_manager.workflows.cleanup_files.workflow') +async def test_cleanup_files_workflow(mock_workflow_module): + """Test the CleanupFiles workflow.""" + from model_manager.runtime_paths import REPORTS_TEMP_DIR + from model_manager.workflows.cleanup_files import CleanupFiles + + # Mock execute_activity_method + mock_workflow_module.execute_activity_method = AsyncMock() + + # Instantiate and run the workflow + workflow_instance = CleanupFiles() + await workflow_instance.run({}) + + # Verify that the activities were called with the correct parameters + calls = mock_workflow_module.execute_activity_method.call_args_list + assert len(calls) == 1 + + # Check cleanup_temp_directories call + local_call_args = calls[0][0][1] + assert local_call_args['temp_path'] == REPORTS_TEMP_DIR + assert local_call_args['metadata']['workflow_name'] == 'cleanup_files' + assert 'pod_id' in local_call_args['metadata'] diff --git a/tests/workflows/test_train_model.py b/tests/workflows/test_train_model.py new file mode 100644 index 0000000..2f94d5a --- /dev/null +++ b/tests/workflows/test_train_model.py @@ -0,0 +1,318 @@ +"""Unit tests for TrainModel workflow.""" + +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from temporalio.exceptions import ApplicationError + +from model_manager.utils.models.experiment_status import ExperimentStatus +from model_manager.utils.models.train_model_params import TrainModelParams + + +@pytest.fixture +def mock_train_params(): + """Minimal mock TrainModelParams.""" + params = Mock(spec=TrainModelParams) + params.experiment_run_id = 123 + params.bucket_name = 'test-bucket' + params.file_name = 'test-file.csv' + params.target_variable = 'target' + params.variable_columns = ['var1', 'var2'] + return params + + +@pytest.fixture +def sample_input_data(): + """Sample workflow input (IDs normalized in run()).""" + return { + 'experiment_run_id': 123, + 'target_variable': 'target', + 'variable_columns': ['var1', 'var2'], + 'train_size': 80, + 'bucket_name': 'test-bucket', + 'file_name': 'test-file.csv', + 'line_separator': ',', + 'decimal_separator': '.', + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', + 'shuffle': True, + 'random_state': 42, + 'model_name': 'Linear Regression', + 'model_type': 'linear_regression', + 'data_model_kwargs': {}, + 'model_kwargs': {}, + 'opt_params': {}, + 'val_file_name': None, + 'model_id': None, + 'model_metadata': {'schemas': {'components': {'schemas': {}}}}, + } + + +def test_validate_experiment_run_id_success(): + from model_manager.workflows.train_model import TrainModel + + wf = TrainModel() + assert wf._validate_experiment_run_id({'experiment_run_id': 123}) == 123 + + +def test_validate_experiment_run_id_string_numeric(): + from model_manager.workflows.train_model import TrainModel + + wf = TrainModel() + assert wf._validate_experiment_run_id({'experiment_run_id': '123'}) == 123 + + +def test_validate_experiment_run_id_missing(): + from model_manager.workflows.train_model import TrainModel + + with pytest.raises(ValueError, match='experiment_run_id is required'): + TrainModel()._validate_experiment_run_id({}) + + +def test_validate_experiment_run_id_invalid_type(): + from model_manager.workflows.train_model import TrainModel + + with pytest.raises(ValueError, match='must be an integer or numeric string'): + TrainModel()._validate_experiment_run_id({'experiment_run_id': 'not_int'}) + + +def test_extract_error_message_simple(): + from model_manager.workflows.train_model import TrainModel + + assert TrainModel()._extract_error_message(ValueError('x')) == 'x' + + +def test_extract_error_message_with_cause(): + from model_manager.workflows.train_model import TrainModel + + cause = ValueError('Root') + exc = RuntimeError('Outer') + exc.__cause__ = cause + msg = TrainModel()._extract_error_message(exc) + assert 'Outer' in msg and 'Root' in msg + + +def test_extract_error_message_empty_message(): + from model_manager.workflows.train_model import TrainModel + + out = TrainModel()._extract_error_message(ValueError('')) + assert 'ValueError' in out + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_validate_training_parameters_success(mock_wf, sample_input_data, mock_train_params): + from model_manager.workflows.train_model import TrainModel + + mock_wf.execute_activity_method = AsyncMock( + side_effect=[ + {'experiment_run_id': 123, 'model_metadata': {}}, + mock_train_params, + None, + ] + ) + meta = {'metadata': {'pod_id': 'p', 'experiment_run_id': 123}} + out = await TrainModel()._validate_training_parameters(sample_input_data, 123, meta) + assert out is mock_train_params + assert mock_wf.execute_activity_method.call_count == 3 + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_validate_training_parameters_load_fails(mock_wf, sample_input_data): + from model_manager.workflows.train_model import TrainModel + + mock_wf.execute_activity_method = AsyncMock(side_effect=[ValueError('load'), None]) + meta = {'metadata': {'pod_id': 'p', 'experiment_run_id': 123}} + with pytest.raises(ValueError, match='load'): + await TrainModel()._validate_training_parameters(sample_input_data, 123, meta) + assert mock_wf.execute_activity_method.call_count == 2 + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_train_model_success(mock_wf, mock_train_params): + from model_manager.workflows.train_model import TrainModel + + tr = {'run_name': 'rn', 'run_id': 'rid', 'run_dir': '/tmp/r'} + mock_wf.execute_activity_method = AsyncMock(side_effect=[tr, None]) + meta = {'metadata': {'pod_id': 'p', 'experiment_run_id': 123}} + out = await TrainModel()._train_model(mock_train_params, 123, meta) + assert out == tr + assert mock_wf.execute_activity_method.call_count == 2 + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_validate_training_parameters_logs_when_db_update_fails(mock_wf, sample_input_data): + """If persisting ORCHESTRATOR_VALIDATION_ERROR fails, workflow logs a warning.""" + from model_manager.workflows.train_model import TrainModel + + mock_wf.execute_activity_method = AsyncMock( + side_effect=[ValueError('validation'), RuntimeError('db')], + ) + mock_wf.logger = Mock() + meta = {'metadata': {'pod_id': 'p', 'experiment_run_id': 123}} + with pytest.raises(ValueError, match='validation'): + await TrainModel()._validate_training_parameters(sample_input_data, 123, meta) + mock_wf.logger.warning.assert_called_once() + assert mock_wf.execute_activity_method.call_count == 2 + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_train_model_logs_when_error_status_persist_fails(mock_wf, mock_train_params): + """If persisting TRAINING_ERROR fails, workflow logs a warning.""" + from model_manager.workflows.train_model import TrainModel + + mock_wf.execute_activity_method = AsyncMock( + side_effect=[RuntimeError('train'), RuntimeError('db')], + ) + mock_wf.logger = Mock() + meta = {'metadata': {'pod_id': 'p', 'experiment_run_id': 123}} + with pytest.raises(RuntimeError, match='train'): + await TrainModel()._train_model(mock_train_params, 123, meta) + mock_wf.logger.warning.assert_called_once() + assert mock_wf.execute_activity_method.call_count == 2 + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_train_model_failure_updates_db(mock_wf, mock_train_params): + from model_manager.workflows.train_model import TrainModel + + mock_wf.execute_activity_method = AsyncMock(side_effect=[RuntimeError('fail'), None]) + meta = {'metadata': {'pod_id': 'p', 'experiment_run_id': 123}} + with pytest.raises(RuntimeError, match='fail'): + await TrainModel()._train_model(mock_train_params, 123, meta) + assert mock_wf.execute_activity_method.call_count == 2 + err_call = mock_wf.execute_activity_method.call_args_list[1] + assert err_call[0][1]['status'] == ExperimentStatus.TRAINING_ERROR + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_cleanup_resources(mock_wf): + from model_manager.workflows.train_model import TrainModel + + mock_wf.execute_activity_method = AsyncMock(return_value=None) + meta = {'metadata': {'pod_id': 'p'}} + await TrainModel()._cleanup_resources('/tmp/x', meta) + assert mock_wf.execute_activity_method.call_count == 1 + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_cleanup_resources_none_skips(mock_wf): + from model_manager.workflows.train_model import TrainModel + + await TrainModel()._cleanup_resources(None, {'metadata': {}}) + mock_wf.execute_activity_method.assert_not_called() + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_run_success_six_activities(mock_wf, sample_input_data, mock_train_params): + from model_manager.workflows.train_model import TrainModel + + tr = {'run_name': 'rn', 'run_id': 'i', 'run_dir': '/tmp/t'} + mock_wf.execute_activity_method = AsyncMock( + side_effect=[ + {'x': 1}, + mock_train_params, + None, + tr, + None, + None, + ] + ) + result = await TrainModel().run(sample_input_data) + assert result == tr + assert mock_wf.execute_activity_method.call_count == 6 + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_run_validation_error(mock_wf, sample_input_data): + from model_manager.workflows.train_model import TrainModel + + mock_wf.execute_activity_method = AsyncMock(side_effect=[ValueError('bad'), None]) + with pytest.raises(ValueError, match='bad'): + await TrainModel().run(sample_input_data) + assert mock_wf.execute_activity_method.call_count == 2 + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_run_cleanup_failure_does_not_fail_workflow( + mock_wf, sample_input_data, mock_train_params +): + """After successful training, cleanup failure is logged, workflow still returns result.""" + from model_manager.workflows.train_model import TrainModel + + tr = {'run_name': 'rn', 'run_id': 'i', 'run_dir': '/tmp/t'} + mock_wf.execute_activity_method = AsyncMock( + side_effect=[ + {'x': 1}, + mock_train_params, + None, + tr, + None, + RuntimeError('cleanup'), + ] + ) + mock_wf.logger = Mock() + out = await TrainModel().run(sample_input_data) + assert out == tr + mock_wf.logger.warning.assert_called_once() + assert mock_wf.execute_activity_method.call_count == 6 + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_run_training_failure_skips_cleanup_activity( + mock_wf, sample_input_data, mock_train_params +): + """When train_model raises, train_result stays None and cleanup activity is not scheduled.""" + from model_manager.workflows.train_model import TrainModel + + mock_wf.execute_activity_method = AsyncMock( + side_effect=[ + {'x': 1}, + mock_train_params, + None, + RuntimeError('train failed'), + ] + ) + with pytest.raises(RuntimeError, match='train failed'): + await TrainModel().run(sample_input_data) + # validate (3) + train activity (1) + TRAINING_ERROR DB update (1); no cleanup (6th) when train_result is unset + assert mock_wf.execute_activity_method.call_count == 5 + + +@pytest.mark.asyncio +@patch('model_manager.workflows.train_model.workflow') +async def test_run_missing_experiment_run_id(mock_wf): + from model_manager.workflows.train_model import TrainModel + + mock_wf.logger = Mock() + with pytest.raises(ApplicationError, match='experiment_run_id is required'): + await TrainModel().run({}) + + +def test_module_constants(): + from model_manager.workflows.train_model import ( + TIMEOUT_DELETE_FILE, + TIMEOUT_TRAIN_MODEL, + TIMEOUT_VALIDATE_PARAMS, + database_retry_policy, + network_retry_policy, + no_retry_policy, + ) + + assert isinstance(TIMEOUT_VALIDATE_PARAMS, int) + assert no_retry_policy.maximum_attempts == 1 + assert network_retry_policy.maximum_attempts == 5 + assert database_retry_policy.maximum_attempts == 5 + assert TIMEOUT_TRAIN_MODEL == 2700 + assert TIMEOUT_DELETE_FILE == 120 diff --git a/validate.sh b/validate.sh new file mode 100755 index 0000000..96beac1 --- /dev/null +++ b/validate.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# Model Manager Code Validation Script +# This script runs all code quality checks before committing or deploying +# +# Usage: +# ./validate.sh # Run all checks including tests (default) +# ./validate.sh --no-tests # Skip unit tests +# ./validate.sh --skip-tests # Skip unit tests (alias) +# ./validate.sh --only-tests # Run only unit tests +# ./validate.sh --fix # Auto-fix formatting and linting, then run validations (no tests) + +set -e # Exit on any error + +# Parse command line arguments +RUN_TESTS=true +ONLY_TESTS=false +FIX_MODE=false + +for arg in "$@"; do + case $arg in + --no-tests|--skip-tests) + RUN_TESTS=false + shift + ;; + --only-tests) + ONLY_TESTS=true + shift + ;; + --fix) + FIX_MODE=true + RUN_TESTS=false + shift + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --no-tests, --skip-tests Skip unit tests (default: run tests)" + echo " --only-tests Run only unit tests" + echo " --fix Auto-fix formatting and linting, then run validations (no tests)" + echo " --help, -h Show this help message" + echo "" + exit 0 + ;; + *) + echo "Unknown option: $arg" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Model Manager - Code Validation Suite ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}" +echo "" + +# Handle --fix mode +if [ "$FIX_MODE" = true ]; then + echo -e "${BLUE}🔧 Running auto-fix mode...${NC}" + echo "" + + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${BLUE}▶ Auto-fixing code formatting (Ruff)${NC}" + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + ruff format model_manager/ tests/ + echo -e "${GREEN}✅ Code formatting applied${NC}" + echo "" + + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${BLUE}▶ Auto-fixing linting issues (Ruff)${NC}" + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + ruff check --fix model_manager/ tests/ + echo -e "${GREEN}✅ Linting fixes applied${NC}" + echo "" + + echo -e "${YELLOW}ℹ️ Now running validations (without tests)...${NC}" + echo "" +fi + +# Handle --only-tests mode +if [ "$ONLY_TESTS" = true ]; then + echo -e "${BLUE}🧪 Running only unit tests...${NC}" + echo "" +fi + +if [ "$RUN_TESTS" = false ] && [ "$ONLY_TESTS" = false ]; then + echo -e "${YELLOW}ℹ️ Unit tests will be skipped${NC}" + echo "" +fi + +# Check if virtual environment is activated +if [[ -z "${VIRTUAL_ENV}" ]] && [[ -z "${CONDA_DEFAULT_ENV}" ]]; then + echo -e "${YELLOW}⚠️ Warning: No virtual environment detected${NC}" + echo -e "${YELLOW} Consider activating your venv/conda environment${NC}" + echo "" +fi + +# Function to run a validation step +run_step() { + local step_name=$1 + local step_command=$2 + + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${BLUE}▶ ${step_name}${NC}" + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + + if eval "$step_command"; then + echo -e "${GREEN}✅ ${step_name} - PASSED${NC}" + echo "" + return 0 + else + echo -e "${RED}❌ ${step_name} - FAILED${NC}" + echo "" + return 1 + fi +} + +# Track failures +FAILED_STEPS=() + +# Handle --only-tests mode +if [ "$ONLY_TESTS" = true ]; then + # Step 5: Unit Tests (pytest) + if ! run_step "Unit Tests (pytest)" "pytest tests/ --cov=model_manager --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=80 -q"; then + FAILED_STEPS+=("Unit Tests") + fi +else + # Step 1: Code Formatting Check (Ruff) + if ! run_step "1. Code Formatting (Ruff)" "ruff format --check model_manager/ tests/"; then + FAILED_STEPS+=("Code Formatting") + fi + + # Step 2: Linting (Ruff) + if ! run_step "2. Code Linting (Ruff)" "ruff check model_manager/ tests/"; then + FAILED_STEPS+=("Linting") + fi + + # Step 3: Type Checking (mypy) + if ! run_step "3. Type Checking (mypy)" "mypy model_manager/"; then + FAILED_STEPS+=("Type Checking") + fi + + # Step 4: Security Analysis (Bandit) + if ! run_step "4. Security Analysis (Bandit)" "bandit -r model_manager/ -ll -q"; then + FAILED_STEPS+=("Security Analysis") + fi + + # Step 5: Unit Tests (pytest) + if [ "$RUN_TESTS" = true ]; then + if ! run_step "5. Unit Tests (pytest)" "pytest tests/ --cov=model_manager --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=80 -q"; then + FAILED_STEPS+=("Unit Tests") + fi + else + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${BLUE}▶ 5. Unit Tests (pytest)${NC}" + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${YELLOW}⏭️ Unit Tests - SKIPPED${NC}" + echo "" + fi +fi + +# Summary +echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Validation Summary ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}" +echo "" + +if [ ${#FAILED_STEPS[@]} -eq 0 ]; then + echo -e "${GREEN}✅ All validation checks passed!${NC}" + echo -e "${GREEN} Your code is ready for commit/deployment.${NC}" + echo "" + exit 0 +else + echo -e "${RED}❌ Validation failed for the following steps:${NC}" + for step in "${FAILED_STEPS[@]}"; do + echo -e "${RED} • ${step}${NC}" + done + echo "" + echo -e "${YELLOW}💡 Tips:${NC}" + echo -e "${YELLOW} • Run 'ruff format model_manager/ tests/' to auto-fix formatting${NC}" + echo -e "${YELLOW} • Run 'ruff check --fix model_manager/ tests/' to auto-fix linting issues${NC}" + echo -e "${YELLOW} • Review mypy errors and add type hints where needed${NC}" + echo -e "${YELLOW} • Check bandit warnings for security issues${NC}" + echo -e "${YELLOW} • Fix failing tests or improve test coverage${NC}" + echo "" + exit 1 +fi diff --git a/values.yaml b/values.yaml new file mode 100644 index 0000000..d1fe44f --- /dev/null +++ b/values.yaml @@ -0,0 +1,388 @@ +# +# Default values for sientia-model-manager using the sientia-module chart (0.6.x). +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +# + +projectName: &projectName "sientia-model-manager" + +# ----------------------------------------------------------------------------- +# Global configuration shared by all runtimes +# ----------------------------------------------------------------------------- +global: + # Namespace used by the chart. + namespace: sientia + + # ----------------------------------------------------------------------------- + # Image configuration (chart-level) + # ----------------------------------------------------------------------------- + # The sientia-module chart allows overriding the image used by all runtimes. + # Per requirement, we deploy using the sientia-module image v1.0.0. + image: + repository: aignosi.azurecr.io/sientia-module + pullPolicy: Always + tag: "1.0.2" + + # Common labels applied to pods (can be extended per project). + commonLabels: {} + + # Resources inherited by all runtimes unless overridden. + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Probes inherited by all runtimes unless overridden. + # More information: + # https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + livenessProbe: + exec: + command: + - python3 + - -c + - "import requests; requests.get('http://localhost:9090/metrics')" + initialDelaySeconds: 20 + periodSeconds: 30 + + readinessProbe: + exec: + command: + - python3 + - -c + - "import requests; requests.get('http://localhost:9090/metrics')" + initialDelaySeconds: 10 + periodSeconds: 15 + + # Autoscaling configuration inherited by all runtimes unless overridden. + # More information: + # https://kubernetes.io/docs/concepts/workloads/autoscaling/ + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + + # Environment variables shared by all runtimes. + env: + # Entrypoint variables + - name: GITHUB_REPO_URL + value: "git@github.com:Aignosi/sientia-dataops-model-manager.git" + - name: GITHUB_BRANCH + value: "release/SIENTIAPDE-1645" + - name: PYTHON_APP + value: "model_manager.worker.worker" + - name: PYPI_SERVER + value: "http://library-distribution-server.library.svc.cluster.local:5000" + + - name: POSTGRES_HOST + value: "paradedb-rw.paradedb.svc.cluster.local" + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_PASSWORD + value: "nFqc81y6kwmr2zuAIx43DhiOosFCVPpeEfTtTWZflkNjB2j1KtEeIANkhFR9mAX3" + - name: POSTGRES_DBNAME + value: "sientia-core-mlops-bff" + - name: POSTGRES_MIN_CONNECTIONS + value: "10" + - name: POSTGRES_MAX_CONNECTIONS + value: "30" + + - name: MLFLOW_URL + value: "http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local:80" + - name: MLFLOW_USERNAME + value: "aignosi" + - name: MLFLOW_PASSWORD + value: "1L0FP50j3ncp123" + + - name: LOG_LEVEL + value: "DEBUG" + - name: HTTP_METRICS_PORT + value: "9090" + - name: HTTP_SDK_METRICS_PORT + value: "9091" + + - name: TEMPORAL_HOST + value: "temporal-frontend.temporal.svc.cluster.local:7233" + - name: TEMPORAL_NAMESPACE + value: "model-manager" + - name: TRAIN_TASK_QUEUE + value: "train_model-queue" + - name: CLEANUP_TASK_QUEUE + value: "cleanup-queue" + - name: TEMPORAL_USE_TLS + value: "false" + + - name: STORE_BASE_URL + value: "http://gitea-http.gitea.svc.cluster.local:3000" + - name: STORE_OWNER + value: "aignosi" + - name: STORE_REPO + value: "suse-model-store" + - name: STORE_USERNAME + valueFrom: + secretKeyRef: + name: sientia-plugin-store-credentials + key: username + - name: STORE_PASSWORD + valueFrom: + secretKeyRef: + name: sientia-plugin-store-credentials + key: password + - name: STORE_CACHE_TTL_SECONDS + value: "3600" + + - name: MONGODB_USERNAME + value: "root" + - name: MONGODB_PASSWORD + value: "wKZDbMNU1c" + - name: MONGODB_URL + value: "my-release-mongodb.mongodb.svc.cluster.local:27017" + - name: MONGODB_DATABASE + value: "sientia" + - name: MONGODB_TTL_INDEX_HOURS + value: "1" + + - name: MINIO_ENDPOINT_URL + value: "http://minio.minio.svc.cluster.local:9000" + - name: MINIO_ACCESS_KEY + value: "model-training-user" + - name: MINIO_SECRET_KEY + value: "modelTrainingUser123" + - name: MINIO_REGION + value: "us-east-1" + - name: MINIO_SECURE + value: "false" + - name: MINIO_MAX_RETRY_ATTEMPTS + value: "3" + - name: MINIO_RETRY_MODE + value: "adaptive" + - name: MINIO_CONNECT_TIMEOUT + value: "10" + - name: MINIO_READ_TIMEOUT + value: "60" + - name: MINIO_DEFAULT_BUCKET + value: "model-training" + + - name: TIMEOUT_VALIDATE_PARAMS + value: "30" + - name: TIMEOUT_TRAIN_MODEL + value: "2700" + - name: TIMEOUT_DELETE_FILE + value: "120" + - name: TIMEOUT_UPDATE_DATABASE + value: "30" + + - name: CLEANUP_RETENTION_HOURS + value: "24" + - name: CLEANUP_DRY_RUN + value: "false" + - name: TIMEOUT_CLEANUP_MINIO + value: "300" + - name: TIMEOUT_CLEANUP_LOCAL + value: "120" + - name: MAX_KEYS_CLEANUP + value: "1000" + - name: DEFAULT_CLEANUP_BUCKET + value: "model-training" + + # Cleanup Schedule Configuration + - name: CLEANUP_SCHEDULE_ID + value: "cleanup-files-daily" + - name: CLEANUP_CRON + value: "0 0 * * *" # Midnight UTC + - name: CLEANUP_TIMEZONE + value: "UTC" + - name: CLEANUP_EXECUTION_TIMEOUT_HOURS + value: "1" + +# ----------------------------------------------------------------------------- +# Runtimes configuration +# ----------------------------------------------------------------------------- +# Each runtime inherits settings from `global` (resources, env, probes, autoscaling) +# unless overridden here. +runtimes: + basic: + # Replicas for this runtime. Replaces the old replicaCount. + replicas: 1 + xgboost: + replicas: 1 + +# ----------------------------------------------------------------------------- +# Chart-level configuration (applies to all runtimes) +# ----------------------------------------------------------------------------- + +# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: + - name: docker-hub-secret + +# This is to override the chart name. +nameOverride: *projectName +fullnameOverride: *projectName + +# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: *projectName + +# This is for setting Kubernetes Annotations to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} +# This is for setting Kubernetes Labels to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# Additional volumes on the output Deployment definition. +volumes: + - name: model-manager-runtime + emptyDir: + sizeLimit: 1Gi + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: + - name: model-manager-runtime + mountPath: "/var/lib/model-manager" + +# Deployment strategy configuration +# More information: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: Recreate + # rollingUpdate: + # maxSurge: 0 + # maxUnavailable: 1 + +# Number of old ReplicaSets to retain +revisionHistoryLimit: 2 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +services: + sdk-metrics: + enabled: true + type: ClusterIP + port: 9091 + targetPort: 9091 + name: sdk-metrics + metrics: + enabled: true + type: ClusterIP + port: 9090 + targetPort: 9090 + name: metrics + +# Configuração do ServiceMonitor para o Prometheus Operator +# ref: https://github.com/prometheus-operator/prometheus-operator +serviceMonitor: + # Se true, um recurso ServiceMonitor será criado. + enabled: true + # O intervalo no qual as métricas devem ser coletadas (ex: 30s, 1m). + endpoints: + - port: metrics + path: /metrics + interval: 30s + relabelings: [] + - port: sdk-metrics + path: /metrics + interval: 30s + relabelings: [] + additionalLabels: + release: kube-prometheus-stack + +ssh: + enabled: true + secretName: git-ssh-key-sientia-model-manager-worker + sshPath: /mnt/.ssh + knownHostsPath: /mnt/known_hosts + +# Configuração para dashboards do Grafana +grafanaDashboard: + # Habilita a criação de ConfigMaps para dashboards + enabled: true + # Namespace onde o Grafana está instalado (ajuste conforme seu ambiente) + namespace: monitoring + # Labels para que o sidecar do Grafana encontre os dashboards + labels: + grafana_dashboard: "1" + # Lista de dashboards para importar + dashboards: + - name: sientia-dataops-model-manager + title: "Sientia DataOps Model Manager" + uid: "sientia-dataops-model-manager" + folder: "Sientia" + jsonFile: "dashboards/sientia-dataops-model-manager.json" + overwrite: true # Sobrescreve dashboard se já existir + version: "1.0.0" # Version inicial do dashboard + +# Configuração para datasources do Grafana +grafanaDatasource: + # Habilita a criação de ConfigMap para datasources + enabled: false + # Namespace onde o Grafana está instalado + namespace: monitoring + # Labels para que o sidecar do Grafana encontre os datasources + labels: + grafana_datasource: "1" + # Lista de datasources para configurar + datasources: [] + # Exemplo de datasource: + # - name: Prometheus + # type: prometheus + # url: http://prometheus-server.monitoring.svc.cluster.local + # isDefault: true + # jsonData: + # timeInterval: "5s" + +# ----------------------------------------------------------------------------- +# Helm usage examples +# ----------------------------------------------------------------------------- +# kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password= +# +# helm upgrade --install sientia-model-manager sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.6.1 +# +# Global/runtimes layout note: +# - Shared configuration lives under `global` (env, probes, autoscaling, namespace). +# - Individual runtimes are defined under `runtimes`, each with its own `name` and `replicas`. +# - Runtimes inherit `global` settings unless overridden at the runtime level. +# +# kubectl create secret generic git-ssh-key-sientia-model-manager-worker \ +# --namespace sientia \ +# --from-file=ssh-privatekey=git_key \ +# --type=kubernetes.io/ssh-auth + +# kubectl create secret generic sientia-plugin-store-credentials \ +# --namespace sientia \ +# --from-literal=username= \ +# --from-literal=password= + +