From 50d0ea6f32710e41876df4d0a237399a9db404c2 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 17 Apr 2026 12:12:58 -0300 Subject: [PATCH] doc update --- README.md | 217 +++++++++++++++++++----------------------------- input-sample.md | 27 +++++- 2 files changed, 110 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index a50e437..d347045 100644 --- a/README.md +++ b/README.md @@ -79,26 +79,24 @@ An enterprise-grade ML model training orchestration platform built on Temporal. ## Features ### Core Functionality -- **ML Model Training Pipeline**: Complete training workflow from validation to deployment using MLFlow -- **Polynomial Regression Support**: Configurable polynomial degree with interaction terms and mandatory scaler validation -- **Automated File Cleanup**: Scheduled cleanup of stale files from local filesystem -- **Temporal Workflow Orchestration**: Robust workflow management with granular retry policies and fault tolerance -- **Parameter Validation**: Defense-in-depth validation with business rules and type checking -- **Experiment Tracking**: Comprehensive status tracking in PostgreSQL database -- **Resource Management**: Automatic cleanup of temporary files and storage -- **Comprehensive Monitoring**: Prometheus metrics and detailed logging for operational visibility -- **Scheduled Jobs**: Automated daily cleanup with configurable cron schedules +- **Model Training Orchestration**: Complete training lifecycle management from validation to MLFlow deployment +- **Model Agnostic Pipeline**: Support for multiple model types via dynamic runtime and wrapper installation +- **Temporal Workflow Management**: Robust orchestration with fault tolerance and granular retry policies +- **Multi-Level Validation**: Defense-in-depth parameter validation with type checking and business rules +- **Experiment Tracking**: Integrated status tracking and metadata persistence in PostgreSQL +- **Interactive ML Reporting**: Automated generation of rich HTML reports (Data Drift, Quality, Performance) using **Evidently** +- **Automated Resource Management**: Efficient handling of temporary local storage and artifact persistence +- **Prometheus Monitoring**: Comprehensive observability with real-time metrics and operational logging +- **Scheduled Maintenance**: Automated lifecycle jobs for filesystem hygiene and stale file cleanup ### Advanced Capabilities -- **Granular Retry Policies**: Different strategies for network, training, MLFlow, database, and filesystem operations -- **Configurable Timeouts**: Environment variable-based timeouts supporting large training files (up to 200MB) -- **Notification System**: Integrated alerting and notification management via MongoDB -- **Scalable Architecture**: Kubernetes-ready deployment with horizontal scaling support -- **MLFlow Integration**: Seamless model and artifact persistence to MLFlow tracking server -- **Per-Variable Lag Configuration**: Flexible lag settings for each variable independently -- **Date Range Filtering**: Filter training data by start/end dates and removed intervals -- **NaN Treatment Options**: Configurable handling of missing values (drop, linear interpolation) -- **RCE Drift Metrics**: Reduced Coulomb Energy metrics for drift detection +- **Dynamic Runtime Provisioning**: Automated installation of required model runtimes from the Plugin Store +- **Granular Retry Policies**: Tailored strategies for network, MLFlow, database, and filesystem operations +- **Scalable Infrastructure**: Kubernetes-ready design with support for horizontal scaling and poller autoscaling +- **Secure Configuration**: Environment-driven connection management with fallback to sensible defaults +- **Notification Framework**: Multi-channel alerting and event notification via MongoDB integration +- **High-Performance Data Loading**: Optimized MinIO connectivity supporting large training datasets (up to 200MB) +- **Extensible Architecture**: Plugin-based system for easy integration of new models and preprocessing logic ### Development & Quality Assurance - **Code Quality Tools**: Ruff (linting/formatting), mypy (type checking), Bandit (security analysis) @@ -150,6 +148,7 @@ The Model Manager system uses a Temporal-based workflow architecture with clear - Task queue configuration and load balancing - Prometheus metrics server initialization - Notification handler setup and configuration + - **Runtime Installation**: Automatically installs the required model runtime from the Plugin Store - **Key Features**: - Automatic scaling with `PollerBehaviorAutoscaling` - Health check endpoints for Kubernetes liveness/readiness probes @@ -181,10 +180,9 @@ The Model Manager system uses a Temporal-based workflow architecture with clear - Receives pre-downloaded files (BytesIO) to avoid memory leaks - Returns success/failure status with TrainModelResult or error message - No exception raising on failure - allows workflow to handle errors gracefully - - Integration with TrainingRepository for business logic separation + - Integration with `DataManagerRepository` for data processing and report generation - MLFlow model saving and artifact management - - **Polynomial Regression**: Support for configurable degree and interaction terms - - **Training Predictions**: Calculates y_train_pred before denormalization for accurate metrics + - Calculates training predictions and performance metrics for reporting - **Cleanup**: Local directory cleanup operations - `cleanup_temp_directories()`: Cleans local temporary directories - Configurable retention period (default: 24 hours) @@ -200,13 +198,12 @@ The Model Manager system uses a Temporal-based workflow architecture with clear #### **Data Services (`model_manager/utils/`)** - **Connectors Config**: Environment variable-based configuration management -- **Repository**: Data access layer for training and MLFlow operations - - `training_repository.py`: Training business logic and operations - - `model_repository.py`: MLFlow artifact generation and model persistence +- **Repository**: Data access layer for training and artifact operations + - `data_manager_repository.py`: Core data loading, feature preparation, metrics calculation, and report generation. - **Models**: Data models and schemas - - `train_model_params.py`: Training parameters model with comprehensive validation - - `train_model_result.py`: Training result model (includes y_train_pred) - - `experiment_status.py`: Experiment status enum + - `train_model_params.py`: Training parameters model with comprehensive validation and 7 business rules. + - `train_model_result.py`: Training result model containing processed data, metrics, and artifact paths. + - `experiment_status.py`: Experiment status enum for tracking workflow progress. - **Key Features**: - Environment variable-based configuration with sensible defaults - Connection pool management and optimization @@ -263,17 +260,18 @@ The **TrainModel** workflow orchestrates the complete ML model training pipeline #### Execution Flow 1. **Validate Experiment Run ID**: Critical validation before any DB updates -2. **Validate Training Parameters**: Type checking + business rules validation -3. **Train Model**: Execute ML model training with validated parameters -4. **Save to MLFlow**: Save trained model and artifacts to MLFlow -5. **Cleanup Resources**: Delete temporary local directories +2. **Load Model Metadata**: Fetch model schemas and metadata from the Plugin Store +3. **Validate Training Parameters**: Type checking + business rules validation +4. **Train Model**: Execute ML model training with validated parameters and data +5. **Save to MLFlow**: Save trained model and artifacts to MLFlow +6. **Cleanup Resources**: Delete temporary local directories #### Key Features - **Granular Retry Policies**: Different strategies for network, training, MLFlow, database, and filesystem operations - **Configurable Timeouts**: Environment variable-based timeouts supporting files up to 200MB - **Idempotent Cleanup**: Safe replay with Temporal workflow replay mechanism - **Structured Logging**: Rich context in error messages for debugging -- **Business Validation**: 10 business rules including range checks, consistency validation, and data integrity +- **Business Validation**: 7 business rules including range checks, consistency validation, and dynamic schema-based validation #### Train model workflow input (sample) @@ -286,14 +284,15 @@ When starting the workflow from a Temporal client, use the same **task queue** a #### Architecture Diagram ```mermaid flowchart TD - A[1. validate_experiment_run_id] --> B[2. validate_train_params] - B --> C[3. train_model] - C --> D[4. cleanup_run_directory] + A[1. validate_experiment_run_id] --> B[2. load_model_metadata] + B --> C[3. validate_train_params] + C --> D[4. train_model] + D --> E[5. cleanup_resources] - B -.-> DB[(PostgreSQL)] - C -.-> Training[ML Training] - C -.-> MLFlow[MLFlow] - D -.-> FS[Filesystem] + C -.-> DB[(PostgreSQL)] + D -.-> Training[ML Training] + D -.-> MLFlow[MLFlow] + E -.-> FS[Filesystem] ``` #### Retry Strategies @@ -310,13 +309,15 @@ The workflow implements 5 different retry policies optimized for each operation `TrainModelParams.validate_business_rules()` runs after type coercion. Notable checks: -1. **train_size**: Between 10 and 100 (percent). -2. **variable_columns**: Non-empty list. -3. **model_metadata**: Required (non-empty) for validation to succeed; may include JSON Schema definitions under `model_metadata.schemas.components.schemas` for `data_model`, `model`, and `opt_params` when you want schema validation of the corresponding kwargs. -4. **target_variable**, **bucket_name**, **file_name**, **model_name**: Non-empty strings (no whitespace-only values). -5. **date_format**: When set, must be an allowed frontend date format (see `validate_frontend_date_format`). +1. **train_size**: Must be between 10 and 100 (percent). +2. **variable_columns**: Must be a non-empty list. +3. **model_metadata**: Required (must be loaded before validation) to provide schemas for keyword arguments. +4. **Dynamic Kwargs Validation**: `data_model_kwargs`, `model_kwargs`, and `opt_params` are validated against JSON Schemas provided in `model_metadata` (if present) using `Draft202012Validator`. +5. **Required Strings**: `target_variable`, `bucket_name`, `file_name`, and `model_name` cannot be empty or whitespace. +6. **date_format**: If provided, must match one of the allowed frontend formats (e.g., `yyyy-MM-dd HH:mm:ss`). +7. **experiment_run_id**: Must be a valid integer or numeric string. -Model-specific rules (for example polynomial degree and scaler requirements) live in the training stack and integration scenarios; see `docs/test-scenarios/` and `scripts/run_training_test.py` for scenario-based examples. +Model-specific rules live in the training stack and integration scenarios; see `docs/test-scenarios/` and `scripts/run_training_test.py` for scenario-based examples. ### Cleanup Files Workflow (`cleanup_files.py`) @@ -855,61 +856,29 @@ pytest tests/workflows/test_train_model.py The project includes integration tests that validate the complete training workflow against a running Temporal cluster. These tests use JSON-based scenario files for easy configuration and maintenance. -#### Running Integration Tests +#### Running Training Smoke Tests + +1. Open `scripts/run_training_test.py` in your IDE. +2. Use the **"Run Cell"** or **"Run Below"** functionality (requires the Python/Jupyter extension). +3. The script will: + - Load configuration from `.env` + - Optionally clean up previous test runs in the database + - Insert a new `experiment_run` record + - Upload a sample dataset to MinIO + - Start the `train_model` workflow and wait for completion ```bash -# List available scenarios -python scripts/run_training_test.py --list - -# Run a specific test scenario -python scripts/run_training_test.py --scenario 01-linear-regression-basic - -# Run with custom CSV data file -python scripts/run_training_test.py --scenario 03-polynomial-regression-degree2 --csv /path/to/data.csv - -# Run ALL scenarios sequentially with summary report -python scripts/run_training_test.py --all - -# Run all scenarios with custom CSV -python scripts/run_training_test.py --all --csv docs/custom-data.csv +# Configuration for local testing is managed via the .env file +# Run the cells in scripts/run_training_test.py for end-to-end validation ``` -#### Batch Execution Output +#### Manual Cleanup Test -When running all scenarios with `--all`, the script provides: -- Progress indicators for each scenario (`[1/10] Running scenario: ...`) -- Status symbols (✓ for passed, ✗ for failed) -- Final summary with total/passed/failed counts -- Detailed error messages for failed scenarios -- Exit code 0 if all pass, 1 if any fail +For manual verification of the file cleanup logic, use the provided utility script: -Example output: -``` -Running 10 scenarios... - -[1/10] Running scenario: 01-linear-regression-basic - Loaded scenario: 01-linear-regression-basic - Uploaded CSV to MinIO: test-model-data-20231219-120000.csv - Created experiment_run with ID: 42 - Workflow started: train-model-test-abc123 -[1/10] ✓ 01-linear-regression-basic - -... - -============================================================ -SUMMARY -============================================================ -Total: 10 | Passed: 9 | Failed: 1 -============================================================ - -✓ PASSED: - - 01-linear-regression-basic - - 02-linear-regression-with-scaler - ... - -✗ FAILED: - - 05-linear-regression-with-lags - Error: Failed to start Temporal workflow: connection refused +```bash +# Run once to clean up stale local directories +python scripts/run_cleanup_test.py ``` #### Test Scenarios @@ -934,33 +903,27 @@ Test scenarios are defined as JSON files in `docs/test-scenarios/`. Each scenari ```json { - "_description": "Human-readable description of the scenario", - "experimentName": "test-experiment-name", - "username": "user@example.com", - "modelName": "Linear Regression", - "targetVariable": "target_column_name", - "variableColumns": ["feature1", "feature2"], - "lagTrain": {"feature1": 0, "feature2": 0}, - "lagVal": {"feature1": 0, "feature2": 0}, - "remStaticWin": false, - "staticThreshold": null, - "lowLim": {}, - "uppLim": {}, - "window": 0, - "useScaler": false, - "includeAr": false, - "trainSize": 80, + "experiment_run_id": 1001, + "variable_columns": ["feature_a", "feature_b"], + "target_variable": "target", + "bucket_name": "model-training", + "file_name": "training_data.csv", + "line_separator": ",", + "decimal_separator": ".", + "train_size": 80, "shuffle": true, - "lineSeparator": ",", - "decimalSeparator": ".", - "removedIntervals": [], - "degree": 1, - "interactionOnly": false, - "nanTreatment": "drop", - "startDate": null, - "endDate": null, - "scalerName": "None", - "supportFilters": {} + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": {"feature_a": 0, "feature_b": 0}, + "lag_val": {"feature_a": 0, "feature_b": 0}, + "nan_treatment": "drop" + }, + "model_kwargs": { + "degree": 1, + "scaler_name": "Standard Scaler" + }, + "opt_params": {} } ``` @@ -1189,22 +1152,16 @@ sientia-dataops-model-manager/ │ │ │ ├── train_model_result.py # Training result model │ │ │ └── experiment_status.py # Experiment status enum │ │ └── repository/ # Data access layer -│ │ ├── training_repository.py # Training business logic -│ │ ├── model_repository.py # MLFlow artifact management -│ │ └── storage_repository.py # MinIO storage operations +│ │ └── data_manager_repository.py # Core data logic & report generation │ ├── sientia/ # Sientia-specific implementations │ │ ├── __init__.py │ │ ├── exceptions.py # Custom exceptions │ │ ├── metrics.py # Business metrics (includes RCE drift detection) -│ │ ├── models.py # ML model implementations (Linear & Polynomial Regression) -│ │ ├── model_serving.py # Model serving utilities -│ │ ├── reports.py # Report generation -│ │ └── utils.py # Utility functions +│ │ ├── reports.py # Report generation logic │ ├── reports/ # Report templates and temporary files -│ │ ├── header.html # HTML report header template │ │ └── temp/ # Temporary report files (cleaned up automatically) │ ├── metrics.py # Prometheus metrics definitions -│ └── __init__.py +│ └── runtime_paths.py # Runtime directory management ├── scripts/ # Test and utility scripts │ ├── run_cleanup_test.py # Manual cleanup workflow test │ └── run_training_test.py # Training test with scenario support (--all for batch) @@ -1267,7 +1224,7 @@ sientia-dataops-model-manager/ - Verify Temporal server is running and reachable at `TEMPORAL_HOST` - Check namespace configuration and permissions - Review server logs for connection issues - - If you see **308 Permanent Redirect** or **invalid compression flag** on connect, the endpoint likely expects **TLS** while `TEMPORAL_USE_TLS` is `false`. Set `TEMPORAL_USE_TLS=true` and point `TEMPORAL_HOST` at the correct TLS gRPC address (host and port depend on your ingress or load balancer) + - If you see **308 Permanent Redirect** or **invalid compression flag** on connect, the endpoint likely expects **TLS** while `TEMPORAL_USE_TLS` is `false`. Set `TEMPORAL_USE_TLS=true` and point `TEMPORAL_HOST` at the correct TLS gRPC address. 2. **MLFlow connection issues** - Verify MLFlow server is running and accessible diff --git a/input-sample.md b/input-sample.md index 0ca71bb..ca6b53d 100644 --- a/input-sample.md +++ b/input-sample.md @@ -19,7 +19,7 @@ VALUES( now(), 'model-training', 'training_data.csv', - '{"experiment_run_id":1001,"variable_columns":["feature_a","feature_b"],"target_variable":"target","bucket_name":"model-training","file_name":"training_data.csv","line_separator":",","decimal_separator":".","train_size":80,"shuffle":true,"model_name":"Linear Regression","model_type":"linear_regression","data_model_kwargs":{},"model_kwargs":{},"opt_params":{},"model_metadata":{"schemas":{"components":{"schemas":{}}}}}', + '{"experiment_run_id":1001,"variable_columns":["feature_a","feature_b"],"target_variable":"target","bucket_name":"model-training","file_name":"training_data.csv","line_separator":",","decimal_separator":".","train_size":80,"shuffle":true,"model_name":"Linear Regression","model_type":"linear_regression","data_model_kwargs":{"lag_train":{"feature_a":0,"feature_b":0},"lag_val":{"feature_a":0,"feature_b":0},"nan_treatment":"drop"},"model_kwargs":{"degree":1,"scaler_name":"Standard Scaler"},"opt_params":{}}', null ); ``` @@ -48,9 +48,28 @@ Keys match `TrainModelParams.from_dict` in `model_manager/utils/models/train_mod "random_state": 42, "model_name": "test-runtime-linear-regression-model", "model_type": "linear_regression", - "model_id": 1001, - "data_model_kwargs": {}, - "model_kwargs": {}, + "data_model_kwargs": { + "lag_train": { + "feature_a": 0, + "feature_b": 0 + }, + "lag_val": { + "feature_a": 0, + "feature_b": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "Standard Scaler" + }, "opt_params": {} } ```