From 06fd08dc708f6bd4df2b75341d31c8de24fa06f3 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Thu, 18 Dec 2025 17:05:10 -0300 Subject: [PATCH] SIENTIAPDE-1430: Introduce comprehensive integration testing with JSON-based scenarios and detailed README documentation. Enhance training workflow to support advanced model configurations, including polynomial regression with mandatory scaler validation. Ensure robust prediction handling by calculating training predictions (y_train_pred) before denormalization and automatically configuring datetime indices for time-series operations. --- README.md | 88 ++++++++++ .../01-linear-regression-basic.json | 28 ++++ .../02-linear-regression-with-scaler.json | 28 ++++ .../03-polynomial-regression-degree2.json | 28 ++++ .../04-polynomial-regression-degree3.json | 28 ++++ .../05-linear-regression-with-lags.json | 28 ++++ ...6-linear-regression-nan-interpolation.json | 28 ++++ ...near-regression-static-window-removal.json | 28 ++++ .../08-linear-regression-with-limits.json | 28 ++++ ...lynomial-degree2-with-scaler-and-lags.json | 28 ++++ .../10-linear-regression-with-ar.json | 28 ++++ .../utils/models/train_model_params.py | 6 + .../utils/models/train_model_result.py | 2 + .../utils/repository/model_repository.py | 8 +- .../utils/repository/training_repository.py | 79 +++++++++ scripts/run_training_test.py | 155 +++++++++++++++--- tests/utils/models/test_train_model_result.py | 5 +- .../repository/test_training_repository.py | 45 ++++- 18 files changed, 631 insertions(+), 37 deletions(-) create mode 100644 docs/test-scenarios/01-linear-regression-basic.json create mode 100644 docs/test-scenarios/02-linear-regression-with-scaler.json create mode 100644 docs/test-scenarios/03-polynomial-regression-degree2.json create mode 100644 docs/test-scenarios/04-polynomial-regression-degree3.json create mode 100644 docs/test-scenarios/05-linear-regression-with-lags.json create mode 100644 docs/test-scenarios/06-linear-regression-nan-interpolation.json create mode 100644 docs/test-scenarios/07-linear-regression-static-window-removal.json create mode 100644 docs/test-scenarios/08-linear-regression-with-limits.json create mode 100644 docs/test-scenarios/09-polynomial-degree2-with-scaler-and-lags.json create mode 100644 docs/test-scenarios/10-linear-regression-with-ar.json diff --git a/README.md b/README.md index 08a5946..60fcbc7 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,12 @@ An enterprise-grade ML model training orchestration platform built on Temporal. - [Testing](#testing) - [Test Structure](#test-structure) - [Test Execution](#test-execution) + - [Integration Tests](#integration-tests) + - [Running Integration Tests](#running-integration-tests) + - [Test Scenarios](#test-scenarios) + - [Scenario File Structure](#scenario-file-structure) + - [Creating New Scenarios](#creating-new-scenarios) + - [Important Validations](#important-validations) - [Monitoring and Metrics](#monitoring-and-metrics) - [Application Health Metrics](#application-health-metrics) - [Training Metrics](#training-metrics) @@ -871,6 +877,88 @@ pytest tests/activities/test_training.py pytest tests/workflows/test_train_model.py ``` +### Integration Tests + +The project includes integration tests that validate the complete training workflow against a running Temporal cluster. These tests use JSON-based scenario files for easy configuration and maintenance. + +#### Running Integration Tests + +```bash +# Run a specific test scenario +python scripts/run_training_test.py --scenario 01-linear-regression-basic + +# Run with custom data file +python scripts/run_training_test.py --scenario 03-polynomial-regression-degree2 --data-file /path/to/data.csv + +# List available scenarios +ls docs/test-scenarios/ +``` + +#### Test Scenarios + +Test scenarios are defined as JSON files in `docs/test-scenarios/`. Each scenario configures a complete training workflow with specific parameters: + +| Scenario | Description | Key Features | +|----------|-------------|--------------| +| `01-linear-regression-basic` | Basic linear regression | No scaler, no lags | +| `02-linear-regression-with-scaler` | Linear regression with normalization | Standard Scaler enabled | +| `03-polynomial-regression-degree2` | Polynomial regression (degree 2) | Requires scaler (mandatory) | +| `04-polynomial-regression-degree3` | Polynomial regression (degree 3) | Requires scaler (mandatory) | +| `05-linear-regression-with-lags` | Linear regression with lag features | Lag train/val configuration | +| `06-linear-regression-nan-interpolation` | Linear regression with NaN handling | `nanTreatment: "interpolate"` | +| `07-linear-regression-static-window-removal` | Linear regression with static window removal | `remStaticWin: true` | +| `08-linear-regression-with-limits` | Linear regression with variable limits | `lowLim`/`uppLim` configuration | +| `09-polynomial-degree2-with-scaler-and-lags` | Complete polynomial scenario | Scaler + lags + degree 2 | +| `10-linear-regression-with-ar` | Linear regression with autoregressive variable | `includeAr: true` | + +#### Scenario File Structure + +```json +{ + "_description": "Human-readable description of the scenario", + "experimentName": "test-experiment-name", + "username": "user@example.com", + "modelName": "Linear Regression", + "targetVariable": "target_column_name", + "variableColumns": ["feature1", "feature2"], + "lagTrain": {"feature1": 0, "feature2": 0}, + "lagVal": {"feature1": 0, "feature2": 0}, + "remStaticWin": false, + "lowLim": {}, + "uppLim": {}, + "window": 0, + "useScaler": false, + "includeAr": false, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "removedIntervals": [], + "degree": 1, + "interactionOnly": false, + "nanTreatment": "drop", + "startDate": null, + "endDate": null, + "scalerName": "None", + "supportFilters": {} +} +``` + +#### Creating New Scenarios + +1. Copy an existing scenario file as a template +2. Modify parameters according to your test case +3. Save with a descriptive name: `XX-description.json` +4. Run with: `python scripts/run_training_test.py --scenario XX-description` + +#### Important Validations + +The training workflow enforces several business rules: + +- **Polynomial Regression requires Scaler**: Models with `degree > 1` must have `useScaler: true` and a valid `scalerName` to prevent numerical overflow +- **Static Window Removal requires DatetimeIndex**: Scenarios with `remStaticWin: true` require data with a timestamp column for the `TimeSeriesDiscontinuityAnalyzer` +- **Variable Limits Consistency**: `lowLim` and `uppLim` must have matching keys, and `lowLim[key] < uppLim[key]` for all variables + ## Monitoring and Metrics The Model Manager system exposes comprehensive Prometheus metrics for operational visibility and performance monitoring: diff --git a/docs/test-scenarios/01-linear-regression-basic.json b/docs/test-scenarios/01-linear-regression-basic.json new file mode 100644 index 0000000..1336f38 --- /dev/null +++ b/docs/test-scenarios/01-linear-regression-basic.json @@ -0,0 +1,28 @@ +{ + "_description": "Cenário básico de regressão linear sem scaler", + "experimentName": "test-linear-regression-basic", + "username": "bruno.domingues@aignosi.com.br", + "modelName": "Linear Regression", + "targetVariable": "03CV020/CORRENTE_N_M1_PV(Value)", + "variableColumns": ["303-WIT-200(Value)"], + "lagTrain": {"303-WIT-200(Value)": 0}, + "lagVal": {"303-WIT-200(Value)": 0}, + "remStaticWin": false, + "lowLim": {}, + "uppLim": {}, + "window": 0, + "useScaler": false, + "includeAr": false, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "removedIntervals": [], + "degree": 1, + "interactionOnly": false, + "nanTreatment": "drop", + "startDate": null, + "endDate": null, + "scalerName": "None", + "supportFilters": {} +} diff --git a/docs/test-scenarios/02-linear-regression-with-scaler.json b/docs/test-scenarios/02-linear-regression-with-scaler.json new file mode 100644 index 0000000..ac9cb3d --- /dev/null +++ b/docs/test-scenarios/02-linear-regression-with-scaler.json @@ -0,0 +1,28 @@ +{ + "_description": "Regressão linear com Standard Scaler habilitado", + "experimentName": "test-linear-regression-scaler", + "username": "bruno.domingues@aignosi.com.br", + "modelName": "Linear Regression", + "targetVariable": "03CV020/CORRENTE_N_M1_PV(Value)", + "variableColumns": ["303-WIT-200(Value)"], + "lagTrain": {"303-WIT-200(Value)": 0}, + "lagVal": {"303-WIT-200(Value)": 0}, + "remStaticWin": false, + "lowLim": {}, + "uppLim": {}, + "window": 0, + "useScaler": true, + "includeAr": false, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "removedIntervals": [], + "degree": 1, + "interactionOnly": false, + "nanTreatment": "drop", + "startDate": null, + "endDate": null, + "scalerName": "Standard Scaler", + "supportFilters": {} +} diff --git a/docs/test-scenarios/03-polynomial-regression-degree2.json b/docs/test-scenarios/03-polynomial-regression-degree2.json new file mode 100644 index 0000000..392ee67 --- /dev/null +++ b/docs/test-scenarios/03-polynomial-regression-degree2.json @@ -0,0 +1,28 @@ +{ + "_description": "Regressão polinomial de grau 2 com scaler (obrigatório para evitar overflow)", + "experimentName": "test-polynomial-degree2", + "username": "bruno.domingues@aignosi.com.br", + "modelName": "Polynomial Regression", + "targetVariable": "03CV020/CORRENTE_N_M1_PV(Value)", + "variableColumns": ["303-WIT-200(Value)"], + "lagTrain": {"303-WIT-200(Value)": 0}, + "lagVal": {"303-WIT-200(Value)": 0}, + "remStaticWin": false, + "lowLim": {}, + "uppLim": {}, + "window": 0, + "useScaler": true, + "includeAr": false, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "removedIntervals": [], + "degree": 2, + "interactionOnly": false, + "nanTreatment": "drop", + "startDate": null, + "endDate": null, + "scalerName": "Standard Scaler", + "supportFilters": {} +} diff --git a/docs/test-scenarios/04-polynomial-regression-degree3.json b/docs/test-scenarios/04-polynomial-regression-degree3.json new file mode 100644 index 0000000..342da07 --- /dev/null +++ b/docs/test-scenarios/04-polynomial-regression-degree3.json @@ -0,0 +1,28 @@ +{ + "_description": "Regressão polinomial de grau 3 com scaler", + "experimentName": "test-polynomial-degree3", + "username": "bruno.domingues@aignosi.com.br", + "modelName": "Polynomial Regression", + "targetVariable": "03CV020/CORRENTE_N_M1_PV(Value)", + "variableColumns": ["303-WIT-200(Value)"], + "lagTrain": {"303-WIT-200(Value)": 0}, + "lagVal": {"303-WIT-200(Value)": 0}, + "remStaticWin": false, + "lowLim": {}, + "uppLim": {}, + "window": 0, + "useScaler": true, + "includeAr": false, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "removedIntervals": [], + "degree": 3, + "interactionOnly": false, + "nanTreatment": "drop", + "startDate": null, + "endDate": null, + "scalerName": "Standard Scaler", + "supportFilters": {} +} diff --git a/docs/test-scenarios/05-linear-regression-with-lags.json b/docs/test-scenarios/05-linear-regression-with-lags.json new file mode 100644 index 0000000..a07a2cb --- /dev/null +++ b/docs/test-scenarios/05-linear-regression-with-lags.json @@ -0,0 +1,28 @@ +{ + "_description": "Regressão linear com lags de treino e validação", + "experimentName": "test-linear-with-lags", + "username": "bruno.domingues@aignosi.com.br", + "modelName": "Linear Regression", + "targetVariable": "03CV020/CORRENTE_N_M1_PV(Value)", + "variableColumns": ["303-WIT-200(Value)"], + "lagTrain": {"303-WIT-200(Value)": 5}, + "lagVal": {"303-WIT-200(Value)": 3}, + "remStaticWin": false, + "lowLim": {}, + "uppLim": {}, + "window": 0, + "useScaler": false, + "includeAr": false, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "removedIntervals": [], + "degree": 1, + "interactionOnly": false, + "nanTreatment": "drop", + "startDate": null, + "endDate": null, + "scalerName": "None", + "supportFilters": {} +} diff --git a/docs/test-scenarios/06-linear-regression-nan-interpolation.json b/docs/test-scenarios/06-linear-regression-nan-interpolation.json new file mode 100644 index 0000000..fe48f73 --- /dev/null +++ b/docs/test-scenarios/06-linear-regression-nan-interpolation.json @@ -0,0 +1,28 @@ +{ + "_description": "Regressão linear com tratamento de NaN por interpolação linear", + "experimentName": "test-linear-nan-interpolation", + "username": "bruno.domingues@aignosi.com.br", + "modelName": "Linear Regression", + "targetVariable": "03CV020/CORRENTE_N_M1_PV(Value)", + "variableColumns": ["303-WIT-200(Value)"], + "lagTrain": {"303-WIT-200(Value)": 0}, + "lagVal": {"303-WIT-200(Value)": 0}, + "remStaticWin": false, + "lowLim": {}, + "uppLim": {}, + "window": 0, + "useScaler": false, + "includeAr": false, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "removedIntervals": [], + "degree": 1, + "interactionOnly": false, + "nanTreatment": "linear interpolation", + "startDate": null, + "endDate": null, + "scalerName": "None", + "supportFilters": {} +} diff --git a/docs/test-scenarios/07-linear-regression-static-window-removal.json b/docs/test-scenarios/07-linear-regression-static-window-removal.json new file mode 100644 index 0000000..6dbb4b9 --- /dev/null +++ b/docs/test-scenarios/07-linear-regression-static-window-removal.json @@ -0,0 +1,28 @@ +{ + "_description": "Regressão linear com remoção de janelas estáticas", + "experimentName": "test-linear-static-removal", + "username": "bruno.domingues@aignosi.com.br", + "modelName": "Linear Regression", + "targetVariable": "03CV020/CORRENTE_N_M1_PV(Value)", + "variableColumns": ["303-WIT-200(Value)"], + "lagTrain": {"303-WIT-200(Value)": 0}, + "lagVal": {"303-WIT-200(Value)": 0}, + "remStaticWin": true, + "lowLim": {}, + "uppLim": {}, + "window": 10, + "useScaler": false, + "includeAr": false, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "removedIntervals": [], + "degree": 1, + "interactionOnly": false, + "nanTreatment": "drop", + "startDate": null, + "endDate": null, + "scalerName": "None", + "supportFilters": {} +} diff --git a/docs/test-scenarios/08-linear-regression-with-limits.json b/docs/test-scenarios/08-linear-regression-with-limits.json new file mode 100644 index 0000000..7fb65db --- /dev/null +++ b/docs/test-scenarios/08-linear-regression-with-limits.json @@ -0,0 +1,28 @@ +{ + "_description": "Regressão linear com limites inferior e superior para variáveis", + "experimentName": "test-linear-with-limits", + "username": "bruno.domingues@aignosi.com.br", + "modelName": "Linear Regression", + "targetVariable": "03CV020/CORRENTE_N_M1_PV(Value)", + "variableColumns": ["303-WIT-200(Value)"], + "lagTrain": {"303-WIT-200(Value)": 0}, + "lagVal": {"303-WIT-200(Value)": 0}, + "remStaticWin": false, + "lowLim": {"303-WIT-200(Value)": 0.0}, + "uppLim": {"303-WIT-200(Value)": 1000.0}, + "window": 0, + "useScaler": false, + "includeAr": false, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "removedIntervals": [], + "degree": 1, + "interactionOnly": false, + "nanTreatment": "drop", + "startDate": null, + "endDate": null, + "scalerName": "None", + "supportFilters": {} +} diff --git a/docs/test-scenarios/09-polynomial-degree2-with-scaler-and-lags.json b/docs/test-scenarios/09-polynomial-degree2-with-scaler-and-lags.json new file mode 100644 index 0000000..14f7199 --- /dev/null +++ b/docs/test-scenarios/09-polynomial-degree2-with-scaler-and-lags.json @@ -0,0 +1,28 @@ +{ + "_description": "Cenário completo: regressão polinomial grau 2 com scaler e lags", + "experimentName": "test-polynomial-complete", + "username": "bruno.domingues@aignosi.com.br", + "modelName": "Polynomial Regression", + "targetVariable": "03CV020/CORRENTE_N_M1_PV(Value)", + "variableColumns": ["303-WIT-200(Value)"], + "lagTrain": {"303-WIT-200(Value)": 3}, + "lagVal": {"303-WIT-200(Value)": 2}, + "remStaticWin": false, + "lowLim": {}, + "uppLim": {}, + "window": 0, + "useScaler": true, + "includeAr": false, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "removedIntervals": [], + "degree": 2, + "interactionOnly": false, + "nanTreatment": "drop", + "startDate": null, + "endDate": null, + "scalerName": "Standard Scaler", + "supportFilters": {} +} diff --git a/docs/test-scenarios/10-linear-regression-with-ar.json b/docs/test-scenarios/10-linear-regression-with-ar.json new file mode 100644 index 0000000..6295b34 --- /dev/null +++ b/docs/test-scenarios/10-linear-regression-with-ar.json @@ -0,0 +1,28 @@ +{ + "_description": "Regressão linear com variável autoregressiva (AR)", + "experimentName": "test-linear-with-ar", + "username": "bruno.domingues@aignosi.com.br", + "modelName": "Linear Regression", + "targetVariable": "03CV020/CORRENTE_N_M1_PV(Value)", + "variableColumns": ["303-WIT-200(Value)"], + "lagTrain": {"303-WIT-200(Value)": 0}, + "lagVal": {"303-WIT-200(Value)": 0}, + "remStaticWin": false, + "lowLim": {}, + "uppLim": {}, + "window": 0, + "useScaler": false, + "includeAr": true, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "removedIntervals": [], + "degree": 1, + "interactionOnly": false, + "nanTreatment": "drop", + "startDate": null, + "endDate": null, + "scalerName": "None", + "supportFilters": {} +} diff --git a/model_manager/utils/models/train_model_params.py b/model_manager/utils/models/train_model_params.py index 994ad06..09f45bc 100644 --- a/model_manager/utils/models/train_model_params.py +++ b/model_manager/utils/models/train_model_params.py @@ -248,6 +248,12 @@ class TrainModelParams: f'degree must be at least 2 for Polynomial Regression, got {self.degree}' ) + if self.model_name == 'Polynomial Regression' and self.scaler_name == 'None': + raise ValueError( + 'scaler_name must be set (e.g., "Standard Scaler") for Polynomial Regression ' + 'to avoid numerical overflow with large feature values' + ) + if self.model_name == 'Linear Regression' and self.degree != 1: raise ValueError(f'degree must be 1 for Linear Regression, got {self.degree}') diff --git a/model_manager/utils/models/train_model_result.py b/model_manager/utils/models/train_model_result.py index 0fab657..049bcd2 100644 --- a/model_manager/utils/models/train_model_result.py +++ b/model_manager/utils/models/train_model_result.py @@ -25,6 +25,7 @@ class TrainModelResult: regr (LinearRegressionModel): The trained linear regression model. scaler_dict (dict): A dictionary containing the scalers used to scale the features and target values. y_pred (pd.Series | None): The predicted target values for the testing dataset. Default is None. + y_train_pred (pd.Series | None): The predicted target values for the training dataset. Default is None. mse_val (float | None): The Mean Squared Error (MSE) of the predictions. Default is None. mae_val (float | None): The Mean Absolute Error (MAE) of the predictions. Default is None. r2_val (float | None): The R-squared (R²) value of the predictions. Default is None. @@ -46,6 +47,7 @@ class TrainModelResult: regr: LinearRegressionModel scaler_dict: dict y_pred: pd.Series | None = None + y_train_pred: pd.Series | None = None mse_val: float | None = None mae_val: float | None = None r2_val: float | None = None diff --git a/model_manager/utils/repository/model_repository.py b/model_manager/utils/repository/model_repository.py index 9d64bd5..47d542e 100644 --- a/model_manager/utils/repository/model_repository.py +++ b/model_manager/utils/repository/model_repository.py @@ -284,10 +284,16 @@ class ModelRepository: self.logger.error(error_msg) raise ValueError(error_msg) + if data.y_train_pred is None: + error_msg = 'Training predictions (y_train_pred) are None' + self.logger.error(error_msg) + raise ValueError(error_msg) + # Prepare reference data (training set) reference_data = pd.concat([data.x_train, data.y_train], axis=1) reference_data = reference_data.rename(columns={data.params.target_variable: 'target'}) - reference_data['prediction'] = data.regr.predict(data.x_train) + # Use pre-calculated predictions (calculated before denormalization to avoid overflow) + reference_data['prediction'] = data.y_train_pred # Prepare current data (test set) current_data = pd.concat([data.x_test, data.y_test], axis=1) diff --git a/model_manager/utils/repository/training_repository.py b/model_manager/utils/repository/training_repository.py index 93d8b5c..cd3019f 100644 --- a/model_manager/utils/repository/training_repository.py +++ b/model_manager/utils/repository/training_repository.py @@ -67,6 +67,11 @@ class TrainingRepository: Exception: If data loading, preprocessing, or training fails """ data = load_data(uploaded_file, params.line_separator, params.decimal_separator) + + # Configure datetime index if timestamp column exists + # Required for TimeSeriesDiscontinuityAnalyzer (static window removal) + data = self._configure_datetime_index(data) + process_data = self._init_data_preprocessor(params) process_data.fit(data) data_view = process_data.transform(data) @@ -128,7 +133,9 @@ class TrainingRepository: TrainModelResult: Updated result with predictions, denormalized data, and metrics (mse_val, mae_val, r2_val) """ + # Calculate predictions BEFORE denormalization (important for polynomial models) y_pred_array = tmr.regr.predict(tmr.x_test) + y_train_pred_array = tmr.regr.predict(tmr.x_train) if params.use_scaler: scaler = tmr.process_data.get_scaler() @@ -142,6 +149,9 @@ class TrainingRepository: tmr.y_train = scaler.denormalize_single_input(tmr.y_train, params.target_variable) tmr.y_test = scaler.denormalize_single_input(tmr.y_test, params.target_variable) y_pred_array = scaler.denormalize_predictions(y_pred_array, params.target_variable) + y_train_pred_array = scaler.denormalize_predictions( + y_train_pred_array, params.target_variable + ) else: # Fallback for sklearn StandardScaler: only inverse-transform features feature_cols = getattr( @@ -158,11 +168,15 @@ class TrainingRepository: tmr.y_pred = pd.Series(y_pred_array, index=tmr.y_test.index) tmr.y_pred.name = f'{params.target_variable}_pred' + tmr.y_train_pred = pd.Series(y_train_pred_array, index=tmr.y_train.index) + tmr.y_train_pred.name = f'{params.target_variable}_pred' + tmr.x_train = tmr.x_train.sort_index() tmr.x_test = tmr.x_test.sort_index() tmr.y_train = tmr.y_train.sort_index() tmr.y_test = tmr.y_test.sort_index() tmr.y_pred = tmr.y_pred.sort_index() + tmr.y_train_pred = tmr.y_train_pred.sort_index() assert tmr.y_pred is not None, 'y_pred should be set at this point' @@ -321,3 +335,68 @@ class TrainingRepository: 'interaction_only': params.interaction_only, 'original_features': params.variable_columns, } + + def _configure_datetime_index(self, data: pd.DataFrame) -> pd.DataFrame: + """ + Configure datetime index for the DataFrame. + + This method attempts to identify a timestamp column and set it as the + DataFrame index with DatetimeIndex type. This is required for + TimeSeriesDiscontinuityAnalyzer (used in static window removal). + + The method looks for common timestamp column names and converts the + first matching column to datetime, then sets it as the index. + + Args: + data: Input DataFrame + + Returns: + pd.DataFrame: DataFrame with DatetimeIndex if timestamp column found, + otherwise returns original DataFrame unchanged + """ + # If index is already DatetimeIndex, just ensure it's sorted + if isinstance(data.index, pd.DatetimeIndex): + self.logger.info('DataFrame already has DatetimeIndex') + return data.sort_index() + + # Common timestamp column names + timestamp_columns = [ + 'timestamp', + 'Timestamp', + 'TIMESTAMP', + 'date', + 'Date', + 'DATE', + 'datetime', + 'DateTime', + ] + + for col in timestamp_columns: + if col in data.columns: + try: + data[col] = pd.to_datetime(data[col]) + data = data.set_index(col) + data = data.sort_index() + self.logger.info(f'Configured datetime index from column: {col}') + return data + except (ValueError, TypeError) as e: + self.logger.warning(f'Failed to convert column {col} to datetime: {e}') + continue + + # If no timestamp column found, check if first column looks like a timestamp + first_col = data.columns[0] + try: + # Try to parse first column as datetime + test_values = data[first_col].head(10).dropna() + if len(test_values) > 0: + pd.to_datetime(test_values) + data[first_col] = pd.to_datetime(data[first_col]) + data = data.set_index(first_col) + data = data.sort_index() + self.logger.info(f'Configured datetime index from first column: {first_col}') + return data + except (ValueError, TypeError): + pass + + self.logger.warning('No timestamp column found - some features may not work correctly') + return data diff --git a/scripts/run_training_test.py b/scripts/run_training_test.py index a978daa..f90a8a7 100644 --- a/scripts/run_training_test.py +++ b/scripts/run_training_test.py @@ -15,6 +15,7 @@ Prerequisites: from __future__ import annotations +import argparse import asyncio import json import os @@ -24,8 +25,8 @@ import uuid from datetime import datetime, timedelta from pathlib import Path -from dotenv import load_dotenv import psycopg2 +from dotenv import load_dotenv from psycopg2.extras import Json from temporalio import client @@ -37,7 +38,8 @@ if ENV_PATH.exists(): load_dotenv(dotenv_path=ENV_PATH) -DOCS_PATH = Path('docs/test-model-data.csv') +DEFAULT_CSV_PATH = Path('docs/test-model-data.csv') +TEST_SCENARIOS_DIR = PROJECT_ROOT / 'docs' / 'test-scenarios' MINIO_ALIAS = 'suse' MINIO_BUCKET = 'model-training' @@ -54,26 +56,45 @@ TEMPORAL_NAMESPACE = os.getenv('TEMPORAL_NAMESPACE') TRAIN_TASK_QUEUE = os.getenv('TRAIN_TASK_QUEUE') TEMPORAL_WORKFLOW = 'train_model' -BASE_REQUEST_DATA = { - 'experimentName': 'model-manager-test-01', - 'username': 'bruno.domingues@aignosi.com.br', - 'modelType': 'Linear Regression', - 'targetVariable': '03CV020/CORRENTE_N_M1_PV(Value)', - 'variableColumns': ['303-WIT-200(Value)'], - 'lagTrain': 0, - 'lagVal': 0, - 'remStaticWin': False, - 'lowLim': {}, - 'uppLim': {}, - 'window': 0, - 'useScaler': False, - 'includeAr': False, - 'trainSize': 80, - 'shuffle': True, - 'lineSeparator': ',', - 'decimalSeparator': '.', - 'removedIntervals': [], -} + +def list_available_scenarios() -> list[str]: + """List all available test scenario files.""" + if not TEST_SCENARIOS_DIR.exists(): + return [] + return sorted([f.stem for f in TEST_SCENARIOS_DIR.glob('*.json')]) + + +def load_scenario(scenario_name: str) -> dict: + """Load a test scenario from JSON file. + + Args: + scenario_name: Name of the scenario (without .json extension) + or full path to a JSON file. + + Returns: + Dictionary with scenario data. + + Raises: + FileNotFoundError: If scenario file doesn't exist. + """ + # Check if it's a full path + scenario_path = Path(scenario_name) + if scenario_path.suffix == '.json' and scenario_path.exists(): + with open(scenario_path) as f: + return json.load(f) + + # Otherwise, look in the test-scenarios directory + scenario_file = TEST_SCENARIOS_DIR / f'{scenario_name}.json' + if not scenario_file.exists(): + available = list_available_scenarios() + available_str = ', '.join(available) if available else 'none' + raise FileNotFoundError( + f"Scenario '{scenario_name}' not found at {scenario_file}.\n" + f'Available scenarios: {available_str}' + ) + + with open(scenario_file) as f: + return json.load(f) def _ensure_source_file(path: Path) -> None: @@ -126,7 +147,7 @@ def insert_experiment_run(file_name: str, request_data: dict) -> int: ( request_data['experimentName'], request_data['username'], - 'ORCHESTRATOR_REQUEST_SENT', + 'ORCHESTRATOR_WAITING_PROC', now, now, MINIO_BUCKET, @@ -149,7 +170,6 @@ def build_workflow_payload( 'experiment_run_id': experiment_run_id, 'experiment_name': request_data['experimentName'], 'username': request_data['username'], - 'model_type': request_data['modelType'], 'target_variable': request_data['targetVariable'], 'variable_columns': request_data['variableColumns'], 'lag_train': request_data['lagTrain'], @@ -167,6 +187,15 @@ def build_workflow_payload( 'line_separator': request_data['lineSeparator'], 'decimal_separator': request_data['decimalSeparator'], 'removed_intervals': request_data['removedIntervals'], + # New parameters + 'model_name': request_data.get('modelName', 'Linear Regression'), + 'degree': request_data.get('degree', 1), + 'interaction_only': request_data.get('interactionOnly', False), + 'nan_treatment': request_data.get('nanTreatment', 'drop'), + 'start_date': request_data.get('startDate'), + 'end_date': request_data.get('endDate'), + 'scaler_name': request_data.get('scalerName', 'None'), + 'support_filters': request_data.get('supportFilters', {}), } @@ -191,9 +220,79 @@ async def trigger_temporal_workflow(workflow_input: dict) -> str: return workflow_id +def parse_args() -> argparse.Namespace: + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description='Run training workflow tests with different scenarios.', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # List available scenarios + python scripts/run_training_test.py --list + + # Run a specific scenario + python scripts/run_training_test.py --scenario linear-regression-basic + + # Run with a custom JSON file + python scripts/run_training_test.py --scenario /path/to/custom-scenario.json + + # Run with a custom CSV data file + python scripts/run_training_test.py --scenario linear-regression-basic --csv docs/other-data.csv +""", + ) + parser.add_argument( + '--scenario', + '-s', + type=str, + help='Name of the test scenario (without .json) or path to a JSON file.', + ) + parser.add_argument( + '--csv', + '-c', + type=Path, + default=DEFAULT_CSV_PATH, + help=f'Path to the CSV data file (default: {DEFAULT_CSV_PATH}).', + ) + parser.add_argument( + '--list', + '-l', + action='store_true', + help='List all available test scenarios and exit.', + ) + return parser.parse_args() + + def main() -> None: + args = parse_args() + + # List scenarios and exit if requested + if args.list: + scenarios = list_available_scenarios() + if scenarios: + print('Available test scenarios:') + for scenario in scenarios: + print(f' - {scenario}') + else: + print(f'No scenarios found in {TEST_SCENARIOS_DIR}') + sys.exit(0) + + # Require scenario argument if not listing + if not args.scenario: + print('Error: --scenario is required. Use --list to see available scenarios.', file=sys.stderr) + sys.exit(1) + + # Load scenario try: - uploaded_file_name = upload_to_minio(DOCS_PATH) + experiment_request = load_scenario(args.scenario) + print(f"Loaded scenario: {args.scenario}") + except FileNotFoundError as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) + + # Upload CSV to MinIO + try: + uploaded_file_name = upload_to_minio(args.csv) + print(f"Uploaded CSV to MinIO: {uploaded_file_name}") except subprocess.CalledProcessError as exc: print(f'Failed to upload file to MinIO: {exc}', file=sys.stderr) sys.exit(1) @@ -201,14 +300,15 @@ def main() -> None: print(str(exc), file=sys.stderr) sys.exit(1) - experiment_request = BASE_REQUEST_DATA.copy() - + # Insert experiment run try: experiment_run_id = insert_experiment_run(uploaded_file_name, experiment_request) + print(f"Created experiment_run with ID: {experiment_run_id}") except psycopg2.Error as exc: print(f'Database error while inserting experiment_run: {exc}', file=sys.stderr) sys.exit(1) + # Build and trigger workflow workflow_payload = build_workflow_payload( experiment_run_id=experiment_run_id, file_name=uploaded_file_name, @@ -224,6 +324,7 @@ def main() -> None: print( json.dumps( { + 'scenario': args.scenario, 'experiment_run_id': experiment_run_id, 's3_object_name': uploaded_file_name, 'workflow_id': workflow_id, diff --git a/tests/utils/models/test_train_model_result.py b/tests/utils/models/test_train_model_result.py index 275a713..bba7b06 100644 --- a/tests/utils/models/test_train_model_result.py +++ b/tests/utils/models/test_train_model_result.py @@ -183,11 +183,11 @@ def test_train_model_result_is_dataclass(sample_params, sample_dataframes): def test_train_model_result_field_count(): - """Test that TrainModelResult has exactly 19 fields.""" + """Test that TrainModelResult has exactly 20 fields.""" from dataclasses import fields result_fields = fields(TrainModelResult) - assert len(result_fields) == 19 + assert len(result_fields) == 20 field_names = {f.name for f in result_fields} expected_fields = { @@ -200,6 +200,7 @@ def test_train_model_result_field_count(): 'regr', 'scaler_dict', 'y_pred', + 'y_train_pred', 'mse_val', 'mae_val', 'r2_val', diff --git a/tests/utils/repository/test_training_repository.py b/tests/utils/repository/test_training_repository.py index 56277fb..e5f90d3 100644 --- a/tests/utils/repository/test_training_repository.py +++ b/tests/utils/repository/test_training_repository.py @@ -325,8 +325,14 @@ class TestAfterTrainCalculation: y_train = pd.Series([100, 200, 300], index=[0, 1, 2], name='target') y_test = pd.Series([400, 500], index=[3, 4], name='target') - # Mock predict to return a simple array - sample_linear_model.predict = MagicMock(return_value=np.array([450.0, 550.0])) + # Mock predict to return arrays with correct length based on input + def mock_predict(data): + if len(data) == 3: # x_train + return np.array([150.0, 250.0, 350.0]) + else: # x_test + return np.array([450.0, 550.0]) + + sample_linear_model.predict = MagicMock(side_effect=mock_predict) return TrainModelResult( params=sample_params, @@ -409,13 +415,24 @@ class TestAfterTrainCalculation: y_train = pd.Series([100, 200, 300], index=[0, 1, 2], name='target') y_test = pd.Series([400, 500], index=[3, 4], name='target') - # Mock predict to return a simple array - sample_linear_model.predict = MagicMock(return_value=np.array([450.0, 550.0])) + # Mock predict to return arrays with correct length based on input + def mock_predict(data): + if len(data) == 3: # x_train + return np.array([150.0, 250.0, 350.0]) + else: # x_test + return np.array([450.0, 550.0]) + + sample_linear_model.predict = MagicMock(side_effect=mock_predict) # Create mock scaler with denormalize methods mock_scaler = MagicMock() mock_scaler.denormalize_single_input = MagicMock(side_effect=lambda x, col: x * 2) - mock_scaler.denormalize_predictions = MagicMock(return_value=np.array([900.0, 1100.0])) + + # denormalize_predictions needs to return correct length based on input + def mock_denormalize_predictions(arr, col): + return arr * 2 + + mock_scaler.denormalize_predictions = MagicMock(side_effect=mock_denormalize_predictions) # Create mock preprocessor mock_process_data = MagicMock() @@ -453,8 +470,14 @@ class TestAfterTrainCalculation: y_train = pd.Series([100, 200, 300], index=[0, 1, 2], name='target') y_test = pd.Series([400, 500], index=[3, 4], name='target') - # Mock predict - sample_linear_model.predict = MagicMock(return_value=np.array([450.0, 550.0])) + # Mock predict to return arrays with correct length based on input + def mock_predict(data): + if len(data) == 3: # x_train + return np.array([150.0, 250.0, 350.0]) + else: # x_test + return np.array([450.0, 550.0]) + + sample_linear_model.predict = MagicMock(side_effect=mock_predict) # Create mock sklearn scaler (without denormalize methods) mock_scaler = MagicMock() @@ -506,6 +529,7 @@ class TestTrain: """ return BytesIO(csv_content.encode('utf-8')) + @patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df) @patch('model_manager.utils.repository.training_repository.split_train_test') @patch('model_manager.utils.repository.training_repository.load_data') def test_train_basic_workflow( @@ -551,6 +575,7 @@ class TestTrain: # Verify split was called assert mock_split_train_test.called + @patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df) @patch('model_manager.utils.repository.training_repository.split_train_test') @patch('model_manager.utils.repository.training_repository.load_data') def test_train_with_scaler( @@ -580,6 +605,7 @@ class TestTrain: assert result is not None assert result.scaler_dict is not None + @patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df) @patch('model_manager.utils.repository.training_repository.split_train_test') @patch('model_manager.utils.repository.training_repository.load_data') def test_train_with_shuffle_enabled( @@ -610,6 +636,7 @@ class TestTrain: call_kwargs = mock_split_train_test.call_args[1] assert call_kwargs['shuffle'] is True + @patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df) @patch('model_manager.utils.repository.training_repository.split_train_test') @patch('model_manager.utils.repository.training_repository.load_data') def test_train_with_different_train_size( @@ -640,6 +667,7 @@ class TestTrain: call_kwargs = mock_split_train_test.call_args[1] assert call_kwargs['train_size'] == 0.7 + @patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df) @patch('model_manager.utils.repository.training_repository.split_train_test') @patch('model_manager.utils.repository.training_repository.load_data') def test_train_raises_on_empty_data_after_transform( @@ -660,6 +688,7 @@ class TestTrain: with pytest.raises(ValueError, match='Data view is empty after transformation'): training_repo.train(sample_csv_data, sample_params) + @patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df) @patch('model_manager.utils.repository.training_repository.split_train_test') @patch('model_manager.utils.repository.training_repository.load_data') def test_train_logs_success( @@ -696,6 +725,7 @@ class TestTrain: 'Model trained successfully' in str(call) for call in mock_logger.info.call_args_list ) + @patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df) @patch('model_manager.utils.repository.training_repository.split_train_test') @patch('model_manager.utils.repository.training_repository.load_data') def test_train_with_custom_separators( @@ -726,6 +756,7 @@ class TestTrain: # Verify load_data was called with custom separators mock_load_data.assert_called_once_with(sample_csv_data, ';', ',') + @patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df) @patch('model_manager.utils.repository.training_repository.split_train_test') @patch('model_manager.utils.repository.training_repository.load_data') def test_train_result_contains_all_fields(