From 4a4043c3545655afc9657a3883e335da36aafafe Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 5 Sep 2025 16:49:38 -0300 Subject: [PATCH 01/29] SIENTIAPDE-1214 Update README.md to include additional PostgreSQL, OPC, and MongoDB configuration options for enhanced clarity and usability --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index b2c37be..330cbb4 100644 --- a/README.md +++ b/README.md @@ -537,15 +537,35 @@ The Laborious system exposes comprehensive Prometheus metrics: | `POSTGRES_USER` | PostgreSQL username | `sientia` | Yes | | `POSTGRES_PASSWORD` | PostgreSQL password | `sientia` | Yes | | `POSTGRES_DBNAME` | PostgreSQL database | `sientia` | Yes | +| `POSTGRES_MIN_CONNECTIONS` | Minimum PostgreSQL connections | `10` | No | +| `POSTGRES_MAX_CONNECTIONS` | Maximum PostgreSQL connections | `30` | No | | `MLFLOW_HOST` | MLFlow server hostname | `localhost` | Yes | | `MLFLOW_PORT` | MLFlow server port | `5000` | Yes | | `MLFLOW_USERNAME` | MLFlow username | `admin` | Yes | | `MLFLOW_PASSWORD` | MLFlow password | `admin` | Yes | | `OPC_CONFIG` | OPC server configuration (JSON) | `{}` | No | +| `OPC_ID` | OPC server identifier | `1` | No | +| `OPC_URL` | OPC server URL | `opc.tcp://localhost:4840` | No | +| `OPC_NAME` | OPC server name | `OPC_Server` | No | +| `OPC_SERVER_URI` | OPC server URI | `urn:opcserver:opcua` | No | +| `OPC_CERT_PATH` | OPC client certificate path | `/path/to/cert.pem` | No | +| `OPC_PRIVATE_KEY_PATH` | OPC private key path | `/path/to/key.pem` | No | +| `OPC_SERVER_CERT_PATH` | OPC server certificate path | `/path/to/server_cert.pem` | No | +| `OPC_RECONNECTION_INTERVAL` | OPC reconnection interval (ms) | `5000` | No | | `MONGODB_URL` | MongoDB connection URI | `localhost:27017` | Yes | +| `MONGODB_USERNAME` | MongoDB username | `root` | Yes | +| `MONGODB_PASSWORD` | MongoDB password | `password` | Yes | +| `MONGODB_DATABASE` | MongoDB database name | `sientia` | Yes | +| `MONGODB_TTL_INDEX_HOURS` | MongoDB TTL index hours | `1` | No | +| `KAFKA_BOOTSTRAP_SERVERS` | Kafka bootstrap servers | `localhost:9092` | No | +| `LOG_LEVEL` | Application log level | `INFO` | No | +| `PROJECT_NAME` | Project name for metrics | `sientia-laborious` | No | | `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No | | `HTTP_SDK_METRICS_PORT` | Temporal SDK metrics port | `9091` | No | + + + ### OPC Configuration For multiple OPC servers, use the `OPC_CONFIG` environment variable: From 6df04b72e59baf2541434d00b12d2fa0016bda8a Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 12 Sep 2025 14:23:35 -0300 Subject: [PATCH 02/29] SIENTIAPDE-1214 SIENTIAPDE-1214: Refactor MLFlow and model repository methods to use model_config dictionary - Updated MLFlow class methods to accept model_config instead of model_retention for improved flexibility. - Modified model_repository methods to handle model_config, extracting necessary parameters for transformation and prediction. - Adjusted predictions_batch and prediction_process workflows to utilize model_config for better configuration management. - Commented out the previous sientia-mlops-library dependency in requirements.txt for clarity. --- laborious/activities/mlflow.py | 12 +- .../utils/repository/model_repository.py | 22 +- laborious/workflows/predictions_batch.py | 2 +- .../sub_workflows/prediction_process.py | 12 +- requirements.txt | 3 +- tests.ipynb | 2 +- .../data_model/transformer_pyfunc/MLmodel | 19 + .../transformer_pyfunc/code/utils/__init__.py | 0 .../code/utils/data/.gitkeep | 0 .../code/utils/data/__init__.py | 0 .../code/utils/data/preprocessing.py | 172 ++++ .../code/utils/data/read_data.py | 56 ++ .../code/utils/data/transformers.py | 788 ++++++++++++++++ .../code/utils/dvc/__init__.py | 0 .../code/utils/dvc/params.py | 51 + .../code/utils/features/.gitkeep | 0 .../code/utils/features/__init__.py | 0 .../code/utils/mlflow/pyfunc_wrappers.py | 325 +++++++ .../code/utils/models/.gitkeep | 0 .../code/utils/models/__init__.py | 24 + .../code/utils/models/arima.py | 389 ++++++++ .../code/utils/models/base.py | 824 ++++++++++++++++ .../code/utils/models/catboost_time_series.py | 510 ++++++++++ .../code/utils/models/evaluation.py | 92 ++ .../code/utils/models/factory.py | 140 +++ .../models/linear_regression_time_series.py | 303 ++++++ .../code/utils/models/neural_prophet_model.py | 888 ++++++++++++++++++ .../code/utils/models/stacking_time_series.py | 695 ++++++++++++++ .../code/utils/visualization/.gitkeep | 0 .../code/utils/visualization/__init__.py | 0 .../data_model/transformer_pyfunc/conda.yaml | 11 + .../transformer_pyfunc/python_env.yaml | 7 + .../transformer_pyfunc/python_model.pkl | Bin 0 -> 123 bytes .../transformer_pyfunc/requirements.txt | 4 + .../transformers/courier_transformers.pkl | Bin 0 -> 29607 bytes 35 files changed, 5333 insertions(+), 18 deletions(-) create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/MLmodel create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/conda.yaml create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/requirements.txt create mode 100644 tmp/artifacts/data_model/transformers/courier_transformers.pkl diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 3e3549c..c0785fd 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -94,7 +94,7 @@ class MLFlow(BaseActivity): self.info('Transforming data...', metadata) data = DataFrame(input_data['data']) model_name = input_data['model_name'] - model_retention = input_data['model_retention'] + model_config = input_data.get('model_config', {}) self.debug("Raw input data:", metadata) self.debug(data, metadata) @@ -117,10 +117,11 @@ class MLFlow(BaseActivity): # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( - model_name, data, model_retention) + model_name, data, model_config + ) self.debug("Transform response data:", metadata) - self.debug(json.dumps(response_data, indent=4), metadata) + self.debug(response_data, metadata) self.info("Data transformed successfully", metadata) @@ -160,7 +161,7 @@ class MLFlow(BaseActivity): self.info('Predicting data...', metadata) data = DataFrame(input_data['data']) model_name = input_data['model_name'] - model_retention = input_data['model_retention'] + model_config = input_data.get('model_config', {}) self.debug(data, metadata) @@ -169,7 +170,8 @@ class MLFlow(BaseActivity): # Request prediction from MLFlow model response_data = self.model_monitoring_repository.predict( - model_name, data, model_retention) + model_name, data, model_config + ) self.debug("Prediction response data:", metadata) self.debug(json.dumps(response_data, indent=4), metadata) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 68322f3..be1efa0 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -25,7 +25,7 @@ class MLFlowRepository(): username=username, password=password, logger=logger) - def transform(self, model_name: str, data: pd.DataFrame, model_retention: int): + def transform(self, model_name: str, data: pd.DataFrame, model_config: dict) -> dict: """ Transform data using a model. @@ -39,11 +39,19 @@ class MLFlowRepository(): """ try: + model_retention = model_config.get('model_retention', 0) + flavor = model_config.get('transform_flavor', 'sklearn') + compressed = model_config.get('is_compressed', False) + retention_target = model_config.get('retention_target', 'model') + transform_keyword = model_config.get( + 'transform_function_keyword', 'predict') return { 'success': True, 'content': self.model_serving.get_cached_transform( - model_name, data, model_retention).to_dict() + model_name, data, model_retention, flavor, + compressed, retention_target, transform_keyword + ).to_dict() } except Exception as e: @@ -55,7 +63,7 @@ class MLFlowRepository(): } } - def predict(self, model_name: str, data: pd.DataFrame, model_retention: int): + def predict(self, model_name: str, data: pd.DataFrame, model_config: dict) -> dict: """ Predict data using a model. @@ -68,11 +76,17 @@ class MLFlowRepository(): - dict: A dictionary containing the predicted data. """ try: + model_retention = model_config.get('retention_minutes', 0) + flavor = model_config.get('predict_flavor', 'pyfunc') + compressed = model_config.get('compressed', False) + retention_target = model_config.get('retention_target', 'model') input_index = data.index start_time = datetime.now() data = self.model_serving.get_cached_predict( - model_name, data, model_retention) + model_name, data, model_retention, flavor, + compressed, retention_target + ) end_time = datetime.now() data = pd.DataFrame(data, columns=['prediction']) diff --git a/laborious/workflows/predictions_batch.py b/laborious/workflows/predictions_batch.py index de1fbbe..cc10def 100644 --- a/laborious/workflows/predictions_batch.py +++ b/laborious/workflows/predictions_batch.py @@ -113,7 +113,7 @@ class PredictionsBatch(): 'POLICY': 'STOP' } }), - 'model_retention': input_data.get('model_retention', 60), + 'model_config': input_data.get('model_config', {}), 'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']), 'opc_output_config': input_data.get('opc_output_config', {}) } diff --git a/laborious/workflows/sub_workflows/prediction_process.py b/laborious/workflows/sub_workflows/prediction_process.py index e1a0162..d78018e 100644 --- a/laborious/workflows/sub_workflows/prediction_process.py +++ b/laborious/workflows/sub_workflows/prediction_process.py @@ -79,7 +79,7 @@ class PredictionProcess(): data = input_data['data'] model_id = input_data['model_id'] model_name = input_data['model_name'] - model_retention = input_data['model_retention'] + model_config = input_data.get('model_config', {}) # Get last timestamp for incremental processing last_timestamp = await workflow.execute_local_activity_method( @@ -120,7 +120,7 @@ class PredictionProcess(): **metadata, 'data': data, 'model_name': model_name, - 'model_retention': model_retention + 'model_config': model_config }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -172,7 +172,7 @@ class PredictionProcess(): **metadata, 'data': transformed_data, 'model_name': model_name, - 'model_retention': model_retention + 'model_config': model_config }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -209,7 +209,7 @@ class PredictionProcess(): 'timestamp': last_timestamp, 'model_id': model_id, 'model_name': model_name, - 'model_retention': model_retention, + 'model_config': model_config, 'opc_output_config': input_data['opc_output_config'], 'schema': input_data['schema'], 'table_name': input_data['table_name'], @@ -248,7 +248,7 @@ class PredictionProcess(): table_name = input_data['table_name'] model_id = input_data['model_id'] model_name = input_data['model_name'] - model_retention = input_data['model_retention'] + model_config = input_data.get('model_config', {}) path_flag = path_flag.upper() if path_flag else '' @@ -282,7 +282,7 @@ class PredictionProcess(): 'timestamp': last_timestamp, 'model_id': model_id, 'model_name': model_name, - 'model_retention': model_retention, + 'model_config': model_config, 'schema': schema, 'table_name': table_name, 'comment': comment, diff --git a/requirements.txt b/requirements.txt index 3f723f7..10995c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,6 @@ sqlalchemy asyncua redis git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.5 -git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.13 +# git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.13 +/home/grezewave/Documents/projects/sientia/sientia-mlops-library prometheus-client diff --git a/tests.ipynb b/tests.ipynb index 74750bf..8ab64e2 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -165,7 +165,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 1, "id": "c61be7ab", "metadata": {}, "outputs": [ diff --git a/tmp/artifacts/data_model/transformer_pyfunc/MLmodel b/tmp/artifacts/data_model/transformer_pyfunc/MLmodel new file mode 100644 index 0000000..3fc50fb --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/MLmodel @@ -0,0 +1,19 @@ +artifact_path: transformer_pyfunc +flavors: + python_function: + artifacts: + transformer: + path: artifacts/training_transformer.pkl + uri: /tmp/tmpnzkqz3v0/training_transformer.pkl + cloudpickle_version: 2.2.1 + code: code + env: + conda: conda.yaml + virtualenv: python_env.yaml + loader_module: mlflow.pyfunc.model + python_model: python_model.pkl + python_version: 3.10.16 +mlflow_version: 2.7.1 +model_uuid: 6a4a99079d234d0da2b8091532d55a34 +run_id: c2edec4dfafd4ad8bba257d62d25cd43 +utc_time_created: '2025-09-09 12:30:04.885248' diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py new file mode 100644 index 0000000..4ee9088 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py @@ -0,0 +1,172 @@ +""" +Module with utility functions for data processing. +""" + +import pandas as pd +from typing import Dict, List +from rich.console import Console + +console = Console() + + +def create_lagged_target( + data: pd.DataFrame, + target_column: str, + lags: List[int], + drop_nans: bool = True, +) -> pd.DataFrame: + """ + Creates lagged versions of a target column in a DataFrame. + + For each lag value in the provided list, a new column is created with + the naming pattern: target_column + "_lag_" + lag_value. + + Args: + data: Input DataFrame containing the target column. + target_column: Name of the target column to create lags for. + lags: List of lag values (integers between 1 and len(data)-1). + drop_nans: Whether rows with nulls generated by the lag creation + process should be dropped. Defaults to True. + Returns: + DataFrame with original columns plus the newly created lag columns. + """ + if data.empty: + console.log("[red]Warning: Input data is empty.") + return data + + result = data.copy() + + if target_column not in result.columns: + raise ValueError(f"Target column '{target_column}' not found in data.") + + # Validate lag values + max_lag = len(data) - 1 + valid_lags = [lag for lag in lags if 1 <= lag <= max_lag] + + if len(valid_lags) < len(lags): + invalid_lags = set(lags) - set(valid_lags) + console.log( + f"[yellow]Warning: Ignoring invalid lag values: {invalid_lags}. " + f"Lags must be between 1 and {max_lag}." + ) + + lag_column_names = [] + for lag in valid_lags: + lag_column_name = f"{target_column}_lag_{lag}" + result[lag_column_name] = result[target_column].shift(lag) + console.log(f"Created lagged column: [cyan]{lag_column_name}") + lag_column_names.append(lag_column_name) + if drop_nans: + result = result.dropna(subset=lag_column_names) + + return result + + +def remove_stopped_windows( + data: pd.DataFrame, + stopped_process_columns: Dict[str, float], + stopped_process_threshold: float, + time_colname: str, +) -> pd.DataFrame: + """ + Removes time windows from the input DataFrame if the proportion of samples + below a column threshold exceeds the specified limit. + + A window is considered "stopped" if *all* specified columns exceed the + stopped sample threshold. + + Args: + data: Input DataFrame with process variables and timestamps. + stopped_process_columns: Dict mapping column names to thresholds. + stopped_process_threshold: Proportion threshold (0-1) for marking a + window as stopped. + time_colname: Base name of the timestamp column + (without 'lab_' prefix). + + Returns: + A DataFrame with stopped windows removed. + """ + if data.empty: + console.log("[red]Warning: Input data is empty.") + return data + + console.log( + "Removing windows where any column exceeds" + + f" {stopped_process_threshold:.2%} of values below threshold" + ) + + masks = [] + + for col, threshold in stopped_process_columns.items(): + console.log( + "Evaluating stopped condition for column:" + + f" [cyan]{col} < {threshold}" + ) + below_threshold = data[[col]].lt(threshold) + + console.log( + "Counting number of samples below threshold for each window" + ) + below_threshold[f"lab_{time_colname}"] = data[f"lab_{time_colname}"] + grouped = below_threshold.groupby(f"lab_{time_colname}")[col].agg( + ["sum", "count"] + ) + stopped_mask = ( + grouped["sum"] / grouped["count"] + ) > stopped_process_threshold + + console.log( + f"[red]{stopped_mask.sum()} windows marked as stopped by {col}" + ) + masks.append(stopped_mask) + + # Combine masks across columns: only drop if all agree + combined_mask = pd.concat(masks, axis=1).all(axis=1) + + num_removed = combined_mask.sum() + total = combined_mask.shape[0] + console.log( + f"Removing [bold red]{num_removed}[/] out of {total}" + + f" windows ({num_removed / total:.2%})" + ) + + to_remove = combined_mask[combined_mask].index + keep_mask = ~data[f"lab_{time_colname}"].isin(to_remove) + + return data[keep_mask] + + +def aggregate_data( + merged_data: pd.DataFrame, + time_colname: str, + target_colname: str, + aggregation_functions: List[str], +) -> pd.DataFrame: + """ + Aggregates a DataFrame by time and target columns using specified + aggregation functions. + + Args: + merged_data: Input DataFrame with raw observations. + time_colname: Name of the timestamp column (no 'lab_' prefix). + target_colname: Name of the target/grouping column. + aggregation_functions: List of aggregation functions to apply + (e.g. "mean", "std"). + + Returns: + Aggregated DataFrame with flattened column names and renamed time + column. + """ + group_by_cols = [f"lab_{time_colname}", target_colname] + + aggregated = merged_data.groupby(group_by_cols).agg(aggregation_functions) + # Flatten MultiIndex columns + aggregated.columns = [ + "_".join(col) if isinstance(col, tuple) else col + for col in aggregated.columns + ] # type: ignore + aggregated = aggregated.reset_index() + + console.log(f"[bold green]Aggregated shape: {aggregated.shape}") + + return aggregated.rename(columns={f"lab_{time_colname}": time_colname}) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py new file mode 100644 index 0000000..76e4e9d --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py @@ -0,0 +1,56 @@ +""" +Module with helper functions to read datasets. +""" + +import pandas as pd +from openpyxl import load_workbook +from typing import Union + + +def read_excel_with_colors( + filepath: str, color_columns: list[str], sheet_name: Union[str, int] = 0 +) -> pd.DataFrame: + """ + Read an Excel file and extract cell fill colors for specified columns. + + Parameters: + filepath (str): Path to the Excel file. + color_columns (List[str]): Column names to extract fill colors from. + sheet_name (str or int): Sheet name or index (default is first sheet). + + Returns: + DataFrame: DataFrame with original data and extra color columns. + """ + df = pd.read_excel(filepath, sheet_name=sheet_name) + + workbook = load_workbook(filepath) + sheet = ( + workbook[sheet_name] + if isinstance(sheet_name, str) + else workbook[workbook.sheetnames[sheet_name]] + ) + + header = next(sheet.iter_rows(min_row=1, max_row=1, values_only=True)) + col_name_to_letter = { + name: chr(65 + idx) for idx, name in enumerate(header) + } + + for col_name in color_columns: + if col_name not in df.columns: + raise ValueError(f"Column '{col_name}' not found in Excel file.") + + col_letter = col_name_to_letter[col_name] + fill_colors: list[Union[str, None]] = [] + + for row in range(2, sheet.max_row + 1): + cell = sheet[f"{col_letter}{row}"] + fill = cell.fill + + if fill.fill_type == "solid" and fill.fgColor.rgb: + fill_colors.append(fill.fgColor.rgb) + else: + fill_colors.append(None) + + df[f"{col_name}_fill_color"] = fill_colors + + return df diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py new file mode 100644 index 0000000..efe43e0 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py @@ -0,0 +1,788 @@ +""" +Module with scikit-learn transformers for training and inference pipelines. + +This module replicates the functionality from the DVC pipeline stages: +- preprocessing.py (stopped process filtering, aggregation) +- filter_columns.py (feature selection, lagged target creation) +""" + +import pandas as pd +from typing import Dict, List, Optional +from sklearn.base import BaseEstimator, TransformerMixin +from rich.console import Console + +console = Console() + + +class CourierTrainingTransformer(BaseEstimator, TransformerMixin): + """ + Training transformer that replicates the DVC pipeline preprocessing. + + Includes: + 1. Remove stopped process windows (training only) + 2. Data aggregation + 3. Feature selection (learns and applies - dictionary-based only) + 4. Create lagged target features + """ + + def __init__( + self, + aggregation_functions: List[str] = ["median", "std", "min", "max"], + stopped_process_columns: Dict[str, float] = { + "305-PIT-170": 9, + "305-PIT-175": 9, + }, + stopped_process_threshold: float = 0.1, + target_lags: List[int] = [2], + time_colname: str = "timestamp", + target_colname: str = "SiO2_conc", + dictionary_df: Optional[pd.DataFrame] = None, + create_lagged_target: bool = True, + drop_nans: bool = True, + ): + """ + Initialize the training transformer. + + Args: + aggregation_functions: List of aggregation functions to apply + stopped_process_columns: Dict mapping column names to thresholds + stopped_process_threshold: Proportion threshold for stopped windows + target_lags: List of lag values for target column + time_colname: Name of timestamp column (without 'lab_' prefix) + target_colname: Name of target column + dictionary_df: DataFrame with domain knowledge + (TAG_fill_color column) + create_lagged_target: Whether to create lagged target features + drop_nans: Whether to drop NaNs after creating lags + """ + self.aggregation_functions = aggregation_functions + self.stopped_process_columns = stopped_process_columns + self.stopped_process_threshold = stopped_process_threshold + self.target_lags = target_lags + self.time_colname = time_colname + self.target_colname = target_colname + self.dictionary_df = dictionary_df + self.create_lagged_target = create_lagged_target + self.drop_nans = drop_nans + + # Will be learned during fit + self.selected_features_ = None + + def _infer_lab_timestamp(self, data: pd.DataFrame) -> pd.DataFrame: + """ + Creates a lab timestamp column by rounding the timestamp up to the next + even hour. + + Args: + data: Input DataFrame + + Returns: + DataFrame with added lab timestamp column + """ + lab_col_name = f"lab_{self.time_colname}" + + if lab_col_name in data.columns: + console.log( + f"[yellow]Lab timestamp column {lab_col_name} already exists, " + + "skipping inference" + ) + return data + + # Check if timestamp is a column or the index + if self.time_colname in data.columns: + # Timestamp is a regular column + result = data.copy() + timestamp_col = pd.to_datetime(result[self.time_colname]) + elif data.index.name == self.time_colname or ( + hasattr(data.index, "names") + and self.time_colname in data.index.names + ): + # Timestamp is the index (or part of a MultiIndex) + result = data.copy() + timestamp_col = pd.to_datetime( + result.index.get_level_values(self.time_colname) + if hasattr(result.index, "names") + and len(result.index.names) > 1 + else result.index + ) + else: + raise ValueError( + f"Timestamp '{self.time_colname}' not found in data columns " + + f"or index. Available columns: {list(data.columns)}, " + + f"index name: {data.index.name}" + ) + + # Round up to next even hour + # Step 1: Floor to the hour to remove minutes/seconds + # Handle both Series (from column) and DatetimeIndex (from index) + if hasattr(timestamp_col, "dt"): + # timestamp_col is a Series + hour_floor = timestamp_col.dt.floor("H") + hour = hour_floor.dt.hour + else: + # timestamp_col is a DatetimeIndex + hour_floor = timestamp_col.floor("H") + hour = hour_floor.hour + + # Step 3: Determine if rounding is needed + # - If hour is odd, round up to next even hour + # - If hour is even but original timestamp had minutes/seconds, + # round up to next even hour + # - If hour is even and original timestamp was exactly on the hour, + # keep it + needs_rounding = (hour % 2 == 1) | (timestamp_col != hour_floor) + + # Calculate next even hour + next_even_hour = ((hour // 2) + 1) * 2 + + # Handle case where next even hour >= 24 (next day) + days_to_add = (next_even_hour >= 24).astype(int) + hour_component = next_even_hour % 24 + + # Create the lab timestamp + if hasattr(timestamp_col, "dt"): + # timestamp_col is a Series + lab_timestamp = hour_floor.where( + ~needs_rounding, + hour_floor.dt.floor("D") + + pd.to_timedelta(days_to_add, unit="D") + + pd.to_timedelta(hour_component, unit="H"), + ) + else: + # timestamp_col is a DatetimeIndex + base_date = hour_floor.floor("D") + next_even_timestamp = ( + base_date + + pd.to_timedelta(days_to_add, unit="D") + + pd.to_timedelta(hour_component, unit="H") + ) + lab_timestamp = pd.Series( + hour_floor.where(~needs_rounding, next_even_timestamp), + index=result.index, + ) + + result[lab_col_name] = lab_timestamp + + console.log( + f"[bold green]Created lab timestamp column: {lab_col_name}" + ) + + return result + + def _remove_stopped_windows( + self, + data: pd.DataFrame, + ) -> pd.DataFrame: + """ + Removes time windows where process was stopped. + Replicates remove_stopped_windows from preprocessing.py + """ + if data.empty: + console.log("[red]Warning: Input data is empty.") + return data + + console.log( + "Removing windows where any column exceeds" + + f" {self.stopped_process_threshold:.2%} of values below" + + " threshold" + ) + + masks = [] + + for col, threshold in self.stopped_process_columns.items(): + console.log( + "Evaluating stopped condition for column:" + + f" [cyan]{col} < {threshold}" + ) + below_threshold = data[[col]].lt(threshold) + + console.log( + "Counting number of samples below threshold for each window" + ) + below_threshold[f"lab_{self.time_colname}"] = data[ + f"lab_{self.time_colname}" + ] + grouped = below_threshold.groupby(f"lab_{self.time_colname}")[ + col + ].agg(["sum", "count"]) + stopped_mask = ( + grouped["sum"] / grouped["count"] + ) > self.stopped_process_threshold + + console.log( + f"[red]{stopped_mask.sum()} windows marked as stopped by {col}" + ) + masks.append(stopped_mask) + + # Combine masks across columns: only drop if all agree + if not masks: + return data + + mask_df = pd.concat(masks, axis=1) + combined_mask = mask_df.all(axis=1) + + num_removed = int(combined_mask.sum()) # type: ignore + total = combined_mask.shape[0] + console.log( + f"Removing [bold red]{num_removed}[/] out of {total}" + + f" windows ({num_removed / total:.2%})" + ) + + to_remove = combined_mask[combined_mask].index + keep_mask = ~data[f"lab_{self.time_colname}"].isin(to_remove) + + filtered_data = data[keep_mask] + return filtered_data # type: ignore + + def _aggregate_data( + self, + merged_data: pd.DataFrame, + ) -> pd.DataFrame: + """ + Aggregates data by time and target columns. + Replicates aggregate_data from preprocessing.py + """ + group_by_cols = [f"lab_{self.time_colname}", self.target_colname] + + aggregated = merged_data.groupby(group_by_cols).agg( + self.aggregation_functions + ) + # Flatten MultiIndex columns + aggregated.columns = [ + "_".join(col) if isinstance(col, tuple) else col + for col in aggregated.columns + ] # type: ignore + aggregated = aggregated.reset_index() + + # Rename timestamp column and ensure it's datetime + aggregated = aggregated.rename( + columns={f"lab_{self.time_colname}": self.time_colname} + ) + aggregated[self.time_colname] = pd.to_datetime( + aggregated[self.time_colname] + ) + + console.log(f"[bold green]Aggregated shape: {aggregated.shape}") + + return aggregated + + def _learn_feature_selection( + self, + data: pd.DataFrame, + ) -> List[str]: + """ + Learn which features to keep based on dictionary only. + Replicates domain knowledge filtering from filter_columns.py + """ + if self.dictionary_df is None: + console.log( + "[yellow]Warning: No dictionary data provided. Using all" + + " features." + ) + return [col for col in data.columns if col != self.time_colname] + + # Domain knowledge filter - only keep columns with TAG_fill_color + columns_to_keep_dict = self.dictionary_df.loc[ + self.dictionary_df["TAG_fill_color"].notna(), "TAG" + ].values + + # Filter data columns to only those that match dictionary tags + available_columns = [ + col for col in data.columns if col != self.time_colname + ] + columns_to_keep = [ + col + for col in available_columns + if col.split("_")[0] in columns_to_keep_dict + ] + + console.log( + f"Keeping {len(columns_to_keep)}/{len(available_columns)}" + + " columns based on dictionary." + ) + + # Always include target + columns_to_keep.append(self.target_colname) + + return columns_to_keep + + def _create_lagged_target( + self, + data: pd.DataFrame, + ) -> pd.DataFrame: + """ + Creates lagged versions of target column. + Replicates create_lagged_target from preprocessing.py + """ + if data.empty: + console.log("[red]Warning: Input data is empty.") + return data + + result = data.copy() + + if self.target_colname not in result.columns: + raise ValueError( + f"Target column '{self.target_colname}' not found in data." + ) + + # Validate lag values + max_lag = len(data) - 1 + valid_lags = [lag for lag in self.target_lags if 1 <= lag <= max_lag] + + if len(valid_lags) < len(self.target_lags): + invalid_lags = set(self.target_lags) - set(valid_lags) + console.log( + f"[yellow]Warning: Ignoring invalid lag values: {invalid_lags}" + + f". Lags must be between 1 and {max_lag}." + ) + + lag_column_names = [] + for lag in valid_lags: + lag_column_name = f"{self.target_colname}_lag_{lag}" + result[lag_column_name] = result[self.target_colname].shift(lag) + console.log(f"Created lagged column: [cyan]{lag_column_name}") + lag_column_names.append(lag_column_name) + + if self.drop_nans and lag_column_names: + result = result.dropna(subset=lag_column_names) + + return result + + def fit(self, X: pd.DataFrame, y=None): + """ + Learn feature selection parameters. + + Args: + X: Input DataFrame with merged process and quality data + y: Not used + + Returns: + self + """ + console.log("[bold blue]Training transformer fit phase") + + # Create a copy for processing + data = X.copy() + + # Step 0: Infer lab timestamp if needed + console.log("[bold blue]Inferring lab timestamp") + data = self._infer_lab_timestamp(data) + + # Step 1: Remove stopped process windows (training only) + console.log("[bold blue]Removing stopped process windows") + data = self._remove_stopped_windows(data) + + # Step 2: Aggregate data + console.log("[bold blue]Aggregating data") + data = self._aggregate_data(data) + + # Step 3: Learn feature selection + console.log("[bold blue]Learning feature selection") + self.selected_features_ = self._learn_feature_selection(data) + + console.log( + f"[bold green]Learned {len(self.selected_features_)}" + + " features for selection" + ) + self._feature_names = self.selected_features_ + + return self + + def transform(self, X: pd.DataFrame) -> pd.DataFrame: + """ + Apply the complete training transformation pipeline. + + Args: + X: Input DataFrame with merged process and quality data + + Returns: + Transformed DataFrame ready for model training + """ + if self.selected_features_ is None: + raise ValueError("Transformer must be fitted before transform.") + + console.log("[bold blue]Training transformer transform phase") + + # Create a copy for processing + data = X.copy() + + # Step 0: Infer lab timestamp if needed + console.log("[bold blue]Inferring lab timestamp") + data = self._infer_lab_timestamp(data) + + # Step 1: Remove stopped process windows (training only) + console.log("[bold blue]Removing stopped process windows") + data = self._remove_stopped_windows(data) + + # Step 2: Aggregate data + console.log("[bold blue]Aggregating data") + data = self._aggregate_data(data) + + # Step 3: Apply feature selection + console.log("[bold blue]Applying feature selection") + # Set timestamp as index for filtering and ensure it's datetime + data[self.time_colname] = pd.to_datetime(data[self.time_colname]) + data = data.set_index(self.time_colname) + data = data[self.selected_features_] + + # Step 4: Create lagged target features + if self.create_lagged_target: + console.log("[bold blue]Creating lagged target features") + data = self._create_lagged_target(data) + + console.log(f"[bold green]Final training data shape: {data.shape}") + + return data + + +class CourierInferenceTransformer(BaseEstimator, TransformerMixin): + """ + Inference transformer that replicates DVC pipeline preprocessing + without training-specific steps. + + Includes: + 1. Data aggregation (higher frequency - no grouping by target) + 2. Feature selection (applies learned selection) + 3. Create lagged target features + + Note: Does NOT include stopped process filtering (training only). + """ + + def __init__( + self, + selected_features: List[str], + aggregation_functions: List[str] = ["median", "std", "min", "max"], + target_lags: List[int] = [2], + time_colname: str = "timestamp", + target_colname: str = "SiO2_conc", + create_lagged_target: bool = True, + drop_nans: bool = True, + ): + """ + Initialize the inference transformer. + + Args: + selected_features: Pre-learned list of features to select + aggregation_functions: List of aggregation functions to apply + target_lags: List of lag values for target column + time_colname: Name of timestamp column (without 'lab_' prefix) + target_colname: Name of target column + create_lagged_target: Whether to create lagged target features + drop_nans: Whether to drop NaNs after creating lags + """ + self.selected_features = selected_features + self.aggregation_functions = aggregation_functions + self.target_lags = target_lags + self.time_colname = time_colname + self.target_colname = target_colname + self.create_lagged_target = create_lagged_target + self.drop_nans = drop_nans + + def _infer_lab_timestamp(self, data: pd.DataFrame) -> pd.DataFrame: + """ + Creates a lab timestamp column by rounding the timestamp up to the next + even hour. + + Args: + data: Input DataFrame + + Returns: + DataFrame with added lab timestamp column + """ + lab_col_name = f"lab_{self.time_colname}" + + if lab_col_name in data.columns: + console.log( + f"[yellow]Lab timestamp column {lab_col_name} already exists, " + + "skipping inference" + ) + return data + + # Check if timestamp is a column or the index + if self.time_colname in data.columns: + # Timestamp is a regular column + result = data.copy() + timestamp_col = pd.to_datetime(result[self.time_colname]) + elif data.index.name == self.time_colname or ( + hasattr(data.index, "names") + and self.time_colname in data.index.names + ): + # Timestamp is the index (or part of a MultiIndex) + result = data.copy() + timestamp_col = pd.to_datetime( + result.index.get_level_values(self.time_colname) + if hasattr(result.index, "names") + and len(result.index.names) > 1 + else result.index + ) + else: + raise ValueError( + f"Timestamp '{self.time_colname}' not found in data columns " + + f"or index. Available columns: {list(data.columns)}, " + + f"index name: {data.index.name}" + ) + + # Round up to next even hour + # Step 1: Floor to the hour to remove minutes/seconds + # Handle both Series (from column) and DatetimeIndex (from index) + if hasattr(timestamp_col, "dt"): + # timestamp_col is a Series + hour_floor = timestamp_col.dt.floor("H") + hour = hour_floor.dt.hour + else: + # timestamp_col is a DatetimeIndex + hour_floor = timestamp_col.floor("H") + hour = hour_floor.hour + + # Step 3: Determine if rounding is needed + # - If hour is odd, round up to next even hour + # - If hour is even but original timestamp had minutes/seconds, + # round up to next even hour + # - If hour is even and original timestamp was exactly on the hour, + # keep it + needs_rounding = (hour % 2 == 1) | (timestamp_col != hour_floor) + + # Calculate next even hour + next_even_hour = ((hour // 2) + 1) * 2 + + # Handle case where next even hour >= 24 (next day) + days_to_add = (next_even_hour >= 24).astype(int) + hour_component = next_even_hour % 24 + + # Create the lab timestamp + if hasattr(timestamp_col, "dt"): + # timestamp_col is a Series + lab_timestamp = hour_floor.where( + ~needs_rounding, + hour_floor.dt.floor("D") + + pd.to_timedelta(days_to_add, unit="D") + + pd.to_timedelta(hour_component, unit="H"), + ) + else: + # timestamp_col is a DatetimeIndex + base_date = hour_floor.floor("D") + next_even_timestamp = ( + base_date + + pd.to_timedelta(days_to_add, unit="D") + + pd.to_timedelta(hour_component, unit="H") + ) + lab_timestamp = pd.Series( + hour_floor.where(~needs_rounding, next_even_timestamp), + index=result.index, + ) + + result[lab_col_name] = lab_timestamp + + console.log( + f"[bold green]Created lab timestamp column: {lab_col_name}" + ) + + return result + + def _aggregate_data( + self, + merged_data: pd.DataFrame, + ) -> pd.DataFrame: + """ + Aggregate data into 2-hour non-overlapping windows. + Each row corresponds to one 2-hour window ending at an even hour. + """ + if merged_data.empty: + console.log("[red]Warning: Input data is empty.") + return merged_data + + lab_col = f"lab_{self.time_colname}" + if lab_col not in merged_data.columns: + raise ValueError( + f"Missing '{lab_col}' column. Call _infer_lab_timestamp first." + ) + + # Get numeric columns only for aggregation + numeric_cols = merged_data.select_dtypes( + include=["number"] + ).columns.tolist() + + # Remove time and target columns if present + cols_to_remove = [self.time_colname, lab_col, self.target_colname] + for col in cols_to_remove: + if col in numeric_cols: + numeric_cols.remove(col) + + groups = merged_data.groupby(lab_col) + + if numeric_cols: + aggregated_numeric = groups[numeric_cols].agg( + self.aggregation_functions + ) + # Flatten MultiIndex columns: (col, func) -> "col_func" + aggregated_numeric.columns = [ + f"{col}_{func}" + for col, func in aggregated_numeric.columns.to_flat_index() + ] + else: + # Create empty frame indexed by the 2-hour windows + aggregated_numeric = groups.size().to_frame(name="__rows__") + aggregated_numeric = aggregated_numeric.drop(columns=["__rows__"]) + + # Add target column as the last non-null value per window + if self.target_colname in merged_data.columns: + target_per_window = groups[self.target_colname].apply( + lambda s: s.dropna().iloc[-1] + if not s.dropna().empty + else None + ) + aggregated_numeric[self.target_colname] = target_per_window + + # Reset index and rename lab timestamp to main time column + result = aggregated_numeric.reset_index().rename( + columns={lab_col: self.time_colname} + ) + + # Ensure timestamp is datetime + result[self.time_colname] = pd.to_datetime(result[self.time_colname]) + + console.log( + f"[bold green]Aggregated to windowed shape: {result.shape}" + ) + + return result + + def _create_lagged_target( + self, + data: pd.DataFrame, + ) -> pd.DataFrame: + """ + Creates lagged target column names with target values for inference. + In inference, we assume the data is already properly lagged, + so we just create the expected column names with the target values. + """ + if data.empty: + console.log("[red]Warning: Input data is empty.") + return data + + result = data.copy() + + if self.target_colname not in result.columns: + raise ValueError( + f"Target column '{self.target_colname}' not found in data." + ) + + # Create lagged column names with target values (no actual shifting) + lag_column_names = [] + + # Get target column as a Series to ensure we have exactly one column + target_series = result[self.target_colname] + if isinstance(target_series, pd.DataFrame): + # If we accidentally got a DataFrame, take the first column + target_values = target_series.iloc[:, 0].values + else: + target_values = target_series.values + + for lag in self.target_lags: + lag_column_name = f"{self.target_colname}_lag_{lag}" + # Copy target values instead of shifting for inference + result[lag_column_name] = target_values + console.log(f"Created lagged column: [cyan]{lag_column_name}") + lag_column_names.append(lag_column_name) + + return result + + def fit(self, X: pd.DataFrame, y=None): + """ + No-op for inference transformer (no learning needed). + + Args: + X: Input DataFrame + y: Not used + + Returns: + self + """ + console.log("[bold blue]Inference transformer fit (no-op)") + return self + + def transform(self, X: pd.DataFrame) -> pd.DataFrame: + """ + Apply the inference transformation pipeline. + + Args: + X: Input DataFrame with merged process and quality data + + Returns: + Transformed DataFrame ready for model inference + """ + console.log("[bold blue]Inference transformer transform phase") + + # Create a copy for processing + data = X.copy() + + # Step 0: Infer lab timestamp if needed + console.log("[bold blue]Inferring lab timestamp") + data = self._infer_lab_timestamp(data) + + # Step 1: Aggregate data (higher frequency - no target grouping) + console.log("[bold blue]Aggregating data") + data = self._aggregate_data(data) + + # Step 2: Apply learned feature selection + console.log("[bold blue]Applying learned feature selection") + # Set timestamp as index for filtering and ensure it's datetime + data[self.time_colname] = pd.to_datetime(data[self.time_colname]) + data = data.set_index(self.time_colname) + + # Filter to selected features (handle missing columns gracefully) + available_features = [ + col for col in self.selected_features if col in data.columns + ] + missing_features = set(self.selected_features) - set( + available_features + ) + + if missing_features: + console.log( + "[yellow]Warning: Missing features in inference data:" + + f" {missing_features}" + ) + + # Ensure target column is included but avoid duplicates + if self.target_colname not in available_features: + available_features.append(self.target_colname) + + data = data[available_features] + + # Step 3: Create lagged target features + if self.create_lagged_target: + console.log("[bold blue]Creating lagged target features") + data = self._create_lagged_target(data) + + console.log(f"[bold green]Final inference data shape: {data.shape}") + + return data + + +def create_transformers_from_training_transformer( + training_transformer: CourierTrainingTransformer, +) -> tuple[CourierTrainingTransformer, CourierInferenceTransformer]: + """ + Create both training and inference transformers with shared parameters. + + Args: + training_transformer: Fitted training transformer + + Returns: + Tuple of (training_transformer, inference_transformer) + """ + if training_transformer.selected_features_ is None: + raise ValueError("Training transformer must be fitted first.") + + inference_transformer = CourierInferenceTransformer( + selected_features=training_transformer.selected_features_, + aggregation_functions=training_transformer.aggregation_functions, + target_lags=training_transformer.target_lags, + time_colname=training_transformer.time_colname, + target_colname=training_transformer.target_colname, + create_lagged_target=training_transformer.create_lagged_target, + drop_nans=training_transformer.drop_nans, + ) + + return training_transformer, inference_transformer diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py new file mode 100644 index 0000000..6fba3eb --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py @@ -0,0 +1,51 @@ +""" +Functions needed to load parameters from params.yaml tracked with DVC +""" + +import sys +import os +from typing import Optional + +import yaml +from rich.console import Console + + +console = Console() + + +def get_params(stage_fn: Optional[str] = None): + """ + Reads parameters for a given DVC stage from params.yaml. + + The stage name is inferred from the name of the python file that calls this + function. + Args: + stage_fn (str): Name of the stage. If None, the name of the file + that calls this function is used. Defaults to None. + Returns: + dict with parameters for the stage + Raises: + KeyError: if the stage name is not found in params.yaml + """ + + if stage_fn is None: + stage_fn = os.path.basename(sys.argv[0]).replace(".py", "") + + try: + params = yaml.safe_load(open("params.yaml"))[stage_fn] + except KeyError as exc: + console.print(f'ERROR: Key "{stage_fn}" not in parameters.yaml.') + raise KeyError( + f"Is the stage file name ({sys.argv[0]}) " + + "the same as the stage name in params.yaml?" + ) from exc + try: + all_params = yaml.safe_load(open("params.yaml"))["all"] + params = {**params, **all_params} + except KeyError: + console.print( + '[orange]WARNING: Key "all" not in parameters.yaml.' + + "Only returning stage parameters." + ) + + return params diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py new file mode 100644 index 0000000..9017a1b --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py @@ -0,0 +1,325 @@ +""" +Module with functions for wrapping time series models for MLflow. +""" + +import os +import tempfile +import pickle +from typing import Optional, Union, Dict, Any, List + +import mlflow.pyfunc +import pandas as pd +import numpy as np +from mlflow.models import ModelSignature +from ..models.stacking_time_series import StackingTimeSeriesModel +from ..data.transformers import ( + create_transformers_from_training_transformer, + CourierTrainingTransformer, +) + + +class StackingWrapper(mlflow.pyfunc.PythonModel): # type: ignore + """ + MLflow wrapper for StackingTimeSeriesModel. + + Allows the model to be saved and served via MLflow's pyfunc interface. + """ + + def __init__(self, model: Optional[StackingTimeSeriesModel] = None): + self.model = model + + @property + def _console(self): + from rich.console import Console + + return Console() + + def load_context(self, context: Any) -> None: + """Load model from artifact path in MLflow context.""" + try: + model_path = context.artifacts["model"] + self.model = StackingTimeSeriesModel.load(model_path) + self._console.log("[green]Model loaded from context[/green]") + except Exception as e: + self._console.print(f"[red]Error loading model: {e}[/red]") + raise + + def predict( + self, + context: Any, + model_input: Union[pd.DataFrame, np.ndarray, Dict[str, Any]], + ) -> Union[pd.Series, pd.DataFrame, np.ndarray]: + """Run inference using the wrapped model.""" + if self.model is None: + raise ValueError("Model not loaded. Call load_context first.") + + # Extract data from input + if isinstance(model_input, dict): + X = model_input.get("data") + if X is None: + raise ValueError("Dict input must contain 'data' key.") + else: + X = model_input + + # Ensure DataFrame input (models expect pandas DataFrames) + if not isinstance(X, pd.DataFrame): + raise ValueError("Input must be a pandas DataFrame.") + + try: + predictions = self.model.predict(X) + return predictions.to_frame(name=self.model.target_col) + except Exception as e: + self._console.print(f"[red]Prediction failed: {e}[/red]") + raise + + def get_model_summary(self) -> str: + """Return human-readable model summary.""" + return self.model.summary() if self.model else "No model loaded" + + def store_model( + self, + path: Optional[str] = None, + artifact_path: str = "stacking_model", + signature: Optional[ModelSignature] = None, + pip_requirements: Optional[Union[str, list]] = None, + code_path: Optional[List[str]] = None, + to_disk: bool = False, + ) -> None: + """ + Store the model using MLflow pyfunc interface. + + Logs to the current MLflow run by default. Optionally saves locally. + + Args: + path: Local path to save model (required if to_disk=True) + artifact_path: MLflow artifact path + signature: Optional MLflow model signature + pip_requirements: pip requirements (list or path) + code_path: List of local Python source files/directories to bundle + to_disk: Save locally if True, otherwise logs to MLflow + """ + if self.model is None: + raise ValueError("No model to store.") + + with tempfile.TemporaryDirectory() as tmp: + model_artifact = os.path.join(tmp, "stacking_model.pkl") + self.model.save(model_artifact, compression="lzma") + + common_args = { + "python_model": self, + "artifacts": {"model": model_artifact}, + } + if signature: + common_args["signature"] = signature + if pip_requirements: + common_args["pip_requirements"] = pip_requirements + if code_path: + common_args["code_path"] = code_path + + if to_disk: + if not path: + raise ValueError("`path` required for to_disk=True.") + mlflow.pyfunc.save_model(path=path, **common_args) + self._console.log( + f"[blue]Model saved locally to {path}[/blue]" + ) + else: + mlflow.pyfunc.log_model( + artifact_path=artifact_path, **common_args + ) + self._console.log( + f"[green]Model logged to MLflow at '{artifact_path}'" + ) + + def __getstate__(self): + state = self.__dict__.copy() + state["model"] = None # avoid double saving + return state + + def __setstate__(self, state): + self.__dict__.update(state) + + +class TransformerWrapper(mlflow.pyfunc.PythonModel): # type: ignore + """ + MLflow wrapper for data transformers. + + Allows transformers to be saved and served via MLflow's pyfunc interface. + Supports both training and inference transformers. + """ + + def __init__( + self, + transformer: Optional[CourierTrainingTransformer] = None, + ): + """ + Initialize the transformer wrapper. + + Args: + transformer: The training transformer to wrap + """ + self.training_transformer = transformer + self.inference_transformer = None + + @property + def _console(self): + from rich.console import Console + + return Console() + + def load_context(self, context: Any) -> None: + """ + Load training transformer from artifact path and create + inference transformer. + """ + try: + transformer_path = context.artifacts["transformer"] + + with open(transformer_path, "rb") as f: + self.training_transformer = pickle.load(f) + + _, self.inference_transformer = ( + create_transformers_from_training_transformer( + self.training_transformer + ) + ) + + self._console.log( + "[green]Training transformer loaded and inference transformer " + + "created from context[/green]" + ) + except Exception as e: + self._console.print(f"[red]Error loading transformer: {e}[/red]") + raise + + def predict( + self, + context: Any, + model_input: Union[pd.DataFrame, np.ndarray, Dict[str, Any]], + transformer_type: str = "inference", + ) -> Union[pd.Series, pd.DataFrame, np.ndarray]: + """ + Transform data using the selected transformer. + + Args: + context: MLflow context + model_input: Input data to transform (pandas DataFrame expected) + transformer_type: Either "training" or "inference" + """ + if transformer_type == "training": + transformer = self.training_transformer + elif transformer_type == "inference": + transformer = self.inference_transformer + else: + raise ValueError( + "transformer_type must be 'training' or 'inference'" + ) + + if transformer is None: + raise ValueError( + f"{transformer_type.title()} transformer not loaded. " + + "Call load_context first." + ) + + # Extract data from input + if isinstance(model_input, dict): + X = model_input.get("data") + if X is None: + raise ValueError("Dict input must contain 'data' key.") + else: + X = model_input + + # Ensure DataFrame input (transformers expect pandas DataFrames) + if not isinstance(X, pd.DataFrame): + raise ValueError("Input must be a pandas DataFrame.") + + try: + # Apply transformer + transformed_data = transformer.transform(X) + return transformed_data + + except Exception as e: + self._console.print(f"[red]Transformation failed: {e}[/red]") + raise + + def get_transformer_summary(self) -> str: + """Return human-readable transformer summary.""" + if self.training_transformer is None: + return "No training transformer loaded" + + training_class = self.training_transformer.__class__.__name__ + inference_status = ( + "available" if self.inference_transformer else "not created" + ) + return ( + f"{training_class} (training loaded, inference {inference_status})" + ) + + def store_transformer( + self, + path: Optional[str] = None, + artifact_path: str = "transformer", + signature: Optional[ModelSignature] = None, + pip_requirements: Optional[Union[str, list]] = None, + code_path: Optional[List[str]] = None, + to_disk: bool = False, + ) -> None: + """ + Store the training transformer using MLflow pyfunc interface. + + Logs to the current MLflow run by default. Optionally saves locally. + + Args: + path: Local path to save transformer (required if to_disk=True) + artifact_path: MLflow artifact path + signature: Optional MLflow model signature + pip_requirements: pip requirements (list or path) + code_path: List of local Python source files/directories to bundle + to_disk: Save locally if True, otherwise logs to MLflow + """ + if self.training_transformer is None: + raise ValueError("No training transformer to store.") + + with tempfile.TemporaryDirectory() as tmp: + transformer_artifact = os.path.join( + tmp, "training_transformer.pkl" + ) + with open(transformer_artifact, "wb") as f: + pickle.dump(self.training_transformer, f) + + common_args = { + "python_model": self, + "artifacts": {"transformer": transformer_artifact}, + } + if signature: + common_args["signature"] = signature + if pip_requirements: + common_args["pip_requirements"] = pip_requirements + if code_path: + common_args["code_path"] = code_path + + if to_disk: + if not path: + raise ValueError("`path` required for to_disk=True.") + mlflow.pyfunc.save_model(path=path, **common_args) + self._console.log( + "[blue]Training transformer saved locally to " + + f"{path}[/blue]" + ) + else: + mlflow.pyfunc.log_model( + artifact_path=artifact_path, **common_args + ) + self._console.log( + "[green]Training transformer logged to MLflow at " + + f"'{artifact_path}'[/green]" + ) + + def __getstate__(self): + state = self.__dict__.copy() + state["training_transformer"] = None # avoid double saving + state["inference_transformer"] = None # avoid double saving + return state + + def __setstate__(self, state): + self.__dict__.update(state) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py new file mode 100644 index 0000000..1701c19 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py @@ -0,0 +1,24 @@ +""" +Models package for time series forecasting. + +This package provides standardized interfaces and implementations for +various time series forecasting models. +""" + +from .base import ( + TimeSeriesModel, + UnivariateTimeSeriesModel, + MultivariateTimeSeriesModel, +) +from .factory import create_model, load_model, get_available_models +from .evaluation import timeseries_metrics + +__all__ = [ + "TimeSeriesModel", + "UnivariateTimeSeriesModel", + "MultivariateTimeSeriesModel", + "create_model", + "load_model", + "get_available_models", + "timeseries_metrics", +] diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py new file mode 100644 index 0000000..abf5d6d --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py @@ -0,0 +1,389 @@ +""" +ARIMA univariate time series forecasting model implementation. +""" + +from typing import Optional, Tuple + +import numpy as np +import pandas as pd +from statsmodels.tsa.arima.model import ARIMA, ARIMAResults +from rich.console import Console + +from .base import UnivariateTimeSeriesModel, ensure_fitted + +console = Console() + + +class ARIMAModel(UnivariateTimeSeriesModel): + """ARIMA model for univariate time series forecasting. + + This class implements an ARIMA model for forecasting univariate time + series data. It provides methods for fitting the model, making predictions, + forecasting future values, and updating the model with new data. + + Attributes: + order (Tuple[int, int, int]): The (p, d, q) order of the ARIMA model. + model_ (Optional[ARIMA]): The ARIMA model instance. + result_ (Optional[ARIMAResults]): The fitted ARIMA model results. + training_series_ (Optional[pd.Series]): The training data used to fit + the model. + """ + + def __init__( + self, + order: Tuple[int, int, int] = (1, 0, 0), + name: Optional[str] = None, + time_col: str = "ds", + target_col: str = "y", + random_seed: int = 42, + forecast_horizon: int = 2, + ) -> None: + """Initializes the ARIMAModel with specified parameters. + + Args: + order (Tuple[int, int, int]): The (p, d, q) order of the ARIMA + model. + name (Optional[str]): The name of the model. + time_col (str): The name of the time column in the input data. + target_col (str): The name of the target column in the input data. + random_seed (int): The random seed for reproducibility. + forecast_horizon (int): The number of steps to forecast ahead. + """ + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + ) + self.order: Tuple[int, int, int] = order + self.model_: Optional[ARIMA] = None + self.result_: Optional[ARIMAResults] = None + self.training_series_: pd.Series = pd.Series(dtype=float) + self.observed_series_: pd.Series = pd.Series(dtype=float) + self.backtest_predictions_: Optional[pd.Series] = None + self.forecast_horizon: int = forecast_horizon + + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """ + Fits the ARIMA model to the provided training data. + + Args: + y: The target time series data. + X: Optional exogenous variables. + X_val: Validation feature matrix (not used for ARIMA). + y_val: Validation target series (not used for ARIMA). + """ + y_array: np.ndarray = self._validate_y(y) + self.training_series_ = y.copy() + self.observed_series_ = y.copy() + self.model_ = ARIMA(y_array, order=self.order) + self.result_ = self.model_.fit() + + @ensure_fitted + def predict(self, X: Optional[pd.DataFrame] = None) -> pd.Series: + """ + Generates in-sample predictions from the fitted ARIMA model. + + After this method is called, if X is provided, the model will be + updated with the new data, but the coefficients will not be refit. + This is useful for generating predictions on new data without + retraining the model. + + Args: + X (Optional[pd.DataFrame]): Optional dataframe with future + measurements of y for in-sample predictions. + If None, the model will predict on the observed data + (observed_series_). + + Returns: + pd.Series: The in-sample predictions. + + Raises: + ValueError: If the model has not been fitted yet. + """ + if self.result_ is None: + raise ValueError("Model is not fitted.") + + if X is None: + fitted_values = self.result_.fittedvalues + if fitted_values is None: + raise ValueError("Fitted values are None") + return pd.Series( + fitted_values, + index=self.training_series_.index[: len(fitted_values)], + name=self.target_col, + ) + + # Validate the input data + target_series = ( + X[self.target_col] if self.target_col in X else X.iloc[:, 0] + ) + if not isinstance(target_series, pd.Series): + target_series = pd.Series(target_series, index=X.index) + + X_validated = self._validate_y(target_series) + + # Update the model with the validated data without refitting + self.update(pd.Series(X_validated, index=X.index), refit=False) + + if self.result_ is None: + raise ValueError("Model result is None after update") + + fitted_values = self.result_.fittedvalues + if fitted_values is None: + raise ValueError("Fitted values are None") + + return_series = pd.Series( + fitted_values[-len(X) :], index=X.index, name=self.target_col + ) + + return return_series + + @ensure_fitted + def forecast(self, forecast_horizon: int) -> np.ndarray: + """Generates out-of-sample forecasts from the fitted ARIMA model. + TODO: Change return to include index of the forecasted values. + + Args: + forecast_horizon (int): The number of steps to forecast ahead. + + Returns: + np.ndarray: The out-of-sample forecasts. + + Raises: + ValueError: If the model has not been fitted yet. + """ + if self.result_ is None: + raise ValueError("Model is not fitted.") + return self.result_.forecast(steps=forecast_horizon) + + @ensure_fitted + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Perform comprehensive backtesting with periodic retraining. + + This method implements walk-forward validation with periodic + retraining, providing robust evaluation of model performance in + production-like scenarios. Uses 1-step ahead forecasting by default. + + Args: + y: Target time series for backtesting + X: Unused (included for base class compatibility) + retrain_every: Number of steps between model retraining + reuse_previous_execution: Whether to reuse previous backtest results + + Returns: + Series of backtested predictions indexed by timestamp + + Raises: + ValueError: If parameters are invalid or data is insufficient + RuntimeError: If backtesting fails + """ + if self.result_ is None: + raise ValueError("Model is not fitted") + if self.training_series_ is None: + raise ValueError("No training series found") + + # Handle reuse of previous execution + if reuse_previous_execution and self.backtest_predictions_ is not None: + expected_index = y.index + if ( + len(self.backtest_predictions_) == len(expected_index) + and (self.backtest_predictions_.index == expected_index).all() + ): + console.log( + "[yellow]Reusing previous backtest results[/yellow]" + ) + return self.backtest_predictions_ + else: + console.log( + "[yellow]Previous results incompatible, running new" + + " backtest[/yellow]" + ) + + try: + console.log( + f"[blue]Starting ARIMA backtest with {self.forecast_horizon}" + + "-step forecasting...[/blue]" + ) + + # Validate and prepare data + y_sorted = y.sort_index() + + # Check for overlapping data + training_series = self.training_series_ + if any(t in training_series.index for t in y_sorted.index): + console.print( + "[yellow]Warning: Backtest data overlaps with training" + + " data[/yellow]" + ) + + # Initialize backtesting + predictions = [] + + # Start with training data + current_series = training_series.copy() + + total_steps = len(y_sorted) + console.log( + f"[blue]Running {total_steps} backtest steps with retraining" + + f" every {retrain_every} steps...[/blue]" + ) + + # Create initial model state + current_model = ARIMA(current_series.values, order=self.order) + current_result = current_model.fit() + + # Perform walk-forward validation + for i, (timestamp, actual_value) in enumerate(y_sorted.items()): + if i % 50 == 0 and i > 0: # Progress logging + console.log( + f"[blue]Backtest progress: {i}/{len(y_sorted)}[/blue]" + ) + + try: + # Check if we need to retrain + if i % retrain_every == 0 and i > 0: + console.log( + f"[blue]Retraining model at step {i}[/blue]" + ) + current_model = ARIMA( + current_series.values, order=self.order + ) + current_result = current_model.fit() + + # Generate forecast_horizon-step ahead forecast + forecast = current_result.forecast( + steps=self.forecast_horizon + )[0] + predictions.append((timestamp, forecast)) + + # Update the series with actual observed value + current_series = pd.concat( + [ + current_series, + pd.Series([actual_value], index=[timestamp]), + ] + ) + + # For ARIMA, we can extend the model without full refit + if i % retrain_every != 0: + try: + current_result = current_result.extend( + [actual_value], refit=False + ) + except Exception: + # If extend fails, do a quick refit + current_model = ARIMA( + current_series.values, order=self.order + ) + current_result = current_model.fit() + + except Exception as step_error: + console.print( + f"[yellow]Error at step {i}: {step_error}, using" + + " NaN[/yellow]" + ) + predictions.append((timestamp, np.nan)) + + # Still update the series for continuity + current_series = pd.concat( + [ + current_series, + pd.Series([actual_value], index=[timestamp]), + ] + ) + + # Create results series + if predictions: + pred_index, pred_values = zip(*predictions) + self.backtest_predictions_ = pd.Series( + pred_values, + index=pd.Index(pred_index), + name=f"{self.target_col}_backtest", + ) + else: + self.backtest_predictions_ = pd.Series( + dtype=float, name=f"{self.target_col}_backtest" + ) + + console.log( + "[green]ARIMA backtest completed: " + + f"{len(self.backtest_predictions_)} predictions[/green]" + ) + return self.backtest_predictions_ + + except Exception as e: + console.print(f"[red]ARIMA backtest failed: {e}[/red]") + raise RuntimeError(f"Failed to perform backtest: {e}") from e + + @ensure_fitted + def summary(self) -> str: + """Generates a summary of the fitted ARIMA model. + + Returns: + str: The summary of the fitted model. + + Raises: + ValueError: If the model has not been fitted yet. + """ + if self.result_ is None: + raise ValueError("Model is not fitted.") + return str(self.result_.summary()) + + @ensure_fitted + def update(self, new_data: pd.Series, refit: bool = True) -> None: + """Updates the ARIMA model with new observed data. + + This method allows for two modes of updating the model: + 1. **Refitting**: The model is retrained on the combined dataset + (original training data + new data). + 2. **Incremental Update**: The model is updated using the new data + without retraining, preserving the original model parameters. + + Args: + new_data (pd.Series): New observed values to update the model with. + refit (bool, optional): If True, the model is retrained on the + combined dataset. Defaults to True. + + Raises: + TypeError: If `new_data` is not a pandas Series. + ValueError: If `new_data` is empty or if the model has not been + fitted yet. + """ + if not isinstance(new_data, pd.Series): + raise TypeError("new_data must be a pandas Series.") + if new_data.empty: + raise ValueError("new_data is empty.") + + if refit: + self.training_series_ = pd.concat( + [self.training_series_, new_data] + ) + self.observed_series_ = self.training_series_.copy() + y_array = self._validate_y(self.training_series_) + self.model_ = ARIMA(y_array, order=self.order) + self.result_ = self.model_.fit() + else: + self.observed_series_ = pd.concat( + [self.observed_series_, new_data] + ) + y_array = self._validate_y(self.observed_series_) + + if self.result_ is None: + raise ValueError("Model is not fitted.") + self.result_ = self.result_.apply(y_array, refit=False) + if self.result_ is not None: + self.model_ = self.result_.model diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py new file mode 100644 index 0000000..4afdfaf --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py @@ -0,0 +1,824 @@ +""" +Base model classes and interfaces. + +This module defines base classes with consistent interfaces for all models, +promoting modular model development. It includes essential functionality +for model fitting, prediction, evaluation, saving, and loading, while +abstracting common behaviors into base classes. + +While full compatibility with scikit-learn is not guaranteed, the base +classes provide a consistent interface for model fitting, prediction, +evaluation, saving, and loading, which should be sufficient for most +use cases. + +Key components: +- **Model**: Abstract base class for all models, providing core utilities and + interfaces. +- **TimeSeriesModel**: Abstract base class for time series models, adding + time-based functionality. +- **UnivariateTimeSeriesModel**: Base class for univariate time series models. +- **MultivariateTimeSeriesModel**: Base class for multivariate time series + models that use exogenous features. + +The module also includes utility functions such as `ensure_fitted`, which + ensures models are fitted before calling certain methods. + +Modules in this package should inherit from these base classes and implement + the required methods. + +TODO: + - Add methods to create lagged features for time series models. +""" + +import uuid +import joblib +from abc import ABC, abstractmethod +from typing import Optional, List, Sequence, Tuple, cast, Any, Protocol +from contextlib import contextmanager + +import pandas as pd +import numpy as np +import shap +import matplotlib.pyplot as plt +from rich.console import Console + +from sklearn.base import BaseEstimator, RegressorMixin +from sklearn.utils.validation import check_array, check_X_y +from sklearn.exceptions import NotFittedError + +console = Console() + + +class PredictorProtocol(Protocol): + """Protocol for models with predict method and optional imputation.""" + + def predict(self, X: Any) -> Any: ... + def _impute_missing_values(self, X: Any) -> Any: ... + + +def ensure_fitted(method): + """ + Decorator to ensure the model is fitted before calling the method. + + Raises: + sklearn.exceptions.NotFittedError: If the model is not fitted. + Usage: + @ensure_fitted + def predict(self, X): # Or other methods requiring fit + pass + """ + + def wrapper(self, *args, **kwargs): + is_fitted = self.__sklearn_is_fitted__() + if not is_fitted: + raise NotFittedError( + f"This {self.__class__.__name__} instance is not fitted yet. " + "Call 'fit' with appropriate arguments before using this " + "method." + ) + return method(self, *args, **kwargs) + + return wrapper + + +class Model(BaseEstimator, ABC): + """ + Abstract base class for all models. + + Provides core utilities, input validation, and interface consistency + for time series models. Compatible with scikit-learn workflows. + """ + + def __init__(self, name: Optional[str] = None, random_seed: int = 42): + """ + Initialize the model with a unique name and random seed. + + Args: + name: Optional identifier; auto-generated if None. + random_seed: Seed for reproducibility. + """ + self.name = name or f"{self.__class__.__name__}_{uuid.uuid4().hex}" + self.random_seed = random_seed + self.feature_names_in_: Optional[List[str]] = None + self.n_features_in_: Optional[int] = None + self._is_fitted = False + + def fit( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> "Model": + """ + Trains the model. + + Handles basic input validation for y and sets internal fitted + state after calling _fit_logic. + + 'X' is optional to account for univariate time series models. + + Args: + y: The target variable. + X: Optional exogenous variables. + X_val: Optional validation feature matrix. + y_val: Optional validation target series. + Raises: + TypeError: If y is not a pandas Series. + If X is provided, it must be a pandas DataFrame. + If X_val and y_val are provided, they must be pandas DataFrames + and Series respectively. + + Returns: + Self for chaining. + """ + if not isinstance(y, pd.Series): + raise TypeError("Input 'y' (target) must be a pandas Series.") + + self._fit_logic(y, X, X_val, y_val) + self._is_fitted = True + return self + + @abstractmethod + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """ + Core fitting logic to be implemented by subclasses with + optional validation data. + + Args: + y: The target variable. + X: Optional exogenous variables. + X_val: Validation feature matrix (optional). + y_val: Validation target series (optional). + """ + raise NotImplementedError("Subclasses must implement _fit_logic().") + + @ensure_fitted + @abstractmethod + def predict(self, X: Optional[pd.DataFrame] = None) -> Sequence: + """ + Predict values. + + Args: + X: Optional features for prediction. For univariate models + not using exogenous variables, this might be None or + contain future timestamps. Multivariate models will + require X. + + Returns: + NumPy array or similar sequence of predictions. + """ + raise NotImplementedError("Subclasses must implement predict().") + + def fit_predict( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> Sequence: + """ + Fits model and returns predictions on the same data. + + Args: + y: The target time series. + X: Optional exogenous variables. + + Returns: + Predictions for the input data. + """ + return self.fit(y, X, X_val, y_val).predict(X) + + def save(self, path: str) -> None: + """ + Saves model to disk using joblib. + """ + joblib.dump(self, path) + + @classmethod + def load(cls, path: str) -> "Model": + """ + Loads model from disk using joblib. + """ + return joblib.load(path) + + def __str__(self) -> str: + return f"{self.__class__.__name__}(name={self.name})" + + def __sklearn_is_fitted__(self) -> bool: + """ + Check fitted status and return a Boolean value. + """ + return hasattr(self, "_is_fitted") and self._is_fitted + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(name={self.name})" + + @contextmanager + def model_state_preservation(self): + """Context manager to preserve model state during operations.""" + original_state = self._get_state_snapshot() + try: + yield + except Exception: + self._restore_state_snapshot(original_state) + raise + + def _get_state_snapshot(self) -> dict: + """Get snapshot of current model state.""" + return { + "name": self.name, + "is_fitted": getattr(self, "_is_fitted", False), + "feature_names": self.feature_names_in_, + "n_features": self.n_features_in_, + } + + def _restore_state_snapshot(self, snapshot: dict) -> None: + """Restore model state from snapshot.""" + self.name = snapshot["name"] + self._is_fitted = snapshot["is_fitted"] + self.feature_names_in_ = snapshot["feature_names"] + self.n_features_in_ = snapshot["n_features"] + + def get_params_dict(self) -> dict: + """Get model parameters as dictionary for logging/serialization.""" + return { + "name": self.name, + "random_seed": self.random_seed, + "n_features_in_": self.n_features_in_, + } + + def summary(self) -> str: + """Generate a summary string of the model.""" + params = self.get_params_dict() + fitted_status = ( + "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" + ) + + summary_lines = [ + f"Model: {self.__class__.__name__}", + f"Status: {fitted_status}", + f"Features: {params.get('n_features_in_', 'Unknown')}", + ] + + return "\n".join(summary_lines) + + +class TimeSeriesModel(Model): + """ + Abstract base class for time series forecasting models. + + Extends the base Model class with specific methods for time series + data handling and evaluation. + """ + + def __init__( + self, + name: Optional[str] = None, + time_col: str = "ds", + target_col: str = "y", + random_seed: int = 42, + n_lags: int = 0, + sampling_freq: Optional[str] = None, + ): + super().__init__(name=name, random_seed=random_seed) + self.time_col = time_col + self.target_col = target_col + self.n_lags = n_lags + self.sampling_freq = sampling_freq + + self.training_series_: Optional[pd.Series] = None + self.model_: Optional[BaseEstimator] = None + + # Validate configuration + self._validate_configuration() + + def _validate_configuration(self) -> None: + """Validate model configuration.""" + if self.n_lags < 0: + raise ValueError("n_lags must be non-negative") + + def _validate_y(self, y: pd.Series) -> np.ndarray: + """ + Validates the target variable (y) for the model. + + Ensures y is a pandas Series and checks its name against + the expected target column name. Converts y to a NumPy array. + The series name can be None, but if it is set, it should match + the expected target column name. + + Args: + y: The target variable as a pandas Series. + + Returns: + A NumPy array of the target variable. + + Raises: + TypeError: If y is not a pandas Series. + """ + # Check if y is a pandas Series + if not isinstance(y, pd.Series): + raise TypeError("Input 'y' (target) must be a pandas Series.") + if (y.name is not None) and (y.name != self.target_col): + raise ValueError( + f"Expected target column name '{self.target_col}', " + f"but got '{y.name}'." + ) + return check_array(y, ensure_2d=False) + + @ensure_fitted + @abstractmethod + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Performs backtesting on the time series data. + + Args: + y: The target time series data. + X: Optional exogenous features. + retrain_every: Number of steps after which to retrain the model. + reuse_previous_execution: Whether to reuse the previous execution + of a backtest. If True, any overlapping data between the + previous execution and the current execution will be used + without retraining the model. + Returns: + Series of predictions for each step in the time series. + """ + + raise NotImplementedError("Subclasses must implement backtest().") + + def get_params_dict(self) -> dict: + """Get model parameters as dictionary for logging/serialization.""" + base_params = super().get_params_dict() + ts_params = { + "time_col": self.time_col, + "target_col": self.target_col, + "n_lags": self.n_lags, + "sampling_freq": self.sampling_freq, + } + return {**base_params, **ts_params} + + +class UnivariateTimeSeriesModel(TimeSeriesModel, RegressorMixin): + """ + Base class for univariate time series models. + + Only supports regression settings. Concrete subclasses + must implement `_fit_logic` and `predict`. + """ + + @abstractmethod + @ensure_fitted + def forecast(self, forecast_horizon: int) -> Sequence: + """ + Forecast into the future for a given number of steps. + + Args: + forecast_horizon: Number of future time steps to forecast. + + Returns: + Sequence of forecasted values. + """ + raise NotImplementedError("Subclasses must implement forecast().") + + +class MultivariateTimeSeriesModel(TimeSeriesModel): + """ + Base class for multivariate time series models. + + This class provides a foundation for time series models that utilize + multiple exogenous features (X) to predict a target variable (y). + It supports both regression and classification tasks. + + Attributes: + selected_features_: List of feature names selected for the model. + learning_task: Type of learning task ('regression', 'binary', + 'multiclass'). + differentiate_target: Whether to apply differencing to make series + stationary. + bins: Bin edges for multiclass classification target + transformation. + + Example: + >>> class MyModel(MultivariateTimeSeriesModel): + ... def _fit_logic(self, y, X=None, **kwargs): + ... # Implementation here + ... pass + ... def predict(self, X=None): + ... # Implementation here + ... return predictions + """ + + def __init__( + self, + name: Optional[str] = None, + time_col: str = "ds", + target_col: str = "y", + random_seed: int = 42, + n_lags: int = 0, + sampling_freq: Optional[str] = None, + differentiate_target: bool = False, + bins: Optional[List[float]] = None, + learning_task: Optional[str] = None, + ): + # Set attributes before calling parent constructor + # This is needed because parent constructor calls + # _validate_configuration + self.selected_features_: Optional[List[str]] = None + self.learning_task: Optional[str] = learning_task + self.differentiate_target = differentiate_target + self.bins = bins + self.model_: Optional[PredictorProtocol] = None + + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + n_lags=n_lags, + sampling_freq=sampling_freq, + ) + + # Additional validation for multivariate models + self._validate_learning_task() + + def _validate_learning_task(self) -> None: + """Validate learning task configuration.""" + valid_tasks = {"regression", "binary", "multiclass", None} + if self.learning_task not in valid_tasks: + raise ValueError( + f"Invalid learning_task: {self.learning_task}. " + + f"Must be one of {valid_tasks}" + ) + + if self.learning_task == "multiclass" and not self.bins: + raise ValueError( + "bins must be provided for multiclass learning_task" + ) + + def _get_default_loss_function( + self, provided_loss: Optional[str] + ) -> str: + """ + Get default loss function based on learning task. + + Args: + provided_loss: User-provided loss function (takes precedence) + + Returns: + str: Appropriate loss function for the learning task + """ + if provided_loss is not None: + return provided_loss + + if self.learning_task == "regression": + return "RMSE" + elif self.learning_task == "binary": + return "Logloss" + elif self.learning_task == "multiclass": + return "MultiClass" + else: + return "RMSE" + + def _validate_configuration(self) -> None: + """Validate model configuration.""" + super()._validate_configuration() + + if self.differentiate_target and self.learning_task in [ + "binary", + "multiclass", + ]: + console.print( + "[yellow]Warning: Using differentiation with classification " + + "tasks may not be appropriate[/yellow]" + ) + + @ensure_fitted + def feature_importance(self) -> Optional[pd.DataFrame]: + """ + Returns feature importance if implemented by subclass. + + Returns: + A DataFrame with feature names and their importance scores, + or None if not applicable. + """ + return None + + def _validate_X_y( + self, X: pd.DataFrame, y: pd.Series, allow_nan: bool = True + ) -> Tuple[np.ndarray, np.ndarray]: + """ + Validates input features (X) and target (y). + + Infers and sets `feature_names_in_` and `n_features_in_`. + This method should be called within the `_fit_logic` of + concrete subclasses that use exogenous features. + + Args: + X: DataFrame of input features. + y: Series for the target variable. + allow_nan: If True, allows NaN values in X and y. + Raises: + TypeError: If X is not a DataFrame or y is not a Series. + ValueError: If the number of features in X does not match + the expected number of features. + + Returns: + Tuple of validated NumPy arrays (X_array, y_array). + """ + if allow_nan: + X_array, y_array = check_X_y(X, y, force_all_finite=False) + else: + X_array, y_array = check_X_y(X, y, force_all_finite=True) + + if hasattr(X, "columns"): + console.log( + f"Validating input features with columns: {X.columns.tolist()}" + ) + self.feature_names_in_ = list(X.columns) + else: + console.log( + "Input features do not have column names, using default names." + ) + self.feature_names_in_ = [ + f"feature_{i}" for i in range(X_array.shape[1]) + ] + + self.n_features_in_ = X_array.shape[1] + return X_array, y_array + + def _validate_X( + self, X: pd.DataFrame, allow_nan: bool = True + ) -> np.ndarray: + """ + Validates input features (X) before prediction or scoring. + + Ensures consistency with features seen during fit. This should + be called by concrete subclasses in `predict`, `score`, etc. + + Args: + X: DataFrame of input features. + allow_nan: If True, allows NaN values in X. + + Returns: + Validated NumPy array of X. + """ + if allow_nan: + X_array = check_array(X, force_all_finite=False) + else: + X_array = check_array(X, force_all_finite=True) + # If the model has been fitted, ensure the input features + # match the features seen during fit. + if self.feature_names_in_ is not None: + if not set(self.feature_names_in_).issubset(X.columns): + raise ValueError( + "Input features do not match the features seen during fit." + + f" Expected features: {self.feature_names_in_}, " + + f"but got: {list(X.columns)}." + ) + + return X_array + + def _transform_target_to_multiclass( + self, y: pd.Series, bins: Optional[List[float]] = None + ) -> pd.Series: + """ + Transforms the target variable into a multiclass classification + target. + + If bins are provided, uses pd.cut to categorize the target into + discrete classes. If not, binarize the target at zero (0). + + Args: + y: The target variable as a pandas Series. + bins: Optional list of bin edges for categorization. + + Returns: + A pandas Series with transformed classification targets. + """ + if bins is not None: + # pd.cut returns a Categorical, convert to Series with integer + # codes + categories = pd.cut(y, bins=bins, labels=False) + return pd.Series(categories, index=y.index) + + return (y > 0).astype(int) + + def _transform_target_to_binary( + self, y: pd.Series, threshold: float = 0.0 + ) -> pd.Series: + """ + Transforms the target variable into a binary classification target. + + Binarizes the target at the specified threshold (default is 0.0). + + Args: + y: The target variable as a pandas Series. + threshold: The threshold for binarization. + + Returns: + A pandas Series with binary classification targets. + """ + return (y > threshold).astype(int) + + def _preprocess_data( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> Tuple[ + pd.Series, + Optional[pd.DataFrame], + Optional[pd.Series], + Optional[pd.DataFrame], + ]: + """ + Internal method to handle common data preprocessing operations. + + Args: + y: The target time series data + X: The feature matrix (including exogenous features) + X_val: Validation feature matrix (optional) + y_val: Validation target series (optional) + + Returns: + A tuple containing: + - processed y series + - processed X dataframe (optional) + - processed y_val series (optional) + - processed X_val dataframe (optional) + """ + # Apply differentiation if enabled + if self.differentiate_target: + y = y.diff().dropna() + if X is not None: + X = X.loc[y.index] + + # Transform target for classification if needed + if self.learning_task == "binary": + y = self._transform_target_to_binary(y) + elif self.learning_task == "multiclass": + y = self._transform_target_to_multiclass(y, self.bins) + + # Process validation data if provided + if y_val is not None: + if X_val is None: + raise ValueError( + "Validation features (X_val) must be provided if " + + "validation target (y_val) is given." + ) + y_val = y_val.loc[X_val.index] + if self.differentiate_target: + y_val = y_val.diff().dropna() + X_val = X_val.loc[y_val.index] + if self.learning_task == "binary": + y_val = self._transform_target_to_binary(y_val) + elif self.learning_task == "multiclass": + y_val = self._transform_target_to_multiclass(y_val, self.bins) + + # Filter features if selected_features_ is set + if X is not None and self.selected_features_ is not None: + X = cast(pd.DataFrame, X[self.selected_features_].copy()) + if X_val is not None: + X_val = cast( + pd.DataFrame, X_val[self.selected_features_].copy() + ) + + return y, X, y_val, X_val + + def _prepare_shap_data(self, X: pd.DataFrame) -> pd.DataFrame: + """Prepare data for SHAP analysis.""" + X_processed = X.copy() + + # Remove target column if present + if self.target_col in X_processed.columns: + X_processed = X_processed.drop(columns=[self.target_col]) + + # Filter selected features + if self.selected_features_ is not None: + X_processed = cast( + pd.DataFrame, X_processed[self.selected_features_].copy() + ) + + return X_processed + + def _create_shap_explainer(self, X: pd.DataFrame) -> Any: + """Create appropriate SHAP explainer based on model type.""" + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + + if hasattr(self.model_, "coef_"): # Linear models + try: + # Handle missing values if model supports it + X_clean = self._handle_missing_values_for_shap(X) + return shap.LinearExplainer(self.model_, X_clean) + except Exception as e: + console.print( + f"[yellow]Warning: Linear explainer failed: {e}, " + + "using KernelExplainer[/yellow]" + ) + background = shap.maskers.Independent(X, max_samples=100) + return shap.KernelExplainer( + self.model_.predict, + background, + ) + else: + # Non-linear models + if self.learning_task == "binary": + return shap.TreeExplainer( + self.model_, X, model_output="probability" + ) + else: + return shap.Explainer(self.model_, X) + + def _handle_missing_values_for_shap(self, X: pd.DataFrame) -> pd.DataFrame: + """Handle missing values for SHAP analysis.""" + # Use type ignore for optional method + if hasattr(self.model_, "_impute_missing_values"): + return self.model_._impute_missing_values(X) # type: ignore + else: + return X.dropna() + + def _generate_and_save_plot( + self, explainer: Any, X: pd.DataFrame, path: str + ) -> None: + """Generate and save SHAP plot.""" + shap_values = explainer(X) + + shap.plots.beeswarm(shap_values, show=False) + shap_fig = plt.gcf() + shap_fig.set_size_inches(10, 6) + shap_fig.suptitle(f"SHAP Beeswarm Plot for {self.name}", fontsize=16) + shap_fig.tight_layout() + shap_fig.savefig(path) + plt.clf() + plt.close() + + @ensure_fitted + def shap_beeswarm_plot(self, X: pd.DataFrame, path: str) -> None: + """ + Generates a SHAP beeswarm plot for the model's predictions. + + Args: + X: DataFrame of input features. + path: Path to save the plot file. + + Raises: + ValueError: If model is not fitted. + Exception: If SHAP plot generation fails. + """ + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + + try: + # Prepare data + X_processed = self._prepare_shap_data(X) + + # Create explainer and generate plot + explainer = self._create_shap_explainer(X_processed) + self._generate_and_save_plot(explainer, X_processed, path) + + except Exception as e: + console.print( + f"[red]Error: Failed to generate SHAP plot: {e}[/red]" + ) + raise + + def get_params_dict(self) -> dict: + """Get model parameters as dictionary for logging/serialization.""" + base_params = super().get_params_dict() + mv_params = { + "learning_task": self.learning_task, + "differentiate_target": self.differentiate_target, + "selected_features_": self.selected_features_, + } + return {**base_params, **mv_params} + + def summary(self) -> str: + """Generate a summary string of the model.""" + params = self.get_params_dict() + fitted_status = ( + "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" + ) + + summary_lines = [ + f"Model: {self.__class__.__name__}", + f"Status: {fitted_status}", + f"Features: {params.get('n_features_in_', 'Unknown')}", + f"Task: {params.get('learning_task', 'regression')}", + f"Selected Features: {len(self.selected_features_) if self.selected_features_ else 'All'}", + ] + + return "\n".join(summary_lines) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py new file mode 100644 index 0000000..3721871 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py @@ -0,0 +1,510 @@ +""" +CatBoost implementation for multivariate time series forecasting. + +TODO: + - Implement support for categorical features. +""" + +from typing import Optional, List, Union, Tuple, Dict, Any, cast + +import pandas as pd +from catboost import CatBoostRegressor, CatBoostClassifier, Pool +from rich.console import Console +import numpy as np + +from .base import MultivariateTimeSeriesModel, ensure_fitted + +console = Console() + + +class CatBoostTimeSeriesModel(MultivariateTimeSeriesModel): + """ + CatBoost implementation for multivariate time series forecasting. + + This class wraps the CatBoost models with additional functionality for + time series forecasting, following the MultivariateTimeSeriesModel + interface. Supports both regression and classification tasks with + comprehensive error handling and type safety. + """ + + def __init__( + self, + name: Optional[str] = None, + learning_task: str = "regression", + differentiate_target: bool = False, + n_lags: int = 0, + iterations: int = 1000, + learning_rate: float = 0.1, + depth: int = 6, + loss_function: Optional[str] = None, + bins: Optional[List[float]] = None, + random_seed: int = 42, + time_col: str = "ds", + target_col: str = "y", + verbose: bool = False, + ) -> None: + """ + Initialize the CatBoost time series model. + + Args: + name: Optional identifier for the model + learning_task: Type of learning task, either 'regression', + 'multiclass' or 'binary'. + differentiate_target: Whether to differentiate the target + series before fitting the model. + n_lags: Number of lagged target values included as features. + These lags are expected to already be present in the same + dataset as the exogenous features. + iterations: Number of boosting iterations + learning_rate: Learning rate for the model + depth: Depth of the tree + loss_function: Loss function to optimize + bins: Optional list of bin edges for multiclass classification + random_seed: Random seed for reproducibility + time_col: Name of the time column + target_col: Name of the target column + verbose: Whether to enable verbose output + """ + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + n_lags=n_lags, + learning_task=learning_task, + differentiate_target=differentiate_target, + bins=bins, + ) + + self.iterations = iterations + self.learning_rate = learning_rate + self.depth = depth + self.verbose = verbose + + # Set default loss function based on learning task + self.loss_function = self._get_default_loss_function(loss_function) + + # Initialize model state + self.model_: Optional[Union[CatBoostRegressor, CatBoostClassifier]] = ( + None + ) + self.training_series_: Optional[pd.Series] = None + self.X_train_: Optional[pd.DataFrame] = None + self.backtest_predictions_: Optional[pd.Series] = None + + if self.verbose: + console.log( + "[green]Initialized CatBoostTimeSeriesModel:" + + f" {self.summary()}[/green]" + ) + + def _create_model(self) -> Union[CatBoostRegressor, CatBoostClassifier]: + """Creates a new instance of CatBoost model with current parameters. + + Returns: + A new CatBoost model instance (Regressor or Classifier). + """ + base_params = { + "iterations": self.iterations, + "learning_rate": self.learning_rate, + "depth": self.depth, + "loss_function": self.loss_function, + "random_seed": self.random_seed, + "verbose": self.verbose, + } + + if self.learning_task in ["binary", "multiclass"]: + return CatBoostClassifier( + auto_class_weights="Balanced", **base_params + ) + else: + return CatBoostRegressor(**base_params) + + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """ + Core fitting logic for CatBoost model with optional validation data. + + Args: + y: The target time series data + X: The feature matrix (including exogenous features) + X_val: Validation feature matrix (optional) + y_val: Validation target series (optional) + """ + if X is None or not isinstance(X, pd.DataFrame): + raise ValueError("Feature matrix X must be a non-empty DataFrame.") + + y_processed, X_processed, y_val_processed, X_val_processed = ( + self._preprocess_data(y, X, X_val, y_val) + ) + + # Ensure X is not None after preprocessing + if X_processed is None: + raise ValueError( + "Feature matrix X cannot be None after preprocessing." + ) + + X_array, y_array = self._validate_X_y(X_processed, y_processed) + + self.training_series_ = y_processed.copy() + self.X_train_ = X_processed.copy() + self.model_ = self._create_model() + + eval_set = None + if X_val_processed is not None and y_val_processed is not None: + X_val_array, y_val_array = self._validate_X_y( + X_val_processed, y_val_processed + ) + eval_set = Pool(data=X_val_array, label=y_val_array) + + train_pool = Pool(data=X_array, label=y_array) + + if self.verbose: + console.log( + f"[blue]Training CatBoost model for {self.iterations}" + + " iterations...[/blue]" + ) + + self.model_.fit(train_pool, eval_set=eval_set) + + if self.verbose: + console.log( + "[green]CatBoost model training completed successfully[/green]" + ) + + @ensure_fitted + def predict(self, X: pd.DataFrame) -> pd.Series: + """ + Generate predictions using the fitted CatBoost model. + + Args: + X: The feature matrix for prediction + + Returns: + pd.Series: Predicted values + """ + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + + # Create a copy to avoid modifying the original DataFrame + X_pred = X.copy() + + if self.selected_features_ is not None: + X_pred = cast(pd.DataFrame, X_pred[self.selected_features_]) + + X_array = self._validate_X(X_pred) + predictions = self.model_.predict(X_array) + + # Convert predictions to numpy array if needed + if hasattr(predictions, "squeeze"): + predictions = predictions.squeeze() + elif isinstance(predictions, list): + predictions = np.array(predictions) + + return_series = pd.Series( + predictions, + index=X.index, + name=self.target_col, + ) + return return_series + + @ensure_fitted + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Performs backtesting (walk-forward validation) with periodic + retraining. + + This method simulates a production scenario by iterating through a test + set, making a one-step-ahead prediction, and then retraining the model + periodically with the newly available data. + + Args: + X: DataFrame with features for the backtesting period. + y: Series with the true target values for the backtesting period. + retrain_every: The frequency of retraining. The model will be + retrained every `retrain_every` steps. + reuse_previous_execution: Whether to reuse the previous execution + of a backtest. If True, any overlapping data between the + previous execution and the current execution will be used + without retraining the model. + Returns: + A series of backtested predictions, indexed by the backtest data's + index. + """ + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + if self.training_series_ is None: + raise ValueError("Training series is not set.") + if self.X_train_ is None: + raise ValueError("Training feature matrix is not set.") + if X is None or not isinstance(X, pd.DataFrame): + raise ValueError("Feature matrix X must be a non-empty DataFrame.") + + if reuse_previous_execution: + if self.backtest_predictions_ is None: + raise ValueError("No previous execution found.") + if (self.backtest_predictions_.shape[0] != y.shape[0]) or ( + not (self.backtest_predictions_.index == y.index).all() + ): + raise ValueError( + "Previous execution index does not match y index." + ) + return self.backtest_predictions_ + # Prepare + total_steps = len(X) + predictions = [] + current_model = self.model_ + y_history = self.training_series_.copy() + X_history = self.X_train_.copy() + + # Iterate in chunks instead of single steps + for start in range(0, total_steps, retrain_every): + end = min(start + retrain_every, total_steps) + + # Batch prediction for current chunk + X_chunk = X.iloc[start:end].copy() + if self.selected_features_: + X_chunk = X_chunk[self.selected_features_] + + X_array = self._validate_X(X_chunk) + preds = current_model.predict(X_array) + + # Handle different prediction formats + if hasattr(preds, "squeeze"): + preds = preds.squeeze() + if preds.ndim == 0: # single point + preds = [preds] + predictions.extend(preds) + + # Update training history + y_chunk = y.iloc[start:end] + y_history = pd.concat([y_history, y_chunk]) + X_history = pd.concat([X_history, X_chunk]) + + # Retrain the model for next chunk (if needed) + if end < total_steps: + if self.verbose: + console.print( + f"[cyan]Backtesting: Retraining at step {end}..." + ) + + current_model = self._create_model() + (y_fit, X_fit, _, _) = ( + self._preprocess_data(y_history, X_history) + ) + + # Ensure X_fit is not None after preprocessing + if X_fit is None: + raise ValueError( + "Feature matrix cannot be None after preprocessing." + ) + + X_fit_array, y_fit_array = self._validate_X_y(X_fit, y_fit) + train_pool = Pool(data=X_fit_array, label=y_fit_array) + current_model.fit(train_pool) + + # Store backtest predictions for potential reuse + self.backtest_predictions_ = pd.Series( + predictions, index=X.index, name=f"{self.target_col}_pred" + ) + return self.backtest_predictions_ + + def select_features( + self, + X: pd.DataFrame, + y: pd.Series, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + features_to_select: Optional[int] = None, + algorithm: str = "RecursiveByShapValues", + steps: int = 1, + verbose: bool = False, + ) -> List[str]: + """Identify and select the most important features. + + Uses CatBoost's built-in feature selection capabilities to determine + feature importance and select the most relevant features. + + Args: + X: The feature matrix + y: The target series + X_val: Optional validation feature matrix + y_val: Optional validation target series + features_to_select: Number of features to select. If None, + will select half of the features. + algorithm: Feature selection algorithm. One of: + 'RecursiveByShapValues', 'RecursiveByPredictionValuesChange' + steps: How many times a full model will be trained. + More steps give more accurate results. + verbose: Whether to print progress + + Returns: + List[str]: List of selected feature names + """ + (y_processed, X_processed, y_val_processed, X_val_processed) = ( + self._preprocess_data(y, X, X_val, y_val) + ) + + # Validate input data + if X_processed is None or not isinstance(X_processed, pd.DataFrame): + raise ValueError("Feature matrix X must be a non-empty DataFrame.") + X_array, y_array = self._validate_X_y(X_processed, y_processed) + + # Set default number of features to select if not specified + if features_to_select is None: + features_to_select = X_processed.shape[1] // 2 + + # Create and prepare model + temp_model = self._create_model() + train_pool = Pool(data=X_array, label=y_array) + + # Prepare validation data if provided + eval_set = None + if X_val_processed is not None and y_val_processed is not None: + X_val_array, y_val_array = self._validate_X_y( + X_val_processed, y_val_processed + ) + eval_set = Pool(data=X_val_array, label=y_val_array) + + # Perform feature selection + selected_features = temp_model.select_features( + train_pool, + eval_set=eval_set, + features_for_select=list(range(X_processed.shape[1])), + num_features_to_select=features_to_select, + algorithm=algorithm, + steps=steps, + logging_level="Verbose" if verbose else "Silent", + train_final_model=False, + ) + + # Map feature indices to feature names with proper type casting + selected_feature_names: List[str] = [ + str(X_processed.columns[idx]) + for idx in selected_features["selected_features"] + ] + self.selected_features_ = selected_feature_names + self.feature_names_in_ = selected_feature_names + self.n_features_in_ = len(selected_feature_names) + + return selected_feature_names + + @classmethod + def tune_hyperparameters( + cls, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + selected_features: Optional[List[str]] = None, + param_grid: Optional[Dict[str, Any]] = None, + n_trials: int = 10, + early_stopping_rounds: Optional[int] = 50, + random_seed: int = 42, + **kwargs, + ) -> Tuple[Dict[str, Any], "CatBoostTimeSeriesModel"]: + """ + Tune hyperparameters for the CatBoost model. + + Args: + y: The target time series data + X: The feature matrix (including exogenous features) + X_val: Validation feature matrix (optional) + y_val: Validation target series (optional) + selected_features: List of features to use for tuning + param_grid: Dictionary of hyperparameters to search + n_trials: Number of trials for hyperparameter tuning + early_stopping_rounds: Number of rounds for early stopping + random_seed: Random seed for reproducibility + **kwargs: Additional keyword arguments for model initialization + + Returns: + Tuple[Dict[str, Any], CatBoostTimeSeriesModel]: Best hyperparameters + and fitted model + """ + if n_trials <= 0: + raise ValueError("n_trials must be a positive integer.") + # Create a temporary model instance to use its preprocessing method + temp_model = cls( + learning_task=kwargs.get("learning_task", "regression"), + differentiate_target=kwargs.get("differentiate_target", False), + bins=kwargs.get("bins", None), + random_seed=random_seed, + ) + if selected_features is not None: + temp_model.selected_features_ = selected_features + + (y_processed, X_processed, _, _) = ( + temp_model._preprocess_data(y, X, X_val, y_val) + ) + + if kwargs.get("learning_task", "regression") == "classification": + search_model = CatBoostClassifier( + random_seed=random_seed, + logging_level="Silent", + early_stopping_rounds=early_stopping_rounds, + loss_function=kwargs.get("loss_function", "Logloss"), + class_weights="Balanced", + ) + else: + search_model = CatBoostRegressor( + random_seed=random_seed, + logging_level="Silent", + early_stopping_rounds=early_stopping_rounds, + loss_function=kwargs.get("loss_function", "RMSE"), + ) + + if param_grid is None: + param_grid = { + "iterations": [100, 500, 1000, 2000], + "learning_rate": [0.01, 0.05, 0.1, 0.2], + "depth": [4, 6, 8], + } + + if X_processed is None: + raise ValueError( + "Feature matrix cannot be None after preprocessing." + ) + + train_pool = Pool(data=X_processed, label=y_processed) + results = search_model.randomized_search( + param_grid, + X=train_pool, + n_iter=n_trials, + verbose=False, + refit=False, + ) + + best_params = results["params"] + + best_model = cls( + iterations=best_params["iterations"], + learning_rate=best_params["learning_rate"], + depth=best_params["depth"], + random_seed=random_seed, + time_col=kwargs.get("time_col", "ds"), + target_col=kwargs.get("target_col", "y"), + n_lags=kwargs.get("n_lags", 0), + name=kwargs.get("name", None), + loss_function=kwargs.get("loss_function", None), + learning_task=kwargs.get("learning_task", "regression"), + bins=kwargs.get("bins", None), + differentiate_target=kwargs.get("differentiate_target", False), + ) + if selected_features is not None: + best_model.selected_features_ = selected_features + + best_model.fit(y, X, X_val, y_val) + + return best_params, best_model diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py new file mode 100644 index 0000000..77331d7 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py @@ -0,0 +1,92 @@ +""" +Module with functions of timeseries evaluation. +""" + +from typing import Optional, List, Sequence + +import numpy as np +import pandas as pd +from rich.console import Console +from sklearn.metrics import ( + mean_absolute_error, + mean_squared_error, + accuracy_score, + f1_score, + confusion_matrix, +) +console = Console() + + +def timeseries_metrics( + y_pred: Sequence[float], y_true: Sequence[float] +) -> dict[str, float]: + """ + Compute MAE, MSE, and trend capture for time series predictions. + + Parameters: + y_pred (ArrayLike): Predicted values. + y_true (ArrayLike): Ground truth values. + + Returns: + dict[str, float]: Dictionary with MAE, MSE, and trend_capture. + """ + if len(y_true) != len(y_pred): + raise ValueError("y_true and y_pred must have the same length") + if len(y_true) == 0: + raise ValueError("y_true and y_pred must not be empty") + y_true_np = np.asarray(y_true) + y_pred_np = np.asarray(y_pred) + + mae = mean_absolute_error(y_true_np, y_pred_np) + mse = mean_squared_error(y_true_np, y_pred_np) + + # Compute directional trend: 1 if up, 0 if down or flat + if len(y_true) == 1: + return {"MAE": mae, "MSE": mse, "trend_capture": 1.0} + + true_trend = np.diff(y_true_np) > 0 + pred_trend = np.diff(y_pred_np) > 0 + + trend_capture = np.mean(true_trend == pred_trend) + + return {"MAE": mae, "MSE": mse, "trend_capture": trend_capture} + + +def timeseries_classification_metrics( + y_pred: Sequence[float], + y_true: Sequence[float], + bins: Optional[List] = None, +) -> dict[str, float]: + """ + Compute accuracy for classification predictions. + + Parameters: + y_pred (ArrayLike): Predicted values. + y_true (ArrayLike): Ground truth values. + bins (List[int], optional): Bin edges for categorizing predictions. + + Returns: + dict[str, float]: Dictionary with accuracy. + """ + if bins is not None: + console.log( + f"Using bins for classification: {bins}" + ) + y_true_binned = pd.cut(y_true, bins=bins, labels=False) + else: + console.log("No bins provided, using default classification (y > 0).") + y_true_binned = (y_true > 0).astype(int) + console.log( + f"y_true_binned: {y_true_binned.value_counts()}" + ) + console.log( + f"y_pred: {pd.Series(y_pred).value_counts()}" + ) + + y_pred = np.array(y_pred).astype(int) + acc = accuracy_score(y_true_binned, y_pred) + f1 = f1_score(y_true_binned, y_pred, average="weighted") + # Calculate the confusion matrix + cm = confusion_matrix(y_true_binned, y_pred) + + return {"accuracy": acc, "f1_score": f1, "confusion_matrix": cm} diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py new file mode 100644 index 0000000..a8d646e --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py @@ -0,0 +1,140 @@ +""" +Model factory for creating, loading, and discovering time series models. + +This module provides centralized utility functions to handle different time +series model implementations based on a string identifier. It uses a +central registry (`SUPPORTED_MODELS`) that maps model type strings (e.g., +'arima') to their corresponding model classes (e.g., ARIMAModel). This +approach allows for easy extension and decouples model instantiation logic +from the code that uses the models. + +Key Functions: + create_model: Creates a new instance of a specified model type by looking + up the type string in the `SUPPORTED_MODELS` registry and + passing keyword arguments to the retrieved model class's + constructor. + load_model: Loads a previously saved model instance from disk. It uses + the provided model type string to find the correct class in + the registry and then calls that class's `.load()` classmethod. + get_available_models: Returns a dictionary listing the registered model + types (keys in `SUPPORTED_MODELS`) and their + descriptions, automatically derived from the model + class docstrings. + +Extensibility: + Adding support for a new model involves the following steps: + 1. Ensure the new model class (e.g., `MyNewModel`) inherits from the + appropriate base class (e.g., `TimeSeriesModel`) and implements all + required abstract methods. + 2. Ensure the new model class has a `.load()` classmethod compatible + with the `save()` method in the base `Model` class (if loading is + to be supported via this factory). + 3. Import the new model class into this factory module. + 4. Add an entry to the `SUPPORTED_MODELS` dictionary, mapping a unique, + lowercase string identifier to the model class itself: + `SUPPORTED_MODELS = {..., "mynewmodel": MyNewModel}` + Once added to the registry, the model can be created and loaded via the + factory functions, and it will automatically appear in the output of + `get_available_models()`. +""" + +from typing import Dict, Type +import logging + +from .base import TimeSeriesModel +from .arima import ARIMAModel +from .neural_prophet_model import NeuralProphetModel +from .catboost_time_series import CatBoostTimeSeriesModel +from .linear_regression_time_series import ElasticNetTimeSeriesModel +from .stacking_time_series import StackingTimeSeriesModel +# from .prophet import ProphetModel # Example for future + +# *** Central registry of supported models +SUPPORTED_MODELS: Dict[str, Type[TimeSeriesModel]] = { + "arima": ARIMAModel, + "neuralprophet": NeuralProphetModel, + "catboost": CatBoostTimeSeriesModel, + "elasticnet": ElasticNetTimeSeriesModel, + "stacking": StackingTimeSeriesModel, + # "prophet": ProphetModel, # Add new models here +} + + +def create_model(model_type: str, **kwargs) -> TimeSeriesModel: + """ + Create a new model instance of the specified type using a registry. + + Args: + model_type: Type of model to create (case-insensitive). + **kwargs: Model-specific parameters passed to its constructor. + + Returns: + New model instance inheriting from TimeSeriesModel. + + Raises: + ValueError: If the model type is not supported or kwargs are invalid. + """ + model_type = model_type.lower() + model_class = SUPPORTED_MODELS.get(model_type) + + if model_class: + try: + instance = model_class(**kwargs) + return instance + except TypeError as e: + logging.error(f"Kwargs issue for {model_type}: {kwargs}") + raise ValueError( + f"Invalid parameters for model type '{model_type}'. Error: {e}" + ) from e + else: + supported_list = ", ".join(f"'{k}'" for k in SUPPORTED_MODELS.keys()) + raise ValueError( + f"Unsupported model type: '{model_type}'. " + f"Currently supported models are: {supported_list}." + ) + + +def load_model(path: str, model_type: str) -> TimeSeriesModel: + """ + Load a model from disk using a registry. + + Args: + path: Path to the saved model. + model_type: Expected type of model to load (case-insensitive). + + Returns: + Loaded model instance. + + Raises: + ValueError: If the model type is not supported. + # Other errors might come from the underlying .load() method + """ + model_type = model_type.lower() + model_class = SUPPORTED_MODELS.get(model_type) + + if model_class: + return model_class.load(path) + else: + supported_list = ", ".join(f"'{k}'" for k in SUPPORTED_MODELS.keys()) + raise ValueError( + f"Unsupported model type: '{model_type}'. " + f"Currently supported models are: {supported_list}." + ) + + +def get_available_models() -> Dict[str, str]: + """ + Dynamically get a dictionary of available model types and their + descriptions from the SUPPORTED_MODELS registry and class docstrings. + + Returns: + Dictionary mapping model type names to descriptions. + """ + available = { + type_name: ( + model_class.__doc__.strip().splitlines()[0] + if model_class.__doc__ else "No description available." + ) + for type_name, model_class in SUPPORTED_MODELS.items() + } + return available diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py new file mode 100644 index 0000000..9468954 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py @@ -0,0 +1,303 @@ +""" +ElasticNet implementation for multivariate time series forecasting. +""" + +from typing import Optional, Sequence + +import pandas as pd +from sklearn.linear_model import ElasticNet +from sklearn.impute import SimpleImputer +from rich.console import Console + +from .base import MultivariateTimeSeriesModel, ensure_fitted + +console = Console() + + +class ElasticNetTimeSeriesModel(MultivariateTimeSeriesModel): + """ + ElasticNet implementation for multivariate time series forecasting. + + This class wraps the scikit-learn ElasticNet model with additional + functionality for time series forecasting, following the + MultivariateTimeSeriesModel interface. It's suitable for regression + tasks where features might be correlated. + """ + + def __init__( + self, + name: Optional[str] = None, + n_lags: int = 1, + alpha: float = 1.0, + l1_ratio: float = 0.5, + fit_intercept: bool = True, + max_iter: int = 1000, + tol: float = 1e-4, + random_seed: int = 42, + time_col: str = "ds", + target_col: str = "y", + differentiate_target: bool = False, + bins: Optional[list] = None, + learning_task: Optional[str] = None, + ) -> None: + """ + Initialize the ElasticNet time series model. + + Args: + name: Optional identifier for the model. + n_lags: Number of lagged target values to include as inputs. + alpha: Constant that multiplies the penalty terms. + l1_ratio: The ElasticNet mixing parameter (0 <= l1_ratio <= 1). + For l1_ratio = 0, it's L2 penalty (Ridge). + For l1_ratio = 1, it's L1 penalty (Lasso). + fit_intercept: Whether to calculate the intercept for this model. + max_iter: Maximum number of iterations. + tol: Tolerance for stopping criteria. + random_seed: Random seed for reproducibility. + time_col: Name of the time column. + target_col: Name of the target column. + differentiate_target: Whether to apply differencing to make series + stationary. + bins: Bin edges for multiclass classification target + transformation. + learning_task: Type of learning task ('regression', 'binary', + 'multiclass'). + """ + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + n_lags=n_lags, + differentiate_target=differentiate_target, + bins=bins, + learning_task=learning_task, + ) + + self.alpha = alpha + self.l1_ratio = l1_ratio + self.fit_intercept = fit_intercept + self.max_iter = max_iter + self.tol = tol + + self.model_: Optional[ElasticNet] = None + self.training_series_: Optional[pd.Series] = None + self.imputer_: Optional[SimpleImputer] = None + self.X_train_: Optional[pd.DataFrame] = None + + def _create_model(self) -> ElasticNet: + """Creates a new instance of ElasticNet with current parameters. + + Returns: + A new ElasticNet model instance. + """ + return ElasticNet( + alpha=self.alpha, + l1_ratio=self.l1_ratio, + fit_intercept=self.fit_intercept, + max_iter=self.max_iter, + tol=self.tol, + random_state=self.random_seed, + ) + + def _impute_missing_values(self, X: pd.DataFrame) -> pd.DataFrame: + """ + Handle missing values in the feature matrix using median imputation. + If the imputer is not fitted, it will be fitted on the data. + + Args: + X: The feature matrix potentially containing missing values. + + Returns: + pd.DataFrame: The feature matrix with imputed values. + """ + if self.imputer_ is None: + self.imputer_ = SimpleImputer( + strategy="median", copy=True, add_indicator=False + ) + # Fit the imputer and transform the data + imputed_values = self.imputer_.fit_transform(X) + else: + # Use the fitted imputer to transform new data + imputed_values = self.imputer_.transform(X) + + # Convert back to DataFrame with original index and column names + return pd.DataFrame(imputed_values, index=X.index, columns=X.columns) + + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """ + Core fitting logic for the ElasticNet model. + + Args: + y: The target time series data. + X: The feature matrix (including exogenous features). + X_val: Validation feature matrix (ignored). + y_val: Validation target series (ignored). + """ + # Use base class preprocessing + y_processed, X_processed, _, _ = self._preprocess_data( + y, X, X_val, y_val + ) + + if X_processed is None or not isinstance(X_processed, pd.DataFrame): + raise ValueError("Feature matrix X must be a non-empty DataFrame.") + + # First impute missing values in X + X_imputed = self._impute_missing_values(X_processed) + + # Validate X and y after imputation + X_array, y_array = self._validate_X_y( + X_imputed, y_processed, allow_nan=False + ) + + self.training_series_ = y_processed.copy() + self.model_ = self._create_model() + self.model_.fit(X_array, y_array) + self.X_train_ = X_processed.copy() + + @ensure_fitted + def predict(self, X: Optional[pd.DataFrame] = None) -> Sequence: + """ + Generate predictions using the fitted ElasticNet model. + + Args: + X: The feature matrix for prediction. + + Returns: + pd.Series: Predicted values with the original index. + """ + if X is None: + raise ValueError("Feature matrix X is required for prediction.") + + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + if self.training_series_ is None: + raise ValueError("Training series is not available.") + if self.imputer_ is None: + raise ValueError("Imputer is not fitted yet.") + + # If target column is present, drop it + X_pred = X.copy() + if self.target_col in X_pred.columns: + X_pred = X_pred.drop(columns=[self.target_col]) + + # For prediction, we need to reconstruct lagged features + # This is a simplified approach - in practice, you'd need + # the historical target values to create proper lags + + # Handle missing values using fitted imputer + X_processed = self._impute_missing_values(X_pred) + + # Validate X after imputation + X_array = self._validate_X(X_processed, allow_nan=False) + + predictions = self.model_.predict(X_array) + + return pd.Series( + predictions, index=X_processed.index, name=self.target_col + ) + + @ensure_fitted + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Performs backtesting (walk-forward validation) with periodic + retraining. + + Args: + y: The target time series data. + X: Optional exogenous features. + retrain_every: Number of steps after which to retrain the model. + reuse_previous_execution: Whether to reuse the previous execution + of a backtest. If True, any overlapping data between the + previous execution and the current execution will be used + without retraining the model. + + Returns: + Series of predictions for each step in the time series. + """ + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + if self.training_series_ is None: + raise ValueError("Training series is not set.") + if self.X_train_ is None: + raise ValueError("Training feature matrix is not set.") + if X is None or not isinstance(X, pd.DataFrame): + raise ValueError("Feature matrix X must be a non-empty DataFrame.") + + total_steps = len(X) + predictions = [] + current_model = self.model_ + y_history = self.training_series_.copy() + X_history = self.X_train_.copy() + + # Iterate in chunks instead of single steps + for start in range(0, total_steps, retrain_every): + end = min(start + retrain_every, total_steps) + + # Batch prediction for current chunk + X_chunk = X.iloc[start:end].copy() + if self.selected_features_: + X_chunk = X_chunk[self.selected_features_] + X_imputed = self._impute_missing_values(X_chunk) + X_array = self._validate_X(X_imputed, allow_nan=False) + preds = current_model.predict(X_array).squeeze() + if preds.ndim == 0: # single point + preds = [preds] + predictions.extend(preds) + + # Update training history + y_chunk = y.iloc[start:end] + y_history = pd.concat([y_history, y_chunk]) + X_history = pd.concat([X_history, X_chunk]) + + # Retrain the model for next chunk (if needed) + if end < total_steps: + console.print( + f"[cyan]Backtesting: Retraining at step {end}...[/cyan]" + ) + + current_model = self._create_model() + y_fit, X_fit, *_ = self._preprocess_data(y_history, X_history) + X_fit_imputed = self._impute_missing_values(X_fit) + X_fit_array, y_fit_array = self._validate_X_y( + X_fit_imputed, y_fit, allow_nan=False + ) + current_model.fit(X_fit_array, y_fit_array) + + return pd.Series( + predictions, index=X.index, name=f"{self.target_col}_pred" + ) + + @ensure_fitted + def feature_importance(self) -> Optional[pd.DataFrame]: + """ + Returns feature importance based on model coefficients. + + Returns: + A DataFrame with feature names and their corresponding + coefficients (importance scores), or None if no features. + """ + if self.model_ is None or self.feature_names_in_ is None: + return None + + importance = self.model_.coef_ + feature_importance_df = pd.DataFrame( + {"Feature": self.feature_names_in_, "Importance": importance} + ) + feature_importance_df = feature_importance_df.sort_values( + by="Importance", key=abs, ascending=False + ).reset_index(drop=True) + + return feature_importance_df diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py new file mode 100644 index 0000000..e79f519 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py @@ -0,0 +1,888 @@ +""" +NeuralProphet implementation for univariate time series forecasting. + +This module provides a comprehensive wrapper around the NeuralProphet library, +implementing enterprise-level features including robust error handling, +parameter validation, type safety, and integration with the base model +architecture. +""" + +import os +from typing import Optional, cast, Tuple, Dict, Any + +import numpy as np +import pandas as pd +import torch +from neuralprophet import NeuralProphet +from rich.console import Console + +from .base import UnivariateTimeSeriesModel, ensure_fitted + +console = Console() + +# Configure PyTorch for optimal performance +torch.set_num_threads(os.cpu_count() or 1) + + +class NeuralProphetModel(UnivariateTimeSeriesModel): + """ + Enterprise-grade NeuralProphet implementation for univariate + time series forecasting. + + This class provides a robust wrapper around the NeuralProphet model with + comprehensive error handling, parameter validation, and integration with + the base model architecture. It includes features like automatic data + validation, performance monitoring, and enterprise-level logging. + + Key Features: + - Comprehensive parameter validation + - Robust error handling with detailed diagnostics + - Memory-efficient data processing + - Integration with base model utilities + - Performance monitoring and logging + - Support for various seasonality patterns + - Flexible forecasting capabilities + + Example: + >>> model = NeuralProphetModel( + ... n_lags=7, + ... n_forecasts=3, + ... epochs=50, + ... weekly_seasonality=True + ... ) + >>> model.fit(y_train) + >>> predictions = model.predict() + >>> future_forecast = model.forecast(forecast_horizon=3) + """ + + # Class constants for validation + VALID_SEASONALITY_MODES = {"additive", "multiplicative"} + VALID_LOSS_FUNCTIONS = {"Huber", "MSE", "MAE"} + VALID_NORMALIZE_OPTIONS = {"auto", "soft", "off", "minmax"} + MIN_EPOCHS = 1 + MAX_EPOCHS = 10000 + MIN_N_LAGS = 0 + MAX_N_LAGS = 365 + MIN_N_FORECASTS = 1 + MAX_N_FORECASTS = 365 + + def __init__( + self, + name: Optional[str] = None, + n_lags: int = 1, + n_forecasts: int = 2, + weekly_seasonality: bool = True, + daily_seasonality: bool = True, + yearly_seasonality: bool = False, + seasonality_mode: str = "additive", + epochs: int = 100, + learning_rate: Optional[float] = None, + batch_size: Optional[int] = None, + loss_func: str = "Huber", + normalize: str = "auto", + impute_missing: bool = True, + drop_missing: bool = False, + time_col: str = "ds", + target_col: str = "y", + random_seed: int = 42, + ): + """ + Initialize the NeuralProphet time series model with comprehensive + validation. + + Args: + name: Optional identifier for the model + n_lags: Number of lagged target values to include as inputs (0-365) + n_forecasts: Number of steps ahead to forecast (1-365) + weekly_seasonality: Whether to include weekly seasonality + daily_seasonality: Whether to include daily seasonality + yearly_seasonality: Whether to include yearly seasonality + seasonality_mode: Type of seasonality ('additive' or + 'multiplicative') + epochs: Number of training epochs (1-10000) + learning_rate: Learning rate for optimizer (auto if None) + batch_size: Training batch size (auto if None) + loss_func: Loss function ('Huber', 'MSE', 'MAE') + normalize: Normalization type ('auto', 'soft', 'off', 'minmax') + impute_missing: Whether to automatically impute missing values + drop_missing: Whether to drop missing values in training data + time_col: Name of the time column + target_col: Name of the target column + random_seed: Random seed for reproducibility + + Raises: + ValueError: If any parameters are invalid + TypeError: If parameters have incorrect types + """ + self._validate_and_set_parameters( + n_lags=n_lags, + n_forecasts=n_forecasts, + seasonality_mode=seasonality_mode, + epochs=epochs, + learning_rate=learning_rate, + batch_size=batch_size, + loss_func=loss_func, + normalize=normalize, + weekly_seasonality=weekly_seasonality, + daily_seasonality=daily_seasonality, + yearly_seasonality=yearly_seasonality, + impute_missing=impute_missing, + drop_missing=drop_missing, + ) + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + n_lags=n_lags, + ) + self.forecast_horizon = n_forecasts + # Initialize model state + self.model_: Optional[NeuralProphet] = None + self.backtest_predictions_: Optional[pd.Series] = None + self._training_metrics: Dict[str, float] = {} + + console.log( + f"[green]Initialized NeuralProphetModel: {self.summary()}[/green]" + ) + + def _validate_and_set_parameters( + self, + n_lags: int, + n_forecasts: int, + seasonality_mode: str, + epochs: int, + learning_rate: Optional[float], + batch_size: Optional[int], + loss_func: str, + normalize: str, + weekly_seasonality: bool, + daily_seasonality: bool, + yearly_seasonality: bool, + impute_missing: bool, + drop_missing: bool, + ) -> None: + """Validate and set model parameters with comprehensive checks.""" + # Validate integer parameters + if not (self.MIN_N_LAGS <= n_lags <= self.MAX_N_LAGS): + raise ValueError( + f"n_lags must be between {self.MIN_N_LAGS} and " + + f"{self.MAX_N_LAGS}, got {n_lags}" + ) + + if not (self.MIN_N_FORECASTS <= n_forecasts <= self.MAX_N_FORECASTS): + raise ValueError( + f"n_forecasts must be between {self.MIN_N_FORECASTS} and " + + f"{self.MAX_N_FORECASTS}, got {n_forecasts}" + ) + + if not (self.MIN_EPOCHS <= epochs <= self.MAX_EPOCHS): + raise ValueError( + f"epochs must be between {self.MIN_EPOCHS} and " + + f"{self.MAX_EPOCHS}, got {epochs}" + ) + + # Validate string parameters + if seasonality_mode not in self.VALID_SEASONALITY_MODES: + raise ValueError( + "seasonality_mode must be one of " + + f"{self.VALID_SEASONALITY_MODES}, got {seasonality_mode}" + ) + + if loss_func not in self.VALID_LOSS_FUNCTIONS: + raise ValueError( + "loss_func must be one of " + + f"{self.VALID_LOSS_FUNCTIONS}, got {loss_func}" + ) + + if normalize not in self.VALID_NORMALIZE_OPTIONS: + raise ValueError( + "normalize must be one of " + + f"{self.VALID_NORMALIZE_OPTIONS}, got {normalize}" + ) + + # Validate optional float parameters + if learning_rate is not None: + if ( + not isinstance(learning_rate, (int, float)) + or learning_rate <= 0 + ): + raise ValueError( + "learning_rate must be a positive number, " + + f"got {learning_rate}" + ) + + if batch_size is not None: + if not isinstance(batch_size, int) or batch_size <= 0: + raise ValueError( + "batch_size must be a positive integer, " + + f"got {batch_size}" + ) + + # Validate boolean parameters + for param_name, param_value in [ + ("weekly_seasonality", weekly_seasonality), + ("daily_seasonality", daily_seasonality), + ("yearly_seasonality", yearly_seasonality), + ("impute_missing", impute_missing), + ("drop_missing", drop_missing), + ]: + if not isinstance(param_value, bool): + raise TypeError( + f"{param_name} must be a boolean, " + + f"got {type(param_value)}" + ) + + # Set validated parameters + self.n_forecasts = n_forecasts + self.weekly_seasonality = weekly_seasonality + self.daily_seasonality = daily_seasonality + self.yearly_seasonality = yearly_seasonality + self.seasonality_mode = seasonality_mode + self.epochs = epochs + self.learning_rate = learning_rate + self.batch_size = batch_size + self.loss_func = loss_func + self.normalize = normalize + self.impute_missing = impute_missing + self.drop_missing = drop_missing + + def _create_model(self) -> NeuralProphet: + """ + Create a new NeuralProphet instance with validated parameters. + + Returns: + A new NeuralProphet model instance configured with current + parameters. + + Raises: + RuntimeError: If model creation fails + """ + try: + model_params = { + "n_lags": self.n_lags, + "n_forecasts": self.n_forecasts, + "weekly_seasonality": self.weekly_seasonality, + "daily_seasonality": self.daily_seasonality, + "yearly_seasonality": self.yearly_seasonality, + "seasonality_mode": self.seasonality_mode, + "loss_func": self.loss_func, + "normalize": self.normalize, + "impute_missing": self.impute_missing, + "drop_missing": self.drop_missing, + "impute_rolling": 1000000, + "impute_linear": 100000, + } + + # Add optional parameters if specified + if self.learning_rate is not None: + model_params["learning_rate"] = self.learning_rate + if self.batch_size is not None: + model_params["batch_size"] = self.batch_size + + console.log( + f"[blue]Creating NeuralProphet with params: " + f"{model_params}[/blue]" + ) + return NeuralProphet(**model_params) + + except Exception as e: + raise RuntimeError( + f"Failed to create NeuralProphet model: {e}" + ) from e + + def _validate_and_prepare_data( + self, y: pd.Series + ) -> Tuple[pd.DataFrame, pd.Series]: + """ + Validate and prepare time series data for NeuralProphet. + + Args: + y: Input time series data + + Returns: + Tuple of (prepared_dataframe, validated_series) + + Raises: + ValueError: If data validation fails + """ + try: + # Validate target series + y_array = self._validate_y(y) + + # Ensure datetime index + if not pd.api.types.is_datetime64_any_dtype(y.index): + try: + y_datetime = y.copy() + y_datetime.index = pd.to_datetime(y.index) + console.log("[yellow]Converted index to datetime[/yellow]") + except Exception as e: + raise ValueError( + "y's index must be a DateTime index or convertible " + f"to DateTime. Conversion failed: {e}" + ) from e + else: + y_datetime = y.copy() + + # Check for minimum data requirements + if len(y_datetime) < max(self.n_lags + 1, 10): + raise ValueError( + "Insufficient data: need at least " + + f"{max(self.n_lags + 1, 10)} observations, " + + f"got {len(y_datetime)}" + ) + + # Create NeuralProphet format DataFrame + df = pd.DataFrame({"ds": y_datetime.index, "y": y_array}) + + # Validate for missing values if not configured to handle them + if not self.impute_missing and bool(df["y"].isna().any()): + raise ValueError( + "Data contains missing values but impute_missing=False. " + "Either set impute_missing=True or clean the data." + ) + + console.log( + f"[green]Data validation successful: {len(df)} " + f"observations[/green]" + ) + return df, y_datetime + + except Exception as e: + console.print(f"[red]Data validation failed: {e}[/red]") + raise + + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """ + Core fitting logic for NeuralProphet model with enhanced error + handling. + + Args: + y: The target time series data with DateTime index + X: Optional DataFrame (unused for univariate model) + X_val: Validation features (unused for NeuralProphet) + y_val: Validation target (unused for NeuralProphet) + + Raises: + ValueError: If data validation fails + RuntimeError: If model fitting fails + """ + try: + console.log("[blue]Starting NeuralProphet model fitting...[/blue]") + + df, y_datetime = self._validate_and_prepare_data(y) + self.training_series_ = y_datetime + self.model_ = self._create_model() + + console.log( + f"[blue]Training model for {self.epochs} epochs...[/blue]" + ) + + fit_result = self.model_.fit(df, epochs=self.epochs) + + # Store training metrics if available + if hasattr(fit_result, "losses") and fit_result is not None: + losses = getattr(fit_result, "losses", None) + if losses: + self._training_metrics = { + "final_loss": float(losses[-1]), + "epochs_trained": len(losses), + } + + console.log( + f"[green]Model fitting completed successfully. " + f"Metrics: {self._training_metrics}[/green]" + ) + + except Exception as e: + console.print(f"[red]Model fitting failed: {e}[/red]") + # Reset model state on failure + self.model_ = None + self.training_series_ = None + raise RuntimeError( + f"Failed to fit NeuralProphet model: {e}" + ) from e + + def _prepare_prediction_data( + self, X: Optional[pd.DataFrame] = None + ) -> Tuple[pd.DataFrame, pd.Series]: + """ + Prepare data for prediction with comprehensive validation. + + Args: + X: Optional DataFrame containing prediction data + + Returns: + Tuple of (prepared_dataframe, prediction_index) + + Raises: + ValueError: If data preparation fails + """ + if self.training_series_ is None: + raise ValueError("No training series available") + + training_series = cast(pd.Series, self.training_series_) + + if X is None: + # Predict on training data + df = pd.DataFrame( + { + "ds": training_series.index, + "y": training_series.to_numpy(), + } + ) + return df, pd.Series(training_series.index) + + # Handle various input formats for X + try: + ds_values, y_values = self._extract_time_and_target_from_X(X) + + # Ensure datetime format + if not pd.api.types.is_datetime64_any_dtype(ds_values): + ds_values = pd.to_datetime(ds_values) + + df = pd.DataFrame( + { + "ds": ds_values, + "y": y_values, + } + ) + + return df, ds_values + + except Exception as e: + raise ValueError( + f"Failed to prepare prediction data: {e}" + ) from e + + def _extract_time_and_target_from_X( + self, X: pd.DataFrame + ) -> Tuple[pd.Series, pd.Series]: + """ + Extract time and target columns from input DataFrame. + + Args: + X: Input DataFrame + + Returns: + Tuple of (time_series, target_series) + + Raises: + ValueError: If extraction fails + """ + # Scenario 1: Explicit time and target columns + if self.time_col in X.columns and self.target_col in X.columns: + return ( + X[self.time_col], + self._validate_y(X[self.target_col]) + ) + + # Scenario 2: DateTime index + elif pd.api.types.is_datetime64_any_dtype(X.index): + ds_values = pd.Series(X.index, name=self.time_col) + + if self.target_col in X.columns: + # DateTime index with explicit target column + return ( + ds_values, + self._validate_y(X[self.target_col]) + ) + elif X.shape[1] == 1: + # DateTime index with single data column + return ds_values, self._validate_y( + X.iloc[:, 0].rename(self.target_col) + ) + elif X.shape[1] == 0: + # Only index, no columns - forecast scenario + y_values = pd.Series( + np.nan, index=X.index, name=self.target_col + ) + return ds_values, y_values + else: + raise ValueError( + "X has DateTime index but cannot identify target column. " + + f"Expected '{self.target_col}' or single column. " + + f"Found: {X.columns.tolist()}" + ) + else: + raise ValueError( + "Cannot determine time and target from X. " + + f"Provide columns '{self.time_col}' and '{self.target_col}' " + + "or use DateTime index." + ) + + @ensure_fitted + def predict(self, X: Optional[pd.DataFrame] = None) -> pd.Series: + """ + Generate in-sample predictions with enhanced error handling. + + Args: + X: Optional DataFrame containing timestamps and target values. + If None, predicts on training data. + + Returns: + Series of predictions indexed by timestamp + + Raises: + ValueError: If model is not fitted or prediction fails + RuntimeError: If prediction computation fails + """ + if self.model_ is None: + raise ValueError("Model is not fitted") + + try: + console.log("[blue]Generating predictions...[/blue]") + + # Prepare prediction data + df, predictions_index = self._prepare_prediction_data(X) + + # Get training context for lagged features + training_series = cast(pd.Series, self.training_series_) + past_values = pd.DataFrame( + { + "ds": training_series.index, + "y": training_series.to_numpy(), + } + ).iloc[-self.n_lags :, :] + + # Combine past and prediction data + combined_df = pd.concat([past_values, df], ignore_index=True) + combined_df = ( + combined_df.sort_values(by="ds") + .reset_index(drop=True) + .drop_duplicates(subset="ds", keep="last") + ) + + # Generate forecast + forecast = self.model_.predict(combined_df) + + # Handle different forecast column formats + forecast_col = f"yhat{self.n_forecasts}" + if forecast_col not in forecast.columns: + forecast = self.model_.get_last_forecast( + forecast, include_previous_forecasts=self.n_forecasts + ) + + # Extract predictions for requested indices + forecast = forecast.set_index("ds") + predictions = forecast.loc[predictions_index, forecast_col] + + console.log( + f"[green]Generated {len(predictions)} predictions[/green]" + ) + return predictions + + except Exception as e: + console.print(f"[red]Prediction failed: {e}[/red]") + raise RuntimeError(f"Failed to generate predictions: {e}") from e + + @ensure_fitted + def forecast(self, forecast_horizon: int) -> np.ndarray: + """ + Generate future forecasts with comprehensive validation. + + Args: + forecast_horizon: Number of steps to forecast ahead + (1 to n_forecasts) + + Returns: + Array of forecasted values, indexed by the forecast horizon + + Raises: + ValueError: If forecast_horizon is invalid or model not fitted + RuntimeError: If forecast generation fails + """ + if self.model_ is None: + raise ValueError("Model is not fitted") + + if not (1 <= forecast_horizon <= self.n_forecasts): + raise ValueError( + "forecast_horizon must be between 1 and " + + f"{self.n_forecasts}, got {forecast_horizon}" + ) + + try: + console.log( + f"[blue]Generating {forecast_horizon}-step forecast...[/blue]" + ) + + training_series = cast(pd.Series, self.training_series_) + + # Create future dataframe + future_df = self.model_.make_future_dataframe( + df=pd.DataFrame( + { + "ds": training_series.index, + "y": training_series.to_numpy(), + } + ), + periods=forecast_horizon, + ) + + # Generate forecasts + forecast = self.model_.predict(future_df) + + # Extract forecasted values for each horizon + forecasted_values = np.empty(forecast_horizon) + for i in range(forecast_horizon): + col_name = f"yhat{i + 1}" + if col_name in forecast.columns: + values = forecast[col_name].dropna() + if len(values) > 0: + forecasted_values[i] = values.iloc[0] + else: + forecasted_values[i] = np.nan + else: + forecasted_values[i] = np.nan + + console.log( + f"[green]Generated forecast: {len(forecasted_values)}" + + " values[/green]" + ) + return forecasted_values + + except Exception as e: + console.print(f"[red]Forecast generation failed: {e}[/red]") + raise RuntimeError(f"Failed to generate forecast: {e}") from e + + @ensure_fitted + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Perform comprehensive backtesting with enhanced monitoring. + + This method implements walk-forward validation with periodic + retraining, providing robust evaluation of model performance in + production-like scenarios. + + Args: + y: Target time series for backtesting (must have DateTime index) + X: Unused (included for base class compatibility) + retrain_every: Unused (model retrains at each step) + reuse_previous_execution: Whether to reuse previous backtest + results + + Returns: + Series of backtested predictions indexed by timestamp + + Raises: + ValueError: If parameters are invalid or data is insufficient + RuntimeError: If backtesting fails + """ + if self.model_ is None: + raise ValueError("Model is not fitted") + if self.training_series_ is None: + raise ValueError("No training series found") + + if not (1 <= self.forecast_horizon <= self.n_forecasts): + raise ValueError( + "forecast_horizon must be between 1 and " + + f"{self.n_forecasts}, got {self.forecast_horizon}" + ) + + # Handle reuse of previous execution + if reuse_previous_execution and self.backtest_predictions_ is not None: + expected_index = y.iloc[self.forecast_horizon:].index + if ( + len(self.backtest_predictions_) == len(expected_index) + and (self.backtest_predictions_.index == expected_index).all() + ): + console.log( + "[yellow]Reusing previous backtest results[/yellow]" + ) + return self.backtest_predictions_ + else: + console.log( + "[yellow]Previous results incompatible, running new" + + " backtest[/yellow]" + ) + + try: + console.log( + f"[blue]Starting backtest with {self.forecast_horizon}-step" + + " horizon...[/blue]" + ) + + # Validate and prepare data + y_sorted = y.sort_index() + + # Check for overlapping data + training_series = cast(pd.Series, self.training_series_) + if any(t in training_series.index for t in y_sorted.index): + console.print( + "[yellow]Warning: Backtest data overlaps with training" + + " data[/yellow]" + ) + + # Initialize backtesting + predictions = [] + training_base = pd.DataFrame( + { + "ds": training_series.index, + "y": training_series.values, + } + ) + + timestamps = y_sorted.index + total_steps = len(timestamps) - self.forecast_horizon + + console.log( + f"[blue]Running {total_steps} backtest steps...[/blue]" + ) + + # Perform walk-forward validation + for i in range(self.forecast_horizon, len(timestamps)): + if i % 50 == 0: # Progress logging + console.log( + f"[blue]Backtest progress: {i}/{len(timestamps)}[/blue]" + ) + + t = timestamps[i] + t_minus_h = timestamps[i - self.forecast_horizon] + + # Prepare training data up to t - h + history = y_sorted.loc[:t_minus_h] + train_df = ( + pd.concat( + [ + training_base, + pd.DataFrame( + {"ds": history.index, "y": history.values} + ), + ], + ignore_index=True, + ) + .drop_duplicates(subset="ds") + .sort_values("ds") + ) + + # Check minimum data requirement + if len(train_df) < max(self.n_lags + 1, 10): + console.print( + f"[yellow]Insufficient data at step {i}, " + + "skipping[/yellow]" + ) + predictions.append((t, np.nan)) + continue + + try: + # Retrain model + model = self._create_model() + model.fit(train_df, epochs=self.epochs) + + # Generate forecast + mask = train_df["ds"] <= t_minus_h + future_df = model.make_future_dataframe( + df=train_df.loc[mask], periods=self.forecast_horizon + ) + forecast = model.predict(future_df, decompose=False) + + # Extract prediction + forecast_col = f"yhat{self.forecast_horizon}" + prediction_rows = forecast[forecast["ds"] == t] + + if ( + len(prediction_rows) > 0 + and forecast_col in forecast.columns + ): + prediction = prediction_rows[forecast_col].iloc[0] + else: + prediction = np.nan + + predictions.append((t, prediction)) + + except Exception as step_error: + console.print( + f"[yellow]Error at step {i}: {step_error}[/yellow]" + ) + predictions.append((t, np.nan)) + + # Create results series + if predictions: + pred_index, pred_values = zip(*predictions) + self.backtest_predictions_ = pd.Series( + pred_values, + index=pd.Index(pred_index), + name=f"yhat{self.forecast_horizon}", + ) + else: + self.backtest_predictions_ = pd.Series( + dtype=float, name=f"yhat{self.forecast_horizon}" + ) + + console.log( + f"[green]Backtest completed: {len(self.backtest_predictions_)}" + + " predictions[/green]" + ) + return self.backtest_predictions_ + + except Exception as e: + console.print(f"[red]Backtest failed: {e}[/red]") + raise RuntimeError(f"Failed to perform backtest: {e}") from e + + def get_params_dict(self) -> Dict[str, Any]: + """Get comprehensive model parameters for logging/serialization.""" + base_params = super().get_params_dict() + neural_prophet_params = { + "n_forecasts": self.n_forecasts, + "weekly_seasonality": self.weekly_seasonality, + "daily_seasonality": self.daily_seasonality, + "yearly_seasonality": self.yearly_seasonality, + "seasonality_mode": self.seasonality_mode, + "epochs": self.epochs, + "learning_rate": self.learning_rate, + "batch_size": self.batch_size, + "loss_func": self.loss_func, + "normalize": self.normalize, + "impute_missing": self.impute_missing, + "drop_missing": self.drop_missing, + "training_metrics": self._training_metrics, + } + return {**base_params, **neural_prophet_params} + + def summary(self) -> str: + """Generate comprehensive model summary.""" + fitted_status = ( + "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" + ) + + seasonality_features = [] + if self.weekly_seasonality: + seasonality_features.append("Weekly") + if self.daily_seasonality: + seasonality_features.append("Daily") + if self.yearly_seasonality: + seasonality_features.append("Yearly") + + seasonality_str = ( + ", ".join(seasonality_features) if seasonality_features else "None" + ) + + summary_lines = [ + f"Model: {self.__class__.__name__}", + f"Status: {fitted_status}", + f"Lags: {self.n_lags}, Forecasts: {self.n_forecasts}", + f"Seasonality: {seasonality_str} ({self.seasonality_mode})", + f"Training: {self.epochs} epochs, {self.loss_func} loss", + f"Data Handling: Impute={self.impute_missing}, " + f"Drop={self.drop_missing}", + ] + + if self._training_metrics: + metrics_str = ", ".join( + f"{k}={v:.4f}" for k, v in self._training_metrics.items() + ) + summary_lines.append(f"Metrics: {metrics_str}") + + return "\n".join(summary_lines) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py new file mode 100644 index 0000000..5e5a7c0 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py @@ -0,0 +1,695 @@ +""" +Stacking implementation for time series forecasting. + +This module provides a stacking regressor implementation that combines multiple +time series models' predictions using a meta-model. The stacking approach +helps improve prediction accuracy by combining the strengths of different +base models through a learned meta-model. + +Key features: +- **Model Stacking**: Combines predictions from multiple base models +- **Time-Series Aware**: Uses proper time-based cross-validation +- **Meta-Model Learning**: Learns optimal combination weights +- **Comprehensive Error Handling**: Robust error handling and validation +- **Rich Logging**: Colored console output for better debugging + +The stacking model loads pre-trained base models and uses their predictions +as features for training a meta-model (CatBoost by default). +""" + +from typing import Optional, List, Dict, Any, Union +import pandas as pd +import numpy as np +from catboost import CatBoostRegressor, CatBoostClassifier, Pool +from rich.console import Console +import gzip +import pickle +import lzma + +from .base import ( + MultivariateTimeSeriesModel, + ensure_fitted, + TimeSeriesModel, +) + +console = Console() + + +class StackingTimeSeriesModel(MultivariateTimeSeriesModel): + """ + Stacking implementation for time series forecasting. + + This class implements stacking of multiple base models, using their + predictions as features for a meta-model. It handles time-based + cross-validation to generate out-of-fold predictions for training. + + The model supports both regression and classification tasks through + the meta-model configuration. + + Attributes: + base_models_: List of loaded base models + model_: The trained meta-model (CatBoost) + training_series_: Copy of training target data + base_predictions_train_: Base model predictions on training data + backtest_predictions_: Stored backtest predictions for reuse + + Example: + >>> stacking_model = StackingTimeSeriesModel( + ... base_model_paths=["model1.pkl", "model2.pkl"], + ... base_model_types=["catboost", "elasticnet"] + ... ) + >>> stacking_model.fit(y=target_series, X=feature_matrix) + >>> predictions = stacking_model.predict(X=test_features) + """ + + def __init__( + self, + base_model_paths: List[str], + base_model_types: List[str], + name: Optional[str] = None, + learning_task: str = "regression", + retrain_every: int = 100, + meta_iterations: int = 1000, + meta_learning_rate: float = 0.1, + meta_depth: int = 6, + early_stopping_rounds: Optional[int] = None, + meta_loss_function: Optional[str] = None, + time_col: str = "ds", + target_col: str = "y", + random_seed: int = 42, + verbose: bool = False, + differentiate_target: bool = False, + bins: Optional[List[float]] = None, + use_predict_for_training: bool = True, + ) -> None: + """ + Initialize the stacking model. + + Args: + base_model_paths: Paths to saved base models + base_model_types: Types of base models (must match order of paths) + name: Optional identifier for the model + learning_task: Type of learning task ('regression', 'binary', + 'multiclass') + retrain_every: Frequency of retraining during backtesting + meta_iterations: Number of iterations for meta-model + meta_learning_rate: Learning rate for meta-model + meta_depth: Tree depth for meta-model + early_stopping_rounds: Early stopping rounds for meta-model + meta_loss_function: Loss function for meta-model + time_col: Name of time column + target_col: Name of target column + random_seed: Random seed + verbose: Whether to print verbose logging + differentiate_target: Whether to differentiate the target series + bins: Bin edges for multiclass classification + use_predict_for_training: If True, use predict() instead of + backtest() for generating base model predictions during + training. This is much faster but may lead to overfitting + since the meta-model trains on in-sample predictions. + """ + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + learning_task=learning_task, + differentiate_target=differentiate_target, + bins=bins, + ) + + # Validate inputs + if not base_model_paths: + raise ValueError("base_model_paths cannot be empty") + if not base_model_types: + raise ValueError("base_model_types cannot be empty") + if len(base_model_paths) != len(base_model_types): + raise ValueError( + "base_model_paths and base_model_types must have same length" + ) + + self.retrain_every = retrain_every + self.base_model_paths = base_model_paths + self.base_model_types = base_model_types + self.meta_iterations = meta_iterations + self.meta_learning_rate = meta_learning_rate + self.meta_depth = meta_depth + self.meta_loss_function = self._get_default_loss_function( + meta_loss_function + ) + self.early_stopping_rounds = early_stopping_rounds + self.verbose = verbose + self.use_predict_for_training = use_predict_for_training + + # Will be set during fit + self.base_models_: List[TimeSeriesModel] = [] + self.model_: Optional[ + Union[CatBoostRegressor, CatBoostClassifier] + ] = None + self.training_series_: Optional[pd.Series] = None + self.base_predictions_train_: Optional[pd.DataFrame] = None + self.backtest_predictions_: Optional[pd.Series] = None + + self._load_base_models() + + if self.verbose: + console.log( + "[green]Initialized StackingTimeSeriesModel: " + + f"{self.summary()}[/green]" + ) + + def _load_base_models(self) -> None: + """Load all base models from their saved paths.""" + from .factory import load_model + + self.base_models_ = [] + for model_path, model_type in zip( + self.base_model_paths, self.base_model_types + ): + try: + model = load_model(model_path, model_type) + self.base_models_.append(model) + if self.verbose: + console.log( + f"[blue]Loaded {model_type} model from " + + f"{model_path}[/blue]" + ) + except Exception as e: + console.print( + f"[red]Error loading model from {model_path}: {e}[/red]" + ) + raise ValueError(f"Failed to load model: {model_path}") from e + + if self.verbose: + console.log( + f"[green]Successfully loaded {len(self.base_models_)} " + + "base models[/green]" + ) + + def _create_meta_model( + self, + ) -> Union[CatBoostRegressor, CatBoostClassifier]: + """Creates a new instance of meta-model. + + Returns: + A new CatBoost model instance (Regressor or Classifier). + """ + base_params = { + "iterations": self.meta_iterations, + "learning_rate": self.meta_learning_rate, + "depth": self.meta_depth, + "loss_function": self.meta_loss_function, + "early_stopping_rounds": self.early_stopping_rounds, + "random_seed": self.random_seed, + "verbose": self.verbose, + } + + if self.learning_task in ["binary", "multiclass"]: + return CatBoostClassifier( + auto_class_weights="Balanced", **base_params + ) + else: + return CatBoostRegressor(**base_params) + + def _get_base_predictions( + self, X: Optional[pd.DataFrame], y: Optional[pd.Series] = None + ) -> pd.DataFrame: + """Get predictions from all base models. + + Args: + X: Feature matrix + y: Target series (optional, used for validation) + + Returns: + DataFrame with predictions in model_0, model_1, etc columns + """ + if X is None: + raise ValueError("Feature matrix X cannot be None") + + all_preds = [] + for i, model in enumerate(self.base_models_): + try: + preds = model.predict(X) + model_name = f"model_{model.name or i}" + all_preds.append( + pd.Series(preds, name=model_name, index=X.index) + ) + + if self.verbose: + console.log( + f"[cyan]Generated predictions from {model_name}[/cyan]" + ) + except Exception as e: + console.print( + f"[red]Error getting predictions from model {i}: {e}[/red]" + ) + raise + + return pd.concat(all_preds, axis=1) + + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """Core fitting logic for stacking model. + + Get predictions from base models on training data, then train + meta-model on those predictions. + + Args: + y: The target time series data + X: The feature matrix (including exogenous features) + X_val: Validation feature matrix (optional) + y_val: Validation target series (optional) + """ + if X is None: + raise ValueError("Feature matrix X must be provided for stacking") + + # Preprocess data using parent class method + y_processed, X_processed, y_val_processed, X_val_processed = ( + self._preprocess_data(y, X, X_val, y_val) + ) + + if X_processed is None: + raise ValueError( + "Feature matrix X cannot be None after preprocessing" + ) + + if self.verbose: + method_name = ( + "predictions" + if self.use_predict_for_training + else "backtest predictions" + ) + console.log( + f"[blue]Generating base model {method_name} " + + "for stacking...[/blue]" + ) + + # Get base predictions - use either predict or backtest based on + # setting + if self.use_predict_for_training: + # Fast approach: use direct predictions (may overfit) + meta_features = self._get_base_predictions(X_processed) + # Align with target data + common_index = meta_features.index.intersection(y_processed.index) + meta_features = meta_features.loc[common_index] + y_aligned = y_processed.loc[common_index] + + if self.verbose: + console.log( + "[yellow]Warning: Using predict() for training may " + + "lead to overfitting since meta-model trains on " + + "in-sample predictions[/yellow]" + ) + else: + # Robust approach: use backtesting to avoid overfitting + all_preds = [] + for i, model in enumerate(self.base_models_): + try: + preds = model.backtest( + y_processed, + X_processed, + retrain_every=self.retrain_every, + ) + model_name = f"model_{model.name or i}" + all_preds.append(pd.Series(preds, name=model_name)) + + if self.verbose: + console.log( + "[cyan]Generated backtest predictions from " + + f"{model_name}[/cyan]" + ) + except Exception as e: + console.print( + f"[red]Error during backtesting for model {i}: " + + f"{e}[/red]" + ) + raise + + meta_features = pd.concat(all_preds, axis=1) + + # Align with target data (backtest might have different length) + common_index = meta_features.index.intersection(y_processed.index) + meta_features = meta_features.loc[common_index] + y_aligned = y_processed.loc[common_index] + + if self.verbose: + console.log( + f"[blue]Training meta-model with {len(meta_features)} " + + f"samples and {meta_features.shape[1]} base model " + + "features[/blue]" + ) + + meta_X, meta_y = self._validate_X_y(meta_features, y_aligned) + train_pool = Pool(data=meta_X, label=meta_y) + + # Prepare validation data if provided + eval_set = None + if X_val_processed is not None and y_val_processed is not None: + val_predictions = self._get_base_predictions( + X_val_processed, y_val_processed + ) + val_X, val_y = self._validate_X_y(val_predictions, y_val_processed) + eval_set = Pool(data=val_X, label=val_y) + + if self.verbose: + console.log( + "[blue]Using validation set with " + + f"{len(val_predictions)} samples[/blue]" + ) + + # Create and train meta-model + self.model_ = self._create_meta_model() + self.model_.fit(train_pool, eval_set=eval_set) + + # Store training data + self.training_series_ = y_processed.copy() + self.base_predictions_train_ = meta_features.copy() + + if self.verbose: + console.log( + "[green]Meta-model training completed successfully[/green]" + ) + + @ensure_fitted + def predict(self, X: pd.DataFrame) -> pd.Series: + """ + Generate predictions using the stacking model. + + Args: + X: Feature matrix for prediction + + Returns: + Series containing predictions + """ + if self.model_ is None: + raise ValueError("Model has not been fitted yet") + + base_predictions = self._get_base_predictions(X) + X_array = self._validate_X(base_predictions) + predictions = self.model_.predict(X_array) + + # Convert predictions to numpy array if needed + if hasattr(predictions, "squeeze"): + predictions = predictions.squeeze() + elif isinstance(predictions, list): + predictions = np.array(predictions) + + return pd.Series(predictions, index=X.index, name=self.target_col) + + @ensure_fitted + def feature_importance(self) -> Optional[pd.DataFrame]: + """ + Returns feature importance from the meta-model. + + Returns: + DataFrame with feature names and their importance scores, + or None if not available. + """ + if self.model_ is None or not hasattr( + self.model_, "feature_importances_" + ): + return None + + if self.base_predictions_train_ is None: + return None + + importances = self.model_.feature_importances_ + feature_names = self.base_predictions_train_.columns + + return pd.DataFrame( + { + "feature": feature_names, + "importance": importances, + } + ).sort_values("importance", ascending=False) + + @ensure_fitted + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Performs backtesting (walk-forward validation) with periodic + retraining. + + This method simulates a production scenario by iterating through a test + set, making a one-step-ahead prediction, and then retraining the model + periodically with the newly available data. + + Args: + y: Series with the true target values for the backtesting period + X: DataFrame with features for the backtesting period + retrain_every: The frequency of retraining. The model will be + retrained every `retrain_every` steps + reuse_previous_execution: Whether to reuse the previous execution + of a backtest. If True, any overlapping data between the + previous execution and the current execution will be used + without retraining the model + + Returns: + A series of backtested predictions, indexed by the backtest data's + index + """ + if self.model_ is None: + raise ValueError("Model is not fitted yet") + if self.training_series_ is None: + raise ValueError("Training series is not set") + if X is None: + raise ValueError("Feature matrix X must be provided") + + if reuse_previous_execution: + if self.backtest_predictions_ is None: + raise ValueError("No previous execution found") + if (self.backtest_predictions_.shape[0] != y.shape[0]) or ( + not (self.backtest_predictions_.index == y.index).all() + ): + raise ValueError( + "Previous execution index does not match y index" + ) + return self.backtest_predictions_ + + if self.verbose: + console.log( + f"[blue]Starting backtest with {len(y)} samples, " + + f"retraining every {retrain_every} steps[/blue]" + ) + + # Get base model predictions for the entire backtest period + all_base_preds = [] + for i, model in enumerate(self.base_models_): + try: + preds = model.backtest( + y, + X, + retrain_every=retrain_every, + reuse_previous_execution=reuse_previous_execution, + ) + model_name = f"model_{model.name or i}" + all_base_preds.append(pd.Series(preds, name=model_name)) + + if self.verbose: + console.log( + f"[cyan]Completed backtest for {model_name}[/cyan]" + ) + except Exception as e: + console.print( + f"[red]Error during backtest for model {i}: {e}[/red]" + ) + raise + + meta_features = pd.concat(all_base_preds, axis=1) + + # Generate meta-model predictions + predictions = self.model_.predict(self._validate_X(meta_features)) + + # Store backtest predictions for potential reuse + self.backtest_predictions_ = pd.Series( + predictions, + index=meta_features.index, + name=f"{self.target_col}_pred", + ) + + if self.verbose: + console.log( + "[green]Backtest completed: " + + f"{len(self.backtest_predictions_)} predictions " + + "generated[/green]" + ) + + return self.backtest_predictions_ + + def get_base_model_names(self) -> List[str]: + """Get names of all base models. + + Returns: + List of base model names + """ + return [ + model.name or f"model_{i}" + for i, model in enumerate(self.base_models_) + ] + + def get_params_dict(self) -> Dict[str, Any]: + """Get model parameters as dictionary for logging/serialization.""" + base_params = super().get_params_dict() + stacking_params = { + "base_model_paths": self.base_model_paths, + "base_model_types": self.base_model_types, + "retrain_every": self.retrain_every, + "meta_iterations": self.meta_iterations, + "meta_learning_rate": self.meta_learning_rate, + "meta_depth": self.meta_depth, + "meta_loss_function": self.meta_loss_function, + "early_stopping_rounds": self.early_stopping_rounds, + "num_base_models": len(self.base_models_), + "use_predict_for_training": self.use_predict_for_training, + } + return {**base_params, **stacking_params} + + def summary(self) -> str: + """Generate a summary string of the model.""" + params = self.get_params_dict() + fitted_status = ( + "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" + ) + + summary_lines = [ + f"Model: {self.__class__.__name__}", + f"Status: {fitted_status}", + f"Base Models: {params.get('num_base_models', 0)}", + f"Task: {params.get('learning_task', 'regression')}", + f"Meta Loss: {params.get('meta_loss_function', 'RMSE')}", + ] + + return "\n".join(summary_lines) + + def _optimize_base_models_for_storage(self) -> None: + """ + Optimizes base models for storage by removing unnecessary data. + This can significantly reduce pickle size, especially for neural + models. + """ + if self.verbose: + console.log("[blue]Optimizing base models for storage...[/blue]") + + for i, model in enumerate(self.base_models_): + try: + # For neuralprophet models, remove training history and + # large artifacts + model_attr = getattr(model, "model", None) + if model_attr is not None and hasattr(model_attr, "trainer"): + trainer = getattr(model_attr, "trainer", None) + if trainer is not None: + # Remove trainer which contains training logs and + # can be very large + if hasattr(trainer, "logged_metrics"): + setattr(trainer, "logged_metrics", {}) + if hasattr(trainer, "progress_bar_metrics"): + setattr(trainer, "progress_bar_metrics", {}) + if hasattr(trainer, "callback_metrics"): + setattr(trainer, "callback_metrics", {}) + + # For any model with training history + if hasattr(model, "training_history_"): + setattr(model, "training_history_", None) + if hasattr(model, "validation_history_"): + setattr(model, "validation_history_", None) + + # Remove cached predictions if they exist + if hasattr(model, "_cached_predictions"): + setattr(model, "_cached_predictions", None) + + if self.verbose: + model_name = getattr(model, "name", f"model_{i}") + console.log( + f"[cyan]Optimized {model_name} for storage[/cyan]" + ) + + except Exception as e: + if self.verbose: + console.log( + f"[yellow]Warning: Could not optimize model {i}: " + + f"{e}[/yellow]" + ) + + def save(self, path: str, compression: str = "gzip") -> None: + """ + Saves model to disk using compression to reduce file size. + + Args: + path: File path to save to + compression: Compression method ('gzip', 'lzma', or 'none') + - 'gzip': Fast compression, ~60-80% size reduction + - 'lzma': Better compression, ~70-90% size reduction, slower + - 'none': No compression + """ + if self.verbose: + console.log( + f"[blue]Saving stacking model with {compression} " + + f"compression to {path}[/blue]" + ) + + # Optimize base models for storage first + self._optimize_base_models_for_storage() + + if compression == "lzma": + # LZMA provides better compression but is slower + with lzma.open(path, "wb", preset=9) as f: + pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) + elif compression == "gzip": + # Gzip is faster with good compression + with gzip.open(path, "wb", compresslevel=9) as f: + pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) + else: + # No compression + with open(path, "wb") as f: + pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) + + if self.verbose: + console.log( + f"[green]Saved compressed stacking model to {path}[/green]" + ) + + @classmethod + def load(cls, path: str) -> "StackingTimeSeriesModel": + """ + Loads model from disk with automatic format detection. + Supports both compressed formats and legacy joblib format. + """ + import joblib + + # Try different formats in order of preference + loading_methods = [ + ("lzma", lambda p: lzma.open(p, "rb")), + ("gzip", lambda p: gzip.open(p, "rb")), + ("pickle", lambda p: open(p, "rb")), + ("joblib", None), # Special case for joblib + ] + + for format_name, open_func in loading_methods: + try: + if format_name == "joblib": + return joblib.load(path) + else: + with open_func(path) as f: + return pickle.load(f) + except ( + lzma.LZMAError, + gzip.BadGzipFile, + OSError, + pickle.UnpicklingError, + ValueError, + ): + continue + + raise ValueError( + f"Could not load model from {path} - unknown or corrupted format" + ) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml b/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml new file mode 100644 index 0000000..2f1d2cb --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml @@ -0,0 +1,11 @@ +channels: +- conda-forge +dependencies: +- python=3.10.16 +- pip<=25.0 +- pip: + - mlflow==2.7.1 + - pandas + - numpy + - scikit-learn +name: mlflow-env diff --git a/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml b/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml new file mode 100644 index 0000000..0a0396b --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml @@ -0,0 +1,7 @@ +python: 3.10.16 +build_dependencies: +- pip==25.0 +- setuptools==79.0.0 +- wheel==0.45.1 +dependencies: +- -r requirements.txt diff --git a/tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl b/tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..aa3f5aa8ef7c82c96b4bd1fda6e43cd695abf3f9 GIT binary patch literal 123 zcmZo*om#*E0X;IMC7C(Jdbv4iIr-&!1(j)~dCBqRMTrFksYS(8dW1rX67!1F@{4j) zi^3tIQzlQ*Y@AX%MWaWgq$n{nFEcMa9>{>Hn&Q_ZnwgiDT9lfXoQf(@nxqE+?KdyV literal 0 HcmV?d00001 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt b/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt new file mode 100644 index 0000000..fd9a283 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt @@ -0,0 +1,4 @@ +mlflow==2.7.1 +pandas +numpy +scikit-learn \ No newline at end of file diff --git a/tmp/artifacts/data_model/transformers/courier_transformers.pkl b/tmp/artifacts/data_model/transformers/courier_transformers.pkl new file mode 100644 index 0000000000000000000000000000000000000000..18fe2203362307ea711c91971c88e266be720d81 GIT binary patch literal 29607 zcmeHQS(6-BR<`!BWJ%U;*|MF3Ev%Vww`yZ)=rO)VP9 zg98SzEkgoBIOhR^2MiArLBs?E@WLBdj}ZI?f_?b~yuo+Q-EU@CT{AkGgBp4*zSjPm*tZFCo?w;pS#H+6?>kPP=+z&|asTUh)n^dt1>)J!~iO zu3mow*H)eGTj8xFicj_Wovp2Cqq^1YtVg|Gb-mLZv|vb_2a37!*z)Y!Sbidx%sp_qD@x+W zso3j>t*ssuCRN%x^W|kk8FkQjj%Vq-swi;H@brC5;8c$ z_c}SdtF29>_&Hm>-tI@;c8HX>!gjbBb&=^OE;c*s?=FaP@<|e(lor)y{d#X)nY%+BVb8u>*R47j7@;Uq)%jL?2u}hbxr_*>&;@swBoiCOX z(c)aHfa{53ZenbDYJM6-c`D{}s#xISq$&z&!74g^$?Zv^G=n;GIq)2!Ubi~Sl~h-k zr4ua97aeiI7sHzC5z26ax#lI;@-O%hgZ>7P)``_@@3t8$CWOc4?hJCAWbrw+efxHhC4KQWJ#~cC>F`&)LZZlNGkfs(y zvMOVotgb1HqKUjK?zcLm7n;i*5?B>gO~e9wsiQ4_QcA z^(yGh^DI*0PGNCkxMQf0JJj`sm;rkVV|_V=Nqr7bCz|=uHaf2gkP$N@n@p(wA?wWB z9OQ(pnKv4}Ao)xfY6^lu&R(sXXPBE1WVPvJN#45GdsPCzDQ>*CV9=s z(VWaBu~?%y5zwSwtmUkt3G!4$?ayrjp75a{2rv~|1O<__gG5OksF2H#tyqIm^_U~M ziHpm*eBu1H%kx-|xsyq(n1PjS%$jI1(OV_48xV0~)U!;L#Nr}LG;UxgOBWp3q$8VD zGHx1bTecG=P6Sx&)mN1;A?X@;r)8BO8&}(s$%R!kiP~ob-wMRW#D?9C2}?@KK<;B~ zkko5`Q~TT6XEX;=GtfYl-7YtYv=>B*X*RGl#4>+KXo!SUKm2Lx+3~YO()3TRU5}`d>Y9-Ef-=$s1o-ntqK1vJMIly#yw1zyd@}?)b7}(6YdIy75^h6E{@wi z6vyp)isN=YMOx~#M|qO%=jr$wCNYryyR#00)mj#5zhxDRu1h5hA4^)vC$;~ufjp9C z$qS-oRo04P&!viGj3P_90?+>0251jy$qTM4Mb{PD*jmaJ%+>#719{h#g6m4jb*1dK zQ>B#D{>>qCT`9P(lw4QJZh|Z2r1q~4nd=H|YOO#@t}A6X!Ier<`!8EP@48ZST`9S) zl-&eZCX(8}*klFQ6?7e2T5??}y9utiL&SBf;JSr{)ggA>D!a+9OeVGewAE=3YbC7c zx>a)BD!a+X`womi%evsYLeIvQwB)){cB7iWX!vhiz2Lf1bX_UAu80S3JyT2|suT23 zn#XKR+tY=7ApT)??$Zgl8_w#$1$*`9#$=≶CS=Q#Vtj$pt=DYgm zYw%$uS|4mh-SF-Y@BT?AiFZ_!#>E|vJvQ_b?^<4+O&XV>6Ypt#_LoGm<80EHA)_mF z;k#kQc<0=kN#k|4Zl0$Xa*bKXDBijCdeV4_wBnLDn7FVvg4OIuyh ztH=~h>_ipsv>+Lc9zPiiI-D zN{Tu3g{1K&dv`XVm(aqycqh7Iyqkge@v}j%(*#)$?`Ib=GBR;et|AuKpO9ig5;qve-W=9o^8BiDrx*8C2i#8%CG#7_>ARCxQP$UMA1gD zP;cU69CV=tbfExCNatqMz0F-n+y>ND4v;hq78V13tTZL`lp|BkdMmin?euz`UNF<_ z^zlT@*i5~xk)y)qa&HTjG@^Q+y4yikBW#*K8#%U&k6wCMy@Kgx*i+SIps4oDweUUC zoSF~Tq1>zA5c)x@NDJA?#rn;t8C+a;r9Koxk6l94r ziAy7hg$({>6KnAa**4R+r~y>fXc&pF-wM#3Ig}(muESGUK<^Y!%D9k3t;~WfeQb=C zJ}R{IAvk3!g={10HNzGqkv_Cji8W}D!xsINtQ%E(l57)f#I=cCMUaE!j2M)tn})5} zI0hHPX4t;_L$vNeDF}EV<1^0c*=jHk7pdn%hJlOW%B?O@)#7N?8~8|ZoKyk|3Kpj? z=@!2_6R?D>0yNqQIhY|J3yq&LHo(WFARnN;E_HhN#IqGlVdohJ(~C4Q9hEc+0u3>7PFc%zWeFsfCwqwjqhX5j-Sg)6MNQ*`dw_bPw9Hwv?#Bxhc}}(K8`hk zO|@7u-HgIE;Xi4*;WR#4Mt$|}`T&W?8cuB?=hGzq=Cufk6axj5*=XhGq#s6W++{mF zwGwnYgUwph4QAE?q|8Im79LW>gaNB{;d!tY;K7en6aX3wi6cZDSr~eE3PbljZ8@OR zM6RGW4wd0i^yy)C^eU5thD_#|<3oVMg^2Y}2+W=|Pvi z(vZZ@Lo64yAsR>9yc)TwPlNC%4Wa>#2y>NQZUd>PFwqRhWj$U?WWi{RGp<)Ag`z!1 ziP0p_j57++n$vtYrObAj4e4W{a8_R0B3ZD=Sd~g&7@>V_cPMYG^S_^SfYC>%M>f7 ziW3-{DICL+@T{6nIb+$bC6fs?SlO}XUhWf+%G8Rk;QCKflvw&$IiThz&umPthUgt9 zsDb$<(xlkC$*El9g`9(bUFIyGn|>>on+V1%sajE$?|Olu)eln%Ey5U|vfWcs;i$DZ z2DEUlE(a@@-=NW9Ew#2}fg4MbDhpJWpIlqu&&8fi%kVl*%8}|P@x>X6f~rz0zBFm^ z140*USp34EH-P_YBt{ezJTqMAG$*B@5(+0!$#F9Zm5-O-fPNNC!M>!oO4R3j5x%2^ zZ2`Uh%atou_(^R)>?+i|~cCmnmBMI5IDuN#>_kg9XTg)yvb|Wo>{u zrBWBO-1IPUiDx-atXdtNh#r>YLz~qdTmibnQ_E^6jCK|ebNm^F*|Jj(jY?rC=>$($Z@H# zvb0K=%%xVV9ng1<&caMQE01Q&*0cD?VelQoGi9V7DG`ASK*O{=cNE8Mh-oh+`caX( zjVlf!6`f%zrPNUC^Rm+h*pB5USJ!ep*-~>Tg40|NjZWka71^>tS>rK|t@Au5YZbU@ znX?17?1(edigyyv2#kh2rZK`WlJW(bU9bsKQ?wN0E0T`*d3RpOAb4>(>aIuaey~dK z;Ceh6WT*}If<9Xr;RT6DdqEo8@PgF7E^@(9;;}30b~Xn6j_q^OIJy&0OfSif9^$zf z!O15}oj{!xOj&3_nqoz=pO7WFxEzv2xw$rLvm}%+oy5~9ZGa@s+d*2c&>Zp!NJ59y zK7AVB5vDg71ehIC4@Mf$?ECR9WWL_6h4jE=3RruNPz}OdYah2>&FVgbO__d2%P(my z$w_HIZ)@qbi>@kLK8Gcfa;V=*v<8Vt+0zn&0FZd zGtqi2#0#QV0<`OM(@R%ZW-qUxXHPFJP6spdOKZW*?BdcY5PJUyc({Uijr{ybl@7K8 zBWs}-&!ZI(_h<#0J&Y5y;ovytXBU?7Fi78SbR)4fgo22~3O21mXOOUCBDcLSoLVU4^<=Nr(9LNZM3SyKP0gMvE`Dg6P z#VAMb8kfF1t~pZSx6-b7P7f4#e3nW9q4J1Rs4MePJ<1}|%o?Tl?2a;Au1(KoS z1(MeA0?ANt1IbW8AZpfv+h(!I?pzrfW8!1Td$2f@?2w8mntS*{J?vg8ZquMiFqTIw ze`$qO(`2Z4O_SE}nkGZRZJG>)VS!N7s5#e4hG9?&BLdMXLjzG7Lj%zY!vfI?2*jFt ztR3} zXl#qS_~<0S9_)tMx#Fdz-bntGZ-xiHsEd~?+cVFl&G~cJ5j9>&n}`~_;v?G;rr-8W z-K{?Y#fu*GX%V8dVo2hOZwz&-O8kxvdv*PZ!x(>v7QKhv`wOEC@fj3tdNb(n;hU^F ziOUQ4)ggbK3Lmz?i~x zZ%~B~?Gm5Ddjpy|=uHBq5qhaA-VZcBpo-l~eAIdmpr+Eshotyby~IyiA&K`GjgKgx zALk`LE?OnHh!stJ`q@hwcgV4fOLQ&^{UTU=M0WjEeD9Thg-m__W2*kgbcs*Pj!IvH z!PHLghU6-rG=7sDyYCA=pu*3)>*6~WOKp5r3tu7qEuwmGm-v{~z2LTdQ?^h3Vn1p8 zHr4m&@(^$$p2KUgP+-pne}%l^&>Cwab-%K58ntwZ|P$o{4QDE_a)xD z8s7cU2JF92_V>QT`!4f778bv!E%+|$!hA+Bl-{Uh-5E2MnxeTg0&~YwQ#5zRZ1mkW zELbH=&j{}S&hzU239>963aM9-SI2gX%(7kOFQVtUJG5x(Fe(Q7D&Bo*DgG)Z=PT&* z1OL8weD2j?G8lU`xJru>K78F6;DY@}4gcYBqho$(^uX(WM2H^&JW9f&Bz$SV1$orj z9(A@yo$b*mcr*$gje_(ji1)r7HJhF9BpISTSGrMzUw$We_)m+|2rHV4R@gRyw%E>1 zMn&2zLuR0?J8h|NMrw@|YKrttmR-F2Vsn5ue?OpCEasA(bfd66@~jF+V-J=$&889K zhKBy1-Ixf?T{=V~Gk0k%jciLvYwH_hc3*eeUf+y1*J=;y##EPP93@L@f0>B!%-r2q z{l<*%M%dnmrkPymZWQPCHEEhME<6~{FR}LMT-s$5&OLL#eNSCYGmSsIy9np)#$UTW zsNtpU+hBab8bQB<#D@KTmky+k_f~K4A=0=#71FM%SKSzFA?;y5!tv7A zqh>F;(;HlmkFR$lfo*Q$Y-@3(baDrkbURzsHjd9m;V6FY6s`9mzX9a|z93ZPmO5&k zpA1LD(b+gaR3EBbo$LR>1GP-%H3)c|@_ zq(%pQ!z-ghhkgsM_@G>zZRt=c&`>GFP-(=Y@&XiUWy>pt_*5{|RN`33g6ow6jaLdW zR2p%qa2T!=AgclNs7OucO6e%C^m)lPl>rr%K@^pRNUzip%6XTHYb@urSC*Re%0e`i zJgM2GQlRllA;v3>Xew?R97Sy9N;z*@&>)6N8)_;Mi_?P2`9dl{sCfYb(Nw}uk4k}l zfHsxVF;whlJ}W?ghDxhxuQZ~);s9|Jnv*L98Y+c&0g|Dn66DSplu5?fYlcdzd0xR# zQ;F)F-Y(OgGrip)zE?2RR9sylD^~zLuSm`HiV#gDyz;0NXs8ros5GLf*ee{I?4(_R zhDsrZN+X&|kXKwP1sW=a7%GiODmv?$4y|_rG@zm~h@!F(kIHF5rDj^tAU>68LDIpg ztp#V`7|`}gs>v2KIv*>$rKgT ztNfq~=l);s!XGxLOiN0uO?Go0r9D9uXqe%^>lKjK;xA{43$Q-R|0Vc zjy{vw3Q()*05zhiXxgDOF&(cIXuML0@k%2emFJaEGfS#ydisL|J{1r_CC?DWdBK)f z7AU>45J_bt9+lmKT1~g0h4@su1+|WyD+~0!f|^LXrQ-)^=Sr*TTv>=uC3AJmoc3It z(8+qzLD?@CVm1$=d|7lQGih0hKz%w=q&QtI4%DKnn50>XKz%w=^ymokyepd7*`fpL z(~+V_N03K{zfqvNzeNYsrz1s=jv$Y&Vw%;W1M1U}qDM!NM_0^j3ZOn6DSC7Sd34qc zWO=7{^)BSok*!BpF@w~y1?qc8ik^1_d35-L5*bYa)TbjwkB%UZuAJEvKz%w=^ymok z=qhGAZ3kOQ(UGD@N03KX${ZCyeeX!oqa(eG>;M@NuHS25dcJF-fOjubsQf;_rXW>Wz5y(2}B zjv$W?e=a6tQ~>qqNYSGs$fK)dt`$IiI#TrL2=eGknN0!Irz1s=jv$Y&oY@pWeL7O~ z=m_%YDw%5qP@j$zJvxFsI{bl~jHUqU(~+V_N03KX&TI;xJ{>7~bOd>HmCUsQs82_V z9vwj*T`99EfckW#=+P16(Umis0;o?%iXI(79-Z}8&>FKqeL7O~=m_%YN|{Xo)Tbjw zkB%UZuAJEvKz%w=^ymok=q57P3ZOn6DSC7Sd32@BrU2^Gk)lUOkVj{|>$RFfz3g=% zpN?!jx{1uS0;umDDSF-!1iIR(C_bd#nB!GC{%~W`_y@f3$7}Z!_4bXZi-V9N=TBSK zcH+2l|tstR1Qy(SMC{D9vE}{{VLXpJ4z1 literal 0 HcmV?d00001 From caca9237176bf087d1dfb9f39dd67468314c6e49 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 12 Sep 2025 14:30:52 -0300 Subject: [PATCH 03/29] SIENTIAPDE-1214 SIENTIAPDE-1214: Update requirements.txt to clarify dependencies and improve project setup - Commented out the previous sientia-mlops-library dependency for better clarity. - Ensured that the requirements.txt reflects the current state of dependencies for easier management. --- ## Problemas no Courier:.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 ## Problemas no Courier:.md diff --git a/## Problemas no Courier:.md b/## Problemas no Courier:.md new file mode 100644 index 0000000..48083e8 --- /dev/null +++ b/## Problemas no Courier:.md @@ -0,0 +1,9 @@ +## Problemas no Courier: +1. Enviamos a coluna timestamp do index para fazer o transform, para poder sincronizar a predição com o pacote que gerou ela, visto que vários modelos podem retornar uma lista de predições em vários casos. No caso do Courier, está vindo um timestamp que começa em 0, estando dessincronizado com os dados que enviamos. Seria possível alterar o comportamento do modelo para retornar o mesmo index que enviamos? + + Segue uma output do transform de exemplo: +``` +'303-WIT-230_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '303-WIT-230_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '303-WIT-230_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '303-WIT-230_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '305-WIT-135_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-135_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-WIT-135_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-135_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-160_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-WIT-160_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-WIT-160_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-WIT-160_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-PIT-170_median': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-170_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-PIT-170_min': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-170_max': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-175_median': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-PIT-175_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-PIT-175_min': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-PIT-175_max': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-FIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-FIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-013_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '305-FIT-013_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-FIT-013_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '305-FIT-013_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '306-PIT-101_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-PIT-101_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-101_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-PIT-101_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-FIT-051_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-FIT-051_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-051_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-FIT-051_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-DIT-001_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-DIT-001_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-DIT-001_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-DIT-001_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-PIT-105_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-PIT-105_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-105_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-PIT-105_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-FIT-052_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-FIT-052_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-052_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-FIT-052_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-DIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-DIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-DIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-DIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-PIT-115_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-PIT-115_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-115_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-PIT-115_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-FIT-004_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-FIT-004_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-004_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-FIT-004_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-PIT-110_median': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-PIT-110_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-110_min': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-PIT-110_max': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-FIT-003_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-FIT-003_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-003_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-FIT-003_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-PIT-125_median': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-PIT-125_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-125_min': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-PIT-125_max': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-PIT-130_median': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-PIT-130_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-130_min': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-PIT-130_max': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-FIT-006_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '306-FIT-006_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-006_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '306-FIT-006_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '307-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-003_median': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIT-003_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-003_min': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIT-003_max': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIC-022_median': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '307-FIC-022_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIC-022_min': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '307-FIC-022_max': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '310-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '310-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '310-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '310-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '307-FIT-008_median': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-008_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-008_min': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-008_max': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-009_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '307-FIT-009_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-009_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '307-FIT-009_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '309-PIT-101_median': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-101_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-101_min': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-101_max': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-105_median': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-105_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-105_min': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-105_max': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-110_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-110_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-110_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-110_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-185_median': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-185_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-185_min': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-185_max': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-190_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-190_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-190_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-190_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-195_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-PIT-195_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-195_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-PIT-195_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-FIT-051_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-051_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-FIT-051_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-051_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-052_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-FIT-052_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-FIT-052_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-FIT-052_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-PIT-001_median': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-001_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-001_min': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-001_max': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '309-PIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '309-PIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '317AIT003.3_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, 'SiO2_conc': {Timestamp('1970-01-01 00:00:01.732971600'): 5.15}}} +``` + +2. No modelo do transform (data_model), o nome do método que faz o transform de fato é "transform", sendo que em nossos modelos, por padrão esse nome é "predict". Seria possível alterar o nome do método manta manter a compatibilidade e o padrão que já temos? From 934298b3c31133482fd2e3753777132a57039d54 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 15 Sep 2025 14:47:24 -0300 Subject: [PATCH 04/29] SIENTIAPDE-1222 SIENTIAPDE-1214: Enhance MLFlow and tests with datetime index handling and logging improvements - Added a new method in MLFlow to detect and parse datetime indices in DataFrames, ensuring proper format and raising errors for invalid types. - Updated prediction workflows to utilize the new datetime index handling, improving data integrity during transformations. - Enhanced logging in model_repository to include detailed data outputs for better traceability. - Adjusted timeout settings in prediction workflows for improved execution time management. - Updated tests.ipynb to include additional checks for index types and outputs for better validation of functionality. --- data.csv | 11 + laborious/activities/mlflow.py | 75 +- .../utils/repository/model_repository.py | 1 + laborious/workflows/predictions_batch.py | 2 +- .../sub_workflows/prediction_process.py | 4 +- response_data.csv | 2 + tests.ipynb | 54 ++ .../artifacts/training_transformer.pkl | Bin 0 -> 29391 bytes .../code/utils/models/base.py | 824 ------------------ 9 files changed, 144 insertions(+), 829 deletions(-) create mode 100644 data.csv create mode 100644 response_data.csv create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl diff --git a/data.csv b/data.csv new file mode 100644 index 0000000..76acc92 --- /dev/null +++ b/data.csv @@ -0,0 +1,11 @@ +timestamp,07BP012/VEL_M1_PV,07BP013/VEL_M1_PV,07BP014/VEL_M1_PV,07FT001_COR_B,07FT001_COR_G,07FT001_COR_R,07FT001_TEXT,07FT007_COR_B,07FT007_COR_G,07FT007_COR_R,07FT007_TEXT,07FT012_COR_B,07FT012_COR_G,07FT012_COR_R,07FT012_TEXT,09BP023/VEL_PV,09BP024/VEL_PV,303-WIT-230,305-AIC-001_PV,305-AIC-002_PV,305-CALC-001,305-FIC-001_PV,305-FIC-003_PV,305-FIC-005_PV,305-FIC-006_PV,305-FIT-002,305-FIT-009,305-FIT-010,305-FIT-011,305-FIT-012,305-FIT-013,305-LIC-001_PV,305-LIC-002_PV,305-PIT-170,305-PIT-175,305-SIC-001_SP,305-SIC-002_SP,305-WIT-135,305-WIT-160,306-CALC-018,306-DIT-001,306-DIT-002,306-FIT-003,306-FIT-004,306-FIT-005,306-FIT-006,306-FIT-051,306-FIT-052,306-LIC-001_PV,306-LIC-002_PV,306-LIC-003_PV,306-PIT-101,306-PIT-105,306-PIT-110,306-PIT-115,306-PIT-125,306-PIT-130,307-CALC-001,307-CALC-002,307-FIC-001_PV,307-FIC-005_PV,307-FIC-006_PV,307-FIC-019_PV,307-FIC-022,307-FIC-101_PV,307-FIC-105_PV,307-FIC-110_PV,307-FIC-115_PV,307-FIC-120_PV,307-FIC-130_PV,307-FIC-135_PV,307-FIC-140_PV,307-FIC-145_PV,307-FIC-150_PV,307-FIC-155_PV,307-FIC-160_PV,307-FIT-003,307-FIT-005,307-FIT-008,307-FIT-009,307-LIC-003_PV,307-LIC-004_PV,307-LIC-101_PV,307-LIC-105_PV,307-LIC-110_PV,307-LIC-115_PV,307-LIC-120_PV,307-LIC-130_PV,307-LIC-135_PV,307-LIC-140_PV,307-LIC-145_PV,307-LIC-150_PV,307-LIC-155_PV,307-LIC-160_PV,307-SIC-006_OUT,307-SIC-007_OUT,309-FIC-013,309-FIC-014,309-FIT-051,309-FIT-052,309-LIC-001_PV,309-LIC-002_PV,309-PIT-001,309-PIT-002,309-PIT-101,309-PIT-105,309-PIT-110,309-PIT-185,309-PIT-190,309-PIT-195,310-AIC-001_PV,310-AIC-002,310-CALC-001,310-CALC-002,310-DIC-002_PV,310-FIC-004_PV,310-FIC-010_PV,310-FIC-110_PV,310-FIC-120_PV,310-FIC-160_PV,310-FIC-170_PV,310-FIT-004,310-FIT-005,310-FIT-006,310-FIT-010,310-FV-032,310-LIC-110_PV,310-LIC-120_PV,310-LIC-160_PV,310-LIC-170_PV,310-LIT-003_PV,310-LIT-004_PV,310-SIC-003_OUT,310-SIC-004_OUT,310-SIC-005_OUT,310-SIC-006_OUT,311-FIC-029_PV,311-FIC-033_PV,311-FIT-033,312-CALC-001,312-CALC-005,312-CALC-006,312-DIC-001_PV,312-DIC-002_PV,312-FIC-001_PV,312-FIC-002_PV,313-CALC-001,313-DIC-001_PV,313-DIC-002_SP,313-FIC-006_PV,317AIT001.2,317AIT002.1,317AIT002.10,317AIT002.11,317AIT002.12,317AIT002.13,317AIT002.14,317AIT002.15,317AIT002.16,317AIT002.17,317AIT002.18,317AIT002.19,317AIT002.2,317AIT002.20,317AIT002.21,317AIT002.22,317AIT002.23,317AIT002.24,317AIT002.25,317AIT002.26,317AIT002.27,317AIT002.28,317AIT002.29,317AIT002.3,317AIT002.30,317AIT002.31,317AIT002.32,317AIT002.33,317AIT002.34,317AIT002.35,317AIT002.36,317AIT002.37,317AIT002.38,317AIT002.39,317AIT002.4,317AIT002.40,317AIT002.41,317AIT002.42,317AIT002.43,317AIT002.44,317AIT002.45,317AIT002.46,317AIT002.47,317AIT002.48,317AIT002.49,317AIT002.5,317AIT002.50,317AIT002.51,317AIT002.52,317AIT002.53,317AIT002.54,317AIT002.55,317AIT002.56,317AIT002.57,317AIT002.58,317AIT002.59,317AIT002.6,317AIT002.60,317AIT002.61,317AIT002.62,317AIT002.63,317AIT002.64,317AIT002.65,317AIT002.66,317AIT002.67,317AIT002.68,317AIT002.69,317AIT002.7,317AIT002.70,317AIT002.71,317AIT002.72,317AIT002.8,317AIT002.9,317AIT003.1,317AIT003.2,317AIT003.3,317AIT003.5,319-CALC-001,319-CALC-013,319-DIC-001_PV,319-DIC-001_SP,319-DIC-002_PV,319-DIC-002_SP,319-FIC-006_PV,319-FIC-006_SP,319-FIC-007_PV,319-FIC-007_SP,319-FIQ-CALC-009_DAY,319-FIT-004,319-FIT-005,319-LIT-201-R,319-PIT-003,319-PIT-004,319-SIC-001_OUT,319-SIC-002_OUT,Fe_conc,G03-07BP102_M1,G03-07BP103_M1,G03-08BP107_M1,G03-10BP104_M1,G03-19BP101_M1,G03-19BP106_M1,G03-19BP110_M1,SOL-CALC-005,SOL-CALC-006,SiO2_conc +2024-12-05 02:16:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2185.6279296875,8.909636497497559,8.824532508850098,2628.087646484375,233.4102020263672,289.2882995605469,2.2411208152771,2.740176200866699,3158.830322265625,687.0074462890625,0.0,287.7219543457031,0.0,3312.36962890625,89.0501937866211,83.04390716552734,12.823511123657228,13.373872756958008,,,1364.865478515625,1239.42919921875,3.005729913711548,1.3840404748916626,1.3935494422912598,2170.908447265625,2189.05615234375,1866.9991455078125,1715.1016845703125,2254.970703125,2276.34033203125,83.4413070678711,94.47754669189452,89.85079956054688,34.401798248291016,33.98549270629883,31.97739601135254,34.75410461425781,43.70286178588867,44.13051223754883,1894.9356689453125,2017.797607421875,900.0048217773438,30.6091537475586,375.9732971191406,0.0,652.0335693359375,765.3798217773438,763.6800537109375,769.5640258789062,739.6097412109375,719.44775390625,706.0026245117188,703.8912353515625,567.9400634765625,593.1154174804688,629.06787109375,595.2117919921875,625.0,1.425487995147705,27.0,1194.873046875,998.1407470703124,105.17456817626952,99.1063003540039,28.07830810546875,21.692842483520508,34.955604553222656,12.697091102600098,40.85791778564453,21.66144752502441,17.217727661132812,47.85460662841797,57.36534118652344,59.91471481323242,48.2706184387207,39.44066619873047,83.20423126220703,66.5,0.0,0.0,1768.509765625,1827.390380859375,85.6709213256836,96.77904510498048,0.0998583808541297,4.900833606719971,23.81103706359864,26.791275024414062,26.24358367919922,23.354522705078125,27.844594955444336,27.89141845703125,10.390382766723633,10.471061706542969,223.273666381836,209.89990234375,1.1763592958450315,2.309485912322998,103.65160369873048,450.9481201171875,427.9828186035156,450.59222412109375,448.7367858886719,17.0,700.9956665039062,0.0,1200.0,1.0,39.16987991333008,37.460086822509766,19.4918155670166,11.98000144958496,95.19344329833984,86.53215789794922,,,86.0,99.30213165283205,22.59608268737793,0.0,400.0,1351.7703857421875,0.0,1459.112548828125,2.163416624069214,1.6798019409179688,997.8304443359376,0.0240168757736682,203.056381225586,1.369994044303894,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.4978864192962646,8.562295913696289,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.253314971923828,13.389976501464844,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.01197052001953,1.7381054162979126,1.1206940412521362,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,59.35650634765625,1.4764769077301023,1.526507019996643,0.4823205173015594,0.7710468769073486,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3361487984657287,0.428571492433548,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,7.89943265914917,5.800000190734863,6.465214729309082,86.5999984741211,830.4485473632812,849.3055419921875,,,1.5839204788208008,1.572644829750061,0.0,600.0,920.1913452148438,921.7091064453124,0.0,10.023720741271973,0.0,11.650277137756348,4.348147869110107,0.0748394280672073,71.11778259277344,100.0,64.71,0.0,4.533299922943115,5.666272640228272,8.773231506347656,4.993200302124023,4.861800193786621,6.432722568511963,50.96606063842773,67.95718383789062,3.52 +2024-12-05 02:18:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2300.107421875,8.874933242797852,8.814615249633789,2379.468017578125,205.70236206054688,299.0367736816406,2.237759590148926,2.737307071685791,3149.414306640625,705.9229125976562,0.0,287.82403564453125,0.0,3319.203857421875,88.97682189941406,82.33074951171875,13.035848617553713,13.303841590881348,,,801.2099609375,1577.420166015625,3.18897008895874,1.3844648599624634,1.3944939374923706,2137.277099609375,2091.429931640625,1866.9801025390625,1714.9752197265625,2248.451904296875,2272.76025390625,85.66764831542969,88.35608673095703,87.84507751464844,34.351341247558594,33.97214126586914,31.98730659484864,32.71706008911133,43.68099594116211,44.0351676940918,1898.191162109375,1618.9393310546875,898.7774047851562,30.54461669921875,374.732666015625,0.0,648.9142456054688,761.4437866210938,761.818115234375,728.4386596679688,741.0633544921875,659.6345825195312,707.1867065429688,684.1427612304688,568.2828369140625,594.7976684570312,633.2908325195312,595.1723022460938,640.0,1.245144605636597,27.0,1194.7637939453125,998.0435791015624,103.9820556640625,99.07530975341795,28.694652557373047,22.18309211730957,31.824302673339844,12.703847885131836,38.318546295166016,21.9743595123291,16.507755279541016,47.80324935913086,57.33625793457031,59.91497421264648,48.0490837097168,40.25405502319336,83.34803771972656,66.5,0.0,0.0,1746.5220947265625,1811.0615234375,85.64175415039062,96.97100067138672,0.0999280512332916,4.900284290313721,23.82648658752441,26.91034507751465,26.227479934692383,23.32929229736328,27.45798110961914,27.736440658569336,10.399251937866213,10.47207736968994,175.54986572265625,211.37567138671875,1.174445867538452,2.112058639526367,103.86756896972656,450.9291076660156,427.931396484375,450.5900268554688,448.73138427734375,17.0,700.9699096679688,10.513471603393556,1200.0,1.0,40.18443298339844,37.6434326171875,19.451520919799805,11.894670486450195,95.58326721191406,86.0557632446289,,,86.0,98.96598815917967,22.82772254943848,0.0,400.0,968.2177734375,0.0,1464.4488525390625,2.163362979888916,1.679788589477539,1003.175537109375,0.0238033775240182,184.37130737304688,1.3698077201843262,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,8.57795238494873,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,13.411721229553224,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1217609643936155,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,59.35650634765625,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7702381610870361,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4283382296562195,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,8.100000381469727,5.800000190734863,6.435592651367188,86.5999984741211,725.95458984375,858.4331665039062,,,1.5835398435592651,1.5726622343063354,0.0,600.0,917.3837280273438,925.9874877929688,0.0,10.049739837646484,0.0,11.572147369384766,4.3472161293029785,0.0748393461108207,71.409423828125,100.0,64.71,0.0,4.533299922943115,5.762217998504639,8.699999809265137,4.993200302124023,4.861800193786621,6.558172702789307,50.968997955322266,67.96240234375,3.52 +2024-12-05 02:20:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2063.14990234375,8.86483097076416,8.804698944091797,2637.2822265625,186.7984161376953,274.3067932128906,2.234398603439331,2.734437942504883,3139.637939453125,738.9068603515625,0.0,287.9261474609375,0.0,3328.472412109375,84.00076293945312,86.40196990966797,12.14862060546875,13.639047622680664,,,1477.9354248046875,1141.748291015625,2.891580104827881,1.3848892450332642,1.3954384326934814,2059.924560546875,2129.599853515625,1866.961181640625,1714.848876953125,2241.93310546875,2266.62158203125,85.53884887695312,87.77095031738281,89.44815063476562,34.300880432128906,33.95878982543945,31.997217178344727,30.499488830566406,43.65913391113281,44.02417755126953,1907.386474609375,1944.6339111328125,903.0270385742188,30.480079650878903,374.2165832519531,0.0,649.119384765625,757.221435546875,759.9561767578125,811.8240356445312,748.2200927734375,740.3225708007812,708.370849609375,673.0714721679688,573.4213256835938,596.4798583984375,631.8908081054688,595.1328735351562,635.0,1.0720911026000977,27.0,1194.6546630859375,997.9464111328124,105.5190887451172,99.0443115234375,28.1169376373291,21.61070251464844,32.5518684387207,12.71060562133789,37.97523498535156,22.30083274841309,17.719982147216797,47.75189208984375,57.30717849731445,59.91522979736328,47.62279891967773,41.22840881347656,83.33912658691406,66.5,0.0,0.0,1749.242431640625,1821.6829833984373,85.61257934570312,97.1629638671875,0.0999977141618728,4.899734973907471,23.841936111450195,26.64358901977539,26.1314697265625,23.415620803833008,27.687856674194336,27.292091369628903,10.39584255218506,10.473093032836914,208.32379150390625,211.2283935546875,1.1736302375793457,1.0507607460021973,104.0835418701172,450.91009521484375,427.87994384765625,450.58782958984375,448.7259826660156,17.0,700.9442138671875,0.0,1200.0,1.0,40.472747802734375,38.068607330322266,19.54105758666992,12.325950622558594,95.0851821899414,85.76668548583984,,,86.0,98.85044860839844,22.84588432312012,0.0,400.0,1176.9923095703125,0.0,1459.0894775390625,2.1633095741271973,1.6797752380371094,997.1138916015624,0.0235898792743682,204.41549682617188,1.3696213960647583,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,8.100000381469727,5.400000095367432,6.405970096588135,86.5999984741211,802.98876953125,856.4476318359375,,,1.583159327507019,1.57267963886261,0.0,600.0,935.9249267578124,930.265869140625,0.0,10.131684303283691,0.0,11.596683502197266,4.34628438949585,0.0748392716050148,71.35333251953125,100.0,64.71,0.0,4.533299922943115,5.731375694274902,8.699999809265137,4.993200302124023,4.861800193786621,6.488824367523193,50.97193908691406,67.96761322021484,3.52 +2024-12-05 02:22:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2344.354736328125,8.854728698730469,8.794782638549805,2443.620849609375,235.85362243652344,289.5343017578125,2.231037378311157,2.7315685749053955,3128.458251953125,678.6727905273438,0.0,288.0282287597656,0.0,3352.32080078125,82.41395568847656,88.41472625732422,12.335658073425291,13.590455055236816,,,1179.0438232421875,1241.806396484375,2.9580600261688232,1.3853135108947754,1.3963829278945925,2020.5482177734373,2175.6435546875,1866.942138671875,1714.722412109375,2241.09228515625,2263.143798828125,85.41004943847656,89.18098449707031,91.12660217285156,34.372066497802734,33.94544219970703,32.00712585449219,30.11072158813477,43.63727188110352,44.04359436035156,1842.406005859375,1904.2274169921875,902.5571899414062,30.415542602539062,374.7893676757813,0.0,651.5109252929688,768.8901977539062,758.0942993164062,720.6381225585938,742.55078125,685.2013549804688,709.5549926757812,730.93359375,572.37548828125,598.1620483398438,634.2003173828125,595.0933837890625,632.0,0.9020777940750122,27.0,1194.54541015625,997.8492431640624,104.99007415771484,99.01332092285156,28.318359375,21.52416229248047,31.58490943908692,12.717362403869627,38.4969482421875,21.7913761138916,17.436378479003906,47.70053482055664,57.278099060058594,59.915489196777344,46.977535247802734,42.46583938598633,83.07581329345703,66.5,0.0,0.0,1748.264892578125,1827.9678955078125,85.58340454101562,97.35491943359376,0.1000673845410347,4.899185180664063,23.85738754272461,26.20622062683105,26.06732177734375,23.50194931030273,27.590068817138672,27.46240234375,10.392244338989258,10.47410774230957,214.98492431640625,205.6874847412109,1.173506498336792,2.398390531539917,104.2995147705078,450.8910827636719,427.8284912109375,450.5856323242188,448.7205810546875,17.0,700.9185180664062,0.0,1200.0,1.0,40.27254486083984,38.66741561889648,19.67565155029297,12.237468719482422,93.86233520507812,85.74674987792969,,,86.0,98.91876983642578,22.30738639831543,0.0,400.0,1275.9793701171875,0.0,1467.3343505859375,2.1632559299468994,1.6797618865966797,998.2872314453124,0.0233763810247182,187.3364105224609,1.3694350719451904,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.708251953125,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.3545424044132232,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3020144402980804,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.100000381469727,5.400000095367432,6.699999809265137,85.80000305175781,747.72314453125,830.0194091796875,,,1.5827786922454834,1.5726970434188845,0.0,600.0,924.1300048828124,934.5442504882812,0.0,10.00216579437256,0.0,11.625198364257812,4.345353126525879,0.0748391896486282,71.52710723876953,100.0,64.71,0.0,4.533299922943115,5.767271518707275,8.699999809265137,4.993200302124023,4.861800193786621,6.6877641677856445,50.97750854492188,67.97283172607422,3.52 +2024-12-05 02:24:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2862.970703125,8.844626426696777,8.774948120117188,2980.736572265625,237.5259704589844,291.2091369628906,2.2276761531829834,2.728699445724488,3153.987548828125,682.5768432617188,0.0,288.1303405761719,0.0,3345.968505859375,87.24137878417969,86.74526977539062,12.552753448486328,13.396173477172852,,,1370.993896484375,1611.220947265625,2.8518500328063965,1.3857378959655762,1.397327542304993,2030.3245849609373,2142.251953125,1866.9232177734373,1714.6802978515625,2248.769775390625,2268.414794921875,85.06763458251953,91.27864837646484,90.73202514648438,34.44520568847656,33.932090759277344,32.01703643798828,31.01426696777344,43.61540985107422,44.06300735473633,1867.2406005859373,2242.3193359375,898.3688354492188,30.811418533325195,375.3621520996094,0.0,647.7444458007812,756.3349609375,759.6863403320312,795.988525390625,743.1278076171875,681.9320678710938,710.7391357421875,760.3604736328125,576.6845092773438,599.8442993164062,620.6749267578125,595.053955078125,640.0,0.7632204294204712,27.0,1194.4361572265625,997.7520751953124,106.02984619140624,98.98233032226562,28.794273376464844,21.658815383911133,32.04315185546875,12.724120140075684,38.84426498413086,21.82027244567871,16.058849334716797,47.64917755126953,57.24901580810547,59.915748596191406,46.44183349609375,41.52559280395508,83.30157470703125,66.5,0.0,0.0,1750.5181884765625,1845.0968017578125,85.55422973632812,97.546875,0.1001370549201965,4.8986358642578125,23.87283706665039,26.4957332611084,26.47050094604492,23.5178451538086,27.697071075439453,27.838388442993164,10.388647079467772,10.475123405456545,243.6897125244141,207.8060302734375,1.1728342771530151,1.2688955068588257,104.5154800415039,450.8720397949219,427.7770690917969,450.58343505859375,448.7151794433594,17.0,700.892822265625,0.0,1200.0,1.0,40.07234191894531,37.74433898925781,19.810243606567383,12.158288955688477,91.9724578857422,85.08606719970703,,,86.0,98.60803985595705,22.23607635498047,0.0,400.0,1401.1199951171875,0.0,1445.149658203125,2.1632025241851807,1.67974853515625,1001.5813598632812,0.0231628827750682,230.4506072998047,1.3692487478256226,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.899999618530273,5.400000095367432,6.699999809265137,85.80000305175781,927.35986328125,843.9348754882812,,,1.5823981761932373,1.5727144479751587,0.0,600.0,943.0855712890624,939.0200805664062,0.0,10.032031059265137,0.0,11.4324369430542,4.34442138671875,0.0748391151428222,71.41386413574219,100.0,64.71,0.0,4.533299922943115,5.681782245635986,8.699999809265137,4.993200302124023,4.861800193786621,6.551279544830322,50.98702239990234,67.97804260253906,3.52 +2024-12-05 02:26:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2654.10888671875,8.834524154663086,8.75146198272705,2392.275390625,235.610580444336,295.8012084960937,2.2243149280548096,2.725830078125,3172.66748046875,694.5460205078125,0.0,288.2324523925781,0.0,3339.616455078125,89.53855895996094,85.1905746459961,12.741495132446287,13.875475883483888,,,1323.8238525390625,1067.496337890625,2.916759967803955,1.386162281036377,1.3982720375061035,2102.00048828125,2161.18212890625,1866.9041748046875,1714.821044921875,2259.66064453125,2275.4189453125,84.60967254638672,90.81041717529295,90.06828308105469,34.51834487915039,33.918739318847656,32.026947021484375,32.243064880371094,43.593544006347656,44.08242416381836,1909.4129638671875,1879.74658203125,902.9876708984376,31.21604347229004,375.9349365234375,0.0,650.2518310546875,751.2490234375,761.6719970703125,715.2727661132812,747.89599609375,747.788818359375,706.99658203125,774.9991455078125,583.5380249023438,601.5264892578125,614.032958984375,595.0144653320312,615.0,0.7071666121482849,27.0,1194.326904296875,997.6549682617188,104.1503677368164,98.95133209228516,28.68854713439941,21.7934684753418,32.87420654296875,12.730876922607422,38.9033317565918,22.107240676879883,17.046525955200195,47.59782028198242,57.21993637084961,59.9160041809082,47.52975463867188,40.58535003662109,83.56719970703125,66.5,0.0,0.0,1742.1806640625,1809.867919921875,85.52505493164062,96.9273681640625,0.1002067178487777,4.8980865478515625,23.88828659057617,26.306615829467773,26.495014190673828,23.27698135375977,27.35708808898925,27.72958755493164,10.385048866271973,10.4761381149292,216.2808380126953,213.6409606933593,1.1716774702072144,2.5998244285583496,104.73145294189452,450.85302734375,427.793212890625,450.5812377929688,448.7097778320313,17.0,700.8671264648438,0.0148877017199993,1200.0,1.0,39.87213897705078,37.95595169067383,19.94483757019043,12.006688117980955,91.47502899169922,85.28714752197266,,,86.0,98.45233917236328,22.66000938415528,0.0,400.0,1256.264892578125,0.0,1452.758544921875,2.163148880004883,1.6797351837158203,1000.3143920898438,0.0229493845254182,190.5460968017578,1.369062423706055,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.899999618530273,3.299999952316284,6.699999809265137,85.80000305175781,762.7355346679688,865.4536743164062,,,1.5816150903701782,1.572731852531433,0.0,600.0,938.32177734375,943.602783203125,0.0,10.15860080718994,0.0,11.596125602722168,4.343489646911621,0.0748390331864357,71.58973693847656,100.0,64.71,0.0,4.533299922943115,5.802553176879883,8.699999809265137,4.993200302124023,4.861800193786621,6.558778762817383,50.99653625488281,67.98326110839844,3.52 +2024-12-05 02:28:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2227.77099609375,8.836018562316895,8.75161361694336,2456.804931640625,238.3328552246093,283.34686279296875,2.220953941345215,2.722960948944092,3188.196533203125,818.465087890625,0.0,288.33453369140625,0.0,3333.26416015625,90.52090454101562,83.3249282836914,12.903297424316406,13.22053337097168,,,1122.2493896484375,1331.5670166015625,3.0864999294281006,1.3865865468978882,1.3992165327072144,2095.321533203125,2129.77685546875,1866.88525390625,1714.9619140625,2259.91357421875,2288.202880859375,84.55709075927734,89.37647247314453,88.79876708984375,34.55312728881836,33.905391693115234,32.03685760498047,31.44712448120117,43.62167739868164,44.101837158203125,2035.30908203125,1930.71044921875,899.136474609375,32.6287956237793,376.5077209472656,0.0,653.2977294921875,750.5552368164062,760.983642578125,770.3090209960938,738.3396606445312,653.3170776367188,701.0360107421875,730.8182983398438,566.891845703125,588.6995239257812,629.2593383789062,594.9750366210938,622.0,0.6648255586624146,27.0,1194.2176513671875,997.5578002929688,105.18731689453124,98.92034149169922,28.36734771728516,21.92812156677246,32.240821838378906,12.737634658813477,38.52233505249024,21.57743263244629,17.07017707824707,47.54646301269531,57.19085311889648,59.916263580322266,48.34513473510742,39.64510345458984,83.13888549804688,66.5,0.0,0.0,1753.1981201171875,1814.801513671875,85.49588012695312,96.34169006347656,0.1002763882279396,4.8975372314453125,23.903738021850582,26.488170623779297,26.35142517089844,23.256576538085938,27.712129592895508,27.556394577026367,10.381451606750488,10.477153778076172,218.41183471679688,225.77491760253903,1.1706732511520386,1.818994283676148,104.94741821289062,450.8340148925781,428.7974853515625,450.57904052734375,448.7043762207031,17.0,700.8414306640625,0.5039713978767395,1200.0,1.0,39.67193984985352,38.82048416137695,20.07943153381348,12.295125961303713,90.39128875732422,85.13963317871094,,,86.0,98.48783111572266,24.05027961730957,0.0,400.0,1260.049560546875,0.0,1470.87890625,2.163095474243164,1.6797218322753906,1004.2244873046876,0.0227358862757682,194.2509765625,1.3688760995864868,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.86962890625,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,7.830216407775879,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042373403906822,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.7188185453414917,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7221007943153381,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.49777603149414,65.73210906982422,8.899999618530273,3.299999952316284,6.5,86.30000305175781,790.286376953125,913.74462890625,,,1.5807862281799316,1.572749376296997,0.0,600.0,945.046875,945.7421264648438,0.0,10.257745742797852,0.0,11.62919807434082,4.342557907104492,0.0748389586806297,71.7236099243164,100.0,64.71,0.0,4.533299922943115,5.762265205383301,8.699999809265137,4.993200302124023,4.861800193786621,6.415340900421143,51.00605010986328,67.98847961425781,3.52 +2024-12-05 02:30:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2773.8876953125,8.840205192565918,8.76478099822998,2499.578125,233.58840942382807,273.2201843261719,2.217592716217041,2.720091819763184,3169.754638671875,718.6017456054688,0.0,288.4366455078125,0.0,3293.60546875,88.42021942138672,80.45106506347656,12.661704063415527,13.48155403137207,,,1310.0872802734375,1188.9476318359375,3.5530900955200195,1.387010931968689,1.4001611471176147,2047.4097900390625,2157.358642578125,1866.8662109375,1715.1026611328125,2255.59716796875,2283.900390625,84.08345794677734,86.43714904785156,89.38142395019531,34.57453536987305,34.07208633422852,32.04676818847656,29.560300827026367,43.759544372558594,44.12125396728516,2028.601806640625,1870.660400390625,895.9712524414062,32.81827163696289,377.08050537109375,0.0,649.1517333984375,770.4885864257812,759.2982788085938,734.395751953125,743.6987915039062,678.6212158203125,700.5547485351562,663.7391967773438,576.5570678710938,589.0308837890625,628.8489990234375,594.935546875,631.0,0.6459924578666687,27.0,1194.1085205078125,997.4606323242188,105.77910614013672,98.88935089111328,28.33451271057129,22.062774658203125,32.428157806396484,12.744391441345217,38.12985610961914,22.12809181213379,17.180856704711914,47.4951057434082,57.16177368164063,59.91652297973633,49.058895111083984,39.42089080810547,83.53347778320312,66.5,0.0,0.0,1742.7498779296875,1810.687255859375,85.46670532226562,97.0500717163086,0.1003460586071014,4.896987915039063,23.919187545776367,26.559722900390625,26.710390090942383,23.56723022460937,27.740917205810547,27.745880126953125,10.377854347229004,10.478169441223145,205.0146484375,225.3446502685547,1.1705199480056765,2.214766025543213,105.16339111328124,450.8149719238281,429.5928039550781,450.5768432617188,448.698974609375,17.0,700.815673828125,0.239432543516159,1200.0,1.0,37.73193740844727,38.19848251342773,20.21402359008789,11.93883228302002,88.80018615722656,84.96267700195312,,,86.0,98.27005767822266,24.427764892578125,0.0,400.0,1261.719482421875,0.0,1463.796630859375,2.163041830062866,1.679708480834961,993.51123046875,0.0225223880261182,192.6799774169922,1.368689775466919,1.3799999952316284,0.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,761.5707397460938,910.4165649414062,,,1.5799574851989746,1.5727667808532717,0.0,600.0,953.1422119140624,947.8602294921876,0.0,10.270873069763184,0.0,11.646549224853516,4.341626167297363,0.0748388767242431,71.84945678710938,100.0,64.71,0.0,4.533299922943115,5.71589994430542,8.699999809265137,4.993200302124023,4.861800193786621,6.311936378479004,51.01556396484375,67.99369049072266,3.52 +2024-12-05 02:32:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,3052.78662109375,8.844391822814941,8.777948379516602,2515.992431640625,229.52468872070312,265.2821044921875,2.214231491088867,2.7172224521636963,3156.99365234375,701.08251953125,0.0,288.5387268066406,0.0,3317.0087890625,86.63294982910156,82.62700653076172,12.9373197555542,12.94904613494873,,,1328.0621337890625,1193.599609375,3.41225004196167,1.3874353170394895,1.4011056423187256,1862.8626708984373,2177.4072265625,1866.84716796875,1715.2435302734375,2251.281005859375,2277.60693359375,84.24079895019531,84.25446319580078,89.87091827392578,34.595943450927734,34.107933044433594,32.05667495727539,24.58193588256836,43.89741134643555,44.14066696166992,2033.9571533203125,1965.69189453125,902.446533203125,32.7397346496582,377.6492614746094,0.0,647.1126708984375,740.5242309570312,757.6129760742188,745.9622192382812,750.0130004882812,749.8059692382812,700.0735473632812,701.1631469726562,570.8465576171875,589.3622436523438,632.1378173828125,594.8961181640625,628.0,0.6863186955451965,27.0,1193.999267578125,997.3634643554688,103.6092300415039,98.8583526611328,28.94314765930176,22.19742774963379,32.61549377441406,12.75114917755127,37.71183013916016,22.179346084594727,17.744470596313477,47.44374847412109,57.132694244384766,59.916778564453125,48.76108932495117,40.08521270751953,83.61792755126953,66.5,0.0,0.0,1746.4300537109375,1854.5777587890625,85.64894104003906,97.2160415649414,0.1004157289862632,4.8964385986328125,23.925472259521484,26.834339141845703,26.46601295471192,23.520160675048828,27.76211738586425,27.71584892272949,10.374256134033203,10.4791841506958,218.3724822998047,225.1285858154297,1.1713463068008425,2.5177488327026367,105.52578735351562,450.79595947265625,429.6829833984375,450.57464599609375,448.6935729980469,17.0,700.7899780273438,1.7021775245666504,1200.0,1.0,36.82808303833008,38.11102294921875,20.06475257873535,11.937515258789062,89.86893463134766,84.6894760131836,,,86.0,98.14891815185548,24.32128524780273,0.0,400.0,1259.343017578125,0.0,1462.4814453125,2.1629884243011475,1.6796951293945312,998.4035034179688,0.0223088879138231,198.8683013916016,1.368503451347351,1.3799999952316284,0.0,,61.264404296875,16.665037155151367,5.333080291748047,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.86962890625,4.85367488861084,73.36161041259766,91.94583892822266,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,7.830216407775879,38.73592758178711,18.59975242614746,17.211471557617188,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042373403906822,1.970276951789856,0.4222961962223053,0.0943225920200347,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.7188185453414917,0.3608308732509613,1.2047821283340454,1.6332342624664309,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7221007943153381,0.2941087782382965,-9999.0,0.5667175650596619,0.7008373737335205,0.8771045207977295,8.49777603149414,65.73210906982422,12.899999618530272,5.300000190734863,6.515639781951904,86.30000305175781,790.82958984375,907.0885620117188,,,1.5791287422180176,1.572784185409546,0.0,600.0,950.763671875,949.9783325195312,0.0,10.146312713623049,0.0,11.57530403137207,4.340694427490234,0.0748387947678566,71.91187286376953,100.0,64.71,0.0,4.533299922943115,5.70755672454834,8.692553520202637,4.993200302124023,4.861800193786621,6.372900485992432,51.02507781982422,67.99890899658203,3.52 +2024-12-05 02:34:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2978.102783203125,8.848578453063965,8.862582206726074,2573.7099609375,234.7885284423828,289.6545104980469,2.2108702659606934,2.714353322982788,3147.3876953125,705.0564575195312,0.0,288.6408386230469,0.0,3398.574462890625,86.58733367919922,89.01890563964844,12.617484092712402,13.247730255126951,,,1347.5814208984375,1234.0552978515625,3.514620065689087,1.387859582901001,1.4020501375198364,1821.7401123046875,2152.696533203125,1866.8282470703125,1715.38427734375,2248.69091796875,2274.983642578125,84.39221954345703,88.17390441894531,90.19397735595705,34.61734771728516,34.1109504699707,32.32330322265625,23.76468849182129,43.90478515625,44.16008377075195,1927.233642578125,1939.23876953125,903.4033203125,31.51068115234375,378.1936645507813,0.0,648.6610107421875,766.79345703125,755.9276123046875,794.9697265625,744.4769287109375,656.0175170898438,699.5923461914062,772.5515747070312,569.7051391601562,589.693603515625,635.4265747070312,594.8566284179688,633.0,0.7698317170143127,27.0,1193.8900146484375,997.2662963867188,103.9173355102539,98.82736206054688,29.941625595092773,22.332080841064453,32.80282974243164,12.757905960083008,37.37025833129883,21.68112564086914,15.733065605163574,47.392391204833984,57.10361099243164,59.91703796386719,48.43916702270508,40.41853332519531,83.20599365234375,66.5,0.0,0.0,1749.7413330078125,1801.275634765625,85.9491195678711,97.38201904296876,0.1004853919148445,4.895888805389404,23.921628952026367,26.72721099853516,26.51607131958008,23.67252349853516,27.53003692626953,27.67759895324707,10.37065887451172,10.479698181152344,220.0579071044922,214.2723999023437,1.172289490699768,2.055137157440185,106.04012298583984,450.7769470214844,429.7731628417969,450.5724487304688,448.6881713867188,17.0,700.7642822265625,0.0,1200.0,1.0,37.02928161621094,38.0235595703125,19.88223648071289,12.143880844116213,91.3222427368164,85.17166900634766,,,86.0,98.23806762695312,23.680456161499023,0.0,400.0,1300.7486572265625,0.0,1472.4622802734375,2.1629347801208496,1.679681658744812,1000.1616821289062,0.0220953896641731,198.59475708007807,1.3683171272277832,1.3799999952316284,0.0,,61.264404296875,,5.333080291748047,,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,,12.386075973510742,,,,,,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.547714233,,,,,,,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,,,,,,,,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,,,,,,,42.07585144042969,,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,,,,,,,,,,,12.899999618530272,5.300000190734863,6.536253452301025,85.9000015258789,806.0762329101562,868.0042114257812,,,1.578299880027771,1.5728015899658203,0.0,600.0,963.1144409179688,951.73193359375,0.0,10.191020965576172,0.0,11.601633071899414,4.3397626876831055,0.0748387202620506,71.49508666992188,100.0,,,,,,,,,51.03459548950195,68.00411987304688, diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index c0785fd..d0d47de 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -1,8 +1,11 @@ -import json from temporalio import activity, workflow with workflow.unsafe.imports_passed_through(): + from datetime import datetime + import json + from pandas import Timestamp, to_datetime + from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ from sientia_do.temporal.activities.base import BaseActivity from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel @@ -60,6 +63,44 @@ class MLFlow(BaseActivity): f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger.base_logger ) + def detect_and_parse_datetime_index(self, data: DataFrame, metadata: dict) -> DataFrame: + """ + Detect and parse datetime index from data. index must be a timestamp like column. + This function must detect the timestamp type (pandas Timestamp or datetime) and convert it to DATETIME_FORMAT_WITH_TZ. + If the index is a string, must be in format DATETIME_FORMAT_WITH_TZ. + If another type or format, must raise an error. + """ + index = data.index + + # Get type of first element of index + index_type = type(index[0]) + + self.info(f"Index type: {index_type}", metadata) + + message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}" + + # Check if all in index are of the same type + if not all(isinstance(i, index_type) for i in index): + raise ValueError( + f"{message}") + + # Check type and converts to DATETIME_FORMAT_WITH_TZ + if index_type == str: + # Validate format of string and return error if not valid + try: + to_datetime(data.index) + except ValueError: + raise ValueError( + f"{message}") + + elif index_type == datetime or index_type == Timestamp: + data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) + else: + raise ValueError( + f"{message}") + + return data + @activity.defn(name="request_transform") async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]: """ @@ -113,13 +154,43 @@ class MLFlow(BaseActivity): data.columns.name = None self.debug("Processed input data:", metadata) - self.debug(data, metadata) + data.to_csv('data.csv') + self.debug(data.to_string(), metadata) # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( model_name, data, model_config ) + self.debug("Raw response data:", metadata) + self.debug(response_data, metadata) + + if response_data['success']: + response_dataframe = DataFrame(response_data['content']) + try: + response_dataframe = self.detect_and_parse_datetime_index( + response_dataframe, metadata) + response_dataframe['timestamp'] = to_datetime( + response_dataframe.index, format=DATETIME_FORMAT_WITH_TZ) + response_dataframe['timestamp'] = response_dataframe['timestamp'].dt.strftime( + DATETIME_FORMAT) + except ValueError as e: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id='TRANSFORM_DATA_INDEX_ERROR', + message=f'Error parsing trasnformed data index: {e}', + block='transform', + level=NotificationLevel.ERROR, + attachment_content=trace + ) + self.error(trace, metadata=metadata) + raise e + + response_dataframe.to_csv('response_data.csv') + + response_data['content'] = response_dataframe.to_dict() + self.debug("Transform response data:", metadata) self.debug(response_data, metadata) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index be1efa0..77da23b 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -90,6 +90,7 @@ class MLFlowRepository(): end_time = datetime.now() data = pd.DataFrame(data, columns=['prediction']) + self.logger.info(f"Data: {data.to_string()}") data.index = input_index data['response_time'] = (end_time - start_time).total_seconds() diff --git a/laborious/workflows/predictions_batch.py b/laborious/workflows/predictions_batch.py index cc10def..ecce063 100644 --- a/laborious/workflows/predictions_batch.py +++ b/laborious/workflows/predictions_batch.py @@ -87,7 +87,7 @@ class PredictionsBatch(): 'datetime_columns': input_data.get('datetime_columns', []) }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=300) ) # Prepare input for prediction_process workflow diff --git a/laborious/workflows/sub_workflows/prediction_process.py b/laborious/workflows/sub_workflows/prediction_process.py index d78018e..c958500 100644 --- a/laborious/workflows/sub_workflows/prediction_process.py +++ b/laborious/workflows/sub_workflows/prediction_process.py @@ -123,7 +123,7 @@ class PredictionProcess(): 'model_config': model_config }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(minutes=1), + start_to_close_timeout=timedelta(minutes=5), ) # Validate MLFlow transform response @@ -175,7 +175,7 @@ class PredictionProcess(): 'model_config': model_config }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(minutes=1), + start_to_close_timeout=timedelta(minutes=5), ) # Validate MLFlow prediction response diff --git a/response_data.csv b/response_data.csv new file mode 100644 index 0000000..cd1d691 --- /dev/null +++ b/response_data.csv @@ -0,0 +1,2 @@ +,303-WIT-230_median,303-WIT-230_std,303-WIT-230_min,303-WIT-230_max,305-WIT-135_median,305-WIT-135_std,305-WIT-135_min,305-WIT-135_max,305-WIT-160_median,305-WIT-160_std,305-WIT-160_min,305-WIT-160_max,305-PIT-170_median,305-PIT-170_std,305-PIT-170_min,305-PIT-170_max,305-PIT-175_median,305-PIT-175_std,305-PIT-175_min,305-PIT-175_max,305-FIT-002_median,305-FIT-002_std,305-FIT-002_min,305-FIT-002_max,305-FIT-013_median,305-FIT-013_std,305-FIT-013_min,305-FIT-013_max,306-PIT-101_median,306-PIT-101_std,306-PIT-101_min,306-PIT-101_max,306-FIT-051_median,306-FIT-051_std,306-FIT-051_min,306-FIT-051_max,306-DIT-001_median,306-DIT-001_std,306-DIT-001_min,306-DIT-001_max,306-PIT-105_median,306-PIT-105_std,306-PIT-105_min,306-PIT-105_max,306-FIT-052_median,306-FIT-052_std,306-FIT-052_min,306-FIT-052_max,306-DIT-002_median,306-DIT-002_std,306-DIT-002_min,306-DIT-002_max,306-PIT-115_median,306-PIT-115_std,306-PIT-115_min,306-PIT-115_max,306-FIT-004_median,306-FIT-004_std,306-FIT-004_min,306-FIT-004_max,306-PIT-110_median,306-PIT-110_std,306-PIT-110_min,306-PIT-110_max,306-FIT-003_median,306-FIT-003_std,306-FIT-003_min,306-FIT-003_max,306-PIT-125_median,306-PIT-125_std,306-PIT-125_min,306-PIT-125_max,306-FIT-005_median,306-FIT-005_std,306-FIT-005_min,306-FIT-005_max,306-PIT-130_median,306-PIT-130_std,306-PIT-130_min,306-PIT-130_max,306-FIT-006_median,306-FIT-006_std,306-FIT-006_min,306-FIT-006_max,307-FIT-005_median,307-FIT-005_std,307-FIT-005_min,307-FIT-005_max,307-FIT-003_median,307-FIT-003_std,307-FIT-003_min,307-FIT-003_max,307-FIC-022_median,307-FIC-022_std,307-FIC-022_min,307-FIC-022_max,310-FIT-005_median,310-FIT-005_std,310-FIT-005_min,310-FIT-005_max,307-FIT-008_median,307-FIT-008_std,307-FIT-008_min,307-FIT-008_max,307-FIT-009_median,307-FIT-009_std,307-FIT-009_min,307-FIT-009_max,309-PIT-101_median,309-PIT-101_std,309-PIT-101_min,309-PIT-101_max,309-PIT-105_median,309-PIT-105_std,309-PIT-105_min,309-PIT-105_max,309-PIT-110_median,309-PIT-110_std,309-PIT-110_min,309-PIT-110_max,309-PIT-185_median,309-PIT-185_std,309-PIT-185_min,309-PIT-185_max,309-PIT-190_median,309-PIT-190_std,309-PIT-190_min,309-PIT-190_max,309-PIT-195_median,309-PIT-195_std,309-PIT-195_min,309-PIT-195_max,309-FIT-051_median,309-FIT-051_std,309-FIT-051_min,309-FIT-051_max,309-FIT-052_median,309-FIT-052_std,309-FIT-052_min,309-FIT-052_max,309-PIT-001_median,309-PIT-001_std,309-PIT-001_min,309-PIT-001_max,309-PIT-002_median,309-PIT-002_std,309-PIT-002_min,309-PIT-002_max,317AIT003.3_median,317AIT003.3_std,317AIT003.3_min,317AIT003.3_max,317AIT003.5_median,317AIT003.5_std,317AIT003.5_min,317AIT003.5_max,317AIT003.1_median,317AIT003.1_std,317AIT003.1_min,317AIT003.1_max,317AIT003.2_median,317AIT003.2_std,317AIT003.2_min,317AIT003.2_max,317AIT002.37_median,317AIT002.37_std,317AIT002.37_min,317AIT002.37_max,317AIT002.49_median,317AIT002.49_std,317AIT002.49_min,317AIT002.49_max,317AIT002.61_median,317AIT002.61_std,317AIT002.61_min,317AIT002.61_max,317AIT002.38_median,317AIT002.38_std,317AIT002.38_min,317AIT002.38_max,317AIT002.50_median,317AIT002.50_std,317AIT002.50_min,317AIT002.50_max,317AIT002.62_median,317AIT002.62_std,317AIT002.62_min,317AIT002.62_max,317AIT002.39_median,317AIT002.39_std,317AIT002.39_min,317AIT002.39_max,317AIT002.51_median,317AIT002.51_std,317AIT002.51_min,317AIT002.51_max,317AIT002.63_median,317AIT002.63_std,317AIT002.63_min,317AIT002.63_max,317AIT002.40_median,317AIT002.40_std,317AIT002.40_min,317AIT002.40_max,317AIT002.52_median,317AIT002.52_std,317AIT002.52_min,317AIT002.52_max,317AIT002.64_median,317AIT002.64_std,317AIT002.64_min,317AIT002.64_max,317AIT002.41_median,317AIT002.41_std,317AIT002.41_min,317AIT002.41_max,317AIT002.53_median,317AIT002.53_std,317AIT002.53_min,317AIT002.53_max,317AIT002.65_median,317AIT002.65_std,317AIT002.65_min,317AIT002.65_max,317AIT002.42_median,317AIT002.42_std,317AIT002.42_min,317AIT002.42_max,317AIT002.54_median,317AIT002.54_std,317AIT002.54_min,317AIT002.54_max,317AIT002.66_median,317AIT002.66_std,317AIT002.66_min,317AIT002.66_max,317AIT002.43_median,317AIT002.43_std,317AIT002.43_min,317AIT002.43_max,317AIT002.55_median,317AIT002.55_std,317AIT002.55_min,317AIT002.55_max,317AIT002.67_median,317AIT002.67_std,317AIT002.67_min,317AIT002.67_max,317AIT002.44_median,317AIT002.44_std,317AIT002.44_min,317AIT002.44_max,317AIT002.56_median,317AIT002.56_std,317AIT002.56_min,317AIT002.56_max,317AIT002.68_median,317AIT002.68_std,317AIT002.68_min,317AIT002.68_max,317AIT002.45_median,317AIT002.45_std,317AIT002.45_min,317AIT002.45_max,317AIT002.57_median,317AIT002.57_std,317AIT002.57_min,317AIT002.57_max,317AIT002.69_median,317AIT002.69_std,317AIT002.69_min,317AIT002.69_max,317AIT002.46_median,317AIT002.46_std,317AIT002.46_min,317AIT002.46_max,317AIT002.58_median,317AIT002.58_std,317AIT002.58_min,317AIT002.58_max,317AIT002.70_median,317AIT002.70_std,317AIT002.70_min,317AIT002.70_max,317AIT002.47_median,317AIT002.47_std,317AIT002.47_min,317AIT002.47_max,317AIT002.59_median,317AIT002.59_std,317AIT002.59_min,317AIT002.59_max,317AIT002.71_median,317AIT002.71_std,317AIT002.71_min,317AIT002.71_max,317AIT002.48_median,317AIT002.48_std,317AIT002.48_min,317AIT002.48_max,317AIT002.60_median,317AIT002.60_std,317AIT002.60_min,317AIT002.60_max,317AIT002.72_median,317AIT002.72_std,317AIT002.72_min,317AIT002.72_max,SiO2_conc,timestamp +2024-12-05 04:00:00+0000,2344.354736328125,347.11413476082527,2063.14990234375,3052.78662109375,1323.8238525390625,199.25517177489738,801.2099609375,1477.9354248046875,1239.42919921875,188.20232896348458,1067.496337890625,1611.220947265625,12.741495132446287,0.2925100433215222,12.14862060546875,13.035848617553713,13.396173477172852,0.2659367570480025,12.94904613494873,13.875475883483888,3156.99365234375,17.90496592640638,3128.458251953125,3188.196533203125,3328.472412109375,18.341293358560456,3293.60546875,3352.32080078125,34.44520568847656,0.10727337707066699,34.300880432128906,34.595943450927734,2251.281005859375,6.914512409267933,2241.09228515625,2259.91357421875,1.3857378959655762,0.001162128441763176,1.3840404748916626,1.3874353170394895,33.95878982543945,0.06905080560339776,33.905391693115234,34.107933044433594,2275.4189453125,8.067717393234235,2263.143798828125,2288.202880859375,1.397327542304993,0.0025866991350633178,1.3935494422912598,1.4011056423187256,31.01426696777344,2.797667558372099,24.58193588256836,34.75410461425781,2157.358642578125,30.47011040139319,2091.429931640625,2189.05615234375,32.01703643798828,0.027139770722953593,31.97739601135254,32.05667495727539,2059.924560546875,88.54574101606774,1862.8626708984373,2170.908447265625,43.65913391113281,0.09432217658823967,43.593544006347656,43.89741134643555,1866.9232177734373,0.05200646609213417,1866.84716796875,1866.9991455078125,44.08242416381836,0.04344513322059127,44.02417755126953,44.14066696166992,1714.9619140625,0.1883131118977155,1714.6802978515625,1715.2435302734375,27.0,0.0,27.0,27.0,0.7632204294204712,0.28421935480519783,0.6459924578666687,1.425487995147705,649.1517333984375,2.0424248177883793,647.1126708984375,653.2977294921875,700.892822265625,0.07040149596071192,700.7899780273438,700.9956665039062,1194.4361572265625,0.29912346849236254,1193.999267578125,1194.873046875,997.7520751953124,0.26607758363281586,997.3634643554688,998.1407470703124,23.87283706665039,0.040718881893752515,23.81103706359864,23.925472259521484,26.559722900390625,0.2379241176870002,26.20622062683105,26.91034507751465,26.35142517089844,0.203982336891631,26.06732177734375,26.710390090942383,23.415620803833008,0.11592992262134358,23.256576538085938,23.56723022460937,27.697071075439453,0.15510993222349723,27.35708808898925,27.844594955444336,27.72958755493164,0.19091973703305398,27.292091369628903,27.89141845703125,1748.264892578125,7.86294792652368,1742.1806640625,1768.509765625,1821.6829833984373,15.953009331205266,1809.867919921875,1854.5777587890625,0.1001370549201965,0.00019079441009214591,0.0998583808541297,0.1004157289862632,4.8986358642578125,0.0015045608215440351,4.8964385986328125,4.900833606719971,6.507819890975952,0.12657004614335984,6.405970096588135,6.699999809265137,86.30000305175781,0.3732080423947149,85.80000305175781,86.5999984741211,8.5,1.6421334224036257,7.89943265914917,12.899999618530272,5.400000095367432,1.0432607765452175,3.299999952316284,5.800000190734863,1.2756186723709106,0.0,1.2756186723709106,1.2756186723709106,0.7121588587760925,0.0,0.7121588587760925,0.7121588587760925,0.4104396104812622,0.0,0.4104396104812622,0.4104396104812622,0.2267737984657287,0.0,0.2267737984657287,0.2267737984657287,1.4764769077301023,0.0,1.4764769077301023,1.4764769077301023,0.6497865319252014,0.0,0.6497865319252014,0.6497865319252014,0.4203702211380005,0.0,0.4203702211380005,0.4203702211380005,1.526507019996643,0.0,1.526507019996643,1.526507019996643,0.6540470123291016,0.0,0.6540470123291016,0.6540470123291016,1.7459523677825928,0.002774316303229609,1.7381054162979126,1.7459523677825928,0.4817045927047729,0.0002177622295436636,0.4817045927047729,0.4823205173015594,0.3364990949630737,0.00012384851434926538,0.3361487984657287,0.3364990949630737,1.1931402683258057,0.033290363097006954,1.1206940412521362,1.1931402683258057,0.7161301374435425,0.025235254887373854,0.7161301374435425,0.7710468769073486,0.4127309918403625,0.007279004051801342,0.4127309918403625,0.428571492433548,1.707435429096222,0.0008729009039302258,1.7066189050674438,1.708251953125,0.3564415574073791,0.002030279951199197,0.3545424044132232,0.358340710401535,0.3023426830768585,0.000350906290820105,0.3020144402980804,0.3026709258556366,-9999.0,0.0,-9999.0,-9999.0,-9999.0,0.0,-9999.0,-9999.0,-9999.0,0.0,-9999.0,-9999.0,0.1042194217443466,8.294721469109035e-06,0.1042194217443466,0.1042373403906822,1.715789794921875,0.001402039007837862,1.715789794921875,1.7188185453414917,0.7188712954521179,0.0014949674798809447,0.7188712954521179,0.7221007943153381,1.970276951789856,0.018573843840262887,1.934388875961304,1.970276951789856,0.3608308732509613,0.004311563730231936,0.3608308732509613,0.3691616058349609,0.2941087782382965,0.0009687919419025886,0.2941087782382965,0.2959806621074676,0.4393351674079895,0.006024186034919744,0.4222961962223053,0.4393351674079895,1.1838626861572266,0.007396139710934239,1.1838626861572266,1.2047821283340454,0.5613290071487427,0.0019051429198136875,0.5613290071487427,0.5667175650596619,0.0910589918494224,0.0011538569058607675,0.0910589918494224,0.0943225920200347,1.6499500274658203,0.0059099153918944995,1.6332342624664309,1.6499500274658203,0.7046993374824524,0.0013654103777831785,0.7008373737335205,0.7046993374824524,0.0354578979313373,0.0,0.0354578979313373,0.0354578979313373,2.292947769165039,0.0,2.292947769165039,2.292947769165039,0.8771045207977295,0.0,0.8771045207977295,0.8771045207977295,3.52,2024-12-05 04:00:00 diff --git a/tests.ipynb b/tests.ipynb index 8ab64e2..6ec555e 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -404,6 +404,60 @@ "print(len(b))\n", "print(b.size)\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3374174", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "All elements in index are of the same type\n" + ] + } + ], + "source": [ + "from pandas import DataFrame\n", + "\n", + "data = DataFrame({\n", + " \"a\": {\"2025-01-01\": 1, \"2025-01-02\": 2, \"2025-01-03\": 3},\n", + " \"b\": {\"2025-01-01\": 4, \"2025-01-02\": 5, \"2025-01-03\": 6},\n", + "})\n", + "\n", + "index = data.index\n", + "\n", + "# Get type of first element of index\n", + "index_type = type(index[0])\n", + "\n", + "print(index_type)\n", + "\n", + "# Check if all in index are of the same type\n", + "if all(isinstance(i, index_type) for i in index):\n", + " print(\"All elements in index are of the same type\")\n", + "else:\n", + " print(\"Elements in index are of different types\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40e72c60", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1fbb3788", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl b/tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..02d4f4a33b863e4e6a36c404e28917079c0a8fad GIT binary patch literal 29391 zcmeHQS(6-BRlnF)O&Hi-feBBSHJz&e@y@WBsnyQdRy_J){3^spZVQUa(K{a$5d4(YR75vI2HSY zsJ)#gN2In9t~HiJ=yo>JW$7I?sX!hyd8C-Ezb1eORet4TZ^KM+$K*+i(0F3vp=a#(&T7u*x7Du zyw!?pH_3*SPj!av?YD(9?O|)sh&5WDKj;lN2Bf%;71QJZoM)Af()q{U zOJ{f8On;QFrswB(%^z5Nao784{maR|&B5E-FMWq*14SPw8DPeWk2wS&VnCZ!+-9hXAx$laWK}0P zSrt{f%*~)mJopQuBPdOg^2~trd}o z*ARs+kt#6}^?Qq`iiyMOxna%ix`-7?s>&o4FS4{`<+QBX~(HAKt7ohgxN$|e#-B_a?lZHYzLO04L#zM!*5Vwl!{$zm+(o*_Ak!s(+_ zrx;e%+Vb^v>4j)Fon2`Yie?q1VnO#l1?c`Mfnq^*M+NvAp|4TO0KQ4t*C_iM6RRaTd=mPLH?9f@m4qX7g zi5)s?*r79Ehb{o0Lxzq(3$o{7AefSv@mPj_D5Ef9(M=&%{hTJmqr~O(Aq%OfUIm>6 zo<+*sDJ)J5cMKJBhq}HHGhk0)tS_f9sm}rGL^D6yM(0%lGGb|4W;ICwwRf0!)P#K`4^;1W{543Jb-FRckP+9&@}fb!nwg z3@_ZcvVirNJDJ2vIat}otceyAy;Tys0TCxgJw<}B|?V(69%?6f+Smuui4Uur_hp$lQ*1JGVKj^8U?7F%yV@GP$ za(HcdL3LUf*!D=LWiP8FLVL8>_Aq$<-#nzZXF zP1^O8XsOd46-joGr{f!##6bG*&N>KIYgwfImQ^gdE|oESENQiv*8jr>ib$Fz4@JwW ztd+!`OBE{^MV4|Ep8c~8&>qr~hpsCn*A?2>TFO<-)&FDzMc0+kb*1dOQgPd;E0D75O2tiZwVKxd%T_PCu9RF?%C0LFH^J4Z zwEizPS?Icgu47Bft}7Kc!BuyNxNe26TUb~fV%M#To9yazTK`X5o%XO+!b+}NW!J5W zn{2%AzzDRgL)R60HnyZ?*OiJJ)f7gq^OWrR=&Q9=!ETF@>m3(L-qwvoUQ? z7mI=ThuM)IRJUP*Hl3#3uD0It2tz!Qb3Ioe3qW|voM zt9aP5%U8YfB_vMC*RQ;>uEkTHxhNZRQ8wmNHQT%xy5_R1&1G4eqb$t#3@+5+!)m-S z+>U$E-Jjh3^KP2#s-?|e-u2jHBQMFGm9=YWa~3+uzV^@lib!^yOPiO;=n7r>FG_g=zDeHh7KhxKh3tfw>zg zFj2A!7ugjiW&i0e*|T+FAf05fan~>yfpNy9;WOMNd(O?pX>-Ai(^Rsd)M%b1z2E+;JUzq2A10+Ukm4MW$$C zC#qyWhk;Jb8Z6{0R*BPKM!;R+(_NB7tMQ<**$u)%;YAGnm8jp3@G>|kg%y;Q6m#f{ zY4bXJcP^lp(89Z9H@af7mx1K*b3wn`0$CsLXO}SJ?p{Kx>|QFR&DY8GH|X*vU4Dfw zze<;1qswQICcEPxeJISI#jA&Bo9voNo4-y;8+*C>TmK_IN%Ph{m1 zl)hOkp{1``!kw_dm!m7)!QH>;Y&3L}cnEY3u5~xH*c2c+LQZq$WqY~+G!AXQ8Z6JL z{97gBWy|aRMkh*>XSY>@^`icEwAt+ic$!y+XKsro%9EYlj640tX0#axdE#8+@)%+v zgMazNT6{{j&Fmd&097>_M&cWH0(55%B~4E1@Dvu%JH@ADTu7o;W z>fHSaTK5Sl2zVglGtSz%TCe~Yspmt6fs5hFtu9g3;%L=d_(*Y*R00YL7H2N&7Qa3h zu!OAwG}{K*3}-Uio?HhtZmJ*^bVv2EFcZ zs~-1)x%B`k^ANO!hm>}gzVcOZbm{%w}nf+jP*rTsBq{#~q z%SCO3#)%HEMsDh}AUw)~Xn^CwT&0)WKq@LsG{Z?*kJl1;FdE~m>y=5NXpeDXG|6*g ztf^I`+^~8LGggjFJY$br9uqVT(ZGc8X>0gn#EG4T5Ggy%L6`?d2ze1(>fVk-gONQv zHnUBx{2BC?AfWdMg%EEn?AejrHNuiFJS&Ggj5qESE@s45G?>i%V4u~~3r%&t^e7f< zM6O{>M?h$e7vG1mC1<1`t7;D5jLz{qsbspDlnm*9R@#}y6m0=ZbWdfOVZ~H&3S%>a zV^|WNSJP={EZeo@GNA@5JNDemeF9RMTG16;|7nU6OCKu-)coX`jmgyzz2g)$Fuz2a z7JD~2m1{gKIQW-j&WeTEHwuNRV8W8B6;=7J7Z_UoFqP0EjPWVkJtGy4T8m>q3+LKO zuzKY+8XeYBYfBcnu_URoKxM`0^+o<%?76HAuj8Z~sXm4;&QKIom0IzoNt+)KdSJui z7X|$x{8u9}rl8=N;X0SM*ki`g}jeceJoApx1we zO4SNKP0qRB#URLf)EF1y_N zJne>FlBL2Fl~wf)&FT)W0Nvr~6}1yaJBx=oehh$(5%CkCfP*z!d`5|o79(;A zow^KR8H6M1A`z_k$?$UA-{6fLJyStH3qtcn*#YsG9kqUA7^8oLr2qkPTq>+QtuiKa zsnzZT^qr$~FcZ(p;A+DnOkRHAOsNFRjGAjkq%i*61BvpC^MH zwb5SCXDeg8An|xFNMi?HklL3-E;vd&cE!E!=5WxpeNLOlcH@cZE3%_Ucy2~;^2t&s zP-g{G7Fv*|Sdr|*vLqLmBeEzr*Jf>&gz}}6cp9Y*ki>aANXr$PLp}jX=!n{<&*D46 z^ag_fvqS2`NCTSvAl`*6G&=Q&9+*r4YtM12L6~dplh&(Q-Dj{V)9+~cC9NfSN*d7H zT6*oGtI8M9`i=*2dU>0DC`pszjk_PWZsSohxb${FbxI}d=K8^Q)Wf6Z9rWM1c%vTS z1<}g^+V%O_`26GF`>%rW$rR6ms^!^Xhjl|Xv3L+9K*sKbTQyR=SJIeAY=F`h;d>BFis5TpS3F&qa3|! zQu^+==17I>4!J9*CKydlj*pST?=e!i?Ie}^5atj)81U}N>SdUqsP&D4@O>Nw=iLq7 zr6B*u$`Qd>d2S5B)z#&-wY(86W3UuN5ZN$o6>mgmNhpkL9xMv_m}$I%8wjRG-c2rC zTY7aSpbf1zptA~@Kyp;PK(ZQMAUO(dAUO(dAYxUUq`BkDwe{s}AX$Y>AUP^tAXyDB zkQ@a!kQ@aBqGm0)Z5E6C&XuDvAwGt@2a7Ywj;M&DxrZ;*!|tWxHVv8tV|mQ-msQ9# zO^%A!G+7O=X>t_YrpZwl6$mwrnscpW7zU*DoO<68MdQf;zo?$IJWBD2|WDrcgWJDv_m_P-mg_K^UknkTb+06 z8pL-7L+tS_|BPFoDja-H@F~LCsg)ZJtacg>R6k8PMGV;c4`;_#PB>(>!*EFT5Pzl~ zT*XY$N}E5xWAqL#$uW9WhlypS*J$7UxJO^K`XQMCB-l5B{^n=B;I2*-=lzjo|ojL zXqDg+Ry6hLXFqMePmbkW;`4dvXTg%=vg@zmd$05>Wa|4rqUwK6m*kA>sPr`$Ozre; zNUri}^8<41zAyNI3P0~|i0@b|ckop$e1-HwqIz(b%IUWMba2g{pf(};0HL;ufi zOoZkx8={e!yR4Q*cBG`W^^GyRuRCq8Z^oNzwFh-$s>?EtlV!C(PsDg;?(VC8W5#!5 zZ0|$UOfGacj&tXlG));79t`K_SbKav>#_;wp1I$-r>>@%#vk5YjPrJrU${AJ;Gkdn z>C?V`tFb``_0B&we_(zWozpwNo38KiAFAd*^_su^e zch2N_k7KB}$o*CN-}1NlQ6qvgCmPdTyq(VPV(OUqlf5+@aE#wa zQ;g~e=Nf(+j4xOt=Ad1RDTq>MUX?Z2p0D4rUMhAVv zE2Be)ehaVopj@17=};-qP$|SvX~d)Q0u*W$%PWQWR4~+3;#kPg^-6)pD}@*;jkr`e z4A%*e)c|@_q^5JFbd*>6ykwiofQrf>ipoNySLz7mqD#d!R*KpyOHF!ZA(~2_)a+6z z(0HW~eGxibc3l5zH$q0(xeS1{C6qB^Iy z%d{6vZ#RhV6$~{MR~P2x3ZUl|skvSeqN#*e9+d(Ol|l@aMl=zB&D}(p}!qB6d&ZC;nP(jTKkir2X zsniL-&V+10l`DfNDhu(boX()s%)l{-Ph|#<)WIZR^>zb#UXhv%(CBypUd;0fphrb& zE)^kefYW(i0raRy&7~s5qssGYI^z`(pUU=%#~@ZuCsztIUMa*-X+(P^5NF`%GnuUb zwVDo4Bbth)9Xb=!@k)WlD}@-ZG~!WtUI{g`q>84eKS>)5%nK;J8qaw3^P9h4@r5S5KPLo=a0YS;rid z{c<5@^B~HXMOQYHmZb>Prz1s*)79cYExM{nnxzQTrz1s=jv&vwl9`7~bOd>HrQD_f>eG>;M@NuHXU#yCcWPJf zLOvbYdURDYNG)5SzIUYPc}I{(hd(Hh(-c5`I#TrL2=eGExlIAorz1s=jv$Y&YPQpM zu;mOLDSC7Sd35F6Q32HVjubsQf;_rPZc_mD=}6I|BgmtxnmN-BR!*8O`19VvQr1bK8-v(2_6t7hm((W4{Cqbuh&1yJ8RQuOEu^62pA zVsb_WP@j$zJvxFsx@zuP0o11>MURdkkFK2C6hM7CQuOEu^5`nLO##%WBSnvnAdjw^ zyH)`8=}6I|Bgmt}AGpbB3ZOn6DSC7Sd32TBrU2^Gk)lUOkVjX|T`Pe4bfoCf5#-U8 zbDILFPe+O#9YG#lCATSn`gEk|(Gld)S#JfcF$>hEBSnvnAdjw`+Y~^3I#TrL2=eGE zxlIAorz1s=jv$Y2DtD~_>eG>;M@NuHSI%t;pgtWbdUOPNbk@6Gt0~mWUKjG|$kwBq n%3Ujf`reVE=N&S;hzzhH3&B^}*E89w3 literal 0 HcmV?d00001 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py index 4afdfaf..e69de29 100644 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py @@ -1,824 +0,0 @@ -""" -Base model classes and interfaces. - -This module defines base classes with consistent interfaces for all models, -promoting modular model development. It includes essential functionality -for model fitting, prediction, evaluation, saving, and loading, while -abstracting common behaviors into base classes. - -While full compatibility with scikit-learn is not guaranteed, the base -classes provide a consistent interface for model fitting, prediction, -evaluation, saving, and loading, which should be sufficient for most -use cases. - -Key components: -- **Model**: Abstract base class for all models, providing core utilities and - interfaces. -- **TimeSeriesModel**: Abstract base class for time series models, adding - time-based functionality. -- **UnivariateTimeSeriesModel**: Base class for univariate time series models. -- **MultivariateTimeSeriesModel**: Base class for multivariate time series - models that use exogenous features. - -The module also includes utility functions such as `ensure_fitted`, which - ensures models are fitted before calling certain methods. - -Modules in this package should inherit from these base classes and implement - the required methods. - -TODO: - - Add methods to create lagged features for time series models. -""" - -import uuid -import joblib -from abc import ABC, abstractmethod -from typing import Optional, List, Sequence, Tuple, cast, Any, Protocol -from contextlib import contextmanager - -import pandas as pd -import numpy as np -import shap -import matplotlib.pyplot as plt -from rich.console import Console - -from sklearn.base import BaseEstimator, RegressorMixin -from sklearn.utils.validation import check_array, check_X_y -from sklearn.exceptions import NotFittedError - -console = Console() - - -class PredictorProtocol(Protocol): - """Protocol for models with predict method and optional imputation.""" - - def predict(self, X: Any) -> Any: ... - def _impute_missing_values(self, X: Any) -> Any: ... - - -def ensure_fitted(method): - """ - Decorator to ensure the model is fitted before calling the method. - - Raises: - sklearn.exceptions.NotFittedError: If the model is not fitted. - Usage: - @ensure_fitted - def predict(self, X): # Or other methods requiring fit - pass - """ - - def wrapper(self, *args, **kwargs): - is_fitted = self.__sklearn_is_fitted__() - if not is_fitted: - raise NotFittedError( - f"This {self.__class__.__name__} instance is not fitted yet. " - "Call 'fit' with appropriate arguments before using this " - "method." - ) - return method(self, *args, **kwargs) - - return wrapper - - -class Model(BaseEstimator, ABC): - """ - Abstract base class for all models. - - Provides core utilities, input validation, and interface consistency - for time series models. Compatible with scikit-learn workflows. - """ - - def __init__(self, name: Optional[str] = None, random_seed: int = 42): - """ - Initialize the model with a unique name and random seed. - - Args: - name: Optional identifier; auto-generated if None. - random_seed: Seed for reproducibility. - """ - self.name = name or f"{self.__class__.__name__}_{uuid.uuid4().hex}" - self.random_seed = random_seed - self.feature_names_in_: Optional[List[str]] = None - self.n_features_in_: Optional[int] = None - self._is_fitted = False - - def fit( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> "Model": - """ - Trains the model. - - Handles basic input validation for y and sets internal fitted - state after calling _fit_logic. - - 'X' is optional to account for univariate time series models. - - Args: - y: The target variable. - X: Optional exogenous variables. - X_val: Optional validation feature matrix. - y_val: Optional validation target series. - Raises: - TypeError: If y is not a pandas Series. - If X is provided, it must be a pandas DataFrame. - If X_val and y_val are provided, they must be pandas DataFrames - and Series respectively. - - Returns: - Self for chaining. - """ - if not isinstance(y, pd.Series): - raise TypeError("Input 'y' (target) must be a pandas Series.") - - self._fit_logic(y, X, X_val, y_val) - self._is_fitted = True - return self - - @abstractmethod - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """ - Core fitting logic to be implemented by subclasses with - optional validation data. - - Args: - y: The target variable. - X: Optional exogenous variables. - X_val: Validation feature matrix (optional). - y_val: Validation target series (optional). - """ - raise NotImplementedError("Subclasses must implement _fit_logic().") - - @ensure_fitted - @abstractmethod - def predict(self, X: Optional[pd.DataFrame] = None) -> Sequence: - """ - Predict values. - - Args: - X: Optional features for prediction. For univariate models - not using exogenous variables, this might be None or - contain future timestamps. Multivariate models will - require X. - - Returns: - NumPy array or similar sequence of predictions. - """ - raise NotImplementedError("Subclasses must implement predict().") - - def fit_predict( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> Sequence: - """ - Fits model and returns predictions on the same data. - - Args: - y: The target time series. - X: Optional exogenous variables. - - Returns: - Predictions for the input data. - """ - return self.fit(y, X, X_val, y_val).predict(X) - - def save(self, path: str) -> None: - """ - Saves model to disk using joblib. - """ - joblib.dump(self, path) - - @classmethod - def load(cls, path: str) -> "Model": - """ - Loads model from disk using joblib. - """ - return joblib.load(path) - - def __str__(self) -> str: - return f"{self.__class__.__name__}(name={self.name})" - - def __sklearn_is_fitted__(self) -> bool: - """ - Check fitted status and return a Boolean value. - """ - return hasattr(self, "_is_fitted") and self._is_fitted - - def __repr__(self) -> str: - return f"{self.__class__.__name__}(name={self.name})" - - @contextmanager - def model_state_preservation(self): - """Context manager to preserve model state during operations.""" - original_state = self._get_state_snapshot() - try: - yield - except Exception: - self._restore_state_snapshot(original_state) - raise - - def _get_state_snapshot(self) -> dict: - """Get snapshot of current model state.""" - return { - "name": self.name, - "is_fitted": getattr(self, "_is_fitted", False), - "feature_names": self.feature_names_in_, - "n_features": self.n_features_in_, - } - - def _restore_state_snapshot(self, snapshot: dict) -> None: - """Restore model state from snapshot.""" - self.name = snapshot["name"] - self._is_fitted = snapshot["is_fitted"] - self.feature_names_in_ = snapshot["feature_names"] - self.n_features_in_ = snapshot["n_features"] - - def get_params_dict(self) -> dict: - """Get model parameters as dictionary for logging/serialization.""" - return { - "name": self.name, - "random_seed": self.random_seed, - "n_features_in_": self.n_features_in_, - } - - def summary(self) -> str: - """Generate a summary string of the model.""" - params = self.get_params_dict() - fitted_status = ( - "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" - ) - - summary_lines = [ - f"Model: {self.__class__.__name__}", - f"Status: {fitted_status}", - f"Features: {params.get('n_features_in_', 'Unknown')}", - ] - - return "\n".join(summary_lines) - - -class TimeSeriesModel(Model): - """ - Abstract base class for time series forecasting models. - - Extends the base Model class with specific methods for time series - data handling and evaluation. - """ - - def __init__( - self, - name: Optional[str] = None, - time_col: str = "ds", - target_col: str = "y", - random_seed: int = 42, - n_lags: int = 0, - sampling_freq: Optional[str] = None, - ): - super().__init__(name=name, random_seed=random_seed) - self.time_col = time_col - self.target_col = target_col - self.n_lags = n_lags - self.sampling_freq = sampling_freq - - self.training_series_: Optional[pd.Series] = None - self.model_: Optional[BaseEstimator] = None - - # Validate configuration - self._validate_configuration() - - def _validate_configuration(self) -> None: - """Validate model configuration.""" - if self.n_lags < 0: - raise ValueError("n_lags must be non-negative") - - def _validate_y(self, y: pd.Series) -> np.ndarray: - """ - Validates the target variable (y) for the model. - - Ensures y is a pandas Series and checks its name against - the expected target column name. Converts y to a NumPy array. - The series name can be None, but if it is set, it should match - the expected target column name. - - Args: - y: The target variable as a pandas Series. - - Returns: - A NumPy array of the target variable. - - Raises: - TypeError: If y is not a pandas Series. - """ - # Check if y is a pandas Series - if not isinstance(y, pd.Series): - raise TypeError("Input 'y' (target) must be a pandas Series.") - if (y.name is not None) and (y.name != self.target_col): - raise ValueError( - f"Expected target column name '{self.target_col}', " - f"but got '{y.name}'." - ) - return check_array(y, ensure_2d=False) - - @ensure_fitted - @abstractmethod - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Performs backtesting on the time series data. - - Args: - y: The target time series data. - X: Optional exogenous features. - retrain_every: Number of steps after which to retrain the model. - reuse_previous_execution: Whether to reuse the previous execution - of a backtest. If True, any overlapping data between the - previous execution and the current execution will be used - without retraining the model. - Returns: - Series of predictions for each step in the time series. - """ - - raise NotImplementedError("Subclasses must implement backtest().") - - def get_params_dict(self) -> dict: - """Get model parameters as dictionary for logging/serialization.""" - base_params = super().get_params_dict() - ts_params = { - "time_col": self.time_col, - "target_col": self.target_col, - "n_lags": self.n_lags, - "sampling_freq": self.sampling_freq, - } - return {**base_params, **ts_params} - - -class UnivariateTimeSeriesModel(TimeSeriesModel, RegressorMixin): - """ - Base class for univariate time series models. - - Only supports regression settings. Concrete subclasses - must implement `_fit_logic` and `predict`. - """ - - @abstractmethod - @ensure_fitted - def forecast(self, forecast_horizon: int) -> Sequence: - """ - Forecast into the future for a given number of steps. - - Args: - forecast_horizon: Number of future time steps to forecast. - - Returns: - Sequence of forecasted values. - """ - raise NotImplementedError("Subclasses must implement forecast().") - - -class MultivariateTimeSeriesModel(TimeSeriesModel): - """ - Base class for multivariate time series models. - - This class provides a foundation for time series models that utilize - multiple exogenous features (X) to predict a target variable (y). - It supports both regression and classification tasks. - - Attributes: - selected_features_: List of feature names selected for the model. - learning_task: Type of learning task ('regression', 'binary', - 'multiclass'). - differentiate_target: Whether to apply differencing to make series - stationary. - bins: Bin edges for multiclass classification target - transformation. - - Example: - >>> class MyModel(MultivariateTimeSeriesModel): - ... def _fit_logic(self, y, X=None, **kwargs): - ... # Implementation here - ... pass - ... def predict(self, X=None): - ... # Implementation here - ... return predictions - """ - - def __init__( - self, - name: Optional[str] = None, - time_col: str = "ds", - target_col: str = "y", - random_seed: int = 42, - n_lags: int = 0, - sampling_freq: Optional[str] = None, - differentiate_target: bool = False, - bins: Optional[List[float]] = None, - learning_task: Optional[str] = None, - ): - # Set attributes before calling parent constructor - # This is needed because parent constructor calls - # _validate_configuration - self.selected_features_: Optional[List[str]] = None - self.learning_task: Optional[str] = learning_task - self.differentiate_target = differentiate_target - self.bins = bins - self.model_: Optional[PredictorProtocol] = None - - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - n_lags=n_lags, - sampling_freq=sampling_freq, - ) - - # Additional validation for multivariate models - self._validate_learning_task() - - def _validate_learning_task(self) -> None: - """Validate learning task configuration.""" - valid_tasks = {"regression", "binary", "multiclass", None} - if self.learning_task not in valid_tasks: - raise ValueError( - f"Invalid learning_task: {self.learning_task}. " - + f"Must be one of {valid_tasks}" - ) - - if self.learning_task == "multiclass" and not self.bins: - raise ValueError( - "bins must be provided for multiclass learning_task" - ) - - def _get_default_loss_function( - self, provided_loss: Optional[str] - ) -> str: - """ - Get default loss function based on learning task. - - Args: - provided_loss: User-provided loss function (takes precedence) - - Returns: - str: Appropriate loss function for the learning task - """ - if provided_loss is not None: - return provided_loss - - if self.learning_task == "regression": - return "RMSE" - elif self.learning_task == "binary": - return "Logloss" - elif self.learning_task == "multiclass": - return "MultiClass" - else: - return "RMSE" - - def _validate_configuration(self) -> None: - """Validate model configuration.""" - super()._validate_configuration() - - if self.differentiate_target and self.learning_task in [ - "binary", - "multiclass", - ]: - console.print( - "[yellow]Warning: Using differentiation with classification " - + "tasks may not be appropriate[/yellow]" - ) - - @ensure_fitted - def feature_importance(self) -> Optional[pd.DataFrame]: - """ - Returns feature importance if implemented by subclass. - - Returns: - A DataFrame with feature names and their importance scores, - or None if not applicable. - """ - return None - - def _validate_X_y( - self, X: pd.DataFrame, y: pd.Series, allow_nan: bool = True - ) -> Tuple[np.ndarray, np.ndarray]: - """ - Validates input features (X) and target (y). - - Infers and sets `feature_names_in_` and `n_features_in_`. - This method should be called within the `_fit_logic` of - concrete subclasses that use exogenous features. - - Args: - X: DataFrame of input features. - y: Series for the target variable. - allow_nan: If True, allows NaN values in X and y. - Raises: - TypeError: If X is not a DataFrame or y is not a Series. - ValueError: If the number of features in X does not match - the expected number of features. - - Returns: - Tuple of validated NumPy arrays (X_array, y_array). - """ - if allow_nan: - X_array, y_array = check_X_y(X, y, force_all_finite=False) - else: - X_array, y_array = check_X_y(X, y, force_all_finite=True) - - if hasattr(X, "columns"): - console.log( - f"Validating input features with columns: {X.columns.tolist()}" - ) - self.feature_names_in_ = list(X.columns) - else: - console.log( - "Input features do not have column names, using default names." - ) - self.feature_names_in_ = [ - f"feature_{i}" for i in range(X_array.shape[1]) - ] - - self.n_features_in_ = X_array.shape[1] - return X_array, y_array - - def _validate_X( - self, X: pd.DataFrame, allow_nan: bool = True - ) -> np.ndarray: - """ - Validates input features (X) before prediction or scoring. - - Ensures consistency with features seen during fit. This should - be called by concrete subclasses in `predict`, `score`, etc. - - Args: - X: DataFrame of input features. - allow_nan: If True, allows NaN values in X. - - Returns: - Validated NumPy array of X. - """ - if allow_nan: - X_array = check_array(X, force_all_finite=False) - else: - X_array = check_array(X, force_all_finite=True) - # If the model has been fitted, ensure the input features - # match the features seen during fit. - if self.feature_names_in_ is not None: - if not set(self.feature_names_in_).issubset(X.columns): - raise ValueError( - "Input features do not match the features seen during fit." - + f" Expected features: {self.feature_names_in_}, " - + f"but got: {list(X.columns)}." - ) - - return X_array - - def _transform_target_to_multiclass( - self, y: pd.Series, bins: Optional[List[float]] = None - ) -> pd.Series: - """ - Transforms the target variable into a multiclass classification - target. - - If bins are provided, uses pd.cut to categorize the target into - discrete classes. If not, binarize the target at zero (0). - - Args: - y: The target variable as a pandas Series. - bins: Optional list of bin edges for categorization. - - Returns: - A pandas Series with transformed classification targets. - """ - if bins is not None: - # pd.cut returns a Categorical, convert to Series with integer - # codes - categories = pd.cut(y, bins=bins, labels=False) - return pd.Series(categories, index=y.index) - - return (y > 0).astype(int) - - def _transform_target_to_binary( - self, y: pd.Series, threshold: float = 0.0 - ) -> pd.Series: - """ - Transforms the target variable into a binary classification target. - - Binarizes the target at the specified threshold (default is 0.0). - - Args: - y: The target variable as a pandas Series. - threshold: The threshold for binarization. - - Returns: - A pandas Series with binary classification targets. - """ - return (y > threshold).astype(int) - - def _preprocess_data( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> Tuple[ - pd.Series, - Optional[pd.DataFrame], - Optional[pd.Series], - Optional[pd.DataFrame], - ]: - """ - Internal method to handle common data preprocessing operations. - - Args: - y: The target time series data - X: The feature matrix (including exogenous features) - X_val: Validation feature matrix (optional) - y_val: Validation target series (optional) - - Returns: - A tuple containing: - - processed y series - - processed X dataframe (optional) - - processed y_val series (optional) - - processed X_val dataframe (optional) - """ - # Apply differentiation if enabled - if self.differentiate_target: - y = y.diff().dropna() - if X is not None: - X = X.loc[y.index] - - # Transform target for classification if needed - if self.learning_task == "binary": - y = self._transform_target_to_binary(y) - elif self.learning_task == "multiclass": - y = self._transform_target_to_multiclass(y, self.bins) - - # Process validation data if provided - if y_val is not None: - if X_val is None: - raise ValueError( - "Validation features (X_val) must be provided if " - + "validation target (y_val) is given." - ) - y_val = y_val.loc[X_val.index] - if self.differentiate_target: - y_val = y_val.diff().dropna() - X_val = X_val.loc[y_val.index] - if self.learning_task == "binary": - y_val = self._transform_target_to_binary(y_val) - elif self.learning_task == "multiclass": - y_val = self._transform_target_to_multiclass(y_val, self.bins) - - # Filter features if selected_features_ is set - if X is not None and self.selected_features_ is not None: - X = cast(pd.DataFrame, X[self.selected_features_].copy()) - if X_val is not None: - X_val = cast( - pd.DataFrame, X_val[self.selected_features_].copy() - ) - - return y, X, y_val, X_val - - def _prepare_shap_data(self, X: pd.DataFrame) -> pd.DataFrame: - """Prepare data for SHAP analysis.""" - X_processed = X.copy() - - # Remove target column if present - if self.target_col in X_processed.columns: - X_processed = X_processed.drop(columns=[self.target_col]) - - # Filter selected features - if self.selected_features_ is not None: - X_processed = cast( - pd.DataFrame, X_processed[self.selected_features_].copy() - ) - - return X_processed - - def _create_shap_explainer(self, X: pd.DataFrame) -> Any: - """Create appropriate SHAP explainer based on model type.""" - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - - if hasattr(self.model_, "coef_"): # Linear models - try: - # Handle missing values if model supports it - X_clean = self._handle_missing_values_for_shap(X) - return shap.LinearExplainer(self.model_, X_clean) - except Exception as e: - console.print( - f"[yellow]Warning: Linear explainer failed: {e}, " - + "using KernelExplainer[/yellow]" - ) - background = shap.maskers.Independent(X, max_samples=100) - return shap.KernelExplainer( - self.model_.predict, - background, - ) - else: - # Non-linear models - if self.learning_task == "binary": - return shap.TreeExplainer( - self.model_, X, model_output="probability" - ) - else: - return shap.Explainer(self.model_, X) - - def _handle_missing_values_for_shap(self, X: pd.DataFrame) -> pd.DataFrame: - """Handle missing values for SHAP analysis.""" - # Use type ignore for optional method - if hasattr(self.model_, "_impute_missing_values"): - return self.model_._impute_missing_values(X) # type: ignore - else: - return X.dropna() - - def _generate_and_save_plot( - self, explainer: Any, X: pd.DataFrame, path: str - ) -> None: - """Generate and save SHAP plot.""" - shap_values = explainer(X) - - shap.plots.beeswarm(shap_values, show=False) - shap_fig = plt.gcf() - shap_fig.set_size_inches(10, 6) - shap_fig.suptitle(f"SHAP Beeswarm Plot for {self.name}", fontsize=16) - shap_fig.tight_layout() - shap_fig.savefig(path) - plt.clf() - plt.close() - - @ensure_fitted - def shap_beeswarm_plot(self, X: pd.DataFrame, path: str) -> None: - """ - Generates a SHAP beeswarm plot for the model's predictions. - - Args: - X: DataFrame of input features. - path: Path to save the plot file. - - Raises: - ValueError: If model is not fitted. - Exception: If SHAP plot generation fails. - """ - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - - try: - # Prepare data - X_processed = self._prepare_shap_data(X) - - # Create explainer and generate plot - explainer = self._create_shap_explainer(X_processed) - self._generate_and_save_plot(explainer, X_processed, path) - - except Exception as e: - console.print( - f"[red]Error: Failed to generate SHAP plot: {e}[/red]" - ) - raise - - def get_params_dict(self) -> dict: - """Get model parameters as dictionary for logging/serialization.""" - base_params = super().get_params_dict() - mv_params = { - "learning_task": self.learning_task, - "differentiate_target": self.differentiate_target, - "selected_features_": self.selected_features_, - } - return {**base_params, **mv_params} - - def summary(self) -> str: - """Generate a summary string of the model.""" - params = self.get_params_dict() - fitted_status = ( - "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" - ) - - summary_lines = [ - f"Model: {self.__class__.__name__}", - f"Status: {fitted_status}", - f"Features: {params.get('n_features_in_', 'Unknown')}", - f"Task: {params.get('learning_task', 'regression')}", - f"Selected Features: {len(self.selected_features_) if self.selected_features_ else 'All'}", - ] - - return "\n".join(summary_lines) From df7ce9e79b6513f99ee2d32217ae820b70024214 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 15 Sep 2025 14:47:46 -0300 Subject: [PATCH 05/29] SIENTIAPDE-1222 Remove obsolete data files: deleted data.csv and response_data.csv to streamline project structure and eliminate unused resources. --- data.csv | 11 ----------- response_data.csv | 2 -- 2 files changed, 13 deletions(-) delete mode 100644 data.csv delete mode 100644 response_data.csv diff --git a/data.csv b/data.csv deleted file mode 100644 index 76acc92..0000000 --- a/data.csv +++ /dev/null @@ -1,11 +0,0 @@ -timestamp,07BP012/VEL_M1_PV,07BP013/VEL_M1_PV,07BP014/VEL_M1_PV,07FT001_COR_B,07FT001_COR_G,07FT001_COR_R,07FT001_TEXT,07FT007_COR_B,07FT007_COR_G,07FT007_COR_R,07FT007_TEXT,07FT012_COR_B,07FT012_COR_G,07FT012_COR_R,07FT012_TEXT,09BP023/VEL_PV,09BP024/VEL_PV,303-WIT-230,305-AIC-001_PV,305-AIC-002_PV,305-CALC-001,305-FIC-001_PV,305-FIC-003_PV,305-FIC-005_PV,305-FIC-006_PV,305-FIT-002,305-FIT-009,305-FIT-010,305-FIT-011,305-FIT-012,305-FIT-013,305-LIC-001_PV,305-LIC-002_PV,305-PIT-170,305-PIT-175,305-SIC-001_SP,305-SIC-002_SP,305-WIT-135,305-WIT-160,306-CALC-018,306-DIT-001,306-DIT-002,306-FIT-003,306-FIT-004,306-FIT-005,306-FIT-006,306-FIT-051,306-FIT-052,306-LIC-001_PV,306-LIC-002_PV,306-LIC-003_PV,306-PIT-101,306-PIT-105,306-PIT-110,306-PIT-115,306-PIT-125,306-PIT-130,307-CALC-001,307-CALC-002,307-FIC-001_PV,307-FIC-005_PV,307-FIC-006_PV,307-FIC-019_PV,307-FIC-022,307-FIC-101_PV,307-FIC-105_PV,307-FIC-110_PV,307-FIC-115_PV,307-FIC-120_PV,307-FIC-130_PV,307-FIC-135_PV,307-FIC-140_PV,307-FIC-145_PV,307-FIC-150_PV,307-FIC-155_PV,307-FIC-160_PV,307-FIT-003,307-FIT-005,307-FIT-008,307-FIT-009,307-LIC-003_PV,307-LIC-004_PV,307-LIC-101_PV,307-LIC-105_PV,307-LIC-110_PV,307-LIC-115_PV,307-LIC-120_PV,307-LIC-130_PV,307-LIC-135_PV,307-LIC-140_PV,307-LIC-145_PV,307-LIC-150_PV,307-LIC-155_PV,307-LIC-160_PV,307-SIC-006_OUT,307-SIC-007_OUT,309-FIC-013,309-FIC-014,309-FIT-051,309-FIT-052,309-LIC-001_PV,309-LIC-002_PV,309-PIT-001,309-PIT-002,309-PIT-101,309-PIT-105,309-PIT-110,309-PIT-185,309-PIT-190,309-PIT-195,310-AIC-001_PV,310-AIC-002,310-CALC-001,310-CALC-002,310-DIC-002_PV,310-FIC-004_PV,310-FIC-010_PV,310-FIC-110_PV,310-FIC-120_PV,310-FIC-160_PV,310-FIC-170_PV,310-FIT-004,310-FIT-005,310-FIT-006,310-FIT-010,310-FV-032,310-LIC-110_PV,310-LIC-120_PV,310-LIC-160_PV,310-LIC-170_PV,310-LIT-003_PV,310-LIT-004_PV,310-SIC-003_OUT,310-SIC-004_OUT,310-SIC-005_OUT,310-SIC-006_OUT,311-FIC-029_PV,311-FIC-033_PV,311-FIT-033,312-CALC-001,312-CALC-005,312-CALC-006,312-DIC-001_PV,312-DIC-002_PV,312-FIC-001_PV,312-FIC-002_PV,313-CALC-001,313-DIC-001_PV,313-DIC-002_SP,313-FIC-006_PV,317AIT001.2,317AIT002.1,317AIT002.10,317AIT002.11,317AIT002.12,317AIT002.13,317AIT002.14,317AIT002.15,317AIT002.16,317AIT002.17,317AIT002.18,317AIT002.19,317AIT002.2,317AIT002.20,317AIT002.21,317AIT002.22,317AIT002.23,317AIT002.24,317AIT002.25,317AIT002.26,317AIT002.27,317AIT002.28,317AIT002.29,317AIT002.3,317AIT002.30,317AIT002.31,317AIT002.32,317AIT002.33,317AIT002.34,317AIT002.35,317AIT002.36,317AIT002.37,317AIT002.38,317AIT002.39,317AIT002.4,317AIT002.40,317AIT002.41,317AIT002.42,317AIT002.43,317AIT002.44,317AIT002.45,317AIT002.46,317AIT002.47,317AIT002.48,317AIT002.49,317AIT002.5,317AIT002.50,317AIT002.51,317AIT002.52,317AIT002.53,317AIT002.54,317AIT002.55,317AIT002.56,317AIT002.57,317AIT002.58,317AIT002.59,317AIT002.6,317AIT002.60,317AIT002.61,317AIT002.62,317AIT002.63,317AIT002.64,317AIT002.65,317AIT002.66,317AIT002.67,317AIT002.68,317AIT002.69,317AIT002.7,317AIT002.70,317AIT002.71,317AIT002.72,317AIT002.8,317AIT002.9,317AIT003.1,317AIT003.2,317AIT003.3,317AIT003.5,319-CALC-001,319-CALC-013,319-DIC-001_PV,319-DIC-001_SP,319-DIC-002_PV,319-DIC-002_SP,319-FIC-006_PV,319-FIC-006_SP,319-FIC-007_PV,319-FIC-007_SP,319-FIQ-CALC-009_DAY,319-FIT-004,319-FIT-005,319-LIT-201-R,319-PIT-003,319-PIT-004,319-SIC-001_OUT,319-SIC-002_OUT,Fe_conc,G03-07BP102_M1,G03-07BP103_M1,G03-08BP107_M1,G03-10BP104_M1,G03-19BP101_M1,G03-19BP106_M1,G03-19BP110_M1,SOL-CALC-005,SOL-CALC-006,SiO2_conc -2024-12-05 02:16:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2185.6279296875,8.909636497497559,8.824532508850098,2628.087646484375,233.4102020263672,289.2882995605469,2.2411208152771,2.740176200866699,3158.830322265625,687.0074462890625,0.0,287.7219543457031,0.0,3312.36962890625,89.0501937866211,83.04390716552734,12.823511123657228,13.373872756958008,,,1364.865478515625,1239.42919921875,3.005729913711548,1.3840404748916626,1.3935494422912598,2170.908447265625,2189.05615234375,1866.9991455078125,1715.1016845703125,2254.970703125,2276.34033203125,83.4413070678711,94.47754669189452,89.85079956054688,34.401798248291016,33.98549270629883,31.97739601135254,34.75410461425781,43.70286178588867,44.13051223754883,1894.9356689453125,2017.797607421875,900.0048217773438,30.6091537475586,375.9732971191406,0.0,652.0335693359375,765.3798217773438,763.6800537109375,769.5640258789062,739.6097412109375,719.44775390625,706.0026245117188,703.8912353515625,567.9400634765625,593.1154174804688,629.06787109375,595.2117919921875,625.0,1.425487995147705,27.0,1194.873046875,998.1407470703124,105.17456817626952,99.1063003540039,28.07830810546875,21.692842483520508,34.955604553222656,12.697091102600098,40.85791778564453,21.66144752502441,17.217727661132812,47.85460662841797,57.36534118652344,59.91471481323242,48.2706184387207,39.44066619873047,83.20423126220703,66.5,0.0,0.0,1768.509765625,1827.390380859375,85.6709213256836,96.77904510498048,0.0998583808541297,4.900833606719971,23.81103706359864,26.791275024414062,26.24358367919922,23.354522705078125,27.844594955444336,27.89141845703125,10.390382766723633,10.471061706542969,223.273666381836,209.89990234375,1.1763592958450315,2.309485912322998,103.65160369873048,450.9481201171875,427.9828186035156,450.59222412109375,448.7367858886719,17.0,700.9956665039062,0.0,1200.0,1.0,39.16987991333008,37.460086822509766,19.4918155670166,11.98000144958496,95.19344329833984,86.53215789794922,,,86.0,99.30213165283205,22.59608268737793,0.0,400.0,1351.7703857421875,0.0,1459.112548828125,2.163416624069214,1.6798019409179688,997.8304443359376,0.0240168757736682,203.056381225586,1.369994044303894,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.4978864192962646,8.562295913696289,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.253314971923828,13.389976501464844,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.01197052001953,1.7381054162979126,1.1206940412521362,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,59.35650634765625,1.4764769077301023,1.526507019996643,0.4823205173015594,0.7710468769073486,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3361487984657287,0.428571492433548,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,7.89943265914917,5.800000190734863,6.465214729309082,86.5999984741211,830.4485473632812,849.3055419921875,,,1.5839204788208008,1.572644829750061,0.0,600.0,920.1913452148438,921.7091064453124,0.0,10.023720741271973,0.0,11.650277137756348,4.348147869110107,0.0748394280672073,71.11778259277344,100.0,64.71,0.0,4.533299922943115,5.666272640228272,8.773231506347656,4.993200302124023,4.861800193786621,6.432722568511963,50.96606063842773,67.95718383789062,3.52 -2024-12-05 02:18:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2300.107421875,8.874933242797852,8.814615249633789,2379.468017578125,205.70236206054688,299.0367736816406,2.237759590148926,2.737307071685791,3149.414306640625,705.9229125976562,0.0,287.82403564453125,0.0,3319.203857421875,88.97682189941406,82.33074951171875,13.035848617553713,13.303841590881348,,,801.2099609375,1577.420166015625,3.18897008895874,1.3844648599624634,1.3944939374923706,2137.277099609375,2091.429931640625,1866.9801025390625,1714.9752197265625,2248.451904296875,2272.76025390625,85.66764831542969,88.35608673095703,87.84507751464844,34.351341247558594,33.97214126586914,31.98730659484864,32.71706008911133,43.68099594116211,44.0351676940918,1898.191162109375,1618.9393310546875,898.7774047851562,30.54461669921875,374.732666015625,0.0,648.9142456054688,761.4437866210938,761.818115234375,728.4386596679688,741.0633544921875,659.6345825195312,707.1867065429688,684.1427612304688,568.2828369140625,594.7976684570312,633.2908325195312,595.1723022460938,640.0,1.245144605636597,27.0,1194.7637939453125,998.0435791015624,103.9820556640625,99.07530975341795,28.694652557373047,22.18309211730957,31.824302673339844,12.703847885131836,38.318546295166016,21.9743595123291,16.507755279541016,47.80324935913086,57.33625793457031,59.91497421264648,48.0490837097168,40.25405502319336,83.34803771972656,66.5,0.0,0.0,1746.5220947265625,1811.0615234375,85.64175415039062,96.97100067138672,0.0999280512332916,4.900284290313721,23.82648658752441,26.91034507751465,26.227479934692383,23.32929229736328,27.45798110961914,27.736440658569336,10.399251937866213,10.47207736968994,175.54986572265625,211.37567138671875,1.174445867538452,2.112058639526367,103.86756896972656,450.9291076660156,427.931396484375,450.5900268554688,448.73138427734375,17.0,700.9699096679688,10.513471603393556,1200.0,1.0,40.18443298339844,37.6434326171875,19.451520919799805,11.894670486450195,95.58326721191406,86.0557632446289,,,86.0,98.96598815917967,22.82772254943848,0.0,400.0,968.2177734375,0.0,1464.4488525390625,2.163362979888916,1.679788589477539,1003.175537109375,0.0238033775240182,184.37130737304688,1.3698077201843262,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,8.57795238494873,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,13.411721229553224,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1217609643936155,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,59.35650634765625,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7702381610870361,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4283382296562195,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,8.100000381469727,5.800000190734863,6.435592651367188,86.5999984741211,725.95458984375,858.4331665039062,,,1.5835398435592651,1.5726622343063354,0.0,600.0,917.3837280273438,925.9874877929688,0.0,10.049739837646484,0.0,11.572147369384766,4.3472161293029785,0.0748393461108207,71.409423828125,100.0,64.71,0.0,4.533299922943115,5.762217998504639,8.699999809265137,4.993200302124023,4.861800193786621,6.558172702789307,50.968997955322266,67.96240234375,3.52 -2024-12-05 02:20:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2063.14990234375,8.86483097076416,8.804698944091797,2637.2822265625,186.7984161376953,274.3067932128906,2.234398603439331,2.734437942504883,3139.637939453125,738.9068603515625,0.0,287.9261474609375,0.0,3328.472412109375,84.00076293945312,86.40196990966797,12.14862060546875,13.639047622680664,,,1477.9354248046875,1141.748291015625,2.891580104827881,1.3848892450332642,1.3954384326934814,2059.924560546875,2129.599853515625,1866.961181640625,1714.848876953125,2241.93310546875,2266.62158203125,85.53884887695312,87.77095031738281,89.44815063476562,34.300880432128906,33.95878982543945,31.997217178344727,30.499488830566406,43.65913391113281,44.02417755126953,1907.386474609375,1944.6339111328125,903.0270385742188,30.480079650878903,374.2165832519531,0.0,649.119384765625,757.221435546875,759.9561767578125,811.8240356445312,748.2200927734375,740.3225708007812,708.370849609375,673.0714721679688,573.4213256835938,596.4798583984375,631.8908081054688,595.1328735351562,635.0,1.0720911026000977,27.0,1194.6546630859375,997.9464111328124,105.5190887451172,99.0443115234375,28.1169376373291,21.61070251464844,32.5518684387207,12.71060562133789,37.97523498535156,22.30083274841309,17.719982147216797,47.75189208984375,57.30717849731445,59.91522979736328,47.62279891967773,41.22840881347656,83.33912658691406,66.5,0.0,0.0,1749.242431640625,1821.6829833984373,85.61257934570312,97.1629638671875,0.0999977141618728,4.899734973907471,23.841936111450195,26.64358901977539,26.1314697265625,23.415620803833008,27.687856674194336,27.292091369628903,10.39584255218506,10.473093032836914,208.32379150390625,211.2283935546875,1.1736302375793457,1.0507607460021973,104.0835418701172,450.91009521484375,427.87994384765625,450.58782958984375,448.7259826660156,17.0,700.9442138671875,0.0,1200.0,1.0,40.472747802734375,38.068607330322266,19.54105758666992,12.325950622558594,95.0851821899414,85.76668548583984,,,86.0,98.85044860839844,22.84588432312012,0.0,400.0,1176.9923095703125,0.0,1459.0894775390625,2.1633095741271973,1.6797752380371094,997.1138916015624,0.0235898792743682,204.41549682617188,1.3696213960647583,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,8.100000381469727,5.400000095367432,6.405970096588135,86.5999984741211,802.98876953125,856.4476318359375,,,1.583159327507019,1.57267963886261,0.0,600.0,935.9249267578124,930.265869140625,0.0,10.131684303283691,0.0,11.596683502197266,4.34628438949585,0.0748392716050148,71.35333251953125,100.0,64.71,0.0,4.533299922943115,5.731375694274902,8.699999809265137,4.993200302124023,4.861800193786621,6.488824367523193,50.97193908691406,67.96761322021484,3.52 -2024-12-05 02:22:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2344.354736328125,8.854728698730469,8.794782638549805,2443.620849609375,235.85362243652344,289.5343017578125,2.231037378311157,2.7315685749053955,3128.458251953125,678.6727905273438,0.0,288.0282287597656,0.0,3352.32080078125,82.41395568847656,88.41472625732422,12.335658073425291,13.590455055236816,,,1179.0438232421875,1241.806396484375,2.9580600261688232,1.3853135108947754,1.3963829278945925,2020.5482177734373,2175.6435546875,1866.942138671875,1714.722412109375,2241.09228515625,2263.143798828125,85.41004943847656,89.18098449707031,91.12660217285156,34.372066497802734,33.94544219970703,32.00712585449219,30.11072158813477,43.63727188110352,44.04359436035156,1842.406005859375,1904.2274169921875,902.5571899414062,30.415542602539062,374.7893676757813,0.0,651.5109252929688,768.8901977539062,758.0942993164062,720.6381225585938,742.55078125,685.2013549804688,709.5549926757812,730.93359375,572.37548828125,598.1620483398438,634.2003173828125,595.0933837890625,632.0,0.9020777940750122,27.0,1194.54541015625,997.8492431640624,104.99007415771484,99.01332092285156,28.318359375,21.52416229248047,31.58490943908692,12.717362403869627,38.4969482421875,21.7913761138916,17.436378479003906,47.70053482055664,57.278099060058594,59.915489196777344,46.977535247802734,42.46583938598633,83.07581329345703,66.5,0.0,0.0,1748.264892578125,1827.9678955078125,85.58340454101562,97.35491943359376,0.1000673845410347,4.899185180664063,23.85738754272461,26.20622062683105,26.06732177734375,23.50194931030273,27.590068817138672,27.46240234375,10.392244338989258,10.47410774230957,214.98492431640625,205.6874847412109,1.173506498336792,2.398390531539917,104.2995147705078,450.8910827636719,427.8284912109375,450.5856323242188,448.7205810546875,17.0,700.9185180664062,0.0,1200.0,1.0,40.27254486083984,38.66741561889648,19.67565155029297,12.237468719482422,93.86233520507812,85.74674987792969,,,86.0,98.91876983642578,22.30738639831543,0.0,400.0,1275.9793701171875,0.0,1467.3343505859375,2.1632559299468994,1.6797618865966797,998.2872314453124,0.0233763810247182,187.3364105224609,1.3694350719451904,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.708251953125,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.3545424044132232,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3020144402980804,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.100000381469727,5.400000095367432,6.699999809265137,85.80000305175781,747.72314453125,830.0194091796875,,,1.5827786922454834,1.5726970434188845,0.0,600.0,924.1300048828124,934.5442504882812,0.0,10.00216579437256,0.0,11.625198364257812,4.345353126525879,0.0748391896486282,71.52710723876953,100.0,64.71,0.0,4.533299922943115,5.767271518707275,8.699999809265137,4.993200302124023,4.861800193786621,6.6877641677856445,50.97750854492188,67.97283172607422,3.52 -2024-12-05 02:24:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2862.970703125,8.844626426696777,8.774948120117188,2980.736572265625,237.5259704589844,291.2091369628906,2.2276761531829834,2.728699445724488,3153.987548828125,682.5768432617188,0.0,288.1303405761719,0.0,3345.968505859375,87.24137878417969,86.74526977539062,12.552753448486328,13.396173477172852,,,1370.993896484375,1611.220947265625,2.8518500328063965,1.3857378959655762,1.397327542304993,2030.3245849609373,2142.251953125,1866.9232177734373,1714.6802978515625,2248.769775390625,2268.414794921875,85.06763458251953,91.27864837646484,90.73202514648438,34.44520568847656,33.932090759277344,32.01703643798828,31.01426696777344,43.61540985107422,44.06300735473633,1867.2406005859373,2242.3193359375,898.3688354492188,30.811418533325195,375.3621520996094,0.0,647.7444458007812,756.3349609375,759.6863403320312,795.988525390625,743.1278076171875,681.9320678710938,710.7391357421875,760.3604736328125,576.6845092773438,599.8442993164062,620.6749267578125,595.053955078125,640.0,0.7632204294204712,27.0,1194.4361572265625,997.7520751953124,106.02984619140624,98.98233032226562,28.794273376464844,21.658815383911133,32.04315185546875,12.724120140075684,38.84426498413086,21.82027244567871,16.058849334716797,47.64917755126953,57.24901580810547,59.915748596191406,46.44183349609375,41.52559280395508,83.30157470703125,66.5,0.0,0.0,1750.5181884765625,1845.0968017578125,85.55422973632812,97.546875,0.1001370549201965,4.8986358642578125,23.87283706665039,26.4957332611084,26.47050094604492,23.5178451538086,27.697071075439453,27.838388442993164,10.388647079467772,10.475123405456545,243.6897125244141,207.8060302734375,1.1728342771530151,1.2688955068588257,104.5154800415039,450.8720397949219,427.7770690917969,450.58343505859375,448.7151794433594,17.0,700.892822265625,0.0,1200.0,1.0,40.07234191894531,37.74433898925781,19.810243606567383,12.158288955688477,91.9724578857422,85.08606719970703,,,86.0,98.60803985595705,22.23607635498047,0.0,400.0,1401.1199951171875,0.0,1445.149658203125,2.1632025241851807,1.67974853515625,1001.5813598632812,0.0231628827750682,230.4506072998047,1.3692487478256226,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.899999618530273,5.400000095367432,6.699999809265137,85.80000305175781,927.35986328125,843.9348754882812,,,1.5823981761932373,1.5727144479751587,0.0,600.0,943.0855712890624,939.0200805664062,0.0,10.032031059265137,0.0,11.4324369430542,4.34442138671875,0.0748391151428222,71.41386413574219,100.0,64.71,0.0,4.533299922943115,5.681782245635986,8.699999809265137,4.993200302124023,4.861800193786621,6.551279544830322,50.98702239990234,67.97804260253906,3.52 -2024-12-05 02:26:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2654.10888671875,8.834524154663086,8.75146198272705,2392.275390625,235.610580444336,295.8012084960937,2.2243149280548096,2.725830078125,3172.66748046875,694.5460205078125,0.0,288.2324523925781,0.0,3339.616455078125,89.53855895996094,85.1905746459961,12.741495132446287,13.875475883483888,,,1323.8238525390625,1067.496337890625,2.916759967803955,1.386162281036377,1.3982720375061035,2102.00048828125,2161.18212890625,1866.9041748046875,1714.821044921875,2259.66064453125,2275.4189453125,84.60967254638672,90.81041717529295,90.06828308105469,34.51834487915039,33.918739318847656,32.026947021484375,32.243064880371094,43.593544006347656,44.08242416381836,1909.4129638671875,1879.74658203125,902.9876708984376,31.21604347229004,375.9349365234375,0.0,650.2518310546875,751.2490234375,761.6719970703125,715.2727661132812,747.89599609375,747.788818359375,706.99658203125,774.9991455078125,583.5380249023438,601.5264892578125,614.032958984375,595.0144653320312,615.0,0.7071666121482849,27.0,1194.326904296875,997.6549682617188,104.1503677368164,98.95133209228516,28.68854713439941,21.7934684753418,32.87420654296875,12.730876922607422,38.9033317565918,22.107240676879883,17.046525955200195,47.59782028198242,57.21993637084961,59.9160041809082,47.52975463867188,40.58535003662109,83.56719970703125,66.5,0.0,0.0,1742.1806640625,1809.867919921875,85.52505493164062,96.9273681640625,0.1002067178487777,4.8980865478515625,23.88828659057617,26.306615829467773,26.495014190673828,23.27698135375977,27.35708808898925,27.72958755493164,10.385048866271973,10.4761381149292,216.2808380126953,213.6409606933593,1.1716774702072144,2.5998244285583496,104.73145294189452,450.85302734375,427.793212890625,450.5812377929688,448.7097778320313,17.0,700.8671264648438,0.0148877017199993,1200.0,1.0,39.87213897705078,37.95595169067383,19.94483757019043,12.006688117980955,91.47502899169922,85.28714752197266,,,86.0,98.45233917236328,22.66000938415528,0.0,400.0,1256.264892578125,0.0,1452.758544921875,2.163148880004883,1.6797351837158203,1000.3143920898438,0.0229493845254182,190.5460968017578,1.369062423706055,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.899999618530273,3.299999952316284,6.699999809265137,85.80000305175781,762.7355346679688,865.4536743164062,,,1.5816150903701782,1.572731852531433,0.0,600.0,938.32177734375,943.602783203125,0.0,10.15860080718994,0.0,11.596125602722168,4.343489646911621,0.0748390331864357,71.58973693847656,100.0,64.71,0.0,4.533299922943115,5.802553176879883,8.699999809265137,4.993200302124023,4.861800193786621,6.558778762817383,50.99653625488281,67.98326110839844,3.52 -2024-12-05 02:28:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2227.77099609375,8.836018562316895,8.75161361694336,2456.804931640625,238.3328552246093,283.34686279296875,2.220953941345215,2.722960948944092,3188.196533203125,818.465087890625,0.0,288.33453369140625,0.0,3333.26416015625,90.52090454101562,83.3249282836914,12.903297424316406,13.22053337097168,,,1122.2493896484375,1331.5670166015625,3.0864999294281006,1.3865865468978882,1.3992165327072144,2095.321533203125,2129.77685546875,1866.88525390625,1714.9619140625,2259.91357421875,2288.202880859375,84.55709075927734,89.37647247314453,88.79876708984375,34.55312728881836,33.905391693115234,32.03685760498047,31.44712448120117,43.62167739868164,44.101837158203125,2035.30908203125,1930.71044921875,899.136474609375,32.6287956237793,376.5077209472656,0.0,653.2977294921875,750.5552368164062,760.983642578125,770.3090209960938,738.3396606445312,653.3170776367188,701.0360107421875,730.8182983398438,566.891845703125,588.6995239257812,629.2593383789062,594.9750366210938,622.0,0.6648255586624146,27.0,1194.2176513671875,997.5578002929688,105.18731689453124,98.92034149169922,28.36734771728516,21.92812156677246,32.240821838378906,12.737634658813477,38.52233505249024,21.57743263244629,17.07017707824707,47.54646301269531,57.19085311889648,59.916263580322266,48.34513473510742,39.64510345458984,83.13888549804688,66.5,0.0,0.0,1753.1981201171875,1814.801513671875,85.49588012695312,96.34169006347656,0.1002763882279396,4.8975372314453125,23.903738021850582,26.488170623779297,26.35142517089844,23.256576538085938,27.712129592895508,27.556394577026367,10.381451606750488,10.477153778076172,218.41183471679688,225.77491760253903,1.1706732511520386,1.818994283676148,104.94741821289062,450.8340148925781,428.7974853515625,450.57904052734375,448.7043762207031,17.0,700.8414306640625,0.5039713978767395,1200.0,1.0,39.67193984985352,38.82048416137695,20.07943153381348,12.295125961303713,90.39128875732422,85.13963317871094,,,86.0,98.48783111572266,24.05027961730957,0.0,400.0,1260.049560546875,0.0,1470.87890625,2.163095474243164,1.6797218322753906,1004.2244873046876,0.0227358862757682,194.2509765625,1.3688760995864868,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.86962890625,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,7.830216407775879,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042373403906822,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.7188185453414917,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7221007943153381,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.49777603149414,65.73210906982422,8.899999618530273,3.299999952316284,6.5,86.30000305175781,790.286376953125,913.74462890625,,,1.5807862281799316,1.572749376296997,0.0,600.0,945.046875,945.7421264648438,0.0,10.257745742797852,0.0,11.62919807434082,4.342557907104492,0.0748389586806297,71.7236099243164,100.0,64.71,0.0,4.533299922943115,5.762265205383301,8.699999809265137,4.993200302124023,4.861800193786621,6.415340900421143,51.00605010986328,67.98847961425781,3.52 -2024-12-05 02:30:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2773.8876953125,8.840205192565918,8.76478099822998,2499.578125,233.58840942382807,273.2201843261719,2.217592716217041,2.720091819763184,3169.754638671875,718.6017456054688,0.0,288.4366455078125,0.0,3293.60546875,88.42021942138672,80.45106506347656,12.661704063415527,13.48155403137207,,,1310.0872802734375,1188.9476318359375,3.5530900955200195,1.387010931968689,1.4001611471176147,2047.4097900390625,2157.358642578125,1866.8662109375,1715.1026611328125,2255.59716796875,2283.900390625,84.08345794677734,86.43714904785156,89.38142395019531,34.57453536987305,34.07208633422852,32.04676818847656,29.560300827026367,43.759544372558594,44.12125396728516,2028.601806640625,1870.660400390625,895.9712524414062,32.81827163696289,377.08050537109375,0.0,649.1517333984375,770.4885864257812,759.2982788085938,734.395751953125,743.6987915039062,678.6212158203125,700.5547485351562,663.7391967773438,576.5570678710938,589.0308837890625,628.8489990234375,594.935546875,631.0,0.6459924578666687,27.0,1194.1085205078125,997.4606323242188,105.77910614013672,98.88935089111328,28.33451271057129,22.062774658203125,32.428157806396484,12.744391441345217,38.12985610961914,22.12809181213379,17.180856704711914,47.4951057434082,57.16177368164063,59.91652297973633,49.058895111083984,39.42089080810547,83.53347778320312,66.5,0.0,0.0,1742.7498779296875,1810.687255859375,85.46670532226562,97.0500717163086,0.1003460586071014,4.896987915039063,23.919187545776367,26.559722900390625,26.710390090942383,23.56723022460937,27.740917205810547,27.745880126953125,10.377854347229004,10.478169441223145,205.0146484375,225.3446502685547,1.1705199480056765,2.214766025543213,105.16339111328124,450.8149719238281,429.5928039550781,450.5768432617188,448.698974609375,17.0,700.815673828125,0.239432543516159,1200.0,1.0,37.73193740844727,38.19848251342773,20.21402359008789,11.93883228302002,88.80018615722656,84.96267700195312,,,86.0,98.27005767822266,24.427764892578125,0.0,400.0,1261.719482421875,0.0,1463.796630859375,2.163041830062866,1.679708480834961,993.51123046875,0.0225223880261182,192.6799774169922,1.368689775466919,1.3799999952316284,0.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,761.5707397460938,910.4165649414062,,,1.5799574851989746,1.5727667808532717,0.0,600.0,953.1422119140624,947.8602294921876,0.0,10.270873069763184,0.0,11.646549224853516,4.341626167297363,0.0748388767242431,71.84945678710938,100.0,64.71,0.0,4.533299922943115,5.71589994430542,8.699999809265137,4.993200302124023,4.861800193786621,6.311936378479004,51.01556396484375,67.99369049072266,3.52 -2024-12-05 02:32:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,3052.78662109375,8.844391822814941,8.777948379516602,2515.992431640625,229.52468872070312,265.2821044921875,2.214231491088867,2.7172224521636963,3156.99365234375,701.08251953125,0.0,288.5387268066406,0.0,3317.0087890625,86.63294982910156,82.62700653076172,12.9373197555542,12.94904613494873,,,1328.0621337890625,1193.599609375,3.41225004196167,1.3874353170394895,1.4011056423187256,1862.8626708984373,2177.4072265625,1866.84716796875,1715.2435302734375,2251.281005859375,2277.60693359375,84.24079895019531,84.25446319580078,89.87091827392578,34.595943450927734,34.107933044433594,32.05667495727539,24.58193588256836,43.89741134643555,44.14066696166992,2033.9571533203125,1965.69189453125,902.446533203125,32.7397346496582,377.6492614746094,0.0,647.1126708984375,740.5242309570312,757.6129760742188,745.9622192382812,750.0130004882812,749.8059692382812,700.0735473632812,701.1631469726562,570.8465576171875,589.3622436523438,632.1378173828125,594.8961181640625,628.0,0.6863186955451965,27.0,1193.999267578125,997.3634643554688,103.6092300415039,98.8583526611328,28.94314765930176,22.19742774963379,32.61549377441406,12.75114917755127,37.71183013916016,22.179346084594727,17.744470596313477,47.44374847412109,57.132694244384766,59.916778564453125,48.76108932495117,40.08521270751953,83.61792755126953,66.5,0.0,0.0,1746.4300537109375,1854.5777587890625,85.64894104003906,97.2160415649414,0.1004157289862632,4.8964385986328125,23.925472259521484,26.834339141845703,26.46601295471192,23.520160675048828,27.76211738586425,27.71584892272949,10.374256134033203,10.4791841506958,218.3724822998047,225.1285858154297,1.1713463068008425,2.5177488327026367,105.52578735351562,450.79595947265625,429.6829833984375,450.57464599609375,448.6935729980469,17.0,700.7899780273438,1.7021775245666504,1200.0,1.0,36.82808303833008,38.11102294921875,20.06475257873535,11.937515258789062,89.86893463134766,84.6894760131836,,,86.0,98.14891815185548,24.32128524780273,0.0,400.0,1259.343017578125,0.0,1462.4814453125,2.1629884243011475,1.6796951293945312,998.4035034179688,0.0223088879138231,198.8683013916016,1.368503451347351,1.3799999952316284,0.0,,61.264404296875,16.665037155151367,5.333080291748047,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.86962890625,4.85367488861084,73.36161041259766,91.94583892822266,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,7.830216407775879,38.73592758178711,18.59975242614746,17.211471557617188,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042373403906822,1.970276951789856,0.4222961962223053,0.0943225920200347,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.7188185453414917,0.3608308732509613,1.2047821283340454,1.6332342624664309,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7221007943153381,0.2941087782382965,-9999.0,0.5667175650596619,0.7008373737335205,0.8771045207977295,8.49777603149414,65.73210906982422,12.899999618530272,5.300000190734863,6.515639781951904,86.30000305175781,790.82958984375,907.0885620117188,,,1.5791287422180176,1.572784185409546,0.0,600.0,950.763671875,949.9783325195312,0.0,10.146312713623049,0.0,11.57530403137207,4.340694427490234,0.0748387947678566,71.91187286376953,100.0,64.71,0.0,4.533299922943115,5.70755672454834,8.692553520202637,4.993200302124023,4.861800193786621,6.372900485992432,51.02507781982422,67.99890899658203,3.52 -2024-12-05 02:34:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2978.102783203125,8.848578453063965,8.862582206726074,2573.7099609375,234.7885284423828,289.6545104980469,2.2108702659606934,2.714353322982788,3147.3876953125,705.0564575195312,0.0,288.6408386230469,0.0,3398.574462890625,86.58733367919922,89.01890563964844,12.617484092712402,13.247730255126951,,,1347.5814208984375,1234.0552978515625,3.514620065689087,1.387859582901001,1.4020501375198364,1821.7401123046875,2152.696533203125,1866.8282470703125,1715.38427734375,2248.69091796875,2274.983642578125,84.39221954345703,88.17390441894531,90.19397735595705,34.61734771728516,34.1109504699707,32.32330322265625,23.76468849182129,43.90478515625,44.16008377075195,1927.233642578125,1939.23876953125,903.4033203125,31.51068115234375,378.1936645507813,0.0,648.6610107421875,766.79345703125,755.9276123046875,794.9697265625,744.4769287109375,656.0175170898438,699.5923461914062,772.5515747070312,569.7051391601562,589.693603515625,635.4265747070312,594.8566284179688,633.0,0.7698317170143127,27.0,1193.8900146484375,997.2662963867188,103.9173355102539,98.82736206054688,29.941625595092773,22.332080841064453,32.80282974243164,12.757905960083008,37.37025833129883,21.68112564086914,15.733065605163574,47.392391204833984,57.10361099243164,59.91703796386719,48.43916702270508,40.41853332519531,83.20599365234375,66.5,0.0,0.0,1749.7413330078125,1801.275634765625,85.9491195678711,97.38201904296876,0.1004853919148445,4.895888805389404,23.921628952026367,26.72721099853516,26.51607131958008,23.67252349853516,27.53003692626953,27.67759895324707,10.37065887451172,10.479698181152344,220.0579071044922,214.2723999023437,1.172289490699768,2.055137157440185,106.04012298583984,450.7769470214844,429.7731628417969,450.5724487304688,448.6881713867188,17.0,700.7642822265625,0.0,1200.0,1.0,37.02928161621094,38.0235595703125,19.88223648071289,12.143880844116213,91.3222427368164,85.17166900634766,,,86.0,98.23806762695312,23.680456161499023,0.0,400.0,1300.7486572265625,0.0,1472.4622802734375,2.1629347801208496,1.679681658744812,1000.1616821289062,0.0220953896641731,198.59475708007807,1.3683171272277832,1.3799999952316284,0.0,,61.264404296875,,5.333080291748047,,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,,12.386075973510742,,,,,,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.547714233,,,,,,,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,,,,,,,,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,,,,,,,42.07585144042969,,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,,,,,,,,,,,12.899999618530272,5.300000190734863,6.536253452301025,85.9000015258789,806.0762329101562,868.0042114257812,,,1.578299880027771,1.5728015899658203,0.0,600.0,963.1144409179688,951.73193359375,0.0,10.191020965576172,0.0,11.601633071899414,4.3397626876831055,0.0748387202620506,71.49508666992188,100.0,,,,,,,,,51.03459548950195,68.00411987304688, diff --git a/response_data.csv b/response_data.csv deleted file mode 100644 index cd1d691..0000000 --- a/response_data.csv +++ /dev/null @@ -1,2 +0,0 @@ -,303-WIT-230_median,303-WIT-230_std,303-WIT-230_min,303-WIT-230_max,305-WIT-135_median,305-WIT-135_std,305-WIT-135_min,305-WIT-135_max,305-WIT-160_median,305-WIT-160_std,305-WIT-160_min,305-WIT-160_max,305-PIT-170_median,305-PIT-170_std,305-PIT-170_min,305-PIT-170_max,305-PIT-175_median,305-PIT-175_std,305-PIT-175_min,305-PIT-175_max,305-FIT-002_median,305-FIT-002_std,305-FIT-002_min,305-FIT-002_max,305-FIT-013_median,305-FIT-013_std,305-FIT-013_min,305-FIT-013_max,306-PIT-101_median,306-PIT-101_std,306-PIT-101_min,306-PIT-101_max,306-FIT-051_median,306-FIT-051_std,306-FIT-051_min,306-FIT-051_max,306-DIT-001_median,306-DIT-001_std,306-DIT-001_min,306-DIT-001_max,306-PIT-105_median,306-PIT-105_std,306-PIT-105_min,306-PIT-105_max,306-FIT-052_median,306-FIT-052_std,306-FIT-052_min,306-FIT-052_max,306-DIT-002_median,306-DIT-002_std,306-DIT-002_min,306-DIT-002_max,306-PIT-115_median,306-PIT-115_std,306-PIT-115_min,306-PIT-115_max,306-FIT-004_median,306-FIT-004_std,306-FIT-004_min,306-FIT-004_max,306-PIT-110_median,306-PIT-110_std,306-PIT-110_min,306-PIT-110_max,306-FIT-003_median,306-FIT-003_std,306-FIT-003_min,306-FIT-003_max,306-PIT-125_median,306-PIT-125_std,306-PIT-125_min,306-PIT-125_max,306-FIT-005_median,306-FIT-005_std,306-FIT-005_min,306-FIT-005_max,306-PIT-130_median,306-PIT-130_std,306-PIT-130_min,306-PIT-130_max,306-FIT-006_median,306-FIT-006_std,306-FIT-006_min,306-FIT-006_max,307-FIT-005_median,307-FIT-005_std,307-FIT-005_min,307-FIT-005_max,307-FIT-003_median,307-FIT-003_std,307-FIT-003_min,307-FIT-003_max,307-FIC-022_median,307-FIC-022_std,307-FIC-022_min,307-FIC-022_max,310-FIT-005_median,310-FIT-005_std,310-FIT-005_min,310-FIT-005_max,307-FIT-008_median,307-FIT-008_std,307-FIT-008_min,307-FIT-008_max,307-FIT-009_median,307-FIT-009_std,307-FIT-009_min,307-FIT-009_max,309-PIT-101_median,309-PIT-101_std,309-PIT-101_min,309-PIT-101_max,309-PIT-105_median,309-PIT-105_std,309-PIT-105_min,309-PIT-105_max,309-PIT-110_median,309-PIT-110_std,309-PIT-110_min,309-PIT-110_max,309-PIT-185_median,309-PIT-185_std,309-PIT-185_min,309-PIT-185_max,309-PIT-190_median,309-PIT-190_std,309-PIT-190_min,309-PIT-190_max,309-PIT-195_median,309-PIT-195_std,309-PIT-195_min,309-PIT-195_max,309-FIT-051_median,309-FIT-051_std,309-FIT-051_min,309-FIT-051_max,309-FIT-052_median,309-FIT-052_std,309-FIT-052_min,309-FIT-052_max,309-PIT-001_median,309-PIT-001_std,309-PIT-001_min,309-PIT-001_max,309-PIT-002_median,309-PIT-002_std,309-PIT-002_min,309-PIT-002_max,317AIT003.3_median,317AIT003.3_std,317AIT003.3_min,317AIT003.3_max,317AIT003.5_median,317AIT003.5_std,317AIT003.5_min,317AIT003.5_max,317AIT003.1_median,317AIT003.1_std,317AIT003.1_min,317AIT003.1_max,317AIT003.2_median,317AIT003.2_std,317AIT003.2_min,317AIT003.2_max,317AIT002.37_median,317AIT002.37_std,317AIT002.37_min,317AIT002.37_max,317AIT002.49_median,317AIT002.49_std,317AIT002.49_min,317AIT002.49_max,317AIT002.61_median,317AIT002.61_std,317AIT002.61_min,317AIT002.61_max,317AIT002.38_median,317AIT002.38_std,317AIT002.38_min,317AIT002.38_max,317AIT002.50_median,317AIT002.50_std,317AIT002.50_min,317AIT002.50_max,317AIT002.62_median,317AIT002.62_std,317AIT002.62_min,317AIT002.62_max,317AIT002.39_median,317AIT002.39_std,317AIT002.39_min,317AIT002.39_max,317AIT002.51_median,317AIT002.51_std,317AIT002.51_min,317AIT002.51_max,317AIT002.63_median,317AIT002.63_std,317AIT002.63_min,317AIT002.63_max,317AIT002.40_median,317AIT002.40_std,317AIT002.40_min,317AIT002.40_max,317AIT002.52_median,317AIT002.52_std,317AIT002.52_min,317AIT002.52_max,317AIT002.64_median,317AIT002.64_std,317AIT002.64_min,317AIT002.64_max,317AIT002.41_median,317AIT002.41_std,317AIT002.41_min,317AIT002.41_max,317AIT002.53_median,317AIT002.53_std,317AIT002.53_min,317AIT002.53_max,317AIT002.65_median,317AIT002.65_std,317AIT002.65_min,317AIT002.65_max,317AIT002.42_median,317AIT002.42_std,317AIT002.42_min,317AIT002.42_max,317AIT002.54_median,317AIT002.54_std,317AIT002.54_min,317AIT002.54_max,317AIT002.66_median,317AIT002.66_std,317AIT002.66_min,317AIT002.66_max,317AIT002.43_median,317AIT002.43_std,317AIT002.43_min,317AIT002.43_max,317AIT002.55_median,317AIT002.55_std,317AIT002.55_min,317AIT002.55_max,317AIT002.67_median,317AIT002.67_std,317AIT002.67_min,317AIT002.67_max,317AIT002.44_median,317AIT002.44_std,317AIT002.44_min,317AIT002.44_max,317AIT002.56_median,317AIT002.56_std,317AIT002.56_min,317AIT002.56_max,317AIT002.68_median,317AIT002.68_std,317AIT002.68_min,317AIT002.68_max,317AIT002.45_median,317AIT002.45_std,317AIT002.45_min,317AIT002.45_max,317AIT002.57_median,317AIT002.57_std,317AIT002.57_min,317AIT002.57_max,317AIT002.69_median,317AIT002.69_std,317AIT002.69_min,317AIT002.69_max,317AIT002.46_median,317AIT002.46_std,317AIT002.46_min,317AIT002.46_max,317AIT002.58_median,317AIT002.58_std,317AIT002.58_min,317AIT002.58_max,317AIT002.70_median,317AIT002.70_std,317AIT002.70_min,317AIT002.70_max,317AIT002.47_median,317AIT002.47_std,317AIT002.47_min,317AIT002.47_max,317AIT002.59_median,317AIT002.59_std,317AIT002.59_min,317AIT002.59_max,317AIT002.71_median,317AIT002.71_std,317AIT002.71_min,317AIT002.71_max,317AIT002.48_median,317AIT002.48_std,317AIT002.48_min,317AIT002.48_max,317AIT002.60_median,317AIT002.60_std,317AIT002.60_min,317AIT002.60_max,317AIT002.72_median,317AIT002.72_std,317AIT002.72_min,317AIT002.72_max,SiO2_conc,timestamp -2024-12-05 04:00:00+0000,2344.354736328125,347.11413476082527,2063.14990234375,3052.78662109375,1323.8238525390625,199.25517177489738,801.2099609375,1477.9354248046875,1239.42919921875,188.20232896348458,1067.496337890625,1611.220947265625,12.741495132446287,0.2925100433215222,12.14862060546875,13.035848617553713,13.396173477172852,0.2659367570480025,12.94904613494873,13.875475883483888,3156.99365234375,17.90496592640638,3128.458251953125,3188.196533203125,3328.472412109375,18.341293358560456,3293.60546875,3352.32080078125,34.44520568847656,0.10727337707066699,34.300880432128906,34.595943450927734,2251.281005859375,6.914512409267933,2241.09228515625,2259.91357421875,1.3857378959655762,0.001162128441763176,1.3840404748916626,1.3874353170394895,33.95878982543945,0.06905080560339776,33.905391693115234,34.107933044433594,2275.4189453125,8.067717393234235,2263.143798828125,2288.202880859375,1.397327542304993,0.0025866991350633178,1.3935494422912598,1.4011056423187256,31.01426696777344,2.797667558372099,24.58193588256836,34.75410461425781,2157.358642578125,30.47011040139319,2091.429931640625,2189.05615234375,32.01703643798828,0.027139770722953593,31.97739601135254,32.05667495727539,2059.924560546875,88.54574101606774,1862.8626708984373,2170.908447265625,43.65913391113281,0.09432217658823967,43.593544006347656,43.89741134643555,1866.9232177734373,0.05200646609213417,1866.84716796875,1866.9991455078125,44.08242416381836,0.04344513322059127,44.02417755126953,44.14066696166992,1714.9619140625,0.1883131118977155,1714.6802978515625,1715.2435302734375,27.0,0.0,27.0,27.0,0.7632204294204712,0.28421935480519783,0.6459924578666687,1.425487995147705,649.1517333984375,2.0424248177883793,647.1126708984375,653.2977294921875,700.892822265625,0.07040149596071192,700.7899780273438,700.9956665039062,1194.4361572265625,0.29912346849236254,1193.999267578125,1194.873046875,997.7520751953124,0.26607758363281586,997.3634643554688,998.1407470703124,23.87283706665039,0.040718881893752515,23.81103706359864,23.925472259521484,26.559722900390625,0.2379241176870002,26.20622062683105,26.91034507751465,26.35142517089844,0.203982336891631,26.06732177734375,26.710390090942383,23.415620803833008,0.11592992262134358,23.256576538085938,23.56723022460937,27.697071075439453,0.15510993222349723,27.35708808898925,27.844594955444336,27.72958755493164,0.19091973703305398,27.292091369628903,27.89141845703125,1748.264892578125,7.86294792652368,1742.1806640625,1768.509765625,1821.6829833984373,15.953009331205266,1809.867919921875,1854.5777587890625,0.1001370549201965,0.00019079441009214591,0.0998583808541297,0.1004157289862632,4.8986358642578125,0.0015045608215440351,4.8964385986328125,4.900833606719971,6.507819890975952,0.12657004614335984,6.405970096588135,6.699999809265137,86.30000305175781,0.3732080423947149,85.80000305175781,86.5999984741211,8.5,1.6421334224036257,7.89943265914917,12.899999618530272,5.400000095367432,1.0432607765452175,3.299999952316284,5.800000190734863,1.2756186723709106,0.0,1.2756186723709106,1.2756186723709106,0.7121588587760925,0.0,0.7121588587760925,0.7121588587760925,0.4104396104812622,0.0,0.4104396104812622,0.4104396104812622,0.2267737984657287,0.0,0.2267737984657287,0.2267737984657287,1.4764769077301023,0.0,1.4764769077301023,1.4764769077301023,0.6497865319252014,0.0,0.6497865319252014,0.6497865319252014,0.4203702211380005,0.0,0.4203702211380005,0.4203702211380005,1.526507019996643,0.0,1.526507019996643,1.526507019996643,0.6540470123291016,0.0,0.6540470123291016,0.6540470123291016,1.7459523677825928,0.002774316303229609,1.7381054162979126,1.7459523677825928,0.4817045927047729,0.0002177622295436636,0.4817045927047729,0.4823205173015594,0.3364990949630737,0.00012384851434926538,0.3361487984657287,0.3364990949630737,1.1931402683258057,0.033290363097006954,1.1206940412521362,1.1931402683258057,0.7161301374435425,0.025235254887373854,0.7161301374435425,0.7710468769073486,0.4127309918403625,0.007279004051801342,0.4127309918403625,0.428571492433548,1.707435429096222,0.0008729009039302258,1.7066189050674438,1.708251953125,0.3564415574073791,0.002030279951199197,0.3545424044132232,0.358340710401535,0.3023426830768585,0.000350906290820105,0.3020144402980804,0.3026709258556366,-9999.0,0.0,-9999.0,-9999.0,-9999.0,0.0,-9999.0,-9999.0,-9999.0,0.0,-9999.0,-9999.0,0.1042194217443466,8.294721469109035e-06,0.1042194217443466,0.1042373403906822,1.715789794921875,0.001402039007837862,1.715789794921875,1.7188185453414917,0.7188712954521179,0.0014949674798809447,0.7188712954521179,0.7221007943153381,1.970276951789856,0.018573843840262887,1.934388875961304,1.970276951789856,0.3608308732509613,0.004311563730231936,0.3608308732509613,0.3691616058349609,0.2941087782382965,0.0009687919419025886,0.2941087782382965,0.2959806621074676,0.4393351674079895,0.006024186034919744,0.4222961962223053,0.4393351674079895,1.1838626861572266,0.007396139710934239,1.1838626861572266,1.2047821283340454,0.5613290071487427,0.0019051429198136875,0.5613290071487427,0.5667175650596619,0.0910589918494224,0.0011538569058607675,0.0910589918494224,0.0943225920200347,1.6499500274658203,0.0059099153918944995,1.6332342624664309,1.6499500274658203,0.7046993374824524,0.0013654103777831785,0.7008373737335205,0.7046993374824524,0.0354578979313373,0.0,0.0354578979313373,0.0354578979313373,2.292947769165039,0.0,2.292947769165039,2.292947769165039,0.8771045207977295,0.0,0.8771045207977295,0.8771045207977295,3.52,2024-12-05 04:00:00 From a817d9855be149d6057880aabb97e02ea80455bb Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 15 Sep 2025 15:16:27 -0300 Subject: [PATCH 06/29] SIENTIAPDE-1222 Refactor MLFlow logging to improve data output clarity - Updated the debug logging to directly capture the output of data.to_csv, enhancing traceability of processed input data. - Removed redundant debug statements for raw response data to streamline logging and focus on essential information. --- laborious/activities/mlflow.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index d0d47de..a3dd54a 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -154,18 +154,15 @@ class MLFlow(BaseActivity): data.columns.name = None self.debug("Processed input data:", metadata) - data.to_csv('data.csv') - self.debug(data.to_string(), metadata) + self.debug(data.to_csv('data.csv'), metadata) # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( model_name, data, model_config ) - self.debug("Raw response data:", metadata) - self.debug(response_data, metadata) - if response_data['success']: + response_dataframe = DataFrame(response_data['content']) try: response_dataframe = self.detect_and_parse_datetime_index( From cc692e6dbd6b408b0531620d2da33dd1510f746b Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 07:52:21 -0300 Subject: [PATCH 07/29] SIENTIAPDE-1222 Update values.yaml and MLFlow logging for courier integration - Changed the image repository to 'sientia-module-courier' and updated the image tag to '0.0.1'. - Modified environment variables for GITHUB_BRANCH and MLFLOW_PASSWORD to reflect new configurations. - Enhanced MLFlow logging to include additional debug statements for raw response data and added a check for empty DataFrames. --- laborious/activities/mlflow.py | 5 +++++ values.yaml | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index a3dd54a..f11a397 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -161,9 +161,14 @@ class MLFlow(BaseActivity): model_name, data, model_config ) + self.debug("Transform raw response data:", metadata) + self.debug(response_data, metadata) + if response_data['success']: response_dataframe = DataFrame(response_data['content']) + if len(response_dataframe) == 0: + return response_data try: response_dataframe = self.detect_and_parse_datetime_index( response_dataframe, metadata) diff --git a/values.yaml b/values.yaml index 716254a..b2ae84e 100644 --- a/values.yaml +++ b/values.yaml @@ -7,11 +7,11 @@ replicaCount: 1 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: - repository: aignosi.azurecr.io/sientia-module + repository: aignosi.azurecr.io/sientia-module-courier # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.4.5" + tag: "0.0.1" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: @@ -151,7 +151,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1182-ajustar-laborious-para-pegar-timestamp-da-resposta-do-mlflow" + value: SIENTIAPDE-1222-ajustar-a-library-para-fazer-o-download-do-courier - name: PYTHON_APP value: "laborious.worker.worker" @@ -178,7 +178,7 @@ env: - name: MLFLOW_USERNAME value: "aignosi" - name: MLFLOW_PASSWORD - value: "aignosi" + value: "1L0FP50j3ncp123" - name: OPC_ID value: "1" From 6d10e4c59758fa5f89145f37b1e8e19b8cb93498 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 07:55:56 -0300 Subject: [PATCH 08/29] SIENTIAPDE-1222 Refactor MLFlow logging to enhance data output clarity - Updated debug logging to use data.to_string() for processed input data, improving readability. - Modified raw response data logging to format the output as a string, ensuring consistent logging format. --- laborious/activities/mlflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index f11a397..5f5b12c 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -154,7 +154,7 @@ class MLFlow(BaseActivity): data.columns.name = None self.debug("Processed input data:", metadata) - self.debug(data.to_csv('data.csv'), metadata) + self.debug(data.to_string(), metadata) # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( @@ -162,7 +162,7 @@ class MLFlow(BaseActivity): ) self.debug("Transform raw response data:", metadata) - self.debug(response_data, metadata) + self.debug(f"{response_data}", metadata) if response_data['success']: From 1a9c1a31c5b7cdfd225800908ebd3b506884c52e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 08:16:30 -0300 Subject: [PATCH 09/29] SIENTIAPDE-1222 SIENTIAPDE-1222 Enhance MLFlow logging with sample dictionary for response data - Introduced a new method to create a sample dictionary for debugging, allowing for better visualization of nested data structures in logs. - Updated debug logging to utilize the new sampling method for raw and transformed response data, improving clarity and reducing output size. - Adjusted logging for processed input data to display only the first few rows, enhancing readability. --- laborious/activities/mlflow.py | 48 +++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 5f5b12c..c11bbaa 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -17,6 +17,39 @@ with workflow.unsafe.imports_passed_through(): import traceback +def create_sample_dict(data: dict, max_items: int = 3, max_depth: int = 2) -> dict: + """ + Create a sample of a dictionary for debugging purposes. + + Args: + data: Dictionary to sample + max_items: Maximum number of items to show per level + max_depth: Maximum depth to traverse nested structures + + Returns: + Dictionary with sampled content + """ + if max_depth <= 0: + return {"...": "max_depth_reached"} + + sample = {} + items = list(data.items())[:max_items] + + for key, value in items: + if isinstance(value, dict): + sample[key] = create_sample_dict(value, max_items, max_depth - 1) + elif isinstance(value, list): + sample[key] = value[:max_items] if len( + value) > max_items else value + else: + sample[key] = value + + if len(data) > max_items: + sample["..."] = f"({len(data) - max_items} more items)" + + return sample + + class MLFlow(BaseActivity): """ MLFlow integration activities for model inference operations. @@ -138,7 +171,7 @@ class MLFlow(BaseActivity): model_config = input_data.get('model_config', {}) self.debug("Raw input data:", metadata) - self.debug(data, metadata) + self.debug(data.head(5).to_string(), metadata) # Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair data = data.sort_values('created_at', ascending=False).drop_duplicates( @@ -154,7 +187,7 @@ class MLFlow(BaseActivity): data.columns.name = None self.debug("Processed input data:", metadata) - self.debug(data.to_string(), metadata) + self.debug(data.head(5).to_string(), metadata) # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( @@ -162,7 +195,8 @@ class MLFlow(BaseActivity): ) self.debug("Transform raw response data:", metadata) - self.debug(f"{response_data}", metadata) + self.debug(create_sample_dict( + response_data), metadata) if response_data['success']: @@ -194,7 +228,8 @@ class MLFlow(BaseActivity): response_data['content'] = response_dataframe.to_dict() self.debug("Transform response data:", metadata) - self.debug(response_data, metadata) + self.debug(create_sample_dict( + response_data), metadata) self.info("Data transformed successfully", metadata) @@ -236,7 +271,7 @@ class MLFlow(BaseActivity): model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) - self.debug(data, metadata) + self.debug(data.head(5).to_string(), metadata) # Convert numpy.nan to None for model compatibility data.replace(np.nan, None, inplace=True) @@ -247,7 +282,8 @@ class MLFlow(BaseActivity): ) self.debug("Prediction response data:", metadata) - self.debug(json.dumps(response_data, indent=4), metadata) + self.debug(create_sample_dict( + response_data), metadata) self.info("Data predicted successfully", metadata) From 41af0a80328dad2f85fcd27ccf55a7b36cc4b811 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 08:54:27 -0300 Subject: [PATCH 10/29] SIENTIAPDE-1222 Update requirements and enhance logging in Gates and MLFlow activities - Updated the sientia-dataops-library and sientia-mlops-library dependencies in requirements.txt to the latest versions. - Improved debug logging in the Gates activity to display a sample of input data and filters, enhancing clarity and reducing output size. - Refactored MLFlow activity logging to utilize the create_sample_dict function for better visualization of nested data structures in logs. --- laborious/activities/gates.py | 18 ++++++++--------- laborious/activities/mlflow.py | 35 +--------------------------------- requirements.txt | 5 ++--- 3 files changed, 12 insertions(+), 46 deletions(-) diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index 34c26f1..cbebebe 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -8,6 +8,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.temporal.activities.base import BaseActivity from sientia_do.observability.logger import Logger from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now + from sientia_do.formatters import create_sample_dict from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter from typing import Any from laborious.utils.filters.conditional_filters import ( @@ -122,15 +123,13 @@ class Gates(BaseActivity): self.info("Performing input gate...", metadata) - self.debug(f"Input data: {input_data}", metadata) - filters = input_data['filters'] data = DataFrame(input_data['data']) path_priority = input_data['path_priority'] filter_output = [] - self.debug(f"Input data:\n {data}", metadata) + self.debug(f"Input data: {data.head(5).to_string()}", metadata) self.debug(f"Filters: {filters}", metadata) # Apply each configured filter @@ -207,7 +206,8 @@ class Gates(BaseActivity): filter_output = [] - self.debug(f"Input data:\n {data}", metadata) + self.debug(create_sample_dict( + data, max_items=5, max_depth=2), metadata) self.debug(f"Filters: {filters}", metadata) comments = [] @@ -292,8 +292,8 @@ class Gates(BaseActivity): filter_output = [] - self.debug(f"Input data:\n {data}", metadata) - self.debug(f"Filters: {filters}", metadata) + self.debug(f"Input data:\n {data.head(5).to_string()}", metadata) + self.debug(create_sample_dict(filters), metadata) for fil, config in filters.items(): if fil not in mlflow_content_filter_functions: @@ -410,7 +410,7 @@ class Gates(BaseActivity): self.debug( f"Prediction store policy: {prediction_store_policy}", metadata) - self.debug(f"Prediction data: {data.to_string()}", metadata) + self.debug(f"Prediction data: {data.head(5).to_string()}", metadata) policy_type, policy_value = self.get_prediction_store_policy( prediction_store_policy, metadata) @@ -445,7 +445,7 @@ class Gates(BaseActivity): data = data.reset_index(drop=True) self.info(f"Prediction formatted: {len(data)} rows", metadata) - self.debug(f"Prediction data: {data.to_string()}", metadata) + self.debug(f"Prediction data: {data.head(5).to_string()}", metadata) return data.to_dict() @@ -521,7 +521,7 @@ class Gates(BaseActivity): data = DataFrame(input_data['data']) - self.debug(f"Input data: {data.to_string()}", metadata) + self.debug(f"Input data: {data.head(5).to_string()}", metadata) if data.empty: return now().strftime(DATETIME_FORMAT_WITH_TZ) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index c11bbaa..5b449e0 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -3,13 +3,13 @@ from temporalio import activity, workflow with workflow.unsafe.imports_passed_through(): from datetime import datetime - import json from pandas import Timestamp, to_datetime from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ from sientia_do.temporal.activities.base import BaseActivity from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger + from sientia_do.formatters import create_sample_dict from laborious.utils.repository.model_repository import MLFlowRepository from typing import Any import numpy as np @@ -17,39 +17,6 @@ with workflow.unsafe.imports_passed_through(): import traceback -def create_sample_dict(data: dict, max_items: int = 3, max_depth: int = 2) -> dict: - """ - Create a sample of a dictionary for debugging purposes. - - Args: - data: Dictionary to sample - max_items: Maximum number of items to show per level - max_depth: Maximum depth to traverse nested structures - - Returns: - Dictionary with sampled content - """ - if max_depth <= 0: - return {"...": "max_depth_reached"} - - sample = {} - items = list(data.items())[:max_items] - - for key, value in items: - if isinstance(value, dict): - sample[key] = create_sample_dict(value, max_items, max_depth - 1) - elif isinstance(value, list): - sample[key] = value[:max_items] if len( - value) > max_items else value - else: - sample[key] = value - - if len(data) > max_items: - sample["..."] = f"({len(data) - max_items} more items)" - - return sample - - class MLFlow(BaseActivity): """ MLFlow integration activities for model inference operations. diff --git a/requirements.txt b/requirements.txt index 10995c8..1ce9f9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ psycopg2-binary sqlalchemy asyncua redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.5 -# git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.13 -/home/grezewave/Documents/projects/sientia/sientia-mlops-library +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.6 +git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.39.0 prometheus-client From eefd0815d8720263c05756eb3a5398bb1e30b0ba Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 10:22:51 -0300 Subject: [PATCH 11/29] SIENTIAPDE-1222 Update image tag in values.yaml and enhance debug logging in Gates and MLFlow activities - Updated the image tag in values.yaml from '0.0.1' to '0.0.2'. - Improved debug logging in the Gates activity to format input data and filters for better readability. - Enhanced MLFlow activity logging to include formatted output for raw and transformed response data, ensuring consistent logging format. --- laborious/activities/gates.py | 6 +++--- laborious/activities/mlflow.py | 15 ++++++--------- laborious/utils/repository/model_repository.py | 4 +++- values.yaml | 4 ++-- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index cbebebe..c88b59a 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -206,8 +206,8 @@ class Gates(BaseActivity): filter_output = [] - self.debug(create_sample_dict( - data, max_items=5, max_depth=2), metadata) + self.debug(f"Input data: \n {create_sample_dict( + data, max_items=5, max_depth=2)}", metadata) self.debug(f"Filters: {filters}", metadata) comments = [] @@ -293,7 +293,7 @@ class Gates(BaseActivity): filter_output = [] self.debug(f"Input data:\n {data.head(5).to_string()}", metadata) - self.debug(create_sample_dict(filters), metadata) + self.debug(f"Filters: \n {create_sample_dict(filters)}", metadata) for fil, config in filters.items(): if fil not in mlflow_content_filter_functions: diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 5b449e0..33d9aeb 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -161,9 +161,8 @@ class MLFlow(BaseActivity): model_name, data, model_config ) - self.debug("Transform raw response data:", metadata) - self.debug(create_sample_dict( - response_data), metadata) + self.debug(f"Transform raw response data: \n {create_sample_dict( + response_data)}", metadata) if response_data['success']: @@ -194,9 +193,8 @@ class MLFlow(BaseActivity): response_data['content'] = response_dataframe.to_dict() - self.debug("Transform response data:", metadata) - self.debug(create_sample_dict( - response_data), metadata) + self.debug(f"Transform response data: \n {create_sample_dict( + response_data)}", metadata) self.info("Data transformed successfully", metadata) @@ -248,9 +246,8 @@ class MLFlow(BaseActivity): model_name, data, model_config ) - self.debug("Prediction response data:", metadata) - self.debug(create_sample_dict( - response_data), metadata) + self.debug(f"Prediction response data: \n {create_sample_dict( + response_data)}", metadata) self.info("Data predicted successfully", metadata) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 77da23b..9d1cf64 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -24,6 +24,7 @@ class MLFlowRepository(): self.model_serving = ModelServing(tracking_uri=host, username=username, password=password, logger=logger) + self.logger = logger def transform(self, model_name: str, data: pd.DataFrame, model_config: dict) -> dict: """ @@ -90,7 +91,8 @@ class MLFlowRepository(): end_time = datetime.now() data = pd.DataFrame(data, columns=['prediction']) - self.logger.info(f"Data: {data.to_string()}") + self.logger.info( + f"Data received from model prediction: {data.to_string()}") data.index = input_index data['response_time'] = (end_time - start_time).total_seconds() diff --git a/values.yaml b/values.yaml index b2ae84e..4eb83d7 100644 --- a/values.yaml +++ b/values.yaml @@ -11,9 +11,9 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.0.1" + tag: "0.0.2" -# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +0# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: - name: docker-hub-secret # This is to override the chart name. From 0ffd96e734486dbdabedf284a3bf38bcea1348ed Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 10:29:35 -0300 Subject: [PATCH 12/29] SIENTIAPDE-1222 Refactor MLFlow debug logging for improved readability - Reformatted debug logging statements in the MLFlow activity to enhance clarity and consistency. - Ensured that the output of raw and transformed response data is presented in a more readable format, maintaining the use of create_sample_dict for better visualization. --- laborious/activities/mlflow.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 33d9aeb..e0e7cb3 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -161,8 +161,8 @@ class MLFlow(BaseActivity): model_name, data, model_config ) - self.debug(f"Transform raw response data: \n {create_sample_dict( - response_data)}", metadata) + self.debug( + f"Transform raw response data: \n {create_sample_dict(response_data)}", metadata) if response_data['success']: @@ -193,8 +193,8 @@ class MLFlow(BaseActivity): response_data['content'] = response_dataframe.to_dict() - self.debug(f"Transform response data: \n {create_sample_dict( - response_data)}", metadata) + self.debug( + f"Transform response data: \n {create_sample_dict(response_data)}", metadata) self.info("Data transformed successfully", metadata) @@ -246,8 +246,8 @@ class MLFlow(BaseActivity): model_name, data, model_config ) - self.debug(f"Prediction response data: \n {create_sample_dict( - response_data)}", metadata) + self.debug( + f"Prediction response data: \n {create_sample_dict(response_data)}", metadata) self.info("Data predicted successfully", metadata) From ce288c3926a827e138ddd3bba491d85ac2d84a78 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 10:31:30 -0300 Subject: [PATCH 13/29] SIENTIAPDE-1222 Refactor debug logging in Gates activity for improved readability - Reformatted the debug logging statement for input data in the Gates activity to enhance clarity and maintain consistency with previous logging improvements. --- laborious/activities/gates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index c88b59a..4a71455 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -206,8 +206,8 @@ class Gates(BaseActivity): filter_output = [] - self.debug(f"Input data: \n {create_sample_dict( - data, max_items=5, max_depth=2)}", metadata) + self.debug( + f"Input data: \n {create_sample_dict(data, max_items=5, max_depth=2)}", metadata) self.debug(f"Filters: {filters}", metadata) comments = [] From 7fb416313e48d877bd842fcaf7020747527c3ea4 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 10:36:43 -0300 Subject: [PATCH 14/29] SIENTIAPDE-1222 Enhance MLFlow debug logging to limit output size - Updated debug logging statements in the MLFlow activity to include a maximum of 5 items and a depth of 5 for the sample dictionary, improving readability and reducing log clutter for raw and transformed response data. --- laborious/activities/mlflow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index e0e7cb3..aae09ee 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -162,7 +162,7 @@ class MLFlow(BaseActivity): ) self.debug( - f"Transform raw response data: \n {create_sample_dict(response_data)}", metadata) + f"Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) if response_data['success']: @@ -194,7 +194,7 @@ class MLFlow(BaseActivity): response_data['content'] = response_dataframe.to_dict() self.debug( - f"Transform response data: \n {create_sample_dict(response_data)}", metadata) + f"Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) self.info("Data transformed successfully", metadata) @@ -247,7 +247,7 @@ class MLFlow(BaseActivity): ) self.debug( - f"Prediction response data: \n {create_sample_dict(response_data)}", metadata) + f"Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) self.info("Data predicted successfully", metadata) From 74f7e6c024e1fd0123e43a6f3983419d758c9f9f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 11:41:27 -0300 Subject: [PATCH 15/29] SIENTIAPDE-1222 Update prediction_store_policy handling in workflows - Added 'prediction_store_policy' to the input data handling in PredictionsBatch, ensuring a default value of 'lts:1' is used when not provided. - Modified FormatAndExportPrediction to directly use 'prediction_store_policy' from input_data, removing the default fallback. - Updated PredictionProcess to include 'prediction_store_policy' in the output data structure, ensuring consistency across workflows. --- laborious/workflows/predictions_batch.py | 4 +++- .../workflows/sub_workflows/format_and_export_prediction.py | 3 +-- laborious/workflows/sub_workflows/prediction_process.py | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/laborious/workflows/predictions_batch.py b/laborious/workflows/predictions_batch.py index ecce063..e522eaa 100644 --- a/laborious/workflows/predictions_batch.py +++ b/laborious/workflows/predictions_batch.py @@ -115,7 +115,9 @@ class PredictionsBatch(): }), 'model_config': input_data.get('model_config', {}), 'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']), - 'opc_output_config': input_data.get('opc_output_config', {}) + 'opc_output_config': input_data.get('opc_output_config', {}), + 'prediction_store_policy': input_data.get( + 'prediction_store_policy', 'lts:1') } # Execute prediction process workflow diff --git a/laborious/workflows/sub_workflows/format_and_export_prediction.py b/laborious/workflows/sub_workflows/format_and_export_prediction.py index ae3ccfb..8e7df07 100644 --- a/laborious/workflows/sub_workflows/format_and_export_prediction.py +++ b/laborious/workflows/sub_workflows/format_and_export_prediction.py @@ -79,8 +79,7 @@ class FormatAndExportPrediction(): 'timestamp': input_data['timestamp'], 'model_id': input_data['model_id'], 'prediction_confidence': prediction_confidence, - 'prediction_store_policy': input_data.get( - 'prediction_store_policy', 'lts:1') + 'prediction_store_policy': input_data['prediction_store_policy'] }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=60) diff --git a/laborious/workflows/sub_workflows/prediction_process.py b/laborious/workflows/sub_workflows/prediction_process.py index c958500..777fa1c 100644 --- a/laborious/workflows/sub_workflows/prediction_process.py +++ b/laborious/workflows/sub_workflows/prediction_process.py @@ -213,7 +213,8 @@ class PredictionProcess(): 'opc_output_config': input_data['opc_output_config'], 'schema': input_data['schema'], 'table_name': input_data['table_name'], - 'comment': comment + 'comment': comment, + 'prediction_store_policy': input_data['prediction_store_policy'] } ) @@ -286,7 +287,8 @@ class PredictionProcess(): 'schema': schema, 'table_name': table_name, 'comment': comment, - 'opc_output_config': input_data['opc_output_config'] + 'opc_output_config': input_data['opc_output_config'], + 'prediction_store_policy': input_data['prediction_store_policy'] } ) return True From b1951ea2c09214f2b250c31b237a08b87c49b04c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 11:54:35 -0300 Subject: [PATCH 16/29] SIENTIAPDE-1222 Enhance debug logging in MLFlowRepository to improve data traceability - Added a debug logging statement to capture received data for model prediction, improving visibility into input data. - Changed an existing info logging statement to debug level for consistency, ensuring all relevant data is logged at the appropriate level. --- laborious/utils/repository/model_repository.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 9d1cf64..18b4620 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -84,6 +84,9 @@ class MLFlowRepository(): input_index = data.index start_time = datetime.now() + + self.logger.debug( + f"Data received for model prediction: {data.to_string()}") data = self.model_serving.get_cached_predict( model_name, data, model_retention, flavor, compressed, retention_target @@ -91,7 +94,7 @@ class MLFlowRepository(): end_time = datetime.now() data = pd.DataFrame(data, columns=['prediction']) - self.logger.info( + self.logger.debug( f"Data received from model prediction: {data.to_string()}") data.index = input_index data['response_time'] = (end_time - start_time).total_seconds() From 2b45c1cbd43c000b105f18f5c535e33b1924127f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 12:34:56 -0300 Subject: [PATCH 17/29] SIENTIAPDE-1222 Update model configuration keys in MLFlowRepository for consistency - Changed 'model_retention' to 'retention_minutes' and 'is_compressed' to 'compressed' in model configuration handling, ensuring alignment with updated configuration standards. --- laborious/utils/repository/model_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 18b4620..d1021b9 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -40,9 +40,9 @@ class MLFlowRepository(): """ try: - model_retention = model_config.get('model_retention', 0) + model_retention = model_config.get('retention_minutes', 0) flavor = model_config.get('transform_flavor', 'sklearn') - compressed = model_config.get('is_compressed', False) + compressed = model_config.get('compressed', False) retention_target = model_config.get('retention_target', 'model') transform_keyword = model_config.get( 'transform_function_keyword', 'predict') From a5fc526f616d774cbc2d71488827d1fdd778a895 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 13:04:43 -0300 Subject: [PATCH 18/29] SIENTIAPDE-1222 Update model configuration key for compression in MLFlowRepository - Changed the key for compression from 'compressed' to 'is_compressed' in model configuration handling to ensure consistency with updated standards. --- laborious/utils/repository/model_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index d1021b9..0145554 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -42,7 +42,7 @@ class MLFlowRepository(): try: model_retention = model_config.get('retention_minutes', 0) flavor = model_config.get('transform_flavor', 'sklearn') - compressed = model_config.get('compressed', False) + compressed = model_config.get('is_compressed', False) retention_target = model_config.get('retention_target', 'model') transform_keyword = model_config.get( 'transform_function_keyword', 'predict') @@ -79,7 +79,7 @@ class MLFlowRepository(): try: model_retention = model_config.get('retention_minutes', 0) flavor = model_config.get('predict_flavor', 'pyfunc') - compressed = model_config.get('compressed', False) + compressed = model_config.get('is_compressed', False) retention_target = model_config.get('retention_target', 'model') input_index = data.index From 25bc4d06ca83c9b239a4c8cb8570168daeacfb45 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 17:07:44 -0300 Subject: [PATCH 19/29] SIENTIAPDE-1222 Update README.md to reflect new features and configuration changes - Added details about two dedicated task queues: `predictions_batch-queue` and `minimal_retrain-queue`. - Enhanced descriptions of activities and workflows, including multiple inheritance patterns and configurable MLFlow model serving. - Updated monitoring metrics section to include new labels and metrics for prediction and OPC export operations. - Revised configuration section with updated default values and added new environment variables for Kubernetes pod identification. - Improved clarity in the Predictions Batch Workflow configuration example, including structured input filters and updated retention policies. --- README.md | 200 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 115 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 330cbb4..500153c 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa - Health check endpoints for Kubernetes liveness/readiness probes - Graceful shutdown with cleanup procedures - Multi-instance deployment support + - Two dedicated task queues: `predictions_batch-queue` and `minimal_retrain-queue` #### **Workflows (`laborious/workflows/`)** - **PredictionsBatch**: Main entry point for batch prediction pipelines @@ -79,25 +80,32 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa - Configurable timeout and retry strategies #### **Activities (`laborious/activities/`)** +- **Activities**: Main activity orchestrator combining all functionality through multiple inheritance - **Gates**: Data quality validation and filtering mechanisms - **MLFlow**: Model transformation and prediction operations - **OPC**: Real-time data export to industrial OPC servers -- **Activities**: Main activity orchestrator and coordination - **Key Features**: + - Multiple inheritance pattern for unified activity interface - Configurable filter policies and validation rules - - MLFlow model serving integration + - MLFlow model serving integration with configurable flavors - OPC UA client with certificate-based authentication - - Comprehensive error handling and notification + - Comprehensive error handling and notification integration + - Support for multiple OPC servers with independent configurations #### **Data Services (`laborious/utils/`)** -- **Connectors**: Database and external service configuration management +- **Connectors Config**: Environment variable-based configuration management - **Repository**: Data access layer for MLFlow and OPC operations + - `model_repository.py`: MLFlow model operations and retraining + - `opc_repository.py`: OPC server communication and data writing - **Filters**: Data quality validation and MLFlow response filtering + - `conditional_filters.py`: Input data validation filters + - `mlflow_filters.py`: MLFlow API response validation filters - **Key Features**: - - Environment variable-based configuration + - Environment variable-based configuration with sensible defaults - Connection pool management and optimization - Security credential management - Configuration validation and error handling + - Support for multiple OPC servers and MLFlow model flavors ### Data Flow Architecture @@ -506,23 +514,32 @@ pytest tests/workflow/test_predictions_batch.py ## 📊 Monitoring and Metrics -The Laborious system exposes comprehensive Prometheus metrics: +The Laborious system exposes comprehensive Prometheus metrics for operational visibility and performance monitoring: -### Application Metrics +### Application Health Metrics - `app_up`: Application health status (1=healthy, 0=unhealthy) -- `laborious_predictions_written_count`: Prediction export operation count -- `laborious_prediction_confidence_monitor`: Prediction confidence monitoring -- `laborious_prediction_response_time_monitor`: Prediction response time monitoring + - Labels: `pod_id` -### MLFlow Metrics -- Model transformation and prediction success rates -- API response times and error rates -- Model retention and versioning metrics +### Prediction Operation Metrics +- `laborious_predictions_written_count`: Counter for successful prediction exports + - Labels: `pod_id`, `model_name`, `pipeline_name` +- `laborious_prediction_confidence_monitor`: Gauge for current prediction confidence levels + - Labels: `pod_id`, `model_name`, `pipeline_name` +- `laborious_prediction_response_time_monitor`: Histogram for prediction response times + - Labels: `pod_id`, `model_name`, `pipeline_name` + - Buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0] -### Export Metrics -- PostgreSQL export operation counts and response times -- OPC server write operations and performance -- Data quality filter pass/fail rates +### OPC Export Metrics +- `laborious_prediction_opc_writing_count`: Counter for OPC server write operations + - Labels: `pod_id`, `model_name`, `pipeline_name`, `opc_server_id` +- `laborious_prediction_opc_writing_response_time_monitor`: Histogram for OPC write response times + - Labels: `pod_id`, `model_name`, `pipeline_name`, `opc_server_id` + - Buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0] + +### Data Quality Metrics +- Filter pass/fail rates through notification system +- MLFlow API response validation metrics +- Data quality gate performance tracking ## ⚙️ Configuration @@ -537,31 +554,30 @@ The Laborious system exposes comprehensive Prometheus metrics: | `POSTGRES_USER` | PostgreSQL username | `sientia` | Yes | | `POSTGRES_PASSWORD` | PostgreSQL password | `sientia` | Yes | | `POSTGRES_DBNAME` | PostgreSQL database | `sientia` | Yes | -| `POSTGRES_MIN_CONNECTIONS` | Minimum PostgreSQL connections | `10` | No | -| `POSTGRES_MAX_CONNECTIONS` | Maximum PostgreSQL connections | `30` | No | -| `MLFLOW_HOST` | MLFlow server hostname | `localhost` | Yes | -| `MLFLOW_PORT` | MLFlow server port | `5000` | Yes | -| `MLFLOW_USERNAME` | MLFlow username | `admin` | Yes | -| `MLFLOW_PASSWORD` | MLFlow password | `admin` | Yes | +| `POSTGRES_MIN_CONNECTIONS` | Minimum PostgreSQL connections | `5` | No | +| `POSTGRES_MAX_CONNECTIONS` | Maximum PostgreSQL connections | `20` | No | +| `MLFLOW_HOST` | MLFlow server hostname | `http://localhost` | Yes | +| `MLFLOW_PORT` | MLFlow server port | `5080` | Yes | +| `MLFLOW_USERNAME` | MLFlow username | `aignosi` | Yes | +| `MLFLOW_PASSWORD` | MLFlow password | `aignosi` | Yes | | `OPC_CONFIG` | OPC server configuration (JSON) | `{}` | No | | `OPC_ID` | OPC server identifier | `1` | No | | `OPC_URL` | OPC server URL | `opc.tcp://localhost:4840` | No | -| `OPC_NAME` | OPC server name | `OPC_Server` | No | -| `OPC_SERVER_URI` | OPC server URI | `urn:opcserver:opcua` | No | -| `OPC_CERT_PATH` | OPC client certificate path | `/path/to/cert.pem` | No | -| `OPC_PRIVATE_KEY_PATH` | OPC private key path | `/path/to/key.pem` | No | -| `OPC_SERVER_CERT_PATH` | OPC server certificate path | `/path/to/server_cert.pem` | No | -| `OPC_RECONNECTION_INTERVAL` | OPC reconnection interval (ms) | `5000` | No | -| `MONGODB_URL` | MongoDB connection URI | `localhost:27017` | Yes | +| `OPC_SERVER_URI` | OPC server URI | `opc.tcp://localhost:4840` | No | +| `OPC_CERT_PATH` | OPC client certificate path | `None` | No | +| `OPC_PRIVATE_KEY_PATH` | OPC private key path | `None` | No | +| `OPC_SERVER_CERT_PATH` | OPC server certificate path | `None` | No | +| `OPC_RECONNECTION_INTERVAL` | OPC reconnection interval (ms) | `120` | No | +| `MONGODB_URL` | MongoDB connection URI | `localhost:27018` | Yes | | `MONGODB_USERNAME` | MongoDB username | `root` | Yes | -| `MONGODB_PASSWORD` | MongoDB password | `password` | Yes | -| `MONGODB_DATABASE` | MongoDB database name | `sientia` | Yes | +| `MONGODB_PASSWORD` | MongoDB password | `wKZDbMNU1c` | Yes | +| `MONGODB_DATABASE_NAME` | MongoDB database name | `sientia` | Yes | | `MONGODB_TTL_INDEX_HOURS` | MongoDB TTL index hours | `1` | No | -| `KAFKA_BOOTSTRAP_SERVERS` | Kafka bootstrap servers | `localhost:9092` | No | | `LOG_LEVEL` | Application log level | `INFO` | No | -| `PROJECT_NAME` | Project name for metrics | `sientia-laborious` | No | +| `PROJECT_NAME` | Project name for metrics | `laborious` | No | | `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No | | `HTTP_SDK_METRICS_PORT` | Temporal SDK metrics port | `9091` | No | +| `POD_ID` | Kubernetes pod identifier | `None` | No | @@ -606,74 +622,88 @@ For single OPC server, use individual environment variables: MongoDB pipeline configuration: -#### Predictions Batch Workflow +#### Predictions Batch Workflow configuration sample + +This is the configuration for the Predictions Batch Workflow, to be inserted into the MongoDB pipeline collection. + ```json { "schedule_name": "laborious-orchestrated-pipeline", "model_id": "1", "workflow_type": "predictions_batch", - "frequency": "30s", # Workflow execution frequency - "max_retry_policy": 1, # Maximum number of retries for the workflow + "frequency": "30s", + "max_retry_policy": 1, "query": "select * from sientia_data.laborious_data where model_id = 1 and \"timestamp\" > NOW() - INTERVAL '5 minutes' order by \"timestamp\" desc limit 30;", - "retention_time": 60, # Retention time for models in minutes "write_tags": [ { - "server_id": "1", - "type": "prediction", # Type of tag to write, can be prediction or confidence + "server_id": "server1", + "type": "prediction", "addr": "ns=2;i=5", "data_type": "double" }, { - "server_id": "1", + "server_id": "server1", "type": "confidence", - "addr": "ns=2;i=5", + "addr": "ns=2;i=6", "data_type": "double" } ], - "input_filters": [ - { - "filter_name": "EMPTY_DATA", # Required filter - "policy": "STOP" - }, - { - "filter_name": "SPECIFIC_VARIABLES_NULL_VALUES", - "policy": "CONTINUE", - "config": { - "variables": [ - "Counter" - ] - } + "input_filters": { + "EMPTY_DATA": {"POLICY": "STOP"}, + "SPECIFIC_VARIABLES_NULL_VALUES": { + "POLICY": "CONTINUE", + "config": {"variables": ["Counter"]} } - ], - "mlflow_transform_filters": [ - { - "filter_name": "API_ERROR", # Required filter - "policy": "REPEAT" - }, - { - "filter_name": "NAN_VALUES", - "policy": "STOP" - } - ], - "mlflow_predict_filters": [ - { - "filter_name": "API_ERROR", # Required filter - "policy": "CONTINUE" - } - ], - "path_priority": [ # In case of multiple filters catch problems, this will determine the path to take - "STOP", - "CONTINUE", - "REPEAT" - ], + }, + "mlflow_transform_filters": { + "API_ERROR": {"POLICY": "REPEAT"}, + "NAN_VALUES": {"POLICY": "STOP"} + }, + "mlflow_predict_filters": { + "API_ERROR": {"POLICY": "CONTINUE"} + }, + "path_priority": ["STOP", "CONTINUE", "REPEAT"], "active": true, - "datetime_columns": [ # Columns in data comming from query that are datetime - "timestamp", - "created_at" - ], "updated_at": { - "$date": "2025-08-27T18:35:01.600Z" - } + "$date": "2025-09-16T10:00:00.000Z" + }, + "datetime_columns": ["timestamp", "created_at"], + "predictions_storage_policy": "lts:1" +} +``` + +This is the configuration created by the Orchestrator in Temporal. + +```json +{ + "datetime_columns":["timestamp","created_at"], + "frequency":"15m", + "input_filters":{"EMPTY_DATA":{"config":{},"policy":"STOP"}}, + "max_retry_policy":1, + "mlflow_predict_filters":{"API_ERROR":{"config":{},"policy":"CONTINUE"}}, + "mlflow_transform_filters":{ + "API_ERROR":{"config":{},"policy":"CONTINUE"}, + "EMPTY_DATA":{"config":{},"policy":"STOP"} + }, + "model_config":{ + "is_compressed":true, + "predict_flavor":"pyfunc", + "retention_minutes":60, + "retention_target":"artifact", + "transform_function_keyword":"transform" + }, + "model_id":"352", + "model_name":"courier", + "opc_output_config":{}, + "path_priority":["STOP","CONTINUE","REPEAT"], + "predictions_storage_policy":"lts:1", + "query":"select * from sientia_data.laborious_data where model_id = 352 order by \"timestamp\" desc limit 300;", + "retention_time":3600, + "schedule_name":"laborious-courier", + "schema":"sientia_data", + "table_name":"predictions", + "updated_at":"2025-09-12 19:35:01.600000+0000", + "workflow_type":"predictions_batch" } ``` @@ -687,7 +717,7 @@ laborious/ │ ├── gates.py # Data quality gates and filtering │ ├── mlflow.py # MLFlow model operations │ └── opc.py # OPC server operations -├── workflow/ # Temporal workflow definitions +├── workflows/ # Temporal workflow definitions │ ├── predictions_batch.py # Main batch prediction workflow │ ├── minimal_retrain.py # Model retraining workflow │ └── sub_workflows/ # Sub-workflow implementations From 2075b302439b4350ebb95fcae926c760e8350ac0 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 08:59:12 -0300 Subject: [PATCH 20/29] SIENTIAPDE-1222 Refactor datetime index handling in MLFlow and MLFlowRepository - Moved the detect_and_parse_datetime_index method from MLFlow to MLFlowRepository for better organization and reusability. - Updated the method to include enhanced logging and error handling for invalid datetime formats. - Adjusted the transform method in MLFlowRepository to utilize the new datetime index parsing logic. - Added unit tests for both valid and invalid datetime index cases to ensure robustness. --- laborious/activities/mlflow.py | 67 ----------- .../utils/repository/model_repository.py | 73 ++++++++++-- tests/laborious/activities/test_mlflow.py | 7 +- .../utils/repository/test_model_repository.py | 109 +++++++++++++++++- 4 files changed, 169 insertions(+), 87 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index aae09ee..896f05a 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -63,44 +63,6 @@ class MLFlow(BaseActivity): f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger.base_logger ) - def detect_and_parse_datetime_index(self, data: DataFrame, metadata: dict) -> DataFrame: - """ - Detect and parse datetime index from data. index must be a timestamp like column. - This function must detect the timestamp type (pandas Timestamp or datetime) and convert it to DATETIME_FORMAT_WITH_TZ. - If the index is a string, must be in format DATETIME_FORMAT_WITH_TZ. - If another type or format, must raise an error. - """ - index = data.index - - # Get type of first element of index - index_type = type(index[0]) - - self.info(f"Index type: {index_type}", metadata) - - message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}" - - # Check if all in index are of the same type - if not all(isinstance(i, index_type) for i in index): - raise ValueError( - f"{message}") - - # Check type and converts to DATETIME_FORMAT_WITH_TZ - if index_type == str: - # Validate format of string and return error if not valid - try: - to_datetime(data.index) - except ValueError: - raise ValueError( - f"{message}") - - elif index_type == datetime or index_type == Timestamp: - data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) - else: - raise ValueError( - f"{message}") - - return data - @activity.defn(name="request_transform") async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]: """ @@ -164,35 +126,6 @@ class MLFlow(BaseActivity): self.debug( f"Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) - if response_data['success']: - - response_dataframe = DataFrame(response_data['content']) - if len(response_dataframe) == 0: - return response_data - try: - response_dataframe = self.detect_and_parse_datetime_index( - response_dataframe, metadata) - response_dataframe['timestamp'] = to_datetime( - response_dataframe.index, format=DATETIME_FORMAT_WITH_TZ) - response_dataframe['timestamp'] = response_dataframe['timestamp'].dt.strftime( - DATETIME_FORMAT) - except ValueError as e: - trace = traceback.format_exc() - self.send_notification( - metadata=metadata, - notification_id='TRANSFORM_DATA_INDEX_ERROR', - message=f'Error parsing trasnformed data index: {e}', - block='transform', - level=NotificationLevel.ERROR, - attachment_content=trace - ) - self.error(trace, metadata=metadata) - raise e - - response_dataframe.to_csv('response_data.csv') - - response_data['content'] = response_dataframe.to_dict() - self.debug( f"Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 0145554..3107d1c 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -16,17 +16,57 @@ import pandas as pd import mlflow from os import makedirs, path, remove from sientia.ModelServing import ModelServing +from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ +from sientia_do.observability.logger import Logger class MLFlowRepository(): - def __init__(self, host, username, password, logger): + def __init__(self, host, username, password, logger: Logger): self.model_serving = ModelServing(tracking_uri=host, username=username, password=password, logger=logger) self.logger = logger - def transform(self, model_name: str, data: pd.DataFrame, model_config: dict) -> dict: + def detect_and_parse_datetime_index(self, data: pd.DataFrame, metadata: dict) -> pd.DataFrame: + """ + Detect and parse datetime index from data. index must be a timestamp like column. + This function must detect the timestamp type (pandas Timestamp or datetime) and convert it to DATETIME_FORMAT_WITH_TZ. + If the index is a string, must be in format DATETIME_FORMAT_WITH_TZ. + If another type or format, must raise an error. + """ + index = data.index + + # Get type of first element of index + index_type = type(index[0]) + + self.logger.custom_info(f"Index type: {index_type}", metadata) + + message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}" + + # Check if all in index are of the same type + if not all(isinstance(i, index_type) for i in index): + raise ValueError( + f"{message}") + + # Check type and converts to DATETIME_FORMAT_WITH_TZ + if index_type == str: + # Validate format of string and return error if not valid + try: + pd.to_datetime(data.index, format=DATETIME_FORMAT_WITH_TZ) + except ValueError: + raise ValueError( + f"{message}") + + elif index_type == datetime or index_type == pd.Timestamp: + data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) + else: + raise ValueError( + f"{message}") + + return data + + def transform(self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict) -> dict: """ Transform data using a model. @@ -40,6 +80,9 @@ class MLFlowRepository(): """ try: + self.logger.custom_debug( + f"Data received for model transformation: {data.to_csv()}", metadata) + model_retention = model_config.get('retention_minutes', 0) flavor = model_config.get('transform_flavor', 'sklearn') compressed = model_config.get('is_compressed', False) @@ -47,12 +90,20 @@ class MLFlowRepository(): transform_keyword = model_config.get( 'transform_function_keyword', 'predict') + transformed_data = self.model_serving.get_cached_transform( + model_name, data, model_retention, flavor, + compressed, retention_target, transform_keyword + ) + + self.logger.custom_debug( + f"Data received from model transformation: {transformed_data.to_csv()}", metadata) + + transformed_data = self.detect_and_parse_datetime_index( + transformed_data, metadata) + return { 'success': True, - 'content': self.model_serving.get_cached_transform( - model_name, data, model_retention, flavor, - compressed, retention_target, transform_keyword - ).to_dict() + 'content': transformed_data.to_dict() } except Exception as e: @@ -64,7 +115,7 @@ class MLFlowRepository(): } } - def predict(self, model_name: str, data: pd.DataFrame, model_config: dict) -> dict: + def predict(self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict) -> dict: """ Predict data using a model. @@ -85,8 +136,8 @@ class MLFlowRepository(): input_index = data.index start_time = datetime.now() - self.logger.debug( - f"Data received for model prediction: {data.to_string()}") + self.logger.custom_debug( + f"Data received for model prediction: {data.to_csv()}", metadata) data = self.model_serving.get_cached_predict( model_name, data, model_retention, flavor, compressed, retention_target @@ -94,8 +145,8 @@ class MLFlowRepository(): end_time = datetime.now() data = pd.DataFrame(data, columns=['prediction']) - self.logger.debug( - f"Data received from model prediction: {data.to_string()}") + self.logger.custom_debug( + f"Data received from model prediction: {data.to_csv()}", metadata) data.index = input_index data['response_time'] = (end_time - start_time).total_seconds() diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index 7aced7c..ed68729 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -1,8 +1,9 @@ +from datetime import datetime from unittest.mock import ANY, MagicMock, patch import numpy as np -from pandas import DataFrame -from pytest import fixture, mark +from pandas import DataFrame, Timestamp +from pytest import fixture, mark, raises from laborious.activities.mlflow import MLFlow from sientia_do.notifications.models import NotificationLevel @@ -58,7 +59,7 @@ metadata = { @mark.asyncio @patch("laborious.activities.mlflow.DataFrame") @patch("laborious.activities.mlflow.max") -async def test_request_transform(mock_max, mock_dataframe, mlflow): +async def test_request_transform_success(mock_max, mock_dataframe, mlflow): mock_max.return_value = '2024-01-02' # Mock input data input_data = { diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 8b3716e..51a3223 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -2,7 +2,8 @@ from unittest.mock import ANY, MagicMock, call, patch import numpy as np from pandas import DataFrame import pytest -from laborious.utils.repository import model_repository +from datetime import datetime, timezone +from pandas import Timestamp from laborious.utils.repository.model_repository import MLFlowRepository @@ -22,18 +23,113 @@ def mlflow_repository(): return repo +metadata = { + "metadata": { + "model_id": "test_model", + "model_name": "test_model", + "workflow_name": "test_workflow", + "schema_name": "test_schedule", + }, +} + + +invalid_cases = [ + ( + { + 'value': { + '2024-01-01 12:00:00': 1, + 2024: 2 + } + } + ), + ( + { + 'value': { + '2024-01-01': 1, + '2024-01-02': 2 + } + } + ), + ( + { + 'value': { + 1: 1, + 2: 2 + } + } + ) +] + + +@pytest.mark.parametrize("data", invalid_cases) +def test_detect_and_parse_datetime_index_error_cases(mlflow_repository, data): + input_data = DataFrame( + data + ) + + with pytest.raises(ValueError) as e: + mlflow_repository.detect_and_parse_datetime_index( + input_data, metadata['metadata']) + + assert str(e) == "Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format %Y-%m-%d %H:%M:%S" + + +valid_cases = [ + ( + { + 'value': { + '2024-01-01 12:00:00+0000': 1, + '2024-01-02 12:00:00+0000': 2 + } + }, ['2024-01-01 12:00:00+0000', '2024-01-02 12:00:00+0000'] + ), + ( + { + 'value': { + datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc): 1, + datetime(2025, 1, 2, 12, 0, 0, tzinfo=timezone.utc): 2 + } + }, ['2025-01-01 12:00:00+0000', '2025-01-02 12:00:00+0000'] + ), + ( + { + 'value': { + Timestamp(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc): 1, + Timestamp(2026, 1, 2, 12, 0, 0, tzinfo=timezone.utc): 2 + } + }, ['2026-01-01 12:00:00+0000', '2026-01-02 12:00:00+0000'] + ), +] + + +@pytest.mark.parametrize("data,expected", valid_cases) +def test_detect_and_parse_datetime_index_valid_format(mlflow_repository, data, expected): + input_data = DataFrame(data) + + response = mlflow_repository.detect_and_parse_datetime_index( + input_data, metadata['metadata']) + + assert response.index.tolist() == expected + + def test_transform_success(mlflow_repository): data = 'data' model_name = 'model' - output = mlflow_repository.transform(model_name, data, 1) + mlflow_repository.detect_and_parse_datetime_index = MagicMock() + + output = mlflow_repository.transform( + model_name, data, {}, metadata['metadata']) mlflow_repository.model_serving.get_cached_transform.assert_called_once_with( - model_name, data, 1) + model_name, data, 0, 'sklearn', False, 'model', 'predict') + + mlflow_repository.detect_and_parse_datetime_index.assert_called_once_with( + mlflow_repository.model_serving.get_cached_transform.return_value, metadata['metadata']) assert output == { 'success': True, - 'content': mlflow_repository.model_serving.get_cached_transform.return_value.to_dict.return_value + 'content': mlflow_repository.detect_and_parse_datetime_index.return_value.to_dict.return_value } @@ -44,10 +140,11 @@ def test_transform_error(mlflow_repository): mlflow_repository.model_serving.get_cached_transform.side_effect = Exception( 'error') - output = mlflow_repository.transform(model_name, data, 1) + output = mlflow_repository.transform( + model_name, data, {}, metadata['metadata']) mlflow_repository.model_serving.get_cached_transform.assert_called_once_with( - model_name, data, 1) + model_name, data, 0, 'sklearn', False, 'model', 'predict') assert output == { 'success': False, From b2ef523c3bd5a6019803ec06aa722ac8ca014ace Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 10:07:32 -0300 Subject: [PATCH 21/29] SIENTIAPDE-1222 Update test cases in model repository and prediction process - Replaced string data with MagicMock in test_transform_success and test_transform_error to improve test isolation. - Updated the predict method calls in test_predict_success and test_predict_error to reflect changes in argument structure. - Added 'prediction_store_policy' to the test_run configuration in test_prediction_process for consistency with recent updates. --- .../utils/repository/test_model_repository.py | 21 ++++++++++++------- .../subworkflows/test_prediction_process.py | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 51a3223..5dab80d 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -113,7 +113,7 @@ def test_detect_and_parse_datetime_index_valid_format(mlflow_repository, data, e def test_transform_success(mlflow_repository): - data = 'data' + data = MagicMock() model_name = 'model' mlflow_repository.detect_and_parse_datetime_index = MagicMock() @@ -134,7 +134,7 @@ def test_transform_success(mlflow_repository): def test_transform_error(mlflow_repository): - data = 'data' + data = MagicMock() model_name = 'model' mlflow_repository.model_serving.get_cached_transform.side_effect = Exception( @@ -167,10 +167,11 @@ def test_predict_success(mlflow_repository): [2, 3] ) - output = mlflow_repository.predict(model_name, data, 1) + output = mlflow_repository.predict( + model_name, data, {}, metadata['metadata']) mlflow_repository.model_serving.get_cached_predict.assert_called_once_with( - model_name, data, 1) + model_name, data, 0, 'pyfunc', False, 'model') assert output['success'] is True assert output['content'] == { @@ -185,17 +186,23 @@ def test_predict_success(mlflow_repository): def test_predict_error(mlflow_repository): - data = 'data' + data = DataFrame({ + 'feat_1': { + 'index_1': 2, + 'index_2': 3 + } + }) model_name = 'model' mlflow_repository.model_serving.get_cached_predict = MagicMock( side_effect=Exception('error') ) - output = mlflow_repository.predict(model_name, data, 1) + output = mlflow_repository.predict( + model_name, data, {}, metadata['metadata']) mlflow_repository.model_serving.get_cached_predict.assert_called_once_with( - model_name, data, 1) + model_name, data, 0, 'pyfunc', False, 'model') assert output == { 'success': False, diff --git a/tests/laborious/workflows/subworkflows/test_prediction_process.py b/tests/laborious/workflows/subworkflows/test_prediction_process.py index e76b586..eea8c53 100644 --- a/tests/laborious/workflows/subworkflows/test_prediction_process.py +++ b/tests/laborious/workflows/subworkflows/test_prediction_process.py @@ -37,6 +37,7 @@ async def test_run(workflow_mock, prediction_process): 'model_retention': '30', 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': 'lts:1' } # Mock the activity responses From 863bbdcc5713f68fb566f6f0b558b665cb5fa2cf Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 10:11:54 -0300 Subject: [PATCH 22/29] SIENTIAPDE-1222 SIENTIAPDE-1222 Update MLFlow methods to include metadata parameter - Modified the transform and predict method calls in the MLFlow class to include a new 'metadata' parameter, enhancing the functionality and data handling capabilities of the model monitoring repository. --- laborious/activities/mlflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 896f05a..28086d5 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -120,7 +120,7 @@ class MLFlow(BaseActivity): # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( - model_name, data, model_config + model_name, data, model_config, metadata ) self.debug( @@ -176,7 +176,7 @@ class MLFlow(BaseActivity): # Request prediction from MLFlow model response_data = self.model_monitoring_repository.predict( - model_name, data, model_config + model_name, data, model_config, metadata ) self.debug( From fc3c4ebb45161fd3a4eb7ba7ff6966c3e712bdaf Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 10:27:02 -0300 Subject: [PATCH 23/29] SIENTIAPDE-1222 Update MLFlow class to pass logger instance directly to MLFlowRepository - Modified the initialization of MLFlowRepository in the MLFlow class to pass the logger instance directly, improving logging capabilities and consistency across the application. --- laborious/activities/mlflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 28086d5..b96cc60 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -60,7 +60,7 @@ class MLFlow(BaseActivity): self.mlflow_password = mlflow_password self.model_monitoring_repository = MLFlowRepository( - f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger.base_logger + f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger ) @activity.defn(name="request_transform") From d6fdbc58bbc7465318f74e21d992660553bc8f6d Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 11:15:05 -0300 Subject: [PATCH 24/29] SIENTIAPDE-1222 Enhance MLFlow data handling by adding timestamp column and improving debug logging - Added a 'timestamp' column to the input data, converting the index to a datetime format for better tracking of predictions. - Improved debug logging to provide clearer context by including the input data preview in the log output. --- laborious/activities/mlflow.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index b96cc60..bd06283 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -169,11 +169,15 @@ class MLFlow(BaseActivity): model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) - self.debug(data.head(5).to_string(), metadata) + self.debug(f"Input data for: \n {data.head(5).to_string()}", metadata) # Convert numpy.nan to None for model compatibility data.replace(np.nan, None, inplace=True) + data['timestamp'] = data.index + data['timestamp'] = to_datetime( + data['timestamp'], format=DATETIME_FORMAT_WITH_TZ).dt.strftime(DATETIME_FORMAT) + # Request prediction from MLFlow model response_data = self.model_monitoring_repository.predict( model_name, data, model_config, metadata From 84c1371d6cd953ea86eab51299d64276272e8a2f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 16:02:54 -0300 Subject: [PATCH 25/29] SIENTIAPDE-1222 Refactor model configuration handling in MLFlow and workflows - Replaced 'model_retention' with 'model_config' to encapsulate retention settings and improve consistency across various components. - Updated test cases to reflect changes in argument structure, ensuring compatibility with the new model configuration format. - Added 'prediction_store_policy' to input data handling in workflows for enhanced configuration management. --- ## Problemas no Courier:.md | 9 -- tests/laborious/activities/test_mlflow.py | 45 +++++-- .../subworkflows/test_prediction_process.py | 115 +++++++++++------- .../workflows/test_predictions_batch.py | 11 +- 4 files changed, 111 insertions(+), 69 deletions(-) delete mode 100644 ## Problemas no Courier:.md diff --git a/## Problemas no Courier:.md b/## Problemas no Courier:.md deleted file mode 100644 index 48083e8..0000000 --- a/## Problemas no Courier:.md +++ /dev/null @@ -1,9 +0,0 @@ -## Problemas no Courier: -1. Enviamos a coluna timestamp do index para fazer o transform, para poder sincronizar a predição com o pacote que gerou ela, visto que vários modelos podem retornar uma lista de predições em vários casos. No caso do Courier, está vindo um timestamp que começa em 0, estando dessincronizado com os dados que enviamos. Seria possível alterar o comportamento do modelo para retornar o mesmo index que enviamos? - - Segue uma output do transform de exemplo: -``` -'303-WIT-230_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '303-WIT-230_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '303-WIT-230_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '303-WIT-230_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '305-WIT-135_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-135_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-WIT-135_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-135_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-160_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-WIT-160_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-WIT-160_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-WIT-160_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-PIT-170_median': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-170_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-PIT-170_min': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-170_max': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-175_median': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-PIT-175_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-PIT-175_min': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-PIT-175_max': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-FIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-FIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-013_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '305-FIT-013_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-FIT-013_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '305-FIT-013_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '306-PIT-101_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-PIT-101_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-101_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-PIT-101_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-FIT-051_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-FIT-051_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-051_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-FIT-051_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-DIT-001_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-DIT-001_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-DIT-001_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-DIT-001_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-PIT-105_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-PIT-105_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-105_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-PIT-105_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-FIT-052_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-FIT-052_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-052_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-FIT-052_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-DIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-DIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-DIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-DIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-PIT-115_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-PIT-115_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-115_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-PIT-115_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-FIT-004_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-FIT-004_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-004_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-FIT-004_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-PIT-110_median': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-PIT-110_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-110_min': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-PIT-110_max': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-FIT-003_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-FIT-003_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-003_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-FIT-003_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-PIT-125_median': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-PIT-125_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-125_min': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-PIT-125_max': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-PIT-130_median': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-PIT-130_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-130_min': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-PIT-130_max': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-FIT-006_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '306-FIT-006_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-006_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '306-FIT-006_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '307-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-003_median': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIT-003_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-003_min': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIT-003_max': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIC-022_median': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '307-FIC-022_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIC-022_min': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '307-FIC-022_max': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '310-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '310-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '310-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '310-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '307-FIT-008_median': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-008_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-008_min': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-008_max': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-009_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '307-FIT-009_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-009_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '307-FIT-009_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '309-PIT-101_median': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-101_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-101_min': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-101_max': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-105_median': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-105_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-105_min': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-105_max': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-110_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-110_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-110_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-110_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-185_median': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-185_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-185_min': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-185_max': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-190_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-190_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-190_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-190_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-195_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-PIT-195_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-195_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-PIT-195_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-FIT-051_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-051_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-FIT-051_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-051_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-052_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-FIT-052_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-FIT-052_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-FIT-052_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-PIT-001_median': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-001_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-001_min': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-001_max': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '309-PIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '309-PIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '317AIT003.3_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, 'SiO2_conc': {Timestamp('1970-01-01 00:00:01.732971600'): 5.15}}} -``` - -2. No modelo do transform (data_model), o nome do método que faz o transform de fato é "transform", sendo que em nossos modelos, por padrão esse nome é "predict". Seria possível alterar o nome do método manta manter a compatibilidade e o padrão que já temos? diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index ed68729..666ec7f 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -4,6 +4,7 @@ from unittest.mock import ANY, MagicMock, patch import numpy as np from pandas import DataFrame, Timestamp from pytest import fixture, mark, raises +from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ from laborious.activities.mlflow import MLFlow from sientia_do.notifications.models import NotificationLevel @@ -79,7 +80,7 @@ async def test_request_transform_success(mock_max, mock_dataframe, mlflow): 'value': 1.0, 'created_at': '2024-01-01 12:00:00'} ], 'model_name': 'test_model', - 'model_retention': 30 + 'model_config': {} } # Mock the transform response @@ -108,26 +109,35 @@ async def test_request_transform_success(mock_max, mock_dataframe, mlflow): # Verify the repository was called with correct arguments mlflow.model_monitoring_repository.transform.assert_called_once_with( - 'test_model', mock_dataframe, 30 + 'test_model', mock_dataframe, {}, metadata['metadata'] ) @mark.asyncio @patch("laborious.activities.mlflow.DataFrame") +@patch("laborious.activities.mlflow.to_datetime") @patch("laborious.activities.mlflow.max") -async def test_request_predict(mock_max, mock_dataframe, mlflow): +async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflow): mock_max.return_value = '2024-01-02' # Mock input data input_data = { **metadata, - 'data': [ - {'timestamp': '2024-01-01', 'variable': 'var1', 'value': 1.0}, - {'timestamp': '2024-01-01', 'variable': 'var2', 'value': 2.0}, - {'timestamp': '2024-01-02', 'variable': 'var1', 'value': 3.0}, - {'timestamp': '2024-01-02', 'variable': 'var2', 'value': 4.0} - ], + 'data': { + "variable": { + "2024-01-01": "var1", + "2024-01-02": "var2", + "2024-01-03": "var1", + "2024-01-04": "var2" + }, + "value": { + "2024-01-01": 1.0, + "2024-01-02": 2.0, + "2024-01-03": 3.0, + "2024-01-04": 4.0 + } + }, 'model_name': 'test_model', - 'model_retention': 30 + 'model_config': {} } # Mock the predict response @@ -141,13 +151,26 @@ async def test_request_predict(mock_max, mock_dataframe, mlflow): mock_dataframe.return_value.replace.assert_called_once_with( np.nan, None, inplace=True ) + mock_dataframe.return_value.__setitem__.assert_any_call( + 'timestamp', mock_to_datetime.return_value.dt.strftime.return_value + ) + mock_dataframe.return_value.__setitem__.assert_any_call( + 'timestamp', mock_to_datetime.return_value.dt.strftime.return_value + ) + + mock_to_datetime.assert_called_once_with( + mock_dataframe.return_value.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ + ) + mock_to_datetime.return_value.dt.strftime.assert_called_once_with( + DATETIME_FORMAT + ) # Verify the response assert response_data == expected_response # Verify the repository was called with correct arguments mlflow.model_monitoring_repository.predict.assert_called_once_with( - 'test_model', mock_dataframe.return_value, 30 + 'test_model', mock_dataframe.return_value, {}, metadata['metadata'] ) diff --git a/tests/laborious/workflows/subworkflows/test_prediction_process.py b/tests/laborious/workflows/subworkflows/test_prediction_process.py index eea8c53..df60ada 100644 --- a/tests/laborious/workflows/subworkflows/test_prediction_process.py +++ b/tests/laborious/workflows/subworkflows/test_prediction_process.py @@ -34,7 +34,9 @@ async def test_run(workflow_mock, prediction_process): 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'}, 'prediction_store_policy': 'lts:1' @@ -62,54 +64,54 @@ async def test_run(workflow_mock, prediction_process): workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { + **metadata, 'data': input_data['data'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.input_gate, { + **metadata, 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { + **metadata, 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], - **metadata + 'model_config': input_data['model_config'], }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_response_gate, { + **metadata, 'filters': input_data['mlflow_transform_filters'], 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_content_gate, { + **metadata, 'filters': input_data['mlflow_transform_filters'], 'data': 'transformed_data', 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_predict, { + **metadata, 'data': 'transformed_data', 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], - **metadata + 'model_config': input_data['model_config'], }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_response_gate, { + **metadata, 'filters': input_data['mlflow_predict_filters'], 'data': {'content': 'predicted_data', 'timestamp': '2024-01-01'}, 'type': 'predict', 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_child_workflow.assert_called_once_with( @@ -122,11 +124,12 @@ async def test_run(workflow_mock, prediction_process): 'timestamp': '2024-01-01', 'model_id': 1, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': input_data['model_config'], 'opc_output_config': input_data['opc_output_config'], 'schema': input_data['schema'], 'table_name': input_data['table_name'], - 'comment': 'Error' + 'comment': 'Error', + 'prediction_store_policy': input_data['prediction_store_policy'] } ) @@ -146,7 +149,9 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process): 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -165,13 +170,13 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process): workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY), call(Activities.input_gate, { 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY) ]) workflow_mock.execute_child_workflow.assert_not_called() @@ -192,7 +197,9 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -213,7 +220,7 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -221,14 +228,14 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY) @@ -261,7 +268,9 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -286,7 +295,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -294,14 +303,14 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -310,7 +319,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_content_gate, { @@ -318,7 +327,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'data': 'transformed_data', 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_child_workflow.assert_not_called() @@ -339,7 +348,9 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -365,7 +376,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -373,14 +384,14 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -389,7 +400,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_content_gate, { @@ -397,13 +408,13 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'data': 'transformed_data', 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_predict, { 'data': 'transformed_data', 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -412,7 +423,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'data': {'content': 'predicted_data', 'timestamp': '2024-01-01'}, 'type': 'predict', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_child_workflow.assert_not_called() @@ -429,7 +440,9 @@ async def test_path_flag_handler_stop(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' + model_config = { + 'retention': '30' + } # Act result = await prediction_process.path_flag_handler( @@ -440,7 +453,7 @@ async def test_path_flag_handler_stop(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention + 'model_config': model_config }, confidence, last_timestamp, "" ) @@ -462,7 +475,9 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' + model_config = { + 'retention': '30' + } # Act result = await prediction_process.path_flag_handler( @@ -473,7 +488,7 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention + 'model_config': model_config }, confidence, last_timestamp, "" ) @@ -506,7 +521,10 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' + model_config = { + 'retention': '30' + } + prediction_store_policy = 'erl:1' # Act result = await prediction_process.path_flag_handler( @@ -517,8 +535,9 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention, - 'opc_output_config': {'test': 'config'} + 'model_config': model_config, + 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': prediction_store_policy }, confidence, last_timestamp, 'Prediction Process' ) @@ -535,11 +554,12 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): 'timestamp': last_timestamp, 'model_id': model, 'model_name': model_name, - 'model_retention': model_retention, + 'model_config': model_config, 'schema': schema, 'table_name': table_name, 'comment': 'Prediction Process', - 'opc_output_config': {'test': 'config'} + 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': prediction_store_policy } ) @@ -556,8 +576,10 @@ async def test_path_flag_handler_unknown(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' - + model_config = { + 'retention': '30' + } + prediction_store_policy = 'erl:1' # Act result = await prediction_process.path_flag_handler( data, path_flag, { @@ -567,8 +589,9 @@ async def test_path_flag_handler_unknown(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention, - 'opc_output_config': {'test': 'config'} + 'model_config': model_config, + 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': prediction_store_policy }, confidence, last_timestamp, "" ) diff --git a/tests/laborious/workflows/test_predictions_batch.py b/tests/laborious/workflows/test_predictions_batch.py index e9b9bb6..90d7d21 100644 --- a/tests/laborious/workflows/test_predictions_batch.py +++ b/tests/laborious/workflows/test_predictions_batch.py @@ -33,7 +33,11 @@ async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch 'schema': 'test_schema', 'table_name': 'test_table', 'opc_output_config': 'test_opc_output_config', - 'datetime_columns': ['timestamp', 'created_at'] + 'datetime_columns': ['timestamp', 'created_at'], + 'prediction_store_policy': 'erl:1', + 'model_config': { + 'retention': '30' + } } await predictions_batch.run(input_data) @@ -72,9 +76,10 @@ async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch 'POLICY': 'STOP' } }), - 'model_retention': input_data.get('model_retention', 60), + 'model_config': input_data.get('model_config', {}), 'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']), - 'opc_output_config': input_data.get('opc_output_config', {}) + 'opc_output_config': input_data.get('opc_output_config', {}), + 'prediction_store_policy': input_data.get('prediction_store_policy', 'erl:1') } workflow_mock.execute_child_workflow.assert_has_calls([ From d6c34dcaf750c6ef5db8e93385bd606e99674023 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 24 Sep 2025 09:59:26 -0300 Subject: [PATCH 26/29] SIENTIAPDE-1222 Add placeholder class 'Any' in test_model_repository.py and update invalid_cases to use it - Introduced a new placeholder class 'Any' to be used in test cases. - Updated the 'invalid_cases' list to replace integer keys with instances of the 'Any' class, enhancing test coverage for key types. --- tests/laborious/utils/repository/test_model_repository.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 5dab80d..cdc59b4 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -33,6 +33,10 @@ metadata = { } +class Any: + pass + + invalid_cases = [ ( { @@ -53,8 +57,8 @@ invalid_cases = [ ( { 'value': { - 1: 1, - 2: 2 + Any(): 1, + Any(): 2 } } ) From 4ebbb97de9cd14a8e846b0bbda67612382db98e2 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 24 Sep 2025 10:30:23 -0300 Subject: [PATCH 27/29] Update tests/laborious/activities/test_mlflow.py Co-authored-by: codeant-ai[bot] <151821869+codeant-ai[bot]@users.noreply.github.com> --- tests/laborious/activities/test_mlflow.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index 666ec7f..85cb7f1 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -154,9 +154,6 @@ async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflo mock_dataframe.return_value.__setitem__.assert_any_call( 'timestamp', mock_to_datetime.return_value.dt.strftime.return_value ) - mock_dataframe.return_value.__setitem__.assert_any_call( - 'timestamp', mock_to_datetime.return_value.dt.strftime.return_value - ) mock_to_datetime.assert_called_once_with( mock_dataframe.return_value.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ From e0720eebf122b823aad0f374caa233a64330459f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 24 Sep 2025 10:32:56 -0300 Subject: [PATCH 28/29] SIENTIAPDE-1222 Remove deprecated files and configurations from transformer_pyfunc module - Deleted conda.yaml, MLmodel, python_env.yaml, requirements.txt, and various utility scripts related to data processing and model handling. - Removed binary files including python_model.pkl and training_transformer.pkl to clean up the artifacts directory. - This cleanup is part of the effort to streamline the transformer_pyfunc module and eliminate unused components. --- .../data_model/transformer_pyfunc/MLmodel | 19 - .../artifacts/training_transformer.pkl | Bin 29391 -> 0 bytes .../transformer_pyfunc/code/utils/__init__.py | 0 .../code/utils/data/.gitkeep | 0 .../code/utils/data/__init__.py | 0 .../code/utils/data/preprocessing.py | 172 ---- .../code/utils/data/read_data.py | 56 -- .../code/utils/data/transformers.py | 788 ---------------- .../code/utils/dvc/__init__.py | 0 .../code/utils/dvc/params.py | 51 - .../code/utils/features/.gitkeep | 0 .../code/utils/features/__init__.py | 0 .../code/utils/mlflow/pyfunc_wrappers.py | 325 ------- .../code/utils/models/.gitkeep | 0 .../code/utils/models/__init__.py | 24 - .../code/utils/models/arima.py | 389 -------- .../code/utils/models/base.py | 0 .../code/utils/models/catboost_time_series.py | 510 ---------- .../code/utils/models/evaluation.py | 92 -- .../code/utils/models/factory.py | 140 --- .../models/linear_regression_time_series.py | 303 ------ .../code/utils/models/neural_prophet_model.py | 888 ------------------ .../code/utils/models/stacking_time_series.py | 695 -------------- .../code/utils/visualization/.gitkeep | 0 .../code/utils/visualization/__init__.py | 0 .../data_model/transformer_pyfunc/conda.yaml | 11 - .../transformer_pyfunc/python_env.yaml | 7 - .../transformer_pyfunc/python_model.pkl | Bin 123 -> 0 bytes .../transformer_pyfunc/requirements.txt | 4 - .../transformers/courier_transformers.pkl | Bin 29607 -> 0 bytes 30 files changed, 4474 deletions(-) delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/MLmodel delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/conda.yaml delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/requirements.txt delete mode 100644 tmp/artifacts/data_model/transformers/courier_transformers.pkl diff --git a/tmp/artifacts/data_model/transformer_pyfunc/MLmodel b/tmp/artifacts/data_model/transformer_pyfunc/MLmodel deleted file mode 100644 index 3fc50fb..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/MLmodel +++ /dev/null @@ -1,19 +0,0 @@ -artifact_path: transformer_pyfunc -flavors: - python_function: - artifacts: - transformer: - path: artifacts/training_transformer.pkl - uri: /tmp/tmpnzkqz3v0/training_transformer.pkl - cloudpickle_version: 2.2.1 - code: code - env: - conda: conda.yaml - virtualenv: python_env.yaml - loader_module: mlflow.pyfunc.model - python_model: python_model.pkl - python_version: 3.10.16 -mlflow_version: 2.7.1 -model_uuid: 6a4a99079d234d0da2b8091532d55a34 -run_id: c2edec4dfafd4ad8bba257d62d25cd43 -utc_time_created: '2025-09-09 12:30:04.885248' diff --git a/tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl b/tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl deleted file mode 100644 index 02d4f4a33b863e4e6a36c404e28917079c0a8fad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29391 zcmeHQS(6-BRlnF)O&Hi-feBBSHJz&e@y@WBsnyQdRy_J){3^spZVQUa(K{a$5d4(YR75vI2HSY zsJ)#gN2In9t~HiJ=yo>JW$7I?sX!hyd8C-Ezb1eORet4TZ^KM+$K*+i(0F3vp=a#(&T7u*x7Du zyw!?pH_3*SPj!av?YD(9?O|)sh&5WDKj;lN2Bf%;71QJZoM)Af()q{U zOJ{f8On;QFrswB(%^z5Nao784{maR|&B5E-FMWq*14SPw8DPeWk2wS&VnCZ!+-9hXAx$laWK}0P zSrt{f%*~)mJopQuBPdOg^2~trd}o z*ARs+kt#6}^?Qq`iiyMOxna%ix`-7?s>&o4FS4{`<+QBX~(HAKt7ohgxN$|e#-B_a?lZHYzLO04L#zM!*5Vwl!{$zm+(o*_Ak!s(+_ zrx;e%+Vb^v>4j)Fon2`Yie?q1VnO#l1?c`Mfnq^*M+NvAp|4TO0KQ4t*C_iM6RRaTd=mPLH?9f@m4qX7g zi5)s?*r79Ehb{o0Lxzq(3$o{7AefSv@mPj_D5Ef9(M=&%{hTJmqr~O(Aq%OfUIm>6 zo<+*sDJ)J5cMKJBhq}HHGhk0)tS_f9sm}rGL^D6yM(0%lGGb|4W;ICwwRf0!)P#K`4^;1W{543Jb-FRckP+9&@}fb!nwg z3@_ZcvVirNJDJ2vIat}otceyAy;Tys0TCxgJw<}B|?V(69%?6f+Smuui4Uur_hp$lQ*1JGVKj^8U?7F%yV@GP$ za(HcdL3LUf*!D=LWiP8FLVL8>_Aq$<-#nzZXF zP1^O8XsOd46-joGr{f!##6bG*&N>KIYgwfImQ^gdE|oESENQiv*8jr>ib$Fz4@JwW ztd+!`OBE{^MV4|Ep8c~8&>qr~hpsCn*A?2>TFO<-)&FDzMc0+kb*1dOQgPd;E0D75O2tiZwVKxd%T_PCu9RF?%C0LFH^J4Z zwEizPS?Icgu47Bft}7Kc!BuyNxNe26TUb~fV%M#To9yazTK`X5o%XO+!b+}NW!J5W zn{2%AzzDRgL)R60HnyZ?*OiJJ)f7gq^OWrR=&Q9=!ETF@>m3(L-qwvoUQ? z7mI=ThuM)IRJUP*Hl3#3uD0It2tz!Qb3Ioe3qW|voM zt9aP5%U8YfB_vMC*RQ;>uEkTHxhNZRQ8wmNHQT%xy5_R1&1G4eqb$t#3@+5+!)m-S z+>U$E-Jjh3^KP2#s-?|e-u2jHBQMFGm9=YWa~3+uzV^@lib!^yOPiO;=n7r>FG_g=zDeHh7KhxKh3tfw>zg zFj2A!7ugjiW&i0e*|T+FAf05fan~>yfpNy9;WOMNd(O?pX>-Ai(^Rsd)M%b1z2E+;JUzq2A10+Ukm4MW$$C zC#qyWhk;Jb8Z6{0R*BPKM!;R+(_NB7tMQ<**$u)%;YAGnm8jp3@G>|kg%y;Q6m#f{ zY4bXJcP^lp(89Z9H@af7mx1K*b3wn`0$CsLXO}SJ?p{Kx>|QFR&DY8GH|X*vU4Dfw zze<;1qswQICcEPxeJISI#jA&Bo9voNo4-y;8+*C>TmK_IN%Ph{m1 zl)hOkp{1``!kw_dm!m7)!QH>;Y&3L}cnEY3u5~xH*c2c+LQZq$WqY~+G!AXQ8Z6JL z{97gBWy|aRMkh*>XSY>@^`icEwAt+ic$!y+XKsro%9EYlj640tX0#axdE#8+@)%+v zgMazNT6{{j&Fmd&097>_M&cWH0(55%B~4E1@Dvu%JH@ADTu7o;W z>fHSaTK5Sl2zVglGtSz%TCe~Yspmt6fs5hFtu9g3;%L=d_(*Y*R00YL7H2N&7Qa3h zu!OAwG}{K*3}-Uio?HhtZmJ*^bVv2EFcZ zs~-1)x%B`k^ANO!hm>}gzVcOZbm{%w}nf+jP*rTsBq{#~q z%SCO3#)%HEMsDh}AUw)~Xn^CwT&0)WKq@LsG{Z?*kJl1;FdE~m>y=5NXpeDXG|6*g ztf^I`+^~8LGggjFJY$br9uqVT(ZGc8X>0gn#EG4T5Ggy%L6`?d2ze1(>fVk-gONQv zHnUBx{2BC?AfWdMg%EEn?AejrHNuiFJS&Ggj5qESE@s45G?>i%V4u~~3r%&t^e7f< zM6O{>M?h$e7vG1mC1<1`t7;D5jLz{qsbspDlnm*9R@#}y6m0=ZbWdfOVZ~H&3S%>a zV^|WNSJP={EZeo@GNA@5JNDemeF9RMTG16;|7nU6OCKu-)coX`jmgyzz2g)$Fuz2a z7JD~2m1{gKIQW-j&WeTEHwuNRV8W8B6;=7J7Z_UoFqP0EjPWVkJtGy4T8m>q3+LKO zuzKY+8XeYBYfBcnu_URoKxM`0^+o<%?76HAuj8Z~sXm4;&QKIom0IzoNt+)KdSJui z7X|$x{8u9}rl8=N;X0SM*ki`g}jeceJoApx1we zO4SNKP0qRB#URLf)EF1y_N zJne>FlBL2Fl~wf)&FT)W0Nvr~6}1yaJBx=oehh$(5%CkCfP*z!d`5|o79(;A zow^KR8H6M1A`z_k$?$UA-{6fLJyStH3qtcn*#YsG9kqUA7^8oLr2qkPTq>+QtuiKa zsnzZT^qr$~FcZ(p;A+DnOkRHAOsNFRjGAjkq%i*61BvpC^MH zwb5SCXDeg8An|xFNMi?HklL3-E;vd&cE!E!=5WxpeNLOlcH@cZE3%_Ucy2~;^2t&s zP-g{G7Fv*|Sdr|*vLqLmBeEzr*Jf>&gz}}6cp9Y*ki>aANXr$PLp}jX=!n{<&*D46 z^ag_fvqS2`NCTSvAl`*6G&=Q&9+*r4YtM12L6~dplh&(Q-Dj{V)9+~cC9NfSN*d7H zT6*oGtI8M9`i=*2dU>0DC`pszjk_PWZsSohxb${FbxI}d=K8^Q)Wf6Z9rWM1c%vTS z1<}g^+V%O_`26GF`>%rW$rR6ms^!^Xhjl|Xv3L+9K*sKbTQyR=SJIeAY=F`h;d>BFis5TpS3F&qa3|! zQu^+==17I>4!J9*CKydlj*pST?=e!i?Ie}^5atj)81U}N>SdUqsP&D4@O>Nw=iLq7 zr6B*u$`Qd>d2S5B)z#&-wY(86W3UuN5ZN$o6>mgmNhpkL9xMv_m}$I%8wjRG-c2rC zTY7aSpbf1zptA~@Kyp;PK(ZQMAUO(dAUO(dAYxUUq`BkDwe{s}AX$Y>AUP^tAXyDB zkQ@a!kQ@aBqGm0)Z5E6C&XuDvAwGt@2a7Ywj;M&DxrZ;*!|tWxHVv8tV|mQ-msQ9# zO^%A!G+7O=X>t_YrpZwl6$mwrnscpW7zU*DoO<68MdQf;zo?$IJWBD2|WDrcgWJDv_m_P-mg_K^UknkTb+06 z8pL-7L+tS_|BPFoDja-H@F~LCsg)ZJtacg>R6k8PMGV;c4`;_#PB>(>!*EFT5Pzl~ zT*XY$N}E5xWAqL#$uW9WhlypS*J$7UxJO^K`XQMCB-l5B{^n=B;I2*-=lzjo|ojL zXqDg+Ry6hLXFqMePmbkW;`4dvXTg%=vg@zmd$05>Wa|4rqUwK6m*kA>sPr`$Ozre; zNUri}^8<41zAyNI3P0~|i0@b|ckop$e1-HwqIz(b%IUWMba2g{pf(};0HL;ufi zOoZkx8={e!yR4Q*cBG`W^^GyRuRCq8Z^oNzwFh-$s>?EtlV!C(PsDg;?(VC8W5#!5 zZ0|$UOfGacj&tXlG));79t`K_SbKav>#_;wp1I$-r>>@%#vk5YjPrJrU${AJ;Gkdn z>C?V`tFb``_0B&we_(zWozpwNo38KiAFAd*^_su^e zch2N_k7KB}$o*CN-}1NlQ6qvgCmPdTyq(VPV(OUqlf5+@aE#wa zQ;g~e=Nf(+j4xOt=Ad1RDTq>MUX?Z2p0D4rUMhAVv zE2Be)ehaVopj@17=};-qP$|SvX~d)Q0u*W$%PWQWR4~+3;#kPg^-6)pD}@*;jkr`e z4A%*e)c|@_q^5JFbd*>6ykwiofQrf>ipoNySLz7mqD#d!R*KpyOHF!ZA(~2_)a+6z z(0HW~eGxibc3l5zH$q0(xeS1{C6qB^Iy z%d{6vZ#RhV6$~{MR~P2x3ZUl|skvSeqN#*e9+d(Ol|l@aMl=zB&D}(p}!qB6d&ZC;nP(jTKkir2X zsniL-&V+10l`DfNDhu(boX()s%)l{-Ph|#<)WIZR^>zb#UXhv%(CBypUd;0fphrb& zE)^kefYW(i0raRy&7~s5qssGYI^z`(pUU=%#~@ZuCsztIUMa*-X+(P^5NF`%GnuUb zwVDo4Bbth)9Xb=!@k)WlD}@-ZG~!WtUI{g`q>84eKS>)5%nK;J8qaw3^P9h4@r5S5KPLo=a0YS;rid z{c<5@^B~HXMOQYHmZb>Prz1s*)79cYExM{nnxzQTrz1s=jv&vwl9`7~bOd>HrQD_f>eG>;M@NuHXU#yCcWPJf zLOvbYdURDYNG)5SzIUYPc}I{(hd(Hh(-c5`I#TrL2=eGExlIAorz1s=jv$Y&YPQpM zu;mOLDSC7Sd35F6Q32HVjubsQf;_rPZc_mD=}6I|BgmtxnmN-BR!*8O`19VvQr1bK8-v(2_6t7hm((W4{Cqbuh&1yJ8RQuOEu^62pA zVsb_WP@j$zJvxFsx@zuP0o11>MURdkkFK2C6hM7CQuOEu^5`nLO##%WBSnvnAdjw^ zyH)`8=}6I|Bgmt}AGpbB3ZOn6DSC7Sd32TBrU2^Gk)lUOkVjX|T`Pe4bfoCf5#-U8 zbDILFPe+O#9YG#lCATSn`gEk|(Gld)S#JfcF$>hEBSnvnAdjw`+Y~^3I#TrL2=eGE zxlIAorz1s=jv$Y2DtD~_>eG>;M@NuHSI%t;pgtWbdUOPNbk@6Gt0~mWUKjG|$kwBq n%3Ujf`reVE=N&S;hzzhH3&B^}*E89w3 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py deleted file mode 100644 index 4ee9088..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py +++ /dev/null @@ -1,172 +0,0 @@ -""" -Module with utility functions for data processing. -""" - -import pandas as pd -from typing import Dict, List -from rich.console import Console - -console = Console() - - -def create_lagged_target( - data: pd.DataFrame, - target_column: str, - lags: List[int], - drop_nans: bool = True, -) -> pd.DataFrame: - """ - Creates lagged versions of a target column in a DataFrame. - - For each lag value in the provided list, a new column is created with - the naming pattern: target_column + "_lag_" + lag_value. - - Args: - data: Input DataFrame containing the target column. - target_column: Name of the target column to create lags for. - lags: List of lag values (integers between 1 and len(data)-1). - drop_nans: Whether rows with nulls generated by the lag creation - process should be dropped. Defaults to True. - Returns: - DataFrame with original columns plus the newly created lag columns. - """ - if data.empty: - console.log("[red]Warning: Input data is empty.") - return data - - result = data.copy() - - if target_column not in result.columns: - raise ValueError(f"Target column '{target_column}' not found in data.") - - # Validate lag values - max_lag = len(data) - 1 - valid_lags = [lag for lag in lags if 1 <= lag <= max_lag] - - if len(valid_lags) < len(lags): - invalid_lags = set(lags) - set(valid_lags) - console.log( - f"[yellow]Warning: Ignoring invalid lag values: {invalid_lags}. " - f"Lags must be between 1 and {max_lag}." - ) - - lag_column_names = [] - for lag in valid_lags: - lag_column_name = f"{target_column}_lag_{lag}" - result[lag_column_name] = result[target_column].shift(lag) - console.log(f"Created lagged column: [cyan]{lag_column_name}") - lag_column_names.append(lag_column_name) - if drop_nans: - result = result.dropna(subset=lag_column_names) - - return result - - -def remove_stopped_windows( - data: pd.DataFrame, - stopped_process_columns: Dict[str, float], - stopped_process_threshold: float, - time_colname: str, -) -> pd.DataFrame: - """ - Removes time windows from the input DataFrame if the proportion of samples - below a column threshold exceeds the specified limit. - - A window is considered "stopped" if *all* specified columns exceed the - stopped sample threshold. - - Args: - data: Input DataFrame with process variables and timestamps. - stopped_process_columns: Dict mapping column names to thresholds. - stopped_process_threshold: Proportion threshold (0-1) for marking a - window as stopped. - time_colname: Base name of the timestamp column - (without 'lab_' prefix). - - Returns: - A DataFrame with stopped windows removed. - """ - if data.empty: - console.log("[red]Warning: Input data is empty.") - return data - - console.log( - "Removing windows where any column exceeds" - + f" {stopped_process_threshold:.2%} of values below threshold" - ) - - masks = [] - - for col, threshold in stopped_process_columns.items(): - console.log( - "Evaluating stopped condition for column:" - + f" [cyan]{col} < {threshold}" - ) - below_threshold = data[[col]].lt(threshold) - - console.log( - "Counting number of samples below threshold for each window" - ) - below_threshold[f"lab_{time_colname}"] = data[f"lab_{time_colname}"] - grouped = below_threshold.groupby(f"lab_{time_colname}")[col].agg( - ["sum", "count"] - ) - stopped_mask = ( - grouped["sum"] / grouped["count"] - ) > stopped_process_threshold - - console.log( - f"[red]{stopped_mask.sum()} windows marked as stopped by {col}" - ) - masks.append(stopped_mask) - - # Combine masks across columns: only drop if all agree - combined_mask = pd.concat(masks, axis=1).all(axis=1) - - num_removed = combined_mask.sum() - total = combined_mask.shape[0] - console.log( - f"Removing [bold red]{num_removed}[/] out of {total}" - + f" windows ({num_removed / total:.2%})" - ) - - to_remove = combined_mask[combined_mask].index - keep_mask = ~data[f"lab_{time_colname}"].isin(to_remove) - - return data[keep_mask] - - -def aggregate_data( - merged_data: pd.DataFrame, - time_colname: str, - target_colname: str, - aggregation_functions: List[str], -) -> pd.DataFrame: - """ - Aggregates a DataFrame by time and target columns using specified - aggregation functions. - - Args: - merged_data: Input DataFrame with raw observations. - time_colname: Name of the timestamp column (no 'lab_' prefix). - target_colname: Name of the target/grouping column. - aggregation_functions: List of aggregation functions to apply - (e.g. "mean", "std"). - - Returns: - Aggregated DataFrame with flattened column names and renamed time - column. - """ - group_by_cols = [f"lab_{time_colname}", target_colname] - - aggregated = merged_data.groupby(group_by_cols).agg(aggregation_functions) - # Flatten MultiIndex columns - aggregated.columns = [ - "_".join(col) if isinstance(col, tuple) else col - for col in aggregated.columns - ] # type: ignore - aggregated = aggregated.reset_index() - - console.log(f"[bold green]Aggregated shape: {aggregated.shape}") - - return aggregated.rename(columns={f"lab_{time_colname}": time_colname}) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py deleted file mode 100644 index 76e4e9d..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -Module with helper functions to read datasets. -""" - -import pandas as pd -from openpyxl import load_workbook -from typing import Union - - -def read_excel_with_colors( - filepath: str, color_columns: list[str], sheet_name: Union[str, int] = 0 -) -> pd.DataFrame: - """ - Read an Excel file and extract cell fill colors for specified columns. - - Parameters: - filepath (str): Path to the Excel file. - color_columns (List[str]): Column names to extract fill colors from. - sheet_name (str or int): Sheet name or index (default is first sheet). - - Returns: - DataFrame: DataFrame with original data and extra color columns. - """ - df = pd.read_excel(filepath, sheet_name=sheet_name) - - workbook = load_workbook(filepath) - sheet = ( - workbook[sheet_name] - if isinstance(sheet_name, str) - else workbook[workbook.sheetnames[sheet_name]] - ) - - header = next(sheet.iter_rows(min_row=1, max_row=1, values_only=True)) - col_name_to_letter = { - name: chr(65 + idx) for idx, name in enumerate(header) - } - - for col_name in color_columns: - if col_name not in df.columns: - raise ValueError(f"Column '{col_name}' not found in Excel file.") - - col_letter = col_name_to_letter[col_name] - fill_colors: list[Union[str, None]] = [] - - for row in range(2, sheet.max_row + 1): - cell = sheet[f"{col_letter}{row}"] - fill = cell.fill - - if fill.fill_type == "solid" and fill.fgColor.rgb: - fill_colors.append(fill.fgColor.rgb) - else: - fill_colors.append(None) - - df[f"{col_name}_fill_color"] = fill_colors - - return df diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py deleted file mode 100644 index efe43e0..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py +++ /dev/null @@ -1,788 +0,0 @@ -""" -Module with scikit-learn transformers for training and inference pipelines. - -This module replicates the functionality from the DVC pipeline stages: -- preprocessing.py (stopped process filtering, aggregation) -- filter_columns.py (feature selection, lagged target creation) -""" - -import pandas as pd -from typing import Dict, List, Optional -from sklearn.base import BaseEstimator, TransformerMixin -from rich.console import Console - -console = Console() - - -class CourierTrainingTransformer(BaseEstimator, TransformerMixin): - """ - Training transformer that replicates the DVC pipeline preprocessing. - - Includes: - 1. Remove stopped process windows (training only) - 2. Data aggregation - 3. Feature selection (learns and applies - dictionary-based only) - 4. Create lagged target features - """ - - def __init__( - self, - aggregation_functions: List[str] = ["median", "std", "min", "max"], - stopped_process_columns: Dict[str, float] = { - "305-PIT-170": 9, - "305-PIT-175": 9, - }, - stopped_process_threshold: float = 0.1, - target_lags: List[int] = [2], - time_colname: str = "timestamp", - target_colname: str = "SiO2_conc", - dictionary_df: Optional[pd.DataFrame] = None, - create_lagged_target: bool = True, - drop_nans: bool = True, - ): - """ - Initialize the training transformer. - - Args: - aggregation_functions: List of aggregation functions to apply - stopped_process_columns: Dict mapping column names to thresholds - stopped_process_threshold: Proportion threshold for stopped windows - target_lags: List of lag values for target column - time_colname: Name of timestamp column (without 'lab_' prefix) - target_colname: Name of target column - dictionary_df: DataFrame with domain knowledge - (TAG_fill_color column) - create_lagged_target: Whether to create lagged target features - drop_nans: Whether to drop NaNs after creating lags - """ - self.aggregation_functions = aggregation_functions - self.stopped_process_columns = stopped_process_columns - self.stopped_process_threshold = stopped_process_threshold - self.target_lags = target_lags - self.time_colname = time_colname - self.target_colname = target_colname - self.dictionary_df = dictionary_df - self.create_lagged_target = create_lagged_target - self.drop_nans = drop_nans - - # Will be learned during fit - self.selected_features_ = None - - def _infer_lab_timestamp(self, data: pd.DataFrame) -> pd.DataFrame: - """ - Creates a lab timestamp column by rounding the timestamp up to the next - even hour. - - Args: - data: Input DataFrame - - Returns: - DataFrame with added lab timestamp column - """ - lab_col_name = f"lab_{self.time_colname}" - - if lab_col_name in data.columns: - console.log( - f"[yellow]Lab timestamp column {lab_col_name} already exists, " - + "skipping inference" - ) - return data - - # Check if timestamp is a column or the index - if self.time_colname in data.columns: - # Timestamp is a regular column - result = data.copy() - timestamp_col = pd.to_datetime(result[self.time_colname]) - elif data.index.name == self.time_colname or ( - hasattr(data.index, "names") - and self.time_colname in data.index.names - ): - # Timestamp is the index (or part of a MultiIndex) - result = data.copy() - timestamp_col = pd.to_datetime( - result.index.get_level_values(self.time_colname) - if hasattr(result.index, "names") - and len(result.index.names) > 1 - else result.index - ) - else: - raise ValueError( - f"Timestamp '{self.time_colname}' not found in data columns " - + f"or index. Available columns: {list(data.columns)}, " - + f"index name: {data.index.name}" - ) - - # Round up to next even hour - # Step 1: Floor to the hour to remove minutes/seconds - # Handle both Series (from column) and DatetimeIndex (from index) - if hasattr(timestamp_col, "dt"): - # timestamp_col is a Series - hour_floor = timestamp_col.dt.floor("H") - hour = hour_floor.dt.hour - else: - # timestamp_col is a DatetimeIndex - hour_floor = timestamp_col.floor("H") - hour = hour_floor.hour - - # Step 3: Determine if rounding is needed - # - If hour is odd, round up to next even hour - # - If hour is even but original timestamp had minutes/seconds, - # round up to next even hour - # - If hour is even and original timestamp was exactly on the hour, - # keep it - needs_rounding = (hour % 2 == 1) | (timestamp_col != hour_floor) - - # Calculate next even hour - next_even_hour = ((hour // 2) + 1) * 2 - - # Handle case where next even hour >= 24 (next day) - days_to_add = (next_even_hour >= 24).astype(int) - hour_component = next_even_hour % 24 - - # Create the lab timestamp - if hasattr(timestamp_col, "dt"): - # timestamp_col is a Series - lab_timestamp = hour_floor.where( - ~needs_rounding, - hour_floor.dt.floor("D") - + pd.to_timedelta(days_to_add, unit="D") - + pd.to_timedelta(hour_component, unit="H"), - ) - else: - # timestamp_col is a DatetimeIndex - base_date = hour_floor.floor("D") - next_even_timestamp = ( - base_date - + pd.to_timedelta(days_to_add, unit="D") - + pd.to_timedelta(hour_component, unit="H") - ) - lab_timestamp = pd.Series( - hour_floor.where(~needs_rounding, next_even_timestamp), - index=result.index, - ) - - result[lab_col_name] = lab_timestamp - - console.log( - f"[bold green]Created lab timestamp column: {lab_col_name}" - ) - - return result - - def _remove_stopped_windows( - self, - data: pd.DataFrame, - ) -> pd.DataFrame: - """ - Removes time windows where process was stopped. - Replicates remove_stopped_windows from preprocessing.py - """ - if data.empty: - console.log("[red]Warning: Input data is empty.") - return data - - console.log( - "Removing windows where any column exceeds" - + f" {self.stopped_process_threshold:.2%} of values below" - + " threshold" - ) - - masks = [] - - for col, threshold in self.stopped_process_columns.items(): - console.log( - "Evaluating stopped condition for column:" - + f" [cyan]{col} < {threshold}" - ) - below_threshold = data[[col]].lt(threshold) - - console.log( - "Counting number of samples below threshold for each window" - ) - below_threshold[f"lab_{self.time_colname}"] = data[ - f"lab_{self.time_colname}" - ] - grouped = below_threshold.groupby(f"lab_{self.time_colname}")[ - col - ].agg(["sum", "count"]) - stopped_mask = ( - grouped["sum"] / grouped["count"] - ) > self.stopped_process_threshold - - console.log( - f"[red]{stopped_mask.sum()} windows marked as stopped by {col}" - ) - masks.append(stopped_mask) - - # Combine masks across columns: only drop if all agree - if not masks: - return data - - mask_df = pd.concat(masks, axis=1) - combined_mask = mask_df.all(axis=1) - - num_removed = int(combined_mask.sum()) # type: ignore - total = combined_mask.shape[0] - console.log( - f"Removing [bold red]{num_removed}[/] out of {total}" - + f" windows ({num_removed / total:.2%})" - ) - - to_remove = combined_mask[combined_mask].index - keep_mask = ~data[f"lab_{self.time_colname}"].isin(to_remove) - - filtered_data = data[keep_mask] - return filtered_data # type: ignore - - def _aggregate_data( - self, - merged_data: pd.DataFrame, - ) -> pd.DataFrame: - """ - Aggregates data by time and target columns. - Replicates aggregate_data from preprocessing.py - """ - group_by_cols = [f"lab_{self.time_colname}", self.target_colname] - - aggregated = merged_data.groupby(group_by_cols).agg( - self.aggregation_functions - ) - # Flatten MultiIndex columns - aggregated.columns = [ - "_".join(col) if isinstance(col, tuple) else col - for col in aggregated.columns - ] # type: ignore - aggregated = aggregated.reset_index() - - # Rename timestamp column and ensure it's datetime - aggregated = aggregated.rename( - columns={f"lab_{self.time_colname}": self.time_colname} - ) - aggregated[self.time_colname] = pd.to_datetime( - aggregated[self.time_colname] - ) - - console.log(f"[bold green]Aggregated shape: {aggregated.shape}") - - return aggregated - - def _learn_feature_selection( - self, - data: pd.DataFrame, - ) -> List[str]: - """ - Learn which features to keep based on dictionary only. - Replicates domain knowledge filtering from filter_columns.py - """ - if self.dictionary_df is None: - console.log( - "[yellow]Warning: No dictionary data provided. Using all" - + " features." - ) - return [col for col in data.columns if col != self.time_colname] - - # Domain knowledge filter - only keep columns with TAG_fill_color - columns_to_keep_dict = self.dictionary_df.loc[ - self.dictionary_df["TAG_fill_color"].notna(), "TAG" - ].values - - # Filter data columns to only those that match dictionary tags - available_columns = [ - col for col in data.columns if col != self.time_colname - ] - columns_to_keep = [ - col - for col in available_columns - if col.split("_")[0] in columns_to_keep_dict - ] - - console.log( - f"Keeping {len(columns_to_keep)}/{len(available_columns)}" - + " columns based on dictionary." - ) - - # Always include target - columns_to_keep.append(self.target_colname) - - return columns_to_keep - - def _create_lagged_target( - self, - data: pd.DataFrame, - ) -> pd.DataFrame: - """ - Creates lagged versions of target column. - Replicates create_lagged_target from preprocessing.py - """ - if data.empty: - console.log("[red]Warning: Input data is empty.") - return data - - result = data.copy() - - if self.target_colname not in result.columns: - raise ValueError( - f"Target column '{self.target_colname}' not found in data." - ) - - # Validate lag values - max_lag = len(data) - 1 - valid_lags = [lag for lag in self.target_lags if 1 <= lag <= max_lag] - - if len(valid_lags) < len(self.target_lags): - invalid_lags = set(self.target_lags) - set(valid_lags) - console.log( - f"[yellow]Warning: Ignoring invalid lag values: {invalid_lags}" - + f". Lags must be between 1 and {max_lag}." - ) - - lag_column_names = [] - for lag in valid_lags: - lag_column_name = f"{self.target_colname}_lag_{lag}" - result[lag_column_name] = result[self.target_colname].shift(lag) - console.log(f"Created lagged column: [cyan]{lag_column_name}") - lag_column_names.append(lag_column_name) - - if self.drop_nans and lag_column_names: - result = result.dropna(subset=lag_column_names) - - return result - - def fit(self, X: pd.DataFrame, y=None): - """ - Learn feature selection parameters. - - Args: - X: Input DataFrame with merged process and quality data - y: Not used - - Returns: - self - """ - console.log("[bold blue]Training transformer fit phase") - - # Create a copy for processing - data = X.copy() - - # Step 0: Infer lab timestamp if needed - console.log("[bold blue]Inferring lab timestamp") - data = self._infer_lab_timestamp(data) - - # Step 1: Remove stopped process windows (training only) - console.log("[bold blue]Removing stopped process windows") - data = self._remove_stopped_windows(data) - - # Step 2: Aggregate data - console.log("[bold blue]Aggregating data") - data = self._aggregate_data(data) - - # Step 3: Learn feature selection - console.log("[bold blue]Learning feature selection") - self.selected_features_ = self._learn_feature_selection(data) - - console.log( - f"[bold green]Learned {len(self.selected_features_)}" - + " features for selection" - ) - self._feature_names = self.selected_features_ - - return self - - def transform(self, X: pd.DataFrame) -> pd.DataFrame: - """ - Apply the complete training transformation pipeline. - - Args: - X: Input DataFrame with merged process and quality data - - Returns: - Transformed DataFrame ready for model training - """ - if self.selected_features_ is None: - raise ValueError("Transformer must be fitted before transform.") - - console.log("[bold blue]Training transformer transform phase") - - # Create a copy for processing - data = X.copy() - - # Step 0: Infer lab timestamp if needed - console.log("[bold blue]Inferring lab timestamp") - data = self._infer_lab_timestamp(data) - - # Step 1: Remove stopped process windows (training only) - console.log("[bold blue]Removing stopped process windows") - data = self._remove_stopped_windows(data) - - # Step 2: Aggregate data - console.log("[bold blue]Aggregating data") - data = self._aggregate_data(data) - - # Step 3: Apply feature selection - console.log("[bold blue]Applying feature selection") - # Set timestamp as index for filtering and ensure it's datetime - data[self.time_colname] = pd.to_datetime(data[self.time_colname]) - data = data.set_index(self.time_colname) - data = data[self.selected_features_] - - # Step 4: Create lagged target features - if self.create_lagged_target: - console.log("[bold blue]Creating lagged target features") - data = self._create_lagged_target(data) - - console.log(f"[bold green]Final training data shape: {data.shape}") - - return data - - -class CourierInferenceTransformer(BaseEstimator, TransformerMixin): - """ - Inference transformer that replicates DVC pipeline preprocessing - without training-specific steps. - - Includes: - 1. Data aggregation (higher frequency - no grouping by target) - 2. Feature selection (applies learned selection) - 3. Create lagged target features - - Note: Does NOT include stopped process filtering (training only). - """ - - def __init__( - self, - selected_features: List[str], - aggregation_functions: List[str] = ["median", "std", "min", "max"], - target_lags: List[int] = [2], - time_colname: str = "timestamp", - target_colname: str = "SiO2_conc", - create_lagged_target: bool = True, - drop_nans: bool = True, - ): - """ - Initialize the inference transformer. - - Args: - selected_features: Pre-learned list of features to select - aggregation_functions: List of aggregation functions to apply - target_lags: List of lag values for target column - time_colname: Name of timestamp column (without 'lab_' prefix) - target_colname: Name of target column - create_lagged_target: Whether to create lagged target features - drop_nans: Whether to drop NaNs after creating lags - """ - self.selected_features = selected_features - self.aggregation_functions = aggregation_functions - self.target_lags = target_lags - self.time_colname = time_colname - self.target_colname = target_colname - self.create_lagged_target = create_lagged_target - self.drop_nans = drop_nans - - def _infer_lab_timestamp(self, data: pd.DataFrame) -> pd.DataFrame: - """ - Creates a lab timestamp column by rounding the timestamp up to the next - even hour. - - Args: - data: Input DataFrame - - Returns: - DataFrame with added lab timestamp column - """ - lab_col_name = f"lab_{self.time_colname}" - - if lab_col_name in data.columns: - console.log( - f"[yellow]Lab timestamp column {lab_col_name} already exists, " - + "skipping inference" - ) - return data - - # Check if timestamp is a column or the index - if self.time_colname in data.columns: - # Timestamp is a regular column - result = data.copy() - timestamp_col = pd.to_datetime(result[self.time_colname]) - elif data.index.name == self.time_colname or ( - hasattr(data.index, "names") - and self.time_colname in data.index.names - ): - # Timestamp is the index (or part of a MultiIndex) - result = data.copy() - timestamp_col = pd.to_datetime( - result.index.get_level_values(self.time_colname) - if hasattr(result.index, "names") - and len(result.index.names) > 1 - else result.index - ) - else: - raise ValueError( - f"Timestamp '{self.time_colname}' not found in data columns " - + f"or index. Available columns: {list(data.columns)}, " - + f"index name: {data.index.name}" - ) - - # Round up to next even hour - # Step 1: Floor to the hour to remove minutes/seconds - # Handle both Series (from column) and DatetimeIndex (from index) - if hasattr(timestamp_col, "dt"): - # timestamp_col is a Series - hour_floor = timestamp_col.dt.floor("H") - hour = hour_floor.dt.hour - else: - # timestamp_col is a DatetimeIndex - hour_floor = timestamp_col.floor("H") - hour = hour_floor.hour - - # Step 3: Determine if rounding is needed - # - If hour is odd, round up to next even hour - # - If hour is even but original timestamp had minutes/seconds, - # round up to next even hour - # - If hour is even and original timestamp was exactly on the hour, - # keep it - needs_rounding = (hour % 2 == 1) | (timestamp_col != hour_floor) - - # Calculate next even hour - next_even_hour = ((hour // 2) + 1) * 2 - - # Handle case where next even hour >= 24 (next day) - days_to_add = (next_even_hour >= 24).astype(int) - hour_component = next_even_hour % 24 - - # Create the lab timestamp - if hasattr(timestamp_col, "dt"): - # timestamp_col is a Series - lab_timestamp = hour_floor.where( - ~needs_rounding, - hour_floor.dt.floor("D") - + pd.to_timedelta(days_to_add, unit="D") - + pd.to_timedelta(hour_component, unit="H"), - ) - else: - # timestamp_col is a DatetimeIndex - base_date = hour_floor.floor("D") - next_even_timestamp = ( - base_date - + pd.to_timedelta(days_to_add, unit="D") - + pd.to_timedelta(hour_component, unit="H") - ) - lab_timestamp = pd.Series( - hour_floor.where(~needs_rounding, next_even_timestamp), - index=result.index, - ) - - result[lab_col_name] = lab_timestamp - - console.log( - f"[bold green]Created lab timestamp column: {lab_col_name}" - ) - - return result - - def _aggregate_data( - self, - merged_data: pd.DataFrame, - ) -> pd.DataFrame: - """ - Aggregate data into 2-hour non-overlapping windows. - Each row corresponds to one 2-hour window ending at an even hour. - """ - if merged_data.empty: - console.log("[red]Warning: Input data is empty.") - return merged_data - - lab_col = f"lab_{self.time_colname}" - if lab_col not in merged_data.columns: - raise ValueError( - f"Missing '{lab_col}' column. Call _infer_lab_timestamp first." - ) - - # Get numeric columns only for aggregation - numeric_cols = merged_data.select_dtypes( - include=["number"] - ).columns.tolist() - - # Remove time and target columns if present - cols_to_remove = [self.time_colname, lab_col, self.target_colname] - for col in cols_to_remove: - if col in numeric_cols: - numeric_cols.remove(col) - - groups = merged_data.groupby(lab_col) - - if numeric_cols: - aggregated_numeric = groups[numeric_cols].agg( - self.aggregation_functions - ) - # Flatten MultiIndex columns: (col, func) -> "col_func" - aggregated_numeric.columns = [ - f"{col}_{func}" - for col, func in aggregated_numeric.columns.to_flat_index() - ] - else: - # Create empty frame indexed by the 2-hour windows - aggregated_numeric = groups.size().to_frame(name="__rows__") - aggregated_numeric = aggregated_numeric.drop(columns=["__rows__"]) - - # Add target column as the last non-null value per window - if self.target_colname in merged_data.columns: - target_per_window = groups[self.target_colname].apply( - lambda s: s.dropna().iloc[-1] - if not s.dropna().empty - else None - ) - aggregated_numeric[self.target_colname] = target_per_window - - # Reset index and rename lab timestamp to main time column - result = aggregated_numeric.reset_index().rename( - columns={lab_col: self.time_colname} - ) - - # Ensure timestamp is datetime - result[self.time_colname] = pd.to_datetime(result[self.time_colname]) - - console.log( - f"[bold green]Aggregated to windowed shape: {result.shape}" - ) - - return result - - def _create_lagged_target( - self, - data: pd.DataFrame, - ) -> pd.DataFrame: - """ - Creates lagged target column names with target values for inference. - In inference, we assume the data is already properly lagged, - so we just create the expected column names with the target values. - """ - if data.empty: - console.log("[red]Warning: Input data is empty.") - return data - - result = data.copy() - - if self.target_colname not in result.columns: - raise ValueError( - f"Target column '{self.target_colname}' not found in data." - ) - - # Create lagged column names with target values (no actual shifting) - lag_column_names = [] - - # Get target column as a Series to ensure we have exactly one column - target_series = result[self.target_colname] - if isinstance(target_series, pd.DataFrame): - # If we accidentally got a DataFrame, take the first column - target_values = target_series.iloc[:, 0].values - else: - target_values = target_series.values - - for lag in self.target_lags: - lag_column_name = f"{self.target_colname}_lag_{lag}" - # Copy target values instead of shifting for inference - result[lag_column_name] = target_values - console.log(f"Created lagged column: [cyan]{lag_column_name}") - lag_column_names.append(lag_column_name) - - return result - - def fit(self, X: pd.DataFrame, y=None): - """ - No-op for inference transformer (no learning needed). - - Args: - X: Input DataFrame - y: Not used - - Returns: - self - """ - console.log("[bold blue]Inference transformer fit (no-op)") - return self - - def transform(self, X: pd.DataFrame) -> pd.DataFrame: - """ - Apply the inference transformation pipeline. - - Args: - X: Input DataFrame with merged process and quality data - - Returns: - Transformed DataFrame ready for model inference - """ - console.log("[bold blue]Inference transformer transform phase") - - # Create a copy for processing - data = X.copy() - - # Step 0: Infer lab timestamp if needed - console.log("[bold blue]Inferring lab timestamp") - data = self._infer_lab_timestamp(data) - - # Step 1: Aggregate data (higher frequency - no target grouping) - console.log("[bold blue]Aggregating data") - data = self._aggregate_data(data) - - # Step 2: Apply learned feature selection - console.log("[bold blue]Applying learned feature selection") - # Set timestamp as index for filtering and ensure it's datetime - data[self.time_colname] = pd.to_datetime(data[self.time_colname]) - data = data.set_index(self.time_colname) - - # Filter to selected features (handle missing columns gracefully) - available_features = [ - col for col in self.selected_features if col in data.columns - ] - missing_features = set(self.selected_features) - set( - available_features - ) - - if missing_features: - console.log( - "[yellow]Warning: Missing features in inference data:" - + f" {missing_features}" - ) - - # Ensure target column is included but avoid duplicates - if self.target_colname not in available_features: - available_features.append(self.target_colname) - - data = data[available_features] - - # Step 3: Create lagged target features - if self.create_lagged_target: - console.log("[bold blue]Creating lagged target features") - data = self._create_lagged_target(data) - - console.log(f"[bold green]Final inference data shape: {data.shape}") - - return data - - -def create_transformers_from_training_transformer( - training_transformer: CourierTrainingTransformer, -) -> tuple[CourierTrainingTransformer, CourierInferenceTransformer]: - """ - Create both training and inference transformers with shared parameters. - - Args: - training_transformer: Fitted training transformer - - Returns: - Tuple of (training_transformer, inference_transformer) - """ - if training_transformer.selected_features_ is None: - raise ValueError("Training transformer must be fitted first.") - - inference_transformer = CourierInferenceTransformer( - selected_features=training_transformer.selected_features_, - aggregation_functions=training_transformer.aggregation_functions, - target_lags=training_transformer.target_lags, - time_colname=training_transformer.time_colname, - target_colname=training_transformer.target_colname, - create_lagged_target=training_transformer.create_lagged_target, - drop_nans=training_transformer.drop_nans, - ) - - return training_transformer, inference_transformer diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py deleted file mode 100644 index 6fba3eb..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Functions needed to load parameters from params.yaml tracked with DVC -""" - -import sys -import os -from typing import Optional - -import yaml -from rich.console import Console - - -console = Console() - - -def get_params(stage_fn: Optional[str] = None): - """ - Reads parameters for a given DVC stage from params.yaml. - - The stage name is inferred from the name of the python file that calls this - function. - Args: - stage_fn (str): Name of the stage. If None, the name of the file - that calls this function is used. Defaults to None. - Returns: - dict with parameters for the stage - Raises: - KeyError: if the stage name is not found in params.yaml - """ - - if stage_fn is None: - stage_fn = os.path.basename(sys.argv[0]).replace(".py", "") - - try: - params = yaml.safe_load(open("params.yaml"))[stage_fn] - except KeyError as exc: - console.print(f'ERROR: Key "{stage_fn}" not in parameters.yaml.') - raise KeyError( - f"Is the stage file name ({sys.argv[0]}) " - + "the same as the stage name in params.yaml?" - ) from exc - try: - all_params = yaml.safe_load(open("params.yaml"))["all"] - params = {**params, **all_params} - except KeyError: - console.print( - '[orange]WARNING: Key "all" not in parameters.yaml.' - + "Only returning stage parameters." - ) - - return params diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py deleted file mode 100644 index 9017a1b..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py +++ /dev/null @@ -1,325 +0,0 @@ -""" -Module with functions for wrapping time series models for MLflow. -""" - -import os -import tempfile -import pickle -from typing import Optional, Union, Dict, Any, List - -import mlflow.pyfunc -import pandas as pd -import numpy as np -from mlflow.models import ModelSignature -from ..models.stacking_time_series import StackingTimeSeriesModel -from ..data.transformers import ( - create_transformers_from_training_transformer, - CourierTrainingTransformer, -) - - -class StackingWrapper(mlflow.pyfunc.PythonModel): # type: ignore - """ - MLflow wrapper for StackingTimeSeriesModel. - - Allows the model to be saved and served via MLflow's pyfunc interface. - """ - - def __init__(self, model: Optional[StackingTimeSeriesModel] = None): - self.model = model - - @property - def _console(self): - from rich.console import Console - - return Console() - - def load_context(self, context: Any) -> None: - """Load model from artifact path in MLflow context.""" - try: - model_path = context.artifacts["model"] - self.model = StackingTimeSeriesModel.load(model_path) - self._console.log("[green]Model loaded from context[/green]") - except Exception as e: - self._console.print(f"[red]Error loading model: {e}[/red]") - raise - - def predict( - self, - context: Any, - model_input: Union[pd.DataFrame, np.ndarray, Dict[str, Any]], - ) -> Union[pd.Series, pd.DataFrame, np.ndarray]: - """Run inference using the wrapped model.""" - if self.model is None: - raise ValueError("Model not loaded. Call load_context first.") - - # Extract data from input - if isinstance(model_input, dict): - X = model_input.get("data") - if X is None: - raise ValueError("Dict input must contain 'data' key.") - else: - X = model_input - - # Ensure DataFrame input (models expect pandas DataFrames) - if not isinstance(X, pd.DataFrame): - raise ValueError("Input must be a pandas DataFrame.") - - try: - predictions = self.model.predict(X) - return predictions.to_frame(name=self.model.target_col) - except Exception as e: - self._console.print(f"[red]Prediction failed: {e}[/red]") - raise - - def get_model_summary(self) -> str: - """Return human-readable model summary.""" - return self.model.summary() if self.model else "No model loaded" - - def store_model( - self, - path: Optional[str] = None, - artifact_path: str = "stacking_model", - signature: Optional[ModelSignature] = None, - pip_requirements: Optional[Union[str, list]] = None, - code_path: Optional[List[str]] = None, - to_disk: bool = False, - ) -> None: - """ - Store the model using MLflow pyfunc interface. - - Logs to the current MLflow run by default. Optionally saves locally. - - Args: - path: Local path to save model (required if to_disk=True) - artifact_path: MLflow artifact path - signature: Optional MLflow model signature - pip_requirements: pip requirements (list or path) - code_path: List of local Python source files/directories to bundle - to_disk: Save locally if True, otherwise logs to MLflow - """ - if self.model is None: - raise ValueError("No model to store.") - - with tempfile.TemporaryDirectory() as tmp: - model_artifact = os.path.join(tmp, "stacking_model.pkl") - self.model.save(model_artifact, compression="lzma") - - common_args = { - "python_model": self, - "artifacts": {"model": model_artifact}, - } - if signature: - common_args["signature"] = signature - if pip_requirements: - common_args["pip_requirements"] = pip_requirements - if code_path: - common_args["code_path"] = code_path - - if to_disk: - if not path: - raise ValueError("`path` required for to_disk=True.") - mlflow.pyfunc.save_model(path=path, **common_args) - self._console.log( - f"[blue]Model saved locally to {path}[/blue]" - ) - else: - mlflow.pyfunc.log_model( - artifact_path=artifact_path, **common_args - ) - self._console.log( - f"[green]Model logged to MLflow at '{artifact_path}'" - ) - - def __getstate__(self): - state = self.__dict__.copy() - state["model"] = None # avoid double saving - return state - - def __setstate__(self, state): - self.__dict__.update(state) - - -class TransformerWrapper(mlflow.pyfunc.PythonModel): # type: ignore - """ - MLflow wrapper for data transformers. - - Allows transformers to be saved and served via MLflow's pyfunc interface. - Supports both training and inference transformers. - """ - - def __init__( - self, - transformer: Optional[CourierTrainingTransformer] = None, - ): - """ - Initialize the transformer wrapper. - - Args: - transformer: The training transformer to wrap - """ - self.training_transformer = transformer - self.inference_transformer = None - - @property - def _console(self): - from rich.console import Console - - return Console() - - def load_context(self, context: Any) -> None: - """ - Load training transformer from artifact path and create - inference transformer. - """ - try: - transformer_path = context.artifacts["transformer"] - - with open(transformer_path, "rb") as f: - self.training_transformer = pickle.load(f) - - _, self.inference_transformer = ( - create_transformers_from_training_transformer( - self.training_transformer - ) - ) - - self._console.log( - "[green]Training transformer loaded and inference transformer " - + "created from context[/green]" - ) - except Exception as e: - self._console.print(f"[red]Error loading transformer: {e}[/red]") - raise - - def predict( - self, - context: Any, - model_input: Union[pd.DataFrame, np.ndarray, Dict[str, Any]], - transformer_type: str = "inference", - ) -> Union[pd.Series, pd.DataFrame, np.ndarray]: - """ - Transform data using the selected transformer. - - Args: - context: MLflow context - model_input: Input data to transform (pandas DataFrame expected) - transformer_type: Either "training" or "inference" - """ - if transformer_type == "training": - transformer = self.training_transformer - elif transformer_type == "inference": - transformer = self.inference_transformer - else: - raise ValueError( - "transformer_type must be 'training' or 'inference'" - ) - - if transformer is None: - raise ValueError( - f"{transformer_type.title()} transformer not loaded. " - + "Call load_context first." - ) - - # Extract data from input - if isinstance(model_input, dict): - X = model_input.get("data") - if X is None: - raise ValueError("Dict input must contain 'data' key.") - else: - X = model_input - - # Ensure DataFrame input (transformers expect pandas DataFrames) - if not isinstance(X, pd.DataFrame): - raise ValueError("Input must be a pandas DataFrame.") - - try: - # Apply transformer - transformed_data = transformer.transform(X) - return transformed_data - - except Exception as e: - self._console.print(f"[red]Transformation failed: {e}[/red]") - raise - - def get_transformer_summary(self) -> str: - """Return human-readable transformer summary.""" - if self.training_transformer is None: - return "No training transformer loaded" - - training_class = self.training_transformer.__class__.__name__ - inference_status = ( - "available" if self.inference_transformer else "not created" - ) - return ( - f"{training_class} (training loaded, inference {inference_status})" - ) - - def store_transformer( - self, - path: Optional[str] = None, - artifact_path: str = "transformer", - signature: Optional[ModelSignature] = None, - pip_requirements: Optional[Union[str, list]] = None, - code_path: Optional[List[str]] = None, - to_disk: bool = False, - ) -> None: - """ - Store the training transformer using MLflow pyfunc interface. - - Logs to the current MLflow run by default. Optionally saves locally. - - Args: - path: Local path to save transformer (required if to_disk=True) - artifact_path: MLflow artifact path - signature: Optional MLflow model signature - pip_requirements: pip requirements (list or path) - code_path: List of local Python source files/directories to bundle - to_disk: Save locally if True, otherwise logs to MLflow - """ - if self.training_transformer is None: - raise ValueError("No training transformer to store.") - - with tempfile.TemporaryDirectory() as tmp: - transformer_artifact = os.path.join( - tmp, "training_transformer.pkl" - ) - with open(transformer_artifact, "wb") as f: - pickle.dump(self.training_transformer, f) - - common_args = { - "python_model": self, - "artifacts": {"transformer": transformer_artifact}, - } - if signature: - common_args["signature"] = signature - if pip_requirements: - common_args["pip_requirements"] = pip_requirements - if code_path: - common_args["code_path"] = code_path - - if to_disk: - if not path: - raise ValueError("`path` required for to_disk=True.") - mlflow.pyfunc.save_model(path=path, **common_args) - self._console.log( - "[blue]Training transformer saved locally to " - + f"{path}[/blue]" - ) - else: - mlflow.pyfunc.log_model( - artifact_path=artifact_path, **common_args - ) - self._console.log( - "[green]Training transformer logged to MLflow at " - + f"'{artifact_path}'[/green]" - ) - - def __getstate__(self): - state = self.__dict__.copy() - state["training_transformer"] = None # avoid double saving - state["inference_transformer"] = None # avoid double saving - return state - - def __setstate__(self, state): - self.__dict__.update(state) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py deleted file mode 100644 index 1701c19..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Models package for time series forecasting. - -This package provides standardized interfaces and implementations for -various time series forecasting models. -""" - -from .base import ( - TimeSeriesModel, - UnivariateTimeSeriesModel, - MultivariateTimeSeriesModel, -) -from .factory import create_model, load_model, get_available_models -from .evaluation import timeseries_metrics - -__all__ = [ - "TimeSeriesModel", - "UnivariateTimeSeriesModel", - "MultivariateTimeSeriesModel", - "create_model", - "load_model", - "get_available_models", - "timeseries_metrics", -] diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py deleted file mode 100644 index abf5d6d..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py +++ /dev/null @@ -1,389 +0,0 @@ -""" -ARIMA univariate time series forecasting model implementation. -""" - -from typing import Optional, Tuple - -import numpy as np -import pandas as pd -from statsmodels.tsa.arima.model import ARIMA, ARIMAResults -from rich.console import Console - -from .base import UnivariateTimeSeriesModel, ensure_fitted - -console = Console() - - -class ARIMAModel(UnivariateTimeSeriesModel): - """ARIMA model for univariate time series forecasting. - - This class implements an ARIMA model for forecasting univariate time - series data. It provides methods for fitting the model, making predictions, - forecasting future values, and updating the model with new data. - - Attributes: - order (Tuple[int, int, int]): The (p, d, q) order of the ARIMA model. - model_ (Optional[ARIMA]): The ARIMA model instance. - result_ (Optional[ARIMAResults]): The fitted ARIMA model results. - training_series_ (Optional[pd.Series]): The training data used to fit - the model. - """ - - def __init__( - self, - order: Tuple[int, int, int] = (1, 0, 0), - name: Optional[str] = None, - time_col: str = "ds", - target_col: str = "y", - random_seed: int = 42, - forecast_horizon: int = 2, - ) -> None: - """Initializes the ARIMAModel with specified parameters. - - Args: - order (Tuple[int, int, int]): The (p, d, q) order of the ARIMA - model. - name (Optional[str]): The name of the model. - time_col (str): The name of the time column in the input data. - target_col (str): The name of the target column in the input data. - random_seed (int): The random seed for reproducibility. - forecast_horizon (int): The number of steps to forecast ahead. - """ - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - ) - self.order: Tuple[int, int, int] = order - self.model_: Optional[ARIMA] = None - self.result_: Optional[ARIMAResults] = None - self.training_series_: pd.Series = pd.Series(dtype=float) - self.observed_series_: pd.Series = pd.Series(dtype=float) - self.backtest_predictions_: Optional[pd.Series] = None - self.forecast_horizon: int = forecast_horizon - - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """ - Fits the ARIMA model to the provided training data. - - Args: - y: The target time series data. - X: Optional exogenous variables. - X_val: Validation feature matrix (not used for ARIMA). - y_val: Validation target series (not used for ARIMA). - """ - y_array: np.ndarray = self._validate_y(y) - self.training_series_ = y.copy() - self.observed_series_ = y.copy() - self.model_ = ARIMA(y_array, order=self.order) - self.result_ = self.model_.fit() - - @ensure_fitted - def predict(self, X: Optional[pd.DataFrame] = None) -> pd.Series: - """ - Generates in-sample predictions from the fitted ARIMA model. - - After this method is called, if X is provided, the model will be - updated with the new data, but the coefficients will not be refit. - This is useful for generating predictions on new data without - retraining the model. - - Args: - X (Optional[pd.DataFrame]): Optional dataframe with future - measurements of y for in-sample predictions. - If None, the model will predict on the observed data - (observed_series_). - - Returns: - pd.Series: The in-sample predictions. - - Raises: - ValueError: If the model has not been fitted yet. - """ - if self.result_ is None: - raise ValueError("Model is not fitted.") - - if X is None: - fitted_values = self.result_.fittedvalues - if fitted_values is None: - raise ValueError("Fitted values are None") - return pd.Series( - fitted_values, - index=self.training_series_.index[: len(fitted_values)], - name=self.target_col, - ) - - # Validate the input data - target_series = ( - X[self.target_col] if self.target_col in X else X.iloc[:, 0] - ) - if not isinstance(target_series, pd.Series): - target_series = pd.Series(target_series, index=X.index) - - X_validated = self._validate_y(target_series) - - # Update the model with the validated data without refitting - self.update(pd.Series(X_validated, index=X.index), refit=False) - - if self.result_ is None: - raise ValueError("Model result is None after update") - - fitted_values = self.result_.fittedvalues - if fitted_values is None: - raise ValueError("Fitted values are None") - - return_series = pd.Series( - fitted_values[-len(X) :], index=X.index, name=self.target_col - ) - - return return_series - - @ensure_fitted - def forecast(self, forecast_horizon: int) -> np.ndarray: - """Generates out-of-sample forecasts from the fitted ARIMA model. - TODO: Change return to include index of the forecasted values. - - Args: - forecast_horizon (int): The number of steps to forecast ahead. - - Returns: - np.ndarray: The out-of-sample forecasts. - - Raises: - ValueError: If the model has not been fitted yet. - """ - if self.result_ is None: - raise ValueError("Model is not fitted.") - return self.result_.forecast(steps=forecast_horizon) - - @ensure_fitted - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Perform comprehensive backtesting with periodic retraining. - - This method implements walk-forward validation with periodic - retraining, providing robust evaluation of model performance in - production-like scenarios. Uses 1-step ahead forecasting by default. - - Args: - y: Target time series for backtesting - X: Unused (included for base class compatibility) - retrain_every: Number of steps between model retraining - reuse_previous_execution: Whether to reuse previous backtest results - - Returns: - Series of backtested predictions indexed by timestamp - - Raises: - ValueError: If parameters are invalid or data is insufficient - RuntimeError: If backtesting fails - """ - if self.result_ is None: - raise ValueError("Model is not fitted") - if self.training_series_ is None: - raise ValueError("No training series found") - - # Handle reuse of previous execution - if reuse_previous_execution and self.backtest_predictions_ is not None: - expected_index = y.index - if ( - len(self.backtest_predictions_) == len(expected_index) - and (self.backtest_predictions_.index == expected_index).all() - ): - console.log( - "[yellow]Reusing previous backtest results[/yellow]" - ) - return self.backtest_predictions_ - else: - console.log( - "[yellow]Previous results incompatible, running new" - + " backtest[/yellow]" - ) - - try: - console.log( - f"[blue]Starting ARIMA backtest with {self.forecast_horizon}" - + "-step forecasting...[/blue]" - ) - - # Validate and prepare data - y_sorted = y.sort_index() - - # Check for overlapping data - training_series = self.training_series_ - if any(t in training_series.index for t in y_sorted.index): - console.print( - "[yellow]Warning: Backtest data overlaps with training" - + " data[/yellow]" - ) - - # Initialize backtesting - predictions = [] - - # Start with training data - current_series = training_series.copy() - - total_steps = len(y_sorted) - console.log( - f"[blue]Running {total_steps} backtest steps with retraining" - + f" every {retrain_every} steps...[/blue]" - ) - - # Create initial model state - current_model = ARIMA(current_series.values, order=self.order) - current_result = current_model.fit() - - # Perform walk-forward validation - for i, (timestamp, actual_value) in enumerate(y_sorted.items()): - if i % 50 == 0 and i > 0: # Progress logging - console.log( - f"[blue]Backtest progress: {i}/{len(y_sorted)}[/blue]" - ) - - try: - # Check if we need to retrain - if i % retrain_every == 0 and i > 0: - console.log( - f"[blue]Retraining model at step {i}[/blue]" - ) - current_model = ARIMA( - current_series.values, order=self.order - ) - current_result = current_model.fit() - - # Generate forecast_horizon-step ahead forecast - forecast = current_result.forecast( - steps=self.forecast_horizon - )[0] - predictions.append((timestamp, forecast)) - - # Update the series with actual observed value - current_series = pd.concat( - [ - current_series, - pd.Series([actual_value], index=[timestamp]), - ] - ) - - # For ARIMA, we can extend the model without full refit - if i % retrain_every != 0: - try: - current_result = current_result.extend( - [actual_value], refit=False - ) - except Exception: - # If extend fails, do a quick refit - current_model = ARIMA( - current_series.values, order=self.order - ) - current_result = current_model.fit() - - except Exception as step_error: - console.print( - f"[yellow]Error at step {i}: {step_error}, using" - + " NaN[/yellow]" - ) - predictions.append((timestamp, np.nan)) - - # Still update the series for continuity - current_series = pd.concat( - [ - current_series, - pd.Series([actual_value], index=[timestamp]), - ] - ) - - # Create results series - if predictions: - pred_index, pred_values = zip(*predictions) - self.backtest_predictions_ = pd.Series( - pred_values, - index=pd.Index(pred_index), - name=f"{self.target_col}_backtest", - ) - else: - self.backtest_predictions_ = pd.Series( - dtype=float, name=f"{self.target_col}_backtest" - ) - - console.log( - "[green]ARIMA backtest completed: " - + f"{len(self.backtest_predictions_)} predictions[/green]" - ) - return self.backtest_predictions_ - - except Exception as e: - console.print(f"[red]ARIMA backtest failed: {e}[/red]") - raise RuntimeError(f"Failed to perform backtest: {e}") from e - - @ensure_fitted - def summary(self) -> str: - """Generates a summary of the fitted ARIMA model. - - Returns: - str: The summary of the fitted model. - - Raises: - ValueError: If the model has not been fitted yet. - """ - if self.result_ is None: - raise ValueError("Model is not fitted.") - return str(self.result_.summary()) - - @ensure_fitted - def update(self, new_data: pd.Series, refit: bool = True) -> None: - """Updates the ARIMA model with new observed data. - - This method allows for two modes of updating the model: - 1. **Refitting**: The model is retrained on the combined dataset - (original training data + new data). - 2. **Incremental Update**: The model is updated using the new data - without retraining, preserving the original model parameters. - - Args: - new_data (pd.Series): New observed values to update the model with. - refit (bool, optional): If True, the model is retrained on the - combined dataset. Defaults to True. - - Raises: - TypeError: If `new_data` is not a pandas Series. - ValueError: If `new_data` is empty or if the model has not been - fitted yet. - """ - if not isinstance(new_data, pd.Series): - raise TypeError("new_data must be a pandas Series.") - if new_data.empty: - raise ValueError("new_data is empty.") - - if refit: - self.training_series_ = pd.concat( - [self.training_series_, new_data] - ) - self.observed_series_ = self.training_series_.copy() - y_array = self._validate_y(self.training_series_) - self.model_ = ARIMA(y_array, order=self.order) - self.result_ = self.model_.fit() - else: - self.observed_series_ = pd.concat( - [self.observed_series_, new_data] - ) - y_array = self._validate_y(self.observed_series_) - - if self.result_ is None: - raise ValueError("Model is not fitted.") - self.result_ = self.result_.apply(y_array, refit=False) - if self.result_ is not None: - self.model_ = self.result_.model diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py deleted file mode 100644 index 3721871..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py +++ /dev/null @@ -1,510 +0,0 @@ -""" -CatBoost implementation for multivariate time series forecasting. - -TODO: - - Implement support for categorical features. -""" - -from typing import Optional, List, Union, Tuple, Dict, Any, cast - -import pandas as pd -from catboost import CatBoostRegressor, CatBoostClassifier, Pool -from rich.console import Console -import numpy as np - -from .base import MultivariateTimeSeriesModel, ensure_fitted - -console = Console() - - -class CatBoostTimeSeriesModel(MultivariateTimeSeriesModel): - """ - CatBoost implementation for multivariate time series forecasting. - - This class wraps the CatBoost models with additional functionality for - time series forecasting, following the MultivariateTimeSeriesModel - interface. Supports both regression and classification tasks with - comprehensive error handling and type safety. - """ - - def __init__( - self, - name: Optional[str] = None, - learning_task: str = "regression", - differentiate_target: bool = False, - n_lags: int = 0, - iterations: int = 1000, - learning_rate: float = 0.1, - depth: int = 6, - loss_function: Optional[str] = None, - bins: Optional[List[float]] = None, - random_seed: int = 42, - time_col: str = "ds", - target_col: str = "y", - verbose: bool = False, - ) -> None: - """ - Initialize the CatBoost time series model. - - Args: - name: Optional identifier for the model - learning_task: Type of learning task, either 'regression', - 'multiclass' or 'binary'. - differentiate_target: Whether to differentiate the target - series before fitting the model. - n_lags: Number of lagged target values included as features. - These lags are expected to already be present in the same - dataset as the exogenous features. - iterations: Number of boosting iterations - learning_rate: Learning rate for the model - depth: Depth of the tree - loss_function: Loss function to optimize - bins: Optional list of bin edges for multiclass classification - random_seed: Random seed for reproducibility - time_col: Name of the time column - target_col: Name of the target column - verbose: Whether to enable verbose output - """ - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - n_lags=n_lags, - learning_task=learning_task, - differentiate_target=differentiate_target, - bins=bins, - ) - - self.iterations = iterations - self.learning_rate = learning_rate - self.depth = depth - self.verbose = verbose - - # Set default loss function based on learning task - self.loss_function = self._get_default_loss_function(loss_function) - - # Initialize model state - self.model_: Optional[Union[CatBoostRegressor, CatBoostClassifier]] = ( - None - ) - self.training_series_: Optional[pd.Series] = None - self.X_train_: Optional[pd.DataFrame] = None - self.backtest_predictions_: Optional[pd.Series] = None - - if self.verbose: - console.log( - "[green]Initialized CatBoostTimeSeriesModel:" - + f" {self.summary()}[/green]" - ) - - def _create_model(self) -> Union[CatBoostRegressor, CatBoostClassifier]: - """Creates a new instance of CatBoost model with current parameters. - - Returns: - A new CatBoost model instance (Regressor or Classifier). - """ - base_params = { - "iterations": self.iterations, - "learning_rate": self.learning_rate, - "depth": self.depth, - "loss_function": self.loss_function, - "random_seed": self.random_seed, - "verbose": self.verbose, - } - - if self.learning_task in ["binary", "multiclass"]: - return CatBoostClassifier( - auto_class_weights="Balanced", **base_params - ) - else: - return CatBoostRegressor(**base_params) - - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """ - Core fitting logic for CatBoost model with optional validation data. - - Args: - y: The target time series data - X: The feature matrix (including exogenous features) - X_val: Validation feature matrix (optional) - y_val: Validation target series (optional) - """ - if X is None or not isinstance(X, pd.DataFrame): - raise ValueError("Feature matrix X must be a non-empty DataFrame.") - - y_processed, X_processed, y_val_processed, X_val_processed = ( - self._preprocess_data(y, X, X_val, y_val) - ) - - # Ensure X is not None after preprocessing - if X_processed is None: - raise ValueError( - "Feature matrix X cannot be None after preprocessing." - ) - - X_array, y_array = self._validate_X_y(X_processed, y_processed) - - self.training_series_ = y_processed.copy() - self.X_train_ = X_processed.copy() - self.model_ = self._create_model() - - eval_set = None - if X_val_processed is not None and y_val_processed is not None: - X_val_array, y_val_array = self._validate_X_y( - X_val_processed, y_val_processed - ) - eval_set = Pool(data=X_val_array, label=y_val_array) - - train_pool = Pool(data=X_array, label=y_array) - - if self.verbose: - console.log( - f"[blue]Training CatBoost model for {self.iterations}" - + " iterations...[/blue]" - ) - - self.model_.fit(train_pool, eval_set=eval_set) - - if self.verbose: - console.log( - "[green]CatBoost model training completed successfully[/green]" - ) - - @ensure_fitted - def predict(self, X: pd.DataFrame) -> pd.Series: - """ - Generate predictions using the fitted CatBoost model. - - Args: - X: The feature matrix for prediction - - Returns: - pd.Series: Predicted values - """ - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - - # Create a copy to avoid modifying the original DataFrame - X_pred = X.copy() - - if self.selected_features_ is not None: - X_pred = cast(pd.DataFrame, X_pred[self.selected_features_]) - - X_array = self._validate_X(X_pred) - predictions = self.model_.predict(X_array) - - # Convert predictions to numpy array if needed - if hasattr(predictions, "squeeze"): - predictions = predictions.squeeze() - elif isinstance(predictions, list): - predictions = np.array(predictions) - - return_series = pd.Series( - predictions, - index=X.index, - name=self.target_col, - ) - return return_series - - @ensure_fitted - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Performs backtesting (walk-forward validation) with periodic - retraining. - - This method simulates a production scenario by iterating through a test - set, making a one-step-ahead prediction, and then retraining the model - periodically with the newly available data. - - Args: - X: DataFrame with features for the backtesting period. - y: Series with the true target values for the backtesting period. - retrain_every: The frequency of retraining. The model will be - retrained every `retrain_every` steps. - reuse_previous_execution: Whether to reuse the previous execution - of a backtest. If True, any overlapping data between the - previous execution and the current execution will be used - without retraining the model. - Returns: - A series of backtested predictions, indexed by the backtest data's - index. - """ - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - if self.training_series_ is None: - raise ValueError("Training series is not set.") - if self.X_train_ is None: - raise ValueError("Training feature matrix is not set.") - if X is None or not isinstance(X, pd.DataFrame): - raise ValueError("Feature matrix X must be a non-empty DataFrame.") - - if reuse_previous_execution: - if self.backtest_predictions_ is None: - raise ValueError("No previous execution found.") - if (self.backtest_predictions_.shape[0] != y.shape[0]) or ( - not (self.backtest_predictions_.index == y.index).all() - ): - raise ValueError( - "Previous execution index does not match y index." - ) - return self.backtest_predictions_ - # Prepare - total_steps = len(X) - predictions = [] - current_model = self.model_ - y_history = self.training_series_.copy() - X_history = self.X_train_.copy() - - # Iterate in chunks instead of single steps - for start in range(0, total_steps, retrain_every): - end = min(start + retrain_every, total_steps) - - # Batch prediction for current chunk - X_chunk = X.iloc[start:end].copy() - if self.selected_features_: - X_chunk = X_chunk[self.selected_features_] - - X_array = self._validate_X(X_chunk) - preds = current_model.predict(X_array) - - # Handle different prediction formats - if hasattr(preds, "squeeze"): - preds = preds.squeeze() - if preds.ndim == 0: # single point - preds = [preds] - predictions.extend(preds) - - # Update training history - y_chunk = y.iloc[start:end] - y_history = pd.concat([y_history, y_chunk]) - X_history = pd.concat([X_history, X_chunk]) - - # Retrain the model for next chunk (if needed) - if end < total_steps: - if self.verbose: - console.print( - f"[cyan]Backtesting: Retraining at step {end}..." - ) - - current_model = self._create_model() - (y_fit, X_fit, _, _) = ( - self._preprocess_data(y_history, X_history) - ) - - # Ensure X_fit is not None after preprocessing - if X_fit is None: - raise ValueError( - "Feature matrix cannot be None after preprocessing." - ) - - X_fit_array, y_fit_array = self._validate_X_y(X_fit, y_fit) - train_pool = Pool(data=X_fit_array, label=y_fit_array) - current_model.fit(train_pool) - - # Store backtest predictions for potential reuse - self.backtest_predictions_ = pd.Series( - predictions, index=X.index, name=f"{self.target_col}_pred" - ) - return self.backtest_predictions_ - - def select_features( - self, - X: pd.DataFrame, - y: pd.Series, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - features_to_select: Optional[int] = None, - algorithm: str = "RecursiveByShapValues", - steps: int = 1, - verbose: bool = False, - ) -> List[str]: - """Identify and select the most important features. - - Uses CatBoost's built-in feature selection capabilities to determine - feature importance and select the most relevant features. - - Args: - X: The feature matrix - y: The target series - X_val: Optional validation feature matrix - y_val: Optional validation target series - features_to_select: Number of features to select. If None, - will select half of the features. - algorithm: Feature selection algorithm. One of: - 'RecursiveByShapValues', 'RecursiveByPredictionValuesChange' - steps: How many times a full model will be trained. - More steps give more accurate results. - verbose: Whether to print progress - - Returns: - List[str]: List of selected feature names - """ - (y_processed, X_processed, y_val_processed, X_val_processed) = ( - self._preprocess_data(y, X, X_val, y_val) - ) - - # Validate input data - if X_processed is None or not isinstance(X_processed, pd.DataFrame): - raise ValueError("Feature matrix X must be a non-empty DataFrame.") - X_array, y_array = self._validate_X_y(X_processed, y_processed) - - # Set default number of features to select if not specified - if features_to_select is None: - features_to_select = X_processed.shape[1] // 2 - - # Create and prepare model - temp_model = self._create_model() - train_pool = Pool(data=X_array, label=y_array) - - # Prepare validation data if provided - eval_set = None - if X_val_processed is not None and y_val_processed is not None: - X_val_array, y_val_array = self._validate_X_y( - X_val_processed, y_val_processed - ) - eval_set = Pool(data=X_val_array, label=y_val_array) - - # Perform feature selection - selected_features = temp_model.select_features( - train_pool, - eval_set=eval_set, - features_for_select=list(range(X_processed.shape[1])), - num_features_to_select=features_to_select, - algorithm=algorithm, - steps=steps, - logging_level="Verbose" if verbose else "Silent", - train_final_model=False, - ) - - # Map feature indices to feature names with proper type casting - selected_feature_names: List[str] = [ - str(X_processed.columns[idx]) - for idx in selected_features["selected_features"] - ] - self.selected_features_ = selected_feature_names - self.feature_names_in_ = selected_feature_names - self.n_features_in_ = len(selected_feature_names) - - return selected_feature_names - - @classmethod - def tune_hyperparameters( - cls, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - selected_features: Optional[List[str]] = None, - param_grid: Optional[Dict[str, Any]] = None, - n_trials: int = 10, - early_stopping_rounds: Optional[int] = 50, - random_seed: int = 42, - **kwargs, - ) -> Tuple[Dict[str, Any], "CatBoostTimeSeriesModel"]: - """ - Tune hyperparameters for the CatBoost model. - - Args: - y: The target time series data - X: The feature matrix (including exogenous features) - X_val: Validation feature matrix (optional) - y_val: Validation target series (optional) - selected_features: List of features to use for tuning - param_grid: Dictionary of hyperparameters to search - n_trials: Number of trials for hyperparameter tuning - early_stopping_rounds: Number of rounds for early stopping - random_seed: Random seed for reproducibility - **kwargs: Additional keyword arguments for model initialization - - Returns: - Tuple[Dict[str, Any], CatBoostTimeSeriesModel]: Best hyperparameters - and fitted model - """ - if n_trials <= 0: - raise ValueError("n_trials must be a positive integer.") - # Create a temporary model instance to use its preprocessing method - temp_model = cls( - learning_task=kwargs.get("learning_task", "regression"), - differentiate_target=kwargs.get("differentiate_target", False), - bins=kwargs.get("bins", None), - random_seed=random_seed, - ) - if selected_features is not None: - temp_model.selected_features_ = selected_features - - (y_processed, X_processed, _, _) = ( - temp_model._preprocess_data(y, X, X_val, y_val) - ) - - if kwargs.get("learning_task", "regression") == "classification": - search_model = CatBoostClassifier( - random_seed=random_seed, - logging_level="Silent", - early_stopping_rounds=early_stopping_rounds, - loss_function=kwargs.get("loss_function", "Logloss"), - class_weights="Balanced", - ) - else: - search_model = CatBoostRegressor( - random_seed=random_seed, - logging_level="Silent", - early_stopping_rounds=early_stopping_rounds, - loss_function=kwargs.get("loss_function", "RMSE"), - ) - - if param_grid is None: - param_grid = { - "iterations": [100, 500, 1000, 2000], - "learning_rate": [0.01, 0.05, 0.1, 0.2], - "depth": [4, 6, 8], - } - - if X_processed is None: - raise ValueError( - "Feature matrix cannot be None after preprocessing." - ) - - train_pool = Pool(data=X_processed, label=y_processed) - results = search_model.randomized_search( - param_grid, - X=train_pool, - n_iter=n_trials, - verbose=False, - refit=False, - ) - - best_params = results["params"] - - best_model = cls( - iterations=best_params["iterations"], - learning_rate=best_params["learning_rate"], - depth=best_params["depth"], - random_seed=random_seed, - time_col=kwargs.get("time_col", "ds"), - target_col=kwargs.get("target_col", "y"), - n_lags=kwargs.get("n_lags", 0), - name=kwargs.get("name", None), - loss_function=kwargs.get("loss_function", None), - learning_task=kwargs.get("learning_task", "regression"), - bins=kwargs.get("bins", None), - differentiate_target=kwargs.get("differentiate_target", False), - ) - if selected_features is not None: - best_model.selected_features_ = selected_features - - best_model.fit(y, X, X_val, y_val) - - return best_params, best_model diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py deleted file mode 100644 index 77331d7..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -Module with functions of timeseries evaluation. -""" - -from typing import Optional, List, Sequence - -import numpy as np -import pandas as pd -from rich.console import Console -from sklearn.metrics import ( - mean_absolute_error, - mean_squared_error, - accuracy_score, - f1_score, - confusion_matrix, -) -console = Console() - - -def timeseries_metrics( - y_pred: Sequence[float], y_true: Sequence[float] -) -> dict[str, float]: - """ - Compute MAE, MSE, and trend capture for time series predictions. - - Parameters: - y_pred (ArrayLike): Predicted values. - y_true (ArrayLike): Ground truth values. - - Returns: - dict[str, float]: Dictionary with MAE, MSE, and trend_capture. - """ - if len(y_true) != len(y_pred): - raise ValueError("y_true and y_pred must have the same length") - if len(y_true) == 0: - raise ValueError("y_true and y_pred must not be empty") - y_true_np = np.asarray(y_true) - y_pred_np = np.asarray(y_pred) - - mae = mean_absolute_error(y_true_np, y_pred_np) - mse = mean_squared_error(y_true_np, y_pred_np) - - # Compute directional trend: 1 if up, 0 if down or flat - if len(y_true) == 1: - return {"MAE": mae, "MSE": mse, "trend_capture": 1.0} - - true_trend = np.diff(y_true_np) > 0 - pred_trend = np.diff(y_pred_np) > 0 - - trend_capture = np.mean(true_trend == pred_trend) - - return {"MAE": mae, "MSE": mse, "trend_capture": trend_capture} - - -def timeseries_classification_metrics( - y_pred: Sequence[float], - y_true: Sequence[float], - bins: Optional[List] = None, -) -> dict[str, float]: - """ - Compute accuracy for classification predictions. - - Parameters: - y_pred (ArrayLike): Predicted values. - y_true (ArrayLike): Ground truth values. - bins (List[int], optional): Bin edges for categorizing predictions. - - Returns: - dict[str, float]: Dictionary with accuracy. - """ - if bins is not None: - console.log( - f"Using bins for classification: {bins}" - ) - y_true_binned = pd.cut(y_true, bins=bins, labels=False) - else: - console.log("No bins provided, using default classification (y > 0).") - y_true_binned = (y_true > 0).astype(int) - console.log( - f"y_true_binned: {y_true_binned.value_counts()}" - ) - console.log( - f"y_pred: {pd.Series(y_pred).value_counts()}" - ) - - y_pred = np.array(y_pred).astype(int) - acc = accuracy_score(y_true_binned, y_pred) - f1 = f1_score(y_true_binned, y_pred, average="weighted") - # Calculate the confusion matrix - cm = confusion_matrix(y_true_binned, y_pred) - - return {"accuracy": acc, "f1_score": f1, "confusion_matrix": cm} diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py deleted file mode 100644 index a8d646e..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -Model factory for creating, loading, and discovering time series models. - -This module provides centralized utility functions to handle different time -series model implementations based on a string identifier. It uses a -central registry (`SUPPORTED_MODELS`) that maps model type strings (e.g., -'arima') to their corresponding model classes (e.g., ARIMAModel). This -approach allows for easy extension and decouples model instantiation logic -from the code that uses the models. - -Key Functions: - create_model: Creates a new instance of a specified model type by looking - up the type string in the `SUPPORTED_MODELS` registry and - passing keyword arguments to the retrieved model class's - constructor. - load_model: Loads a previously saved model instance from disk. It uses - the provided model type string to find the correct class in - the registry and then calls that class's `.load()` classmethod. - get_available_models: Returns a dictionary listing the registered model - types (keys in `SUPPORTED_MODELS`) and their - descriptions, automatically derived from the model - class docstrings. - -Extensibility: - Adding support for a new model involves the following steps: - 1. Ensure the new model class (e.g., `MyNewModel`) inherits from the - appropriate base class (e.g., `TimeSeriesModel`) and implements all - required abstract methods. - 2. Ensure the new model class has a `.load()` classmethod compatible - with the `save()` method in the base `Model` class (if loading is - to be supported via this factory). - 3. Import the new model class into this factory module. - 4. Add an entry to the `SUPPORTED_MODELS` dictionary, mapping a unique, - lowercase string identifier to the model class itself: - `SUPPORTED_MODELS = {..., "mynewmodel": MyNewModel}` - Once added to the registry, the model can be created and loaded via the - factory functions, and it will automatically appear in the output of - `get_available_models()`. -""" - -from typing import Dict, Type -import logging - -from .base import TimeSeriesModel -from .arima import ARIMAModel -from .neural_prophet_model import NeuralProphetModel -from .catboost_time_series import CatBoostTimeSeriesModel -from .linear_regression_time_series import ElasticNetTimeSeriesModel -from .stacking_time_series import StackingTimeSeriesModel -# from .prophet import ProphetModel # Example for future - -# *** Central registry of supported models -SUPPORTED_MODELS: Dict[str, Type[TimeSeriesModel]] = { - "arima": ARIMAModel, - "neuralprophet": NeuralProphetModel, - "catboost": CatBoostTimeSeriesModel, - "elasticnet": ElasticNetTimeSeriesModel, - "stacking": StackingTimeSeriesModel, - # "prophet": ProphetModel, # Add new models here -} - - -def create_model(model_type: str, **kwargs) -> TimeSeriesModel: - """ - Create a new model instance of the specified type using a registry. - - Args: - model_type: Type of model to create (case-insensitive). - **kwargs: Model-specific parameters passed to its constructor. - - Returns: - New model instance inheriting from TimeSeriesModel. - - Raises: - ValueError: If the model type is not supported or kwargs are invalid. - """ - model_type = model_type.lower() - model_class = SUPPORTED_MODELS.get(model_type) - - if model_class: - try: - instance = model_class(**kwargs) - return instance - except TypeError as e: - logging.error(f"Kwargs issue for {model_type}: {kwargs}") - raise ValueError( - f"Invalid parameters for model type '{model_type}'. Error: {e}" - ) from e - else: - supported_list = ", ".join(f"'{k}'" for k in SUPPORTED_MODELS.keys()) - raise ValueError( - f"Unsupported model type: '{model_type}'. " - f"Currently supported models are: {supported_list}." - ) - - -def load_model(path: str, model_type: str) -> TimeSeriesModel: - """ - Load a model from disk using a registry. - - Args: - path: Path to the saved model. - model_type: Expected type of model to load (case-insensitive). - - Returns: - Loaded model instance. - - Raises: - ValueError: If the model type is not supported. - # Other errors might come from the underlying .load() method - """ - model_type = model_type.lower() - model_class = SUPPORTED_MODELS.get(model_type) - - if model_class: - return model_class.load(path) - else: - supported_list = ", ".join(f"'{k}'" for k in SUPPORTED_MODELS.keys()) - raise ValueError( - f"Unsupported model type: '{model_type}'. " - f"Currently supported models are: {supported_list}." - ) - - -def get_available_models() -> Dict[str, str]: - """ - Dynamically get a dictionary of available model types and their - descriptions from the SUPPORTED_MODELS registry and class docstrings. - - Returns: - Dictionary mapping model type names to descriptions. - """ - available = { - type_name: ( - model_class.__doc__.strip().splitlines()[0] - if model_class.__doc__ else "No description available." - ) - for type_name, model_class in SUPPORTED_MODELS.items() - } - return available diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py deleted file mode 100644 index 9468954..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py +++ /dev/null @@ -1,303 +0,0 @@ -""" -ElasticNet implementation for multivariate time series forecasting. -""" - -from typing import Optional, Sequence - -import pandas as pd -from sklearn.linear_model import ElasticNet -from sklearn.impute import SimpleImputer -from rich.console import Console - -from .base import MultivariateTimeSeriesModel, ensure_fitted - -console = Console() - - -class ElasticNetTimeSeriesModel(MultivariateTimeSeriesModel): - """ - ElasticNet implementation for multivariate time series forecasting. - - This class wraps the scikit-learn ElasticNet model with additional - functionality for time series forecasting, following the - MultivariateTimeSeriesModel interface. It's suitable for regression - tasks where features might be correlated. - """ - - def __init__( - self, - name: Optional[str] = None, - n_lags: int = 1, - alpha: float = 1.0, - l1_ratio: float = 0.5, - fit_intercept: bool = True, - max_iter: int = 1000, - tol: float = 1e-4, - random_seed: int = 42, - time_col: str = "ds", - target_col: str = "y", - differentiate_target: bool = False, - bins: Optional[list] = None, - learning_task: Optional[str] = None, - ) -> None: - """ - Initialize the ElasticNet time series model. - - Args: - name: Optional identifier for the model. - n_lags: Number of lagged target values to include as inputs. - alpha: Constant that multiplies the penalty terms. - l1_ratio: The ElasticNet mixing parameter (0 <= l1_ratio <= 1). - For l1_ratio = 0, it's L2 penalty (Ridge). - For l1_ratio = 1, it's L1 penalty (Lasso). - fit_intercept: Whether to calculate the intercept for this model. - max_iter: Maximum number of iterations. - tol: Tolerance for stopping criteria. - random_seed: Random seed for reproducibility. - time_col: Name of the time column. - target_col: Name of the target column. - differentiate_target: Whether to apply differencing to make series - stationary. - bins: Bin edges for multiclass classification target - transformation. - learning_task: Type of learning task ('regression', 'binary', - 'multiclass'). - """ - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - n_lags=n_lags, - differentiate_target=differentiate_target, - bins=bins, - learning_task=learning_task, - ) - - self.alpha = alpha - self.l1_ratio = l1_ratio - self.fit_intercept = fit_intercept - self.max_iter = max_iter - self.tol = tol - - self.model_: Optional[ElasticNet] = None - self.training_series_: Optional[pd.Series] = None - self.imputer_: Optional[SimpleImputer] = None - self.X_train_: Optional[pd.DataFrame] = None - - def _create_model(self) -> ElasticNet: - """Creates a new instance of ElasticNet with current parameters. - - Returns: - A new ElasticNet model instance. - """ - return ElasticNet( - alpha=self.alpha, - l1_ratio=self.l1_ratio, - fit_intercept=self.fit_intercept, - max_iter=self.max_iter, - tol=self.tol, - random_state=self.random_seed, - ) - - def _impute_missing_values(self, X: pd.DataFrame) -> pd.DataFrame: - """ - Handle missing values in the feature matrix using median imputation. - If the imputer is not fitted, it will be fitted on the data. - - Args: - X: The feature matrix potentially containing missing values. - - Returns: - pd.DataFrame: The feature matrix with imputed values. - """ - if self.imputer_ is None: - self.imputer_ = SimpleImputer( - strategy="median", copy=True, add_indicator=False - ) - # Fit the imputer and transform the data - imputed_values = self.imputer_.fit_transform(X) - else: - # Use the fitted imputer to transform new data - imputed_values = self.imputer_.transform(X) - - # Convert back to DataFrame with original index and column names - return pd.DataFrame(imputed_values, index=X.index, columns=X.columns) - - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """ - Core fitting logic for the ElasticNet model. - - Args: - y: The target time series data. - X: The feature matrix (including exogenous features). - X_val: Validation feature matrix (ignored). - y_val: Validation target series (ignored). - """ - # Use base class preprocessing - y_processed, X_processed, _, _ = self._preprocess_data( - y, X, X_val, y_val - ) - - if X_processed is None or not isinstance(X_processed, pd.DataFrame): - raise ValueError("Feature matrix X must be a non-empty DataFrame.") - - # First impute missing values in X - X_imputed = self._impute_missing_values(X_processed) - - # Validate X and y after imputation - X_array, y_array = self._validate_X_y( - X_imputed, y_processed, allow_nan=False - ) - - self.training_series_ = y_processed.copy() - self.model_ = self._create_model() - self.model_.fit(X_array, y_array) - self.X_train_ = X_processed.copy() - - @ensure_fitted - def predict(self, X: Optional[pd.DataFrame] = None) -> Sequence: - """ - Generate predictions using the fitted ElasticNet model. - - Args: - X: The feature matrix for prediction. - - Returns: - pd.Series: Predicted values with the original index. - """ - if X is None: - raise ValueError("Feature matrix X is required for prediction.") - - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - if self.training_series_ is None: - raise ValueError("Training series is not available.") - if self.imputer_ is None: - raise ValueError("Imputer is not fitted yet.") - - # If target column is present, drop it - X_pred = X.copy() - if self.target_col in X_pred.columns: - X_pred = X_pred.drop(columns=[self.target_col]) - - # For prediction, we need to reconstruct lagged features - # This is a simplified approach - in practice, you'd need - # the historical target values to create proper lags - - # Handle missing values using fitted imputer - X_processed = self._impute_missing_values(X_pred) - - # Validate X after imputation - X_array = self._validate_X(X_processed, allow_nan=False) - - predictions = self.model_.predict(X_array) - - return pd.Series( - predictions, index=X_processed.index, name=self.target_col - ) - - @ensure_fitted - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Performs backtesting (walk-forward validation) with periodic - retraining. - - Args: - y: The target time series data. - X: Optional exogenous features. - retrain_every: Number of steps after which to retrain the model. - reuse_previous_execution: Whether to reuse the previous execution - of a backtest. If True, any overlapping data between the - previous execution and the current execution will be used - without retraining the model. - - Returns: - Series of predictions for each step in the time series. - """ - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - if self.training_series_ is None: - raise ValueError("Training series is not set.") - if self.X_train_ is None: - raise ValueError("Training feature matrix is not set.") - if X is None or not isinstance(X, pd.DataFrame): - raise ValueError("Feature matrix X must be a non-empty DataFrame.") - - total_steps = len(X) - predictions = [] - current_model = self.model_ - y_history = self.training_series_.copy() - X_history = self.X_train_.copy() - - # Iterate in chunks instead of single steps - for start in range(0, total_steps, retrain_every): - end = min(start + retrain_every, total_steps) - - # Batch prediction for current chunk - X_chunk = X.iloc[start:end].copy() - if self.selected_features_: - X_chunk = X_chunk[self.selected_features_] - X_imputed = self._impute_missing_values(X_chunk) - X_array = self._validate_X(X_imputed, allow_nan=False) - preds = current_model.predict(X_array).squeeze() - if preds.ndim == 0: # single point - preds = [preds] - predictions.extend(preds) - - # Update training history - y_chunk = y.iloc[start:end] - y_history = pd.concat([y_history, y_chunk]) - X_history = pd.concat([X_history, X_chunk]) - - # Retrain the model for next chunk (if needed) - if end < total_steps: - console.print( - f"[cyan]Backtesting: Retraining at step {end}...[/cyan]" - ) - - current_model = self._create_model() - y_fit, X_fit, *_ = self._preprocess_data(y_history, X_history) - X_fit_imputed = self._impute_missing_values(X_fit) - X_fit_array, y_fit_array = self._validate_X_y( - X_fit_imputed, y_fit, allow_nan=False - ) - current_model.fit(X_fit_array, y_fit_array) - - return pd.Series( - predictions, index=X.index, name=f"{self.target_col}_pred" - ) - - @ensure_fitted - def feature_importance(self) -> Optional[pd.DataFrame]: - """ - Returns feature importance based on model coefficients. - - Returns: - A DataFrame with feature names and their corresponding - coefficients (importance scores), or None if no features. - """ - if self.model_ is None or self.feature_names_in_ is None: - return None - - importance = self.model_.coef_ - feature_importance_df = pd.DataFrame( - {"Feature": self.feature_names_in_, "Importance": importance} - ) - feature_importance_df = feature_importance_df.sort_values( - by="Importance", key=abs, ascending=False - ).reset_index(drop=True) - - return feature_importance_df diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py deleted file mode 100644 index e79f519..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py +++ /dev/null @@ -1,888 +0,0 @@ -""" -NeuralProphet implementation for univariate time series forecasting. - -This module provides a comprehensive wrapper around the NeuralProphet library, -implementing enterprise-level features including robust error handling, -parameter validation, type safety, and integration with the base model -architecture. -""" - -import os -from typing import Optional, cast, Tuple, Dict, Any - -import numpy as np -import pandas as pd -import torch -from neuralprophet import NeuralProphet -from rich.console import Console - -from .base import UnivariateTimeSeriesModel, ensure_fitted - -console = Console() - -# Configure PyTorch for optimal performance -torch.set_num_threads(os.cpu_count() or 1) - - -class NeuralProphetModel(UnivariateTimeSeriesModel): - """ - Enterprise-grade NeuralProphet implementation for univariate - time series forecasting. - - This class provides a robust wrapper around the NeuralProphet model with - comprehensive error handling, parameter validation, and integration with - the base model architecture. It includes features like automatic data - validation, performance monitoring, and enterprise-level logging. - - Key Features: - - Comprehensive parameter validation - - Robust error handling with detailed diagnostics - - Memory-efficient data processing - - Integration with base model utilities - - Performance monitoring and logging - - Support for various seasonality patterns - - Flexible forecasting capabilities - - Example: - >>> model = NeuralProphetModel( - ... n_lags=7, - ... n_forecasts=3, - ... epochs=50, - ... weekly_seasonality=True - ... ) - >>> model.fit(y_train) - >>> predictions = model.predict() - >>> future_forecast = model.forecast(forecast_horizon=3) - """ - - # Class constants for validation - VALID_SEASONALITY_MODES = {"additive", "multiplicative"} - VALID_LOSS_FUNCTIONS = {"Huber", "MSE", "MAE"} - VALID_NORMALIZE_OPTIONS = {"auto", "soft", "off", "minmax"} - MIN_EPOCHS = 1 - MAX_EPOCHS = 10000 - MIN_N_LAGS = 0 - MAX_N_LAGS = 365 - MIN_N_FORECASTS = 1 - MAX_N_FORECASTS = 365 - - def __init__( - self, - name: Optional[str] = None, - n_lags: int = 1, - n_forecasts: int = 2, - weekly_seasonality: bool = True, - daily_seasonality: bool = True, - yearly_seasonality: bool = False, - seasonality_mode: str = "additive", - epochs: int = 100, - learning_rate: Optional[float] = None, - batch_size: Optional[int] = None, - loss_func: str = "Huber", - normalize: str = "auto", - impute_missing: bool = True, - drop_missing: bool = False, - time_col: str = "ds", - target_col: str = "y", - random_seed: int = 42, - ): - """ - Initialize the NeuralProphet time series model with comprehensive - validation. - - Args: - name: Optional identifier for the model - n_lags: Number of lagged target values to include as inputs (0-365) - n_forecasts: Number of steps ahead to forecast (1-365) - weekly_seasonality: Whether to include weekly seasonality - daily_seasonality: Whether to include daily seasonality - yearly_seasonality: Whether to include yearly seasonality - seasonality_mode: Type of seasonality ('additive' or - 'multiplicative') - epochs: Number of training epochs (1-10000) - learning_rate: Learning rate for optimizer (auto if None) - batch_size: Training batch size (auto if None) - loss_func: Loss function ('Huber', 'MSE', 'MAE') - normalize: Normalization type ('auto', 'soft', 'off', 'minmax') - impute_missing: Whether to automatically impute missing values - drop_missing: Whether to drop missing values in training data - time_col: Name of the time column - target_col: Name of the target column - random_seed: Random seed for reproducibility - - Raises: - ValueError: If any parameters are invalid - TypeError: If parameters have incorrect types - """ - self._validate_and_set_parameters( - n_lags=n_lags, - n_forecasts=n_forecasts, - seasonality_mode=seasonality_mode, - epochs=epochs, - learning_rate=learning_rate, - batch_size=batch_size, - loss_func=loss_func, - normalize=normalize, - weekly_seasonality=weekly_seasonality, - daily_seasonality=daily_seasonality, - yearly_seasonality=yearly_seasonality, - impute_missing=impute_missing, - drop_missing=drop_missing, - ) - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - n_lags=n_lags, - ) - self.forecast_horizon = n_forecasts - # Initialize model state - self.model_: Optional[NeuralProphet] = None - self.backtest_predictions_: Optional[pd.Series] = None - self._training_metrics: Dict[str, float] = {} - - console.log( - f"[green]Initialized NeuralProphetModel: {self.summary()}[/green]" - ) - - def _validate_and_set_parameters( - self, - n_lags: int, - n_forecasts: int, - seasonality_mode: str, - epochs: int, - learning_rate: Optional[float], - batch_size: Optional[int], - loss_func: str, - normalize: str, - weekly_seasonality: bool, - daily_seasonality: bool, - yearly_seasonality: bool, - impute_missing: bool, - drop_missing: bool, - ) -> None: - """Validate and set model parameters with comprehensive checks.""" - # Validate integer parameters - if not (self.MIN_N_LAGS <= n_lags <= self.MAX_N_LAGS): - raise ValueError( - f"n_lags must be between {self.MIN_N_LAGS} and " - + f"{self.MAX_N_LAGS}, got {n_lags}" - ) - - if not (self.MIN_N_FORECASTS <= n_forecasts <= self.MAX_N_FORECASTS): - raise ValueError( - f"n_forecasts must be between {self.MIN_N_FORECASTS} and " - + f"{self.MAX_N_FORECASTS}, got {n_forecasts}" - ) - - if not (self.MIN_EPOCHS <= epochs <= self.MAX_EPOCHS): - raise ValueError( - f"epochs must be between {self.MIN_EPOCHS} and " - + f"{self.MAX_EPOCHS}, got {epochs}" - ) - - # Validate string parameters - if seasonality_mode not in self.VALID_SEASONALITY_MODES: - raise ValueError( - "seasonality_mode must be one of " - + f"{self.VALID_SEASONALITY_MODES}, got {seasonality_mode}" - ) - - if loss_func not in self.VALID_LOSS_FUNCTIONS: - raise ValueError( - "loss_func must be one of " - + f"{self.VALID_LOSS_FUNCTIONS}, got {loss_func}" - ) - - if normalize not in self.VALID_NORMALIZE_OPTIONS: - raise ValueError( - "normalize must be one of " - + f"{self.VALID_NORMALIZE_OPTIONS}, got {normalize}" - ) - - # Validate optional float parameters - if learning_rate is not None: - if ( - not isinstance(learning_rate, (int, float)) - or learning_rate <= 0 - ): - raise ValueError( - "learning_rate must be a positive number, " - + f"got {learning_rate}" - ) - - if batch_size is not None: - if not isinstance(batch_size, int) or batch_size <= 0: - raise ValueError( - "batch_size must be a positive integer, " - + f"got {batch_size}" - ) - - # Validate boolean parameters - for param_name, param_value in [ - ("weekly_seasonality", weekly_seasonality), - ("daily_seasonality", daily_seasonality), - ("yearly_seasonality", yearly_seasonality), - ("impute_missing", impute_missing), - ("drop_missing", drop_missing), - ]: - if not isinstance(param_value, bool): - raise TypeError( - f"{param_name} must be a boolean, " - + f"got {type(param_value)}" - ) - - # Set validated parameters - self.n_forecasts = n_forecasts - self.weekly_seasonality = weekly_seasonality - self.daily_seasonality = daily_seasonality - self.yearly_seasonality = yearly_seasonality - self.seasonality_mode = seasonality_mode - self.epochs = epochs - self.learning_rate = learning_rate - self.batch_size = batch_size - self.loss_func = loss_func - self.normalize = normalize - self.impute_missing = impute_missing - self.drop_missing = drop_missing - - def _create_model(self) -> NeuralProphet: - """ - Create a new NeuralProphet instance with validated parameters. - - Returns: - A new NeuralProphet model instance configured with current - parameters. - - Raises: - RuntimeError: If model creation fails - """ - try: - model_params = { - "n_lags": self.n_lags, - "n_forecasts": self.n_forecasts, - "weekly_seasonality": self.weekly_seasonality, - "daily_seasonality": self.daily_seasonality, - "yearly_seasonality": self.yearly_seasonality, - "seasonality_mode": self.seasonality_mode, - "loss_func": self.loss_func, - "normalize": self.normalize, - "impute_missing": self.impute_missing, - "drop_missing": self.drop_missing, - "impute_rolling": 1000000, - "impute_linear": 100000, - } - - # Add optional parameters if specified - if self.learning_rate is not None: - model_params["learning_rate"] = self.learning_rate - if self.batch_size is not None: - model_params["batch_size"] = self.batch_size - - console.log( - f"[blue]Creating NeuralProphet with params: " - f"{model_params}[/blue]" - ) - return NeuralProphet(**model_params) - - except Exception as e: - raise RuntimeError( - f"Failed to create NeuralProphet model: {e}" - ) from e - - def _validate_and_prepare_data( - self, y: pd.Series - ) -> Tuple[pd.DataFrame, pd.Series]: - """ - Validate and prepare time series data for NeuralProphet. - - Args: - y: Input time series data - - Returns: - Tuple of (prepared_dataframe, validated_series) - - Raises: - ValueError: If data validation fails - """ - try: - # Validate target series - y_array = self._validate_y(y) - - # Ensure datetime index - if not pd.api.types.is_datetime64_any_dtype(y.index): - try: - y_datetime = y.copy() - y_datetime.index = pd.to_datetime(y.index) - console.log("[yellow]Converted index to datetime[/yellow]") - except Exception as e: - raise ValueError( - "y's index must be a DateTime index or convertible " - f"to DateTime. Conversion failed: {e}" - ) from e - else: - y_datetime = y.copy() - - # Check for minimum data requirements - if len(y_datetime) < max(self.n_lags + 1, 10): - raise ValueError( - "Insufficient data: need at least " - + f"{max(self.n_lags + 1, 10)} observations, " - + f"got {len(y_datetime)}" - ) - - # Create NeuralProphet format DataFrame - df = pd.DataFrame({"ds": y_datetime.index, "y": y_array}) - - # Validate for missing values if not configured to handle them - if not self.impute_missing and bool(df["y"].isna().any()): - raise ValueError( - "Data contains missing values but impute_missing=False. " - "Either set impute_missing=True or clean the data." - ) - - console.log( - f"[green]Data validation successful: {len(df)} " - f"observations[/green]" - ) - return df, y_datetime - - except Exception as e: - console.print(f"[red]Data validation failed: {e}[/red]") - raise - - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """ - Core fitting logic for NeuralProphet model with enhanced error - handling. - - Args: - y: The target time series data with DateTime index - X: Optional DataFrame (unused for univariate model) - X_val: Validation features (unused for NeuralProphet) - y_val: Validation target (unused for NeuralProphet) - - Raises: - ValueError: If data validation fails - RuntimeError: If model fitting fails - """ - try: - console.log("[blue]Starting NeuralProphet model fitting...[/blue]") - - df, y_datetime = self._validate_and_prepare_data(y) - self.training_series_ = y_datetime - self.model_ = self._create_model() - - console.log( - f"[blue]Training model for {self.epochs} epochs...[/blue]" - ) - - fit_result = self.model_.fit(df, epochs=self.epochs) - - # Store training metrics if available - if hasattr(fit_result, "losses") and fit_result is not None: - losses = getattr(fit_result, "losses", None) - if losses: - self._training_metrics = { - "final_loss": float(losses[-1]), - "epochs_trained": len(losses), - } - - console.log( - f"[green]Model fitting completed successfully. " - f"Metrics: {self._training_metrics}[/green]" - ) - - except Exception as e: - console.print(f"[red]Model fitting failed: {e}[/red]") - # Reset model state on failure - self.model_ = None - self.training_series_ = None - raise RuntimeError( - f"Failed to fit NeuralProphet model: {e}" - ) from e - - def _prepare_prediction_data( - self, X: Optional[pd.DataFrame] = None - ) -> Tuple[pd.DataFrame, pd.Series]: - """ - Prepare data for prediction with comprehensive validation. - - Args: - X: Optional DataFrame containing prediction data - - Returns: - Tuple of (prepared_dataframe, prediction_index) - - Raises: - ValueError: If data preparation fails - """ - if self.training_series_ is None: - raise ValueError("No training series available") - - training_series = cast(pd.Series, self.training_series_) - - if X is None: - # Predict on training data - df = pd.DataFrame( - { - "ds": training_series.index, - "y": training_series.to_numpy(), - } - ) - return df, pd.Series(training_series.index) - - # Handle various input formats for X - try: - ds_values, y_values = self._extract_time_and_target_from_X(X) - - # Ensure datetime format - if not pd.api.types.is_datetime64_any_dtype(ds_values): - ds_values = pd.to_datetime(ds_values) - - df = pd.DataFrame( - { - "ds": ds_values, - "y": y_values, - } - ) - - return df, ds_values - - except Exception as e: - raise ValueError( - f"Failed to prepare prediction data: {e}" - ) from e - - def _extract_time_and_target_from_X( - self, X: pd.DataFrame - ) -> Tuple[pd.Series, pd.Series]: - """ - Extract time and target columns from input DataFrame. - - Args: - X: Input DataFrame - - Returns: - Tuple of (time_series, target_series) - - Raises: - ValueError: If extraction fails - """ - # Scenario 1: Explicit time and target columns - if self.time_col in X.columns and self.target_col in X.columns: - return ( - X[self.time_col], - self._validate_y(X[self.target_col]) - ) - - # Scenario 2: DateTime index - elif pd.api.types.is_datetime64_any_dtype(X.index): - ds_values = pd.Series(X.index, name=self.time_col) - - if self.target_col in X.columns: - # DateTime index with explicit target column - return ( - ds_values, - self._validate_y(X[self.target_col]) - ) - elif X.shape[1] == 1: - # DateTime index with single data column - return ds_values, self._validate_y( - X.iloc[:, 0].rename(self.target_col) - ) - elif X.shape[1] == 0: - # Only index, no columns - forecast scenario - y_values = pd.Series( - np.nan, index=X.index, name=self.target_col - ) - return ds_values, y_values - else: - raise ValueError( - "X has DateTime index but cannot identify target column. " - + f"Expected '{self.target_col}' or single column. " - + f"Found: {X.columns.tolist()}" - ) - else: - raise ValueError( - "Cannot determine time and target from X. " - + f"Provide columns '{self.time_col}' and '{self.target_col}' " - + "or use DateTime index." - ) - - @ensure_fitted - def predict(self, X: Optional[pd.DataFrame] = None) -> pd.Series: - """ - Generate in-sample predictions with enhanced error handling. - - Args: - X: Optional DataFrame containing timestamps and target values. - If None, predicts on training data. - - Returns: - Series of predictions indexed by timestamp - - Raises: - ValueError: If model is not fitted or prediction fails - RuntimeError: If prediction computation fails - """ - if self.model_ is None: - raise ValueError("Model is not fitted") - - try: - console.log("[blue]Generating predictions...[/blue]") - - # Prepare prediction data - df, predictions_index = self._prepare_prediction_data(X) - - # Get training context for lagged features - training_series = cast(pd.Series, self.training_series_) - past_values = pd.DataFrame( - { - "ds": training_series.index, - "y": training_series.to_numpy(), - } - ).iloc[-self.n_lags :, :] - - # Combine past and prediction data - combined_df = pd.concat([past_values, df], ignore_index=True) - combined_df = ( - combined_df.sort_values(by="ds") - .reset_index(drop=True) - .drop_duplicates(subset="ds", keep="last") - ) - - # Generate forecast - forecast = self.model_.predict(combined_df) - - # Handle different forecast column formats - forecast_col = f"yhat{self.n_forecasts}" - if forecast_col not in forecast.columns: - forecast = self.model_.get_last_forecast( - forecast, include_previous_forecasts=self.n_forecasts - ) - - # Extract predictions for requested indices - forecast = forecast.set_index("ds") - predictions = forecast.loc[predictions_index, forecast_col] - - console.log( - f"[green]Generated {len(predictions)} predictions[/green]" - ) - return predictions - - except Exception as e: - console.print(f"[red]Prediction failed: {e}[/red]") - raise RuntimeError(f"Failed to generate predictions: {e}") from e - - @ensure_fitted - def forecast(self, forecast_horizon: int) -> np.ndarray: - """ - Generate future forecasts with comprehensive validation. - - Args: - forecast_horizon: Number of steps to forecast ahead - (1 to n_forecasts) - - Returns: - Array of forecasted values, indexed by the forecast horizon - - Raises: - ValueError: If forecast_horizon is invalid or model not fitted - RuntimeError: If forecast generation fails - """ - if self.model_ is None: - raise ValueError("Model is not fitted") - - if not (1 <= forecast_horizon <= self.n_forecasts): - raise ValueError( - "forecast_horizon must be between 1 and " - + f"{self.n_forecasts}, got {forecast_horizon}" - ) - - try: - console.log( - f"[blue]Generating {forecast_horizon}-step forecast...[/blue]" - ) - - training_series = cast(pd.Series, self.training_series_) - - # Create future dataframe - future_df = self.model_.make_future_dataframe( - df=pd.DataFrame( - { - "ds": training_series.index, - "y": training_series.to_numpy(), - } - ), - periods=forecast_horizon, - ) - - # Generate forecasts - forecast = self.model_.predict(future_df) - - # Extract forecasted values for each horizon - forecasted_values = np.empty(forecast_horizon) - for i in range(forecast_horizon): - col_name = f"yhat{i + 1}" - if col_name in forecast.columns: - values = forecast[col_name].dropna() - if len(values) > 0: - forecasted_values[i] = values.iloc[0] - else: - forecasted_values[i] = np.nan - else: - forecasted_values[i] = np.nan - - console.log( - f"[green]Generated forecast: {len(forecasted_values)}" - + " values[/green]" - ) - return forecasted_values - - except Exception as e: - console.print(f"[red]Forecast generation failed: {e}[/red]") - raise RuntimeError(f"Failed to generate forecast: {e}") from e - - @ensure_fitted - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Perform comprehensive backtesting with enhanced monitoring. - - This method implements walk-forward validation with periodic - retraining, providing robust evaluation of model performance in - production-like scenarios. - - Args: - y: Target time series for backtesting (must have DateTime index) - X: Unused (included for base class compatibility) - retrain_every: Unused (model retrains at each step) - reuse_previous_execution: Whether to reuse previous backtest - results - - Returns: - Series of backtested predictions indexed by timestamp - - Raises: - ValueError: If parameters are invalid or data is insufficient - RuntimeError: If backtesting fails - """ - if self.model_ is None: - raise ValueError("Model is not fitted") - if self.training_series_ is None: - raise ValueError("No training series found") - - if not (1 <= self.forecast_horizon <= self.n_forecasts): - raise ValueError( - "forecast_horizon must be between 1 and " - + f"{self.n_forecasts}, got {self.forecast_horizon}" - ) - - # Handle reuse of previous execution - if reuse_previous_execution and self.backtest_predictions_ is not None: - expected_index = y.iloc[self.forecast_horizon:].index - if ( - len(self.backtest_predictions_) == len(expected_index) - and (self.backtest_predictions_.index == expected_index).all() - ): - console.log( - "[yellow]Reusing previous backtest results[/yellow]" - ) - return self.backtest_predictions_ - else: - console.log( - "[yellow]Previous results incompatible, running new" - + " backtest[/yellow]" - ) - - try: - console.log( - f"[blue]Starting backtest with {self.forecast_horizon}-step" - + " horizon...[/blue]" - ) - - # Validate and prepare data - y_sorted = y.sort_index() - - # Check for overlapping data - training_series = cast(pd.Series, self.training_series_) - if any(t in training_series.index for t in y_sorted.index): - console.print( - "[yellow]Warning: Backtest data overlaps with training" - + " data[/yellow]" - ) - - # Initialize backtesting - predictions = [] - training_base = pd.DataFrame( - { - "ds": training_series.index, - "y": training_series.values, - } - ) - - timestamps = y_sorted.index - total_steps = len(timestamps) - self.forecast_horizon - - console.log( - f"[blue]Running {total_steps} backtest steps...[/blue]" - ) - - # Perform walk-forward validation - for i in range(self.forecast_horizon, len(timestamps)): - if i % 50 == 0: # Progress logging - console.log( - f"[blue]Backtest progress: {i}/{len(timestamps)}[/blue]" - ) - - t = timestamps[i] - t_minus_h = timestamps[i - self.forecast_horizon] - - # Prepare training data up to t - h - history = y_sorted.loc[:t_minus_h] - train_df = ( - pd.concat( - [ - training_base, - pd.DataFrame( - {"ds": history.index, "y": history.values} - ), - ], - ignore_index=True, - ) - .drop_duplicates(subset="ds") - .sort_values("ds") - ) - - # Check minimum data requirement - if len(train_df) < max(self.n_lags + 1, 10): - console.print( - f"[yellow]Insufficient data at step {i}, " - + "skipping[/yellow]" - ) - predictions.append((t, np.nan)) - continue - - try: - # Retrain model - model = self._create_model() - model.fit(train_df, epochs=self.epochs) - - # Generate forecast - mask = train_df["ds"] <= t_minus_h - future_df = model.make_future_dataframe( - df=train_df.loc[mask], periods=self.forecast_horizon - ) - forecast = model.predict(future_df, decompose=False) - - # Extract prediction - forecast_col = f"yhat{self.forecast_horizon}" - prediction_rows = forecast[forecast["ds"] == t] - - if ( - len(prediction_rows) > 0 - and forecast_col in forecast.columns - ): - prediction = prediction_rows[forecast_col].iloc[0] - else: - prediction = np.nan - - predictions.append((t, prediction)) - - except Exception as step_error: - console.print( - f"[yellow]Error at step {i}: {step_error}[/yellow]" - ) - predictions.append((t, np.nan)) - - # Create results series - if predictions: - pred_index, pred_values = zip(*predictions) - self.backtest_predictions_ = pd.Series( - pred_values, - index=pd.Index(pred_index), - name=f"yhat{self.forecast_horizon}", - ) - else: - self.backtest_predictions_ = pd.Series( - dtype=float, name=f"yhat{self.forecast_horizon}" - ) - - console.log( - f"[green]Backtest completed: {len(self.backtest_predictions_)}" - + " predictions[/green]" - ) - return self.backtest_predictions_ - - except Exception as e: - console.print(f"[red]Backtest failed: {e}[/red]") - raise RuntimeError(f"Failed to perform backtest: {e}") from e - - def get_params_dict(self) -> Dict[str, Any]: - """Get comprehensive model parameters for logging/serialization.""" - base_params = super().get_params_dict() - neural_prophet_params = { - "n_forecasts": self.n_forecasts, - "weekly_seasonality": self.weekly_seasonality, - "daily_seasonality": self.daily_seasonality, - "yearly_seasonality": self.yearly_seasonality, - "seasonality_mode": self.seasonality_mode, - "epochs": self.epochs, - "learning_rate": self.learning_rate, - "batch_size": self.batch_size, - "loss_func": self.loss_func, - "normalize": self.normalize, - "impute_missing": self.impute_missing, - "drop_missing": self.drop_missing, - "training_metrics": self._training_metrics, - } - return {**base_params, **neural_prophet_params} - - def summary(self) -> str: - """Generate comprehensive model summary.""" - fitted_status = ( - "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" - ) - - seasonality_features = [] - if self.weekly_seasonality: - seasonality_features.append("Weekly") - if self.daily_seasonality: - seasonality_features.append("Daily") - if self.yearly_seasonality: - seasonality_features.append("Yearly") - - seasonality_str = ( - ", ".join(seasonality_features) if seasonality_features else "None" - ) - - summary_lines = [ - f"Model: {self.__class__.__name__}", - f"Status: {fitted_status}", - f"Lags: {self.n_lags}, Forecasts: {self.n_forecasts}", - f"Seasonality: {seasonality_str} ({self.seasonality_mode})", - f"Training: {self.epochs} epochs, {self.loss_func} loss", - f"Data Handling: Impute={self.impute_missing}, " - f"Drop={self.drop_missing}", - ] - - if self._training_metrics: - metrics_str = ", ".join( - f"{k}={v:.4f}" for k, v in self._training_metrics.items() - ) - summary_lines.append(f"Metrics: {metrics_str}") - - return "\n".join(summary_lines) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py deleted file mode 100644 index 5e5a7c0..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py +++ /dev/null @@ -1,695 +0,0 @@ -""" -Stacking implementation for time series forecasting. - -This module provides a stacking regressor implementation that combines multiple -time series models' predictions using a meta-model. The stacking approach -helps improve prediction accuracy by combining the strengths of different -base models through a learned meta-model. - -Key features: -- **Model Stacking**: Combines predictions from multiple base models -- **Time-Series Aware**: Uses proper time-based cross-validation -- **Meta-Model Learning**: Learns optimal combination weights -- **Comprehensive Error Handling**: Robust error handling and validation -- **Rich Logging**: Colored console output for better debugging - -The stacking model loads pre-trained base models and uses their predictions -as features for training a meta-model (CatBoost by default). -""" - -from typing import Optional, List, Dict, Any, Union -import pandas as pd -import numpy as np -from catboost import CatBoostRegressor, CatBoostClassifier, Pool -from rich.console import Console -import gzip -import pickle -import lzma - -from .base import ( - MultivariateTimeSeriesModel, - ensure_fitted, - TimeSeriesModel, -) - -console = Console() - - -class StackingTimeSeriesModel(MultivariateTimeSeriesModel): - """ - Stacking implementation for time series forecasting. - - This class implements stacking of multiple base models, using their - predictions as features for a meta-model. It handles time-based - cross-validation to generate out-of-fold predictions for training. - - The model supports both regression and classification tasks through - the meta-model configuration. - - Attributes: - base_models_: List of loaded base models - model_: The trained meta-model (CatBoost) - training_series_: Copy of training target data - base_predictions_train_: Base model predictions on training data - backtest_predictions_: Stored backtest predictions for reuse - - Example: - >>> stacking_model = StackingTimeSeriesModel( - ... base_model_paths=["model1.pkl", "model2.pkl"], - ... base_model_types=["catboost", "elasticnet"] - ... ) - >>> stacking_model.fit(y=target_series, X=feature_matrix) - >>> predictions = stacking_model.predict(X=test_features) - """ - - def __init__( - self, - base_model_paths: List[str], - base_model_types: List[str], - name: Optional[str] = None, - learning_task: str = "regression", - retrain_every: int = 100, - meta_iterations: int = 1000, - meta_learning_rate: float = 0.1, - meta_depth: int = 6, - early_stopping_rounds: Optional[int] = None, - meta_loss_function: Optional[str] = None, - time_col: str = "ds", - target_col: str = "y", - random_seed: int = 42, - verbose: bool = False, - differentiate_target: bool = False, - bins: Optional[List[float]] = None, - use_predict_for_training: bool = True, - ) -> None: - """ - Initialize the stacking model. - - Args: - base_model_paths: Paths to saved base models - base_model_types: Types of base models (must match order of paths) - name: Optional identifier for the model - learning_task: Type of learning task ('regression', 'binary', - 'multiclass') - retrain_every: Frequency of retraining during backtesting - meta_iterations: Number of iterations for meta-model - meta_learning_rate: Learning rate for meta-model - meta_depth: Tree depth for meta-model - early_stopping_rounds: Early stopping rounds for meta-model - meta_loss_function: Loss function for meta-model - time_col: Name of time column - target_col: Name of target column - random_seed: Random seed - verbose: Whether to print verbose logging - differentiate_target: Whether to differentiate the target series - bins: Bin edges for multiclass classification - use_predict_for_training: If True, use predict() instead of - backtest() for generating base model predictions during - training. This is much faster but may lead to overfitting - since the meta-model trains on in-sample predictions. - """ - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - learning_task=learning_task, - differentiate_target=differentiate_target, - bins=bins, - ) - - # Validate inputs - if not base_model_paths: - raise ValueError("base_model_paths cannot be empty") - if not base_model_types: - raise ValueError("base_model_types cannot be empty") - if len(base_model_paths) != len(base_model_types): - raise ValueError( - "base_model_paths and base_model_types must have same length" - ) - - self.retrain_every = retrain_every - self.base_model_paths = base_model_paths - self.base_model_types = base_model_types - self.meta_iterations = meta_iterations - self.meta_learning_rate = meta_learning_rate - self.meta_depth = meta_depth - self.meta_loss_function = self._get_default_loss_function( - meta_loss_function - ) - self.early_stopping_rounds = early_stopping_rounds - self.verbose = verbose - self.use_predict_for_training = use_predict_for_training - - # Will be set during fit - self.base_models_: List[TimeSeriesModel] = [] - self.model_: Optional[ - Union[CatBoostRegressor, CatBoostClassifier] - ] = None - self.training_series_: Optional[pd.Series] = None - self.base_predictions_train_: Optional[pd.DataFrame] = None - self.backtest_predictions_: Optional[pd.Series] = None - - self._load_base_models() - - if self.verbose: - console.log( - "[green]Initialized StackingTimeSeriesModel: " - + f"{self.summary()}[/green]" - ) - - def _load_base_models(self) -> None: - """Load all base models from their saved paths.""" - from .factory import load_model - - self.base_models_ = [] - for model_path, model_type in zip( - self.base_model_paths, self.base_model_types - ): - try: - model = load_model(model_path, model_type) - self.base_models_.append(model) - if self.verbose: - console.log( - f"[blue]Loaded {model_type} model from " - + f"{model_path}[/blue]" - ) - except Exception as e: - console.print( - f"[red]Error loading model from {model_path}: {e}[/red]" - ) - raise ValueError(f"Failed to load model: {model_path}") from e - - if self.verbose: - console.log( - f"[green]Successfully loaded {len(self.base_models_)} " - + "base models[/green]" - ) - - def _create_meta_model( - self, - ) -> Union[CatBoostRegressor, CatBoostClassifier]: - """Creates a new instance of meta-model. - - Returns: - A new CatBoost model instance (Regressor or Classifier). - """ - base_params = { - "iterations": self.meta_iterations, - "learning_rate": self.meta_learning_rate, - "depth": self.meta_depth, - "loss_function": self.meta_loss_function, - "early_stopping_rounds": self.early_stopping_rounds, - "random_seed": self.random_seed, - "verbose": self.verbose, - } - - if self.learning_task in ["binary", "multiclass"]: - return CatBoostClassifier( - auto_class_weights="Balanced", **base_params - ) - else: - return CatBoostRegressor(**base_params) - - def _get_base_predictions( - self, X: Optional[pd.DataFrame], y: Optional[pd.Series] = None - ) -> pd.DataFrame: - """Get predictions from all base models. - - Args: - X: Feature matrix - y: Target series (optional, used for validation) - - Returns: - DataFrame with predictions in model_0, model_1, etc columns - """ - if X is None: - raise ValueError("Feature matrix X cannot be None") - - all_preds = [] - for i, model in enumerate(self.base_models_): - try: - preds = model.predict(X) - model_name = f"model_{model.name or i}" - all_preds.append( - pd.Series(preds, name=model_name, index=X.index) - ) - - if self.verbose: - console.log( - f"[cyan]Generated predictions from {model_name}[/cyan]" - ) - except Exception as e: - console.print( - f"[red]Error getting predictions from model {i}: {e}[/red]" - ) - raise - - return pd.concat(all_preds, axis=1) - - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """Core fitting logic for stacking model. - - Get predictions from base models on training data, then train - meta-model on those predictions. - - Args: - y: The target time series data - X: The feature matrix (including exogenous features) - X_val: Validation feature matrix (optional) - y_val: Validation target series (optional) - """ - if X is None: - raise ValueError("Feature matrix X must be provided for stacking") - - # Preprocess data using parent class method - y_processed, X_processed, y_val_processed, X_val_processed = ( - self._preprocess_data(y, X, X_val, y_val) - ) - - if X_processed is None: - raise ValueError( - "Feature matrix X cannot be None after preprocessing" - ) - - if self.verbose: - method_name = ( - "predictions" - if self.use_predict_for_training - else "backtest predictions" - ) - console.log( - f"[blue]Generating base model {method_name} " - + "for stacking...[/blue]" - ) - - # Get base predictions - use either predict or backtest based on - # setting - if self.use_predict_for_training: - # Fast approach: use direct predictions (may overfit) - meta_features = self._get_base_predictions(X_processed) - # Align with target data - common_index = meta_features.index.intersection(y_processed.index) - meta_features = meta_features.loc[common_index] - y_aligned = y_processed.loc[common_index] - - if self.verbose: - console.log( - "[yellow]Warning: Using predict() for training may " - + "lead to overfitting since meta-model trains on " - + "in-sample predictions[/yellow]" - ) - else: - # Robust approach: use backtesting to avoid overfitting - all_preds = [] - for i, model in enumerate(self.base_models_): - try: - preds = model.backtest( - y_processed, - X_processed, - retrain_every=self.retrain_every, - ) - model_name = f"model_{model.name or i}" - all_preds.append(pd.Series(preds, name=model_name)) - - if self.verbose: - console.log( - "[cyan]Generated backtest predictions from " - + f"{model_name}[/cyan]" - ) - except Exception as e: - console.print( - f"[red]Error during backtesting for model {i}: " - + f"{e}[/red]" - ) - raise - - meta_features = pd.concat(all_preds, axis=1) - - # Align with target data (backtest might have different length) - common_index = meta_features.index.intersection(y_processed.index) - meta_features = meta_features.loc[common_index] - y_aligned = y_processed.loc[common_index] - - if self.verbose: - console.log( - f"[blue]Training meta-model with {len(meta_features)} " - + f"samples and {meta_features.shape[1]} base model " - + "features[/blue]" - ) - - meta_X, meta_y = self._validate_X_y(meta_features, y_aligned) - train_pool = Pool(data=meta_X, label=meta_y) - - # Prepare validation data if provided - eval_set = None - if X_val_processed is not None and y_val_processed is not None: - val_predictions = self._get_base_predictions( - X_val_processed, y_val_processed - ) - val_X, val_y = self._validate_X_y(val_predictions, y_val_processed) - eval_set = Pool(data=val_X, label=val_y) - - if self.verbose: - console.log( - "[blue]Using validation set with " - + f"{len(val_predictions)} samples[/blue]" - ) - - # Create and train meta-model - self.model_ = self._create_meta_model() - self.model_.fit(train_pool, eval_set=eval_set) - - # Store training data - self.training_series_ = y_processed.copy() - self.base_predictions_train_ = meta_features.copy() - - if self.verbose: - console.log( - "[green]Meta-model training completed successfully[/green]" - ) - - @ensure_fitted - def predict(self, X: pd.DataFrame) -> pd.Series: - """ - Generate predictions using the stacking model. - - Args: - X: Feature matrix for prediction - - Returns: - Series containing predictions - """ - if self.model_ is None: - raise ValueError("Model has not been fitted yet") - - base_predictions = self._get_base_predictions(X) - X_array = self._validate_X(base_predictions) - predictions = self.model_.predict(X_array) - - # Convert predictions to numpy array if needed - if hasattr(predictions, "squeeze"): - predictions = predictions.squeeze() - elif isinstance(predictions, list): - predictions = np.array(predictions) - - return pd.Series(predictions, index=X.index, name=self.target_col) - - @ensure_fitted - def feature_importance(self) -> Optional[pd.DataFrame]: - """ - Returns feature importance from the meta-model. - - Returns: - DataFrame with feature names and their importance scores, - or None if not available. - """ - if self.model_ is None or not hasattr( - self.model_, "feature_importances_" - ): - return None - - if self.base_predictions_train_ is None: - return None - - importances = self.model_.feature_importances_ - feature_names = self.base_predictions_train_.columns - - return pd.DataFrame( - { - "feature": feature_names, - "importance": importances, - } - ).sort_values("importance", ascending=False) - - @ensure_fitted - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Performs backtesting (walk-forward validation) with periodic - retraining. - - This method simulates a production scenario by iterating through a test - set, making a one-step-ahead prediction, and then retraining the model - periodically with the newly available data. - - Args: - y: Series with the true target values for the backtesting period - X: DataFrame with features for the backtesting period - retrain_every: The frequency of retraining. The model will be - retrained every `retrain_every` steps - reuse_previous_execution: Whether to reuse the previous execution - of a backtest. If True, any overlapping data between the - previous execution and the current execution will be used - without retraining the model - - Returns: - A series of backtested predictions, indexed by the backtest data's - index - """ - if self.model_ is None: - raise ValueError("Model is not fitted yet") - if self.training_series_ is None: - raise ValueError("Training series is not set") - if X is None: - raise ValueError("Feature matrix X must be provided") - - if reuse_previous_execution: - if self.backtest_predictions_ is None: - raise ValueError("No previous execution found") - if (self.backtest_predictions_.shape[0] != y.shape[0]) or ( - not (self.backtest_predictions_.index == y.index).all() - ): - raise ValueError( - "Previous execution index does not match y index" - ) - return self.backtest_predictions_ - - if self.verbose: - console.log( - f"[blue]Starting backtest with {len(y)} samples, " - + f"retraining every {retrain_every} steps[/blue]" - ) - - # Get base model predictions for the entire backtest period - all_base_preds = [] - for i, model in enumerate(self.base_models_): - try: - preds = model.backtest( - y, - X, - retrain_every=retrain_every, - reuse_previous_execution=reuse_previous_execution, - ) - model_name = f"model_{model.name or i}" - all_base_preds.append(pd.Series(preds, name=model_name)) - - if self.verbose: - console.log( - f"[cyan]Completed backtest for {model_name}[/cyan]" - ) - except Exception as e: - console.print( - f"[red]Error during backtest for model {i}: {e}[/red]" - ) - raise - - meta_features = pd.concat(all_base_preds, axis=1) - - # Generate meta-model predictions - predictions = self.model_.predict(self._validate_X(meta_features)) - - # Store backtest predictions for potential reuse - self.backtest_predictions_ = pd.Series( - predictions, - index=meta_features.index, - name=f"{self.target_col}_pred", - ) - - if self.verbose: - console.log( - "[green]Backtest completed: " - + f"{len(self.backtest_predictions_)} predictions " - + "generated[/green]" - ) - - return self.backtest_predictions_ - - def get_base_model_names(self) -> List[str]: - """Get names of all base models. - - Returns: - List of base model names - """ - return [ - model.name or f"model_{i}" - for i, model in enumerate(self.base_models_) - ] - - def get_params_dict(self) -> Dict[str, Any]: - """Get model parameters as dictionary for logging/serialization.""" - base_params = super().get_params_dict() - stacking_params = { - "base_model_paths": self.base_model_paths, - "base_model_types": self.base_model_types, - "retrain_every": self.retrain_every, - "meta_iterations": self.meta_iterations, - "meta_learning_rate": self.meta_learning_rate, - "meta_depth": self.meta_depth, - "meta_loss_function": self.meta_loss_function, - "early_stopping_rounds": self.early_stopping_rounds, - "num_base_models": len(self.base_models_), - "use_predict_for_training": self.use_predict_for_training, - } - return {**base_params, **stacking_params} - - def summary(self) -> str: - """Generate a summary string of the model.""" - params = self.get_params_dict() - fitted_status = ( - "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" - ) - - summary_lines = [ - f"Model: {self.__class__.__name__}", - f"Status: {fitted_status}", - f"Base Models: {params.get('num_base_models', 0)}", - f"Task: {params.get('learning_task', 'regression')}", - f"Meta Loss: {params.get('meta_loss_function', 'RMSE')}", - ] - - return "\n".join(summary_lines) - - def _optimize_base_models_for_storage(self) -> None: - """ - Optimizes base models for storage by removing unnecessary data. - This can significantly reduce pickle size, especially for neural - models. - """ - if self.verbose: - console.log("[blue]Optimizing base models for storage...[/blue]") - - for i, model in enumerate(self.base_models_): - try: - # For neuralprophet models, remove training history and - # large artifacts - model_attr = getattr(model, "model", None) - if model_attr is not None and hasattr(model_attr, "trainer"): - trainer = getattr(model_attr, "trainer", None) - if trainer is not None: - # Remove trainer which contains training logs and - # can be very large - if hasattr(trainer, "logged_metrics"): - setattr(trainer, "logged_metrics", {}) - if hasattr(trainer, "progress_bar_metrics"): - setattr(trainer, "progress_bar_metrics", {}) - if hasattr(trainer, "callback_metrics"): - setattr(trainer, "callback_metrics", {}) - - # For any model with training history - if hasattr(model, "training_history_"): - setattr(model, "training_history_", None) - if hasattr(model, "validation_history_"): - setattr(model, "validation_history_", None) - - # Remove cached predictions if they exist - if hasattr(model, "_cached_predictions"): - setattr(model, "_cached_predictions", None) - - if self.verbose: - model_name = getattr(model, "name", f"model_{i}") - console.log( - f"[cyan]Optimized {model_name} for storage[/cyan]" - ) - - except Exception as e: - if self.verbose: - console.log( - f"[yellow]Warning: Could not optimize model {i}: " - + f"{e}[/yellow]" - ) - - def save(self, path: str, compression: str = "gzip") -> None: - """ - Saves model to disk using compression to reduce file size. - - Args: - path: File path to save to - compression: Compression method ('gzip', 'lzma', or 'none') - - 'gzip': Fast compression, ~60-80% size reduction - - 'lzma': Better compression, ~70-90% size reduction, slower - - 'none': No compression - """ - if self.verbose: - console.log( - f"[blue]Saving stacking model with {compression} " - + f"compression to {path}[/blue]" - ) - - # Optimize base models for storage first - self._optimize_base_models_for_storage() - - if compression == "lzma": - # LZMA provides better compression but is slower - with lzma.open(path, "wb", preset=9) as f: - pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) - elif compression == "gzip": - # Gzip is faster with good compression - with gzip.open(path, "wb", compresslevel=9) as f: - pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) - else: - # No compression - with open(path, "wb") as f: - pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) - - if self.verbose: - console.log( - f"[green]Saved compressed stacking model to {path}[/green]" - ) - - @classmethod - def load(cls, path: str) -> "StackingTimeSeriesModel": - """ - Loads model from disk with automatic format detection. - Supports both compressed formats and legacy joblib format. - """ - import joblib - - # Try different formats in order of preference - loading_methods = [ - ("lzma", lambda p: lzma.open(p, "rb")), - ("gzip", lambda p: gzip.open(p, "rb")), - ("pickle", lambda p: open(p, "rb")), - ("joblib", None), # Special case for joblib - ] - - for format_name, open_func in loading_methods: - try: - if format_name == "joblib": - return joblib.load(path) - else: - with open_func(path) as f: - return pickle.load(f) - except ( - lzma.LZMAError, - gzip.BadGzipFile, - OSError, - pickle.UnpicklingError, - ValueError, - ): - continue - - raise ValueError( - f"Could not load model from {path} - unknown or corrupted format" - ) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml b/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml deleted file mode 100644 index 2f1d2cb..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml +++ /dev/null @@ -1,11 +0,0 @@ -channels: -- conda-forge -dependencies: -- python=3.10.16 -- pip<=25.0 -- pip: - - mlflow==2.7.1 - - pandas - - numpy - - scikit-learn -name: mlflow-env diff --git a/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml b/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml deleted file mode 100644 index 0a0396b..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml +++ /dev/null @@ -1,7 +0,0 @@ -python: 3.10.16 -build_dependencies: -- pip==25.0 -- setuptools==79.0.0 -- wheel==0.45.1 -dependencies: -- -r requirements.txt diff --git a/tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl b/tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl deleted file mode 100644 index aa3f5aa8ef7c82c96b4bd1fda6e43cd695abf3f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 123 zcmZo*om#*E0X;IMC7C(Jdbv4iIr-&!1(j)~dCBqRMTrFksYS(8dW1rX67!1F@{4j) zi^3tIQzlQ*Y@AX%MWaWgq$n{nFEcMa9>{>Hn&Q_ZnwgiDT9lfXoQf(@nxqE+?KdyV diff --git a/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt b/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt deleted file mode 100644 index fd9a283..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -mlflow==2.7.1 -pandas -numpy -scikit-learn \ No newline at end of file diff --git a/tmp/artifacts/data_model/transformers/courier_transformers.pkl b/tmp/artifacts/data_model/transformers/courier_transformers.pkl deleted file mode 100644 index 18fe2203362307ea711c91971c88e266be720d81..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29607 zcmeHQS(6-BR<`!BWJ%U;*|MF3Ev%Vww`yZ)=rO)VP9 zg98SzEkgoBIOhR^2MiArLBs?E@WLBdj}ZI?f_?b~yuo+Q-EU@CT{AkGgBp4*zSjPm*tZFCo?w;pS#H+6?>kPP=+z&|asTUh)n^dt1>)J!~iO zu3mow*H)eGTj8xFicj_Wovp2Cqq^1YtVg|Gb-mLZv|vb_2a37!*z)Y!Sbidx%sp_qD@x+W zso3j>t*ssuCRN%x^W|kk8FkQjj%Vq-swi;H@brC5;8c$ z_c}SdtF29>_&Hm>-tI@;c8HX>!gjbBb&=^OE;c*s?=FaP@<|e(lor)y{d#X)nY%+BVb8u>*R47j7@;Uq)%jL?2u}hbxr_*>&;@swBoiCOX z(c)aHfa{53ZenbDYJM6-c`D{}s#xISq$&z&!74g^$?Zv^G=n;GIq)2!Ubi~Sl~h-k zr4ua97aeiI7sHzC5z26ax#lI;@-O%hgZ>7P)``_@@3t8$CWOc4?hJCAWbrw+efxHhC4KQWJ#~cC>F`&)LZZlNGkfs(y zvMOVotgb1HqKUjK?zcLm7n;i*5?B>gO~e9wsiQ4_QcA z^(yGh^DI*0PGNCkxMQf0JJj`sm;rkVV|_V=Nqr7bCz|=uHaf2gkP$N@n@p(wA?wWB z9OQ(pnKv4}Ao)xfY6^lu&R(sXXPBE1WVPvJN#45GdsPCzDQ>*CV9=s z(VWaBu~?%y5zwSwtmUkt3G!4$?ayrjp75a{2rv~|1O<__gG5OksF2H#tyqIm^_U~M ziHpm*eBu1H%kx-|xsyq(n1PjS%$jI1(OV_48xV0~)U!;L#Nr}LG;UxgOBWp3q$8VD zGHx1bTecG=P6Sx&)mN1;A?X@;r)8BO8&}(s$%R!kiP~ob-wMRW#D?9C2}?@KK<;B~ zkko5`Q~TT6XEX;=GtfYl-7YtYv=>B*X*RGl#4>+KXo!SUKm2Lx+3~YO()3TRU5}`d>Y9-Ef-=$s1o-ntqK1vJMIly#yw1zyd@}?)b7}(6YdIy75^h6E{@wi z6vyp)isN=YMOx~#M|qO%=jr$wCNYryyR#00)mj#5zhxDRu1h5hA4^)vC$;~ufjp9C z$qS-oRo04P&!viGj3P_90?+>0251jy$qTM4Mb{PD*jmaJ%+>#719{h#g6m4jb*1dK zQ>B#D{>>qCT`9P(lw4QJZh|Z2r1q~4nd=H|YOO#@t}A6X!Ier<`!8EP@48ZST`9S) zl-&eZCX(8}*klFQ6?7e2T5??}y9utiL&SBf;JSr{)ggA>D!a+9OeVGewAE=3YbC7c zx>a)BD!a+X`womi%evsYLeIvQwB)){cB7iWX!vhiz2Lf1bX_UAu80S3JyT2|suT23 zn#XKR+tY=7ApT)??$Zgl8_w#$1$*`9#$=≶CS=Q#Vtj$pt=DYgm zYw%$uS|4mh-SF-Y@BT?AiFZ_!#>E|vJvQ_b?^<4+O&XV>6Ypt#_LoGm<80EHA)_mF z;k#kQc<0=kN#k|4Zl0$Xa*bKXDBijCdeV4_wBnLDn7FVvg4OIuyh ztH=~h>_ipsv>+Lc9zPiiI-D zN{Tu3g{1K&dv`XVm(aqycqh7Iyqkge@v}j%(*#)$?`Ib=GBR;et|AuKpO9ig5;qve-W=9o^8BiDrx*8C2i#8%CG#7_>ARCxQP$UMA1gD zP;cU69CV=tbfExCNatqMz0F-n+y>ND4v;hq78V13tTZL`lp|BkdMmin?euz`UNF<_ z^zlT@*i5~xk)y)qa&HTjG@^Q+y4yikBW#*K8#%U&k6wCMy@Kgx*i+SIps4oDweUUC zoSF~Tq1>zA5c)x@NDJA?#rn;t8C+a;r9Koxk6l94r ziAy7hg$({>6KnAa**4R+r~y>fXc&pF-wM#3Ig}(muESGUK<^Y!%D9k3t;~WfeQb=C zJ}R{IAvk3!g={10HNzGqkv_Cji8W}D!xsINtQ%E(l57)f#I=cCMUaE!j2M)tn})5} zI0hHPX4t;_L$vNeDF}EV<1^0c*=jHk7pdn%hJlOW%B?O@)#7N?8~8|ZoKyk|3Kpj? z=@!2_6R?D>0yNqQIhY|J3yq&LHo(WFARnN;E_HhN#IqGlVdohJ(~C4Q9hEc+0u3>7PFc%zWeFsfCwqwjqhX5j-Sg)6MNQ*`dw_bPw9Hwv?#Bxhc}}(K8`hk zO|@7u-HgIE;Xi4*;WR#4Mt$|}`T&W?8cuB?=hGzq=Cufk6axj5*=XhGq#s6W++{mF zwGwnYgUwph4QAE?q|8Im79LW>gaNB{;d!tY;K7en6aX3wi6cZDSr~eE3PbljZ8@OR zM6RGW4wd0i^yy)C^eU5thD_#|<3oVMg^2Y}2+W=|Pvi z(vZZ@Lo64yAsR>9yc)TwPlNC%4Wa>#2y>NQZUd>PFwqRhWj$U?WWi{RGp<)Ag`z!1 ziP0p_j57++n$vtYrObAj4e4W{a8_R0B3ZD=Sd~g&7@>V_cPMYG^S_^SfYC>%M>f7 ziW3-{DICL+@T{6nIb+$bC6fs?SlO}XUhWf+%G8Rk;QCKflvw&$IiThz&umPthUgt9 zsDb$<(xlkC$*El9g`9(bUFIyGn|>>on+V1%sajE$?|Olu)eln%Ey5U|vfWcs;i$DZ z2DEUlE(a@@-=NW9Ew#2}fg4MbDhpJWpIlqu&&8fi%kVl*%8}|P@x>X6f~rz0zBFm^ z140*USp34EH-P_YBt{ezJTqMAG$*B@5(+0!$#F9Zm5-O-fPNNC!M>!oO4R3j5x%2^ zZ2`Uh%atou_(^R)>?+i|~cCmnmBMI5IDuN#>_kg9XTg)yvb|Wo>{u zrBWBO-1IPUiDx-atXdtNh#r>YLz~qdTmibnQ_E^6jCK|ebNm^F*|Jj(jY?rC=>$($Z@H# zvb0K=%%xVV9ng1<&caMQE01Q&*0cD?VelQoGi9V7DG`ASK*O{=cNE8Mh-oh+`caX( zjVlf!6`f%zrPNUC^Rm+h*pB5USJ!ep*-~>Tg40|NjZWka71^>tS>rK|t@Au5YZbU@ znX?17?1(edigyyv2#kh2rZK`WlJW(bU9bsKQ?wN0E0T`*d3RpOAb4>(>aIuaey~dK z;Ceh6WT*}If<9Xr;RT6DdqEo8@PgF7E^@(9;;}30b~Xn6j_q^OIJy&0OfSif9^$zf z!O15}oj{!xOj&3_nqoz=pO7WFxEzv2xw$rLvm}%+oy5~9ZGa@s+d*2c&>Zp!NJ59y zK7AVB5vDg71ehIC4@Mf$?ECR9WWL_6h4jE=3RruNPz}OdYah2>&FVgbO__d2%P(my z$w_HIZ)@qbi>@kLK8Gcfa;V=*v<8Vt+0zn&0FZd zGtqi2#0#QV0<`OM(@R%ZW-qUxXHPFJP6spdOKZW*?BdcY5PJUyc({Uijr{ybl@7K8 zBWs}-&!ZI(_h<#0J&Y5y;ovytXBU?7Fi78SbR)4fgo22~3O21mXOOUCBDcLSoLVU4^<=Nr(9LNZM3SyKP0gMvE`Dg6P z#VAMb8kfF1t~pZSx6-b7P7f4#e3nW9q4J1Rs4MePJ<1}|%o?Tl?2a;Au1(KoS z1(MeA0?ANt1IbW8AZpfv+h(!I?pzrfW8!1Td$2f@?2w8mntS*{J?vg8ZquMiFqTIw ze`$qO(`2Z4O_SE}nkGZRZJG>)VS!N7s5#e4hG9?&BLdMXLjzG7Lj%zY!vfI?2*jFt ztR3} zXl#qS_~<0S9_)tMx#Fdz-bntGZ-xiHsEd~?+cVFl&G~cJ5j9>&n}`~_;v?G;rr-8W z-K{?Y#fu*GX%V8dVo2hOZwz&-O8kxvdv*PZ!x(>v7QKhv`wOEC@fj3tdNb(n;hU^F ziOUQ4)ggbK3Lmz?i~x zZ%~B~?Gm5Ddjpy|=uHBq5qhaA-VZcBpo-l~eAIdmpr+Eshotyby~IyiA&K`GjgKgx zALk`LE?OnHh!stJ`q@hwcgV4fOLQ&^{UTU=M0WjEeD9Thg-m__W2*kgbcs*Pj!IvH z!PHLghU6-rG=7sDyYCA=pu*3)>*6~WOKp5r3tu7qEuwmGm-v{~z2LTdQ?^h3Vn1p8 zHr4m&@(^$$p2KUgP+-pne}%l^&>Cwab-%K58ntwZ|P$o{4QDE_a)xD z8s7cU2JF92_V>QT`!4f778bv!E%+|$!hA+Bl-{Uh-5E2MnxeTg0&~YwQ#5zRZ1mkW zELbH=&j{}S&hzU239>963aM9-SI2gX%(7kOFQVtUJG5x(Fe(Q7D&Bo*DgG)Z=PT&* z1OL8weD2j?G8lU`xJru>K78F6;DY@}4gcYBqho$(^uX(WM2H^&JW9f&Bz$SV1$orj z9(A@yo$b*mcr*$gje_(ji1)r7HJhF9BpISTSGrMzUw$We_)m+|2rHV4R@gRyw%E>1 zMn&2zLuR0?J8h|NMrw@|YKrttmR-F2Vsn5ue?OpCEasA(bfd66@~jF+V-J=$&889K zhKBy1-Ixf?T{=V~Gk0k%jciLvYwH_hc3*eeUf+y1*J=;y##EPP93@L@f0>B!%-r2q z{l<*%M%dnmrkPymZWQPCHEEhME<6~{FR}LMT-s$5&OLL#eNSCYGmSsIy9np)#$UTW zsNtpU+hBab8bQB<#D@KTmky+k_f~K4A=0=#71FM%SKSzFA?;y5!tv7A zqh>F;(;HlmkFR$lfo*Q$Y-@3(baDrkbURzsHjd9m;V6FY6s`9mzX9a|z93ZPmO5&k zpA1LD(b+gaR3EBbo$LR>1GP-%H3)c|@_ zq(%pQ!z-ghhkgsM_@G>zZRt=c&`>GFP-(=Y@&XiUWy>pt_*5{|RN`33g6ow6jaLdW zR2p%qa2T!=AgclNs7OucO6e%C^m)lPl>rr%K@^pRNUzip%6XTHYb@urSC*Re%0e`i zJgM2GQlRllA;v3>Xew?R97Sy9N;z*@&>)6N8)_;Mi_?P2`9dl{sCfYb(Nw}uk4k}l zfHsxVF;whlJ}W?ghDxhxuQZ~);s9|Jnv*L98Y+c&0g|Dn66DSplu5?fYlcdzd0xR# zQ;F)F-Y(OgGrip)zE?2RR9sylD^~zLuSm`HiV#gDyz;0NXs8ros5GLf*ee{I?4(_R zhDsrZN+X&|kXKwP1sW=a7%GiODmv?$4y|_rG@zm~h@!F(kIHF5rDj^tAU>68LDIpg ztp#V`7|`}gs>v2KIv*>$rKgT ztNfq~=l);s!XGxLOiN0uO?Go0r9D9uXqe%^>lKjK;xA{43$Q-R|0Vc zjy{vw3Q()*05zhiXxgDOF&(cIXuML0@k%2emFJaEGfS#ydisL|J{1r_CC?DWdBK)f z7AU>45J_bt9+lmKT1~g0h4@su1+|WyD+~0!f|^LXrQ-)^=Sr*TTv>=uC3AJmoc3It z(8+qzLD?@CVm1$=d|7lQGih0hKz%w=q&QtI4%DKnn50>XKz%w=^ymokyepd7*`fpL z(~+V_N03K{zfqvNzeNYsrz1s=jv$Y&Vw%;W1M1U}qDM!NM_0^j3ZOn6DSC7Sd34qc zWO=7{^)BSok*!BpF@w~y1?qc8ik^1_d35-L5*bYa)TbjwkB%UZuAJEvKz%w=^ymok z=qhGAZ3kOQ(UGD@N03KX${ZCyeeX!oqa(eG>;M@NuHS25dcJF-fOjubsQf;_rXW>Wz5y(2}B zjv$W?e=a6tQ~>qqNYSGs$fK)dt`$IiI#TrL2=eGknN0!Irz1s=jv$Y&oY@pWeL7O~ z=m_%YDw%5qP@j$zJvxFsI{bl~jHUqU(~+V_N03KX&TI;xJ{>7~bOd>HmCUsQs82_V z9vwj*T`99EfckW#=+P16(Umis0;o?%iXI(79-Z}8&>FKqeL7O~=m_%YN|{Xo)Tbjw zkB%UZuAJEvKz%w=^ymok=q57P3ZOn6DSC7Sd32@BrU2^Gk)lUOkVj{|>$RFfz3g=% zpN?!jx{1uS0;umDDSF-!1iIR(C_bd#nB!GC{%~W`_y@f3$7}Z!_4bXZi-V9N=TBSK zcH+2l|tstR1Qy(SMC{D9vE}{{VLXpJ4z1 From 58f8cb9720764e47c3ee2b995518fb031579f085 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 24 Sep 2025 10:33:18 -0300 Subject: [PATCH 29/29] SIENTIAPDE-1222 Update .gitignore to include 'tmp/' directory and ensure '.env' is listed - Added 'tmp/' to the .gitignore file to prevent temporary files from being tracked. - Confirmed that '.env' is included to avoid committing sensitive environment variables. --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b14c9ac..cfaf446 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,6 @@ git_key* git_log -.env \ No newline at end of file +.env + +tmp/ \ No newline at end of file