From ec0a193c325b92b40fef162ea31acd8ac091eaff Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 23 Sep 2025 16:06:10 -0300 Subject: [PATCH 01/52] SIENTIAPDE-1231 Refactor OPC and model repository for improved functionality and clarity - Updated OPC server logging to handle missing prediction and confidence tags gracefully. - Corrected documentation for OPC reconnection interval from milliseconds to seconds. - Enhanced MLFlowRepository with new methods for model retrieval, caching, and transformation, improving model management and retraining workflows. --- encode.sh | 13 + encrypt.py | 112 +++ laborious/activities/opc.py | 2 +- laborious/utils/connectors_config.py | 2 +- .../utils/repository/model_repository.py | 899 ++++++++++++++---- model_convert.ipynb | 106 +++ 6 files changed, 969 insertions(+), 165 deletions(-) create mode 100755 encode.sh create mode 100644 encrypt.py create mode 100644 model_convert.ipynb diff --git a/encode.sh b/encode.sh new file mode 100755 index 0000000..2beb0d1 --- /dev/null +++ b/encode.sh @@ -0,0 +1,13 @@ +source ./venv/bin/activate + +pip install pathspec +pip install pyyaml + +echo " +.git" >> .gitignore + +python encrypt.py ./ code --ignore .gitignore --chunk-size 100000 + +sed -i '/.git/d' .gitignore + +xdg-open . \ No newline at end of file diff --git a/encrypt.py b/encrypt.py new file mode 100644 index 0000000..6aff252 --- /dev/null +++ b/encrypt.py @@ -0,0 +1,112 @@ +import os +import argparse +from pathspec import PathSpec +import yaml + +''' +Usage: + python .\encrypt.py path_to_dir output_file --ignore ignore_file --chunk-size 100000 +''' + + +def load_ignore_patterns(ignore_file, include_library): + # Ensure the .gitignore file exists + if not os.path.exists(ignore_file): + raise FileNotFoundError(f"Ignore file not found at {ignore_file}") + + # Load and parse the .gitignore patterns + with open(ignore_file, 'r') as file: + patterns = file.readlines() + if not include_library: + patterns.append('**/deploy/library/') + + spec = PathSpec.from_lines('gitwildmatch', patterns) + return spec + + +def is_ignored(file_path, spec): + """Check if a file should be ignored based on the ignore patterns.""" + return spec.match_file(file_path) if spec else False + + +def encode_file_tree_to_yaml(directory, ignore_file, include_library): + """Encode the file tree into a single YAML file.""" + ignore_patterns = load_ignore_patterns( + ignore_file, include_library) if ignore_file else None + file_tree = {} + + for root, dirs, files in os.walk(directory): + # Skip ignored directories + dirs[:] = [d for d in dirs if not is_ignored( + os.path.join(root, d), ignore_patterns)] + + for file in files: + file_path = os.path.join(root, file) + + # Skip ignored files + if is_ignored(file_path, ignore_patterns): + continue + + # Read file content + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + except Exception as e: + print(f"Error reading file {file_path}: {e}") + raise + + # Create nested dictionary structure + path_parts = os.path.relpath(file_path, directory).split(os.sep) + current_level = file_tree + + # all except the last part (the file name) + for part in path_parts[:-1]: + current_level = current_level.setdefault(part, {}) + + # Add the file and its content + current_level[path_parts[-1]] = content + return yaml.dump(file_tree, default_flow_style=False) + + +def chunk_and_write_file_tree_to_yaml(yaml_content, output_file, chunk_size=None): + """Chunk the YAML content and write it to the output file.""" + + chunks = [yaml_content] if chunk_size is None else [ + yaml_content[i:i + chunk_size] for i in range(0, len(yaml_content), chunk_size)] + + for i, chunk in enumerate(chunks): + chunk_file = f"{output_file}_{i}.yaml" + # Write the file tree to the output YAML file + with open(chunk_file, 'w', encoding='utf-8') as yaml_file: + yaml_file.write(chunk) + + +def main(): + parser = argparse.ArgumentParser( + description="Encrypts file tree to yaml file") + parser.add_argument("input_directory", help="Directory to encode") + parser.add_argument("output_yaml_file", help="Output YAML file") + parser.add_argument("--ignore", default=None, + help="Path to the ignore file") + parser.add_argument("--chunk-size", type=int, default=None, + help="Chunk size for the output YAML file") + parser.add_argument("--library", type=bool, default=False, + help="Incude the library in the output YAML file") + + # Parse arguments + args = parser.parse_args() + + # Example usage + directory_to_encode = args.input_directory + ignore_file_path = args.ignore + output_yaml_file = args.output_yaml_file + include_library = args.library + + content = encode_file_tree_to_yaml( + directory_to_encode, ignore_file_path, include_library) + chunk_and_write_file_tree_to_yaml( + content, output_yaml_file, args.chunk_size) + + +if __name__ == "__main__": + main() diff --git a/laborious/activities/opc.py b/laborious/activities/opc.py index 0b4ec9f..2c427e0 100644 --- a/laborious/activities/opc.py +++ b/laborious/activities/opc.py @@ -289,7 +289,7 @@ class OPC(BaseActivity): success = success and local_success self.info( - f"Process completed for OPC server {server_id}: {local_count} of {len(config['prediction_tags'])} prediction tags and {len(config['confidence_tags'])} confidence tags", metadata) + f"Process completed for OPC server {server_id}: {local_count} of {len(config.get('prediction_tags', []))} prediction tags and {len(config.get('confidence_tags', []))} confidence tags", metadata) return self.process_confidence(data, success, metadata) diff --git a/laborious/utils/connectors_config.py b/laborious/utils/connectors_config.py index 145f248..41c933f 100644 --- a/laborious/utils/connectors_config.py +++ b/laborious/utils/connectors_config.py @@ -75,7 +75,7 @@ def build_opc_config() -> Dict[str, Any]: OPC_CERT_PATH: Client certificate path (fallback, default: None) OPC_PRIVATE_KEY_PATH: Client private key path (fallback, default: None) OPC_SERVER_CERT_PATH: Server certificate path (fallback, default: None) - OPC_RECONNECTION_INTERVAL: Reconnection interval in milliseconds (fallback, default: 120) + OPC_RECONNECTION_INTERVAL: Reconnection interval in seconds (fallback, default: 120) Returns: dict: OPC server configuration dictionary diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 68322f3..07f0fc5 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -10,88 +10,75 @@ requests using the Model Monitoring API functions. By Monitoring we mean the evaluation of the performance of models, the generation of reports. """ -from datetime import datetime +from datetime import datetime, timedelta import traceback import pandas as pd import mlflow -from os import makedirs, path, remove -from sientia.ModelServing import ModelServing +from os import makedirs, path, remove, environ +from sys import path as sys_path +from sientia_do.observability.logger import Logger +import lzma +import gzip +import pickle + +ARTIFACTS_PATH = "./tmp/artifacts" class MLFlowRepository(): - def __init__(self, host, username, password, logger): + def __init__(self, host: str, username: str, password: str, logger: Logger): - self.model_serving = ModelServing(tracking_uri=host, - username=username, password=password, - logger=logger) + # set tracking uri + mlflow.set_tracking_uri(host) - def transform(self, model_name: str, data: pd.DataFrame, model_retention: int): + environ["MLFLOW_TRACKING_USERNAME"] = username + environ["MLFLOW_TRACKING_PASSWORD"] = password + # Create an MLflow client + self.client = mlflow.tracking.MlflowClient() + + self.model_cache = {} + + """ + Functions related to get model registry parameters + """ + + def get_model_uri(self, run_id: str, prediction: bool = True): """ - Transform data using a model. + Get the model URI based on the run_id. - Parameters: - - model_name (str): The name of the model to use for transformation. - - data (pandas.DataFrame): The data to transform. - - model_retention (int): The number of minutes to keep the model. + Args: + run_id (str): The run_id of the model. Returns: - - dict: A dictionary containing the transformed data. + str: The model URI. """ + run_info = mlflow.get_run(run_id) + if prediction: + model_uri = run_info.info.artifact_uri + "/prediction_model" + else: + model_uri = run_info.info.artifact_uri + "/data_model" + return model_uri - try: - - return { - 'success': True, - 'content': self.model_serving.get_cached_transform( - model_name, data, model_retention).to_dict() - } - - except Exception as e: - return { - 'success': False, - 'content': { - 'message': str(e), - 'traceback': traceback.format_exc() - } - } - - def predict(self, model_name: str, data: pd.DataFrame, model_retention: int): + def get_model_run_id(self, model_name: str, stage: str = "Production"): """ - Predict data using a model. + Get the run_id of a model based on its name and stage. - Parameters: - - model_name (str): The name of the model to use for prediction. - - data (pandas.DataFrame): The data to predict. - - model_retention (int): The number of minutes to keep the model. + Args: + model_name (str): The name of the model. + stage (str): The stage of the model. Returns: - - dict: A dictionary containing the predicted data. + str: The run_id of the model. """ - try: - - input_index = data.index - start_time = datetime.now() - data = self.model_serving.get_cached_predict( - model_name, data, model_retention) - - end_time = datetime.now() - data = pd.DataFrame(data, columns=['prediction']) - data.index = input_index - data['response_time'] = (end_time - start_time).total_seconds() - - return { - 'success': True, - 'content': data.to_dict() - } - - except Exception as e: - return { - 'success': False, - 'content': { - 'message': str(e), - 'traceback': traceback.format_exc() - } - } + latest_versions = self.client.get_latest_versions( + name=model_name, stages=[stage] + ) + if not latest_versions: + raise mlflow.exceptions.MlflowException( + f"Model '{model_name}' in stage '{stage}' not found in the Model Registry." + ) + else: + run_id = latest_versions[0].source.split("/") + return run_id[2] def get_experiment_by_run_id(self, run_id: str) -> dict: # Get the run information using the run_id @@ -124,6 +111,491 @@ class MLFlowRepository(): next_run_number = len(runs) + 1 return f"{model_name}-{next_run_number}" + def get_experiment(self, experiment_name: str) -> int: + """ + Retrieve MLFlow experiment ID by experiment name. + + This method searches for an MLFlow experiment by name and + returns its unique identifier. It provides error handling + for non-existent experiments. + + Args: + experiment_name (str): Name of the MLFlow experiment + + Returns: + int: MLFlow experiment ID + + Raises: + ValueError: If the experiment name is not found + """ + experiment = mlflow.get_experiment_by_name(experiment_name) + + if experiment is None: + raise ValueError(f'Experiment {experiment_name} not found') + + return int(experiment.experiment_id) + + def get_experiment_last_run(self, experiment_id: int) -> str: + """ + Retrieve the most recent retraining run ID for an experiment. + + This method searches for the latest run in an MLFlow experiment + that has been marked as a retraining run. It filters runs by + the 'retrain' parameter and orders them by completion time. + + Args: + experiment_id (int): MLFlow experiment ID + + Returns: + str: MLFlow run ID of the most recent retraining run + + Raises: + ValueError: If runs data is not in expected DataFrame format + """ + runs = mlflow.search_runs( + experiment_ids=[experiment_id], + filter_string="", # Sem filtro no MLflow ainda + output_format="pandas" + ) + + if not isinstance(runs, pd.DataFrame): + raise ValueError('Runs is not a pandas DataFrame') + + # Filtrar apenas as runs onde params.retrain == True + filtered_runs = runs[runs["params.retrain"] == 'True'] + + # Converter a coluna 'end_time' para datetime + filtered_runs['end_time'] = pd.to_datetime(filtered_runs['end_time']) + + # Ordenar o DataFrame de forma descendente pela coluna 'end_time' + filtered_runs = filtered_runs.sort_values( + by='end_time', ascending=False) + + # Pegar a última run_id do DataFrame filtrado e ordenado + latest_run_id = filtered_runs.iloc[0]['run_id'] + + return latest_run_id + + """ + Functions related to download and load models + """ + + def dowload_artifacts(self, model_name: str, artifact_path: str = "data_model") -> str: + """ + Downloads artifacts from a specific MLFlow run. + """ + run_id = self.get_model_run_id( + model_name=model_name, stage="Production" + ) + output_dir = f"{ARTIFACTS_PATH}/{model_name}" + + if not path.exists(output_dir): + makedirs(output_dir) + + return self.client.download_artifacts( + run_id, + artifact_path, + output_dir + ) + + def load_predict_model(self, model_name: str, flavor: str = 'pyfunc', + artifact_path: str | None = None): + """ + Downloads a predictive model from the MLflow Model Registry. + Args: + model_name (str): The name of the model to download from the registry. + Returns: + mlflow.pyfunc.PyFuncModel: The loaded predictive model. + Notes: + - The model is fetched from the "production" stage of the MLflow Model Registry. + - Warnings during the model loading process are suppressed. + """ + + model_uri = f"models:/{model_name}/production" + + if artifact_path: + self.logger.info( + f"Prediction model {model_name} is not compressed, loading from {artifact_path}") + + model = self.load_model_with_compression( + artifact_path, "prediction") + + else: + if flavor == 'pyfunc': + model = mlflow.pyfunc.load_model(model_uri) + elif flavor == 'sklearn': + model = mlflow.sklearn.load_model(model_uri) + elif flavor == 'pytorch': + model = mlflow.pytorch.load_model(model_uri) + else: + raise ValueError( + "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") + + return model + + def load_transform_model(self, model_name: str, flavor: str, + artifact_path: str | None = None): + """ + Downloads the latest production version of a specified model. + + This method retrieves the latest production model run ID for the given + model name, constructs the model URI, and loads the model using MLflow. + + Args: + model_name (str): The name of the model to download. + + Returns: + Any: The loaded model object, as returned by `mlflow.sklearn.load_model`. + + Raises: + Exception: If the model run ID or URI cannot be retrieved, or if the + model cannot be loaded. + """ + latest_production_id = self.get_model_run_id( + model_name=model_name, stage="Production" + ) + model_uri = self.get_model_uri( + latest_production_id, prediction=False) + + self.logger.debug( + f"Model {model_name} is at {model_uri} and latest production id is {latest_production_id}") + + if artifact_path: + # Download model artifacts + self.logger.info( + f"Data model {model_name} is not compressed, loading from {artifact_path}") + + model = self.load_model_with_compression( + artifact_path, "transformer") + else: + if flavor == 'sklearn': + model = mlflow.sklearn.load_model(model_uri) + elif flavor == 'pyfunc': + model = mlflow.pyfunc.load_model(model_uri) + elif flavor == 'pytorch': + model = mlflow.pytorch.load_model(model_uri) + else: + raise ValueError( + "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") + return model + + def load_model_with_compression(self, artifact_path: str, type: str): + """ + Load model from pickle file trying different compression methods. + + Args: + pickle_path (str): Path to the pickle file + + Returns: + Any: Loaded model object + + Raises: + ValueError: If model cannot be loaded with any compression method + """ + + if type == "transformer": + code_path = path.join( + artifact_path, "transformer_pyfunc", "code") + pickle_path = path.join( + artifact_path, "transformer_pyfunc", "artifacts", "training_transformer.pkl") + + elif type == "prediction": + code_path = path.join( + artifact_path, "stacking_model", "code") + pickle_path = path.join( + artifact_path, "stacking_model", "artifacts", "stacking_model.pkl") + + if code_path not in sys_path: + sys_path.insert(0, code_path) + self.logger.info( + f"Added {code_path} to Python path") + + loading_methods = [ + ("lzma", lambda p: lzma.open(p, "rb")), + ("gzip", lambda p: gzip.open(p, "rb")), + ("pickle", lambda p: open(p, "rb")), + ] + + for format_name, open_func in loading_methods: + try: + self.logger.info(f"Trying to load with {format_name}...") + with open_func(pickle_path) as f: + model = pickle.load(f) + self.logger.info( + f"Successfully loaded with {format_name}!") + return model + except (lzma.LZMAError, gzip.BadGzipFile, OSError, pickle.UnpicklingError, ValueError) as e: + self.logger.info( + f"Failed with {format_name}: {e.__class__.__name__}:{e}") + continue + + raise ValueError( + f"Could not load model from {pickle_path} - unknown or corrupted format") + + def download_model(self, model_name: str, model_type: str, flavor: str, + compressed: bool = False) -> dict: + """ + Download model based on type (predict or transform). + + Args: + model_name (str): Name of the model to download + model_type (str): Type of model ('predict' or 'transform') + flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') + compressed (bool): Whether model is compressed + + Returns: + dict: Model configuration with model and artifact paths + """ + if model_type == "predict": + if compressed: + artifact_path = self.dowload_artifacts( + model_name, "prediction_model") + else: + artifact_path = None + model = self.load_predict_model(model_name, flavor, artifact_path) + + elif model_type == "transform": + if compressed: + self.logger.info( + f"Model {model_name} is compressed, downloading artifacts") + + artifact_path = self.dowload_artifacts( + model_name, "data_model") + + self.logger.info( + f"Artifacts downloaded at path {artifact_path}") + else: + artifact_path = None + model = self.load_transform_model( + model_name, flavor, artifact_path) + else: + raise ValueError( + "Invalid model_type. Use 'predict' or 'transform'.") + + return { + "model": model, + "artifact_path": artifact_path + } + + """ + Functions related to cache management of models + """ + + def check_cache_config(self, cache: dict, new_config: dict) -> bool: + """ + Check if cache configuration matches new configuration. + + Args: + cache (dict): Cached model configuration + new_config (dict): New configuration to compare + + Returns: + bool: True if configurations match, False otherwise + """ + old_config = cache['config'] + if old_config != new_config: + return False + return True + + def check_cache_retention(self, cache: dict, retention: int) -> bool: + """ + Check if cache is still valid based on retention time. + + Args: + cache (dict): Cached model data + retention (int): Retention time in minutes + + Returns: + bool: True if cache is still valid, False if expired + """ + current_time = self.now() + cache_time = cache['timestamp'] + if current_time - cache_time >= timedelta(minutes=retention): + return False + return True + + def handle_valid_model(self, model_name: str, model_type: str, + compressed: bool, retention_target: str, + cache: dict) -> dict: + """ + Handle valid cached model by returning appropriate model configuration. + + Args: + model_name (str): Name of the model + model_type (str): Type of model ('predict' or 'transform') + compressed (bool): Whether model is compressed + retention_target (str): Retention target ('model' or 'artifact') + cache (dict): Cached model data + + Returns: + dict: Model configuration with model and artifact path + """ + if self.logger: + self.logger.debug( + f"Model {model_name} is still valid, using cached version") + + # If model is compressed and retention target is artifact, load the model from pkl + if compressed and retention_target == "artifact": + model = self.load_model_with_compression( + cache['target']['artifact_path'], model_type) + return { + 'model': model, + 'artifact_path': cache['target']['artifact_path'] + } + else: + return cache['target'] + + def handle_outdated_model(self, model_name: str, model_key: str) -> dict: + """ + Clean up outdated cached model and its artifacts. + + Args: + model_name (str): Name of the model + model_key (str): Cache key for the model + + Returns: + dict: Empty dictionary (cleanup operation) + """ + if self.logger: + self.logger.debug( + f"Model {model_name} is outdated, downloading a new one") + + del self.model_cache[model_key]['target']['model'] + if path.exists(self.model_cache[model_key]['target']['artifact_path']): + remove(self.model_cache[model_key] + ['target']['artifact_path']) + del self.model_cache[model_key] + + def get_model(self, model_name: str, retention: int, model_type: str, flavor: str, + compressed: bool = False, retention_target: str = "model"): + """ + Get model with caching support based on retention policy. + + Args: + model_name (str): Name of the model to retrieve + retention (int): Cache retention time in minutes (0 = no cache) + model_type (str): Type of model ('predict' or 'transform') + flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') + compressed (bool): Whether model is compressed + retention_target (str): What to cache ('model' or 'artifact') + + Returns: + dict: Model configuration with model and artifact paths + """ + # Retention is 0, download a new model + if retention <= 0: + model_config = self.download_model( + model_name, model_type, flavor, compressed) + return model_config + + model_key = f'{model_name}_{model_type}' + config = { + 'compressed': compressed, + 'retention_target': retention_target + } + + if model_key in self.model_cache: + cache = self.model_cache[model_key] + + # Check if config has changed or is outdated + if self.check_cache_config(cache, config) or self.check_cache_retention(cache, retention): + return self.handle_valid_model( + model_name, model_type, compressed, + retention_target, cache) + else: + # Model is outdated, delete old model files + self.handle_outdated_model(model_name, model_key) + else: + if self.logger: + self.logger.debug( + f"Model {model_name} is not in the cache, downloading a new one") + + # Donwload new model + model_config = self.download_model(model_name, model_type, flavor, + compressed) + # If model is compressed and retention target is artifact, + # dont save the model in the cache + if compressed and retention_target == "artifact": + model_config_to_cache = { + 'artifact_path': model_config['artifact_path'], + 'model': None + } + else: + model_config_to_cache = { + **model_config + } + cache = { + 'target': model_config_to_cache, + 'config': config, + 'timestamp': self.now() + } + + self.model_cache[model_key] = cache + + return model_config + + def get_cached_transform(self, model_name: str, data: pd.DataFrame, retention: int, flavor: str, + compressed: bool = False, retention_target: str = "model", keyword: str = "predict") -> pd.DataFrame: + """ + Get transformed data using cached transform model. + + Args: + model_name (str): Name of the transform model + data (pd.DataFrame): Data to transform + retention (int): Cache retention time in minutes + flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') + compressed (bool): Whether model is compressed + retention_target (str): What to cache ('model' or 'artifact') + keyword (str): Method name to call on model (default: 'predict') + + Returns: + pd.DataFrame: Transformed data + """ + model_config = self.get_model(model_name, retention, + "transform", flavor, compressed, retention_target) + model = model_config['model'] + + # Use getattr to dynamically call the method specified by keyword + method = getattr(model, keyword) + transformed_data = method(data) + + # If model is compressed and retention target is artifact, + # delete the model after the prediction + if compressed and retention_target == "artifact": + del model + + # If retention is 0, delete the artifacts after the prediction + if retention == 0 and model_config['artifact_path'] is not None: + if path.exists(model_config['artifact_path']): + remove(model_config['artifact_path']) + + return transformed_data + + def get_cached_predict(self, model_name: str, data: pd.DataFrame, retention: int, flavor: str, + compressed: bool = False, retention_target: str = "model") -> pd.DataFrame: + """ + Get predictions using cached prediction model. + + Args: + model_name (str): Name of the prediction model + data (pd.DataFrame): Data to make predictions on + retention (int): Cache retention time in minutes + flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') + compressed (bool): Whether model is compressed + retention_target (str): What to cache ('model' or 'artifact') + + Returns: + pd.DataFrame: Model predictions + """ + model_config = self.get_model(model_name, retention, "predict", + flavor, compressed, retention_target) + model = model_config['model'] + return model.predict(data) + + """ + Functions related to model retraining + """ + def create_model_experiment(self, model_name: str, data: pd.DataFrame) -> tuple: """ Create a new MLFlow experiment for model retraining. @@ -148,10 +620,10 @@ class MLFlowRepository(): # load predictor model predictor_uri = f"models:/{model_name}/production" # load transform model - latest_production_id = self.model_serving.get_model_run_id( + latest_production_id = self.get_model_run_id( model_name, stage="Production" ) - transform_uri = self.model_serving.get_model_uri( + transform_uri = self.get_model_uri( latest_production_id, prediction=False ) # load @@ -235,96 +707,6 @@ class MLFlowRepository(): return "Model retrained successfully", experiment - def retrain_model(self, data: pd.DataFrame, model_name: str) -> tuple: - """ - Orchestrate the complete model retraining workflow. - - This method coordinates the entire model retraining process by: - 1. Creating the MLFlow experiment environment - 2. Loading existing production models - 3. Executing the retraining process - 4. Returning comprehensive retraining results - - Args: - data (pd.DataFrame): Training data for model retraining - model_name (str): Name of the MLFlow model to retrain - - Returns: - tuple: (status_message, experiment_name) - - status_message (str): Retraining operation status - - experiment_name (str): MLFlow experiment identifier - """ - prediction_model, data_model, experiment = self.create_model_experiment( - model_name, data) - retrain_result = self.perform_model_retrain( - prediction_model, data_model, experiment, model_name, data) - return retrain_result - - def get_experiment(self, experiment_name: str) -> int: - """ - Retrieve MLFlow experiment ID by experiment name. - - This method searches for an MLFlow experiment by name and - returns its unique identifier. It provides error handling - for non-existent experiments. - - Args: - experiment_name (str): Name of the MLFlow experiment - - Returns: - int: MLFlow experiment ID - - Raises: - ValueError: If the experiment name is not found - """ - experiment = mlflow.get_experiment_by_name(experiment_name) - - if experiment is None: - raise ValueError(f'Experiment {experiment_name} not found') - - return int(experiment.experiment_id) - - def get_experiment_last_run(self, experiment_id: int) -> str: - """ - Retrieve the most recent retraining run ID for an experiment. - - This method searches for the latest run in an MLFlow experiment - that has been marked as a retraining run. It filters runs by - the 'retrain' parameter and orders them by completion time. - - Args: - experiment_id (int): MLFlow experiment ID - - Returns: - str: MLFlow run ID of the most recent retraining run - - Raises: - ValueError: If runs data is not in expected DataFrame format - """ - runs = mlflow.search_runs( - experiment_ids=[experiment_id], - filter_string="", # Sem filtro no MLflow ainda - output_format="pandas" - ) - - if not isinstance(runs, pd.DataFrame): - raise ValueError('Runs is not a pandas DataFrame') - - # Filtrar apenas as runs onde params.retrain == True - filtered_runs = runs[runs["params.retrain"] == 'True'] - - # Converter a coluna 'end_time' para datetime - filtered_runs['end_time'] = pd.to_datetime(filtered_runs['end_time']) - - # Ordenar o DataFrame de forma descendente pela coluna 'end_time' - filtered_runs = filtered_runs.sort_values( - by='end_time', ascending=False) - - # Pegar a última run_id do DataFrame filtrado e ordenado - latest_run_id = filtered_runs.iloc[0]['run_id'] - - return latest_run_id - def update_production_model_by_run_id(self, run_id: str, model_name: str) -> dict: """ Update production model with a specific MLFlow run. @@ -382,24 +764,215 @@ class MLFlowRepository(): 'mlflow_run_id': run_id } + """ + Functions that provide the interface to model operations + """ + + def transform(self, model_name: str, data: pd.DataFrame, model_retention: int): + """ + Transform data using a cached transformation model. + + This method provides a high-level interface for data transformation operations + using MLFlow models. It handles model caching, error management, and data + format conversion automatically. + + Process Flow: + 1. Retrieves or downloads the transformation model using caching mechanism + 2. Applies the transformation model to the input data + 3. Converts the transformed data to dictionary format for API response + 4. Handles any exceptions and returns structured error information + 5. Manages model lifecycle based on retention policy (cleanup artifacts if needed) + + Parameters: + model_name (str): The name of the MLFlow model to use for transformation. + data (pd.DataFrame): The input data to be transformed by the model. + model_retention (int): Cache retention time in minutes (0 = no caching). + + Returns: + dict: Response dictionary containing: + - success (bool): Operation success status + - content (dict): Transformed data as dictionary, or error information + if operation failed. Error content includes: + - message (str): Error description + - traceback (str): Full exception traceback + + Raises: + Exception: Any exception during model loading or transformation is caught + and returned in the response structure rather than propagated. + """ + + try: + + return { + 'success': True, + 'content': self.get_cached_transform( + model_name, data, model_retention).to_dict() + } + + except Exception as e: + return { + 'success': False, + 'content': { + 'message': str(e), + 'traceback': traceback.format_exc() + } + } + + def predict(self, model_name: str, data: pd.DataFrame, model_retention: int): + """ + Generate predictions using a cached prediction model. + + This method provides a high-level interface for model prediction operations + using MLFlow models. It handles model caching, performance monitoring, + response formatting, and error management automatically. + + Process Flow: + 1. Preserves input data index for result alignment + 2. Records prediction start time for performance measurement + 3. Retrieves or downloads the prediction model using caching mechanism + 4. Executes model prediction on the input data + 5. Formats predictions into DataFrame with proper column naming + 6. Restores original data index to maintain data alignment + 7. Calculates and adds response time measurement + 8. Converts results to dictionary format for API response + 9. Handles any exceptions and returns structured error information + + Parameters: + model_name (str): The name of the MLFlow model to use for prediction. + data (pd.DataFrame): The input data to make predictions on. + model_retention (int): Cache retention time in minutes (0 = no caching). + + Returns: + dict: Response dictionary containing: + - success (bool): Operation success status + - content (dict): Prediction results as dictionary with: + - prediction: Model predictions array + - response_time: Prediction execution time in seconds + Or error information if operation failed: + - message (str): Error description + - traceback (str): Full exception traceback + + Raises: + Exception: Any exception during model loading or prediction is caught + and returned in the response structure rather than propagated. + """ + try: + + input_index = data.index + start_time = datetime.now() + data = self.get_cached_predict( + model_name, data, model_retention) + + end_time = datetime.now() + data = pd.DataFrame(data, columns=['prediction']) + data.index = input_index + data['response_time'] = (end_time - start_time).total_seconds() + + return { + 'success': True, + 'content': data.to_dict() + } + + except Exception as e: + return { + 'success': False, + 'content': { + 'message': str(e), + 'traceback': traceback.format_exc() + } + } + + def retrain_model(self, data: pd.DataFrame, model_name: str) -> tuple: + """ + Orchestrate the complete model retraining workflow. + + This method coordinates the entire model retraining process by managing + the MLFlow experiment lifecycle, model loading, retraining execution, + and artifact management. It provides a comprehensive retraining solution + that maintains model versioning and experiment tracking. + + Process Flow: + 1. Creates MLFlow experiment environment: + - Loads current production prediction model + - Loads current production transformation model + - Fits transformation model with new training data + - Prepares transformed data for prediction model retraining + - Sets up MLFlow experiment context + 2. Executes model retraining: + - Starts new MLFlow run with descriptive metadata + - Logs model parameters and hyperparameters + - Retrains both prediction and transformation models + - Logs training data as artifacts + - Saves retrained models to MLFlow registry + - Cleans up temporary files + 3. Returns comprehensive retraining results + + Args: + data (pd.DataFrame): Training data for model retraining. Must contain + all features required by both transformation and + prediction models, including target variable. + model_name (str): Name of the MLFlow model to retrain. Must exist + in the MLFlow Model Registry in Production stage. + + Returns: + tuple: Retraining operation results containing: + - status_message (str): Success confirmation message or error details + - experiment_name (str): MLFlow experiment identifier for tracking + + Raises: + mlflow.exceptions.MlflowException: If model not found in registry + ValueError: If experiment cannot be created or models cannot be loaded + Exception: Any other exception during the retraining process + """ + prediction_model, data_model, experiment = self.create_model_experiment( + model_name, data) + retrain_result = self.perform_model_retrain( + prediction_model, data_model, experiment, model_name, data) + return retrain_result + def update_production_model(self, experiment: str, model_name: str) -> dict: """ Update production model using the latest retraining run. - This method orchestrates the complete production model update - process by identifying the most recent retraining run and - promoting it to production stage. + This method orchestrates the complete production model update process by + identifying the most recent retraining run and promoting it to production + stage. It handles model registration, versioning, and stage transitions + with comprehensive metadata tracking. + + Process Flow: + 1. Retrieves experiment information: + - Converts experiment name to MLFlow experiment ID + - Searches for the most recent retraining run in the experiment + - Filters runs by 'retrain' parameter and orders by completion time + 2. Promotes model to production: + - Registers the model from the specified run to MLFlow Model Registry + - Retrieves the latest model version number + - Transitions the model to 'Production' stage + - Archives existing production versions automatically + 3. Returns comprehensive update metadata Args: - experiment (str): MLFlow experiment name - model_name (str): Name of the MLFlow model + experiment (str): MLFlow experiment name containing the retraining runs. + Must be a valid experiment that exists in MLFlow. + model_name (str): Name of the MLFlow model to update. Must exist + in the MLFlow Model Registry. Returns: dict: Complete model update metadata containing: - model_name (str): Name of the updated model - - version (str): New model version number - - mlflow_run_id (str): Source run ID - - mlflow_experiment_id (int): Experiment ID + - version (str): New model version number (incremented automatically) + - mlflow_run_id (str): Source run ID of the promoted model + - mlflow_experiment_id (int): Experiment ID for tracking + + Raises: + ValueError: If experiment not found or model versions are invalid + mlflow.exceptions.MlflowException: If model registration or stage + transition fails + Exception: Any other exception during the update process + + Note: + This operation is irreversible. The previous production model will + be automatically archived when the new version is promoted. """ experiment_id = self.get_experiment(experiment) run_id = self.get_experiment_last_run(experiment_id) diff --git a/model_convert.ipynb b/model_convert.ipynb new file mode 100644 index 0000000..42f6c3d --- /dev/null +++ b/model_convert.ipynb @@ -0,0 +1,106 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 23, + "id": "e838ff21", + "metadata": {}, + "outputs": [], + "source": [ + "import csv\n", + "\n", + "def csv_to_tag_lists(csv_path: str) -> dict:\n", + " read_tags = []\n", + " write_tags = []\n", + "\n", + " def to_float(val):\n", + " try:\n", + " return float(str(val).strip())\n", + " except Exception:\n", + " return None\n", + "\n", + " with open(csv_path, newline=\"\", encoding=\"utf-8\") as f:\n", + " reader = csv.DictReader(f)\n", + " for row in reader:\n", + " # Basic normalization\n", + " op = (row.get(\"operation\") or \"\").strip()\n", + "\n", + " if op == \"READ\":\n", + " # Build common tag payload with required mappings\n", + " tag = {\n", + " \"server_id\": \"1\",\n", + " \"tag_address\": row.get(\"opc_tag\"),\n", + " \"tag_name\": row.get(\"name\"),\n", + " \"data_range\": [to_float(row.get(\"min_value\")), to_float(row.get(\"max_value\"))],\n", + " \"aggr_func\": row.get(\"aggregation_func\").lower(),\n", + " # keep other fields with their original names\n", + " \"frequency\": row.get(\"frequency\"),\n", + " \"local\": row.get(\"local\"),\n", + " \"area\": row.get(\"area\"),\n", + " \"description\": row.get(\"description\"),\n", + " }\n", + "\n", + " read_tags.append(tag)\n", + "\n", + " else:\n", + " tag = {\n", + " \"server_id\": \"1\",\n", + " \"addr\": row.get(\"opc_tag\"),\n", + " \"tag_name\": row.get(\"name\"),\n", + " \"local\": row.get(\"local\"),\n", + " \"area\": row.get(\"area\"),\n", + " \"description\": row.get(\"description\"),\n", + " }\n", + " \n", + " if op == \"WRITE_PREDICTION\":\n", + " tag[\"type\"] = \"prediction\"\n", + " write_tags.append(tag)\n", + " elif op == \"WRITE_CONFIDENCE\":\n", + " tag[\"type\"] = \"confidence\"\n", + " write_tags.append(tag)\n", + " # ignore any other operation values silently\n", + "\n", + " return {\"read_tags\": read_tags, \"write_tags\": write_tags}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4621cd43", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "file_names = [\"Courier - Página1.csv\"]\n", + "\n", + "for file_name in file_names:\n", + " write_file = file_name.replace(\".csv\", \".json\")\n", + "\n", + " with open(write_file, \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(csv_to_tag_lists(file_name), f, indent=2, ensure_ascii=False)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From d55b8bb1877da6c24b4ff1485ded878b8c1c3a27 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 5 Sep 2025 16:49:38 -0300 Subject: [PATCH 02/52] SIENTIAPDE-1214 Update README.md to include additional PostgreSQL, OPC, and MongoDB configuration options for enhanced clarity and usability --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index b2c37be..330cbb4 100644 --- a/README.md +++ b/README.md @@ -537,15 +537,35 @@ The Laborious system exposes comprehensive Prometheus metrics: | `POSTGRES_USER` | PostgreSQL username | `sientia` | Yes | | `POSTGRES_PASSWORD` | PostgreSQL password | `sientia` | Yes | | `POSTGRES_DBNAME` | PostgreSQL database | `sientia` | Yes | +| `POSTGRES_MIN_CONNECTIONS` | Minimum PostgreSQL connections | `10` | No | +| `POSTGRES_MAX_CONNECTIONS` | Maximum PostgreSQL connections | `30` | No | | `MLFLOW_HOST` | MLFlow server hostname | `localhost` | Yes | | `MLFLOW_PORT` | MLFlow server port | `5000` | Yes | | `MLFLOW_USERNAME` | MLFlow username | `admin` | Yes | | `MLFLOW_PASSWORD` | MLFlow password | `admin` | Yes | | `OPC_CONFIG` | OPC server configuration (JSON) | `{}` | No | +| `OPC_ID` | OPC server identifier | `1` | No | +| `OPC_URL` | OPC server URL | `opc.tcp://localhost:4840` | No | +| `OPC_NAME` | OPC server name | `OPC_Server` | No | +| `OPC_SERVER_URI` | OPC server URI | `urn:opcserver:opcua` | No | +| `OPC_CERT_PATH` | OPC client certificate path | `/path/to/cert.pem` | No | +| `OPC_PRIVATE_KEY_PATH` | OPC private key path | `/path/to/key.pem` | No | +| `OPC_SERVER_CERT_PATH` | OPC server certificate path | `/path/to/server_cert.pem` | No | +| `OPC_RECONNECTION_INTERVAL` | OPC reconnection interval (ms) | `5000` | No | | `MONGODB_URL` | MongoDB connection URI | `localhost:27017` | Yes | +| `MONGODB_USERNAME` | MongoDB username | `root` | Yes | +| `MONGODB_PASSWORD` | MongoDB password | `password` | Yes | +| `MONGODB_DATABASE` | MongoDB database name | `sientia` | Yes | +| `MONGODB_TTL_INDEX_HOURS` | MongoDB TTL index hours | `1` | No | +| `KAFKA_BOOTSTRAP_SERVERS` | Kafka bootstrap servers | `localhost:9092` | No | +| `LOG_LEVEL` | Application log level | `INFO` | No | +| `PROJECT_NAME` | Project name for metrics | `sientia-laborious` | No | | `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No | | `HTTP_SDK_METRICS_PORT` | Temporal SDK metrics port | `9091` | No | + + + ### OPC Configuration For multiple OPC servers, use the `OPC_CONFIG` environment variable: From f7217d400f7f0049ba53d3be49296dcfa9598060 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 12 Sep 2025 14:23:35 -0300 Subject: [PATCH 03/52] SIENTIAPDE-1214 SIENTIAPDE-1214: Refactor MLFlow and model repository methods to use model_config dictionary - Updated MLFlow class methods to accept model_config instead of model_retention for improved flexibility. - Modified model_repository methods to handle model_config, extracting necessary parameters for transformation and prediction. - Adjusted predictions_batch and prediction_process workflows to utilize model_config for better configuration management. - Commented out the previous sientia-mlops-library dependency in requirements.txt for clarity. --- laborious/activities/mlflow.py | 12 +- laborious/workflows/predictions_batch.py | 2 +- .../sub_workflows/prediction_process.py | 12 +- requirements.txt | 3 +- tests.ipynb | 2 +- .../data_model/transformer_pyfunc/MLmodel | 19 + .../transformer_pyfunc/code/utils/__init__.py | 0 .../code/utils/data/.gitkeep | 0 .../code/utils/data/__init__.py | 0 .../code/utils/data/preprocessing.py | 172 ++++ .../code/utils/data/read_data.py | 56 ++ .../code/utils/data/transformers.py | 788 ++++++++++++++++ .../code/utils/dvc/__init__.py | 0 .../code/utils/dvc/params.py | 51 + .../code/utils/features/.gitkeep | 0 .../code/utils/features/__init__.py | 0 .../code/utils/mlflow/pyfunc_wrappers.py | 325 +++++++ .../code/utils/models/.gitkeep | 0 .../code/utils/models/__init__.py | 24 + .../code/utils/models/arima.py | 389 ++++++++ .../code/utils/models/base.py | 824 ++++++++++++++++ .../code/utils/models/catboost_time_series.py | 510 ++++++++++ .../code/utils/models/evaluation.py | 92 ++ .../code/utils/models/factory.py | 140 +++ .../models/linear_regression_time_series.py | 303 ++++++ .../code/utils/models/neural_prophet_model.py | 888 ++++++++++++++++++ .../code/utils/models/stacking_time_series.py | 695 ++++++++++++++ .../code/utils/visualization/.gitkeep | 0 .../code/utils/visualization/__init__.py | 0 .../data_model/transformer_pyfunc/conda.yaml | 11 + .../transformer_pyfunc/python_env.yaml | 7 + .../transformer_pyfunc/python_model.pkl | Bin 0 -> 123 bytes .../transformer_pyfunc/requirements.txt | 4 + .../transformers/courier_transformers.pkl | Bin 0 -> 29607 bytes 34 files changed, 5315 insertions(+), 14 deletions(-) create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/MLmodel create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/conda.yaml create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/requirements.txt create mode 100644 tmp/artifacts/data_model/transformers/courier_transformers.pkl diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 3e3549c..c0785fd 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -94,7 +94,7 @@ class MLFlow(BaseActivity): self.info('Transforming data...', metadata) data = DataFrame(input_data['data']) model_name = input_data['model_name'] - model_retention = input_data['model_retention'] + model_config = input_data.get('model_config', {}) self.debug("Raw input data:", metadata) self.debug(data, metadata) @@ -117,10 +117,11 @@ class MLFlow(BaseActivity): # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( - model_name, data, model_retention) + model_name, data, model_config + ) self.debug("Transform response data:", metadata) - self.debug(json.dumps(response_data, indent=4), metadata) + self.debug(response_data, metadata) self.info("Data transformed successfully", metadata) @@ -160,7 +161,7 @@ class MLFlow(BaseActivity): self.info('Predicting data...', metadata) data = DataFrame(input_data['data']) model_name = input_data['model_name'] - model_retention = input_data['model_retention'] + model_config = input_data.get('model_config', {}) self.debug(data, metadata) @@ -169,7 +170,8 @@ class MLFlow(BaseActivity): # Request prediction from MLFlow model response_data = self.model_monitoring_repository.predict( - model_name, data, model_retention) + model_name, data, model_config + ) self.debug("Prediction response data:", metadata) self.debug(json.dumps(response_data, indent=4), metadata) diff --git a/laborious/workflows/predictions_batch.py b/laborious/workflows/predictions_batch.py index de1fbbe..cc10def 100644 --- a/laborious/workflows/predictions_batch.py +++ b/laborious/workflows/predictions_batch.py @@ -113,7 +113,7 @@ class PredictionsBatch(): 'POLICY': 'STOP' } }), - 'model_retention': input_data.get('model_retention', 60), + 'model_config': input_data.get('model_config', {}), 'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']), 'opc_output_config': input_data.get('opc_output_config', {}) } diff --git a/laborious/workflows/sub_workflows/prediction_process.py b/laborious/workflows/sub_workflows/prediction_process.py index e1a0162..d78018e 100644 --- a/laborious/workflows/sub_workflows/prediction_process.py +++ b/laborious/workflows/sub_workflows/prediction_process.py @@ -79,7 +79,7 @@ class PredictionProcess(): data = input_data['data'] model_id = input_data['model_id'] model_name = input_data['model_name'] - model_retention = input_data['model_retention'] + model_config = input_data.get('model_config', {}) # Get last timestamp for incremental processing last_timestamp = await workflow.execute_local_activity_method( @@ -120,7 +120,7 @@ class PredictionProcess(): **metadata, 'data': data, 'model_name': model_name, - 'model_retention': model_retention + 'model_config': model_config }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -172,7 +172,7 @@ class PredictionProcess(): **metadata, 'data': transformed_data, 'model_name': model_name, - 'model_retention': model_retention + 'model_config': model_config }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -209,7 +209,7 @@ class PredictionProcess(): 'timestamp': last_timestamp, 'model_id': model_id, 'model_name': model_name, - 'model_retention': model_retention, + 'model_config': model_config, 'opc_output_config': input_data['opc_output_config'], 'schema': input_data['schema'], 'table_name': input_data['table_name'], @@ -248,7 +248,7 @@ class PredictionProcess(): table_name = input_data['table_name'] model_id = input_data['model_id'] model_name = input_data['model_name'] - model_retention = input_data['model_retention'] + model_config = input_data.get('model_config', {}) path_flag = path_flag.upper() if path_flag else '' @@ -282,7 +282,7 @@ class PredictionProcess(): 'timestamp': last_timestamp, 'model_id': model_id, 'model_name': model_name, - 'model_retention': model_retention, + 'model_config': model_config, 'schema': schema, 'table_name': table_name, 'comment': comment, diff --git a/requirements.txt b/requirements.txt index 3f723f7..10995c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,6 @@ sqlalchemy asyncua redis git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.5 -git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.13 +# git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.13 +/home/grezewave/Documents/projects/sientia/sientia-mlops-library prometheus-client diff --git a/tests.ipynb b/tests.ipynb index 74750bf..8ab64e2 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -165,7 +165,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 1, "id": "c61be7ab", "metadata": {}, "outputs": [ diff --git a/tmp/artifacts/data_model/transformer_pyfunc/MLmodel b/tmp/artifacts/data_model/transformer_pyfunc/MLmodel new file mode 100644 index 0000000..3fc50fb --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/MLmodel @@ -0,0 +1,19 @@ +artifact_path: transformer_pyfunc +flavors: + python_function: + artifacts: + transformer: + path: artifacts/training_transformer.pkl + uri: /tmp/tmpnzkqz3v0/training_transformer.pkl + cloudpickle_version: 2.2.1 + code: code + env: + conda: conda.yaml + virtualenv: python_env.yaml + loader_module: mlflow.pyfunc.model + python_model: python_model.pkl + python_version: 3.10.16 +mlflow_version: 2.7.1 +model_uuid: 6a4a99079d234d0da2b8091532d55a34 +run_id: c2edec4dfafd4ad8bba257d62d25cd43 +utc_time_created: '2025-09-09 12:30:04.885248' diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py new file mode 100644 index 0000000..4ee9088 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py @@ -0,0 +1,172 @@ +""" +Module with utility functions for data processing. +""" + +import pandas as pd +from typing import Dict, List +from rich.console import Console + +console = Console() + + +def create_lagged_target( + data: pd.DataFrame, + target_column: str, + lags: List[int], + drop_nans: bool = True, +) -> pd.DataFrame: + """ + Creates lagged versions of a target column in a DataFrame. + + For each lag value in the provided list, a new column is created with + the naming pattern: target_column + "_lag_" + lag_value. + + Args: + data: Input DataFrame containing the target column. + target_column: Name of the target column to create lags for. + lags: List of lag values (integers between 1 and len(data)-1). + drop_nans: Whether rows with nulls generated by the lag creation + process should be dropped. Defaults to True. + Returns: + DataFrame with original columns plus the newly created lag columns. + """ + if data.empty: + console.log("[red]Warning: Input data is empty.") + return data + + result = data.copy() + + if target_column not in result.columns: + raise ValueError(f"Target column '{target_column}' not found in data.") + + # Validate lag values + max_lag = len(data) - 1 + valid_lags = [lag for lag in lags if 1 <= lag <= max_lag] + + if len(valid_lags) < len(lags): + invalid_lags = set(lags) - set(valid_lags) + console.log( + f"[yellow]Warning: Ignoring invalid lag values: {invalid_lags}. " + f"Lags must be between 1 and {max_lag}." + ) + + lag_column_names = [] + for lag in valid_lags: + lag_column_name = f"{target_column}_lag_{lag}" + result[lag_column_name] = result[target_column].shift(lag) + console.log(f"Created lagged column: [cyan]{lag_column_name}") + lag_column_names.append(lag_column_name) + if drop_nans: + result = result.dropna(subset=lag_column_names) + + return result + + +def remove_stopped_windows( + data: pd.DataFrame, + stopped_process_columns: Dict[str, float], + stopped_process_threshold: float, + time_colname: str, +) -> pd.DataFrame: + """ + Removes time windows from the input DataFrame if the proportion of samples + below a column threshold exceeds the specified limit. + + A window is considered "stopped" if *all* specified columns exceed the + stopped sample threshold. + + Args: + data: Input DataFrame with process variables and timestamps. + stopped_process_columns: Dict mapping column names to thresholds. + stopped_process_threshold: Proportion threshold (0-1) for marking a + window as stopped. + time_colname: Base name of the timestamp column + (without 'lab_' prefix). + + Returns: + A DataFrame with stopped windows removed. + """ + if data.empty: + console.log("[red]Warning: Input data is empty.") + return data + + console.log( + "Removing windows where any column exceeds" + + f" {stopped_process_threshold:.2%} of values below threshold" + ) + + masks = [] + + for col, threshold in stopped_process_columns.items(): + console.log( + "Evaluating stopped condition for column:" + + f" [cyan]{col} < {threshold}" + ) + below_threshold = data[[col]].lt(threshold) + + console.log( + "Counting number of samples below threshold for each window" + ) + below_threshold[f"lab_{time_colname}"] = data[f"lab_{time_colname}"] + grouped = below_threshold.groupby(f"lab_{time_colname}")[col].agg( + ["sum", "count"] + ) + stopped_mask = ( + grouped["sum"] / grouped["count"] + ) > stopped_process_threshold + + console.log( + f"[red]{stopped_mask.sum()} windows marked as stopped by {col}" + ) + masks.append(stopped_mask) + + # Combine masks across columns: only drop if all agree + combined_mask = pd.concat(masks, axis=1).all(axis=1) + + num_removed = combined_mask.sum() + total = combined_mask.shape[0] + console.log( + f"Removing [bold red]{num_removed}[/] out of {total}" + + f" windows ({num_removed / total:.2%})" + ) + + to_remove = combined_mask[combined_mask].index + keep_mask = ~data[f"lab_{time_colname}"].isin(to_remove) + + return data[keep_mask] + + +def aggregate_data( + merged_data: pd.DataFrame, + time_colname: str, + target_colname: str, + aggregation_functions: List[str], +) -> pd.DataFrame: + """ + Aggregates a DataFrame by time and target columns using specified + aggregation functions. + + Args: + merged_data: Input DataFrame with raw observations. + time_colname: Name of the timestamp column (no 'lab_' prefix). + target_colname: Name of the target/grouping column. + aggregation_functions: List of aggregation functions to apply + (e.g. "mean", "std"). + + Returns: + Aggregated DataFrame with flattened column names and renamed time + column. + """ + group_by_cols = [f"lab_{time_colname}", target_colname] + + aggregated = merged_data.groupby(group_by_cols).agg(aggregation_functions) + # Flatten MultiIndex columns + aggregated.columns = [ + "_".join(col) if isinstance(col, tuple) else col + for col in aggregated.columns + ] # type: ignore + aggregated = aggregated.reset_index() + + console.log(f"[bold green]Aggregated shape: {aggregated.shape}") + + return aggregated.rename(columns={f"lab_{time_colname}": time_colname}) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py new file mode 100644 index 0000000..76e4e9d --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py @@ -0,0 +1,56 @@ +""" +Module with helper functions to read datasets. +""" + +import pandas as pd +from openpyxl import load_workbook +from typing import Union + + +def read_excel_with_colors( + filepath: str, color_columns: list[str], sheet_name: Union[str, int] = 0 +) -> pd.DataFrame: + """ + Read an Excel file and extract cell fill colors for specified columns. + + Parameters: + filepath (str): Path to the Excel file. + color_columns (List[str]): Column names to extract fill colors from. + sheet_name (str or int): Sheet name or index (default is first sheet). + + Returns: + DataFrame: DataFrame with original data and extra color columns. + """ + df = pd.read_excel(filepath, sheet_name=sheet_name) + + workbook = load_workbook(filepath) + sheet = ( + workbook[sheet_name] + if isinstance(sheet_name, str) + else workbook[workbook.sheetnames[sheet_name]] + ) + + header = next(sheet.iter_rows(min_row=1, max_row=1, values_only=True)) + col_name_to_letter = { + name: chr(65 + idx) for idx, name in enumerate(header) + } + + for col_name in color_columns: + if col_name not in df.columns: + raise ValueError(f"Column '{col_name}' not found in Excel file.") + + col_letter = col_name_to_letter[col_name] + fill_colors: list[Union[str, None]] = [] + + for row in range(2, sheet.max_row + 1): + cell = sheet[f"{col_letter}{row}"] + fill = cell.fill + + if fill.fill_type == "solid" and fill.fgColor.rgb: + fill_colors.append(fill.fgColor.rgb) + else: + fill_colors.append(None) + + df[f"{col_name}_fill_color"] = fill_colors + + return df diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py new file mode 100644 index 0000000..efe43e0 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py @@ -0,0 +1,788 @@ +""" +Module with scikit-learn transformers for training and inference pipelines. + +This module replicates the functionality from the DVC pipeline stages: +- preprocessing.py (stopped process filtering, aggregation) +- filter_columns.py (feature selection, lagged target creation) +""" + +import pandas as pd +from typing import Dict, List, Optional +from sklearn.base import BaseEstimator, TransformerMixin +from rich.console import Console + +console = Console() + + +class CourierTrainingTransformer(BaseEstimator, TransformerMixin): + """ + Training transformer that replicates the DVC pipeline preprocessing. + + Includes: + 1. Remove stopped process windows (training only) + 2. Data aggregation + 3. Feature selection (learns and applies - dictionary-based only) + 4. Create lagged target features + """ + + def __init__( + self, + aggregation_functions: List[str] = ["median", "std", "min", "max"], + stopped_process_columns: Dict[str, float] = { + "305-PIT-170": 9, + "305-PIT-175": 9, + }, + stopped_process_threshold: float = 0.1, + target_lags: List[int] = [2], + time_colname: str = "timestamp", + target_colname: str = "SiO2_conc", + dictionary_df: Optional[pd.DataFrame] = None, + create_lagged_target: bool = True, + drop_nans: bool = True, + ): + """ + Initialize the training transformer. + + Args: + aggregation_functions: List of aggregation functions to apply + stopped_process_columns: Dict mapping column names to thresholds + stopped_process_threshold: Proportion threshold for stopped windows + target_lags: List of lag values for target column + time_colname: Name of timestamp column (without 'lab_' prefix) + target_colname: Name of target column + dictionary_df: DataFrame with domain knowledge + (TAG_fill_color column) + create_lagged_target: Whether to create lagged target features + drop_nans: Whether to drop NaNs after creating lags + """ + self.aggregation_functions = aggregation_functions + self.stopped_process_columns = stopped_process_columns + self.stopped_process_threshold = stopped_process_threshold + self.target_lags = target_lags + self.time_colname = time_colname + self.target_colname = target_colname + self.dictionary_df = dictionary_df + self.create_lagged_target = create_lagged_target + self.drop_nans = drop_nans + + # Will be learned during fit + self.selected_features_ = None + + def _infer_lab_timestamp(self, data: pd.DataFrame) -> pd.DataFrame: + """ + Creates a lab timestamp column by rounding the timestamp up to the next + even hour. + + Args: + data: Input DataFrame + + Returns: + DataFrame with added lab timestamp column + """ + lab_col_name = f"lab_{self.time_colname}" + + if lab_col_name in data.columns: + console.log( + f"[yellow]Lab timestamp column {lab_col_name} already exists, " + + "skipping inference" + ) + return data + + # Check if timestamp is a column or the index + if self.time_colname in data.columns: + # Timestamp is a regular column + result = data.copy() + timestamp_col = pd.to_datetime(result[self.time_colname]) + elif data.index.name == self.time_colname or ( + hasattr(data.index, "names") + and self.time_colname in data.index.names + ): + # Timestamp is the index (or part of a MultiIndex) + result = data.copy() + timestamp_col = pd.to_datetime( + result.index.get_level_values(self.time_colname) + if hasattr(result.index, "names") + and len(result.index.names) > 1 + else result.index + ) + else: + raise ValueError( + f"Timestamp '{self.time_colname}' not found in data columns " + + f"or index. Available columns: {list(data.columns)}, " + + f"index name: {data.index.name}" + ) + + # Round up to next even hour + # Step 1: Floor to the hour to remove minutes/seconds + # Handle both Series (from column) and DatetimeIndex (from index) + if hasattr(timestamp_col, "dt"): + # timestamp_col is a Series + hour_floor = timestamp_col.dt.floor("H") + hour = hour_floor.dt.hour + else: + # timestamp_col is a DatetimeIndex + hour_floor = timestamp_col.floor("H") + hour = hour_floor.hour + + # Step 3: Determine if rounding is needed + # - If hour is odd, round up to next even hour + # - If hour is even but original timestamp had minutes/seconds, + # round up to next even hour + # - If hour is even and original timestamp was exactly on the hour, + # keep it + needs_rounding = (hour % 2 == 1) | (timestamp_col != hour_floor) + + # Calculate next even hour + next_even_hour = ((hour // 2) + 1) * 2 + + # Handle case where next even hour >= 24 (next day) + days_to_add = (next_even_hour >= 24).astype(int) + hour_component = next_even_hour % 24 + + # Create the lab timestamp + if hasattr(timestamp_col, "dt"): + # timestamp_col is a Series + lab_timestamp = hour_floor.where( + ~needs_rounding, + hour_floor.dt.floor("D") + + pd.to_timedelta(days_to_add, unit="D") + + pd.to_timedelta(hour_component, unit="H"), + ) + else: + # timestamp_col is a DatetimeIndex + base_date = hour_floor.floor("D") + next_even_timestamp = ( + base_date + + pd.to_timedelta(days_to_add, unit="D") + + pd.to_timedelta(hour_component, unit="H") + ) + lab_timestamp = pd.Series( + hour_floor.where(~needs_rounding, next_even_timestamp), + index=result.index, + ) + + result[lab_col_name] = lab_timestamp + + console.log( + f"[bold green]Created lab timestamp column: {lab_col_name}" + ) + + return result + + def _remove_stopped_windows( + self, + data: pd.DataFrame, + ) -> pd.DataFrame: + """ + Removes time windows where process was stopped. + Replicates remove_stopped_windows from preprocessing.py + """ + if data.empty: + console.log("[red]Warning: Input data is empty.") + return data + + console.log( + "Removing windows where any column exceeds" + + f" {self.stopped_process_threshold:.2%} of values below" + + " threshold" + ) + + masks = [] + + for col, threshold in self.stopped_process_columns.items(): + console.log( + "Evaluating stopped condition for column:" + + f" [cyan]{col} < {threshold}" + ) + below_threshold = data[[col]].lt(threshold) + + console.log( + "Counting number of samples below threshold for each window" + ) + below_threshold[f"lab_{self.time_colname}"] = data[ + f"lab_{self.time_colname}" + ] + grouped = below_threshold.groupby(f"lab_{self.time_colname}")[ + col + ].agg(["sum", "count"]) + stopped_mask = ( + grouped["sum"] / grouped["count"] + ) > self.stopped_process_threshold + + console.log( + f"[red]{stopped_mask.sum()} windows marked as stopped by {col}" + ) + masks.append(stopped_mask) + + # Combine masks across columns: only drop if all agree + if not masks: + return data + + mask_df = pd.concat(masks, axis=1) + combined_mask = mask_df.all(axis=1) + + num_removed = int(combined_mask.sum()) # type: ignore + total = combined_mask.shape[0] + console.log( + f"Removing [bold red]{num_removed}[/] out of {total}" + + f" windows ({num_removed / total:.2%})" + ) + + to_remove = combined_mask[combined_mask].index + keep_mask = ~data[f"lab_{self.time_colname}"].isin(to_remove) + + filtered_data = data[keep_mask] + return filtered_data # type: ignore + + def _aggregate_data( + self, + merged_data: pd.DataFrame, + ) -> pd.DataFrame: + """ + Aggregates data by time and target columns. + Replicates aggregate_data from preprocessing.py + """ + group_by_cols = [f"lab_{self.time_colname}", self.target_colname] + + aggregated = merged_data.groupby(group_by_cols).agg( + self.aggregation_functions + ) + # Flatten MultiIndex columns + aggregated.columns = [ + "_".join(col) if isinstance(col, tuple) else col + for col in aggregated.columns + ] # type: ignore + aggregated = aggregated.reset_index() + + # Rename timestamp column and ensure it's datetime + aggregated = aggregated.rename( + columns={f"lab_{self.time_colname}": self.time_colname} + ) + aggregated[self.time_colname] = pd.to_datetime( + aggregated[self.time_colname] + ) + + console.log(f"[bold green]Aggregated shape: {aggregated.shape}") + + return aggregated + + def _learn_feature_selection( + self, + data: pd.DataFrame, + ) -> List[str]: + """ + Learn which features to keep based on dictionary only. + Replicates domain knowledge filtering from filter_columns.py + """ + if self.dictionary_df is None: + console.log( + "[yellow]Warning: No dictionary data provided. Using all" + + " features." + ) + return [col for col in data.columns if col != self.time_colname] + + # Domain knowledge filter - only keep columns with TAG_fill_color + columns_to_keep_dict = self.dictionary_df.loc[ + self.dictionary_df["TAG_fill_color"].notna(), "TAG" + ].values + + # Filter data columns to only those that match dictionary tags + available_columns = [ + col for col in data.columns if col != self.time_colname + ] + columns_to_keep = [ + col + for col in available_columns + if col.split("_")[0] in columns_to_keep_dict + ] + + console.log( + f"Keeping {len(columns_to_keep)}/{len(available_columns)}" + + " columns based on dictionary." + ) + + # Always include target + columns_to_keep.append(self.target_colname) + + return columns_to_keep + + def _create_lagged_target( + self, + data: pd.DataFrame, + ) -> pd.DataFrame: + """ + Creates lagged versions of target column. + Replicates create_lagged_target from preprocessing.py + """ + if data.empty: + console.log("[red]Warning: Input data is empty.") + return data + + result = data.copy() + + if self.target_colname not in result.columns: + raise ValueError( + f"Target column '{self.target_colname}' not found in data." + ) + + # Validate lag values + max_lag = len(data) - 1 + valid_lags = [lag for lag in self.target_lags if 1 <= lag <= max_lag] + + if len(valid_lags) < len(self.target_lags): + invalid_lags = set(self.target_lags) - set(valid_lags) + console.log( + f"[yellow]Warning: Ignoring invalid lag values: {invalid_lags}" + + f". Lags must be between 1 and {max_lag}." + ) + + lag_column_names = [] + for lag in valid_lags: + lag_column_name = f"{self.target_colname}_lag_{lag}" + result[lag_column_name] = result[self.target_colname].shift(lag) + console.log(f"Created lagged column: [cyan]{lag_column_name}") + lag_column_names.append(lag_column_name) + + if self.drop_nans and lag_column_names: + result = result.dropna(subset=lag_column_names) + + return result + + def fit(self, X: pd.DataFrame, y=None): + """ + Learn feature selection parameters. + + Args: + X: Input DataFrame with merged process and quality data + y: Not used + + Returns: + self + """ + console.log("[bold blue]Training transformer fit phase") + + # Create a copy for processing + data = X.copy() + + # Step 0: Infer lab timestamp if needed + console.log("[bold blue]Inferring lab timestamp") + data = self._infer_lab_timestamp(data) + + # Step 1: Remove stopped process windows (training only) + console.log("[bold blue]Removing stopped process windows") + data = self._remove_stopped_windows(data) + + # Step 2: Aggregate data + console.log("[bold blue]Aggregating data") + data = self._aggregate_data(data) + + # Step 3: Learn feature selection + console.log("[bold blue]Learning feature selection") + self.selected_features_ = self._learn_feature_selection(data) + + console.log( + f"[bold green]Learned {len(self.selected_features_)}" + + " features for selection" + ) + self._feature_names = self.selected_features_ + + return self + + def transform(self, X: pd.DataFrame) -> pd.DataFrame: + """ + Apply the complete training transformation pipeline. + + Args: + X: Input DataFrame with merged process and quality data + + Returns: + Transformed DataFrame ready for model training + """ + if self.selected_features_ is None: + raise ValueError("Transformer must be fitted before transform.") + + console.log("[bold blue]Training transformer transform phase") + + # Create a copy for processing + data = X.copy() + + # Step 0: Infer lab timestamp if needed + console.log("[bold blue]Inferring lab timestamp") + data = self._infer_lab_timestamp(data) + + # Step 1: Remove stopped process windows (training only) + console.log("[bold blue]Removing stopped process windows") + data = self._remove_stopped_windows(data) + + # Step 2: Aggregate data + console.log("[bold blue]Aggregating data") + data = self._aggregate_data(data) + + # Step 3: Apply feature selection + console.log("[bold blue]Applying feature selection") + # Set timestamp as index for filtering and ensure it's datetime + data[self.time_colname] = pd.to_datetime(data[self.time_colname]) + data = data.set_index(self.time_colname) + data = data[self.selected_features_] + + # Step 4: Create lagged target features + if self.create_lagged_target: + console.log("[bold blue]Creating lagged target features") + data = self._create_lagged_target(data) + + console.log(f"[bold green]Final training data shape: {data.shape}") + + return data + + +class CourierInferenceTransformer(BaseEstimator, TransformerMixin): + """ + Inference transformer that replicates DVC pipeline preprocessing + without training-specific steps. + + Includes: + 1. Data aggregation (higher frequency - no grouping by target) + 2. Feature selection (applies learned selection) + 3. Create lagged target features + + Note: Does NOT include stopped process filtering (training only). + """ + + def __init__( + self, + selected_features: List[str], + aggregation_functions: List[str] = ["median", "std", "min", "max"], + target_lags: List[int] = [2], + time_colname: str = "timestamp", + target_colname: str = "SiO2_conc", + create_lagged_target: bool = True, + drop_nans: bool = True, + ): + """ + Initialize the inference transformer. + + Args: + selected_features: Pre-learned list of features to select + aggregation_functions: List of aggregation functions to apply + target_lags: List of lag values for target column + time_colname: Name of timestamp column (without 'lab_' prefix) + target_colname: Name of target column + create_lagged_target: Whether to create lagged target features + drop_nans: Whether to drop NaNs after creating lags + """ + self.selected_features = selected_features + self.aggregation_functions = aggregation_functions + self.target_lags = target_lags + self.time_colname = time_colname + self.target_colname = target_colname + self.create_lagged_target = create_lagged_target + self.drop_nans = drop_nans + + def _infer_lab_timestamp(self, data: pd.DataFrame) -> pd.DataFrame: + """ + Creates a lab timestamp column by rounding the timestamp up to the next + even hour. + + Args: + data: Input DataFrame + + Returns: + DataFrame with added lab timestamp column + """ + lab_col_name = f"lab_{self.time_colname}" + + if lab_col_name in data.columns: + console.log( + f"[yellow]Lab timestamp column {lab_col_name} already exists, " + + "skipping inference" + ) + return data + + # Check if timestamp is a column or the index + if self.time_colname in data.columns: + # Timestamp is a regular column + result = data.copy() + timestamp_col = pd.to_datetime(result[self.time_colname]) + elif data.index.name == self.time_colname or ( + hasattr(data.index, "names") + and self.time_colname in data.index.names + ): + # Timestamp is the index (or part of a MultiIndex) + result = data.copy() + timestamp_col = pd.to_datetime( + result.index.get_level_values(self.time_colname) + if hasattr(result.index, "names") + and len(result.index.names) > 1 + else result.index + ) + else: + raise ValueError( + f"Timestamp '{self.time_colname}' not found in data columns " + + f"or index. Available columns: {list(data.columns)}, " + + f"index name: {data.index.name}" + ) + + # Round up to next even hour + # Step 1: Floor to the hour to remove minutes/seconds + # Handle both Series (from column) and DatetimeIndex (from index) + if hasattr(timestamp_col, "dt"): + # timestamp_col is a Series + hour_floor = timestamp_col.dt.floor("H") + hour = hour_floor.dt.hour + else: + # timestamp_col is a DatetimeIndex + hour_floor = timestamp_col.floor("H") + hour = hour_floor.hour + + # Step 3: Determine if rounding is needed + # - If hour is odd, round up to next even hour + # - If hour is even but original timestamp had minutes/seconds, + # round up to next even hour + # - If hour is even and original timestamp was exactly on the hour, + # keep it + needs_rounding = (hour % 2 == 1) | (timestamp_col != hour_floor) + + # Calculate next even hour + next_even_hour = ((hour // 2) + 1) * 2 + + # Handle case where next even hour >= 24 (next day) + days_to_add = (next_even_hour >= 24).astype(int) + hour_component = next_even_hour % 24 + + # Create the lab timestamp + if hasattr(timestamp_col, "dt"): + # timestamp_col is a Series + lab_timestamp = hour_floor.where( + ~needs_rounding, + hour_floor.dt.floor("D") + + pd.to_timedelta(days_to_add, unit="D") + + pd.to_timedelta(hour_component, unit="H"), + ) + else: + # timestamp_col is a DatetimeIndex + base_date = hour_floor.floor("D") + next_even_timestamp = ( + base_date + + pd.to_timedelta(days_to_add, unit="D") + + pd.to_timedelta(hour_component, unit="H") + ) + lab_timestamp = pd.Series( + hour_floor.where(~needs_rounding, next_even_timestamp), + index=result.index, + ) + + result[lab_col_name] = lab_timestamp + + console.log( + f"[bold green]Created lab timestamp column: {lab_col_name}" + ) + + return result + + def _aggregate_data( + self, + merged_data: pd.DataFrame, + ) -> pd.DataFrame: + """ + Aggregate data into 2-hour non-overlapping windows. + Each row corresponds to one 2-hour window ending at an even hour. + """ + if merged_data.empty: + console.log("[red]Warning: Input data is empty.") + return merged_data + + lab_col = f"lab_{self.time_colname}" + if lab_col not in merged_data.columns: + raise ValueError( + f"Missing '{lab_col}' column. Call _infer_lab_timestamp first." + ) + + # Get numeric columns only for aggregation + numeric_cols = merged_data.select_dtypes( + include=["number"] + ).columns.tolist() + + # Remove time and target columns if present + cols_to_remove = [self.time_colname, lab_col, self.target_colname] + for col in cols_to_remove: + if col in numeric_cols: + numeric_cols.remove(col) + + groups = merged_data.groupby(lab_col) + + if numeric_cols: + aggregated_numeric = groups[numeric_cols].agg( + self.aggregation_functions + ) + # Flatten MultiIndex columns: (col, func) -> "col_func" + aggregated_numeric.columns = [ + f"{col}_{func}" + for col, func in aggregated_numeric.columns.to_flat_index() + ] + else: + # Create empty frame indexed by the 2-hour windows + aggregated_numeric = groups.size().to_frame(name="__rows__") + aggregated_numeric = aggregated_numeric.drop(columns=["__rows__"]) + + # Add target column as the last non-null value per window + if self.target_colname in merged_data.columns: + target_per_window = groups[self.target_colname].apply( + lambda s: s.dropna().iloc[-1] + if not s.dropna().empty + else None + ) + aggregated_numeric[self.target_colname] = target_per_window + + # Reset index and rename lab timestamp to main time column + result = aggregated_numeric.reset_index().rename( + columns={lab_col: self.time_colname} + ) + + # Ensure timestamp is datetime + result[self.time_colname] = pd.to_datetime(result[self.time_colname]) + + console.log( + f"[bold green]Aggregated to windowed shape: {result.shape}" + ) + + return result + + def _create_lagged_target( + self, + data: pd.DataFrame, + ) -> pd.DataFrame: + """ + Creates lagged target column names with target values for inference. + In inference, we assume the data is already properly lagged, + so we just create the expected column names with the target values. + """ + if data.empty: + console.log("[red]Warning: Input data is empty.") + return data + + result = data.copy() + + if self.target_colname not in result.columns: + raise ValueError( + f"Target column '{self.target_colname}' not found in data." + ) + + # Create lagged column names with target values (no actual shifting) + lag_column_names = [] + + # Get target column as a Series to ensure we have exactly one column + target_series = result[self.target_colname] + if isinstance(target_series, pd.DataFrame): + # If we accidentally got a DataFrame, take the first column + target_values = target_series.iloc[:, 0].values + else: + target_values = target_series.values + + for lag in self.target_lags: + lag_column_name = f"{self.target_colname}_lag_{lag}" + # Copy target values instead of shifting for inference + result[lag_column_name] = target_values + console.log(f"Created lagged column: [cyan]{lag_column_name}") + lag_column_names.append(lag_column_name) + + return result + + def fit(self, X: pd.DataFrame, y=None): + """ + No-op for inference transformer (no learning needed). + + Args: + X: Input DataFrame + y: Not used + + Returns: + self + """ + console.log("[bold blue]Inference transformer fit (no-op)") + return self + + def transform(self, X: pd.DataFrame) -> pd.DataFrame: + """ + Apply the inference transformation pipeline. + + Args: + X: Input DataFrame with merged process and quality data + + Returns: + Transformed DataFrame ready for model inference + """ + console.log("[bold blue]Inference transformer transform phase") + + # Create a copy for processing + data = X.copy() + + # Step 0: Infer lab timestamp if needed + console.log("[bold blue]Inferring lab timestamp") + data = self._infer_lab_timestamp(data) + + # Step 1: Aggregate data (higher frequency - no target grouping) + console.log("[bold blue]Aggregating data") + data = self._aggregate_data(data) + + # Step 2: Apply learned feature selection + console.log("[bold blue]Applying learned feature selection") + # Set timestamp as index for filtering and ensure it's datetime + data[self.time_colname] = pd.to_datetime(data[self.time_colname]) + data = data.set_index(self.time_colname) + + # Filter to selected features (handle missing columns gracefully) + available_features = [ + col for col in self.selected_features if col in data.columns + ] + missing_features = set(self.selected_features) - set( + available_features + ) + + if missing_features: + console.log( + "[yellow]Warning: Missing features in inference data:" + + f" {missing_features}" + ) + + # Ensure target column is included but avoid duplicates + if self.target_colname not in available_features: + available_features.append(self.target_colname) + + data = data[available_features] + + # Step 3: Create lagged target features + if self.create_lagged_target: + console.log("[bold blue]Creating lagged target features") + data = self._create_lagged_target(data) + + console.log(f"[bold green]Final inference data shape: {data.shape}") + + return data + + +def create_transformers_from_training_transformer( + training_transformer: CourierTrainingTransformer, +) -> tuple[CourierTrainingTransformer, CourierInferenceTransformer]: + """ + Create both training and inference transformers with shared parameters. + + Args: + training_transformer: Fitted training transformer + + Returns: + Tuple of (training_transformer, inference_transformer) + """ + if training_transformer.selected_features_ is None: + raise ValueError("Training transformer must be fitted first.") + + inference_transformer = CourierInferenceTransformer( + selected_features=training_transformer.selected_features_, + aggregation_functions=training_transformer.aggregation_functions, + target_lags=training_transformer.target_lags, + time_colname=training_transformer.time_colname, + target_colname=training_transformer.target_colname, + create_lagged_target=training_transformer.create_lagged_target, + drop_nans=training_transformer.drop_nans, + ) + + return training_transformer, inference_transformer diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py new file mode 100644 index 0000000..6fba3eb --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py @@ -0,0 +1,51 @@ +""" +Functions needed to load parameters from params.yaml tracked with DVC +""" + +import sys +import os +from typing import Optional + +import yaml +from rich.console import Console + + +console = Console() + + +def get_params(stage_fn: Optional[str] = None): + """ + Reads parameters for a given DVC stage from params.yaml. + + The stage name is inferred from the name of the python file that calls this + function. + Args: + stage_fn (str): Name of the stage. If None, the name of the file + that calls this function is used. Defaults to None. + Returns: + dict with parameters for the stage + Raises: + KeyError: if the stage name is not found in params.yaml + """ + + if stage_fn is None: + stage_fn = os.path.basename(sys.argv[0]).replace(".py", "") + + try: + params = yaml.safe_load(open("params.yaml"))[stage_fn] + except KeyError as exc: + console.print(f'ERROR: Key "{stage_fn}" not in parameters.yaml.') + raise KeyError( + f"Is the stage file name ({sys.argv[0]}) " + + "the same as the stage name in params.yaml?" + ) from exc + try: + all_params = yaml.safe_load(open("params.yaml"))["all"] + params = {**params, **all_params} + except KeyError: + console.print( + '[orange]WARNING: Key "all" not in parameters.yaml.' + + "Only returning stage parameters." + ) + + return params diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py new file mode 100644 index 0000000..9017a1b --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py @@ -0,0 +1,325 @@ +""" +Module with functions for wrapping time series models for MLflow. +""" + +import os +import tempfile +import pickle +from typing import Optional, Union, Dict, Any, List + +import mlflow.pyfunc +import pandas as pd +import numpy as np +from mlflow.models import ModelSignature +from ..models.stacking_time_series import StackingTimeSeriesModel +from ..data.transformers import ( + create_transformers_from_training_transformer, + CourierTrainingTransformer, +) + + +class StackingWrapper(mlflow.pyfunc.PythonModel): # type: ignore + """ + MLflow wrapper for StackingTimeSeriesModel. + + Allows the model to be saved and served via MLflow's pyfunc interface. + """ + + def __init__(self, model: Optional[StackingTimeSeriesModel] = None): + self.model = model + + @property + def _console(self): + from rich.console import Console + + return Console() + + def load_context(self, context: Any) -> None: + """Load model from artifact path in MLflow context.""" + try: + model_path = context.artifacts["model"] + self.model = StackingTimeSeriesModel.load(model_path) + self._console.log("[green]Model loaded from context[/green]") + except Exception as e: + self._console.print(f"[red]Error loading model: {e}[/red]") + raise + + def predict( + self, + context: Any, + model_input: Union[pd.DataFrame, np.ndarray, Dict[str, Any]], + ) -> Union[pd.Series, pd.DataFrame, np.ndarray]: + """Run inference using the wrapped model.""" + if self.model is None: + raise ValueError("Model not loaded. Call load_context first.") + + # Extract data from input + if isinstance(model_input, dict): + X = model_input.get("data") + if X is None: + raise ValueError("Dict input must contain 'data' key.") + else: + X = model_input + + # Ensure DataFrame input (models expect pandas DataFrames) + if not isinstance(X, pd.DataFrame): + raise ValueError("Input must be a pandas DataFrame.") + + try: + predictions = self.model.predict(X) + return predictions.to_frame(name=self.model.target_col) + except Exception as e: + self._console.print(f"[red]Prediction failed: {e}[/red]") + raise + + def get_model_summary(self) -> str: + """Return human-readable model summary.""" + return self.model.summary() if self.model else "No model loaded" + + def store_model( + self, + path: Optional[str] = None, + artifact_path: str = "stacking_model", + signature: Optional[ModelSignature] = None, + pip_requirements: Optional[Union[str, list]] = None, + code_path: Optional[List[str]] = None, + to_disk: bool = False, + ) -> None: + """ + Store the model using MLflow pyfunc interface. + + Logs to the current MLflow run by default. Optionally saves locally. + + Args: + path: Local path to save model (required if to_disk=True) + artifact_path: MLflow artifact path + signature: Optional MLflow model signature + pip_requirements: pip requirements (list or path) + code_path: List of local Python source files/directories to bundle + to_disk: Save locally if True, otherwise logs to MLflow + """ + if self.model is None: + raise ValueError("No model to store.") + + with tempfile.TemporaryDirectory() as tmp: + model_artifact = os.path.join(tmp, "stacking_model.pkl") + self.model.save(model_artifact, compression="lzma") + + common_args = { + "python_model": self, + "artifacts": {"model": model_artifact}, + } + if signature: + common_args["signature"] = signature + if pip_requirements: + common_args["pip_requirements"] = pip_requirements + if code_path: + common_args["code_path"] = code_path + + if to_disk: + if not path: + raise ValueError("`path` required for to_disk=True.") + mlflow.pyfunc.save_model(path=path, **common_args) + self._console.log( + f"[blue]Model saved locally to {path}[/blue]" + ) + else: + mlflow.pyfunc.log_model( + artifact_path=artifact_path, **common_args + ) + self._console.log( + f"[green]Model logged to MLflow at '{artifact_path}'" + ) + + def __getstate__(self): + state = self.__dict__.copy() + state["model"] = None # avoid double saving + return state + + def __setstate__(self, state): + self.__dict__.update(state) + + +class TransformerWrapper(mlflow.pyfunc.PythonModel): # type: ignore + """ + MLflow wrapper for data transformers. + + Allows transformers to be saved and served via MLflow's pyfunc interface. + Supports both training and inference transformers. + """ + + def __init__( + self, + transformer: Optional[CourierTrainingTransformer] = None, + ): + """ + Initialize the transformer wrapper. + + Args: + transformer: The training transformer to wrap + """ + self.training_transformer = transformer + self.inference_transformer = None + + @property + def _console(self): + from rich.console import Console + + return Console() + + def load_context(self, context: Any) -> None: + """ + Load training transformer from artifact path and create + inference transformer. + """ + try: + transformer_path = context.artifacts["transformer"] + + with open(transformer_path, "rb") as f: + self.training_transformer = pickle.load(f) + + _, self.inference_transformer = ( + create_transformers_from_training_transformer( + self.training_transformer + ) + ) + + self._console.log( + "[green]Training transformer loaded and inference transformer " + + "created from context[/green]" + ) + except Exception as e: + self._console.print(f"[red]Error loading transformer: {e}[/red]") + raise + + def predict( + self, + context: Any, + model_input: Union[pd.DataFrame, np.ndarray, Dict[str, Any]], + transformer_type: str = "inference", + ) -> Union[pd.Series, pd.DataFrame, np.ndarray]: + """ + Transform data using the selected transformer. + + Args: + context: MLflow context + model_input: Input data to transform (pandas DataFrame expected) + transformer_type: Either "training" or "inference" + """ + if transformer_type == "training": + transformer = self.training_transformer + elif transformer_type == "inference": + transformer = self.inference_transformer + else: + raise ValueError( + "transformer_type must be 'training' or 'inference'" + ) + + if transformer is None: + raise ValueError( + f"{transformer_type.title()} transformer not loaded. " + + "Call load_context first." + ) + + # Extract data from input + if isinstance(model_input, dict): + X = model_input.get("data") + if X is None: + raise ValueError("Dict input must contain 'data' key.") + else: + X = model_input + + # Ensure DataFrame input (transformers expect pandas DataFrames) + if not isinstance(X, pd.DataFrame): + raise ValueError("Input must be a pandas DataFrame.") + + try: + # Apply transformer + transformed_data = transformer.transform(X) + return transformed_data + + except Exception as e: + self._console.print(f"[red]Transformation failed: {e}[/red]") + raise + + def get_transformer_summary(self) -> str: + """Return human-readable transformer summary.""" + if self.training_transformer is None: + return "No training transformer loaded" + + training_class = self.training_transformer.__class__.__name__ + inference_status = ( + "available" if self.inference_transformer else "not created" + ) + return ( + f"{training_class} (training loaded, inference {inference_status})" + ) + + def store_transformer( + self, + path: Optional[str] = None, + artifact_path: str = "transformer", + signature: Optional[ModelSignature] = None, + pip_requirements: Optional[Union[str, list]] = None, + code_path: Optional[List[str]] = None, + to_disk: bool = False, + ) -> None: + """ + Store the training transformer using MLflow pyfunc interface. + + Logs to the current MLflow run by default. Optionally saves locally. + + Args: + path: Local path to save transformer (required if to_disk=True) + artifact_path: MLflow artifact path + signature: Optional MLflow model signature + pip_requirements: pip requirements (list or path) + code_path: List of local Python source files/directories to bundle + to_disk: Save locally if True, otherwise logs to MLflow + """ + if self.training_transformer is None: + raise ValueError("No training transformer to store.") + + with tempfile.TemporaryDirectory() as tmp: + transformer_artifact = os.path.join( + tmp, "training_transformer.pkl" + ) + with open(transformer_artifact, "wb") as f: + pickle.dump(self.training_transformer, f) + + common_args = { + "python_model": self, + "artifacts": {"transformer": transformer_artifact}, + } + if signature: + common_args["signature"] = signature + if pip_requirements: + common_args["pip_requirements"] = pip_requirements + if code_path: + common_args["code_path"] = code_path + + if to_disk: + if not path: + raise ValueError("`path` required for to_disk=True.") + mlflow.pyfunc.save_model(path=path, **common_args) + self._console.log( + "[blue]Training transformer saved locally to " + + f"{path}[/blue]" + ) + else: + mlflow.pyfunc.log_model( + artifact_path=artifact_path, **common_args + ) + self._console.log( + "[green]Training transformer logged to MLflow at " + + f"'{artifact_path}'[/green]" + ) + + def __getstate__(self): + state = self.__dict__.copy() + state["training_transformer"] = None # avoid double saving + state["inference_transformer"] = None # avoid double saving + return state + + def __setstate__(self, state): + self.__dict__.update(state) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py new file mode 100644 index 0000000..1701c19 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py @@ -0,0 +1,24 @@ +""" +Models package for time series forecasting. + +This package provides standardized interfaces and implementations for +various time series forecasting models. +""" + +from .base import ( + TimeSeriesModel, + UnivariateTimeSeriesModel, + MultivariateTimeSeriesModel, +) +from .factory import create_model, load_model, get_available_models +from .evaluation import timeseries_metrics + +__all__ = [ + "TimeSeriesModel", + "UnivariateTimeSeriesModel", + "MultivariateTimeSeriesModel", + "create_model", + "load_model", + "get_available_models", + "timeseries_metrics", +] diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py new file mode 100644 index 0000000..abf5d6d --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py @@ -0,0 +1,389 @@ +""" +ARIMA univariate time series forecasting model implementation. +""" + +from typing import Optional, Tuple + +import numpy as np +import pandas as pd +from statsmodels.tsa.arima.model import ARIMA, ARIMAResults +from rich.console import Console + +from .base import UnivariateTimeSeriesModel, ensure_fitted + +console = Console() + + +class ARIMAModel(UnivariateTimeSeriesModel): + """ARIMA model for univariate time series forecasting. + + This class implements an ARIMA model for forecasting univariate time + series data. It provides methods for fitting the model, making predictions, + forecasting future values, and updating the model with new data. + + Attributes: + order (Tuple[int, int, int]): The (p, d, q) order of the ARIMA model. + model_ (Optional[ARIMA]): The ARIMA model instance. + result_ (Optional[ARIMAResults]): The fitted ARIMA model results. + training_series_ (Optional[pd.Series]): The training data used to fit + the model. + """ + + def __init__( + self, + order: Tuple[int, int, int] = (1, 0, 0), + name: Optional[str] = None, + time_col: str = "ds", + target_col: str = "y", + random_seed: int = 42, + forecast_horizon: int = 2, + ) -> None: + """Initializes the ARIMAModel with specified parameters. + + Args: + order (Tuple[int, int, int]): The (p, d, q) order of the ARIMA + model. + name (Optional[str]): The name of the model. + time_col (str): The name of the time column in the input data. + target_col (str): The name of the target column in the input data. + random_seed (int): The random seed for reproducibility. + forecast_horizon (int): The number of steps to forecast ahead. + """ + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + ) + self.order: Tuple[int, int, int] = order + self.model_: Optional[ARIMA] = None + self.result_: Optional[ARIMAResults] = None + self.training_series_: pd.Series = pd.Series(dtype=float) + self.observed_series_: pd.Series = pd.Series(dtype=float) + self.backtest_predictions_: Optional[pd.Series] = None + self.forecast_horizon: int = forecast_horizon + + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """ + Fits the ARIMA model to the provided training data. + + Args: + y: The target time series data. + X: Optional exogenous variables. + X_val: Validation feature matrix (not used for ARIMA). + y_val: Validation target series (not used for ARIMA). + """ + y_array: np.ndarray = self._validate_y(y) + self.training_series_ = y.copy() + self.observed_series_ = y.copy() + self.model_ = ARIMA(y_array, order=self.order) + self.result_ = self.model_.fit() + + @ensure_fitted + def predict(self, X: Optional[pd.DataFrame] = None) -> pd.Series: + """ + Generates in-sample predictions from the fitted ARIMA model. + + After this method is called, if X is provided, the model will be + updated with the new data, but the coefficients will not be refit. + This is useful for generating predictions on new data without + retraining the model. + + Args: + X (Optional[pd.DataFrame]): Optional dataframe with future + measurements of y for in-sample predictions. + If None, the model will predict on the observed data + (observed_series_). + + Returns: + pd.Series: The in-sample predictions. + + Raises: + ValueError: If the model has not been fitted yet. + """ + if self.result_ is None: + raise ValueError("Model is not fitted.") + + if X is None: + fitted_values = self.result_.fittedvalues + if fitted_values is None: + raise ValueError("Fitted values are None") + return pd.Series( + fitted_values, + index=self.training_series_.index[: len(fitted_values)], + name=self.target_col, + ) + + # Validate the input data + target_series = ( + X[self.target_col] if self.target_col in X else X.iloc[:, 0] + ) + if not isinstance(target_series, pd.Series): + target_series = pd.Series(target_series, index=X.index) + + X_validated = self._validate_y(target_series) + + # Update the model with the validated data without refitting + self.update(pd.Series(X_validated, index=X.index), refit=False) + + if self.result_ is None: + raise ValueError("Model result is None after update") + + fitted_values = self.result_.fittedvalues + if fitted_values is None: + raise ValueError("Fitted values are None") + + return_series = pd.Series( + fitted_values[-len(X) :], index=X.index, name=self.target_col + ) + + return return_series + + @ensure_fitted + def forecast(self, forecast_horizon: int) -> np.ndarray: + """Generates out-of-sample forecasts from the fitted ARIMA model. + TODO: Change return to include index of the forecasted values. + + Args: + forecast_horizon (int): The number of steps to forecast ahead. + + Returns: + np.ndarray: The out-of-sample forecasts. + + Raises: + ValueError: If the model has not been fitted yet. + """ + if self.result_ is None: + raise ValueError("Model is not fitted.") + return self.result_.forecast(steps=forecast_horizon) + + @ensure_fitted + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Perform comprehensive backtesting with periodic retraining. + + This method implements walk-forward validation with periodic + retraining, providing robust evaluation of model performance in + production-like scenarios. Uses 1-step ahead forecasting by default. + + Args: + y: Target time series for backtesting + X: Unused (included for base class compatibility) + retrain_every: Number of steps between model retraining + reuse_previous_execution: Whether to reuse previous backtest results + + Returns: + Series of backtested predictions indexed by timestamp + + Raises: + ValueError: If parameters are invalid or data is insufficient + RuntimeError: If backtesting fails + """ + if self.result_ is None: + raise ValueError("Model is not fitted") + if self.training_series_ is None: + raise ValueError("No training series found") + + # Handle reuse of previous execution + if reuse_previous_execution and self.backtest_predictions_ is not None: + expected_index = y.index + if ( + len(self.backtest_predictions_) == len(expected_index) + and (self.backtest_predictions_.index == expected_index).all() + ): + console.log( + "[yellow]Reusing previous backtest results[/yellow]" + ) + return self.backtest_predictions_ + else: + console.log( + "[yellow]Previous results incompatible, running new" + + " backtest[/yellow]" + ) + + try: + console.log( + f"[blue]Starting ARIMA backtest with {self.forecast_horizon}" + + "-step forecasting...[/blue]" + ) + + # Validate and prepare data + y_sorted = y.sort_index() + + # Check for overlapping data + training_series = self.training_series_ + if any(t in training_series.index for t in y_sorted.index): + console.print( + "[yellow]Warning: Backtest data overlaps with training" + + " data[/yellow]" + ) + + # Initialize backtesting + predictions = [] + + # Start with training data + current_series = training_series.copy() + + total_steps = len(y_sorted) + console.log( + f"[blue]Running {total_steps} backtest steps with retraining" + + f" every {retrain_every} steps...[/blue]" + ) + + # Create initial model state + current_model = ARIMA(current_series.values, order=self.order) + current_result = current_model.fit() + + # Perform walk-forward validation + for i, (timestamp, actual_value) in enumerate(y_sorted.items()): + if i % 50 == 0 and i > 0: # Progress logging + console.log( + f"[blue]Backtest progress: {i}/{len(y_sorted)}[/blue]" + ) + + try: + # Check if we need to retrain + if i % retrain_every == 0 and i > 0: + console.log( + f"[blue]Retraining model at step {i}[/blue]" + ) + current_model = ARIMA( + current_series.values, order=self.order + ) + current_result = current_model.fit() + + # Generate forecast_horizon-step ahead forecast + forecast = current_result.forecast( + steps=self.forecast_horizon + )[0] + predictions.append((timestamp, forecast)) + + # Update the series with actual observed value + current_series = pd.concat( + [ + current_series, + pd.Series([actual_value], index=[timestamp]), + ] + ) + + # For ARIMA, we can extend the model without full refit + if i % retrain_every != 0: + try: + current_result = current_result.extend( + [actual_value], refit=False + ) + except Exception: + # If extend fails, do a quick refit + current_model = ARIMA( + current_series.values, order=self.order + ) + current_result = current_model.fit() + + except Exception as step_error: + console.print( + f"[yellow]Error at step {i}: {step_error}, using" + + " NaN[/yellow]" + ) + predictions.append((timestamp, np.nan)) + + # Still update the series for continuity + current_series = pd.concat( + [ + current_series, + pd.Series([actual_value], index=[timestamp]), + ] + ) + + # Create results series + if predictions: + pred_index, pred_values = zip(*predictions) + self.backtest_predictions_ = pd.Series( + pred_values, + index=pd.Index(pred_index), + name=f"{self.target_col}_backtest", + ) + else: + self.backtest_predictions_ = pd.Series( + dtype=float, name=f"{self.target_col}_backtest" + ) + + console.log( + "[green]ARIMA backtest completed: " + + f"{len(self.backtest_predictions_)} predictions[/green]" + ) + return self.backtest_predictions_ + + except Exception as e: + console.print(f"[red]ARIMA backtest failed: {e}[/red]") + raise RuntimeError(f"Failed to perform backtest: {e}") from e + + @ensure_fitted + def summary(self) -> str: + """Generates a summary of the fitted ARIMA model. + + Returns: + str: The summary of the fitted model. + + Raises: + ValueError: If the model has not been fitted yet. + """ + if self.result_ is None: + raise ValueError("Model is not fitted.") + return str(self.result_.summary()) + + @ensure_fitted + def update(self, new_data: pd.Series, refit: bool = True) -> None: + """Updates the ARIMA model with new observed data. + + This method allows for two modes of updating the model: + 1. **Refitting**: The model is retrained on the combined dataset + (original training data + new data). + 2. **Incremental Update**: The model is updated using the new data + without retraining, preserving the original model parameters. + + Args: + new_data (pd.Series): New observed values to update the model with. + refit (bool, optional): If True, the model is retrained on the + combined dataset. Defaults to True. + + Raises: + TypeError: If `new_data` is not a pandas Series. + ValueError: If `new_data` is empty or if the model has not been + fitted yet. + """ + if not isinstance(new_data, pd.Series): + raise TypeError("new_data must be a pandas Series.") + if new_data.empty: + raise ValueError("new_data is empty.") + + if refit: + self.training_series_ = pd.concat( + [self.training_series_, new_data] + ) + self.observed_series_ = self.training_series_.copy() + y_array = self._validate_y(self.training_series_) + self.model_ = ARIMA(y_array, order=self.order) + self.result_ = self.model_.fit() + else: + self.observed_series_ = pd.concat( + [self.observed_series_, new_data] + ) + y_array = self._validate_y(self.observed_series_) + + if self.result_ is None: + raise ValueError("Model is not fitted.") + self.result_ = self.result_.apply(y_array, refit=False) + if self.result_ is not None: + self.model_ = self.result_.model diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py new file mode 100644 index 0000000..4afdfaf --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py @@ -0,0 +1,824 @@ +""" +Base model classes and interfaces. + +This module defines base classes with consistent interfaces for all models, +promoting modular model development. It includes essential functionality +for model fitting, prediction, evaluation, saving, and loading, while +abstracting common behaviors into base classes. + +While full compatibility with scikit-learn is not guaranteed, the base +classes provide a consistent interface for model fitting, prediction, +evaluation, saving, and loading, which should be sufficient for most +use cases. + +Key components: +- **Model**: Abstract base class for all models, providing core utilities and + interfaces. +- **TimeSeriesModel**: Abstract base class for time series models, adding + time-based functionality. +- **UnivariateTimeSeriesModel**: Base class for univariate time series models. +- **MultivariateTimeSeriesModel**: Base class for multivariate time series + models that use exogenous features. + +The module also includes utility functions such as `ensure_fitted`, which + ensures models are fitted before calling certain methods. + +Modules in this package should inherit from these base classes and implement + the required methods. + +TODO: + - Add methods to create lagged features for time series models. +""" + +import uuid +import joblib +from abc import ABC, abstractmethod +from typing import Optional, List, Sequence, Tuple, cast, Any, Protocol +from contextlib import contextmanager + +import pandas as pd +import numpy as np +import shap +import matplotlib.pyplot as plt +from rich.console import Console + +from sklearn.base import BaseEstimator, RegressorMixin +from sklearn.utils.validation import check_array, check_X_y +from sklearn.exceptions import NotFittedError + +console = Console() + + +class PredictorProtocol(Protocol): + """Protocol for models with predict method and optional imputation.""" + + def predict(self, X: Any) -> Any: ... + def _impute_missing_values(self, X: Any) -> Any: ... + + +def ensure_fitted(method): + """ + Decorator to ensure the model is fitted before calling the method. + + Raises: + sklearn.exceptions.NotFittedError: If the model is not fitted. + Usage: + @ensure_fitted + def predict(self, X): # Or other methods requiring fit + pass + """ + + def wrapper(self, *args, **kwargs): + is_fitted = self.__sklearn_is_fitted__() + if not is_fitted: + raise NotFittedError( + f"This {self.__class__.__name__} instance is not fitted yet. " + "Call 'fit' with appropriate arguments before using this " + "method." + ) + return method(self, *args, **kwargs) + + return wrapper + + +class Model(BaseEstimator, ABC): + """ + Abstract base class for all models. + + Provides core utilities, input validation, and interface consistency + for time series models. Compatible with scikit-learn workflows. + """ + + def __init__(self, name: Optional[str] = None, random_seed: int = 42): + """ + Initialize the model with a unique name and random seed. + + Args: + name: Optional identifier; auto-generated if None. + random_seed: Seed for reproducibility. + """ + self.name = name or f"{self.__class__.__name__}_{uuid.uuid4().hex}" + self.random_seed = random_seed + self.feature_names_in_: Optional[List[str]] = None + self.n_features_in_: Optional[int] = None + self._is_fitted = False + + def fit( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> "Model": + """ + Trains the model. + + Handles basic input validation for y and sets internal fitted + state after calling _fit_logic. + + 'X' is optional to account for univariate time series models. + + Args: + y: The target variable. + X: Optional exogenous variables. + X_val: Optional validation feature matrix. + y_val: Optional validation target series. + Raises: + TypeError: If y is not a pandas Series. + If X is provided, it must be a pandas DataFrame. + If X_val and y_val are provided, they must be pandas DataFrames + and Series respectively. + + Returns: + Self for chaining. + """ + if not isinstance(y, pd.Series): + raise TypeError("Input 'y' (target) must be a pandas Series.") + + self._fit_logic(y, X, X_val, y_val) + self._is_fitted = True + return self + + @abstractmethod + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """ + Core fitting logic to be implemented by subclasses with + optional validation data. + + Args: + y: The target variable. + X: Optional exogenous variables. + X_val: Validation feature matrix (optional). + y_val: Validation target series (optional). + """ + raise NotImplementedError("Subclasses must implement _fit_logic().") + + @ensure_fitted + @abstractmethod + def predict(self, X: Optional[pd.DataFrame] = None) -> Sequence: + """ + Predict values. + + Args: + X: Optional features for prediction. For univariate models + not using exogenous variables, this might be None or + contain future timestamps. Multivariate models will + require X. + + Returns: + NumPy array or similar sequence of predictions. + """ + raise NotImplementedError("Subclasses must implement predict().") + + def fit_predict( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> Sequence: + """ + Fits model and returns predictions on the same data. + + Args: + y: The target time series. + X: Optional exogenous variables. + + Returns: + Predictions for the input data. + """ + return self.fit(y, X, X_val, y_val).predict(X) + + def save(self, path: str) -> None: + """ + Saves model to disk using joblib. + """ + joblib.dump(self, path) + + @classmethod + def load(cls, path: str) -> "Model": + """ + Loads model from disk using joblib. + """ + return joblib.load(path) + + def __str__(self) -> str: + return f"{self.__class__.__name__}(name={self.name})" + + def __sklearn_is_fitted__(self) -> bool: + """ + Check fitted status and return a Boolean value. + """ + return hasattr(self, "_is_fitted") and self._is_fitted + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(name={self.name})" + + @contextmanager + def model_state_preservation(self): + """Context manager to preserve model state during operations.""" + original_state = self._get_state_snapshot() + try: + yield + except Exception: + self._restore_state_snapshot(original_state) + raise + + def _get_state_snapshot(self) -> dict: + """Get snapshot of current model state.""" + return { + "name": self.name, + "is_fitted": getattr(self, "_is_fitted", False), + "feature_names": self.feature_names_in_, + "n_features": self.n_features_in_, + } + + def _restore_state_snapshot(self, snapshot: dict) -> None: + """Restore model state from snapshot.""" + self.name = snapshot["name"] + self._is_fitted = snapshot["is_fitted"] + self.feature_names_in_ = snapshot["feature_names"] + self.n_features_in_ = snapshot["n_features"] + + def get_params_dict(self) -> dict: + """Get model parameters as dictionary for logging/serialization.""" + return { + "name": self.name, + "random_seed": self.random_seed, + "n_features_in_": self.n_features_in_, + } + + def summary(self) -> str: + """Generate a summary string of the model.""" + params = self.get_params_dict() + fitted_status = ( + "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" + ) + + summary_lines = [ + f"Model: {self.__class__.__name__}", + f"Status: {fitted_status}", + f"Features: {params.get('n_features_in_', 'Unknown')}", + ] + + return "\n".join(summary_lines) + + +class TimeSeriesModel(Model): + """ + Abstract base class for time series forecasting models. + + Extends the base Model class with specific methods for time series + data handling and evaluation. + """ + + def __init__( + self, + name: Optional[str] = None, + time_col: str = "ds", + target_col: str = "y", + random_seed: int = 42, + n_lags: int = 0, + sampling_freq: Optional[str] = None, + ): + super().__init__(name=name, random_seed=random_seed) + self.time_col = time_col + self.target_col = target_col + self.n_lags = n_lags + self.sampling_freq = sampling_freq + + self.training_series_: Optional[pd.Series] = None + self.model_: Optional[BaseEstimator] = None + + # Validate configuration + self._validate_configuration() + + def _validate_configuration(self) -> None: + """Validate model configuration.""" + if self.n_lags < 0: + raise ValueError("n_lags must be non-negative") + + def _validate_y(self, y: pd.Series) -> np.ndarray: + """ + Validates the target variable (y) for the model. + + Ensures y is a pandas Series and checks its name against + the expected target column name. Converts y to a NumPy array. + The series name can be None, but if it is set, it should match + the expected target column name. + + Args: + y: The target variable as a pandas Series. + + Returns: + A NumPy array of the target variable. + + Raises: + TypeError: If y is not a pandas Series. + """ + # Check if y is a pandas Series + if not isinstance(y, pd.Series): + raise TypeError("Input 'y' (target) must be a pandas Series.") + if (y.name is not None) and (y.name != self.target_col): + raise ValueError( + f"Expected target column name '{self.target_col}', " + f"but got '{y.name}'." + ) + return check_array(y, ensure_2d=False) + + @ensure_fitted + @abstractmethod + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Performs backtesting on the time series data. + + Args: + y: The target time series data. + X: Optional exogenous features. + retrain_every: Number of steps after which to retrain the model. + reuse_previous_execution: Whether to reuse the previous execution + of a backtest. If True, any overlapping data between the + previous execution and the current execution will be used + without retraining the model. + Returns: + Series of predictions for each step in the time series. + """ + + raise NotImplementedError("Subclasses must implement backtest().") + + def get_params_dict(self) -> dict: + """Get model parameters as dictionary for logging/serialization.""" + base_params = super().get_params_dict() + ts_params = { + "time_col": self.time_col, + "target_col": self.target_col, + "n_lags": self.n_lags, + "sampling_freq": self.sampling_freq, + } + return {**base_params, **ts_params} + + +class UnivariateTimeSeriesModel(TimeSeriesModel, RegressorMixin): + """ + Base class for univariate time series models. + + Only supports regression settings. Concrete subclasses + must implement `_fit_logic` and `predict`. + """ + + @abstractmethod + @ensure_fitted + def forecast(self, forecast_horizon: int) -> Sequence: + """ + Forecast into the future for a given number of steps. + + Args: + forecast_horizon: Number of future time steps to forecast. + + Returns: + Sequence of forecasted values. + """ + raise NotImplementedError("Subclasses must implement forecast().") + + +class MultivariateTimeSeriesModel(TimeSeriesModel): + """ + Base class for multivariate time series models. + + This class provides a foundation for time series models that utilize + multiple exogenous features (X) to predict a target variable (y). + It supports both regression and classification tasks. + + Attributes: + selected_features_: List of feature names selected for the model. + learning_task: Type of learning task ('regression', 'binary', + 'multiclass'). + differentiate_target: Whether to apply differencing to make series + stationary. + bins: Bin edges for multiclass classification target + transformation. + + Example: + >>> class MyModel(MultivariateTimeSeriesModel): + ... def _fit_logic(self, y, X=None, **kwargs): + ... # Implementation here + ... pass + ... def predict(self, X=None): + ... # Implementation here + ... return predictions + """ + + def __init__( + self, + name: Optional[str] = None, + time_col: str = "ds", + target_col: str = "y", + random_seed: int = 42, + n_lags: int = 0, + sampling_freq: Optional[str] = None, + differentiate_target: bool = False, + bins: Optional[List[float]] = None, + learning_task: Optional[str] = None, + ): + # Set attributes before calling parent constructor + # This is needed because parent constructor calls + # _validate_configuration + self.selected_features_: Optional[List[str]] = None + self.learning_task: Optional[str] = learning_task + self.differentiate_target = differentiate_target + self.bins = bins + self.model_: Optional[PredictorProtocol] = None + + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + n_lags=n_lags, + sampling_freq=sampling_freq, + ) + + # Additional validation for multivariate models + self._validate_learning_task() + + def _validate_learning_task(self) -> None: + """Validate learning task configuration.""" + valid_tasks = {"regression", "binary", "multiclass", None} + if self.learning_task not in valid_tasks: + raise ValueError( + f"Invalid learning_task: {self.learning_task}. " + + f"Must be one of {valid_tasks}" + ) + + if self.learning_task == "multiclass" and not self.bins: + raise ValueError( + "bins must be provided for multiclass learning_task" + ) + + def _get_default_loss_function( + self, provided_loss: Optional[str] + ) -> str: + """ + Get default loss function based on learning task. + + Args: + provided_loss: User-provided loss function (takes precedence) + + Returns: + str: Appropriate loss function for the learning task + """ + if provided_loss is not None: + return provided_loss + + if self.learning_task == "regression": + return "RMSE" + elif self.learning_task == "binary": + return "Logloss" + elif self.learning_task == "multiclass": + return "MultiClass" + else: + return "RMSE" + + def _validate_configuration(self) -> None: + """Validate model configuration.""" + super()._validate_configuration() + + if self.differentiate_target and self.learning_task in [ + "binary", + "multiclass", + ]: + console.print( + "[yellow]Warning: Using differentiation with classification " + + "tasks may not be appropriate[/yellow]" + ) + + @ensure_fitted + def feature_importance(self) -> Optional[pd.DataFrame]: + """ + Returns feature importance if implemented by subclass. + + Returns: + A DataFrame with feature names and their importance scores, + or None if not applicable. + """ + return None + + def _validate_X_y( + self, X: pd.DataFrame, y: pd.Series, allow_nan: bool = True + ) -> Tuple[np.ndarray, np.ndarray]: + """ + Validates input features (X) and target (y). + + Infers and sets `feature_names_in_` and `n_features_in_`. + This method should be called within the `_fit_logic` of + concrete subclasses that use exogenous features. + + Args: + X: DataFrame of input features. + y: Series for the target variable. + allow_nan: If True, allows NaN values in X and y. + Raises: + TypeError: If X is not a DataFrame or y is not a Series. + ValueError: If the number of features in X does not match + the expected number of features. + + Returns: + Tuple of validated NumPy arrays (X_array, y_array). + """ + if allow_nan: + X_array, y_array = check_X_y(X, y, force_all_finite=False) + else: + X_array, y_array = check_X_y(X, y, force_all_finite=True) + + if hasattr(X, "columns"): + console.log( + f"Validating input features with columns: {X.columns.tolist()}" + ) + self.feature_names_in_ = list(X.columns) + else: + console.log( + "Input features do not have column names, using default names." + ) + self.feature_names_in_ = [ + f"feature_{i}" for i in range(X_array.shape[1]) + ] + + self.n_features_in_ = X_array.shape[1] + return X_array, y_array + + def _validate_X( + self, X: pd.DataFrame, allow_nan: bool = True + ) -> np.ndarray: + """ + Validates input features (X) before prediction or scoring. + + Ensures consistency with features seen during fit. This should + be called by concrete subclasses in `predict`, `score`, etc. + + Args: + X: DataFrame of input features. + allow_nan: If True, allows NaN values in X. + + Returns: + Validated NumPy array of X. + """ + if allow_nan: + X_array = check_array(X, force_all_finite=False) + else: + X_array = check_array(X, force_all_finite=True) + # If the model has been fitted, ensure the input features + # match the features seen during fit. + if self.feature_names_in_ is not None: + if not set(self.feature_names_in_).issubset(X.columns): + raise ValueError( + "Input features do not match the features seen during fit." + + f" Expected features: {self.feature_names_in_}, " + + f"but got: {list(X.columns)}." + ) + + return X_array + + def _transform_target_to_multiclass( + self, y: pd.Series, bins: Optional[List[float]] = None + ) -> pd.Series: + """ + Transforms the target variable into a multiclass classification + target. + + If bins are provided, uses pd.cut to categorize the target into + discrete classes. If not, binarize the target at zero (0). + + Args: + y: The target variable as a pandas Series. + bins: Optional list of bin edges for categorization. + + Returns: + A pandas Series with transformed classification targets. + """ + if bins is not None: + # pd.cut returns a Categorical, convert to Series with integer + # codes + categories = pd.cut(y, bins=bins, labels=False) + return pd.Series(categories, index=y.index) + + return (y > 0).astype(int) + + def _transform_target_to_binary( + self, y: pd.Series, threshold: float = 0.0 + ) -> pd.Series: + """ + Transforms the target variable into a binary classification target. + + Binarizes the target at the specified threshold (default is 0.0). + + Args: + y: The target variable as a pandas Series. + threshold: The threshold for binarization. + + Returns: + A pandas Series with binary classification targets. + """ + return (y > threshold).astype(int) + + def _preprocess_data( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> Tuple[ + pd.Series, + Optional[pd.DataFrame], + Optional[pd.Series], + Optional[pd.DataFrame], + ]: + """ + Internal method to handle common data preprocessing operations. + + Args: + y: The target time series data + X: The feature matrix (including exogenous features) + X_val: Validation feature matrix (optional) + y_val: Validation target series (optional) + + Returns: + A tuple containing: + - processed y series + - processed X dataframe (optional) + - processed y_val series (optional) + - processed X_val dataframe (optional) + """ + # Apply differentiation if enabled + if self.differentiate_target: + y = y.diff().dropna() + if X is not None: + X = X.loc[y.index] + + # Transform target for classification if needed + if self.learning_task == "binary": + y = self._transform_target_to_binary(y) + elif self.learning_task == "multiclass": + y = self._transform_target_to_multiclass(y, self.bins) + + # Process validation data if provided + if y_val is not None: + if X_val is None: + raise ValueError( + "Validation features (X_val) must be provided if " + + "validation target (y_val) is given." + ) + y_val = y_val.loc[X_val.index] + if self.differentiate_target: + y_val = y_val.diff().dropna() + X_val = X_val.loc[y_val.index] + if self.learning_task == "binary": + y_val = self._transform_target_to_binary(y_val) + elif self.learning_task == "multiclass": + y_val = self._transform_target_to_multiclass(y_val, self.bins) + + # Filter features if selected_features_ is set + if X is not None and self.selected_features_ is not None: + X = cast(pd.DataFrame, X[self.selected_features_].copy()) + if X_val is not None: + X_val = cast( + pd.DataFrame, X_val[self.selected_features_].copy() + ) + + return y, X, y_val, X_val + + def _prepare_shap_data(self, X: pd.DataFrame) -> pd.DataFrame: + """Prepare data for SHAP analysis.""" + X_processed = X.copy() + + # Remove target column if present + if self.target_col in X_processed.columns: + X_processed = X_processed.drop(columns=[self.target_col]) + + # Filter selected features + if self.selected_features_ is not None: + X_processed = cast( + pd.DataFrame, X_processed[self.selected_features_].copy() + ) + + return X_processed + + def _create_shap_explainer(self, X: pd.DataFrame) -> Any: + """Create appropriate SHAP explainer based on model type.""" + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + + if hasattr(self.model_, "coef_"): # Linear models + try: + # Handle missing values if model supports it + X_clean = self._handle_missing_values_for_shap(X) + return shap.LinearExplainer(self.model_, X_clean) + except Exception as e: + console.print( + f"[yellow]Warning: Linear explainer failed: {e}, " + + "using KernelExplainer[/yellow]" + ) + background = shap.maskers.Independent(X, max_samples=100) + return shap.KernelExplainer( + self.model_.predict, + background, + ) + else: + # Non-linear models + if self.learning_task == "binary": + return shap.TreeExplainer( + self.model_, X, model_output="probability" + ) + else: + return shap.Explainer(self.model_, X) + + def _handle_missing_values_for_shap(self, X: pd.DataFrame) -> pd.DataFrame: + """Handle missing values for SHAP analysis.""" + # Use type ignore for optional method + if hasattr(self.model_, "_impute_missing_values"): + return self.model_._impute_missing_values(X) # type: ignore + else: + return X.dropna() + + def _generate_and_save_plot( + self, explainer: Any, X: pd.DataFrame, path: str + ) -> None: + """Generate and save SHAP plot.""" + shap_values = explainer(X) + + shap.plots.beeswarm(shap_values, show=False) + shap_fig = plt.gcf() + shap_fig.set_size_inches(10, 6) + shap_fig.suptitle(f"SHAP Beeswarm Plot for {self.name}", fontsize=16) + shap_fig.tight_layout() + shap_fig.savefig(path) + plt.clf() + plt.close() + + @ensure_fitted + def shap_beeswarm_plot(self, X: pd.DataFrame, path: str) -> None: + """ + Generates a SHAP beeswarm plot for the model's predictions. + + Args: + X: DataFrame of input features. + path: Path to save the plot file. + + Raises: + ValueError: If model is not fitted. + Exception: If SHAP plot generation fails. + """ + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + + try: + # Prepare data + X_processed = self._prepare_shap_data(X) + + # Create explainer and generate plot + explainer = self._create_shap_explainer(X_processed) + self._generate_and_save_plot(explainer, X_processed, path) + + except Exception as e: + console.print( + f"[red]Error: Failed to generate SHAP plot: {e}[/red]" + ) + raise + + def get_params_dict(self) -> dict: + """Get model parameters as dictionary for logging/serialization.""" + base_params = super().get_params_dict() + mv_params = { + "learning_task": self.learning_task, + "differentiate_target": self.differentiate_target, + "selected_features_": self.selected_features_, + } + return {**base_params, **mv_params} + + def summary(self) -> str: + """Generate a summary string of the model.""" + params = self.get_params_dict() + fitted_status = ( + "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" + ) + + summary_lines = [ + f"Model: {self.__class__.__name__}", + f"Status: {fitted_status}", + f"Features: {params.get('n_features_in_', 'Unknown')}", + f"Task: {params.get('learning_task', 'regression')}", + f"Selected Features: {len(self.selected_features_) if self.selected_features_ else 'All'}", + ] + + return "\n".join(summary_lines) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py new file mode 100644 index 0000000..3721871 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py @@ -0,0 +1,510 @@ +""" +CatBoost implementation for multivariate time series forecasting. + +TODO: + - Implement support for categorical features. +""" + +from typing import Optional, List, Union, Tuple, Dict, Any, cast + +import pandas as pd +from catboost import CatBoostRegressor, CatBoostClassifier, Pool +from rich.console import Console +import numpy as np + +from .base import MultivariateTimeSeriesModel, ensure_fitted + +console = Console() + + +class CatBoostTimeSeriesModel(MultivariateTimeSeriesModel): + """ + CatBoost implementation for multivariate time series forecasting. + + This class wraps the CatBoost models with additional functionality for + time series forecasting, following the MultivariateTimeSeriesModel + interface. Supports both regression and classification tasks with + comprehensive error handling and type safety. + """ + + def __init__( + self, + name: Optional[str] = None, + learning_task: str = "regression", + differentiate_target: bool = False, + n_lags: int = 0, + iterations: int = 1000, + learning_rate: float = 0.1, + depth: int = 6, + loss_function: Optional[str] = None, + bins: Optional[List[float]] = None, + random_seed: int = 42, + time_col: str = "ds", + target_col: str = "y", + verbose: bool = False, + ) -> None: + """ + Initialize the CatBoost time series model. + + Args: + name: Optional identifier for the model + learning_task: Type of learning task, either 'regression', + 'multiclass' or 'binary'. + differentiate_target: Whether to differentiate the target + series before fitting the model. + n_lags: Number of lagged target values included as features. + These lags are expected to already be present in the same + dataset as the exogenous features. + iterations: Number of boosting iterations + learning_rate: Learning rate for the model + depth: Depth of the tree + loss_function: Loss function to optimize + bins: Optional list of bin edges for multiclass classification + random_seed: Random seed for reproducibility + time_col: Name of the time column + target_col: Name of the target column + verbose: Whether to enable verbose output + """ + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + n_lags=n_lags, + learning_task=learning_task, + differentiate_target=differentiate_target, + bins=bins, + ) + + self.iterations = iterations + self.learning_rate = learning_rate + self.depth = depth + self.verbose = verbose + + # Set default loss function based on learning task + self.loss_function = self._get_default_loss_function(loss_function) + + # Initialize model state + self.model_: Optional[Union[CatBoostRegressor, CatBoostClassifier]] = ( + None + ) + self.training_series_: Optional[pd.Series] = None + self.X_train_: Optional[pd.DataFrame] = None + self.backtest_predictions_: Optional[pd.Series] = None + + if self.verbose: + console.log( + "[green]Initialized CatBoostTimeSeriesModel:" + + f" {self.summary()}[/green]" + ) + + def _create_model(self) -> Union[CatBoostRegressor, CatBoostClassifier]: + """Creates a new instance of CatBoost model with current parameters. + + Returns: + A new CatBoost model instance (Regressor or Classifier). + """ + base_params = { + "iterations": self.iterations, + "learning_rate": self.learning_rate, + "depth": self.depth, + "loss_function": self.loss_function, + "random_seed": self.random_seed, + "verbose": self.verbose, + } + + if self.learning_task in ["binary", "multiclass"]: + return CatBoostClassifier( + auto_class_weights="Balanced", **base_params + ) + else: + return CatBoostRegressor(**base_params) + + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """ + Core fitting logic for CatBoost model with optional validation data. + + Args: + y: The target time series data + X: The feature matrix (including exogenous features) + X_val: Validation feature matrix (optional) + y_val: Validation target series (optional) + """ + if X is None or not isinstance(X, pd.DataFrame): + raise ValueError("Feature matrix X must be a non-empty DataFrame.") + + y_processed, X_processed, y_val_processed, X_val_processed = ( + self._preprocess_data(y, X, X_val, y_val) + ) + + # Ensure X is not None after preprocessing + if X_processed is None: + raise ValueError( + "Feature matrix X cannot be None after preprocessing." + ) + + X_array, y_array = self._validate_X_y(X_processed, y_processed) + + self.training_series_ = y_processed.copy() + self.X_train_ = X_processed.copy() + self.model_ = self._create_model() + + eval_set = None + if X_val_processed is not None and y_val_processed is not None: + X_val_array, y_val_array = self._validate_X_y( + X_val_processed, y_val_processed + ) + eval_set = Pool(data=X_val_array, label=y_val_array) + + train_pool = Pool(data=X_array, label=y_array) + + if self.verbose: + console.log( + f"[blue]Training CatBoost model for {self.iterations}" + + " iterations...[/blue]" + ) + + self.model_.fit(train_pool, eval_set=eval_set) + + if self.verbose: + console.log( + "[green]CatBoost model training completed successfully[/green]" + ) + + @ensure_fitted + def predict(self, X: pd.DataFrame) -> pd.Series: + """ + Generate predictions using the fitted CatBoost model. + + Args: + X: The feature matrix for prediction + + Returns: + pd.Series: Predicted values + """ + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + + # Create a copy to avoid modifying the original DataFrame + X_pred = X.copy() + + if self.selected_features_ is not None: + X_pred = cast(pd.DataFrame, X_pred[self.selected_features_]) + + X_array = self._validate_X(X_pred) + predictions = self.model_.predict(X_array) + + # Convert predictions to numpy array if needed + if hasattr(predictions, "squeeze"): + predictions = predictions.squeeze() + elif isinstance(predictions, list): + predictions = np.array(predictions) + + return_series = pd.Series( + predictions, + index=X.index, + name=self.target_col, + ) + return return_series + + @ensure_fitted + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Performs backtesting (walk-forward validation) with periodic + retraining. + + This method simulates a production scenario by iterating through a test + set, making a one-step-ahead prediction, and then retraining the model + periodically with the newly available data. + + Args: + X: DataFrame with features for the backtesting period. + y: Series with the true target values for the backtesting period. + retrain_every: The frequency of retraining. The model will be + retrained every `retrain_every` steps. + reuse_previous_execution: Whether to reuse the previous execution + of a backtest. If True, any overlapping data between the + previous execution and the current execution will be used + without retraining the model. + Returns: + A series of backtested predictions, indexed by the backtest data's + index. + """ + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + if self.training_series_ is None: + raise ValueError("Training series is not set.") + if self.X_train_ is None: + raise ValueError("Training feature matrix is not set.") + if X is None or not isinstance(X, pd.DataFrame): + raise ValueError("Feature matrix X must be a non-empty DataFrame.") + + if reuse_previous_execution: + if self.backtest_predictions_ is None: + raise ValueError("No previous execution found.") + if (self.backtest_predictions_.shape[0] != y.shape[0]) or ( + not (self.backtest_predictions_.index == y.index).all() + ): + raise ValueError( + "Previous execution index does not match y index." + ) + return self.backtest_predictions_ + # Prepare + total_steps = len(X) + predictions = [] + current_model = self.model_ + y_history = self.training_series_.copy() + X_history = self.X_train_.copy() + + # Iterate in chunks instead of single steps + for start in range(0, total_steps, retrain_every): + end = min(start + retrain_every, total_steps) + + # Batch prediction for current chunk + X_chunk = X.iloc[start:end].copy() + if self.selected_features_: + X_chunk = X_chunk[self.selected_features_] + + X_array = self._validate_X(X_chunk) + preds = current_model.predict(X_array) + + # Handle different prediction formats + if hasattr(preds, "squeeze"): + preds = preds.squeeze() + if preds.ndim == 0: # single point + preds = [preds] + predictions.extend(preds) + + # Update training history + y_chunk = y.iloc[start:end] + y_history = pd.concat([y_history, y_chunk]) + X_history = pd.concat([X_history, X_chunk]) + + # Retrain the model for next chunk (if needed) + if end < total_steps: + if self.verbose: + console.print( + f"[cyan]Backtesting: Retraining at step {end}..." + ) + + current_model = self._create_model() + (y_fit, X_fit, _, _) = ( + self._preprocess_data(y_history, X_history) + ) + + # Ensure X_fit is not None after preprocessing + if X_fit is None: + raise ValueError( + "Feature matrix cannot be None after preprocessing." + ) + + X_fit_array, y_fit_array = self._validate_X_y(X_fit, y_fit) + train_pool = Pool(data=X_fit_array, label=y_fit_array) + current_model.fit(train_pool) + + # Store backtest predictions for potential reuse + self.backtest_predictions_ = pd.Series( + predictions, index=X.index, name=f"{self.target_col}_pred" + ) + return self.backtest_predictions_ + + def select_features( + self, + X: pd.DataFrame, + y: pd.Series, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + features_to_select: Optional[int] = None, + algorithm: str = "RecursiveByShapValues", + steps: int = 1, + verbose: bool = False, + ) -> List[str]: + """Identify and select the most important features. + + Uses CatBoost's built-in feature selection capabilities to determine + feature importance and select the most relevant features. + + Args: + X: The feature matrix + y: The target series + X_val: Optional validation feature matrix + y_val: Optional validation target series + features_to_select: Number of features to select. If None, + will select half of the features. + algorithm: Feature selection algorithm. One of: + 'RecursiveByShapValues', 'RecursiveByPredictionValuesChange' + steps: How many times a full model will be trained. + More steps give more accurate results. + verbose: Whether to print progress + + Returns: + List[str]: List of selected feature names + """ + (y_processed, X_processed, y_val_processed, X_val_processed) = ( + self._preprocess_data(y, X, X_val, y_val) + ) + + # Validate input data + if X_processed is None or not isinstance(X_processed, pd.DataFrame): + raise ValueError("Feature matrix X must be a non-empty DataFrame.") + X_array, y_array = self._validate_X_y(X_processed, y_processed) + + # Set default number of features to select if not specified + if features_to_select is None: + features_to_select = X_processed.shape[1] // 2 + + # Create and prepare model + temp_model = self._create_model() + train_pool = Pool(data=X_array, label=y_array) + + # Prepare validation data if provided + eval_set = None + if X_val_processed is not None and y_val_processed is not None: + X_val_array, y_val_array = self._validate_X_y( + X_val_processed, y_val_processed + ) + eval_set = Pool(data=X_val_array, label=y_val_array) + + # Perform feature selection + selected_features = temp_model.select_features( + train_pool, + eval_set=eval_set, + features_for_select=list(range(X_processed.shape[1])), + num_features_to_select=features_to_select, + algorithm=algorithm, + steps=steps, + logging_level="Verbose" if verbose else "Silent", + train_final_model=False, + ) + + # Map feature indices to feature names with proper type casting + selected_feature_names: List[str] = [ + str(X_processed.columns[idx]) + for idx in selected_features["selected_features"] + ] + self.selected_features_ = selected_feature_names + self.feature_names_in_ = selected_feature_names + self.n_features_in_ = len(selected_feature_names) + + return selected_feature_names + + @classmethod + def tune_hyperparameters( + cls, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + selected_features: Optional[List[str]] = None, + param_grid: Optional[Dict[str, Any]] = None, + n_trials: int = 10, + early_stopping_rounds: Optional[int] = 50, + random_seed: int = 42, + **kwargs, + ) -> Tuple[Dict[str, Any], "CatBoostTimeSeriesModel"]: + """ + Tune hyperparameters for the CatBoost model. + + Args: + y: The target time series data + X: The feature matrix (including exogenous features) + X_val: Validation feature matrix (optional) + y_val: Validation target series (optional) + selected_features: List of features to use for tuning + param_grid: Dictionary of hyperparameters to search + n_trials: Number of trials for hyperparameter tuning + early_stopping_rounds: Number of rounds for early stopping + random_seed: Random seed for reproducibility + **kwargs: Additional keyword arguments for model initialization + + Returns: + Tuple[Dict[str, Any], CatBoostTimeSeriesModel]: Best hyperparameters + and fitted model + """ + if n_trials <= 0: + raise ValueError("n_trials must be a positive integer.") + # Create a temporary model instance to use its preprocessing method + temp_model = cls( + learning_task=kwargs.get("learning_task", "regression"), + differentiate_target=kwargs.get("differentiate_target", False), + bins=kwargs.get("bins", None), + random_seed=random_seed, + ) + if selected_features is not None: + temp_model.selected_features_ = selected_features + + (y_processed, X_processed, _, _) = ( + temp_model._preprocess_data(y, X, X_val, y_val) + ) + + if kwargs.get("learning_task", "regression") == "classification": + search_model = CatBoostClassifier( + random_seed=random_seed, + logging_level="Silent", + early_stopping_rounds=early_stopping_rounds, + loss_function=kwargs.get("loss_function", "Logloss"), + class_weights="Balanced", + ) + else: + search_model = CatBoostRegressor( + random_seed=random_seed, + logging_level="Silent", + early_stopping_rounds=early_stopping_rounds, + loss_function=kwargs.get("loss_function", "RMSE"), + ) + + if param_grid is None: + param_grid = { + "iterations": [100, 500, 1000, 2000], + "learning_rate": [0.01, 0.05, 0.1, 0.2], + "depth": [4, 6, 8], + } + + if X_processed is None: + raise ValueError( + "Feature matrix cannot be None after preprocessing." + ) + + train_pool = Pool(data=X_processed, label=y_processed) + results = search_model.randomized_search( + param_grid, + X=train_pool, + n_iter=n_trials, + verbose=False, + refit=False, + ) + + best_params = results["params"] + + best_model = cls( + iterations=best_params["iterations"], + learning_rate=best_params["learning_rate"], + depth=best_params["depth"], + random_seed=random_seed, + time_col=kwargs.get("time_col", "ds"), + target_col=kwargs.get("target_col", "y"), + n_lags=kwargs.get("n_lags", 0), + name=kwargs.get("name", None), + loss_function=kwargs.get("loss_function", None), + learning_task=kwargs.get("learning_task", "regression"), + bins=kwargs.get("bins", None), + differentiate_target=kwargs.get("differentiate_target", False), + ) + if selected_features is not None: + best_model.selected_features_ = selected_features + + best_model.fit(y, X, X_val, y_val) + + return best_params, best_model diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py new file mode 100644 index 0000000..77331d7 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py @@ -0,0 +1,92 @@ +""" +Module with functions of timeseries evaluation. +""" + +from typing import Optional, List, Sequence + +import numpy as np +import pandas as pd +from rich.console import Console +from sklearn.metrics import ( + mean_absolute_error, + mean_squared_error, + accuracy_score, + f1_score, + confusion_matrix, +) +console = Console() + + +def timeseries_metrics( + y_pred: Sequence[float], y_true: Sequence[float] +) -> dict[str, float]: + """ + Compute MAE, MSE, and trend capture for time series predictions. + + Parameters: + y_pred (ArrayLike): Predicted values. + y_true (ArrayLike): Ground truth values. + + Returns: + dict[str, float]: Dictionary with MAE, MSE, and trend_capture. + """ + if len(y_true) != len(y_pred): + raise ValueError("y_true and y_pred must have the same length") + if len(y_true) == 0: + raise ValueError("y_true and y_pred must not be empty") + y_true_np = np.asarray(y_true) + y_pred_np = np.asarray(y_pred) + + mae = mean_absolute_error(y_true_np, y_pred_np) + mse = mean_squared_error(y_true_np, y_pred_np) + + # Compute directional trend: 1 if up, 0 if down or flat + if len(y_true) == 1: + return {"MAE": mae, "MSE": mse, "trend_capture": 1.0} + + true_trend = np.diff(y_true_np) > 0 + pred_trend = np.diff(y_pred_np) > 0 + + trend_capture = np.mean(true_trend == pred_trend) + + return {"MAE": mae, "MSE": mse, "trend_capture": trend_capture} + + +def timeseries_classification_metrics( + y_pred: Sequence[float], + y_true: Sequence[float], + bins: Optional[List] = None, +) -> dict[str, float]: + """ + Compute accuracy for classification predictions. + + Parameters: + y_pred (ArrayLike): Predicted values. + y_true (ArrayLike): Ground truth values. + bins (List[int], optional): Bin edges for categorizing predictions. + + Returns: + dict[str, float]: Dictionary with accuracy. + """ + if bins is not None: + console.log( + f"Using bins for classification: {bins}" + ) + y_true_binned = pd.cut(y_true, bins=bins, labels=False) + else: + console.log("No bins provided, using default classification (y > 0).") + y_true_binned = (y_true > 0).astype(int) + console.log( + f"y_true_binned: {y_true_binned.value_counts()}" + ) + console.log( + f"y_pred: {pd.Series(y_pred).value_counts()}" + ) + + y_pred = np.array(y_pred).astype(int) + acc = accuracy_score(y_true_binned, y_pred) + f1 = f1_score(y_true_binned, y_pred, average="weighted") + # Calculate the confusion matrix + cm = confusion_matrix(y_true_binned, y_pred) + + return {"accuracy": acc, "f1_score": f1, "confusion_matrix": cm} diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py new file mode 100644 index 0000000..a8d646e --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py @@ -0,0 +1,140 @@ +""" +Model factory for creating, loading, and discovering time series models. + +This module provides centralized utility functions to handle different time +series model implementations based on a string identifier. It uses a +central registry (`SUPPORTED_MODELS`) that maps model type strings (e.g., +'arima') to their corresponding model classes (e.g., ARIMAModel). This +approach allows for easy extension and decouples model instantiation logic +from the code that uses the models. + +Key Functions: + create_model: Creates a new instance of a specified model type by looking + up the type string in the `SUPPORTED_MODELS` registry and + passing keyword arguments to the retrieved model class's + constructor. + load_model: Loads a previously saved model instance from disk. It uses + the provided model type string to find the correct class in + the registry and then calls that class's `.load()` classmethod. + get_available_models: Returns a dictionary listing the registered model + types (keys in `SUPPORTED_MODELS`) and their + descriptions, automatically derived from the model + class docstrings. + +Extensibility: + Adding support for a new model involves the following steps: + 1. Ensure the new model class (e.g., `MyNewModel`) inherits from the + appropriate base class (e.g., `TimeSeriesModel`) and implements all + required abstract methods. + 2. Ensure the new model class has a `.load()` classmethod compatible + with the `save()` method in the base `Model` class (if loading is + to be supported via this factory). + 3. Import the new model class into this factory module. + 4. Add an entry to the `SUPPORTED_MODELS` dictionary, mapping a unique, + lowercase string identifier to the model class itself: + `SUPPORTED_MODELS = {..., "mynewmodel": MyNewModel}` + Once added to the registry, the model can be created and loaded via the + factory functions, and it will automatically appear in the output of + `get_available_models()`. +""" + +from typing import Dict, Type +import logging + +from .base import TimeSeriesModel +from .arima import ARIMAModel +from .neural_prophet_model import NeuralProphetModel +from .catboost_time_series import CatBoostTimeSeriesModel +from .linear_regression_time_series import ElasticNetTimeSeriesModel +from .stacking_time_series import StackingTimeSeriesModel +# from .prophet import ProphetModel # Example for future + +# *** Central registry of supported models +SUPPORTED_MODELS: Dict[str, Type[TimeSeriesModel]] = { + "arima": ARIMAModel, + "neuralprophet": NeuralProphetModel, + "catboost": CatBoostTimeSeriesModel, + "elasticnet": ElasticNetTimeSeriesModel, + "stacking": StackingTimeSeriesModel, + # "prophet": ProphetModel, # Add new models here +} + + +def create_model(model_type: str, **kwargs) -> TimeSeriesModel: + """ + Create a new model instance of the specified type using a registry. + + Args: + model_type: Type of model to create (case-insensitive). + **kwargs: Model-specific parameters passed to its constructor. + + Returns: + New model instance inheriting from TimeSeriesModel. + + Raises: + ValueError: If the model type is not supported or kwargs are invalid. + """ + model_type = model_type.lower() + model_class = SUPPORTED_MODELS.get(model_type) + + if model_class: + try: + instance = model_class(**kwargs) + return instance + except TypeError as e: + logging.error(f"Kwargs issue for {model_type}: {kwargs}") + raise ValueError( + f"Invalid parameters for model type '{model_type}'. Error: {e}" + ) from e + else: + supported_list = ", ".join(f"'{k}'" for k in SUPPORTED_MODELS.keys()) + raise ValueError( + f"Unsupported model type: '{model_type}'. " + f"Currently supported models are: {supported_list}." + ) + + +def load_model(path: str, model_type: str) -> TimeSeriesModel: + """ + Load a model from disk using a registry. + + Args: + path: Path to the saved model. + model_type: Expected type of model to load (case-insensitive). + + Returns: + Loaded model instance. + + Raises: + ValueError: If the model type is not supported. + # Other errors might come from the underlying .load() method + """ + model_type = model_type.lower() + model_class = SUPPORTED_MODELS.get(model_type) + + if model_class: + return model_class.load(path) + else: + supported_list = ", ".join(f"'{k}'" for k in SUPPORTED_MODELS.keys()) + raise ValueError( + f"Unsupported model type: '{model_type}'. " + f"Currently supported models are: {supported_list}." + ) + + +def get_available_models() -> Dict[str, str]: + """ + Dynamically get a dictionary of available model types and their + descriptions from the SUPPORTED_MODELS registry and class docstrings. + + Returns: + Dictionary mapping model type names to descriptions. + """ + available = { + type_name: ( + model_class.__doc__.strip().splitlines()[0] + if model_class.__doc__ else "No description available." + ) + for type_name, model_class in SUPPORTED_MODELS.items() + } + return available diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py new file mode 100644 index 0000000..9468954 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py @@ -0,0 +1,303 @@ +""" +ElasticNet implementation for multivariate time series forecasting. +""" + +from typing import Optional, Sequence + +import pandas as pd +from sklearn.linear_model import ElasticNet +from sklearn.impute import SimpleImputer +from rich.console import Console + +from .base import MultivariateTimeSeriesModel, ensure_fitted + +console = Console() + + +class ElasticNetTimeSeriesModel(MultivariateTimeSeriesModel): + """ + ElasticNet implementation for multivariate time series forecasting. + + This class wraps the scikit-learn ElasticNet model with additional + functionality for time series forecasting, following the + MultivariateTimeSeriesModel interface. It's suitable for regression + tasks where features might be correlated. + """ + + def __init__( + self, + name: Optional[str] = None, + n_lags: int = 1, + alpha: float = 1.0, + l1_ratio: float = 0.5, + fit_intercept: bool = True, + max_iter: int = 1000, + tol: float = 1e-4, + random_seed: int = 42, + time_col: str = "ds", + target_col: str = "y", + differentiate_target: bool = False, + bins: Optional[list] = None, + learning_task: Optional[str] = None, + ) -> None: + """ + Initialize the ElasticNet time series model. + + Args: + name: Optional identifier for the model. + n_lags: Number of lagged target values to include as inputs. + alpha: Constant that multiplies the penalty terms. + l1_ratio: The ElasticNet mixing parameter (0 <= l1_ratio <= 1). + For l1_ratio = 0, it's L2 penalty (Ridge). + For l1_ratio = 1, it's L1 penalty (Lasso). + fit_intercept: Whether to calculate the intercept for this model. + max_iter: Maximum number of iterations. + tol: Tolerance for stopping criteria. + random_seed: Random seed for reproducibility. + time_col: Name of the time column. + target_col: Name of the target column. + differentiate_target: Whether to apply differencing to make series + stationary. + bins: Bin edges for multiclass classification target + transformation. + learning_task: Type of learning task ('regression', 'binary', + 'multiclass'). + """ + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + n_lags=n_lags, + differentiate_target=differentiate_target, + bins=bins, + learning_task=learning_task, + ) + + self.alpha = alpha + self.l1_ratio = l1_ratio + self.fit_intercept = fit_intercept + self.max_iter = max_iter + self.tol = tol + + self.model_: Optional[ElasticNet] = None + self.training_series_: Optional[pd.Series] = None + self.imputer_: Optional[SimpleImputer] = None + self.X_train_: Optional[pd.DataFrame] = None + + def _create_model(self) -> ElasticNet: + """Creates a new instance of ElasticNet with current parameters. + + Returns: + A new ElasticNet model instance. + """ + return ElasticNet( + alpha=self.alpha, + l1_ratio=self.l1_ratio, + fit_intercept=self.fit_intercept, + max_iter=self.max_iter, + tol=self.tol, + random_state=self.random_seed, + ) + + def _impute_missing_values(self, X: pd.DataFrame) -> pd.DataFrame: + """ + Handle missing values in the feature matrix using median imputation. + If the imputer is not fitted, it will be fitted on the data. + + Args: + X: The feature matrix potentially containing missing values. + + Returns: + pd.DataFrame: The feature matrix with imputed values. + """ + if self.imputer_ is None: + self.imputer_ = SimpleImputer( + strategy="median", copy=True, add_indicator=False + ) + # Fit the imputer and transform the data + imputed_values = self.imputer_.fit_transform(X) + else: + # Use the fitted imputer to transform new data + imputed_values = self.imputer_.transform(X) + + # Convert back to DataFrame with original index and column names + return pd.DataFrame(imputed_values, index=X.index, columns=X.columns) + + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """ + Core fitting logic for the ElasticNet model. + + Args: + y: The target time series data. + X: The feature matrix (including exogenous features). + X_val: Validation feature matrix (ignored). + y_val: Validation target series (ignored). + """ + # Use base class preprocessing + y_processed, X_processed, _, _ = self._preprocess_data( + y, X, X_val, y_val + ) + + if X_processed is None or not isinstance(X_processed, pd.DataFrame): + raise ValueError("Feature matrix X must be a non-empty DataFrame.") + + # First impute missing values in X + X_imputed = self._impute_missing_values(X_processed) + + # Validate X and y after imputation + X_array, y_array = self._validate_X_y( + X_imputed, y_processed, allow_nan=False + ) + + self.training_series_ = y_processed.copy() + self.model_ = self._create_model() + self.model_.fit(X_array, y_array) + self.X_train_ = X_processed.copy() + + @ensure_fitted + def predict(self, X: Optional[pd.DataFrame] = None) -> Sequence: + """ + Generate predictions using the fitted ElasticNet model. + + Args: + X: The feature matrix for prediction. + + Returns: + pd.Series: Predicted values with the original index. + """ + if X is None: + raise ValueError("Feature matrix X is required for prediction.") + + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + if self.training_series_ is None: + raise ValueError("Training series is not available.") + if self.imputer_ is None: + raise ValueError("Imputer is not fitted yet.") + + # If target column is present, drop it + X_pred = X.copy() + if self.target_col in X_pred.columns: + X_pred = X_pred.drop(columns=[self.target_col]) + + # For prediction, we need to reconstruct lagged features + # This is a simplified approach - in practice, you'd need + # the historical target values to create proper lags + + # Handle missing values using fitted imputer + X_processed = self._impute_missing_values(X_pred) + + # Validate X after imputation + X_array = self._validate_X(X_processed, allow_nan=False) + + predictions = self.model_.predict(X_array) + + return pd.Series( + predictions, index=X_processed.index, name=self.target_col + ) + + @ensure_fitted + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Performs backtesting (walk-forward validation) with periodic + retraining. + + Args: + y: The target time series data. + X: Optional exogenous features. + retrain_every: Number of steps after which to retrain the model. + reuse_previous_execution: Whether to reuse the previous execution + of a backtest. If True, any overlapping data between the + previous execution and the current execution will be used + without retraining the model. + + Returns: + Series of predictions for each step in the time series. + """ + if self.model_ is None: + raise ValueError("Model is not fitted yet.") + if self.training_series_ is None: + raise ValueError("Training series is not set.") + if self.X_train_ is None: + raise ValueError("Training feature matrix is not set.") + if X is None or not isinstance(X, pd.DataFrame): + raise ValueError("Feature matrix X must be a non-empty DataFrame.") + + total_steps = len(X) + predictions = [] + current_model = self.model_ + y_history = self.training_series_.copy() + X_history = self.X_train_.copy() + + # Iterate in chunks instead of single steps + for start in range(0, total_steps, retrain_every): + end = min(start + retrain_every, total_steps) + + # Batch prediction for current chunk + X_chunk = X.iloc[start:end].copy() + if self.selected_features_: + X_chunk = X_chunk[self.selected_features_] + X_imputed = self._impute_missing_values(X_chunk) + X_array = self._validate_X(X_imputed, allow_nan=False) + preds = current_model.predict(X_array).squeeze() + if preds.ndim == 0: # single point + preds = [preds] + predictions.extend(preds) + + # Update training history + y_chunk = y.iloc[start:end] + y_history = pd.concat([y_history, y_chunk]) + X_history = pd.concat([X_history, X_chunk]) + + # Retrain the model for next chunk (if needed) + if end < total_steps: + console.print( + f"[cyan]Backtesting: Retraining at step {end}...[/cyan]" + ) + + current_model = self._create_model() + y_fit, X_fit, *_ = self._preprocess_data(y_history, X_history) + X_fit_imputed = self._impute_missing_values(X_fit) + X_fit_array, y_fit_array = self._validate_X_y( + X_fit_imputed, y_fit, allow_nan=False + ) + current_model.fit(X_fit_array, y_fit_array) + + return pd.Series( + predictions, index=X.index, name=f"{self.target_col}_pred" + ) + + @ensure_fitted + def feature_importance(self) -> Optional[pd.DataFrame]: + """ + Returns feature importance based on model coefficients. + + Returns: + A DataFrame with feature names and their corresponding + coefficients (importance scores), or None if no features. + """ + if self.model_ is None or self.feature_names_in_ is None: + return None + + importance = self.model_.coef_ + feature_importance_df = pd.DataFrame( + {"Feature": self.feature_names_in_, "Importance": importance} + ) + feature_importance_df = feature_importance_df.sort_values( + by="Importance", key=abs, ascending=False + ).reset_index(drop=True) + + return feature_importance_df diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py new file mode 100644 index 0000000..e79f519 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py @@ -0,0 +1,888 @@ +""" +NeuralProphet implementation for univariate time series forecasting. + +This module provides a comprehensive wrapper around the NeuralProphet library, +implementing enterprise-level features including robust error handling, +parameter validation, type safety, and integration with the base model +architecture. +""" + +import os +from typing import Optional, cast, Tuple, Dict, Any + +import numpy as np +import pandas as pd +import torch +from neuralprophet import NeuralProphet +from rich.console import Console + +from .base import UnivariateTimeSeriesModel, ensure_fitted + +console = Console() + +# Configure PyTorch for optimal performance +torch.set_num_threads(os.cpu_count() or 1) + + +class NeuralProphetModel(UnivariateTimeSeriesModel): + """ + Enterprise-grade NeuralProphet implementation for univariate + time series forecasting. + + This class provides a robust wrapper around the NeuralProphet model with + comprehensive error handling, parameter validation, and integration with + the base model architecture. It includes features like automatic data + validation, performance monitoring, and enterprise-level logging. + + Key Features: + - Comprehensive parameter validation + - Robust error handling with detailed diagnostics + - Memory-efficient data processing + - Integration with base model utilities + - Performance monitoring and logging + - Support for various seasonality patterns + - Flexible forecasting capabilities + + Example: + >>> model = NeuralProphetModel( + ... n_lags=7, + ... n_forecasts=3, + ... epochs=50, + ... weekly_seasonality=True + ... ) + >>> model.fit(y_train) + >>> predictions = model.predict() + >>> future_forecast = model.forecast(forecast_horizon=3) + """ + + # Class constants for validation + VALID_SEASONALITY_MODES = {"additive", "multiplicative"} + VALID_LOSS_FUNCTIONS = {"Huber", "MSE", "MAE"} + VALID_NORMALIZE_OPTIONS = {"auto", "soft", "off", "minmax"} + MIN_EPOCHS = 1 + MAX_EPOCHS = 10000 + MIN_N_LAGS = 0 + MAX_N_LAGS = 365 + MIN_N_FORECASTS = 1 + MAX_N_FORECASTS = 365 + + def __init__( + self, + name: Optional[str] = None, + n_lags: int = 1, + n_forecasts: int = 2, + weekly_seasonality: bool = True, + daily_seasonality: bool = True, + yearly_seasonality: bool = False, + seasonality_mode: str = "additive", + epochs: int = 100, + learning_rate: Optional[float] = None, + batch_size: Optional[int] = None, + loss_func: str = "Huber", + normalize: str = "auto", + impute_missing: bool = True, + drop_missing: bool = False, + time_col: str = "ds", + target_col: str = "y", + random_seed: int = 42, + ): + """ + Initialize the NeuralProphet time series model with comprehensive + validation. + + Args: + name: Optional identifier for the model + n_lags: Number of lagged target values to include as inputs (0-365) + n_forecasts: Number of steps ahead to forecast (1-365) + weekly_seasonality: Whether to include weekly seasonality + daily_seasonality: Whether to include daily seasonality + yearly_seasonality: Whether to include yearly seasonality + seasonality_mode: Type of seasonality ('additive' or + 'multiplicative') + epochs: Number of training epochs (1-10000) + learning_rate: Learning rate for optimizer (auto if None) + batch_size: Training batch size (auto if None) + loss_func: Loss function ('Huber', 'MSE', 'MAE') + normalize: Normalization type ('auto', 'soft', 'off', 'minmax') + impute_missing: Whether to automatically impute missing values + drop_missing: Whether to drop missing values in training data + time_col: Name of the time column + target_col: Name of the target column + random_seed: Random seed for reproducibility + + Raises: + ValueError: If any parameters are invalid + TypeError: If parameters have incorrect types + """ + self._validate_and_set_parameters( + n_lags=n_lags, + n_forecasts=n_forecasts, + seasonality_mode=seasonality_mode, + epochs=epochs, + learning_rate=learning_rate, + batch_size=batch_size, + loss_func=loss_func, + normalize=normalize, + weekly_seasonality=weekly_seasonality, + daily_seasonality=daily_seasonality, + yearly_seasonality=yearly_seasonality, + impute_missing=impute_missing, + drop_missing=drop_missing, + ) + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + n_lags=n_lags, + ) + self.forecast_horizon = n_forecasts + # Initialize model state + self.model_: Optional[NeuralProphet] = None + self.backtest_predictions_: Optional[pd.Series] = None + self._training_metrics: Dict[str, float] = {} + + console.log( + f"[green]Initialized NeuralProphetModel: {self.summary()}[/green]" + ) + + def _validate_and_set_parameters( + self, + n_lags: int, + n_forecasts: int, + seasonality_mode: str, + epochs: int, + learning_rate: Optional[float], + batch_size: Optional[int], + loss_func: str, + normalize: str, + weekly_seasonality: bool, + daily_seasonality: bool, + yearly_seasonality: bool, + impute_missing: bool, + drop_missing: bool, + ) -> None: + """Validate and set model parameters with comprehensive checks.""" + # Validate integer parameters + if not (self.MIN_N_LAGS <= n_lags <= self.MAX_N_LAGS): + raise ValueError( + f"n_lags must be between {self.MIN_N_LAGS} and " + + f"{self.MAX_N_LAGS}, got {n_lags}" + ) + + if not (self.MIN_N_FORECASTS <= n_forecasts <= self.MAX_N_FORECASTS): + raise ValueError( + f"n_forecasts must be between {self.MIN_N_FORECASTS} and " + + f"{self.MAX_N_FORECASTS}, got {n_forecasts}" + ) + + if not (self.MIN_EPOCHS <= epochs <= self.MAX_EPOCHS): + raise ValueError( + f"epochs must be between {self.MIN_EPOCHS} and " + + f"{self.MAX_EPOCHS}, got {epochs}" + ) + + # Validate string parameters + if seasonality_mode not in self.VALID_SEASONALITY_MODES: + raise ValueError( + "seasonality_mode must be one of " + + f"{self.VALID_SEASONALITY_MODES}, got {seasonality_mode}" + ) + + if loss_func not in self.VALID_LOSS_FUNCTIONS: + raise ValueError( + "loss_func must be one of " + + f"{self.VALID_LOSS_FUNCTIONS}, got {loss_func}" + ) + + if normalize not in self.VALID_NORMALIZE_OPTIONS: + raise ValueError( + "normalize must be one of " + + f"{self.VALID_NORMALIZE_OPTIONS}, got {normalize}" + ) + + # Validate optional float parameters + if learning_rate is not None: + if ( + not isinstance(learning_rate, (int, float)) + or learning_rate <= 0 + ): + raise ValueError( + "learning_rate must be a positive number, " + + f"got {learning_rate}" + ) + + if batch_size is not None: + if not isinstance(batch_size, int) or batch_size <= 0: + raise ValueError( + "batch_size must be a positive integer, " + + f"got {batch_size}" + ) + + # Validate boolean parameters + for param_name, param_value in [ + ("weekly_seasonality", weekly_seasonality), + ("daily_seasonality", daily_seasonality), + ("yearly_seasonality", yearly_seasonality), + ("impute_missing", impute_missing), + ("drop_missing", drop_missing), + ]: + if not isinstance(param_value, bool): + raise TypeError( + f"{param_name} must be a boolean, " + + f"got {type(param_value)}" + ) + + # Set validated parameters + self.n_forecasts = n_forecasts + self.weekly_seasonality = weekly_seasonality + self.daily_seasonality = daily_seasonality + self.yearly_seasonality = yearly_seasonality + self.seasonality_mode = seasonality_mode + self.epochs = epochs + self.learning_rate = learning_rate + self.batch_size = batch_size + self.loss_func = loss_func + self.normalize = normalize + self.impute_missing = impute_missing + self.drop_missing = drop_missing + + def _create_model(self) -> NeuralProphet: + """ + Create a new NeuralProphet instance with validated parameters. + + Returns: + A new NeuralProphet model instance configured with current + parameters. + + Raises: + RuntimeError: If model creation fails + """ + try: + model_params = { + "n_lags": self.n_lags, + "n_forecasts": self.n_forecasts, + "weekly_seasonality": self.weekly_seasonality, + "daily_seasonality": self.daily_seasonality, + "yearly_seasonality": self.yearly_seasonality, + "seasonality_mode": self.seasonality_mode, + "loss_func": self.loss_func, + "normalize": self.normalize, + "impute_missing": self.impute_missing, + "drop_missing": self.drop_missing, + "impute_rolling": 1000000, + "impute_linear": 100000, + } + + # Add optional parameters if specified + if self.learning_rate is not None: + model_params["learning_rate"] = self.learning_rate + if self.batch_size is not None: + model_params["batch_size"] = self.batch_size + + console.log( + f"[blue]Creating NeuralProphet with params: " + f"{model_params}[/blue]" + ) + return NeuralProphet(**model_params) + + except Exception as e: + raise RuntimeError( + f"Failed to create NeuralProphet model: {e}" + ) from e + + def _validate_and_prepare_data( + self, y: pd.Series + ) -> Tuple[pd.DataFrame, pd.Series]: + """ + Validate and prepare time series data for NeuralProphet. + + Args: + y: Input time series data + + Returns: + Tuple of (prepared_dataframe, validated_series) + + Raises: + ValueError: If data validation fails + """ + try: + # Validate target series + y_array = self._validate_y(y) + + # Ensure datetime index + if not pd.api.types.is_datetime64_any_dtype(y.index): + try: + y_datetime = y.copy() + y_datetime.index = pd.to_datetime(y.index) + console.log("[yellow]Converted index to datetime[/yellow]") + except Exception as e: + raise ValueError( + "y's index must be a DateTime index or convertible " + f"to DateTime. Conversion failed: {e}" + ) from e + else: + y_datetime = y.copy() + + # Check for minimum data requirements + if len(y_datetime) < max(self.n_lags + 1, 10): + raise ValueError( + "Insufficient data: need at least " + + f"{max(self.n_lags + 1, 10)} observations, " + + f"got {len(y_datetime)}" + ) + + # Create NeuralProphet format DataFrame + df = pd.DataFrame({"ds": y_datetime.index, "y": y_array}) + + # Validate for missing values if not configured to handle them + if not self.impute_missing and bool(df["y"].isna().any()): + raise ValueError( + "Data contains missing values but impute_missing=False. " + "Either set impute_missing=True or clean the data." + ) + + console.log( + f"[green]Data validation successful: {len(df)} " + f"observations[/green]" + ) + return df, y_datetime + + except Exception as e: + console.print(f"[red]Data validation failed: {e}[/red]") + raise + + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """ + Core fitting logic for NeuralProphet model with enhanced error + handling. + + Args: + y: The target time series data with DateTime index + X: Optional DataFrame (unused for univariate model) + X_val: Validation features (unused for NeuralProphet) + y_val: Validation target (unused for NeuralProphet) + + Raises: + ValueError: If data validation fails + RuntimeError: If model fitting fails + """ + try: + console.log("[blue]Starting NeuralProphet model fitting...[/blue]") + + df, y_datetime = self._validate_and_prepare_data(y) + self.training_series_ = y_datetime + self.model_ = self._create_model() + + console.log( + f"[blue]Training model for {self.epochs} epochs...[/blue]" + ) + + fit_result = self.model_.fit(df, epochs=self.epochs) + + # Store training metrics if available + if hasattr(fit_result, "losses") and fit_result is not None: + losses = getattr(fit_result, "losses", None) + if losses: + self._training_metrics = { + "final_loss": float(losses[-1]), + "epochs_trained": len(losses), + } + + console.log( + f"[green]Model fitting completed successfully. " + f"Metrics: {self._training_metrics}[/green]" + ) + + except Exception as e: + console.print(f"[red]Model fitting failed: {e}[/red]") + # Reset model state on failure + self.model_ = None + self.training_series_ = None + raise RuntimeError( + f"Failed to fit NeuralProphet model: {e}" + ) from e + + def _prepare_prediction_data( + self, X: Optional[pd.DataFrame] = None + ) -> Tuple[pd.DataFrame, pd.Series]: + """ + Prepare data for prediction with comprehensive validation. + + Args: + X: Optional DataFrame containing prediction data + + Returns: + Tuple of (prepared_dataframe, prediction_index) + + Raises: + ValueError: If data preparation fails + """ + if self.training_series_ is None: + raise ValueError("No training series available") + + training_series = cast(pd.Series, self.training_series_) + + if X is None: + # Predict on training data + df = pd.DataFrame( + { + "ds": training_series.index, + "y": training_series.to_numpy(), + } + ) + return df, pd.Series(training_series.index) + + # Handle various input formats for X + try: + ds_values, y_values = self._extract_time_and_target_from_X(X) + + # Ensure datetime format + if not pd.api.types.is_datetime64_any_dtype(ds_values): + ds_values = pd.to_datetime(ds_values) + + df = pd.DataFrame( + { + "ds": ds_values, + "y": y_values, + } + ) + + return df, ds_values + + except Exception as e: + raise ValueError( + f"Failed to prepare prediction data: {e}" + ) from e + + def _extract_time_and_target_from_X( + self, X: pd.DataFrame + ) -> Tuple[pd.Series, pd.Series]: + """ + Extract time and target columns from input DataFrame. + + Args: + X: Input DataFrame + + Returns: + Tuple of (time_series, target_series) + + Raises: + ValueError: If extraction fails + """ + # Scenario 1: Explicit time and target columns + if self.time_col in X.columns and self.target_col in X.columns: + return ( + X[self.time_col], + self._validate_y(X[self.target_col]) + ) + + # Scenario 2: DateTime index + elif pd.api.types.is_datetime64_any_dtype(X.index): + ds_values = pd.Series(X.index, name=self.time_col) + + if self.target_col in X.columns: + # DateTime index with explicit target column + return ( + ds_values, + self._validate_y(X[self.target_col]) + ) + elif X.shape[1] == 1: + # DateTime index with single data column + return ds_values, self._validate_y( + X.iloc[:, 0].rename(self.target_col) + ) + elif X.shape[1] == 0: + # Only index, no columns - forecast scenario + y_values = pd.Series( + np.nan, index=X.index, name=self.target_col + ) + return ds_values, y_values + else: + raise ValueError( + "X has DateTime index but cannot identify target column. " + + f"Expected '{self.target_col}' or single column. " + + f"Found: {X.columns.tolist()}" + ) + else: + raise ValueError( + "Cannot determine time and target from X. " + + f"Provide columns '{self.time_col}' and '{self.target_col}' " + + "or use DateTime index." + ) + + @ensure_fitted + def predict(self, X: Optional[pd.DataFrame] = None) -> pd.Series: + """ + Generate in-sample predictions with enhanced error handling. + + Args: + X: Optional DataFrame containing timestamps and target values. + If None, predicts on training data. + + Returns: + Series of predictions indexed by timestamp + + Raises: + ValueError: If model is not fitted or prediction fails + RuntimeError: If prediction computation fails + """ + if self.model_ is None: + raise ValueError("Model is not fitted") + + try: + console.log("[blue]Generating predictions...[/blue]") + + # Prepare prediction data + df, predictions_index = self._prepare_prediction_data(X) + + # Get training context for lagged features + training_series = cast(pd.Series, self.training_series_) + past_values = pd.DataFrame( + { + "ds": training_series.index, + "y": training_series.to_numpy(), + } + ).iloc[-self.n_lags :, :] + + # Combine past and prediction data + combined_df = pd.concat([past_values, df], ignore_index=True) + combined_df = ( + combined_df.sort_values(by="ds") + .reset_index(drop=True) + .drop_duplicates(subset="ds", keep="last") + ) + + # Generate forecast + forecast = self.model_.predict(combined_df) + + # Handle different forecast column formats + forecast_col = f"yhat{self.n_forecasts}" + if forecast_col not in forecast.columns: + forecast = self.model_.get_last_forecast( + forecast, include_previous_forecasts=self.n_forecasts + ) + + # Extract predictions for requested indices + forecast = forecast.set_index("ds") + predictions = forecast.loc[predictions_index, forecast_col] + + console.log( + f"[green]Generated {len(predictions)} predictions[/green]" + ) + return predictions + + except Exception as e: + console.print(f"[red]Prediction failed: {e}[/red]") + raise RuntimeError(f"Failed to generate predictions: {e}") from e + + @ensure_fitted + def forecast(self, forecast_horizon: int) -> np.ndarray: + """ + Generate future forecasts with comprehensive validation. + + Args: + forecast_horizon: Number of steps to forecast ahead + (1 to n_forecasts) + + Returns: + Array of forecasted values, indexed by the forecast horizon + + Raises: + ValueError: If forecast_horizon is invalid or model not fitted + RuntimeError: If forecast generation fails + """ + if self.model_ is None: + raise ValueError("Model is not fitted") + + if not (1 <= forecast_horizon <= self.n_forecasts): + raise ValueError( + "forecast_horizon must be between 1 and " + + f"{self.n_forecasts}, got {forecast_horizon}" + ) + + try: + console.log( + f"[blue]Generating {forecast_horizon}-step forecast...[/blue]" + ) + + training_series = cast(pd.Series, self.training_series_) + + # Create future dataframe + future_df = self.model_.make_future_dataframe( + df=pd.DataFrame( + { + "ds": training_series.index, + "y": training_series.to_numpy(), + } + ), + periods=forecast_horizon, + ) + + # Generate forecasts + forecast = self.model_.predict(future_df) + + # Extract forecasted values for each horizon + forecasted_values = np.empty(forecast_horizon) + for i in range(forecast_horizon): + col_name = f"yhat{i + 1}" + if col_name in forecast.columns: + values = forecast[col_name].dropna() + if len(values) > 0: + forecasted_values[i] = values.iloc[0] + else: + forecasted_values[i] = np.nan + else: + forecasted_values[i] = np.nan + + console.log( + f"[green]Generated forecast: {len(forecasted_values)}" + + " values[/green]" + ) + return forecasted_values + + except Exception as e: + console.print(f"[red]Forecast generation failed: {e}[/red]") + raise RuntimeError(f"Failed to generate forecast: {e}") from e + + @ensure_fitted + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Perform comprehensive backtesting with enhanced monitoring. + + This method implements walk-forward validation with periodic + retraining, providing robust evaluation of model performance in + production-like scenarios. + + Args: + y: Target time series for backtesting (must have DateTime index) + X: Unused (included for base class compatibility) + retrain_every: Unused (model retrains at each step) + reuse_previous_execution: Whether to reuse previous backtest + results + + Returns: + Series of backtested predictions indexed by timestamp + + Raises: + ValueError: If parameters are invalid or data is insufficient + RuntimeError: If backtesting fails + """ + if self.model_ is None: + raise ValueError("Model is not fitted") + if self.training_series_ is None: + raise ValueError("No training series found") + + if not (1 <= self.forecast_horizon <= self.n_forecasts): + raise ValueError( + "forecast_horizon must be between 1 and " + + f"{self.n_forecasts}, got {self.forecast_horizon}" + ) + + # Handle reuse of previous execution + if reuse_previous_execution and self.backtest_predictions_ is not None: + expected_index = y.iloc[self.forecast_horizon:].index + if ( + len(self.backtest_predictions_) == len(expected_index) + and (self.backtest_predictions_.index == expected_index).all() + ): + console.log( + "[yellow]Reusing previous backtest results[/yellow]" + ) + return self.backtest_predictions_ + else: + console.log( + "[yellow]Previous results incompatible, running new" + + " backtest[/yellow]" + ) + + try: + console.log( + f"[blue]Starting backtest with {self.forecast_horizon}-step" + + " horizon...[/blue]" + ) + + # Validate and prepare data + y_sorted = y.sort_index() + + # Check for overlapping data + training_series = cast(pd.Series, self.training_series_) + if any(t in training_series.index for t in y_sorted.index): + console.print( + "[yellow]Warning: Backtest data overlaps with training" + + " data[/yellow]" + ) + + # Initialize backtesting + predictions = [] + training_base = pd.DataFrame( + { + "ds": training_series.index, + "y": training_series.values, + } + ) + + timestamps = y_sorted.index + total_steps = len(timestamps) - self.forecast_horizon + + console.log( + f"[blue]Running {total_steps} backtest steps...[/blue]" + ) + + # Perform walk-forward validation + for i in range(self.forecast_horizon, len(timestamps)): + if i % 50 == 0: # Progress logging + console.log( + f"[blue]Backtest progress: {i}/{len(timestamps)}[/blue]" + ) + + t = timestamps[i] + t_minus_h = timestamps[i - self.forecast_horizon] + + # Prepare training data up to t - h + history = y_sorted.loc[:t_minus_h] + train_df = ( + pd.concat( + [ + training_base, + pd.DataFrame( + {"ds": history.index, "y": history.values} + ), + ], + ignore_index=True, + ) + .drop_duplicates(subset="ds") + .sort_values("ds") + ) + + # Check minimum data requirement + if len(train_df) < max(self.n_lags + 1, 10): + console.print( + f"[yellow]Insufficient data at step {i}, " + + "skipping[/yellow]" + ) + predictions.append((t, np.nan)) + continue + + try: + # Retrain model + model = self._create_model() + model.fit(train_df, epochs=self.epochs) + + # Generate forecast + mask = train_df["ds"] <= t_minus_h + future_df = model.make_future_dataframe( + df=train_df.loc[mask], periods=self.forecast_horizon + ) + forecast = model.predict(future_df, decompose=False) + + # Extract prediction + forecast_col = f"yhat{self.forecast_horizon}" + prediction_rows = forecast[forecast["ds"] == t] + + if ( + len(prediction_rows) > 0 + and forecast_col in forecast.columns + ): + prediction = prediction_rows[forecast_col].iloc[0] + else: + prediction = np.nan + + predictions.append((t, prediction)) + + except Exception as step_error: + console.print( + f"[yellow]Error at step {i}: {step_error}[/yellow]" + ) + predictions.append((t, np.nan)) + + # Create results series + if predictions: + pred_index, pred_values = zip(*predictions) + self.backtest_predictions_ = pd.Series( + pred_values, + index=pd.Index(pred_index), + name=f"yhat{self.forecast_horizon}", + ) + else: + self.backtest_predictions_ = pd.Series( + dtype=float, name=f"yhat{self.forecast_horizon}" + ) + + console.log( + f"[green]Backtest completed: {len(self.backtest_predictions_)}" + + " predictions[/green]" + ) + return self.backtest_predictions_ + + except Exception as e: + console.print(f"[red]Backtest failed: {e}[/red]") + raise RuntimeError(f"Failed to perform backtest: {e}") from e + + def get_params_dict(self) -> Dict[str, Any]: + """Get comprehensive model parameters for logging/serialization.""" + base_params = super().get_params_dict() + neural_prophet_params = { + "n_forecasts": self.n_forecasts, + "weekly_seasonality": self.weekly_seasonality, + "daily_seasonality": self.daily_seasonality, + "yearly_seasonality": self.yearly_seasonality, + "seasonality_mode": self.seasonality_mode, + "epochs": self.epochs, + "learning_rate": self.learning_rate, + "batch_size": self.batch_size, + "loss_func": self.loss_func, + "normalize": self.normalize, + "impute_missing": self.impute_missing, + "drop_missing": self.drop_missing, + "training_metrics": self._training_metrics, + } + return {**base_params, **neural_prophet_params} + + def summary(self) -> str: + """Generate comprehensive model summary.""" + fitted_status = ( + "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" + ) + + seasonality_features = [] + if self.weekly_seasonality: + seasonality_features.append("Weekly") + if self.daily_seasonality: + seasonality_features.append("Daily") + if self.yearly_seasonality: + seasonality_features.append("Yearly") + + seasonality_str = ( + ", ".join(seasonality_features) if seasonality_features else "None" + ) + + summary_lines = [ + f"Model: {self.__class__.__name__}", + f"Status: {fitted_status}", + f"Lags: {self.n_lags}, Forecasts: {self.n_forecasts}", + f"Seasonality: {seasonality_str} ({self.seasonality_mode})", + f"Training: {self.epochs} epochs, {self.loss_func} loss", + f"Data Handling: Impute={self.impute_missing}, " + f"Drop={self.drop_missing}", + ] + + if self._training_metrics: + metrics_str = ", ".join( + f"{k}={v:.4f}" for k, v in self._training_metrics.items() + ) + summary_lines.append(f"Metrics: {metrics_str}") + + return "\n".join(summary_lines) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py new file mode 100644 index 0000000..5e5a7c0 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py @@ -0,0 +1,695 @@ +""" +Stacking implementation for time series forecasting. + +This module provides a stacking regressor implementation that combines multiple +time series models' predictions using a meta-model. The stacking approach +helps improve prediction accuracy by combining the strengths of different +base models through a learned meta-model. + +Key features: +- **Model Stacking**: Combines predictions from multiple base models +- **Time-Series Aware**: Uses proper time-based cross-validation +- **Meta-Model Learning**: Learns optimal combination weights +- **Comprehensive Error Handling**: Robust error handling and validation +- **Rich Logging**: Colored console output for better debugging + +The stacking model loads pre-trained base models and uses their predictions +as features for training a meta-model (CatBoost by default). +""" + +from typing import Optional, List, Dict, Any, Union +import pandas as pd +import numpy as np +from catboost import CatBoostRegressor, CatBoostClassifier, Pool +from rich.console import Console +import gzip +import pickle +import lzma + +from .base import ( + MultivariateTimeSeriesModel, + ensure_fitted, + TimeSeriesModel, +) + +console = Console() + + +class StackingTimeSeriesModel(MultivariateTimeSeriesModel): + """ + Stacking implementation for time series forecasting. + + This class implements stacking of multiple base models, using their + predictions as features for a meta-model. It handles time-based + cross-validation to generate out-of-fold predictions for training. + + The model supports both regression and classification tasks through + the meta-model configuration. + + Attributes: + base_models_: List of loaded base models + model_: The trained meta-model (CatBoost) + training_series_: Copy of training target data + base_predictions_train_: Base model predictions on training data + backtest_predictions_: Stored backtest predictions for reuse + + Example: + >>> stacking_model = StackingTimeSeriesModel( + ... base_model_paths=["model1.pkl", "model2.pkl"], + ... base_model_types=["catboost", "elasticnet"] + ... ) + >>> stacking_model.fit(y=target_series, X=feature_matrix) + >>> predictions = stacking_model.predict(X=test_features) + """ + + def __init__( + self, + base_model_paths: List[str], + base_model_types: List[str], + name: Optional[str] = None, + learning_task: str = "regression", + retrain_every: int = 100, + meta_iterations: int = 1000, + meta_learning_rate: float = 0.1, + meta_depth: int = 6, + early_stopping_rounds: Optional[int] = None, + meta_loss_function: Optional[str] = None, + time_col: str = "ds", + target_col: str = "y", + random_seed: int = 42, + verbose: bool = False, + differentiate_target: bool = False, + bins: Optional[List[float]] = None, + use_predict_for_training: bool = True, + ) -> None: + """ + Initialize the stacking model. + + Args: + base_model_paths: Paths to saved base models + base_model_types: Types of base models (must match order of paths) + name: Optional identifier for the model + learning_task: Type of learning task ('regression', 'binary', + 'multiclass') + retrain_every: Frequency of retraining during backtesting + meta_iterations: Number of iterations for meta-model + meta_learning_rate: Learning rate for meta-model + meta_depth: Tree depth for meta-model + early_stopping_rounds: Early stopping rounds for meta-model + meta_loss_function: Loss function for meta-model + time_col: Name of time column + target_col: Name of target column + random_seed: Random seed + verbose: Whether to print verbose logging + differentiate_target: Whether to differentiate the target series + bins: Bin edges for multiclass classification + use_predict_for_training: If True, use predict() instead of + backtest() for generating base model predictions during + training. This is much faster but may lead to overfitting + since the meta-model trains on in-sample predictions. + """ + super().__init__( + name=name, + time_col=time_col, + target_col=target_col, + random_seed=random_seed, + learning_task=learning_task, + differentiate_target=differentiate_target, + bins=bins, + ) + + # Validate inputs + if not base_model_paths: + raise ValueError("base_model_paths cannot be empty") + if not base_model_types: + raise ValueError("base_model_types cannot be empty") + if len(base_model_paths) != len(base_model_types): + raise ValueError( + "base_model_paths and base_model_types must have same length" + ) + + self.retrain_every = retrain_every + self.base_model_paths = base_model_paths + self.base_model_types = base_model_types + self.meta_iterations = meta_iterations + self.meta_learning_rate = meta_learning_rate + self.meta_depth = meta_depth + self.meta_loss_function = self._get_default_loss_function( + meta_loss_function + ) + self.early_stopping_rounds = early_stopping_rounds + self.verbose = verbose + self.use_predict_for_training = use_predict_for_training + + # Will be set during fit + self.base_models_: List[TimeSeriesModel] = [] + self.model_: Optional[ + Union[CatBoostRegressor, CatBoostClassifier] + ] = None + self.training_series_: Optional[pd.Series] = None + self.base_predictions_train_: Optional[pd.DataFrame] = None + self.backtest_predictions_: Optional[pd.Series] = None + + self._load_base_models() + + if self.verbose: + console.log( + "[green]Initialized StackingTimeSeriesModel: " + + f"{self.summary()}[/green]" + ) + + def _load_base_models(self) -> None: + """Load all base models from their saved paths.""" + from .factory import load_model + + self.base_models_ = [] + for model_path, model_type in zip( + self.base_model_paths, self.base_model_types + ): + try: + model = load_model(model_path, model_type) + self.base_models_.append(model) + if self.verbose: + console.log( + f"[blue]Loaded {model_type} model from " + + f"{model_path}[/blue]" + ) + except Exception as e: + console.print( + f"[red]Error loading model from {model_path}: {e}[/red]" + ) + raise ValueError(f"Failed to load model: {model_path}") from e + + if self.verbose: + console.log( + f"[green]Successfully loaded {len(self.base_models_)} " + + "base models[/green]" + ) + + def _create_meta_model( + self, + ) -> Union[CatBoostRegressor, CatBoostClassifier]: + """Creates a new instance of meta-model. + + Returns: + A new CatBoost model instance (Regressor or Classifier). + """ + base_params = { + "iterations": self.meta_iterations, + "learning_rate": self.meta_learning_rate, + "depth": self.meta_depth, + "loss_function": self.meta_loss_function, + "early_stopping_rounds": self.early_stopping_rounds, + "random_seed": self.random_seed, + "verbose": self.verbose, + } + + if self.learning_task in ["binary", "multiclass"]: + return CatBoostClassifier( + auto_class_weights="Balanced", **base_params + ) + else: + return CatBoostRegressor(**base_params) + + def _get_base_predictions( + self, X: Optional[pd.DataFrame], y: Optional[pd.Series] = None + ) -> pd.DataFrame: + """Get predictions from all base models. + + Args: + X: Feature matrix + y: Target series (optional, used for validation) + + Returns: + DataFrame with predictions in model_0, model_1, etc columns + """ + if X is None: + raise ValueError("Feature matrix X cannot be None") + + all_preds = [] + for i, model in enumerate(self.base_models_): + try: + preds = model.predict(X) + model_name = f"model_{model.name or i}" + all_preds.append( + pd.Series(preds, name=model_name, index=X.index) + ) + + if self.verbose: + console.log( + f"[cyan]Generated predictions from {model_name}[/cyan]" + ) + except Exception as e: + console.print( + f"[red]Error getting predictions from model {i}: {e}[/red]" + ) + raise + + return pd.concat(all_preds, axis=1) + + def _fit_logic( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + X_val: Optional[pd.DataFrame] = None, + y_val: Optional[pd.Series] = None, + ) -> None: + """Core fitting logic for stacking model. + + Get predictions from base models on training data, then train + meta-model on those predictions. + + Args: + y: The target time series data + X: The feature matrix (including exogenous features) + X_val: Validation feature matrix (optional) + y_val: Validation target series (optional) + """ + if X is None: + raise ValueError("Feature matrix X must be provided for stacking") + + # Preprocess data using parent class method + y_processed, X_processed, y_val_processed, X_val_processed = ( + self._preprocess_data(y, X, X_val, y_val) + ) + + if X_processed is None: + raise ValueError( + "Feature matrix X cannot be None after preprocessing" + ) + + if self.verbose: + method_name = ( + "predictions" + if self.use_predict_for_training + else "backtest predictions" + ) + console.log( + f"[blue]Generating base model {method_name} " + + "for stacking...[/blue]" + ) + + # Get base predictions - use either predict or backtest based on + # setting + if self.use_predict_for_training: + # Fast approach: use direct predictions (may overfit) + meta_features = self._get_base_predictions(X_processed) + # Align with target data + common_index = meta_features.index.intersection(y_processed.index) + meta_features = meta_features.loc[common_index] + y_aligned = y_processed.loc[common_index] + + if self.verbose: + console.log( + "[yellow]Warning: Using predict() for training may " + + "lead to overfitting since meta-model trains on " + + "in-sample predictions[/yellow]" + ) + else: + # Robust approach: use backtesting to avoid overfitting + all_preds = [] + for i, model in enumerate(self.base_models_): + try: + preds = model.backtest( + y_processed, + X_processed, + retrain_every=self.retrain_every, + ) + model_name = f"model_{model.name or i}" + all_preds.append(pd.Series(preds, name=model_name)) + + if self.verbose: + console.log( + "[cyan]Generated backtest predictions from " + + f"{model_name}[/cyan]" + ) + except Exception as e: + console.print( + f"[red]Error during backtesting for model {i}: " + + f"{e}[/red]" + ) + raise + + meta_features = pd.concat(all_preds, axis=1) + + # Align with target data (backtest might have different length) + common_index = meta_features.index.intersection(y_processed.index) + meta_features = meta_features.loc[common_index] + y_aligned = y_processed.loc[common_index] + + if self.verbose: + console.log( + f"[blue]Training meta-model with {len(meta_features)} " + + f"samples and {meta_features.shape[1]} base model " + + "features[/blue]" + ) + + meta_X, meta_y = self._validate_X_y(meta_features, y_aligned) + train_pool = Pool(data=meta_X, label=meta_y) + + # Prepare validation data if provided + eval_set = None + if X_val_processed is not None and y_val_processed is not None: + val_predictions = self._get_base_predictions( + X_val_processed, y_val_processed + ) + val_X, val_y = self._validate_X_y(val_predictions, y_val_processed) + eval_set = Pool(data=val_X, label=val_y) + + if self.verbose: + console.log( + "[blue]Using validation set with " + + f"{len(val_predictions)} samples[/blue]" + ) + + # Create and train meta-model + self.model_ = self._create_meta_model() + self.model_.fit(train_pool, eval_set=eval_set) + + # Store training data + self.training_series_ = y_processed.copy() + self.base_predictions_train_ = meta_features.copy() + + if self.verbose: + console.log( + "[green]Meta-model training completed successfully[/green]" + ) + + @ensure_fitted + def predict(self, X: pd.DataFrame) -> pd.Series: + """ + Generate predictions using the stacking model. + + Args: + X: Feature matrix for prediction + + Returns: + Series containing predictions + """ + if self.model_ is None: + raise ValueError("Model has not been fitted yet") + + base_predictions = self._get_base_predictions(X) + X_array = self._validate_X(base_predictions) + predictions = self.model_.predict(X_array) + + # Convert predictions to numpy array if needed + if hasattr(predictions, "squeeze"): + predictions = predictions.squeeze() + elif isinstance(predictions, list): + predictions = np.array(predictions) + + return pd.Series(predictions, index=X.index, name=self.target_col) + + @ensure_fitted + def feature_importance(self) -> Optional[pd.DataFrame]: + """ + Returns feature importance from the meta-model. + + Returns: + DataFrame with feature names and their importance scores, + or None if not available. + """ + if self.model_ is None or not hasattr( + self.model_, "feature_importances_" + ): + return None + + if self.base_predictions_train_ is None: + return None + + importances = self.model_.feature_importances_ + feature_names = self.base_predictions_train_.columns + + return pd.DataFrame( + { + "feature": feature_names, + "importance": importances, + } + ).sort_values("importance", ascending=False) + + @ensure_fitted + def backtest( + self, + y: pd.Series, + X: Optional[pd.DataFrame] = None, + retrain_every: int = 50, + reuse_previous_execution: bool = False, + ) -> pd.Series: + """ + Performs backtesting (walk-forward validation) with periodic + retraining. + + This method simulates a production scenario by iterating through a test + set, making a one-step-ahead prediction, and then retraining the model + periodically with the newly available data. + + Args: + y: Series with the true target values for the backtesting period + X: DataFrame with features for the backtesting period + retrain_every: The frequency of retraining. The model will be + retrained every `retrain_every` steps + reuse_previous_execution: Whether to reuse the previous execution + of a backtest. If True, any overlapping data between the + previous execution and the current execution will be used + without retraining the model + + Returns: + A series of backtested predictions, indexed by the backtest data's + index + """ + if self.model_ is None: + raise ValueError("Model is not fitted yet") + if self.training_series_ is None: + raise ValueError("Training series is not set") + if X is None: + raise ValueError("Feature matrix X must be provided") + + if reuse_previous_execution: + if self.backtest_predictions_ is None: + raise ValueError("No previous execution found") + if (self.backtest_predictions_.shape[0] != y.shape[0]) or ( + not (self.backtest_predictions_.index == y.index).all() + ): + raise ValueError( + "Previous execution index does not match y index" + ) + return self.backtest_predictions_ + + if self.verbose: + console.log( + f"[blue]Starting backtest with {len(y)} samples, " + + f"retraining every {retrain_every} steps[/blue]" + ) + + # Get base model predictions for the entire backtest period + all_base_preds = [] + for i, model in enumerate(self.base_models_): + try: + preds = model.backtest( + y, + X, + retrain_every=retrain_every, + reuse_previous_execution=reuse_previous_execution, + ) + model_name = f"model_{model.name or i}" + all_base_preds.append(pd.Series(preds, name=model_name)) + + if self.verbose: + console.log( + f"[cyan]Completed backtest for {model_name}[/cyan]" + ) + except Exception as e: + console.print( + f"[red]Error during backtest for model {i}: {e}[/red]" + ) + raise + + meta_features = pd.concat(all_base_preds, axis=1) + + # Generate meta-model predictions + predictions = self.model_.predict(self._validate_X(meta_features)) + + # Store backtest predictions for potential reuse + self.backtest_predictions_ = pd.Series( + predictions, + index=meta_features.index, + name=f"{self.target_col}_pred", + ) + + if self.verbose: + console.log( + "[green]Backtest completed: " + + f"{len(self.backtest_predictions_)} predictions " + + "generated[/green]" + ) + + return self.backtest_predictions_ + + def get_base_model_names(self) -> List[str]: + """Get names of all base models. + + Returns: + List of base model names + """ + return [ + model.name or f"model_{i}" + for i, model in enumerate(self.base_models_) + ] + + def get_params_dict(self) -> Dict[str, Any]: + """Get model parameters as dictionary for logging/serialization.""" + base_params = super().get_params_dict() + stacking_params = { + "base_model_paths": self.base_model_paths, + "base_model_types": self.base_model_types, + "retrain_every": self.retrain_every, + "meta_iterations": self.meta_iterations, + "meta_learning_rate": self.meta_learning_rate, + "meta_depth": self.meta_depth, + "meta_loss_function": self.meta_loss_function, + "early_stopping_rounds": self.early_stopping_rounds, + "num_base_models": len(self.base_models_), + "use_predict_for_training": self.use_predict_for_training, + } + return {**base_params, **stacking_params} + + def summary(self) -> str: + """Generate a summary string of the model.""" + params = self.get_params_dict() + fitted_status = ( + "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" + ) + + summary_lines = [ + f"Model: {self.__class__.__name__}", + f"Status: {fitted_status}", + f"Base Models: {params.get('num_base_models', 0)}", + f"Task: {params.get('learning_task', 'regression')}", + f"Meta Loss: {params.get('meta_loss_function', 'RMSE')}", + ] + + return "\n".join(summary_lines) + + def _optimize_base_models_for_storage(self) -> None: + """ + Optimizes base models for storage by removing unnecessary data. + This can significantly reduce pickle size, especially for neural + models. + """ + if self.verbose: + console.log("[blue]Optimizing base models for storage...[/blue]") + + for i, model in enumerate(self.base_models_): + try: + # For neuralprophet models, remove training history and + # large artifacts + model_attr = getattr(model, "model", None) + if model_attr is not None and hasattr(model_attr, "trainer"): + trainer = getattr(model_attr, "trainer", None) + if trainer is not None: + # Remove trainer which contains training logs and + # can be very large + if hasattr(trainer, "logged_metrics"): + setattr(trainer, "logged_metrics", {}) + if hasattr(trainer, "progress_bar_metrics"): + setattr(trainer, "progress_bar_metrics", {}) + if hasattr(trainer, "callback_metrics"): + setattr(trainer, "callback_metrics", {}) + + # For any model with training history + if hasattr(model, "training_history_"): + setattr(model, "training_history_", None) + if hasattr(model, "validation_history_"): + setattr(model, "validation_history_", None) + + # Remove cached predictions if they exist + if hasattr(model, "_cached_predictions"): + setattr(model, "_cached_predictions", None) + + if self.verbose: + model_name = getattr(model, "name", f"model_{i}") + console.log( + f"[cyan]Optimized {model_name} for storage[/cyan]" + ) + + except Exception as e: + if self.verbose: + console.log( + f"[yellow]Warning: Could not optimize model {i}: " + + f"{e}[/yellow]" + ) + + def save(self, path: str, compression: str = "gzip") -> None: + """ + Saves model to disk using compression to reduce file size. + + Args: + path: File path to save to + compression: Compression method ('gzip', 'lzma', or 'none') + - 'gzip': Fast compression, ~60-80% size reduction + - 'lzma': Better compression, ~70-90% size reduction, slower + - 'none': No compression + """ + if self.verbose: + console.log( + f"[blue]Saving stacking model with {compression} " + + f"compression to {path}[/blue]" + ) + + # Optimize base models for storage first + self._optimize_base_models_for_storage() + + if compression == "lzma": + # LZMA provides better compression but is slower + with lzma.open(path, "wb", preset=9) as f: + pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) + elif compression == "gzip": + # Gzip is faster with good compression + with gzip.open(path, "wb", compresslevel=9) as f: + pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) + else: + # No compression + with open(path, "wb") as f: + pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) + + if self.verbose: + console.log( + f"[green]Saved compressed stacking model to {path}[/green]" + ) + + @classmethod + def load(cls, path: str) -> "StackingTimeSeriesModel": + """ + Loads model from disk with automatic format detection. + Supports both compressed formats and legacy joblib format. + """ + import joblib + + # Try different formats in order of preference + loading_methods = [ + ("lzma", lambda p: lzma.open(p, "rb")), + ("gzip", lambda p: gzip.open(p, "rb")), + ("pickle", lambda p: open(p, "rb")), + ("joblib", None), # Special case for joblib + ] + + for format_name, open_func in loading_methods: + try: + if format_name == "joblib": + return joblib.load(path) + else: + with open_func(path) as f: + return pickle.load(f) + except ( + lzma.LZMAError, + gzip.BadGzipFile, + OSError, + pickle.UnpicklingError, + ValueError, + ): + continue + + raise ValueError( + f"Could not load model from {path} - unknown or corrupted format" + ) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml b/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml new file mode 100644 index 0000000..2f1d2cb --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml @@ -0,0 +1,11 @@ +channels: +- conda-forge +dependencies: +- python=3.10.16 +- pip<=25.0 +- pip: + - mlflow==2.7.1 + - pandas + - numpy + - scikit-learn +name: mlflow-env diff --git a/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml b/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml new file mode 100644 index 0000000..0a0396b --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml @@ -0,0 +1,7 @@ +python: 3.10.16 +build_dependencies: +- pip==25.0 +- setuptools==79.0.0 +- wheel==0.45.1 +dependencies: +- -r requirements.txt diff --git a/tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl b/tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..aa3f5aa8ef7c82c96b4bd1fda6e43cd695abf3f9 GIT binary patch literal 123 zcmZo*om#*E0X;IMC7C(Jdbv4iIr-&!1(j)~dCBqRMTrFksYS(8dW1rX67!1F@{4j) zi^3tIQzlQ*Y@AX%MWaWgq$n{nFEcMa9>{>Hn&Q_ZnwgiDT9lfXoQf(@nxqE+?KdyV literal 0 HcmV?d00001 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt b/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt new file mode 100644 index 0000000..fd9a283 --- /dev/null +++ b/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt @@ -0,0 +1,4 @@ +mlflow==2.7.1 +pandas +numpy +scikit-learn \ No newline at end of file diff --git a/tmp/artifacts/data_model/transformers/courier_transformers.pkl b/tmp/artifacts/data_model/transformers/courier_transformers.pkl new file mode 100644 index 0000000000000000000000000000000000000000..18fe2203362307ea711c91971c88e266be720d81 GIT binary patch literal 29607 zcmeHQS(6-BR<`!BWJ%U;*|MF3Ev%Vww`yZ)=rO)VP9 zg98SzEkgoBIOhR^2MiArLBs?E@WLBdj}ZI?f_?b~yuo+Q-EU@CT{AkGgBp4*zSjPm*tZFCo?w;pS#H+6?>kPP=+z&|asTUh)n^dt1>)J!~iO zu3mow*H)eGTj8xFicj_Wovp2Cqq^1YtVg|Gb-mLZv|vb_2a37!*z)Y!Sbidx%sp_qD@x+W zso3j>t*ssuCRN%x^W|kk8FkQjj%Vq-swi;H@brC5;8c$ z_c}SdtF29>_&Hm>-tI@;c8HX>!gjbBb&=^OE;c*s?=FaP@<|e(lor)y{d#X)nY%+BVb8u>*R47j7@;Uq)%jL?2u}hbxr_*>&;@swBoiCOX z(c)aHfa{53ZenbDYJM6-c`D{}s#xISq$&z&!74g^$?Zv^G=n;GIq)2!Ubi~Sl~h-k zr4ua97aeiI7sHzC5z26ax#lI;@-O%hgZ>7P)``_@@3t8$CWOc4?hJCAWbrw+efxHhC4KQWJ#~cC>F`&)LZZlNGkfs(y zvMOVotgb1HqKUjK?zcLm7n;i*5?B>gO~e9wsiQ4_QcA z^(yGh^DI*0PGNCkxMQf0JJj`sm;rkVV|_V=Nqr7bCz|=uHaf2gkP$N@n@p(wA?wWB z9OQ(pnKv4}Ao)xfY6^lu&R(sXXPBE1WVPvJN#45GdsPCzDQ>*CV9=s z(VWaBu~?%y5zwSwtmUkt3G!4$?ayrjp75a{2rv~|1O<__gG5OksF2H#tyqIm^_U~M ziHpm*eBu1H%kx-|xsyq(n1PjS%$jI1(OV_48xV0~)U!;L#Nr}LG;UxgOBWp3q$8VD zGHx1bTecG=P6Sx&)mN1;A?X@;r)8BO8&}(s$%R!kiP~ob-wMRW#D?9C2}?@KK<;B~ zkko5`Q~TT6XEX;=GtfYl-7YtYv=>B*X*RGl#4>+KXo!SUKm2Lx+3~YO()3TRU5}`d>Y9-Ef-=$s1o-ntqK1vJMIly#yw1zyd@}?)b7}(6YdIy75^h6E{@wi z6vyp)isN=YMOx~#M|qO%=jr$wCNYryyR#00)mj#5zhxDRu1h5hA4^)vC$;~ufjp9C z$qS-oRo04P&!viGj3P_90?+>0251jy$qTM4Mb{PD*jmaJ%+>#719{h#g6m4jb*1dK zQ>B#D{>>qCT`9P(lw4QJZh|Z2r1q~4nd=H|YOO#@t}A6X!Ier<`!8EP@48ZST`9S) zl-&eZCX(8}*klFQ6?7e2T5??}y9utiL&SBf;JSr{)ggA>D!a+9OeVGewAE=3YbC7c zx>a)BD!a+X`womi%evsYLeIvQwB)){cB7iWX!vhiz2Lf1bX_UAu80S3JyT2|suT23 zn#XKR+tY=7ApT)??$Zgl8_w#$1$*`9#$=≶CS=Q#Vtj$pt=DYgm zYw%$uS|4mh-SF-Y@BT?AiFZ_!#>E|vJvQ_b?^<4+O&XV>6Ypt#_LoGm<80EHA)_mF z;k#kQc<0=kN#k|4Zl0$Xa*bKXDBijCdeV4_wBnLDn7FVvg4OIuyh ztH=~h>_ipsv>+Lc9zPiiI-D zN{Tu3g{1K&dv`XVm(aqycqh7Iyqkge@v}j%(*#)$?`Ib=GBR;et|AuKpO9ig5;qve-W=9o^8BiDrx*8C2i#8%CG#7_>ARCxQP$UMA1gD zP;cU69CV=tbfExCNatqMz0F-n+y>ND4v;hq78V13tTZL`lp|BkdMmin?euz`UNF<_ z^zlT@*i5~xk)y)qa&HTjG@^Q+y4yikBW#*K8#%U&k6wCMy@Kgx*i+SIps4oDweUUC zoSF~Tq1>zA5c)x@NDJA?#rn;t8C+a;r9Koxk6l94r ziAy7hg$({>6KnAa**4R+r~y>fXc&pF-wM#3Ig}(muESGUK<^Y!%D9k3t;~WfeQb=C zJ}R{IAvk3!g={10HNzGqkv_Cji8W}D!xsINtQ%E(l57)f#I=cCMUaE!j2M)tn})5} zI0hHPX4t;_L$vNeDF}EV<1^0c*=jHk7pdn%hJlOW%B?O@)#7N?8~8|ZoKyk|3Kpj? z=@!2_6R?D>0yNqQIhY|J3yq&LHo(WFARnN;E_HhN#IqGlVdohJ(~C4Q9hEc+0u3>7PFc%zWeFsfCwqwjqhX5j-Sg)6MNQ*`dw_bPw9Hwv?#Bxhc}}(K8`hk zO|@7u-HgIE;Xi4*;WR#4Mt$|}`T&W?8cuB?=hGzq=Cufk6axj5*=XhGq#s6W++{mF zwGwnYgUwph4QAE?q|8Im79LW>gaNB{;d!tY;K7en6aX3wi6cZDSr~eE3PbljZ8@OR zM6RGW4wd0i^yy)C^eU5thD_#|<3oVMg^2Y}2+W=|Pvi z(vZZ@Lo64yAsR>9yc)TwPlNC%4Wa>#2y>NQZUd>PFwqRhWj$U?WWi{RGp<)Ag`z!1 ziP0p_j57++n$vtYrObAj4e4W{a8_R0B3ZD=Sd~g&7@>V_cPMYG^S_^SfYC>%M>f7 ziW3-{DICL+@T{6nIb+$bC6fs?SlO}XUhWf+%G8Rk;QCKflvw&$IiThz&umPthUgt9 zsDb$<(xlkC$*El9g`9(bUFIyGn|>>on+V1%sajE$?|Olu)eln%Ey5U|vfWcs;i$DZ z2DEUlE(a@@-=NW9Ew#2}fg4MbDhpJWpIlqu&&8fi%kVl*%8}|P@x>X6f~rz0zBFm^ z140*USp34EH-P_YBt{ezJTqMAG$*B@5(+0!$#F9Zm5-O-fPNNC!M>!oO4R3j5x%2^ zZ2`Uh%atou_(^R)>?+i|~cCmnmBMI5IDuN#>_kg9XTg)yvb|Wo>{u zrBWBO-1IPUiDx-atXdtNh#r>YLz~qdTmibnQ_E^6jCK|ebNm^F*|Jj(jY?rC=>$($Z@H# zvb0K=%%xVV9ng1<&caMQE01Q&*0cD?VelQoGi9V7DG`ASK*O{=cNE8Mh-oh+`caX( zjVlf!6`f%zrPNUC^Rm+h*pB5USJ!ep*-~>Tg40|NjZWka71^>tS>rK|t@Au5YZbU@ znX?17?1(edigyyv2#kh2rZK`WlJW(bU9bsKQ?wN0E0T`*d3RpOAb4>(>aIuaey~dK z;Ceh6WT*}If<9Xr;RT6DdqEo8@PgF7E^@(9;;}30b~Xn6j_q^OIJy&0OfSif9^$zf z!O15}oj{!xOj&3_nqoz=pO7WFxEzv2xw$rLvm}%+oy5~9ZGa@s+d*2c&>Zp!NJ59y zK7AVB5vDg71ehIC4@Mf$?ECR9WWL_6h4jE=3RruNPz}OdYah2>&FVgbO__d2%P(my z$w_HIZ)@qbi>@kLK8Gcfa;V=*v<8Vt+0zn&0FZd zGtqi2#0#QV0<`OM(@R%ZW-qUxXHPFJP6spdOKZW*?BdcY5PJUyc({Uijr{ybl@7K8 zBWs}-&!ZI(_h<#0J&Y5y;ovytXBU?7Fi78SbR)4fgo22~3O21mXOOUCBDcLSoLVU4^<=Nr(9LNZM3SyKP0gMvE`Dg6P z#VAMb8kfF1t~pZSx6-b7P7f4#e3nW9q4J1Rs4MePJ<1}|%o?Tl?2a;Au1(KoS z1(MeA0?ANt1IbW8AZpfv+h(!I?pzrfW8!1Td$2f@?2w8mntS*{J?vg8ZquMiFqTIw ze`$qO(`2Z4O_SE}nkGZRZJG>)VS!N7s5#e4hG9?&BLdMXLjzG7Lj%zY!vfI?2*jFt ztR3} zXl#qS_~<0S9_)tMx#Fdz-bntGZ-xiHsEd~?+cVFl&G~cJ5j9>&n}`~_;v?G;rr-8W z-K{?Y#fu*GX%V8dVo2hOZwz&-O8kxvdv*PZ!x(>v7QKhv`wOEC@fj3tdNb(n;hU^F ziOUQ4)ggbK3Lmz?i~x zZ%~B~?Gm5Ddjpy|=uHBq5qhaA-VZcBpo-l~eAIdmpr+Eshotyby~IyiA&K`GjgKgx zALk`LE?OnHh!stJ`q@hwcgV4fOLQ&^{UTU=M0WjEeD9Thg-m__W2*kgbcs*Pj!IvH z!PHLghU6-rG=7sDyYCA=pu*3)>*6~WOKp5r3tu7qEuwmGm-v{~z2LTdQ?^h3Vn1p8 zHr4m&@(^$$p2KUgP+-pne}%l^&>Cwab-%K58ntwZ|P$o{4QDE_a)xD z8s7cU2JF92_V>QT`!4f778bv!E%+|$!hA+Bl-{Uh-5E2MnxeTg0&~YwQ#5zRZ1mkW zELbH=&j{}S&hzU239>963aM9-SI2gX%(7kOFQVtUJG5x(Fe(Q7D&Bo*DgG)Z=PT&* z1OL8weD2j?G8lU`xJru>K78F6;DY@}4gcYBqho$(^uX(WM2H^&JW9f&Bz$SV1$orj z9(A@yo$b*mcr*$gje_(ji1)r7HJhF9BpISTSGrMzUw$We_)m+|2rHV4R@gRyw%E>1 zMn&2zLuR0?J8h|NMrw@|YKrttmR-F2Vsn5ue?OpCEasA(bfd66@~jF+V-J=$&889K zhKBy1-Ixf?T{=V~Gk0k%jciLvYwH_hc3*eeUf+y1*J=;y##EPP93@L@f0>B!%-r2q z{l<*%M%dnmrkPymZWQPCHEEhME<6~{FR}LMT-s$5&OLL#eNSCYGmSsIy9np)#$UTW zsNtpU+hBab8bQB<#D@KTmky+k_f~K4A=0=#71FM%SKSzFA?;y5!tv7A zqh>F;(;HlmkFR$lfo*Q$Y-@3(baDrkbURzsHjd9m;V6FY6s`9mzX9a|z93ZPmO5&k zpA1LD(b+gaR3EBbo$LR>1GP-%H3)c|@_ zq(%pQ!z-ghhkgsM_@G>zZRt=c&`>GFP-(=Y@&XiUWy>pt_*5{|RN`33g6ow6jaLdW zR2p%qa2T!=AgclNs7OucO6e%C^m)lPl>rr%K@^pRNUzip%6XTHYb@urSC*Re%0e`i zJgM2GQlRllA;v3>Xew?R97Sy9N;z*@&>)6N8)_;Mi_?P2`9dl{sCfYb(Nw}uk4k}l zfHsxVF;whlJ}W?ghDxhxuQZ~);s9|Jnv*L98Y+c&0g|Dn66DSplu5?fYlcdzd0xR# zQ;F)F-Y(OgGrip)zE?2RR9sylD^~zLuSm`HiV#gDyz;0NXs8ros5GLf*ee{I?4(_R zhDsrZN+X&|kXKwP1sW=a7%GiODmv?$4y|_rG@zm~h@!F(kIHF5rDj^tAU>68LDIpg ztp#V`7|`}gs>v2KIv*>$rKgT ztNfq~=l);s!XGxLOiN0uO?Go0r9D9uXqe%^>lKjK;xA{43$Q-R|0Vc zjy{vw3Q()*05zhiXxgDOF&(cIXuML0@k%2emFJaEGfS#ydisL|J{1r_CC?DWdBK)f z7AU>45J_bt9+lmKT1~g0h4@su1+|WyD+~0!f|^LXrQ-)^=Sr*TTv>=uC3AJmoc3It z(8+qzLD?@CVm1$=d|7lQGih0hKz%w=q&QtI4%DKnn50>XKz%w=^ymokyepd7*`fpL z(~+V_N03K{zfqvNzeNYsrz1s=jv$Y&Vw%;W1M1U}qDM!NM_0^j3ZOn6DSC7Sd34qc zWO=7{^)BSok*!BpF@w~y1?qc8ik^1_d35-L5*bYa)TbjwkB%UZuAJEvKz%w=^ymok z=qhGAZ3kOQ(UGD@N03KX${ZCyeeX!oqa(eG>;M@NuHS25dcJF-fOjubsQf;_rXW>Wz5y(2}B zjv$W?e=a6tQ~>qqNYSGs$fK)dt`$IiI#TrL2=eGknN0!Irz1s=jv$Y&oY@pWeL7O~ z=m_%YDw%5qP@j$zJvxFsI{bl~jHUqU(~+V_N03KX&TI;xJ{>7~bOd>HmCUsQs82_V z9vwj*T`99EfckW#=+P16(Umis0;o?%iXI(79-Z}8&>FKqeL7O~=m_%YN|{Xo)Tbjw zkB%UZuAJEvKz%w=^ymok=q57P3ZOn6DSC7Sd32@BrU2^Gk)lUOkVj{|>$RFfz3g=% zpN?!jx{1uS0;umDDSF-!1iIR(C_bd#nB!GC{%~W`_y@f3$7}Z!_4bXZi-V9N=TBSK zcH+2l|tstR1Qy(SMC{D9vE}{{VLXpJ4z1 literal 0 HcmV?d00001 From fc47c1bb3dd9a9f9c57ffbcb3c3a677bccda035b Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 12 Sep 2025 14:30:52 -0300 Subject: [PATCH 04/52] SIENTIAPDE-1214 SIENTIAPDE-1214: Update requirements.txt to clarify dependencies and improve project setup - Commented out the previous sientia-mlops-library dependency for better clarity. - Ensured that the requirements.txt reflects the current state of dependencies for easier management. --- ## Problemas no Courier:.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 ## Problemas no Courier:.md diff --git a/## Problemas no Courier:.md b/## Problemas no Courier:.md new file mode 100644 index 0000000..48083e8 --- /dev/null +++ b/## Problemas no Courier:.md @@ -0,0 +1,9 @@ +## Problemas no Courier: +1. Enviamos a coluna timestamp do index para fazer o transform, para poder sincronizar a predição com o pacote que gerou ela, visto que vários modelos podem retornar uma lista de predições em vários casos. No caso do Courier, está vindo um timestamp que começa em 0, estando dessincronizado com os dados que enviamos. Seria possível alterar o comportamento do modelo para retornar o mesmo index que enviamos? + + Segue uma output do transform de exemplo: +``` +'303-WIT-230_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '303-WIT-230_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '303-WIT-230_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '303-WIT-230_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '305-WIT-135_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-135_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-WIT-135_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-135_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-160_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-WIT-160_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-WIT-160_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-WIT-160_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-PIT-170_median': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-170_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-PIT-170_min': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-170_max': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-175_median': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-PIT-175_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-PIT-175_min': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-PIT-175_max': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-FIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-FIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-013_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '305-FIT-013_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-FIT-013_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '305-FIT-013_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '306-PIT-101_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-PIT-101_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-101_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-PIT-101_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-FIT-051_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-FIT-051_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-051_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-FIT-051_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-DIT-001_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-DIT-001_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-DIT-001_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-DIT-001_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-PIT-105_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-PIT-105_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-105_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-PIT-105_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-FIT-052_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-FIT-052_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-052_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-FIT-052_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-DIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-DIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-DIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-DIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-PIT-115_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-PIT-115_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-115_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-PIT-115_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-FIT-004_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-FIT-004_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-004_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-FIT-004_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-PIT-110_median': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-PIT-110_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-110_min': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-PIT-110_max': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-FIT-003_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-FIT-003_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-003_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-FIT-003_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-PIT-125_median': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-PIT-125_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-125_min': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-PIT-125_max': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-PIT-130_median': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-PIT-130_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-130_min': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-PIT-130_max': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-FIT-006_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '306-FIT-006_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-006_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '306-FIT-006_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '307-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-003_median': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIT-003_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-003_min': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIT-003_max': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIC-022_median': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '307-FIC-022_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIC-022_min': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '307-FIC-022_max': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '310-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '310-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '310-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '310-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '307-FIT-008_median': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-008_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-008_min': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-008_max': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-009_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '307-FIT-009_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-009_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '307-FIT-009_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '309-PIT-101_median': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-101_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-101_min': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-101_max': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-105_median': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-105_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-105_min': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-105_max': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-110_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-110_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-110_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-110_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-185_median': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-185_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-185_min': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-185_max': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-190_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-190_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-190_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-190_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-195_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-PIT-195_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-195_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-PIT-195_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-FIT-051_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-051_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-FIT-051_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-051_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-052_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-FIT-052_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-FIT-052_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-FIT-052_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-PIT-001_median': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-001_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-001_min': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-001_max': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '309-PIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '309-PIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '317AIT003.3_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, 'SiO2_conc': {Timestamp('1970-01-01 00:00:01.732971600'): 5.15}}} +``` + +2. No modelo do transform (data_model), o nome do método que faz o transform de fato é "transform", sendo que em nossos modelos, por padrão esse nome é "predict". Seria possível alterar o nome do método manta manter a compatibilidade e o padrão que já temos? From 45263c10e5949b1d70b01d900994891a4ff7c85c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 15 Sep 2025 14:47:24 -0300 Subject: [PATCH 05/52] SIENTIAPDE-1222 SIENTIAPDE-1214: Enhance MLFlow and tests with datetime index handling and logging improvements - Added a new method in MLFlow to detect and parse datetime indices in DataFrames, ensuring proper format and raising errors for invalid types. - Updated prediction workflows to utilize the new datetime index handling, improving data integrity during transformations. - Enhanced logging in model_repository to include detailed data outputs for better traceability. - Adjusted timeout settings in prediction workflows for improved execution time management. - Updated tests.ipynb to include additional checks for index types and outputs for better validation of functionality. --- data.csv | 11 + laborious/activities/mlflow.py | 75 +- laborious/workflows/predictions_batch.py | 2 +- .../sub_workflows/prediction_process.py | 4 +- response_data.csv | 2 + tests.ipynb | 54 ++ .../artifacts/training_transformer.pkl | Bin 0 -> 29391 bytes .../code/utils/models/base.py | 824 ------------------ 8 files changed, 143 insertions(+), 829 deletions(-) create mode 100644 data.csv create mode 100644 response_data.csv create mode 100644 tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl diff --git a/data.csv b/data.csv new file mode 100644 index 0000000..76acc92 --- /dev/null +++ b/data.csv @@ -0,0 +1,11 @@ +timestamp,07BP012/VEL_M1_PV,07BP013/VEL_M1_PV,07BP014/VEL_M1_PV,07FT001_COR_B,07FT001_COR_G,07FT001_COR_R,07FT001_TEXT,07FT007_COR_B,07FT007_COR_G,07FT007_COR_R,07FT007_TEXT,07FT012_COR_B,07FT012_COR_G,07FT012_COR_R,07FT012_TEXT,09BP023/VEL_PV,09BP024/VEL_PV,303-WIT-230,305-AIC-001_PV,305-AIC-002_PV,305-CALC-001,305-FIC-001_PV,305-FIC-003_PV,305-FIC-005_PV,305-FIC-006_PV,305-FIT-002,305-FIT-009,305-FIT-010,305-FIT-011,305-FIT-012,305-FIT-013,305-LIC-001_PV,305-LIC-002_PV,305-PIT-170,305-PIT-175,305-SIC-001_SP,305-SIC-002_SP,305-WIT-135,305-WIT-160,306-CALC-018,306-DIT-001,306-DIT-002,306-FIT-003,306-FIT-004,306-FIT-005,306-FIT-006,306-FIT-051,306-FIT-052,306-LIC-001_PV,306-LIC-002_PV,306-LIC-003_PV,306-PIT-101,306-PIT-105,306-PIT-110,306-PIT-115,306-PIT-125,306-PIT-130,307-CALC-001,307-CALC-002,307-FIC-001_PV,307-FIC-005_PV,307-FIC-006_PV,307-FIC-019_PV,307-FIC-022,307-FIC-101_PV,307-FIC-105_PV,307-FIC-110_PV,307-FIC-115_PV,307-FIC-120_PV,307-FIC-130_PV,307-FIC-135_PV,307-FIC-140_PV,307-FIC-145_PV,307-FIC-150_PV,307-FIC-155_PV,307-FIC-160_PV,307-FIT-003,307-FIT-005,307-FIT-008,307-FIT-009,307-LIC-003_PV,307-LIC-004_PV,307-LIC-101_PV,307-LIC-105_PV,307-LIC-110_PV,307-LIC-115_PV,307-LIC-120_PV,307-LIC-130_PV,307-LIC-135_PV,307-LIC-140_PV,307-LIC-145_PV,307-LIC-150_PV,307-LIC-155_PV,307-LIC-160_PV,307-SIC-006_OUT,307-SIC-007_OUT,309-FIC-013,309-FIC-014,309-FIT-051,309-FIT-052,309-LIC-001_PV,309-LIC-002_PV,309-PIT-001,309-PIT-002,309-PIT-101,309-PIT-105,309-PIT-110,309-PIT-185,309-PIT-190,309-PIT-195,310-AIC-001_PV,310-AIC-002,310-CALC-001,310-CALC-002,310-DIC-002_PV,310-FIC-004_PV,310-FIC-010_PV,310-FIC-110_PV,310-FIC-120_PV,310-FIC-160_PV,310-FIC-170_PV,310-FIT-004,310-FIT-005,310-FIT-006,310-FIT-010,310-FV-032,310-LIC-110_PV,310-LIC-120_PV,310-LIC-160_PV,310-LIC-170_PV,310-LIT-003_PV,310-LIT-004_PV,310-SIC-003_OUT,310-SIC-004_OUT,310-SIC-005_OUT,310-SIC-006_OUT,311-FIC-029_PV,311-FIC-033_PV,311-FIT-033,312-CALC-001,312-CALC-005,312-CALC-006,312-DIC-001_PV,312-DIC-002_PV,312-FIC-001_PV,312-FIC-002_PV,313-CALC-001,313-DIC-001_PV,313-DIC-002_SP,313-FIC-006_PV,317AIT001.2,317AIT002.1,317AIT002.10,317AIT002.11,317AIT002.12,317AIT002.13,317AIT002.14,317AIT002.15,317AIT002.16,317AIT002.17,317AIT002.18,317AIT002.19,317AIT002.2,317AIT002.20,317AIT002.21,317AIT002.22,317AIT002.23,317AIT002.24,317AIT002.25,317AIT002.26,317AIT002.27,317AIT002.28,317AIT002.29,317AIT002.3,317AIT002.30,317AIT002.31,317AIT002.32,317AIT002.33,317AIT002.34,317AIT002.35,317AIT002.36,317AIT002.37,317AIT002.38,317AIT002.39,317AIT002.4,317AIT002.40,317AIT002.41,317AIT002.42,317AIT002.43,317AIT002.44,317AIT002.45,317AIT002.46,317AIT002.47,317AIT002.48,317AIT002.49,317AIT002.5,317AIT002.50,317AIT002.51,317AIT002.52,317AIT002.53,317AIT002.54,317AIT002.55,317AIT002.56,317AIT002.57,317AIT002.58,317AIT002.59,317AIT002.6,317AIT002.60,317AIT002.61,317AIT002.62,317AIT002.63,317AIT002.64,317AIT002.65,317AIT002.66,317AIT002.67,317AIT002.68,317AIT002.69,317AIT002.7,317AIT002.70,317AIT002.71,317AIT002.72,317AIT002.8,317AIT002.9,317AIT003.1,317AIT003.2,317AIT003.3,317AIT003.5,319-CALC-001,319-CALC-013,319-DIC-001_PV,319-DIC-001_SP,319-DIC-002_PV,319-DIC-002_SP,319-FIC-006_PV,319-FIC-006_SP,319-FIC-007_PV,319-FIC-007_SP,319-FIQ-CALC-009_DAY,319-FIT-004,319-FIT-005,319-LIT-201-R,319-PIT-003,319-PIT-004,319-SIC-001_OUT,319-SIC-002_OUT,Fe_conc,G03-07BP102_M1,G03-07BP103_M1,G03-08BP107_M1,G03-10BP104_M1,G03-19BP101_M1,G03-19BP106_M1,G03-19BP110_M1,SOL-CALC-005,SOL-CALC-006,SiO2_conc +2024-12-05 02:16:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2185.6279296875,8.909636497497559,8.824532508850098,2628.087646484375,233.4102020263672,289.2882995605469,2.2411208152771,2.740176200866699,3158.830322265625,687.0074462890625,0.0,287.7219543457031,0.0,3312.36962890625,89.0501937866211,83.04390716552734,12.823511123657228,13.373872756958008,,,1364.865478515625,1239.42919921875,3.005729913711548,1.3840404748916626,1.3935494422912598,2170.908447265625,2189.05615234375,1866.9991455078125,1715.1016845703125,2254.970703125,2276.34033203125,83.4413070678711,94.47754669189452,89.85079956054688,34.401798248291016,33.98549270629883,31.97739601135254,34.75410461425781,43.70286178588867,44.13051223754883,1894.9356689453125,2017.797607421875,900.0048217773438,30.6091537475586,375.9732971191406,0.0,652.0335693359375,765.3798217773438,763.6800537109375,769.5640258789062,739.6097412109375,719.44775390625,706.0026245117188,703.8912353515625,567.9400634765625,593.1154174804688,629.06787109375,595.2117919921875,625.0,1.425487995147705,27.0,1194.873046875,998.1407470703124,105.17456817626952,99.1063003540039,28.07830810546875,21.692842483520508,34.955604553222656,12.697091102600098,40.85791778564453,21.66144752502441,17.217727661132812,47.85460662841797,57.36534118652344,59.91471481323242,48.2706184387207,39.44066619873047,83.20423126220703,66.5,0.0,0.0,1768.509765625,1827.390380859375,85.6709213256836,96.77904510498048,0.0998583808541297,4.900833606719971,23.81103706359864,26.791275024414062,26.24358367919922,23.354522705078125,27.844594955444336,27.89141845703125,10.390382766723633,10.471061706542969,223.273666381836,209.89990234375,1.1763592958450315,2.309485912322998,103.65160369873048,450.9481201171875,427.9828186035156,450.59222412109375,448.7367858886719,17.0,700.9956665039062,0.0,1200.0,1.0,39.16987991333008,37.460086822509766,19.4918155670166,11.98000144958496,95.19344329833984,86.53215789794922,,,86.0,99.30213165283205,22.59608268737793,0.0,400.0,1351.7703857421875,0.0,1459.112548828125,2.163416624069214,1.6798019409179688,997.8304443359376,0.0240168757736682,203.056381225586,1.369994044303894,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.4978864192962646,8.562295913696289,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.253314971923828,13.389976501464844,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.01197052001953,1.7381054162979126,1.1206940412521362,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,59.35650634765625,1.4764769077301023,1.526507019996643,0.4823205173015594,0.7710468769073486,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3361487984657287,0.428571492433548,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,7.89943265914917,5.800000190734863,6.465214729309082,86.5999984741211,830.4485473632812,849.3055419921875,,,1.5839204788208008,1.572644829750061,0.0,600.0,920.1913452148438,921.7091064453124,0.0,10.023720741271973,0.0,11.650277137756348,4.348147869110107,0.0748394280672073,71.11778259277344,100.0,64.71,0.0,4.533299922943115,5.666272640228272,8.773231506347656,4.993200302124023,4.861800193786621,6.432722568511963,50.96606063842773,67.95718383789062,3.52 +2024-12-05 02:18:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2300.107421875,8.874933242797852,8.814615249633789,2379.468017578125,205.70236206054688,299.0367736816406,2.237759590148926,2.737307071685791,3149.414306640625,705.9229125976562,0.0,287.82403564453125,0.0,3319.203857421875,88.97682189941406,82.33074951171875,13.035848617553713,13.303841590881348,,,801.2099609375,1577.420166015625,3.18897008895874,1.3844648599624634,1.3944939374923706,2137.277099609375,2091.429931640625,1866.9801025390625,1714.9752197265625,2248.451904296875,2272.76025390625,85.66764831542969,88.35608673095703,87.84507751464844,34.351341247558594,33.97214126586914,31.98730659484864,32.71706008911133,43.68099594116211,44.0351676940918,1898.191162109375,1618.9393310546875,898.7774047851562,30.54461669921875,374.732666015625,0.0,648.9142456054688,761.4437866210938,761.818115234375,728.4386596679688,741.0633544921875,659.6345825195312,707.1867065429688,684.1427612304688,568.2828369140625,594.7976684570312,633.2908325195312,595.1723022460938,640.0,1.245144605636597,27.0,1194.7637939453125,998.0435791015624,103.9820556640625,99.07530975341795,28.694652557373047,22.18309211730957,31.824302673339844,12.703847885131836,38.318546295166016,21.9743595123291,16.507755279541016,47.80324935913086,57.33625793457031,59.91497421264648,48.0490837097168,40.25405502319336,83.34803771972656,66.5,0.0,0.0,1746.5220947265625,1811.0615234375,85.64175415039062,96.97100067138672,0.0999280512332916,4.900284290313721,23.82648658752441,26.91034507751465,26.227479934692383,23.32929229736328,27.45798110961914,27.736440658569336,10.399251937866213,10.47207736968994,175.54986572265625,211.37567138671875,1.174445867538452,2.112058639526367,103.86756896972656,450.9291076660156,427.931396484375,450.5900268554688,448.73138427734375,17.0,700.9699096679688,10.513471603393556,1200.0,1.0,40.18443298339844,37.6434326171875,19.451520919799805,11.894670486450195,95.58326721191406,86.0557632446289,,,86.0,98.96598815917967,22.82772254943848,0.0,400.0,968.2177734375,0.0,1464.4488525390625,2.163362979888916,1.679788589477539,1003.175537109375,0.0238033775240182,184.37130737304688,1.3698077201843262,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,8.57795238494873,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,13.411721229553224,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1217609643936155,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,59.35650634765625,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7702381610870361,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4283382296562195,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,8.100000381469727,5.800000190734863,6.435592651367188,86.5999984741211,725.95458984375,858.4331665039062,,,1.5835398435592651,1.5726622343063354,0.0,600.0,917.3837280273438,925.9874877929688,0.0,10.049739837646484,0.0,11.572147369384766,4.3472161293029785,0.0748393461108207,71.409423828125,100.0,64.71,0.0,4.533299922943115,5.762217998504639,8.699999809265137,4.993200302124023,4.861800193786621,6.558172702789307,50.968997955322266,67.96240234375,3.52 +2024-12-05 02:20:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2063.14990234375,8.86483097076416,8.804698944091797,2637.2822265625,186.7984161376953,274.3067932128906,2.234398603439331,2.734437942504883,3139.637939453125,738.9068603515625,0.0,287.9261474609375,0.0,3328.472412109375,84.00076293945312,86.40196990966797,12.14862060546875,13.639047622680664,,,1477.9354248046875,1141.748291015625,2.891580104827881,1.3848892450332642,1.3954384326934814,2059.924560546875,2129.599853515625,1866.961181640625,1714.848876953125,2241.93310546875,2266.62158203125,85.53884887695312,87.77095031738281,89.44815063476562,34.300880432128906,33.95878982543945,31.997217178344727,30.499488830566406,43.65913391113281,44.02417755126953,1907.386474609375,1944.6339111328125,903.0270385742188,30.480079650878903,374.2165832519531,0.0,649.119384765625,757.221435546875,759.9561767578125,811.8240356445312,748.2200927734375,740.3225708007812,708.370849609375,673.0714721679688,573.4213256835938,596.4798583984375,631.8908081054688,595.1328735351562,635.0,1.0720911026000977,27.0,1194.6546630859375,997.9464111328124,105.5190887451172,99.0443115234375,28.1169376373291,21.61070251464844,32.5518684387207,12.71060562133789,37.97523498535156,22.30083274841309,17.719982147216797,47.75189208984375,57.30717849731445,59.91522979736328,47.62279891967773,41.22840881347656,83.33912658691406,66.5,0.0,0.0,1749.242431640625,1821.6829833984373,85.61257934570312,97.1629638671875,0.0999977141618728,4.899734973907471,23.841936111450195,26.64358901977539,26.1314697265625,23.415620803833008,27.687856674194336,27.292091369628903,10.39584255218506,10.473093032836914,208.32379150390625,211.2283935546875,1.1736302375793457,1.0507607460021973,104.0835418701172,450.91009521484375,427.87994384765625,450.58782958984375,448.7259826660156,17.0,700.9442138671875,0.0,1200.0,1.0,40.472747802734375,38.068607330322266,19.54105758666992,12.325950622558594,95.0851821899414,85.76668548583984,,,86.0,98.85044860839844,22.84588432312012,0.0,400.0,1176.9923095703125,0.0,1459.0894775390625,2.1633095741271973,1.6797752380371094,997.1138916015624,0.0235898792743682,204.41549682617188,1.3696213960647583,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,8.100000381469727,5.400000095367432,6.405970096588135,86.5999984741211,802.98876953125,856.4476318359375,,,1.583159327507019,1.57267963886261,0.0,600.0,935.9249267578124,930.265869140625,0.0,10.131684303283691,0.0,11.596683502197266,4.34628438949585,0.0748392716050148,71.35333251953125,100.0,64.71,0.0,4.533299922943115,5.731375694274902,8.699999809265137,4.993200302124023,4.861800193786621,6.488824367523193,50.97193908691406,67.96761322021484,3.52 +2024-12-05 02:22:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2344.354736328125,8.854728698730469,8.794782638549805,2443.620849609375,235.85362243652344,289.5343017578125,2.231037378311157,2.7315685749053955,3128.458251953125,678.6727905273438,0.0,288.0282287597656,0.0,3352.32080078125,82.41395568847656,88.41472625732422,12.335658073425291,13.590455055236816,,,1179.0438232421875,1241.806396484375,2.9580600261688232,1.3853135108947754,1.3963829278945925,2020.5482177734373,2175.6435546875,1866.942138671875,1714.722412109375,2241.09228515625,2263.143798828125,85.41004943847656,89.18098449707031,91.12660217285156,34.372066497802734,33.94544219970703,32.00712585449219,30.11072158813477,43.63727188110352,44.04359436035156,1842.406005859375,1904.2274169921875,902.5571899414062,30.415542602539062,374.7893676757813,0.0,651.5109252929688,768.8901977539062,758.0942993164062,720.6381225585938,742.55078125,685.2013549804688,709.5549926757812,730.93359375,572.37548828125,598.1620483398438,634.2003173828125,595.0933837890625,632.0,0.9020777940750122,27.0,1194.54541015625,997.8492431640624,104.99007415771484,99.01332092285156,28.318359375,21.52416229248047,31.58490943908692,12.717362403869627,38.4969482421875,21.7913761138916,17.436378479003906,47.70053482055664,57.278099060058594,59.915489196777344,46.977535247802734,42.46583938598633,83.07581329345703,66.5,0.0,0.0,1748.264892578125,1827.9678955078125,85.58340454101562,97.35491943359376,0.1000673845410347,4.899185180664063,23.85738754272461,26.20622062683105,26.06732177734375,23.50194931030273,27.590068817138672,27.46240234375,10.392244338989258,10.47410774230957,214.98492431640625,205.6874847412109,1.173506498336792,2.398390531539917,104.2995147705078,450.8910827636719,427.8284912109375,450.5856323242188,448.7205810546875,17.0,700.9185180664062,0.0,1200.0,1.0,40.27254486083984,38.66741561889648,19.67565155029297,12.237468719482422,93.86233520507812,85.74674987792969,,,86.0,98.91876983642578,22.30738639831543,0.0,400.0,1275.9793701171875,0.0,1467.3343505859375,2.1632559299468994,1.6797618865966797,998.2872314453124,0.0233763810247182,187.3364105224609,1.3694350719451904,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.708251953125,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.3545424044132232,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3020144402980804,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.100000381469727,5.400000095367432,6.699999809265137,85.80000305175781,747.72314453125,830.0194091796875,,,1.5827786922454834,1.5726970434188845,0.0,600.0,924.1300048828124,934.5442504882812,0.0,10.00216579437256,0.0,11.625198364257812,4.345353126525879,0.0748391896486282,71.52710723876953,100.0,64.71,0.0,4.533299922943115,5.767271518707275,8.699999809265137,4.993200302124023,4.861800193786621,6.6877641677856445,50.97750854492188,67.97283172607422,3.52 +2024-12-05 02:24:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2862.970703125,8.844626426696777,8.774948120117188,2980.736572265625,237.5259704589844,291.2091369628906,2.2276761531829834,2.728699445724488,3153.987548828125,682.5768432617188,0.0,288.1303405761719,0.0,3345.968505859375,87.24137878417969,86.74526977539062,12.552753448486328,13.396173477172852,,,1370.993896484375,1611.220947265625,2.8518500328063965,1.3857378959655762,1.397327542304993,2030.3245849609373,2142.251953125,1866.9232177734373,1714.6802978515625,2248.769775390625,2268.414794921875,85.06763458251953,91.27864837646484,90.73202514648438,34.44520568847656,33.932090759277344,32.01703643798828,31.01426696777344,43.61540985107422,44.06300735473633,1867.2406005859373,2242.3193359375,898.3688354492188,30.811418533325195,375.3621520996094,0.0,647.7444458007812,756.3349609375,759.6863403320312,795.988525390625,743.1278076171875,681.9320678710938,710.7391357421875,760.3604736328125,576.6845092773438,599.8442993164062,620.6749267578125,595.053955078125,640.0,0.7632204294204712,27.0,1194.4361572265625,997.7520751953124,106.02984619140624,98.98233032226562,28.794273376464844,21.658815383911133,32.04315185546875,12.724120140075684,38.84426498413086,21.82027244567871,16.058849334716797,47.64917755126953,57.24901580810547,59.915748596191406,46.44183349609375,41.52559280395508,83.30157470703125,66.5,0.0,0.0,1750.5181884765625,1845.0968017578125,85.55422973632812,97.546875,0.1001370549201965,4.8986358642578125,23.87283706665039,26.4957332611084,26.47050094604492,23.5178451538086,27.697071075439453,27.838388442993164,10.388647079467772,10.475123405456545,243.6897125244141,207.8060302734375,1.1728342771530151,1.2688955068588257,104.5154800415039,450.8720397949219,427.7770690917969,450.58343505859375,448.7151794433594,17.0,700.892822265625,0.0,1200.0,1.0,40.07234191894531,37.74433898925781,19.810243606567383,12.158288955688477,91.9724578857422,85.08606719970703,,,86.0,98.60803985595705,22.23607635498047,0.0,400.0,1401.1199951171875,0.0,1445.149658203125,2.1632025241851807,1.67974853515625,1001.5813598632812,0.0231628827750682,230.4506072998047,1.3692487478256226,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.899999618530273,5.400000095367432,6.699999809265137,85.80000305175781,927.35986328125,843.9348754882812,,,1.5823981761932373,1.5727144479751587,0.0,600.0,943.0855712890624,939.0200805664062,0.0,10.032031059265137,0.0,11.4324369430542,4.34442138671875,0.0748391151428222,71.41386413574219,100.0,64.71,0.0,4.533299922943115,5.681782245635986,8.699999809265137,4.993200302124023,4.861800193786621,6.551279544830322,50.98702239990234,67.97804260253906,3.52 +2024-12-05 02:26:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2654.10888671875,8.834524154663086,8.75146198272705,2392.275390625,235.610580444336,295.8012084960937,2.2243149280548096,2.725830078125,3172.66748046875,694.5460205078125,0.0,288.2324523925781,0.0,3339.616455078125,89.53855895996094,85.1905746459961,12.741495132446287,13.875475883483888,,,1323.8238525390625,1067.496337890625,2.916759967803955,1.386162281036377,1.3982720375061035,2102.00048828125,2161.18212890625,1866.9041748046875,1714.821044921875,2259.66064453125,2275.4189453125,84.60967254638672,90.81041717529295,90.06828308105469,34.51834487915039,33.918739318847656,32.026947021484375,32.243064880371094,43.593544006347656,44.08242416381836,1909.4129638671875,1879.74658203125,902.9876708984376,31.21604347229004,375.9349365234375,0.0,650.2518310546875,751.2490234375,761.6719970703125,715.2727661132812,747.89599609375,747.788818359375,706.99658203125,774.9991455078125,583.5380249023438,601.5264892578125,614.032958984375,595.0144653320312,615.0,0.7071666121482849,27.0,1194.326904296875,997.6549682617188,104.1503677368164,98.95133209228516,28.68854713439941,21.7934684753418,32.87420654296875,12.730876922607422,38.9033317565918,22.107240676879883,17.046525955200195,47.59782028198242,57.21993637084961,59.9160041809082,47.52975463867188,40.58535003662109,83.56719970703125,66.5,0.0,0.0,1742.1806640625,1809.867919921875,85.52505493164062,96.9273681640625,0.1002067178487777,4.8980865478515625,23.88828659057617,26.306615829467773,26.495014190673828,23.27698135375977,27.35708808898925,27.72958755493164,10.385048866271973,10.4761381149292,216.2808380126953,213.6409606933593,1.1716774702072144,2.5998244285583496,104.73145294189452,450.85302734375,427.793212890625,450.5812377929688,448.7097778320313,17.0,700.8671264648438,0.0148877017199993,1200.0,1.0,39.87213897705078,37.95595169067383,19.94483757019043,12.006688117980955,91.47502899169922,85.28714752197266,,,86.0,98.45233917236328,22.66000938415528,0.0,400.0,1256.264892578125,0.0,1452.758544921875,2.163148880004883,1.6797351837158203,1000.3143920898438,0.0229493845254182,190.5460968017578,1.369062423706055,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.899999618530273,3.299999952316284,6.699999809265137,85.80000305175781,762.7355346679688,865.4536743164062,,,1.5816150903701782,1.572731852531433,0.0,600.0,938.32177734375,943.602783203125,0.0,10.15860080718994,0.0,11.596125602722168,4.343489646911621,0.0748390331864357,71.58973693847656,100.0,64.71,0.0,4.533299922943115,5.802553176879883,8.699999809265137,4.993200302124023,4.861800193786621,6.558778762817383,50.99653625488281,67.98326110839844,3.52 +2024-12-05 02:28:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2227.77099609375,8.836018562316895,8.75161361694336,2456.804931640625,238.3328552246093,283.34686279296875,2.220953941345215,2.722960948944092,3188.196533203125,818.465087890625,0.0,288.33453369140625,0.0,3333.26416015625,90.52090454101562,83.3249282836914,12.903297424316406,13.22053337097168,,,1122.2493896484375,1331.5670166015625,3.0864999294281006,1.3865865468978882,1.3992165327072144,2095.321533203125,2129.77685546875,1866.88525390625,1714.9619140625,2259.91357421875,2288.202880859375,84.55709075927734,89.37647247314453,88.79876708984375,34.55312728881836,33.905391693115234,32.03685760498047,31.44712448120117,43.62167739868164,44.101837158203125,2035.30908203125,1930.71044921875,899.136474609375,32.6287956237793,376.5077209472656,0.0,653.2977294921875,750.5552368164062,760.983642578125,770.3090209960938,738.3396606445312,653.3170776367188,701.0360107421875,730.8182983398438,566.891845703125,588.6995239257812,629.2593383789062,594.9750366210938,622.0,0.6648255586624146,27.0,1194.2176513671875,997.5578002929688,105.18731689453124,98.92034149169922,28.36734771728516,21.92812156677246,32.240821838378906,12.737634658813477,38.52233505249024,21.57743263244629,17.07017707824707,47.54646301269531,57.19085311889648,59.916263580322266,48.34513473510742,39.64510345458984,83.13888549804688,66.5,0.0,0.0,1753.1981201171875,1814.801513671875,85.49588012695312,96.34169006347656,0.1002763882279396,4.8975372314453125,23.903738021850582,26.488170623779297,26.35142517089844,23.256576538085938,27.712129592895508,27.556394577026367,10.381451606750488,10.477153778076172,218.41183471679688,225.77491760253903,1.1706732511520386,1.818994283676148,104.94741821289062,450.8340148925781,428.7974853515625,450.57904052734375,448.7043762207031,17.0,700.8414306640625,0.5039713978767395,1200.0,1.0,39.67193984985352,38.82048416137695,20.07943153381348,12.295125961303713,90.39128875732422,85.13963317871094,,,86.0,98.48783111572266,24.05027961730957,0.0,400.0,1260.049560546875,0.0,1470.87890625,2.163095474243164,1.6797218322753906,1004.2244873046876,0.0227358862757682,194.2509765625,1.3688760995864868,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.86962890625,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,7.830216407775879,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042373403906822,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.7188185453414917,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7221007943153381,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.49777603149414,65.73210906982422,8.899999618530273,3.299999952316284,6.5,86.30000305175781,790.286376953125,913.74462890625,,,1.5807862281799316,1.572749376296997,0.0,600.0,945.046875,945.7421264648438,0.0,10.257745742797852,0.0,11.62919807434082,4.342557907104492,0.0748389586806297,71.7236099243164,100.0,64.71,0.0,4.533299922943115,5.762265205383301,8.699999809265137,4.993200302124023,4.861800193786621,6.415340900421143,51.00605010986328,67.98847961425781,3.52 +2024-12-05 02:30:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2773.8876953125,8.840205192565918,8.76478099822998,2499.578125,233.58840942382807,273.2201843261719,2.217592716217041,2.720091819763184,3169.754638671875,718.6017456054688,0.0,288.4366455078125,0.0,3293.60546875,88.42021942138672,80.45106506347656,12.661704063415527,13.48155403137207,,,1310.0872802734375,1188.9476318359375,3.5530900955200195,1.387010931968689,1.4001611471176147,2047.4097900390625,2157.358642578125,1866.8662109375,1715.1026611328125,2255.59716796875,2283.900390625,84.08345794677734,86.43714904785156,89.38142395019531,34.57453536987305,34.07208633422852,32.04676818847656,29.560300827026367,43.759544372558594,44.12125396728516,2028.601806640625,1870.660400390625,895.9712524414062,32.81827163696289,377.08050537109375,0.0,649.1517333984375,770.4885864257812,759.2982788085938,734.395751953125,743.6987915039062,678.6212158203125,700.5547485351562,663.7391967773438,576.5570678710938,589.0308837890625,628.8489990234375,594.935546875,631.0,0.6459924578666687,27.0,1194.1085205078125,997.4606323242188,105.77910614013672,98.88935089111328,28.33451271057129,22.062774658203125,32.428157806396484,12.744391441345217,38.12985610961914,22.12809181213379,17.180856704711914,47.4951057434082,57.16177368164063,59.91652297973633,49.058895111083984,39.42089080810547,83.53347778320312,66.5,0.0,0.0,1742.7498779296875,1810.687255859375,85.46670532226562,97.0500717163086,0.1003460586071014,4.896987915039063,23.919187545776367,26.559722900390625,26.710390090942383,23.56723022460937,27.740917205810547,27.745880126953125,10.377854347229004,10.478169441223145,205.0146484375,225.3446502685547,1.1705199480056765,2.214766025543213,105.16339111328124,450.8149719238281,429.5928039550781,450.5768432617188,448.698974609375,17.0,700.815673828125,0.239432543516159,1200.0,1.0,37.73193740844727,38.19848251342773,20.21402359008789,11.93883228302002,88.80018615722656,84.96267700195312,,,86.0,98.27005767822266,24.427764892578125,0.0,400.0,1261.719482421875,0.0,1463.796630859375,2.163041830062866,1.679708480834961,993.51123046875,0.0225223880261182,192.6799774169922,1.368689775466919,1.3799999952316284,0.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,761.5707397460938,910.4165649414062,,,1.5799574851989746,1.5727667808532717,0.0,600.0,953.1422119140624,947.8602294921876,0.0,10.270873069763184,0.0,11.646549224853516,4.341626167297363,0.0748388767242431,71.84945678710938,100.0,64.71,0.0,4.533299922943115,5.71589994430542,8.699999809265137,4.993200302124023,4.861800193786621,6.311936378479004,51.01556396484375,67.99369049072266,3.52 +2024-12-05 02:32:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,3052.78662109375,8.844391822814941,8.777948379516602,2515.992431640625,229.52468872070312,265.2821044921875,2.214231491088867,2.7172224521636963,3156.99365234375,701.08251953125,0.0,288.5387268066406,0.0,3317.0087890625,86.63294982910156,82.62700653076172,12.9373197555542,12.94904613494873,,,1328.0621337890625,1193.599609375,3.41225004196167,1.3874353170394895,1.4011056423187256,1862.8626708984373,2177.4072265625,1866.84716796875,1715.2435302734375,2251.281005859375,2277.60693359375,84.24079895019531,84.25446319580078,89.87091827392578,34.595943450927734,34.107933044433594,32.05667495727539,24.58193588256836,43.89741134643555,44.14066696166992,2033.9571533203125,1965.69189453125,902.446533203125,32.7397346496582,377.6492614746094,0.0,647.1126708984375,740.5242309570312,757.6129760742188,745.9622192382812,750.0130004882812,749.8059692382812,700.0735473632812,701.1631469726562,570.8465576171875,589.3622436523438,632.1378173828125,594.8961181640625,628.0,0.6863186955451965,27.0,1193.999267578125,997.3634643554688,103.6092300415039,98.8583526611328,28.94314765930176,22.19742774963379,32.61549377441406,12.75114917755127,37.71183013916016,22.179346084594727,17.744470596313477,47.44374847412109,57.132694244384766,59.916778564453125,48.76108932495117,40.08521270751953,83.61792755126953,66.5,0.0,0.0,1746.4300537109375,1854.5777587890625,85.64894104003906,97.2160415649414,0.1004157289862632,4.8964385986328125,23.925472259521484,26.834339141845703,26.46601295471192,23.520160675048828,27.76211738586425,27.71584892272949,10.374256134033203,10.4791841506958,218.3724822998047,225.1285858154297,1.1713463068008425,2.5177488327026367,105.52578735351562,450.79595947265625,429.6829833984375,450.57464599609375,448.6935729980469,17.0,700.7899780273438,1.7021775245666504,1200.0,1.0,36.82808303833008,38.11102294921875,20.06475257873535,11.937515258789062,89.86893463134766,84.6894760131836,,,86.0,98.14891815185548,24.32128524780273,0.0,400.0,1259.343017578125,0.0,1462.4814453125,2.1629884243011475,1.6796951293945312,998.4035034179688,0.0223088879138231,198.8683013916016,1.368503451347351,1.3799999952316284,0.0,,61.264404296875,16.665037155151367,5.333080291748047,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.86962890625,4.85367488861084,73.36161041259766,91.94583892822266,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,7.830216407775879,38.73592758178711,18.59975242614746,17.211471557617188,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042373403906822,1.970276951789856,0.4222961962223053,0.0943225920200347,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.7188185453414917,0.3608308732509613,1.2047821283340454,1.6332342624664309,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7221007943153381,0.2941087782382965,-9999.0,0.5667175650596619,0.7008373737335205,0.8771045207977295,8.49777603149414,65.73210906982422,12.899999618530272,5.300000190734863,6.515639781951904,86.30000305175781,790.82958984375,907.0885620117188,,,1.5791287422180176,1.572784185409546,0.0,600.0,950.763671875,949.9783325195312,0.0,10.146312713623049,0.0,11.57530403137207,4.340694427490234,0.0748387947678566,71.91187286376953,100.0,64.71,0.0,4.533299922943115,5.70755672454834,8.692553520202637,4.993200302124023,4.861800193786621,6.372900485992432,51.02507781982422,67.99890899658203,3.52 +2024-12-05 02:34:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2978.102783203125,8.848578453063965,8.862582206726074,2573.7099609375,234.7885284423828,289.6545104980469,2.2108702659606934,2.714353322982788,3147.3876953125,705.0564575195312,0.0,288.6408386230469,0.0,3398.574462890625,86.58733367919922,89.01890563964844,12.617484092712402,13.247730255126951,,,1347.5814208984375,1234.0552978515625,3.514620065689087,1.387859582901001,1.4020501375198364,1821.7401123046875,2152.696533203125,1866.8282470703125,1715.38427734375,2248.69091796875,2274.983642578125,84.39221954345703,88.17390441894531,90.19397735595705,34.61734771728516,34.1109504699707,32.32330322265625,23.76468849182129,43.90478515625,44.16008377075195,1927.233642578125,1939.23876953125,903.4033203125,31.51068115234375,378.1936645507813,0.0,648.6610107421875,766.79345703125,755.9276123046875,794.9697265625,744.4769287109375,656.0175170898438,699.5923461914062,772.5515747070312,569.7051391601562,589.693603515625,635.4265747070312,594.8566284179688,633.0,0.7698317170143127,27.0,1193.8900146484375,997.2662963867188,103.9173355102539,98.82736206054688,29.941625595092773,22.332080841064453,32.80282974243164,12.757905960083008,37.37025833129883,21.68112564086914,15.733065605163574,47.392391204833984,57.10361099243164,59.91703796386719,48.43916702270508,40.41853332519531,83.20599365234375,66.5,0.0,0.0,1749.7413330078125,1801.275634765625,85.9491195678711,97.38201904296876,0.1004853919148445,4.895888805389404,23.921628952026367,26.72721099853516,26.51607131958008,23.67252349853516,27.53003692626953,27.67759895324707,10.37065887451172,10.479698181152344,220.0579071044922,214.2723999023437,1.172289490699768,2.055137157440185,106.04012298583984,450.7769470214844,429.7731628417969,450.5724487304688,448.6881713867188,17.0,700.7642822265625,0.0,1200.0,1.0,37.02928161621094,38.0235595703125,19.88223648071289,12.143880844116213,91.3222427368164,85.17166900634766,,,86.0,98.23806762695312,23.680456161499023,0.0,400.0,1300.7486572265625,0.0,1472.4622802734375,2.1629347801208496,1.679681658744812,1000.1616821289062,0.0220953896641731,198.59475708007807,1.3683171272277832,1.3799999952316284,0.0,,61.264404296875,,5.333080291748047,,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,,12.386075973510742,,,,,,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.547714233,,,,,,,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,,,,,,,,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,,,,,,,42.07585144042969,,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,,,,,,,,,,,12.899999618530272,5.300000190734863,6.536253452301025,85.9000015258789,806.0762329101562,868.0042114257812,,,1.578299880027771,1.5728015899658203,0.0,600.0,963.1144409179688,951.73193359375,0.0,10.191020965576172,0.0,11.601633071899414,4.3397626876831055,0.0748387202620506,71.49508666992188,100.0,,,,,,,,,51.03459548950195,68.00411987304688, diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index c0785fd..d0d47de 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -1,8 +1,11 @@ -import json from temporalio import activity, workflow with workflow.unsafe.imports_passed_through(): + from datetime import datetime + import json + from pandas import Timestamp, to_datetime + from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ from sientia_do.temporal.activities.base import BaseActivity from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel @@ -60,6 +63,44 @@ class MLFlow(BaseActivity): f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger.base_logger ) + def detect_and_parse_datetime_index(self, data: DataFrame, metadata: dict) -> DataFrame: + """ + Detect and parse datetime index from data. index must be a timestamp like column. + This function must detect the timestamp type (pandas Timestamp or datetime) and convert it to DATETIME_FORMAT_WITH_TZ. + If the index is a string, must be in format DATETIME_FORMAT_WITH_TZ. + If another type or format, must raise an error. + """ + index = data.index + + # Get type of first element of index + index_type = type(index[0]) + + self.info(f"Index type: {index_type}", metadata) + + message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}" + + # Check if all in index are of the same type + if not all(isinstance(i, index_type) for i in index): + raise ValueError( + f"{message}") + + # Check type and converts to DATETIME_FORMAT_WITH_TZ + if index_type == str: + # Validate format of string and return error if not valid + try: + to_datetime(data.index) + except ValueError: + raise ValueError( + f"{message}") + + elif index_type == datetime or index_type == Timestamp: + data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) + else: + raise ValueError( + f"{message}") + + return data + @activity.defn(name="request_transform") async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]: """ @@ -113,13 +154,43 @@ class MLFlow(BaseActivity): data.columns.name = None self.debug("Processed input data:", metadata) - self.debug(data, metadata) + data.to_csv('data.csv') + self.debug(data.to_string(), metadata) # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( model_name, data, model_config ) + self.debug("Raw response data:", metadata) + self.debug(response_data, metadata) + + if response_data['success']: + response_dataframe = DataFrame(response_data['content']) + try: + response_dataframe = self.detect_and_parse_datetime_index( + response_dataframe, metadata) + response_dataframe['timestamp'] = to_datetime( + response_dataframe.index, format=DATETIME_FORMAT_WITH_TZ) + response_dataframe['timestamp'] = response_dataframe['timestamp'].dt.strftime( + DATETIME_FORMAT) + except ValueError as e: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id='TRANSFORM_DATA_INDEX_ERROR', + message=f'Error parsing trasnformed data index: {e}', + block='transform', + level=NotificationLevel.ERROR, + attachment_content=trace + ) + self.error(trace, metadata=metadata) + raise e + + response_dataframe.to_csv('response_data.csv') + + response_data['content'] = response_dataframe.to_dict() + self.debug("Transform response data:", metadata) self.debug(response_data, metadata) diff --git a/laborious/workflows/predictions_batch.py b/laborious/workflows/predictions_batch.py index cc10def..ecce063 100644 --- a/laborious/workflows/predictions_batch.py +++ b/laborious/workflows/predictions_batch.py @@ -87,7 +87,7 @@ class PredictionsBatch(): 'datetime_columns': input_data.get('datetime_columns', []) }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=300) ) # Prepare input for prediction_process workflow diff --git a/laborious/workflows/sub_workflows/prediction_process.py b/laborious/workflows/sub_workflows/prediction_process.py index d78018e..c958500 100644 --- a/laborious/workflows/sub_workflows/prediction_process.py +++ b/laborious/workflows/sub_workflows/prediction_process.py @@ -123,7 +123,7 @@ class PredictionProcess(): 'model_config': model_config }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(minutes=1), + start_to_close_timeout=timedelta(minutes=5), ) # Validate MLFlow transform response @@ -175,7 +175,7 @@ class PredictionProcess(): 'model_config': model_config }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(minutes=1), + start_to_close_timeout=timedelta(minutes=5), ) # Validate MLFlow prediction response diff --git a/response_data.csv b/response_data.csv new file mode 100644 index 0000000..cd1d691 --- /dev/null +++ b/response_data.csv @@ -0,0 +1,2 @@ +,303-WIT-230_median,303-WIT-230_std,303-WIT-230_min,303-WIT-230_max,305-WIT-135_median,305-WIT-135_std,305-WIT-135_min,305-WIT-135_max,305-WIT-160_median,305-WIT-160_std,305-WIT-160_min,305-WIT-160_max,305-PIT-170_median,305-PIT-170_std,305-PIT-170_min,305-PIT-170_max,305-PIT-175_median,305-PIT-175_std,305-PIT-175_min,305-PIT-175_max,305-FIT-002_median,305-FIT-002_std,305-FIT-002_min,305-FIT-002_max,305-FIT-013_median,305-FIT-013_std,305-FIT-013_min,305-FIT-013_max,306-PIT-101_median,306-PIT-101_std,306-PIT-101_min,306-PIT-101_max,306-FIT-051_median,306-FIT-051_std,306-FIT-051_min,306-FIT-051_max,306-DIT-001_median,306-DIT-001_std,306-DIT-001_min,306-DIT-001_max,306-PIT-105_median,306-PIT-105_std,306-PIT-105_min,306-PIT-105_max,306-FIT-052_median,306-FIT-052_std,306-FIT-052_min,306-FIT-052_max,306-DIT-002_median,306-DIT-002_std,306-DIT-002_min,306-DIT-002_max,306-PIT-115_median,306-PIT-115_std,306-PIT-115_min,306-PIT-115_max,306-FIT-004_median,306-FIT-004_std,306-FIT-004_min,306-FIT-004_max,306-PIT-110_median,306-PIT-110_std,306-PIT-110_min,306-PIT-110_max,306-FIT-003_median,306-FIT-003_std,306-FIT-003_min,306-FIT-003_max,306-PIT-125_median,306-PIT-125_std,306-PIT-125_min,306-PIT-125_max,306-FIT-005_median,306-FIT-005_std,306-FIT-005_min,306-FIT-005_max,306-PIT-130_median,306-PIT-130_std,306-PIT-130_min,306-PIT-130_max,306-FIT-006_median,306-FIT-006_std,306-FIT-006_min,306-FIT-006_max,307-FIT-005_median,307-FIT-005_std,307-FIT-005_min,307-FIT-005_max,307-FIT-003_median,307-FIT-003_std,307-FIT-003_min,307-FIT-003_max,307-FIC-022_median,307-FIC-022_std,307-FIC-022_min,307-FIC-022_max,310-FIT-005_median,310-FIT-005_std,310-FIT-005_min,310-FIT-005_max,307-FIT-008_median,307-FIT-008_std,307-FIT-008_min,307-FIT-008_max,307-FIT-009_median,307-FIT-009_std,307-FIT-009_min,307-FIT-009_max,309-PIT-101_median,309-PIT-101_std,309-PIT-101_min,309-PIT-101_max,309-PIT-105_median,309-PIT-105_std,309-PIT-105_min,309-PIT-105_max,309-PIT-110_median,309-PIT-110_std,309-PIT-110_min,309-PIT-110_max,309-PIT-185_median,309-PIT-185_std,309-PIT-185_min,309-PIT-185_max,309-PIT-190_median,309-PIT-190_std,309-PIT-190_min,309-PIT-190_max,309-PIT-195_median,309-PIT-195_std,309-PIT-195_min,309-PIT-195_max,309-FIT-051_median,309-FIT-051_std,309-FIT-051_min,309-FIT-051_max,309-FIT-052_median,309-FIT-052_std,309-FIT-052_min,309-FIT-052_max,309-PIT-001_median,309-PIT-001_std,309-PIT-001_min,309-PIT-001_max,309-PIT-002_median,309-PIT-002_std,309-PIT-002_min,309-PIT-002_max,317AIT003.3_median,317AIT003.3_std,317AIT003.3_min,317AIT003.3_max,317AIT003.5_median,317AIT003.5_std,317AIT003.5_min,317AIT003.5_max,317AIT003.1_median,317AIT003.1_std,317AIT003.1_min,317AIT003.1_max,317AIT003.2_median,317AIT003.2_std,317AIT003.2_min,317AIT003.2_max,317AIT002.37_median,317AIT002.37_std,317AIT002.37_min,317AIT002.37_max,317AIT002.49_median,317AIT002.49_std,317AIT002.49_min,317AIT002.49_max,317AIT002.61_median,317AIT002.61_std,317AIT002.61_min,317AIT002.61_max,317AIT002.38_median,317AIT002.38_std,317AIT002.38_min,317AIT002.38_max,317AIT002.50_median,317AIT002.50_std,317AIT002.50_min,317AIT002.50_max,317AIT002.62_median,317AIT002.62_std,317AIT002.62_min,317AIT002.62_max,317AIT002.39_median,317AIT002.39_std,317AIT002.39_min,317AIT002.39_max,317AIT002.51_median,317AIT002.51_std,317AIT002.51_min,317AIT002.51_max,317AIT002.63_median,317AIT002.63_std,317AIT002.63_min,317AIT002.63_max,317AIT002.40_median,317AIT002.40_std,317AIT002.40_min,317AIT002.40_max,317AIT002.52_median,317AIT002.52_std,317AIT002.52_min,317AIT002.52_max,317AIT002.64_median,317AIT002.64_std,317AIT002.64_min,317AIT002.64_max,317AIT002.41_median,317AIT002.41_std,317AIT002.41_min,317AIT002.41_max,317AIT002.53_median,317AIT002.53_std,317AIT002.53_min,317AIT002.53_max,317AIT002.65_median,317AIT002.65_std,317AIT002.65_min,317AIT002.65_max,317AIT002.42_median,317AIT002.42_std,317AIT002.42_min,317AIT002.42_max,317AIT002.54_median,317AIT002.54_std,317AIT002.54_min,317AIT002.54_max,317AIT002.66_median,317AIT002.66_std,317AIT002.66_min,317AIT002.66_max,317AIT002.43_median,317AIT002.43_std,317AIT002.43_min,317AIT002.43_max,317AIT002.55_median,317AIT002.55_std,317AIT002.55_min,317AIT002.55_max,317AIT002.67_median,317AIT002.67_std,317AIT002.67_min,317AIT002.67_max,317AIT002.44_median,317AIT002.44_std,317AIT002.44_min,317AIT002.44_max,317AIT002.56_median,317AIT002.56_std,317AIT002.56_min,317AIT002.56_max,317AIT002.68_median,317AIT002.68_std,317AIT002.68_min,317AIT002.68_max,317AIT002.45_median,317AIT002.45_std,317AIT002.45_min,317AIT002.45_max,317AIT002.57_median,317AIT002.57_std,317AIT002.57_min,317AIT002.57_max,317AIT002.69_median,317AIT002.69_std,317AIT002.69_min,317AIT002.69_max,317AIT002.46_median,317AIT002.46_std,317AIT002.46_min,317AIT002.46_max,317AIT002.58_median,317AIT002.58_std,317AIT002.58_min,317AIT002.58_max,317AIT002.70_median,317AIT002.70_std,317AIT002.70_min,317AIT002.70_max,317AIT002.47_median,317AIT002.47_std,317AIT002.47_min,317AIT002.47_max,317AIT002.59_median,317AIT002.59_std,317AIT002.59_min,317AIT002.59_max,317AIT002.71_median,317AIT002.71_std,317AIT002.71_min,317AIT002.71_max,317AIT002.48_median,317AIT002.48_std,317AIT002.48_min,317AIT002.48_max,317AIT002.60_median,317AIT002.60_std,317AIT002.60_min,317AIT002.60_max,317AIT002.72_median,317AIT002.72_std,317AIT002.72_min,317AIT002.72_max,SiO2_conc,timestamp +2024-12-05 04:00:00+0000,2344.354736328125,347.11413476082527,2063.14990234375,3052.78662109375,1323.8238525390625,199.25517177489738,801.2099609375,1477.9354248046875,1239.42919921875,188.20232896348458,1067.496337890625,1611.220947265625,12.741495132446287,0.2925100433215222,12.14862060546875,13.035848617553713,13.396173477172852,0.2659367570480025,12.94904613494873,13.875475883483888,3156.99365234375,17.90496592640638,3128.458251953125,3188.196533203125,3328.472412109375,18.341293358560456,3293.60546875,3352.32080078125,34.44520568847656,0.10727337707066699,34.300880432128906,34.595943450927734,2251.281005859375,6.914512409267933,2241.09228515625,2259.91357421875,1.3857378959655762,0.001162128441763176,1.3840404748916626,1.3874353170394895,33.95878982543945,0.06905080560339776,33.905391693115234,34.107933044433594,2275.4189453125,8.067717393234235,2263.143798828125,2288.202880859375,1.397327542304993,0.0025866991350633178,1.3935494422912598,1.4011056423187256,31.01426696777344,2.797667558372099,24.58193588256836,34.75410461425781,2157.358642578125,30.47011040139319,2091.429931640625,2189.05615234375,32.01703643798828,0.027139770722953593,31.97739601135254,32.05667495727539,2059.924560546875,88.54574101606774,1862.8626708984373,2170.908447265625,43.65913391113281,0.09432217658823967,43.593544006347656,43.89741134643555,1866.9232177734373,0.05200646609213417,1866.84716796875,1866.9991455078125,44.08242416381836,0.04344513322059127,44.02417755126953,44.14066696166992,1714.9619140625,0.1883131118977155,1714.6802978515625,1715.2435302734375,27.0,0.0,27.0,27.0,0.7632204294204712,0.28421935480519783,0.6459924578666687,1.425487995147705,649.1517333984375,2.0424248177883793,647.1126708984375,653.2977294921875,700.892822265625,0.07040149596071192,700.7899780273438,700.9956665039062,1194.4361572265625,0.29912346849236254,1193.999267578125,1194.873046875,997.7520751953124,0.26607758363281586,997.3634643554688,998.1407470703124,23.87283706665039,0.040718881893752515,23.81103706359864,23.925472259521484,26.559722900390625,0.2379241176870002,26.20622062683105,26.91034507751465,26.35142517089844,0.203982336891631,26.06732177734375,26.710390090942383,23.415620803833008,0.11592992262134358,23.256576538085938,23.56723022460937,27.697071075439453,0.15510993222349723,27.35708808898925,27.844594955444336,27.72958755493164,0.19091973703305398,27.292091369628903,27.89141845703125,1748.264892578125,7.86294792652368,1742.1806640625,1768.509765625,1821.6829833984373,15.953009331205266,1809.867919921875,1854.5777587890625,0.1001370549201965,0.00019079441009214591,0.0998583808541297,0.1004157289862632,4.8986358642578125,0.0015045608215440351,4.8964385986328125,4.900833606719971,6.507819890975952,0.12657004614335984,6.405970096588135,6.699999809265137,86.30000305175781,0.3732080423947149,85.80000305175781,86.5999984741211,8.5,1.6421334224036257,7.89943265914917,12.899999618530272,5.400000095367432,1.0432607765452175,3.299999952316284,5.800000190734863,1.2756186723709106,0.0,1.2756186723709106,1.2756186723709106,0.7121588587760925,0.0,0.7121588587760925,0.7121588587760925,0.4104396104812622,0.0,0.4104396104812622,0.4104396104812622,0.2267737984657287,0.0,0.2267737984657287,0.2267737984657287,1.4764769077301023,0.0,1.4764769077301023,1.4764769077301023,0.6497865319252014,0.0,0.6497865319252014,0.6497865319252014,0.4203702211380005,0.0,0.4203702211380005,0.4203702211380005,1.526507019996643,0.0,1.526507019996643,1.526507019996643,0.6540470123291016,0.0,0.6540470123291016,0.6540470123291016,1.7459523677825928,0.002774316303229609,1.7381054162979126,1.7459523677825928,0.4817045927047729,0.0002177622295436636,0.4817045927047729,0.4823205173015594,0.3364990949630737,0.00012384851434926538,0.3361487984657287,0.3364990949630737,1.1931402683258057,0.033290363097006954,1.1206940412521362,1.1931402683258057,0.7161301374435425,0.025235254887373854,0.7161301374435425,0.7710468769073486,0.4127309918403625,0.007279004051801342,0.4127309918403625,0.428571492433548,1.707435429096222,0.0008729009039302258,1.7066189050674438,1.708251953125,0.3564415574073791,0.002030279951199197,0.3545424044132232,0.358340710401535,0.3023426830768585,0.000350906290820105,0.3020144402980804,0.3026709258556366,-9999.0,0.0,-9999.0,-9999.0,-9999.0,0.0,-9999.0,-9999.0,-9999.0,0.0,-9999.0,-9999.0,0.1042194217443466,8.294721469109035e-06,0.1042194217443466,0.1042373403906822,1.715789794921875,0.001402039007837862,1.715789794921875,1.7188185453414917,0.7188712954521179,0.0014949674798809447,0.7188712954521179,0.7221007943153381,1.970276951789856,0.018573843840262887,1.934388875961304,1.970276951789856,0.3608308732509613,0.004311563730231936,0.3608308732509613,0.3691616058349609,0.2941087782382965,0.0009687919419025886,0.2941087782382965,0.2959806621074676,0.4393351674079895,0.006024186034919744,0.4222961962223053,0.4393351674079895,1.1838626861572266,0.007396139710934239,1.1838626861572266,1.2047821283340454,0.5613290071487427,0.0019051429198136875,0.5613290071487427,0.5667175650596619,0.0910589918494224,0.0011538569058607675,0.0910589918494224,0.0943225920200347,1.6499500274658203,0.0059099153918944995,1.6332342624664309,1.6499500274658203,0.7046993374824524,0.0013654103777831785,0.7008373737335205,0.7046993374824524,0.0354578979313373,0.0,0.0354578979313373,0.0354578979313373,2.292947769165039,0.0,2.292947769165039,2.292947769165039,0.8771045207977295,0.0,0.8771045207977295,0.8771045207977295,3.52,2024-12-05 04:00:00 diff --git a/tests.ipynb b/tests.ipynb index 8ab64e2..6ec555e 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -404,6 +404,60 @@ "print(len(b))\n", "print(b.size)\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3374174", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "All elements in index are of the same type\n" + ] + } + ], + "source": [ + "from pandas import DataFrame\n", + "\n", + "data = DataFrame({\n", + " \"a\": {\"2025-01-01\": 1, \"2025-01-02\": 2, \"2025-01-03\": 3},\n", + " \"b\": {\"2025-01-01\": 4, \"2025-01-02\": 5, \"2025-01-03\": 6},\n", + "})\n", + "\n", + "index = data.index\n", + "\n", + "# Get type of first element of index\n", + "index_type = type(index[0])\n", + "\n", + "print(index_type)\n", + "\n", + "# Check if all in index are of the same type\n", + "if all(isinstance(i, index_type) for i in index):\n", + " print(\"All elements in index are of the same type\")\n", + "else:\n", + " print(\"Elements in index are of different types\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40e72c60", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1fbb3788", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl b/tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..02d4f4a33b863e4e6a36c404e28917079c0a8fad GIT binary patch literal 29391 zcmeHQS(6-BRlnF)O&Hi-feBBSHJz&e@y@WBsnyQdRy_J){3^spZVQUa(K{a$5d4(YR75vI2HSY zsJ)#gN2In9t~HiJ=yo>JW$7I?sX!hyd8C-Ezb1eORet4TZ^KM+$K*+i(0F3vp=a#(&T7u*x7Du zyw!?pH_3*SPj!av?YD(9?O|)sh&5WDKj;lN2Bf%;71QJZoM)Af()q{U zOJ{f8On;QFrswB(%^z5Nao784{maR|&B5E-FMWq*14SPw8DPeWk2wS&VnCZ!+-9hXAx$laWK}0P zSrt{f%*~)mJopQuBPdOg^2~trd}o z*ARs+kt#6}^?Qq`iiyMOxna%ix`-7?s>&o4FS4{`<+QBX~(HAKt7ohgxN$|e#-B_a?lZHYzLO04L#zM!*5Vwl!{$zm+(o*_Ak!s(+_ zrx;e%+Vb^v>4j)Fon2`Yie?q1VnO#l1?c`Mfnq^*M+NvAp|4TO0KQ4t*C_iM6RRaTd=mPLH?9f@m4qX7g zi5)s?*r79Ehb{o0Lxzq(3$o{7AefSv@mPj_D5Ef9(M=&%{hTJmqr~O(Aq%OfUIm>6 zo<+*sDJ)J5cMKJBhq}HHGhk0)tS_f9sm}rGL^D6yM(0%lGGb|4W;ICwwRf0!)P#K`4^;1W{543Jb-FRckP+9&@}fb!nwg z3@_ZcvVirNJDJ2vIat}otceyAy;Tys0TCxgJw<}B|?V(69%?6f+Smuui4Uur_hp$lQ*1JGVKj^8U?7F%yV@GP$ za(HcdL3LUf*!D=LWiP8FLVL8>_Aq$<-#nzZXF zP1^O8XsOd46-joGr{f!##6bG*&N>KIYgwfImQ^gdE|oESENQiv*8jr>ib$Fz4@JwW ztd+!`OBE{^MV4|Ep8c~8&>qr~hpsCn*A?2>TFO<-)&FDzMc0+kb*1dOQgPd;E0D75O2tiZwVKxd%T_PCu9RF?%C0LFH^J4Z zwEizPS?Icgu47Bft}7Kc!BuyNxNe26TUb~fV%M#To9yazTK`X5o%XO+!b+}NW!J5W zn{2%AzzDRgL)R60HnyZ?*OiJJ)f7gq^OWrR=&Q9=!ETF@>m3(L-qwvoUQ? z7mI=ThuM)IRJUP*Hl3#3uD0It2tz!Qb3Ioe3qW|voM zt9aP5%U8YfB_vMC*RQ;>uEkTHxhNZRQ8wmNHQT%xy5_R1&1G4eqb$t#3@+5+!)m-S z+>U$E-Jjh3^KP2#s-?|e-u2jHBQMFGm9=YWa~3+uzV^@lib!^yOPiO;=n7r>FG_g=zDeHh7KhxKh3tfw>zg zFj2A!7ugjiW&i0e*|T+FAf05fan~>yfpNy9;WOMNd(O?pX>-Ai(^Rsd)M%b1z2E+;JUzq2A10+Ukm4MW$$C zC#qyWhk;Jb8Z6{0R*BPKM!;R+(_NB7tMQ<**$u)%;YAGnm8jp3@G>|kg%y;Q6m#f{ zY4bXJcP^lp(89Z9H@af7mx1K*b3wn`0$CsLXO}SJ?p{Kx>|QFR&DY8GH|X*vU4Dfw zze<;1qswQICcEPxeJISI#jA&Bo9voNo4-y;8+*C>TmK_IN%Ph{m1 zl)hOkp{1``!kw_dm!m7)!QH>;Y&3L}cnEY3u5~xH*c2c+LQZq$WqY~+G!AXQ8Z6JL z{97gBWy|aRMkh*>XSY>@^`icEwAt+ic$!y+XKsro%9EYlj640tX0#axdE#8+@)%+v zgMazNT6{{j&Fmd&097>_M&cWH0(55%B~4E1@Dvu%JH@ADTu7o;W z>fHSaTK5Sl2zVglGtSz%TCe~Yspmt6fs5hFtu9g3;%L=d_(*Y*R00YL7H2N&7Qa3h zu!OAwG}{K*3}-Uio?HhtZmJ*^bVv2EFcZ zs~-1)x%B`k^ANO!hm>}gzVcOZbm{%w}nf+jP*rTsBq{#~q z%SCO3#)%HEMsDh}AUw)~Xn^CwT&0)WKq@LsG{Z?*kJl1;FdE~m>y=5NXpeDXG|6*g ztf^I`+^~8LGggjFJY$br9uqVT(ZGc8X>0gn#EG4T5Ggy%L6`?d2ze1(>fVk-gONQv zHnUBx{2BC?AfWdMg%EEn?AejrHNuiFJS&Ggj5qESE@s45G?>i%V4u~~3r%&t^e7f< zM6O{>M?h$e7vG1mC1<1`t7;D5jLz{qsbspDlnm*9R@#}y6m0=ZbWdfOVZ~H&3S%>a zV^|WNSJP={EZeo@GNA@5JNDemeF9RMTG16;|7nU6OCKu-)coX`jmgyzz2g)$Fuz2a z7JD~2m1{gKIQW-j&WeTEHwuNRV8W8B6;=7J7Z_UoFqP0EjPWVkJtGy4T8m>q3+LKO zuzKY+8XeYBYfBcnu_URoKxM`0^+o<%?76HAuj8Z~sXm4;&QKIom0IzoNt+)KdSJui z7X|$x{8u9}rl8=N;X0SM*ki`g}jeceJoApx1we zO4SNKP0qRB#URLf)EF1y_N zJne>FlBL2Fl~wf)&FT)W0Nvr~6}1yaJBx=oehh$(5%CkCfP*z!d`5|o79(;A zow^KR8H6M1A`z_k$?$UA-{6fLJyStH3qtcn*#YsG9kqUA7^8oLr2qkPTq>+QtuiKa zsnzZT^qr$~FcZ(p;A+DnOkRHAOsNFRjGAjkq%i*61BvpC^MH zwb5SCXDeg8An|xFNMi?HklL3-E;vd&cE!E!=5WxpeNLOlcH@cZE3%_Ucy2~;^2t&s zP-g{G7Fv*|Sdr|*vLqLmBeEzr*Jf>&gz}}6cp9Y*ki>aANXr$PLp}jX=!n{<&*D46 z^ag_fvqS2`NCTSvAl`*6G&=Q&9+*r4YtM12L6~dplh&(Q-Dj{V)9+~cC9NfSN*d7H zT6*oGtI8M9`i=*2dU>0DC`pszjk_PWZsSohxb${FbxI}d=K8^Q)Wf6Z9rWM1c%vTS z1<}g^+V%O_`26GF`>%rW$rR6ms^!^Xhjl|Xv3L+9K*sKbTQyR=SJIeAY=F`h;d>BFis5TpS3F&qa3|! zQu^+==17I>4!J9*CKydlj*pST?=e!i?Ie}^5atj)81U}N>SdUqsP&D4@O>Nw=iLq7 zr6B*u$`Qd>d2S5B)z#&-wY(86W3UuN5ZN$o6>mgmNhpkL9xMv_m}$I%8wjRG-c2rC zTY7aSpbf1zptA~@Kyp;PK(ZQMAUO(dAUO(dAYxUUq`BkDwe{s}AX$Y>AUP^tAXyDB zkQ@a!kQ@aBqGm0)Z5E6C&XuDvAwGt@2a7Ywj;M&DxrZ;*!|tWxHVv8tV|mQ-msQ9# zO^%A!G+7O=X>t_YrpZwl6$mwrnscpW7zU*DoO<68MdQf;zo?$IJWBD2|WDrcgWJDv_m_P-mg_K^UknkTb+06 z8pL-7L+tS_|BPFoDja-H@F~LCsg)ZJtacg>R6k8PMGV;c4`;_#PB>(>!*EFT5Pzl~ zT*XY$N}E5xWAqL#$uW9WhlypS*J$7UxJO^K`XQMCB-l5B{^n=B;I2*-=lzjo|ojL zXqDg+Ry6hLXFqMePmbkW;`4dvXTg%=vg@zmd$05>Wa|4rqUwK6m*kA>sPr`$Ozre; zNUri}^8<41zAyNI3P0~|i0@b|ckop$e1-HwqIz(b%IUWMba2g{pf(};0HL;ufi zOoZkx8={e!yR4Q*cBG`W^^GyRuRCq8Z^oNzwFh-$s>?EtlV!C(PsDg;?(VC8W5#!5 zZ0|$UOfGacj&tXlG));79t`K_SbKav>#_;wp1I$-r>>@%#vk5YjPrJrU${AJ;Gkdn z>C?V`tFb``_0B&we_(zWozpwNo38KiAFAd*^_su^e zch2N_k7KB}$o*CN-}1NlQ6qvgCmPdTyq(VPV(OUqlf5+@aE#wa zQ;g~e=Nf(+j4xOt=Ad1RDTq>MUX?Z2p0D4rUMhAVv zE2Be)ehaVopj@17=};-qP$|SvX~d)Q0u*W$%PWQWR4~+3;#kPg^-6)pD}@*;jkr`e z4A%*e)c|@_q^5JFbd*>6ykwiofQrf>ipoNySLz7mqD#d!R*KpyOHF!ZA(~2_)a+6z z(0HW~eGxibc3l5zH$q0(xeS1{C6qB^Iy z%d{6vZ#RhV6$~{MR~P2x3ZUl|skvSeqN#*e9+d(Ol|l@aMl=zB&D}(p}!qB6d&ZC;nP(jTKkir2X zsniL-&V+10l`DfNDhu(boX()s%)l{-Ph|#<)WIZR^>zb#UXhv%(CBypUd;0fphrb& zE)^kefYW(i0raRy&7~s5qssGYI^z`(pUU=%#~@ZuCsztIUMa*-X+(P^5NF`%GnuUb zwVDo4Bbth)9Xb=!@k)WlD}@-ZG~!WtUI{g`q>84eKS>)5%nK;J8qaw3^P9h4@r5S5KPLo=a0YS;rid z{c<5@^B~HXMOQYHmZb>Prz1s*)79cYExM{nnxzQTrz1s=jv&vwl9`7~bOd>HrQD_f>eG>;M@NuHXU#yCcWPJf zLOvbYdURDYNG)5SzIUYPc}I{(hd(Hh(-c5`I#TrL2=eGExlIAorz1s=jv$Y&YPQpM zu;mOLDSC7Sd35F6Q32HVjubsQf;_rPZc_mD=}6I|BgmtxnmN-BR!*8O`19VvQr1bK8-v(2_6t7hm((W4{Cqbuh&1yJ8RQuOEu^62pA zVsb_WP@j$zJvxFsx@zuP0o11>MURdkkFK2C6hM7CQuOEu^5`nLO##%WBSnvnAdjw^ zyH)`8=}6I|Bgmt}AGpbB3ZOn6DSC7Sd32TBrU2^Gk)lUOkVjX|T`Pe4bfoCf5#-U8 zbDILFPe+O#9YG#lCATSn`gEk|(Gld)S#JfcF$>hEBSnvnAdjw`+Y~^3I#TrL2=eGE zxlIAorz1s=jv$Y2DtD~_>eG>;M@NuHSI%t;pgtWbdUOPNbk@6Gt0~mWUKjG|$kwBq n%3Ujf`reVE=N&S;hzzhH3&B^}*E89w3 literal 0 HcmV?d00001 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py index 4afdfaf..e69de29 100644 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py +++ b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py @@ -1,824 +0,0 @@ -""" -Base model classes and interfaces. - -This module defines base classes with consistent interfaces for all models, -promoting modular model development. It includes essential functionality -for model fitting, prediction, evaluation, saving, and loading, while -abstracting common behaviors into base classes. - -While full compatibility with scikit-learn is not guaranteed, the base -classes provide a consistent interface for model fitting, prediction, -evaluation, saving, and loading, which should be sufficient for most -use cases. - -Key components: -- **Model**: Abstract base class for all models, providing core utilities and - interfaces. -- **TimeSeriesModel**: Abstract base class for time series models, adding - time-based functionality. -- **UnivariateTimeSeriesModel**: Base class for univariate time series models. -- **MultivariateTimeSeriesModel**: Base class for multivariate time series - models that use exogenous features. - -The module also includes utility functions such as `ensure_fitted`, which - ensures models are fitted before calling certain methods. - -Modules in this package should inherit from these base classes and implement - the required methods. - -TODO: - - Add methods to create lagged features for time series models. -""" - -import uuid -import joblib -from abc import ABC, abstractmethod -from typing import Optional, List, Sequence, Tuple, cast, Any, Protocol -from contextlib import contextmanager - -import pandas as pd -import numpy as np -import shap -import matplotlib.pyplot as plt -from rich.console import Console - -from sklearn.base import BaseEstimator, RegressorMixin -from sklearn.utils.validation import check_array, check_X_y -from sklearn.exceptions import NotFittedError - -console = Console() - - -class PredictorProtocol(Protocol): - """Protocol for models with predict method and optional imputation.""" - - def predict(self, X: Any) -> Any: ... - def _impute_missing_values(self, X: Any) -> Any: ... - - -def ensure_fitted(method): - """ - Decorator to ensure the model is fitted before calling the method. - - Raises: - sklearn.exceptions.NotFittedError: If the model is not fitted. - Usage: - @ensure_fitted - def predict(self, X): # Or other methods requiring fit - pass - """ - - def wrapper(self, *args, **kwargs): - is_fitted = self.__sklearn_is_fitted__() - if not is_fitted: - raise NotFittedError( - f"This {self.__class__.__name__} instance is not fitted yet. " - "Call 'fit' with appropriate arguments before using this " - "method." - ) - return method(self, *args, **kwargs) - - return wrapper - - -class Model(BaseEstimator, ABC): - """ - Abstract base class for all models. - - Provides core utilities, input validation, and interface consistency - for time series models. Compatible with scikit-learn workflows. - """ - - def __init__(self, name: Optional[str] = None, random_seed: int = 42): - """ - Initialize the model with a unique name and random seed. - - Args: - name: Optional identifier; auto-generated if None. - random_seed: Seed for reproducibility. - """ - self.name = name or f"{self.__class__.__name__}_{uuid.uuid4().hex}" - self.random_seed = random_seed - self.feature_names_in_: Optional[List[str]] = None - self.n_features_in_: Optional[int] = None - self._is_fitted = False - - def fit( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> "Model": - """ - Trains the model. - - Handles basic input validation for y and sets internal fitted - state after calling _fit_logic. - - 'X' is optional to account for univariate time series models. - - Args: - y: The target variable. - X: Optional exogenous variables. - X_val: Optional validation feature matrix. - y_val: Optional validation target series. - Raises: - TypeError: If y is not a pandas Series. - If X is provided, it must be a pandas DataFrame. - If X_val and y_val are provided, they must be pandas DataFrames - and Series respectively. - - Returns: - Self for chaining. - """ - if not isinstance(y, pd.Series): - raise TypeError("Input 'y' (target) must be a pandas Series.") - - self._fit_logic(y, X, X_val, y_val) - self._is_fitted = True - return self - - @abstractmethod - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """ - Core fitting logic to be implemented by subclasses with - optional validation data. - - Args: - y: The target variable. - X: Optional exogenous variables. - X_val: Validation feature matrix (optional). - y_val: Validation target series (optional). - """ - raise NotImplementedError("Subclasses must implement _fit_logic().") - - @ensure_fitted - @abstractmethod - def predict(self, X: Optional[pd.DataFrame] = None) -> Sequence: - """ - Predict values. - - Args: - X: Optional features for prediction. For univariate models - not using exogenous variables, this might be None or - contain future timestamps. Multivariate models will - require X. - - Returns: - NumPy array or similar sequence of predictions. - """ - raise NotImplementedError("Subclasses must implement predict().") - - def fit_predict( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> Sequence: - """ - Fits model and returns predictions on the same data. - - Args: - y: The target time series. - X: Optional exogenous variables. - - Returns: - Predictions for the input data. - """ - return self.fit(y, X, X_val, y_val).predict(X) - - def save(self, path: str) -> None: - """ - Saves model to disk using joblib. - """ - joblib.dump(self, path) - - @classmethod - def load(cls, path: str) -> "Model": - """ - Loads model from disk using joblib. - """ - return joblib.load(path) - - def __str__(self) -> str: - return f"{self.__class__.__name__}(name={self.name})" - - def __sklearn_is_fitted__(self) -> bool: - """ - Check fitted status and return a Boolean value. - """ - return hasattr(self, "_is_fitted") and self._is_fitted - - def __repr__(self) -> str: - return f"{self.__class__.__name__}(name={self.name})" - - @contextmanager - def model_state_preservation(self): - """Context manager to preserve model state during operations.""" - original_state = self._get_state_snapshot() - try: - yield - except Exception: - self._restore_state_snapshot(original_state) - raise - - def _get_state_snapshot(self) -> dict: - """Get snapshot of current model state.""" - return { - "name": self.name, - "is_fitted": getattr(self, "_is_fitted", False), - "feature_names": self.feature_names_in_, - "n_features": self.n_features_in_, - } - - def _restore_state_snapshot(self, snapshot: dict) -> None: - """Restore model state from snapshot.""" - self.name = snapshot["name"] - self._is_fitted = snapshot["is_fitted"] - self.feature_names_in_ = snapshot["feature_names"] - self.n_features_in_ = snapshot["n_features"] - - def get_params_dict(self) -> dict: - """Get model parameters as dictionary for logging/serialization.""" - return { - "name": self.name, - "random_seed": self.random_seed, - "n_features_in_": self.n_features_in_, - } - - def summary(self) -> str: - """Generate a summary string of the model.""" - params = self.get_params_dict() - fitted_status = ( - "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" - ) - - summary_lines = [ - f"Model: {self.__class__.__name__}", - f"Status: {fitted_status}", - f"Features: {params.get('n_features_in_', 'Unknown')}", - ] - - return "\n".join(summary_lines) - - -class TimeSeriesModel(Model): - """ - Abstract base class for time series forecasting models. - - Extends the base Model class with specific methods for time series - data handling and evaluation. - """ - - def __init__( - self, - name: Optional[str] = None, - time_col: str = "ds", - target_col: str = "y", - random_seed: int = 42, - n_lags: int = 0, - sampling_freq: Optional[str] = None, - ): - super().__init__(name=name, random_seed=random_seed) - self.time_col = time_col - self.target_col = target_col - self.n_lags = n_lags - self.sampling_freq = sampling_freq - - self.training_series_: Optional[pd.Series] = None - self.model_: Optional[BaseEstimator] = None - - # Validate configuration - self._validate_configuration() - - def _validate_configuration(self) -> None: - """Validate model configuration.""" - if self.n_lags < 0: - raise ValueError("n_lags must be non-negative") - - def _validate_y(self, y: pd.Series) -> np.ndarray: - """ - Validates the target variable (y) for the model. - - Ensures y is a pandas Series and checks its name against - the expected target column name. Converts y to a NumPy array. - The series name can be None, but if it is set, it should match - the expected target column name. - - Args: - y: The target variable as a pandas Series. - - Returns: - A NumPy array of the target variable. - - Raises: - TypeError: If y is not a pandas Series. - """ - # Check if y is a pandas Series - if not isinstance(y, pd.Series): - raise TypeError("Input 'y' (target) must be a pandas Series.") - if (y.name is not None) and (y.name != self.target_col): - raise ValueError( - f"Expected target column name '{self.target_col}', " - f"but got '{y.name}'." - ) - return check_array(y, ensure_2d=False) - - @ensure_fitted - @abstractmethod - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Performs backtesting on the time series data. - - Args: - y: The target time series data. - X: Optional exogenous features. - retrain_every: Number of steps after which to retrain the model. - reuse_previous_execution: Whether to reuse the previous execution - of a backtest. If True, any overlapping data between the - previous execution and the current execution will be used - without retraining the model. - Returns: - Series of predictions for each step in the time series. - """ - - raise NotImplementedError("Subclasses must implement backtest().") - - def get_params_dict(self) -> dict: - """Get model parameters as dictionary for logging/serialization.""" - base_params = super().get_params_dict() - ts_params = { - "time_col": self.time_col, - "target_col": self.target_col, - "n_lags": self.n_lags, - "sampling_freq": self.sampling_freq, - } - return {**base_params, **ts_params} - - -class UnivariateTimeSeriesModel(TimeSeriesModel, RegressorMixin): - """ - Base class for univariate time series models. - - Only supports regression settings. Concrete subclasses - must implement `_fit_logic` and `predict`. - """ - - @abstractmethod - @ensure_fitted - def forecast(self, forecast_horizon: int) -> Sequence: - """ - Forecast into the future for a given number of steps. - - Args: - forecast_horizon: Number of future time steps to forecast. - - Returns: - Sequence of forecasted values. - """ - raise NotImplementedError("Subclasses must implement forecast().") - - -class MultivariateTimeSeriesModel(TimeSeriesModel): - """ - Base class for multivariate time series models. - - This class provides a foundation for time series models that utilize - multiple exogenous features (X) to predict a target variable (y). - It supports both regression and classification tasks. - - Attributes: - selected_features_: List of feature names selected for the model. - learning_task: Type of learning task ('regression', 'binary', - 'multiclass'). - differentiate_target: Whether to apply differencing to make series - stationary. - bins: Bin edges for multiclass classification target - transformation. - - Example: - >>> class MyModel(MultivariateTimeSeriesModel): - ... def _fit_logic(self, y, X=None, **kwargs): - ... # Implementation here - ... pass - ... def predict(self, X=None): - ... # Implementation here - ... return predictions - """ - - def __init__( - self, - name: Optional[str] = None, - time_col: str = "ds", - target_col: str = "y", - random_seed: int = 42, - n_lags: int = 0, - sampling_freq: Optional[str] = None, - differentiate_target: bool = False, - bins: Optional[List[float]] = None, - learning_task: Optional[str] = None, - ): - # Set attributes before calling parent constructor - # This is needed because parent constructor calls - # _validate_configuration - self.selected_features_: Optional[List[str]] = None - self.learning_task: Optional[str] = learning_task - self.differentiate_target = differentiate_target - self.bins = bins - self.model_: Optional[PredictorProtocol] = None - - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - n_lags=n_lags, - sampling_freq=sampling_freq, - ) - - # Additional validation for multivariate models - self._validate_learning_task() - - def _validate_learning_task(self) -> None: - """Validate learning task configuration.""" - valid_tasks = {"regression", "binary", "multiclass", None} - if self.learning_task not in valid_tasks: - raise ValueError( - f"Invalid learning_task: {self.learning_task}. " - + f"Must be one of {valid_tasks}" - ) - - if self.learning_task == "multiclass" and not self.bins: - raise ValueError( - "bins must be provided for multiclass learning_task" - ) - - def _get_default_loss_function( - self, provided_loss: Optional[str] - ) -> str: - """ - Get default loss function based on learning task. - - Args: - provided_loss: User-provided loss function (takes precedence) - - Returns: - str: Appropriate loss function for the learning task - """ - if provided_loss is not None: - return provided_loss - - if self.learning_task == "regression": - return "RMSE" - elif self.learning_task == "binary": - return "Logloss" - elif self.learning_task == "multiclass": - return "MultiClass" - else: - return "RMSE" - - def _validate_configuration(self) -> None: - """Validate model configuration.""" - super()._validate_configuration() - - if self.differentiate_target and self.learning_task in [ - "binary", - "multiclass", - ]: - console.print( - "[yellow]Warning: Using differentiation with classification " - + "tasks may not be appropriate[/yellow]" - ) - - @ensure_fitted - def feature_importance(self) -> Optional[pd.DataFrame]: - """ - Returns feature importance if implemented by subclass. - - Returns: - A DataFrame with feature names and their importance scores, - or None if not applicable. - """ - return None - - def _validate_X_y( - self, X: pd.DataFrame, y: pd.Series, allow_nan: bool = True - ) -> Tuple[np.ndarray, np.ndarray]: - """ - Validates input features (X) and target (y). - - Infers and sets `feature_names_in_` and `n_features_in_`. - This method should be called within the `_fit_logic` of - concrete subclasses that use exogenous features. - - Args: - X: DataFrame of input features. - y: Series for the target variable. - allow_nan: If True, allows NaN values in X and y. - Raises: - TypeError: If X is not a DataFrame or y is not a Series. - ValueError: If the number of features in X does not match - the expected number of features. - - Returns: - Tuple of validated NumPy arrays (X_array, y_array). - """ - if allow_nan: - X_array, y_array = check_X_y(X, y, force_all_finite=False) - else: - X_array, y_array = check_X_y(X, y, force_all_finite=True) - - if hasattr(X, "columns"): - console.log( - f"Validating input features with columns: {X.columns.tolist()}" - ) - self.feature_names_in_ = list(X.columns) - else: - console.log( - "Input features do not have column names, using default names." - ) - self.feature_names_in_ = [ - f"feature_{i}" for i in range(X_array.shape[1]) - ] - - self.n_features_in_ = X_array.shape[1] - return X_array, y_array - - def _validate_X( - self, X: pd.DataFrame, allow_nan: bool = True - ) -> np.ndarray: - """ - Validates input features (X) before prediction or scoring. - - Ensures consistency with features seen during fit. This should - be called by concrete subclasses in `predict`, `score`, etc. - - Args: - X: DataFrame of input features. - allow_nan: If True, allows NaN values in X. - - Returns: - Validated NumPy array of X. - """ - if allow_nan: - X_array = check_array(X, force_all_finite=False) - else: - X_array = check_array(X, force_all_finite=True) - # If the model has been fitted, ensure the input features - # match the features seen during fit. - if self.feature_names_in_ is not None: - if not set(self.feature_names_in_).issubset(X.columns): - raise ValueError( - "Input features do not match the features seen during fit." - + f" Expected features: {self.feature_names_in_}, " - + f"but got: {list(X.columns)}." - ) - - return X_array - - def _transform_target_to_multiclass( - self, y: pd.Series, bins: Optional[List[float]] = None - ) -> pd.Series: - """ - Transforms the target variable into a multiclass classification - target. - - If bins are provided, uses pd.cut to categorize the target into - discrete classes. If not, binarize the target at zero (0). - - Args: - y: The target variable as a pandas Series. - bins: Optional list of bin edges for categorization. - - Returns: - A pandas Series with transformed classification targets. - """ - if bins is not None: - # pd.cut returns a Categorical, convert to Series with integer - # codes - categories = pd.cut(y, bins=bins, labels=False) - return pd.Series(categories, index=y.index) - - return (y > 0).astype(int) - - def _transform_target_to_binary( - self, y: pd.Series, threshold: float = 0.0 - ) -> pd.Series: - """ - Transforms the target variable into a binary classification target. - - Binarizes the target at the specified threshold (default is 0.0). - - Args: - y: The target variable as a pandas Series. - threshold: The threshold for binarization. - - Returns: - A pandas Series with binary classification targets. - """ - return (y > threshold).astype(int) - - def _preprocess_data( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> Tuple[ - pd.Series, - Optional[pd.DataFrame], - Optional[pd.Series], - Optional[pd.DataFrame], - ]: - """ - Internal method to handle common data preprocessing operations. - - Args: - y: The target time series data - X: The feature matrix (including exogenous features) - X_val: Validation feature matrix (optional) - y_val: Validation target series (optional) - - Returns: - A tuple containing: - - processed y series - - processed X dataframe (optional) - - processed y_val series (optional) - - processed X_val dataframe (optional) - """ - # Apply differentiation if enabled - if self.differentiate_target: - y = y.diff().dropna() - if X is not None: - X = X.loc[y.index] - - # Transform target for classification if needed - if self.learning_task == "binary": - y = self._transform_target_to_binary(y) - elif self.learning_task == "multiclass": - y = self._transform_target_to_multiclass(y, self.bins) - - # Process validation data if provided - if y_val is not None: - if X_val is None: - raise ValueError( - "Validation features (X_val) must be provided if " - + "validation target (y_val) is given." - ) - y_val = y_val.loc[X_val.index] - if self.differentiate_target: - y_val = y_val.diff().dropna() - X_val = X_val.loc[y_val.index] - if self.learning_task == "binary": - y_val = self._transform_target_to_binary(y_val) - elif self.learning_task == "multiclass": - y_val = self._transform_target_to_multiclass(y_val, self.bins) - - # Filter features if selected_features_ is set - if X is not None and self.selected_features_ is not None: - X = cast(pd.DataFrame, X[self.selected_features_].copy()) - if X_val is not None: - X_val = cast( - pd.DataFrame, X_val[self.selected_features_].copy() - ) - - return y, X, y_val, X_val - - def _prepare_shap_data(self, X: pd.DataFrame) -> pd.DataFrame: - """Prepare data for SHAP analysis.""" - X_processed = X.copy() - - # Remove target column if present - if self.target_col in X_processed.columns: - X_processed = X_processed.drop(columns=[self.target_col]) - - # Filter selected features - if self.selected_features_ is not None: - X_processed = cast( - pd.DataFrame, X_processed[self.selected_features_].copy() - ) - - return X_processed - - def _create_shap_explainer(self, X: pd.DataFrame) -> Any: - """Create appropriate SHAP explainer based on model type.""" - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - - if hasattr(self.model_, "coef_"): # Linear models - try: - # Handle missing values if model supports it - X_clean = self._handle_missing_values_for_shap(X) - return shap.LinearExplainer(self.model_, X_clean) - except Exception as e: - console.print( - f"[yellow]Warning: Linear explainer failed: {e}, " - + "using KernelExplainer[/yellow]" - ) - background = shap.maskers.Independent(X, max_samples=100) - return shap.KernelExplainer( - self.model_.predict, - background, - ) - else: - # Non-linear models - if self.learning_task == "binary": - return shap.TreeExplainer( - self.model_, X, model_output="probability" - ) - else: - return shap.Explainer(self.model_, X) - - def _handle_missing_values_for_shap(self, X: pd.DataFrame) -> pd.DataFrame: - """Handle missing values for SHAP analysis.""" - # Use type ignore for optional method - if hasattr(self.model_, "_impute_missing_values"): - return self.model_._impute_missing_values(X) # type: ignore - else: - return X.dropna() - - def _generate_and_save_plot( - self, explainer: Any, X: pd.DataFrame, path: str - ) -> None: - """Generate and save SHAP plot.""" - shap_values = explainer(X) - - shap.plots.beeswarm(shap_values, show=False) - shap_fig = plt.gcf() - shap_fig.set_size_inches(10, 6) - shap_fig.suptitle(f"SHAP Beeswarm Plot for {self.name}", fontsize=16) - shap_fig.tight_layout() - shap_fig.savefig(path) - plt.clf() - plt.close() - - @ensure_fitted - def shap_beeswarm_plot(self, X: pd.DataFrame, path: str) -> None: - """ - Generates a SHAP beeswarm plot for the model's predictions. - - Args: - X: DataFrame of input features. - path: Path to save the plot file. - - Raises: - ValueError: If model is not fitted. - Exception: If SHAP plot generation fails. - """ - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - - try: - # Prepare data - X_processed = self._prepare_shap_data(X) - - # Create explainer and generate plot - explainer = self._create_shap_explainer(X_processed) - self._generate_and_save_plot(explainer, X_processed, path) - - except Exception as e: - console.print( - f"[red]Error: Failed to generate SHAP plot: {e}[/red]" - ) - raise - - def get_params_dict(self) -> dict: - """Get model parameters as dictionary for logging/serialization.""" - base_params = super().get_params_dict() - mv_params = { - "learning_task": self.learning_task, - "differentiate_target": self.differentiate_target, - "selected_features_": self.selected_features_, - } - return {**base_params, **mv_params} - - def summary(self) -> str: - """Generate a summary string of the model.""" - params = self.get_params_dict() - fitted_status = ( - "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" - ) - - summary_lines = [ - f"Model: {self.__class__.__name__}", - f"Status: {fitted_status}", - f"Features: {params.get('n_features_in_', 'Unknown')}", - f"Task: {params.get('learning_task', 'regression')}", - f"Selected Features: {len(self.selected_features_) if self.selected_features_ else 'All'}", - ] - - return "\n".join(summary_lines) From c7544d0082a3afb549db9ad18e0d45ab53c3026c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 15 Sep 2025 14:47:46 -0300 Subject: [PATCH 06/52] SIENTIAPDE-1222 Remove obsolete data files: deleted data.csv and response_data.csv to streamline project structure and eliminate unused resources. --- data.csv | 11 ----------- response_data.csv | 2 -- 2 files changed, 13 deletions(-) delete mode 100644 data.csv delete mode 100644 response_data.csv diff --git a/data.csv b/data.csv deleted file mode 100644 index 76acc92..0000000 --- a/data.csv +++ /dev/null @@ -1,11 +0,0 @@ -timestamp,07BP012/VEL_M1_PV,07BP013/VEL_M1_PV,07BP014/VEL_M1_PV,07FT001_COR_B,07FT001_COR_G,07FT001_COR_R,07FT001_TEXT,07FT007_COR_B,07FT007_COR_G,07FT007_COR_R,07FT007_TEXT,07FT012_COR_B,07FT012_COR_G,07FT012_COR_R,07FT012_TEXT,09BP023/VEL_PV,09BP024/VEL_PV,303-WIT-230,305-AIC-001_PV,305-AIC-002_PV,305-CALC-001,305-FIC-001_PV,305-FIC-003_PV,305-FIC-005_PV,305-FIC-006_PV,305-FIT-002,305-FIT-009,305-FIT-010,305-FIT-011,305-FIT-012,305-FIT-013,305-LIC-001_PV,305-LIC-002_PV,305-PIT-170,305-PIT-175,305-SIC-001_SP,305-SIC-002_SP,305-WIT-135,305-WIT-160,306-CALC-018,306-DIT-001,306-DIT-002,306-FIT-003,306-FIT-004,306-FIT-005,306-FIT-006,306-FIT-051,306-FIT-052,306-LIC-001_PV,306-LIC-002_PV,306-LIC-003_PV,306-PIT-101,306-PIT-105,306-PIT-110,306-PIT-115,306-PIT-125,306-PIT-130,307-CALC-001,307-CALC-002,307-FIC-001_PV,307-FIC-005_PV,307-FIC-006_PV,307-FIC-019_PV,307-FIC-022,307-FIC-101_PV,307-FIC-105_PV,307-FIC-110_PV,307-FIC-115_PV,307-FIC-120_PV,307-FIC-130_PV,307-FIC-135_PV,307-FIC-140_PV,307-FIC-145_PV,307-FIC-150_PV,307-FIC-155_PV,307-FIC-160_PV,307-FIT-003,307-FIT-005,307-FIT-008,307-FIT-009,307-LIC-003_PV,307-LIC-004_PV,307-LIC-101_PV,307-LIC-105_PV,307-LIC-110_PV,307-LIC-115_PV,307-LIC-120_PV,307-LIC-130_PV,307-LIC-135_PV,307-LIC-140_PV,307-LIC-145_PV,307-LIC-150_PV,307-LIC-155_PV,307-LIC-160_PV,307-SIC-006_OUT,307-SIC-007_OUT,309-FIC-013,309-FIC-014,309-FIT-051,309-FIT-052,309-LIC-001_PV,309-LIC-002_PV,309-PIT-001,309-PIT-002,309-PIT-101,309-PIT-105,309-PIT-110,309-PIT-185,309-PIT-190,309-PIT-195,310-AIC-001_PV,310-AIC-002,310-CALC-001,310-CALC-002,310-DIC-002_PV,310-FIC-004_PV,310-FIC-010_PV,310-FIC-110_PV,310-FIC-120_PV,310-FIC-160_PV,310-FIC-170_PV,310-FIT-004,310-FIT-005,310-FIT-006,310-FIT-010,310-FV-032,310-LIC-110_PV,310-LIC-120_PV,310-LIC-160_PV,310-LIC-170_PV,310-LIT-003_PV,310-LIT-004_PV,310-SIC-003_OUT,310-SIC-004_OUT,310-SIC-005_OUT,310-SIC-006_OUT,311-FIC-029_PV,311-FIC-033_PV,311-FIT-033,312-CALC-001,312-CALC-005,312-CALC-006,312-DIC-001_PV,312-DIC-002_PV,312-FIC-001_PV,312-FIC-002_PV,313-CALC-001,313-DIC-001_PV,313-DIC-002_SP,313-FIC-006_PV,317AIT001.2,317AIT002.1,317AIT002.10,317AIT002.11,317AIT002.12,317AIT002.13,317AIT002.14,317AIT002.15,317AIT002.16,317AIT002.17,317AIT002.18,317AIT002.19,317AIT002.2,317AIT002.20,317AIT002.21,317AIT002.22,317AIT002.23,317AIT002.24,317AIT002.25,317AIT002.26,317AIT002.27,317AIT002.28,317AIT002.29,317AIT002.3,317AIT002.30,317AIT002.31,317AIT002.32,317AIT002.33,317AIT002.34,317AIT002.35,317AIT002.36,317AIT002.37,317AIT002.38,317AIT002.39,317AIT002.4,317AIT002.40,317AIT002.41,317AIT002.42,317AIT002.43,317AIT002.44,317AIT002.45,317AIT002.46,317AIT002.47,317AIT002.48,317AIT002.49,317AIT002.5,317AIT002.50,317AIT002.51,317AIT002.52,317AIT002.53,317AIT002.54,317AIT002.55,317AIT002.56,317AIT002.57,317AIT002.58,317AIT002.59,317AIT002.6,317AIT002.60,317AIT002.61,317AIT002.62,317AIT002.63,317AIT002.64,317AIT002.65,317AIT002.66,317AIT002.67,317AIT002.68,317AIT002.69,317AIT002.7,317AIT002.70,317AIT002.71,317AIT002.72,317AIT002.8,317AIT002.9,317AIT003.1,317AIT003.2,317AIT003.3,317AIT003.5,319-CALC-001,319-CALC-013,319-DIC-001_PV,319-DIC-001_SP,319-DIC-002_PV,319-DIC-002_SP,319-FIC-006_PV,319-FIC-006_SP,319-FIC-007_PV,319-FIC-007_SP,319-FIQ-CALC-009_DAY,319-FIT-004,319-FIT-005,319-LIT-201-R,319-PIT-003,319-PIT-004,319-SIC-001_OUT,319-SIC-002_OUT,Fe_conc,G03-07BP102_M1,G03-07BP103_M1,G03-08BP107_M1,G03-10BP104_M1,G03-19BP101_M1,G03-19BP106_M1,G03-19BP110_M1,SOL-CALC-005,SOL-CALC-006,SiO2_conc -2024-12-05 02:16:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2185.6279296875,8.909636497497559,8.824532508850098,2628.087646484375,233.4102020263672,289.2882995605469,2.2411208152771,2.740176200866699,3158.830322265625,687.0074462890625,0.0,287.7219543457031,0.0,3312.36962890625,89.0501937866211,83.04390716552734,12.823511123657228,13.373872756958008,,,1364.865478515625,1239.42919921875,3.005729913711548,1.3840404748916626,1.3935494422912598,2170.908447265625,2189.05615234375,1866.9991455078125,1715.1016845703125,2254.970703125,2276.34033203125,83.4413070678711,94.47754669189452,89.85079956054688,34.401798248291016,33.98549270629883,31.97739601135254,34.75410461425781,43.70286178588867,44.13051223754883,1894.9356689453125,2017.797607421875,900.0048217773438,30.6091537475586,375.9732971191406,0.0,652.0335693359375,765.3798217773438,763.6800537109375,769.5640258789062,739.6097412109375,719.44775390625,706.0026245117188,703.8912353515625,567.9400634765625,593.1154174804688,629.06787109375,595.2117919921875,625.0,1.425487995147705,27.0,1194.873046875,998.1407470703124,105.17456817626952,99.1063003540039,28.07830810546875,21.692842483520508,34.955604553222656,12.697091102600098,40.85791778564453,21.66144752502441,17.217727661132812,47.85460662841797,57.36534118652344,59.91471481323242,48.2706184387207,39.44066619873047,83.20423126220703,66.5,0.0,0.0,1768.509765625,1827.390380859375,85.6709213256836,96.77904510498048,0.0998583808541297,4.900833606719971,23.81103706359864,26.791275024414062,26.24358367919922,23.354522705078125,27.844594955444336,27.89141845703125,10.390382766723633,10.471061706542969,223.273666381836,209.89990234375,1.1763592958450315,2.309485912322998,103.65160369873048,450.9481201171875,427.9828186035156,450.59222412109375,448.7367858886719,17.0,700.9956665039062,0.0,1200.0,1.0,39.16987991333008,37.460086822509766,19.4918155670166,11.98000144958496,95.19344329833984,86.53215789794922,,,86.0,99.30213165283205,22.59608268737793,0.0,400.0,1351.7703857421875,0.0,1459.112548828125,2.163416624069214,1.6798019409179688,997.8304443359376,0.0240168757736682,203.056381225586,1.369994044303894,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.4978864192962646,8.562295913696289,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.253314971923828,13.389976501464844,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.01197052001953,1.7381054162979126,1.1206940412521362,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,59.35650634765625,1.4764769077301023,1.526507019996643,0.4823205173015594,0.7710468769073486,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3361487984657287,0.428571492433548,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,7.89943265914917,5.800000190734863,6.465214729309082,86.5999984741211,830.4485473632812,849.3055419921875,,,1.5839204788208008,1.572644829750061,0.0,600.0,920.1913452148438,921.7091064453124,0.0,10.023720741271973,0.0,11.650277137756348,4.348147869110107,0.0748394280672073,71.11778259277344,100.0,64.71,0.0,4.533299922943115,5.666272640228272,8.773231506347656,4.993200302124023,4.861800193786621,6.432722568511963,50.96606063842773,67.95718383789062,3.52 -2024-12-05 02:18:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2300.107421875,8.874933242797852,8.814615249633789,2379.468017578125,205.70236206054688,299.0367736816406,2.237759590148926,2.737307071685791,3149.414306640625,705.9229125976562,0.0,287.82403564453125,0.0,3319.203857421875,88.97682189941406,82.33074951171875,13.035848617553713,13.303841590881348,,,801.2099609375,1577.420166015625,3.18897008895874,1.3844648599624634,1.3944939374923706,2137.277099609375,2091.429931640625,1866.9801025390625,1714.9752197265625,2248.451904296875,2272.76025390625,85.66764831542969,88.35608673095703,87.84507751464844,34.351341247558594,33.97214126586914,31.98730659484864,32.71706008911133,43.68099594116211,44.0351676940918,1898.191162109375,1618.9393310546875,898.7774047851562,30.54461669921875,374.732666015625,0.0,648.9142456054688,761.4437866210938,761.818115234375,728.4386596679688,741.0633544921875,659.6345825195312,707.1867065429688,684.1427612304688,568.2828369140625,594.7976684570312,633.2908325195312,595.1723022460938,640.0,1.245144605636597,27.0,1194.7637939453125,998.0435791015624,103.9820556640625,99.07530975341795,28.694652557373047,22.18309211730957,31.824302673339844,12.703847885131836,38.318546295166016,21.9743595123291,16.507755279541016,47.80324935913086,57.33625793457031,59.91497421264648,48.0490837097168,40.25405502319336,83.34803771972656,66.5,0.0,0.0,1746.5220947265625,1811.0615234375,85.64175415039062,96.97100067138672,0.0999280512332916,4.900284290313721,23.82648658752441,26.91034507751465,26.227479934692383,23.32929229736328,27.45798110961914,27.736440658569336,10.399251937866213,10.47207736968994,175.54986572265625,211.37567138671875,1.174445867538452,2.112058639526367,103.86756896972656,450.9291076660156,427.931396484375,450.5900268554688,448.73138427734375,17.0,700.9699096679688,10.513471603393556,1200.0,1.0,40.18443298339844,37.6434326171875,19.451520919799805,11.894670486450195,95.58326721191406,86.0557632446289,,,86.0,98.96598815917967,22.82772254943848,0.0,400.0,968.2177734375,0.0,1464.4488525390625,2.163362979888916,1.679788589477539,1003.175537109375,0.0238033775240182,184.37130737304688,1.3698077201843262,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,8.57795238494873,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,13.411721229553224,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1217609643936155,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,59.35650634765625,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7702381610870361,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4283382296562195,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,8.100000381469727,5.800000190734863,6.435592651367188,86.5999984741211,725.95458984375,858.4331665039062,,,1.5835398435592651,1.5726622343063354,0.0,600.0,917.3837280273438,925.9874877929688,0.0,10.049739837646484,0.0,11.572147369384766,4.3472161293029785,0.0748393461108207,71.409423828125,100.0,64.71,0.0,4.533299922943115,5.762217998504639,8.699999809265137,4.993200302124023,4.861800193786621,6.558172702789307,50.968997955322266,67.96240234375,3.52 -2024-12-05 02:20:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2063.14990234375,8.86483097076416,8.804698944091797,2637.2822265625,186.7984161376953,274.3067932128906,2.234398603439331,2.734437942504883,3139.637939453125,738.9068603515625,0.0,287.9261474609375,0.0,3328.472412109375,84.00076293945312,86.40196990966797,12.14862060546875,13.639047622680664,,,1477.9354248046875,1141.748291015625,2.891580104827881,1.3848892450332642,1.3954384326934814,2059.924560546875,2129.599853515625,1866.961181640625,1714.848876953125,2241.93310546875,2266.62158203125,85.53884887695312,87.77095031738281,89.44815063476562,34.300880432128906,33.95878982543945,31.997217178344727,30.499488830566406,43.65913391113281,44.02417755126953,1907.386474609375,1944.6339111328125,903.0270385742188,30.480079650878903,374.2165832519531,0.0,649.119384765625,757.221435546875,759.9561767578125,811.8240356445312,748.2200927734375,740.3225708007812,708.370849609375,673.0714721679688,573.4213256835938,596.4798583984375,631.8908081054688,595.1328735351562,635.0,1.0720911026000977,27.0,1194.6546630859375,997.9464111328124,105.5190887451172,99.0443115234375,28.1169376373291,21.61070251464844,32.5518684387207,12.71060562133789,37.97523498535156,22.30083274841309,17.719982147216797,47.75189208984375,57.30717849731445,59.91522979736328,47.62279891967773,41.22840881347656,83.33912658691406,66.5,0.0,0.0,1749.242431640625,1821.6829833984373,85.61257934570312,97.1629638671875,0.0999977141618728,4.899734973907471,23.841936111450195,26.64358901977539,26.1314697265625,23.415620803833008,27.687856674194336,27.292091369628903,10.39584255218506,10.473093032836914,208.32379150390625,211.2283935546875,1.1736302375793457,1.0507607460021973,104.0835418701172,450.91009521484375,427.87994384765625,450.58782958984375,448.7259826660156,17.0,700.9442138671875,0.0,1200.0,1.0,40.472747802734375,38.068607330322266,19.54105758666992,12.325950622558594,95.0851821899414,85.76668548583984,,,86.0,98.85044860839844,22.84588432312012,0.0,400.0,1176.9923095703125,0.0,1459.0894775390625,2.1633095741271973,1.6797752380371094,997.1138916015624,0.0235898792743682,204.41549682617188,1.3696213960647583,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,8.100000381469727,5.400000095367432,6.405970096588135,86.5999984741211,802.98876953125,856.4476318359375,,,1.583159327507019,1.57267963886261,0.0,600.0,935.9249267578124,930.265869140625,0.0,10.131684303283691,0.0,11.596683502197266,4.34628438949585,0.0748392716050148,71.35333251953125,100.0,64.71,0.0,4.533299922943115,5.731375694274902,8.699999809265137,4.993200302124023,4.861800193786621,6.488824367523193,50.97193908691406,67.96761322021484,3.52 -2024-12-05 02:22:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2344.354736328125,8.854728698730469,8.794782638549805,2443.620849609375,235.85362243652344,289.5343017578125,2.231037378311157,2.7315685749053955,3128.458251953125,678.6727905273438,0.0,288.0282287597656,0.0,3352.32080078125,82.41395568847656,88.41472625732422,12.335658073425291,13.590455055236816,,,1179.0438232421875,1241.806396484375,2.9580600261688232,1.3853135108947754,1.3963829278945925,2020.5482177734373,2175.6435546875,1866.942138671875,1714.722412109375,2241.09228515625,2263.143798828125,85.41004943847656,89.18098449707031,91.12660217285156,34.372066497802734,33.94544219970703,32.00712585449219,30.11072158813477,43.63727188110352,44.04359436035156,1842.406005859375,1904.2274169921875,902.5571899414062,30.415542602539062,374.7893676757813,0.0,651.5109252929688,768.8901977539062,758.0942993164062,720.6381225585938,742.55078125,685.2013549804688,709.5549926757812,730.93359375,572.37548828125,598.1620483398438,634.2003173828125,595.0933837890625,632.0,0.9020777940750122,27.0,1194.54541015625,997.8492431640624,104.99007415771484,99.01332092285156,28.318359375,21.52416229248047,31.58490943908692,12.717362403869627,38.4969482421875,21.7913761138916,17.436378479003906,47.70053482055664,57.278099060058594,59.915489196777344,46.977535247802734,42.46583938598633,83.07581329345703,66.5,0.0,0.0,1748.264892578125,1827.9678955078125,85.58340454101562,97.35491943359376,0.1000673845410347,4.899185180664063,23.85738754272461,26.20622062683105,26.06732177734375,23.50194931030273,27.590068817138672,27.46240234375,10.392244338989258,10.47410774230957,214.98492431640625,205.6874847412109,1.173506498336792,2.398390531539917,104.2995147705078,450.8910827636719,427.8284912109375,450.5856323242188,448.7205810546875,17.0,700.9185180664062,0.0,1200.0,1.0,40.27254486083984,38.66741561889648,19.67565155029297,12.237468719482422,93.86233520507812,85.74674987792969,,,86.0,98.91876983642578,22.30738639831543,0.0,400.0,1275.9793701171875,0.0,1467.3343505859375,2.1632559299468994,1.6797618865966797,998.2872314453124,0.0233763810247182,187.3364105224609,1.3694350719451904,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.708251953125,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.3545424044132232,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3020144402980804,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.100000381469727,5.400000095367432,6.699999809265137,85.80000305175781,747.72314453125,830.0194091796875,,,1.5827786922454834,1.5726970434188845,0.0,600.0,924.1300048828124,934.5442504882812,0.0,10.00216579437256,0.0,11.625198364257812,4.345353126525879,0.0748391896486282,71.52710723876953,100.0,64.71,0.0,4.533299922943115,5.767271518707275,8.699999809265137,4.993200302124023,4.861800193786621,6.6877641677856445,50.97750854492188,67.97283172607422,3.52 -2024-12-05 02:24:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2862.970703125,8.844626426696777,8.774948120117188,2980.736572265625,237.5259704589844,291.2091369628906,2.2276761531829834,2.728699445724488,3153.987548828125,682.5768432617188,0.0,288.1303405761719,0.0,3345.968505859375,87.24137878417969,86.74526977539062,12.552753448486328,13.396173477172852,,,1370.993896484375,1611.220947265625,2.8518500328063965,1.3857378959655762,1.397327542304993,2030.3245849609373,2142.251953125,1866.9232177734373,1714.6802978515625,2248.769775390625,2268.414794921875,85.06763458251953,91.27864837646484,90.73202514648438,34.44520568847656,33.932090759277344,32.01703643798828,31.01426696777344,43.61540985107422,44.06300735473633,1867.2406005859373,2242.3193359375,898.3688354492188,30.811418533325195,375.3621520996094,0.0,647.7444458007812,756.3349609375,759.6863403320312,795.988525390625,743.1278076171875,681.9320678710938,710.7391357421875,760.3604736328125,576.6845092773438,599.8442993164062,620.6749267578125,595.053955078125,640.0,0.7632204294204712,27.0,1194.4361572265625,997.7520751953124,106.02984619140624,98.98233032226562,28.794273376464844,21.658815383911133,32.04315185546875,12.724120140075684,38.84426498413086,21.82027244567871,16.058849334716797,47.64917755126953,57.24901580810547,59.915748596191406,46.44183349609375,41.52559280395508,83.30157470703125,66.5,0.0,0.0,1750.5181884765625,1845.0968017578125,85.55422973632812,97.546875,0.1001370549201965,4.8986358642578125,23.87283706665039,26.4957332611084,26.47050094604492,23.5178451538086,27.697071075439453,27.838388442993164,10.388647079467772,10.475123405456545,243.6897125244141,207.8060302734375,1.1728342771530151,1.2688955068588257,104.5154800415039,450.8720397949219,427.7770690917969,450.58343505859375,448.7151794433594,17.0,700.892822265625,0.0,1200.0,1.0,40.07234191894531,37.74433898925781,19.810243606567383,12.158288955688477,91.9724578857422,85.08606719970703,,,86.0,98.60803985595705,22.23607635498047,0.0,400.0,1401.1199951171875,0.0,1445.149658203125,2.1632025241851807,1.67974853515625,1001.5813598632812,0.0231628827750682,230.4506072998047,1.3692487478256226,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.899999618530273,5.400000095367432,6.699999809265137,85.80000305175781,927.35986328125,843.9348754882812,,,1.5823981761932373,1.5727144479751587,0.0,600.0,943.0855712890624,939.0200805664062,0.0,10.032031059265137,0.0,11.4324369430542,4.34442138671875,0.0748391151428222,71.41386413574219,100.0,64.71,0.0,4.533299922943115,5.681782245635986,8.699999809265137,4.993200302124023,4.861800193786621,6.551279544830322,50.98702239990234,67.97804260253906,3.52 -2024-12-05 02:26:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2654.10888671875,8.834524154663086,8.75146198272705,2392.275390625,235.610580444336,295.8012084960937,2.2243149280548096,2.725830078125,3172.66748046875,694.5460205078125,0.0,288.2324523925781,0.0,3339.616455078125,89.53855895996094,85.1905746459961,12.741495132446287,13.875475883483888,,,1323.8238525390625,1067.496337890625,2.916759967803955,1.386162281036377,1.3982720375061035,2102.00048828125,2161.18212890625,1866.9041748046875,1714.821044921875,2259.66064453125,2275.4189453125,84.60967254638672,90.81041717529295,90.06828308105469,34.51834487915039,33.918739318847656,32.026947021484375,32.243064880371094,43.593544006347656,44.08242416381836,1909.4129638671875,1879.74658203125,902.9876708984376,31.21604347229004,375.9349365234375,0.0,650.2518310546875,751.2490234375,761.6719970703125,715.2727661132812,747.89599609375,747.788818359375,706.99658203125,774.9991455078125,583.5380249023438,601.5264892578125,614.032958984375,595.0144653320312,615.0,0.7071666121482849,27.0,1194.326904296875,997.6549682617188,104.1503677368164,98.95133209228516,28.68854713439941,21.7934684753418,32.87420654296875,12.730876922607422,38.9033317565918,22.107240676879883,17.046525955200195,47.59782028198242,57.21993637084961,59.9160041809082,47.52975463867188,40.58535003662109,83.56719970703125,66.5,0.0,0.0,1742.1806640625,1809.867919921875,85.52505493164062,96.9273681640625,0.1002067178487777,4.8980865478515625,23.88828659057617,26.306615829467773,26.495014190673828,23.27698135375977,27.35708808898925,27.72958755493164,10.385048866271973,10.4761381149292,216.2808380126953,213.6409606933593,1.1716774702072144,2.5998244285583496,104.73145294189452,450.85302734375,427.793212890625,450.5812377929688,448.7097778320313,17.0,700.8671264648438,0.0148877017199993,1200.0,1.0,39.87213897705078,37.95595169067383,19.94483757019043,12.006688117980955,91.47502899169922,85.28714752197266,,,86.0,98.45233917236328,22.66000938415528,0.0,400.0,1256.264892578125,0.0,1452.758544921875,2.163148880004883,1.6797351837158203,1000.3143920898438,0.0229493845254182,190.5460968017578,1.369062423706055,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.899999618530273,3.299999952316284,6.699999809265137,85.80000305175781,762.7355346679688,865.4536743164062,,,1.5816150903701782,1.572731852531433,0.0,600.0,938.32177734375,943.602783203125,0.0,10.15860080718994,0.0,11.596125602722168,4.343489646911621,0.0748390331864357,71.58973693847656,100.0,64.71,0.0,4.533299922943115,5.802553176879883,8.699999809265137,4.993200302124023,4.861800193786621,6.558778762817383,50.99653625488281,67.98326110839844,3.52 -2024-12-05 02:28:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2227.77099609375,8.836018562316895,8.75161361694336,2456.804931640625,238.3328552246093,283.34686279296875,2.220953941345215,2.722960948944092,3188.196533203125,818.465087890625,0.0,288.33453369140625,0.0,3333.26416015625,90.52090454101562,83.3249282836914,12.903297424316406,13.22053337097168,,,1122.2493896484375,1331.5670166015625,3.0864999294281006,1.3865865468978882,1.3992165327072144,2095.321533203125,2129.77685546875,1866.88525390625,1714.9619140625,2259.91357421875,2288.202880859375,84.55709075927734,89.37647247314453,88.79876708984375,34.55312728881836,33.905391693115234,32.03685760498047,31.44712448120117,43.62167739868164,44.101837158203125,2035.30908203125,1930.71044921875,899.136474609375,32.6287956237793,376.5077209472656,0.0,653.2977294921875,750.5552368164062,760.983642578125,770.3090209960938,738.3396606445312,653.3170776367188,701.0360107421875,730.8182983398438,566.891845703125,588.6995239257812,629.2593383789062,594.9750366210938,622.0,0.6648255586624146,27.0,1194.2176513671875,997.5578002929688,105.18731689453124,98.92034149169922,28.36734771728516,21.92812156677246,32.240821838378906,12.737634658813477,38.52233505249024,21.57743263244629,17.07017707824707,47.54646301269531,57.19085311889648,59.916263580322266,48.34513473510742,39.64510345458984,83.13888549804688,66.5,0.0,0.0,1753.1981201171875,1814.801513671875,85.49588012695312,96.34169006347656,0.1002763882279396,4.8975372314453125,23.903738021850582,26.488170623779297,26.35142517089844,23.256576538085938,27.712129592895508,27.556394577026367,10.381451606750488,10.477153778076172,218.41183471679688,225.77491760253903,1.1706732511520386,1.818994283676148,104.94741821289062,450.8340148925781,428.7974853515625,450.57904052734375,448.7043762207031,17.0,700.8414306640625,0.5039713978767395,1200.0,1.0,39.67193984985352,38.82048416137695,20.07943153381348,12.295125961303713,90.39128875732422,85.13963317871094,,,86.0,98.48783111572266,24.05027961730957,0.0,400.0,1260.049560546875,0.0,1470.87890625,2.163095474243164,1.6797218322753906,1004.2244873046876,0.0227358862757682,194.2509765625,1.3688760995864868,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.86962890625,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,7.830216407775879,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042373403906822,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.7188185453414917,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7221007943153381,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.49777603149414,65.73210906982422,8.899999618530273,3.299999952316284,6.5,86.30000305175781,790.286376953125,913.74462890625,,,1.5807862281799316,1.572749376296997,0.0,600.0,945.046875,945.7421264648438,0.0,10.257745742797852,0.0,11.62919807434082,4.342557907104492,0.0748389586806297,71.7236099243164,100.0,64.71,0.0,4.533299922943115,5.762265205383301,8.699999809265137,4.993200302124023,4.861800193786621,6.415340900421143,51.00605010986328,67.98847961425781,3.52 -2024-12-05 02:30:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2773.8876953125,8.840205192565918,8.76478099822998,2499.578125,233.58840942382807,273.2201843261719,2.217592716217041,2.720091819763184,3169.754638671875,718.6017456054688,0.0,288.4366455078125,0.0,3293.60546875,88.42021942138672,80.45106506347656,12.661704063415527,13.48155403137207,,,1310.0872802734375,1188.9476318359375,3.5530900955200195,1.387010931968689,1.4001611471176147,2047.4097900390625,2157.358642578125,1866.8662109375,1715.1026611328125,2255.59716796875,2283.900390625,84.08345794677734,86.43714904785156,89.38142395019531,34.57453536987305,34.07208633422852,32.04676818847656,29.560300827026367,43.759544372558594,44.12125396728516,2028.601806640625,1870.660400390625,895.9712524414062,32.81827163696289,377.08050537109375,0.0,649.1517333984375,770.4885864257812,759.2982788085938,734.395751953125,743.6987915039062,678.6212158203125,700.5547485351562,663.7391967773438,576.5570678710938,589.0308837890625,628.8489990234375,594.935546875,631.0,0.6459924578666687,27.0,1194.1085205078125,997.4606323242188,105.77910614013672,98.88935089111328,28.33451271057129,22.062774658203125,32.428157806396484,12.744391441345217,38.12985610961914,22.12809181213379,17.180856704711914,47.4951057434082,57.16177368164063,59.91652297973633,49.058895111083984,39.42089080810547,83.53347778320312,66.5,0.0,0.0,1742.7498779296875,1810.687255859375,85.46670532226562,97.0500717163086,0.1003460586071014,4.896987915039063,23.919187545776367,26.559722900390625,26.710390090942383,23.56723022460937,27.740917205810547,27.745880126953125,10.377854347229004,10.478169441223145,205.0146484375,225.3446502685547,1.1705199480056765,2.214766025543213,105.16339111328124,450.8149719238281,429.5928039550781,450.5768432617188,448.698974609375,17.0,700.815673828125,0.239432543516159,1200.0,1.0,37.73193740844727,38.19848251342773,20.21402359008789,11.93883228302002,88.80018615722656,84.96267700195312,,,86.0,98.27005767822266,24.427764892578125,0.0,400.0,1261.719482421875,0.0,1463.796630859375,2.163041830062866,1.679708480834961,993.51123046875,0.0225223880261182,192.6799774169922,1.368689775466919,1.3799999952316284,0.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,761.5707397460938,910.4165649414062,,,1.5799574851989746,1.5727667808532717,0.0,600.0,953.1422119140624,947.8602294921876,0.0,10.270873069763184,0.0,11.646549224853516,4.341626167297363,0.0748388767242431,71.84945678710938,100.0,64.71,0.0,4.533299922943115,5.71589994430542,8.699999809265137,4.993200302124023,4.861800193786621,6.311936378479004,51.01556396484375,67.99369049072266,3.52 -2024-12-05 02:32:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,3052.78662109375,8.844391822814941,8.777948379516602,2515.992431640625,229.52468872070312,265.2821044921875,2.214231491088867,2.7172224521636963,3156.99365234375,701.08251953125,0.0,288.5387268066406,0.0,3317.0087890625,86.63294982910156,82.62700653076172,12.9373197555542,12.94904613494873,,,1328.0621337890625,1193.599609375,3.41225004196167,1.3874353170394895,1.4011056423187256,1862.8626708984373,2177.4072265625,1866.84716796875,1715.2435302734375,2251.281005859375,2277.60693359375,84.24079895019531,84.25446319580078,89.87091827392578,34.595943450927734,34.107933044433594,32.05667495727539,24.58193588256836,43.89741134643555,44.14066696166992,2033.9571533203125,1965.69189453125,902.446533203125,32.7397346496582,377.6492614746094,0.0,647.1126708984375,740.5242309570312,757.6129760742188,745.9622192382812,750.0130004882812,749.8059692382812,700.0735473632812,701.1631469726562,570.8465576171875,589.3622436523438,632.1378173828125,594.8961181640625,628.0,0.6863186955451965,27.0,1193.999267578125,997.3634643554688,103.6092300415039,98.8583526611328,28.94314765930176,22.19742774963379,32.61549377441406,12.75114917755127,37.71183013916016,22.179346084594727,17.744470596313477,47.44374847412109,57.132694244384766,59.916778564453125,48.76108932495117,40.08521270751953,83.61792755126953,66.5,0.0,0.0,1746.4300537109375,1854.5777587890625,85.64894104003906,97.2160415649414,0.1004157289862632,4.8964385986328125,23.925472259521484,26.834339141845703,26.46601295471192,23.520160675048828,27.76211738586425,27.71584892272949,10.374256134033203,10.4791841506958,218.3724822998047,225.1285858154297,1.1713463068008425,2.5177488327026367,105.52578735351562,450.79595947265625,429.6829833984375,450.57464599609375,448.6935729980469,17.0,700.7899780273438,1.7021775245666504,1200.0,1.0,36.82808303833008,38.11102294921875,20.06475257873535,11.937515258789062,89.86893463134766,84.6894760131836,,,86.0,98.14891815185548,24.32128524780273,0.0,400.0,1259.343017578125,0.0,1462.4814453125,2.1629884243011475,1.6796951293945312,998.4035034179688,0.0223088879138231,198.8683013916016,1.368503451347351,1.3799999952316284,0.0,,61.264404296875,16.665037155151367,5.333080291748047,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.86962890625,4.85367488861084,73.36161041259766,91.94583892822266,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,7.830216407775879,38.73592758178711,18.59975242614746,17.211471557617188,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042373403906822,1.970276951789856,0.4222961962223053,0.0943225920200347,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.7188185453414917,0.3608308732509613,1.2047821283340454,1.6332342624664309,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7221007943153381,0.2941087782382965,-9999.0,0.5667175650596619,0.7008373737335205,0.8771045207977295,8.49777603149414,65.73210906982422,12.899999618530272,5.300000190734863,6.515639781951904,86.30000305175781,790.82958984375,907.0885620117188,,,1.5791287422180176,1.572784185409546,0.0,600.0,950.763671875,949.9783325195312,0.0,10.146312713623049,0.0,11.57530403137207,4.340694427490234,0.0748387947678566,71.91187286376953,100.0,64.71,0.0,4.533299922943115,5.70755672454834,8.692553520202637,4.993200302124023,4.861800193786621,6.372900485992432,51.02507781982422,67.99890899658203,3.52 -2024-12-05 02:34:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2978.102783203125,8.848578453063965,8.862582206726074,2573.7099609375,234.7885284423828,289.6545104980469,2.2108702659606934,2.714353322982788,3147.3876953125,705.0564575195312,0.0,288.6408386230469,0.0,3398.574462890625,86.58733367919922,89.01890563964844,12.617484092712402,13.247730255126951,,,1347.5814208984375,1234.0552978515625,3.514620065689087,1.387859582901001,1.4020501375198364,1821.7401123046875,2152.696533203125,1866.8282470703125,1715.38427734375,2248.69091796875,2274.983642578125,84.39221954345703,88.17390441894531,90.19397735595705,34.61734771728516,34.1109504699707,32.32330322265625,23.76468849182129,43.90478515625,44.16008377075195,1927.233642578125,1939.23876953125,903.4033203125,31.51068115234375,378.1936645507813,0.0,648.6610107421875,766.79345703125,755.9276123046875,794.9697265625,744.4769287109375,656.0175170898438,699.5923461914062,772.5515747070312,569.7051391601562,589.693603515625,635.4265747070312,594.8566284179688,633.0,0.7698317170143127,27.0,1193.8900146484375,997.2662963867188,103.9173355102539,98.82736206054688,29.941625595092773,22.332080841064453,32.80282974243164,12.757905960083008,37.37025833129883,21.68112564086914,15.733065605163574,47.392391204833984,57.10361099243164,59.91703796386719,48.43916702270508,40.41853332519531,83.20599365234375,66.5,0.0,0.0,1749.7413330078125,1801.275634765625,85.9491195678711,97.38201904296876,0.1004853919148445,4.895888805389404,23.921628952026367,26.72721099853516,26.51607131958008,23.67252349853516,27.53003692626953,27.67759895324707,10.37065887451172,10.479698181152344,220.0579071044922,214.2723999023437,1.172289490699768,2.055137157440185,106.04012298583984,450.7769470214844,429.7731628417969,450.5724487304688,448.6881713867188,17.0,700.7642822265625,0.0,1200.0,1.0,37.02928161621094,38.0235595703125,19.88223648071289,12.143880844116213,91.3222427368164,85.17166900634766,,,86.0,98.23806762695312,23.680456161499023,0.0,400.0,1300.7486572265625,0.0,1472.4622802734375,2.1629347801208496,1.679681658744812,1000.1616821289062,0.0220953896641731,198.59475708007807,1.3683171272277832,1.3799999952316284,0.0,,61.264404296875,,5.333080291748047,,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,,12.386075973510742,,,,,,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.547714233,,,,,,,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,,,,,,,,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,,,,,,,42.07585144042969,,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,,,,,,,,,,,12.899999618530272,5.300000190734863,6.536253452301025,85.9000015258789,806.0762329101562,868.0042114257812,,,1.578299880027771,1.5728015899658203,0.0,600.0,963.1144409179688,951.73193359375,0.0,10.191020965576172,0.0,11.601633071899414,4.3397626876831055,0.0748387202620506,71.49508666992188,100.0,,,,,,,,,51.03459548950195,68.00411987304688, diff --git a/response_data.csv b/response_data.csv deleted file mode 100644 index cd1d691..0000000 --- a/response_data.csv +++ /dev/null @@ -1,2 +0,0 @@ -,303-WIT-230_median,303-WIT-230_std,303-WIT-230_min,303-WIT-230_max,305-WIT-135_median,305-WIT-135_std,305-WIT-135_min,305-WIT-135_max,305-WIT-160_median,305-WIT-160_std,305-WIT-160_min,305-WIT-160_max,305-PIT-170_median,305-PIT-170_std,305-PIT-170_min,305-PIT-170_max,305-PIT-175_median,305-PIT-175_std,305-PIT-175_min,305-PIT-175_max,305-FIT-002_median,305-FIT-002_std,305-FIT-002_min,305-FIT-002_max,305-FIT-013_median,305-FIT-013_std,305-FIT-013_min,305-FIT-013_max,306-PIT-101_median,306-PIT-101_std,306-PIT-101_min,306-PIT-101_max,306-FIT-051_median,306-FIT-051_std,306-FIT-051_min,306-FIT-051_max,306-DIT-001_median,306-DIT-001_std,306-DIT-001_min,306-DIT-001_max,306-PIT-105_median,306-PIT-105_std,306-PIT-105_min,306-PIT-105_max,306-FIT-052_median,306-FIT-052_std,306-FIT-052_min,306-FIT-052_max,306-DIT-002_median,306-DIT-002_std,306-DIT-002_min,306-DIT-002_max,306-PIT-115_median,306-PIT-115_std,306-PIT-115_min,306-PIT-115_max,306-FIT-004_median,306-FIT-004_std,306-FIT-004_min,306-FIT-004_max,306-PIT-110_median,306-PIT-110_std,306-PIT-110_min,306-PIT-110_max,306-FIT-003_median,306-FIT-003_std,306-FIT-003_min,306-FIT-003_max,306-PIT-125_median,306-PIT-125_std,306-PIT-125_min,306-PIT-125_max,306-FIT-005_median,306-FIT-005_std,306-FIT-005_min,306-FIT-005_max,306-PIT-130_median,306-PIT-130_std,306-PIT-130_min,306-PIT-130_max,306-FIT-006_median,306-FIT-006_std,306-FIT-006_min,306-FIT-006_max,307-FIT-005_median,307-FIT-005_std,307-FIT-005_min,307-FIT-005_max,307-FIT-003_median,307-FIT-003_std,307-FIT-003_min,307-FIT-003_max,307-FIC-022_median,307-FIC-022_std,307-FIC-022_min,307-FIC-022_max,310-FIT-005_median,310-FIT-005_std,310-FIT-005_min,310-FIT-005_max,307-FIT-008_median,307-FIT-008_std,307-FIT-008_min,307-FIT-008_max,307-FIT-009_median,307-FIT-009_std,307-FIT-009_min,307-FIT-009_max,309-PIT-101_median,309-PIT-101_std,309-PIT-101_min,309-PIT-101_max,309-PIT-105_median,309-PIT-105_std,309-PIT-105_min,309-PIT-105_max,309-PIT-110_median,309-PIT-110_std,309-PIT-110_min,309-PIT-110_max,309-PIT-185_median,309-PIT-185_std,309-PIT-185_min,309-PIT-185_max,309-PIT-190_median,309-PIT-190_std,309-PIT-190_min,309-PIT-190_max,309-PIT-195_median,309-PIT-195_std,309-PIT-195_min,309-PIT-195_max,309-FIT-051_median,309-FIT-051_std,309-FIT-051_min,309-FIT-051_max,309-FIT-052_median,309-FIT-052_std,309-FIT-052_min,309-FIT-052_max,309-PIT-001_median,309-PIT-001_std,309-PIT-001_min,309-PIT-001_max,309-PIT-002_median,309-PIT-002_std,309-PIT-002_min,309-PIT-002_max,317AIT003.3_median,317AIT003.3_std,317AIT003.3_min,317AIT003.3_max,317AIT003.5_median,317AIT003.5_std,317AIT003.5_min,317AIT003.5_max,317AIT003.1_median,317AIT003.1_std,317AIT003.1_min,317AIT003.1_max,317AIT003.2_median,317AIT003.2_std,317AIT003.2_min,317AIT003.2_max,317AIT002.37_median,317AIT002.37_std,317AIT002.37_min,317AIT002.37_max,317AIT002.49_median,317AIT002.49_std,317AIT002.49_min,317AIT002.49_max,317AIT002.61_median,317AIT002.61_std,317AIT002.61_min,317AIT002.61_max,317AIT002.38_median,317AIT002.38_std,317AIT002.38_min,317AIT002.38_max,317AIT002.50_median,317AIT002.50_std,317AIT002.50_min,317AIT002.50_max,317AIT002.62_median,317AIT002.62_std,317AIT002.62_min,317AIT002.62_max,317AIT002.39_median,317AIT002.39_std,317AIT002.39_min,317AIT002.39_max,317AIT002.51_median,317AIT002.51_std,317AIT002.51_min,317AIT002.51_max,317AIT002.63_median,317AIT002.63_std,317AIT002.63_min,317AIT002.63_max,317AIT002.40_median,317AIT002.40_std,317AIT002.40_min,317AIT002.40_max,317AIT002.52_median,317AIT002.52_std,317AIT002.52_min,317AIT002.52_max,317AIT002.64_median,317AIT002.64_std,317AIT002.64_min,317AIT002.64_max,317AIT002.41_median,317AIT002.41_std,317AIT002.41_min,317AIT002.41_max,317AIT002.53_median,317AIT002.53_std,317AIT002.53_min,317AIT002.53_max,317AIT002.65_median,317AIT002.65_std,317AIT002.65_min,317AIT002.65_max,317AIT002.42_median,317AIT002.42_std,317AIT002.42_min,317AIT002.42_max,317AIT002.54_median,317AIT002.54_std,317AIT002.54_min,317AIT002.54_max,317AIT002.66_median,317AIT002.66_std,317AIT002.66_min,317AIT002.66_max,317AIT002.43_median,317AIT002.43_std,317AIT002.43_min,317AIT002.43_max,317AIT002.55_median,317AIT002.55_std,317AIT002.55_min,317AIT002.55_max,317AIT002.67_median,317AIT002.67_std,317AIT002.67_min,317AIT002.67_max,317AIT002.44_median,317AIT002.44_std,317AIT002.44_min,317AIT002.44_max,317AIT002.56_median,317AIT002.56_std,317AIT002.56_min,317AIT002.56_max,317AIT002.68_median,317AIT002.68_std,317AIT002.68_min,317AIT002.68_max,317AIT002.45_median,317AIT002.45_std,317AIT002.45_min,317AIT002.45_max,317AIT002.57_median,317AIT002.57_std,317AIT002.57_min,317AIT002.57_max,317AIT002.69_median,317AIT002.69_std,317AIT002.69_min,317AIT002.69_max,317AIT002.46_median,317AIT002.46_std,317AIT002.46_min,317AIT002.46_max,317AIT002.58_median,317AIT002.58_std,317AIT002.58_min,317AIT002.58_max,317AIT002.70_median,317AIT002.70_std,317AIT002.70_min,317AIT002.70_max,317AIT002.47_median,317AIT002.47_std,317AIT002.47_min,317AIT002.47_max,317AIT002.59_median,317AIT002.59_std,317AIT002.59_min,317AIT002.59_max,317AIT002.71_median,317AIT002.71_std,317AIT002.71_min,317AIT002.71_max,317AIT002.48_median,317AIT002.48_std,317AIT002.48_min,317AIT002.48_max,317AIT002.60_median,317AIT002.60_std,317AIT002.60_min,317AIT002.60_max,317AIT002.72_median,317AIT002.72_std,317AIT002.72_min,317AIT002.72_max,SiO2_conc,timestamp -2024-12-05 04:00:00+0000,2344.354736328125,347.11413476082527,2063.14990234375,3052.78662109375,1323.8238525390625,199.25517177489738,801.2099609375,1477.9354248046875,1239.42919921875,188.20232896348458,1067.496337890625,1611.220947265625,12.741495132446287,0.2925100433215222,12.14862060546875,13.035848617553713,13.396173477172852,0.2659367570480025,12.94904613494873,13.875475883483888,3156.99365234375,17.90496592640638,3128.458251953125,3188.196533203125,3328.472412109375,18.341293358560456,3293.60546875,3352.32080078125,34.44520568847656,0.10727337707066699,34.300880432128906,34.595943450927734,2251.281005859375,6.914512409267933,2241.09228515625,2259.91357421875,1.3857378959655762,0.001162128441763176,1.3840404748916626,1.3874353170394895,33.95878982543945,0.06905080560339776,33.905391693115234,34.107933044433594,2275.4189453125,8.067717393234235,2263.143798828125,2288.202880859375,1.397327542304993,0.0025866991350633178,1.3935494422912598,1.4011056423187256,31.01426696777344,2.797667558372099,24.58193588256836,34.75410461425781,2157.358642578125,30.47011040139319,2091.429931640625,2189.05615234375,32.01703643798828,0.027139770722953593,31.97739601135254,32.05667495727539,2059.924560546875,88.54574101606774,1862.8626708984373,2170.908447265625,43.65913391113281,0.09432217658823967,43.593544006347656,43.89741134643555,1866.9232177734373,0.05200646609213417,1866.84716796875,1866.9991455078125,44.08242416381836,0.04344513322059127,44.02417755126953,44.14066696166992,1714.9619140625,0.1883131118977155,1714.6802978515625,1715.2435302734375,27.0,0.0,27.0,27.0,0.7632204294204712,0.28421935480519783,0.6459924578666687,1.425487995147705,649.1517333984375,2.0424248177883793,647.1126708984375,653.2977294921875,700.892822265625,0.07040149596071192,700.7899780273438,700.9956665039062,1194.4361572265625,0.29912346849236254,1193.999267578125,1194.873046875,997.7520751953124,0.26607758363281586,997.3634643554688,998.1407470703124,23.87283706665039,0.040718881893752515,23.81103706359864,23.925472259521484,26.559722900390625,0.2379241176870002,26.20622062683105,26.91034507751465,26.35142517089844,0.203982336891631,26.06732177734375,26.710390090942383,23.415620803833008,0.11592992262134358,23.256576538085938,23.56723022460937,27.697071075439453,0.15510993222349723,27.35708808898925,27.844594955444336,27.72958755493164,0.19091973703305398,27.292091369628903,27.89141845703125,1748.264892578125,7.86294792652368,1742.1806640625,1768.509765625,1821.6829833984373,15.953009331205266,1809.867919921875,1854.5777587890625,0.1001370549201965,0.00019079441009214591,0.0998583808541297,0.1004157289862632,4.8986358642578125,0.0015045608215440351,4.8964385986328125,4.900833606719971,6.507819890975952,0.12657004614335984,6.405970096588135,6.699999809265137,86.30000305175781,0.3732080423947149,85.80000305175781,86.5999984741211,8.5,1.6421334224036257,7.89943265914917,12.899999618530272,5.400000095367432,1.0432607765452175,3.299999952316284,5.800000190734863,1.2756186723709106,0.0,1.2756186723709106,1.2756186723709106,0.7121588587760925,0.0,0.7121588587760925,0.7121588587760925,0.4104396104812622,0.0,0.4104396104812622,0.4104396104812622,0.2267737984657287,0.0,0.2267737984657287,0.2267737984657287,1.4764769077301023,0.0,1.4764769077301023,1.4764769077301023,0.6497865319252014,0.0,0.6497865319252014,0.6497865319252014,0.4203702211380005,0.0,0.4203702211380005,0.4203702211380005,1.526507019996643,0.0,1.526507019996643,1.526507019996643,0.6540470123291016,0.0,0.6540470123291016,0.6540470123291016,1.7459523677825928,0.002774316303229609,1.7381054162979126,1.7459523677825928,0.4817045927047729,0.0002177622295436636,0.4817045927047729,0.4823205173015594,0.3364990949630737,0.00012384851434926538,0.3361487984657287,0.3364990949630737,1.1931402683258057,0.033290363097006954,1.1206940412521362,1.1931402683258057,0.7161301374435425,0.025235254887373854,0.7161301374435425,0.7710468769073486,0.4127309918403625,0.007279004051801342,0.4127309918403625,0.428571492433548,1.707435429096222,0.0008729009039302258,1.7066189050674438,1.708251953125,0.3564415574073791,0.002030279951199197,0.3545424044132232,0.358340710401535,0.3023426830768585,0.000350906290820105,0.3020144402980804,0.3026709258556366,-9999.0,0.0,-9999.0,-9999.0,-9999.0,0.0,-9999.0,-9999.0,-9999.0,0.0,-9999.0,-9999.0,0.1042194217443466,8.294721469109035e-06,0.1042194217443466,0.1042373403906822,1.715789794921875,0.001402039007837862,1.715789794921875,1.7188185453414917,0.7188712954521179,0.0014949674798809447,0.7188712954521179,0.7221007943153381,1.970276951789856,0.018573843840262887,1.934388875961304,1.970276951789856,0.3608308732509613,0.004311563730231936,0.3608308732509613,0.3691616058349609,0.2941087782382965,0.0009687919419025886,0.2941087782382965,0.2959806621074676,0.4393351674079895,0.006024186034919744,0.4222961962223053,0.4393351674079895,1.1838626861572266,0.007396139710934239,1.1838626861572266,1.2047821283340454,0.5613290071487427,0.0019051429198136875,0.5613290071487427,0.5667175650596619,0.0910589918494224,0.0011538569058607675,0.0910589918494224,0.0943225920200347,1.6499500274658203,0.0059099153918944995,1.6332342624664309,1.6499500274658203,0.7046993374824524,0.0013654103777831785,0.7008373737335205,0.7046993374824524,0.0354578979313373,0.0,0.0354578979313373,0.0354578979313373,2.292947769165039,0.0,2.292947769165039,2.292947769165039,0.8771045207977295,0.0,0.8771045207977295,0.8771045207977295,3.52,2024-12-05 04:00:00 From dcfd0572d116b9e77b0b59904cfbf370a7aab205 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 15 Sep 2025 15:16:27 -0300 Subject: [PATCH 07/52] SIENTIAPDE-1222 Refactor MLFlow logging to improve data output clarity - Updated the debug logging to directly capture the output of data.to_csv, enhancing traceability of processed input data. - Removed redundant debug statements for raw response data to streamline logging and focus on essential information. --- laborious/activities/mlflow.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index d0d47de..a3dd54a 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -154,18 +154,15 @@ class MLFlow(BaseActivity): data.columns.name = None self.debug("Processed input data:", metadata) - data.to_csv('data.csv') - self.debug(data.to_string(), metadata) + self.debug(data.to_csv('data.csv'), metadata) # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( model_name, data, model_config ) - self.debug("Raw response data:", metadata) - self.debug(response_data, metadata) - if response_data['success']: + response_dataframe = DataFrame(response_data['content']) try: response_dataframe = self.detect_and_parse_datetime_index( From 16f28c4d635813b05bbe23df092d1b2015c16fc2 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 07:52:21 -0300 Subject: [PATCH 08/52] SIENTIAPDE-1222 Update values.yaml and MLFlow logging for courier integration - Changed the image repository to 'sientia-module-courier' and updated the image tag to '0.0.1'. - Modified environment variables for GITHUB_BRANCH and MLFLOW_PASSWORD to reflect new configurations. - Enhanced MLFlow logging to include additional debug statements for raw response data and added a check for empty DataFrames. --- laborious/activities/mlflow.py | 5 +++++ values.yaml | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index a3dd54a..f11a397 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -161,9 +161,14 @@ class MLFlow(BaseActivity): model_name, data, model_config ) + self.debug("Transform raw response data:", metadata) + self.debug(response_data, metadata) + if response_data['success']: response_dataframe = DataFrame(response_data['content']) + if len(response_dataframe) == 0: + return response_data try: response_dataframe = self.detect_and_parse_datetime_index( response_dataframe, metadata) diff --git a/values.yaml b/values.yaml index 716254a..b2ae84e 100644 --- a/values.yaml +++ b/values.yaml @@ -7,11 +7,11 @@ replicaCount: 1 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: - repository: aignosi.azurecr.io/sientia-module + repository: aignosi.azurecr.io/sientia-module-courier # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.4.5" + tag: "0.0.1" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: @@ -151,7 +151,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1182-ajustar-laborious-para-pegar-timestamp-da-resposta-do-mlflow" + value: SIENTIAPDE-1222-ajustar-a-library-para-fazer-o-download-do-courier - name: PYTHON_APP value: "laborious.worker.worker" @@ -178,7 +178,7 @@ env: - name: MLFLOW_USERNAME value: "aignosi" - name: MLFLOW_PASSWORD - value: "aignosi" + value: "1L0FP50j3ncp123" - name: OPC_ID value: "1" From ecffa0c301d7f8d60ce11043b89d25a38688a5da Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 07:55:56 -0300 Subject: [PATCH 09/52] SIENTIAPDE-1222 Refactor MLFlow logging to enhance data output clarity - Updated debug logging to use data.to_string() for processed input data, improving readability. - Modified raw response data logging to format the output as a string, ensuring consistent logging format. --- laborious/activities/mlflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index f11a397..5f5b12c 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -154,7 +154,7 @@ class MLFlow(BaseActivity): data.columns.name = None self.debug("Processed input data:", metadata) - self.debug(data.to_csv('data.csv'), metadata) + self.debug(data.to_string(), metadata) # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( @@ -162,7 +162,7 @@ class MLFlow(BaseActivity): ) self.debug("Transform raw response data:", metadata) - self.debug(response_data, metadata) + self.debug(f"{response_data}", metadata) if response_data['success']: From 1de44e7ed1fcc3f404811b0a37940f5d07ee73e4 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 08:16:30 -0300 Subject: [PATCH 10/52] SIENTIAPDE-1222 SIENTIAPDE-1222 Enhance MLFlow logging with sample dictionary for response data - Introduced a new method to create a sample dictionary for debugging, allowing for better visualization of nested data structures in logs. - Updated debug logging to utilize the new sampling method for raw and transformed response data, improving clarity and reducing output size. - Adjusted logging for processed input data to display only the first few rows, enhancing readability. --- laborious/activities/mlflow.py | 48 +++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 5f5b12c..c11bbaa 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -17,6 +17,39 @@ with workflow.unsafe.imports_passed_through(): import traceback +def create_sample_dict(data: dict, max_items: int = 3, max_depth: int = 2) -> dict: + """ + Create a sample of a dictionary for debugging purposes. + + Args: + data: Dictionary to sample + max_items: Maximum number of items to show per level + max_depth: Maximum depth to traverse nested structures + + Returns: + Dictionary with sampled content + """ + if max_depth <= 0: + return {"...": "max_depth_reached"} + + sample = {} + items = list(data.items())[:max_items] + + for key, value in items: + if isinstance(value, dict): + sample[key] = create_sample_dict(value, max_items, max_depth - 1) + elif isinstance(value, list): + sample[key] = value[:max_items] if len( + value) > max_items else value + else: + sample[key] = value + + if len(data) > max_items: + sample["..."] = f"({len(data) - max_items} more items)" + + return sample + + class MLFlow(BaseActivity): """ MLFlow integration activities for model inference operations. @@ -138,7 +171,7 @@ class MLFlow(BaseActivity): model_config = input_data.get('model_config', {}) self.debug("Raw input data:", metadata) - self.debug(data, metadata) + self.debug(data.head(5).to_string(), metadata) # Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair data = data.sort_values('created_at', ascending=False).drop_duplicates( @@ -154,7 +187,7 @@ class MLFlow(BaseActivity): data.columns.name = None self.debug("Processed input data:", metadata) - self.debug(data.to_string(), metadata) + self.debug(data.head(5).to_string(), metadata) # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( @@ -162,7 +195,8 @@ class MLFlow(BaseActivity): ) self.debug("Transform raw response data:", metadata) - self.debug(f"{response_data}", metadata) + self.debug(create_sample_dict( + response_data), metadata) if response_data['success']: @@ -194,7 +228,8 @@ class MLFlow(BaseActivity): response_data['content'] = response_dataframe.to_dict() self.debug("Transform response data:", metadata) - self.debug(response_data, metadata) + self.debug(create_sample_dict( + response_data), metadata) self.info("Data transformed successfully", metadata) @@ -236,7 +271,7 @@ class MLFlow(BaseActivity): model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) - self.debug(data, metadata) + self.debug(data.head(5).to_string(), metadata) # Convert numpy.nan to None for model compatibility data.replace(np.nan, None, inplace=True) @@ -247,7 +282,8 @@ class MLFlow(BaseActivity): ) self.debug("Prediction response data:", metadata) - self.debug(json.dumps(response_data, indent=4), metadata) + self.debug(create_sample_dict( + response_data), metadata) self.info("Data predicted successfully", metadata) From 31e7e94a955d70d0971cd84903f52a56d2b51322 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 08:54:27 -0300 Subject: [PATCH 11/52] SIENTIAPDE-1222 Update requirements and enhance logging in Gates and MLFlow activities - Updated the sientia-dataops-library and sientia-mlops-library dependencies in requirements.txt to the latest versions. - Improved debug logging in the Gates activity to display a sample of input data and filters, enhancing clarity and reducing output size. - Refactored MLFlow activity logging to utilize the create_sample_dict function for better visualization of nested data structures in logs. --- laborious/activities/gates.py | 18 ++++++++--------- laborious/activities/mlflow.py | 35 +--------------------------------- requirements.txt | 5 ++--- 3 files changed, 12 insertions(+), 46 deletions(-) diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index 34c26f1..cbebebe 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -8,6 +8,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.temporal.activities.base import BaseActivity from sientia_do.observability.logger import Logger from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now + from sientia_do.formatters import create_sample_dict from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter from typing import Any from laborious.utils.filters.conditional_filters import ( @@ -122,15 +123,13 @@ class Gates(BaseActivity): self.info("Performing input gate...", metadata) - self.debug(f"Input data: {input_data}", metadata) - filters = input_data['filters'] data = DataFrame(input_data['data']) path_priority = input_data['path_priority'] filter_output = [] - self.debug(f"Input data:\n {data}", metadata) + self.debug(f"Input data: {data.head(5).to_string()}", metadata) self.debug(f"Filters: {filters}", metadata) # Apply each configured filter @@ -207,7 +206,8 @@ class Gates(BaseActivity): filter_output = [] - self.debug(f"Input data:\n {data}", metadata) + self.debug(create_sample_dict( + data, max_items=5, max_depth=2), metadata) self.debug(f"Filters: {filters}", metadata) comments = [] @@ -292,8 +292,8 @@ class Gates(BaseActivity): filter_output = [] - self.debug(f"Input data:\n {data}", metadata) - self.debug(f"Filters: {filters}", metadata) + self.debug(f"Input data:\n {data.head(5).to_string()}", metadata) + self.debug(create_sample_dict(filters), metadata) for fil, config in filters.items(): if fil not in mlflow_content_filter_functions: @@ -410,7 +410,7 @@ class Gates(BaseActivity): self.debug( f"Prediction store policy: {prediction_store_policy}", metadata) - self.debug(f"Prediction data: {data.to_string()}", metadata) + self.debug(f"Prediction data: {data.head(5).to_string()}", metadata) policy_type, policy_value = self.get_prediction_store_policy( prediction_store_policy, metadata) @@ -445,7 +445,7 @@ class Gates(BaseActivity): data = data.reset_index(drop=True) self.info(f"Prediction formatted: {len(data)} rows", metadata) - self.debug(f"Prediction data: {data.to_string()}", metadata) + self.debug(f"Prediction data: {data.head(5).to_string()}", metadata) return data.to_dict() @@ -521,7 +521,7 @@ class Gates(BaseActivity): data = DataFrame(input_data['data']) - self.debug(f"Input data: {data.to_string()}", metadata) + self.debug(f"Input data: {data.head(5).to_string()}", metadata) if data.empty: return now().strftime(DATETIME_FORMAT_WITH_TZ) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index c11bbaa..5b449e0 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -3,13 +3,13 @@ from temporalio import activity, workflow with workflow.unsafe.imports_passed_through(): from datetime import datetime - import json from pandas import Timestamp, to_datetime from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ from sientia_do.temporal.activities.base import BaseActivity from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger + from sientia_do.formatters import create_sample_dict from laborious.utils.repository.model_repository import MLFlowRepository from typing import Any import numpy as np @@ -17,39 +17,6 @@ with workflow.unsafe.imports_passed_through(): import traceback -def create_sample_dict(data: dict, max_items: int = 3, max_depth: int = 2) -> dict: - """ - Create a sample of a dictionary for debugging purposes. - - Args: - data: Dictionary to sample - max_items: Maximum number of items to show per level - max_depth: Maximum depth to traverse nested structures - - Returns: - Dictionary with sampled content - """ - if max_depth <= 0: - return {"...": "max_depth_reached"} - - sample = {} - items = list(data.items())[:max_items] - - for key, value in items: - if isinstance(value, dict): - sample[key] = create_sample_dict(value, max_items, max_depth - 1) - elif isinstance(value, list): - sample[key] = value[:max_items] if len( - value) > max_items else value - else: - sample[key] = value - - if len(data) > max_items: - sample["..."] = f"({len(data) - max_items} more items)" - - return sample - - class MLFlow(BaseActivity): """ MLFlow integration activities for model inference operations. diff --git a/requirements.txt b/requirements.txt index 10995c8..1ce9f9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ psycopg2-binary sqlalchemy asyncua redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.5 -# git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.13 -/home/grezewave/Documents/projects/sientia/sientia-mlops-library +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.6 +git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.39.0 prometheus-client From 0bdbef00b781275ab78dc453188550416c1f9911 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 10:22:51 -0300 Subject: [PATCH 12/52] SIENTIAPDE-1222 Update image tag in values.yaml and enhance debug logging in Gates and MLFlow activities - Updated the image tag in values.yaml from '0.0.1' to '0.0.2'. - Improved debug logging in the Gates activity to format input data and filters for better readability. - Enhanced MLFlow activity logging to include formatted output for raw and transformed response data, ensuring consistent logging format. --- laborious/activities/gates.py | 6 +++--- laborious/activities/mlflow.py | 15 ++++++--------- laborious/utils/repository/model_repository.py | 8 ++++++++ values.yaml | 4 ++-- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index cbebebe..c88b59a 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -206,8 +206,8 @@ class Gates(BaseActivity): filter_output = [] - self.debug(create_sample_dict( - data, max_items=5, max_depth=2), metadata) + self.debug(f"Input data: \n {create_sample_dict( + data, max_items=5, max_depth=2)}", metadata) self.debug(f"Filters: {filters}", metadata) comments = [] @@ -293,7 +293,7 @@ class Gates(BaseActivity): filter_output = [] self.debug(f"Input data:\n {data.head(5).to_string()}", metadata) - self.debug(create_sample_dict(filters), metadata) + self.debug(f"Filters: \n {create_sample_dict(filters)}", metadata) for fil, config in filters.items(): if fil not in mlflow_content_filter_functions: diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 5b449e0..33d9aeb 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -161,9 +161,8 @@ class MLFlow(BaseActivity): model_name, data, model_config ) - self.debug("Transform raw response data:", metadata) - self.debug(create_sample_dict( - response_data), metadata) + self.debug(f"Transform raw response data: \n {create_sample_dict( + response_data)}", metadata) if response_data['success']: @@ -194,9 +193,8 @@ class MLFlow(BaseActivity): response_data['content'] = response_dataframe.to_dict() - self.debug("Transform response data:", metadata) - self.debug(create_sample_dict( - response_data), metadata) + self.debug(f"Transform response data: \n {create_sample_dict( + response_data)}", metadata) self.info("Data transformed successfully", metadata) @@ -248,9 +246,8 @@ class MLFlow(BaseActivity): model_name, data, model_config ) - self.debug("Prediction response data:", metadata) - self.debug(create_sample_dict( - response_data), metadata) + self.debug(f"Prediction response data: \n {create_sample_dict( + response_data)}", metadata) self.info("Data predicted successfully", metadata) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 07f0fc5..a4bd28f 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -27,8 +27,16 @@ ARTIFACTS_PATH = "./tmp/artifacts" class MLFlowRepository(): def __init__(self, host: str, username: str, password: str, logger: Logger): + +<< << << < HEAD # set tracking uri mlflow.set_tracking_uri(host) +== == == = + self.model_serving = ModelServing(tracking_uri=host, + username=username, password=password, + logger=logger) + self.logger = logger +>>>>>> > eefd081(SIENTIAPDE-1222) environ["MLFLOW_TRACKING_USERNAME"] = username environ["MLFLOW_TRACKING_PASSWORD"] = password diff --git a/values.yaml b/values.yaml index b2ae84e..4eb83d7 100644 --- a/values.yaml +++ b/values.yaml @@ -11,9 +11,9 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.0.1" + tag: "0.0.2" -# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +0# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: - name: docker-hub-secret # This is to override the chart name. From 8095e37710544e56b3a99285d079c42812a3bde4 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 10:29:35 -0300 Subject: [PATCH 13/52] SIENTIAPDE-1222 Refactor MLFlow debug logging for improved readability - Reformatted debug logging statements in the MLFlow activity to enhance clarity and consistency. - Ensured that the output of raw and transformed response data is presented in a more readable format, maintaining the use of create_sample_dict for better visualization. --- laborious/activities/mlflow.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 33d9aeb..e0e7cb3 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -161,8 +161,8 @@ class MLFlow(BaseActivity): model_name, data, model_config ) - self.debug(f"Transform raw response data: \n {create_sample_dict( - response_data)}", metadata) + self.debug( + f"Transform raw response data: \n {create_sample_dict(response_data)}", metadata) if response_data['success']: @@ -193,8 +193,8 @@ class MLFlow(BaseActivity): response_data['content'] = response_dataframe.to_dict() - self.debug(f"Transform response data: \n {create_sample_dict( - response_data)}", metadata) + self.debug( + f"Transform response data: \n {create_sample_dict(response_data)}", metadata) self.info("Data transformed successfully", metadata) @@ -246,8 +246,8 @@ class MLFlow(BaseActivity): model_name, data, model_config ) - self.debug(f"Prediction response data: \n {create_sample_dict( - response_data)}", metadata) + self.debug( + f"Prediction response data: \n {create_sample_dict(response_data)}", metadata) self.info("Data predicted successfully", metadata) From 3752170c3aab014e5ea1eecf2b09523ae9675ee2 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 10:31:30 -0300 Subject: [PATCH 14/52] SIENTIAPDE-1222 Refactor debug logging in Gates activity for improved readability - Reformatted the debug logging statement for input data in the Gates activity to enhance clarity and maintain consistency with previous logging improvements. --- laborious/activities/gates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index c88b59a..4a71455 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -206,8 +206,8 @@ class Gates(BaseActivity): filter_output = [] - self.debug(f"Input data: \n {create_sample_dict( - data, max_items=5, max_depth=2)}", metadata) + self.debug( + f"Input data: \n {create_sample_dict(data, max_items=5, max_depth=2)}", metadata) self.debug(f"Filters: {filters}", metadata) comments = [] From 39aa0893848d93c6bc55a433b06ae796aea2320d Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 10:36:43 -0300 Subject: [PATCH 15/52] SIENTIAPDE-1222 Enhance MLFlow debug logging to limit output size - Updated debug logging statements in the MLFlow activity to include a maximum of 5 items and a depth of 5 for the sample dictionary, improving readability and reducing log clutter for raw and transformed response data. --- laborious/activities/mlflow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index e0e7cb3..aae09ee 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -162,7 +162,7 @@ class MLFlow(BaseActivity): ) self.debug( - f"Transform raw response data: \n {create_sample_dict(response_data)}", metadata) + f"Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) if response_data['success']: @@ -194,7 +194,7 @@ class MLFlow(BaseActivity): response_data['content'] = response_dataframe.to_dict() self.debug( - f"Transform response data: \n {create_sample_dict(response_data)}", metadata) + f"Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) self.info("Data transformed successfully", metadata) @@ -247,7 +247,7 @@ class MLFlow(BaseActivity): ) self.debug( - f"Prediction response data: \n {create_sample_dict(response_data)}", metadata) + f"Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) self.info("Data predicted successfully", metadata) From 73b03e6749a7c3ff30e85d7d53665dd85e9261c1 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 11:41:27 -0300 Subject: [PATCH 16/52] SIENTIAPDE-1222 Update prediction_store_policy handling in workflows - Added 'prediction_store_policy' to the input data handling in PredictionsBatch, ensuring a default value of 'lts:1' is used when not provided. - Modified FormatAndExportPrediction to directly use 'prediction_store_policy' from input_data, removing the default fallback. - Updated PredictionProcess to include 'prediction_store_policy' in the output data structure, ensuring consistency across workflows. --- laborious/workflows/predictions_batch.py | 4 +++- .../workflows/sub_workflows/format_and_export_prediction.py | 3 +-- laborious/workflows/sub_workflows/prediction_process.py | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/laborious/workflows/predictions_batch.py b/laborious/workflows/predictions_batch.py index ecce063..e522eaa 100644 --- a/laborious/workflows/predictions_batch.py +++ b/laborious/workflows/predictions_batch.py @@ -115,7 +115,9 @@ class PredictionsBatch(): }), 'model_config': input_data.get('model_config', {}), 'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']), - 'opc_output_config': input_data.get('opc_output_config', {}) + 'opc_output_config': input_data.get('opc_output_config', {}), + 'prediction_store_policy': input_data.get( + 'prediction_store_policy', 'lts:1') } # Execute prediction process workflow diff --git a/laborious/workflows/sub_workflows/format_and_export_prediction.py b/laborious/workflows/sub_workflows/format_and_export_prediction.py index ae3ccfb..8e7df07 100644 --- a/laborious/workflows/sub_workflows/format_and_export_prediction.py +++ b/laborious/workflows/sub_workflows/format_and_export_prediction.py @@ -79,8 +79,7 @@ class FormatAndExportPrediction(): 'timestamp': input_data['timestamp'], 'model_id': input_data['model_id'], 'prediction_confidence': prediction_confidence, - 'prediction_store_policy': input_data.get( - 'prediction_store_policy', 'lts:1') + 'prediction_store_policy': input_data['prediction_store_policy'] }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=60) diff --git a/laborious/workflows/sub_workflows/prediction_process.py b/laborious/workflows/sub_workflows/prediction_process.py index c958500..777fa1c 100644 --- a/laborious/workflows/sub_workflows/prediction_process.py +++ b/laborious/workflows/sub_workflows/prediction_process.py @@ -213,7 +213,8 @@ class PredictionProcess(): 'opc_output_config': input_data['opc_output_config'], 'schema': input_data['schema'], 'table_name': input_data['table_name'], - 'comment': comment + 'comment': comment, + 'prediction_store_policy': input_data['prediction_store_policy'] } ) @@ -286,7 +287,8 @@ class PredictionProcess(): 'schema': schema, 'table_name': table_name, 'comment': comment, - 'opc_output_config': input_data['opc_output_config'] + 'opc_output_config': input_data['opc_output_config'], + 'prediction_store_policy': input_data['prediction_store_policy'] } ) return True From 12b56cb71fb5d3cdab26996e07d5d8d30ea2c83c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 12:34:56 -0300 Subject: [PATCH 17/52] SIENTIAPDE-1222 Update model configuration keys in MLFlowRepository for consistency - Changed 'model_retention' to 'retention_minutes' and 'is_compressed' to 'compressed' in model configuration handling, ensuring alignment with updated configuration standards. --- laborious/utils/repository/model_repository.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index a4bd28f..07f0fc5 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -27,16 +27,8 @@ ARTIFACTS_PATH = "./tmp/artifacts" class MLFlowRepository(): def __init__(self, host: str, username: str, password: str, logger: Logger): - -<< << << < HEAD # set tracking uri mlflow.set_tracking_uri(host) -== == == = - self.model_serving = ModelServing(tracking_uri=host, - username=username, password=password, - logger=logger) - self.logger = logger ->>>>>> > eefd081(SIENTIAPDE-1222) environ["MLFLOW_TRACKING_USERNAME"] = username environ["MLFLOW_TRACKING_PASSWORD"] = password From ee2ac5a3651164d5f83f52a1d8f906c91f9bbed1 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 17:07:44 -0300 Subject: [PATCH 18/52] SIENTIAPDE-1222 Update README.md to reflect new features and configuration changes - Added details about two dedicated task queues: `predictions_batch-queue` and `minimal_retrain-queue`. - Enhanced descriptions of activities and workflows, including multiple inheritance patterns and configurable MLFlow model serving. - Updated monitoring metrics section to include new labels and metrics for prediction and OPC export operations. - Revised configuration section with updated default values and added new environment variables for Kubernetes pod identification. - Improved clarity in the Predictions Batch Workflow configuration example, including structured input filters and updated retention policies. --- README.md | 200 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 115 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 330cbb4..500153c 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa - Health check endpoints for Kubernetes liveness/readiness probes - Graceful shutdown with cleanup procedures - Multi-instance deployment support + - Two dedicated task queues: `predictions_batch-queue` and `minimal_retrain-queue` #### **Workflows (`laborious/workflows/`)** - **PredictionsBatch**: Main entry point for batch prediction pipelines @@ -79,25 +80,32 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa - Configurable timeout and retry strategies #### **Activities (`laborious/activities/`)** +- **Activities**: Main activity orchestrator combining all functionality through multiple inheritance - **Gates**: Data quality validation and filtering mechanisms - **MLFlow**: Model transformation and prediction operations - **OPC**: Real-time data export to industrial OPC servers -- **Activities**: Main activity orchestrator and coordination - **Key Features**: + - Multiple inheritance pattern for unified activity interface - Configurable filter policies and validation rules - - MLFlow model serving integration + - MLFlow model serving integration with configurable flavors - OPC UA client with certificate-based authentication - - Comprehensive error handling and notification + - Comprehensive error handling and notification integration + - Support for multiple OPC servers with independent configurations #### **Data Services (`laborious/utils/`)** -- **Connectors**: Database and external service configuration management +- **Connectors Config**: Environment variable-based configuration management - **Repository**: Data access layer for MLFlow and OPC operations + - `model_repository.py`: MLFlow model operations and retraining + - `opc_repository.py`: OPC server communication and data writing - **Filters**: Data quality validation and MLFlow response filtering + - `conditional_filters.py`: Input data validation filters + - `mlflow_filters.py`: MLFlow API response validation filters - **Key Features**: - - Environment variable-based configuration + - Environment variable-based configuration with sensible defaults - Connection pool management and optimization - Security credential management - Configuration validation and error handling + - Support for multiple OPC servers and MLFlow model flavors ### Data Flow Architecture @@ -506,23 +514,32 @@ pytest tests/workflow/test_predictions_batch.py ## 📊 Monitoring and Metrics -The Laborious system exposes comprehensive Prometheus metrics: +The Laborious system exposes comprehensive Prometheus metrics for operational visibility and performance monitoring: -### Application Metrics +### Application Health Metrics - `app_up`: Application health status (1=healthy, 0=unhealthy) -- `laborious_predictions_written_count`: Prediction export operation count -- `laborious_prediction_confidence_monitor`: Prediction confidence monitoring -- `laborious_prediction_response_time_monitor`: Prediction response time monitoring + - Labels: `pod_id` -### MLFlow Metrics -- Model transformation and prediction success rates -- API response times and error rates -- Model retention and versioning metrics +### Prediction Operation Metrics +- `laborious_predictions_written_count`: Counter for successful prediction exports + - Labels: `pod_id`, `model_name`, `pipeline_name` +- `laborious_prediction_confidence_monitor`: Gauge for current prediction confidence levels + - Labels: `pod_id`, `model_name`, `pipeline_name` +- `laborious_prediction_response_time_monitor`: Histogram for prediction response times + - Labels: `pod_id`, `model_name`, `pipeline_name` + - Buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0] -### Export Metrics -- PostgreSQL export operation counts and response times -- OPC server write operations and performance -- Data quality filter pass/fail rates +### OPC Export Metrics +- `laborious_prediction_opc_writing_count`: Counter for OPC server write operations + - Labels: `pod_id`, `model_name`, `pipeline_name`, `opc_server_id` +- `laborious_prediction_opc_writing_response_time_monitor`: Histogram for OPC write response times + - Labels: `pod_id`, `model_name`, `pipeline_name`, `opc_server_id` + - Buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0] + +### Data Quality Metrics +- Filter pass/fail rates through notification system +- MLFlow API response validation metrics +- Data quality gate performance tracking ## ⚙️ Configuration @@ -537,31 +554,30 @@ The Laborious system exposes comprehensive Prometheus metrics: | `POSTGRES_USER` | PostgreSQL username | `sientia` | Yes | | `POSTGRES_PASSWORD` | PostgreSQL password | `sientia` | Yes | | `POSTGRES_DBNAME` | PostgreSQL database | `sientia` | Yes | -| `POSTGRES_MIN_CONNECTIONS` | Minimum PostgreSQL connections | `10` | No | -| `POSTGRES_MAX_CONNECTIONS` | Maximum PostgreSQL connections | `30` | No | -| `MLFLOW_HOST` | MLFlow server hostname | `localhost` | Yes | -| `MLFLOW_PORT` | MLFlow server port | `5000` | Yes | -| `MLFLOW_USERNAME` | MLFlow username | `admin` | Yes | -| `MLFLOW_PASSWORD` | MLFlow password | `admin` | Yes | +| `POSTGRES_MIN_CONNECTIONS` | Minimum PostgreSQL connections | `5` | No | +| `POSTGRES_MAX_CONNECTIONS` | Maximum PostgreSQL connections | `20` | No | +| `MLFLOW_HOST` | MLFlow server hostname | `http://localhost` | Yes | +| `MLFLOW_PORT` | MLFlow server port | `5080` | Yes | +| `MLFLOW_USERNAME` | MLFlow username | `aignosi` | Yes | +| `MLFLOW_PASSWORD` | MLFlow password | `aignosi` | Yes | | `OPC_CONFIG` | OPC server configuration (JSON) | `{}` | No | | `OPC_ID` | OPC server identifier | `1` | No | | `OPC_URL` | OPC server URL | `opc.tcp://localhost:4840` | No | -| `OPC_NAME` | OPC server name | `OPC_Server` | No | -| `OPC_SERVER_URI` | OPC server URI | `urn:opcserver:opcua` | No | -| `OPC_CERT_PATH` | OPC client certificate path | `/path/to/cert.pem` | No | -| `OPC_PRIVATE_KEY_PATH` | OPC private key path | `/path/to/key.pem` | No | -| `OPC_SERVER_CERT_PATH` | OPC server certificate path | `/path/to/server_cert.pem` | No | -| `OPC_RECONNECTION_INTERVAL` | OPC reconnection interval (ms) | `5000` | No | -| `MONGODB_URL` | MongoDB connection URI | `localhost:27017` | Yes | +| `OPC_SERVER_URI` | OPC server URI | `opc.tcp://localhost:4840` | No | +| `OPC_CERT_PATH` | OPC client certificate path | `None` | No | +| `OPC_PRIVATE_KEY_PATH` | OPC private key path | `None` | No | +| `OPC_SERVER_CERT_PATH` | OPC server certificate path | `None` | No | +| `OPC_RECONNECTION_INTERVAL` | OPC reconnection interval (ms) | `120` | No | +| `MONGODB_URL` | MongoDB connection URI | `localhost:27018` | Yes | | `MONGODB_USERNAME` | MongoDB username | `root` | Yes | -| `MONGODB_PASSWORD` | MongoDB password | `password` | Yes | -| `MONGODB_DATABASE` | MongoDB database name | `sientia` | Yes | +| `MONGODB_PASSWORD` | MongoDB password | `wKZDbMNU1c` | Yes | +| `MONGODB_DATABASE_NAME` | MongoDB database name | `sientia` | Yes | | `MONGODB_TTL_INDEX_HOURS` | MongoDB TTL index hours | `1` | No | -| `KAFKA_BOOTSTRAP_SERVERS` | Kafka bootstrap servers | `localhost:9092` | No | | `LOG_LEVEL` | Application log level | `INFO` | No | -| `PROJECT_NAME` | Project name for metrics | `sientia-laborious` | No | +| `PROJECT_NAME` | Project name for metrics | `laborious` | No | | `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No | | `HTTP_SDK_METRICS_PORT` | Temporal SDK metrics port | `9091` | No | +| `POD_ID` | Kubernetes pod identifier | `None` | No | @@ -606,74 +622,88 @@ For single OPC server, use individual environment variables: MongoDB pipeline configuration: -#### Predictions Batch Workflow +#### Predictions Batch Workflow configuration sample + +This is the configuration for the Predictions Batch Workflow, to be inserted into the MongoDB pipeline collection. + ```json { "schedule_name": "laborious-orchestrated-pipeline", "model_id": "1", "workflow_type": "predictions_batch", - "frequency": "30s", # Workflow execution frequency - "max_retry_policy": 1, # Maximum number of retries for the workflow + "frequency": "30s", + "max_retry_policy": 1, "query": "select * from sientia_data.laborious_data where model_id = 1 and \"timestamp\" > NOW() - INTERVAL '5 minutes' order by \"timestamp\" desc limit 30;", - "retention_time": 60, # Retention time for models in minutes "write_tags": [ { - "server_id": "1", - "type": "prediction", # Type of tag to write, can be prediction or confidence + "server_id": "server1", + "type": "prediction", "addr": "ns=2;i=5", "data_type": "double" }, { - "server_id": "1", + "server_id": "server1", "type": "confidence", - "addr": "ns=2;i=5", + "addr": "ns=2;i=6", "data_type": "double" } ], - "input_filters": [ - { - "filter_name": "EMPTY_DATA", # Required filter - "policy": "STOP" - }, - { - "filter_name": "SPECIFIC_VARIABLES_NULL_VALUES", - "policy": "CONTINUE", - "config": { - "variables": [ - "Counter" - ] - } + "input_filters": { + "EMPTY_DATA": {"POLICY": "STOP"}, + "SPECIFIC_VARIABLES_NULL_VALUES": { + "POLICY": "CONTINUE", + "config": {"variables": ["Counter"]} } - ], - "mlflow_transform_filters": [ - { - "filter_name": "API_ERROR", # Required filter - "policy": "REPEAT" - }, - { - "filter_name": "NAN_VALUES", - "policy": "STOP" - } - ], - "mlflow_predict_filters": [ - { - "filter_name": "API_ERROR", # Required filter - "policy": "CONTINUE" - } - ], - "path_priority": [ # In case of multiple filters catch problems, this will determine the path to take - "STOP", - "CONTINUE", - "REPEAT" - ], + }, + "mlflow_transform_filters": { + "API_ERROR": {"POLICY": "REPEAT"}, + "NAN_VALUES": {"POLICY": "STOP"} + }, + "mlflow_predict_filters": { + "API_ERROR": {"POLICY": "CONTINUE"} + }, + "path_priority": ["STOP", "CONTINUE", "REPEAT"], "active": true, - "datetime_columns": [ # Columns in data comming from query that are datetime - "timestamp", - "created_at" - ], "updated_at": { - "$date": "2025-08-27T18:35:01.600Z" - } + "$date": "2025-09-16T10:00:00.000Z" + }, + "datetime_columns": ["timestamp", "created_at"], + "predictions_storage_policy": "lts:1" +} +``` + +This is the configuration created by the Orchestrator in Temporal. + +```json +{ + "datetime_columns":["timestamp","created_at"], + "frequency":"15m", + "input_filters":{"EMPTY_DATA":{"config":{},"policy":"STOP"}}, + "max_retry_policy":1, + "mlflow_predict_filters":{"API_ERROR":{"config":{},"policy":"CONTINUE"}}, + "mlflow_transform_filters":{ + "API_ERROR":{"config":{},"policy":"CONTINUE"}, + "EMPTY_DATA":{"config":{},"policy":"STOP"} + }, + "model_config":{ + "is_compressed":true, + "predict_flavor":"pyfunc", + "retention_minutes":60, + "retention_target":"artifact", + "transform_function_keyword":"transform" + }, + "model_id":"352", + "model_name":"courier", + "opc_output_config":{}, + "path_priority":["STOP","CONTINUE","REPEAT"], + "predictions_storage_policy":"lts:1", + "query":"select * from sientia_data.laborious_data where model_id = 352 order by \"timestamp\" desc limit 300;", + "retention_time":3600, + "schedule_name":"laborious-courier", + "schema":"sientia_data", + "table_name":"predictions", + "updated_at":"2025-09-12 19:35:01.600000+0000", + "workflow_type":"predictions_batch" } ``` @@ -687,7 +717,7 @@ laborious/ │ ├── gates.py # Data quality gates and filtering │ ├── mlflow.py # MLFlow model operations │ └── opc.py # OPC server operations -├── workflow/ # Temporal workflow definitions +├── workflows/ # Temporal workflow definitions │ ├── predictions_batch.py # Main batch prediction workflow │ ├── minimal_retrain.py # Model retraining workflow │ └── sub_workflows/ # Sub-workflow implementations From 20fc938cc0af18fc3ff39f8e780fb9d84ec154e6 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 08:59:12 -0300 Subject: [PATCH 19/52] SIENTIAPDE-1222 Refactor datetime index handling in MLFlow and MLFlowRepository - Moved the detect_and_parse_datetime_index method from MLFlow to MLFlowRepository for better organization and reusability. - Updated the method to include enhanced logging and error handling for invalid datetime formats. - Adjusted the transform method in MLFlowRepository to utilize the new datetime index parsing logic. - Added unit tests for both valid and invalid datetime index cases to ensure robustness. --- laborious/activities/mlflow.py | 67 ----------- tests/laborious/activities/test_mlflow.py | 7 +- .../utils/repository/test_model_repository.py | 109 +++++++++++++++++- 3 files changed, 107 insertions(+), 76 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index aae09ee..896f05a 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -63,44 +63,6 @@ class MLFlow(BaseActivity): f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger.base_logger ) - def detect_and_parse_datetime_index(self, data: DataFrame, metadata: dict) -> DataFrame: - """ - Detect and parse datetime index from data. index must be a timestamp like column. - This function must detect the timestamp type (pandas Timestamp or datetime) and convert it to DATETIME_FORMAT_WITH_TZ. - If the index is a string, must be in format DATETIME_FORMAT_WITH_TZ. - If another type or format, must raise an error. - """ - index = data.index - - # Get type of first element of index - index_type = type(index[0]) - - self.info(f"Index type: {index_type}", metadata) - - message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}" - - # Check if all in index are of the same type - if not all(isinstance(i, index_type) for i in index): - raise ValueError( - f"{message}") - - # Check type and converts to DATETIME_FORMAT_WITH_TZ - if index_type == str: - # Validate format of string and return error if not valid - try: - to_datetime(data.index) - except ValueError: - raise ValueError( - f"{message}") - - elif index_type == datetime or index_type == Timestamp: - data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) - else: - raise ValueError( - f"{message}") - - return data - @activity.defn(name="request_transform") async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]: """ @@ -164,35 +126,6 @@ class MLFlow(BaseActivity): self.debug( f"Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) - if response_data['success']: - - response_dataframe = DataFrame(response_data['content']) - if len(response_dataframe) == 0: - return response_data - try: - response_dataframe = self.detect_and_parse_datetime_index( - response_dataframe, metadata) - response_dataframe['timestamp'] = to_datetime( - response_dataframe.index, format=DATETIME_FORMAT_WITH_TZ) - response_dataframe['timestamp'] = response_dataframe['timestamp'].dt.strftime( - DATETIME_FORMAT) - except ValueError as e: - trace = traceback.format_exc() - self.send_notification( - metadata=metadata, - notification_id='TRANSFORM_DATA_INDEX_ERROR', - message=f'Error parsing trasnformed data index: {e}', - block='transform', - level=NotificationLevel.ERROR, - attachment_content=trace - ) - self.error(trace, metadata=metadata) - raise e - - response_dataframe.to_csv('response_data.csv') - - response_data['content'] = response_dataframe.to_dict() - self.debug( f"Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index 7aced7c..ed68729 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -1,8 +1,9 @@ +from datetime import datetime from unittest.mock import ANY, MagicMock, patch import numpy as np -from pandas import DataFrame -from pytest import fixture, mark +from pandas import DataFrame, Timestamp +from pytest import fixture, mark, raises from laborious.activities.mlflow import MLFlow from sientia_do.notifications.models import NotificationLevel @@ -58,7 +59,7 @@ metadata = { @mark.asyncio @patch("laborious.activities.mlflow.DataFrame") @patch("laborious.activities.mlflow.max") -async def test_request_transform(mock_max, mock_dataframe, mlflow): +async def test_request_transform_success(mock_max, mock_dataframe, mlflow): mock_max.return_value = '2024-01-02' # Mock input data input_data = { diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 8b3716e..51a3223 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -2,7 +2,8 @@ from unittest.mock import ANY, MagicMock, call, patch import numpy as np from pandas import DataFrame import pytest -from laborious.utils.repository import model_repository +from datetime import datetime, timezone +from pandas import Timestamp from laborious.utils.repository.model_repository import MLFlowRepository @@ -22,18 +23,113 @@ def mlflow_repository(): return repo +metadata = { + "metadata": { + "model_id": "test_model", + "model_name": "test_model", + "workflow_name": "test_workflow", + "schema_name": "test_schedule", + }, +} + + +invalid_cases = [ + ( + { + 'value': { + '2024-01-01 12:00:00': 1, + 2024: 2 + } + } + ), + ( + { + 'value': { + '2024-01-01': 1, + '2024-01-02': 2 + } + } + ), + ( + { + 'value': { + 1: 1, + 2: 2 + } + } + ) +] + + +@pytest.mark.parametrize("data", invalid_cases) +def test_detect_and_parse_datetime_index_error_cases(mlflow_repository, data): + input_data = DataFrame( + data + ) + + with pytest.raises(ValueError) as e: + mlflow_repository.detect_and_parse_datetime_index( + input_data, metadata['metadata']) + + assert str(e) == "Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format %Y-%m-%d %H:%M:%S" + + +valid_cases = [ + ( + { + 'value': { + '2024-01-01 12:00:00+0000': 1, + '2024-01-02 12:00:00+0000': 2 + } + }, ['2024-01-01 12:00:00+0000', '2024-01-02 12:00:00+0000'] + ), + ( + { + 'value': { + datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc): 1, + datetime(2025, 1, 2, 12, 0, 0, tzinfo=timezone.utc): 2 + } + }, ['2025-01-01 12:00:00+0000', '2025-01-02 12:00:00+0000'] + ), + ( + { + 'value': { + Timestamp(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc): 1, + Timestamp(2026, 1, 2, 12, 0, 0, tzinfo=timezone.utc): 2 + } + }, ['2026-01-01 12:00:00+0000', '2026-01-02 12:00:00+0000'] + ), +] + + +@pytest.mark.parametrize("data,expected", valid_cases) +def test_detect_and_parse_datetime_index_valid_format(mlflow_repository, data, expected): + input_data = DataFrame(data) + + response = mlflow_repository.detect_and_parse_datetime_index( + input_data, metadata['metadata']) + + assert response.index.tolist() == expected + + def test_transform_success(mlflow_repository): data = 'data' model_name = 'model' - output = mlflow_repository.transform(model_name, data, 1) + mlflow_repository.detect_and_parse_datetime_index = MagicMock() + + output = mlflow_repository.transform( + model_name, data, {}, metadata['metadata']) mlflow_repository.model_serving.get_cached_transform.assert_called_once_with( - model_name, data, 1) + model_name, data, 0, 'sklearn', False, 'model', 'predict') + + mlflow_repository.detect_and_parse_datetime_index.assert_called_once_with( + mlflow_repository.model_serving.get_cached_transform.return_value, metadata['metadata']) assert output == { 'success': True, - 'content': mlflow_repository.model_serving.get_cached_transform.return_value.to_dict.return_value + 'content': mlflow_repository.detect_and_parse_datetime_index.return_value.to_dict.return_value } @@ -44,10 +140,11 @@ def test_transform_error(mlflow_repository): mlflow_repository.model_serving.get_cached_transform.side_effect = Exception( 'error') - output = mlflow_repository.transform(model_name, data, 1) + output = mlflow_repository.transform( + model_name, data, {}, metadata['metadata']) mlflow_repository.model_serving.get_cached_transform.assert_called_once_with( - model_name, data, 1) + model_name, data, 0, 'sklearn', False, 'model', 'predict') assert output == { 'success': False, From 73c8593993227f881449c5953793e88f5d0b8fc4 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 10:07:32 -0300 Subject: [PATCH 20/52] SIENTIAPDE-1222 Update test cases in model repository and prediction process - Replaced string data with MagicMock in test_transform_success and test_transform_error to improve test isolation. - Updated the predict method calls in test_predict_success and test_predict_error to reflect changes in argument structure. - Added 'prediction_store_policy' to the test_run configuration in test_prediction_process for consistency with recent updates. --- .../utils/repository/test_model_repository.py | 21 ++++++++++++------- .../subworkflows/test_prediction_process.py | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 51a3223..5dab80d 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -113,7 +113,7 @@ def test_detect_and_parse_datetime_index_valid_format(mlflow_repository, data, e def test_transform_success(mlflow_repository): - data = 'data' + data = MagicMock() model_name = 'model' mlflow_repository.detect_and_parse_datetime_index = MagicMock() @@ -134,7 +134,7 @@ def test_transform_success(mlflow_repository): def test_transform_error(mlflow_repository): - data = 'data' + data = MagicMock() model_name = 'model' mlflow_repository.model_serving.get_cached_transform.side_effect = Exception( @@ -167,10 +167,11 @@ def test_predict_success(mlflow_repository): [2, 3] ) - output = mlflow_repository.predict(model_name, data, 1) + output = mlflow_repository.predict( + model_name, data, {}, metadata['metadata']) mlflow_repository.model_serving.get_cached_predict.assert_called_once_with( - model_name, data, 1) + model_name, data, 0, 'pyfunc', False, 'model') assert output['success'] is True assert output['content'] == { @@ -185,17 +186,23 @@ def test_predict_success(mlflow_repository): def test_predict_error(mlflow_repository): - data = 'data' + data = DataFrame({ + 'feat_1': { + 'index_1': 2, + 'index_2': 3 + } + }) model_name = 'model' mlflow_repository.model_serving.get_cached_predict = MagicMock( side_effect=Exception('error') ) - output = mlflow_repository.predict(model_name, data, 1) + output = mlflow_repository.predict( + model_name, data, {}, metadata['metadata']) mlflow_repository.model_serving.get_cached_predict.assert_called_once_with( - model_name, data, 1) + model_name, data, 0, 'pyfunc', False, 'model') assert output == { 'success': False, diff --git a/tests/laborious/workflows/subworkflows/test_prediction_process.py b/tests/laborious/workflows/subworkflows/test_prediction_process.py index e76b586..eea8c53 100644 --- a/tests/laborious/workflows/subworkflows/test_prediction_process.py +++ b/tests/laborious/workflows/subworkflows/test_prediction_process.py @@ -37,6 +37,7 @@ async def test_run(workflow_mock, prediction_process): 'model_retention': '30', 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': 'lts:1' } # Mock the activity responses From 03ac6a8f8d69ff655aaf230981a7bb85dd2004e0 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 10:11:54 -0300 Subject: [PATCH 21/52] SIENTIAPDE-1222 SIENTIAPDE-1222 Update MLFlow methods to include metadata parameter - Modified the transform and predict method calls in the MLFlow class to include a new 'metadata' parameter, enhancing the functionality and data handling capabilities of the model monitoring repository. --- laborious/activities/mlflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 896f05a..28086d5 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -120,7 +120,7 @@ class MLFlow(BaseActivity): # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( - model_name, data, model_config + model_name, data, model_config, metadata ) self.debug( @@ -176,7 +176,7 @@ class MLFlow(BaseActivity): # Request prediction from MLFlow model response_data = self.model_monitoring_repository.predict( - model_name, data, model_config + model_name, data, model_config, metadata ) self.debug( From 295cab9f73c34e6f6cb3711ed23e6bc21d8117af Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 10:27:02 -0300 Subject: [PATCH 22/52] SIENTIAPDE-1222 Update MLFlow class to pass logger instance directly to MLFlowRepository - Modified the initialization of MLFlowRepository in the MLFlow class to pass the logger instance directly, improving logging capabilities and consistency across the application. --- laborious/activities/mlflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 28086d5..b96cc60 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -60,7 +60,7 @@ class MLFlow(BaseActivity): self.mlflow_password = mlflow_password self.model_monitoring_repository = MLFlowRepository( - f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger.base_logger + f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger ) @activity.defn(name="request_transform") From b279244160fbd15bda2b3e592312b29295b9805c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 11:15:05 -0300 Subject: [PATCH 23/52] SIENTIAPDE-1222 Enhance MLFlow data handling by adding timestamp column and improving debug logging - Added a 'timestamp' column to the input data, converting the index to a datetime format for better tracking of predictions. - Improved debug logging to provide clearer context by including the input data preview in the log output. --- laborious/activities/mlflow.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index b96cc60..bd06283 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -169,11 +169,15 @@ class MLFlow(BaseActivity): model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) - self.debug(data.head(5).to_string(), metadata) + self.debug(f"Input data for: \n {data.head(5).to_string()}", metadata) # Convert numpy.nan to None for model compatibility data.replace(np.nan, None, inplace=True) + data['timestamp'] = data.index + data['timestamp'] = to_datetime( + data['timestamp'], format=DATETIME_FORMAT_WITH_TZ).dt.strftime(DATETIME_FORMAT) + # Request prediction from MLFlow model response_data = self.model_monitoring_repository.predict( model_name, data, model_config, metadata From 673fc79df33a92a7490e90f9165595d88f224ddc Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 16:02:54 -0300 Subject: [PATCH 24/52] SIENTIAPDE-1222 Refactor model configuration handling in MLFlow and workflows - Replaced 'model_retention' with 'model_config' to encapsulate retention settings and improve consistency across various components. - Updated test cases to reflect changes in argument structure, ensuring compatibility with the new model configuration format. - Added 'prediction_store_policy' to input data handling in workflows for enhanced configuration management. --- ## Problemas no Courier:.md | 9 -- tests/laborious/activities/test_mlflow.py | 45 +++++-- .../subworkflows/test_prediction_process.py | 115 +++++++++++------- .../workflows/test_predictions_batch.py | 11 +- 4 files changed, 111 insertions(+), 69 deletions(-) delete mode 100644 ## Problemas no Courier:.md diff --git a/## Problemas no Courier:.md b/## Problemas no Courier:.md deleted file mode 100644 index 48083e8..0000000 --- a/## Problemas no Courier:.md +++ /dev/null @@ -1,9 +0,0 @@ -## Problemas no Courier: -1. Enviamos a coluna timestamp do index para fazer o transform, para poder sincronizar a predição com o pacote que gerou ela, visto que vários modelos podem retornar uma lista de predições em vários casos. No caso do Courier, está vindo um timestamp que começa em 0, estando dessincronizado com os dados que enviamos. Seria possível alterar o comportamento do modelo para retornar o mesmo index que enviamos? - - Segue uma output do transform de exemplo: -``` -'303-WIT-230_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '303-WIT-230_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '303-WIT-230_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '303-WIT-230_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '305-WIT-135_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-135_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-WIT-135_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-135_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-160_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-WIT-160_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-WIT-160_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-WIT-160_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-PIT-170_median': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-170_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-PIT-170_min': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-170_max': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-175_median': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-PIT-175_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-PIT-175_min': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-PIT-175_max': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-FIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-FIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-013_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '305-FIT-013_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-FIT-013_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '305-FIT-013_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '306-PIT-101_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-PIT-101_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-101_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-PIT-101_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-FIT-051_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-FIT-051_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-051_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-FIT-051_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-DIT-001_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-DIT-001_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-DIT-001_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-DIT-001_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-PIT-105_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-PIT-105_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-105_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-PIT-105_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-FIT-052_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-FIT-052_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-052_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-FIT-052_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-DIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-DIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-DIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-DIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-PIT-115_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-PIT-115_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-115_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-PIT-115_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-FIT-004_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-FIT-004_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-004_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-FIT-004_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-PIT-110_median': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-PIT-110_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-110_min': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-PIT-110_max': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-FIT-003_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-FIT-003_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-003_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-FIT-003_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-PIT-125_median': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-PIT-125_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-125_min': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-PIT-125_max': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-PIT-130_median': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-PIT-130_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-130_min': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-PIT-130_max': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-FIT-006_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '306-FIT-006_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-006_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '306-FIT-006_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '307-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-003_median': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIT-003_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-003_min': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIT-003_max': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIC-022_median': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '307-FIC-022_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIC-022_min': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '307-FIC-022_max': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '310-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '310-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '310-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '310-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '307-FIT-008_median': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-008_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-008_min': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-008_max': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-009_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '307-FIT-009_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-009_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '307-FIT-009_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '309-PIT-101_median': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-101_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-101_min': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-101_max': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-105_median': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-105_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-105_min': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-105_max': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-110_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-110_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-110_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-110_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-185_median': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-185_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-185_min': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-185_max': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-190_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-190_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-190_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-190_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-195_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-PIT-195_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-195_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-PIT-195_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-FIT-051_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-051_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-FIT-051_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-051_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-052_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-FIT-052_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-FIT-052_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-FIT-052_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-PIT-001_median': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-001_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-001_min': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-001_max': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '309-PIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '309-PIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '317AIT003.3_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, 'SiO2_conc': {Timestamp('1970-01-01 00:00:01.732971600'): 5.15}}} -``` - -2. No modelo do transform (data_model), o nome do método que faz o transform de fato é "transform", sendo que em nossos modelos, por padrão esse nome é "predict". Seria possível alterar o nome do método manta manter a compatibilidade e o padrão que já temos? diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index ed68729..666ec7f 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -4,6 +4,7 @@ from unittest.mock import ANY, MagicMock, patch import numpy as np from pandas import DataFrame, Timestamp from pytest import fixture, mark, raises +from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ from laborious.activities.mlflow import MLFlow from sientia_do.notifications.models import NotificationLevel @@ -79,7 +80,7 @@ async def test_request_transform_success(mock_max, mock_dataframe, mlflow): 'value': 1.0, 'created_at': '2024-01-01 12:00:00'} ], 'model_name': 'test_model', - 'model_retention': 30 + 'model_config': {} } # Mock the transform response @@ -108,26 +109,35 @@ async def test_request_transform_success(mock_max, mock_dataframe, mlflow): # Verify the repository was called with correct arguments mlflow.model_monitoring_repository.transform.assert_called_once_with( - 'test_model', mock_dataframe, 30 + 'test_model', mock_dataframe, {}, metadata['metadata'] ) @mark.asyncio @patch("laborious.activities.mlflow.DataFrame") +@patch("laborious.activities.mlflow.to_datetime") @patch("laborious.activities.mlflow.max") -async def test_request_predict(mock_max, mock_dataframe, mlflow): +async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflow): mock_max.return_value = '2024-01-02' # Mock input data input_data = { **metadata, - 'data': [ - {'timestamp': '2024-01-01', 'variable': 'var1', 'value': 1.0}, - {'timestamp': '2024-01-01', 'variable': 'var2', 'value': 2.0}, - {'timestamp': '2024-01-02', 'variable': 'var1', 'value': 3.0}, - {'timestamp': '2024-01-02', 'variable': 'var2', 'value': 4.0} - ], + 'data': { + "variable": { + "2024-01-01": "var1", + "2024-01-02": "var2", + "2024-01-03": "var1", + "2024-01-04": "var2" + }, + "value": { + "2024-01-01": 1.0, + "2024-01-02": 2.0, + "2024-01-03": 3.0, + "2024-01-04": 4.0 + } + }, 'model_name': 'test_model', - 'model_retention': 30 + 'model_config': {} } # Mock the predict response @@ -141,13 +151,26 @@ async def test_request_predict(mock_max, mock_dataframe, mlflow): mock_dataframe.return_value.replace.assert_called_once_with( np.nan, None, inplace=True ) + mock_dataframe.return_value.__setitem__.assert_any_call( + 'timestamp', mock_to_datetime.return_value.dt.strftime.return_value + ) + mock_dataframe.return_value.__setitem__.assert_any_call( + 'timestamp', mock_to_datetime.return_value.dt.strftime.return_value + ) + + mock_to_datetime.assert_called_once_with( + mock_dataframe.return_value.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ + ) + mock_to_datetime.return_value.dt.strftime.assert_called_once_with( + DATETIME_FORMAT + ) # Verify the response assert response_data == expected_response # Verify the repository was called with correct arguments mlflow.model_monitoring_repository.predict.assert_called_once_with( - 'test_model', mock_dataframe.return_value, 30 + 'test_model', mock_dataframe.return_value, {}, metadata['metadata'] ) diff --git a/tests/laborious/workflows/subworkflows/test_prediction_process.py b/tests/laborious/workflows/subworkflows/test_prediction_process.py index eea8c53..df60ada 100644 --- a/tests/laborious/workflows/subworkflows/test_prediction_process.py +++ b/tests/laborious/workflows/subworkflows/test_prediction_process.py @@ -34,7 +34,9 @@ async def test_run(workflow_mock, prediction_process): 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'}, 'prediction_store_policy': 'lts:1' @@ -62,54 +64,54 @@ async def test_run(workflow_mock, prediction_process): workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { + **metadata, 'data': input_data['data'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.input_gate, { + **metadata, 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { + **metadata, 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], - **metadata + 'model_config': input_data['model_config'], }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_response_gate, { + **metadata, 'filters': input_data['mlflow_transform_filters'], 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_content_gate, { + **metadata, 'filters': input_data['mlflow_transform_filters'], 'data': 'transformed_data', 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_predict, { + **metadata, 'data': 'transformed_data', 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], - **metadata + 'model_config': input_data['model_config'], }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_response_gate, { + **metadata, 'filters': input_data['mlflow_predict_filters'], 'data': {'content': 'predicted_data', 'timestamp': '2024-01-01'}, 'type': 'predict', 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_child_workflow.assert_called_once_with( @@ -122,11 +124,12 @@ async def test_run(workflow_mock, prediction_process): 'timestamp': '2024-01-01', 'model_id': 1, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': input_data['model_config'], 'opc_output_config': input_data['opc_output_config'], 'schema': input_data['schema'], 'table_name': input_data['table_name'], - 'comment': 'Error' + 'comment': 'Error', + 'prediction_store_policy': input_data['prediction_store_policy'] } ) @@ -146,7 +149,9 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process): 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -165,13 +170,13 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process): workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY), call(Activities.input_gate, { 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY) ]) workflow_mock.execute_child_workflow.assert_not_called() @@ -192,7 +197,9 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -213,7 +220,7 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -221,14 +228,14 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY) @@ -261,7 +268,9 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -286,7 +295,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -294,14 +303,14 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -310,7 +319,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_content_gate, { @@ -318,7 +327,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'data': 'transformed_data', 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_child_workflow.assert_not_called() @@ -339,7 +348,9 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -365,7 +376,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -373,14 +384,14 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -389,7 +400,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_content_gate, { @@ -397,13 +408,13 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'data': 'transformed_data', 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_predict, { 'data': 'transformed_data', 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -412,7 +423,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'data': {'content': 'predicted_data', 'timestamp': '2024-01-01'}, 'type': 'predict', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_child_workflow.assert_not_called() @@ -429,7 +440,9 @@ async def test_path_flag_handler_stop(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' + model_config = { + 'retention': '30' + } # Act result = await prediction_process.path_flag_handler( @@ -440,7 +453,7 @@ async def test_path_flag_handler_stop(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention + 'model_config': model_config }, confidence, last_timestamp, "" ) @@ -462,7 +475,9 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' + model_config = { + 'retention': '30' + } # Act result = await prediction_process.path_flag_handler( @@ -473,7 +488,7 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention + 'model_config': model_config }, confidence, last_timestamp, "" ) @@ -506,7 +521,10 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' + model_config = { + 'retention': '30' + } + prediction_store_policy = 'erl:1' # Act result = await prediction_process.path_flag_handler( @@ -517,8 +535,9 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention, - 'opc_output_config': {'test': 'config'} + 'model_config': model_config, + 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': prediction_store_policy }, confidence, last_timestamp, 'Prediction Process' ) @@ -535,11 +554,12 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): 'timestamp': last_timestamp, 'model_id': model, 'model_name': model_name, - 'model_retention': model_retention, + 'model_config': model_config, 'schema': schema, 'table_name': table_name, 'comment': 'Prediction Process', - 'opc_output_config': {'test': 'config'} + 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': prediction_store_policy } ) @@ -556,8 +576,10 @@ async def test_path_flag_handler_unknown(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' - + model_config = { + 'retention': '30' + } + prediction_store_policy = 'erl:1' # Act result = await prediction_process.path_flag_handler( data, path_flag, { @@ -567,8 +589,9 @@ async def test_path_flag_handler_unknown(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention, - 'opc_output_config': {'test': 'config'} + 'model_config': model_config, + 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': prediction_store_policy }, confidence, last_timestamp, "" ) diff --git a/tests/laborious/workflows/test_predictions_batch.py b/tests/laborious/workflows/test_predictions_batch.py index e9b9bb6..90d7d21 100644 --- a/tests/laborious/workflows/test_predictions_batch.py +++ b/tests/laborious/workflows/test_predictions_batch.py @@ -33,7 +33,11 @@ async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch 'schema': 'test_schema', 'table_name': 'test_table', 'opc_output_config': 'test_opc_output_config', - 'datetime_columns': ['timestamp', 'created_at'] + 'datetime_columns': ['timestamp', 'created_at'], + 'prediction_store_policy': 'erl:1', + 'model_config': { + 'retention': '30' + } } await predictions_batch.run(input_data) @@ -72,9 +76,10 @@ async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch 'POLICY': 'STOP' } }), - 'model_retention': input_data.get('model_retention', 60), + 'model_config': input_data.get('model_config', {}), 'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']), - 'opc_output_config': input_data.get('opc_output_config', {}) + 'opc_output_config': input_data.get('opc_output_config', {}), + 'prediction_store_policy': input_data.get('prediction_store_policy', 'erl:1') } workflow_mock.execute_child_workflow.assert_has_calls([ From bbae728e42d11486944d16d7423e05f07baec99b Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 24 Sep 2025 09:59:26 -0300 Subject: [PATCH 25/52] SIENTIAPDE-1222 Add placeholder class 'Any' in test_model_repository.py and update invalid_cases to use it - Introduced a new placeholder class 'Any' to be used in test cases. - Updated the 'invalid_cases' list to replace integer keys with instances of the 'Any' class, enhancing test coverage for key types. --- tests/laborious/utils/repository/test_model_repository.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 5dab80d..cdc59b4 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -33,6 +33,10 @@ metadata = { } +class Any: + pass + + invalid_cases = [ ( { @@ -53,8 +57,8 @@ invalid_cases = [ ( { 'value': { - 1: 1, - 2: 2 + Any(): 1, + Any(): 2 } } ) From bb13c9a539e6581ddfbceeb8a41579289a4889c7 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 24 Sep 2025 10:30:23 -0300 Subject: [PATCH 26/52] Update tests/laborious/activities/test_mlflow.py Co-authored-by: codeant-ai[bot] <151821869+codeant-ai[bot]@users.noreply.github.com> --- tests/laborious/activities/test_mlflow.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index 666ec7f..85cb7f1 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -154,9 +154,6 @@ async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflo mock_dataframe.return_value.__setitem__.assert_any_call( 'timestamp', mock_to_datetime.return_value.dt.strftime.return_value ) - mock_dataframe.return_value.__setitem__.assert_any_call( - 'timestamp', mock_to_datetime.return_value.dt.strftime.return_value - ) mock_to_datetime.assert_called_once_with( mock_dataframe.return_value.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ From 8b2218e56aa4ea94b86a41f480e1ee347d3bd704 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 24 Sep 2025 10:32:56 -0300 Subject: [PATCH 27/52] SIENTIAPDE-1222 Remove deprecated files and configurations from transformer_pyfunc module - Deleted conda.yaml, MLmodel, python_env.yaml, requirements.txt, and various utility scripts related to data processing and model handling. - Removed binary files including python_model.pkl and training_transformer.pkl to clean up the artifacts directory. - This cleanup is part of the effort to streamline the transformer_pyfunc module and eliminate unused components. --- .../data_model/transformer_pyfunc/MLmodel | 19 - .../artifacts/training_transformer.pkl | Bin 29391 -> 0 bytes .../transformer_pyfunc/code/utils/__init__.py | 0 .../code/utils/data/.gitkeep | 0 .../code/utils/data/__init__.py | 0 .../code/utils/data/preprocessing.py | 172 ---- .../code/utils/data/read_data.py | 56 -- .../code/utils/data/transformers.py | 788 ---------------- .../code/utils/dvc/__init__.py | 0 .../code/utils/dvc/params.py | 51 - .../code/utils/features/.gitkeep | 0 .../code/utils/features/__init__.py | 0 .../code/utils/mlflow/pyfunc_wrappers.py | 325 ------- .../code/utils/models/.gitkeep | 0 .../code/utils/models/__init__.py | 24 - .../code/utils/models/arima.py | 389 -------- .../code/utils/models/base.py | 0 .../code/utils/models/catboost_time_series.py | 510 ---------- .../code/utils/models/evaluation.py | 92 -- .../code/utils/models/factory.py | 140 --- .../models/linear_regression_time_series.py | 303 ------ .../code/utils/models/neural_prophet_model.py | 888 ------------------ .../code/utils/models/stacking_time_series.py | 695 -------------- .../code/utils/visualization/.gitkeep | 0 .../code/utils/visualization/__init__.py | 0 .../data_model/transformer_pyfunc/conda.yaml | 11 - .../transformer_pyfunc/python_env.yaml | 7 - .../transformer_pyfunc/python_model.pkl | Bin 123 -> 0 bytes .../transformer_pyfunc/requirements.txt | 4 - .../transformers/courier_transformers.pkl | Bin 29607 -> 0 bytes 30 files changed, 4474 deletions(-) delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/MLmodel delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/conda.yaml delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl delete mode 100644 tmp/artifacts/data_model/transformer_pyfunc/requirements.txt delete mode 100644 tmp/artifacts/data_model/transformers/courier_transformers.pkl diff --git a/tmp/artifacts/data_model/transformer_pyfunc/MLmodel b/tmp/artifacts/data_model/transformer_pyfunc/MLmodel deleted file mode 100644 index 3fc50fb..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/MLmodel +++ /dev/null @@ -1,19 +0,0 @@ -artifact_path: transformer_pyfunc -flavors: - python_function: - artifacts: - transformer: - path: artifacts/training_transformer.pkl - uri: /tmp/tmpnzkqz3v0/training_transformer.pkl - cloudpickle_version: 2.2.1 - code: code - env: - conda: conda.yaml - virtualenv: python_env.yaml - loader_module: mlflow.pyfunc.model - python_model: python_model.pkl - python_version: 3.10.16 -mlflow_version: 2.7.1 -model_uuid: 6a4a99079d234d0da2b8091532d55a34 -run_id: c2edec4dfafd4ad8bba257d62d25cd43 -utc_time_created: '2025-09-09 12:30:04.885248' diff --git a/tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl b/tmp/artifacts/data_model/transformer_pyfunc/artifacts/training_transformer.pkl deleted file mode 100644 index 02d4f4a33b863e4e6a36c404e28917079c0a8fad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29391 zcmeHQS(6-BRlnF)O&Hi-feBBSHJz&e@y@WBsnyQdRy_J){3^spZVQUa(K{a$5d4(YR75vI2HSY zsJ)#gN2In9t~HiJ=yo>JW$7I?sX!hyd8C-Ezb1eORet4TZ^KM+$K*+i(0F3vp=a#(&T7u*x7Du zyw!?pH_3*SPj!av?YD(9?O|)sh&5WDKj;lN2Bf%;71QJZoM)Af()q{U zOJ{f8On;QFrswB(%^z5Nao784{maR|&B5E-FMWq*14SPw8DPeWk2wS&VnCZ!+-9hXAx$laWK}0P zSrt{f%*~)mJopQuBPdOg^2~trd}o z*ARs+kt#6}^?Qq`iiyMOxna%ix`-7?s>&o4FS4{`<+QBX~(HAKt7ohgxN$|e#-B_a?lZHYzLO04L#zM!*5Vwl!{$zm+(o*_Ak!s(+_ zrx;e%+Vb^v>4j)Fon2`Yie?q1VnO#l1?c`Mfnq^*M+NvAp|4TO0KQ4t*C_iM6RRaTd=mPLH?9f@m4qX7g zi5)s?*r79Ehb{o0Lxzq(3$o{7AefSv@mPj_D5Ef9(M=&%{hTJmqr~O(Aq%OfUIm>6 zo<+*sDJ)J5cMKJBhq}HHGhk0)tS_f9sm}rGL^D6yM(0%lGGb|4W;ICwwRf0!)P#K`4^;1W{543Jb-FRckP+9&@}fb!nwg z3@_ZcvVirNJDJ2vIat}otceyAy;Tys0TCxgJw<}B|?V(69%?6f+Smuui4Uur_hp$lQ*1JGVKj^8U?7F%yV@GP$ za(HcdL3LUf*!D=LWiP8FLVL8>_Aq$<-#nzZXF zP1^O8XsOd46-joGr{f!##6bG*&N>KIYgwfImQ^gdE|oESENQiv*8jr>ib$Fz4@JwW ztd+!`OBE{^MV4|Ep8c~8&>qr~hpsCn*A?2>TFO<-)&FDzMc0+kb*1dOQgPd;E0D75O2tiZwVKxd%T_PCu9RF?%C0LFH^J4Z zwEizPS?Icgu47Bft}7Kc!BuyNxNe26TUb~fV%M#To9yazTK`X5o%XO+!b+}NW!J5W zn{2%AzzDRgL)R60HnyZ?*OiJJ)f7gq^OWrR=&Q9=!ETF@>m3(L-qwvoUQ? z7mI=ThuM)IRJUP*Hl3#3uD0It2tz!Qb3Ioe3qW|voM zt9aP5%U8YfB_vMC*RQ;>uEkTHxhNZRQ8wmNHQT%xy5_R1&1G4eqb$t#3@+5+!)m-S z+>U$E-Jjh3^KP2#s-?|e-u2jHBQMFGm9=YWa~3+uzV^@lib!^yOPiO;=n7r>FG_g=zDeHh7KhxKh3tfw>zg zFj2A!7ugjiW&i0e*|T+FAf05fan~>yfpNy9;WOMNd(O?pX>-Ai(^Rsd)M%b1z2E+;JUzq2A10+Ukm4MW$$C zC#qyWhk;Jb8Z6{0R*BPKM!;R+(_NB7tMQ<**$u)%;YAGnm8jp3@G>|kg%y;Q6m#f{ zY4bXJcP^lp(89Z9H@af7mx1K*b3wn`0$CsLXO}SJ?p{Kx>|QFR&DY8GH|X*vU4Dfw zze<;1qswQICcEPxeJISI#jA&Bo9voNo4-y;8+*C>TmK_IN%Ph{m1 zl)hOkp{1``!kw_dm!m7)!QH>;Y&3L}cnEY3u5~xH*c2c+LQZq$WqY~+G!AXQ8Z6JL z{97gBWy|aRMkh*>XSY>@^`icEwAt+ic$!y+XKsro%9EYlj640tX0#axdE#8+@)%+v zgMazNT6{{j&Fmd&097>_M&cWH0(55%B~4E1@Dvu%JH@ADTu7o;W z>fHSaTK5Sl2zVglGtSz%TCe~Yspmt6fs5hFtu9g3;%L=d_(*Y*R00YL7H2N&7Qa3h zu!OAwG}{K*3}-Uio?HhtZmJ*^bVv2EFcZ zs~-1)x%B`k^ANO!hm>}gzVcOZbm{%w}nf+jP*rTsBq{#~q z%SCO3#)%HEMsDh}AUw)~Xn^CwT&0)WKq@LsG{Z?*kJl1;FdE~m>y=5NXpeDXG|6*g ztf^I`+^~8LGggjFJY$br9uqVT(ZGc8X>0gn#EG4T5Ggy%L6`?d2ze1(>fVk-gONQv zHnUBx{2BC?AfWdMg%EEn?AejrHNuiFJS&Ggj5qESE@s45G?>i%V4u~~3r%&t^e7f< zM6O{>M?h$e7vG1mC1<1`t7;D5jLz{qsbspDlnm*9R@#}y6m0=ZbWdfOVZ~H&3S%>a zV^|WNSJP={EZeo@GNA@5JNDemeF9RMTG16;|7nU6OCKu-)coX`jmgyzz2g)$Fuz2a z7JD~2m1{gKIQW-j&WeTEHwuNRV8W8B6;=7J7Z_UoFqP0EjPWVkJtGy4T8m>q3+LKO zuzKY+8XeYBYfBcnu_URoKxM`0^+o<%?76HAuj8Z~sXm4;&QKIom0IzoNt+)KdSJui z7X|$x{8u9}rl8=N;X0SM*ki`g}jeceJoApx1we zO4SNKP0qRB#URLf)EF1y_N zJne>FlBL2Fl~wf)&FT)W0Nvr~6}1yaJBx=oehh$(5%CkCfP*z!d`5|o79(;A zow^KR8H6M1A`z_k$?$UA-{6fLJyStH3qtcn*#YsG9kqUA7^8oLr2qkPTq>+QtuiKa zsnzZT^qr$~FcZ(p;A+DnOkRHAOsNFRjGAjkq%i*61BvpC^MH zwb5SCXDeg8An|xFNMi?HklL3-E;vd&cE!E!=5WxpeNLOlcH@cZE3%_Ucy2~;^2t&s zP-g{G7Fv*|Sdr|*vLqLmBeEzr*Jf>&gz}}6cp9Y*ki>aANXr$PLp}jX=!n{<&*D46 z^ag_fvqS2`NCTSvAl`*6G&=Q&9+*r4YtM12L6~dplh&(Q-Dj{V)9+~cC9NfSN*d7H zT6*oGtI8M9`i=*2dU>0DC`pszjk_PWZsSohxb${FbxI}d=K8^Q)Wf6Z9rWM1c%vTS z1<}g^+V%O_`26GF`>%rW$rR6ms^!^Xhjl|Xv3L+9K*sKbTQyR=SJIeAY=F`h;d>BFis5TpS3F&qa3|! zQu^+==17I>4!J9*CKydlj*pST?=e!i?Ie}^5atj)81U}N>SdUqsP&D4@O>Nw=iLq7 zr6B*u$`Qd>d2S5B)z#&-wY(86W3UuN5ZN$o6>mgmNhpkL9xMv_m}$I%8wjRG-c2rC zTY7aSpbf1zptA~@Kyp;PK(ZQMAUO(dAUO(dAYxUUq`BkDwe{s}AX$Y>AUP^tAXyDB zkQ@a!kQ@aBqGm0)Z5E6C&XuDvAwGt@2a7Ywj;M&DxrZ;*!|tWxHVv8tV|mQ-msQ9# zO^%A!G+7O=X>t_YrpZwl6$mwrnscpW7zU*DoO<68MdQf;zo?$IJWBD2|WDrcgWJDv_m_P-mg_K^UknkTb+06 z8pL-7L+tS_|BPFoDja-H@F~LCsg)ZJtacg>R6k8PMGV;c4`;_#PB>(>!*EFT5Pzl~ zT*XY$N}E5xWAqL#$uW9WhlypS*J$7UxJO^K`XQMCB-l5B{^n=B;I2*-=lzjo|ojL zXqDg+Ry6hLXFqMePmbkW;`4dvXTg%=vg@zmd$05>Wa|4rqUwK6m*kA>sPr`$Ozre; zNUri}^8<41zAyNI3P0~|i0@b|ckop$e1-HwqIz(b%IUWMba2g{pf(};0HL;ufi zOoZkx8={e!yR4Q*cBG`W^^GyRuRCq8Z^oNzwFh-$s>?EtlV!C(PsDg;?(VC8W5#!5 zZ0|$UOfGacj&tXlG));79t`K_SbKav>#_;wp1I$-r>>@%#vk5YjPrJrU${AJ;Gkdn z>C?V`tFb``_0B&we_(zWozpwNo38KiAFAd*^_su^e zch2N_k7KB}$o*CN-}1NlQ6qvgCmPdTyq(VPV(OUqlf5+@aE#wa zQ;g~e=Nf(+j4xOt=Ad1RDTq>MUX?Z2p0D4rUMhAVv zE2Be)ehaVopj@17=};-qP$|SvX~d)Q0u*W$%PWQWR4~+3;#kPg^-6)pD}@*;jkr`e z4A%*e)c|@_q^5JFbd*>6ykwiofQrf>ipoNySLz7mqD#d!R*KpyOHF!ZA(~2_)a+6z z(0HW~eGxibc3l5zH$q0(xeS1{C6qB^Iy z%d{6vZ#RhV6$~{MR~P2x3ZUl|skvSeqN#*e9+d(Ol|l@aMl=zB&D}(p}!qB6d&ZC;nP(jTKkir2X zsniL-&V+10l`DfNDhu(boX()s%)l{-Ph|#<)WIZR^>zb#UXhv%(CBypUd;0fphrb& zE)^kefYW(i0raRy&7~s5qssGYI^z`(pUU=%#~@ZuCsztIUMa*-X+(P^5NF`%GnuUb zwVDo4Bbth)9Xb=!@k)WlD}@-ZG~!WtUI{g`q>84eKS>)5%nK;J8qaw3^P9h4@r5S5KPLo=a0YS;rid z{c<5@^B~HXMOQYHmZb>Prz1s*)79cYExM{nnxzQTrz1s=jv&vwl9`7~bOd>HrQD_f>eG>;M@NuHXU#yCcWPJf zLOvbYdURDYNG)5SzIUYPc}I{(hd(Hh(-c5`I#TrL2=eGExlIAorz1s=jv$Y&YPQpM zu;mOLDSC7Sd35F6Q32HVjubsQf;_rPZc_mD=}6I|BgmtxnmN-BR!*8O`19VvQr1bK8-v(2_6t7hm((W4{Cqbuh&1yJ8RQuOEu^62pA zVsb_WP@j$zJvxFsx@zuP0o11>MURdkkFK2C6hM7CQuOEu^5`nLO##%WBSnvnAdjw^ zyH)`8=}6I|Bgmt}AGpbB3ZOn6DSC7Sd32TBrU2^Gk)lUOkVjX|T`Pe4bfoCf5#-U8 zbDILFPe+O#9YG#lCATSn`gEk|(Gld)S#JfcF$>hEBSnvnAdjw`+Y~^3I#TrL2=eGE zxlIAorz1s=jv$Y2DtD~_>eG>;M@NuHSI%t;pgtWbdUOPNbk@6Gt0~mWUKjG|$kwBq n%3Ujf`reVE=N&S;hzzhH3&B^}*E89w3 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py deleted file mode 100644 index 4ee9088..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/preprocessing.py +++ /dev/null @@ -1,172 +0,0 @@ -""" -Module with utility functions for data processing. -""" - -import pandas as pd -from typing import Dict, List -from rich.console import Console - -console = Console() - - -def create_lagged_target( - data: pd.DataFrame, - target_column: str, - lags: List[int], - drop_nans: bool = True, -) -> pd.DataFrame: - """ - Creates lagged versions of a target column in a DataFrame. - - For each lag value in the provided list, a new column is created with - the naming pattern: target_column + "_lag_" + lag_value. - - Args: - data: Input DataFrame containing the target column. - target_column: Name of the target column to create lags for. - lags: List of lag values (integers between 1 and len(data)-1). - drop_nans: Whether rows with nulls generated by the lag creation - process should be dropped. Defaults to True. - Returns: - DataFrame with original columns plus the newly created lag columns. - """ - if data.empty: - console.log("[red]Warning: Input data is empty.") - return data - - result = data.copy() - - if target_column not in result.columns: - raise ValueError(f"Target column '{target_column}' not found in data.") - - # Validate lag values - max_lag = len(data) - 1 - valid_lags = [lag for lag in lags if 1 <= lag <= max_lag] - - if len(valid_lags) < len(lags): - invalid_lags = set(lags) - set(valid_lags) - console.log( - f"[yellow]Warning: Ignoring invalid lag values: {invalid_lags}. " - f"Lags must be between 1 and {max_lag}." - ) - - lag_column_names = [] - for lag in valid_lags: - lag_column_name = f"{target_column}_lag_{lag}" - result[lag_column_name] = result[target_column].shift(lag) - console.log(f"Created lagged column: [cyan]{lag_column_name}") - lag_column_names.append(lag_column_name) - if drop_nans: - result = result.dropna(subset=lag_column_names) - - return result - - -def remove_stopped_windows( - data: pd.DataFrame, - stopped_process_columns: Dict[str, float], - stopped_process_threshold: float, - time_colname: str, -) -> pd.DataFrame: - """ - Removes time windows from the input DataFrame if the proportion of samples - below a column threshold exceeds the specified limit. - - A window is considered "stopped" if *all* specified columns exceed the - stopped sample threshold. - - Args: - data: Input DataFrame with process variables and timestamps. - stopped_process_columns: Dict mapping column names to thresholds. - stopped_process_threshold: Proportion threshold (0-1) for marking a - window as stopped. - time_colname: Base name of the timestamp column - (without 'lab_' prefix). - - Returns: - A DataFrame with stopped windows removed. - """ - if data.empty: - console.log("[red]Warning: Input data is empty.") - return data - - console.log( - "Removing windows where any column exceeds" - + f" {stopped_process_threshold:.2%} of values below threshold" - ) - - masks = [] - - for col, threshold in stopped_process_columns.items(): - console.log( - "Evaluating stopped condition for column:" - + f" [cyan]{col} < {threshold}" - ) - below_threshold = data[[col]].lt(threshold) - - console.log( - "Counting number of samples below threshold for each window" - ) - below_threshold[f"lab_{time_colname}"] = data[f"lab_{time_colname}"] - grouped = below_threshold.groupby(f"lab_{time_colname}")[col].agg( - ["sum", "count"] - ) - stopped_mask = ( - grouped["sum"] / grouped["count"] - ) > stopped_process_threshold - - console.log( - f"[red]{stopped_mask.sum()} windows marked as stopped by {col}" - ) - masks.append(stopped_mask) - - # Combine masks across columns: only drop if all agree - combined_mask = pd.concat(masks, axis=1).all(axis=1) - - num_removed = combined_mask.sum() - total = combined_mask.shape[0] - console.log( - f"Removing [bold red]{num_removed}[/] out of {total}" - + f" windows ({num_removed / total:.2%})" - ) - - to_remove = combined_mask[combined_mask].index - keep_mask = ~data[f"lab_{time_colname}"].isin(to_remove) - - return data[keep_mask] - - -def aggregate_data( - merged_data: pd.DataFrame, - time_colname: str, - target_colname: str, - aggregation_functions: List[str], -) -> pd.DataFrame: - """ - Aggregates a DataFrame by time and target columns using specified - aggregation functions. - - Args: - merged_data: Input DataFrame with raw observations. - time_colname: Name of the timestamp column (no 'lab_' prefix). - target_colname: Name of the target/grouping column. - aggregation_functions: List of aggregation functions to apply - (e.g. "mean", "std"). - - Returns: - Aggregated DataFrame with flattened column names and renamed time - column. - """ - group_by_cols = [f"lab_{time_colname}", target_colname] - - aggregated = merged_data.groupby(group_by_cols).agg(aggregation_functions) - # Flatten MultiIndex columns - aggregated.columns = [ - "_".join(col) if isinstance(col, tuple) else col - for col in aggregated.columns - ] # type: ignore - aggregated = aggregated.reset_index() - - console.log(f"[bold green]Aggregated shape: {aggregated.shape}") - - return aggregated.rename(columns={f"lab_{time_colname}": time_colname}) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py deleted file mode 100644 index 76e4e9d..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/read_data.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -Module with helper functions to read datasets. -""" - -import pandas as pd -from openpyxl import load_workbook -from typing import Union - - -def read_excel_with_colors( - filepath: str, color_columns: list[str], sheet_name: Union[str, int] = 0 -) -> pd.DataFrame: - """ - Read an Excel file and extract cell fill colors for specified columns. - - Parameters: - filepath (str): Path to the Excel file. - color_columns (List[str]): Column names to extract fill colors from. - sheet_name (str or int): Sheet name or index (default is first sheet). - - Returns: - DataFrame: DataFrame with original data and extra color columns. - """ - df = pd.read_excel(filepath, sheet_name=sheet_name) - - workbook = load_workbook(filepath) - sheet = ( - workbook[sheet_name] - if isinstance(sheet_name, str) - else workbook[workbook.sheetnames[sheet_name]] - ) - - header = next(sheet.iter_rows(min_row=1, max_row=1, values_only=True)) - col_name_to_letter = { - name: chr(65 + idx) for idx, name in enumerate(header) - } - - for col_name in color_columns: - if col_name not in df.columns: - raise ValueError(f"Column '{col_name}' not found in Excel file.") - - col_letter = col_name_to_letter[col_name] - fill_colors: list[Union[str, None]] = [] - - for row in range(2, sheet.max_row + 1): - cell = sheet[f"{col_letter}{row}"] - fill = cell.fill - - if fill.fill_type == "solid" and fill.fgColor.rgb: - fill_colors.append(fill.fgColor.rgb) - else: - fill_colors.append(None) - - df[f"{col_name}_fill_color"] = fill_colors - - return df diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py deleted file mode 100644 index efe43e0..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/data/transformers.py +++ /dev/null @@ -1,788 +0,0 @@ -""" -Module with scikit-learn transformers for training and inference pipelines. - -This module replicates the functionality from the DVC pipeline stages: -- preprocessing.py (stopped process filtering, aggregation) -- filter_columns.py (feature selection, lagged target creation) -""" - -import pandas as pd -from typing import Dict, List, Optional -from sklearn.base import BaseEstimator, TransformerMixin -from rich.console import Console - -console = Console() - - -class CourierTrainingTransformer(BaseEstimator, TransformerMixin): - """ - Training transformer that replicates the DVC pipeline preprocessing. - - Includes: - 1. Remove stopped process windows (training only) - 2. Data aggregation - 3. Feature selection (learns and applies - dictionary-based only) - 4. Create lagged target features - """ - - def __init__( - self, - aggregation_functions: List[str] = ["median", "std", "min", "max"], - stopped_process_columns: Dict[str, float] = { - "305-PIT-170": 9, - "305-PIT-175": 9, - }, - stopped_process_threshold: float = 0.1, - target_lags: List[int] = [2], - time_colname: str = "timestamp", - target_colname: str = "SiO2_conc", - dictionary_df: Optional[pd.DataFrame] = None, - create_lagged_target: bool = True, - drop_nans: bool = True, - ): - """ - Initialize the training transformer. - - Args: - aggregation_functions: List of aggregation functions to apply - stopped_process_columns: Dict mapping column names to thresholds - stopped_process_threshold: Proportion threshold for stopped windows - target_lags: List of lag values for target column - time_colname: Name of timestamp column (without 'lab_' prefix) - target_colname: Name of target column - dictionary_df: DataFrame with domain knowledge - (TAG_fill_color column) - create_lagged_target: Whether to create lagged target features - drop_nans: Whether to drop NaNs after creating lags - """ - self.aggregation_functions = aggregation_functions - self.stopped_process_columns = stopped_process_columns - self.stopped_process_threshold = stopped_process_threshold - self.target_lags = target_lags - self.time_colname = time_colname - self.target_colname = target_colname - self.dictionary_df = dictionary_df - self.create_lagged_target = create_lagged_target - self.drop_nans = drop_nans - - # Will be learned during fit - self.selected_features_ = None - - def _infer_lab_timestamp(self, data: pd.DataFrame) -> pd.DataFrame: - """ - Creates a lab timestamp column by rounding the timestamp up to the next - even hour. - - Args: - data: Input DataFrame - - Returns: - DataFrame with added lab timestamp column - """ - lab_col_name = f"lab_{self.time_colname}" - - if lab_col_name in data.columns: - console.log( - f"[yellow]Lab timestamp column {lab_col_name} already exists, " - + "skipping inference" - ) - return data - - # Check if timestamp is a column or the index - if self.time_colname in data.columns: - # Timestamp is a regular column - result = data.copy() - timestamp_col = pd.to_datetime(result[self.time_colname]) - elif data.index.name == self.time_colname or ( - hasattr(data.index, "names") - and self.time_colname in data.index.names - ): - # Timestamp is the index (or part of a MultiIndex) - result = data.copy() - timestamp_col = pd.to_datetime( - result.index.get_level_values(self.time_colname) - if hasattr(result.index, "names") - and len(result.index.names) > 1 - else result.index - ) - else: - raise ValueError( - f"Timestamp '{self.time_colname}' not found in data columns " - + f"or index. Available columns: {list(data.columns)}, " - + f"index name: {data.index.name}" - ) - - # Round up to next even hour - # Step 1: Floor to the hour to remove minutes/seconds - # Handle both Series (from column) and DatetimeIndex (from index) - if hasattr(timestamp_col, "dt"): - # timestamp_col is a Series - hour_floor = timestamp_col.dt.floor("H") - hour = hour_floor.dt.hour - else: - # timestamp_col is a DatetimeIndex - hour_floor = timestamp_col.floor("H") - hour = hour_floor.hour - - # Step 3: Determine if rounding is needed - # - If hour is odd, round up to next even hour - # - If hour is even but original timestamp had minutes/seconds, - # round up to next even hour - # - If hour is even and original timestamp was exactly on the hour, - # keep it - needs_rounding = (hour % 2 == 1) | (timestamp_col != hour_floor) - - # Calculate next even hour - next_even_hour = ((hour // 2) + 1) * 2 - - # Handle case where next even hour >= 24 (next day) - days_to_add = (next_even_hour >= 24).astype(int) - hour_component = next_even_hour % 24 - - # Create the lab timestamp - if hasattr(timestamp_col, "dt"): - # timestamp_col is a Series - lab_timestamp = hour_floor.where( - ~needs_rounding, - hour_floor.dt.floor("D") - + pd.to_timedelta(days_to_add, unit="D") - + pd.to_timedelta(hour_component, unit="H"), - ) - else: - # timestamp_col is a DatetimeIndex - base_date = hour_floor.floor("D") - next_even_timestamp = ( - base_date - + pd.to_timedelta(days_to_add, unit="D") - + pd.to_timedelta(hour_component, unit="H") - ) - lab_timestamp = pd.Series( - hour_floor.where(~needs_rounding, next_even_timestamp), - index=result.index, - ) - - result[lab_col_name] = lab_timestamp - - console.log( - f"[bold green]Created lab timestamp column: {lab_col_name}" - ) - - return result - - def _remove_stopped_windows( - self, - data: pd.DataFrame, - ) -> pd.DataFrame: - """ - Removes time windows where process was stopped. - Replicates remove_stopped_windows from preprocessing.py - """ - if data.empty: - console.log("[red]Warning: Input data is empty.") - return data - - console.log( - "Removing windows where any column exceeds" - + f" {self.stopped_process_threshold:.2%} of values below" - + " threshold" - ) - - masks = [] - - for col, threshold in self.stopped_process_columns.items(): - console.log( - "Evaluating stopped condition for column:" - + f" [cyan]{col} < {threshold}" - ) - below_threshold = data[[col]].lt(threshold) - - console.log( - "Counting number of samples below threshold for each window" - ) - below_threshold[f"lab_{self.time_colname}"] = data[ - f"lab_{self.time_colname}" - ] - grouped = below_threshold.groupby(f"lab_{self.time_colname}")[ - col - ].agg(["sum", "count"]) - stopped_mask = ( - grouped["sum"] / grouped["count"] - ) > self.stopped_process_threshold - - console.log( - f"[red]{stopped_mask.sum()} windows marked as stopped by {col}" - ) - masks.append(stopped_mask) - - # Combine masks across columns: only drop if all agree - if not masks: - return data - - mask_df = pd.concat(masks, axis=1) - combined_mask = mask_df.all(axis=1) - - num_removed = int(combined_mask.sum()) # type: ignore - total = combined_mask.shape[0] - console.log( - f"Removing [bold red]{num_removed}[/] out of {total}" - + f" windows ({num_removed / total:.2%})" - ) - - to_remove = combined_mask[combined_mask].index - keep_mask = ~data[f"lab_{self.time_colname}"].isin(to_remove) - - filtered_data = data[keep_mask] - return filtered_data # type: ignore - - def _aggregate_data( - self, - merged_data: pd.DataFrame, - ) -> pd.DataFrame: - """ - Aggregates data by time and target columns. - Replicates aggregate_data from preprocessing.py - """ - group_by_cols = [f"lab_{self.time_colname}", self.target_colname] - - aggregated = merged_data.groupby(group_by_cols).agg( - self.aggregation_functions - ) - # Flatten MultiIndex columns - aggregated.columns = [ - "_".join(col) if isinstance(col, tuple) else col - for col in aggregated.columns - ] # type: ignore - aggregated = aggregated.reset_index() - - # Rename timestamp column and ensure it's datetime - aggregated = aggregated.rename( - columns={f"lab_{self.time_colname}": self.time_colname} - ) - aggregated[self.time_colname] = pd.to_datetime( - aggregated[self.time_colname] - ) - - console.log(f"[bold green]Aggregated shape: {aggregated.shape}") - - return aggregated - - def _learn_feature_selection( - self, - data: pd.DataFrame, - ) -> List[str]: - """ - Learn which features to keep based on dictionary only. - Replicates domain knowledge filtering from filter_columns.py - """ - if self.dictionary_df is None: - console.log( - "[yellow]Warning: No dictionary data provided. Using all" - + " features." - ) - return [col for col in data.columns if col != self.time_colname] - - # Domain knowledge filter - only keep columns with TAG_fill_color - columns_to_keep_dict = self.dictionary_df.loc[ - self.dictionary_df["TAG_fill_color"].notna(), "TAG" - ].values - - # Filter data columns to only those that match dictionary tags - available_columns = [ - col for col in data.columns if col != self.time_colname - ] - columns_to_keep = [ - col - for col in available_columns - if col.split("_")[0] in columns_to_keep_dict - ] - - console.log( - f"Keeping {len(columns_to_keep)}/{len(available_columns)}" - + " columns based on dictionary." - ) - - # Always include target - columns_to_keep.append(self.target_colname) - - return columns_to_keep - - def _create_lagged_target( - self, - data: pd.DataFrame, - ) -> pd.DataFrame: - """ - Creates lagged versions of target column. - Replicates create_lagged_target from preprocessing.py - """ - if data.empty: - console.log("[red]Warning: Input data is empty.") - return data - - result = data.copy() - - if self.target_colname not in result.columns: - raise ValueError( - f"Target column '{self.target_colname}' not found in data." - ) - - # Validate lag values - max_lag = len(data) - 1 - valid_lags = [lag for lag in self.target_lags if 1 <= lag <= max_lag] - - if len(valid_lags) < len(self.target_lags): - invalid_lags = set(self.target_lags) - set(valid_lags) - console.log( - f"[yellow]Warning: Ignoring invalid lag values: {invalid_lags}" - + f". Lags must be between 1 and {max_lag}." - ) - - lag_column_names = [] - for lag in valid_lags: - lag_column_name = f"{self.target_colname}_lag_{lag}" - result[lag_column_name] = result[self.target_colname].shift(lag) - console.log(f"Created lagged column: [cyan]{lag_column_name}") - lag_column_names.append(lag_column_name) - - if self.drop_nans and lag_column_names: - result = result.dropna(subset=lag_column_names) - - return result - - def fit(self, X: pd.DataFrame, y=None): - """ - Learn feature selection parameters. - - Args: - X: Input DataFrame with merged process and quality data - y: Not used - - Returns: - self - """ - console.log("[bold blue]Training transformer fit phase") - - # Create a copy for processing - data = X.copy() - - # Step 0: Infer lab timestamp if needed - console.log("[bold blue]Inferring lab timestamp") - data = self._infer_lab_timestamp(data) - - # Step 1: Remove stopped process windows (training only) - console.log("[bold blue]Removing stopped process windows") - data = self._remove_stopped_windows(data) - - # Step 2: Aggregate data - console.log("[bold blue]Aggregating data") - data = self._aggregate_data(data) - - # Step 3: Learn feature selection - console.log("[bold blue]Learning feature selection") - self.selected_features_ = self._learn_feature_selection(data) - - console.log( - f"[bold green]Learned {len(self.selected_features_)}" - + " features for selection" - ) - self._feature_names = self.selected_features_ - - return self - - def transform(self, X: pd.DataFrame) -> pd.DataFrame: - """ - Apply the complete training transformation pipeline. - - Args: - X: Input DataFrame with merged process and quality data - - Returns: - Transformed DataFrame ready for model training - """ - if self.selected_features_ is None: - raise ValueError("Transformer must be fitted before transform.") - - console.log("[bold blue]Training transformer transform phase") - - # Create a copy for processing - data = X.copy() - - # Step 0: Infer lab timestamp if needed - console.log("[bold blue]Inferring lab timestamp") - data = self._infer_lab_timestamp(data) - - # Step 1: Remove stopped process windows (training only) - console.log("[bold blue]Removing stopped process windows") - data = self._remove_stopped_windows(data) - - # Step 2: Aggregate data - console.log("[bold blue]Aggregating data") - data = self._aggregate_data(data) - - # Step 3: Apply feature selection - console.log("[bold blue]Applying feature selection") - # Set timestamp as index for filtering and ensure it's datetime - data[self.time_colname] = pd.to_datetime(data[self.time_colname]) - data = data.set_index(self.time_colname) - data = data[self.selected_features_] - - # Step 4: Create lagged target features - if self.create_lagged_target: - console.log("[bold blue]Creating lagged target features") - data = self._create_lagged_target(data) - - console.log(f"[bold green]Final training data shape: {data.shape}") - - return data - - -class CourierInferenceTransformer(BaseEstimator, TransformerMixin): - """ - Inference transformer that replicates DVC pipeline preprocessing - without training-specific steps. - - Includes: - 1. Data aggregation (higher frequency - no grouping by target) - 2. Feature selection (applies learned selection) - 3. Create lagged target features - - Note: Does NOT include stopped process filtering (training only). - """ - - def __init__( - self, - selected_features: List[str], - aggregation_functions: List[str] = ["median", "std", "min", "max"], - target_lags: List[int] = [2], - time_colname: str = "timestamp", - target_colname: str = "SiO2_conc", - create_lagged_target: bool = True, - drop_nans: bool = True, - ): - """ - Initialize the inference transformer. - - Args: - selected_features: Pre-learned list of features to select - aggregation_functions: List of aggregation functions to apply - target_lags: List of lag values for target column - time_colname: Name of timestamp column (without 'lab_' prefix) - target_colname: Name of target column - create_lagged_target: Whether to create lagged target features - drop_nans: Whether to drop NaNs after creating lags - """ - self.selected_features = selected_features - self.aggregation_functions = aggregation_functions - self.target_lags = target_lags - self.time_colname = time_colname - self.target_colname = target_colname - self.create_lagged_target = create_lagged_target - self.drop_nans = drop_nans - - def _infer_lab_timestamp(self, data: pd.DataFrame) -> pd.DataFrame: - """ - Creates a lab timestamp column by rounding the timestamp up to the next - even hour. - - Args: - data: Input DataFrame - - Returns: - DataFrame with added lab timestamp column - """ - lab_col_name = f"lab_{self.time_colname}" - - if lab_col_name in data.columns: - console.log( - f"[yellow]Lab timestamp column {lab_col_name} already exists, " - + "skipping inference" - ) - return data - - # Check if timestamp is a column or the index - if self.time_colname in data.columns: - # Timestamp is a regular column - result = data.copy() - timestamp_col = pd.to_datetime(result[self.time_colname]) - elif data.index.name == self.time_colname or ( - hasattr(data.index, "names") - and self.time_colname in data.index.names - ): - # Timestamp is the index (or part of a MultiIndex) - result = data.copy() - timestamp_col = pd.to_datetime( - result.index.get_level_values(self.time_colname) - if hasattr(result.index, "names") - and len(result.index.names) > 1 - else result.index - ) - else: - raise ValueError( - f"Timestamp '{self.time_colname}' not found in data columns " - + f"or index. Available columns: {list(data.columns)}, " - + f"index name: {data.index.name}" - ) - - # Round up to next even hour - # Step 1: Floor to the hour to remove minutes/seconds - # Handle both Series (from column) and DatetimeIndex (from index) - if hasattr(timestamp_col, "dt"): - # timestamp_col is a Series - hour_floor = timestamp_col.dt.floor("H") - hour = hour_floor.dt.hour - else: - # timestamp_col is a DatetimeIndex - hour_floor = timestamp_col.floor("H") - hour = hour_floor.hour - - # Step 3: Determine if rounding is needed - # - If hour is odd, round up to next even hour - # - If hour is even but original timestamp had minutes/seconds, - # round up to next even hour - # - If hour is even and original timestamp was exactly on the hour, - # keep it - needs_rounding = (hour % 2 == 1) | (timestamp_col != hour_floor) - - # Calculate next even hour - next_even_hour = ((hour // 2) + 1) * 2 - - # Handle case where next even hour >= 24 (next day) - days_to_add = (next_even_hour >= 24).astype(int) - hour_component = next_even_hour % 24 - - # Create the lab timestamp - if hasattr(timestamp_col, "dt"): - # timestamp_col is a Series - lab_timestamp = hour_floor.where( - ~needs_rounding, - hour_floor.dt.floor("D") - + pd.to_timedelta(days_to_add, unit="D") - + pd.to_timedelta(hour_component, unit="H"), - ) - else: - # timestamp_col is a DatetimeIndex - base_date = hour_floor.floor("D") - next_even_timestamp = ( - base_date - + pd.to_timedelta(days_to_add, unit="D") - + pd.to_timedelta(hour_component, unit="H") - ) - lab_timestamp = pd.Series( - hour_floor.where(~needs_rounding, next_even_timestamp), - index=result.index, - ) - - result[lab_col_name] = lab_timestamp - - console.log( - f"[bold green]Created lab timestamp column: {lab_col_name}" - ) - - return result - - def _aggregate_data( - self, - merged_data: pd.DataFrame, - ) -> pd.DataFrame: - """ - Aggregate data into 2-hour non-overlapping windows. - Each row corresponds to one 2-hour window ending at an even hour. - """ - if merged_data.empty: - console.log("[red]Warning: Input data is empty.") - return merged_data - - lab_col = f"lab_{self.time_colname}" - if lab_col not in merged_data.columns: - raise ValueError( - f"Missing '{lab_col}' column. Call _infer_lab_timestamp first." - ) - - # Get numeric columns only for aggregation - numeric_cols = merged_data.select_dtypes( - include=["number"] - ).columns.tolist() - - # Remove time and target columns if present - cols_to_remove = [self.time_colname, lab_col, self.target_colname] - for col in cols_to_remove: - if col in numeric_cols: - numeric_cols.remove(col) - - groups = merged_data.groupby(lab_col) - - if numeric_cols: - aggregated_numeric = groups[numeric_cols].agg( - self.aggregation_functions - ) - # Flatten MultiIndex columns: (col, func) -> "col_func" - aggregated_numeric.columns = [ - f"{col}_{func}" - for col, func in aggregated_numeric.columns.to_flat_index() - ] - else: - # Create empty frame indexed by the 2-hour windows - aggregated_numeric = groups.size().to_frame(name="__rows__") - aggregated_numeric = aggregated_numeric.drop(columns=["__rows__"]) - - # Add target column as the last non-null value per window - if self.target_colname in merged_data.columns: - target_per_window = groups[self.target_colname].apply( - lambda s: s.dropna().iloc[-1] - if not s.dropna().empty - else None - ) - aggregated_numeric[self.target_colname] = target_per_window - - # Reset index and rename lab timestamp to main time column - result = aggregated_numeric.reset_index().rename( - columns={lab_col: self.time_colname} - ) - - # Ensure timestamp is datetime - result[self.time_colname] = pd.to_datetime(result[self.time_colname]) - - console.log( - f"[bold green]Aggregated to windowed shape: {result.shape}" - ) - - return result - - def _create_lagged_target( - self, - data: pd.DataFrame, - ) -> pd.DataFrame: - """ - Creates lagged target column names with target values for inference. - In inference, we assume the data is already properly lagged, - so we just create the expected column names with the target values. - """ - if data.empty: - console.log("[red]Warning: Input data is empty.") - return data - - result = data.copy() - - if self.target_colname not in result.columns: - raise ValueError( - f"Target column '{self.target_colname}' not found in data." - ) - - # Create lagged column names with target values (no actual shifting) - lag_column_names = [] - - # Get target column as a Series to ensure we have exactly one column - target_series = result[self.target_colname] - if isinstance(target_series, pd.DataFrame): - # If we accidentally got a DataFrame, take the first column - target_values = target_series.iloc[:, 0].values - else: - target_values = target_series.values - - for lag in self.target_lags: - lag_column_name = f"{self.target_colname}_lag_{lag}" - # Copy target values instead of shifting for inference - result[lag_column_name] = target_values - console.log(f"Created lagged column: [cyan]{lag_column_name}") - lag_column_names.append(lag_column_name) - - return result - - def fit(self, X: pd.DataFrame, y=None): - """ - No-op for inference transformer (no learning needed). - - Args: - X: Input DataFrame - y: Not used - - Returns: - self - """ - console.log("[bold blue]Inference transformer fit (no-op)") - return self - - def transform(self, X: pd.DataFrame) -> pd.DataFrame: - """ - Apply the inference transformation pipeline. - - Args: - X: Input DataFrame with merged process and quality data - - Returns: - Transformed DataFrame ready for model inference - """ - console.log("[bold blue]Inference transformer transform phase") - - # Create a copy for processing - data = X.copy() - - # Step 0: Infer lab timestamp if needed - console.log("[bold blue]Inferring lab timestamp") - data = self._infer_lab_timestamp(data) - - # Step 1: Aggregate data (higher frequency - no target grouping) - console.log("[bold blue]Aggregating data") - data = self._aggregate_data(data) - - # Step 2: Apply learned feature selection - console.log("[bold blue]Applying learned feature selection") - # Set timestamp as index for filtering and ensure it's datetime - data[self.time_colname] = pd.to_datetime(data[self.time_colname]) - data = data.set_index(self.time_colname) - - # Filter to selected features (handle missing columns gracefully) - available_features = [ - col for col in self.selected_features if col in data.columns - ] - missing_features = set(self.selected_features) - set( - available_features - ) - - if missing_features: - console.log( - "[yellow]Warning: Missing features in inference data:" - + f" {missing_features}" - ) - - # Ensure target column is included but avoid duplicates - if self.target_colname not in available_features: - available_features.append(self.target_colname) - - data = data[available_features] - - # Step 3: Create lagged target features - if self.create_lagged_target: - console.log("[bold blue]Creating lagged target features") - data = self._create_lagged_target(data) - - console.log(f"[bold green]Final inference data shape: {data.shape}") - - return data - - -def create_transformers_from_training_transformer( - training_transformer: CourierTrainingTransformer, -) -> tuple[CourierTrainingTransformer, CourierInferenceTransformer]: - """ - Create both training and inference transformers with shared parameters. - - Args: - training_transformer: Fitted training transformer - - Returns: - Tuple of (training_transformer, inference_transformer) - """ - if training_transformer.selected_features_ is None: - raise ValueError("Training transformer must be fitted first.") - - inference_transformer = CourierInferenceTransformer( - selected_features=training_transformer.selected_features_, - aggregation_functions=training_transformer.aggregation_functions, - target_lags=training_transformer.target_lags, - time_colname=training_transformer.time_colname, - target_colname=training_transformer.target_colname, - create_lagged_target=training_transformer.create_lagged_target, - drop_nans=training_transformer.drop_nans, - ) - - return training_transformer, inference_transformer diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py deleted file mode 100644 index 6fba3eb..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/dvc/params.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Functions needed to load parameters from params.yaml tracked with DVC -""" - -import sys -import os -from typing import Optional - -import yaml -from rich.console import Console - - -console = Console() - - -def get_params(stage_fn: Optional[str] = None): - """ - Reads parameters for a given DVC stage from params.yaml. - - The stage name is inferred from the name of the python file that calls this - function. - Args: - stage_fn (str): Name of the stage. If None, the name of the file - that calls this function is used. Defaults to None. - Returns: - dict with parameters for the stage - Raises: - KeyError: if the stage name is not found in params.yaml - """ - - if stage_fn is None: - stage_fn = os.path.basename(sys.argv[0]).replace(".py", "") - - try: - params = yaml.safe_load(open("params.yaml"))[stage_fn] - except KeyError as exc: - console.print(f'ERROR: Key "{stage_fn}" not in parameters.yaml.') - raise KeyError( - f"Is the stage file name ({sys.argv[0]}) " - + "the same as the stage name in params.yaml?" - ) from exc - try: - all_params = yaml.safe_load(open("params.yaml"))["all"] - params = {**params, **all_params} - except KeyError: - console.print( - '[orange]WARNING: Key "all" not in parameters.yaml.' - + "Only returning stage parameters." - ) - - return params diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/features/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py deleted file mode 100644 index 9017a1b..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/mlflow/pyfunc_wrappers.py +++ /dev/null @@ -1,325 +0,0 @@ -""" -Module with functions for wrapping time series models for MLflow. -""" - -import os -import tempfile -import pickle -from typing import Optional, Union, Dict, Any, List - -import mlflow.pyfunc -import pandas as pd -import numpy as np -from mlflow.models import ModelSignature -from ..models.stacking_time_series import StackingTimeSeriesModel -from ..data.transformers import ( - create_transformers_from_training_transformer, - CourierTrainingTransformer, -) - - -class StackingWrapper(mlflow.pyfunc.PythonModel): # type: ignore - """ - MLflow wrapper for StackingTimeSeriesModel. - - Allows the model to be saved and served via MLflow's pyfunc interface. - """ - - def __init__(self, model: Optional[StackingTimeSeriesModel] = None): - self.model = model - - @property - def _console(self): - from rich.console import Console - - return Console() - - def load_context(self, context: Any) -> None: - """Load model from artifact path in MLflow context.""" - try: - model_path = context.artifacts["model"] - self.model = StackingTimeSeriesModel.load(model_path) - self._console.log("[green]Model loaded from context[/green]") - except Exception as e: - self._console.print(f"[red]Error loading model: {e}[/red]") - raise - - def predict( - self, - context: Any, - model_input: Union[pd.DataFrame, np.ndarray, Dict[str, Any]], - ) -> Union[pd.Series, pd.DataFrame, np.ndarray]: - """Run inference using the wrapped model.""" - if self.model is None: - raise ValueError("Model not loaded. Call load_context first.") - - # Extract data from input - if isinstance(model_input, dict): - X = model_input.get("data") - if X is None: - raise ValueError("Dict input must contain 'data' key.") - else: - X = model_input - - # Ensure DataFrame input (models expect pandas DataFrames) - if not isinstance(X, pd.DataFrame): - raise ValueError("Input must be a pandas DataFrame.") - - try: - predictions = self.model.predict(X) - return predictions.to_frame(name=self.model.target_col) - except Exception as e: - self._console.print(f"[red]Prediction failed: {e}[/red]") - raise - - def get_model_summary(self) -> str: - """Return human-readable model summary.""" - return self.model.summary() if self.model else "No model loaded" - - def store_model( - self, - path: Optional[str] = None, - artifact_path: str = "stacking_model", - signature: Optional[ModelSignature] = None, - pip_requirements: Optional[Union[str, list]] = None, - code_path: Optional[List[str]] = None, - to_disk: bool = False, - ) -> None: - """ - Store the model using MLflow pyfunc interface. - - Logs to the current MLflow run by default. Optionally saves locally. - - Args: - path: Local path to save model (required if to_disk=True) - artifact_path: MLflow artifact path - signature: Optional MLflow model signature - pip_requirements: pip requirements (list or path) - code_path: List of local Python source files/directories to bundle - to_disk: Save locally if True, otherwise logs to MLflow - """ - if self.model is None: - raise ValueError("No model to store.") - - with tempfile.TemporaryDirectory() as tmp: - model_artifact = os.path.join(tmp, "stacking_model.pkl") - self.model.save(model_artifact, compression="lzma") - - common_args = { - "python_model": self, - "artifacts": {"model": model_artifact}, - } - if signature: - common_args["signature"] = signature - if pip_requirements: - common_args["pip_requirements"] = pip_requirements - if code_path: - common_args["code_path"] = code_path - - if to_disk: - if not path: - raise ValueError("`path` required for to_disk=True.") - mlflow.pyfunc.save_model(path=path, **common_args) - self._console.log( - f"[blue]Model saved locally to {path}[/blue]" - ) - else: - mlflow.pyfunc.log_model( - artifact_path=artifact_path, **common_args - ) - self._console.log( - f"[green]Model logged to MLflow at '{artifact_path}'" - ) - - def __getstate__(self): - state = self.__dict__.copy() - state["model"] = None # avoid double saving - return state - - def __setstate__(self, state): - self.__dict__.update(state) - - -class TransformerWrapper(mlflow.pyfunc.PythonModel): # type: ignore - """ - MLflow wrapper for data transformers. - - Allows transformers to be saved and served via MLflow's pyfunc interface. - Supports both training and inference transformers. - """ - - def __init__( - self, - transformer: Optional[CourierTrainingTransformer] = None, - ): - """ - Initialize the transformer wrapper. - - Args: - transformer: The training transformer to wrap - """ - self.training_transformer = transformer - self.inference_transformer = None - - @property - def _console(self): - from rich.console import Console - - return Console() - - def load_context(self, context: Any) -> None: - """ - Load training transformer from artifact path and create - inference transformer. - """ - try: - transformer_path = context.artifacts["transformer"] - - with open(transformer_path, "rb") as f: - self.training_transformer = pickle.load(f) - - _, self.inference_transformer = ( - create_transformers_from_training_transformer( - self.training_transformer - ) - ) - - self._console.log( - "[green]Training transformer loaded and inference transformer " - + "created from context[/green]" - ) - except Exception as e: - self._console.print(f"[red]Error loading transformer: {e}[/red]") - raise - - def predict( - self, - context: Any, - model_input: Union[pd.DataFrame, np.ndarray, Dict[str, Any]], - transformer_type: str = "inference", - ) -> Union[pd.Series, pd.DataFrame, np.ndarray]: - """ - Transform data using the selected transformer. - - Args: - context: MLflow context - model_input: Input data to transform (pandas DataFrame expected) - transformer_type: Either "training" or "inference" - """ - if transformer_type == "training": - transformer = self.training_transformer - elif transformer_type == "inference": - transformer = self.inference_transformer - else: - raise ValueError( - "transformer_type must be 'training' or 'inference'" - ) - - if transformer is None: - raise ValueError( - f"{transformer_type.title()} transformer not loaded. " - + "Call load_context first." - ) - - # Extract data from input - if isinstance(model_input, dict): - X = model_input.get("data") - if X is None: - raise ValueError("Dict input must contain 'data' key.") - else: - X = model_input - - # Ensure DataFrame input (transformers expect pandas DataFrames) - if not isinstance(X, pd.DataFrame): - raise ValueError("Input must be a pandas DataFrame.") - - try: - # Apply transformer - transformed_data = transformer.transform(X) - return transformed_data - - except Exception as e: - self._console.print(f"[red]Transformation failed: {e}[/red]") - raise - - def get_transformer_summary(self) -> str: - """Return human-readable transformer summary.""" - if self.training_transformer is None: - return "No training transformer loaded" - - training_class = self.training_transformer.__class__.__name__ - inference_status = ( - "available" if self.inference_transformer else "not created" - ) - return ( - f"{training_class} (training loaded, inference {inference_status})" - ) - - def store_transformer( - self, - path: Optional[str] = None, - artifact_path: str = "transformer", - signature: Optional[ModelSignature] = None, - pip_requirements: Optional[Union[str, list]] = None, - code_path: Optional[List[str]] = None, - to_disk: bool = False, - ) -> None: - """ - Store the training transformer using MLflow pyfunc interface. - - Logs to the current MLflow run by default. Optionally saves locally. - - Args: - path: Local path to save transformer (required if to_disk=True) - artifact_path: MLflow artifact path - signature: Optional MLflow model signature - pip_requirements: pip requirements (list or path) - code_path: List of local Python source files/directories to bundle - to_disk: Save locally if True, otherwise logs to MLflow - """ - if self.training_transformer is None: - raise ValueError("No training transformer to store.") - - with tempfile.TemporaryDirectory() as tmp: - transformer_artifact = os.path.join( - tmp, "training_transformer.pkl" - ) - with open(transformer_artifact, "wb") as f: - pickle.dump(self.training_transformer, f) - - common_args = { - "python_model": self, - "artifacts": {"transformer": transformer_artifact}, - } - if signature: - common_args["signature"] = signature - if pip_requirements: - common_args["pip_requirements"] = pip_requirements - if code_path: - common_args["code_path"] = code_path - - if to_disk: - if not path: - raise ValueError("`path` required for to_disk=True.") - mlflow.pyfunc.save_model(path=path, **common_args) - self._console.log( - "[blue]Training transformer saved locally to " - + f"{path}[/blue]" - ) - else: - mlflow.pyfunc.log_model( - artifact_path=artifact_path, **common_args - ) - self._console.log( - "[green]Training transformer logged to MLflow at " - + f"'{artifact_path}'[/green]" - ) - - def __getstate__(self): - state = self.__dict__.copy() - state["training_transformer"] = None # avoid double saving - state["inference_transformer"] = None # avoid double saving - return state - - def __setstate__(self, state): - self.__dict__.update(state) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py deleted file mode 100644 index 1701c19..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Models package for time series forecasting. - -This package provides standardized interfaces and implementations for -various time series forecasting models. -""" - -from .base import ( - TimeSeriesModel, - UnivariateTimeSeriesModel, - MultivariateTimeSeriesModel, -) -from .factory import create_model, load_model, get_available_models -from .evaluation import timeseries_metrics - -__all__ = [ - "TimeSeriesModel", - "UnivariateTimeSeriesModel", - "MultivariateTimeSeriesModel", - "create_model", - "load_model", - "get_available_models", - "timeseries_metrics", -] diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py deleted file mode 100644 index abf5d6d..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/arima.py +++ /dev/null @@ -1,389 +0,0 @@ -""" -ARIMA univariate time series forecasting model implementation. -""" - -from typing import Optional, Tuple - -import numpy as np -import pandas as pd -from statsmodels.tsa.arima.model import ARIMA, ARIMAResults -from rich.console import Console - -from .base import UnivariateTimeSeriesModel, ensure_fitted - -console = Console() - - -class ARIMAModel(UnivariateTimeSeriesModel): - """ARIMA model for univariate time series forecasting. - - This class implements an ARIMA model for forecasting univariate time - series data. It provides methods for fitting the model, making predictions, - forecasting future values, and updating the model with new data. - - Attributes: - order (Tuple[int, int, int]): The (p, d, q) order of the ARIMA model. - model_ (Optional[ARIMA]): The ARIMA model instance. - result_ (Optional[ARIMAResults]): The fitted ARIMA model results. - training_series_ (Optional[pd.Series]): The training data used to fit - the model. - """ - - def __init__( - self, - order: Tuple[int, int, int] = (1, 0, 0), - name: Optional[str] = None, - time_col: str = "ds", - target_col: str = "y", - random_seed: int = 42, - forecast_horizon: int = 2, - ) -> None: - """Initializes the ARIMAModel with specified parameters. - - Args: - order (Tuple[int, int, int]): The (p, d, q) order of the ARIMA - model. - name (Optional[str]): The name of the model. - time_col (str): The name of the time column in the input data. - target_col (str): The name of the target column in the input data. - random_seed (int): The random seed for reproducibility. - forecast_horizon (int): The number of steps to forecast ahead. - """ - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - ) - self.order: Tuple[int, int, int] = order - self.model_: Optional[ARIMA] = None - self.result_: Optional[ARIMAResults] = None - self.training_series_: pd.Series = pd.Series(dtype=float) - self.observed_series_: pd.Series = pd.Series(dtype=float) - self.backtest_predictions_: Optional[pd.Series] = None - self.forecast_horizon: int = forecast_horizon - - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """ - Fits the ARIMA model to the provided training data. - - Args: - y: The target time series data. - X: Optional exogenous variables. - X_val: Validation feature matrix (not used for ARIMA). - y_val: Validation target series (not used for ARIMA). - """ - y_array: np.ndarray = self._validate_y(y) - self.training_series_ = y.copy() - self.observed_series_ = y.copy() - self.model_ = ARIMA(y_array, order=self.order) - self.result_ = self.model_.fit() - - @ensure_fitted - def predict(self, X: Optional[pd.DataFrame] = None) -> pd.Series: - """ - Generates in-sample predictions from the fitted ARIMA model. - - After this method is called, if X is provided, the model will be - updated with the new data, but the coefficients will not be refit. - This is useful for generating predictions on new data without - retraining the model. - - Args: - X (Optional[pd.DataFrame]): Optional dataframe with future - measurements of y for in-sample predictions. - If None, the model will predict on the observed data - (observed_series_). - - Returns: - pd.Series: The in-sample predictions. - - Raises: - ValueError: If the model has not been fitted yet. - """ - if self.result_ is None: - raise ValueError("Model is not fitted.") - - if X is None: - fitted_values = self.result_.fittedvalues - if fitted_values is None: - raise ValueError("Fitted values are None") - return pd.Series( - fitted_values, - index=self.training_series_.index[: len(fitted_values)], - name=self.target_col, - ) - - # Validate the input data - target_series = ( - X[self.target_col] if self.target_col in X else X.iloc[:, 0] - ) - if not isinstance(target_series, pd.Series): - target_series = pd.Series(target_series, index=X.index) - - X_validated = self._validate_y(target_series) - - # Update the model with the validated data without refitting - self.update(pd.Series(X_validated, index=X.index), refit=False) - - if self.result_ is None: - raise ValueError("Model result is None after update") - - fitted_values = self.result_.fittedvalues - if fitted_values is None: - raise ValueError("Fitted values are None") - - return_series = pd.Series( - fitted_values[-len(X) :], index=X.index, name=self.target_col - ) - - return return_series - - @ensure_fitted - def forecast(self, forecast_horizon: int) -> np.ndarray: - """Generates out-of-sample forecasts from the fitted ARIMA model. - TODO: Change return to include index of the forecasted values. - - Args: - forecast_horizon (int): The number of steps to forecast ahead. - - Returns: - np.ndarray: The out-of-sample forecasts. - - Raises: - ValueError: If the model has not been fitted yet. - """ - if self.result_ is None: - raise ValueError("Model is not fitted.") - return self.result_.forecast(steps=forecast_horizon) - - @ensure_fitted - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Perform comprehensive backtesting with periodic retraining. - - This method implements walk-forward validation with periodic - retraining, providing robust evaluation of model performance in - production-like scenarios. Uses 1-step ahead forecasting by default. - - Args: - y: Target time series for backtesting - X: Unused (included for base class compatibility) - retrain_every: Number of steps between model retraining - reuse_previous_execution: Whether to reuse previous backtest results - - Returns: - Series of backtested predictions indexed by timestamp - - Raises: - ValueError: If parameters are invalid or data is insufficient - RuntimeError: If backtesting fails - """ - if self.result_ is None: - raise ValueError("Model is not fitted") - if self.training_series_ is None: - raise ValueError("No training series found") - - # Handle reuse of previous execution - if reuse_previous_execution and self.backtest_predictions_ is not None: - expected_index = y.index - if ( - len(self.backtest_predictions_) == len(expected_index) - and (self.backtest_predictions_.index == expected_index).all() - ): - console.log( - "[yellow]Reusing previous backtest results[/yellow]" - ) - return self.backtest_predictions_ - else: - console.log( - "[yellow]Previous results incompatible, running new" - + " backtest[/yellow]" - ) - - try: - console.log( - f"[blue]Starting ARIMA backtest with {self.forecast_horizon}" - + "-step forecasting...[/blue]" - ) - - # Validate and prepare data - y_sorted = y.sort_index() - - # Check for overlapping data - training_series = self.training_series_ - if any(t in training_series.index for t in y_sorted.index): - console.print( - "[yellow]Warning: Backtest data overlaps with training" - + " data[/yellow]" - ) - - # Initialize backtesting - predictions = [] - - # Start with training data - current_series = training_series.copy() - - total_steps = len(y_sorted) - console.log( - f"[blue]Running {total_steps} backtest steps with retraining" - + f" every {retrain_every} steps...[/blue]" - ) - - # Create initial model state - current_model = ARIMA(current_series.values, order=self.order) - current_result = current_model.fit() - - # Perform walk-forward validation - for i, (timestamp, actual_value) in enumerate(y_sorted.items()): - if i % 50 == 0 and i > 0: # Progress logging - console.log( - f"[blue]Backtest progress: {i}/{len(y_sorted)}[/blue]" - ) - - try: - # Check if we need to retrain - if i % retrain_every == 0 and i > 0: - console.log( - f"[blue]Retraining model at step {i}[/blue]" - ) - current_model = ARIMA( - current_series.values, order=self.order - ) - current_result = current_model.fit() - - # Generate forecast_horizon-step ahead forecast - forecast = current_result.forecast( - steps=self.forecast_horizon - )[0] - predictions.append((timestamp, forecast)) - - # Update the series with actual observed value - current_series = pd.concat( - [ - current_series, - pd.Series([actual_value], index=[timestamp]), - ] - ) - - # For ARIMA, we can extend the model without full refit - if i % retrain_every != 0: - try: - current_result = current_result.extend( - [actual_value], refit=False - ) - except Exception: - # If extend fails, do a quick refit - current_model = ARIMA( - current_series.values, order=self.order - ) - current_result = current_model.fit() - - except Exception as step_error: - console.print( - f"[yellow]Error at step {i}: {step_error}, using" - + " NaN[/yellow]" - ) - predictions.append((timestamp, np.nan)) - - # Still update the series for continuity - current_series = pd.concat( - [ - current_series, - pd.Series([actual_value], index=[timestamp]), - ] - ) - - # Create results series - if predictions: - pred_index, pred_values = zip(*predictions) - self.backtest_predictions_ = pd.Series( - pred_values, - index=pd.Index(pred_index), - name=f"{self.target_col}_backtest", - ) - else: - self.backtest_predictions_ = pd.Series( - dtype=float, name=f"{self.target_col}_backtest" - ) - - console.log( - "[green]ARIMA backtest completed: " - + f"{len(self.backtest_predictions_)} predictions[/green]" - ) - return self.backtest_predictions_ - - except Exception as e: - console.print(f"[red]ARIMA backtest failed: {e}[/red]") - raise RuntimeError(f"Failed to perform backtest: {e}") from e - - @ensure_fitted - def summary(self) -> str: - """Generates a summary of the fitted ARIMA model. - - Returns: - str: The summary of the fitted model. - - Raises: - ValueError: If the model has not been fitted yet. - """ - if self.result_ is None: - raise ValueError("Model is not fitted.") - return str(self.result_.summary()) - - @ensure_fitted - def update(self, new_data: pd.Series, refit: bool = True) -> None: - """Updates the ARIMA model with new observed data. - - This method allows for two modes of updating the model: - 1. **Refitting**: The model is retrained on the combined dataset - (original training data + new data). - 2. **Incremental Update**: The model is updated using the new data - without retraining, preserving the original model parameters. - - Args: - new_data (pd.Series): New observed values to update the model with. - refit (bool, optional): If True, the model is retrained on the - combined dataset. Defaults to True. - - Raises: - TypeError: If `new_data` is not a pandas Series. - ValueError: If `new_data` is empty or if the model has not been - fitted yet. - """ - if not isinstance(new_data, pd.Series): - raise TypeError("new_data must be a pandas Series.") - if new_data.empty: - raise ValueError("new_data is empty.") - - if refit: - self.training_series_ = pd.concat( - [self.training_series_, new_data] - ) - self.observed_series_ = self.training_series_.copy() - y_array = self._validate_y(self.training_series_) - self.model_ = ARIMA(y_array, order=self.order) - self.result_ = self.model_.fit() - else: - self.observed_series_ = pd.concat( - [self.observed_series_, new_data] - ) - y_array = self._validate_y(self.observed_series_) - - if self.result_ is None: - raise ValueError("Model is not fitted.") - self.result_ = self.result_.apply(y_array, refit=False) - if self.result_ is not None: - self.model_ = self.result_.model diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/base.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py deleted file mode 100644 index 3721871..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/catboost_time_series.py +++ /dev/null @@ -1,510 +0,0 @@ -""" -CatBoost implementation for multivariate time series forecasting. - -TODO: - - Implement support for categorical features. -""" - -from typing import Optional, List, Union, Tuple, Dict, Any, cast - -import pandas as pd -from catboost import CatBoostRegressor, CatBoostClassifier, Pool -from rich.console import Console -import numpy as np - -from .base import MultivariateTimeSeriesModel, ensure_fitted - -console = Console() - - -class CatBoostTimeSeriesModel(MultivariateTimeSeriesModel): - """ - CatBoost implementation for multivariate time series forecasting. - - This class wraps the CatBoost models with additional functionality for - time series forecasting, following the MultivariateTimeSeriesModel - interface. Supports both regression and classification tasks with - comprehensive error handling and type safety. - """ - - def __init__( - self, - name: Optional[str] = None, - learning_task: str = "regression", - differentiate_target: bool = False, - n_lags: int = 0, - iterations: int = 1000, - learning_rate: float = 0.1, - depth: int = 6, - loss_function: Optional[str] = None, - bins: Optional[List[float]] = None, - random_seed: int = 42, - time_col: str = "ds", - target_col: str = "y", - verbose: bool = False, - ) -> None: - """ - Initialize the CatBoost time series model. - - Args: - name: Optional identifier for the model - learning_task: Type of learning task, either 'regression', - 'multiclass' or 'binary'. - differentiate_target: Whether to differentiate the target - series before fitting the model. - n_lags: Number of lagged target values included as features. - These lags are expected to already be present in the same - dataset as the exogenous features. - iterations: Number of boosting iterations - learning_rate: Learning rate for the model - depth: Depth of the tree - loss_function: Loss function to optimize - bins: Optional list of bin edges for multiclass classification - random_seed: Random seed for reproducibility - time_col: Name of the time column - target_col: Name of the target column - verbose: Whether to enable verbose output - """ - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - n_lags=n_lags, - learning_task=learning_task, - differentiate_target=differentiate_target, - bins=bins, - ) - - self.iterations = iterations - self.learning_rate = learning_rate - self.depth = depth - self.verbose = verbose - - # Set default loss function based on learning task - self.loss_function = self._get_default_loss_function(loss_function) - - # Initialize model state - self.model_: Optional[Union[CatBoostRegressor, CatBoostClassifier]] = ( - None - ) - self.training_series_: Optional[pd.Series] = None - self.X_train_: Optional[pd.DataFrame] = None - self.backtest_predictions_: Optional[pd.Series] = None - - if self.verbose: - console.log( - "[green]Initialized CatBoostTimeSeriesModel:" - + f" {self.summary()}[/green]" - ) - - def _create_model(self) -> Union[CatBoostRegressor, CatBoostClassifier]: - """Creates a new instance of CatBoost model with current parameters. - - Returns: - A new CatBoost model instance (Regressor or Classifier). - """ - base_params = { - "iterations": self.iterations, - "learning_rate": self.learning_rate, - "depth": self.depth, - "loss_function": self.loss_function, - "random_seed": self.random_seed, - "verbose": self.verbose, - } - - if self.learning_task in ["binary", "multiclass"]: - return CatBoostClassifier( - auto_class_weights="Balanced", **base_params - ) - else: - return CatBoostRegressor(**base_params) - - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """ - Core fitting logic for CatBoost model with optional validation data. - - Args: - y: The target time series data - X: The feature matrix (including exogenous features) - X_val: Validation feature matrix (optional) - y_val: Validation target series (optional) - """ - if X is None or not isinstance(X, pd.DataFrame): - raise ValueError("Feature matrix X must be a non-empty DataFrame.") - - y_processed, X_processed, y_val_processed, X_val_processed = ( - self._preprocess_data(y, X, X_val, y_val) - ) - - # Ensure X is not None after preprocessing - if X_processed is None: - raise ValueError( - "Feature matrix X cannot be None after preprocessing." - ) - - X_array, y_array = self._validate_X_y(X_processed, y_processed) - - self.training_series_ = y_processed.copy() - self.X_train_ = X_processed.copy() - self.model_ = self._create_model() - - eval_set = None - if X_val_processed is not None and y_val_processed is not None: - X_val_array, y_val_array = self._validate_X_y( - X_val_processed, y_val_processed - ) - eval_set = Pool(data=X_val_array, label=y_val_array) - - train_pool = Pool(data=X_array, label=y_array) - - if self.verbose: - console.log( - f"[blue]Training CatBoost model for {self.iterations}" - + " iterations...[/blue]" - ) - - self.model_.fit(train_pool, eval_set=eval_set) - - if self.verbose: - console.log( - "[green]CatBoost model training completed successfully[/green]" - ) - - @ensure_fitted - def predict(self, X: pd.DataFrame) -> pd.Series: - """ - Generate predictions using the fitted CatBoost model. - - Args: - X: The feature matrix for prediction - - Returns: - pd.Series: Predicted values - """ - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - - # Create a copy to avoid modifying the original DataFrame - X_pred = X.copy() - - if self.selected_features_ is not None: - X_pred = cast(pd.DataFrame, X_pred[self.selected_features_]) - - X_array = self._validate_X(X_pred) - predictions = self.model_.predict(X_array) - - # Convert predictions to numpy array if needed - if hasattr(predictions, "squeeze"): - predictions = predictions.squeeze() - elif isinstance(predictions, list): - predictions = np.array(predictions) - - return_series = pd.Series( - predictions, - index=X.index, - name=self.target_col, - ) - return return_series - - @ensure_fitted - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Performs backtesting (walk-forward validation) with periodic - retraining. - - This method simulates a production scenario by iterating through a test - set, making a one-step-ahead prediction, and then retraining the model - periodically with the newly available data. - - Args: - X: DataFrame with features for the backtesting period. - y: Series with the true target values for the backtesting period. - retrain_every: The frequency of retraining. The model will be - retrained every `retrain_every` steps. - reuse_previous_execution: Whether to reuse the previous execution - of a backtest. If True, any overlapping data between the - previous execution and the current execution will be used - without retraining the model. - Returns: - A series of backtested predictions, indexed by the backtest data's - index. - """ - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - if self.training_series_ is None: - raise ValueError("Training series is not set.") - if self.X_train_ is None: - raise ValueError("Training feature matrix is not set.") - if X is None or not isinstance(X, pd.DataFrame): - raise ValueError("Feature matrix X must be a non-empty DataFrame.") - - if reuse_previous_execution: - if self.backtest_predictions_ is None: - raise ValueError("No previous execution found.") - if (self.backtest_predictions_.shape[0] != y.shape[0]) or ( - not (self.backtest_predictions_.index == y.index).all() - ): - raise ValueError( - "Previous execution index does not match y index." - ) - return self.backtest_predictions_ - # Prepare - total_steps = len(X) - predictions = [] - current_model = self.model_ - y_history = self.training_series_.copy() - X_history = self.X_train_.copy() - - # Iterate in chunks instead of single steps - for start in range(0, total_steps, retrain_every): - end = min(start + retrain_every, total_steps) - - # Batch prediction for current chunk - X_chunk = X.iloc[start:end].copy() - if self.selected_features_: - X_chunk = X_chunk[self.selected_features_] - - X_array = self._validate_X(X_chunk) - preds = current_model.predict(X_array) - - # Handle different prediction formats - if hasattr(preds, "squeeze"): - preds = preds.squeeze() - if preds.ndim == 0: # single point - preds = [preds] - predictions.extend(preds) - - # Update training history - y_chunk = y.iloc[start:end] - y_history = pd.concat([y_history, y_chunk]) - X_history = pd.concat([X_history, X_chunk]) - - # Retrain the model for next chunk (if needed) - if end < total_steps: - if self.verbose: - console.print( - f"[cyan]Backtesting: Retraining at step {end}..." - ) - - current_model = self._create_model() - (y_fit, X_fit, _, _) = ( - self._preprocess_data(y_history, X_history) - ) - - # Ensure X_fit is not None after preprocessing - if X_fit is None: - raise ValueError( - "Feature matrix cannot be None after preprocessing." - ) - - X_fit_array, y_fit_array = self._validate_X_y(X_fit, y_fit) - train_pool = Pool(data=X_fit_array, label=y_fit_array) - current_model.fit(train_pool) - - # Store backtest predictions for potential reuse - self.backtest_predictions_ = pd.Series( - predictions, index=X.index, name=f"{self.target_col}_pred" - ) - return self.backtest_predictions_ - - def select_features( - self, - X: pd.DataFrame, - y: pd.Series, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - features_to_select: Optional[int] = None, - algorithm: str = "RecursiveByShapValues", - steps: int = 1, - verbose: bool = False, - ) -> List[str]: - """Identify and select the most important features. - - Uses CatBoost's built-in feature selection capabilities to determine - feature importance and select the most relevant features. - - Args: - X: The feature matrix - y: The target series - X_val: Optional validation feature matrix - y_val: Optional validation target series - features_to_select: Number of features to select. If None, - will select half of the features. - algorithm: Feature selection algorithm. One of: - 'RecursiveByShapValues', 'RecursiveByPredictionValuesChange' - steps: How many times a full model will be trained. - More steps give more accurate results. - verbose: Whether to print progress - - Returns: - List[str]: List of selected feature names - """ - (y_processed, X_processed, y_val_processed, X_val_processed) = ( - self._preprocess_data(y, X, X_val, y_val) - ) - - # Validate input data - if X_processed is None or not isinstance(X_processed, pd.DataFrame): - raise ValueError("Feature matrix X must be a non-empty DataFrame.") - X_array, y_array = self._validate_X_y(X_processed, y_processed) - - # Set default number of features to select if not specified - if features_to_select is None: - features_to_select = X_processed.shape[1] // 2 - - # Create and prepare model - temp_model = self._create_model() - train_pool = Pool(data=X_array, label=y_array) - - # Prepare validation data if provided - eval_set = None - if X_val_processed is not None and y_val_processed is not None: - X_val_array, y_val_array = self._validate_X_y( - X_val_processed, y_val_processed - ) - eval_set = Pool(data=X_val_array, label=y_val_array) - - # Perform feature selection - selected_features = temp_model.select_features( - train_pool, - eval_set=eval_set, - features_for_select=list(range(X_processed.shape[1])), - num_features_to_select=features_to_select, - algorithm=algorithm, - steps=steps, - logging_level="Verbose" if verbose else "Silent", - train_final_model=False, - ) - - # Map feature indices to feature names with proper type casting - selected_feature_names: List[str] = [ - str(X_processed.columns[idx]) - for idx in selected_features["selected_features"] - ] - self.selected_features_ = selected_feature_names - self.feature_names_in_ = selected_feature_names - self.n_features_in_ = len(selected_feature_names) - - return selected_feature_names - - @classmethod - def tune_hyperparameters( - cls, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - selected_features: Optional[List[str]] = None, - param_grid: Optional[Dict[str, Any]] = None, - n_trials: int = 10, - early_stopping_rounds: Optional[int] = 50, - random_seed: int = 42, - **kwargs, - ) -> Tuple[Dict[str, Any], "CatBoostTimeSeriesModel"]: - """ - Tune hyperparameters for the CatBoost model. - - Args: - y: The target time series data - X: The feature matrix (including exogenous features) - X_val: Validation feature matrix (optional) - y_val: Validation target series (optional) - selected_features: List of features to use for tuning - param_grid: Dictionary of hyperparameters to search - n_trials: Number of trials for hyperparameter tuning - early_stopping_rounds: Number of rounds for early stopping - random_seed: Random seed for reproducibility - **kwargs: Additional keyword arguments for model initialization - - Returns: - Tuple[Dict[str, Any], CatBoostTimeSeriesModel]: Best hyperparameters - and fitted model - """ - if n_trials <= 0: - raise ValueError("n_trials must be a positive integer.") - # Create a temporary model instance to use its preprocessing method - temp_model = cls( - learning_task=kwargs.get("learning_task", "regression"), - differentiate_target=kwargs.get("differentiate_target", False), - bins=kwargs.get("bins", None), - random_seed=random_seed, - ) - if selected_features is not None: - temp_model.selected_features_ = selected_features - - (y_processed, X_processed, _, _) = ( - temp_model._preprocess_data(y, X, X_val, y_val) - ) - - if kwargs.get("learning_task", "regression") == "classification": - search_model = CatBoostClassifier( - random_seed=random_seed, - logging_level="Silent", - early_stopping_rounds=early_stopping_rounds, - loss_function=kwargs.get("loss_function", "Logloss"), - class_weights="Balanced", - ) - else: - search_model = CatBoostRegressor( - random_seed=random_seed, - logging_level="Silent", - early_stopping_rounds=early_stopping_rounds, - loss_function=kwargs.get("loss_function", "RMSE"), - ) - - if param_grid is None: - param_grid = { - "iterations": [100, 500, 1000, 2000], - "learning_rate": [0.01, 0.05, 0.1, 0.2], - "depth": [4, 6, 8], - } - - if X_processed is None: - raise ValueError( - "Feature matrix cannot be None after preprocessing." - ) - - train_pool = Pool(data=X_processed, label=y_processed) - results = search_model.randomized_search( - param_grid, - X=train_pool, - n_iter=n_trials, - verbose=False, - refit=False, - ) - - best_params = results["params"] - - best_model = cls( - iterations=best_params["iterations"], - learning_rate=best_params["learning_rate"], - depth=best_params["depth"], - random_seed=random_seed, - time_col=kwargs.get("time_col", "ds"), - target_col=kwargs.get("target_col", "y"), - n_lags=kwargs.get("n_lags", 0), - name=kwargs.get("name", None), - loss_function=kwargs.get("loss_function", None), - learning_task=kwargs.get("learning_task", "regression"), - bins=kwargs.get("bins", None), - differentiate_target=kwargs.get("differentiate_target", False), - ) - if selected_features is not None: - best_model.selected_features_ = selected_features - - best_model.fit(y, X, X_val, y_val) - - return best_params, best_model diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py deleted file mode 100644 index 77331d7..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/evaluation.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -Module with functions of timeseries evaluation. -""" - -from typing import Optional, List, Sequence - -import numpy as np -import pandas as pd -from rich.console import Console -from sklearn.metrics import ( - mean_absolute_error, - mean_squared_error, - accuracy_score, - f1_score, - confusion_matrix, -) -console = Console() - - -def timeseries_metrics( - y_pred: Sequence[float], y_true: Sequence[float] -) -> dict[str, float]: - """ - Compute MAE, MSE, and trend capture for time series predictions. - - Parameters: - y_pred (ArrayLike): Predicted values. - y_true (ArrayLike): Ground truth values. - - Returns: - dict[str, float]: Dictionary with MAE, MSE, and trend_capture. - """ - if len(y_true) != len(y_pred): - raise ValueError("y_true and y_pred must have the same length") - if len(y_true) == 0: - raise ValueError("y_true and y_pred must not be empty") - y_true_np = np.asarray(y_true) - y_pred_np = np.asarray(y_pred) - - mae = mean_absolute_error(y_true_np, y_pred_np) - mse = mean_squared_error(y_true_np, y_pred_np) - - # Compute directional trend: 1 if up, 0 if down or flat - if len(y_true) == 1: - return {"MAE": mae, "MSE": mse, "trend_capture": 1.0} - - true_trend = np.diff(y_true_np) > 0 - pred_trend = np.diff(y_pred_np) > 0 - - trend_capture = np.mean(true_trend == pred_trend) - - return {"MAE": mae, "MSE": mse, "trend_capture": trend_capture} - - -def timeseries_classification_metrics( - y_pred: Sequence[float], - y_true: Sequence[float], - bins: Optional[List] = None, -) -> dict[str, float]: - """ - Compute accuracy for classification predictions. - - Parameters: - y_pred (ArrayLike): Predicted values. - y_true (ArrayLike): Ground truth values. - bins (List[int], optional): Bin edges for categorizing predictions. - - Returns: - dict[str, float]: Dictionary with accuracy. - """ - if bins is not None: - console.log( - f"Using bins for classification: {bins}" - ) - y_true_binned = pd.cut(y_true, bins=bins, labels=False) - else: - console.log("No bins provided, using default classification (y > 0).") - y_true_binned = (y_true > 0).astype(int) - console.log( - f"y_true_binned: {y_true_binned.value_counts()}" - ) - console.log( - f"y_pred: {pd.Series(y_pred).value_counts()}" - ) - - y_pred = np.array(y_pred).astype(int) - acc = accuracy_score(y_true_binned, y_pred) - f1 = f1_score(y_true_binned, y_pred, average="weighted") - # Calculate the confusion matrix - cm = confusion_matrix(y_true_binned, y_pred) - - return {"accuracy": acc, "f1_score": f1, "confusion_matrix": cm} diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py deleted file mode 100644 index a8d646e..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/factory.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -Model factory for creating, loading, and discovering time series models. - -This module provides centralized utility functions to handle different time -series model implementations based on a string identifier. It uses a -central registry (`SUPPORTED_MODELS`) that maps model type strings (e.g., -'arima') to their corresponding model classes (e.g., ARIMAModel). This -approach allows for easy extension and decouples model instantiation logic -from the code that uses the models. - -Key Functions: - create_model: Creates a new instance of a specified model type by looking - up the type string in the `SUPPORTED_MODELS` registry and - passing keyword arguments to the retrieved model class's - constructor. - load_model: Loads a previously saved model instance from disk. It uses - the provided model type string to find the correct class in - the registry and then calls that class's `.load()` classmethod. - get_available_models: Returns a dictionary listing the registered model - types (keys in `SUPPORTED_MODELS`) and their - descriptions, automatically derived from the model - class docstrings. - -Extensibility: - Adding support for a new model involves the following steps: - 1. Ensure the new model class (e.g., `MyNewModel`) inherits from the - appropriate base class (e.g., `TimeSeriesModel`) and implements all - required abstract methods. - 2. Ensure the new model class has a `.load()` classmethod compatible - with the `save()` method in the base `Model` class (if loading is - to be supported via this factory). - 3. Import the new model class into this factory module. - 4. Add an entry to the `SUPPORTED_MODELS` dictionary, mapping a unique, - lowercase string identifier to the model class itself: - `SUPPORTED_MODELS = {..., "mynewmodel": MyNewModel}` - Once added to the registry, the model can be created and loaded via the - factory functions, and it will automatically appear in the output of - `get_available_models()`. -""" - -from typing import Dict, Type -import logging - -from .base import TimeSeriesModel -from .arima import ARIMAModel -from .neural_prophet_model import NeuralProphetModel -from .catboost_time_series import CatBoostTimeSeriesModel -from .linear_regression_time_series import ElasticNetTimeSeriesModel -from .stacking_time_series import StackingTimeSeriesModel -# from .prophet import ProphetModel # Example for future - -# *** Central registry of supported models -SUPPORTED_MODELS: Dict[str, Type[TimeSeriesModel]] = { - "arima": ARIMAModel, - "neuralprophet": NeuralProphetModel, - "catboost": CatBoostTimeSeriesModel, - "elasticnet": ElasticNetTimeSeriesModel, - "stacking": StackingTimeSeriesModel, - # "prophet": ProphetModel, # Add new models here -} - - -def create_model(model_type: str, **kwargs) -> TimeSeriesModel: - """ - Create a new model instance of the specified type using a registry. - - Args: - model_type: Type of model to create (case-insensitive). - **kwargs: Model-specific parameters passed to its constructor. - - Returns: - New model instance inheriting from TimeSeriesModel. - - Raises: - ValueError: If the model type is not supported or kwargs are invalid. - """ - model_type = model_type.lower() - model_class = SUPPORTED_MODELS.get(model_type) - - if model_class: - try: - instance = model_class(**kwargs) - return instance - except TypeError as e: - logging.error(f"Kwargs issue for {model_type}: {kwargs}") - raise ValueError( - f"Invalid parameters for model type '{model_type}'. Error: {e}" - ) from e - else: - supported_list = ", ".join(f"'{k}'" for k in SUPPORTED_MODELS.keys()) - raise ValueError( - f"Unsupported model type: '{model_type}'. " - f"Currently supported models are: {supported_list}." - ) - - -def load_model(path: str, model_type: str) -> TimeSeriesModel: - """ - Load a model from disk using a registry. - - Args: - path: Path to the saved model. - model_type: Expected type of model to load (case-insensitive). - - Returns: - Loaded model instance. - - Raises: - ValueError: If the model type is not supported. - # Other errors might come from the underlying .load() method - """ - model_type = model_type.lower() - model_class = SUPPORTED_MODELS.get(model_type) - - if model_class: - return model_class.load(path) - else: - supported_list = ", ".join(f"'{k}'" for k in SUPPORTED_MODELS.keys()) - raise ValueError( - f"Unsupported model type: '{model_type}'. " - f"Currently supported models are: {supported_list}." - ) - - -def get_available_models() -> Dict[str, str]: - """ - Dynamically get a dictionary of available model types and their - descriptions from the SUPPORTED_MODELS registry and class docstrings. - - Returns: - Dictionary mapping model type names to descriptions. - """ - available = { - type_name: ( - model_class.__doc__.strip().splitlines()[0] - if model_class.__doc__ else "No description available." - ) - for type_name, model_class in SUPPORTED_MODELS.items() - } - return available diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py deleted file mode 100644 index 9468954..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/linear_regression_time_series.py +++ /dev/null @@ -1,303 +0,0 @@ -""" -ElasticNet implementation for multivariate time series forecasting. -""" - -from typing import Optional, Sequence - -import pandas as pd -from sklearn.linear_model import ElasticNet -from sklearn.impute import SimpleImputer -from rich.console import Console - -from .base import MultivariateTimeSeriesModel, ensure_fitted - -console = Console() - - -class ElasticNetTimeSeriesModel(MultivariateTimeSeriesModel): - """ - ElasticNet implementation for multivariate time series forecasting. - - This class wraps the scikit-learn ElasticNet model with additional - functionality for time series forecasting, following the - MultivariateTimeSeriesModel interface. It's suitable for regression - tasks where features might be correlated. - """ - - def __init__( - self, - name: Optional[str] = None, - n_lags: int = 1, - alpha: float = 1.0, - l1_ratio: float = 0.5, - fit_intercept: bool = True, - max_iter: int = 1000, - tol: float = 1e-4, - random_seed: int = 42, - time_col: str = "ds", - target_col: str = "y", - differentiate_target: bool = False, - bins: Optional[list] = None, - learning_task: Optional[str] = None, - ) -> None: - """ - Initialize the ElasticNet time series model. - - Args: - name: Optional identifier for the model. - n_lags: Number of lagged target values to include as inputs. - alpha: Constant that multiplies the penalty terms. - l1_ratio: The ElasticNet mixing parameter (0 <= l1_ratio <= 1). - For l1_ratio = 0, it's L2 penalty (Ridge). - For l1_ratio = 1, it's L1 penalty (Lasso). - fit_intercept: Whether to calculate the intercept for this model. - max_iter: Maximum number of iterations. - tol: Tolerance for stopping criteria. - random_seed: Random seed for reproducibility. - time_col: Name of the time column. - target_col: Name of the target column. - differentiate_target: Whether to apply differencing to make series - stationary. - bins: Bin edges for multiclass classification target - transformation. - learning_task: Type of learning task ('regression', 'binary', - 'multiclass'). - """ - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - n_lags=n_lags, - differentiate_target=differentiate_target, - bins=bins, - learning_task=learning_task, - ) - - self.alpha = alpha - self.l1_ratio = l1_ratio - self.fit_intercept = fit_intercept - self.max_iter = max_iter - self.tol = tol - - self.model_: Optional[ElasticNet] = None - self.training_series_: Optional[pd.Series] = None - self.imputer_: Optional[SimpleImputer] = None - self.X_train_: Optional[pd.DataFrame] = None - - def _create_model(self) -> ElasticNet: - """Creates a new instance of ElasticNet with current parameters. - - Returns: - A new ElasticNet model instance. - """ - return ElasticNet( - alpha=self.alpha, - l1_ratio=self.l1_ratio, - fit_intercept=self.fit_intercept, - max_iter=self.max_iter, - tol=self.tol, - random_state=self.random_seed, - ) - - def _impute_missing_values(self, X: pd.DataFrame) -> pd.DataFrame: - """ - Handle missing values in the feature matrix using median imputation. - If the imputer is not fitted, it will be fitted on the data. - - Args: - X: The feature matrix potentially containing missing values. - - Returns: - pd.DataFrame: The feature matrix with imputed values. - """ - if self.imputer_ is None: - self.imputer_ = SimpleImputer( - strategy="median", copy=True, add_indicator=False - ) - # Fit the imputer and transform the data - imputed_values = self.imputer_.fit_transform(X) - else: - # Use the fitted imputer to transform new data - imputed_values = self.imputer_.transform(X) - - # Convert back to DataFrame with original index and column names - return pd.DataFrame(imputed_values, index=X.index, columns=X.columns) - - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """ - Core fitting logic for the ElasticNet model. - - Args: - y: The target time series data. - X: The feature matrix (including exogenous features). - X_val: Validation feature matrix (ignored). - y_val: Validation target series (ignored). - """ - # Use base class preprocessing - y_processed, X_processed, _, _ = self._preprocess_data( - y, X, X_val, y_val - ) - - if X_processed is None or not isinstance(X_processed, pd.DataFrame): - raise ValueError("Feature matrix X must be a non-empty DataFrame.") - - # First impute missing values in X - X_imputed = self._impute_missing_values(X_processed) - - # Validate X and y after imputation - X_array, y_array = self._validate_X_y( - X_imputed, y_processed, allow_nan=False - ) - - self.training_series_ = y_processed.copy() - self.model_ = self._create_model() - self.model_.fit(X_array, y_array) - self.X_train_ = X_processed.copy() - - @ensure_fitted - def predict(self, X: Optional[pd.DataFrame] = None) -> Sequence: - """ - Generate predictions using the fitted ElasticNet model. - - Args: - X: The feature matrix for prediction. - - Returns: - pd.Series: Predicted values with the original index. - """ - if X is None: - raise ValueError("Feature matrix X is required for prediction.") - - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - if self.training_series_ is None: - raise ValueError("Training series is not available.") - if self.imputer_ is None: - raise ValueError("Imputer is not fitted yet.") - - # If target column is present, drop it - X_pred = X.copy() - if self.target_col in X_pred.columns: - X_pred = X_pred.drop(columns=[self.target_col]) - - # For prediction, we need to reconstruct lagged features - # This is a simplified approach - in practice, you'd need - # the historical target values to create proper lags - - # Handle missing values using fitted imputer - X_processed = self._impute_missing_values(X_pred) - - # Validate X after imputation - X_array = self._validate_X(X_processed, allow_nan=False) - - predictions = self.model_.predict(X_array) - - return pd.Series( - predictions, index=X_processed.index, name=self.target_col - ) - - @ensure_fitted - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Performs backtesting (walk-forward validation) with periodic - retraining. - - Args: - y: The target time series data. - X: Optional exogenous features. - retrain_every: Number of steps after which to retrain the model. - reuse_previous_execution: Whether to reuse the previous execution - of a backtest. If True, any overlapping data between the - previous execution and the current execution will be used - without retraining the model. - - Returns: - Series of predictions for each step in the time series. - """ - if self.model_ is None: - raise ValueError("Model is not fitted yet.") - if self.training_series_ is None: - raise ValueError("Training series is not set.") - if self.X_train_ is None: - raise ValueError("Training feature matrix is not set.") - if X is None or not isinstance(X, pd.DataFrame): - raise ValueError("Feature matrix X must be a non-empty DataFrame.") - - total_steps = len(X) - predictions = [] - current_model = self.model_ - y_history = self.training_series_.copy() - X_history = self.X_train_.copy() - - # Iterate in chunks instead of single steps - for start in range(0, total_steps, retrain_every): - end = min(start + retrain_every, total_steps) - - # Batch prediction for current chunk - X_chunk = X.iloc[start:end].copy() - if self.selected_features_: - X_chunk = X_chunk[self.selected_features_] - X_imputed = self._impute_missing_values(X_chunk) - X_array = self._validate_X(X_imputed, allow_nan=False) - preds = current_model.predict(X_array).squeeze() - if preds.ndim == 0: # single point - preds = [preds] - predictions.extend(preds) - - # Update training history - y_chunk = y.iloc[start:end] - y_history = pd.concat([y_history, y_chunk]) - X_history = pd.concat([X_history, X_chunk]) - - # Retrain the model for next chunk (if needed) - if end < total_steps: - console.print( - f"[cyan]Backtesting: Retraining at step {end}...[/cyan]" - ) - - current_model = self._create_model() - y_fit, X_fit, *_ = self._preprocess_data(y_history, X_history) - X_fit_imputed = self._impute_missing_values(X_fit) - X_fit_array, y_fit_array = self._validate_X_y( - X_fit_imputed, y_fit, allow_nan=False - ) - current_model.fit(X_fit_array, y_fit_array) - - return pd.Series( - predictions, index=X.index, name=f"{self.target_col}_pred" - ) - - @ensure_fitted - def feature_importance(self) -> Optional[pd.DataFrame]: - """ - Returns feature importance based on model coefficients. - - Returns: - A DataFrame with feature names and their corresponding - coefficients (importance scores), or None if no features. - """ - if self.model_ is None or self.feature_names_in_ is None: - return None - - importance = self.model_.coef_ - feature_importance_df = pd.DataFrame( - {"Feature": self.feature_names_in_, "Importance": importance} - ) - feature_importance_df = feature_importance_df.sort_values( - by="Importance", key=abs, ascending=False - ).reset_index(drop=True) - - return feature_importance_df diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py deleted file mode 100644 index e79f519..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/neural_prophet_model.py +++ /dev/null @@ -1,888 +0,0 @@ -""" -NeuralProphet implementation for univariate time series forecasting. - -This module provides a comprehensive wrapper around the NeuralProphet library, -implementing enterprise-level features including robust error handling, -parameter validation, type safety, and integration with the base model -architecture. -""" - -import os -from typing import Optional, cast, Tuple, Dict, Any - -import numpy as np -import pandas as pd -import torch -from neuralprophet import NeuralProphet -from rich.console import Console - -from .base import UnivariateTimeSeriesModel, ensure_fitted - -console = Console() - -# Configure PyTorch for optimal performance -torch.set_num_threads(os.cpu_count() or 1) - - -class NeuralProphetModel(UnivariateTimeSeriesModel): - """ - Enterprise-grade NeuralProphet implementation for univariate - time series forecasting. - - This class provides a robust wrapper around the NeuralProphet model with - comprehensive error handling, parameter validation, and integration with - the base model architecture. It includes features like automatic data - validation, performance monitoring, and enterprise-level logging. - - Key Features: - - Comprehensive parameter validation - - Robust error handling with detailed diagnostics - - Memory-efficient data processing - - Integration with base model utilities - - Performance monitoring and logging - - Support for various seasonality patterns - - Flexible forecasting capabilities - - Example: - >>> model = NeuralProphetModel( - ... n_lags=7, - ... n_forecasts=3, - ... epochs=50, - ... weekly_seasonality=True - ... ) - >>> model.fit(y_train) - >>> predictions = model.predict() - >>> future_forecast = model.forecast(forecast_horizon=3) - """ - - # Class constants for validation - VALID_SEASONALITY_MODES = {"additive", "multiplicative"} - VALID_LOSS_FUNCTIONS = {"Huber", "MSE", "MAE"} - VALID_NORMALIZE_OPTIONS = {"auto", "soft", "off", "minmax"} - MIN_EPOCHS = 1 - MAX_EPOCHS = 10000 - MIN_N_LAGS = 0 - MAX_N_LAGS = 365 - MIN_N_FORECASTS = 1 - MAX_N_FORECASTS = 365 - - def __init__( - self, - name: Optional[str] = None, - n_lags: int = 1, - n_forecasts: int = 2, - weekly_seasonality: bool = True, - daily_seasonality: bool = True, - yearly_seasonality: bool = False, - seasonality_mode: str = "additive", - epochs: int = 100, - learning_rate: Optional[float] = None, - batch_size: Optional[int] = None, - loss_func: str = "Huber", - normalize: str = "auto", - impute_missing: bool = True, - drop_missing: bool = False, - time_col: str = "ds", - target_col: str = "y", - random_seed: int = 42, - ): - """ - Initialize the NeuralProphet time series model with comprehensive - validation. - - Args: - name: Optional identifier for the model - n_lags: Number of lagged target values to include as inputs (0-365) - n_forecasts: Number of steps ahead to forecast (1-365) - weekly_seasonality: Whether to include weekly seasonality - daily_seasonality: Whether to include daily seasonality - yearly_seasonality: Whether to include yearly seasonality - seasonality_mode: Type of seasonality ('additive' or - 'multiplicative') - epochs: Number of training epochs (1-10000) - learning_rate: Learning rate for optimizer (auto if None) - batch_size: Training batch size (auto if None) - loss_func: Loss function ('Huber', 'MSE', 'MAE') - normalize: Normalization type ('auto', 'soft', 'off', 'minmax') - impute_missing: Whether to automatically impute missing values - drop_missing: Whether to drop missing values in training data - time_col: Name of the time column - target_col: Name of the target column - random_seed: Random seed for reproducibility - - Raises: - ValueError: If any parameters are invalid - TypeError: If parameters have incorrect types - """ - self._validate_and_set_parameters( - n_lags=n_lags, - n_forecasts=n_forecasts, - seasonality_mode=seasonality_mode, - epochs=epochs, - learning_rate=learning_rate, - batch_size=batch_size, - loss_func=loss_func, - normalize=normalize, - weekly_seasonality=weekly_seasonality, - daily_seasonality=daily_seasonality, - yearly_seasonality=yearly_seasonality, - impute_missing=impute_missing, - drop_missing=drop_missing, - ) - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - n_lags=n_lags, - ) - self.forecast_horizon = n_forecasts - # Initialize model state - self.model_: Optional[NeuralProphet] = None - self.backtest_predictions_: Optional[pd.Series] = None - self._training_metrics: Dict[str, float] = {} - - console.log( - f"[green]Initialized NeuralProphetModel: {self.summary()}[/green]" - ) - - def _validate_and_set_parameters( - self, - n_lags: int, - n_forecasts: int, - seasonality_mode: str, - epochs: int, - learning_rate: Optional[float], - batch_size: Optional[int], - loss_func: str, - normalize: str, - weekly_seasonality: bool, - daily_seasonality: bool, - yearly_seasonality: bool, - impute_missing: bool, - drop_missing: bool, - ) -> None: - """Validate and set model parameters with comprehensive checks.""" - # Validate integer parameters - if not (self.MIN_N_LAGS <= n_lags <= self.MAX_N_LAGS): - raise ValueError( - f"n_lags must be between {self.MIN_N_LAGS} and " - + f"{self.MAX_N_LAGS}, got {n_lags}" - ) - - if not (self.MIN_N_FORECASTS <= n_forecasts <= self.MAX_N_FORECASTS): - raise ValueError( - f"n_forecasts must be between {self.MIN_N_FORECASTS} and " - + f"{self.MAX_N_FORECASTS}, got {n_forecasts}" - ) - - if not (self.MIN_EPOCHS <= epochs <= self.MAX_EPOCHS): - raise ValueError( - f"epochs must be between {self.MIN_EPOCHS} and " - + f"{self.MAX_EPOCHS}, got {epochs}" - ) - - # Validate string parameters - if seasonality_mode not in self.VALID_SEASONALITY_MODES: - raise ValueError( - "seasonality_mode must be one of " - + f"{self.VALID_SEASONALITY_MODES}, got {seasonality_mode}" - ) - - if loss_func not in self.VALID_LOSS_FUNCTIONS: - raise ValueError( - "loss_func must be one of " - + f"{self.VALID_LOSS_FUNCTIONS}, got {loss_func}" - ) - - if normalize not in self.VALID_NORMALIZE_OPTIONS: - raise ValueError( - "normalize must be one of " - + f"{self.VALID_NORMALIZE_OPTIONS}, got {normalize}" - ) - - # Validate optional float parameters - if learning_rate is not None: - if ( - not isinstance(learning_rate, (int, float)) - or learning_rate <= 0 - ): - raise ValueError( - "learning_rate must be a positive number, " - + f"got {learning_rate}" - ) - - if batch_size is not None: - if not isinstance(batch_size, int) or batch_size <= 0: - raise ValueError( - "batch_size must be a positive integer, " - + f"got {batch_size}" - ) - - # Validate boolean parameters - for param_name, param_value in [ - ("weekly_seasonality", weekly_seasonality), - ("daily_seasonality", daily_seasonality), - ("yearly_seasonality", yearly_seasonality), - ("impute_missing", impute_missing), - ("drop_missing", drop_missing), - ]: - if not isinstance(param_value, bool): - raise TypeError( - f"{param_name} must be a boolean, " - + f"got {type(param_value)}" - ) - - # Set validated parameters - self.n_forecasts = n_forecasts - self.weekly_seasonality = weekly_seasonality - self.daily_seasonality = daily_seasonality - self.yearly_seasonality = yearly_seasonality - self.seasonality_mode = seasonality_mode - self.epochs = epochs - self.learning_rate = learning_rate - self.batch_size = batch_size - self.loss_func = loss_func - self.normalize = normalize - self.impute_missing = impute_missing - self.drop_missing = drop_missing - - def _create_model(self) -> NeuralProphet: - """ - Create a new NeuralProphet instance with validated parameters. - - Returns: - A new NeuralProphet model instance configured with current - parameters. - - Raises: - RuntimeError: If model creation fails - """ - try: - model_params = { - "n_lags": self.n_lags, - "n_forecasts": self.n_forecasts, - "weekly_seasonality": self.weekly_seasonality, - "daily_seasonality": self.daily_seasonality, - "yearly_seasonality": self.yearly_seasonality, - "seasonality_mode": self.seasonality_mode, - "loss_func": self.loss_func, - "normalize": self.normalize, - "impute_missing": self.impute_missing, - "drop_missing": self.drop_missing, - "impute_rolling": 1000000, - "impute_linear": 100000, - } - - # Add optional parameters if specified - if self.learning_rate is not None: - model_params["learning_rate"] = self.learning_rate - if self.batch_size is not None: - model_params["batch_size"] = self.batch_size - - console.log( - f"[blue]Creating NeuralProphet with params: " - f"{model_params}[/blue]" - ) - return NeuralProphet(**model_params) - - except Exception as e: - raise RuntimeError( - f"Failed to create NeuralProphet model: {e}" - ) from e - - def _validate_and_prepare_data( - self, y: pd.Series - ) -> Tuple[pd.DataFrame, pd.Series]: - """ - Validate and prepare time series data for NeuralProphet. - - Args: - y: Input time series data - - Returns: - Tuple of (prepared_dataframe, validated_series) - - Raises: - ValueError: If data validation fails - """ - try: - # Validate target series - y_array = self._validate_y(y) - - # Ensure datetime index - if not pd.api.types.is_datetime64_any_dtype(y.index): - try: - y_datetime = y.copy() - y_datetime.index = pd.to_datetime(y.index) - console.log("[yellow]Converted index to datetime[/yellow]") - except Exception as e: - raise ValueError( - "y's index must be a DateTime index or convertible " - f"to DateTime. Conversion failed: {e}" - ) from e - else: - y_datetime = y.copy() - - # Check for minimum data requirements - if len(y_datetime) < max(self.n_lags + 1, 10): - raise ValueError( - "Insufficient data: need at least " - + f"{max(self.n_lags + 1, 10)} observations, " - + f"got {len(y_datetime)}" - ) - - # Create NeuralProphet format DataFrame - df = pd.DataFrame({"ds": y_datetime.index, "y": y_array}) - - # Validate for missing values if not configured to handle them - if not self.impute_missing and bool(df["y"].isna().any()): - raise ValueError( - "Data contains missing values but impute_missing=False. " - "Either set impute_missing=True or clean the data." - ) - - console.log( - f"[green]Data validation successful: {len(df)} " - f"observations[/green]" - ) - return df, y_datetime - - except Exception as e: - console.print(f"[red]Data validation failed: {e}[/red]") - raise - - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """ - Core fitting logic for NeuralProphet model with enhanced error - handling. - - Args: - y: The target time series data with DateTime index - X: Optional DataFrame (unused for univariate model) - X_val: Validation features (unused for NeuralProphet) - y_val: Validation target (unused for NeuralProphet) - - Raises: - ValueError: If data validation fails - RuntimeError: If model fitting fails - """ - try: - console.log("[blue]Starting NeuralProphet model fitting...[/blue]") - - df, y_datetime = self._validate_and_prepare_data(y) - self.training_series_ = y_datetime - self.model_ = self._create_model() - - console.log( - f"[blue]Training model for {self.epochs} epochs...[/blue]" - ) - - fit_result = self.model_.fit(df, epochs=self.epochs) - - # Store training metrics if available - if hasattr(fit_result, "losses") and fit_result is not None: - losses = getattr(fit_result, "losses", None) - if losses: - self._training_metrics = { - "final_loss": float(losses[-1]), - "epochs_trained": len(losses), - } - - console.log( - f"[green]Model fitting completed successfully. " - f"Metrics: {self._training_metrics}[/green]" - ) - - except Exception as e: - console.print(f"[red]Model fitting failed: {e}[/red]") - # Reset model state on failure - self.model_ = None - self.training_series_ = None - raise RuntimeError( - f"Failed to fit NeuralProphet model: {e}" - ) from e - - def _prepare_prediction_data( - self, X: Optional[pd.DataFrame] = None - ) -> Tuple[pd.DataFrame, pd.Series]: - """ - Prepare data for prediction with comprehensive validation. - - Args: - X: Optional DataFrame containing prediction data - - Returns: - Tuple of (prepared_dataframe, prediction_index) - - Raises: - ValueError: If data preparation fails - """ - if self.training_series_ is None: - raise ValueError("No training series available") - - training_series = cast(pd.Series, self.training_series_) - - if X is None: - # Predict on training data - df = pd.DataFrame( - { - "ds": training_series.index, - "y": training_series.to_numpy(), - } - ) - return df, pd.Series(training_series.index) - - # Handle various input formats for X - try: - ds_values, y_values = self._extract_time_and_target_from_X(X) - - # Ensure datetime format - if not pd.api.types.is_datetime64_any_dtype(ds_values): - ds_values = pd.to_datetime(ds_values) - - df = pd.DataFrame( - { - "ds": ds_values, - "y": y_values, - } - ) - - return df, ds_values - - except Exception as e: - raise ValueError( - f"Failed to prepare prediction data: {e}" - ) from e - - def _extract_time_and_target_from_X( - self, X: pd.DataFrame - ) -> Tuple[pd.Series, pd.Series]: - """ - Extract time and target columns from input DataFrame. - - Args: - X: Input DataFrame - - Returns: - Tuple of (time_series, target_series) - - Raises: - ValueError: If extraction fails - """ - # Scenario 1: Explicit time and target columns - if self.time_col in X.columns and self.target_col in X.columns: - return ( - X[self.time_col], - self._validate_y(X[self.target_col]) - ) - - # Scenario 2: DateTime index - elif pd.api.types.is_datetime64_any_dtype(X.index): - ds_values = pd.Series(X.index, name=self.time_col) - - if self.target_col in X.columns: - # DateTime index with explicit target column - return ( - ds_values, - self._validate_y(X[self.target_col]) - ) - elif X.shape[1] == 1: - # DateTime index with single data column - return ds_values, self._validate_y( - X.iloc[:, 0].rename(self.target_col) - ) - elif X.shape[1] == 0: - # Only index, no columns - forecast scenario - y_values = pd.Series( - np.nan, index=X.index, name=self.target_col - ) - return ds_values, y_values - else: - raise ValueError( - "X has DateTime index but cannot identify target column. " - + f"Expected '{self.target_col}' or single column. " - + f"Found: {X.columns.tolist()}" - ) - else: - raise ValueError( - "Cannot determine time and target from X. " - + f"Provide columns '{self.time_col}' and '{self.target_col}' " - + "or use DateTime index." - ) - - @ensure_fitted - def predict(self, X: Optional[pd.DataFrame] = None) -> pd.Series: - """ - Generate in-sample predictions with enhanced error handling. - - Args: - X: Optional DataFrame containing timestamps and target values. - If None, predicts on training data. - - Returns: - Series of predictions indexed by timestamp - - Raises: - ValueError: If model is not fitted or prediction fails - RuntimeError: If prediction computation fails - """ - if self.model_ is None: - raise ValueError("Model is not fitted") - - try: - console.log("[blue]Generating predictions...[/blue]") - - # Prepare prediction data - df, predictions_index = self._prepare_prediction_data(X) - - # Get training context for lagged features - training_series = cast(pd.Series, self.training_series_) - past_values = pd.DataFrame( - { - "ds": training_series.index, - "y": training_series.to_numpy(), - } - ).iloc[-self.n_lags :, :] - - # Combine past and prediction data - combined_df = pd.concat([past_values, df], ignore_index=True) - combined_df = ( - combined_df.sort_values(by="ds") - .reset_index(drop=True) - .drop_duplicates(subset="ds", keep="last") - ) - - # Generate forecast - forecast = self.model_.predict(combined_df) - - # Handle different forecast column formats - forecast_col = f"yhat{self.n_forecasts}" - if forecast_col not in forecast.columns: - forecast = self.model_.get_last_forecast( - forecast, include_previous_forecasts=self.n_forecasts - ) - - # Extract predictions for requested indices - forecast = forecast.set_index("ds") - predictions = forecast.loc[predictions_index, forecast_col] - - console.log( - f"[green]Generated {len(predictions)} predictions[/green]" - ) - return predictions - - except Exception as e: - console.print(f"[red]Prediction failed: {e}[/red]") - raise RuntimeError(f"Failed to generate predictions: {e}") from e - - @ensure_fitted - def forecast(self, forecast_horizon: int) -> np.ndarray: - """ - Generate future forecasts with comprehensive validation. - - Args: - forecast_horizon: Number of steps to forecast ahead - (1 to n_forecasts) - - Returns: - Array of forecasted values, indexed by the forecast horizon - - Raises: - ValueError: If forecast_horizon is invalid or model not fitted - RuntimeError: If forecast generation fails - """ - if self.model_ is None: - raise ValueError("Model is not fitted") - - if not (1 <= forecast_horizon <= self.n_forecasts): - raise ValueError( - "forecast_horizon must be between 1 and " - + f"{self.n_forecasts}, got {forecast_horizon}" - ) - - try: - console.log( - f"[blue]Generating {forecast_horizon}-step forecast...[/blue]" - ) - - training_series = cast(pd.Series, self.training_series_) - - # Create future dataframe - future_df = self.model_.make_future_dataframe( - df=pd.DataFrame( - { - "ds": training_series.index, - "y": training_series.to_numpy(), - } - ), - periods=forecast_horizon, - ) - - # Generate forecasts - forecast = self.model_.predict(future_df) - - # Extract forecasted values for each horizon - forecasted_values = np.empty(forecast_horizon) - for i in range(forecast_horizon): - col_name = f"yhat{i + 1}" - if col_name in forecast.columns: - values = forecast[col_name].dropna() - if len(values) > 0: - forecasted_values[i] = values.iloc[0] - else: - forecasted_values[i] = np.nan - else: - forecasted_values[i] = np.nan - - console.log( - f"[green]Generated forecast: {len(forecasted_values)}" - + " values[/green]" - ) - return forecasted_values - - except Exception as e: - console.print(f"[red]Forecast generation failed: {e}[/red]") - raise RuntimeError(f"Failed to generate forecast: {e}") from e - - @ensure_fitted - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Perform comprehensive backtesting with enhanced monitoring. - - This method implements walk-forward validation with periodic - retraining, providing robust evaluation of model performance in - production-like scenarios. - - Args: - y: Target time series for backtesting (must have DateTime index) - X: Unused (included for base class compatibility) - retrain_every: Unused (model retrains at each step) - reuse_previous_execution: Whether to reuse previous backtest - results - - Returns: - Series of backtested predictions indexed by timestamp - - Raises: - ValueError: If parameters are invalid or data is insufficient - RuntimeError: If backtesting fails - """ - if self.model_ is None: - raise ValueError("Model is not fitted") - if self.training_series_ is None: - raise ValueError("No training series found") - - if not (1 <= self.forecast_horizon <= self.n_forecasts): - raise ValueError( - "forecast_horizon must be between 1 and " - + f"{self.n_forecasts}, got {self.forecast_horizon}" - ) - - # Handle reuse of previous execution - if reuse_previous_execution and self.backtest_predictions_ is not None: - expected_index = y.iloc[self.forecast_horizon:].index - if ( - len(self.backtest_predictions_) == len(expected_index) - and (self.backtest_predictions_.index == expected_index).all() - ): - console.log( - "[yellow]Reusing previous backtest results[/yellow]" - ) - return self.backtest_predictions_ - else: - console.log( - "[yellow]Previous results incompatible, running new" - + " backtest[/yellow]" - ) - - try: - console.log( - f"[blue]Starting backtest with {self.forecast_horizon}-step" - + " horizon...[/blue]" - ) - - # Validate and prepare data - y_sorted = y.sort_index() - - # Check for overlapping data - training_series = cast(pd.Series, self.training_series_) - if any(t in training_series.index for t in y_sorted.index): - console.print( - "[yellow]Warning: Backtest data overlaps with training" - + " data[/yellow]" - ) - - # Initialize backtesting - predictions = [] - training_base = pd.DataFrame( - { - "ds": training_series.index, - "y": training_series.values, - } - ) - - timestamps = y_sorted.index - total_steps = len(timestamps) - self.forecast_horizon - - console.log( - f"[blue]Running {total_steps} backtest steps...[/blue]" - ) - - # Perform walk-forward validation - for i in range(self.forecast_horizon, len(timestamps)): - if i % 50 == 0: # Progress logging - console.log( - f"[blue]Backtest progress: {i}/{len(timestamps)}[/blue]" - ) - - t = timestamps[i] - t_minus_h = timestamps[i - self.forecast_horizon] - - # Prepare training data up to t - h - history = y_sorted.loc[:t_minus_h] - train_df = ( - pd.concat( - [ - training_base, - pd.DataFrame( - {"ds": history.index, "y": history.values} - ), - ], - ignore_index=True, - ) - .drop_duplicates(subset="ds") - .sort_values("ds") - ) - - # Check minimum data requirement - if len(train_df) < max(self.n_lags + 1, 10): - console.print( - f"[yellow]Insufficient data at step {i}, " - + "skipping[/yellow]" - ) - predictions.append((t, np.nan)) - continue - - try: - # Retrain model - model = self._create_model() - model.fit(train_df, epochs=self.epochs) - - # Generate forecast - mask = train_df["ds"] <= t_minus_h - future_df = model.make_future_dataframe( - df=train_df.loc[mask], periods=self.forecast_horizon - ) - forecast = model.predict(future_df, decompose=False) - - # Extract prediction - forecast_col = f"yhat{self.forecast_horizon}" - prediction_rows = forecast[forecast["ds"] == t] - - if ( - len(prediction_rows) > 0 - and forecast_col in forecast.columns - ): - prediction = prediction_rows[forecast_col].iloc[0] - else: - prediction = np.nan - - predictions.append((t, prediction)) - - except Exception as step_error: - console.print( - f"[yellow]Error at step {i}: {step_error}[/yellow]" - ) - predictions.append((t, np.nan)) - - # Create results series - if predictions: - pred_index, pred_values = zip(*predictions) - self.backtest_predictions_ = pd.Series( - pred_values, - index=pd.Index(pred_index), - name=f"yhat{self.forecast_horizon}", - ) - else: - self.backtest_predictions_ = pd.Series( - dtype=float, name=f"yhat{self.forecast_horizon}" - ) - - console.log( - f"[green]Backtest completed: {len(self.backtest_predictions_)}" - + " predictions[/green]" - ) - return self.backtest_predictions_ - - except Exception as e: - console.print(f"[red]Backtest failed: {e}[/red]") - raise RuntimeError(f"Failed to perform backtest: {e}") from e - - def get_params_dict(self) -> Dict[str, Any]: - """Get comprehensive model parameters for logging/serialization.""" - base_params = super().get_params_dict() - neural_prophet_params = { - "n_forecasts": self.n_forecasts, - "weekly_seasonality": self.weekly_seasonality, - "daily_seasonality": self.daily_seasonality, - "yearly_seasonality": self.yearly_seasonality, - "seasonality_mode": self.seasonality_mode, - "epochs": self.epochs, - "learning_rate": self.learning_rate, - "batch_size": self.batch_size, - "loss_func": self.loss_func, - "normalize": self.normalize, - "impute_missing": self.impute_missing, - "drop_missing": self.drop_missing, - "training_metrics": self._training_metrics, - } - return {**base_params, **neural_prophet_params} - - def summary(self) -> str: - """Generate comprehensive model summary.""" - fitted_status = ( - "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" - ) - - seasonality_features = [] - if self.weekly_seasonality: - seasonality_features.append("Weekly") - if self.daily_seasonality: - seasonality_features.append("Daily") - if self.yearly_seasonality: - seasonality_features.append("Yearly") - - seasonality_str = ( - ", ".join(seasonality_features) if seasonality_features else "None" - ) - - summary_lines = [ - f"Model: {self.__class__.__name__}", - f"Status: {fitted_status}", - f"Lags: {self.n_lags}, Forecasts: {self.n_forecasts}", - f"Seasonality: {seasonality_str} ({self.seasonality_mode})", - f"Training: {self.epochs} epochs, {self.loss_func} loss", - f"Data Handling: Impute={self.impute_missing}, " - f"Drop={self.drop_missing}", - ] - - if self._training_metrics: - metrics_str = ", ".join( - f"{k}={v:.4f}" for k, v in self._training_metrics.items() - ) - summary_lines.append(f"Metrics: {metrics_str}") - - return "\n".join(summary_lines) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py deleted file mode 100644 index 5e5a7c0..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/models/stacking_time_series.py +++ /dev/null @@ -1,695 +0,0 @@ -""" -Stacking implementation for time series forecasting. - -This module provides a stacking regressor implementation that combines multiple -time series models' predictions using a meta-model. The stacking approach -helps improve prediction accuracy by combining the strengths of different -base models through a learned meta-model. - -Key features: -- **Model Stacking**: Combines predictions from multiple base models -- **Time-Series Aware**: Uses proper time-based cross-validation -- **Meta-Model Learning**: Learns optimal combination weights -- **Comprehensive Error Handling**: Robust error handling and validation -- **Rich Logging**: Colored console output for better debugging - -The stacking model loads pre-trained base models and uses their predictions -as features for training a meta-model (CatBoost by default). -""" - -from typing import Optional, List, Dict, Any, Union -import pandas as pd -import numpy as np -from catboost import CatBoostRegressor, CatBoostClassifier, Pool -from rich.console import Console -import gzip -import pickle -import lzma - -from .base import ( - MultivariateTimeSeriesModel, - ensure_fitted, - TimeSeriesModel, -) - -console = Console() - - -class StackingTimeSeriesModel(MultivariateTimeSeriesModel): - """ - Stacking implementation for time series forecasting. - - This class implements stacking of multiple base models, using their - predictions as features for a meta-model. It handles time-based - cross-validation to generate out-of-fold predictions for training. - - The model supports both regression and classification tasks through - the meta-model configuration. - - Attributes: - base_models_: List of loaded base models - model_: The trained meta-model (CatBoost) - training_series_: Copy of training target data - base_predictions_train_: Base model predictions on training data - backtest_predictions_: Stored backtest predictions for reuse - - Example: - >>> stacking_model = StackingTimeSeriesModel( - ... base_model_paths=["model1.pkl", "model2.pkl"], - ... base_model_types=["catboost", "elasticnet"] - ... ) - >>> stacking_model.fit(y=target_series, X=feature_matrix) - >>> predictions = stacking_model.predict(X=test_features) - """ - - def __init__( - self, - base_model_paths: List[str], - base_model_types: List[str], - name: Optional[str] = None, - learning_task: str = "regression", - retrain_every: int = 100, - meta_iterations: int = 1000, - meta_learning_rate: float = 0.1, - meta_depth: int = 6, - early_stopping_rounds: Optional[int] = None, - meta_loss_function: Optional[str] = None, - time_col: str = "ds", - target_col: str = "y", - random_seed: int = 42, - verbose: bool = False, - differentiate_target: bool = False, - bins: Optional[List[float]] = None, - use_predict_for_training: bool = True, - ) -> None: - """ - Initialize the stacking model. - - Args: - base_model_paths: Paths to saved base models - base_model_types: Types of base models (must match order of paths) - name: Optional identifier for the model - learning_task: Type of learning task ('regression', 'binary', - 'multiclass') - retrain_every: Frequency of retraining during backtesting - meta_iterations: Number of iterations for meta-model - meta_learning_rate: Learning rate for meta-model - meta_depth: Tree depth for meta-model - early_stopping_rounds: Early stopping rounds for meta-model - meta_loss_function: Loss function for meta-model - time_col: Name of time column - target_col: Name of target column - random_seed: Random seed - verbose: Whether to print verbose logging - differentiate_target: Whether to differentiate the target series - bins: Bin edges for multiclass classification - use_predict_for_training: If True, use predict() instead of - backtest() for generating base model predictions during - training. This is much faster but may lead to overfitting - since the meta-model trains on in-sample predictions. - """ - super().__init__( - name=name, - time_col=time_col, - target_col=target_col, - random_seed=random_seed, - learning_task=learning_task, - differentiate_target=differentiate_target, - bins=bins, - ) - - # Validate inputs - if not base_model_paths: - raise ValueError("base_model_paths cannot be empty") - if not base_model_types: - raise ValueError("base_model_types cannot be empty") - if len(base_model_paths) != len(base_model_types): - raise ValueError( - "base_model_paths and base_model_types must have same length" - ) - - self.retrain_every = retrain_every - self.base_model_paths = base_model_paths - self.base_model_types = base_model_types - self.meta_iterations = meta_iterations - self.meta_learning_rate = meta_learning_rate - self.meta_depth = meta_depth - self.meta_loss_function = self._get_default_loss_function( - meta_loss_function - ) - self.early_stopping_rounds = early_stopping_rounds - self.verbose = verbose - self.use_predict_for_training = use_predict_for_training - - # Will be set during fit - self.base_models_: List[TimeSeriesModel] = [] - self.model_: Optional[ - Union[CatBoostRegressor, CatBoostClassifier] - ] = None - self.training_series_: Optional[pd.Series] = None - self.base_predictions_train_: Optional[pd.DataFrame] = None - self.backtest_predictions_: Optional[pd.Series] = None - - self._load_base_models() - - if self.verbose: - console.log( - "[green]Initialized StackingTimeSeriesModel: " - + f"{self.summary()}[/green]" - ) - - def _load_base_models(self) -> None: - """Load all base models from their saved paths.""" - from .factory import load_model - - self.base_models_ = [] - for model_path, model_type in zip( - self.base_model_paths, self.base_model_types - ): - try: - model = load_model(model_path, model_type) - self.base_models_.append(model) - if self.verbose: - console.log( - f"[blue]Loaded {model_type} model from " - + f"{model_path}[/blue]" - ) - except Exception as e: - console.print( - f"[red]Error loading model from {model_path}: {e}[/red]" - ) - raise ValueError(f"Failed to load model: {model_path}") from e - - if self.verbose: - console.log( - f"[green]Successfully loaded {len(self.base_models_)} " - + "base models[/green]" - ) - - def _create_meta_model( - self, - ) -> Union[CatBoostRegressor, CatBoostClassifier]: - """Creates a new instance of meta-model. - - Returns: - A new CatBoost model instance (Regressor or Classifier). - """ - base_params = { - "iterations": self.meta_iterations, - "learning_rate": self.meta_learning_rate, - "depth": self.meta_depth, - "loss_function": self.meta_loss_function, - "early_stopping_rounds": self.early_stopping_rounds, - "random_seed": self.random_seed, - "verbose": self.verbose, - } - - if self.learning_task in ["binary", "multiclass"]: - return CatBoostClassifier( - auto_class_weights="Balanced", **base_params - ) - else: - return CatBoostRegressor(**base_params) - - def _get_base_predictions( - self, X: Optional[pd.DataFrame], y: Optional[pd.Series] = None - ) -> pd.DataFrame: - """Get predictions from all base models. - - Args: - X: Feature matrix - y: Target series (optional, used for validation) - - Returns: - DataFrame with predictions in model_0, model_1, etc columns - """ - if X is None: - raise ValueError("Feature matrix X cannot be None") - - all_preds = [] - for i, model in enumerate(self.base_models_): - try: - preds = model.predict(X) - model_name = f"model_{model.name or i}" - all_preds.append( - pd.Series(preds, name=model_name, index=X.index) - ) - - if self.verbose: - console.log( - f"[cyan]Generated predictions from {model_name}[/cyan]" - ) - except Exception as e: - console.print( - f"[red]Error getting predictions from model {i}: {e}[/red]" - ) - raise - - return pd.concat(all_preds, axis=1) - - def _fit_logic( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - X_val: Optional[pd.DataFrame] = None, - y_val: Optional[pd.Series] = None, - ) -> None: - """Core fitting logic for stacking model. - - Get predictions from base models on training data, then train - meta-model on those predictions. - - Args: - y: The target time series data - X: The feature matrix (including exogenous features) - X_val: Validation feature matrix (optional) - y_val: Validation target series (optional) - """ - if X is None: - raise ValueError("Feature matrix X must be provided for stacking") - - # Preprocess data using parent class method - y_processed, X_processed, y_val_processed, X_val_processed = ( - self._preprocess_data(y, X, X_val, y_val) - ) - - if X_processed is None: - raise ValueError( - "Feature matrix X cannot be None after preprocessing" - ) - - if self.verbose: - method_name = ( - "predictions" - if self.use_predict_for_training - else "backtest predictions" - ) - console.log( - f"[blue]Generating base model {method_name} " - + "for stacking...[/blue]" - ) - - # Get base predictions - use either predict or backtest based on - # setting - if self.use_predict_for_training: - # Fast approach: use direct predictions (may overfit) - meta_features = self._get_base_predictions(X_processed) - # Align with target data - common_index = meta_features.index.intersection(y_processed.index) - meta_features = meta_features.loc[common_index] - y_aligned = y_processed.loc[common_index] - - if self.verbose: - console.log( - "[yellow]Warning: Using predict() for training may " - + "lead to overfitting since meta-model trains on " - + "in-sample predictions[/yellow]" - ) - else: - # Robust approach: use backtesting to avoid overfitting - all_preds = [] - for i, model in enumerate(self.base_models_): - try: - preds = model.backtest( - y_processed, - X_processed, - retrain_every=self.retrain_every, - ) - model_name = f"model_{model.name or i}" - all_preds.append(pd.Series(preds, name=model_name)) - - if self.verbose: - console.log( - "[cyan]Generated backtest predictions from " - + f"{model_name}[/cyan]" - ) - except Exception as e: - console.print( - f"[red]Error during backtesting for model {i}: " - + f"{e}[/red]" - ) - raise - - meta_features = pd.concat(all_preds, axis=1) - - # Align with target data (backtest might have different length) - common_index = meta_features.index.intersection(y_processed.index) - meta_features = meta_features.loc[common_index] - y_aligned = y_processed.loc[common_index] - - if self.verbose: - console.log( - f"[blue]Training meta-model with {len(meta_features)} " - + f"samples and {meta_features.shape[1]} base model " - + "features[/blue]" - ) - - meta_X, meta_y = self._validate_X_y(meta_features, y_aligned) - train_pool = Pool(data=meta_X, label=meta_y) - - # Prepare validation data if provided - eval_set = None - if X_val_processed is not None and y_val_processed is not None: - val_predictions = self._get_base_predictions( - X_val_processed, y_val_processed - ) - val_X, val_y = self._validate_X_y(val_predictions, y_val_processed) - eval_set = Pool(data=val_X, label=val_y) - - if self.verbose: - console.log( - "[blue]Using validation set with " - + f"{len(val_predictions)} samples[/blue]" - ) - - # Create and train meta-model - self.model_ = self._create_meta_model() - self.model_.fit(train_pool, eval_set=eval_set) - - # Store training data - self.training_series_ = y_processed.copy() - self.base_predictions_train_ = meta_features.copy() - - if self.verbose: - console.log( - "[green]Meta-model training completed successfully[/green]" - ) - - @ensure_fitted - def predict(self, X: pd.DataFrame) -> pd.Series: - """ - Generate predictions using the stacking model. - - Args: - X: Feature matrix for prediction - - Returns: - Series containing predictions - """ - if self.model_ is None: - raise ValueError("Model has not been fitted yet") - - base_predictions = self._get_base_predictions(X) - X_array = self._validate_X(base_predictions) - predictions = self.model_.predict(X_array) - - # Convert predictions to numpy array if needed - if hasattr(predictions, "squeeze"): - predictions = predictions.squeeze() - elif isinstance(predictions, list): - predictions = np.array(predictions) - - return pd.Series(predictions, index=X.index, name=self.target_col) - - @ensure_fitted - def feature_importance(self) -> Optional[pd.DataFrame]: - """ - Returns feature importance from the meta-model. - - Returns: - DataFrame with feature names and their importance scores, - or None if not available. - """ - if self.model_ is None or not hasattr( - self.model_, "feature_importances_" - ): - return None - - if self.base_predictions_train_ is None: - return None - - importances = self.model_.feature_importances_ - feature_names = self.base_predictions_train_.columns - - return pd.DataFrame( - { - "feature": feature_names, - "importance": importances, - } - ).sort_values("importance", ascending=False) - - @ensure_fitted - def backtest( - self, - y: pd.Series, - X: Optional[pd.DataFrame] = None, - retrain_every: int = 50, - reuse_previous_execution: bool = False, - ) -> pd.Series: - """ - Performs backtesting (walk-forward validation) with periodic - retraining. - - This method simulates a production scenario by iterating through a test - set, making a one-step-ahead prediction, and then retraining the model - periodically with the newly available data. - - Args: - y: Series with the true target values for the backtesting period - X: DataFrame with features for the backtesting period - retrain_every: The frequency of retraining. The model will be - retrained every `retrain_every` steps - reuse_previous_execution: Whether to reuse the previous execution - of a backtest. If True, any overlapping data between the - previous execution and the current execution will be used - without retraining the model - - Returns: - A series of backtested predictions, indexed by the backtest data's - index - """ - if self.model_ is None: - raise ValueError("Model is not fitted yet") - if self.training_series_ is None: - raise ValueError("Training series is not set") - if X is None: - raise ValueError("Feature matrix X must be provided") - - if reuse_previous_execution: - if self.backtest_predictions_ is None: - raise ValueError("No previous execution found") - if (self.backtest_predictions_.shape[0] != y.shape[0]) or ( - not (self.backtest_predictions_.index == y.index).all() - ): - raise ValueError( - "Previous execution index does not match y index" - ) - return self.backtest_predictions_ - - if self.verbose: - console.log( - f"[blue]Starting backtest with {len(y)} samples, " - + f"retraining every {retrain_every} steps[/blue]" - ) - - # Get base model predictions for the entire backtest period - all_base_preds = [] - for i, model in enumerate(self.base_models_): - try: - preds = model.backtest( - y, - X, - retrain_every=retrain_every, - reuse_previous_execution=reuse_previous_execution, - ) - model_name = f"model_{model.name or i}" - all_base_preds.append(pd.Series(preds, name=model_name)) - - if self.verbose: - console.log( - f"[cyan]Completed backtest for {model_name}[/cyan]" - ) - except Exception as e: - console.print( - f"[red]Error during backtest for model {i}: {e}[/red]" - ) - raise - - meta_features = pd.concat(all_base_preds, axis=1) - - # Generate meta-model predictions - predictions = self.model_.predict(self._validate_X(meta_features)) - - # Store backtest predictions for potential reuse - self.backtest_predictions_ = pd.Series( - predictions, - index=meta_features.index, - name=f"{self.target_col}_pred", - ) - - if self.verbose: - console.log( - "[green]Backtest completed: " - + f"{len(self.backtest_predictions_)} predictions " - + "generated[/green]" - ) - - return self.backtest_predictions_ - - def get_base_model_names(self) -> List[str]: - """Get names of all base models. - - Returns: - List of base model names - """ - return [ - model.name or f"model_{i}" - for i, model in enumerate(self.base_models_) - ] - - def get_params_dict(self) -> Dict[str, Any]: - """Get model parameters as dictionary for logging/serialization.""" - base_params = super().get_params_dict() - stacking_params = { - "base_model_paths": self.base_model_paths, - "base_model_types": self.base_model_types, - "retrain_every": self.retrain_every, - "meta_iterations": self.meta_iterations, - "meta_learning_rate": self.meta_learning_rate, - "meta_depth": self.meta_depth, - "meta_loss_function": self.meta_loss_function, - "early_stopping_rounds": self.early_stopping_rounds, - "num_base_models": len(self.base_models_), - "use_predict_for_training": self.use_predict_for_training, - } - return {**base_params, **stacking_params} - - def summary(self) -> str: - """Generate a summary string of the model.""" - params = self.get_params_dict() - fitted_status = ( - "✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted" - ) - - summary_lines = [ - f"Model: {self.__class__.__name__}", - f"Status: {fitted_status}", - f"Base Models: {params.get('num_base_models', 0)}", - f"Task: {params.get('learning_task', 'regression')}", - f"Meta Loss: {params.get('meta_loss_function', 'RMSE')}", - ] - - return "\n".join(summary_lines) - - def _optimize_base_models_for_storage(self) -> None: - """ - Optimizes base models for storage by removing unnecessary data. - This can significantly reduce pickle size, especially for neural - models. - """ - if self.verbose: - console.log("[blue]Optimizing base models for storage...[/blue]") - - for i, model in enumerate(self.base_models_): - try: - # For neuralprophet models, remove training history and - # large artifacts - model_attr = getattr(model, "model", None) - if model_attr is not None and hasattr(model_attr, "trainer"): - trainer = getattr(model_attr, "trainer", None) - if trainer is not None: - # Remove trainer which contains training logs and - # can be very large - if hasattr(trainer, "logged_metrics"): - setattr(trainer, "logged_metrics", {}) - if hasattr(trainer, "progress_bar_metrics"): - setattr(trainer, "progress_bar_metrics", {}) - if hasattr(trainer, "callback_metrics"): - setattr(trainer, "callback_metrics", {}) - - # For any model with training history - if hasattr(model, "training_history_"): - setattr(model, "training_history_", None) - if hasattr(model, "validation_history_"): - setattr(model, "validation_history_", None) - - # Remove cached predictions if they exist - if hasattr(model, "_cached_predictions"): - setattr(model, "_cached_predictions", None) - - if self.verbose: - model_name = getattr(model, "name", f"model_{i}") - console.log( - f"[cyan]Optimized {model_name} for storage[/cyan]" - ) - - except Exception as e: - if self.verbose: - console.log( - f"[yellow]Warning: Could not optimize model {i}: " - + f"{e}[/yellow]" - ) - - def save(self, path: str, compression: str = "gzip") -> None: - """ - Saves model to disk using compression to reduce file size. - - Args: - path: File path to save to - compression: Compression method ('gzip', 'lzma', or 'none') - - 'gzip': Fast compression, ~60-80% size reduction - - 'lzma': Better compression, ~70-90% size reduction, slower - - 'none': No compression - """ - if self.verbose: - console.log( - f"[blue]Saving stacking model with {compression} " - + f"compression to {path}[/blue]" - ) - - # Optimize base models for storage first - self._optimize_base_models_for_storage() - - if compression == "lzma": - # LZMA provides better compression but is slower - with lzma.open(path, "wb", preset=9) as f: - pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) - elif compression == "gzip": - # Gzip is faster with good compression - with gzip.open(path, "wb", compresslevel=9) as f: - pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) - else: - # No compression - with open(path, "wb") as f: - pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) - - if self.verbose: - console.log( - f"[green]Saved compressed stacking model to {path}[/green]" - ) - - @classmethod - def load(cls, path: str) -> "StackingTimeSeriesModel": - """ - Loads model from disk with automatic format detection. - Supports both compressed formats and legacy joblib format. - """ - import joblib - - # Try different formats in order of preference - loading_methods = [ - ("lzma", lambda p: lzma.open(p, "rb")), - ("gzip", lambda p: gzip.open(p, "rb")), - ("pickle", lambda p: open(p, "rb")), - ("joblib", None), # Special case for joblib - ] - - for format_name, open_func in loading_methods: - try: - if format_name == "joblib": - return joblib.load(path) - else: - with open_func(path) as f: - return pickle.load(f) - except ( - lzma.LZMAError, - gzip.BadGzipFile, - OSError, - pickle.UnpicklingError, - ValueError, - ): - continue - - raise ValueError( - f"Could not load model from {path} - unknown or corrupted format" - ) diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py b/tmp/artifacts/data_model/transformer_pyfunc/code/utils/visualization/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml b/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml deleted file mode 100644 index 2f1d2cb..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/conda.yaml +++ /dev/null @@ -1,11 +0,0 @@ -channels: -- conda-forge -dependencies: -- python=3.10.16 -- pip<=25.0 -- pip: - - mlflow==2.7.1 - - pandas - - numpy - - scikit-learn -name: mlflow-env diff --git a/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml b/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml deleted file mode 100644 index 0a0396b..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/python_env.yaml +++ /dev/null @@ -1,7 +0,0 @@ -python: 3.10.16 -build_dependencies: -- pip==25.0 -- setuptools==79.0.0 -- wheel==0.45.1 -dependencies: -- -r requirements.txt diff --git a/tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl b/tmp/artifacts/data_model/transformer_pyfunc/python_model.pkl deleted file mode 100644 index aa3f5aa8ef7c82c96b4bd1fda6e43cd695abf3f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 123 zcmZo*om#*E0X;IMC7C(Jdbv4iIr-&!1(j)~dCBqRMTrFksYS(8dW1rX67!1F@{4j) zi^3tIQzlQ*Y@AX%MWaWgq$n{nFEcMa9>{>Hn&Q_ZnwgiDT9lfXoQf(@nxqE+?KdyV diff --git a/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt b/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt deleted file mode 100644 index fd9a283..0000000 --- a/tmp/artifacts/data_model/transformer_pyfunc/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -mlflow==2.7.1 -pandas -numpy -scikit-learn \ No newline at end of file diff --git a/tmp/artifacts/data_model/transformers/courier_transformers.pkl b/tmp/artifacts/data_model/transformers/courier_transformers.pkl deleted file mode 100644 index 18fe2203362307ea711c91971c88e266be720d81..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29607 zcmeHQS(6-BR<`!BWJ%U;*|MF3Ev%Vww`yZ)=rO)VP9 zg98SzEkgoBIOhR^2MiArLBs?E@WLBdj}ZI?f_?b~yuo+Q-EU@CT{AkGgBp4*zSjPm*tZFCo?w;pS#H+6?>kPP=+z&|asTUh)n^dt1>)J!~iO zu3mow*H)eGTj8xFicj_Wovp2Cqq^1YtVg|Gb-mLZv|vb_2a37!*z)Y!Sbidx%sp_qD@x+W zso3j>t*ssuCRN%x^W|kk8FkQjj%Vq-swi;H@brC5;8c$ z_c}SdtF29>_&Hm>-tI@;c8HX>!gjbBb&=^OE;c*s?=FaP@<|e(lor)y{d#X)nY%+BVb8u>*R47j7@;Uq)%jL?2u}hbxr_*>&;@swBoiCOX z(c)aHfa{53ZenbDYJM6-c`D{}s#xISq$&z&!74g^$?Zv^G=n;GIq)2!Ubi~Sl~h-k zr4ua97aeiI7sHzC5z26ax#lI;@-O%hgZ>7P)``_@@3t8$CWOc4?hJCAWbrw+efxHhC4KQWJ#~cC>F`&)LZZlNGkfs(y zvMOVotgb1HqKUjK?zcLm7n;i*5?B>gO~e9wsiQ4_QcA z^(yGh^DI*0PGNCkxMQf0JJj`sm;rkVV|_V=Nqr7bCz|=uHaf2gkP$N@n@p(wA?wWB z9OQ(pnKv4}Ao)xfY6^lu&R(sXXPBE1WVPvJN#45GdsPCzDQ>*CV9=s z(VWaBu~?%y5zwSwtmUkt3G!4$?ayrjp75a{2rv~|1O<__gG5OksF2H#tyqIm^_U~M ziHpm*eBu1H%kx-|xsyq(n1PjS%$jI1(OV_48xV0~)U!;L#Nr}LG;UxgOBWp3q$8VD zGHx1bTecG=P6Sx&)mN1;A?X@;r)8BO8&}(s$%R!kiP~ob-wMRW#D?9C2}?@KK<;B~ zkko5`Q~TT6XEX;=GtfYl-7YtYv=>B*X*RGl#4>+KXo!SUKm2Lx+3~YO()3TRU5}`d>Y9-Ef-=$s1o-ntqK1vJMIly#yw1zyd@}?)b7}(6YdIy75^h6E{@wi z6vyp)isN=YMOx~#M|qO%=jr$wCNYryyR#00)mj#5zhxDRu1h5hA4^)vC$;~ufjp9C z$qS-oRo04P&!viGj3P_90?+>0251jy$qTM4Mb{PD*jmaJ%+>#719{h#g6m4jb*1dK zQ>B#D{>>qCT`9P(lw4QJZh|Z2r1q~4nd=H|YOO#@t}A6X!Ier<`!8EP@48ZST`9S) zl-&eZCX(8}*klFQ6?7e2T5??}y9utiL&SBf;JSr{)ggA>D!a+9OeVGewAE=3YbC7c zx>a)BD!a+X`womi%evsYLeIvQwB)){cB7iWX!vhiz2Lf1bX_UAu80S3JyT2|suT23 zn#XKR+tY=7ApT)??$Zgl8_w#$1$*`9#$=≶CS=Q#Vtj$pt=DYgm zYw%$uS|4mh-SF-Y@BT?AiFZ_!#>E|vJvQ_b?^<4+O&XV>6Ypt#_LoGm<80EHA)_mF z;k#kQc<0=kN#k|4Zl0$Xa*bKXDBijCdeV4_wBnLDn7FVvg4OIuyh ztH=~h>_ipsv>+Lc9zPiiI-D zN{Tu3g{1K&dv`XVm(aqycqh7Iyqkge@v}j%(*#)$?`Ib=GBR;et|AuKpO9ig5;qve-W=9o^8BiDrx*8C2i#8%CG#7_>ARCxQP$UMA1gD zP;cU69CV=tbfExCNatqMz0F-n+y>ND4v;hq78V13tTZL`lp|BkdMmin?euz`UNF<_ z^zlT@*i5~xk)y)qa&HTjG@^Q+y4yikBW#*K8#%U&k6wCMy@Kgx*i+SIps4oDweUUC zoSF~Tq1>zA5c)x@NDJA?#rn;t8C+a;r9Koxk6l94r ziAy7hg$({>6KnAa**4R+r~y>fXc&pF-wM#3Ig}(muESGUK<^Y!%D9k3t;~WfeQb=C zJ}R{IAvk3!g={10HNzGqkv_Cji8W}D!xsINtQ%E(l57)f#I=cCMUaE!j2M)tn})5} zI0hHPX4t;_L$vNeDF}EV<1^0c*=jHk7pdn%hJlOW%B?O@)#7N?8~8|ZoKyk|3Kpj? z=@!2_6R?D>0yNqQIhY|J3yq&LHo(WFARnN;E_HhN#IqGlVdohJ(~C4Q9hEc+0u3>7PFc%zWeFsfCwqwjqhX5j-Sg)6MNQ*`dw_bPw9Hwv?#Bxhc}}(K8`hk zO|@7u-HgIE;Xi4*;WR#4Mt$|}`T&W?8cuB?=hGzq=Cufk6axj5*=XhGq#s6W++{mF zwGwnYgUwph4QAE?q|8Im79LW>gaNB{;d!tY;K7en6aX3wi6cZDSr~eE3PbljZ8@OR zM6RGW4wd0i^yy)C^eU5thD_#|<3oVMg^2Y}2+W=|Pvi z(vZZ@Lo64yAsR>9yc)TwPlNC%4Wa>#2y>NQZUd>PFwqRhWj$U?WWi{RGp<)Ag`z!1 ziP0p_j57++n$vtYrObAj4e4W{a8_R0B3ZD=Sd~g&7@>V_cPMYG^S_^SfYC>%M>f7 ziW3-{DICL+@T{6nIb+$bC6fs?SlO}XUhWf+%G8Rk;QCKflvw&$IiThz&umPthUgt9 zsDb$<(xlkC$*El9g`9(bUFIyGn|>>on+V1%sajE$?|Olu)eln%Ey5U|vfWcs;i$DZ z2DEUlE(a@@-=NW9Ew#2}fg4MbDhpJWpIlqu&&8fi%kVl*%8}|P@x>X6f~rz0zBFm^ z140*USp34EH-P_YBt{ezJTqMAG$*B@5(+0!$#F9Zm5-O-fPNNC!M>!oO4R3j5x%2^ zZ2`Uh%atou_(^R)>?+i|~cCmnmBMI5IDuN#>_kg9XTg)yvb|Wo>{u zrBWBO-1IPUiDx-atXdtNh#r>YLz~qdTmibnQ_E^6jCK|ebNm^F*|Jj(jY?rC=>$($Z@H# zvb0K=%%xVV9ng1<&caMQE01Q&*0cD?VelQoGi9V7DG`ASK*O{=cNE8Mh-oh+`caX( zjVlf!6`f%zrPNUC^Rm+h*pB5USJ!ep*-~>Tg40|NjZWka71^>tS>rK|t@Au5YZbU@ znX?17?1(edigyyv2#kh2rZK`WlJW(bU9bsKQ?wN0E0T`*d3RpOAb4>(>aIuaey~dK z;Ceh6WT*}If<9Xr;RT6DdqEo8@PgF7E^@(9;;}30b~Xn6j_q^OIJy&0OfSif9^$zf z!O15}oj{!xOj&3_nqoz=pO7WFxEzv2xw$rLvm}%+oy5~9ZGa@s+d*2c&>Zp!NJ59y zK7AVB5vDg71ehIC4@Mf$?ECR9WWL_6h4jE=3RruNPz}OdYah2>&FVgbO__d2%P(my z$w_HIZ)@qbi>@kLK8Gcfa;V=*v<8Vt+0zn&0FZd zGtqi2#0#QV0<`OM(@R%ZW-qUxXHPFJP6spdOKZW*?BdcY5PJUyc({Uijr{ybl@7K8 zBWs}-&!ZI(_h<#0J&Y5y;ovytXBU?7Fi78SbR)4fgo22~3O21mXOOUCBDcLSoLVU4^<=Nr(9LNZM3SyKP0gMvE`Dg6P z#VAMb8kfF1t~pZSx6-b7P7f4#e3nW9q4J1Rs4MePJ<1}|%o?Tl?2a;Au1(KoS z1(MeA0?ANt1IbW8AZpfv+h(!I?pzrfW8!1Td$2f@?2w8mntS*{J?vg8ZquMiFqTIw ze`$qO(`2Z4O_SE}nkGZRZJG>)VS!N7s5#e4hG9?&BLdMXLjzG7Lj%zY!vfI?2*jFt ztR3} zXl#qS_~<0S9_)tMx#Fdz-bntGZ-xiHsEd~?+cVFl&G~cJ5j9>&n}`~_;v?G;rr-8W z-K{?Y#fu*GX%V8dVo2hOZwz&-O8kxvdv*PZ!x(>v7QKhv`wOEC@fj3tdNb(n;hU^F ziOUQ4)ggbK3Lmz?i~x zZ%~B~?Gm5Ddjpy|=uHBq5qhaA-VZcBpo-l~eAIdmpr+Eshotyby~IyiA&K`GjgKgx zALk`LE?OnHh!stJ`q@hwcgV4fOLQ&^{UTU=M0WjEeD9Thg-m__W2*kgbcs*Pj!IvH z!PHLghU6-rG=7sDyYCA=pu*3)>*6~WOKp5r3tu7qEuwmGm-v{~z2LTdQ?^h3Vn1p8 zHr4m&@(^$$p2KUgP+-pne}%l^&>Cwab-%K58ntwZ|P$o{4QDE_a)xD z8s7cU2JF92_V>QT`!4f778bv!E%+|$!hA+Bl-{Uh-5E2MnxeTg0&~YwQ#5zRZ1mkW zELbH=&j{}S&hzU239>963aM9-SI2gX%(7kOFQVtUJG5x(Fe(Q7D&Bo*DgG)Z=PT&* z1OL8weD2j?G8lU`xJru>K78F6;DY@}4gcYBqho$(^uX(WM2H^&JW9f&Bz$SV1$orj z9(A@yo$b*mcr*$gje_(ji1)r7HJhF9BpISTSGrMzUw$We_)m+|2rHV4R@gRyw%E>1 zMn&2zLuR0?J8h|NMrw@|YKrttmR-F2Vsn5ue?OpCEasA(bfd66@~jF+V-J=$&889K zhKBy1-Ixf?T{=V~Gk0k%jciLvYwH_hc3*eeUf+y1*J=;y##EPP93@L@f0>B!%-r2q z{l<*%M%dnmrkPymZWQPCHEEhME<6~{FR}LMT-s$5&OLL#eNSCYGmSsIy9np)#$UTW zsNtpU+hBab8bQB<#D@KTmky+k_f~K4A=0=#71FM%SKSzFA?;y5!tv7A zqh>F;(;HlmkFR$lfo*Q$Y-@3(baDrkbURzsHjd9m;V6FY6s`9mzX9a|z93ZPmO5&k zpA1LD(b+gaR3EBbo$LR>1GP-%H3)c|@_ zq(%pQ!z-ghhkgsM_@G>zZRt=c&`>GFP-(=Y@&XiUWy>pt_*5{|RN`33g6ow6jaLdW zR2p%qa2T!=AgclNs7OucO6e%C^m)lPl>rr%K@^pRNUzip%6XTHYb@urSC*Re%0e`i zJgM2GQlRllA;v3>Xew?R97Sy9N;z*@&>)6N8)_;Mi_?P2`9dl{sCfYb(Nw}uk4k}l zfHsxVF;whlJ}W?ghDxhxuQZ~);s9|Jnv*L98Y+c&0g|Dn66DSplu5?fYlcdzd0xR# zQ;F)F-Y(OgGrip)zE?2RR9sylD^~zLuSm`HiV#gDyz;0NXs8ros5GLf*ee{I?4(_R zhDsrZN+X&|kXKwP1sW=a7%GiODmv?$4y|_rG@zm~h@!F(kIHF5rDj^tAU>68LDIpg ztp#V`7|`}gs>v2KIv*>$rKgT ztNfq~=l);s!XGxLOiN0uO?Go0r9D9uXqe%^>lKjK;xA{43$Q-R|0Vc zjy{vw3Q()*05zhiXxgDOF&(cIXuML0@k%2emFJaEGfS#ydisL|J{1r_CC?DWdBK)f z7AU>45J_bt9+lmKT1~g0h4@su1+|WyD+~0!f|^LXrQ-)^=Sr*TTv>=uC3AJmoc3It z(8+qzLD?@CVm1$=d|7lQGih0hKz%w=q&QtI4%DKnn50>XKz%w=^ymokyepd7*`fpL z(~+V_N03K{zfqvNzeNYsrz1s=jv$Y&Vw%;W1M1U}qDM!NM_0^j3ZOn6DSC7Sd34qc zWO=7{^)BSok*!BpF@w~y1?qc8ik^1_d35-L5*bYa)TbjwkB%UZuAJEvKz%w=^ymok z=qhGAZ3kOQ(UGD@N03KX${ZCyeeX!oqa(eG>;M@NuHS25dcJF-fOjubsQf;_rXW>Wz5y(2}B zjv$W?e=a6tQ~>qqNYSGs$fK)dt`$IiI#TrL2=eGknN0!Irz1s=jv$Y&oY@pWeL7O~ z=m_%YDw%5qP@j$zJvxFsI{bl~jHUqU(~+V_N03KX&TI;xJ{>7~bOd>HmCUsQs82_V z9vwj*T`99EfckW#=+P16(Umis0;o?%iXI(79-Z}8&>FKqeL7O~=m_%YN|{Xo)Tbjw zkB%UZuAJEvKz%w=^ymok=q57P3ZOn6DSC7Sd32@BrU2^Gk)lUOkVj{|>$RFfz3g=% zpN?!jx{1uS0;umDDSF-!1iIR(C_bd#nB!GC{%~W`_y@f3$7}Z!_4bXZi-V9N=TBSK zcH+2l|tstR1Qy(SMC{D9vE}{{VLXpJ4z1 From 24ca22b75283929a98e2ca334c7b2a70bf9b1a2c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 24 Sep 2025 10:33:18 -0300 Subject: [PATCH 28/52] SIENTIAPDE-1222 Update .gitignore to include 'tmp/' directory and ensure '.env' is listed - Added 'tmp/' to the .gitignore file to prevent temporary files from being tracked. - Confirmed that '.env' is included to avoid committing sensitive environment variables. --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b14c9ac..cfaf446 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,6 @@ git_key* git_log -.env \ No newline at end of file +.env + +tmp/ \ No newline at end of file From 1aede51dc1910b17f5d8d6f5dc406ed9f1d8095c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 24 Sep 2025 17:04:40 -0300 Subject: [PATCH 29/52] SIENTIAPDE-1231 SIENTIAPDE-1222 Enhance MLFlow and MLFlowRepository with model configuration support - Introduced `model_config` parameter in MLFlow methods to streamline model handling and configuration management. - Updated `retrain_model`, `transform`, and `predict` methods to accept `model_config` and `metadata` for improved flexibility and logging. - Added `detect_and_parse_datetime_index` method to handle datetime index parsing with enhanced error handling and logging. - Refactored model experiment creation to include transformation and prediction flavors, along with compression options. - Improved documentation and type hints across methods for better clarity and usability. --- laborious/activities/mlflow.py | 4 +- .../utils/repository/model_repository.py | 209 ++++++++++++++---- 2 files changed, 165 insertions(+), 48 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index bd06283..d6dc98d 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -225,6 +225,7 @@ class MLFlow(BaseActivity): metadata = input_data['metadata'] data = DataFrame(input_data['data']) model_name = input_data['model_name'] + model_config = input_data.get('model_config', {}) self.info(f'Retraining model {model_name}...', metadata) @@ -245,7 +246,8 @@ class MLFlow(BaseActivity): try: retrain_output, experiment = self.model_monitoring_repository.retrain_model( data=data, - model_name=model_name + model_name=model_name, + model_config=model_config ) return { diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 07f0fc5..7ce7b32 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -1,14 +1,17 @@ """ -Model Monitoring Repository +MLFlow Repository -This module contains the ModelMonitoringRepository class, -which is responsible for handling the communication with the Model Monitoring API. +This module contains the MLFlowRepository class, +which is responsible for handling the communication with MLFlow tracking server. -It includes the methods that are used to answer ModelMonitoringService -requests using the Model Monitoring API functions. - -By Monitoring we mean the evaluation of the performance of models, the generation of reports. +It includes methods for model management, caching, retraining, and serving operations +using MLFlow's tracking and model registry capabilities. +The repository provides comprehensive functionality for: +- Model loading and caching with retention policies +- Data transformation and prediction operations +- Model retraining workflows +- Production model updates and versioning """ from datetime import datetime, timedelta import traceback @@ -21,6 +24,8 @@ import lzma import gzip import pickle +from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + ARTIFACTS_PATH = "./tmp/artifacts" @@ -36,6 +41,7 @@ class MLFlowRepository(): self.client = mlflow.tracking.MlflowClient() self.model_cache = {} + self.logger = logger """ Functions related to get model registry parameters @@ -47,6 +53,7 @@ class MLFlowRepository(): Args: run_id (str): The run_id of the model. + prediction (bool): Whether to get prediction model URI (default: True) Returns: str: The model URI. @@ -80,7 +87,16 @@ class MLFlowRepository(): run_id = latest_versions[0].source.split("/") return run_id[2] - def get_experiment_by_run_id(self, run_id: str) -> dict: + def get_experiment_by_run_id(self, run_id: str) -> str: + """ + Get experiment name by run ID. + + Args: + run_id (str): The MLFlow run ID + + Returns: + str: The experiment name + """ # Get the run information using the run_id run = mlflow.get_run(run_id) @@ -183,6 +199,13 @@ class MLFlowRepository(): def dowload_artifacts(self, model_name: str, artifact_path: str = "data_model") -> str: """ Downloads artifacts from a specific MLFlow run. + + Args: + model_name (str): Name of the model + artifact_path (str): Path to the artifact within the run + + Returns: + str: Path to the downloaded artifacts """ run_id = self.get_model_run_id( model_name=model_name, stage="Production" @@ -202,10 +225,15 @@ class MLFlowRepository(): artifact_path: str | None = None): """ Downloads a predictive model from the MLflow Model Registry. + Args: model_name (str): The name of the model to download from the registry. + flavor (str): Model flavor ('pyfunc', 'sklearn', 'pytorch') + artifact_path (str | None): Path to compressed artifacts if model is compressed + Returns: mlflow.pyfunc.PyFuncModel: The loaded predictive model. + Notes: - The model is fetched from the "production" stage of the MLflow Model Registry. - Warnings during the model loading process are suppressed. @@ -236,13 +264,15 @@ class MLFlowRepository(): def load_transform_model(self, model_name: str, flavor: str, artifact_path: str | None = None): """ - Downloads the latest production version of a specified model. + Downloads the latest production version of a specified transformation model. This method retrieves the latest production model run ID for the given model name, constructs the model URI, and loads the model using MLflow. Args: model_name (str): The name of the model to download. + flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') + artifact_path (str | None): Path to compressed artifacts if model is compressed Returns: Any: The loaded model object, as returned by `mlflow.sklearn.load_model`. @@ -284,7 +314,8 @@ class MLFlowRepository(): Load model from pickle file trying different compression methods. Args: - pickle_path (str): Path to the pickle file + artifact_path (str): Path to the artifact directory + type (str): Type of model ('transformer' or 'prediction') Returns: Any: Loaded model object @@ -346,37 +377,80 @@ class MLFlowRepository(): Returns: dict: Model configuration with model and artifact paths """ + + if model_type not in ["predict", "transform"]: + raise ValueError( + "Invalid model_type. Use 'predict' or 'transform'.") + + if compressed: + target = "prediction_model" if model_type == "predict" else "data_model" + + artifact_path = self.dowload_artifacts( + model_name, target) + else: + artifact_path = None + if model_type == "predict": - if compressed: - artifact_path = self.dowload_artifacts( - model_name, "prediction_model") - else: - artifact_path = None model = self.load_predict_model(model_name, flavor, artifact_path) elif model_type == "transform": - if compressed: - self.logger.info( - f"Model {model_name} is compressed, downloading artifacts") - - artifact_path = self.dowload_artifacts( - model_name, "data_model") - - self.logger.info( - f"Artifacts downloaded at path {artifact_path}") - else: - artifact_path = None model = self.load_transform_model( model_name, flavor, artifact_path) - else: - raise ValueError( - "Invalid model_type. Use 'predict' or 'transform'.") return { "model": model, "artifact_path": artifact_path } + """ + Functions related to data format + """ + + def detect_and_parse_datetime_index(self, data: pd.DataFrame, metadata: dict) -> pd.DataFrame: + """ + Detect and parse datetime index from data. index must be a timestamp like column. + This function must detect the timestamp type (pandas Timestamp or datetime) and convert it to DATETIME_FORMAT_WITH_TZ. + If the index is a string, must be in format DATETIME_FORMAT_WITH_TZ. + If another type or format, must raise an error. + + Args: + data (pd.DataFrame): DataFrame with timestamp index + metadata (dict): Metadata for logging + + Returns: + pd.DataFrame: DataFrame with converted datetime index + """ + index = data.index + + # Get type of first element of index + index_type = type(index[0]) + + self.logger.custom_info(f"Index type: {index_type}", metadata) + + message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}" + + # Check if all in index are of the same type + if not all(isinstance(i, index_type) for i in index): + raise ValueError( + f"{message}") + + # Check type and converts to DATETIME_FORMAT_WITH_TZ + if index_type == str: + # Validate format of string and return error if not valid + try: + pd.to_datetime(data.index, format=DATETIME_FORMAT_WITH_TZ) + except ValueError: + raise ValueError( + f"{message}") + + elif index_type == datetime or index_type == pd.Timestamp: + data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) + else: + raise ValueError( + f"{message}") + + return data + """ Functions related to cache management of models """ @@ -445,7 +519,7 @@ class MLFlowRepository(): else: return cache['target'] - def handle_outdated_model(self, model_name: str, model_key: str) -> dict: + def handle_outdated_model(self, model_name: str, model_key: str) -> None: """ Clean up outdated cached model and its artifacts. @@ -454,7 +528,7 @@ class MLFlowRepository(): model_key (str): Cache key for the model Returns: - dict: Empty dictionary (cleanup operation) + None """ if self.logger: self.logger.debug( @@ -596,7 +670,9 @@ class MLFlowRepository(): Functions related to model retraining """ - def create_model_experiment(self, model_name: str, data: pd.DataFrame) -> tuple: + def create_model_experiment(self, model_name: str, data: pd.DataFrame, + transform_flavor: str = 'sklearn', predict_flavor: str = 'pyfunc', + compressed: bool = False) -> tuple: """ Create a new MLFlow experiment for model retraining. @@ -610,6 +686,9 @@ class MLFlowRepository(): Args: model_name (str): Name of the MLFlow model to retrain data (pd.DataFrame): Training data for model retraining + transform_flavor (str): Flavor for transformation model + predict_flavor (str): Flavor for prediction model + compressed (bool): Whether models are compressed Returns: tuple: (prediction_model, data_model, experiment) @@ -617,18 +696,15 @@ class MLFlowRepository(): - data_model: Fitted transformation model - experiment: MLFlow experiment name """ - # load predictor model - predictor_uri = f"models:/{model_name}/production" - # load transform model latest_production_id = self.get_model_run_id( model_name, stage="Production" ) - transform_uri = self.get_model_uri( - latest_production_id, prediction=False + data_model = self.download_model( + model_name, "transform", transform_flavor, compressed + ) + prediction_model = self.download_model( + model_name, "predict", predict_flavor, compressed ) - # load - data_model = mlflow.sklearn.load_model(transform_uri) - prediction_model = mlflow.sklearn.load_model(predictor_uri) data_model = data_model.fit(data) treated_data = data_model.predict(data) @@ -768,7 +844,8 @@ class MLFlowRepository(): Functions that provide the interface to model operations """ - def transform(self, model_name: str, data: pd.DataFrame, model_retention: int): + def transform(self, model_name: str, data: pd.DataFrame, model_retention: int, + model_config: dict, metadata: dict): """ Transform data using a cached transformation model. @@ -787,6 +864,8 @@ class MLFlowRepository(): model_name (str): The name of the MLFlow model to use for transformation. data (pd.DataFrame): The input data to be transformed by the model. model_retention (int): Cache retention time in minutes (0 = no caching). + model_config (dict): Model configuration parameters + metadata (dict): Metadata for logging Returns: dict: Response dictionary containing: @@ -800,13 +879,31 @@ class MLFlowRepository(): Exception: Any exception during model loading or transformation is caught and returned in the response structure rather than propagated. """ + self.logger.custom_debug( + f"Data received for model transformation: {data.to_csv()}", metadata) + + model_retention = model_config.get('retention_minutes', 0) + flavor = model_config.get('transform_flavor', 'sklearn') + compressed = model_config.get('is_compressed', False) + retention_target = model_config.get('retention_target', 'model') + transform_keyword = model_config.get( + 'transform_function_keyword', 'predict') try: + transformed_data = self.get_cached_transform( + model_name, data, model_retention, flavor, + compressed, retention_target, transform_keyword + ) + + self.logger.custom_debug( + f"Data received from model transformation: {transformed_data.to_csv()}", metadata) + + transformed_data = self.detect_and_parse_datetime_index( + transformed_data, metadata) return { 'success': True, - 'content': self.get_cached_transform( - model_name, data, model_retention).to_dict() + 'content': transformed_data.to_dict() } except Exception as e: @@ -818,7 +915,8 @@ class MLFlowRepository(): } } - def predict(self, model_name: str, data: pd.DataFrame, model_retention: int): + def predict(self, model_name: str, data: pd.DataFrame, model_retention: int, + model_config: dict, metadata: dict): """ Generate predictions using a cached prediction model. @@ -841,6 +939,8 @@ class MLFlowRepository(): model_name (str): The name of the MLFlow model to use for prediction. data (pd.DataFrame): The input data to make predictions on. model_retention (int): Cache retention time in minutes (0 = no caching). + model_config (dict): Model configuration parameters + metadata (dict): Metadata for logging Returns: dict: Response dictionary containing: @@ -856,15 +956,24 @@ class MLFlowRepository(): Exception: Any exception during model loading or prediction is caught and returned in the response structure rather than propagated. """ + + model_retention = model_config.get('retention_minutes', 0) + flavor = model_config.get('predict_flavor', 'pyfunc') + try: input_index = data.index start_time = datetime.now() + + self.logger.custom_debug( + f"Data received for model prediction: {data.to_csv()}", metadata) data = self.get_cached_predict( - model_name, data, model_retention) + model_name, data, model_retention, flavor) end_time = datetime.now() data = pd.DataFrame(data, columns=['prediction']) + self.logger.custom_debug( + f"Data received from model prediction: {data.to_csv()}", metadata) data.index = input_index data['response_time'] = (end_time - start_time).total_seconds() @@ -882,7 +991,8 @@ class MLFlowRepository(): } } - def retrain_model(self, data: pd.DataFrame, model_name: str) -> tuple: + def retrain_model(self, data: pd.DataFrame, model_name: str, + model_config: dict) -> tuple: """ Orchestrate the complete model retraining workflow. @@ -913,6 +1023,7 @@ class MLFlowRepository(): prediction models, including target variable. model_name (str): Name of the MLFlow model to retrain. Must exist in the MLFlow Model Registry in Production stage. + model_config (dict): Model configuration parameters Returns: tuple: Retraining operation results containing: @@ -924,8 +1035,12 @@ class MLFlowRepository(): ValueError: If experiment cannot be created or models cannot be loaded Exception: Any other exception during the retraining process """ + transform_flavor = model_config.get('transform_flavor', 'sklearn') + predict_flavor = model_config.get('predict_flavor', 'pyfunc') + compressed = model_config.get('is_compressed', False) + prediction_model, data_model, experiment = self.create_model_experiment( - model_name, data) + model_name, data, transform_flavor, predict_flavor, compressed) retrain_result = self.perform_model_retrain( prediction_model, data_model, experiment, model_name, data) return retrain_result From c6f004d20dcc46c125979f36ad5eca2edb148f1b Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 29 Sep 2025 17:34:47 -0300 Subject: [PATCH 30/52] SIENTIAPDE-1231 Update model retraining and reporting functionality - Changed the GITHUB_BRANCH value in values.yaml to reflect the latest adjustments for retraining the courier. - Enhanced the Gates class with a new method `format_retrain_report` to format retraining report data according to storage policies. - Refactored the MLFlow class to improve error handling during model retraining and return structured output. - Updated the model_repository to utilize the latest MLFlow API for retrieving model versions and improved logging. - Modified the minimal_retrain workflow to conditionally update the production model based on retraining success. --- clean_job.yaml | 45 ++++++++ laborious/activities/gates.py | 35 +++++- laborious/activities/mlflow.py | 41 +++---- .../utils/filters/conditional_filters.py | 4 + .../utils/repository/model_repository.py | 107 +++++++++++++----- laborious/workflows/minimal_retrain.py | 22 +++- values.yaml | 2 +- 7 files changed, 197 insertions(+), 59 deletions(-) create mode 100644 clean_job.yaml diff --git a/clean_job.yaml b/clean_job.yaml new file mode 100644 index 0000000..00422a1 --- /dev/null +++ b/clean_job.yaml @@ -0,0 +1,45 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: delete-old-rows + namespace: sientia +spec: + template: + spec: + containers: + - name: delete-old-rows + image: docker.io/bitnami/postgresql:16.2.0-debian-12-r10 + env: + - name: PGPASSWORD + value: "asidhsd@!#!@@!ASD!@#!ASDQ@#!FSDTRYJG#@@$#@%" + - name: PGUSER + value: temporal + - name: PGHOST + value: "paradedb-rw.paradedb.svc.cluster.local" + - name: PGDATABASE + value: "temporal_visibility" + command: + - "sh" + - "-c" + - | + # COMANDO CORRIGIDO - Excluir apenas workflows COMPLETED/FAILED antigos + # Preserva schedules (que ficam RUNNING) e workflows recentes + psql -h $PGHOST -U $PGUSER -d $PGDATABASE -c " + DELETE FROM public.executions_visibility + WHERE start_time < NOW() - INTERVAL '1 minutes' + AND status IN (2, 3, 4, 5, 7);" + + # Comando para executar VACUUM FULL após a exclusão + psql -h $PGHOST -U $PGUSER -d $PGDATABASE -c "VACUUM FULL public.executions_visibility;" + envFrom: + - secretRef: + name: postgres-credentials + restartPolicy: Never + backoffLimit: 0 + ttlSecondsAfterFinished: 3600 + +# kubectl apply -f clean_job.yaml -n sientia + +# kubectl create secret generic postgres-credentials --from-literal=postgres-password=sientia --from-literal=postgres-username=sientia -n sientia4 + +# drop database temporal; drop database temporal_visibility; create database temporal owner temporal; create database temporal_visibility owner temporal; \ No newline at end of file diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index 4a71455..832cfbb 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -207,13 +207,12 @@ class Gates(BaseActivity): filter_output = [] self.debug( - f"Input data: \n {create_sample_dict(data, max_items=5, max_depth=2)}", metadata) + f"Input data: \n {create_sample_dict(data, max_items=5, max_depth=5)}", metadata) self.debug(f"Filters: {filters}", metadata) comments = [] for fil, config in filters.items(): if fil not in mlflow_response_filter_functions: - self.error(f"Filter {fil} not found", metadata) continue try: if mlflow_response_filter_functions[fil](data, config): @@ -293,7 +292,7 @@ class Gates(BaseActivity): filter_output = [] self.debug(f"Input data:\n {data.head(5).to_string()}", metadata) - self.debug(f"Filters: \n {create_sample_dict(filters)}", metadata) + self.debug(f"Filters: \n {filters}", metadata) for fil, config in filters.items(): if fil not in mlflow_content_filter_functions: @@ -492,6 +491,36 @@ class Gates(BaseActivity): self.info(f"Default prediction formatted: {data.size} rows", metadata) return data.to_dict() + @activity.defn(name="format_retrain_report") + async def format_retrain_report(self, input_data: dict[str, Any]) -> dict[Any, Any]: + """ + Format retrain report data according to configured storage policies. + """ + metadata = input_data['metadata'] + self.info("Formatting retrain report...", metadata) + + experiment_response = input_data['experiment_response'] + update_report = input_data['update_report'] + model_id = input_data['model_id'] + model_name = input_data['model_name'] + + report = DataFrame({ + 'model_id': [model_id], + 'model_name': [model_name], + 'timestamp': [experiment_response['timestamp']], + 'status': [experiment_response['message']] + }) + + if experiment_response['success']: + # Retrain was successfull + report['version'] = update_report['version'] + report['mlflow_run_id'] = update_report['mlflow_run_id'] + report['mlflow_experiment_id'] = update_report['mlflow_experiment_id'] + + self.debug(f"Retrain report: {report.to_csv()}", metadata) + + return report.to_dict() + @activity.defn(name="get_last_timestamp") async def get_last_timestamp(self, input_data: dict[str, Any]) -> str: """ diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index d6dc98d..7ad795b 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -243,30 +243,29 @@ class MLFlow(BaseActivity): data = data.dropna() data.columns.name = None - try: - retrain_output, experiment = self.model_monitoring_repository.retrain_model( - data=data, - model_name=model_name, - model_config=model_config - ) + retrain_output = self.model_monitoring_repository.retrain_model( + data=data, + model_name=model_name, + model_config=model_config + ) - return { - 'status': retrain_output, - 'timestamp': timestamp, - 'experiment': experiment - } - except Exception as e: - trace = traceback.format_exc() + if not retrain_output['success']: + + trace = retrain_output['traceback'] self.send_notification( metadata=metadata, notification_id='RETRAIN_MODEL_ERROR', - message=f'Error retraining model {model_name}: {e}', + message=f'Error retraining model {model_name}: {retrain_output['message']}', block='retrain_model', level=NotificationLevel.ERROR, attachment_content=trace ) self.error(trace, metadata=metadata) - raise e + + return { + **retrain_output, + 'timestamp': timestamp + } @activity.defn(name="update_production_model") async def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]: @@ -307,11 +306,7 @@ class MLFlow(BaseActivity): """ metadata = input_data['metadata'] model_name = input_data['model_name'] - model_id = input_data['model_id'] experiment = input_data['experiment'] - timestamp = input_data['timestamp'] - status = input_data['status'] - self.info( f'Updating production model {model_name} from experiment {experiment}...', metadata) @@ -321,15 +316,9 @@ class MLFlow(BaseActivity): model_name=model_name ) - report = DataFrame([response]) - report['model_id'] = model_id - report['model_name'] = model_name - report['timestamp'] = timestamp - report['status'] = status - self.info( f'Production model {model_name} updated successfully', metadata) - return report.to_dict() + return response except Exception as e: trace = traceback.format_exc() diff --git a/laborious/utils/filters/conditional_filters.py b/laborious/utils/filters/conditional_filters.py index 2c51805..6caf1e5 100644 --- a/laborious/utils/filters/conditional_filters.py +++ b/laborious/utils/filters/conditional_filters.py @@ -20,6 +20,10 @@ def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool False if none of the specified variables contain null values. """ + + if data.empty: + return False + return not data[ data['variable'].isin(config['variables']) & data['value'].isna()].empty diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 7ce7b32..1816c80 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -76,16 +76,30 @@ class MLFlowRepository(): Returns: str: The run_id of the model. """ - latest_versions = self.client.get_latest_versions( - name=model_name, stages=[stage] + # Use search_registered_models instead of deprecated get_latest_versions + registered_models = self.client.search_registered_models( + filter_string=f"name='{model_name}'" ) - if not latest_versions: + + if not registered_models: + raise mlflow.exceptions.MlflowException( + f"Model '{model_name}' not found in the Model Registry." + ) + + # Get the latest version in the specified stage + model_versions = self.client.search_model_versions( + filter_string=f"name='{model_name}' and stage='{stage}'" + ) + + if not model_versions: raise mlflow.exceptions.MlflowException( f"Model '{model_name}' in stage '{stage}' not found in the Model Registry." ) - else: - run_id = latest_versions[0].source.split("/") - return run_id[2] + + # Sort by version number to get the latest + latest_version = max(model_versions, key=lambda v: int(v.version)) + run_id = latest_version.source.split("/") + return run_id[2] def get_experiment_by_run_id(self, run_id: str) -> str: """ @@ -482,7 +496,7 @@ class MLFlowRepository(): Returns: bool: True if cache is still valid, False if expired """ - current_time = self.now() + current_time = datetime.now() cache_time = cache['timestamp'] if current_time - cache_time >= timedelta(minutes=retention): return False @@ -601,7 +615,7 @@ class MLFlowRepository(): cache = { 'target': model_config_to_cache, 'config': config, - 'timestamp': self.now() + 'timestamp': datetime.now() } self.model_cache[model_key] = cache @@ -672,7 +686,7 @@ class MLFlowRepository(): def create_model_experiment(self, model_name: str, data: pd.DataFrame, transform_flavor: str = 'sklearn', predict_flavor: str = 'pyfunc', - compressed: bool = False) -> tuple: + compressed: bool = False, fit_config: dict = {}, target_name: str = None) -> tuple: """ Create a new MLFlow experiment for model retraining. @@ -701,18 +715,34 @@ class MLFlowRepository(): ) data_model = self.download_model( model_name, "transform", transform_flavor, compressed - ) + )['model'] prediction_model = self.download_model( model_name, "predict", predict_flavor, compressed - ) + )['model'] data_model = data_model.fit(data) treated_data = data_model.predict(data) - target_name = data_model.target_variable - y = data[target_name] - treated_data = pd.merge( - treated_data, y, left_index=True, right_index=True) - prediction_model = prediction_model.fit(treated_data) + if target_name is None: + target_name = data_model.target_variable + + if fit_config.get('y_type', 'series').lower() == 'series': + y = treated_data[target_name] + else: + y = treated_data[[target_name]] + + if not fit_config.get('split_fit_data', False): + # Merge treated data with target + treated_data = pd.merge( + treated_data, y, left_index=True, right_index=True) + + prediction_model = prediction_model.fit(treated_data) + else: + # Keep data separated + if fit_config.get('split_fit_first', 'x').lower() == 'x': + prediction_model = prediction_model.fit(treated_data, y) + else: + prediction_model = prediction_model.fit(y, treated_data) + experiment = self.get_experiment_by_run_id(latest_production_id) mlflow.set_experiment(experiment) @@ -781,7 +811,7 @@ class MLFlowRepository(): if path.exists(file_path): remove(file_path) - return "Model retrained successfully", experiment + return experiment def update_production_model_by_run_id(self, run_id: str, model_name: str) -> dict: """ @@ -844,7 +874,7 @@ class MLFlowRepository(): Functions that provide the interface to model operations """ - def transform(self, model_name: str, data: pd.DataFrame, model_retention: int, + def transform(self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict): """ Transform data using a cached transformation model. @@ -863,7 +893,6 @@ class MLFlowRepository(): Parameters: model_name (str): The name of the MLFlow model to use for transformation. data (pd.DataFrame): The input data to be transformed by the model. - model_retention (int): Cache retention time in minutes (0 = no caching). model_config (dict): Model configuration parameters metadata (dict): Metadata for logging @@ -915,7 +944,7 @@ class MLFlowRepository(): } } - def predict(self, model_name: str, data: pd.DataFrame, model_retention: int, + def predict(self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict): """ Generate predictions using a cached prediction model. @@ -992,7 +1021,7 @@ class MLFlowRepository(): } def retrain_model(self, data: pd.DataFrame, model_name: str, - model_config: dict) -> tuple: + model_config: dict, metadata: dict) -> tuple: """ Orchestrate the complete model retraining workflow. @@ -1035,15 +1064,41 @@ class MLFlowRepository(): ValueError: If experiment cannot be created or models cannot be loaded Exception: Any other exception during the retraining process """ + + self.logger.debug( + f"Data received for model retraining: {data.to_csv()}", metadata) + transform_flavor = model_config.get('transform_flavor', 'sklearn') predict_flavor = model_config.get('predict_flavor', 'pyfunc') compressed = model_config.get('is_compressed', False) + target_name = model_config.get('target', None) - prediction_model, data_model, experiment = self.create_model_experiment( - model_name, data, transform_flavor, predict_flavor, compressed) - retrain_result = self.perform_model_retrain( - prediction_model, data_model, experiment, model_name, data) - return retrain_result + fit_config = { + 'split_fit_data': model_config.get('split_fit_data', False), + 'split_fit_first': model_config.get('split_fit_first', 'x').lower(), + 'y_type': model_config.get('y_type', 'series').lower() + } + + try: + + prediction_model, data_model, experiment = self.create_model_experiment( + model_name, data, transform_flavor, predict_flavor, compressed, + fit_config, target_name) + experiment = self.perform_model_retrain( + prediction_model, data_model, experiment, model_name, data) + + return { + 'success': True, + 'experiment': experiment, + 'message': 'Model retrained successfully.' + } + except Exception as e: + return { + 'success': False, + 'experiment': None, + 'message': f'Error retraining model {model_name}: {e}', + 'traceback': traceback.format_exc() + } def update_production_model(self, experiment: str, model_name: str) -> dict: """ diff --git a/laborious/workflows/minimal_retrain.py b/laborious/workflows/minimal_retrain.py index 1893e25..98c1e16 100644 --- a/laborious/workflows/minimal_retrain.py +++ b/laborious/workflows/minimal_retrain.py @@ -91,13 +91,29 @@ class MinimalRetrain(): start_to_close_timeout=timedelta(seconds=60) ) + if experiment_response['success']: + + update_report = await workflow.execute_activity_method( + Activities.update_production_model, + { + **metadata, + 'model_name': model_name, + **experiment_response + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60) + ) + else: + update_report = {} + report = await workflow.execute_activity_method( - Activities.update_production_model, + Activities.format_retrain_report, { **metadata, - 'model_name': model_name, + 'experiment_response': experiment_response, 'model_id': input_data['model_id'], - **experiment_response + 'model_name': model_name, + 'update_report': update_report }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=60) diff --git a/values.yaml b/values.yaml index 4eb83d7..c441f65 100644 --- a/values.yaml +++ b/values.yaml @@ -151,7 +151,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git" - name: GITHUB_BRANCH - value: SIENTIAPDE-1222-ajustar-a-library-para-fazer-o-download-do-courier + value: SIENTIAPDE-1231-ajustar-o-retreino-do-courier-no-laborious - name: PYTHON_APP value: "laborious.worker.worker" From 9b71ad7556f08a706b91511b2ce7ca03f8652c48 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 2 Oct 2025 16:36:18 -0300 Subject: [PATCH 31/52] SIENTIAPDE-1231 Update model retraining and logging enhancements - Changed the GITHUB_BRANCH value in values.yaml to 'main' for consistency. - Refactored MLFlow class to improve timestamp handling and error messaging during model retraining. - Enhanced MLFlowRepository methods to include metadata logging and improved model version retrieval. - Updated minimal_retrain workflow to support extended timeout for activities and include model configuration in input data. --- laborious/activities/mlflow.py | 12 +- .../utils/repository/model_repository.py | 323 ++++++++++-------- laborious/worker/worker.py | 3 +- laborious/workflows/minimal_retrain.py | 14 +- values.yaml | 2 +- 5 files changed, 204 insertions(+), 150 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 7ad795b..3e4b492 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -240,13 +240,19 @@ class MLFlow(BaseActivity): data.sort_index(inplace=True) data.reset_index(inplace=True) - data = data.dropna() + data['timestamp'] = to_datetime( + data['timestamp'], format=DATETIME_FORMAT_WITH_TZ).dt.strftime(DATETIME_FORMAT) + data['timestamp'] = to_datetime( + data['timestamp'], format=DATETIME_FORMAT) + + # data = data.dropna() data.columns.name = None retrain_output = self.model_monitoring_repository.retrain_model( data=data, model_name=model_name, - model_config=model_config + model_config=model_config, + metadata=metadata ) if not retrain_output['success']: @@ -255,7 +261,7 @@ class MLFlow(BaseActivity): self.send_notification( metadata=metadata, notification_id='RETRAIN_MODEL_ERROR', - message=f'Error retraining model {model_name}: {retrain_output['message']}', + message=f"Error retraining model {model_name}: {retrain_output['message']}", block='retrain_model', level=NotificationLevel.ERROR, attachment_content=trace diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 1816c80..89922a8 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -23,6 +23,8 @@ from sientia_do.observability.logger import Logger import lzma import gzip import pickle +from numpy import ndarray +from typing import Any from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ @@ -86,18 +88,22 @@ class MLFlowRepository(): f"Model '{model_name}' not found in the Model Registry." ) - # Get the latest version in the specified stage + # Get all versions of the model and filter by stage model_versions = self.client.search_model_versions( - filter_string=f"name='{model_name}' and stage='{stage}'" + filter_string=f"name='{model_name}'" ) - if not model_versions: + # Filter versions by the desired stage using current_stage attribute + stage_versions = [ + mv for mv in model_versions if mv.current_stage == stage] + + if not stage_versions: raise mlflow.exceptions.MlflowException( f"Model '{model_name}' in stage '{stage}' not found in the Model Registry." ) # Sort by version number to get the latest - latest_version = max(model_versions, key=lambda v: int(v.version)) + latest_version = max(stage_versions, key=lambda v: int(v.version)) run_id = latest_version.source.split("/") return run_id[2] @@ -236,7 +242,7 @@ class MLFlowRepository(): ) def load_predict_model(self, model_name: str, flavor: str = 'pyfunc', - artifact_path: str | None = None): + artifact_path: str | None = None) -> Any: """ Downloads a predictive model from the MLflow Model Registry. @@ -257,12 +263,14 @@ class MLFlowRepository(): if artifact_path: self.logger.info( - f"Prediction model {model_name} is not compressed, loading from {artifact_path}") + f"Prediction model {model_name} is compressed, loading from {artifact_path}") model = self.load_model_with_compression( artifact_path, "prediction") else: + self.logger.info( + f"Prediction model {model_name} is not compressed, loading from {model_uri}") if flavor == 'pyfunc': model = mlflow.pyfunc.load_model(model_uri) elif flavor == 'sklearn': @@ -276,7 +284,7 @@ class MLFlowRepository(): return model def load_transform_model(self, model_name: str, flavor: str, - artifact_path: str | None = None): + artifact_path: str | None = None) -> Any: """ Downloads the latest production version of a specified transformation model. @@ -307,11 +315,13 @@ class MLFlowRepository(): if artifact_path: # Download model artifacts self.logger.info( - f"Data model {model_name} is not compressed, loading from {artifact_path}") + f"Data model {model_name} is compressed, loading from {artifact_path}") model = self.load_model_with_compression( artifact_path, "transformer") else: + self.logger.info( + f"Data model {model_name} is not compressed, loading from {model_uri}") if flavor == 'sklearn': model = mlflow.sklearn.load_model(model_uri) elif flavor == 'pyfunc': @@ -323,7 +333,7 @@ class MLFlowRepository(): "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") return model - def load_model_with_compression(self, artifact_path: str, type: str): + def load_model_with_compression(self, artifact_path: str, type: str) -> Any: """ Load model from pickle file trying different compression methods. @@ -378,7 +388,7 @@ class MLFlowRepository(): f"Could not load model from {pickle_path} - unknown or corrupted format") def download_model(self, model_name: str, model_type: str, flavor: str, - compressed: bool = False) -> dict: + download_artifacts: bool = False) -> Any: """ Download model based on type (predict or transform). @@ -386,17 +396,20 @@ class MLFlowRepository(): model_name (str): Name of the model to download model_type (str): Type of model ('predict' or 'transform') flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') - compressed (bool): Whether model is compressed + download_artifacts (bool): Whether to download artifacts Returns: - dict: Model configuration with model and artifact paths + Any: Model object """ + self.logger.info( + f"Downloading {model_type} model {model_name} with flavor {flavor} and download_artifacts {download_artifacts}") + if model_type not in ["predict", "transform"]: raise ValueError( "Invalid model_type. Use 'predict' or 'transform'.") - if compressed: + if download_artifacts: target = "prediction_model" if model_type == "predict" else "data_model" artifact_path = self.dowload_artifacts( @@ -411,10 +424,7 @@ class MLFlowRepository(): model = self.load_transform_model( model_name, flavor, artifact_path) - return { - "model": model, - "artifact_path": artifact_path - } + return model """ Functions related to data format @@ -469,22 +479,6 @@ class MLFlowRepository(): Functions related to cache management of models """ - def check_cache_config(self, cache: dict, new_config: dict) -> bool: - """ - Check if cache configuration matches new configuration. - - Args: - cache (dict): Cached model configuration - new_config (dict): New configuration to compare - - Returns: - bool: True if configurations match, False otherwise - """ - old_config = cache['config'] - if old_config != new_config: - return False - return True - def check_cache_retention(self, cache: dict, retention: int) -> bool: """ Check if cache is still valid based on retention time. @@ -503,7 +497,6 @@ class MLFlowRepository(): return True def handle_valid_model(self, model_name: str, model_type: str, - compressed: bool, retention_target: str, cache: dict) -> dict: """ Handle valid cached model by returning appropriate model configuration. @@ -522,16 +515,7 @@ class MLFlowRepository(): self.logger.debug( f"Model {model_name} is still valid, using cached version") - # If model is compressed and retention target is artifact, load the model from pkl - if compressed and retention_target == "artifact": - model = self.load_model_with_compression( - cache['target']['artifact_path'], model_type) - return { - 'model': model, - 'artifact_path': cache['target']['artifact_path'] - } - else: - return cache['target'] + return cache['target'] def handle_outdated_model(self, model_name: str, model_key: str) -> None: """ @@ -549,13 +533,9 @@ class MLFlowRepository(): f"Model {model_name} is outdated, downloading a new one") del self.model_cache[model_key]['target']['model'] - if path.exists(self.model_cache[model_key]['target']['artifact_path']): - remove(self.model_cache[model_key] - ['target']['artifact_path']) del self.model_cache[model_key] - def get_model(self, model_name: str, retention: int, model_type: str, flavor: str, - compressed: bool = False, retention_target: str = "model"): + def get_model(self, model_name: str, retention: int, model_type: str, flavor: str): """ Get model with caching support based on retention policy. @@ -564,7 +544,6 @@ class MLFlowRepository(): retention (int): Cache retention time in minutes (0 = no cache) model_type (str): Type of model ('predict' or 'transform') flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') - compressed (bool): Whether model is compressed retention_target (str): What to cache ('model' or 'artifact') Returns: @@ -572,58 +551,42 @@ class MLFlowRepository(): """ # Retention is 0, download a new model if retention <= 0: - model_config = self.download_model( - model_name, model_type, flavor, compressed) - return model_config + return self.download_model( + model_name=model_name, model_type=model_type, flavor=flavor, download_artifacts=False) model_key = f'{model_name}_{model_type}' - config = { - 'compressed': compressed, - 'retention_target': retention_target - } if model_key in self.model_cache: cache = self.model_cache[model_key] # Check if config has changed or is outdated - if self.check_cache_config(cache, config) or self.check_cache_retention(cache, retention): + if self.check_cache_retention(cache, retention): return self.handle_valid_model( - model_name, model_type, compressed, - retention_target, cache) + model_name=model_name, model_type=model_type, cache=cache) else: # Model is outdated, delete old model files self.handle_outdated_model(model_name, model_key) else: if self.logger: self.logger.debug( - f"Model {model_name} is not in the cache, downloading a new one") + f"Model {model_name} is not in {model_type} cache, downloading a new one") # Donwload new model - model_config = self.download_model(model_name, model_type, flavor, - compressed) - # If model is compressed and retention target is artifact, - # dont save the model in the cache - if compressed and retention_target == "artifact": - model_config_to_cache = { - 'artifact_path': model_config['artifact_path'], - 'model': None - } - else: - model_config_to_cache = { - **model_config - } + model = self.download_model( + model_name=model_name, model_type=model_type, flavor=flavor, + download_artifacts=False) + cache = { - 'target': model_config_to_cache, - 'config': config, + 'target': model, 'timestamp': datetime.now() } self.model_cache[model_key] = cache - return model_config + return model - def get_cached_transform(self, model_name: str, data: pd.DataFrame, retention: int, flavor: str, - compressed: bool = False, retention_target: str = "model", keyword: str = "predict") -> pd.DataFrame: + def get_cached_transform(self, model_name: str, data: pd.DataFrame, + retention: int, flavor: str) -> pd.DataFrame: """ Get transformed data using cached transform model. @@ -632,35 +595,18 @@ class MLFlowRepository(): data (pd.DataFrame): Data to transform retention (int): Cache retention time in minutes flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') - compressed (bool): Whether model is compressed - retention_target (str): What to cache ('model' or 'artifact') - keyword (str): Method name to call on model (default: 'predict') Returns: pd.DataFrame: Transformed data """ - model_config = self.get_model(model_name, retention, - "transform", flavor, compressed, retention_target) - model = model_config['model'] + model = self.get_model( + model_name=model_name, retention=retention, + model_type="transform", flavor=flavor) - # Use getattr to dynamically call the method specified by keyword - method = getattr(model, keyword) - transformed_data = method(data) - - # If model is compressed and retention target is artifact, - # delete the model after the prediction - if compressed and retention_target == "artifact": - del model - - # If retention is 0, delete the artifacts after the prediction - if retention == 0 and model_config['artifact_path'] is not None: - if path.exists(model_config['artifact_path']): - remove(model_config['artifact_path']) - - return transformed_data + return model.predict(data) def get_cached_predict(self, model_name: str, data: pd.DataFrame, retention: int, flavor: str, - compressed: bool = False, retention_target: str = "model") -> pd.DataFrame: + compressed: bool = False, retention_target: str = "model") -> pd.DataFrame | ndarray: """ Get predictions using cached prediction model. @@ -675,9 +621,10 @@ class MLFlowRepository(): Returns: pd.DataFrame: Model predictions """ - model_config = self.get_model(model_name, retention, "predict", - flavor, compressed, retention_target) - model = model_config['model'] + model = self.get_model( + model_name=model_name, retention=retention, + model_type="predict", flavor=flavor) + return model.predict(data) """ @@ -686,7 +633,8 @@ class MLFlowRepository(): def create_model_experiment(self, model_name: str, data: pd.DataFrame, transform_flavor: str = 'sklearn', predict_flavor: str = 'pyfunc', - compressed: bool = False, fit_config: dict = {}, target_name: str = None) -> tuple: + fit_config: dict = {}, target_name: str = None, + metadata: dict = {}) -> tuple: """ Create a new MLFlow experiment for model retraining. @@ -702,7 +650,9 @@ class MLFlowRepository(): data (pd.DataFrame): Training data for model retraining transform_flavor (str): Flavor for transformation model predict_flavor (str): Flavor for prediction model - compressed (bool): Whether models are compressed + fit_config (dict): Fit configuration + target_name (str): Target name + metadata (dict): Metadata for logging Returns: tuple: (prediction_model, data_model, experiment) @@ -710,42 +660,94 @@ class MLFlowRepository(): - data_model: Fitted transformation model - experiment: MLFlow experiment name """ + self.logger.custom_info( + f"Starting model experiment creation for {model_name}", metadata) + self.logger.custom_debug( + f"Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, fit_config: {fit_config}, target_name: {target_name}", metadata) + latest_production_id = self.get_model_run_id( model_name, stage="Production" ) + + self.logger.custom_info( + f"Retrieved latest production run ID: {latest_production_id}", metadata) + self.logger.custom_info( + f"Loading transformation model for {model_name}", metadata) + data_model = self.download_model( - model_name, "transform", transform_flavor, compressed - )['model'] + model_name=model_name, model_type="transform", flavor=transform_flavor, + download_artifacts=(transform_flavor == 'pyfunc') + ) + + self.logger.custom_info( + f"Loading prediction model for {model_name}", metadata) + prediction_model = self.download_model( - model_name, "predict", predict_flavor, compressed - )['model'] + model_name=model_name, model_type="predict", flavor=predict_flavor, + download_artifacts=(predict_flavor == 'pyfunc') + ) + + self.logger.custom_debug( + f"Fitting transformation model with training data (shape: {data.shape})", metadata) + data_model = data_model.fit(data) + + self.logger.custom_debug( + f"Applying transformation to training data", metadata) + treated_data = data_model.predict(data) + self.logger.custom_debug( + f"Transformed data shape: {treated_data.shape}", metadata) + if target_name is None: target_name = data_model.target_variable + self.logger.custom_debug( + f"Using target variable from data model: {target_name}", metadata) + else: + self.logger.custom_debug( + f"Using provided target variable: {target_name}", metadata) if fit_config.get('y_type', 'series').lower() == 'series': - y = treated_data[target_name] + y = data[target_name] + self.logger.custom_debug( + f"Extracting target as series, shape: {y.shape}", metadata) else: - y = treated_data[[target_name]] + y = data[[target_name]] + self.logger.custom_debug( + f"Extracting target as dataframe, shape: {y.shape}", metadata) if not fit_config.get('split_fit_data', False): # Merge treated data with target + self.logger.custom_debug( + "Merging treated data with target for combined fit", metadata) treated_data = pd.merge( treated_data, y, left_index=True, right_index=True) + self.logger.custom_debug( + f"Combined data shape for prediction model fit: {treated_data.shape}", metadata) prediction_model = prediction_model.fit(treated_data) + self.logger.custom_debug( + "Prediction model fitted with combined data", metadata) else: # Keep data separated - if fit_config.get('split_fit_first', 'x').lower() == 'x': + fit_order = fit_config.get('split_fit_first', 'x').lower() + self.logger.custom_debug( + f"Fitting prediction model with separated data, order: {fit_order}", metadata) + if fit_order == 'x': prediction_model = prediction_model.fit(treated_data, y) + self.logger.custom_debug( + "Prediction model fitted with X, y order", metadata) else: prediction_model = prediction_model.fit(y, treated_data) + self.logger.custom_debug( + "Prediction model fitted with y, X order", metadata) experiment = self.get_experiment_by_run_id(latest_production_id) mlflow.set_experiment(experiment) + self.logger.custom_info( + f"Model experiment creation completed successfully for {model_name}", metadata) return prediction_model, data_model, experiment def perform_model_retrain(self, @@ -753,7 +755,8 @@ class MLFlowRepository(): data_model, experiment: str, model_name: str, - data: pd.DataFrame): + data: pd.DataFrame, + metadata: dict = {}): """ Execute the complete model retraining process in MLFlow. @@ -770,12 +773,16 @@ class MLFlowRepository(): experiment (str): MLFlow experiment name for the retraining model_name (str): Name of the model being retrained data (pd.DataFrame): Training data used for retraining + metadata (dict): Metadata for logging Returns: tuple: (status_message, experiment_name) - status_message (str): Success confirmation message - experiment_name (str): Name of the experiment """ + self.logger.custom_info( + f"Starting model retraining process for {model_name} in experiment {experiment}", metadata) + pred_model_atributes = vars(prediction_model) # load class attributes data_model_atributes = vars(data_model) # load class attributes experiment_description = f"Retrain model {model_name} with new data" @@ -813,7 +820,7 @@ class MLFlowRepository(): return experiment - def update_production_model_by_run_id(self, run_id: str, model_name: str) -> dict: + def update_production_model_by_run_id(self, run_id: str, model_name: str, metadata: dict = {}) -> dict: """ Update production model with a specific MLFlow run. @@ -824,6 +831,7 @@ class MLFlowRepository(): Args: run_id (str): MLFlow run ID containing the model to promote model_name (str): Name of the MLFlow model + metadata (dict): Metadata for logging Returns: dict: Model update metadata containing: @@ -837,6 +845,9 @@ class MLFlowRepository(): 3. Transitions the model to 'Production' stage 4. Archives existing production versions """ + self.logger.custom_info( + f"Starting production model update for {model_name} with run ID: {run_id}", metadata) + # Registrar o modelo # Aqui estamos assumindo que você já tem um modelo salvo, caso contrário você precisará treiná-lo e salvá-lo primeiro. # Se o modelo já está registrado, você pode usar o método register_model() ou pyfunc.load_model() para isso. @@ -909,23 +920,24 @@ class MLFlowRepository(): and returned in the response structure rather than propagated. """ self.logger.custom_debug( - f"Data received for model transformation: {data.to_csv()}", metadata) + f"Data received for model transformation: {data.head(5).to_csv()}", metadata) + + data.to_csv( + f"tmp/data_{model_name}.csv", index=True) model_retention = model_config.get('retention_minutes', 0) flavor = model_config.get('transform_flavor', 'sklearn') - compressed = model_config.get('is_compressed', False) - retention_target = model_config.get('retention_target', 'model') - transform_keyword = model_config.get( - 'transform_function_keyword', 'predict') try: transformed_data = self.get_cached_transform( - model_name, data, model_retention, flavor, - compressed, retention_target, transform_keyword + model_name, data, model_retention, flavor ) self.logger.custom_debug( - f"Data received from model transformation: {transformed_data.to_csv()}", metadata) + f"Data received from model transformation: {transformed_data.head(5).to_csv()}", metadata) + + transformed_data.to_csv( + f"tmp/transformed_data_{model_name}.csv", index=True) transformed_data = self.detect_and_parse_datetime_index( transformed_data, metadata) @@ -995,14 +1007,29 @@ class MLFlowRepository(): start_time = datetime.now() self.logger.custom_debug( - f"Data received for model prediction: {data.to_csv()}", metadata) + f"Data received for model prediction: {data.head(5).to_csv()}", metadata) + + data.to_csv( + f"tmp/treated_data_{model_name}.csv", index=True) + data = self.get_cached_predict( model_name, data, model_retention, flavor) end_time = datetime.now() - data = pd.DataFrame(data, columns=['prediction']) - self.logger.custom_debug( - f"Data received from model prediction: {data.to_csv()}", metadata) + + if isinstance(data, pd.DataFrame): + self.logger.custom_debug( + f"Data received from model prediction: {data.head(5).to_csv()}", metadata) + + data.to_csv( + f"tmp/predicted_data_{model_name}.csv", index=True) + data.columns = ['prediction'] + + else: + data = pd.DataFrame(data, columns=['prediction']) + data.to_csv( + f"tmp/predicted_data_{model_name}.csv", index=True) + data.index = input_index data['response_time'] = (end_time - start_time).total_seconds() @@ -1053,6 +1080,7 @@ class MLFlowRepository(): model_name (str): Name of the MLFlow model to retrain. Must exist in the MLFlow Model Registry in Production stage. model_config (dict): Model configuration parameters + metadata (dict): Metadata for logging Returns: tuple: Retraining operation results containing: @@ -1065,13 +1093,15 @@ class MLFlowRepository(): Exception: Any other exception during the retraining process """ - self.logger.debug( + self.logger.custom_info( + f"Starting model retraining workflow for {model_name}", metadata) + self.logger.custom_debug( f"Data received for model retraining: {data.to_csv()}", metadata) + target_name = model_config.get('target', None) + transform_flavor = model_config.get('transform_flavor', 'sklearn') predict_flavor = model_config.get('predict_flavor', 'pyfunc') - compressed = model_config.get('is_compressed', False) - target_name = model_config.get('target', None) fit_config = { 'split_fit_data': model_config.get('split_fit_data', False), @@ -1079,13 +1109,24 @@ class MLFlowRepository(): 'y_type': model_config.get('y_type', 'series').lower() } - try: + self.logger.custom_debug( + f"Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, fit_config: {fit_config}, target_name: {target_name}", metadata) + try: + self.logger.custom_info( + "Creating model experiment environment", metadata) prediction_model, data_model, experiment = self.create_model_experiment( - model_name, data, transform_flavor, predict_flavor, compressed, - fit_config, target_name) + model_name=model_name, data=data, transform_flavor=transform_flavor, + predict_flavor=predict_flavor, fit_config=fit_config, target_name=target_name, + metadata=metadata) + self.logger.custom_info( + f"Model experiment created successfully: {experiment}", metadata) + + self.logger.custom_info("Performing model retraining", metadata) experiment = self.perform_model_retrain( - prediction_model, data_model, experiment, model_name, data) + prediction_model, data_model, experiment, model_name, data, metadata) + self.logger.custom_info( + f"Model retraining completed successfully for experiment: {experiment}", metadata) return { 'success': True, @@ -1093,14 +1134,16 @@ class MLFlowRepository(): 'message': 'Model retrained successfully.' } except Exception as e: + error_msg = f'Error retraining model {model_name}: {e}' + self.logger.custom_info(error_msg, metadata) return { 'success': False, 'experiment': None, - 'message': f'Error retraining model {model_name}: {e}', + 'message': error_msg, 'traceback': traceback.format_exc() } - def update_production_model(self, experiment: str, model_name: str) -> dict: + def update_production_model(self, experiment: str, model_name: str, metadata: dict = {}) -> dict: """ Update production model using the latest retraining run. @@ -1126,6 +1169,7 @@ class MLFlowRepository(): Must be a valid experiment that exists in MLFlow. model_name (str): Name of the MLFlow model to update. Must exist in the MLFlow Model Registry. + metadata (dict): Metadata for logging Returns: dict: Complete model update metadata containing: @@ -1146,8 +1190,9 @@ class MLFlowRepository(): """ experiment_id = self.get_experiment(experiment) run_id = self.get_experiment_last_run(experiment_id) - metadata = self.update_production_model_by_run_id(run_id, model_name) + metadata_result = self.update_production_model_by_run_id( + run_id, model_name, metadata) - metadata['mlflow_experiment_id'] = experiment_id + metadata_result['mlflow_experiment_id'] = experiment_id - return metadata + return metadata_result diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 567a257..81c716b 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -142,7 +142,8 @@ async def main(): activities.load_custom_query, activities.retrain_model, activities.update_production_model, - activities.export_data_to_postgres + activities.format_retrain_report, + activities.export_data_to_postgres, ], max_concurrent_workflow_tasks=50, max_concurrent_activities=50, diff --git a/laborious/workflows/minimal_retrain.py b/laborious/workflows/minimal_retrain.py index 98c1e16..607f3a5 100644 --- a/laborious/workflows/minimal_retrain.py +++ b/laborious/workflows/minimal_retrain.py @@ -68,6 +68,7 @@ class MinimalRetrain(): } model_name = input_data['model_name'] + model_config = input_data.get('model_config', {}) data = await workflow.execute_local_activity_method( Activities.load_custom_query, @@ -77,7 +78,7 @@ class MinimalRetrain(): 'datetime_columns': input_data.get('datetime_columns', []) }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=600) ) experiment_response = await workflow.execute_activity_method( @@ -85,10 +86,11 @@ class MinimalRetrain(): { **metadata, 'data': data, - 'model_name': model_name + 'model_name': model_name, + 'model_config': model_config }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=600) ) if experiment_response['success']: @@ -101,7 +103,7 @@ class MinimalRetrain(): **experiment_response }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=600) ) else: update_report = {} @@ -116,7 +118,7 @@ class MinimalRetrain(): 'update_report': update_report }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=600) ) await workflow.execute_activity_method( @@ -128,5 +130,5 @@ class MinimalRetrain(): 'table_name': input_data['table_name'] }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=600) ) diff --git a/values.yaml b/values.yaml index c441f65..ef86d89 100644 --- a/values.yaml +++ b/values.yaml @@ -151,7 +151,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git" - name: GITHUB_BRANCH - value: SIENTIAPDE-1231-ajustar-o-retreino-do-courier-no-laborious + value: main - name: PYTHON_APP value: "laborious.worker.worker" From 36af84f0568a50472bfd87bc0b3f5a5c5308c739 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 6 Oct 2025 13:32:36 -0300 Subject: [PATCH 32/52] SIENTIAPDE-1231 Enhance MLFlow and MLFlowRepository with improved data handling and logging - Refactored MLFlow class to sort data by 'created_at' and drop duplicates for better input preparation. - Updated MLFlowRepository methods to include detailed logging for artifact downloads and model predictions. - Introduced LzmaPayloadCodec for efficient payload compression in the worker, optimizing data handling for large payloads. - Enhanced timestamp handling in treated data to ensure compatibility with model expectations. --- laborious/activities/mlflow.py | 17 +++- .../utils/repository/model_repository.py | 85 +++++++++++++------ laborious/worker/worker.py | 50 +++++++++++ 3 files changed, 123 insertions(+), 29 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 3e4b492..87f6ea3 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -232,14 +232,23 @@ class MLFlow(BaseActivity): timestamp = data['timestamp'].max() self.debug(f'Timestamp: {timestamp}', metadata) + # Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair + data = data.sort_values('created_at', ascending=False).drop_duplicates( + subset=['variable', 'timestamp'], keep='first' + ) + data.drop(columns=['model_id'], inplace=True, errors='ignore') data.drop(columns=['created_at'], inplace=True, errors='ignore') - data = data.pivot(index='timestamp', columns='variable', - values='value') - data.sort_index(inplace=True) - data.reset_index(inplace=True) + # Pivot data for model input format + data = data.pivot( + index='timestamp', columns='variable', + values='value') + data.fillna(np.nan, inplace=True) + # data.reset_index(inplace=True) + data.columns.name = None + data['timestamp'] = data.index data['timestamp'] = to_datetime( data['timestamp'], format=DATETIME_FORMAT_WITH_TZ).dt.strftime(DATETIME_FORMAT) data['timestamp'] = to_datetime( diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 89922a8..a58857d 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -235,6 +235,9 @@ class MLFlowRepository(): if not path.exists(output_dir): makedirs(output_dir) + self.logger.info( + f"Downloading artifacts from {run_id} to {output_dir}") + return self.client.download_artifacts( run_id, artifact_path, @@ -347,18 +350,13 @@ class MLFlowRepository(): Raises: ValueError: If model cannot be loaded with any compression method """ + code_path = path.join( + artifact_path, "code") - if type == "transformer": - code_path = path.join( - artifact_path, "transformer_pyfunc", "code") - pickle_path = path.join( - artifact_path, "transformer_pyfunc", "artifacts", "training_transformer.pkl") + pickle_file = "training_transformer.pkl" if type == "transformer" else "stacking_model.pkl" - elif type == "prediction": - code_path = path.join( - artifact_path, "stacking_model", "code") - pickle_path = path.join( - artifact_path, "stacking_model", "artifacts", "stacking_model.pkl") + pickle_path = path.join( + artifact_path, "artifacts", pickle_file) if code_path not in sys_path: sys_path.insert(0, code_path) @@ -468,6 +466,9 @@ class MLFlowRepository(): f"{message}") elif index_type == datetime or index_type == pd.Timestamp: + if data.index.tz is None: + data.index = data.index.tz_localize('UTC') + data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) else: raise ValueError( @@ -690,6 +691,9 @@ class MLFlowRepository(): self.logger.custom_debug( f"Fitting transformation model with training data (shape: {data.shape})", metadata) + data.to_csv( + f"tmp/retrain_data_{model_name}.csv", index=True) + data_model = data_model.fit(data) self.logger.custom_debug( @@ -697,6 +701,20 @@ class MLFlowRepository(): treated_data = data_model.predict(data) + # Stores current index as timestamp, Courier model expects a timestamp column + # with specific format + treated_data['timestamp'] = treated_data.index + + # Parses timestamp column to datetime format to align with data + treated_data = self.detect_and_parse_datetime_index( + treated_data, metadata) + + self.logger.custom_debug( + f"Treated data index: {treated_data.index}", metadata) + + treated_data.to_csv( + f"tmp/retrain_treated_data_{model_name}.csv", index=True) + self.logger.custom_debug( f"Transformed data shape: {treated_data.shape}", metadata) @@ -708,12 +726,24 @@ class MLFlowRepository(): self.logger.custom_debug( f"Using provided target variable: {target_name}", metadata) + # Check if treated_data contains target variable + if target_name not in treated_data.columns: + self.logger.custom_debug( + f"Target variable {target_name} not found in treated data, aligning data with treated data indexes", metadata) + # Aligns data with treated data indexes to get target variable + aligned_data = data.loc[treated_data.index] + else: + # Uses target variable from treated data + self.logger.custom_debug( + f"Target variable {target_name} found in treated data, using it", metadata) + aligned_data = treated_data + if fit_config.get('y_type', 'series').lower() == 'series': - y = data[target_name] + y = aligned_data[target_name] self.logger.custom_debug( f"Extracting target as series, shape: {y.shape}", metadata) else: - y = data[[target_name]] + y = aligned_data[[target_name]] self.logger.custom_debug( f"Extracting target as dataframe, shape: {y.shape}", metadata) @@ -726,22 +756,25 @@ class MLFlowRepository(): self.logger.custom_debug( f"Combined data shape for prediction model fit: {treated_data.shape}", metadata) + treated_data.to_csv( + f"tmp/retrain_treated_data_combined_{model_name}.csv", index=True) + prediction_model = prediction_model.fit(treated_data) self.logger.custom_debug( "Prediction model fitted with combined data", metadata) else: # Keep data separated fit_order = fit_config.get('split_fit_first', 'x').lower() + + y.to_csv( + f"tmp/retrain_y_{model_name}.csv", index=True) + self.logger.custom_debug( - f"Fitting prediction model with separated data, order: {fit_order}", metadata) + f"Fitting prediction model with separated data, first arg: {fit_order}", metadata) if fit_order == 'x': prediction_model = prediction_model.fit(treated_data, y) - self.logger.custom_debug( - "Prediction model fitted with X, y order", metadata) else: prediction_model = prediction_model.fit(y, treated_data) - self.logger.custom_debug( - "Prediction model fitted with y, X order", metadata) experiment = self.get_experiment_by_run_id(latest_production_id) mlflow.set_experiment(experiment) @@ -1012,30 +1045,32 @@ class MLFlowRepository(): data.to_csv( f"tmp/treated_data_{model_name}.csv", index=True) - data = self.get_cached_predict( + predict_data = self.get_cached_predict( model_name, data, model_retention, flavor) end_time = datetime.now() - if isinstance(data, pd.DataFrame): + if isinstance(predict_data, pd.DataFrame): self.logger.custom_debug( f"Data received from model prediction: {data.head(5).to_csv()}", metadata) - data.to_csv( + predict_data.to_csv( f"tmp/predicted_data_{model_name}.csv", index=True) data.columns = ['prediction'] else: - data = pd.DataFrame(data, columns=['prediction']) - data.to_csv( + predict_data = pd.DataFrame( + predict_data, columns=['prediction']) + predict_data.to_csv( f"tmp/predicted_data_{model_name}.csv", index=True) - data.index = input_index - data['response_time'] = (end_time - start_time).total_seconds() + predict_data.index = input_index + predict_data['response_time'] = ( + end_time - start_time).total_seconds() return { 'success': True, - 'content': data.to_dict() + 'content': predict_data.to_dict() } except Exception as e: diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 81c716b..ad2e6c0 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -28,6 +28,8 @@ Environment Variables: from temporalio import workflow, client from temporalio.worker import Worker, PollerBehaviorAutoscaling from temporalio.runtime import Runtime, TelemetryConfig, PrometheusConfig +from temporalio.converter import PayloadCodec, DataConverter +from temporalio.api.common.v1 import Payload with workflow.unsafe.imports_passed_through(): import os @@ -49,9 +51,54 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.logger import get_logger from laborious import metrics from prometheus_client import start_http_server + import lzma + import dataclasses + POD_ID = os.getenv('POD_ID') SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', "9091")) +LZMA_MIN_MB = float(os.getenv('LZMA_MIN_MB', "1.5")) + + +class LzmaPayloadCodec(PayloadCodec): + async def encode(self, payloads): + out = [] + for p in payloads: + if p.data: + old_len = len(p.data) / 1000000 + # Only compress payloads larger than 1.5 MB + if old_len > LZMA_MIN_MB: + compressed_data = lzma.compress(p.data) + new_len = len(compressed_data) / 1000000 + ratio = new_len / old_len if old_len else 0 + print( + f"[codec] encode lzma: {old_len} MB -> {new_len} MB ({ratio:.2f}x)") + meta = dict(p.metadata or {}) + meta[b"codec"] = b"lzma" + out.append(Payload(metadata=meta, data=compressed_data)) + else: + out.append(p) + + else: + out.append(p) + return out + + async def decode(self, payloads): + out = [] + for p in payloads: + if p.data and (p.metadata or {}).get(b"codec") == b"lzma": + # comp_len = len(p.data) + decomp = lzma.decompress(p.data) + # decomp_len = len(decomp) + # ratio = (decomp_len / comp_len) if comp_len else 0 + # print( + # f"[codec] decode lzma: {comp_len} B -> {decomp_len} B ({ratio:.2f}x)") + meta = dict(p.metadata or {}) + meta.pop(b"codec", None) + out.append(Payload(metadata=meta, data=decomp)) + else: + out.append(p) + return out async def main(): @@ -125,9 +172,12 @@ async def main(): logger.custom_info(f'Starting Temporal Client at {host}...', metadata) + codec_dc = DataConverter(payload_codec=LzmaPayloadCodec()) + temporal_client = await client.Client.connect( target_host=host, namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'), + data_converter=codec_dc, runtime=new_runtime ) From 16a3aa7022849a9852fc3df7c09dd55cc833697e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 6 Oct 2025 16:35:53 -0300 Subject: [PATCH 33/52] SIENTIAPDE-1231 Update .gitignore, values.yaml, and model_repository.py for improved data handling and logging - Added 'catboost_info/' to .gitignore to prevent tracking of additional temporary files. - Updated GITHUB_BRANCH in values.yaml to reflect the current branch for model retraining. - Enhanced model_repository.py to drop duplicate timestamps in treated data and streamline attribute logging during model retraining. --- .gitignore | 3 +- .../utils/repository/model_repository.py | 32 +++++++++++++------ values.yaml | 2 +- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index cfaf446..2c8240d 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,5 @@ git_log .env -tmp/ \ No newline at end of file +tmp/ +catboost_info/ \ No newline at end of file diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index a58857d..54b1985 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -709,6 +709,9 @@ class MLFlowRepository(): treated_data = self.detect_and_parse_datetime_index( treated_data, metadata) + treated_data = treated_data.drop_duplicates( + subset=['timestamp'], keep='first') + self.logger.custom_debug( f"Treated data index: {treated_data.index}", metadata) @@ -820,24 +823,33 @@ class MLFlowRepository(): data_model_atributes = vars(data_model) # load class attributes experiment_description = f"Retrain model {model_name} with new data" current_run_name = self.get_next_run_name(experiment) + + attributes = {} + + for name_atribute, val_atribute in pred_model_atributes.items(): + if name_atribute != "model": + attributes[name_atribute] = val_atribute + for name_atribute, val_atribute in data_model_atributes.items(): + if name_atribute != "model": + attributes[name_atribute] = val_atribute + + self.logger.custom_debug( + f"Attributes: {attributes}", metadata) + with mlflow.start_run( run_name=current_run_name, description=experiment_description ) as _run: # update transfomation model # fixed parameters - for name_atribute, val_atribute in pred_model_atributes.items(): - if name_atribute != "model": - mlflow.log_param(name_atribute, val_atribute) - # update prediction model - for name_atribute, val_atribute in data_model_atributes.items(): - if name_atribute != "model": - mlflow.log_param(name_atribute, val_atribute) + for name_atribute, val_atribute in attributes.items(): + mlflow.log_param(name_atribute, val_atribute) + # dynamic parameters, including model itself mlflow.sklearn.log_model(data_model, "data_model") - makedirs("temp", exist_ok=True) + makedirs("tmp/retrain_data", exist_ok=True) - file_path = f"temp/raw_data_{model_name}.csv" + file_path = f"tmp/retrain_data/retrain_data_{model_name}.csv" data.to_csv(file_path, index=True) # log the data raw @@ -1157,7 +1169,7 @@ class MLFlowRepository(): self.logger.custom_info( f"Model experiment created successfully: {experiment}", metadata) - self.logger.custom_info("Performing model retraining", metadata) + self.logger.custom_info("Saving model retrain", metadata) experiment = self.perform_model_retrain( prediction_model, data_model, experiment, model_name, data, metadata) self.logger.custom_info( diff --git a/values.yaml b/values.yaml index ef86d89..c441f65 100644 --- a/values.yaml +++ b/values.yaml @@ -151,7 +151,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git" - name: GITHUB_BRANCH - value: main + value: SIENTIAPDE-1231-ajustar-o-retreino-do-courier-no-laborious - name: PYTHON_APP value: "laborious.worker.worker" From 7512963e19799761eab255bad3db4cd1e8af5599 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 7 Oct 2025 16:56:16 -0300 Subject: [PATCH 34/52] SIENTIAPDE-1231 Enhance MinIO integration and update environment configurations - Added MinIO configuration parameters to .env.example and values.yaml for improved storage management. - Updated requirements.txt to include necessary libraries for MinIO support. - Refactored Activities class to utilize Storage for MinIO interactions. - Enhanced MLFlow class to integrate MinIO for data retrieval during model retraining. - Introduced build_minio_config function to streamline MinIO configuration setup. - Updated minimal_retrain workflow to support data storage in MinIO. --- .env.example | 8 +- laborious/activities/activities.py | 27 ++-- laborious/activities/mlflow.py | 58 ++++++- laborious/activities/storage.py | 142 ++++++++++++++++++ laborious/utils/connectors_config.py | 23 +++ .../utils/repository/minio_repository.py | 115 ++++++++++++++ .../utils/repository/model_repository.py | 99 ++++++++---- laborious/worker/worker.py | 4 + laborious/workflows/minimal_retrain.py | 13 +- requirements.txt | 4 + values.yaml | 11 ++ 11 files changed, 454 insertions(+), 50 deletions(-) create mode 100644 laborious/activities/storage.py create mode 100644 laborious/utils/repository/minio_repository.py diff --git a/.env.example b/.env.example index 865a73b..6e621e9 100644 --- a/.env.example +++ b/.env.example @@ -26,4 +26,10 @@ MONGODB_USERNAME="mongo_user" MONGODB_PASSWORD="mongo_db_password" MONGODB_URL="my-release-mongodb.mongodb.svc.cluster.local:27017" MONGODB_DATABASE="sientia" -MONGODB_TTL_INDEX_HOURS="1" \ No newline at end of file +MONGODB_TTL_INDEX_HOURS="1" + +MINIO_ENDPOINT_URL="http://localhost:9000" +MINIO_ACCESS_KEY="sientia" +MINIO_SECRET_KEY="sientia" +MINIO_REGION_NAME="sa-east-1" +MINIO_DEFAULT_BUCKET="sientia" \ No newline at end of file diff --git a/laborious/activities/activities.py b/laborious/activities/activities.py index ec5ae46..d028064 100644 --- a/laborious/activities/activities.py +++ b/laborious/activities/activities.py @@ -1,7 +1,7 @@ from temporalio import activity, workflow with workflow.unsafe.imports_passed_through(): - from sientia_do.temporal.activities.postgres import Postgres + from laborious.activities.storage import Storage from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import Logger from laborious.activities.mlflow import MLFlow @@ -10,7 +10,7 @@ with workflow.unsafe.imports_passed_through(): from typing import Any -class Activities(Postgres, MLFlow, Gates, OPC): +class Activities(Storage, MLFlow, Gates, OPC): """ Main activities orchestrator for the Laborious system. @@ -35,6 +35,7 @@ class Activities(Postgres, MLFlow, Gates, OPC): def __init__(self, postgres_config: dict[str, Any], mlflow_config: dict[str, Any], + minio_config: dict[str, Any], opc_config: dict[str, Any], logger: Logger, notification_handler: NotificationHandler): @@ -58,20 +59,22 @@ class Activities(Postgres, MLFlow, Gates, OPC): Exception: If any parent class initialization fails """ # Initialize parent classes - Postgres.__init__(self, host=postgres_config['host'], - port=postgres_config['port'], - user=postgres_config['user'], - password=postgres_config['password'], - dbname=postgres_config['dbname'], - min_connections=postgres_config['min_connections'], - max_connections=postgres_config['max_connections'], - logger=logger, - notification_handler=notification_handler) + Storage.__init__(self, host=postgres_config['host'], + port=postgres_config['port'], + user=postgres_config['user'], + password=postgres_config['password'], + dbname=postgres_config['dbname'], + min_connections=postgres_config['min_connections'], + max_connections=postgres_config['max_connections'], + minio_config=minio_config, + logger=logger, + notification_handler=notification_handler) MLFlow.__init__(self, mlflow_host=mlflow_config['host'], mlflow_port=mlflow_config['port'], mlflow_username=mlflow_config['username'], mlflow_password=mlflow_config['password'], + minio_config=minio_config, logger=logger, notification_handler=notification_handler) @@ -95,5 +98,5 @@ class Activities(Postgres, MLFlow, Gates, OPC): The method should be called before the application terminates to ensure proper resource cleanup and prevent resource leaks. """ - Postgres.close(self) + Storage.close(self) await OPC.shutdown(self) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 87f6ea3..a09c297 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -1,9 +1,9 @@ +from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ, now from temporalio import activity, workflow with workflow.unsafe.imports_passed_through(): - from datetime import datetime - from pandas import Timestamp, to_datetime + from pandas import to_datetime from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ from sientia_do.temporal.activities.base import BaseActivity from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler @@ -15,6 +15,7 @@ with workflow.unsafe.imports_passed_through(): import numpy as np from pandas import DataFrame import traceback + from laborious.utils.repository.minio_repository import MinioRepository class MLFlow(BaseActivity): @@ -37,7 +38,8 @@ class MLFlow(BaseActivity): """ def __init__(self, mlflow_host: str, mlflow_port: int, mlflow_username: str, - mlflow_password: str, logger: Logger, notification_handler: NotificationHandler): + minio_config: dict[str, Any], mlflow_password: str, + logger: Logger, notification_handler: NotificationHandler): """ Initialize MLFlow activities with server configuration. @@ -63,6 +65,26 @@ class MLFlow(BaseActivity): f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger ) + if not hasattr(self, 'minio_repository'): + self.minio_repository = MinioRepository( + logger=logger, + notification_handler=notification_handler, + minio_endpoint_url=minio_config['endpoint_url'], + minio_access_key=minio_config['access_key'], + minio_secret_key=minio_config['secret_key'], + minio_region_name=minio_config['region_name'], + minio_default_bucket=minio_config['default_bucket']) + + if self.minio_repository is None: + self.minio_repository = MinioRepository( + logger=logger, + notification_handler=notification_handler, + minio_endpoint_url=minio_config['endpoint_url'], + minio_access_key=minio_config['access_key'], + minio_secret_key=minio_config['secret_key'], + minio_region_name=minio_config['region_name'], + minio_default_bucket=minio_config['default_bucket']) + @activity.defn(name="request_transform") async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]: """ @@ -223,7 +245,35 @@ class MLFlow(BaseActivity): Exception: If retraining fails or encounters critical errors """ metadata = input_data['metadata'] - data = DataFrame(input_data['data']) + object_key = input_data['object_key'] + + self.info(f'Loading retrain data from Key: {object_key}', metadata) + + try: + + data = self.minio_repository.get_parquet_as_dataframe( + object_key=object_key, metadata=metadata) + except Exception as e: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id='ERROR_LOADING_RETRAIN_DATA', + message=f'Error loading retrain data: {e}', + block='retrain_model', + level=NotificationLevel.ERROR, + attachment_content=trace + ) + self.error(trace, metadata) + return { + 'success': False, + 'message': f'Error loading retrain data: {e}', + 'traceback': trace, + 'timestamp': now().strftime(DATETIME_FORMAT_MS_WITH_TZ) + } + + self.debug( + f'Retrain data loaded successfully: shape {data.shape}', metadata) + model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) diff --git a/laborious/activities/storage.py b/laborious/activities/storage.py new file mode 100644 index 0000000..ef61fb4 --- /dev/null +++ b/laborious/activities/storage.py @@ -0,0 +1,142 @@ +from temporalio import activity, workflow + +from laborious.utils.repository.minio_repository import MinioRepository + + +with workflow.unsafe.imports_passed_through(): + # Extend the Temporal Postgres activities for convenient query -> MinIO export + from sientia_do.temporal.activities.postgres import Postgres + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.observability.logger import Logger + from sientia_do.temporal.constants import now + from sientia_do.notifications.models import NotificationLevel + from typing import Any + import traceback + import pandas as pd + +DATETIME_FILENAME_FORMAT = "%Y-%m-%d_%H-%M-%S" + + +class Storage(Postgres): + """ + Extensions for Postgres activities with a helper to export query results + directly to MinIO as Parquet and return the object name. + """ + + def __init__(self, + host: str, + port: int, + user: str, + password: str, + dbname: str, + min_connections: int, + max_connections: int, + minio_config: dict[str, Any], + logger: Logger, + notification_handler: NotificationHandler): + super().__init__(host=host, + port=port, + user=user, + password=password, + dbname=dbname, + min_connections=min_connections, + max_connections=max_connections, + logger=logger, + notification_handler=notification_handler) + + if not hasattr(self, 'minio_repository'): + self.minio_repository = MinioRepository( + logger=logger, + notification_handler=notification_handler, + minio_endpoint_url=minio_config['endpoint_url'], + minio_access_key=minio_config['access_key'], + minio_secret_key=minio_config['secret_key'], + minio_region_name=minio_config['region_name'], + minio_default_bucket=minio_config['default_bucket']) + + if self.minio_repository is None: + self.minio_repository = MinioRepository( + logger=logger, + notification_handler=notification_handler, + minio_endpoint_url=minio_config['endpoint_url'], + minio_access_key=minio_config['access_key'], + minio_secret_key=minio_config['secret_key'], + minio_region_name=minio_config['region_name'], + minio_default_bucket=minio_config['default_bucket']) + + @activity.defn(name='query_to_minio') + async def query_to_minio(self, input_data: dict[str, Any]) -> dict[str, Any]: + """ + Execute SQL query, write result as Parquet to MinIO, and return object name. + + Args (input_data): + metadata (dict): Workflow metadata + query (str): SQL query + model_name (str): Model name for object naming + object_prefix (str, optional): Prefix inside bucket (default: datasets/retrain) + + Returns: + dict: { success: bool, object_name: str, uri: str } + """ + + metadata = input_data.get('metadata', {}) + object_prefix = input_data.get('object_prefix', 'datasets/retrain') + + timestamp = now().strftime(DATETIME_FILENAME_FORMAT) + object_name = f"{object_prefix}_{timestamp}.parquet" + uri = f"s3://{self.minio_repository.minio_bucket}/{object_name}" + + try: + data = await self.load_custom_query(input_data) + if not data: + self.error( + f"query_to_minio failed: No data returned from query", metadata) + return {"success": False, "message": "No data returned from query"} + + # Ensure we have a DataFrame + data = pd.DataFrame(data) + + # Write parquet to memory and upload via persistent client + self.minio_repository.store_dataframe_as_parquet( + dataframe=data, + uri=uri, + object_name=object_name, + metadata=metadata + ) + + return {"success": True, "object_key": object_name, "uri": uri} + except Exception as e: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id="ERROR_LOADING_CUSTOM_QUERY", + message=f"Error fetching data from query: {e}", + block="load_custom_query", + level=NotificationLevel.ERROR, + attachment_content=trace + ) + + self.error(trace, metadata) + + return {"success": False, "message": str(e)} + + def close(self) -> None: + """Close Storage resources (MinIO client and Postgres engine).""" + try: + if hasattr(self, 's3_client') and self.s3_client is not None: + try: + self.s3_client.close() + finally: + self.s3_client = None + finally: + # Ensure Postgres resources are disposed as well + try: + super().close() + except Exception: + pass + + def __del__(self): + try: + self.close() + except Exception: + pass diff --git a/laborious/utils/connectors_config.py b/laborious/utils/connectors_config.py index 41c933f..969b95c 100644 --- a/laborious/utils/connectors_config.py +++ b/laborious/utils/connectors_config.py @@ -127,3 +127,26 @@ def build_mongodb_config() -> Dict[str, Any]: 'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'), 'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600 } + + +def build_minio_config() -> Dict[str, Any]: + """ + Build MinIO (S3-compatible) configuration from environment variables. + + Environment Variables: + MINIO_ENDPOINT: MinIO endpoint including scheme (default: http://localhost:9000) + MINIO_ACCESS_KEY: Access key (default: minioadmin) + MINIO_SECRET_KEY: Secret key (default: minioadmin) + MINIO_REGION: Region name for S3 client (default: us-east-1) + MINIO_BUCKET_DEFAULT: Default bucket for uploads (default: laborious) + + Returns: + dict: MinIO configuration dictionary + """ + return { + 'endpoint_url': getenv('MINIO_ENDPOINT_URL', 'http://localhost:9000'), + 'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'), + 'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'), + 'region_name': getenv('MINIO_REGION_NAME', 'us-east-1'), + 'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious') + } diff --git a/laborious/utils/repository/minio_repository.py b/laborious/utils/repository/minio_repository.py new file mode 100644 index 0000000..d957e65 --- /dev/null +++ b/laborious/utils/repository/minio_repository.py @@ -0,0 +1,115 @@ +from io import BytesIO +import traceback +import boto3 +from botocore.config import Config +from pandas import DataFrame, read_parquet +from sientia_do.observability.logger import Logger +from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler +from sientia_do.notifications.models import NotificationLevel +from typing import Any +from botocore.exceptions import ClientError + + +class MinioRepository(): + def __init__(self, + minio_endpoint_url: str, + minio_access_key: str, + minio_secret_key: str, + minio_region_name: str, + minio_default_bucket: str, + logger: Logger, + notification_handler: NotificationHandler): + + # MinIO settings shared with pandas s3fs + self.storage_options = { + 'key': minio_access_key, + 'secret': minio_secret_key, + 'client_kwargs': {'endpoint_url': minio_endpoint_url} + } + self.minio_bucket = minio_default_bucket + self.minio_endpoint_url = minio_endpoint_url + self.minio_region_name = minio_region_name + + logger.info( + f"Connecting to MinIO at {self.minio_endpoint_url}, default bucket: {self.minio_bucket}") + + # Reusable MinIO client + self.s3_client = boto3.client( + 's3', + endpoint_url=self.minio_endpoint_url, + aws_access_key_id=self.storage_options['key'], + aws_secret_access_key=self.storage_options['secret'], + region_name=self.minio_region_name, + config=Config( + signature_version='s3v4', + s3={'addressing_style': 'path'}, + retries={'max_attempts': 5, 'mode': 'standard'}, + connect_timeout=5, + read_timeout=120, + ), + ) + self._bucket_checked = False + + self.logger = logger + self.notification_handler = notification_handler + + def ensure_bucket_exists(self, metadata: dict[str, Any]) -> bool: + """ + Ensure the MinIO bucket exists; create it if necessary. + """ + if self._bucket_checked: + return True + + try: + self.logger.custom_info( + f"Checking if bucket '{self.minio_bucket}' exists", metadata) + self.s3_client.head_bucket(Bucket=self.minio_bucket) + self._bucket_checked = True + return True + except ClientError: + try: + self.logger.custom_info( + f"Creating bucket '{self.minio_bucket}'", metadata) + self.s3_client.create_bucket(Bucket=self.minio_bucket) + self._bucket_checked = True + return True + except ClientError as ce: + trace = traceback.format_exc() + self.notification_handler.send_notification( + metadata=metadata, + notification_id="ERROR_CREATING_MINIO_BUCKET", + message=f"Failed to ensure bucket '{self.minio_bucket}': {ce}", + block="ensure_bucket_exists", + level=NotificationLevel.ERROR, + attachment_content=str(ce) + ) + self.logger.custom_error(trace, metadata) + return False + + def store_dataframe_as_parquet(self, dataframe: DataFrame, uri: str, + object_name: str, metadata: dict[str, Any]): + + self.ensure_bucket_exists(metadata) + + self.logger.custom_info( + f"Storing dataframe as parquet in {uri}", metadata) + + buffer = BytesIO() + dataframe.to_parquet(buffer, engine='pyarrow', index=True) + buffer.seek(0) + self.s3_client.put_object( + Bucket=self.minio_bucket, Key=object_name, Body=buffer.getvalue()) + + self.logger.custom_info( + f"Dataframe stored as parquet in {uri}", metadata) + + def get_parquet_as_dataframe(self, object_key: str, metadata: dict[str, Any]) -> DataFrame: + self.logger.custom_info( + f"Getting parquet as dataframe from {object_key}", metadata) + + response = self.s3_client.get_object( + Bucket=self.minio_bucket, Key=object_key) + + # Read the content into a BytesIO buffer to support seek operations + buffer = BytesIO(response['Body'].read()) + return read_parquet(buffer) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 54b1985..b194a69 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -244,7 +244,7 @@ class MLFlowRepository(): output_dir ) - def load_predict_model(self, model_name: str, flavor: str = 'pyfunc', + def load_predict_model(self, model_name: str, flavor: str = 'sklearn', artifact_path: str | None = None) -> Any: """ Downloads a predictive model from the MLflow Model Registry. @@ -386,7 +386,7 @@ class MLFlowRepository(): f"Could not load model from {pickle_path} - unknown or corrupted format") def download_model(self, model_name: str, model_type: str, flavor: str, - download_artifacts: bool = False) -> Any: + download_artifacts: bool = False) -> tuple[Any, str]: """ Download model based on type (predict or transform). @@ -397,7 +397,7 @@ class MLFlowRepository(): download_artifacts (bool): Whether to download artifacts Returns: - Any: Model object + tuple[Any, str]: Model object and artifact path if model is compressed """ self.logger.info( @@ -422,7 +422,7 @@ class MLFlowRepository(): model = self.load_transform_model( model_name, flavor, artifact_path) - return model + return model, artifact_path """ Functions related to data format @@ -633,9 +633,9 @@ class MLFlowRepository(): """ def create_model_experiment(self, model_name: str, data: pd.DataFrame, - transform_flavor: str = 'sklearn', predict_flavor: str = 'pyfunc', + transform_config: dict = {}, predict_config: dict = {}, fit_config: dict = {}, target_name: str = None, - metadata: dict = {}) -> tuple: + is_compressed: bool = False, metadata: dict = {}) -> tuple: """ Create a new MLFlow experiment for model retraining. @@ -649,10 +649,11 @@ class MLFlowRepository(): Args: model_name (str): Name of the MLFlow model to retrain data (pd.DataFrame): Training data for model retraining - transform_flavor (str): Flavor for transformation model - predict_flavor (str): Flavor for prediction model + transform_config (dict): Configuration for transformation model + predict_config (dict): Configuration for prediction model fit_config (dict): Fit configuration target_name (str): Target name + is_compressed (bool): Whether model is compressed metadata (dict): Metadata for logging Returns: @@ -664,7 +665,7 @@ class MLFlowRepository(): self.logger.custom_info( f"Starting model experiment creation for {model_name}", metadata) self.logger.custom_debug( - f"Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, fit_config: {fit_config}, target_name: {target_name}", metadata) + f"Model configuration - transform_config: {transform_config}, predict_config: {predict_config}, fit_config: {fit_config}, target_name: {target_name}", metadata) latest_production_id = self.get_model_run_id( model_name, stage="Production" @@ -675,17 +676,24 @@ class MLFlowRepository(): self.logger.custom_info( f"Loading transformation model for {model_name}", metadata) - data_model = self.download_model( + transform_flavor = transform_config.get('flavor', 'sklearn') + transformed_is_compressed = transform_config.get( + 'is_compressed', False) + + data_model, data_artifact_path = self.download_model( model_name=model_name, model_type="transform", flavor=transform_flavor, - download_artifacts=(transform_flavor == 'pyfunc') + download_artifacts=transformed_is_compressed ) self.logger.custom_info( f"Loading prediction model for {model_name}", metadata) - prediction_model = self.download_model( + predict_flavor = predict_config.get('flavor', 'pyfunc') + predicted_is_compressed = predict_config.get('is_compressed', False) + + prediction_model, prediction_artifact_path = self.download_model( model_name=model_name, model_type="predict", flavor=predict_flavor, - download_artifacts=(predict_flavor == 'pyfunc') + download_artifacts=predicted_is_compressed ) self.logger.custom_debug( @@ -784,14 +792,36 @@ class MLFlowRepository(): self.logger.custom_info( f"Model experiment creation completed successfully for {model_name}", metadata) - return prediction_model, data_model, experiment + + retrain_data = { + 'prediction_model': prediction_model, + 'prediction_artifact_path': prediction_artifact_path, + 'data_model': data_model, + 'data_artifact_path': data_artifact_path, + 'experiment': experiment + } + return retrain_data + + def log_model(self, model: Any, artifact_local_path: str, flavor: str, model_type: str): + if artifact_local_path: + mlflow.log_artifact(artifact_local_path, artifact_path="") + else: + if flavor == 'sklearn': + mlflow.sklearn.log_model(model, model_type) + elif flavor == 'pyfunc': + mlflow.pyfunc.log_model(model, model_type) + elif flavor == 'pytorch': + mlflow.pytorch.log_model(model, model_type) + else: + raise ValueError( + "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") def perform_model_retrain(self, - prediction_model, - data_model, - experiment: str, model_name: str, data: pd.DataFrame, + retrain_data: dict, + transform_config: dict = {}, + predict_config: dict = {}, metadata: dict = {}): """ Execute the complete model retraining process in MLFlow. @@ -809,6 +839,7 @@ class MLFlowRepository(): experiment (str): MLFlow experiment name for the retraining model_name (str): Name of the model being retrained data (pd.DataFrame): Training data used for retraining + is_compressed (bool): Whether model is compressed metadata (dict): Metadata for logging Returns: @@ -816,9 +847,19 @@ class MLFlowRepository(): - status_message (str): Success confirmation message - experiment_name (str): Name of the experiment """ + + prediction_model = retrain_data['prediction_model'] + data_model = retrain_data['data_model'] + experiment = retrain_data['experiment'] + prediction_artifact_path = retrain_data['prediction_artifact_path'] + data_artifact_path = retrain_data['data_artifact_path'] + self.logger.custom_info( f"Starting model retraining process for {model_name} in experiment {experiment}", metadata) + transform_flavor = transform_config.get('flavor', 'sklearn') + predict_flavor = predict_config.get('flavor', 'sklearn') + pred_model_atributes = vars(prediction_model) # load class attributes data_model_atributes = vars(data_model) # load class attributes experiment_description = f"Retrain model {model_name} with new data" @@ -845,7 +886,8 @@ class MLFlowRepository(): mlflow.log_param(name_atribute, val_atribute) # dynamic parameters, including model itself - mlflow.sklearn.log_model(data_model, "data_model") + self.log_model(data_model, data_artifact_path, + transform_flavor, "data_model") makedirs("tmp/retrain_data", exist_ok=True) @@ -856,7 +898,8 @@ class MLFlowRepository(): mlflow.log_artifact(file_path) # dynamic parameters, including model itself - mlflow.sklearn.log_model(prediction_model, "prediction_model") + self.log_model(prediction_model, prediction_artifact_path, + predict_flavor, "prediction_model") mlflow.log_param("retrain", True) # clear temp file @@ -1044,7 +1087,7 @@ class MLFlowRepository(): """ model_retention = model_config.get('retention_minutes', 0) - flavor = model_config.get('predict_flavor', 'pyfunc') + flavor = model_config.get('predict_flavor', 'sklearn') try: @@ -1147,8 +1190,8 @@ class MLFlowRepository(): target_name = model_config.get('target', None) - transform_flavor = model_config.get('transform_flavor', 'sklearn') - predict_flavor = model_config.get('predict_flavor', 'pyfunc') + transform_config = model_config.get('transform_config', {}) + predict_config = model_config.get('predict_config', {}) fit_config = { 'split_fit_data': model_config.get('split_fit_data', False), @@ -1157,21 +1200,19 @@ class MLFlowRepository(): } self.logger.custom_debug( - f"Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, fit_config: {fit_config}, target_name: {target_name}", metadata) + f"Model configuration - transform_config: {transform_config}, predict_config: {predict_config}, fit_config: {fit_config}, target_name: {target_name}", metadata) try: self.logger.custom_info( "Creating model experiment environment", metadata) - prediction_model, data_model, experiment = self.create_model_experiment( - model_name=model_name, data=data, transform_flavor=transform_flavor, - predict_flavor=predict_flavor, fit_config=fit_config, target_name=target_name, - metadata=metadata) + retrain_data = self.create_model_experiment( + model_name=model_name, data=data, transform_config=transform_config, predict_config=predict_config, fit_config=fit_config, target_name=target_name, metadata=metadata) self.logger.custom_info( - f"Model experiment created successfully: {experiment}", metadata) + f"Model experiment created successfully: {retrain_data}", metadata) self.logger.custom_info("Saving model retrain", metadata) experiment = self.perform_model_retrain( - prediction_model, data_model, experiment, model_name, data, metadata) + model_name, data, retrain_data, transform_config, predict_config, metadata) self.logger.custom_info( f"Model retraining completed successfully for experiment: {experiment}", metadata) diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index ad2e6c0..bb74fb3 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -44,6 +44,7 @@ with workflow.unsafe.imports_passed_through(): from laborious.utils.connectors_config import ( build_postgres_config, build_mlflow_config, + build_minio_config, build_opc_config, build_mongodb_config ) @@ -152,6 +153,7 @@ async def main(): activities = Activities( postgres_config=build_postgres_config(), mlflow_config=build_mlflow_config(), + minio_config=build_minio_config(), opc_config=build_opc_config(), logger=logger, notification_handler=notification_handler @@ -190,6 +192,7 @@ async def main(): workflows=[MinimalRetrain], activities=[ activities.load_custom_query, + activities.query_to_minio, activities.retrain_model, activities.update_production_model, activities.format_retrain_report, @@ -211,6 +214,7 @@ async def main(): # MLFlow activities.request_predict, activities.request_transform, + activities.query_to_minio, # Gates activities.input_gate, activities.mlflow_response_gate, diff --git a/laborious/workflows/minimal_retrain.py b/laborious/workflows/minimal_retrain.py index 607f3a5..ebdb0b2 100644 --- a/laborious/workflows/minimal_retrain.py +++ b/laborious/workflows/minimal_retrain.py @@ -70,22 +70,27 @@ class MinimalRetrain(): model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) - data = await workflow.execute_local_activity_method( - Activities.load_custom_query, + storage_result = await workflow.execute_local_activity_method( + Activities.query_to_minio, { **metadata, 'query': input_data['query'], - 'datetime_columns': input_data.get('datetime_columns', []) + 'datetime_columns': input_data.get('datetime_columns', []), + 'model_name': model_name, + 'object_prefix': f'retrain_datasets/{model_name}/data' }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=600) ) + if not storage_result['success']: + return + experiment_response = await workflow.execute_activity_method( Activities.retrain_model, { **metadata, - 'data': data, + 'object_key': storage_result['object_key'], 'model_name': model_name, 'model_config': model_config }, diff --git a/requirements.txt b/requirements.txt index 1ce9f9c..3ba54e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,7 @@ redis git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.6 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.39.0 prometheus-client +botocore +boto3 +s3fs +pyarrow diff --git a/values.yaml b/values.yaml index c441f65..3f2b9f8 100644 --- a/values.yaml +++ b/values.yaml @@ -213,6 +213,17 @@ env: - name: MONGODB_TTL_INDEX_HOURS value: "1" + - name: MINIO_ENDPOINT_URL + value: "http://sientia-minio-minio.sientia.svc.cluster.local:9000" + - name: MINIO_ACCESS_KEY + value: "admin" + - name: MINIO_SECRET_KEY + value: "FvcxOPX55j" + - name: MINIO_REGION_NAME + value: "sa-east-1" + - name: MINIO_DEFAULT_BUCKET + value: "sientia" + ssh: enabled: true secretName: git-ssh-key-sientia-laborious-worker From 49b6e504aeb4266692ae3870ba281e7a7867178d Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 13 Oct 2025 10:36:11 -0300 Subject: [PATCH 35/52] SIENTIAPDE-1231 Enhance Gates and MLFlowRepository with new functionalities and improvements - Added a new method `clean_tmp_files` in the Gates class to remove temporary files associated with model retraining. - Updated MLFlowRepository methods to improve experiment handling, including dynamic parameter logging and model retrieval. - Refactored model loading methods to streamline the process and enhance error handling. - Improved logging for model operations and added support for model parameter retrieval. - Adjusted minimal_retrain workflow to extend timeouts for activities and ensure proper model configuration handling. --- laborious/activities/gates.py | 18 + laborious/activities/mlflow.py | 1 - .../utils/repository/model_repository.py | 493 ++++++++++-------- laborious/worker/worker.py | 47 -- laborious/workflows/minimal_retrain.py | 6 +- 5 files changed, 291 insertions(+), 274 deletions(-) diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index 832cfbb..7e6f785 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -17,6 +17,8 @@ with workflow.unsafe.imports_passed_through(): ) from pandas import DataFrame from laborious import metrics + from os import path + from shutil import rmtree # Input filter function mappings input_filter_functions = { @@ -614,3 +616,19 @@ class Gates(BaseActivity): self.info( f"Metrics written for model {metadata['model_name']}", metadata) + + @activity.defn(name="clean_tmp_files") + async def clean_tmp_files(self, input_data: dict[str, Any]): + """ + Clean temporary files in the tmp directory. + """ + model_name = input_data['model_name'] + metadata = input_data['metadata'] + self.info(f"Cleaning tmp files for model {model_name}...", metadata) + + if path.exists(f"tmp/retrain_data/{model_name}"): + rmtree(f"tmp/retrain_data/{model_name}") + if path.exists(f"tmp/artifacts/{model_name}"): + rmtree(f"tmp/artifacts/{model_name}") + + self.info("Tmp files cleaned", metadata) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index a09c297..822cd69 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -304,7 +304,6 @@ class MLFlow(BaseActivity): data['timestamp'] = to_datetime( data['timestamp'], format=DATETIME_FORMAT) - # data = data.dropna() data.columns.name = None retrain_output = self.model_monitoring_repository.retrain_model( diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index b194a69..880602c 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -15,9 +15,11 @@ The repository provides comprehensive functionality for: """ from datetime import datetime, timedelta import traceback +from mlflow.entities import Experiment, experiment import pandas as pd import mlflow from os import makedirs, path, remove, environ +from shutil import rmtree from sys import path as sys_path from sientia_do.observability.logger import Logger import lzma @@ -29,6 +31,8 @@ from typing import Any from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ ARTIFACTS_PATH = "./tmp/artifacts" +TRANSFORMED_COMPRESSED_PATH = "artifacts/training_transformer.pkl" +PREDICTION_COMPRESSED_PATH = "artifacts/stacking_model.pkl" class MLFlowRepository(): @@ -107,7 +111,7 @@ class MLFlowRepository(): run_id = latest_version.source.split("/") return run_id[2] - def get_experiment_by_run_id(self, run_id: str) -> str: + def get_experiment_by_run_id(self, run_id: str) -> Experiment: """ Get experiment name by run ID. @@ -124,9 +128,7 @@ class MLFlowRepository(): experiment_id = run.info.experiment_id # Get the experiment details using the experiment ID - experiment = mlflow.get_experiment(experiment_id) - experiment_name = experiment.name - return experiment_name + return mlflow.get_experiment(experiment_id) def get_next_run_name(self, model_name: str) -> str: """ @@ -147,7 +149,7 @@ class MLFlowRepository(): next_run_number = len(runs) + 1 return f"{model_name}-{next_run_number}" - def get_experiment(self, experiment_name: str) -> int: + def get_experiment(self, experiment_name: str, create_if_not_exists: bool = False) -> Experiment: """ Retrieve MLFlow experiment ID by experiment name. @@ -167,9 +169,12 @@ class MLFlowRepository(): experiment = mlflow.get_experiment_by_name(experiment_name) if experiment is None: - raise ValueError(f'Experiment {experiment_name} not found') + if create_if_not_exists: + experiment = mlflow.create_experiment(experiment_name) + else: + raise ValueError(f'Experiment {experiment_name} not found') - return int(experiment.experiment_id) + return experiment def get_experiment_last_run(self, experiment_id: int) -> str: """ @@ -212,6 +217,11 @@ class MLFlowRepository(): return latest_run_id + def get_model_params(self, run_id: str): + """Obtém os parâmetros de uma run""" + run_info = mlflow.get_run(run_id) + return run_info.data.params + """ Functions related to download and load models """ @@ -232,8 +242,12 @@ class MLFlowRepository(): ) output_dir = f"{ARTIFACTS_PATH}/{model_name}" - if not path.exists(output_dir): - makedirs(output_dir) + full_path = path.join(output_dir, artifact_path) + + if path.exists(full_path): + # Remove the directory and create a new one + rmtree(full_path) + makedirs(output_dir, exist_ok=True) self.logger.info( f"Downloading artifacts from {run_id} to {output_dir}") @@ -244,8 +258,7 @@ class MLFlowRepository(): output_dir ) - def load_predict_model(self, model_name: str, flavor: str = 'sklearn', - artifact_path: str | None = None) -> Any: + def load_predict_model(self, model_name: str, flavor: str = 'sklearn') -> Any: """ Downloads a predictive model from the MLflow Model Registry. @@ -261,33 +274,22 @@ class MLFlowRepository(): - The model is fetched from the "production" stage of the MLflow Model Registry. - Warnings during the model loading process are suppressed. """ - model_uri = f"models:/{model_name}/production" - - if artifact_path: - self.logger.info( - f"Prediction model {model_name} is compressed, loading from {artifact_path}") - - model = self.load_model_with_compression( - artifact_path, "prediction") - + self.logger.info( + f"Loading prediction model {model_name} from {model_uri}") + if flavor == 'pyfunc': + model = mlflow.pyfunc.load_model(model_uri) + elif flavor == 'sklearn': + model = mlflow.sklearn.load_model(model_uri) + elif flavor == 'pytorch': + model = mlflow.pytorch.load_model(model_uri) else: - self.logger.info( - f"Prediction model {model_name} is not compressed, loading from {model_uri}") - if flavor == 'pyfunc': - model = mlflow.pyfunc.load_model(model_uri) - elif flavor == 'sklearn': - model = mlflow.sklearn.load_model(model_uri) - elif flavor == 'pytorch': - model = mlflow.pytorch.load_model(model_uri) - else: - raise ValueError( - "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") + raise ValueError( + "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") return model - def load_transform_model(self, model_name: str, flavor: str, - artifact_path: str | None = None) -> Any: + def load_transform_model(self, model_name: str, flavor: str) -> Any: """ Downloads the latest production version of a specified transformation model. @@ -306,37 +308,27 @@ class MLFlowRepository(): Exception: If the model run ID or URI cannot be retrieved, or if the model cannot be loaded. """ + latest_production_id = self.get_model_run_id( model_name=model_name, stage="Production" ) model_uri = self.get_model_uri( latest_production_id, prediction=False) - self.logger.debug( - f"Model {model_name} is at {model_uri} and latest production id is {latest_production_id}") - - if artifact_path: - # Download model artifacts - self.logger.info( - f"Data model {model_name} is compressed, loading from {artifact_path}") - - model = self.load_model_with_compression( - artifact_path, "transformer") + self.logger.info( + f"Loading data model {model_name} from {model_uri}") + if flavor == 'sklearn': + model = mlflow.sklearn.load_model(model_uri) + elif flavor == 'pyfunc': + model = mlflow.pyfunc.load_model(model_uri) + elif flavor == 'pytorch': + model = mlflow.pytorch.load_model(model_uri) else: - self.logger.info( - f"Data model {model_name} is not compressed, loading from {model_uri}") - if flavor == 'sklearn': - model = mlflow.sklearn.load_model(model_uri) - elif flavor == 'pyfunc': - model = mlflow.pyfunc.load_model(model_uri) - elif flavor == 'pytorch': - model = mlflow.pytorch.load_model(model_uri) - else: - raise ValueError( - "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") + raise ValueError( + "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") return model - def load_model_with_compression(self, artifact_path: str, type: str) -> Any: + def load_model_with_compression(self, artifact_path: str, model_type: str) -> Any: """ Load model from pickle file trying different compression methods. @@ -353,10 +345,12 @@ class MLFlowRepository(): code_path = path.join( artifact_path, "code") - pickle_file = "training_transformer.pkl" if type == "transformer" else "stacking_model.pkl" + pickle_file = TRANSFORMED_COMPRESSED_PATH if model_type == "transform" else PREDICTION_COMPRESSED_PATH - pickle_path = path.join( - artifact_path, "artifacts", pickle_file) + pickle_path = path.join(artifact_path, pickle_file) + + self.logger.info( + f"Loading model with type {model_type} from {pickle_path}") if code_path not in sys_path: sys_path.insert(0, code_path) @@ -371,7 +365,8 @@ class MLFlowRepository(): for format_name, open_func in loading_methods: try: - self.logger.info(f"Trying to load with {format_name}...") + self.logger.inf + (f"Trying to load with {format_name}...") with open_func(pickle_path) as f: model = pickle.load(f) self.logger.info( @@ -386,7 +381,7 @@ class MLFlowRepository(): f"Could not load model from {pickle_path} - unknown or corrupted format") def download_model(self, model_name: str, model_type: str, flavor: str, - download_artifacts: bool = False) -> tuple[Any, str]: + load_wrapper: bool = False) -> tuple[Any, str]: """ Download model based on type (predict or transform). @@ -394,33 +389,44 @@ class MLFlowRepository(): model_name (str): Name of the model to download model_type (str): Type of model ('predict' or 'transform') flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') - download_artifacts (bool): Whether to download artifacts + load_wrapper (bool): Whether to load wrapper Returns: tuple[Any, str]: Model object and artifact path if model is compressed """ self.logger.info( - f"Downloading {model_type} model {model_name} with flavor {flavor} and download_artifacts {download_artifacts}") + f"Downloading {model_type} model {model_name} with flavor {flavor} and load_wrapper {load_wrapper}") if model_type not in ["predict", "transform"]: raise ValueError( "Invalid model_type. Use 'predict' or 'transform'.") - if download_artifacts: + artifact_path = None + + if load_wrapper: + self.logger.info( + f"Loading wrapper for {model_type} model {model_name} with flavor {flavor}") + target = "prediction_model" if model_type == "predict" else "data_model" artifact_path = self.dowload_artifacts( model_name, target) + + self.logger.info( + f"Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}") + + raw_model = mlflow.pyfunc.load_model(artifact_path) + model = raw_model._model_impl.python_model else: - artifact_path = None - if model_type == "predict": - model = self.load_predict_model(model_name, flavor, artifact_path) + if model_type == "predict": + model = self.load_predict_model( + model_name, flavor) - elif model_type == "transform": - model = self.load_transform_model( - model_name, flavor, artifact_path) + elif model_type == "transform": + model = self.load_transform_model( + model_name, flavor) return model, artifact_path @@ -536,7 +542,7 @@ class MLFlowRepository(): del self.model_cache[model_key]['target']['model'] del self.model_cache[model_key] - def get_model(self, model_name: str, retention: int, model_type: str, flavor: str): + def get_model(self, model_name: str, retention: int, model_type: str, flavor: str) -> Any: """ Get model with caching support based on retention policy. @@ -545,15 +551,15 @@ class MLFlowRepository(): retention (int): Cache retention time in minutes (0 = no cache) model_type (str): Type of model ('predict' or 'transform') flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') - retention_target (str): What to cache ('model' or 'artifact') Returns: - dict: Model configuration with model and artifact paths + Any: Model object """ # Retention is 0, download a new model if retention <= 0: - return self.download_model( - model_name=model_name, model_type=model_type, flavor=flavor, download_artifacts=False) + model, _artifact_path = self.download_model( + model_name=model_name, model_type=model_type, flavor=flavor, load_wrapper=False) + return model model_key = f'{model_name}_{model_type}' @@ -573,9 +579,9 @@ class MLFlowRepository(): f"Model {model_name} is not in {model_type} cache, downloading a new one") # Donwload new model - model = self.download_model( + model, _artifact_path = self.download_model( model_name=model_name, model_type=model_type, flavor=flavor, - download_artifacts=False) + load_wrapper=False) cache = { 'target': model, @@ -604,10 +610,15 @@ class MLFlowRepository(): model_name=model_name, retention=retention, model_type="transform", flavor=flavor) - return model.predict(data) + prediction = model.predict(data) - def get_cached_predict(self, model_name: str, data: pd.DataFrame, retention: int, flavor: str, - compressed: bool = False, retention_target: str = "model") -> pd.DataFrame | ndarray: + if retention == 0: + del model + + return prediction + + def get_cached_predict(self, model_name: str, data: pd.DataFrame, retention: int, + flavor: str) -> ndarray: """ Get predictions using cached prediction model. @@ -616,8 +627,6 @@ class MLFlowRepository(): data (pd.DataFrame): Data to make predictions on retention (int): Cache retention time in minutes flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') - compressed (bool): Whether model is compressed - retention_target (str): What to cache ('model' or 'artifact') Returns: pd.DataFrame: Model predictions @@ -626,16 +635,21 @@ class MLFlowRepository(): model_name=model_name, retention=retention, model_type="predict", flavor=flavor) - return model.predict(data) + prediction = model.predict(data) + + if retention == 0: + del model + + return prediction """ Functions related to model retraining """ - def create_model_experiment(self, model_name: str, data: pd.DataFrame, - transform_config: dict = {}, predict_config: dict = {}, + def create_model_experiment(self, model_name: str, data: pd.DataFrame, latest_production_id: str, + transform_flavor: str = 'sklearn', predict_flavor: str = 'sklearn', fit_config: dict = {}, target_name: str = None, - is_compressed: bool = False, metadata: dict = {}) -> tuple: + metadata: dict = {}) -> tuple: """ Create a new MLFlow experiment for model retraining. @@ -649,11 +663,10 @@ class MLFlowRepository(): Args: model_name (str): Name of the MLFlow model to retrain data (pd.DataFrame): Training data for model retraining - transform_config (dict): Configuration for transformation model - predict_config (dict): Configuration for prediction model + transform_flavor (str): Flavor for transformation model + predict_flavor (str): Flavor for prediction model fit_config (dict): Fit configuration target_name (str): Target name - is_compressed (bool): Whether model is compressed metadata (dict): Metadata for logging Returns: @@ -665,49 +678,34 @@ class MLFlowRepository(): self.logger.custom_info( f"Starting model experiment creation for {model_name}", metadata) self.logger.custom_debug( - f"Model configuration - transform_config: {transform_config}, predict_config: {predict_config}, fit_config: {fit_config}, target_name: {target_name}", metadata) + f"Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, fit_config: {fit_config}, target_name: {target_name}", metadata) - latest_production_id = self.get_model_run_id( - model_name, stage="Production" - ) + data.to_csv( + f"tmp/retrain_data_{model_name}.csv", index=True) self.logger.custom_info( f"Retrieved latest production run ID: {latest_production_id}", metadata) self.logger.custom_info( f"Loading transformation model for {model_name}", metadata) - transform_flavor = transform_config.get('flavor', 'sklearn') - transformed_is_compressed = transform_config.get( - 'is_compressed', False) + load_transform_wrapper = transform_flavor == 'pyfunc' data_model, data_artifact_path = self.download_model( model_name=model_name, model_type="transform", flavor=transform_flavor, - download_artifacts=transformed_is_compressed + load_wrapper=load_transform_wrapper ) self.logger.custom_info( f"Loading prediction model for {model_name}", metadata) - predict_flavor = predict_config.get('flavor', 'pyfunc') - predicted_is_compressed = predict_config.get('is_compressed', False) + load_predict_wrapper = predict_flavor == 'pyfunc' prediction_model, prediction_artifact_path = self.download_model( model_name=model_name, model_type="predict", flavor=predict_flavor, - download_artifacts=predicted_is_compressed + load_wrapper=load_predict_wrapper ) - self.logger.custom_debug( - f"Fitting transformation model with training data (shape: {data.shape})", metadata) - - data.to_csv( - f"tmp/retrain_data_{model_name}.csv", index=True) - - data_model = data_model.fit(data) - - self.logger.custom_debug( - f"Applying transformation to training data", metadata) - - treated_data = data_model.predict(data) + treated_data = data_model.fit(data) # Stores current index as timestamp, Courier model expects a timestamp column # with specific format @@ -743,86 +741,105 @@ class MLFlowRepository(): f"Target variable {target_name} not found in treated data, aligning data with treated data indexes", metadata) # Aligns data with treated data indexes to get target variable aligned_data = data.loc[treated_data.index] + retrain_dataset = pd.merge( + treated_data, aligned_data, left_index=True, right_index=True) else: # Uses target variable from treated data self.logger.custom_debug( f"Target variable {target_name} found in treated data, using it", metadata) - aligned_data = treated_data + retrain_dataset = treated_data - if fit_config.get('y_type', 'series').lower() == 'series': - y = aligned_data[target_name] - self.logger.custom_debug( - f"Extracting target as series, shape: {y.shape}", metadata) - else: - y = aligned_data[[target_name]] - self.logger.custom_debug( - f"Extracting target as dataframe, shape: {y.shape}", metadata) + retrain_dataset.to_csv( + f"tmp/retrain_retrain_dataset_{model_name}.csv", index=True) - if not fit_config.get('split_fit_data', False): - # Merge treated data with target - self.logger.custom_debug( - "Merging treated data with target for combined fit", metadata) - treated_data = pd.merge( - treated_data, y, left_index=True, right_index=True) - self.logger.custom_debug( - f"Combined data shape for prediction model fit: {treated_data.shape}", metadata) - - treated_data.to_csv( - f"tmp/retrain_treated_data_combined_{model_name}.csv", index=True) - - prediction_model = prediction_model.fit(treated_data) - self.logger.custom_debug( - "Prediction model fitted with combined data", metadata) - else: - # Keep data separated - fit_order = fit_config.get('split_fit_first', 'x').lower() - - y.to_csv( - f"tmp/retrain_y_{model_name}.csv", index=True) - - self.logger.custom_debug( - f"Fitting prediction model with separated data, first arg: {fit_order}", metadata) - if fit_order == 'x': - prediction_model = prediction_model.fit(treated_data, y) - else: - prediction_model = prediction_model.fit(y, treated_data) - - experiment = self.get_experiment_by_run_id(latest_production_id) - mlflow.set_experiment(experiment) + prediction_model.fit(retrain_dataset) self.logger.custom_info( f"Model experiment creation completed successfully for {model_name}", metadata) retrain_data = { - 'prediction_model': prediction_model, - 'prediction_artifact_path': prediction_artifact_path, - 'data_model': data_model, - 'data_artifact_path': data_artifact_path, - 'experiment': experiment + 'prediction_model': { + 'model': prediction_model, + 'artifact_path': prediction_artifact_path + }, + 'data_model': { + 'model': data_model, + 'artifact_path': data_artifact_path + } } return retrain_data - def log_model(self, model: Any, artifact_local_path: str, flavor: str, model_type: str): + def export_model_to_pkl(self, model_data: dict, model_type: str, metadata: dict = {}): + artifact_local_path = model_data['artifact_path'] + model = model_data['model'] + if artifact_local_path: - mlflow.log_artifact(artifact_local_path, artifact_path="") + pickle_file = TRANSFORMED_COMPRESSED_PATH if model_type == "data_model" else PREDICTION_COMPRESSED_PATH + pickle_path = path.join(artifact_local_path, pickle_file) + + self.logger.custom_info( + f"Saving model to {pickle_path}", metadata) + + with open(pickle_path, "wb") as f: + pickle.dump(model, f) + + def export_model_to_lzma(self, model_data: dict, model_type: str, metadata: dict = {}): + artifact_local_path = model_data['artifact_path'] + model = model_data['model'] + + if artifact_local_path: + pickle_file = TRANSFORMED_COMPRESSED_PATH if model_type == "data_model" else PREDICTION_COMPRESSED_PATH + pickle_path = path.join(artifact_local_path, pickle_file) + + self.logger.custom_info( + f"Saving model to {pickle_path}", metadata) + + if path.exists(pickle_path): + remove(pickle_path) + + with lzma.open(pickle_path + ".xz", "wb") as f: + self.logger.custom_info( + "Compressing model", metadata) + pickle.dump(model, f) + + def log_model(self, model_data: dict, flavor: str, model_type: str, + metadata: dict = {}): + + model = model_data['model'] + + self.logger.custom_debug( + f"Logging {model_type} model to {model_type}", metadata) + if flavor == 'sklearn': + mlflow.sklearn.log_model(model, model_type) + elif flavor == 'pyfunc': + code_path = [path.join( + model_data['artifact_path'], 'code', "utils")] + + self.logger.custom_debug( + f"Code path: {code_path}", metadata) + + model.store_model( + artifact_path=model_type, + code_path=code_path, + to_disk=False + ) + + self.logger.custom_debug( + f"Model uploaded successfully", metadata) + elif flavor == 'pytorch': + mlflow.pytorch.log_model(model, model_type) else: - if flavor == 'sklearn': - mlflow.sklearn.log_model(model, model_type) - elif flavor == 'pyfunc': - mlflow.pyfunc.log_model(model, model_type) - elif flavor == 'pytorch': - mlflow.pytorch.log_model(model, model_type) - else: - raise ValueError( - "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") + raise ValueError( + "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") def perform_model_retrain(self, model_name: str, data: pd.DataFrame, retrain_data: dict, - transform_config: dict = {}, - predict_config: dict = {}, - metadata: dict = {}): + transform_flavor: str = 'sklearn', + predict_flavor: str = 'sklearn', + metadata: dict = {}, + latest_production_id: str = None) -> dict: """ Execute the complete model retraining process in MLFlow. @@ -839,7 +856,8 @@ class MLFlowRepository(): experiment (str): MLFlow experiment name for the retraining model_name (str): Name of the model being retrained data (pd.DataFrame): Training data used for retraining - is_compressed (bool): Whether model is compressed + transform_flavor (str): Flavor for transformation model + predict_flavor (str): Flavor for prediction model metadata (dict): Metadata for logging Returns: @@ -850,63 +868,90 @@ class MLFlowRepository(): prediction_model = retrain_data['prediction_model'] data_model = retrain_data['data_model'] - experiment = retrain_data['experiment'] - prediction_artifact_path = retrain_data['prediction_artifact_path'] - data_artifact_path = retrain_data['data_artifact_path'] + + model_temp_path = path.join( + ARTIFACTS_PATH, model_name) self.logger.custom_info( - f"Starting model retraining process for {model_name} in experiment {experiment}", metadata) + f"Starting model retraining process for {model_name}", metadata) - transform_flavor = transform_config.get('flavor', 'sklearn') - predict_flavor = predict_config.get('flavor', 'sklearn') - - pred_model_atributes = vars(prediction_model) # load class attributes - data_model_atributes = vars(data_model) # load class attributes + original_params = self.get_model_params(latest_production_id) + retrain_params = { + **original_params, + "retrain": True, + 'retrain_date': datetime.now().isoformat(), + 'source_run_id': latest_production_id, + 'retrain_samples': str(data.shape), + } experiment_description = f"Retrain model {model_name} with new data" - current_run_name = self.get_next_run_name(experiment) - attributes = {} + experiment = self.get_experiment(model_name, + create_if_not_exists=True) + experiment_name = experiment.name - for name_atribute, val_atribute in pred_model_atributes.items(): - if name_atribute != "model": - attributes[name_atribute] = val_atribute - for name_atribute, val_atribute in data_model_atributes.items(): - if name_atribute != "model": - attributes[name_atribute] = val_atribute + current_run_name = self.get_next_run_name(experiment_name) self.logger.custom_debug( - f"Attributes: {attributes}", metadata) + f"Attributes: {retrain_params}", metadata) + + data_path = f"{model_temp_path}/retrain_data.csv" + + data.to_csv(data_path, index=True) + + self.logger.custom_info( + f"Starting model upload for {experiment_name} with run name {current_run_name}", metadata) with mlflow.start_run( - run_name=current_run_name, description=experiment_description + experiment_id=experiment.experiment_id, + run_name=current_run_name, + description=experiment_description ) as _run: + run_id = _run.info.run_id + self.logger.custom_info( + f"Logging data model", metadata) + # dynamic parameters, including model itself + self.log_model(data_model, transform_flavor, + "data_model", metadata) + + # dynamic parameters, including model itself + self.logger.custom_info( + f"Logging prediction model", metadata) + self.log_model(prediction_model, predict_flavor, + "prediction_model", metadata) + + self.logger.custom_info( + f"Model logged successfully for {model_name}", metadata) + + self.logger.custom_info( + f"Logging remaining parameters for {model_name}", metadata) + # update transfomation model # fixed parameters - for name_atribute, val_atribute in attributes.items(): - mlflow.log_param(name_atribute, val_atribute) - - # dynamic parameters, including model itself - self.log_model(data_model, data_artifact_path, - transform_flavor, "data_model") - - makedirs("tmp/retrain_data", exist_ok=True) - - file_path = f"tmp/retrain_data/retrain_data_{model_name}.csv" - data.to_csv(file_path, index=True) + mlflow.log_params(retrain_params) # log the data raw - mlflow.log_artifact(file_path) + mlflow.log_artifact(data_path) - # dynamic parameters, including model itself - self.log_model(prediction_model, prediction_artifact_path, - predict_flavor, "prediction_model") - mlflow.log_param("retrain", True) + self.logger.custom_info( + f"Deleting model from filesystem", metadata) + if path.exists(model_temp_path): + rmtree(model_temp_path) - # clear temp file - if path.exists(file_path): - remove(file_path) + self.logger.custom_info( + f"Deleting prediction model from memory", metadata) + del prediction_model['model'] + del prediction_model - return experiment + self.logger.custom_info( + f"Deleting data model from memory", metadata) + del data_model['model'] + del data_model + + return { + 'run_id': run_id, + 'experiment_id': experiment.experiment_id, + 'experiment_name': experiment.name + } def update_production_model_by_run_id(self, run_id: str, model_name: str, metadata: dict = {}) -> dict: """ @@ -942,12 +987,8 @@ class MLFlowRepository(): mlflow.register_model( f"runs:/{run_id}/prediction_model", model_name) - # Colocar a versão do modelo em produção - # Depois de registrar o modelo, precisamos pegar a versão mais recente do modelo e movê-lo para o estágio 'Production' - client = mlflow.tracking.MlflowClient() - # Obter a versão mais recente registrada do modelo - model_versions = client.get_registered_model( + model_versions = self.client.get_registered_model( model_name).latest_versions if not isinstance(model_versions, list): @@ -956,7 +997,7 @@ class MLFlowRepository(): max_version = max(model_versions, key=lambda x: int(x.version)).version # Mover a versão mais recente do modelo para o estágio de 'Production' - client.transition_model_version_stage( + self.client.transition_model_version_stage( name=model_name, version=max_version, stage="Production", @@ -1190,8 +1231,8 @@ class MLFlowRepository(): target_name = model_config.get('target', None) - transform_config = model_config.get('transform_config', {}) - predict_config = model_config.get('predict_config', {}) + transform_flavor = model_config.get('transform_flavor', 'sklearn') + predict_flavor = model_config.get('predict_flavor', 'sklearn') fit_config = { 'split_fit_data': model_config.get('split_fit_data', False), @@ -1200,19 +1241,25 @@ class MLFlowRepository(): } self.logger.custom_debug( - f"Model configuration - transform_config: {transform_config}, predict_config: {predict_config}, fit_config: {fit_config}, target_name: {target_name}", metadata) + f"Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, fit_config: {fit_config}, target_name: {target_name}", metadata) try: + latest_production_id = self.get_model_run_id( + model_name, stage="Production" + ) self.logger.custom_info( "Creating model experiment environment", metadata) retrain_data = self.create_model_experiment( - model_name=model_name, data=data, transform_config=transform_config, predict_config=predict_config, fit_config=fit_config, target_name=target_name, metadata=metadata) + model_name=model_name, data=data, transform_flavor=transform_flavor, + predict_flavor=predict_flavor, fit_config=fit_config, target_name=target_name, + metadata=metadata, latest_production_id=latest_production_id) self.logger.custom_info( f"Model experiment created successfully: {retrain_data}", metadata) self.logger.custom_info("Saving model retrain", metadata) experiment = self.perform_model_retrain( - model_name, data, retrain_data, transform_config, predict_config, metadata) + model_name=model_name, data=data, retrain_data=retrain_data, + transform_flavor=transform_flavor, predict_flavor=predict_flavor, metadata=metadata, latest_production_id=latest_production_id) self.logger.custom_info( f"Model retraining completed successfully for experiment: {experiment}", metadata) @@ -1276,8 +1323,8 @@ class MLFlowRepository(): This operation is irreversible. The previous production model will be automatically archived when the new version is promoted. """ - experiment_id = self.get_experiment(experiment) - run_id = self.get_experiment_last_run(experiment_id) + run_id = experiment['run_id'] + experiment_id = experiment['experiment_id'] metadata_result = self.update_production_model_by_run_id( run_id, model_name, metadata) diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index bb74fb3..4658524 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -28,8 +28,6 @@ Environment Variables: from temporalio import workflow, client from temporalio.worker import Worker, PollerBehaviorAutoscaling from temporalio.runtime import Runtime, TelemetryConfig, PrometheusConfig -from temporalio.converter import PayloadCodec, DataConverter -from temporalio.api.common.v1 import Payload with workflow.unsafe.imports_passed_through(): import os @@ -58,48 +56,6 @@ with workflow.unsafe.imports_passed_through(): POD_ID = os.getenv('POD_ID') SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', "9091")) -LZMA_MIN_MB = float(os.getenv('LZMA_MIN_MB', "1.5")) - - -class LzmaPayloadCodec(PayloadCodec): - async def encode(self, payloads): - out = [] - for p in payloads: - if p.data: - old_len = len(p.data) / 1000000 - # Only compress payloads larger than 1.5 MB - if old_len > LZMA_MIN_MB: - compressed_data = lzma.compress(p.data) - new_len = len(compressed_data) / 1000000 - ratio = new_len / old_len if old_len else 0 - print( - f"[codec] encode lzma: {old_len} MB -> {new_len} MB ({ratio:.2f}x)") - meta = dict(p.metadata or {}) - meta[b"codec"] = b"lzma" - out.append(Payload(metadata=meta, data=compressed_data)) - else: - out.append(p) - - else: - out.append(p) - return out - - async def decode(self, payloads): - out = [] - for p in payloads: - if p.data and (p.metadata or {}).get(b"codec") == b"lzma": - # comp_len = len(p.data) - decomp = lzma.decompress(p.data) - # decomp_len = len(decomp) - # ratio = (decomp_len / comp_len) if comp_len else 0 - # print( - # f"[codec] decode lzma: {comp_len} B -> {decomp_len} B ({ratio:.2f}x)") - meta = dict(p.metadata or {}) - meta.pop(b"codec", None) - out.append(Payload(metadata=meta, data=decomp)) - else: - out.append(p) - return out async def main(): @@ -174,12 +130,9 @@ async def main(): logger.custom_info(f'Starting Temporal Client at {host}...', metadata) - codec_dc = DataConverter(payload_codec=LzmaPayloadCodec()) - temporal_client = await client.Client.connect( target_host=host, namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'), - data_converter=codec_dc, runtime=new_runtime ) diff --git a/laborious/workflows/minimal_retrain.py b/laborious/workflows/minimal_retrain.py index ebdb0b2..71f72ee 100644 --- a/laborious/workflows/minimal_retrain.py +++ b/laborious/workflows/minimal_retrain.py @@ -95,7 +95,7 @@ class MinimalRetrain(): 'model_config': model_config }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=600) + start_to_close_timeout=timedelta(hours=1) ) if experiment_response['success']: @@ -108,7 +108,7 @@ class MinimalRetrain(): **experiment_response }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=600) + start_to_close_timeout=timedelta(seconds=60) ) else: update_report = {} @@ -123,7 +123,7 @@ class MinimalRetrain(): 'update_report': update_report }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=600) + start_to_close_timeout=timedelta(seconds=60) ) await workflow.execute_activity_method( From ac9b2e0ec63aebab7e5fb75ba1ad6a4fedd1b1b2 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 13 Oct 2025 11:35:18 -0300 Subject: [PATCH 36/52] SIENTIAPDE-1231 Comment out CSV export lines in MLFlowRepository to prevent temporary file creation during model operations. This change enhances data handling by avoiding unnecessary file writes while maintaining logging functionality. --- .../utils/repository/model_repository.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 880602c..3652c25 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -680,8 +680,8 @@ class MLFlowRepository(): self.logger.custom_debug( f"Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, fit_config: {fit_config}, target_name: {target_name}", metadata) - data.to_csv( - f"tmp/retrain_data_{model_name}.csv", index=True) + # data.to_csv( + # f"tmp/retrain_data_{model_name}.csv", index=True) self.logger.custom_info( f"Retrieved latest production run ID: {latest_production_id}", metadata) @@ -721,8 +721,8 @@ class MLFlowRepository(): self.logger.custom_debug( f"Treated data index: {treated_data.index}", metadata) - treated_data.to_csv( - f"tmp/retrain_treated_data_{model_name}.csv", index=True) + # treated_data.to_csv( + # f"tmp/retrain_treated_data_{model_name}.csv", index=True) self.logger.custom_debug( f"Transformed data shape: {treated_data.shape}", metadata) @@ -749,8 +749,8 @@ class MLFlowRepository(): f"Target variable {target_name} found in treated data, using it", metadata) retrain_dataset = treated_data - retrain_dataset.to_csv( - f"tmp/retrain_retrain_dataset_{model_name}.csv", index=True) + # retrain_dataset.to_csv( + # f"tmp/retrain_retrain_dataset_{model_name}.csv", index=True) prediction_model.fit(retrain_dataset) @@ -896,7 +896,7 @@ class MLFlowRepository(): data_path = f"{model_temp_path}/retrain_data.csv" - data.to_csv(data_path, index=True) + # data.to_csv(data_path, index=True) self.logger.custom_info( f"Starting model upload for {experiment_name} with run name {current_run_name}", metadata) @@ -1051,8 +1051,8 @@ class MLFlowRepository(): self.logger.custom_debug( f"Data received for model transformation: {data.head(5).to_csv()}", metadata) - data.to_csv( - f"tmp/data_{model_name}.csv", index=True) + # data.to_csv( + # f"tmp/data_{model_name}.csv", index=True) model_retention = model_config.get('retention_minutes', 0) flavor = model_config.get('transform_flavor', 'sklearn') @@ -1065,8 +1065,8 @@ class MLFlowRepository(): self.logger.custom_debug( f"Data received from model transformation: {transformed_data.head(5).to_csv()}", metadata) - transformed_data.to_csv( - f"tmp/transformed_data_{model_name}.csv", index=True) + # transformed_data.to_csv( + # f"tmp/transformed_data_{model_name}.csv", index=True) transformed_data = self.detect_and_parse_datetime_index( transformed_data, metadata) @@ -1138,8 +1138,8 @@ class MLFlowRepository(): self.logger.custom_debug( f"Data received for model prediction: {data.head(5).to_csv()}", metadata) - data.to_csv( - f"tmp/treated_data_{model_name}.csv", index=True) + # data.to_csv( + # f"tmp/treated_data_{model_name}.csv", index=True) predict_data = self.get_cached_predict( model_name, data, model_retention, flavor) @@ -1150,15 +1150,15 @@ class MLFlowRepository(): self.logger.custom_debug( f"Data received from model prediction: {data.head(5).to_csv()}", metadata) - predict_data.to_csv( - f"tmp/predicted_data_{model_name}.csv", index=True) + # predict_data.to_csv( + # f"tmp/predicted_data_{model_name}.csv", index=True) data.columns = ['prediction'] else: predict_data = pd.DataFrame( predict_data, columns=['prediction']) - predict_data.to_csv( - f"tmp/predicted_data_{model_name}.csv", index=True) + # predict_data.to_csv( + # f"tmp/predicted_data_{model_name}.csv", index=True) predict_data.index = input_index predict_data['response_time'] = ( From fdaf48ddc9e9745f10c162c8e9be157b29d13be1 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 13 Oct 2025 15:12:20 -0300 Subject: [PATCH 37/52] SIENTIAPDE-1231 Update values.yaml and model_repository.py for configuration and error handling improvements - Increased replicaCount from 1 to 2 in values.yaml for enhanced scalability. - Updated image tag from "0.0.2" to "0.0.3" in values.yaml to reflect the latest version. - Enhanced error messaging in model_repository.py to include the actual index type when raising ValueError for index type validation. --- laborious/utils/repository/model_repository.py | 6 +++--- values.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 3652c25..33611bb 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -455,12 +455,12 @@ class MLFlowRepository(): self.logger.custom_info(f"Index type: {index_type}", metadata) - message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}" + message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}. Got {index_type}." # Check if all in index are of the same type if not all(isinstance(i, index_type) for i in index): raise ValueError( - f"{message}") + f"{message}. Elements are {','.join(map(type, index))}") # Check type and converts to DATETIME_FORMAT_WITH_TZ if index_type == str: @@ -896,7 +896,7 @@ class MLFlowRepository(): data_path = f"{model_temp_path}/retrain_data.csv" - # data.to_csv(data_path, index=True) + data.to_csv(data_path, index=True) self.logger.custom_info( f"Starting model upload for {experiment_name} with run name {current_run_name}", metadata) diff --git a/values.yaml b/values.yaml index 3f2b9f8..1300546 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 1 +replicaCount: 2 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.0.2" + tag: "0.0.3" 0# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: @@ -214,7 +214,7 @@ env: value: "1" - name: MINIO_ENDPOINT_URL - value: "http://sientia-minio-minio.sientia.svc.cluster.local:9000" + value: "http://minio.minio.svc.cluster.local:9000" - name: MINIO_ACCESS_KEY value: "admin" - name: MINIO_SECRET_KEY From 331df2dd18d6c5c4726c806be4a62a93b0efb553 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 13 Oct 2025 15:31:04 -0300 Subject: [PATCH 38/52] SIENTIAPDE-1231 Enhance MLFlowRepository memory management by adding garbage collection and logging for model deletion - Introduced garbage collection after model deletion to optimize memory usage. - Added logging to inform when a model is deleted from memory, improving traceability during predictions. --- laborious/utils/repository/model_repository.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 33611bb..11944cc 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -27,7 +27,7 @@ import gzip import pickle from numpy import ndarray from typing import Any - +import gc from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ ARTIFACTS_PATH = "./tmp/artifacts" @@ -613,8 +613,12 @@ class MLFlowRepository(): prediction = model.predict(data) if retention == 0: + self.logger.info( + f"Deleting model {model_name} from memory") del model + gc.collect() + return prediction def get_cached_predict(self, model_name: str, data: pd.DataFrame, retention: int, @@ -638,8 +642,12 @@ class MLFlowRepository(): prediction = model.predict(data) if retention == 0: + self.logger.info( + f"Deleting model {model_name} from memory") del model + gc.collect() + return prediction """ From 143dd7759ec890d0028fac7ba485d18042a3b6e2 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 13 Oct 2025 15:50:37 -0300 Subject: [PATCH 39/52] SIENTIAPDE-1231 Add memory management function to model_repository.py - Introduced `force_memory_release` function to enhance memory management by triggering garbage collection and attempting to release unused memory. - Utilized `ctypes` to call `malloc_trim` for further memory optimization, improving overall performance during model operations. --- laborious/utils/repository/model_repository.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 11944cc..904e271 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -29,12 +29,22 @@ from numpy import ndarray from typing import Any import gc from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ +import ctypes ARTIFACTS_PATH = "./tmp/artifacts" TRANSFORMED_COMPRESSED_PATH = "artifacts/training_transformer.pkl" PREDICTION_COMPRESSED_PATH = "artifacts/stacking_model.pkl" +def force_memory_release(): + gc.collect() + + try: + ctypes.CDLL("libc.so.6").malloc_trim(0) + except: + pass + + class MLFlowRepository(): def __init__(self, host: str, username: str, password: str, logger: Logger): From a3989b84af54500dc80a9dfea979347ae69ce3fe Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 13 Oct 2025 15:57:23 -0300 Subject: [PATCH 40/52] SIENTIAPDE-1231 Refactor memory management in MLFlowRepository to use `force_memory_release` function - Replaced direct calls to `gc.collect()` with `force_memory_release()` for improved memory optimization after model deletion. - This change enhances memory management during model operations, ensuring more efficient resource handling. --- laborious/utils/repository/model_repository.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 904e271..25345ce 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -36,13 +36,16 @@ TRANSFORMED_COMPRESSED_PATH = "artifacts/training_transformer.pkl" PREDICTION_COMPRESSED_PATH = "artifacts/stacking_model.pkl" -def force_memory_release(): +def force_memory_release(logger: Logger, metadata: dict): gc.collect() try: ctypes.CDLL("libc.so.6").malloc_trim(0) - except: - pass + logger.custom_info( + f"Memory released", metadata) + except Exception as e: + logger.custom_info( + f"Memory release failed: {e}", metadata) class MLFlowRepository(): @@ -627,7 +630,7 @@ class MLFlowRepository(): f"Deleting model {model_name} from memory") del model - gc.collect() + force_memory_release() return prediction @@ -656,7 +659,7 @@ class MLFlowRepository(): f"Deleting model {model_name} from memory") del model - gc.collect() + force_memory_release() return prediction From a5d2b0d3fda9425584bc44a5571df41a38957164 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 13 Oct 2025 15:59:10 -0300 Subject: [PATCH 41/52] SIENTIAPDE-1231 Refactor `force_memory_release` function in model_repository.py to improve logging - Updated the `force_memory_release` function to remove metadata parameter and enhance logging by using `logger.info()` instead of `logger.custom_info()`. - Adjusted calls to `force_memory_release` in the MLFlowRepository to pass the logger instance, ensuring consistent logging during memory management operations. --- laborious/utils/repository/model_repository.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 25345ce..691857a 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -36,16 +36,16 @@ TRANSFORMED_COMPRESSED_PATH = "artifacts/training_transformer.pkl" PREDICTION_COMPRESSED_PATH = "artifacts/stacking_model.pkl" -def force_memory_release(logger: Logger, metadata: dict): +def force_memory_release(logger: Logger): gc.collect() try: ctypes.CDLL("libc.so.6").malloc_trim(0) - logger.custom_info( - f"Memory released", metadata) + logger.info( + f"Memory released") except Exception as e: - logger.custom_info( - f"Memory release failed: {e}", metadata) + logger.info( + f"Memory release failed: {e}") class MLFlowRepository(): @@ -630,7 +630,7 @@ class MLFlowRepository(): f"Deleting model {model_name} from memory") del model - force_memory_release() + force_memory_release(self.logger) return prediction @@ -659,7 +659,7 @@ class MLFlowRepository(): f"Deleting model {model_name} from memory") del model - force_memory_release() + force_memory_release(self.logger) return prediction From ac795c7c5344f5f96e4191c41ece1843685cf0dd Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 15 Oct 2025 16:00:18 -0300 Subject: [PATCH 42/52] SIENTIAPDE-1231 Update .gitignore and refactor metrics.py for improved logging and consistency - Added coverage.xml to .gitignore to prevent tracking of coverage reports. - Refactored metric labels in metrics.py for consistency in string formatting and improved readability. - Enhanced logging messages in various activities to ensure uniformity in message formatting. --- .gitignore | 1 + laborious/activities/activities.py | 76 +- laborious/activities/gates.py | 320 +++-- laborious/activities/mlflow.py | 144 +- laborious/activities/opc.py | 119 +- laborious/activities/storage.py | 121 +- laborious/metrics.py | 38 +- laborious/utils/connectors_config.py | 24 +- .../utils/filters/conditional_filters.py | 3 +- laborious/utils/filters/mlflow_filters.py | 7 +- .../utils/repository/minio_repository.py | 90 +- .../utils/repository/model_repository.py | 765 ++++------ laborious/utils/repository/opc_repository.py | 182 ++- laborious/worker/worker.py | 77 +- laborious/workflows/minimal_retrain.py | 44 +- laborious/workflows/predictions_batch.py | 46 +- .../format_and_export_prediction.py | 42 +- .../sub_workflows/prediction_process.py | 55 +- mlruns/0/meta.yaml | 6 + mlruns/586524947870967910/meta.yaml | 6 + mlruns/models/test/meta.yaml | 5 + pyproject.toml | 159 +++ requirements-dev.txt | 19 + tests/laborious/activities/test_activities.py | 79 +- tests/laborious/activities/test_gates.py | 237 ++-- tests/laborious/activities/test_mlflow.py | 414 ++++-- tests/laborious/activities/test_opc.py | 339 ++--- tests/laborious/activities/test_storage.py | 203 +++ .../utils/filters/test_conditional_filters.py | 31 +- .../utils/filters/test_mlflow_filters.py | 11 +- .../utils/repository/test_minio_repository.py | 134 ++ .../utils/repository/test_model_repository.py | 1245 ++++++++++++----- .../utils/repository/test_opc_repository.py | 203 +-- .../laborious/utils/test_connectors_config.py | 15 +- .../test_format_and_export_prediction.py | 253 ++-- .../subworkflows/test_prediction_process.py | 720 ++++++---- .../workflows/test_minimal_retrain.py | 313 ++++- .../workflows/test_predictions_batch.py | 79 +- validate.sh | 99 ++ 39 files changed, 4122 insertions(+), 2602 deletions(-) create mode 100644 mlruns/0/meta.yaml create mode 100644 mlruns/586524947870967910/meta.yaml create mode 100644 mlruns/models/test/meta.yaml create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 tests/laborious/activities/test_storage.py create mode 100644 tests/laborious/utils/repository/test_minio_repository.py create mode 100755 validate.sh diff --git a/.gitignore b/.gitignore index 2c8240d..00fc074 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ __pycache__/ # Ignorar coverage htmlcov/ .coverage +coverage.xml # git keys git_key* diff --git a/laborious/activities/activities.py b/laborious/activities/activities.py index d028064..5487d2a 100644 --- a/laborious/activities/activities.py +++ b/laborious/activities/activities.py @@ -1,13 +1,15 @@ -from temporalio import activity, workflow +from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.storage import Storage + from typing import Any + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import Logger - from laborious.activities.mlflow import MLFlow + from laborious.activities.gates import Gates + from laborious.activities.mlflow import MLFlow from laborious.activities.opc import OPC - from typing import Any + from laborious.activities.storage import Storage class Activities(Storage, MLFlow, Gates, OPC): @@ -32,13 +34,15 @@ class Activities(Storage, MLFlow, Gates, OPC): notification_handler (NotificationHandler): Notification management instance """ - def __init__(self, - postgres_config: dict[str, Any], - mlflow_config: dict[str, Any], - minio_config: dict[str, Any], - opc_config: dict[str, Any], - logger: Logger, - notification_handler: NotificationHandler): + def __init__( + self, + postgres_config: dict[str, Any], + mlflow_config: dict[str, Any], + minio_config: dict[str, Any], + opc_config: dict[str, Any], + logger: Logger, + notification_handler: NotificationHandler, + ): """ Initialize the Activities orchestrator with all required configurations. @@ -59,32 +63,36 @@ class Activities(Storage, MLFlow, Gates, OPC): Exception: If any parent class initialization fails """ # Initialize parent classes - Storage.__init__(self, host=postgres_config['host'], - port=postgres_config['port'], - user=postgres_config['user'], - password=postgres_config['password'], - dbname=postgres_config['dbname'], - min_connections=postgres_config['min_connections'], - max_connections=postgres_config['max_connections'], - minio_config=minio_config, - logger=logger, - notification_handler=notification_handler) + Storage.__init__( + self, + host=postgres_config['host'], + port=postgres_config['port'], + user=postgres_config['user'], + password=postgres_config['password'], + dbname=postgres_config['dbname'], + min_connections=postgres_config['min_connections'], + max_connections=postgres_config['max_connections'], + minio_config=minio_config, + logger=logger, + notification_handler=notification_handler, + ) - MLFlow.__init__(self, mlflow_host=mlflow_config['host'], - mlflow_port=mlflow_config['port'], - mlflow_username=mlflow_config['username'], - mlflow_password=mlflow_config['password'], - minio_config=minio_config, - logger=logger, - notification_handler=notification_handler) + MLFlow.__init__( + self, + mlflow_host=mlflow_config['host'], + mlflow_port=mlflow_config['port'], + mlflow_username=mlflow_config['username'], + mlflow_password=mlflow_config['password'], + minio_config=minio_config, + logger=logger, + notification_handler=notification_handler, + ) - Gates.__init__(self, logger=logger, - notification_handler=notification_handler) + Gates.__init__(self, logger=logger, notification_handler=notification_handler) - OPC.__init__(self, - opc_servers=opc_config, - logger=logger, - notification_handler=notification_handler) + OPC.__init__( + self, opc_servers=opc_config, logger=logger, notification_handler=notification_handler + ) async def shutdown(self): """ diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index 7e6f785..0653161 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -1,55 +1,66 @@ from temporalio import activity, workflow - with workflow.unsafe.imports_passed_through(): import traceback - from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler - from sientia_do.notifications.models import NotificationLevel - from sientia_do.temporal.activities.base import BaseActivity - from sientia_do.observability.logger import Logger - from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now - from sientia_do.formatters import create_sample_dict - from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter - from typing import Any - from laborious.utils.filters.conditional_filters import ( - filter_empty_data, - filter_specific_variables_null_values - ) - from pandas import DataFrame - from laborious import metrics + from collections.abc import Callable, Mapping from os import path from shutil import rmtree + from typing import Any + + from pandas import DataFrame + from sientia_do.formatters import create_sample_dict + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.temporal.activities.base import BaseActivity + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now + + from laborious import metrics + from laborious.utils.filters.conditional_filters import ( + filter_empty_data, + filter_specific_variables_null_values, + ) + from laborious.utils.filters.mlflow_filters import api_error_filter, nan_values_filter + +# Strongly-typed filter function signatures +InputFilterFunc = Callable[[DataFrame, dict[str, Any]], bool] +ResponseFilterFunc = Callable[[dict[str, Any], dict[str, Any]], bool] +ContentFilterFunc = Callable[[DataFrame, dict[str, Any]], bool] # Input filter function mappings -input_filter_functions = { +input_filter_functions: dict[str, InputFilterFunc] = { 'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values, 'EMPTY_DATA': filter_empty_data, - 'path_confidence': { - 'STOP': -1, - 'CONTINUE': 2, - 'REPEAT': -1 - } +} + +# Confidence mappings kept separate from function maps to avoid Union types +input_path_confidence: Mapping[str, int] = { + 'STOP': -1, + 'CONTINUE': 2, + 'REPEAT': -1, } # MLFlow response filter function mappings -mlflow_response_filter_functions = { +mlflow_response_filter_functions: dict[str, ResponseFilterFunc] = { 'API_ERROR': api_error_filter, - 'path_confidence': { - 'STOP': -1, - 'CONTINUE': 10, - 'REPEAT': -1 - }, +} + +mlflow_response_path_confidence: Mapping[str, int] = { + 'STOP': -1, + 'CONTINUE': 10, + 'REPEAT': -1, } # MLFlow content filter function mappings -mlflow_content_filter_functions = { +mlflow_content_filter_functions: dict[str, ContentFilterFunc] = { 'NAN_VALUES': nan_values_filter, 'EMPTY_DATA': filter_empty_data, - 'path_confidence': { - 'STOP': -1, - 'CONTINUE': 18, - 'REPEAT': -1 - } +} + +mlflow_content_path_confidence: Mapping[str, int] = { + 'STOP': -1, + 'CONTINUE': 18, + 'REPEAT': -1, } @@ -84,10 +95,9 @@ class Gates(BaseActivity): Raises: Exception: If BaseActivity initialization fails """ - BaseActivity.__init__( - self, logger, notification_handler, set_error_counter=True) + BaseActivity.__init__(self, logger, notification_handler, set_error_counter=True) - @activity.defn(name="input_gate") + @activity.defn(name='input_gate') async def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]: """ Apply input data quality filters and validation. @@ -123,7 +133,7 @@ class Gates(BaseActivity): """ metadata = input_data['metadata'] - self.info("Performing input gate...", metadata) + self.info('Performing input gate...', metadata) filters = input_data['filters'] data = DataFrame(input_data['data']) @@ -131,40 +141,38 @@ class Gates(BaseActivity): filter_output = [] - self.debug(f"Input data: {data.head(5).to_string()}", metadata) - self.debug(f"Filters: {filters}", metadata) + self.debug(f'Input data: {data.head(5).to_string()}', metadata) + self.debug(f'Filters: {filters}', metadata) # Apply each configured filter for fil, config in filters.items(): if fil not in input_filter_functions: - self.error(f"Filter {fil} not found", metadata) + self.error(f'Filter {fil} not found', metadata) continue try: if input_filter_functions[fil](data, config['config']): - self.debug( - f"Data not passed the input filter {fil}:{config}", metadata) + self.debug(f'Data not passed the input filter {fil}:{config}', metadata) filter_output.append(config['policy']) except Exception as e: trace = traceback.format_exc() self.send_notification( metadata=metadata, - notification_id=f"INTPUT_GATE_ERROR__{fil}", - message=f"Error in filter {fil}:{config}: \n {e}", - block="input_gate", + notification_id=f'INTPUT_GATE_ERROR__{fil}', + message=f'Error in filter {fil}:{config}: \n {e}', + block='input_gate', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) for path_flag in path_priority: if path_flag in filter_output: - self.info(f"Input gate result: {path_flag}", metadata) - return path_flag, input_filter_functions['path_confidence'][path_flag], \ - "Input data with bad quality" + self.info(f'Input gate result: {path_flag}', metadata) + return path_flag, input_path_confidence[path_flag], 'Input data with bad quality' - self.info("Nothing was filtered by the input gate", metadata) - return None, 0, "" + self.info('Nothing was filtered by the input gate', metadata) + return None, 0, '' - @activity.defn(name="mlflow_response_gate") + @activity.defn(name='mlflow_response_gate') async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]: """ Validate MLFlow API response quality and integrity. @@ -199,7 +207,7 @@ class Gates(BaseActivity): Exception: If response validation fails or configuration is invalid """ metadata = input_data['metadata'] - self.info("Performing mlflow response gate...", metadata) + self.info('Performing mlflow response gate...', metadata) filters = input_data['filters'] data = input_data['data'] @@ -208,9 +216,8 @@ class Gates(BaseActivity): filter_output = [] - self.debug( - f"Input data: \n {create_sample_dict(data, max_items=5, max_depth=5)}", metadata) - self.debug(f"Filters: {filters}", metadata) + self.debug(f'Input data: \n {create_sample_dict(data, max_items=5, max_depth=5)}', metadata) + self.debug(f'Filters: {filters}', metadata) comments = [] for fil, config in filters.items(): @@ -222,34 +229,32 @@ class Gates(BaseActivity): comments.append(data['content']['message']) self.send_notification( metadata=metadata, - notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}", + notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}', message=data['content']['message'], - block="mlflow_gate", + block='mlflow_gate', level=NotificationLevel.ERROR, - attachment_content=data['content']['traceback'] + attachment_content=data['content']['traceback'], ) except Exception as e: trace = traceback.format_exc() self.send_notification( metadata=metadata, - notification_id=f"MLFLOW_GATE_RESPONSE_FILTER__{fil}", - message=f"Error in filter {fil}:{config}: \n {e}", - block="mlflow_gate", + notification_id=f'MLFLOW_GATE_RESPONSE_FILTER__{fil}', + message=f'Error in filter {fil}:{config}: \n {e}', + block='mlflow_gate', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) for path_flag in path_priority: if path_flag in filter_output: - self.info( - f"Mlflow response gate result: {path_flag}", metadata) - return path_flag, mlflow_response_filter_functions['path_confidence'][path_flag], \ - ", ".join(comments) + self.info(f'Mlflow response gate result: {path_flag}', metadata) + return path_flag, mlflow_response_path_confidence[path_flag], ', '.join(comments) - self.info("Nothing was filtered by the mlflow response gate", metadata) - return None, 0, "" + self.info('Nothing was filtered by the mlflow response gate', metadata) + return None, 0, '' - @activity.defn(name="mlflow_content_gate") + @activity.defn(name='mlflow_content_gate') async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]: """ Validate MLFlow prediction content quality and integrity. @@ -284,7 +289,7 @@ class Gates(BaseActivity): Exception: If content validation fails or configuration is invalid """ metadata = input_data['metadata'] - self.info("Performing mlflow content gate...", metadata) + self.info('Performing mlflow content gate...', metadata) filters = input_data['filters'] data = DataFrame(input_data['data']) @@ -293,8 +298,8 @@ class Gates(BaseActivity): filter_output = [] - self.debug(f"Input data:\n {data.head(5).to_string()}", metadata) - self.debug(f"Filters: \n {filters}", metadata) + self.debug(f'Input data:\n {data.head(5).to_string()}', metadata) + self.debug(f'Filters: \n {filters}', metadata) for fil, config in filters.items(): if fil not in mlflow_content_filter_functions: @@ -304,36 +309,38 @@ class Gates(BaseActivity): filter_output.append(config['policy']) self.send_notification( metadata=metadata, - notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}", - message=f"Data not passed the content filter {fil}:{config}", - block="mlflow_gate", + notification_id=f'{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}', + message=f'Data not passed the content filter {fil}:{config}', + block='mlflow_gate', level=NotificationLevel.WARNING, - attachment_content=data.to_string() + attachment_content=data.to_string(), ) except Exception as e: trace = traceback.format_exc() self.send_notification( metadata=metadata, - notification_id=f"MLFLOW_GATE_CONTENT_FILTER__{fil}", - message=f"Error in filter {fil}:{config}: \n {e}", - block="mlflow_gate", + notification_id=f'MLFLOW_GATE_CONTENT_FILTER__{fil}', + message=f'Error in filter {fil}:{config}: \n {e}', + block='mlflow_gate', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) for path_flag in path_priority: if path_flag in filter_output: - self.info( - f"Mlflow content gate result: {path_flag}", metadata) - return path_flag, mlflow_content_filter_functions['path_confidence'][path_flag], \ - "Transformed data not passed the content filter" + self.info(f'Mlflow content gate result: {path_flag}', metadata) + return ( + path_flag, + mlflow_content_path_confidence[path_flag], + 'Transformed data not passed the content filter', + ) - self.info("Nothing was filtered by the mlflow content gate", metadata) - return None, 0, "" + self.info('Nothing was filtered by the mlflow content gate', metadata) + return None, 0, '' - def get_prediction_store_policy(self, - prediction_store_policy: str, - metadata: dict[str, Any]) -> tuple[str, int]: + def get_prediction_store_policy( + self, prediction_store_policy: str, metadata: dict[str, Any] + ) -> tuple[str, int]: """ Parse and validate prediction store policy configuration. @@ -359,7 +366,9 @@ class Gates(BaseActivity): if len(policy_elements) < 2: self.error( - f"Invalid prediction store policy: {prediction_store_policy}, using default policy", metadata) + f'Invalid prediction store policy: {prediction_store_policy}, using default policy', + metadata, + ) return 'lts', 1 policy_type = policy_elements[0] @@ -367,14 +376,20 @@ class Gates(BaseActivity): # If the policy_type is not lts or erl, we use the default policy # If the policty_value is not a number or 0, we use the default policy - if policy_type not in ['lts', 'erl'] or not policy_value.isdigit() or int(policy_value) == 0: + if ( + policy_type not in ['lts', 'erl'] + or not policy_value.isdigit() + or int(policy_value) == 0 + ): self.error( - f"Invalid prediction store policy: {prediction_store_policy}, using default policy", metadata) + f'Invalid prediction store policy: {prediction_store_policy}, using default policy', + metadata, + ) return 'lts', 1 return policy_type, int(policy_value) - @activity.defn(name="format_prediction") + @activity.defn(name='format_prediction') async def format_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]: """ Format prediction data according to configured storage policies. @@ -401,7 +416,7 @@ class Gates(BaseActivity): """ metadata = input_data['metadata'] prediction_store_policy = input_data['prediction_store_policy'] - self.info("Formatting prediction...", metadata) + self.info('Formatting prediction...', metadata) data = DataFrame(input_data['data']) @@ -409,48 +424,45 @@ class Gates(BaseActivity): data['timestamp'] = data.index data = data.reset_index(drop=True) - self.debug( - f"Prediction store policy: {prediction_store_policy}", metadata) - self.debug(f"Prediction data: {data.head(5).to_string()}", metadata) + self.debug(f'Prediction store policy: {prediction_store_policy}', metadata) + self.debug(f'Prediction data: {data.head(5).to_string()}', metadata) policy_type, policy_value = self.get_prediction_store_policy( - prediction_store_policy, metadata) + prediction_store_policy, metadata + ) # If data has no timestamp, we use the default timestamp and not sort the data self.info( - f"Sorting data by timestamp and applying policy: {policy_type}:{policy_value}", metadata) + f'Sorting data by timestamp and applying policy: {policy_type}:{policy_value}', metadata + ) # If policy_type is lts, we need to sort the data by timestamp descending and take the first policy_value rows if policy_type == 'lts': - self.debug( - "Sorting data by timestamp descending", metadata) + self.debug('Sorting data by timestamp descending', metadata) data = data.sort_values(by='timestamp', ascending=False) # If policy_type is erl, we need to sort the data by timestamp ascending and take the first policy_value rows elif policy_type == 'erl': - self.debug( - "Sorting data by timestamp ascending", metadata) + self.debug('Sorting data by timestamp ascending', metadata) data = data.sort_values(by='timestamp', ascending=True) else: - self.error( - f"Invalid policy type: {policy_type}, using default policy", metadata) - raise ValueError( - f"Invalid policy type: {policy_type}") + self.error(f'Invalid policy type: {policy_type}, using default policy', metadata) + raise ValueError(f'Invalid policy type: {policy_type}') data = data.head(int(policy_value)) data['model_id'] = input_data['model_id'] data['prediction_confidence'] = input_data['prediction_confidence'] data['prediction_status'] = 'Good' - data['comments'] = "" + data['comments'] = '' data = data.sort_values(by='timestamp', ascending=False) data = data.reset_index(drop=True) - self.info(f"Prediction formatted: {len(data)} rows", metadata) - self.debug(f"Prediction data: {data.head(5).to_string()}", metadata) + self.info(f'Prediction formatted: {len(data)} rows', metadata) + self.debug(f'Prediction data: {data.head(5).to_string()}', metadata) return data.to_dict() - @activity.defn(name="format_default_prediction") + @activity.defn(name='format_default_prediction') async def format_default_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]: """ Create and format default prediction data for error conditions. @@ -478,40 +490,44 @@ class Gates(BaseActivity): """ metadata = input_data['metadata'] - self.debug("Formatting default prediction...", metadata) + self.debug('Formatting default prediction...', metadata) - data = DataFrame({ - 'prediction': [0], - 'response_time': [0], - 'timestamp': [input_data['timestamp']], - 'model_id': [input_data['model_id']], - 'prediction_confidence': [input_data['prediction_confidence']], - 'prediction_status': ['Bad'], - 'comments': [input_data['comment']] - }) + data = DataFrame( + { + 'prediction': [0], + 'response_time': [0], + 'timestamp': [input_data['timestamp']], + 'model_id': [input_data['model_id']], + 'prediction_confidence': [input_data['prediction_confidence']], + 'prediction_status': ['Bad'], + 'comments': [input_data['comment']], + } + ) - self.info(f"Default prediction formatted: {data.size} rows", metadata) + self.info(f'Default prediction formatted: {data.size} rows', metadata) return data.to_dict() - @activity.defn(name="format_retrain_report") + @activity.defn(name='format_retrain_report') async def format_retrain_report(self, input_data: dict[str, Any]) -> dict[Any, Any]: """ Format retrain report data according to configured storage policies. """ metadata = input_data['metadata'] - self.info("Formatting retrain report...", metadata) + self.info('Formatting retrain report...', metadata) experiment_response = input_data['experiment_response'] update_report = input_data['update_report'] model_id = input_data['model_id'] model_name = input_data['model_name'] - report = DataFrame({ - 'model_id': [model_id], - 'model_name': [model_name], - 'timestamp': [experiment_response['timestamp']], - 'status': [experiment_response['message']] - }) + report = DataFrame( + { + 'model_id': [model_id], + 'model_name': [model_name], + 'timestamp': [experiment_response['timestamp']], + 'status': [experiment_response['message']], + } + ) if experiment_response['success']: # Retrain was successfull @@ -519,11 +535,11 @@ class Gates(BaseActivity): report['mlflow_run_id'] = update_report['mlflow_run_id'] report['mlflow_experiment_id'] = update_report['mlflow_experiment_id'] - self.debug(f"Retrain report: {report.to_csv()}", metadata) + self.debug(f'Retrain report: {report.to_csv()}', metadata) return report.to_dict() - @activity.defn(name="get_last_timestamp") + @activity.defn(name='get_last_timestamp') async def get_last_timestamp(self, input_data: dict[str, Any]) -> str: """ Extract the most recent timestamp from prediction data. @@ -548,24 +564,22 @@ class Gates(BaseActivity): """ metadata = input_data['metadata'] - self.info("Getting last timestamp...", metadata) + self.info('Getting last timestamp...', metadata) data = DataFrame(input_data['data']) - self.debug(f"Input data: {data.head(5).to_string()}", metadata) + self.debug(f'Input data: {data.head(5).to_string()}', metadata) if data.empty: return now().strftime(DATETIME_FORMAT_WITH_TZ) - max_timestamp = max( - data['timestamp'].values.tolist()) + max_timestamp = max(data['timestamp'].values.tolist()) - self.info( - f"Last timestamp: {max_timestamp}", metadata) + self.info(f'Last timestamp: {max_timestamp}', metadata) return max_timestamp - @activity.defn(name="write_metrics") + @activity.defn(name='write_metrics') async def write_metrics(self, input_data: dict[str, Any]): """ Write prediction performance metrics to Prometheus monitoring system. @@ -593,42 +607,40 @@ class Gates(BaseActivity): prediction_confidence = prediction['prediction_confidence'].values[0] response_time = prediction['response_time'].values[0] - self.info( - f"Writing metrics for model {metadata['model_name']}", metadata) + self.info(f'Writing metrics for model {metadata["model_name"]}', metadata) metrics.PREDICTIONS_WRITTEN_COUNT.labels( pod_id=self.pod_id, model_name=metadata['model_name'], - pipeline_name=metadata['workflow_name'] + pipeline_name=metadata['workflow_name'], ).inc() metrics.PREDICTION_CONFIDENCE_MONITOR.labels( pod_id=self.pod_id, model_name=metadata['model_name'], - pipeline_name=metadata['workflow_name'] + pipeline_name=metadata['workflow_name'], ).set(prediction_confidence) metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels( pod_id=self.pod_id, model_name=metadata['model_name'], - pipeline_name=metadata['workflow_name'] + pipeline_name=metadata['workflow_name'], ).observe(response_time) - self.info( - f"Metrics written for model {metadata['model_name']}", metadata) + self.info(f'Metrics written for model {metadata["model_name"]}', metadata) - @activity.defn(name="clean_tmp_files") + @activity.defn(name='clean_tmp_files') async def clean_tmp_files(self, input_data: dict[str, Any]): """ Clean temporary files in the tmp directory. """ model_name = input_data['model_name'] metadata = input_data['metadata'] - self.info(f"Cleaning tmp files for model {model_name}...", metadata) + self.info(f'Cleaning tmp files for model {model_name}...', metadata) - if path.exists(f"tmp/retrain_data/{model_name}"): - rmtree(f"tmp/retrain_data/{model_name}") - if path.exists(f"tmp/artifacts/{model_name}"): - rmtree(f"tmp/artifacts/{model_name}") + if path.exists(f'tmp/retrain_data/{model_name}'): + rmtree(f'tmp/retrain_data/{model_name}') + if path.exists(f'tmp/artifacts/{model_name}'): + rmtree(f'tmp/artifacts/{model_name}') - self.info("Tmp files cleaned", metadata) + self.info('Tmp files cleaned', metadata) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 822cd69..a289c9f 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -1,21 +1,25 @@ -from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ, now from temporalio import activity, workflow - with workflow.unsafe.imports_passed_through(): - from pandas import to_datetime - from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ - from sientia_do.temporal.activities.base import BaseActivity + import traceback + from typing import Any + + import numpy as np + from pandas import DataFrame, to_datetime + from sientia_do.formatters import create_sample_dict from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger - from sientia_do.formatters import create_sample_dict - from laborious.utils.repository.model_repository import MLFlowRepository - from typing import Any - import numpy as np - from pandas import DataFrame - import traceback + from sientia_do.temporal.activities.base import BaseActivity + from sientia_do.temporal.constants import ( + DATETIME_FORMAT, + DATETIME_FORMAT_MS_WITH_TZ, + DATETIME_FORMAT_WITH_TZ, + now, + ) + from laborious.utils.repository.minio_repository import MinioRepository + from laborious.utils.repository.model_repository import MLFlowRepository class MLFlow(BaseActivity): @@ -37,9 +41,16 @@ class MLFlow(BaseActivity): model_monitoring_repository (MLFlowRepository): Repository for MLFlow operations """ - def __init__(self, mlflow_host: str, mlflow_port: int, mlflow_username: str, - minio_config: dict[str, Any], mlflow_password: str, - logger: Logger, notification_handler: NotificationHandler): + def __init__( + self, + mlflow_host: str, + mlflow_port: int, + mlflow_username: str, + minio_config: dict[str, Any], + mlflow_password: str, + logger: Logger, + notification_handler: NotificationHandler, + ): """ Initialize MLFlow activities with server configuration. @@ -54,26 +65,18 @@ class MLFlow(BaseActivity): Raises: Exception: If MLFlowRepository initialization fails """ - BaseActivity.__init__( - self, logger, notification_handler, set_error_counter=True) + BaseActivity.__init__(self, logger, notification_handler, set_error_counter=True) self.mlflow_host = mlflow_host self.mlflow_port = mlflow_port self.mlflow_username = mlflow_username self.mlflow_password = mlflow_password self.model_monitoring_repository = MLFlowRepository( - f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger + f'{mlflow_host}:{mlflow_port}', mlflow_username, mlflow_password, logger ) if not hasattr(self, 'minio_repository'): - self.minio_repository = MinioRepository( - logger=logger, - notification_handler=notification_handler, - minio_endpoint_url=minio_config['endpoint_url'], - minio_access_key=minio_config['access_key'], - minio_secret_key=minio_config['secret_key'], - minio_region_name=minio_config['region_name'], - minio_default_bucket=minio_config['default_bucket']) + self.minio_repository: MinioRepository | None = None if self.minio_repository is None: self.minio_repository = MinioRepository( @@ -83,9 +86,10 @@ class MLFlow(BaseActivity): minio_access_key=minio_config['access_key'], minio_secret_key=minio_config['secret_key'], minio_region_name=minio_config['region_name'], - minio_default_bucket=minio_config['default_bucket']) + minio_default_bucket=minio_config['default_bucket'], + ) - @activity.defn(name="request_transform") + @activity.defn(name='request_transform') async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]: """ Transform input data using MLFlow models. @@ -121,7 +125,7 @@ class MLFlow(BaseActivity): model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) - self.debug("Raw input data:", metadata) + self.debug('Raw input data:', metadata) self.debug(data.head(5).to_string(), metadata) # Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair @@ -130,14 +134,12 @@ class MLFlow(BaseActivity): ) # Pivot data for model input format - data = data.pivot( - index='timestamp', columns='variable', - values='value') + data = data.pivot(index='timestamp', columns='variable', values='value') data.fillna(np.nan, inplace=True) # data.reset_index(inplace=True) data.columns.name = None - self.debug("Processed input data:", metadata) + self.debug('Processed input data:', metadata) self.debug(data.head(5).to_string(), metadata) # Request transformation from MLFlow model @@ -146,16 +148,20 @@ class MLFlow(BaseActivity): ) self.debug( - f"Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) + f'Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}', + metadata, + ) self.debug( - f"Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) + f'Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}', + metadata, + ) - self.info("Data transformed successfully", metadata) + self.info('Data transformed successfully', metadata) return response_data - @activity.defn(name="request_predict") + @activity.defn(name='request_predict') async def request_predict(self, input_data: dict[str, Any]) -> dict[str, Any]: """ Execute predictions using MLFlow models. @@ -191,14 +197,15 @@ class MLFlow(BaseActivity): model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) - self.debug(f"Input data for: \n {data.head(5).to_string()}", metadata) + self.debug(f'Input data for: \n {data.head(5).to_string()}', metadata) # Convert numpy.nan to None for model compatibility data.replace(np.nan, None, inplace=True) data['timestamp'] = data.index data['timestamp'] = to_datetime( - data['timestamp'], format=DATETIME_FORMAT_WITH_TZ).dt.strftime(DATETIME_FORMAT) + data['timestamp'], format=DATETIME_FORMAT_WITH_TZ + ).dt.strftime(DATETIME_FORMAT) # Request prediction from MLFlow model response_data = self.model_monitoring_repository.predict( @@ -206,13 +213,15 @@ class MLFlow(BaseActivity): ) self.debug( - f"Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) + f'Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}', + metadata, + ) - self.info("Data predicted successfully", metadata) + self.info('Data predicted successfully', metadata) return response_data - @activity.defn(name="retrain_model") + @activity.defn(name='retrain_model') async def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]: """ Retrain MLFlow models with updated training data. @@ -244,15 +253,19 @@ class MLFlow(BaseActivity): Raises: Exception: If retraining fails or encounters critical errors """ + + if self.minio_repository is None: + raise ValueError('Minio repository not initialized') + metadata = input_data['metadata'] object_key = input_data['object_key'] self.info(f'Loading retrain data from Key: {object_key}', metadata) try: - data = self.minio_repository.get_parquet_as_dataframe( - object_key=object_key, metadata=metadata) + object_key=object_key, metadata=metadata + ) except Exception as e: trace = traceback.format_exc() self.send_notification( @@ -261,18 +274,17 @@ class MLFlow(BaseActivity): message=f'Error loading retrain data: {e}', block='retrain_model', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) self.error(trace, metadata) return { 'success': False, 'message': f'Error loading retrain data: {e}', 'traceback': trace, - 'timestamp': now().strftime(DATETIME_FORMAT_MS_WITH_TZ) + 'timestamp': now().strftime(DATETIME_FORMAT_MS_WITH_TZ), } - self.debug( - f'Retrain data loaded successfully: shape {data.shape}', metadata) + self.debug(f'Retrain data loaded successfully: shape {data.shape}', metadata) model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) @@ -291,47 +303,38 @@ class MLFlow(BaseActivity): data.drop(columns=['created_at'], inplace=True, errors='ignore') # Pivot data for model input format - data = data.pivot( - index='timestamp', columns='variable', - values='value') + data = data.pivot(index='timestamp', columns='variable', values='value') data.fillna(np.nan, inplace=True) # data.reset_index(inplace=True) data.columns.name = None data['timestamp'] = data.index data['timestamp'] = to_datetime( - data['timestamp'], format=DATETIME_FORMAT_WITH_TZ).dt.strftime(DATETIME_FORMAT) - data['timestamp'] = to_datetime( - data['timestamp'], format=DATETIME_FORMAT) + data['timestamp'], format=DATETIME_FORMAT_WITH_TZ + ).dt.strftime(DATETIME_FORMAT) + data['timestamp'] = to_datetime(data['timestamp'], format=DATETIME_FORMAT) data.columns.name = None retrain_output = self.model_monitoring_repository.retrain_model( - data=data, - model_name=model_name, - model_config=model_config, - metadata=metadata + data=data, model_name=model_name, model_config=model_config, metadata=metadata ) if not retrain_output['success']: - trace = retrain_output['traceback'] self.send_notification( metadata=metadata, notification_id='RETRAIN_MODEL_ERROR', - message=f"Error retraining model {model_name}: {retrain_output['message']}", + message=f'Error retraining model {model_name}: {retrain_output["message"]}', block='retrain_model', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) self.error(trace, metadata=metadata) - return { - **retrain_output, - 'timestamp': timestamp - } + return {**retrain_output, 'timestamp': timestamp} - @activity.defn(name="update_production_model") + @activity.defn(name='update_production_model') async def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]: """ Update production model with newly trained model version. @@ -372,16 +375,15 @@ class MLFlow(BaseActivity): model_name = input_data['model_name'] experiment = input_data['experiment'] self.info( - f'Updating production model {model_name} from experiment {experiment}...', metadata) + f'Updating production model {model_name} from experiment {experiment}...', metadata + ) try: response = self.model_monitoring_repository.update_production_model( - experiment=experiment, - model_name=model_name + experiment=experiment, model_name=model_name, metadata=metadata ) - self.info( - f'Production model {model_name} updated successfully', metadata) + self.info(f'Production model {model_name} updated successfully', metadata) return response except Exception as e: @@ -392,7 +394,7 @@ class MLFlow(BaseActivity): message=f'Error updating production model {model_name}: {e}', block='update_production_model', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) self.error(trace, metadata=metadata) raise e diff --git a/laborious/activities/opc.py b/laborious/activities/opc.py index 2c427e0..b4604d8 100644 --- a/laborious/activities/opc.py +++ b/laborious/activities/opc.py @@ -1,15 +1,16 @@ from temporalio import activity, workflow - with workflow.unsafe.imports_passed_through(): + import traceback + from typing import Any + + from pandas import DataFrame from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel - from sientia_do.temporal.activities.base import BaseActivity from sientia_do.observability.logger import Logger + from sientia_do.temporal.activities.base import BaseActivity + from laborious.utils.repository.opc_repository import OpcRepository - from typing import Any - import traceback - from pandas import DataFrame OPC_WRITTING_ERROR_CONFIDENCE = 12 @@ -33,15 +34,17 @@ class OPC(BaseActivity): notification_handler (NotificationHandler): Notification management instance """ - def __init__(self, opc_servers: dict[str, dict[str, Any]], - logger: Logger, notification_handler: NotificationHandler): - + def __init__( + self, + opc_servers: dict[str, dict[str, Any]], + logger: Logger, + notification_handler: NotificationHandler, + ): self.logger = logger self.notification_handler = notification_handler self.opc_servers = opc_servers - BaseActivity.__init__( - self, logger, notification_handler, set_error_counter=True) + BaseActivity.__init__(self, logger, notification_handler, set_error_counter=True) self.opc_repository: dict[str, OpcRepository] = {} self.opc_servers = opc_servers @@ -70,10 +73,10 @@ class OPC(BaseActivity): the initialization of other OPC servers. Each server is handled independently to ensure maximum availability. """ - self.logger.info("Initializing OPC servers...") - for id, server in self.opc_servers.items(): - self.opc_repository[id] = OpcRepository( - id=server['id'], + self.logger.info('Initializing OPC servers...') + for opc_id, server in self.opc_servers.items(): + self.opc_repository[opc_id] = OpcRepository( + opc_id=server['id'], url=server['url'], logger=self.logger, server_uri=server['server_uri'], @@ -82,30 +85,35 @@ class OPC(BaseActivity): server_cert_path=server['server_cert_path'], notification_handler=self.notification_handler, reconnection_interval=server['reconnection_interval'], - pod_id=self.pod_id + pod_id=self.pod_id, ) - is_connected, error_data = await self.opc_repository[id].connect() + is_connected, error_data = await self.opc_repository[opc_id].connect() if not is_connected: self.send_notification( metadata={ 'model_id': '-', 'model_name': '-', 'workflow_name': '-', - 'schedule_name': 'INITIALIZATION' + 'schedule_name': 'INITIALIZATION', }, notification_id=error_data['notification_id'], message=error_data['message'], block=error_data['block'], level=error_data.get('level', NotificationLevel.ERROR), - attachment_content=error_data.get( - 'attachment_content', None) + attachment_content=error_data.get('attachment_content', None), ) else: - self.logger.info( - f"OPC server {id} connected successfully.") + self.logger.info(f'OPC server {opc_id} connected successfully.') - async def write_data(self, server_id: str, tag: str, data: Any, - data_type: str, tag_type: str, metadata: dict[str, Any]) -> bool: + async def write_data( + self, + server_id: str, + tag: str, + data: Any, + data_type: str, + tag_type: str, + metadata: dict[str, Any], + ) -> bool: """ Write data to a specific OPC server tag with comprehensive error handling. @@ -127,7 +135,8 @@ class OPC(BaseActivity): try: is_success, error_data = await self.opc_repository[server_id].write_data( - tag, data, data_type, self.logger, metadata) + tag, data, data_type, self.logger, metadata + ) if not is_success: self.send_notification( metadata=metadata, @@ -135,8 +144,7 @@ class OPC(BaseActivity): message=error_data['message'], block=error_data['block'], level=error_data.get('level', NotificationLevel.ERROR), - attachment_content=error_data.get( - 'attachment_content', None) + attachment_content=error_data.get('attachment_content', None), ) return False return True @@ -144,11 +152,11 @@ class OPC(BaseActivity): trace = traceback.format_exc() self.send_notification( metadata=metadata, - notification_id=f"WRITE_OPC_{tag_type.upper()}_ERROR", - message=f"Error writing data to OPC server: {e}", - block="write_opc_data", + notification_id=f'WRITE_OPC_{tag_type.upper()}_ERROR', + message=f'Error writing data to OPC server: {e}', + block='write_opc_data', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) raise e @@ -174,21 +182,26 @@ class OPC(BaseActivity): This helps operators quickly identify configuration issues. """ if self.opc_repository.get(server_id) is None: - message = f"OPC server {server_id} not found to perform write operation." + message = f'OPC server {server_id} not found to perform write operation.' self.send_notification( metadata=metadata, - notification_id="OPC_SERVER_NOT_FOUND", + notification_id='OPC_SERVER_NOT_FOUND', message=message, - block="write_opc_data", + block='write_opc_data', level=NotificationLevel.ERROR, - attachment_content=f"OPC servers: {list(self.opc_repository.keys())}" + attachment_content=f'OPC servers: {list(self.opc_repository.keys())}', ) return False return True async def manage_output_tags( - self, server_id: str, config: dict[str, Any], data: DataFrame, - metadata: dict[str, Any], success: bool) -> tuple[bool, int]: + self, + server_id: str, + config: dict[str, Any], + data: DataFrame, + metadata: dict[str, Any], + success: bool, + ) -> tuple[bool, int]: """ Manage the writing of prediction and confidence data to OPC server tags. @@ -225,11 +238,13 @@ class OPC(BaseActivity): data=data.head(1)['prediction'].values[0], data_type=tag_config['data_type'], tag_type='prediction', - metadata=metadata + metadata=metadata, ) if local_success: self.info( - f"Prediction data written to OPC server {server_id} for tag {tag}.", metadata) + f'Prediction data written to OPC server {server_id} for tag {tag}.', + metadata, + ) count += 1 success = success and local_success @@ -241,11 +256,13 @@ class OPC(BaseActivity): data=data.head(1)['prediction_confidence'].values[0], data_type=tag_config['data_type'], tag_type='confidence', - metadata=metadata + metadata=metadata, ) if local_success: self.info( - f"Confidence data written to OPC server {server_id} for tag {tag}.", metadata) + f'Confidence data written to OPC server {server_id} for tag {tag}.', + metadata, + ) count += 1 success = success and local_success @@ -271,29 +288,33 @@ class OPC(BaseActivity): """ metadata = input_data['metadata'] - self.info("Writing data to OPC servers...", metadata) + self.info('Writing data to OPC servers...', metadata) data = DataFrame(input_data['data']) opc_output_config = input_data['opc_output_config'] - self.info(f"Data to write: {data.size} rows", metadata) + self.info(f'Data to write: {data.size} rows', metadata) success = True for server_id, config in opc_output_config.items(): - if not self.validate_server(server_id, metadata): success = False continue local_success, local_count = await self.manage_output_tags( - server_id, config, data, metadata, success) + server_id, config, data, metadata, success + ) success = success and local_success self.info( - f"Process completed for OPC server {server_id}: {local_count} of {len(config.get('prediction_tags', []))} prediction tags and {len(config.get('confidence_tags', []))} confidence tags", metadata) + f'Process completed for OPC server {server_id}: {local_count} of {len(config.get("prediction_tags", []))} prediction tags and {len(config.get("confidence_tags", []))} confidence tags', + metadata, + ) return self.process_confidence(data, success, metadata) - def process_confidence(self, data: DataFrame, success: bool, metadata: dict[str, Any]) -> dict[Any, Any]: + def process_confidence( + self, data: DataFrame, success: bool, metadata: dict[str, Any] + ) -> dict[Any, Any]: """ Process prediction confidence based on OPC write operation success. @@ -323,12 +344,12 @@ class OPC(BaseActivity): if not success: data['prediction_confidence'] = OPC_WRITTING_ERROR_CONFIDENCE self.debug( - f"Some data could not be written to OPC servers, setting confidence to {OPC_WRITTING_ERROR_CONFIDENCE}.", - metadata + f'Some data could not be written to OPC servers, setting confidence to {OPC_WRITTING_ERROR_CONFIDENCE}.', + metadata, ) else: - self.debug("Data written to OPC servers successfully.", metadata) + self.debug('Data written to OPC servers successfully.', metadata) return data.to_dict() diff --git a/laborious/activities/storage.py b/laborious/activities/storage.py index ef61fb4..8aec756 100644 --- a/laborious/activities/storage.py +++ b/laborious/activities/storage.py @@ -1,20 +1,21 @@ from temporalio import activity, workflow -from laborious.utils.repository.minio_repository import MinioRepository - - with workflow.unsafe.imports_passed_through(): # Extend the Temporal Postgres activities for convenient query -> MinIO export - from sientia_do.temporal.activities.postgres import Postgres - from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler - from sientia_do.observability.logger import Logger - from sientia_do.temporal.constants import now - from sientia_do.notifications.models import NotificationLevel - from typing import Any import traceback - import pandas as pd + from typing import Any -DATETIME_FILENAME_FORMAT = "%Y-%m-%d_%H-%M-%S" + import pandas as pd + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.temporal.activities.postgres import Postgres + from sientia_do.temporal.constants import now + + from laborious.utils.repository.minio_repository import MinioRepository + + +DATETIME_FILENAME_FORMAT = '%Y-%m-%d_%H-%M-%S' class Storage(Postgres): @@ -23,36 +24,33 @@ class Storage(Postgres): directly to MinIO as Parquet and return the object name. """ - def __init__(self, - host: str, - port: int, - user: str, - password: str, - dbname: str, - min_connections: int, - max_connections: int, - minio_config: dict[str, Any], - logger: Logger, - notification_handler: NotificationHandler): - super().__init__(host=host, - port=port, - user=user, - password=password, - dbname=dbname, - min_connections=min_connections, - max_connections=max_connections, - logger=logger, - notification_handler=notification_handler) + def __init__( + self, + host: str, + port: int, + user: str, + password: str, + dbname: str, + min_connections: int, + max_connections: int, + minio_config: dict[str, Any], + logger: Logger, + notification_handler: NotificationHandler, + ): + super().__init__( + host=host, + port=port, + user=user, + password=password, + dbname=dbname, + min_connections=min_connections, + max_connections=max_connections, + logger=logger, + notification_handler=notification_handler, + ) if not hasattr(self, 'minio_repository'): - self.minio_repository = MinioRepository( - logger=logger, - notification_handler=notification_handler, - minio_endpoint_url=minio_config['endpoint_url'], - minio_access_key=minio_config['access_key'], - minio_secret_key=minio_config['secret_key'], - minio_region_name=minio_config['region_name'], - minio_default_bucket=minio_config['default_bucket']) + self.minio_repository: MinioRepository | None = None if self.minio_repository is None: self.minio_repository = MinioRepository( @@ -62,7 +60,8 @@ class Storage(Postgres): minio_access_key=minio_config['access_key'], minio_secret_key=minio_config['secret_key'], minio_region_name=minio_config['region_name'], - minio_default_bucket=minio_config['default_bucket']) + minio_default_bucket=minio_config['default_bucket'], + ) @activity.defn(name='query_to_minio') async def query_to_minio(self, input_data: dict[str, Any]) -> dict[str, Any]: @@ -79,64 +78,60 @@ class Storage(Postgres): dict: { success: bool, object_name: str, uri: str } """ + if self.minio_repository is None: + raise ValueError('Minio repository not initialized') + metadata = input_data.get('metadata', {}) object_prefix = input_data.get('object_prefix', 'datasets/retrain') timestamp = now().strftime(DATETIME_FILENAME_FORMAT) - object_name = f"{object_prefix}_{timestamp}.parquet" - uri = f"s3://{self.minio_repository.minio_bucket}/{object_name}" + object_name = f'{object_prefix}_{timestamp}.parquet' + uri = f's3://{self.minio_repository.minio_bucket}/{object_name}' try: data = await self.load_custom_query(input_data) if not data: - self.error( - f"query_to_minio failed: No data returned from query", metadata) - return {"success": False, "message": "No data returned from query"} + self.error('query_to_minio failed: No data returned from query', metadata) + return {'success': False, 'message': 'No data returned from query'} # Ensure we have a DataFrame data = pd.DataFrame(data) # Write parquet to memory and upload via persistent client self.minio_repository.store_dataframe_as_parquet( - dataframe=data, - uri=uri, - object_name=object_name, - metadata=metadata + dataframe=data, uri=uri, object_name=object_name, metadata=metadata ) - return {"success": True, "object_key": object_name, "uri": uri} + return {'success': True, 'object_key': object_name, 'uri': uri} except Exception as e: trace = traceback.format_exc() self.send_notification( metadata=metadata, - notification_id="ERROR_LOADING_CUSTOM_QUERY", - message=f"Error fetching data from query: {e}", - block="load_custom_query", + notification_id='ERROR_STORING_QUERY_TO_MINIO', + message=f'Error storing query to MinIO: {e}', + block='query_to_minio', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) self.error(trace, metadata) - return {"success": False, "message": str(e)} + return {'success': False, 'message': str(e)} def close(self) -> None: """Close Storage resources (MinIO client and Postgres engine).""" try: - if hasattr(self, 's3_client') and self.s3_client is not None: + if hasattr(self, 'minio_repository') and self.minio_repository is not None: try: - self.s3_client.close() + self.minio_repository.close() finally: - self.s3_client = None + self.minio_repository = None finally: # Ensure Postgres resources are disposed as well try: super().close() except Exception: - pass + self.logger.error('Error closing Postgres resources') def __del__(self): - try: - self.close() - except Exception: - pass + self.close() diff --git a/laborious/metrics.py b/laborious/metrics.py index 97f7bb9..7826fac 100644 --- a/laborious/metrics.py +++ b/laborious/metrics.py @@ -23,50 +23,50 @@ Metric Labels: - opc_server_id: Identifier for OPC server operations """ -from prometheus_client import Gauge, Counter, Histogram +from prometheus_client import Counter, Gauge, Histogram # Application health metric APP_UP = Gauge( - "app_up", - "Indicates if the application is running (1) or shutting down (0)", - ["pod_id"], + 'app_up', + 'Indicates if the application is running (1) or shutting down (0)', + ['pod_id'], ) # Core labels used across multiple metrics -CORE_LABELS = ["pod_id", "model_name", "pipeline_name"] +CORE_LABELS = ['pod_id', 'model_name', 'pipeline_name'] # Prediction operation metrics PREDICTIONS_WRITTEN_COUNT = Counter( - "laborious_predictions_written_count", - "Number of predictions written to the database table predictions", + 'laborious_predictions_written_count', + 'Number of predictions written to the database table predictions', CORE_LABELS, ) # Prediction quality metrics PREDICTION_CONFIDENCE_MONITOR = Gauge( - "laborious_prediction_confidence_monitor", - "Current confidence of each prediction", + 'laborious_prediction_confidence_monitor', + 'Current confidence of each prediction', CORE_LABELS, ) # Performance monitoring metrics PREDICTION_RESPONSE_TIME_MONITOR = Histogram( - "laborious_prediction_response_time_monitor", - "Current response time of each prediction", + 'laborious_prediction_response_time_monitor', + 'Current response time of each prediction', CORE_LABELS, - buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0] + buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], ) # OPC export metrics PREDICTION_OPC_WRITING_COUNT = Counter( - "laborious_prediction_opc_writing_count", - "Number of predictions written to the OPC server", - [*CORE_LABELS, "opc_server_id"], + 'laborious_prediction_opc_writing_count', + 'Number of predictions written to the OPC server', + [*CORE_LABELS, 'opc_server_id'], ) PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR = Histogram( - "laborious_prediction_opc_writing_response_time_monitor", - "Current response time of each prediction written to the OPC server", - [*CORE_LABELS, "opc_server_id"], - buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0] + 'laborious_prediction_opc_writing_response_time_monitor', + 'Current response time of each prediction written to the OPC server', + [*CORE_LABELS, 'opc_server_id'], + buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], ) diff --git a/laborious/utils/connectors_config.py b/laborious/utils/connectors_config.py index 969b95c..7501a07 100644 --- a/laborious/utils/connectors_config.py +++ b/laborious/utils/connectors_config.py @@ -1,9 +1,9 @@ -from os import getenv import json -from typing import Dict, Any +from os import getenv +from typing import Any -def build_postgres_config() -> Dict[str, Any]: +def build_postgres_config() -> dict[str, Any]: """ Build PostgreSQL database configuration from environment variables. @@ -30,11 +30,11 @@ def build_postgres_config() -> Dict[str, Any]: 'password': getenv('POSTGRES_PASSWORD', 'sientia'), 'dbname': getenv('POSTGRES_DBNAME', 'sientia'), 'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')), - 'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20')) + 'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20')), } -def build_mlflow_config() -> Dict[str, Any]: +def build_mlflow_config() -> dict[str, Any]: """ Build MLFlow server configuration from environment variables. @@ -55,11 +55,11 @@ def build_mlflow_config() -> Dict[str, Any]: 'host': getenv('MLFLOW_HOST', 'http://localhost'), 'port': int(getenv('MLFLOW_PORT', '5080')), 'username': getenv('MLFLOW_USERNAME', 'aignosi'), - 'password': getenv('MLFLOW_PASSWORD', 'aignosi') + 'password': getenv('MLFLOW_PASSWORD', 'aignosi'), } -def build_opc_config() -> Dict[str, Any]: +def build_opc_config() -> dict[str, Any]: """ Build OPC server configuration from environment variables. @@ -93,12 +93,12 @@ def build_opc_config() -> Dict[str, Any]: 'cert_path': getenv('OPC_CERT_PATH', None), 'private_key_path': getenv('OPC_PRIVATE_KEY_PATH', None), 'server_cert_path': getenv('OPC_SERVER_CERT_PATH', None), - 'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120')) + 'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120')), } } -def build_mongodb_config() -> Dict[str, Any]: +def build_mongodb_config() -> dict[str, Any]: """ Build MongoDB configuration from environment variables. @@ -125,11 +125,11 @@ def build_mongodb_config() -> Dict[str, Any]: return { 'connection_string': connection_string, 'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'), - 'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600 + 'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600, } -def build_minio_config() -> Dict[str, Any]: +def build_minio_config() -> dict[str, Any]: """ Build MinIO (S3-compatible) configuration from environment variables. @@ -148,5 +148,5 @@ def build_minio_config() -> Dict[str, Any]: 'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'), 'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'), 'region_name': getenv('MINIO_REGION_NAME', 'us-east-1'), - 'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious') + 'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious'), } diff --git a/laborious/utils/filters/conditional_filters.py b/laborious/utils/filters/conditional_filters.py index 6caf1e5..967e52c 100644 --- a/laborious/utils/filters/conditional_filters.py +++ b/laborious/utils/filters/conditional_filters.py @@ -24,8 +24,7 @@ def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool if data.empty: return False - return not data[ - data['variable'].isin(config['variables']) & data['value'].isna()].empty + return not data[data['variable'].isin(config['variables']) & data['value'].isna()].empty def filter_empty_data(data: DataFrame, _config: dict) -> bool: diff --git a/laborious/utils/filters/mlflow_filters.py b/laborious/utils/filters/mlflow_filters.py index f792a0e..82970e8 100644 --- a/laborious/utils/filters/mlflow_filters.py +++ b/laborious/utils/filters/mlflow_filters.py @@ -52,8 +52,11 @@ def nan_values_filter(predictions: DataFrame, _config: dict) -> bool: bool: True if data should be filtered (too many NaN values), False otherwise """ - data = predictions.replace({None: np.nan}).drop( - columns=['timestamp'], errors='ignore').infer_objects() + data = ( + predictions.replace({None: np.nan}) + .drop(columns=['timestamp'], errors='ignore') + .infer_objects() + ) if data.isna().all().all(): return True diff --git a/laborious/utils/repository/minio_repository.py b/laborious/utils/repository/minio_repository.py index d957e65..df20a06 100644 --- a/laborious/utils/repository/minio_repository.py +++ b/laborious/utils/repository/minio_repository.py @@ -1,40 +1,41 @@ from io import BytesIO -import traceback +from typing import Any + import boto3 from botocore.config import Config -from pandas import DataFrame, read_parquet -from sientia_do.observability.logger import Logger -from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler -from sientia_do.notifications.models import NotificationLevel -from typing import Any from botocore.exceptions import ClientError +from pandas import DataFrame, read_parquet +from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler +from sientia_do.observability.logger import Logger -class MinioRepository(): - def __init__(self, - minio_endpoint_url: str, - minio_access_key: str, - minio_secret_key: str, - minio_region_name: str, - minio_default_bucket: str, - logger: Logger, - notification_handler: NotificationHandler): - +class MinioRepository: + def __init__( + self, + minio_endpoint_url: str, + minio_access_key: str, + minio_secret_key: str, + minio_region_name: str, + minio_default_bucket: str, + logger: Logger, + notification_handler: NotificationHandler, + ): # MinIO settings shared with pandas s3fs self.storage_options = { 'key': minio_access_key, 'secret': minio_secret_key, - 'client_kwargs': {'endpoint_url': minio_endpoint_url} + 'client_kwargs': {'endpoint_url': minio_endpoint_url}, } self.minio_bucket = minio_default_bucket self.minio_endpoint_url = minio_endpoint_url self.minio_region_name = minio_region_name logger.info( - f"Connecting to MinIO at {self.minio_endpoint_url}, default bucket: {self.minio_bucket}") + f'Connecting to MinIO at {self.minio_endpoint_url}, default bucket: {self.minio_bucket}' + ) # Reusable MinIO client - self.s3_client = boto3.client( + self.s3_client: Any = boto3.client( 's3', endpoint_url=self.minio_endpoint_url, aws_access_key_id=self.storage_options['key'], @@ -48,67 +49,46 @@ class MinioRepository(): read_timeout=120, ), ) - self._bucket_checked = False self.logger = logger self.notification_handler = notification_handler + def close(self): + self.s3_client.close() + def ensure_bucket_exists(self, metadata: dict[str, Any]) -> bool: """ Ensure the MinIO bucket exists; create it if necessary. """ - if self._bucket_checked: - return True try: - self.logger.custom_info( - f"Checking if bucket '{self.minio_bucket}' exists", metadata) + self.logger.custom_info(f"Checking if bucket '{self.minio_bucket}' exists", metadata) self.s3_client.head_bucket(Bucket=self.minio_bucket) - self._bucket_checked = True return True except ClientError: - try: - self.logger.custom_info( - f"Creating bucket '{self.minio_bucket}'", metadata) - self.s3_client.create_bucket(Bucket=self.minio_bucket) - self._bucket_checked = True - return True - except ClientError as ce: - trace = traceback.format_exc() - self.notification_handler.send_notification( - metadata=metadata, - notification_id="ERROR_CREATING_MINIO_BUCKET", - message=f"Failed to ensure bucket '{self.minio_bucket}': {ce}", - block="ensure_bucket_exists", - level=NotificationLevel.ERROR, - attachment_content=str(ce) - ) - self.logger.custom_error(trace, metadata) - return False + self.logger.custom_info(f"Creating bucket '{self.minio_bucket}'", metadata) + self.s3_client.create_bucket(Bucket=self.minio_bucket) - def store_dataframe_as_parquet(self, dataframe: DataFrame, uri: str, - object_name: str, metadata: dict[str, Any]): + return True + def store_dataframe_as_parquet( + self, dataframe: DataFrame, uri: str, object_name: str, metadata: dict[str, Any] + ): self.ensure_bucket_exists(metadata) - self.logger.custom_info( - f"Storing dataframe as parquet in {uri}", metadata) + self.logger.custom_info(f'Storing dataframe as parquet in {uri}', metadata) buffer = BytesIO() dataframe.to_parquet(buffer, engine='pyarrow', index=True) buffer.seek(0) - self.s3_client.put_object( - Bucket=self.minio_bucket, Key=object_name, Body=buffer.getvalue()) + self.s3_client.put_object(Bucket=self.minio_bucket, Key=object_name, Body=buffer.getvalue()) - self.logger.custom_info( - f"Dataframe stored as parquet in {uri}", metadata) + self.logger.custom_info(f'Dataframe stored as parquet in {uri}', metadata) def get_parquet_as_dataframe(self, object_key: str, metadata: dict[str, Any]) -> DataFrame: - self.logger.custom_info( - f"Getting parquet as dataframe from {object_key}", metadata) + self.logger.custom_info(f'Getting parquet as dataframe from {object_key}', metadata) - response = self.s3_client.get_object( - Bucket=self.minio_bucket, Key=object_key) + response = self.s3_client.get_object(Bucket=self.minio_bucket, Key=object_key) # Read the content into a BytesIO buffer to support seek operations buffer = BytesIO(response['Body'].read()) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 691857a..d84f527 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -13,53 +13,48 @@ The repository provides comprehensive functionality for: - Model retraining workflows - Production model updates and versioning """ -from datetime import datetime, timedelta -import traceback -from mlflow.entities import Experiment, experiment -import pandas as pd -import mlflow -from os import makedirs, path, remove, environ -from shutil import rmtree -from sys import path as sys_path -from sientia_do.observability.logger import Logger -import lzma -import gzip -import pickle -from numpy import ndarray -from typing import Any -import gc -from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ -import ctypes -ARTIFACTS_PATH = "./tmp/artifacts" -TRANSFORMED_COMPRESSED_PATH = "artifacts/training_transformer.pkl" -PREDICTION_COMPRESSED_PATH = "artifacts/stacking_model.pkl" +import ctypes +import gc +import traceback +from datetime import datetime, timedelta +from os import environ, makedirs, path +from shutil import rmtree +from typing import Any, Literal, overload + +import mlflow +import pandas as pd +from mlflow.entities import Experiment +from numpy import ndarray +from sientia_do.observability.logger import Logger +from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + +ARTIFACTS_PATH = './tmp/artifacts' +TRANSFORMED_COMPRESSED_PATH = 'artifacts/training_transformer.pkl' +PREDICTION_COMPRESSED_PATH = 'artifacts/stacking_model.pkl' def force_memory_release(logger: Logger): gc.collect() try: - ctypes.CDLL("libc.so.6").malloc_trim(0) - logger.info( - f"Memory released") + ctypes.CDLL('libc.so.6').malloc_trim(0) + logger.info('Memory released') except Exception as e: - logger.info( - f"Memory release failed: {e}") + logger.info(f'Memory release failed: {e}') -class MLFlowRepository(): +class MLFlowRepository: def __init__(self, host: str, username: str, password: str, logger: Logger): - # set tracking uri mlflow.set_tracking_uri(host) - environ["MLFLOW_TRACKING_USERNAME"] = username - environ["MLFLOW_TRACKING_PASSWORD"] = password + environ['MLFLOW_TRACKING_USERNAME'] = username + environ['MLFLOW_TRACKING_PASSWORD'] = password # Create an MLflow client self.client = mlflow.tracking.MlflowClient() - self.model_cache = {} + self.model_cache: dict[str, Any] = {} self.logger = logger """ @@ -79,12 +74,12 @@ class MLFlowRepository(): """ run_info = mlflow.get_run(run_id) if prediction: - model_uri = run_info.info.artifact_uri + "/prediction_model" + model_uri = run_info.info.artifact_uri + '/prediction_model' else: - model_uri = run_info.info.artifact_uri + "/data_model" + model_uri = run_info.info.artifact_uri + '/data_model' return model_uri - def get_model_run_id(self, model_name: str, stage: str = "Production"): + def get_model_run_id(self, model_name: str, stage: str = 'Production') -> str: """ Get the run_id of a model based on its name and stage. @@ -106,13 +101,10 @@ class MLFlowRepository(): ) # Get all versions of the model and filter by stage - model_versions = self.client.search_model_versions( - filter_string=f"name='{model_name}'" - ) + model_versions = self.client.search_model_versions(filter_string=f"name='{model_name}'") # Filter versions by the desired stage using current_stage attribute - stage_versions = [ - mv for mv in model_versions if mv.current_stage == stage] + stage_versions = [mv for mv in model_versions if mv.current_stage == stage] if not stage_versions: raise mlflow.exceptions.MlflowException( @@ -121,28 +113,9 @@ class MLFlowRepository(): # Sort by version number to get the latest latest_version = max(stage_versions, key=lambda v: int(v.version)) - run_id = latest_version.source.split("/") + run_id = latest_version.source.split('/') return run_id[2] - def get_experiment_by_run_id(self, run_id: str) -> Experiment: - """ - Get experiment name by run ID. - - Args: - run_id (str): The MLFlow run ID - - Returns: - str: The experiment name - """ - # Get the run information using the run_id - run = mlflow.get_run(run_id) - - # Extract the experiment ID from the run - experiment_id = run.info.experiment_id - - # Get the experiment details using the experiment ID - return mlflow.get_experiment(experiment_id) - def get_next_run_name(self, model_name: str) -> str: """ Generate the next run name for a specific MLFlow model. @@ -157,12 +130,13 @@ class MLFlowRepository(): Returns: str: The next run name in format 'model_name-run_number' """ - runs = mlflow.search_runs( - experiment_names=[model_name], order_by=["start_time desc"]) + runs = mlflow.search_runs(experiment_names=[model_name], order_by=['start_time desc']) next_run_number = len(runs) + 1 - return f"{model_name}-{next_run_number}" + return f'{model_name}-{next_run_number}' - def get_experiment(self, experiment_name: str, create_if_not_exists: bool = False) -> Experiment: + def get_experiment( + self, experiment_name: str, create_if_not_exists: bool = False + ) -> Experiment: """ Retrieve MLFlow experiment ID by experiment name. @@ -189,47 +163,6 @@ class MLFlowRepository(): return experiment - def get_experiment_last_run(self, experiment_id: int) -> str: - """ - Retrieve the most recent retraining run ID for an experiment. - - This method searches for the latest run in an MLFlow experiment - that has been marked as a retraining run. It filters runs by - the 'retrain' parameter and orders them by completion time. - - Args: - experiment_id (int): MLFlow experiment ID - - Returns: - str: MLFlow run ID of the most recent retraining run - - Raises: - ValueError: If runs data is not in expected DataFrame format - """ - runs = mlflow.search_runs( - experiment_ids=[experiment_id], - filter_string="", # Sem filtro no MLflow ainda - output_format="pandas" - ) - - if not isinstance(runs, pd.DataFrame): - raise ValueError('Runs is not a pandas DataFrame') - - # Filtrar apenas as runs onde params.retrain == True - filtered_runs = runs[runs["params.retrain"] == 'True'] - - # Converter a coluna 'end_time' para datetime - filtered_runs['end_time'] = pd.to_datetime(filtered_runs['end_time']) - - # Ordenar o DataFrame de forma descendente pela coluna 'end_time' - filtered_runs = filtered_runs.sort_values( - by='end_time', ascending=False) - - # Pegar a última run_id do DataFrame filtrado e ordenado - latest_run_id = filtered_runs.iloc[0]['run_id'] - - return latest_run_id - def get_model_params(self, run_id: str): """Obtém os parâmetros de uma run""" run_info = mlflow.get_run(run_id) @@ -239,7 +172,7 @@ class MLFlowRepository(): Functions related to download and load models """ - def dowload_artifacts(self, model_name: str, artifact_path: str = "data_model") -> str: + def dowload_artifacts(self, model_name: str, artifact_path: str = 'data_model') -> str: """ Downloads artifacts from a specific MLFlow run. @@ -250,10 +183,8 @@ class MLFlowRepository(): Returns: str: Path to the downloaded artifacts """ - run_id = self.get_model_run_id( - model_name=model_name, stage="Production" - ) - output_dir = f"{ARTIFACTS_PATH}/{model_name}" + run_id = self.get_model_run_id(model_name=model_name, stage='Production') + output_dir = f'{ARTIFACTS_PATH}/{model_name}' full_path = path.join(output_dir, artifact_path) @@ -262,14 +193,9 @@ class MLFlowRepository(): rmtree(full_path) makedirs(output_dir, exist_ok=True) - self.logger.info( - f"Downloading artifacts from {run_id} to {output_dir}") + self.logger.info(f'Downloading artifacts from {run_id} to {output_dir}') - return self.client.download_artifacts( - run_id, - artifact_path, - output_dir - ) + return self.client.download_artifacts(run_id, artifact_path, output_dir) def load_predict_model(self, model_name: str, flavor: str = 'sklearn') -> Any: """ @@ -287,9 +213,8 @@ class MLFlowRepository(): - The model is fetched from the "production" stage of the MLflow Model Registry. - Warnings during the model loading process are suppressed. """ - model_uri = f"models:/{model_name}/production" - self.logger.info( - f"Loading prediction model {model_name} from {model_uri}") + model_uri = f'models:/{model_name}/production' + self.logger.info(f'Loading prediction model {model_name} from {model_uri}') if flavor == 'pyfunc': model = mlflow.pyfunc.load_model(model_uri) elif flavor == 'sklearn': @@ -297,8 +222,7 @@ class MLFlowRepository(): elif flavor == 'pytorch': model = mlflow.pytorch.load_model(model_uri) else: - raise ValueError( - "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") + raise ValueError("Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") return model @@ -322,14 +246,10 @@ class MLFlowRepository(): model cannot be loaded. """ - latest_production_id = self.get_model_run_id( - model_name=model_name, stage="Production" - ) - model_uri = self.get_model_uri( - latest_production_id, prediction=False) + latest_production_id = self.get_model_run_id(model_name=model_name, stage='Production') + model_uri = self.get_model_uri(latest_production_id, prediction=False) - self.logger.info( - f"Loading data model {model_name} from {model_uri}") + self.logger.info(f'Loading data model {model_name} from {model_uri}') if flavor == 'sklearn': model = mlflow.sklearn.load_model(model_uri) elif flavor == 'pyfunc': @@ -337,64 +257,12 @@ class MLFlowRepository(): elif flavor == 'pytorch': model = mlflow.pytorch.load_model(model_uri) else: - raise ValueError( - "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") + raise ValueError("Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") return model - def load_model_with_compression(self, artifact_path: str, model_type: str) -> Any: - """ - Load model from pickle file trying different compression methods. - - Args: - artifact_path (str): Path to the artifact directory - type (str): Type of model ('transformer' or 'prediction') - - Returns: - Any: Loaded model object - - Raises: - ValueError: If model cannot be loaded with any compression method - """ - code_path = path.join( - artifact_path, "code") - - pickle_file = TRANSFORMED_COMPRESSED_PATH if model_type == "transform" else PREDICTION_COMPRESSED_PATH - - pickle_path = path.join(artifact_path, pickle_file) - - self.logger.info( - f"Loading model with type {model_type} from {pickle_path}") - - if code_path not in sys_path: - sys_path.insert(0, code_path) - self.logger.info( - f"Added {code_path} to Python path") - - loading_methods = [ - ("lzma", lambda p: lzma.open(p, "rb")), - ("gzip", lambda p: gzip.open(p, "rb")), - ("pickle", lambda p: open(p, "rb")), - ] - - for format_name, open_func in loading_methods: - try: - self.logger.inf - (f"Trying to load with {format_name}...") - with open_func(pickle_path) as f: - model = pickle.load(f) - self.logger.info( - f"Successfully loaded with {format_name}!") - return model - except (lzma.LZMAError, gzip.BadGzipFile, OSError, pickle.UnpicklingError, ValueError) as e: - self.logger.info( - f"Failed with {format_name}: {e.__class__.__name__}:{e}") - continue - - raise ValueError( - f"Could not load model from {pickle_path} - unknown or corrupted format") - - def download_model(self, model_name: str, model_type: str, flavor: str, - load_wrapper: bool = False) -> tuple[Any, str]: + def download_model( + self, model_name: str, model_type: str, flavor: str, load_wrapper: bool = False + ) -> tuple[Any, str | None]: """ Download model based on type (predict or transform). @@ -409,37 +277,35 @@ class MLFlowRepository(): """ self.logger.info( - f"Downloading {model_type} model {model_name} with flavor {flavor} and load_wrapper {load_wrapper}") + f'Downloading {model_type} model {model_name} with flavor {flavor} and load_wrapper {load_wrapper}' + ) - if model_type not in ["predict", "transform"]: - raise ValueError( - "Invalid model_type. Use 'predict' or 'transform'.") + if model_type not in ['predict', 'transform']: + raise ValueError("Invalid model_type. Use 'predict' or 'transform'.") artifact_path = None if load_wrapper: self.logger.info( - f"Loading wrapper for {model_type} model {model_name} with flavor {flavor}") + f'Loading wrapper for {model_type} model {model_name} with flavor {flavor}' + ) - target = "prediction_model" if model_type == "predict" else "data_model" + target = 'prediction_model' if model_type == 'predict' else 'data_model' - artifact_path = self.dowload_artifacts( - model_name, target) + artifact_path = self.dowload_artifacts(model_name, target) self.logger.info( - f"Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}") + f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}' + ) raw_model = mlflow.pyfunc.load_model(artifact_path) model = raw_model._model_impl.python_model else: + if model_type == 'predict': + model = self.load_predict_model(model_name, flavor) - if model_type == "predict": - model = self.load_predict_model( - model_name, flavor) - - elif model_type == "transform": - model = self.load_transform_model( - model_name, flavor) + elif model_type == 'transform': + model = self.load_transform_model(model_name, flavor) return model, artifact_path @@ -466,32 +332,31 @@ class MLFlowRepository(): # Get type of first element of index index_type = type(index[0]) - self.logger.custom_info(f"Index type: {index_type}", metadata) + self.logger.custom_info(f'Index type: {index_type}', metadata) - message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}. Got {index_type}." + message = f'Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}.' # Check if all in index are of the same type if not all(isinstance(i, index_type) for i in index): - raise ValueError( - f"{message}. Elements are {','.join(map(type, index))}") + types = map(str, map(type, index)) + raise ValueError(f'{message}. Elements are {",".join(types)}') # Check type and converts to DATETIME_FORMAT_WITH_TZ - if index_type == str: + if index_type is str: # Validate format of string and return error if not valid try: pd.to_datetime(data.index, format=DATETIME_FORMAT_WITH_TZ) - except ValueError: - raise ValueError( - f"{message}") + except ValueError as e: + raise ValueError(f'{message}. Unable to parse given date format: {e}') from e elif index_type == datetime or index_type == pd.Timestamp: - if data.index.tz is None: - data.index = data.index.tz_localize('UTC') + index = data.index + if hasattr(index, 'tz') and index.tz is None: + data.index = index.tz_localize('UTC') # type: ignore[attr-defined] - data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) + data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) # type: ignore[attr-defined] else: - raise ValueError( - f"{message}") + raise ValueError(f'{message}. Got {index_type}.') return data @@ -516,8 +381,7 @@ class MLFlowRepository(): return False return True - def handle_valid_model(self, model_name: str, model_type: str, - cache: dict) -> dict: + def handle_valid_model(self, model_name: str, cache: dict) -> dict: """ Handle valid cached model by returning appropriate model configuration. @@ -531,9 +395,7 @@ class MLFlowRepository(): Returns: dict: Model configuration with model and artifact path """ - if self.logger: - self.logger.debug( - f"Model {model_name} is still valid, using cached version") + self.logger.debug(f'Model {model_name} is still valid, using cached version') return cache['target'] @@ -548,9 +410,7 @@ class MLFlowRepository(): Returns: None """ - if self.logger: - self.logger.debug( - f"Model {model_name} is outdated, downloading a new one") + self.logger.debug(f'Model {model_name} is outdated, downloading a new one') del self.model_cache[model_key]['target']['model'] del self.model_cache[model_key] @@ -571,7 +431,8 @@ class MLFlowRepository(): # Retention is 0, download a new model if retention <= 0: model, _artifact_path = self.download_model( - model_name=model_name, model_type=model_type, flavor=flavor, load_wrapper=False) + model_name=model_name, model_type=model_type, flavor=flavor, load_wrapper=False + ) return model model_key = f'{model_name}_{model_type}' @@ -581,32 +442,49 @@ class MLFlowRepository(): # Check if config has changed or is outdated if self.check_cache_retention(cache, retention): - return self.handle_valid_model( - model_name=model_name, model_type=model_type, cache=cache) + return self.handle_valid_model(model_name=model_name, cache=cache) else: # Model is outdated, delete old model files - self.handle_outdated_model(model_name, model_key) + self.handle_outdated_model(model_name=model_name, model_key=model_key) else: - if self.logger: - self.logger.debug( - f"Model {model_name} is not in {model_type} cache, downloading a new one") + self.logger.debug( + f'Model {model_name} is not in {model_type} cache, downloading a new one' + ) # Donwload new model model, _artifact_path = self.download_model( - model_name=model_name, model_type=model_type, flavor=flavor, - load_wrapper=False) + model_name=model_name, model_type=model_type, flavor=flavor, load_wrapper=False + ) - cache = { - 'target': model, - 'timestamp': datetime.now() - } + cache = {'target': model, 'timestamp': datetime.now()} self.model_cache[model_key] = cache return model - def get_cached_transform(self, model_name: str, data: pd.DataFrame, - retention: int, flavor: str) -> pd.DataFrame: + @overload + def get_cached_operation( + self, + model_name: str, + data: pd.DataFrame, + operation: Literal['transform'], + retention: int, + flavor: str, + ) -> pd.DataFrame: ... + + @overload + def get_cached_operation( + self, + model_name: str, + data: pd.DataFrame, + operation: Literal['predict'], + retention: int, + flavor: str, + ) -> pd.DataFrame | ndarray: ... + + def get_cached_operation( + self, model_name: str, data: pd.DataFrame, operation: str, retention: int, flavor: str + ) -> pd.DataFrame | ndarray: """ Get transformed data using cached transform model. @@ -619,44 +497,17 @@ class MLFlowRepository(): Returns: pd.DataFrame: Transformed data """ + if operation not in ['transform', 'predict']: + raise ValueError("Invalid operation. Use 'transform' or 'predict'.") + model = self.get_model( - model_name=model_name, retention=retention, - model_type="transform", flavor=flavor) + model_name=model_name, retention=retention, model_type=operation, flavor=flavor + ) prediction = model.predict(data) if retention == 0: - self.logger.info( - f"Deleting model {model_name} from memory") - del model - - force_memory_release(self.logger) - - return prediction - - def get_cached_predict(self, model_name: str, data: pd.DataFrame, retention: int, - flavor: str) -> ndarray: - """ - Get predictions using cached prediction model. - - Args: - model_name (str): Name of the prediction model - data (pd.DataFrame): Data to make predictions on - retention (int): Cache retention time in minutes - flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') - - Returns: - pd.DataFrame: Model predictions - """ - model = self.get_model( - model_name=model_name, retention=retention, - model_type="predict", flavor=flavor) - - prediction = model.predict(data) - - if retention == 0: - self.logger.info( - f"Deleting model {model_name} from memory") + self.logger.info(f'Deleting model {model_name}:{operation} from memory') del model force_memory_release(self.logger) @@ -667,10 +518,16 @@ class MLFlowRepository(): Functions related to model retraining """ - def create_model_experiment(self, model_name: str, data: pd.DataFrame, latest_production_id: str, - transform_flavor: str = 'sklearn', predict_flavor: str = 'sklearn', - fit_config: dict = {}, target_name: str = None, - metadata: dict = {}) -> tuple: + def fit_models( + self, + model_name: str, + data: pd.DataFrame, + latest_production_id: str, + metadata: dict, + transform_flavor: str = 'sklearn', + predict_flavor: str = 'sklearn', + target_name: str | None = None, + ) -> dict[str, dict[str, Any]]: """ Create a new MLFlow experiment for model retraining. @@ -686,7 +543,6 @@ class MLFlowRepository(): data (pd.DataFrame): Training data for model retraining transform_flavor (str): Flavor for transformation model predict_flavor (str): Flavor for prediction model - fit_config (dict): Fit configuration target_name (str): Target name metadata (dict): Metadata for logging @@ -696,78 +552,90 @@ class MLFlowRepository(): - data_model: Fitted transformation model - experiment: MLFlow experiment name """ - self.logger.custom_info( - f"Starting model experiment creation for {model_name}", metadata) + + self.logger.custom_info(f'Starting model experiment creation for {model_name}', metadata) self.logger.custom_debug( - f"Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, fit_config: {fit_config}, target_name: {target_name}", metadata) + f'Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, target_name: {target_name}', + metadata, + ) # data.to_csv( # f"tmp/retrain_data_{model_name}.csv", index=True) self.logger.custom_info( - f"Retrieved latest production run ID: {latest_production_id}", metadata) - self.logger.custom_info( - f"Loading transformation model for {model_name}", metadata) + f'Retrieved latest production run ID: {latest_production_id}', metadata + ) + self.logger.custom_info(f'Loading transformation model for {model_name}', metadata) load_transform_wrapper = transform_flavor == 'pyfunc' data_model, data_artifact_path = self.download_model( - model_name=model_name, model_type="transform", flavor=transform_flavor, - load_wrapper=load_transform_wrapper + model_name=model_name, + model_type='transform', + flavor=transform_flavor, + load_wrapper=load_transform_wrapper, ) - self.logger.custom_info( - f"Loading prediction model for {model_name}", metadata) + self.logger.custom_info(f'Loading prediction model for {model_name}', metadata) load_predict_wrapper = predict_flavor == 'pyfunc' prediction_model, prediction_artifact_path = self.download_model( - model_name=model_name, model_type="predict", flavor=predict_flavor, - load_wrapper=load_predict_wrapper + model_name=model_name, + model_type='predict', + flavor=predict_flavor, + load_wrapper=load_predict_wrapper, ) - treated_data = data_model.fit(data) + treated_data_candidate = data_model.fit(data) + + if not isinstance(treated_data_candidate, pd.DataFrame): + data_model = treated_data_candidate + treated_data = data_model.predict(data) + else: + treated_data = treated_data_candidate # Stores current index as timestamp, Courier model expects a timestamp column # with specific format treated_data['timestamp'] = treated_data.index # Parses timestamp column to datetime format to align with data - treated_data = self.detect_and_parse_datetime_index( - treated_data, metadata) + treated_data = self.detect_and_parse_datetime_index(treated_data, metadata) - treated_data = treated_data.drop_duplicates( - subset=['timestamp'], keep='first') + treated_data = treated_data.drop_duplicates(subset=['timestamp'], keep='first') - self.logger.custom_debug( - f"Treated data index: {treated_data.index}", metadata) + self.logger.custom_debug(f'Treated data index: {treated_data.index}', metadata) # treated_data.to_csv( # f"tmp/retrain_treated_data_{model_name}.csv", index=True) - self.logger.custom_debug( - f"Transformed data shape: {treated_data.shape}", metadata) + self.logger.custom_debug(f'Transformed data shape: {treated_data.shape}', metadata) if target_name is None: target_name = data_model.target_variable self.logger.custom_debug( - f"Using target variable from data model: {target_name}", metadata) + f'Using target variable from data model: {target_name}', metadata + ) else: - self.logger.custom_debug( - f"Using provided target variable: {target_name}", metadata) + self.logger.custom_debug(f'Using provided target variable: {target_name}', metadata) # Check if treated_data contains target variable if target_name not in treated_data.columns: self.logger.custom_debug( - f"Target variable {target_name} not found in treated data, aligning data with treated data indexes", metadata) + f'Target variable {target_name} not found in treated data, aligning data with treated data indexes', + metadata, + ) # Aligns data with treated data indexes to get target variable aligned_data = data.loc[treated_data.index] + aligned_series = aligned_data[target_name] retrain_dataset = pd.merge( - treated_data, aligned_data, left_index=True, right_index=True) + treated_data, aligned_series, left_index=True, right_index=True + ) else: # Uses target variable from treated data self.logger.custom_debug( - f"Target variable {target_name} found in treated data, using it", metadata) + f'Target variable {target_name} found in treated data, using it', metadata + ) retrain_dataset = treated_data # retrain_dataset.to_csv( @@ -776,91 +644,47 @@ class MLFlowRepository(): prediction_model.fit(retrain_dataset) self.logger.custom_info( - f"Model experiment creation completed successfully for {model_name}", metadata) + f'Model experiment creation completed successfully for {model_name}', metadata + ) retrain_data = { 'prediction_model': { 'model': prediction_model, - 'artifact_path': prediction_artifact_path + 'artifact_path': prediction_artifact_path, }, - 'data_model': { - 'model': data_model, - 'artifact_path': data_artifact_path - } + 'data_model': {'model': data_model, 'artifact_path': data_artifact_path}, } return retrain_data - def export_model_to_pkl(self, model_data: dict, model_type: str, metadata: dict = {}): - artifact_local_path = model_data['artifact_path'] + def log_model(self, model_data: dict, flavor: str, model_type: str, metadata: dict): model = model_data['model'] - if artifact_local_path: - pickle_file = TRANSFORMED_COMPRESSED_PATH if model_type == "data_model" else PREDICTION_COMPRESSED_PATH - pickle_path = path.join(artifact_local_path, pickle_file) - - self.logger.custom_info( - f"Saving model to {pickle_path}", metadata) - - with open(pickle_path, "wb") as f: - pickle.dump(model, f) - - def export_model_to_lzma(self, model_data: dict, model_type: str, metadata: dict = {}): - artifact_local_path = model_data['artifact_path'] - model = model_data['model'] - - if artifact_local_path: - pickle_file = TRANSFORMED_COMPRESSED_PATH if model_type == "data_model" else PREDICTION_COMPRESSED_PATH - pickle_path = path.join(artifact_local_path, pickle_file) - - self.logger.custom_info( - f"Saving model to {pickle_path}", metadata) - - if path.exists(pickle_path): - remove(pickle_path) - - with lzma.open(pickle_path + ".xz", "wb") as f: - self.logger.custom_info( - "Compressing model", metadata) - pickle.dump(model, f) - - def log_model(self, model_data: dict, flavor: str, model_type: str, - metadata: dict = {}): - - model = model_data['model'] - - self.logger.custom_debug( - f"Logging {model_type} model to {model_type}", metadata) + self.logger.custom_debug(f'Logging {model_type} model to {model_type}', metadata) if flavor == 'sklearn': mlflow.sklearn.log_model(model, model_type) elif flavor == 'pyfunc': - code_path = [path.join( - model_data['artifact_path'], 'code', "utils")] + code_path = [path.join(model_data['artifact_path'], 'code', 'utils')] - self.logger.custom_debug( - f"Code path: {code_path}", metadata) + self.logger.custom_debug(f'Code path: {code_path}', metadata) - model.store_model( - artifact_path=model_type, - code_path=code_path, - to_disk=False - ) + model.store_model(artifact_path=model_type, code_path=code_path, to_disk=False) - self.logger.custom_debug( - f"Model uploaded successfully", metadata) + self.logger.custom_debug('Model uploaded successfully', metadata) elif flavor == 'pytorch': mlflow.pytorch.log_model(model, model_type) else: - raise ValueError( - "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") + raise ValueError("Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") - def perform_model_retrain(self, - model_name: str, - data: pd.DataFrame, - retrain_data: dict, - transform_flavor: str = 'sklearn', - predict_flavor: str = 'sklearn', - metadata: dict = {}, - latest_production_id: str = None) -> dict: + def create_new_experiment( + self, + model_name: str, + data: pd.DataFrame, + retrain_data: dict, + latest_production_id: str, + metadata: dict, + transform_flavor: str = 'sklearn', + predict_flavor: str = 'sklearn', + ) -> dict: """ Execute the complete model retraining process in MLFlow. @@ -890,61 +714,53 @@ class MLFlowRepository(): prediction_model = retrain_data['prediction_model'] data_model = retrain_data['data_model'] - model_temp_path = path.join( - ARTIFACTS_PATH, model_name) + model_temp_path = path.join(ARTIFACTS_PATH, model_name) - self.logger.custom_info( - f"Starting model retraining process for {model_name}", metadata) + self.logger.custom_info(f'Starting model retraining process for {model_name}', metadata) original_params = self.get_model_params(latest_production_id) retrain_params = { **original_params, - "retrain": True, + 'retrain': True, 'retrain_date': datetime.now().isoformat(), 'source_run_id': latest_production_id, 'retrain_samples': str(data.shape), } - experiment_description = f"Retrain model {model_name} with new data" + experiment_description = f'Retrain model {model_name} with new data' - experiment = self.get_experiment(model_name, - create_if_not_exists=True) + experiment = self.get_experiment(model_name, create_if_not_exists=True) experiment_name = experiment.name current_run_name = self.get_next_run_name(experiment_name) - self.logger.custom_debug( - f"Attributes: {retrain_params}", metadata) + self.logger.custom_debug(f'Attributes: {retrain_params}', metadata) - data_path = f"{model_temp_path}/retrain_data.csv" + data_path = f'{model_temp_path}/retrain_data.csv' data.to_csv(data_path, index=True) self.logger.custom_info( - f"Starting model upload for {experiment_name} with run name {current_run_name}", metadata) + f'Starting model upload for {experiment_name} with run name {current_run_name}', + metadata, + ) with mlflow.start_run( experiment_id=experiment.experiment_id, run_name=current_run_name, - description=experiment_description + description=experiment_description, ) as _run: run_id = _run.info.run_id - self.logger.custom_info( - f"Logging data model", metadata) + self.logger.custom_info('Logging data model', metadata) # dynamic parameters, including model itself - self.log_model(data_model, transform_flavor, - "data_model", metadata) + self.log_model(data_model, transform_flavor, 'data_model', metadata) # dynamic parameters, including model itself - self.logger.custom_info( - f"Logging prediction model", metadata) - self.log_model(prediction_model, predict_flavor, - "prediction_model", metadata) + self.logger.custom_info('Logging prediction model', metadata) + self.log_model(prediction_model, predict_flavor, 'prediction_model', metadata) - self.logger.custom_info( - f"Model logged successfully for {model_name}", metadata) + self.logger.custom_info(f'Model logged successfully for {model_name}', metadata) - self.logger.custom_info( - f"Logging remaining parameters for {model_name}", metadata) + self.logger.custom_info(f'Logging remaining parameters for {model_name}', metadata) # update transfomation model # fixed parameters @@ -953,28 +769,29 @@ class MLFlowRepository(): # log the data raw mlflow.log_artifact(data_path) - self.logger.custom_info( - f"Deleting model from filesystem", metadata) + self.logger.custom_info('Deleting model from filesystem', metadata) if path.exists(model_temp_path): rmtree(model_temp_path) - self.logger.custom_info( - f"Deleting prediction model from memory", metadata) + self.logger.custom_info('Deleting prediction model from memory', metadata) del prediction_model['model'] del prediction_model - self.logger.custom_info( - f"Deleting data model from memory", metadata) + self.logger.custom_info('Deleting data model from memory', metadata) del data_model['model'] del data_model + force_memory_release(self.logger) + return { 'run_id': run_id, 'experiment_id': experiment.experiment_id, - 'experiment_name': experiment.name + 'experiment_name': experiment.name, } - def update_production_model_by_run_id(self, run_id: str, model_name: str, metadata: dict = {}) -> dict: + def update_production_model_by_run_id( + self, run_id: str, model_name: str, metadata: dict + ) -> dict: """ Update production model with a specific MLFlow run. @@ -999,18 +816,18 @@ class MLFlowRepository(): 3. Transitions the model to 'Production' stage 4. Archives existing production versions """ + self.logger.custom_info( - f"Starting production model update for {model_name} with run ID: {run_id}", metadata) + f'Starting production model update for {model_name} with run ID: {run_id}', metadata + ) # Registrar o modelo # Aqui estamos assumindo que você já tem um modelo salvo, caso contrário você precisará treiná-lo e salvá-lo primeiro. # Se o modelo já está registrado, você pode usar o método register_model() ou pyfunc.load_model() para isso. - mlflow.register_model( - f"runs:/{run_id}/prediction_model", model_name) + mlflow.register_model(f'runs:/{run_id}/prediction_model', model_name) # Obter a versão mais recente registrada do modelo - model_versions = self.client.get_registered_model( - model_name).latest_versions + model_versions = self.client.get_registered_model(model_name).latest_versions if not isinstance(model_versions, list): raise ValueError('Model versions is not a list') @@ -1019,24 +836,16 @@ class MLFlowRepository(): # Mover a versão mais recente do modelo para o estágio de 'Production' self.client.transition_model_version_stage( - name=model_name, - version=max_version, - stage="Production", - archive_existing_versions=True + name=model_name, version=max_version, stage='Production', archive_existing_versions=True ) - return { - 'model_name': model_name, - 'version': max_version, - 'mlflow_run_id': run_id - } + return {'model_name': model_name, 'version': max_version, 'mlflow_run_id': run_id} """ Functions that provide the interface to model operations """ - def transform(self, model_name: str, data: pd.DataFrame, - model_config: dict, metadata: dict): + def transform(self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict): """ Transform data using a cached transformation model. @@ -1069,8 +878,10 @@ class MLFlowRepository(): Exception: Any exception during model loading or transformation is caught and returned in the response structure rather than propagated. """ + self.logger.custom_debug( - f"Data received for model transformation: {data.head(5).to_csv()}", metadata) + f'Data received for model transformation: {data.head(5).to_csv()}', metadata + ) # data.to_csv( # f"tmp/data_{model_name}.csv", index=True) @@ -1079,35 +890,29 @@ class MLFlowRepository(): flavor = model_config.get('transform_flavor', 'sklearn') try: - transformed_data = self.get_cached_transform( - model_name, data, model_retention, flavor + transformed_data: pd.DataFrame = self.get_cached_operation( + model_name, data, 'transform', model_retention, flavor ) self.logger.custom_debug( - f"Data received from model transformation: {transformed_data.head(5).to_csv()}", metadata) + f'Data received from model transformation: {transformed_data.head(5).to_csv()}', + metadata, + ) # transformed_data.to_csv( # f"tmp/transformed_data_{model_name}.csv", index=True) - transformed_data = self.detect_and_parse_datetime_index( - transformed_data, metadata) + transformed_data = self.detect_and_parse_datetime_index(transformed_data, metadata) - return { - 'success': True, - 'content': transformed_data.to_dict() - } + return {'success': True, 'content': transformed_data.to_dict()} except Exception as e: return { 'success': False, - 'content': { - 'message': str(e), - 'traceback': traceback.format_exc() - } + 'content': {'message': str(e), 'traceback': traceback.format_exc()}, } - def predict(self, model_name: str, data: pd.DataFrame, - model_config: dict, metadata: dict): + def predict(self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict): """ Generate predictions using a cached prediction model. @@ -1152,55 +957,50 @@ class MLFlowRepository(): flavor = model_config.get('predict_flavor', 'sklearn') try: - input_index = data.index start_time = datetime.now() self.logger.custom_debug( - f"Data received for model prediction: {data.head(5).to_csv()}", metadata) + f'Data received for model prediction: {data.head(5).to_csv()}', metadata + ) # data.to_csv( # f"tmp/treated_data_{model_name}.csv", index=True) - predict_data = self.get_cached_predict( - model_name, data, model_retention, flavor) + predict_data = self.get_cached_operation( + model_name, data, 'predict', model_retention, flavor + ) end_time = datetime.now() if isinstance(predict_data, pd.DataFrame): self.logger.custom_debug( - f"Data received from model prediction: {data.head(5).to_csv()}", metadata) + f'Data received from model prediction: {data.head(5).to_csv()}', metadata + ) # predict_data.to_csv( # f"tmp/predicted_data_{model_name}.csv", index=True) - data.columns = ['prediction'] + predict_data.columns = ['prediction'] else: - predict_data = pd.DataFrame( - predict_data, columns=['prediction']) + predict_data = pd.DataFrame(predict_data, columns=['prediction']) # predict_data.to_csv( # f"tmp/predicted_data_{model_name}.csv", index=True) predict_data.index = input_index - predict_data['response_time'] = ( - end_time - start_time).total_seconds() + predict_data['response_time'] = (end_time - start_time).total_seconds() - return { - 'success': True, - 'content': predict_data.to_dict() - } + return {'success': True, 'content': predict_data.to_dict()} except Exception as e: return { 'success': False, - 'content': { - 'message': str(e), - 'traceback': traceback.format_exc() - } + 'content': {'message': str(e), 'traceback': traceback.format_exc()}, } - def retrain_model(self, data: pd.DataFrame, model_name: str, - model_config: dict, metadata: dict) -> tuple: + def retrain_model( + self, data: pd.DataFrame, model_name: str, model_config: dict, metadata: dict + ) -> dict[str, Any]: """ Orchestrate the complete model retraining workflow. @@ -1245,49 +1045,53 @@ class MLFlowRepository(): Exception: Any other exception during the retraining process """ - self.logger.custom_info( - f"Starting model retraining workflow for {model_name}", metadata) - self.logger.custom_debug( - f"Data received for model retraining: {data.to_csv()}", metadata) + self.logger.custom_info(f'Starting model retraining workflow for {model_name}', metadata) + self.logger.custom_debug(f'Data received for model retraining: {data.to_csv()}', metadata) target_name = model_config.get('target', None) transform_flavor = model_config.get('transform_flavor', 'sklearn') predict_flavor = model_config.get('predict_flavor', 'sklearn') - fit_config = { - 'split_fit_data': model_config.get('split_fit_data', False), - 'split_fit_first': model_config.get('split_fit_first', 'x').lower(), - 'y_type': model_config.get('y_type', 'series').lower() - } - self.logger.custom_debug( - f"Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, fit_config: {fit_config}, target_name: {target_name}", metadata) + f'Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, target_name: {target_name}', + metadata, + ) try: - latest_production_id = self.get_model_run_id( - model_name, stage="Production" + latest_production_id = self.get_model_run_id(model_name, stage='Production') + self.logger.custom_info('Creating model experiment environment', metadata) + retrain_data = self.fit_models( + model_name=model_name, + data=data, + transform_flavor=transform_flavor, + predict_flavor=predict_flavor, + target_name=target_name, + metadata=metadata, + latest_production_id=latest_production_id, ) self.logger.custom_info( - "Creating model experiment environment", metadata) - retrain_data = self.create_model_experiment( - model_name=model_name, data=data, transform_flavor=transform_flavor, - predict_flavor=predict_flavor, fit_config=fit_config, target_name=target_name, - metadata=metadata, latest_production_id=latest_production_id) - self.logger.custom_info( - f"Model experiment created successfully: {retrain_data}", metadata) + f'Model experiment created successfully: {retrain_data}', metadata + ) - self.logger.custom_info("Saving model retrain", metadata) - experiment = self.perform_model_retrain( - model_name=model_name, data=data, retrain_data=retrain_data, - transform_flavor=transform_flavor, predict_flavor=predict_flavor, metadata=metadata, latest_production_id=latest_production_id) + self.logger.custom_info('Saving model retrain', metadata) + experiment = self.create_new_experiment( + model_name=model_name, + data=data, + retrain_data=retrain_data, + transform_flavor=transform_flavor, + predict_flavor=predict_flavor, + metadata=metadata, + latest_production_id=latest_production_id, + ) self.logger.custom_info( - f"Model retraining completed successfully for experiment: {experiment}", metadata) + f'Model retraining completed successfully for experiment: {experiment}', metadata + ) return { 'success': True, 'experiment': experiment, - 'message': 'Model retrained successfully.' + 'message': 'Model retrained successfully.', } except Exception as e: error_msg = f'Error retraining model {model_name}: {e}' @@ -1296,10 +1100,12 @@ class MLFlowRepository(): 'success': False, 'experiment': None, 'message': error_msg, - 'traceback': traceback.format_exc() + 'traceback': traceback.format_exc(), } - def update_production_model(self, experiment: str, model_name: str, metadata: dict = {}) -> dict: + def update_production_model( + self, experiment: dict[str, Any], model_name: str, metadata: dict + ) -> dict: """ Update production model using the latest retraining run. @@ -1346,8 +1152,7 @@ class MLFlowRepository(): """ run_id = experiment['run_id'] experiment_id = experiment['experiment_id'] - metadata_result = self.update_production_model_by_run_id( - run_id, model_name, metadata) + metadata_result = self.update_production_model_by_run_id(run_id, model_name, metadata) metadata_result['mlflow_experiment_id'] = experiment_id diff --git a/laborious/utils/repository/opc_repository.py b/laborious/utils/repository/opc_repository.py index 96ecfef..072274a 100644 --- a/laborious/utils/repository/opc_repository.py +++ b/laborious/utils/repository/opc_repository.py @@ -1,16 +1,16 @@ -import asyncio -import traceback import time +import traceback from datetime import datetime from pathlib import Path from typing import Any + from asyncua import Client from asyncua.crypto.security_policies import SecurityPolicyBasic256 -from asyncua.ua import DataValue, Variant, VariantType, DateTime -from regex import F +from asyncua.ua import DataValue, DateTime, Variant, VariantType from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger + from laborious import metrics data_type_map = { @@ -33,17 +33,26 @@ data_type_map = { 'str': { 'converter': str, 'opc_type': VariantType.String, - } + }, } -class OpcRepository(): - def __init__(self, id: str, url: str, logger: Logger, - notification_handler: NotificationHandler, - reconnection_interval: int = 60, server_uri: str = None, cert_path: str = None, - private_key_path: str = None, server_cert_path: str = None, pod_id: str = None): +class OpcRepository: + def __init__( + self, + opc_id: str, + url: str, + logger: Logger, + notification_handler: NotificationHandler, + reconnection_interval: int = 60, + server_uri: str | None = None, + cert_path: str | None = None, + private_key_path: str | None = None, + server_cert_path: str | None = None, + pod_id: str | None = None, + ): self.url = url - self.id = id + self.id = opc_id self.server_uri = server_uri self.cert_path = cert_path self.private_key_path = private_key_path @@ -51,16 +60,16 @@ class OpcRepository(): self.logger = logger self.error_count = 0 self.reconnection_interval = reconnection_interval - self.last_reconnection_time = None + self.last_reconnection_time: None | datetime = None self.notification_handler = notification_handler - self.client = None + self.client: None | Client = None self.pod_id = pod_id self.metadata = { 'model_name': '-', 'model_id': '-', 'workflow_name': 'opc_repository', - 'schedule_name': '-' + 'schedule_name': '-', } async def set_security(self): @@ -85,11 +94,18 @@ class OpcRepository(): if not all([self.cert_path, self.private_key_path]): raise ValueError( - "Certificate and private key paths must be provided for secure connection.") + 'Certificate and private key paths must be provided for secure connection.' + ) + + if self.cert_path is None or self.private_key_path is None: + raise ValueError('Certificate and private key paths cannot be None') + cert = Path(self.cert_path) private_key = Path(self.private_key_path) - server_cert = Path( - self.server_cert_path) if self.server_cert_path else None + server_cert = Path(self.server_cert_path) if self.server_cert_path else None + + if self.client is None: + raise ValueError('Client must be initialized before setting security') self.client.application_uri = self.server_uri self.logger.custom_info('Setting security...', self.metadata) @@ -97,7 +113,7 @@ class OpcRepository(): SecurityPolicyBasic256, certificate=str(cert), private_key=str(private_key), - server_certificate=str(server_cert) + server_certificate=str(server_cert) if server_cert else None, ) self.client.secure_channel_timeout = 10000000 self.client.session_timeout = 10000000 @@ -115,8 +131,7 @@ class OpcRepository(): self.client = Client(self.url) if self.cert_path: await self.set_security() - self.logger.custom_info( - f'Starting connection to OPC server {self.id}...', self.metadata) + self.logger.custom_info(f'Starting connection to OPC server {self.id}...', self.metadata) return await self.try_connect() async def try_connect(self) -> tuple[bool, dict[str, Any]]: @@ -136,6 +151,13 @@ class OpcRepository(): try: self.last_reconnection_time = datetime.now() + if self.client is None: + return False, { + 'notification_id': f'OPC_CONNECTION_ERROR_{self.id}', + 'message': 'Client is not initialized', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + } await self.client.connect() return True, {} except Exception as e: @@ -143,11 +165,11 @@ class OpcRepository(): self.logger.custom_error(trace, self.metadata) return False, { - "notification_id": f"OPC_CONNECTION_ERROR_{self.id}", - "message": f"Failed to connect to OPC server: {e}", - "block": "opc_repository", - "level": NotificationLevel.ERROR, - "attachment_content": trace + 'notification_id': f'OPC_CONNECTION_ERROR_{self.id}', + 'message': f'Failed to connect to OPC server: {e}', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + 'attachment_content': trace, } async def disconnect(self): @@ -162,11 +184,9 @@ class OpcRepository(): return try: await self.client.disconnect() - self.logger.custom_info( - 'Disconnected from OPC server', self.metadata) + self.logger.custom_info('Disconnected from OPC server', self.metadata) except Exception as e: - self.logger.custom_error( - f"Failed to disconnect from OPC server: {e}", self.metadata) + self.logger.custom_error(f'Failed to disconnect from OPC server: {e}', self.metadata) self.client = None async def validate_connection(self) -> tuple[bool, dict[str, Any]]: @@ -203,52 +223,62 @@ class OpcRepository(): if self.error_count > 5: self.logger.custom_warning( - f"OPC server {self.id} will be disconnected due to multiple errors", self.metadata) + f'OPC server {self.id} will be disconnected due to multiple errors', self.metadata + ) try: await self.disconnect() except Exception as e: trace = traceback.format_exc() self.logger.custom_error( - f"Failed to disconnect from OPC server: {e}", self.metadata) + f'Failed to disconnect from OPC server: {e}', self.metadata + ) self.logger.custom_error(trace, self.metadata) self.logger.custom_info( - f"Attempting to reconnect to OPC server {self.id}...", self.metadata) + f'Attempting to reconnect to OPC server {self.id}...', self.metadata + ) return await self.connect() # Check if client is connected using asyncua's connection state try: - if self.client.uaclient.protocol is None or self.client.uaclient.protocol.state == "closed": + if ( + self.client.uaclient.protocol is None + or self.client.uaclient.protocol.state == 'closed' + ): # OPC server is not connected - self.logger.custom_error( - f"OPC server {self.id} is not connected", self.metadata) - if self.last_reconnection_time is None or (datetime.now() - self.last_reconnection_time).total_seconds( - ) > self.reconnection_interval: + self.logger.custom_error(f'OPC server {self.id} is not connected', self.metadata) + if ( + self.last_reconnection_time is None + or (datetime.now() - self.last_reconnection_time).total_seconds() + > self.reconnection_interval + ): await self.disconnect() self.logger.custom_info( - f"Trying to reconnect to OPC server {self.id}...", self.metadata) + f'Trying to reconnect to OPC server {self.id}...', self.metadata + ) return await self.connect() return False, { - "notification_id": f"OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}", - "message": f"OPC server {self.id} is not connected, waiting for next reconnection window...", - "block": "opc_repository", - "level": NotificationLevel.WARNING + 'notification_id': f'OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}', + 'message': f'OPC server {self.id} is not connected, waiting for next reconnection window...', + 'block': 'opc_repository', + 'level': NotificationLevel.WARNING, } return True, {} except Exception as e: trace = traceback.format_exc() - message = f"Failed to validate connection to OPC server: {e}" + message = f'Failed to validate connection to OPC server: {e}' self.logger.custom_error(message, self.metadata) return False, { - "notification_id": f"OPC_CONNECTION_CHECK_ERROR_{self.id}", - "message": message, - "block": "opc_repository", - "level": NotificationLevel.ERROR, - "attachment_content": trace + 'notification_id': f'OPC_CONNECTION_CHECK_ERROR_{self.id}', + 'message': message, + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + 'attachment_content': trace, } - async def write_data(self, node: str, value: Any, data_type: str, - logger: Logger, metadata: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + async def write_data( + self, node: str, value: Any, data_type: str, logger: Logger, metadata: dict[str, Any] + ) -> tuple[bool, dict[str, Any]]: """ Write data to OPC server with comprehensive validation and monitoring. @@ -286,42 +316,42 @@ class OpcRepository(): start_time = time.time() try: + if self.client is None: + return False, { + 'notification_id': f'OPC_WRITE_GET_NODE_ERROR_{self.id}', + 'message': 'Client is not initialized', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + } node_obj = self.client.get_node(node) except Exception as e: trace = traceback.format_exc() logger.custom_error(trace, metadata.get('schedule_name', 'N/A')) self.error_count += 1 return False, { - "notification_id": f"OPC_WRITE_GET_NODE_ERROR_{self.id}", - "message": f"Failed to get node from OPC server: {e} | metadata: {metadata}", - "block": "opc_repository", - "level": NotificationLevel.ERROR, - "attachment_content": trace + 'notification_id': f'OPC_WRITE_GET_NODE_ERROR_{self.id}', + 'message': f'Failed to get node from OPC server: {e} | metadata: {metadata}', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + 'attachment_content': trace, } if data_type not in data_type_map: return False, { - "notification_id": f"OPC_WRITE_DATA_TYPE_ERROR_{self.id}", - "message": f"Unsupported data type: {data_type} | metadata: {metadata}", - "block": "opc_repository", - "level": NotificationLevel.ERROR + 'notification_id': f'OPC_WRITE_DATA_TYPE_ERROR_{self.id}', + 'message': f'Unsupported data type: {data_type} | metadata: {metadata}', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, } data = data_type_map[data_type]['converter'](value) - logger.custom_info( - f'Writing {data} - {type(data)} to {node}', metadata) + logger.custom_info(f'Writing {data} - {type(data)} to {node}', metadata) now = datetime.now() ua_data = DataValue( Variant(data, data_type_map[data_type]['opc_type']), SourceTimestamp=DateTime( - now.year, - now.month, - now.day, - now.hour, - now.minute, - now.second, - now.microsecond - ) + now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond + ), ) try: @@ -331,7 +361,7 @@ class OpcRepository(): pod_id=self.pod_id, model_name=metadata['model_name'], pipeline_name=metadata['workflow_name'], - opc_server_id=self.id + opc_server_id=self.id, ).inc() end_time = time.time() @@ -340,7 +370,7 @@ class OpcRepository(): pod_id=self.pod_id, model_name=metadata['model_name'], pipeline_name=metadata['workflow_name'], - opc_server_id=self.id + opc_server_id=self.id, ).observe(response_time) except Exception as e: @@ -348,11 +378,11 @@ class OpcRepository(): logger.custom_error(trace, metadata) self.error_count += 1 return False, { - "notification_id": f"OPC_WRITE_DATA_ERROR_{self.id}", - "message": f"Failed to write data to OPC server: {e} | metadata: {metadata}", - "block": "opc_repository", - "level": NotificationLevel.ERROR, - "attachment_content": trace + 'notification_id': f'OPC_WRITE_DATA_ERROR_{self.id}', + 'message': f'Failed to write data to OPC server: {e} | metadata: {metadata}', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + 'attachment_content': trace, } self.error_count = 0 diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 4658524..781ad55 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -25,37 +25,37 @@ Environment Variables: - PROJECT_NAME: Project name for notifications (default: laborious) """ -from temporalio import workflow, client -from temporalio.worker import Worker, PollerBehaviorAutoscaling -from temporalio.runtime import Runtime, TelemetryConfig, PrometheusConfig +from temporalio import client, workflow +from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig +from temporalio.worker import PollerBehaviorAutoscaling, Worker with workflow.unsafe.imports_passed_through(): + import asyncio import os import sys - import asyncio - from laborious.workflows.minimal_retrain import MinimalRetrain - from laborious.workflows.predictions_batch import PredictionsBatch - from laborious.workflows.sub_workflows.prediction_process import PredictionProcess - from laborious.workflows.sub_workflows.format_and_export_prediction import \ - FormatAndExportPrediction - from laborious.activities.activities import Activities - from laborious.utils.connectors_config import ( - build_postgres_config, - build_mlflow_config, - build_minio_config, - build_opc_config, - build_mongodb_config - ) + + from prometheus_client import start_http_server from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import get_logger - from laborious import metrics - from prometheus_client import start_http_server - import lzma - import dataclasses + from laborious import metrics + from laborious.activities.activities import Activities + from laborious.utils.connectors_config import ( + build_minio_config, + build_mlflow_config, + build_mongodb_config, + build_opc_config, + build_postgres_config, + ) + from laborious.workflows.minimal_retrain import MinimalRetrain + from laborious.workflows.predictions_batch import PredictionsBatch + from laborious.workflows.sub_workflows.format_and_export_prediction import ( + FormatAndExportPrediction, + ) + from laborious.workflows.sub_workflows.prediction_process import PredictionProcess POD_ID = os.getenv('POD_ID') -SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', "9091")) +SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091')) async def main(): @@ -91,7 +91,7 @@ async def main(): logger.custom_info(f'Starting Worker with POD_ID: {POD_ID}', metadata) - logger.custom_info("Starting prometheus client...", metadata) + logger.custom_info('Starting prometheus client...', metadata) start_prometheus_server() logger.custom_info('Starting Notification Handler...', metadata) @@ -101,7 +101,7 @@ async def main(): connection_string=mongo_config['connection_string'], database=mongo_config['database_name'], logger=logger, - project_name=os.getenv('PROJECT_NAME', 'laborious') + project_name=os.getenv('PROJECT_NAME', 'laborious'), ) logger.custom_info('Starting Activities...', metadata) @@ -112,19 +112,17 @@ async def main(): minio_config=build_minio_config(), opc_config=build_opc_config(), logger=logger, - notification_handler=notification_handler + notification_handler=notification_handler, ) logger.custom_info('Initializing OPC...', metadata) await activities.init_opc() - logger.custom_info( - f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata) + logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata) new_runtime = Runtime( telemetry=TelemetryConfig( - metrics=PrometheusConfig( - bind_address=f"0.0.0.0:{SDK_METRICS_PORT}") + metrics=PrometheusConfig(bind_address=f'0.0.0.0:{SDK_METRICS_PORT}') ) ) @@ -133,7 +131,7 @@ async def main(): temporal_client = await client.Client.connect( target_host=host, namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'), - runtime=new_runtime + runtime=new_runtime, ) logger.custom_info('Starting Workers...', metadata) @@ -156,13 +154,12 @@ async def main(): max_concurrent_local_activities=50, max_cached_workflows=200, workflow_task_poller_behavior=PollerBehaviorAutoscaling(), - activity_task_poller_behavior=PollerBehaviorAutoscaling() + activity_task_poller_behavior=PollerBehaviorAutoscaling(), ), Worker( temporal_client, task_queue='predictions_batch-queue', - workflows=[PredictionsBatch, PredictionProcess, - FormatAndExportPrediction], + workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction], activities=[ # MLFlow activities.request_predict, @@ -181,15 +178,15 @@ async def main(): activities.load_custom_query, activities.repeat_last_prediction, activities.export_data_to_postgres, - activities.write_metrics + activities.write_metrics, ], max_concurrent_workflow_tasks=50, max_concurrent_activities=50, max_concurrent_local_activities=50, max_cached_workflows=200, workflow_task_poller_behavior=PollerBehaviorAutoscaling(), - activity_task_poller_behavior=PollerBehaviorAutoscaling() - ) + activity_task_poller_behavior=PollerBehaviorAutoscaling(), + ), ] handlers = [] @@ -203,7 +200,7 @@ async def main(): # If an exception occurs in any of the worker handlers, it will be propagated here. await asyncio.gather(*handlers) except BaseException as e: # NOSONAR - logger.custom_error(f"An unhandled exception occurred: {e}", metadata) + logger.custom_error(f'An unhandled exception occurred: {e}', metadata) finally: if notification_handler: notification_handler.shutdown() @@ -232,12 +229,12 @@ def start_prometheus_server(): SystemExit: If the metrics server fails to start """ try: - port = int(os.getenv("HTTP_METRICS_PORT", 9090)) + port = int(os.getenv('HTTP_METRICS_PORT', 9090)) start_http_server(port) - print(f"Prometheus server started on port {port}.") + print(f'Prometheus server started on port {port}.') metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP except Exception as e: - print(f"Failed to start Prometheus server: {e}") + print(f'Failed to start Prometheus server: {e}') os._exit(1) diff --git a/laborious/workflows/minimal_retrain.py b/laborious/workflows/minimal_retrain.py index 71f72ee..5497432 100644 --- a/laborious/workflows/minimal_retrain.py +++ b/laborious/workflows/minimal_retrain.py @@ -1,14 +1,16 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities - from typing import Any - from sientia_do.temporal.policies import retry_policy from datetime import timedelta + from typing import Any + + from sientia_do.temporal.policies import retry_policy + + from laborious.activities.activities import Activities -@workflow.defn(name="minimal_retrain") -class MinimalRetrain(): +@workflow.defn(name='minimal_retrain') +class MinimalRetrain: """ Automated model retraining workflow for the Laborious system. @@ -63,24 +65,24 @@ class MinimalRetrain(): 'schedule_name': input_data['schedule_name'], 'model_name': input_data['model_name'], 'model_id': input_data['model_id'], - 'workflow_name': 'minimal_retrain' + 'workflow_name': 'minimal_retrain', } } model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) - storage_result = await workflow.execute_local_activity_method( + storage_result = await workflow.execute_activity_method( Activities.query_to_minio, { **metadata, 'query': input_data['query'], 'datetime_columns': input_data.get('datetime_columns', []), 'model_name': model_name, - 'object_prefix': f'retrain_datasets/{model_name}/data' + 'object_prefix': f'retrain_datasets/{model_name}/data', }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=600) + start_to_close_timeout=timedelta(seconds=600), ) if not storage_result['success']: @@ -92,38 +94,32 @@ class MinimalRetrain(): **metadata, 'object_key': storage_result['object_key'], 'model_name': model_name, - 'model_config': model_config + 'model_config': model_config, }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(hours=1) + start_to_close_timeout=timedelta(hours=1), ) if experiment_response['success']: - update_report = await workflow.execute_activity_method( Activities.update_production_model, - { - **metadata, - 'model_name': model_name, - **experiment_response - }, + {**metadata, 'model_name': model_name, **experiment_response}, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), ) else: update_report = {} - report = await workflow.execute_activity_method( + report = await workflow.execute_local_activity_method( Activities.format_retrain_report, { **metadata, 'experiment_response': experiment_response, - 'model_id': input_data['model_id'], 'model_name': model_name, - 'update_report': update_report + 'update_report': update_report, }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), ) await workflow.execute_activity_method( @@ -132,8 +128,8 @@ class MinimalRetrain(): **metadata, 'data': report, 'schema': input_data['schema'], - 'table_name': input_data['table_name'] + 'table_name': input_data['table_name'], }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=600) + start_to_close_timeout=timedelta(seconds=600), ) diff --git a/laborious/workflows/predictions_batch.py b/laborious/workflows/predictions_batch.py index e522eaa..f79206f 100644 --- a/laborious/workflows/predictions_batch.py +++ b/laborious/workflows/predictions_batch.py @@ -1,14 +1,16 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities - from typing import Any - from sientia_do.temporal.policies import retry_policy from datetime import timedelta + from typing import Any + + from sientia_do.temporal.policies import retry_policy + + from laborious.activities.activities import Activities -@workflow.defn(name="predictions_batch") -class PredictionsBatch(): +@workflow.defn(name='predictions_batch') +class PredictionsBatch: """ Main batch prediction workflow for the Laborious system. @@ -74,7 +76,7 @@ class PredictionsBatch(): 'schedule_name': input_data['schedule_name'], 'model_name': input_data['model_name'], 'model_id': input_data['model_id'], - 'workflow_name': 'predictions_batch' + 'workflow_name': 'predictions_batch', } } @@ -84,10 +86,10 @@ class PredictionsBatch(): { **metadata, 'query': input_data['query'], - 'datetime_columns': input_data.get('datetime_columns', []) + 'datetime_columns': input_data.get('datetime_columns', []), }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=300) + start_to_close_timeout=timedelta(seconds=300), ) # Prepare input for prediction_process workflow @@ -98,28 +100,18 @@ class PredictionsBatch(): 'table_name': input_data['table_name'], 'model_id': input_data['model_id'], 'model_name': input_data['model_name'], - 'input_filters': input_data.get('input_filters', { - 'EMPTY_DATA': { - 'POLICY': 'STOP' - } - }), - 'mlflow_transform_filters': input_data.get('mlflow_transform_filters', { - 'API_ERROR': { - 'POLICY': 'STOP' - } - }), - 'mlflow_predict_filters': input_data.get('mlflow_predict_filters', { - 'API_ERROR': { - 'POLICY': 'STOP' - } - }), + 'input_filters': input_data.get('input_filters', {'EMPTY_DATA': {'POLICY': 'STOP'}}), + 'mlflow_transform_filters': input_data.get( + 'mlflow_transform_filters', {'API_ERROR': {'POLICY': 'STOP'}} + ), + 'mlflow_predict_filters': input_data.get( + 'mlflow_predict_filters', {'API_ERROR': {'POLICY': 'STOP'}} + ), 'model_config': input_data.get('model_config', {}), 'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']), 'opc_output_config': input_data.get('opc_output_config', {}), - 'prediction_store_policy': input_data.get( - 'prediction_store_policy', 'lts:1') + 'prediction_store_policy': input_data.get('prediction_store_policy', 'lts:1'), } # Execute prediction process workflow - await workflow.execute_child_workflow( - 'prediction_process', prediction_input) + await workflow.execute_child_workflow('prediction_process', prediction_input) diff --git a/laborious/workflows/sub_workflows/format_and_export_prediction.py b/laborious/workflows/sub_workflows/format_and_export_prediction.py index 8e7df07..72ff12d 100644 --- a/laborious/workflows/sub_workflows/format_and_export_prediction.py +++ b/laborious/workflows/sub_workflows/format_and_export_prediction.py @@ -1,15 +1,17 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities - from typing import Any from datetime import timedelta - from sientia_do.temporal.policies import retry_policy + from typing import Any + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + from sientia_do.temporal.policies import retry_policy + + from laborious.activities.activities import Activities -@workflow.defn(name="format_and_export_prediction") -class FormatAndExportPrediction(): +@workflow.defn(name='format_and_export_prediction') +class FormatAndExportPrediction: """ Data formatting and export workflow for prediction results. @@ -79,10 +81,10 @@ class FormatAndExportPrediction(): 'timestamp': input_data['timestamp'], 'model_id': input_data['model_id'], 'prediction_confidence': prediction_confidence, - 'prediction_store_policy': input_data['prediction_store_policy'] + 'prediction_store_policy': input_data['prediction_store_policy'], }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), ) else: @@ -94,22 +96,18 @@ class FormatAndExportPrediction(): 'timestamp': input_data['timestamp'], 'model_id': input_data['model_id'], 'prediction_confidence': prediction_confidence, - 'comment': input_data['comment'] + 'comment': input_data['comment'], }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), ) # write to opc prediction = await workflow.execute_activity_method( Activities.write_opc_data, - { - **metadata, - 'opc_output_config': input_data['opc_output_config'], - 'data': prediction - }, + {**metadata, 'opc_output_config': input_data['opc_output_config'], 'data': prediction}, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), ) # write to postgres @@ -120,21 +118,15 @@ class FormatAndExportPrediction(): 'schema': input_data['schema'], 'table_name': input_data['table_name'], 'data': prediction, - 'timestamp_conversion': { - 'column': 'timestamp', - 'format': DATETIME_FORMAT_WITH_TZ - } + 'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ}, }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), ) await workflow.execute_activity_method( Activities.write_metrics, - { - **metadata, - 'prediction': prediction - }, + {**metadata, 'prediction': prediction}, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), ) diff --git a/laborious/workflows/sub_workflows/prediction_process.py b/laborious/workflows/sub_workflows/prediction_process.py index 777fa1c..7bd36e2 100644 --- a/laborious/workflows/sub_workflows/prediction_process.py +++ b/laborious/workflows/sub_workflows/prediction_process.py @@ -1,14 +1,16 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities - from typing import Any - from sientia_do.temporal.policies import retry_policy from datetime import timedelta + from typing import Any + + from sientia_do.temporal.policies import retry_policy + + from laborious.activities.activities import Activities -@workflow.defn(name="prediction_process") -class PredictionProcess(): +@workflow.defn(name='prediction_process') +class PredictionProcess: """ Core prediction processing workflow for the Laborious system. @@ -84,10 +86,7 @@ class PredictionProcess(): # Get last timestamp for incremental processing last_timestamp = await workflow.execute_local_activity_method( Activities.get_last_timestamp, - { - **metadata, - 'data': data - }, + {**metadata, 'data': data}, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), ) @@ -97,7 +96,7 @@ class PredictionProcess(): **metadata, 'filters': input_data['input_filters'], 'data': data, - 'path_priority': input_data['path_priority'] + 'path_priority': input_data['path_priority'], } path_flag, confidence, comment = await workflow.execute_local_activity_method( @@ -116,12 +115,7 @@ class PredictionProcess(): # Request MLFlow model transformation response_data = await workflow.execute_local_activity_method( Activities.request_transform, - { - **metadata, - 'data': data, - 'model_name': model_name, - 'model_config': model_config - }, + {**metadata, 'data': data, 'model_name': model_name, 'model_config': model_config}, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=5), ) @@ -134,7 +128,7 @@ class PredictionProcess(): 'filters': input_data['mlflow_transform_filters'], 'data': response_data, 'type': 'transform', - 'path_priority': input_data['path_priority'] + 'path_priority': input_data['path_priority'], }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -155,7 +149,7 @@ class PredictionProcess(): 'filters': input_data['mlflow_transform_filters'], 'data': transformed_data, 'type': 'transform', - 'path_priority': input_data['path_priority'] + 'path_priority': input_data['path_priority'], }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -172,7 +166,7 @@ class PredictionProcess(): **metadata, 'data': transformed_data, 'model_name': model_name, - 'model_config': model_config + 'model_config': model_config, }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=5), @@ -186,7 +180,7 @@ class PredictionProcess(): 'filters': input_data['mlflow_predict_filters'], 'data': response_data, 'type': 'predict', - 'path_priority': input_data['path_priority'] + 'path_priority': input_data['path_priority'], }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -214,12 +208,19 @@ class PredictionProcess(): 'schema': input_data['schema'], 'table_name': input_data['table_name'], 'comment': comment, - 'prediction_store_policy': input_data['prediction_store_policy'] - } + 'prediction_store_policy': input_data['prediction_store_policy'], + }, ) - async def path_flag_handler(self, data: dict, path_flag: str, input_data: dict, - confidence: int, last_timestamp: str, comment: str) -> bool: + async def path_flag_handler( + self, + data: dict, + path_flag: str, + input_data: dict, + confidence: int, + last_timestamp: str, + comment: str, + ) -> bool: """ Handle path decisions based on filter results and confidence levels. @@ -265,7 +266,7 @@ class PredictionProcess(): 'schema': schema, 'table_name': table_name, 'model': model_id, - 'last_timestamp': last_timestamp + 'last_timestamp': last_timestamp, }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -288,8 +289,8 @@ class PredictionProcess(): 'table_name': table_name, 'comment': comment, 'opc_output_config': input_data['opc_output_config'], - 'prediction_store_policy': input_data['prediction_store_policy'] - } + 'prediction_store_policy': input_data['prediction_store_policy'], + }, ) return True diff --git a/mlruns/0/meta.yaml b/mlruns/0/meta.yaml new file mode 100644 index 0000000..2a5def3 --- /dev/null +++ b/mlruns/0/meta.yaml @@ -0,0 +1,6 @@ +artifact_location: file:///home/grezewave/Documents/projects/sientia/sientia-dataops-laborious_temporal/mlruns/0 +creation_time: 1760447041053 +experiment_id: '0' +last_update_time: 1760447041053 +lifecycle_stage: active +name: Default diff --git a/mlruns/586524947870967910/meta.yaml b/mlruns/586524947870967910/meta.yaml new file mode 100644 index 0000000..7221134 --- /dev/null +++ b/mlruns/586524947870967910/meta.yaml @@ -0,0 +1,6 @@ +artifact_location: file:///home/grezewave/Documents/projects/sientia/sientia-dataops-laborious_temporal/mlruns/586524947870967910 +creation_time: 1760447067255 +experiment_id: '586524947870967910' +last_update_time: 1760447067255 +lifecycle_stage: active +name: test diff --git a/mlruns/models/test/meta.yaml b/mlruns/models/test/meta.yaml new file mode 100644 index 0000000..9d225cc --- /dev/null +++ b/mlruns/models/test/meta.yaml @@ -0,0 +1,5 @@ +aliases: {} +creation_timestamp: 1760447068191 +description: null +last_updated_timestamp: 1760447068191 +name: test diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5b153a3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,159 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "laborious" +version = "0.0.0" +description = "Sientia DataOps Laborious - ML Model Orchestration System" +readme = "README.md" +requires-python = ">=3.11" +authors = [ + {name = "Aignosi", email = "dev@aignosi.com"} +] + +[tool.ruff] +line-length = 100 +target-version = "py311" +exclude = [ + ".git", + ".venv", + "venv", + "__pycache__", + "*.pyc", + ".pytest_cache", + "htmlcov", +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade + "N", # pep8-naming + "YTT", # flake8-2020 + "S", # flake8-bandit + "BLE", # flake8-blind-except + "A", # flake8-builtins + "C90", # mccabe complexity +] + +ignore = [ + "BLE001", # ignore blind except, we need to send notifications with any error + "E501", # line too long (handled by formatter) + "S101", # use of assert (needed for tests) + "S105", # possible hardcoded password (false positives) + "S106", # possible hardcoded password (false positives) + "N802", # function name should be lowercase (temporal decorators) + "N806", # variable in function should be lowercase +] + +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = [ + "S101", # assert allowed in tests + "S105", # hardcoded passwords ok in tests + "S106", # hardcoded passwords ok in tests +] + +[tool.ruff.lint.mccabe] +max-complexity = 15 + +[tool.ruff.format] +quote-style = "single" +indent-style = "space" +line-ending = "auto" + +[tool.mypy] +python_version = "3.11" +warn_return_any = false +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = false +warn_no_return = true +strict_equality = true +ignore_missing_imports = true + +# Ignore missing imports for external packages +[[tool.mypy.overrides]] +module = "temporalio.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sientia_do.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mlflow.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "prometheus_client.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sientia.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pandas.*" +ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-v", + "--strict-markers", + "--cov=model_manager", + "--cov-report=term-missing", + "--cov-report=html", + "--cov-report=xml", +] +markers = [ + "asyncio: marks tests as async", + "integration: marks tests as integration tests", + "unit: marks tests as unit tests", +] + +[tool.coverage.run] +source = ["model_manager"] +omit = [ + "*/tests/*", + "*/venv/*", + "*/__pycache__/*", + "*/site-packages/*", +] +branch = true + +[tool.coverage.report] +precision = 2 +show_missing = true +skip_covered = false +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "def __str__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", + "class .*\\bProtocol\\):", + "@(abc\\.)?abstractmethod", +] + +[tool.coverage.html] +directory = "htmlcov" + +[tool.bandit] +exclude_dirs = ["tests", "venv", ".venv"] +skips = ["B101", "B601"] # Skip assert and shell injection in controlled environments \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..56ab376 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,19 @@ +# Development and Testing Dependencies +# These packages are only needed for development, testing, and code quality checks +# Install with: pip install -r requirements-dev.txt + +# Code Quality & Linting +ruff>=0.1.0 # Fast Python linter and formatter (replaces flake8, black, isort) +mypy>=1.7.0 # Static type checker +bandit>=1.7.5 # Security vulnerability scanner +pandas-stubs>=2.0.0 # Type stubs for pandas +types-requests>=2.31.0 # Type stubs for requests + +# Testing +pytest>=7.4.0 # Testing framework +pytest-cov>=4.1.0 # Coverage plugin for pytest +pytest-asyncio>=0.21.0 # Async test support (already in main requirements) + +# Development Tools +ipython>=8.12.0 # Enhanced Python shell +ipdb>=0.13.13 # IPython debugger \ No newline at end of file diff --git a/tests/laborious/activities/test_activities.py b/tests/laborious/activities/test_activities.py index de418be..1999e91 100644 --- a/tests/laborious/activities/test_activities.py +++ b/tests/laborious/activities/test_activities.py @@ -1,18 +1,19 @@ +from unittest.mock import ANY, MagicMock, patch + from pytest import mark -from unittest.mock import patch, MagicMock, ANY -from sientia_do.temporal.activities.postgres import Postgres + from laborious.activities.activities import Activities -from laborious.activities.mlflow import MLFlow from laborious.activities.gates import Gates +from laborious.activities.mlflow import MLFlow from laborious.activities.opc import OPC +from laborious.activities.storage import Storage -@patch('laborious.activities.activities.Postgres.__init__') +@patch('laborious.activities.activities.Storage.__init__') @patch('laborious.activities.activities.MLFlow.__init__') @patch('laborious.activities.activities.OPC.__init__') @patch('laborious.activities.activities.Gates.__init__') -def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgres_init): - +def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_storage_init): postgres_config = { 'host': 'localhost', 'port': 5432, @@ -20,20 +21,23 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre 'password': 'postgres', 'dbname': 'postgres', 'min_connections': 1, - 'max_connections': 10 + 'max_connections': 10, } - mlflow_config = { - 'host': 'localhost', - 'port': 5000, - 'username': 'mlflow', - 'password': 'mlflow' + minio_config = { + 'endpoint_url': 'localhost:9000', + 'access_key': 'minio', + 'secret_key': 'minio123', + 'region_name': 'us-east-1', + 'default_bucket': 'test', } + mlflow_config = {'host': 'localhost', 'port': 5000, 'username': 'mlflow', 'password': 'mlflow'} + opc_config = { 'bootstrap_servers': 'localhost:9092', 'polling_time': 1000, - 'group_id': 'test-group' + 'group_id': 'test-group', } logger = MagicMock() @@ -42,18 +46,19 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre activities = Activities( postgres_config=postgres_config, mlflow_config=mlflow_config, + minio_config=minio_config, opc_config=opc_config, logger=logger, - notification_handler=notification_handler + notification_handler=notification_handler, ) assert isinstance(activities, Activities) - assert isinstance(activities, Postgres) + assert isinstance(activities, Storage) assert isinstance(activities, MLFlow) assert isinstance(activities, OPC) assert isinstance(activities, Gates) - mock_postgres_init.assert_called_once_with( + mock_storage_init.assert_called_once_with( ANY, host=postgres_config['host'], port=postgres_config['port'], @@ -62,8 +67,9 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre dbname=postgres_config['dbname'], min_connections=postgres_config['min_connections'], max_connections=postgres_config['max_connections'], + minio_config=minio_config, logger=logger, - notification_handler=notification_handler + notification_handler=notification_handler, ) mock_mlflow_init.assert_called_once_with( @@ -72,30 +78,25 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre mlflow_port=mlflow_config['port'], mlflow_username=mlflow_config['username'], mlflow_password=mlflow_config['password'], + minio_config=minio_config, logger=logger, - notification_handler=notification_handler + notification_handler=notification_handler, ) mock_opc_init.assert_called_once_with( - ANY, - opc_servers=opc_config, - logger=logger, - notification_handler=notification_handler + ANY, opc_servers=opc_config, logger=logger, notification_handler=notification_handler ) mock_gates_init.assert_called_once_with( - ANY, - logger=logger, - notification_handler=notification_handler + ANY, logger=logger, notification_handler=notification_handler ) @mark.asyncio -@patch('laborious.activities.activities.Postgres', return_value=MagicMock()) +@patch('laborious.activities.activities.Storage', return_value=MagicMock()) @patch('laborious.activities.activities.MLFlow', return_value=MagicMock()) @patch('laborious.activities.activities.OPC', return_value=MagicMock()) -async def test_shutdown(mock_opc_init, - _mock_mlflow_init, mock_postgres_init): +async def test_shutdown(mock_opc_init, _mock_mlflow_init, mock_storage_init): postgres_config = { 'host': 'localhost', 'port': 5432, @@ -103,20 +104,23 @@ async def test_shutdown(mock_opc_init, 'password': 'postgres', 'dbname': 'postgres', 'min_connections': 1, - 'max_connections': 10 + 'max_connections': 10, } - mlflow_config = { - 'host': 'localhost', - 'port': 5000, - 'username': 'mlflow', - 'password': 'mlflow' + minio_config = { + 'endpoint_url': 'localhost:9000', + 'access_key': 'minio', + 'secret_key': 'minio123', + 'region_name': 'us-east-1', + 'default_bucket': 'test', } + mlflow_config = {'host': 'localhost', 'port': 5000, 'username': 'mlflow', 'password': 'mlflow'} + opc_config = { 'bootstrap_servers': 'localhost:9092', 'polling_time': 1000, - 'group_id': 'test-group' + 'group_id': 'test-group', } logger = MagicMock() @@ -125,11 +129,12 @@ async def test_shutdown(mock_opc_init, activities = Activities( postgres_config=postgres_config, mlflow_config=mlflow_config, + minio_config=minio_config, opc_config=opc_config, logger=logger, - notification_handler=notification_handler + notification_handler=notification_handler, ) await activities.shutdown() mock_opc_init.shutdown.assert_called_once() - mock_postgres_init.close.assert_called_once() + mock_storage_init.close.assert_called_once() diff --git a/tests/laborious/activities/test_gates.py b/tests/laborious/activities/test_gates.py index a65c6d4..1a376e2 100644 --- a/tests/laborious/activities/test_gates.py +++ b/tests/laborious/activities/test_gates.py @@ -1,6 +1,8 @@ -from unittest.mock import MagicMock, ANY, patch +from unittest.mock import ANY, MagicMock, patch + from pytest import fixture, mark from sientia_do.notifications.models import NotificationLevel + from laborious.activities.gates import Gates @@ -20,11 +22,11 @@ def gates_activity(): metadata = { - "metadata": { - "model_id": "test_model", - "model_name": "test_model", - "workflow_name": "test_workflow", - "schema_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', }, } @@ -34,20 +36,18 @@ async def test_input_gate_invalid_filter(gates_activity): # Arrange input_data = { **metadata, - 'filters': { - 'INVALID_FILTER': {'POLICY': 'STOP'} - }, + 'filters': {'INVALID_FILTER': {'POLICY': 'STOP'}}, 'data': {'value': [1, 2, 3]}, - 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'] + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], } # Act result = await gates_activity.input_gate(input_data) # Assert - assert result == (None, 0, "") + assert result == (None, 0, '') gates_activity.error.assert_called_once_with( - "Filter INVALID_FILTER not found", metadata['metadata'] + 'Filter INVALID_FILTER not found', metadata['metadata'] ) @@ -57,28 +57,27 @@ async def test_input_gate_filter_exception(mock_input_filter_functions, gates_ac # Arrange mock_input_filter_functions.__contains__.return_value = True mock_input_filter_functions.__getitem__.return_value = MagicMock( - side_effect=Exception("Test error")) + side_effect=Exception('Test error') + ) input_data = { **metadata, - 'filters': { - 'EMPTY_DATA': {'policy': 'STOP', 'config': {}} - }, + 'filters': {'EMPTY_DATA': {'policy': 'STOP', 'config': {}}}, 'data': {'value': []}, - 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'] + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], } # Act result = await gates_activity.input_gate(input_data) # Assert - assert result == (None, 0, "") + assert result == (None, 0, '') gates_activity.send_notification.assert_called_once_with( metadata=metadata['metadata'], - notification_id="INTPUT_GATE_ERROR__EMPTY_DATA", + notification_id='INTPUT_GATE_ERROR__EMPTY_DATA', message="Error in filter EMPTY_DATA:{'policy': 'STOP', 'config': {}}: \n Test error", - block="input_gate", + block='input_gate', level=NotificationLevel.ERROR, - attachment_content=ANY + attachment_content=ANY, ) @@ -89,14 +88,14 @@ async def test_input_gate_no_filters(gates_activity): **metadata, 'filters': {}, 'data': {'value': [1, 2, 3]}, - 'path_priority': ['CONTINUE', 'STOP', 'REPEAT'] + 'path_priority': ['CONTINUE', 'STOP', 'REPEAT'], } # Act result = await gates_activity.input_gate(input_data) # Assert - assert result == (None, 0, "") + assert result == (None, 0, '') gates_activity.debug.assert_called() @@ -105,18 +104,16 @@ async def test_input_gate_with_filter(gates_activity): # Arrange input_data = { **metadata, - 'filters': { - 'EMPTY_DATA': {'policy': 'STOP', 'config': {}} - }, + 'filters': {'EMPTY_DATA': {'policy': 'STOP', 'config': {}}}, 'data': {'value': []}, - 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'] + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], } # Act result = await gates_activity.input_gate(input_data) # Assert - assert result == ('STOP', -1, "Input data with bad quality") + assert result == ('STOP', -1, 'Input data with bad quality') gates_activity.debug.assert_called() @@ -125,51 +122,49 @@ async def test_mlflow_response_gate_invalid_filter(gates_activity): # Arrange input_data = { **metadata, - 'filters': { - 'INVALID_FILTER': {'POLICY': 'STOP'} - }, + 'filters': {'INVALID_FILTER': {'POLICY': 'STOP'}}, 'data': {'content': {'message': 'success'}}, 'type': 'test', - 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'] + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], } # Act result = await gates_activity.mlflow_response_gate(input_data) # Assert - assert result == (None, 0, "") + assert result == (None, 0, '') @mark.asyncio @patch('laborious.activities.gates.mlflow_response_filter_functions') -async def test_mlflow_response_gate_filter_exception(mock_mlflow_response_filter_functions, - gates_activity): +async def test_mlflow_response_gate_filter_exception( + mock_mlflow_response_filter_functions, gates_activity +): # Arrange mock_mlflow_response_filter_functions.__contains__.return_value = True mock_mlflow_response_filter_functions.__getitem__.return_value = MagicMock( - side_effect=Exception("Test error")) + side_effect=Exception('Test error') + ) input_data = { **metadata, - 'filters': { - 'INVALID_FILTER': {'POLICY': 'STOP'} - }, + 'filters': {'INVALID_FILTER': {'POLICY': 'STOP'}}, 'data': {'content': {'message': 'success'}}, 'type': 'test', - 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'] + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], } # Act result = await gates_activity.mlflow_response_gate(input_data) # Assert - assert result == (None, 0, "") + assert result == (None, 0, '') gates_activity.send_notification.assert_called_once_with( metadata=metadata['metadata'], - notification_id="MLFLOW_GATE_RESPONSE_FILTER__INVALID_FILTER", + notification_id='MLFLOW_GATE_RESPONSE_FILTER__INVALID_FILTER', message="Error in filter INVALID_FILTER:{'POLICY': 'STOP'}: \n Test error", - block="mlflow_gate", + block='mlflow_gate', level=NotificationLevel.ERROR, - attachment_content=ANY + attachment_content=ANY, ) @@ -181,14 +176,14 @@ async def test_mlflow_response_gate_no_filters(gates_activity): 'filters': {}, 'data': {'content': {'message': 'success'}}, 'type': 'test', - 'path_priority': ['CONTINUE', 'STOP', 'REPEAT'] + 'path_priority': ['CONTINUE', 'STOP', 'REPEAT'], } # Act result = await gates_activity.mlflow_response_gate(input_data) # Assert - assert result == (None, 0, "") + assert result == (None, 0, '') gates_activity.debug.assert_called() @@ -197,25 +192,20 @@ async def test_mlflow_response_gate_with_filter(gates_activity): # Arrange input_data = { **metadata, - 'filters': { - 'API_ERROR': {'policy': 'STOP'} - }, + 'filters': {'API_ERROR': {'policy': 'STOP'}}, 'data': { 'success': False, - 'content': { - 'message': 'API error occurred', - 'traceback': 'error trace' - } + 'content': {'message': 'API error occurred', 'traceback': 'error trace'}, }, 'type': 'test', - 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'] + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], } # Act result = await gates_activity.mlflow_response_gate(input_data) # Assert - assert result == ('STOP', -1, "API error occurred") + assert result == ('STOP', -1, 'API error occurred') gates_activity.debug.assert_called() gates_activity.send_notification.assert_called() @@ -225,58 +215,53 @@ async def test_mlflow_content_gate_invalid_filter(gates_activity): # Arrange input_data = { **metadata, - 'filters': { - 'INVALID_FILTER': {'POLICY': 'STOP'} - }, + 'filters': {'INVALID_FILTER': {'POLICY': 'STOP'}}, 'data': {'value': [1, 2, 3]}, 'type': 'test', - 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'] + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], } # Act result = await gates_activity.mlflow_content_gate(input_data) # Assert - assert result == (None, 0, "") + assert result == (None, 0, '') @mark.asyncio @patch('laborious.activities.gates.mlflow_content_filter_functions') -async def test_mlflow_content_gate_filter_exception(mock_mlflow_content_filter_functions, - gates_activity): +async def test_mlflow_content_gate_filter_exception( + mock_mlflow_content_filter_functions, gates_activity +): # Arrange mock_mlflow_content_filter_functions.__contains__.return_value = True mock_mlflow_content_filter_functions.__getitem__.return_value = MagicMock( - side_effect=Exception("Test error")) + side_effect=Exception('Test error') + ) input_data = { **metadata, - 'filters': { - 'API_ERROR': {'POLICY': 'STOP'} - }, + 'filters': {'API_ERROR': {'POLICY': 'STOP'}}, 'data': { 'success': False, - 'content': { - 'message': 'API error occurred', - 'traceback': 'error trace' - } + 'content': {'message': 'API error occurred', 'traceback': 'error trace'}, }, 'type': 'test', - 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'] + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], } # Act result = await gates_activity.mlflow_content_gate(input_data) # Assert - assert result == (None, 0, "") + assert result == (None, 0, '') gates_activity.debug.assert_called() gates_activity.send_notification.assert_called_once_with( metadata=metadata['metadata'], - notification_id="MLFLOW_GATE_CONTENT_FILTER__API_ERROR", + notification_id='MLFLOW_GATE_CONTENT_FILTER__API_ERROR', message="Error in filter API_ERROR:{'POLICY': 'STOP'}: \n Test error", - block="mlflow_gate", + block='mlflow_gate', level=NotificationLevel.ERROR, - attachment_content=ANY + attachment_content=ANY, ) @@ -288,14 +273,14 @@ async def test_mlflow_content_gate_no_filters(gates_activity): 'filters': {}, 'data': {'value': [1, 2, 3]}, 'type': 'test', - 'path_priority': ['CONTINUE', 'STOP', 'REPEAT'] + 'path_priority': ['CONTINUE', 'STOP', 'REPEAT'], } # Act result = await gates_activity.mlflow_content_gate(input_data) # Assert - assert result == (None, 0, "") + assert result == (None, 0, '') gates_activity.debug.assert_called() @@ -304,20 +289,17 @@ async def test_mlflow_content_gate_with_filter(gates_activity): # Arrange input_data = { **metadata, - 'filters': { - 'NAN_VALUES': {'policy': 'STOP', 'config': {}} - }, + 'filters': {'NAN_VALUES': {'policy': 'STOP', 'config': {}}}, 'data': {'value': [None, None, None]}, 'type': 'test', - 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'] + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], } # Act result = await gates_activity.mlflow_content_gate(input_data) # Assert - assert result == ( - 'STOP', -1, "Transformed data not passed the content filter") + assert result == ('STOP', -1, 'Transformed data not passed the content filter') gates_activity.debug.assert_called() gates_activity.send_notification.assert_called() @@ -328,7 +310,8 @@ def test_get_prediction_store_policy_invalid_policy(gates_activity): # Act policy_type, policy_value = gates_activity.get_prediction_store_policy( - prediction_store_policy, metadata) + prediction_store_policy, metadata + ) # Assert assert policy_type == 'lts' @@ -341,7 +324,8 @@ def test_get_prediction_store_policy_invalid_policy_value(gates_activity): # Act policy_type, policy_value = gates_activity.get_prediction_store_policy( - prediction_store_policy, metadata) + prediction_store_policy, metadata + ) # Assert assert policy_type == 'lts' @@ -354,7 +338,8 @@ def test_get_prediction_store_policy_valid_policy_type(gates_activity): # Act policy_type, policy_value = gates_activity.get_prediction_store_policy( - prediction_store_policy, metadata) + prediction_store_policy, metadata + ) # Assert assert policy_type == 'lts' @@ -367,7 +352,8 @@ def test_get_prediction_store_policy_valid_policy(gates_activity): # Act policy_type, policy_value = gates_activity.get_prediction_store_policy( - prediction_store_policy, metadata) + prediction_store_policy, metadata + ) # Assert assert policy_type == 'erl' @@ -380,16 +366,12 @@ async def test_format_prediction_no_timestamp(gates_activity): input_data = { **metadata, 'data': { - 'prediction': { - '2023-05-26 11:12:27': 1 - }, - 'response_time': { - '2023-05-26 11:12:27': 0.1 - } + 'prediction': {'2023-05-26 11:12:27': 1}, + 'response_time': {'2023-05-26 11:12:27': 0.1}, }, 'model_id': 'test_model', 'prediction_confidence': 0.9, - 'prediction_store_policy': 'lts:1' + 'prediction_store_policy': 'lts:1', } # Act @@ -402,7 +384,7 @@ async def test_format_prediction_no_timestamp(gates_activity): assert result['model_id'] == {0: 'test_model'} assert result['prediction_confidence'] == {0: 0.9} assert result['prediction_status'] == {0: 'Good'} - assert result['comments'] == {0: ""} + assert result['comments'] == {0: ''} @mark.asyncio @@ -420,11 +402,11 @@ async def test_format_prediction_with_timestamp_erl(gates_activity): '2023-05-26 11:12:27': 0.1, '2023-05-26 11:12:28': 0.2, '2023-05-26 11:12:29': 0.3, - } + }, }, 'model_id': 'test_model', 'prediction_confidence': 0.9, - 'prediction_store_policy': 'erl:2' + 'prediction_store_policy': 'erl:2', } # Act @@ -433,12 +415,11 @@ async def test_format_prediction_with_timestamp_erl(gates_activity): # Assert assert result['prediction'] == {0: 2, 1: 1} assert result['response_time'] == {0: 0.2, 1: 0.1} - assert result['timestamp'] == { - 0: '2023-05-26 11:12:28', 1: '2023-05-26 11:12:27'} + assert result['timestamp'] == {0: '2023-05-26 11:12:28', 1: '2023-05-26 11:12:27'} assert result['model_id'] == {0: 'test_model', 1: 'test_model'} assert result['prediction_confidence'] == {0: 0.9, 1: 0.9} assert result['prediction_status'] == {0: 'Good', 1: 'Good'} - assert result['comments'] == {0: "", 1: ""} + assert result['comments'] == {0: '', 1: ''} @mark.asyncio @@ -456,11 +437,11 @@ async def test_format_prediction_with_timestamp_lts(gates_activity): '2023-05-26 11:12:27': 0.1, '2023-05-26 11:12:28': 0.2, '2023-05-26 11:12:29': 0.3, - } + }, }, 'model_id': 'test_model', 'prediction_confidence': 0.9, - 'prediction_store_policy': 'lts:2' + 'prediction_store_policy': 'lts:2', } # Act @@ -469,12 +450,11 @@ async def test_format_prediction_with_timestamp_lts(gates_activity): # Assert assert result['prediction'] == {0: 3, 1: 2} assert result['response_time'] == {0: 0.3, 1: 0.2} - assert result['timestamp'] == { - 0: '2023-05-26 11:12:29', 1: '2023-05-26 11:12:28'} + assert result['timestamp'] == {0: '2023-05-26 11:12:29', 1: '2023-05-26 11:12:28'} assert result['model_id'] == {0: 'test_model', 1: 'test_model'} assert result['prediction_confidence'] == {0: 0.9, 1: 0.9} assert result['prediction_status'] == {0: 'Good', 1: 'Good'} - assert result['comments'] == {0: "", 1: ""} + assert result['comments'] == {0: '', 1: ''} @mark.asyncio @@ -482,22 +462,23 @@ async def test_format_prediction_with_timestamp_invalid_policy(gates_activity): # Arrange input_data = { **metadata, - 'data': {'prediction': [1, 2, 3], - 'response_time': [0.1, 0.2, 0.3], - 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29']}, + 'data': { + 'prediction': [1, 2, 3], + 'response_time': [0.1, 0.2, 0.3], + 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'], + }, 'model_id': 'test_model', 'prediction_confidence': 0.9, - 'prediction_store_policy': 'lts:2' + 'prediction_store_policy': 'lts:2', } - gates_activity.get_prediction_store_policy = MagicMock( - return_value=('invalid', 1)) + gates_activity.get_prediction_store_policy = MagicMock(return_value=('invalid', 1)) try: - result = await gates_activity.format_prediction(input_data) + await gates_activity.format_prediction(input_data) except ValueError as e: - assert str(e) == "Invalid policy type: invalid" + assert str(e) == 'Invalid policy type: invalid' else: - assert False, "Expected ValueError" + raise AssertionError('Expected ValueError') @mark.asyncio @@ -508,7 +489,7 @@ async def test_format_default_prediction(gates_activity): 'timestamp': '2023-05-26 11:12:27', 'model_id': 'test_model', 'prediction_confidence': 0.1, - 'comment': 'Test comment' + 'comment': 'Test comment', } # Act @@ -528,12 +509,7 @@ async def test_format_default_prediction(gates_activity): @mark.asyncio async def test_get_last_timestamp_with_data(gates_activity): # Arrange - input_data = { - **metadata, - 'data': { - 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'] - } - } + input_data = {**metadata, 'data': {'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28']}} # Act result = await gates_activity.get_last_timestamp(input_data) @@ -545,10 +521,7 @@ async def test_get_last_timestamp_with_data(gates_activity): @mark.asyncio async def test_get_last_timestamp_no_data(gates_activity): # Arrange - input_data = { - 'data': {}, - **metadata - } + input_data = {'data': {}, **metadata} # Act result = await gates_activity.get_last_timestamp(input_data) @@ -567,30 +540,28 @@ async def test_write_metrics(mock_metrics, gates_activity): 'prediction': { 'prediction': [1, 2, 3], 'prediction_confidence': [0.9, 0.8, 0.7], - 'response_time': [0.1, 0.2, 0.3] - } + 'response_time': [0.1, 0.2, 0.3], + }, } await gates_activity.write_metrics(input_data) mock_metrics.PREDICTIONS_WRITTEN_COUNT.labels.assert_called_once_with( pod_id=gates_activity.pod_id, model_name=metadata['metadata']['model_name'], - pipeline_name=metadata['metadata']['workflow_name'] + pipeline_name=metadata['metadata']['workflow_name'], ) mock_metrics.PREDICTIONS_WRITTEN_COUNT.labels.return_value.inc.assert_called_once_with() mock_metrics.PREDICTION_CONFIDENCE_MONITOR.labels.assert_called_once_with( pod_id=gates_activity.pod_id, model_name=metadata['metadata']['model_name'], - pipeline_name=metadata['metadata']['workflow_name'] - ) - mock_metrics.PREDICTION_CONFIDENCE_MONITOR.labels.return_value.set.assert_called_once_with( - 0.9 + pipeline_name=metadata['metadata']['workflow_name'], ) + mock_metrics.PREDICTION_CONFIDENCE_MONITOR.labels.return_value.set.assert_called_once_with(0.9) mock_metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels.assert_called_once_with( pod_id=gates_activity.pod_id, model_name=metadata['metadata']['model_name'], - pipeline_name=metadata['metadata']['workflow_name'] + pipeline_name=metadata['metadata']['workflow_name'], ) mock_metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels.return_value.observe.assert_called_once_with( 0.1 diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index 85cb7f1..cfb66a1 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -1,45 +1,68 @@ -from datetime import datetime -from unittest.mock import ANY, MagicMock, patch +from unittest.mock import ANY, MagicMock, call, patch import numpy as np -from pandas import DataFrame, Timestamp -from pytest import fixture, mark, raises -from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ -from laborious.activities.mlflow import MLFlow +from pytest import fixture, mark from sientia_do.notifications.models import NotificationLevel +from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ + +from laborious.activities.mlflow import MLFlow -@patch("laborious.activities.mlflow.MLFlowRepository") -def test___init__(mock_mlflow_repository): +@patch('laborious.activities.mlflow.MLFlowRepository') +@patch('laborious.activities.mlflow.MinioRepository') +def test___init__(mock_minio_repository, mock_mlflow_repository): mlflow = MLFlow( - mlflow_host="http://localhost", + mlflow_host='http://localhost', mlflow_port=5000, - mlflow_username="admin", - mlflow_password="admin", + mlflow_username='admin', + mlflow_password='admin', + minio_config={ + 'endpoint_url': 'http://localhost:9000', + 'access_key': 'minio', + 'secret_key': 'minio123', + 'region_name': 'us-east-1', + 'default_bucket': 'test', + }, logger=MagicMock(), - notification_handler=MagicMock() + notification_handler=MagicMock(), ) - assert mlflow.mlflow_host == "http://localhost" + assert mlflow.mlflow_host == 'http://localhost' assert mlflow.mlflow_port == 5000 - assert mlflow.mlflow_username == "admin" - assert mlflow.mlflow_password == "admin" + assert mlflow.mlflow_username == 'admin' + assert mlflow.mlflow_password == 'admin' - mock_mlflow_repository.assert_called_once_with( - "http://localhost:5000", "admin", "admin", ANY + mock_mlflow_repository.assert_called_once_with('http://localhost:5000', 'admin', 'admin', ANY) + + mock_minio_repository.assert_called_once_with( + logger=ANY, + notification_handler=ANY, + minio_endpoint_url='http://localhost:9000', + minio_access_key='minio', + minio_secret_key='minio123', + minio_region_name='us-east-1', + minio_default_bucket='test', ) @fixture -@patch("laborious.activities.mlflow.MLFlowRepository") -def mlflow(mock_mlflow_repository): +@patch('laborious.activities.mlflow.MLFlowRepository') +@patch('laborious.activities.mlflow.MinioRepository') +def mlflow(mock_minio_repository, mock_mlflow_repository): mlflow = MLFlow( - mlflow_host="http://localhost:5000", + mlflow_host='http://localhost:5000', mlflow_port=5000, - mlflow_username="admin", - mlflow_password="admin", + mlflow_username='admin', + mlflow_password='admin', + minio_config={ + 'endpoint_url': 'http://localhost:9000', + 'access_key': 'minio', + 'secret_key': 'minio123', + 'region_name': 'us-east-1', + 'default_bucket': 'test', + }, logger=MagicMock(), - notification_handler=MagicMock() + notification_handler=MagicMock(), ) mlflow.send_notification = MagicMock() @@ -48,44 +71,67 @@ def mlflow(mock_mlflow_repository): metadata = { - "metadata": { - "model_id": "test_model", - "model_name": "test_model", - "workflow_name": "test_workflow", - "schema_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', }, } @mark.asyncio -@patch("laborious.activities.mlflow.DataFrame") -@patch("laborious.activities.mlflow.max") +@patch('laborious.activities.mlflow.DataFrame') +@patch('laborious.activities.mlflow.max') async def test_request_transform_success(mock_max, mock_dataframe, mlflow): mock_max.return_value = '2024-01-02' # Mock input data input_data = { **metadata, 'data': [ - {'timestamp': '2024-01-01', 'variable': 'var1', - 'value': 1.0, 'created_at': '2024-01-01 12:00:00'}, - {'timestamp': '2024-01-01', 'variable': 'var2', - 'value': 2.0, 'created_at': '2024-01-01 12:00:00'}, - {'timestamp': '2024-01-02', 'variable': 'var1', - 'value': 3.0, 'created_at': '2024-01-02 12:00:00'}, - {'timestamp': '2024-01-02', 'variable': 'var2', - 'value': 4.0, 'created_at': '2024-01-02 12:00:00'}, - {'timestamp': '2024-01-02', 'variable': 'var1', - 'value': 1.0, 'created_at': '2024-01-01 12:00:00'}, - {'timestamp': '2024-01-02', 'variable': 'var2', - 'value': 1.0, 'created_at': '2024-01-01 12:00:00'} + { + 'timestamp': '2024-01-01', + 'variable': 'var1', + 'value': 1.0, + 'created_at': '2024-01-01 12:00:00', + }, + { + 'timestamp': '2024-01-01', + 'variable': 'var2', + 'value': 2.0, + 'created_at': '2024-01-01 12:00:00', + }, + { + 'timestamp': '2024-01-02', + 'variable': 'var1', + 'value': 3.0, + 'created_at': '2024-01-02 12:00:00', + }, + { + 'timestamp': '2024-01-02', + 'variable': 'var2', + 'value': 4.0, + 'created_at': '2024-01-02 12:00:00', + }, + { + 'timestamp': '2024-01-02', + 'variable': 'var1', + 'value': 1.0, + 'created_at': '2024-01-01 12:00:00', + }, + { + 'timestamp': '2024-01-02', + 'variable': 'var2', + 'value': 1.0, + 'created_at': '2024-01-01 12:00:00', + }, ], 'model_name': 'test_model', - 'model_config': {} + 'model_config': {}, } # Mock the transform response - expected_response = {'prediction': [0.5, 0.6], 'timestamp': [ - '2024-01-01', '2024-01-02']} + expected_response = {'prediction': [0.5, 0.6], 'timestamp': ['2024-01-01', '2024-01-02']} mlflow.model_monitoring_repository.transform.return_value = expected_response mock_dataframe.return_value.sort_values.return_value = mock_dataframe.return_value @@ -114,30 +160,25 @@ async def test_request_transform_success(mock_max, mock_dataframe, mlflow): @mark.asyncio -@patch("laborious.activities.mlflow.DataFrame") -@patch("laborious.activities.mlflow.to_datetime") -@patch("laborious.activities.mlflow.max") +@patch('laborious.activities.mlflow.DataFrame') +@patch('laborious.activities.mlflow.to_datetime') +@patch('laborious.activities.mlflow.max') async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflow): mock_max.return_value = '2024-01-02' # Mock input data input_data = { **metadata, 'data': { - "variable": { - "2024-01-01": "var1", - "2024-01-02": "var2", - "2024-01-03": "var1", - "2024-01-04": "var2" + 'variable': { + '2024-01-01': 'var1', + '2024-01-02': 'var2', + '2024-01-03': 'var1', + '2024-01-04': 'var2', }, - "value": { - "2024-01-01": 1.0, - "2024-01-02": 2.0, - "2024-01-03": 3.0, - "2024-01-04": 4.0 - } + 'value': {'2024-01-01': 1.0, '2024-01-02': 2.0, '2024-01-03': 3.0, '2024-01-04': 4.0}, }, 'model_name': 'test_model', - 'model_config': {} + 'model_config': {}, } # Mock the predict response @@ -148,9 +189,7 @@ async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflo response_data = await mlflow.request_predict(input_data) mock_dataframe.assert_called_once_with(input_data['data']) - mock_dataframe.return_value.replace.assert_called_once_with( - np.nan, None, inplace=True - ) + mock_dataframe.return_value.replace.assert_called_once_with(np.nan, None, inplace=True) mock_dataframe.return_value.__setitem__.assert_any_call( 'timestamp', mock_to_datetime.return_value.dt.strftime.return_value ) @@ -158,9 +197,7 @@ async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflo mock_to_datetime.assert_called_once_with( mock_dataframe.return_value.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ ) - mock_to_datetime.return_value.dt.strftime.assert_called_once_with( - DATETIME_FORMAT - ) + mock_to_datetime.return_value.dt.strftime.assert_called_once_with(DATETIME_FORMAT) # Verify the response assert response_data == expected_response @@ -172,98 +209,211 @@ async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflo @mark.asyncio -async def test_retrain_model(mlflow): - data = { - "model_id": [4, 5, 6, 7], - "created_at": [1, 2, 3, 4], - "timestamp": [1, 1, 2, 2], - "variable": ["var1", "var2", "var1", "var2"], - "value": [1, 2, 3, 4] +@patch('laborious.activities.mlflow.to_datetime') +async def test_retrain_model_success_data_success_retrain(mock_to_datetime, mlflow): + mlflow.model_monitoring_repository.retrain_model.return_value = { + 'success': True, + 'experiment': 'test_experiment', + 'message': 'Model retrained successfully.', } - mlflow.model_monitoring_repository.retrain_model.return_value = ( - 'Model retrained successfully', 'test') + response = await mlflow.retrain_model( + { + **metadata, + 'object_key': 'test_object_key', + 'model_name': 'test_model', + 'model_config': { + 'target': 'target', + 'transform_flavor': 'sklearn', + 'predict_flavor': 'pyfunc', + }, + } + ) - response = await mlflow.retrain_model({ - **metadata, - 'data': data, - 'model_name': 'test_model' - }) + raw_data = mlflow.minio_repository.get_parquet_as_dataframe.return_value - mlflow.model_monitoring_repository.retrain_model.assert_called_once() + timestamp = raw_data.__getitem__.return_value.max.return_value + + raw_data.sort_values.assert_called_once_with('created_at', ascending=False) + raw_data.sort_values.return_value.drop_duplicates.assert_called_once_with( + subset=['variable', 'timestamp'], keep='first' + ) + raw_data = raw_data.sort_values.return_value.drop_duplicates.return_value + + raw_data.drop.assert_has_calls( + [ + call(columns=['model_id'], inplace=True, errors='ignore'), + call(columns=['created_at'], inplace=True, errors='ignore'), + ] + ) + raw_data.pivot.assert_called_once_with(index='timestamp', columns='variable', values='value') + raw_data.pivot.return_value.fillna.assert_called_once_with(np.nan, inplace=True) + + raw_data = raw_data.pivot.return_value + + raw_data.__setitem__.assert_has_calls( + [ + call('timestamp', raw_data.index), + call('timestamp', mock_to_datetime.return_value.dt.strftime.return_value), + call('timestamp', mock_to_datetime.return_value), + ] + ) + mock_to_datetime.assert_has_calls( + [call(raw_data.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ)] + ) + mock_to_datetime.assert_has_calls( + [call(raw_data.__getitem__.return_value, format=DATETIME_FORMAT)] + ) + + mlflow.model_monitoring_repository.retrain_model.assert_called_once_with( + data=raw_data, + model_name='test_model', + model_config={ + 'target': 'target', + 'transform_flavor': 'sklearn', + 'predict_flavor': 'pyfunc', + }, + metadata=metadata['metadata'], + ) assert response == { - "status": 'Model retrained successfully', - "timestamp": 2, - "experiment": 'test' + 'success': True, + 'experiment': 'test_experiment', + 'message': 'Model retrained successfully.', + 'timestamp': timestamp, } @mark.asyncio -async def test_retrain_model_error(mlflow): - mlflow.model_monitoring_repository.retrain_model.side_effect = Exception( - 'Error retraining model' - ) - - data = { - "model_id": [4, 5, 6, 7], - "created_at": [1, 2, 3, 4], - "timestamp": [1, 1, 2, 2], - "variable": ["var1", "var2", "var1", "var2"], - "value": [1, 2, 3, 4] +@patch('laborious.activities.mlflow.to_datetime') +async def test_retrain_model_success_data_fail_retrain(mock_to_datetime, mlflow): + mlflow.model_monitoring_repository.retrain_model.return_value = { + 'success': False, + 'traceback': 'test_traceback', + 'message': 'Model retrained failed.', } - try: - await mlflow.retrain_model({ + response = await mlflow.retrain_model( + { **metadata, - 'data': data, - 'model_name': 'test_model' - }) - except Exception as e: - assert str(e) == 'Error retraining model' - mlflow.send_notification.assert_called_once_with( - metadata=metadata['metadata'], - notification_id='RETRAIN_MODEL_ERROR', - message='Error retraining model test_model: Error retraining model', - block='retrain_model', - level=NotificationLevel.ERROR, - attachment_content=ANY - ) - else: - assert False, "No exception raised" + 'object_key': 'test_object_key', + 'model_name': 'test_model', + 'model_config': { + 'target': 'target', + 'transform_flavor': 'sklearn', + 'predict_flavor': 'pyfunc', + }, + } + ) + + raw_data = mlflow.minio_repository.get_parquet_as_dataframe.return_value + + timestamp = raw_data.__getitem__.return_value.max.return_value + + raw_data.sort_values.assert_called_once_with('created_at', ascending=False) + raw_data.sort_values.return_value.drop_duplicates.assert_called_once_with( + subset=['variable', 'timestamp'], keep='first' + ) + raw_data = raw_data.sort_values.return_value.drop_duplicates.return_value + + raw_data.drop.assert_has_calls( + [ + call(columns=['model_id'], inplace=True, errors='ignore'), + call(columns=['created_at'], inplace=True, errors='ignore'), + ] + ) + raw_data.pivot.assert_called_once_with(index='timestamp', columns='variable', values='value') + raw_data.pivot.return_value.fillna.assert_called_once_with(np.nan, inplace=True) + + raw_data = raw_data.pivot.return_value + + raw_data.__setitem__.assert_has_calls( + [ + call('timestamp', raw_data.index), + call('timestamp', mock_to_datetime.return_value.dt.strftime.return_value), + call('timestamp', mock_to_datetime.return_value), + ] + ) + mock_to_datetime.assert_has_calls( + [call(raw_data.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ)] + ) + mock_to_datetime.assert_has_calls( + [call(raw_data.__getitem__.return_value, format=DATETIME_FORMAT)] + ) + + mlflow.model_monitoring_repository.retrain_model.assert_called_once_with( + data=raw_data, + model_name='test_model', + model_config={ + 'target': 'target', + 'transform_flavor': 'sklearn', + 'predict_flavor': 'pyfunc', + }, + metadata=metadata['metadata'], + ) + + mlflow.send_notification.assert_called_once_with( + metadata=metadata['metadata'], + notification_id='RETRAIN_MODEL_ERROR', + message='Error retraining model test_model: Model retrained failed.', + block='retrain_model', + level=NotificationLevel.ERROR, + attachment_content=ANY, + ) + + assert response == { + 'success': False, + 'traceback': 'test_traceback', + 'message': 'Model retrained failed.', + 'timestamp': timestamp, + } + + +@mark.asyncio +async def test_retrain_model_data_error(mlflow): + mlflow.minio_repository.get_parquet_as_dataframe.side_effect = Exception( + 'Error loading retrain data' + ) + + response = await mlflow.retrain_model( + { + **metadata, + 'object_key': 'test_object_key', + 'model_name': 'test_model', + 'model_config': { + 'target': 'target', + 'transform_flavor': 'sklearn', + 'predict_flavor': 'pyfunc', + }, + } + ) + + assert response == { + 'success': False, + 'message': 'Error loading retrain data: Error loading retrain data', + 'traceback': ANY, + 'timestamp': ANY, + } @mark.asyncio async def test_update_production_model(mlflow): - mlflow.model_monitoring_repository.update_production_model.return_value = ( - { - "data1": 1, - "data2": 2 - } - ) - input_data = { **metadata, 'model_name': 'test_model', 'model_id': 1, 'experiment': 'test', 'timestamp': 2, - 'status': 'success' + 'status': 'success', } response = await mlflow.update_production_model(input_data) mlflow.model_monitoring_repository.update_production_model.assert_called_once_with( - experiment='test', model_name='test_model') + experiment='test', model_name='test_model', metadata=metadata['metadata'] + ) - assert response == { - 'data1': {0: 1}, - 'data2': {0: 2}, - 'model_id': {0: 1}, - 'model_name': {0: 'test_model'}, - 'timestamp': {0: 2}, - 'status': {0: 'success'} - } + assert response == mlflow.model_monitoring_repository.update_production_model.return_value @mark.asyncio @@ -278,7 +428,7 @@ async def test_update_production_model_error(mlflow): 'model_id': 1, 'experiment': 'test', 'timestamp': 2, - 'status': 'success' + 'status': 'success', } try: @@ -291,7 +441,7 @@ async def test_update_production_model_error(mlflow): message='Error updating production model test_model: Error updating production model', block='update_production_model', level=NotificationLevel.ERROR, - attachment_content=ANY + attachment_content=ANY, ) else: - assert False, "No exception raised" + raise AssertionError('No exception raised') diff --git a/tests/laborious/activities/test_opc.py b/tests/laborious/activities/test_opc.py index 07f00b8..815df27 100644 --- a/tests/laborious/activities/test_opc.py +++ b/tests/laborious/activities/test_opc.py @@ -1,57 +1,55 @@ -from unittest.mock import patch, MagicMock, ANY, call, AsyncMock -from pandas import DataFrame -from pytest import fixture, mark +from unittest.mock import ANY, AsyncMock, MagicMock, call, patch + import pytest_asyncio +from pandas import DataFrame +from pytest import mark from sientia_do.notifications.models import NotificationLevel from laborious.activities.opc import OPC metadata = { - "metadata": { - "model_id": "test_model", - "model_name": "test_model", - "workflow_name": "test_workflow", - "schema_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', }, } def test__init__(): - servers = { - 'server1': 'config' - } - opc = OPC( - opc_servers=servers, - logger=MagicMock(), - notification_handler=MagicMock() - ) + servers = {'server1': 'config'} + opc = OPC(opc_servers=servers, logger=MagicMock(), notification_handler=MagicMock()) assert opc.opc_servers == servers assert opc.opc_repository == {} @mark.asyncio -@patch("laborious.activities.opc.OpcRepository") -@patch("laborious.activities.opc.OPC.send_notification") +@patch('laborious.activities.opc.OpcRepository') +@patch('laborious.activities.opc.OPC.send_notification') async def test_init_opc(mock_send_notification, mock_opc_repository): mock_logger = MagicMock() server1 = MagicMock( - connect=AsyncMock(return_value=(True, {})), - write_data=AsyncMock(return_value=(True, {})) + connect=AsyncMock(return_value=(True, {})), write_data=AsyncMock(return_value=(True, {})) ) server2 = MagicMock( - connect=AsyncMock(return_value=(True, {})), - write_data=AsyncMock(return_value=(True, {})) + connect=AsyncMock(return_value=(True, {})), write_data=AsyncMock(return_value=(True, {})) ) server3 = MagicMock( - connect=AsyncMock(return_value=(False, { - 'notification_id': 'OPC_CONNECTION_ERROR_server3', - 'message': 'Failed to connect to OPC server: Test error', - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - 'attachment_content': 'Test error' - })), - write_data=AsyncMock(return_value=(True, {})) + connect=AsyncMock( + return_value=( + False, + { + 'notification_id': 'OPC_CONNECTION_ERROR_server3', + 'message': 'Failed to connect to OPC server: Test error', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + 'attachment_content': 'Test error', + }, + ) + ), + write_data=AsyncMock(return_value=(True, {})), ) mock_opc_repository.side_effect = [server1, server2, server3] mock_notification_handler = MagicMock() @@ -82,12 +80,10 @@ async def test_init_opc(mock_send_notification, mock_opc_repository): 'private_key_path': '', 'server_cert_path': '', 'reconnection_interval': 60, - } + }, } opc = OPC( - opc_servers=servers, - logger=mock_logger, - notification_handler=mock_notification_handler + opc_servers=servers, logger=mock_logger, notification_handler=mock_notification_handler ) await opc.init_opc() @@ -97,57 +93,63 @@ async def test_init_opc(mock_send_notification, mock_opc_repository): assert opc.opc_repository['server1'] == server1 assert opc.opc_repository['server2'] == server2 - mock_opc_repository.assert_has_calls([ - call( - id="server1", - url="http://localhost:8080", - logger=mock_logger, - server_uri="opc.tcp://localhost:4840", - cert_path="", - private_key_path="", - server_cert_path="", - notification_handler=mock_notification_handler, - reconnection_interval=60, - pod_id='localhost' - ), - ]) - mock_opc_repository.assert_has_calls([ - call( - id="server2", - url="http://localhost:8080", - logger=mock_logger, - server_uri="opc.tcp://localhost:4840", - cert_path="", - private_key_path="", - server_cert_path="", - notification_handler=mock_notification_handler, - reconnection_interval=60, - pod_id='localhost' - ) - ]) + mock_opc_repository.assert_has_calls( + [ + call( + opc_id='server1', + url='http://localhost:8080', + logger=mock_logger, + server_uri='opc.tcp://localhost:4840', + cert_path='', + private_key_path='', + server_cert_path='', + notification_handler=mock_notification_handler, + reconnection_interval=60, + pod_id='localhost', + ), + ] + ) + mock_opc_repository.assert_has_calls( + [ + call( + opc_id='server2', + url='http://localhost:8080', + logger=mock_logger, + server_uri='opc.tcp://localhost:4840', + cert_path='', + private_key_path='', + server_cert_path='', + notification_handler=mock_notification_handler, + reconnection_interval=60, + pod_id='localhost', + ) + ] + ) server1.connect.assert_called_once() server2.connect.assert_called_once() - mock_send_notification.assert_has_calls([ - call( - metadata={ - 'model_id': '-', - 'model_name': '-', - 'workflow_name': '-', - 'schedule_name': 'INITIALIZATION' - }, - notification_id="OPC_CONNECTION_ERROR_server3", - message="Failed to connect to OPC server: Test error", - block="opc_repository", - level=NotificationLevel.ERROR, - attachment_content=ANY - ) - ]) + mock_send_notification.assert_has_calls( + [ + call( + metadata={ + 'model_id': '-', + 'model_name': '-', + 'workflow_name': '-', + 'schedule_name': 'INITIALIZATION', + }, + notification_id='OPC_CONNECTION_ERROR_server3', + message='Failed to connect to OPC server: Test error', + block='opc_repository', + level=NotificationLevel.ERROR, + attachment_content=ANY, + ) + ] + ) @pytest_asyncio.fixture -@patch("laborious.activities.opc.OpcRepository") +@patch('laborious.activities.opc.OpcRepository') async def opc(mock_opc_repository): servers = { 'server1': { @@ -161,17 +163,9 @@ async def opc(mock_opc_repository): } } - mock_opc_repository.return_value.write_data = AsyncMock( - return_value=(True, {}) - ) - mock_opc_repository.return_value.connect = AsyncMock( - return_value=(True, {}) - ) - opc = OPC( - opc_servers=servers, - logger=MagicMock(), - notification_handler=MagicMock() - ) + mock_opc_repository.return_value.write_data = AsyncMock(return_value=(True, {})) + mock_opc_repository.return_value.connect = AsyncMock(return_value=(True, {})) + opc = OPC(opc_servers=servers, logger=MagicMock(), notification_handler=MagicMock()) await opc.init_opc() opc.send_notification = MagicMock() return opc @@ -188,58 +182,79 @@ WRITE_DATA_CASES = [ @mark.parametrize('tag,data_type,data', WRITE_DATA_CASES) @mark.asyncio async def test_write_data_success(opc, tag, data_type, data): - result = await opc.write_data(server_id='server1', tag=tag, data=data, - data_type=data_type, tag_type='prediction', metadata=metadata) + result = await opc.write_data( + server_id='server1', + tag=tag, + data=data, + data_type=data_type, + tag_type='prediction', + metadata=metadata, + ) assert result is True opc.opc_repository['server1'].write_data.assert_called_once_with( - tag, data, data_type, opc.logger, metadata) + tag, data, data_type, opc.logger, metadata + ) @mark.asyncio async def test_write_data_failed(opc): - opc.opc_repository['server1'].write_data.return_value = (False, { - 'notification_id': 'OPC_WRITE_DATA_ERROR_server1', - 'message': 'Failed to write data to OPC server: Test error', - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - 'attachment_content': 'Test error' - }) + opc.opc_repository['server1'].write_data.return_value = ( + False, + { + 'notification_id': 'OPC_WRITE_DATA_ERROR_server1', + 'message': 'Failed to write data to OPC server: Test error', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + 'attachment_content': 'Test error', + }, + ) - result = await opc.write_data(server_id='server1', tag='tag1', data=50, - data_type='int', tag_type='prediction', metadata=metadata) + result = await opc.write_data( + server_id='server1', + tag='tag1', + data=50, + data_type='int', + tag_type='prediction', + metadata=metadata, + ) assert result is False opc.send_notification.assert_called_once_with( metadata=metadata, - notification_id="OPC_WRITE_DATA_ERROR_server1", - message="Failed to write data to OPC server: Test error", - block="opc_repository", + notification_id='OPC_WRITE_DATA_ERROR_server1', + message='Failed to write data to OPC server: Test error', + block='opc_repository', level=NotificationLevel.ERROR, - attachment_content=ANY + attachment_content=ANY, ) @mark.asyncio async def test_write_data_exception(opc): - opc.opc_repository['server1'].write_data.side_effect = Exception( - "Test error") + opc.opc_repository['server1'].write_data.side_effect = Exception('Test error') try: - await opc.write_data(server_id='server1', tag='tag1', data=50, - data_type='int', tag_type='prediction', metadata=metadata) + await opc.write_data( + server_id='server1', + tag='tag1', + data=50, + data_type='int', + tag_type='prediction', + metadata=metadata, + ) except Exception: opc.send_notification.assert_called_once_with( metadata=metadata, - notification_id="WRITE_OPC_PREDICTION_ERROR", - message="Error writing data to OPC server: Test error", - block="write_opc_data", + notification_id='WRITE_OPC_PREDICTION_ERROR', + message='Error writing data to OPC server: Test error', + block='write_opc_data', level=NotificationLevel.ERROR, - attachment_content=ANY + attachment_content=ANY, ) else: - assert False, "Expected an exception to be raised" + raise AssertionError('Expected an exception to be raised') @mark.asyncio @@ -247,20 +262,13 @@ async def test_write_opc_data_success(opc): # Arrange input_data = { **metadata, - 'data': { - 'prediction': [0.75], - 'prediction_confidence': [0.95] - }, + 'data': {'prediction': [0.75], 'prediction_confidence': [0.95]}, 'opc_output_config': { 'server1': { - 'prediction_tags': { - 'tag1': {'data_type': 'float'} - }, - 'confidence_tags': { - 'tag2': {'data_type': 'float'} - } + 'prediction_tags': {'tag1': {'data_type': 'float'}}, + 'confidence_tags': {'tag2': {'data_type': 'float'}}, } - } + }, } # Act @@ -270,25 +278,30 @@ async def test_write_opc_data_success(opc): # Assert assert output == {'data': 'data'} - opc.write_data.assert_has_calls([ - call( - server_id='server1', - tag='tag1', - data=0.75, - data_type='float', - tag_type='prediction', - metadata=metadata['metadata'] - )]) - opc.write_data.assert_has_calls([ - call( - server_id='server1', - tag='tag2', - data=0.95, - data_type='float', - tag_type='confidence', - metadata=metadata['metadata'] - ) - ]) + opc.write_data.assert_has_calls( + [ + call( + server_id='server1', + tag='tag1', + data=0.75, + data_type='float', + tag_type='prediction', + metadata=metadata['metadata'], + ) + ] + ) + opc.write_data.assert_has_calls( + [ + call( + server_id='server1', + tag='tag2', + data=0.95, + data_type='float', + tag_type='confidence', + metadata=metadata['metadata'], + ) + ] + ) assert opc.write_data.call_count == 2 @@ -297,17 +310,9 @@ async def test_write_opc_data_empty_config(opc): # Arrange input_data = { **metadata, - 'data': { - 'prediction': [0.75], - 'prediction_confidence': [0.95] - }, + 'data': {'prediction': [0.75], 'prediction_confidence': [0.95]}, 'opc_servers': ['server1'], - 'opc_output_config': { - 'server1': { - 'prediction_tags': {}, - 'confidence_tags': {} - } - } + 'opc_output_config': {'server1': {'prediction_tags': {}, 'confidence_tags': {}}}, } # Act @@ -322,20 +327,13 @@ async def test_write_opc_data_no_validate_server(opc): opc.validate_server = MagicMock(return_value=False) input_data = { **metadata, - 'data': { - 'prediction': [0.75], - 'prediction_confidence': [0.95] - }, + 'data': {'prediction': [0.75], 'prediction_confidence': [0.95]}, 'opc_output_config': { 'server1': { - 'prediction_tags': { - 'tag1': {'data_type': 'float'} - }, - 'confidence_tags': { - 'tag2': {'data_type': 'float'} - } + 'prediction_tags': {'tag1': {'data_type': 'float'}}, + 'confidence_tags': {'tag2': {'data_type': 'float'}}, } - } + }, } # Act @@ -345,10 +343,13 @@ async def test_write_opc_data_no_validate_server(opc): opc.opc_repository['server1'].write_data.assert_not_called() -@mark.parametrize('data,success,expected', [ - (DataFrame({'prediction_confidence': [0]}), True, 0), - (DataFrame({'prediction_confidence': [0]}), False, 12), -]) +@mark.parametrize( + 'data,success,expected', + [ + (DataFrame({'prediction_confidence': [0]}), True, 0), + (DataFrame({'prediction_confidence': [0]}), False, 12), + ], +) def test_process_confidence(opc, data, success, expected): # Act result = opc.process_confidence(data, success, metadata) diff --git a/tests/laborious/activities/test_storage.py b/tests/laborious/activities/test_storage.py new file mode 100644 index 0000000..87c28f0 --- /dev/null +++ b/tests/laborious/activities/test_storage.py @@ -0,0 +1,203 @@ +import datetime +from unittest.mock import ANY, AsyncMock, MagicMock, patch + +from pytest import fixture, mark +from sientia_do.notifications.models import NotificationLevel +from sientia_do.temporal.activities.postgres import Postgres + +from laborious.activities.storage import Storage + +metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + } +} + + +@fixture +@patch('laborious.activities.storage.MinioRepository') +def storage(mock_minio_repository): + return Storage( + host='localhost', + port=5432, + user='postgres', + password='postgres', + dbname='postgres', + min_connections=1, + max_connections=10, + minio_config={ + 'endpoint_url': 'localhost:9000', + 'access_key': 'minio', + 'secret_key': 'minio123', + 'region_name': 'us-east-1', + 'default_bucket': 'test', + }, + logger=MagicMock(), + notification_handler=MagicMock(), + ) + + +@patch('laborious.activities.storage.MinioRepository') +def test___init___not_hasattr(mock_minio_repository): + logger = MagicMock() + notification_handler = MagicMock() + storage = Storage( + host='localhost', + port=5432, + user='postgres', + password='postgres', + dbname='postgres', + min_connections=1, + max_connections=10, + minio_config={ + 'endpoint_url': 'localhost:9000', + 'access_key': 'minio', + 'secret_key': 'minio123', + 'region_name': 'us-east-1', + 'default_bucket': 'test', + }, + logger=logger, + notification_handler=notification_handler, + ) + assert isinstance(storage, Postgres) + + mock_minio_repository.assert_called_once_with( + logger=logger, + notification_handler=notification_handler, + minio_endpoint_url='localhost:9000', + minio_access_key='minio', + minio_secret_key='minio123', + minio_region_name='us-east-1', + minio_default_bucket='test', + ) + + +@patch('laborious.activities.storage.MinioRepository') +def test___init___none_minio_repository(mock_minio_repository, storage): + storage.minio_repository = None + logger = MagicMock() + notification_handler = MagicMock() + + storage.__init__( + host='localhost', + port=5432, + user='postgres', + password='postgres', + dbname='postgres', + min_connections=1, + max_connections=10, + minio_config={ + 'endpoint_url': 'localhost:9000', + 'access_key': 'minio', + 'secret_key': 'minio123', + 'region_name': 'us-east-1', + 'default_bucket': 'test', + }, + logger=logger, + notification_handler=notification_handler, + ) + + mock_minio_repository.assert_called_once_with( + logger=logger, + notification_handler=notification_handler, + minio_endpoint_url='localhost:9000', + minio_access_key='minio', + minio_secret_key='minio123', + minio_region_name='us-east-1', + minio_default_bucket='test', + ) + + +@patch('laborious.activities.storage.MinioRepository') +def test___init___done_repository(mock_minio_repository, storage): + storage.__init__( + host='localhost', + port=5432, + user='postgres', + password='postgres', + dbname='postgres', + min_connections=1, + max_connections=10, + minio_config={ + 'endpoint_url': 'localhost:9000', + 'access_key': 'minio', + 'secret_key': 'minio123', + 'region_name': 'us-east-1', + 'default_bucket': 'test', + }, + logger=MagicMock(), + notification_handler=MagicMock(), + ) + mock_minio_repository.assert_not_called() + assert storage.minio_repository is not None + + +@mark.asyncio +async def test_query_to_minio_not_data(storage): + storage.load_custom_query = AsyncMock(return_value=None) + result = await storage.query_to_minio({}) + + storage.load_custom_query.assert_called_once_with({}) + assert result['success'] is False + assert result['message'] == 'No data returned from query' + + +@mark.asyncio +@patch('laborious.activities.storage.pd.DataFrame') +@patch('laborious.activities.storage.now') +async def test_query_to_minio_success(now, dataframe, storage): + data = [{'a': 1}, {'a': 2}, {'a': 3}] + storage.load_custom_query = AsyncMock(return_value=data) + now.return_value = datetime.datetime(2024, 1, 1, 0, 0, 0) + storage.minio_repository.minio_bucket = 'test' + + result = await storage.query_to_minio({'object_prefix': 'test', **metadata}) + + dataframe.assert_called_once_with(data) + + storage.minio_repository.store_dataframe_as_parquet.assert_called_once_with( + dataframe=dataframe.return_value, + uri='s3://test/test_2024-01-01_00-00-00.parquet', + object_name='test_2024-01-01_00-00-00.parquet', + metadata=metadata['metadata'], + ) + + assert result['success'] is True + assert result['object_key'] == 'test_2024-01-01_00-00-00.parquet' + assert result['uri'] == 's3://test/test_2024-01-01_00-00-00.parquet' + + +@mark.asyncio +async def test_query_to_minio_error(storage): + storage.send_notification = MagicMock() + storage.load_custom_query = AsyncMock(side_effect=Exception('test')) + result = await storage.query_to_minio({**metadata, 'object_prefix': 'test'}) + assert result['success'] is False + assert result['message'] == 'test' + storage.send_notification.assert_called_once_with( + metadata=metadata['metadata'], + notification_id='ERROR_STORING_QUERY_TO_MINIO', + message='Error storing query to MinIO: test', + block='query_to_minio', + level=NotificationLevel.ERROR, + attachment_content=ANY, + ) + + +def test_close(storage): + storage.minio_repository = MagicMock() + + storage.close() + + assert storage.minio_repository is None + + +def test___del__(storage): + storage.close = MagicMock() + + storage.__del__() + + storage.close.assert_called_once() diff --git a/tests/laborious/utils/filters/test_conditional_filters.py b/tests/laborious/utils/filters/test_conditional_filters.py index 405bc9b..de25b7b 100644 --- a/tests/laborious/utils/filters/test_conditional_filters.py +++ b/tests/laborious/utils/filters/test_conditional_filters.py @@ -1,23 +1,29 @@ from pandas import DataFrame from laborious.utils.filters.conditional_filters import ( + filter_empty_data, filter_specific_variables_null_values, - filter_empty_data ) def test_filter_specific_variables_null_values(): - assert filter_specific_variables_null_values( - DataFrame( - {'variable': ['variable1', 'variable2'], 'value': [1, 2]}), - config={'variables': ['variable2']}) is False + assert ( + filter_specific_variables_null_values( + DataFrame({'variable': ['variable1', 'variable2'], 'value': [1, 2]}), + config={'variables': ['variable2']}, + ) + is False + ) def test_filter_specific_variables_null_values_with_null_values(): - assert filter_specific_variables_null_values( - DataFrame( - {'variable': ['variable1', 'variable2'], 'value': [1, None]}), - config={'variables': ['variable2']}) is True + assert ( + filter_specific_variables_null_values( + DataFrame({'variable': ['variable1', 'variable2'], 'value': [1, None]}), + config={'variables': ['variable2']}, + ) + is True + ) def test_filter_empty_data(): @@ -25,6 +31,7 @@ def test_filter_empty_data(): def test_filter_empty_data_with_data(): - assert filter_empty_data( - DataFrame({'variable': ['variable1', 'variable2'], 'value': [1, 2]}), - {}) is False + assert ( + filter_empty_data(DataFrame({'variable': ['variable1', 'variable2'], 'value': [1, 2]}), {}) + is False + ) diff --git a/tests/laborious/utils/filters/test_mlflow_filters.py b/tests/laborious/utils/filters/test_mlflow_filters.py index f9c61e9..5353a6f 100644 --- a/tests/laborious/utils/filters/test_mlflow_filters.py +++ b/tests/laborious/utils/filters/test_mlflow_filters.py @@ -1,22 +1,23 @@ from pandas import DataFrame + from laborious.utils.filters.mlflow_filters import api_error_filter, nan_values_filter def test_api_error_filter_invalid_response(): - assert api_error_filter(None, {}) == True # NOSONAR + assert api_error_filter(None, {}) is True # NOSONAR def test_api_error_filter_valid_response_fail(): - assert api_error_filter({'success': False}, {}) == True + assert api_error_filter({'success': False}, {}) is True def test_api_error_filter_valid_response_success(): - assert api_error_filter({'success': True}, {}) == False + assert api_error_filter({'success': True}, {}) is False def test_nan_values_filter_all_nan_values(): - assert nan_values_filter(DataFrame({'variable': [None, None]}), {}) == True + assert nan_values_filter(DataFrame({'variable': [None, None]}), {}) is True def test_nan_values_filter_no_nan_values(): - assert nan_values_filter(DataFrame({'variable': [1, 2]}), {}) == False + assert nan_values_filter(DataFrame({'variable': [1, 2]}), {}) is False diff --git a/tests/laborious/utils/repository/test_minio_repository.py b/tests/laborious/utils/repository/test_minio_repository.py new file mode 100644 index 0000000..ec279d2 --- /dev/null +++ b/tests/laborious/utils/repository/test_minio_repository.py @@ -0,0 +1,134 @@ +from unittest.mock import MagicMock, patch + +from botocore.utils import ClientError +from pytest import fixture, raises + +from laborious.utils.repository.minio_repository import MinioRepository + + +@patch('laborious.utils.repository.minio_repository.boto3') +@patch('laborious.utils.repository.minio_repository.Config') +def test___init___(mock_config, mock_boto3): + minio_repository = MinioRepository( + minio_endpoint_url='localhost:9000', + minio_access_key='minio', + minio_secret_key='minio123', + minio_region_name='us-east-1', + minio_default_bucket='test', + logger=MagicMock(), + notification_handler=MagicMock(), + ) + + assert minio_repository.storage_options == { + 'key': 'minio', + 'secret': 'minio123', + 'client_kwargs': {'endpoint_url': 'localhost:9000'}, + } + assert minio_repository.minio_bucket == 'test' + assert minio_repository.minio_endpoint_url == 'localhost:9000' + assert minio_repository.minio_region_name == 'us-east-1' + + mock_config.assert_called_once_with( + signature_version='s3v4', + s3={'addressing_style': 'path'}, + retries={'max_attempts': 5, 'mode': 'standard'}, + connect_timeout=5, + read_timeout=120, + ) + + mock_boto3.client.assert_called_once_with( + 's3', + endpoint_url='localhost:9000', + aws_access_key_id='minio', + aws_secret_access_key='minio123', + region_name='us-east-1', + config=mock_config.return_value, + ) + + +@fixture +@patch('laborious.utils.repository.minio_repository.Config') +@patch('laborious.utils.repository.minio_repository.boto3') +def minio_repository(mock_boto3, mock_config): + return MinioRepository( + minio_endpoint_url='localhost:9000', + minio_access_key='minio', + minio_secret_key='minio123', + minio_region_name='us-east-1', + minio_default_bucket='test', + logger=MagicMock(), + notification_handler=MagicMock(), + ) + + +def test_ensure_bucket_exists_bucket_exists(minio_repository): + assert minio_repository.ensure_bucket_exists({}) is True + + minio_repository.s3_client.head_bucket.assert_called_once_with(Bucket='test') + + +def test_ensure_bucket_exists_bucket_not_exists_create_success(minio_repository): + minio_repository.s3_client.head_bucket.side_effect = ClientError( + error_response={'Error': {'Code': '404'}}, operation_name='head_bucket' + ) + + assert minio_repository.ensure_bucket_exists({}) is True + + minio_repository.s3_client.head_bucket.assert_called_once_with(Bucket='test') + minio_repository.s3_client.create_bucket.assert_called_once_with(Bucket='test') + + +def test_ensure_bucket_exists_bucket_not_exists_create_error(minio_repository): + minio_repository.send_notification = MagicMock() + + minio_repository.s3_client.head_bucket.side_effect = ClientError( + error_response={'Error': {'Code': '404'}}, operation_name='head_bucket' + ) + minio_repository.s3_client.create_bucket.side_effect = ClientError( + error_response={'Error': {'Code': '404'}}, operation_name='create_bucket' + ) + + with raises(ClientError): + minio_repository.ensure_bucket_exists({}) + + minio_repository.s3_client.head_bucket.assert_called_once_with(Bucket='test') + minio_repository.s3_client.create_bucket.assert_called_once_with(Bucket='test') + + +@patch('laborious.utils.repository.minio_repository.BytesIO') +def test_store_dataframe_as_parquet(mock_bytesio, minio_repository): + input_data = MagicMock() + + minio_repository.ensure_bucket_exists = MagicMock(return_value=True) + + minio_repository.store_dataframe_as_parquet( + dataframe=input_data, uri='s3://test/test.parquet', object_name='test.parquet', metadata={} + ) + + minio_repository.ensure_bucket_exists.assert_called_once_with({}) + mock_bytesio.assert_called_once() + + input_data.to_parquet.assert_called_once_with( + mock_bytesio.return_value, engine='pyarrow', index=True + ) + mock_bytesio.return_value.seek.assert_called_once_with(0) + minio_repository.s3_client.put_object.assert_called_once_with( + Bucket='test', Key='test.parquet', Body=mock_bytesio.return_value.getvalue.return_value + ) + + +@patch('laborious.utils.repository.minio_repository.BytesIO') +@patch('laborious.utils.repository.minio_repository.read_parquet') +def test_get_parquet_as_dataframe(mock_read_parquet, mock_bytesio, minio_repository): + input_data = {'Body': MagicMock(read=MagicMock(return_value=b'test'))} + + minio_repository.s3_client.get_object.return_value = input_data + + output = minio_repository.get_parquet_as_dataframe(object_key='test.parquet', metadata={}) + + minio_repository.s3_client.get_object.assert_called_once_with(Bucket='test', Key='test.parquet') + + mock_bytesio.assert_called_once_with(input_data['Body'].read.return_value) + mock_read_parquet.assert_called_once_with(mock_bytesio.return_value) + + assert output == mock_read_parquet.return_value diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index cdc59b4..0e77bdf 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -1,34 +1,60 @@ +from datetime import UTC, datetime from unittest.mock import ANY, MagicMock, call, patch + +import mlflow as mlflow_lib import numpy as np -from pandas import DataFrame import pytest -from datetime import datetime, timezone -from pandas import Timestamp -from laborious.utils.repository.model_repository import MLFlowRepository +from pandas import DataFrame, Timestamp + +from laborious.utils.repository.model_repository import MLFlowRepository, force_memory_release + + +@patch('laborious.utils.repository.model_repository.ctypes') +@patch('laborious.utils.repository.model_repository.gc') +def test_force_memory_release_success(gc, ctypes): + logger = MagicMock() + + force_memory_release(logger) + + gc.collect.assert_called_once() + ctypes.CDLL.return_value.malloc_trim.assert_called_once_with(0) + logger.info.assert_called_once_with('Memory released') + + +@patch('laborious.utils.repository.model_repository.ctypes') +@patch('laborious.utils.repository.model_repository.gc') +def test_force_memory_release_error(gc, ctypes): + logger = MagicMock() + ctypes.CDLL.return_value.malloc_trim.side_effect = Exception('error') + force_memory_release(logger) + + gc.collect.assert_called_once() + ctypes.CDLL.return_value.malloc_trim.assert_called_once_with(0) + + logger.info.assert_called_once_with('Memory release failed: error') @pytest.fixture def mlflow_repository(): - with patch('laborious.utils.repository.model_repository.ModelServing', - autospec=True) as mock_model_serving: - mock_instance = mock_model_serving.return_value - mock_instance.get_transformed_data = MagicMock() - + with patch('laborious.utils.repository.model_repository.mlflow'): repo = MLFlowRepository( - host='http://localhost:5000', - username='admin', - password='admin', - logger=MagicMock() + host='http://localhost:5000', username='admin', password='admin', logger=MagicMock() ) return repo +@pytest.fixture +def mlflow(): + with patch('laborious.utils.repository.model_repository.mlflow') as mlflow: + yield mlflow + + metadata = { - "metadata": { - "model_id": "test_model", - "model_name": "test_model", - "workflow_name": "test_workflow", - "schema_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', }, } @@ -37,203 +63,75 @@ class Any: pass -invalid_cases = [ - ( - { - 'value': { - '2024-01-01 12:00:00': 1, - 2024: 2 - } - } - ), - ( - { - 'value': { - '2024-01-01': 1, - '2024-01-02': 2 - } - } - ), - ( - { - 'value': { - Any(): 1, - Any(): 2 - } - } - ) -] - - -@pytest.mark.parametrize("data", invalid_cases) -def test_detect_and_parse_datetime_index_error_cases(mlflow_repository, data): - input_data = DataFrame( - data - ) - - with pytest.raises(ValueError) as e: - mlflow_repository.detect_and_parse_datetime_index( - input_data, metadata['metadata']) - - assert str(e) == "Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format %Y-%m-%d %H:%M:%S" - - -valid_cases = [ - ( - { - 'value': { - '2024-01-01 12:00:00+0000': 1, - '2024-01-02 12:00:00+0000': 2 - } - }, ['2024-01-01 12:00:00+0000', '2024-01-02 12:00:00+0000'] - ), - ( - { - 'value': { - datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc): 1, - datetime(2025, 1, 2, 12, 0, 0, tzinfo=timezone.utc): 2 - } - }, ['2025-01-01 12:00:00+0000', '2025-01-02 12:00:00+0000'] - ), - ( - { - 'value': { - Timestamp(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc): 1, - Timestamp(2026, 1, 2, 12, 0, 0, tzinfo=timezone.utc): 2 - } - }, ['2026-01-01 12:00:00+0000', '2026-01-02 12:00:00+0000'] - ), -] - - -@pytest.mark.parametrize("data,expected", valid_cases) -def test_detect_and_parse_datetime_index_valid_format(mlflow_repository, data, expected): - input_data = DataFrame(data) - - response = mlflow_repository.detect_and_parse_datetime_index( - input_data, metadata['metadata']) - - assert response.index.tolist() == expected - - -def test_transform_success(mlflow_repository): - data = MagicMock() - model_name = 'model' - - mlflow_repository.detect_and_parse_datetime_index = MagicMock() - - output = mlflow_repository.transform( - model_name, data, {}, metadata['metadata']) - - mlflow_repository.model_serving.get_cached_transform.assert_called_once_with( - model_name, data, 0, 'sklearn', False, 'model', 'predict') - - mlflow_repository.detect_and_parse_datetime_index.assert_called_once_with( - mlflow_repository.model_serving.get_cached_transform.return_value, metadata['metadata']) - - assert output == { - 'success': True, - 'content': mlflow_repository.detect_and_parse_datetime_index.return_value.to_dict.return_value - } - - -def test_transform_error(mlflow_repository): - data = MagicMock() - model_name = 'model' - - mlflow_repository.model_serving.get_cached_transform.side_effect = Exception( - 'error') - - output = mlflow_repository.transform( - model_name, data, {}, metadata['metadata']) - - mlflow_repository.model_serving.get_cached_transform.assert_called_once_with( - model_name, data, 0, 'sklearn', False, 'model', 'predict') - - assert output == { - 'success': False, - 'content': { - 'message': 'error', - 'traceback': ANY - } - } - - -def test_predict_success(mlflow_repository): - data = DataFrame({ - 'feat_1': { - 'index_1': 2, - 'index_2': 3 - } - }) - model_name = 'model' - mlflow_repository.model_serving.get_cached_predict.return_value = np.array( - [2, 3] - ) - - output = mlflow_repository.predict( - model_name, data, {}, metadata['metadata']) - - mlflow_repository.model_serving.get_cached_predict.assert_called_once_with( - model_name, data, 0, 'pyfunc', False, 'model') - - assert output['success'] is True - assert output['content'] == { - 'prediction': { - 'index_1': 2, - 'index_2': 3 - }, 'response_time': { - 'index_1': ANY, - 'index_2': ANY - } - } - - -def test_predict_error(mlflow_repository): - data = DataFrame({ - 'feat_1': { - 'index_1': 2, - 'index_2': 3 - } - }) - model_name = 'model' - - mlflow_repository.model_serving.get_cached_predict = MagicMock( - side_effect=Exception('error') - ) - - output = mlflow_repository.predict( - model_name, data, {}, metadata['metadata']) - - mlflow_repository.model_serving.get_cached_predict.assert_called_once_with( - model_name, data, 0, 'pyfunc', False, 'model') - - assert output == { - 'success': False, - 'content': { - 'message': 'error', - 'traceback': ANY - } - } - - -@patch('laborious.utils.repository.model_repository.mlflow') -def test_get_experiment_by_run_id(mlflow, mlflow_repository): - mlflow.get_run.return_value = MagicMock( - info=MagicMock( - experiment_id='0', - ) - ) - mlflow.get_experiment.return_value = MagicMock() - mlflow.get_experiment.return_value.name = 'test' - - output = mlflow_repository.get_experiment_by_run_id('0') - assert output == 'test' +def test_get_model_uri_prediction(mlflow, mlflow_repository): + mlflow.get_run.return_value = MagicMock(info=MagicMock(artifact_uri='test')) + output = mlflow_repository.get_model_uri('0', prediction=True) + assert output == 'test/prediction_model' mlflow.get_run.assert_called_once_with('0') - mlflow.get_experiment.assert_called_once_with('0') -@patch('laborious.utils.repository.model_repository.mlflow') +def test_get_model_uri_transform(mlflow, mlflow_repository): + mlflow.get_run.return_value = MagicMock(info=MagicMock(artifact_uri='test')) + output = mlflow_repository.get_model_uri('0', prediction=False) + assert output == 'test/data_model' + mlflow.get_run.assert_called_once_with('0') + + +def test_get_model_run_id_not_registered_models(mlflow_repository): + mlflow_repository.client.search_registered_models.return_value = [] + with pytest.raises(mlflow_lib.exceptions.MlflowException) as e: + mlflow_repository.get_model_run_id('test') + + mlflow_repository.client.search_registered_models.assert_called_once_with( + filter_string="name='test'" + ) + + assert str(e.value) == "Model 'test' not found in the Model Registry." + + +def test_get_model_run_id_not_stage_versions(mlflow_repository): + mlflow_repository.client.search_registered_models.return_value = [MagicMock(name='test')] + + mlflow_repository.client.search_model_versions.return_value = [ + MagicMock(current_stage='Staging'), + MagicMock(current_stage='Staging'), + MagicMock(current_stage='Archived'), + ] + + with pytest.raises(mlflow_lib.exceptions.MlflowException) as e: + mlflow_repository.get_model_run_id('test') + + mlflow_repository.client.search_registered_models.assert_called_once_with( + filter_string="name='test'" + ) + mlflow_repository.client.search_model_versions.assert_called_once_with( + filter_string="name='test'" + ) + + assert str(e.value) == "Model 'test' in stage 'Production' not found in the Model Registry." + + +def test_get_model_run_id_success(mlflow_repository): + mlflow_repository.client.search_registered_models.return_value = [MagicMock(name='test')] + + mlflow_repository.client.search_model_versions.return_value = [ + MagicMock(current_stage='Production', version='1'), + MagicMock(current_stage='Production', version='2', source='runs/test/1'), + MagicMock(current_stage='Archived', version='3'), + ] + + output = mlflow_repository.get_model_run_id('test') + + mlflow_repository.client.search_registered_models.assert_called_once_with( + filter_string="name='test'" + ) + mlflow_repository.client.search_model_versions.assert_called_once_with( + filter_string="name='test'" + ) + + assert output == '1' + + def test_get_next_run_name(mlflow, mlflow_repository): mlflow.search_runs.return_value = [1, 2, 3] output = mlflow_repository.get_next_run_name('run') @@ -244,17 +142,66 @@ def test_get_next_run_name(mlflow, mlflow_repository): ) -@patch('laborious.utils.repository.model_repository.mlflow') -def test_get_experiment_success(mlflow, mlflow_repository): - mlflow.get_experiment_by_name.return_value = MagicMock( - experiment_id='0') +def test_get_experiment_experiment_exists(mlflow, mlflow_repository): + experiment = MagicMock(experiment_id='0') + + mlflow.get_experiment_by_name.return_value = experiment output = mlflow_repository.get_experiment('test') - assert output == 0 + assert output == experiment + + +def test_get_experiment_none_create(mlflow, mlflow_repository): + experiment = MagicMock(experiment_id='0') + + mlflow.get_experiment_by_name.return_value = None + + mlflow.create_experiment.return_value = experiment + + output = mlflow_repository.get_experiment('test', create_if_not_exists=True) + + assert output == experiment + + +def test_get_experiment_none_not_create(mlflow, mlflow_repository): + mlflow.get_experiment_by_name.return_value = None + + with pytest.raises(ValueError) as e: + mlflow_repository.get_experiment('test', create_if_not_exists=False) + + assert str(e) == 'Experiment test not found' + + +@patch('laborious.utils.repository.model_repository.path') +@patch('laborious.utils.repository.model_repository.rmtree') +@patch('laborious.utils.repository.model_repository.makedirs') +def test_download_artifacts_success(makedirs, rmtree, path, mlflow_repository): + mlflow_repository.get_model_run_id = MagicMock(return_value='test') + + path.exists.return_value = True + + output = mlflow_repository.dowload_artifacts('test', 'path') + + mlflow_repository.get_model_run_id.assert_called_once_with( + model_name='test', stage='Production' + ) + + path.join.assert_called_once_with('./tmp/artifacts/test', 'path') + + path.exists.assert_called_once_with(path.join.return_value) + + rmtree.assert_called_once_with(path.join.return_value) + + makedirs.assert_called_once_with('./tmp/artifacts/test', exist_ok=True) + + mlflow_repository.client.download_artifacts.assert_called_once_with( + mlflow_repository.get_model_run_id.return_value, 'path', './tmp/artifacts/test' + ) + + assert output == mlflow_repository.client.download_artifacts.return_value -@patch('laborious.utils.repository.model_repository.mlflow') def test_get_experiment_error(mlflow, mlflow_repository): mlflow.get_experiment_by_name.return_value = None @@ -263,188 +210,611 @@ def test_get_experiment_error(mlflow, mlflow_repository): except ValueError as e: assert str(e) == 'Experiment test not found' else: - assert False + raise AssertionError('Expected ValueError') -@patch('laborious.utils.repository.model_repository.mlflow') -def test_get_experiment_last_run(mlflow, mlflow_repository): - mlflow.search_runs.return_value = DataFrame({ - 'params.retrain': ['True', 'False', 'True', 'False'], - 'end_time': ['2021-01-01', '2021-01-02', '2021-01-03', '2021-01-04'], - 'run_id': ['0', '1', '2', '3'], - }) +def test_load_predict_model_sklearn(mlflow, mlflow_repository): + result = mlflow_repository.load_predict_model('test_model', 'sklearn') - output = mlflow_repository.get_experiment_last_run(0) + assert result == mlflow.sklearn.load_model.return_value + mlflow.sklearn.load_model.assert_called_once_with('models:/test_model/production') - mlflow.search_runs.assert_called_once_with( - experiment_ids=[0], - filter_string="", - output_format="pandas", + +def test_load_predict_model_pyfunc(mlflow, mlflow_repository): + result = mlflow_repository.load_predict_model('test_model', 'pyfunc') + assert result == mlflow.pyfunc.load_model.return_value + mlflow.pyfunc.load_model.assert_called_once_with('models:/test_model/production') + + +def test_load_predict_model_pytorch(mlflow, mlflow_repository): + result = mlflow_repository.load_predict_model('test_model', 'pytorch') + assert result == mlflow.pytorch.load_model.return_value + mlflow.pytorch.load_model.assert_called_once_with('models:/test_model/production') + + +def test_load_predict_model_error(mlflow_repository): + with pytest.raises(ValueError) as e: + mlflow_repository.load_predict_model('test_model', 'invalid') + assert str(e) == "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'." + + +def validate_common_load_transform_model_mocks(mlflow_repository, model_name): + mlflow_repository.get_model_run_id.assert_called_once_with( + model_name=model_name, stage='Production' + ) + mlflow_repository.get_model_uri.assert_called_once_with( + mlflow_repository.get_model_run_id.return_value, prediction=False ) - assert output == '2' + +def test_load_transform_model_sklearn(mlflow, mlflow_repository): + mlflow_repository.get_model_run_id = MagicMock() + mlflow_repository.get_model_uri = MagicMock() + + result = mlflow_repository.load_transform_model('test_model', 'sklearn') + + validate_common_load_transform_model_mocks(mlflow_repository, 'test_model') + + assert result == mlflow.sklearn.load_model.return_value + mlflow.sklearn.load_model.assert_called_once_with(mlflow_repository.get_model_uri.return_value) -@patch('laborious.utils.repository.model_repository.mlflow') -def test_get_experiment_last_run_error(mlflow, mlflow_repository): - mlflow.search_runs.return_value = [] +def test_load_transform_model_pyfunc(mlflow, mlflow_repository): + mlflow_repository.get_model_run_id = MagicMock() + mlflow_repository.get_model_uri = MagicMock() - try: - mlflow_repository.get_experiment_last_run(0) - except ValueError as e: - assert str(e) == 'Runs is not a pandas DataFrame' - else: - assert False + result = mlflow_repository.load_transform_model('test_model', 'pyfunc') + + validate_common_load_transform_model_mocks(mlflow_repository, 'test_model') + + assert result == mlflow.pyfunc.load_model.return_value + mlflow.pyfunc.load_model.assert_called_once_with(mlflow_repository.get_model_uri.return_value) -@patch('laborious.utils.repository.model_repository.mlflow.sklearn') -@patch('laborious.utils.repository.model_repository.mlflow.set_experiment') -def test_create_model_experiment(set_experiment, sklearn, mlflow_repository): +def test_load_transform_model_pytorch(mlflow, mlflow_repository): + mlflow_repository.get_model_run_id = MagicMock() + mlflow_repository.get_model_uri = MagicMock() - mlflow_repository.model_serving.get_model_run_id = MagicMock( - return_value='0') - mlflow_repository.model_serving.get_model_uri = MagicMock( - return_value='test') - mlflow_repository.get_experiment_by_run_id = MagicMock() + result = mlflow_repository.load_transform_model('test_model', 'pytorch') + validate_common_load_transform_model_mocks(mlflow_repository, 'test_model') - data_model_mock = MagicMock() - prediction_model_mock = MagicMock() + assert result == mlflow.pytorch.load_model.return_value + mlflow.pytorch.load_model.assert_called_once_with(mlflow_repository.get_model_uri.return_value) - sklearn.load_model.side_effect = [data_model_mock, prediction_model_mock] - data_model_mock.fit.return_value = data_model_mock - data_model_mock.predict.return_value = DataFrame({ - 'x': [10, 20, 30], - }) - data_model_mock.target_variable = 'y' +def test_load_transform_model_error(mlflow_repository): + mlflow_repository.get_model_run_id = MagicMock() + mlflow_repository.get_model_uri = MagicMock() - prediction_model_mock.fit.return_value = prediction_model_mock + with pytest.raises(ValueError) as e: + mlflow_repository.load_transform_model('test_model', 'invalid') + assert str(e) == "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'." - data = DataFrame({ - 'x': [1, 2, 3], - 'y': [4, 5, 6] - }) - output = mlflow_repository.create_model_experiment('test', data) +def test_download_model_invalid_model_type(mlflow_repository): + with pytest.raises(ValueError) as e: + mlflow_repository.download_model('test_model', 'invalid', 'sklearn') + assert str(e) == "Invalid model_type. Use 'predict' or 'transform'." - mlflow_repository.model_serving.get_model_run_id.assert_called_once_with( - 'test', stage='Production') - mlflow_repository.model_serving.get_model_uri.assert_called_once_with( - '0', prediction=False) - sklearn.load_model.assert_has_calls([ - call(mlflow_repository.model_serving.get_model_uri.return_value), - call("models:/test/production"), - ]) - assert sklearn.load_model.call_count == 2 +@pytest.mark.parametrize( + 'model_type', [('predict', 'prediction_model'), ('transform', 'data_model')] +) +def test_download_model_load_wrapper(mlflow, mlflow_repository, model_type): + mlflow_repository.dowload_artifacts = MagicMock() - data_model_mock.fit.assert_called_once_with(data) - data_model_mock.predict.assert_called_once_with(data) + result = mlflow_repository.download_model('test_model', model_type[0], 'pyfunc', True) - fit_args = prediction_model_mock.fit.call_args[0][0] - assert fit_args.equals( - DataFrame({ - 'x': [10, 20, 30], - 'y': [4, 5, 6], - }) + mlflow_repository.dowload_artifacts.assert_called_once_with('test_model', model_type[1]) + + mlflow.pyfunc.load_model.assert_called_once_with( + mlflow_repository.dowload_artifacts.return_value ) - mlflow_repository.get_experiment_by_run_id.assert_called_once_with('0') - - set_experiment.assert_called_once_with( - mlflow_repository.get_experiment_by_run_id.return_value + assert result == ( + mlflow.pyfunc.load_model.return_value._model_impl.python_model, + mlflow_repository.dowload_artifacts.return_value, ) - assert output == (prediction_model_mock, - data_model_mock, - mlflow_repository.get_experiment_by_run_id.return_value) + +def test_download_model_predict(mlflow_repository): + mlflow_repository.load_predict_model = MagicMock() + mlflow_repository.load_transform_model = MagicMock() + + result = mlflow_repository.download_model('test_model', 'predict', 'pyfunc', False) + + mlflow_repository.load_predict_model.assert_called_once_with('test_model', 'pyfunc') + mlflow_repository.load_transform_model.assert_not_called() + + assert result == (mlflow_repository.load_predict_model.return_value, None) -@patch('laborious.utils.repository.model_repository.mlflow.start_run') -@patch('laborious.utils.repository.model_repository.mlflow.log_param') -@patch('laborious.utils.repository.model_repository.mlflow.sklearn.log_model') -@patch('laborious.utils.repository.model_repository.mlflow.log_artifact') -def test_perform_model_retrain(log_artifact, log_model, log_param, start_run, mlflow_repository): +def test_download_model_transform(mlflow_repository): + mlflow_repository.load_predict_model = MagicMock() + mlflow_repository.load_transform_model = MagicMock() + + result = mlflow_repository.download_model('test_model', 'transform', 'pyfunc', False) + + mlflow_repository.load_predict_model.assert_not_called() + mlflow_repository.load_transform_model.assert_called_once_with('test_model', 'pyfunc') + + assert result == (mlflow_repository.load_transform_model.return_value, None) + + +invalid_cases = [ + ( + {'value': {'2024-01-01 12:00:00': 1, 2024: 2}}, + "Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format %Y-%m-%d %H:%M:%S. Elements are , .", + ), + ( + {'value': {'2024-01-01': 1, '2024-01-02': 2}}, + 'Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format %Y-%m-%d %H:%M:%S. Unable to parse given date format', + ), + ( + {'value': {Any(): 1, Any(): 2}}, + 'Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format %Y-%m-%d %H:%M:%S. Got .', + ), +] + + +@pytest.mark.parametrize('data', invalid_cases) +def test_detect_and_parse_datetime_index_error_cases(mlflow_repository, data): + input_data = DataFrame(data[0]) + + message = data[1] + + with pytest.raises(ValueError) as e: + mlflow_repository.detect_and_parse_datetime_index(input_data, metadata['metadata']) + + assert str(e) == message + + +valid_cases = [ + ( + {'value': {'2024-01-01 12:00:00+0000': 1, '2024-01-02 12:00:00+0000': 2}}, + ['2024-01-01 12:00:00+0000', '2024-01-02 12:00:00+0000'], + ), + ( + { + 'value': { + datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC): 1, + datetime(2025, 1, 2, 12, 0, 0, tzinfo=UTC): 2, + } + }, + ['2025-01-01 12:00:00+0000', '2025-01-02 12:00:00+0000'], + ), + ( + { + 'value': { + Timestamp(2026, 1, 1, 12, 0, 0, tzinfo=UTC): 1, + Timestamp(2026, 1, 2, 12, 0, 0, tzinfo=UTC): 2, + } + }, + ['2026-01-01 12:00:00+0000', '2026-01-02 12:00:00+0000'], + ), +] + + +@pytest.mark.parametrize('data,expected', valid_cases) +def test_detect_and_parse_datetime_index_valid_format(mlflow_repository, data, expected): + input_data = DataFrame(data) + + response = mlflow_repository.detect_and_parse_datetime_index(input_data, metadata['metadata']) + + assert response.index.tolist() == expected + + +@patch('laborious.utils.repository.model_repository.datetime') +def test_check_cache_retention_false(datetime_mock, mlflow_repository): + datetime_mock.now = MagicMock(return_value=datetime.strptime('2025-01-02', '%Y-%m-%d')) + cache = {'timestamp': datetime.strptime('2025-01-01', '%Y-%m-%d')} + + assert mlflow_repository.check_cache_retention(cache, 1) is False + + +@patch('laborious.utils.repository.model_repository.datetime') +def test_check_cache_retention_true(datetime_mock, mlflow_repository): + datetime_mock.now = MagicMock(return_value=datetime.strptime('2025-01-01', '%Y-%m-%d')) + cache = {'timestamp': datetime.strptime('2025-01-01', '%Y-%m-%d')} + assert mlflow_repository.check_cache_retention(cache, 1) is True + + +def test_handle_valid_model(mlflow_repository): + cache = {'target': {'model': 'model', 'artifact_path': 'test_artifact_path'}} + output = mlflow_repository.handle_valid_model('model_name', cache) + assert output == {'model': 'model', 'artifact_path': 'test_artifact_path'} + + +def test_handle_outdated_model(mlflow_repository): + mlflow_repository.model_cache = { + 'model_name_transform': { + 'target': {'model': 'model', 'artifact_path': 'test_artifact_path'} + } + } + mlflow_repository.handle_outdated_model('model_name', 'model_name_transform') + assert mlflow_repository.model_cache == {} + + +def test_get_model_retention_0(mlflow_repository): + model = MagicMock() + + mlflow_repository.download_model = MagicMock(return_value=(model, 'artifact_path')) + output = mlflow_repository.get_model('model_name', 0, 'predict', 'pyfunc') + + assert output == model + mlflow_repository.download_model.assert_called_once_with( + model_name='model_name', model_type='predict', flavor='pyfunc', load_wrapper=False + ) + + +def test_get_model_cached_valid(mlflow_repository): + mlflow_repository.check_cache_retention = MagicMock(return_value=True) + mlflow_repository.handle_valid_model = MagicMock() + mlflow_repository.handle_outdated_model = MagicMock() + mlflow_repository.model_cache = { + 'model_name_predict': { + 'target': 'cached_model', + } + } + + output = mlflow_repository.get_model('model_name', 1, 'predict', 'pyfunc') + + assert output == mlflow_repository.handle_valid_model.return_value + mlflow_repository.check_cache_retention.assert_called_once_with( + mlflow_repository.model_cache['model_name_predict'], 1 + ) + + mlflow_repository.handle_valid_model.assert_called_once_with( + model_name='model_name', cache=mlflow_repository.model_cache['model_name_predict'] + ) + + mlflow_repository.handle_outdated_model.assert_not_called() + + +def test_get_model_cached_outdated(mlflow_repository): + mlflow_repository.check_cache_retention = MagicMock(return_value=False) + mlflow_repository.handle_valid_model = MagicMock() + mlflow_repository.handle_outdated_model = MagicMock() + model = MagicMock() + mlflow_repository.download_model = MagicMock(return_value=(model, 'artifact_path')) + cache = { + 'model_name_predict': { + 'target': 'cached_model', + } + } + mlflow_repository.model_cache = cache + + output = mlflow_repository.get_model('model_name', 1, 'predict', 'pyfunc') + assert output == model + mlflow_repository.check_cache_retention.assert_called_once_with( + { + 'target': 'cached_model', + }, + 1, + ) + mlflow_repository.handle_valid_model.assert_not_called() + mlflow_repository.handle_outdated_model.assert_called_once_with( + model_name='model_name', model_key='model_name_predict' + ) + + +def test_get_model_cached_not_found(mlflow_repository): + mlflow_repository.check_cache_retention = MagicMock(return_value=False) + mlflow_repository.handle_valid_model = MagicMock() + mlflow_repository.handle_outdated_model = MagicMock() + mlflow_repository.model_cache = {} + model = MagicMock() + mlflow_repository.download_model = MagicMock(return_value=(model, 'artifact_path')) + output = mlflow_repository.get_model('model_name', 1, 'predict', 'pyfunc') + assert output == model + mlflow_repository.check_cache_retention.assert_not_called() + mlflow_repository.handle_valid_model.assert_not_called() + mlflow_repository.handle_outdated_model.assert_not_called() + + +@patch('laborious.utils.repository.model_repository.force_memory_release') +def test_get_cached_operation_retention_0(force_memory_release, mlflow_repository): + model = MagicMock() + data = MagicMock() + mlflow_repository.get_model = MagicMock(return_value=model) + output = mlflow_repository.get_cached_operation('model_name', data, 'transform', 0, 'sklearn') + assert output == model.predict.return_value + force_memory_release.assert_called_once_with(mlflow_repository.logger) + + +@patch('laborious.utils.repository.model_repository.force_memory_release') +def test_get_cached_predict_retention_not_0(force_memory_release, mlflow_repository): + model = MagicMock() + data = MagicMock() + mlflow_repository.get_model = MagicMock(return_value=model) + output = mlflow_repository.get_cached_operation('model_name', data, 'predict', 1, 'sklearn') + assert output == model.predict.return_value + force_memory_release.assert_not_called() + + +@patch('laborious.utils.repository.model_repository.force_memory_release') +def test_get_cached_operation_invalid_operation(force_memory_release, mlflow_repository): + data = MagicMock() + with pytest.raises(ValueError) as e: + mlflow_repository.get_cached_operation('model_name', data, 'invalid', 0, 'sklearn') + assert str(e) == "Invalid operation. Use 'transform' or 'predict'." + + +@patch('laborious.utils.repository.model_repository.pd.merge') +@patch('laborious.utils.repository.model_repository.isinstance') +def test_fit_models_not_df_target_name_none_and_not_in_model( + isinstance_mock, pd_merge, mlflow_repository +): + isinstance_mock.return_value = False + + data_model = MagicMock() + prediction_model = MagicMock() + mlflow_repository.download_model = MagicMock( + side_effect=[(data_model, 'artifact_path'), (prediction_model, 'artifact_path')], + ) + mlflow_repository.detect_and_parse_datetime_index = MagicMock( + return_value=MagicMock(drop_duplicates=MagicMock(return_value=MagicMock(columns=[]))) + ) - prediction_model_mock = MagicMock() - data_model_mock = MagicMock() - experiment = 'test' - model_name = 'test' data = MagicMock() - mlflow_repository.get_next_run_name = MagicMock( - return_value='test-1') - run = MagicMock() - start_run.__enter__.return_value = run + output = mlflow_repository.fit_models( + 'model_name', data, 'latest_production_id', metadata['metadata'], 'sklearn', 'pyfunc', None + ) - output = mlflow_repository.perform_model_retrain( - prediction_model_mock, data_model_mock, experiment, model_name, data) + mlflow_repository.download_model.assert_has_calls( + [ + call( + model_name='model_name', + model_type='transform', + flavor='sklearn', + load_wrapper=False, + ), + call(model_name='model_name', model_type='predict', flavor='pyfunc', load_wrapper=True), + ] + ) - mlflow_repository.get_next_run_name.assert_called_once_with(experiment) - start_run.assert_called_once_with( - run_name='test-1', description='Retrain model test with new data') + data_model.fit.assert_called_once_with(data) - log_model.assert_has_calls([ - call(data_model_mock, "data_model"), - call(prediction_model_mock, "prediction_model"), - ]) + data_model.fit.return_value.predict.assert_called_once_with(data) - data.to_csv.assert_called_once_with( - "temp/raw_data_test.csv", index=True) + transformed_data = data_model.fit.return_value.predict.return_value - log_artifact.assert_called_once_with( - "temp/raw_data_test.csv") + transformed_data.__setitem__.assert_called_once_with('timestamp', transformed_data.index) - log_param.assert_has_calls([ - call("retrain", True), - ]) + mlflow_repository.detect_and_parse_datetime_index.assert_called_once_with( + transformed_data, metadata['metadata'] + ) - assert output == ("Model retrained successfully", experiment) + transformed_data = mlflow_repository.detect_and_parse_datetime_index.return_value + + transformed_data.drop_duplicates.assert_called_once_with(subset=['timestamp'], keep='first') + + transformed_data = transformed_data.drop_duplicates.return_value + + data.loc.__getitem__.assert_called_once_with(transformed_data.index) + + aligned_data = data.loc.__getitem__.return_value + + aligned_data.__getitem__.assert_called_once_with(data_model.fit.return_value.target_variable) + + pd_merge.assert_called_once_with( + transformed_data, aligned_data.__getitem__.return_value, left_index=True, right_index=True + ) + + prediction_model.fit.assert_called_once_with(pd_merge.return_value) + + assert output == { + 'prediction_model': {'model': prediction_model, 'artifact_path': 'artifact_path'}, + 'data_model': {'model': data_model.fit.return_value, 'artifact_path': 'artifact_path'}, + } -def test_retrain_model(mlflow_repository): +@patch('laborious.utils.repository.model_repository.pd.merge') +@patch('laborious.utils.repository.model_repository.isinstance') +def test_fit_models_df_target_name_not_none_and_in_model( + isinstance_mock, pd_merge, mlflow_repository +): + isinstance_mock.return_value = True + + data_model = MagicMock( + target_variable='feat_2', + ) + prediction_model = MagicMock() + mlflow_repository.download_model = MagicMock( + side_effect=[(data_model, 'artifact_path'), (prediction_model, 'artifact_path')], + ) + mlflow_repository.detect_and_parse_datetime_index = MagicMock( + return_value=MagicMock( + drop_duplicates=MagicMock(return_value=MagicMock(columns=['feat_1'])) + ) + ) + data = MagicMock() - model_name = 'test' - mlflow_repository.create_model_experiment = MagicMock( - return_value=('data_model', 'prediction_model', '0')) + output = mlflow_repository.fit_models( + 'model_name', + data, + 'latest_production_id', + metadata['metadata'], + 'sklearn', + 'pyfunc', + 'feat_1', + ) - mlflow_repository.perform_model_retrain = MagicMock( - return_value='Model retrained successfully') + mlflow_repository.download_model.assert_has_calls( + [ + call( + model_name='model_name', + model_type='transform', + flavor='sklearn', + load_wrapper=False, + ), + call(model_name='model_name', model_type='predict', flavor='pyfunc', load_wrapper=True), + ] + ) - output = mlflow_repository.retrain_model(data, model_name) + data_model.fit.assert_called_once_with(data) - mlflow_repository.create_model_experiment.assert_called_once_with( - model_name, data) + transformed_data = data_model.fit.return_value - mlflow_repository.perform_model_retrain.assert_called_once_with( - 'data_model', 'prediction_model', '0', model_name, data) + transformed_data.__setitem__.assert_called_once_with('timestamp', transformed_data.index) - assert output == 'Model retrained successfully' + mlflow_repository.detect_and_parse_datetime_index.assert_called_once_with( + transformed_data, metadata['metadata'] + ) + + transformed_data = mlflow_repository.detect_and_parse_datetime_index.return_value + + transformed_data.drop_duplicates.assert_called_once_with(subset=['timestamp'], keep='first') + + transformed_data = transformed_data.drop_duplicates.return_value + + data.loc.__getitem__.assert_not_called() + + pd_merge.assert_not_called() + + prediction_model.fit.assert_called_once_with(transformed_data) + + assert output == { + 'prediction_model': {'model': prediction_model, 'artifact_path': 'artifact_path'}, + 'data_model': {'model': data_model, 'artifact_path': 'artifact_path'}, + } + + +def test_log_model_sklearn(mlflow, mlflow_repository): + model_data = {'model': MagicMock(), 'artifact_path': 'artifact_path'} + mlflow_repository.log_model(model_data, 'sklearn', 'prediction_model', metadata['metadata']) + mlflow.sklearn.log_model.assert_called_once_with(model_data['model'], 'prediction_model') + + +@patch('laborious.utils.repository.model_repository.path') +def test_log_model_pyfunc(path, mlflow, mlflow_repository): + model_data = {'model': MagicMock(), 'artifact_path': 'artifact_path'} + mlflow_repository.log_model(model_data, 'pyfunc', 'prediction_model', metadata['metadata']) + + mlflow.pyfunc.log_model.assert_not_called() + + path.join.assert_called_once_with('artifact_path', 'code', 'utils') + + model_data['model'].store_model.assert_called_once_with( + artifact_path='prediction_model', code_path=[path.join.return_value], to_disk=False + ) + + +def test_log_model_pytorch(mlflow, mlflow_repository): + model_data = {'model': MagicMock(), 'artifact_path': 'artifact_path'} + mlflow_repository.log_model(model_data, 'pytorch', 'prediction_model', metadata['metadata']) + mlflow.pytorch.log_model.assert_called_once_with(model_data['model'], 'prediction_model') + + +def test_log_model_error(mlflow_repository): + model_data = {'model': MagicMock(), 'artifact_path': 'artifact_path'} + with pytest.raises(ValueError) as e: + mlflow_repository.log_model(model_data, 'invalid', 'prediction_model', metadata['metadata']) + assert str(e) == "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'." + + +@patch('laborious.utils.repository.model_repository.force_memory_release') +@patch('laborious.utils.repository.model_repository.path') +@patch('laborious.utils.repository.model_repository.rmtree') +def test_create_new_experiment(_rmtree, path, force_memory_release, mlflow, mlflow_repository): + model_name = 'model_name' + data = MagicMock() + retrain_data = { + 'prediction_model': {'model': MagicMock(), 'artifact_path': 'artifact_path'}, + 'data_model': {'model': MagicMock(), 'artifact_path': 'artifact_path'}, + } + + mlflow_repository.get_model_params = MagicMock( + return_value={ + 'transform_flavor': 'sklearn', + 'predict_flavor': 'pyfunc', + 'target_name': 'target_name', + } + ) + + mlflow_repository.get_experiment = MagicMock() + mlflow_repository.get_next_run_name = MagicMock() + mlflow_repository.log_model = MagicMock() + path.exists.return_value = True + path.join.return_value = './tmp/artifacts/model_name' + + report = mlflow_repository.create_new_experiment( + model_name, + data, + retrain_data, + 'latest_production_id', + metadata['metadata'], + 'sklearn', + 'pyfunc', + ) + + path.join.assert_called_once_with('./tmp/artifacts', 'model_name') + + mlflow_repository.get_model_params.assert_called_once_with('latest_production_id') + mlflow_repository.get_experiment.assert_called_once_with(model_name, create_if_not_exists=True) + mlflow_repository.get_next_run_name.assert_called_once_with( + mlflow_repository.get_experiment.return_value.name + ) + + data.to_csv.assert_called_once_with('./tmp/artifacts/model_name/retrain_data.csv', index=True) + + mlflow.start_run.assert_called_once_with( + experiment_id=mlflow_repository.get_experiment.return_value.experiment_id, + run_name=mlflow_repository.get_next_run_name.return_value, + description='Retrain model model_name with new data', + ) + + mlflow_repository.log_model.assert_has_calls( + [ + call(retrain_data['data_model'], 'sklearn', 'data_model', metadata['metadata']), + call( + retrain_data['prediction_model'], 'pyfunc', 'prediction_model', metadata['metadata'] + ), + ] + ) + + mlflow.log_params.assert_called_once_with( + { + 'transform_flavor': 'sklearn', + 'predict_flavor': 'pyfunc', + 'target_name': 'target_name', + 'retrain': True, + 'retrain_date': ANY, + 'source_run_id': 'latest_production_id', + 'retrain_samples': data.shape.__str__.return_value, + } + ) + + mlflow.log_artifact.assert_called_once_with('./tmp/artifacts/model_name/retrain_data.csv') + + force_memory_release.assert_called_once_with(mlflow_repository.logger) + + assert report == { + 'run_id': mlflow.start_run.return_value.__enter__.return_value.info.run_id, + 'experiment_id': mlflow_repository.get_experiment.return_value.experiment_id, + 'experiment_name': mlflow_repository.get_experiment.return_value.name, + } -@patch('laborious.utils.repository.model_repository.mlflow') def test_update_production_model_by_run_id(mlflow, mlflow_repository): - client_mock = MagicMock() - mlflow.tracking.MlflowClient.return_value = client_mock - - client_mock.get_registered_model.return_value = MagicMock( + mlflow_repository.client.get_registered_model.return_value = MagicMock( latest_versions=[ MagicMock(version='1'), MagicMock(version='2'), MagicMock(version='3'), ] ) - output = mlflow_repository.update_production_model_by_run_id('0', 'test') + output = mlflow_repository.update_production_model_by_run_id('0', 'test', metadata['metadata']) mlflow.register_model.assert_called_once_with( - "runs:/0/prediction_model", + 'runs:/0/prediction_model', 'test', ) - mlflow.tracking.MlflowClient.assert_called_once() - client_mock.get_registered_model.assert_called_once_with('test') - client_mock.transition_model_version_stage.assert_called_once_with( + mlflow_repository.client.get_registered_model.assert_called_once_with('test') + mlflow_repository.client.transition_model_version_stage.assert_called_once_with( name='test', version='3', stage='Production', @@ -458,45 +828,184 @@ def test_update_production_model_by_run_id(mlflow, mlflow_repository): } -@patch('laborious.utils.repository.model_repository.mlflow') def test_update_production_model_by_run_id_error(mlflow, mlflow_repository): - mlflow.tracking.MlflowClient.return_value = MagicMock( - get_registered_model=MagicMock( - return_value=MagicMock( - latest_versions={} - ) - ) + mlflow_repository.client.get_registered_model.return_value = MagicMock( + get_registered_model=MagicMock(return_value=MagicMock(latest_versions={})) ) try: - mlflow_repository.update_production_model_by_run_id('0', 'test') + mlflow_repository.update_production_model_by_run_id('0', 'test', metadata['metadata']) except Exception as e: assert str(e) == 'Model versions is not a list' else: - assert False + raise AssertionError('Expected Exception') + + +def test_transform_success(mlflow_repository): + data = MagicMock() + model_name = 'model' + model_config = { + 'retention_minutes': 60, + 'transform_flavor': 'sklearn', + 'predict_flavor': 'pyfunc', + } + + mlflow_repository.get_cached_operation = MagicMock() + + mlflow_repository.detect_and_parse_datetime_index = MagicMock() + + output = mlflow_repository.transform(model_name, data, model_config, metadata['metadata']) + + mlflow_repository.get_cached_operation.assert_called_once_with( + model_name, data, 'transform', 60, 'sklearn' + ) + + mlflow_repository.detect_and_parse_datetime_index.assert_called_once_with( + mlflow_repository.get_cached_operation.return_value, metadata['metadata'] + ) + + assert output == { + 'success': True, + 'content': mlflow_repository.detect_and_parse_datetime_index.return_value.to_dict.return_value, + } + + +def test_transform_error(mlflow_repository): + data = MagicMock() + model_name = 'model' + model_config = { + 'retention_minutes': 60, + 'transform_flavor': 'sklearn', + 'predict_flavor': 'pyfunc', + } + + mlflow_repository.get_cached_operation = MagicMock(side_effect=Exception('error')) + + output = mlflow_repository.transform(model_name, data, model_config, metadata['metadata']) + + mlflow_repository.get_cached_operation.assert_called_once_with( + model_name, data, 'transform', 60, 'sklearn' + ) + + assert output == {'success': False, 'content': {'message': 'error', 'traceback': ANY}} + + +def test_predict_success_array(mlflow_repository): + data = DataFrame({'feat_1': {'index_1': 2, 'index_2': 3}}) + model_config = {'retention_minutes': 60, 'predict_flavor': 'pyfunc'} + model_name = 'model' + mlflow_repository.get_cached_operation = MagicMock(return_value=np.array([2, 3])) + + output = mlflow_repository.predict(model_name, data, model_config, metadata['metadata']) + + mlflow_repository.get_cached_operation.assert_called_once_with( + model_name, data, 'predict', 60, 'pyfunc' + ) + + assert output['success'] is True + assert output['content'] == { + 'prediction': {'index_1': 2, 'index_2': 3}, + 'response_time': {'index_1': ANY, 'index_2': ANY}, + } + + +def test_predict_success_df(mlflow_repository): + data = DataFrame({'feat_1': {'index_1': 2, 'index_2': 3}}) + model_config = {'retention_minutes': 60, 'predict_flavor': 'pyfunc'} + model_name = 'model' + + mlflow_repository.get_cached_operation = MagicMock( + return_value=DataFrame({'feat_1': {'index_3': 2, 'index_4': 3}}) + ) + + output = mlflow_repository.predict(model_name, data, model_config, metadata['metadata']) + + mlflow_repository.get_cached_operation.assert_called_once_with( + model_name, data, 'predict', 60, 'pyfunc' + ) + + assert output['success'] is True + assert output['content'] == { + 'prediction': {'index_1': 2, 'index_2': 3}, + 'response_time': {'index_1': ANY, 'index_2': ANY}, + } + + +def test_predict_error(mlflow_repository): + data = DataFrame({'feat_1': {'index_1': 2, 'index_2': 3}}) + model_name = 'model' + model_config = {'retention_minutes': 60, 'predict_flavor': 'pyfunc'} + + mlflow_repository.get_cached_operation = MagicMock(side_effect=Exception('error')) + + output = mlflow_repository.predict(model_name, data, model_config, metadata['metadata']) + + mlflow_repository.get_cached_operation.assert_called_once_with( + model_name, data, 'predict', 60, 'pyfunc' + ) + + assert output == {'success': False, 'content': {'message': 'error', 'traceback': ANY}} + + +def test_retrain_model(mlflow_repository): + data = MagicMock() + model_name = 'test' + model_config = {'target': 'target', 'transform_flavor': 'sklearn', 'predict_flavor': 'pyfunc'} + + mlflow_repository.get_model_run_id = MagicMock() + mlflow_repository.fit_models = MagicMock() + mlflow_repository.create_new_experiment = MagicMock() + + output = mlflow_repository.retrain_model(data, model_name, model_config, metadata['metadata']) + + mlflow_repository.get_model_run_id.assert_called_once_with(model_name, stage='Production') + + mlflow_repository.fit_models.assert_called_once_with( + model_name=model_name, + data=data, + transform_flavor='sklearn', + predict_flavor='pyfunc', + target_name='target', + metadata=metadata['metadata'], + latest_production_id=mlflow_repository.get_model_run_id.return_value, + ) + + mlflow_repository.create_new_experiment.assert_called_once_with( + model_name=model_name, + data=data, + retrain_data=mlflow_repository.fit_models.return_value, + transform_flavor='sklearn', + predict_flavor='pyfunc', + metadata=metadata['metadata'], + latest_production_id=mlflow_repository.get_model_run_id.return_value, + ) + + assert output == { + 'success': True, + 'experiment': mlflow_repository.create_new_experiment.return_value, + 'message': 'Model retrained successfully.', + } def test_update_production_model(mlflow_repository): - connector = mlflow_repository + experiment = {'run_id': '0', 'experiment_id': '0'} + model_name = 'test' + mlflow_repository.update_production_model_by_run_id = MagicMock() + mlflow_repository.update_production_model_by_run_id.return_value = { + 'model_name': 'test', + 'version': '3', + 'mlflow_run_id': '0', + } - with patch.object(connector, 'get_experiment', - return_value='0') as get_experiment: - with patch.object(connector, 'get_experiment_last_run', - return_value='2') as get_experiment_last_run: - with patch.object(connector, 'update_production_model_by_run_id', - return_value={'model_name': 'test', 'version': '3', - 'mlflow_run_id': '0'}) as update_production_model_by_run_id: + output = mlflow_repository.update_production_model(experiment, model_name, metadata['metadata']) - output = connector.update_production_model('0', 'test') + mlflow_repository.update_production_model_by_run_id.assert_called_once_with( + '0', 'test', metadata['metadata'] + ) - get_experiment.assert_called_once_with('0') - get_experiment_last_run.assert_called_once_with('0') - update_production_model_by_run_id.assert_called_once_with( - '2', 'test') - - assert output == { - 'model_name': 'test', - 'version': '3', - 'mlflow_run_id': '0', - 'mlflow_experiment_id': '0', - } + assert output == { + 'model_name': 'test', + 'version': '3', + 'mlflow_run_id': '0', + 'mlflow_experiment_id': '0', + } diff --git a/tests/laborious/utils/repository/test_opc_repository.py b/tests/laborious/utils/repository/test_opc_repository.py index bc84db8..a0048e8 100644 --- a/tests/laborious/utils/repository/test_opc_repository.py +++ b/tests/laborious/utils/repository/test_opc_repository.py @@ -1,9 +1,11 @@ -import pytest -from unittest.mock import AsyncMock, Mock, patch, MagicMock, ANY, call -from asyncua.crypto.security_policies import SecurityPolicyBasic256 -from laborious.utils.repository.opc_repository import OpcRepository -from sientia_do.notifications.models import NotificationLevel from datetime import datetime +from unittest.mock import ANY, AsyncMock, MagicMock, Mock, call, patch + +import pytest +from asyncua.crypto.security_policies import SecurityPolicyBasic256 +from sientia_do.notifications.models import NotificationLevel + +from laborious.utils.repository.opc_repository import OpcRepository @pytest.fixture @@ -14,15 +16,15 @@ def mock_logger(): @pytest.fixture def opc_repository(mock_logger): return OpcRepository( - id="test_repo", - url="opc.tcp://localhost:4840", + opc_id='test_repo', + url='opc.tcp://localhost:4840', logger=mock_logger, notification_handler=Mock(), reconnection_interval=60, - server_uri="urn:test:server", - cert_path="/path/to/cert.pem", - private_key_path="/path/to/key.pem", - server_cert_path="/path/to/server_cert.pem" + server_uri='urn:test:server', + cert_path='/path/to/cert.pem', + private_key_path='/path/to/key.pem', + server_cert_path='/path/to/server_cert.pem', ) @@ -35,22 +37,22 @@ def mock_client(): metadata = { - "metadata": { - "model_id": "test_model", - "model_name": "test_model", - "workflow_name": "test_workflow", - "schema_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', }, } def test_init(opc_repository): - assert opc_repository.id == "test_repo" - assert opc_repository.url == "opc.tcp://localhost:4840" - assert opc_repository.server_uri == "urn:test:server" - assert opc_repository.cert_path == "/path/to/cert.pem" - assert opc_repository.private_key_path == "/path/to/key.pem" - assert opc_repository.server_cert_path == "/path/to/server_cert.pem" + assert opc_repository.id == 'test_repo' + assert opc_repository.url == 'opc.tcp://localhost:4840' + assert opc_repository.server_uri == 'urn:test:server' + assert opc_repository.cert_path == '/path/to/cert.pem' + assert opc_repository.private_key_path == '/path/to/key.pem' + assert opc_repository.server_cert_path == '/path/to/server_cert.pem' assert opc_repository.reconnection_interval == 60 assert opc_repository.client is None assert opc_repository.last_reconnection_time is None @@ -62,12 +64,12 @@ async def test_set_security(opc_repository, mock_client): opc_repository.client = mock_client await opc_repository.set_security() - mock_client.application_uri = "urn:test:server" + mock_client.application_uri = 'urn:test:server' mock_client.set_security.assert_called_once_with( SecurityPolicyBasic256, - certificate="/path/to/cert.pem", - private_key="/path/to/key.pem", - server_certificate="/path/to/server_cert.pem" + certificate='/path/to/cert.pem', + private_key='/path/to/key.pem', + server_certificate='/path/to/server_cert.pem', ) assert mock_client.secure_channel_timeout == 10000000 assert mock_client.session_timeout == 10000000 @@ -81,8 +83,7 @@ async def test_set_security_missing_certificates(opc_repository): try: await opc_repository.set_security() except ValueError as e: - assert str( - e) == "Certificate and private key paths must be provided for secure connection." + assert str(e) == 'Certificate and private key paths must be provided for secure connection.' @pytest.mark.asyncio @@ -123,15 +124,15 @@ async def test_try_connect_success(opc_repository): async def test_try_connect_fail(opc_repository): opc_repository.last_reconnection_time = None opc_repository.client = MagicMock() - opc_repository.client.connect.side_effect = Exception("Test error") + opc_repository.client.connect.side_effect = Exception('Test error') is_connected, error_data = await opc_repository.try_connect() opc_repository.client.connect.assert_called_once() assert is_connected is False - assert error_data['notification_id'] == f"OPC_CONNECTION_ERROR_{opc_repository.id}" - assert error_data['message'] == "Failed to connect to OPC server: Test error" - assert error_data['block'] == "opc_repository" + assert error_data['notification_id'] == f'OPC_CONNECTION_ERROR_{opc_repository.id}' + assert error_data['message'] == 'Failed to connect to OPC server: Test error' + assert error_data['block'] == 'opc_repository' assert error_data['level'] == NotificationLevel.ERROR assert error_data['attachment_content'] is not None @@ -154,12 +155,11 @@ async def test_disconnect_no_client(opc_repository): @pytest.mark.asyncio async def test_disconnect_error(opc_repository, mock_client): opc_repository.client = mock_client - mock_client.disconnect.side_effect = Exception("Test error") + mock_client.disconnect.side_effect = Exception('Test error') await opc_repository.disconnect() opc_repository.logger.custom_error.assert_called_once_with( - "Failed to disconnect from OPC server: Test error", - ANY + 'Failed to disconnect from OPC server: Test error', ANY ) assert opc_repository.client is None @@ -177,9 +177,7 @@ async def test_validate_connection_none_client(opc_repository): async def test_validate_connection_error_count_disconnect_error(opc_repository): opc_repository.error_count = 6 opc_repository.client = AsyncMock() - opc_repository.disconnect = AsyncMock( - side_effect=Exception("Test error") - ) + opc_repository.disconnect = AsyncMock(side_effect=Exception('Test error')) opc_repository.connect = AsyncMock(return_value=(True, {})) response = await opc_repository.validate_connection() @@ -188,34 +186,34 @@ async def test_validate_connection_error_count_disconnect_error(opc_repository): opc_repository.connect.assert_called_once() opc_repository.logger.custom_error.assert_has_calls( [ - call("Failed to disconnect from OPC server: Test error", ANY), + call('Failed to disconnect from OPC server: Test error', ANY), ] ) @pytest.mark.asyncio async def test_validate_connection_error_validate_connection_error(opc_repository): - opc_repository.client = MagicMock( - uaclient=Exception("Test error") - ) + opc_repository.client = MagicMock(uaclient=Exception('Test error')) opc_repository.error_count = 0 response = await opc_repository.validate_connection() - assert response == (False, { - "notification_id": f"OPC_CONNECTION_CHECK_ERROR_{opc_repository.id}", - "message": "Failed to validate connection to OPC server: 'Exception' object has no attribute 'protocol'", - "block": "opc_repository", - "level": NotificationLevel.ERROR, - "attachment_content": ANY - }) + assert response == ( + False, + { + 'notification_id': f'OPC_CONNECTION_CHECK_ERROR_{opc_repository.id}', + 'message': "Failed to validate connection to OPC server: 'Exception' object has no attribute 'protocol'", + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + 'attachment_content': ANY, + }, + ) @pytest.mark.asyncio @patch('laborious.utils.repository.opc_repository.datetime') async def test_validate_connection_lost_not_time_to_reconnect(_mock_datetime, opc_repository): - _mock_datetime.now = MagicMock( - return_value=datetime(2025, 1, 1, 0, 0, 0)) + _mock_datetime.now = MagicMock(return_value=datetime(2025, 1, 1, 0, 0, 0)) opc_repository.error_count = 0 opc_repository.client = MagicMock() opc_repository.client.uaclient.protocol = None @@ -224,19 +222,21 @@ async def test_validate_connection_lost_not_time_to_reconnect(_mock_datetime, op response = await opc_repository.validate_connection() opc_repository.connect.assert_not_called() - assert response == (False, { - "notification_id": f"OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{opc_repository.id}", - "message": f"OPC server {opc_repository.id} is not connected, waiting for next reconnection window...", - "block": "opc_repository", - "level": NotificationLevel.WARNING - }) + assert response == ( + False, + { + 'notification_id': f'OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{opc_repository.id}', + 'message': f'OPC server {opc_repository.id} is not connected, waiting for next reconnection window...', + 'block': 'opc_repository', + 'level': NotificationLevel.WARNING, + }, + ) @pytest.mark.asyncio @patch('laborious.utils.repository.opc_repository.datetime') async def test_validate_connection_lost_time_to_reconnect(mock_datetime, opc_repository): - mock_datetime.now = MagicMock( - return_value=datetime(2025, 1, 1, 1, 0, 0)) + mock_datetime.now = MagicMock(return_value=datetime(2025, 1, 1, 1, 0, 0)) opc_repository.error_count = 0 opc_repository.client = AsyncMock() opc_repository.client.uaclient.protocol = None @@ -253,7 +253,7 @@ async def test_validate_connection_success(opc_repository): opc_repository.client = MagicMock() opc_repository.error_count = 0 opc_repository.client.uaclient.protocol = MagicMock() - opc_repository.client.uaclient.protocol.state = "open" + opc_repository.client.uaclient.protocol.state = 'open' output = await opc_repository.validate_connection() assert output == (True, {}) @@ -262,17 +262,16 @@ async def test_validate_connection_success(opc_repository): @pytest.mark.asyncio async def test_write_data_validate_connection_do_nothing(opc_repository): opc_repository.validate_connection = AsyncMock(return_value=(True, {})) - opc_repository.client = AsyncMock( - get_node=MagicMock() - ) + opc_repository.client = AsyncMock(get_node=MagicMock()) mock_node = AsyncMock() opc_repository.client.get_node.return_value = mock_node - result = await opc_repository.write_data("ns=2;s=TestNode", 42.0, - "float", opc_repository.logger, metadata['metadata']) + result = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', opc_repository.logger, metadata['metadata'] + ) opc_repository.validate_connection.assert_called_once() - opc_repository.client.get_node.assert_called_once_with("ns=2;s=TestNode") + opc_repository.client.get_node.assert_called_once_with('ns=2;s=TestNode') assert result == (True, {}) @@ -282,8 +281,9 @@ async def test_write_data_validate_connection_failed(opc_repository): opc_repository.client = AsyncMock() opc_repository.error_count = 0 - result = await opc_repository.write_data("ns=2;s=TestNode", 42.0, - "float", opc_repository.logger, metadata['metadata']) + result = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', opc_repository.logger, metadata['metadata'] + ) opc_repository.validate_connection.assert_called_once() opc_repository.client.get_node.assert_not_called() @@ -295,18 +295,21 @@ async def test_write_data_get_node_failed(opc_repository): opc_repository.validate_connection = AsyncMock(return_value=(True, {})) opc_repository.client = AsyncMock() opc_repository.error_count = 0 - opc_repository.client.get_node = MagicMock( - side_effect=Exception("Test error")) + opc_repository.client.get_node = MagicMock(side_effect=Exception('Test error')) - is_success, error_data = await opc_repository.write_data("ns=2;s=TestNode", 42.0, - "float", opc_repository.logger, metadata['metadata']) + is_success, error_data = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', opc_repository.logger, metadata['metadata'] + ) opc_repository.validate_connection.assert_called_once() - opc_repository.client.get_node.assert_called_once_with("ns=2;s=TestNode") + opc_repository.client.get_node.assert_called_once_with('ns=2;s=TestNode') assert is_success is False - assert error_data['notification_id'] == f"OPC_WRITE_GET_NODE_ERROR_{opc_repository.id}" - assert error_data['message'] == "Failed to get node from OPC server: Test error | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}" - assert error_data['block'] == "opc_repository" + assert error_data['notification_id'] == f'OPC_WRITE_GET_NODE_ERROR_{opc_repository.id}' + assert ( + error_data['message'] + == "Failed to get node from OPC server: Test error | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}" + ) + assert error_data['block'] == 'opc_repository' assert error_data['level'] == NotificationLevel.ERROR assert error_data['attachment_content'] is not None @@ -318,16 +321,20 @@ async def test_write_data_invalid_data_type(opc_repository, mock_client): mock_node = AsyncMock() mock_client.get_node = MagicMock(return_value=mock_node) - is_success, error_data = await opc_repository.write_data("ns=2;s=TestNode", 42.0, - "invalid_type", opc_repository.logger, metadata['metadata']) + is_success, error_data = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'invalid_type', opc_repository.logger, metadata['metadata'] + ) opc_repository.validate_connection.assert_called_once() - mock_client.get_node.assert_called_once_with("ns=2;s=TestNode") + mock_client.get_node.assert_called_once_with('ns=2;s=TestNode') assert is_success is False - assert error_data['notification_id'] == f"OPC_WRITE_DATA_TYPE_ERROR_{opc_repository.id}" - assert error_data['message'] == "Unsupported data type: invalid_type | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}" - assert error_data['block'] == "opc_repository" + assert error_data['notification_id'] == f'OPC_WRITE_DATA_TYPE_ERROR_{opc_repository.id}' + assert ( + error_data['message'] + == "Unsupported data type: invalid_type | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}" + ) + assert error_data['block'] == 'opc_repository' assert error_data['level'] == NotificationLevel.ERROR assert error_data.get('attachment_content') is None @@ -340,10 +347,11 @@ async def test_write_data(mock_metrics, opc_repository, mock_client): mock_node = AsyncMock() mock_client.get_node = MagicMock(return_value=mock_node) - result = await opc_repository.write_data("ns=2;s=TestNode", 42.0, - "float", opc_repository.logger, metadata['metadata']) + result = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', opc_repository.logger, metadata['metadata'] + ) - mock_client.get_node.assert_called_once_with("ns=2;s=TestNode") + mock_client.get_node.assert_called_once_with('ns=2;s=TestNode') mock_node.write_value.assert_called_once() assert result == (True, {}) @@ -351,7 +359,7 @@ async def test_write_data(mock_metrics, opc_repository, mock_client): pod_id=opc_repository.pod_id, model_name=metadata['metadata']['model_name'], pipeline_name=metadata['metadata']['workflow_name'], - opc_server_id=opc_repository.id + opc_server_id=opc_repository.id, ) mock_metrics.PREDICTION_OPC_WRITING_COUNT.labels.return_value.inc.assert_called_once_with() @@ -359,10 +367,11 @@ async def test_write_data(mock_metrics, opc_repository, mock_client): pod_id=opc_repository.pod_id, model_name=metadata['metadata']['model_name'], pipeline_name=metadata['metadata']['workflow_name'], - opc_server_id=opc_repository.id + opc_server_id=opc_repository.id, ) mock_metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR.labels.return_value.observe.assert_called_once_with( - ANY) + ANY + ) @pytest.mark.asyncio @@ -372,17 +381,21 @@ async def test_write_data_write_value_failed(opc_repository, mock_client): mock_node = AsyncMock() opc_repository.error_count = 0 mock_client.get_node = MagicMock(return_value=mock_node) - mock_node.write_value.side_effect = Exception("Test error") + mock_node.write_value.side_effect = Exception('Test error') - is_success, error_data = await opc_repository.write_data("ns=2;s=TestNode", 42.0, - "float", opc_repository.logger, metadata['metadata']) + is_success, error_data = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', opc_repository.logger, metadata['metadata'] + ) opc_repository.validate_connection.assert_called_once() - mock_client.get_node.assert_called_once_with("ns=2;s=TestNode") + mock_client.get_node.assert_called_once_with('ns=2;s=TestNode') mock_node.write_value.assert_called_once() assert is_success is False - assert error_data['notification_id'] == f"OPC_WRITE_DATA_ERROR_{opc_repository.id}" - assert error_data['message'] == "Failed to write data to OPC server: Test error | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}" - assert error_data['block'] == "opc_repository" + assert error_data['notification_id'] == f'OPC_WRITE_DATA_ERROR_{opc_repository.id}' + assert ( + error_data['message'] + == "Failed to write data to OPC server: Test error | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}" + ) + assert error_data['block'] == 'opc_repository' assert error_data['level'] == NotificationLevel.ERROR assert error_data['attachment_content'] is not None diff --git a/tests/laborious/utils/test_connectors_config.py b/tests/laborious/utils/test_connectors_config.py index 910439c..cf2a6b8 100644 --- a/tests/laborious/utils/test_connectors_config.py +++ b/tests/laborious/utils/test_connectors_config.py @@ -1,8 +1,11 @@ from os import environ -from laborious.utils.connectors_config import (build_mlflow_config, - build_opc_config, - build_postgres_config, - build_mongodb_config) + +from laborious.utils.connectors_config import ( + build_mlflow_config, + build_mongodb_config, + build_opc_config, + build_postgres_config, +) def test_build_mlflow_config_with_env_vars(): @@ -144,7 +147,7 @@ def test_build_mongo_db_config_with_env_vars(): assert build_mongodb_config() == { 'connection_string': 'mongodb://sientia1:sientia1@localhost:27018', 'database_name': 'test_db', - 'ttl_index_seconds': 3600 + 'ttl_index_seconds': 3600, } @@ -157,5 +160,5 @@ def test_build_mongo_db_config_with_defaults(): assert build_mongodb_config() == { 'connection_string': 'mongodb://root:wKZDbMNU1c@localhost:27018', 'database_name': 'sientia', - 'ttl_index_seconds': 3600 + 'ttl_index_seconds': 3600, } diff --git a/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py b/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py index a8e6e20..006ef4e 100644 --- a/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py +++ b/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py @@ -1,9 +1,10 @@ -from unittest.mock import call, patch, AsyncMock, ANY -from pytest import mark, fixture +from unittest.mock import ANY, AsyncMock, call, patch + +from pytest import fixture, mark +from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ from laborious.activities.activities import Activities from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction -from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ @fixture @@ -12,149 +13,167 @@ def format_and_export_prediction(): metadata = { - "metadata": { - "model_id": "test_model", - "model_name": "test_model", - "workflow_name": "test_workflow", - "schema_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', }, } @mark.asyncio -@patch("laborious.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock) +@patch( + 'laborious.workflows.sub_workflows.format_and_export_prediction.workflow', + new_callable=AsyncMock, +) async def test_run_none_path_flag(workflow_mock, format_and_export_prediction): - input_data = { 'metadata': metadata, - "path_flag": None, - "data": {"test": "data"}, - "timestamp": "2021-01-01", - "model_id": 1, - "prediction_confidence": 0, - "schema": "test_schema", - "table_name": "test_table", - "opc_servers": ["test_server"], - "opc_output_config": {"test": "config"}, - "prediction_store_policy": "erl:1" + 'path_flag': None, + 'data': {'test': 'data'}, + 'timestamp': '2021-01-01', + 'model_id': 1, + 'prediction_confidence': 0, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'opc_servers': ['test_server'], + 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': 'erl:1', } await format_and_export_prediction.run(input_data) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call( - Activities.format_prediction, - { - 'data': input_data['data'], - 'timestamp': input_data['timestamp'], - 'model_id': input_data['model_id'], - 'prediction_confidence': input_data['prediction_confidence'], - 'prediction_store_policy': input_data['prediction_store_policy'], - **metadata - }, - retry_policy=ANY, - start_to_close_timeout=ANY - )]) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.format_prediction, + { + 'data': input_data['data'], + 'timestamp': input_data['timestamp'], + 'model_id': input_data['model_id'], + 'prediction_confidence': input_data['prediction_confidence'], + 'prediction_store_policy': input_data['prediction_store_policy'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) - workflow_mock.execute_activity_method.assert_has_calls([ - call( - Activities.write_opc_data, - { - 'opc_output_config': input_data['opc_output_config'], - 'data': workflow_mock.execute_local_activity_method.return_value, - **metadata - }, - retry_policy=ANY, - start_to_close_timeout=ANY - ) - ]) + workflow_mock.execute_activity_method.assert_has_calls( + [ + call( + Activities.write_opc_data, + { + 'opc_output_config': input_data['opc_output_config'], + 'data': workflow_mock.execute_local_activity_method.return_value, + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) - workflow_mock.execute_activity_method.assert_has_calls([ - call( - Activities.export_data_to_postgres, - { - 'schema': input_data['schema'], - 'table_name': input_data['table_name'], - 'data': workflow_mock.execute_activity_method.return_value, - **metadata, - 'timestamp_conversion': { - 'column': 'timestamp', - 'format': DATETIME_FORMAT_WITH_TZ - } - }, - retry_policy=ANY, - start_to_close_timeout=ANY - )]) + workflow_mock.execute_activity_method.assert_has_calls( + [ + call( + Activities.export_data_to_postgres, + { + 'schema': input_data['schema'], + 'table_name': input_data['table_name'], + 'data': workflow_mock.execute_activity_method.return_value, + **metadata, + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_WITH_TZ, + }, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) assert workflow_mock.execute_activity_method.call_count == 3 assert workflow_mock.execute_local_activity_method.call_count == 1 @mark.asyncio -@patch("laborious.workflows.sub_workflows.format_and_export_prediction.workflow", new_callable=AsyncMock) +@patch( + 'laborious.workflows.sub_workflows.format_and_export_prediction.workflow', + new_callable=AsyncMock, +) async def test_run_default_path_flag(workflow_mock, format_and_export_prediction): - input_data = { 'metadata': metadata, - "path_flag": "default", - "data": {"test": "data"}, - "timestamp": "2021-01-01", - "model_id": 1, - "prediction_confidence": 0, - "schema": "test_schema", - "table_name": "test_table", - "opc_servers": ["test_server"], - "opc_output_config": {"test": "config"}, - "comment": "test_comment" + 'path_flag': 'default', + 'data': {'test': 'data'}, + 'timestamp': '2021-01-01', + 'model_id': 1, + 'prediction_confidence': 0, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'opc_servers': ['test_server'], + 'opc_output_config': {'test': 'config'}, + 'comment': 'test_comment', } await format_and_export_prediction.run(input_data) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call( - Activities.format_default_prediction, - { - 'timestamp': input_data['timestamp'], - 'model_id': input_data['model_id'], - 'prediction_confidence': input_data['prediction_confidence'], - 'comment': input_data['comment'], - **metadata - }, - retry_policy=ANY, - start_to_close_timeout=ANY - ) - ]) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.format_default_prediction, + { + 'timestamp': input_data['timestamp'], + 'model_id': input_data['model_id'], + 'prediction_confidence': input_data['prediction_confidence'], + 'comment': input_data['comment'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) - workflow_mock.execute_activity_method.assert_has_calls([ - call( - Activities.write_opc_data, - { - 'opc_output_config': input_data['opc_output_config'], - 'data': workflow_mock.execute_local_activity_method.return_value, - **metadata - }, - retry_policy=ANY, - start_to_close_timeout=ANY - ) - ]) + workflow_mock.execute_activity_method.assert_has_calls( + [ + call( + Activities.write_opc_data, + { + 'opc_output_config': input_data['opc_output_config'], + 'data': workflow_mock.execute_local_activity_method.return_value, + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) - workflow_mock.execute_activity_method.assert_has_calls([ - call( - Activities.export_data_to_postgres, - { - 'schema': input_data['schema'], - 'table_name': input_data['table_name'], - 'data': workflow_mock.execute_activity_method.return_value, - **metadata, - 'timestamp_conversion': { - 'column': 'timestamp', - 'format': DATETIME_FORMAT_WITH_TZ - } - }, - retry_policy=ANY, - start_to_close_timeout=ANY - ) - ]) + workflow_mock.execute_activity_method.assert_has_calls( + [ + call( + Activities.export_data_to_postgres, + { + 'schema': input_data['schema'], + 'table_name': input_data['table_name'], + 'data': workflow_mock.execute_activity_method.return_value, + **metadata, + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_WITH_TZ, + }, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) assert workflow_mock.execute_activity_method.call_count == 3 assert workflow_mock.execute_local_activity_method.call_count == 1 diff --git a/tests/laborious/workflows/subworkflows/test_prediction_process.py b/tests/laborious/workflows/subworkflows/test_prediction_process.py index df60ada..6baec30 100644 --- a/tests/laborious/workflows/subworkflows/test_prediction_process.py +++ b/tests/laborious/workflows/subworkflows/test_prediction_process.py @@ -1,5 +1,7 @@ -from unittest.mock import AsyncMock, patch, call, ANY +from unittest.mock import ANY, AsyncMock, call, patch + from pytest import fixture, mark + from laborious.activities.activities import Activities from laborious.workflows.sub_workflows.prediction_process import PredictionProcess @@ -10,17 +12,17 @@ def prediction_process(): metadata = { - "metadata": { - "model_id": "test_model", - "model_name": "test_model", - "workflow_name": "test_workflow", - "schema_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', }, } @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock) async def test_run(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock(return_value=False) # Arrange @@ -34,26 +36,24 @@ async def test_run(workflow_mock, prediction_process): 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_config': { - 'retention': '30' - }, + 'model_config': {'retention': '30'}, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'}, - 'prediction_store_policy': 'lts:1' + 'prediction_store_policy': 'lts:1', } # Mock the activity responses workflow_mock.execute_local_activity_method.side_effect = [ '2024-01-01', # get_last_timestamp - ('continue', 0.95, "Input data with bad quality"), # input_gate + ('continue', 0.95, 'Input data with bad quality'), # input_gate {'content': 'transformed_data', 'timestamp': '2024-01-01'}, # transform_data # mlflow_response_gate (transform) - ('continue', 0.95, "Error"), + ('continue', 0.95, 'Error'), # mlflow_content_gate (transform) - ('continue', 0.95, "Transformed data not passed the content filter"), + ('continue', 0.95, 'Transformed data not passed the content filter'), {'content': 'predicted_data', 'timestamp': '2024-01-01'}, # request_predict # mlflow_response_gate (predict) - ('continue', 0.95, "Error"), + ('continue', 0.95, 'Error'), ] # Act @@ -62,57 +62,112 @@ async def test_run(workflow_mock, prediction_process): # Assert assert workflow_mock.execute_local_activity_method.call_count == 7 - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.get_last_timestamp, { - **metadata, - 'data': input_data['data'], - }, - retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.input_gate, { - **metadata, - 'filters': input_data['input_filters'], - 'data': input_data['data'], - 'path_priority': input_data['path_priority'], - }, retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.request_transform, { - **metadata, - 'data': input_data['data'], - 'model_name': input_data['model_name'], - 'model_config': input_data['model_config'], - }, retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.mlflow_response_gate, { - **metadata, - 'filters': input_data['mlflow_transform_filters'], - 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, - 'type': 'transform', - 'path_priority': input_data['path_priority'], - }, retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.mlflow_content_gate, { - **metadata, - 'filters': input_data['mlflow_transform_filters'], - 'data': 'transformed_data', - 'type': 'transform', - 'path_priority': input_data['path_priority'], - }, retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.request_predict, { - **metadata, - 'data': 'transformed_data', - 'model_name': input_data['model_name'], - 'model_config': input_data['model_config'], - }, retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.mlflow_response_gate, { - **metadata, - 'filters': input_data['mlflow_predict_filters'], - 'data': {'content': 'predicted_data', 'timestamp': '2024-01-01'}, - 'type': 'predict', - 'path_priority': input_data['path_priority'], - }, retry_policy=ANY, start_to_close_timeout=ANY)]) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.get_last_timestamp, + { + **metadata, + 'data': input_data['data'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.input_gate, + { + **metadata, + 'filters': input_data['input_filters'], + 'data': input_data['data'], + 'path_priority': input_data['path_priority'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.request_transform, + { + **metadata, + 'data': input_data['data'], + 'model_name': input_data['model_name'], + 'model_config': input_data['model_config'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.mlflow_response_gate, + { + **metadata, + 'filters': input_data['mlflow_transform_filters'], + 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, + 'type': 'transform', + 'path_priority': input_data['path_priority'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.mlflow_content_gate, + { + **metadata, + 'filters': input_data['mlflow_transform_filters'], + 'data': 'transformed_data', + 'type': 'transform', + 'path_priority': input_data['path_priority'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.request_predict, + { + **metadata, + 'data': 'transformed_data', + 'model_name': input_data['model_name'], + 'model_config': input_data['model_config'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.mlflow_response_gate, + { + **metadata, + 'filters': input_data['mlflow_predict_filters'], + 'data': {'content': 'predicted_data', 'timestamp': '2024-01-01'}, + 'type': 'predict', + 'path_priority': input_data['path_priority'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) workflow_mock.execute_child_workflow.assert_called_once_with( 'format_and_export_prediction', @@ -129,13 +184,13 @@ async def test_run(workflow_mock, prediction_process): 'schema': input_data['schema'], 'table_name': input_data['table_name'], 'comment': 'Error', - 'prediction_store_policy': input_data['prediction_store_policy'] - } + 'prediction_store_policy': input_data['prediction_store_policy'], + }, ) @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock) async def test_run_stop_at_input_gate(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock(return_value=True) # Arrange @@ -149,17 +204,15 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process): 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_config': { - 'retention': '30' - }, + 'model_config': {'retention': '30'}, 'path_priority': ['continue', 'repeat', 'stop'], - 'opc_output_config': {'test': 'config'} + 'opc_output_config': {'test': 'config'}, } # Mock the activity responses workflow_mock.execute_local_activity_method.side_effect = [ '2024-01-01', # get_last_timestamp - ('stop', 0.95, "Input data with bad quality"), # input_gate + ('stop', 0.95, 'Input data with bad quality'), # input_gate ] # Act @@ -167,23 +220,35 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process): # Assert assert workflow_mock.execute_local_activity_method.call_count == 2 - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.get_last_timestamp, { - 'data': input_data['data'], - **metadata, - }, retry_policy=ANY, start_to_close_timeout=ANY), - call(Activities.input_gate, { - 'filters': input_data['input_filters'], - 'data': input_data['data'], - 'path_priority': input_data['path_priority'], - **metadata, - }, retry_policy=ANY, start_to_close_timeout=ANY) - ]) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.get_last_timestamp, + { + 'data': input_data['data'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ), + call( + Activities.input_gate, + { + 'filters': input_data['input_filters'], + 'data': input_data['data'], + 'path_priority': input_data['path_priority'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ), + ] + ) workflow_mock.execute_child_workflow.assert_not_called() @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock) async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_process): prediction_process.path_flag_handler = AsyncMock(side_effect=[False, True]) # Arrange @@ -197,19 +262,17 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_config': { - 'retention': '30' - }, + 'model_config': {'retention': '30'}, 'path_priority': ['continue', 'repeat', 'stop'], - 'opc_output_config': {'test': 'config'} + 'opc_output_config': {'test': 'config'}, } # Mock the activity responses workflow_mock.execute_local_activity_method.side_effect = [ '2024-01-01', # get_last_timestamp - ('repeat', 0.95, "Input data with bad quality"), # input_gate + ('repeat', 0.95, 'Input data with bad quality'), # input_gate {'content': 'transformed_data', 'timestamp': '2024-01-01'}, # transform_data - ('continue', 0.95, "Error"), # mlflow_response_gate (transform) + ('continue', 0.95, 'Error'), # mlflow_response_gate (transform) ] # Act @@ -217,46 +280,72 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ # Assert assert workflow_mock.execute_local_activity_method.call_count == 4 - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.get_last_timestamp, { - 'data': input_data['data'], - **metadata, - }, - retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.input_gate, { - 'filters': input_data['input_filters'], - 'data': input_data['data'], - 'path_priority': input_data['path_priority'], - **metadata, - }, - retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.request_transform, { - 'data': input_data['data'], - 'model_name': input_data['model_name'], - 'model_config': input_data['model_config'], - **metadata - }, - retry_policy=ANY, start_to_close_timeout=ANY) - ]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.mlflow_response_gate, { - 'filters': input_data['mlflow_transform_filters'], - 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, - 'type': 'transform', - 'path_priority': input_data['path_priority'], - **metadata - }, retry_policy=ANY, start_to_close_timeout=ANY) - ]) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.get_last_timestamp, + { + 'data': input_data['data'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.input_gate, + { + 'filters': input_data['input_filters'], + 'data': input_data['data'], + 'path_priority': input_data['path_priority'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.request_transform, + { + 'data': input_data['data'], + 'model_name': input_data['model_name'], + 'model_config': input_data['model_config'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.mlflow_response_gate, + { + 'filters': input_data['mlflow_transform_filters'], + 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, + 'type': 'transform', + 'path_priority': input_data['path_priority'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) workflow_mock.execute_child_workflow.assert_not_called() @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock) async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process): - prediction_process.path_flag_handler = AsyncMock( - side_effect=[False, False, True]) + prediction_process.path_flag_handler = AsyncMock(side_effect=[False, False, True]) # Arrange input_data = { 'metadata': metadata, @@ -268,22 +357,20 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_config': { - 'retention': '30' - }, + 'model_config': {'retention': '30'}, 'path_priority': ['continue', 'repeat', 'stop'], - 'opc_output_config': {'test': 'config'} + 'opc_output_config': {'test': 'config'}, } # Mock the activity responses workflow_mock.execute_local_activity_method.side_effect = [ '2024-01-01', # get_last_timestamp - ('continue', 0.95, "Input data with bad quality"), # input_gate + ('continue', 0.95, 'Input data with bad quality'), # input_gate {'content': 'transformed_data', 'timestamp': '2024-01-01'}, # transform_data # mlflow_response_gate (transform) - ('continue', 0.95, "Error"), + ('continue', 0.95, 'Error'), # mlflow_content_gate (transform) - ('continue', 0.95, "Transformed data not passed the content filter"), + ('continue', 0.95, 'Transformed data not passed the content filter'), ] # Act @@ -292,51 +379,88 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process # Assert assert workflow_mock.execute_local_activity_method.call_count == 5 - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.get_last_timestamp, { - 'data': input_data['data'], - **metadata, - }, - retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.input_gate, { - 'filters': input_data['input_filters'], - 'data': input_data['data'], - 'path_priority': input_data['path_priority'], - **metadata, - }, - retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.request_transform, { - 'data': input_data['data'], - 'model_name': input_data['model_name'], - 'model_config': input_data['model_config'], - **metadata - }, retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.mlflow_response_gate, { - 'filters': input_data['mlflow_transform_filters'], - 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, - 'type': 'transform', - 'path_priority': input_data['path_priority'], - **metadata, - }, retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.mlflow_content_gate, { - 'filters': input_data['mlflow_transform_filters'], - 'data': 'transformed_data', - 'type': 'transform', - 'path_priority': input_data['path_priority'], - **metadata, - }, retry_policy=ANY, start_to_close_timeout=ANY)]) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.get_last_timestamp, + { + 'data': input_data['data'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.input_gate, + { + 'filters': input_data['input_filters'], + 'data': input_data['data'], + 'path_priority': input_data['path_priority'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.request_transform, + { + 'data': input_data['data'], + 'model_name': input_data['model_name'], + 'model_config': input_data['model_config'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.mlflow_response_gate, + { + 'filters': input_data['mlflow_transform_filters'], + 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, + 'type': 'transform', + 'path_priority': input_data['path_priority'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.mlflow_content_gate, + { + 'filters': input_data['mlflow_transform_filters'], + 'data': 'transformed_data', + 'type': 'transform', + 'path_priority': input_data['path_priority'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) workflow_mock.execute_child_workflow.assert_not_called() @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock) async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_process): - prediction_process.path_flag_handler = AsyncMock( - side_effect=[False, False, False, True]) + prediction_process.path_flag_handler = AsyncMock(side_effect=[False, False, False, True]) # Arrange input_data = { 'metadata': metadata, @@ -348,24 +472,22 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_config': { - 'retention': '30' - }, + 'model_config': {'retention': '30'}, 'path_priority': ['continue', 'repeat', 'stop'], - 'opc_output_config': {'test': 'config'} + 'opc_output_config': {'test': 'config'}, } # Mock the activity responses workflow_mock.execute_local_activity_method.side_effect = [ '2024-01-01', # get_last_timestamp - ('continue', 0.95, "Input data with bad quality"), # input_gate + ('continue', 0.95, 'Input data with bad quality'), # input_gate {'content': 'transformed_data', 'timestamp': '2024-01-01'}, # transform_data # mlflow_response_gate (transform) - ('continue', 0.95, "Error"), + ('continue', 0.95, 'Error'), # mlflow_content_gate (transform) - ('continue', 0.95, "Transformed data not passed the content filter"), + ('continue', 0.95, 'Transformed data not passed the content filter'), {'content': 'predicted_data', 'timestamp': '2024-01-01'}, # request_predict - ('continue', 0.95, "Error"), # mlflow_response_gate (predict) + ('continue', 0.95, 'Error'), # mlflow_response_gate (predict) ] # Act @@ -373,63 +495,117 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p # Assert assert workflow_mock.execute_local_activity_method.call_count == 7 - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.get_last_timestamp, { - 'data': input_data['data'], - **metadata, - }, - retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.input_gate, { - 'filters': input_data['input_filters'], - 'data': input_data['data'], - 'path_priority': input_data['path_priority'], - **metadata, - }, - retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.request_transform, { - 'data': input_data['data'], - 'model_name': input_data['model_name'], - 'model_config': input_data['model_config'], - **metadata - }, retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.mlflow_response_gate, { - 'filters': input_data['mlflow_transform_filters'], - 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, - 'type': 'transform', - 'path_priority': input_data['path_priority'], - **metadata, - }, retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.mlflow_content_gate, { - 'filters': input_data['mlflow_transform_filters'], - 'data': 'transformed_data', - 'type': 'transform', - 'path_priority': input_data['path_priority'], - **metadata, - }, retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.request_predict, { - 'data': 'transformed_data', - 'model_name': input_data['model_name'], - 'model_config': input_data['model_config'], - **metadata - }, retry_policy=ANY, start_to_close_timeout=ANY)]) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call(Activities.mlflow_response_gate, { - 'filters': input_data['mlflow_predict_filters'], - 'data': {'content': 'predicted_data', 'timestamp': '2024-01-01'}, - 'type': 'predict', - 'path_priority': input_data['path_priority'], - **metadata, - }, retry_policy=ANY, start_to_close_timeout=ANY)]) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.get_last_timestamp, + { + 'data': input_data['data'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.input_gate, + { + 'filters': input_data['input_filters'], + 'data': input_data['data'], + 'path_priority': input_data['path_priority'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.request_transform, + { + 'data': input_data['data'], + 'model_name': input_data['model_name'], + 'model_config': input_data['model_config'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.mlflow_response_gate, + { + 'filters': input_data['mlflow_transform_filters'], + 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, + 'type': 'transform', + 'path_priority': input_data['path_priority'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.mlflow_content_gate, + { + 'filters': input_data['mlflow_transform_filters'], + 'data': 'transformed_data', + 'type': 'transform', + 'path_priority': input_data['path_priority'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.request_predict, + { + 'data': 'transformed_data', + 'model_name': input_data['model_name'], + 'model_config': input_data['model_config'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.mlflow_response_gate, + { + 'filters': input_data['mlflow_predict_filters'], + 'data': {'content': 'predicted_data', 'timestamp': '2024-01-01'}, + 'type': 'predict', + 'path_priority': input_data['path_priority'], + **metadata, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) workflow_mock.execute_child_workflow.assert_not_called() @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock) async def test_path_flag_handler_stop(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} @@ -440,21 +616,24 @@ async def test_path_flag_handler_stop(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_config = { - 'retention': '30' - } + model_config = {'retention': '30'} # Act result = await prediction_process.path_flag_handler( - data, path_flag, { + data, + path_flag, + { 'metadata': metadata, 'schema': schema, 'table_name': table_name, 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_config': model_config - }, confidence, last_timestamp, "" + 'model_config': model_config, + }, + confidence, + last_timestamp, + '', ) # Assert @@ -464,7 +643,7 @@ async def test_path_flag_handler_stop(workflow_mock, prediction_process): @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock) async def test_path_flag_handler_repeat(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} @@ -475,21 +654,24 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_config = { - 'retention': '30' - } + model_config = {'retention': '30'} # Act result = await prediction_process.path_flag_handler( - data, path_flag, { + data, + path_flag, + { 'metadata': metadata, 'schema': schema, 'table_name': table_name, 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_config': model_config - }, confidence, last_timestamp, "" + 'model_config': model_config, + }, + confidence, + last_timestamp, + '', ) # Assert @@ -504,13 +686,13 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process): 'last_timestamp': last_timestamp, }, retry_policy=ANY, - start_to_close_timeout=ANY + start_to_close_timeout=ANY, ) workflow_mock.execute_child_workflow.assert_not_called() @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock) async def test_path_flag_handler_continue(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} @@ -521,14 +703,14 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_config = { - 'retention': '30' - } + model_config = {'retention': '30'} prediction_store_policy = 'erl:1' # Act result = await prediction_process.path_flag_handler( - data, path_flag, { + data, + path_flag, + { 'metadata': metadata, 'schema': schema, 'table_name': table_name, @@ -537,8 +719,11 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): 'model_name': model_name, 'model_config': model_config, 'opc_output_config': {'test': 'config'}, - 'prediction_store_policy': prediction_store_policy - }, confidence, last_timestamp, 'Prediction Process' + 'prediction_store_policy': prediction_store_policy, + }, + confidence, + last_timestamp, + 'Prediction Process', ) # Assert @@ -559,13 +744,13 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): 'table_name': table_name, 'comment': 'Prediction Process', 'opc_output_config': {'test': 'config'}, - 'prediction_store_policy': prediction_store_policy - } + 'prediction_store_policy': prediction_store_policy, + }, ) @mark.asyncio -@patch("laborious.workflows.sub_workflows.prediction_process.workflow", new_callable=AsyncMock) +@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock) async def test_path_flag_handler_unknown(workflow_mock, prediction_process): # Arrange data = {'test': 'data'} @@ -576,13 +761,13 @@ async def test_path_flag_handler_unknown(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_config = { - 'retention': '30' - } + model_config = {'retention': '30'} prediction_store_policy = 'erl:1' # Act result = await prediction_process.path_flag_handler( - data, path_flag, { + data, + path_flag, + { **metadata, 'schema': schema, 'table_name': table_name, @@ -591,8 +776,11 @@ async def test_path_flag_handler_unknown(workflow_mock, prediction_process): 'model_name': model_name, 'model_config': model_config, 'opc_output_config': {'test': 'config'}, - 'prediction_store_policy': prediction_store_policy - }, confidence, last_timestamp, "" + 'prediction_store_policy': prediction_store_policy, + }, + confidence, + last_timestamp, + '', ) # Assert diff --git a/tests/laborious/workflows/test_minimal_retrain.py b/tests/laborious/workflows/test_minimal_retrain.py index b3b03b5..c088702 100644 --- a/tests/laborious/workflows/test_minimal_retrain.py +++ b/tests/laborious/workflows/test_minimal_retrain.py @@ -1,5 +1,7 @@ -from unittest.mock import AsyncMock, MagicMock, call, patch, ANY +from unittest.mock import ANY, AsyncMock, call, patch + from pytest import fixture, mark + from laborious.activities.activities import Activities from laborious.workflows.minimal_retrain import MinimalRetrain @@ -10,11 +12,11 @@ def minimal_retrain() -> MinimalRetrain: metadata = { - "metadata": { - "model_id": "test_model_id", - "model_name": "test_model", - "workflow_name": "minimal_retrain", - "schedule_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'workflow_name': 'minimal_retrain', + 'schedule_name': 'test_schedule', }, } @@ -23,76 +25,271 @@ metadata = { @patch('laborious.workflows.minimal_retrain.workflow', new_callable=AsyncMock) async def test_run(workflow_mock: AsyncMock, minimal_retrain: MinimalRetrain): input_data = { - "model_id": "test_model_id", - "model_name": "test_model", - "workflow_name": "minimal_retrain", - "schedule_name": "test_schedule", - "query": "test_query", - "schema": "test_schema", - "table_name": "test_table", + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'workflow_name': 'minimal_retrain', + 'schedule_name': 'test_schedule', + 'query': 'test_query', + 'schema': 'test_schema', + 'table_name': 'test_table', + 'model_config': { + 'target': 'test_target', + 'transform_flavor': 'test_transform_flavor', + 'predict_flavor': 'test_predict_flavor', + }, } workflow_mock.execute_activity_method = AsyncMock( - return_value={ - "data1": "1", - "data2": "2", - } + side_effect=[ + {'success': True, 'object_key': 'test_object_key'}, + {'success': True, 'experiment': 'test_experiment'}, + { + 'success': True, + 'version': 'test_version', + 'mlflow_run_id': 'test_mlflow_run_id', + 'mlflow_experiment_id': 'test_mlflow_experiment_id', + }, + {'report': 'test_report'}, + ] ) await minimal_retrain.run(input_data) - workflow_mock.execute_local_activity_method.assert_has_calls( + workflow_mock.execute_activity_method.assert_has_calls( [ call( - Activities.load_custom_query, + Activities.query_to_minio, { **metadata, - "query": input_data["query"], - 'datetime_columns': input_data.get('datetime_columns', []) + 'query': input_data['query'], + 'datetime_columns': input_data.get('datetime_columns', []), + 'model_name': input_data['model_name'], + 'object_prefix': f'retrain_datasets/{input_data["model_name"]}/data', }, retry_policy=ANY, - start_to_close_timeout=ANY + start_to_close_timeout=ANY, ) ] ) - workflow_mock.execute_activity_method.assert_has_calls([ - call( - Activities.retrain_model, - { - **metadata, - 'data': workflow_mock.execute_local_activity_method.return_value, - 'model_name': input_data['model_name'], - }, - retry_policy=ANY, - start_to_close_timeout=ANY - ) - ]) + workflow_mock.execute_activity_method.assert_has_calls( + [ + call( + Activities.retrain_model, + { + **metadata, + 'object_key': 'test_object_key', + 'model_name': input_data['model_name'], + 'model_config': input_data['model_config'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) - workflow_mock.execute_activity_method.assert_has_calls([ - call( - Activities.update_production_model, - { - **metadata, - 'model_name': input_data['model_name'], - 'model_id': input_data['model_id'], - **workflow_mock.execute_activity_method.return_value, - }, - retry_policy=ANY, - start_to_close_timeout=ANY - ) - ]) + workflow_mock.execute_activity_method.assert_has_calls( + [ + call( + Activities.update_production_model, + { + **metadata, + 'model_name': input_data['model_name'], + 'success': True, + 'experiment': 'test_experiment', + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) - workflow_mock.execute_activity_method.assert_has_calls([ - call( - Activities.export_data_to_postgres, + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.format_retrain_report, + { + **metadata, + 'experiment_response': {'success': True, 'experiment': 'test_experiment'}, + 'model_name': input_data['model_name'], + 'update_report': { + 'success': True, + 'version': 'test_version', + 'mlflow_run_id': 'test_mlflow_run_id', + 'mlflow_experiment_id': 'test_mlflow_experiment_id', + }, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + workflow_mock.execute_activity_method.assert_has_calls( + [ + call( + Activities.export_data_to_postgres, + { + **metadata, + 'data': workflow_mock.execute_local_activity_method.return_value, + 'schema': input_data['schema'], + 'table_name': input_data['table_name'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + +@mark.asyncio +@patch('laborious.workflows.minimal_retrain.workflow', new_callable=AsyncMock) +async def test_run_storage_fail(workflow_mock: AsyncMock, minimal_retrain: MinimalRetrain): + input_data = { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'workflow_name': 'minimal_retrain', + 'schedule_name': 'test_schedule', + 'query': 'test_query', + 'schema': 'test_schema', + 'table_name': 'test_table', + 'model_config': { + 'target': 'test_target', + 'transform_flavor': 'test_transform_flavor', + 'predict_flavor': 'test_predict_flavor', + }, + } + + workflow_mock.execute_activity_method = AsyncMock( + side_effect=[ + {'success': False, 'object_key': 'test_object_key'}, + {'success': True, 'experiment': 'test_experiment'}, { - **metadata, - 'data': workflow_mock.execute_activity_method.return_value, - 'schema': input_data['schema'], - 'table_name': input_data['table_name'], + 'success': True, + 'version': 'test_version', + 'mlflow_run_id': 'test_mlflow_run_id', + 'mlflow_experiment_id': 'test_mlflow_experiment_id', }, - retry_policy=ANY, - start_to_close_timeout=ANY - ) - ]) + {'report': 'test_report'}, + ] + ) + + await minimal_retrain.run(input_data) + + workflow_mock.execute_activity_method.assert_called_once_with( + Activities.query_to_minio, + { + **metadata, + 'query': input_data['query'], + 'datetime_columns': input_data.get('datetime_columns', []), + 'model_name': input_data['model_name'], + 'object_prefix': f'retrain_datasets/{input_data["model_name"]}/data', + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + + workflow_mock.execute_local_activity_method.assert_not_called() + + +@mark.asyncio +@patch('laborious.workflows.minimal_retrain.workflow', new_callable=AsyncMock) +async def test_run_fail_retrain(workflow_mock: AsyncMock, minimal_retrain: MinimalRetrain): + input_data = { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'workflow_name': 'minimal_retrain', + 'schedule_name': 'test_schedule', + 'query': 'test_query', + 'schema': 'test_schema', + 'table_name': 'test_table', + 'model_config': { + 'target': 'test_target', + 'transform_flavor': 'test_transform_flavor', + 'predict_flavor': 'test_predict_flavor', + }, + } + + workflow_mock.execute_activity_method = AsyncMock( + side_effect=[ + {'success': True, 'object_key': 'test_object_key'}, + {'success': False, 'experiment': 'test_experiment'}, + { + 'success': True, + 'version': 'test_version', + 'mlflow_run_id': 'test_mlflow_run_id', + 'mlflow_experiment_id': 'test_mlflow_experiment_id', + }, + {'report': 'test_report'}, + ] + ) + + await minimal_retrain.run(input_data) + + workflow_mock.execute_activity_method.assert_has_calls( + [ + call( + Activities.query_to_minio, + { + **metadata, + 'query': input_data['query'], + 'datetime_columns': input_data.get('datetime_columns', []), + 'model_name': input_data['model_name'], + 'object_prefix': f'retrain_datasets/{input_data["model_name"]}/data', + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + workflow_mock.execute_activity_method.assert_has_calls( + [ + call( + Activities.retrain_model, + { + **metadata, + 'object_key': 'test_object_key', + 'model_name': input_data['model_name'], + 'model_config': input_data['model_config'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.format_retrain_report, + { + **metadata, + 'experiment_response': {'success': False, 'experiment': 'test_experiment'}, + 'model_name': input_data['model_name'], + 'update_report': {}, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + workflow_mock.execute_activity_method.assert_has_calls( + [ + call( + Activities.export_data_to_postgres, + { + **metadata, + 'data': workflow_mock.execute_local_activity_method.return_value, + 'schema': input_data['schema'], + 'table_name': input_data['table_name'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + assert workflow_mock.execute_activity_method.call_count == 3 + assert workflow_mock.execute_local_activity_method.call_count == 1 diff --git a/tests/laborious/workflows/test_predictions_batch.py b/tests/laborious/workflows/test_predictions_batch.py index 90d7d21..62cc43c 100644 --- a/tests/laborious/workflows/test_predictions_batch.py +++ b/tests/laborious/workflows/test_predictions_batch.py @@ -1,5 +1,7 @@ -from unittest.mock import AsyncMock, call, patch, ANY +from unittest.mock import ANY, AsyncMock, call, patch + from pytest import fixture, mark + from laborious.activities.activities import Activities from laborious.workflows.predictions_batch import PredictionsBatch @@ -10,11 +12,11 @@ def predictions_batch() -> PredictionsBatch: metadata = { - "metadata": { - "model_id": "test_model_id", - "model_name": "test_model", - "workflow_name": "predictions_batch", - "schedule_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'workflow_name': 'predictions_batch', + 'schedule_name': 'test_schedule', }, } @@ -22,9 +24,7 @@ metadata = { @mark.asyncio @patch('laborious.workflows.predictions_batch.workflow', new_callable=AsyncMock) async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch): - workflow_mock.execute_local_activity_method.return_value = { - 'data': 'test_data' - } + workflow_mock.execute_local_activity_method.return_value = {'data': 'test_data'} input_data = { 'schedule_name': 'test_schedule', 'model_name': 'test_model', @@ -35,25 +35,25 @@ async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch 'opc_output_config': 'test_opc_output_config', 'datetime_columns': ['timestamp', 'created_at'], 'prediction_store_policy': 'erl:1', - 'model_config': { - 'retention': '30' - } + 'model_config': {'retention': '30'}, } await predictions_batch.run(input_data) - workflow_mock.execute_local_activity_method.assert_has_calls([ - call( - Activities.load_custom_query, - { - **metadata, - 'query': input_data['query'], - 'datetime_columns': input_data.get('datetime_columns', []) - }, - retry_policy=ANY, - start_to_close_timeout=ANY - ) - ]) + workflow_mock.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.load_custom_query, + { + **metadata, + 'query': input_data['query'], + 'datetime_columns': input_data.get('datetime_columns', []), + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) prediction_input = { 'metadata': metadata, 'data': {'data': 'test_data'}, @@ -61,28 +61,19 @@ async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch 'table_name': input_data['table_name'], 'model_id': input_data['model_id'], 'model_name': input_data['model_name'], - 'input_filters': input_data.get('input_filters', { - 'EMPTY_DATA': { - 'POLICY': 'STOP' - } - }), - 'mlflow_transform_filters': input_data.get('mlflow_transform_filters', { - 'API_ERROR': { - 'POLICY': 'STOP' - } - }), - 'mlflow_predict_filters': input_data.get('mlflow_predict_filters', { - 'API_ERROR': { - 'POLICY': 'STOP' - } - }), + 'input_filters': input_data.get('input_filters', {'EMPTY_DATA': {'POLICY': 'STOP'}}), + 'mlflow_transform_filters': input_data.get( + 'mlflow_transform_filters', {'API_ERROR': {'POLICY': 'STOP'}} + ), + 'mlflow_predict_filters': input_data.get( + 'mlflow_predict_filters', {'API_ERROR': {'POLICY': 'STOP'}} + ), 'model_config': input_data.get('model_config', {}), 'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']), 'opc_output_config': input_data.get('opc_output_config', {}), - 'prediction_store_policy': input_data.get('prediction_store_policy', 'erl:1') + 'prediction_store_policy': input_data.get('prediction_store_policy', 'erl:1'), } - workflow_mock.execute_child_workflow.assert_has_calls([ - call( - 'prediction_process', prediction_input) - ]) + workflow_mock.execute_child_workflow.assert_has_calls( + [call('prediction_process', prediction_input)] + ) diff --git a/validate.sh b/validate.sh new file mode 100755 index 0000000..8728312 --- /dev/null +++ b/validate.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Model Manager Code Validation Script +# This script runs all code quality checks before committing or deploying + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Model Manager - Code Validation Suite ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}" +echo "" + +# Check if virtual environment is activated +if [[ -z "${VIRTUAL_ENV}" ]] && [[ -z "${CONDA_DEFAULT_ENV}" ]]; then + echo -e "${YELLOW}⚠️ Warning: No virtual environment detected${NC}" + echo -e "${YELLOW} Consider activating your venv/conda environment${NC}" + echo "" +fi + +# Function to run a validation step +run_step() { + local step_name=$1 + local step_command=$2 + + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${BLUE}▶ ${step_name}${NC}" + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + + if eval "$step_command"; then + echo -e "${GREEN}✅ ${step_name} - PASSED${NC}" + echo "" + return 0 + else + echo -e "${RED}❌ ${step_name} - FAILED${NC}" + echo "" + return 1 + fi +} + +# Track failures +FAILED_STEPS=() + +# Step 1: Code Formatting Check (Ruff) +if ! run_step "1. Code Formatting (Ruff)" "ruff format --check laborious/ tests/"; then + FAILED_STEPS+=("Code Formatting") +fi + +# Step 2: Linting (Ruff) +if ! run_step "2. Code Linting (Ruff)" "ruff check laborious/ tests/"; then + FAILED_STEPS+=("Linting") +fi + +# Step 3: Type Checking (mypy) +if ! run_step "3. Type Checking (mypy)" "mypy laborious/"; then + FAILED_STEPS+=("Type Checking") +fi + +# Step 4: Security Analysis (Bandit) +if ! run_step "4. Security Analysis (Bandit)" "bandit -r laborious/ -ll -q"; then + FAILED_STEPS+=("Security Analysis") +fi + +# Step 5: Unit Tests (pytest) +if ! run_step "5. Unit Tests (pytest)" "pytest tests/ --cov=laborious --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=80 -q"; then + FAILED_STEPS+=("Unit Tests") +fi + +# Summary +echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Validation Summary ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}" +echo "" + +if [ ${#FAILED_STEPS[@]} -eq 0 ]; then + echo -e "${GREEN}✅ All validation checks passed!${NC}" + echo -e "${GREEN} Your code is ready for commit/deployment.${NC}" + echo "" + exit 0 +else + echo -e "${RED}❌ Validation failed for the following steps:${NC}" + for step in "${FAILED_STEPS[@]}"; do + echo -e "${RED} • ${step}${NC}" + done + echo "" + echo -e "${YELLOW}💡 Tips:${NC}" + echo -e "${YELLOW} • Run 'ruff format laborious/ tests/' to auto-fix formatting${NC}" + echo -e "${YELLOW} • Run 'ruff check --fix laborious/ tests/' to auto-fix linting issues${NC}" + echo -e "${YELLOW} • Review mypy errors and add type hints where needed${NC}" + echo -e "${YELLOW} • Check bandit warnings for security issues${NC}" + echo -e "${YELLOW} • Fix failing tests or improve test coverage${NC}" + echo "" + exit 1 +fi \ No newline at end of file From f0fb9b854ebfc01cf01ae6ed08b10f17d5358cd6 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 15 Oct 2025 16:50:38 -0300 Subject: [PATCH 43/52] SIENTIAPDE-1231 Enhance validation script and refactor code in various modules - Updated the validation script to include automatic code formatting and linting fixes using Ruff. - Removed the `clean_tmp_files` method from the Gates class to streamline functionality. - Simplified conditional checks in the OpcRepository for better clarity and error handling. - Added model ID to the minimal retrain workflow for improved tracking. - Introduced new test cases for error handling in MLFlow and storage operations, ensuring robustness in repository interactions. --- laborious/activities/gates.py | 18 ---- .../utils/repository/model_repository.py | 2 +- laborious/utils/repository/opc_repository.py | 15 +-- laborious/workflows/minimal_retrain.py | 1 + tests/laborious/activities/test_gates.py | 96 ++++++++++++++++++- tests/laborious/activities/test_mlflow.py | 23 ++++- tests/laborious/activities/test_storage.py | 12 ++- .../utils/filters/test_conditional_filters.py | 7 ++ .../utils/repository/test_minio_repository.py | 5 + .../utils/repository/test_model_repository.py | 57 +++++++++++ .../utils/repository/test_opc_repository.py | 24 +++++ .../laborious/utils/test_connectors_config.py | 31 ++++++ .../workflows/test_minimal_retrain.py | 2 + validate.sh | 4 +- 14 files changed, 261 insertions(+), 36 deletions(-) diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index 0653161..8cf4afa 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -3,8 +3,6 @@ from temporalio import activity, workflow with workflow.unsafe.imports_passed_through(): import traceback from collections.abc import Callable, Mapping - from os import path - from shutil import rmtree from typing import Any from pandas import DataFrame @@ -628,19 +626,3 @@ class Gates(BaseActivity): ).observe(response_time) self.info(f'Metrics written for model {metadata["model_name"]}', metadata) - - @activity.defn(name='clean_tmp_files') - async def clean_tmp_files(self, input_data: dict[str, Any]): - """ - Clean temporary files in the tmp directory. - """ - model_name = input_data['model_name'] - metadata = input_data['metadata'] - self.info(f'Cleaning tmp files for model {model_name}...', metadata) - - if path.exists(f'tmp/retrain_data/{model_name}'): - rmtree(f'tmp/retrain_data/{model_name}') - if path.exists(f'tmp/artifacts/{model_name}'): - rmtree(f'tmp/artifacts/{model_name}') - - self.info('Tmp files cleaned', metadata) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index d84f527..3e786c7 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -304,7 +304,7 @@ class MLFlowRepository: if model_type == 'predict': model = self.load_predict_model(model_name, flavor) - elif model_type == 'transform': + else: model = self.load_transform_model(model_name, flavor) return model, artifact_path diff --git a/laborious/utils/repository/opc_repository.py b/laborious/utils/repository/opc_repository.py index 072274a..be4edb5 100644 --- a/laborious/utils/repository/opc_repository.py +++ b/laborious/utils/repository/opc_repository.py @@ -92,14 +92,11 @@ class OpcRepository: - Session Timeout: 10,000,000 ms """ - if not all([self.cert_path, self.private_key_path]): + if self.cert_path is None or self.private_key_path is None: raise ValueError( 'Certificate and private key paths must be provided for secure connection.' ) - if self.cert_path is None or self.private_key_path is None: - raise ValueError('Certificate and private key paths cannot be None') - cert = Path(self.cert_path) private_key = Path(self.private_key_path) server_cert = Path(self.server_cert_path) if self.server_cert_path else None @@ -316,14 +313,8 @@ class OpcRepository: start_time = time.time() try: - if self.client is None: - return False, { - 'notification_id': f'OPC_WRITE_GET_NODE_ERROR_{self.id}', - 'message': 'Client is not initialized', - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - } - node_obj = self.client.get_node(node) + # ignored because self.validate_connection is called before, so we know self.client is not None + node_obj = self.client.get_node(node) # type: ignore[union-attr] except Exception as e: trace = traceback.format_exc() logger.custom_error(trace, metadata.get('schedule_name', 'N/A')) diff --git a/laborious/workflows/minimal_retrain.py b/laborious/workflows/minimal_retrain.py index 5497432..02878cf 100644 --- a/laborious/workflows/minimal_retrain.py +++ b/laborious/workflows/minimal_retrain.py @@ -116,6 +116,7 @@ class MinimalRetrain: **metadata, 'experiment_response': experiment_response, 'model_name': model_name, + 'model_id': input_data['model_id'], 'update_report': update_report, }, retry_policy=retry_policy, diff --git a/tests/laborious/activities/test_gates.py b/tests/laborious/activities/test_gates.py index 1a376e2..8d8822e 100644 --- a/tests/laborious/activities/test_gates.py +++ b/tests/laborious/activities/test_gates.py @@ -117,6 +117,24 @@ async def test_input_gate_with_filter(gates_activity): gates_activity.debug.assert_called() +@mark.asyncio +async def test_input_gate_with_filter_not_caught(gates_activity): + # Arrange + input_data = { + **metadata, + 'filters': {'EMPTY_DATA': {'policy': 'STOP', 'config': {}}}, + 'data': {'value': [1, 2, 3]}, + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], + } + + # Act + result = await gates_activity.input_gate(input_data) + + # Assert + assert result == (None, 0, '') + gates_activity.debug.assert_called() + + @mark.asyncio async def test_mlflow_response_gate_invalid_filter(gates_activity): # Arrange @@ -210,13 +228,38 @@ async def test_mlflow_response_gate_with_filter(gates_activity): gates_activity.send_notification.assert_called() +@mark.asyncio +async def test_mlflow_response_gate_with_filter_not_caught(gates_activity): + # Arrange + input_data = { + **metadata, + 'filters': {'API_ERROR': {'policy': 'STOP'}}, + 'data': { + 'success': True, + 'content': {'message': 'success'}, + }, + 'type': 'test', + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], + } + + # Act + result = await gates_activity.mlflow_response_gate(input_data) + + # Assert + assert result == (None, 0, '') + gates_activity.debug.assert_called() + + @mark.asyncio async def test_mlflow_content_gate_invalid_filter(gates_activity): # Arrange input_data = { **metadata, 'filters': {'INVALID_FILTER': {'POLICY': 'STOP'}}, - 'data': {'value': [1, 2, 3]}, + 'data': { + 'success': True, + 'content': {'message': 'success'}, + }, 'type': 'test', 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], } @@ -304,6 +347,25 @@ async def test_mlflow_content_gate_with_filter(gates_activity): gates_activity.send_notification.assert_called() +@mark.asyncio +async def test_mlflow_content_gate_with_filter_not_caught(gates_activity): + # Arrange + input_data = { + **metadata, + 'filters': {'API_ERROR': {'POLICY': 'STOP'}}, + 'data': {'content': {'message': 'success'}}, + 'type': 'test', + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], + } + + # Act + result = await gates_activity.mlflow_content_gate(input_data) + + # Assert + assert result == (None, 0, '') + gates_activity.debug.assert_called() + + def test_get_prediction_store_policy_invalid_policy(gates_activity): # Arrange prediction_store_policy = 'INVALID_POLICY' @@ -506,6 +568,38 @@ async def test_format_default_prediction(gates_activity): gates_activity.debug.assert_called() +@mark.asyncio +async def test_format_retrain_report(gates_activity): + # Arrange + input_data = { + **metadata, + 'experiment_response': { + 'success': True, + 'timestamp': '2023-05-26 11:12:27', + 'message': 'success', + }, + 'update_report': { + 'version': '1.0.0', + 'mlflow_run_id': 'test_mlflow_run_id', + 'mlflow_experiment_id': 'test_mlflow_experiment_id', + }, + 'model_id': 'test_model', + 'model_name': 'test_model', + } + + # Act + result = await gates_activity.format_retrain_report(input_data) + + # Assert + assert result['model_id'] == {0: 'test_model'} + assert result['model_name'] == {0: 'test_model'} + assert result['timestamp'] == {0: '2023-05-26 11:12:27'} + assert result['status'] == {0: 'success'} + assert result['version'] == {0: '1.0.0'} + assert result['mlflow_run_id'] == {0: 'test_mlflow_run_id'} + assert result['mlflow_experiment_id'] == {0: 'test_mlflow_experiment_id'} + + @mark.asyncio async def test_get_last_timestamp_with_data(gates_activity): # Arrange diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index cfb66a1..a9f54f8 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -1,7 +1,7 @@ from unittest.mock import ANY, MagicMock, call, patch import numpy as np -from pytest import fixture, mark +from pytest import fixture, mark, raises from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ @@ -396,6 +396,27 @@ async def test_retrain_model_data_error(mlflow): } +@mark.asyncio +async def test_retrain_model_data_error_no_minio_repository(mlflow): + mlflow.minio_repository = None + + with raises(ValueError) as e: + await mlflow.retrain_model( + { + **metadata, + 'object_key': 'test_object_key', + 'model_name': 'test_model', + 'model_config': { + 'target': 'target', + 'transform_flavor': 'sklearn', + 'predict_flavor': 'pyfunc', + }, + } + ) + + assert str(e.value) == 'Minio repository not initialized' + + @mark.asyncio async def test_update_production_model(mlflow): input_data = { diff --git a/tests/laborious/activities/test_storage.py b/tests/laborious/activities/test_storage.py index 87c28f0..76b83e5 100644 --- a/tests/laborious/activities/test_storage.py +++ b/tests/laborious/activities/test_storage.py @@ -1,7 +1,7 @@ import datetime from unittest.mock import ANY, AsyncMock, MagicMock, patch -from pytest import fixture, mark +from pytest import fixture, mark, raises from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.postgres import Postgres @@ -135,6 +135,16 @@ def test___init___done_repository(mock_minio_repository, storage): assert storage.minio_repository is not None +@mark.asyncio +async def test_query_to_minio_minio_repository_not_initialized(storage): + storage.minio_repository = None + + with raises(ValueError) as e: + await storage.query_to_minio({}) + + assert str(e.value) == 'Minio repository not initialized' + + @mark.asyncio async def test_query_to_minio_not_data(storage): storage.load_custom_query = AsyncMock(return_value=None) diff --git a/tests/laborious/utils/filters/test_conditional_filters.py b/tests/laborious/utils/filters/test_conditional_filters.py index de25b7b..8a35801 100644 --- a/tests/laborious/utils/filters/test_conditional_filters.py +++ b/tests/laborious/utils/filters/test_conditional_filters.py @@ -16,6 +16,13 @@ def test_filter_specific_variables_null_values(): ) +def test_filter_specific_variables_null_values_with_empty_data(): + assert ( + filter_specific_variables_null_values(DataFrame(), config={'variables': ['variable2']}) + is False + ) + + def test_filter_specific_variables_null_values_with_null_values(): assert ( filter_specific_variables_null_values( diff --git a/tests/laborious/utils/repository/test_minio_repository.py b/tests/laborious/utils/repository/test_minio_repository.py index ec279d2..7883c1c 100644 --- a/tests/laborious/utils/repository/test_minio_repository.py +++ b/tests/laborious/utils/repository/test_minio_repository.py @@ -61,6 +61,11 @@ def minio_repository(mock_boto3, mock_config): ) +def test_close(minio_repository): + minio_repository.close() + minio_repository.s3_client.close.assert_called_once() + + def test_ensure_bucket_exists_bucket_exists(minio_repository): assert minio_repository.ensure_bucket_exists({}) is True diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 0e77bdf..a437653 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -173,6 +173,11 @@ def test_get_experiment_none_not_create(mlflow, mlflow_repository): assert str(e) == 'Experiment test not found' +def test_get_model_params(mlflow, mlflow_repository): + output = mlflow_repository.get_model_params('test') + assert output == mlflow.get_run.return_value.data.params + + @patch('laborious.utils.repository.model_repository.path') @patch('laborious.utils.repository.model_repository.rmtree') @patch('laborious.utils.repository.model_repository.makedirs') @@ -202,6 +207,35 @@ def test_download_artifacts_success(makedirs, rmtree, path, mlflow_repository): assert output == mlflow_repository.client.download_artifacts.return_value +@patch('laborious.utils.repository.model_repository.path') +@patch('laborious.utils.repository.model_repository.rmtree') +@patch('laborious.utils.repository.model_repository.makedirs') +def test_download_artifacts_success_path_false(makedirs, rmtree, path, mlflow_repository): + mlflow_repository.get_model_run_id = MagicMock(return_value='test') + + path.exists.return_value = False + + output = mlflow_repository.dowload_artifacts('test', 'path') + + mlflow_repository.get_model_run_id.assert_called_once_with( + model_name='test', stage='Production' + ) + + path.join.assert_called_once_with('./tmp/artifacts/test', 'path') + + path.exists.assert_called_once_with(path.join.return_value) + + rmtree.assert_not_called() + + makedirs.assert_called_once_with('./tmp/artifacts/test', exist_ok=True) + + mlflow_repository.client.download_artifacts.assert_called_once_with( + mlflow_repository.get_model_run_id.return_value, 'path', './tmp/artifacts/test' + ) + + assert output == mlflow_repository.client.download_artifacts.return_value + + def test_get_experiment_error(mlflow, mlflow_repository): mlflow.get_experiment_by_name.return_value = None @@ -383,6 +417,15 @@ valid_cases = [ }, ['2025-01-01 12:00:00+0000', '2025-01-02 12:00:00+0000'], ), + ( + { + 'value': { + datetime(2025, 1, 1, 12, 0, 0, tzinfo=None): 1, + datetime(2025, 1, 2, 12, 0, 0, tzinfo=None): 2, + } + }, + ['2025-01-01 12:00:00+0000', '2025-01-02 12:00:00+0000'], + ), ( { 'value': { @@ -987,6 +1030,20 @@ def test_retrain_model(mlflow_repository): } +def test_retrain_model_error(mlflow_repository): + data = MagicMock() + model_name = 'test' + model_config = {'target': 'target', 'transform_flavor': 'sklearn', 'predict_flavor': 'pyfunc'} + mlflow_repository.get_model_run_id = MagicMock(side_effect=Exception('error')) + output = mlflow_repository.retrain_model(data, model_name, model_config, metadata['metadata']) + assert output == { + 'success': False, + 'experiment': None, + 'message': 'Error retraining model test: error', + 'traceback': ANY, + } + + def test_update_production_model(mlflow_repository): experiment = {'run_id': '0', 'experiment_id': '0'} model_name = 'test' diff --git a/tests/laborious/utils/repository/test_opc_repository.py b/tests/laborious/utils/repository/test_opc_repository.py index a0048e8..6ec680d 100644 --- a/tests/laborious/utils/repository/test_opc_repository.py +++ b/tests/laborious/utils/repository/test_opc_repository.py @@ -86,6 +86,15 @@ async def test_set_security_missing_certificates(opc_repository): assert str(e) == 'Certificate and private key paths must be provided for secure connection.' +@pytest.mark.asyncio +async def test_set_security_missing_client(opc_repository): + opc_repository.client = None + try: + await opc_repository.set_security() + except ValueError as e: + assert str(e) == 'Client must be initialized before setting security' + + @pytest.mark.asyncio async def test_connect_with_security(opc_repository, mock_client): opc_repository.try_connect = AsyncMock(return_value=(True, {})) @@ -137,6 +146,21 @@ async def test_try_connect_fail(opc_repository): assert error_data['attachment_content'] is not None +@pytest.mark.asyncio +async def test_try_connect_no_client(opc_repository): + opc_repository.client = None + result = await opc_repository.try_connect() + assert result == ( + False, + { + 'notification_id': f'OPC_CONNECTION_ERROR_{opc_repository.id}', + 'message': 'Client is not initialized', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + }, + ) + + @pytest.mark.asyncio async def test_disconnect(opc_repository, mock_client): opc_repository.client = mock_client diff --git a/tests/laborious/utils/test_connectors_config.py b/tests/laborious/utils/test_connectors_config.py index cf2a6b8..80516ad 100644 --- a/tests/laborious/utils/test_connectors_config.py +++ b/tests/laborious/utils/test_connectors_config.py @@ -1,6 +1,7 @@ from os import environ from laborious.utils.connectors_config import ( + build_minio_config, build_mlflow_config, build_mongodb_config, build_opc_config, @@ -162,3 +163,33 @@ def test_build_mongo_db_config_with_defaults(): 'database_name': 'sientia', 'ttl_index_seconds': 3600, } + + +def test_build_minio_config_with_env_vars(): + environ['MINIO_ENDPOINT_URL'] = 'http://test-host' + environ['MINIO_ACCESS_KEY'] = 'test-key' + environ['MINIO_SECRET_KEY'] = 'test-secret' + environ['MINIO_REGION_NAME'] = 'test-region' + environ['MINIO_DEFAULT_BUCKET'] = 'test-bucket' + assert build_minio_config() == { + 'endpoint_url': 'http://test-host', + 'access_key': 'test-key', + 'secret_key': 'test-secret', + 'region_name': 'test-region', + 'default_bucket': 'test-bucket', + } + + +def test_build_minio_config_with_defaults(): + environ.pop('MINIO_ENDPOINT_URL', None) + environ.pop('MINIO_ACCESS_KEY', None) + environ.pop('MINIO_SECRET_KEY', None) + environ.pop('MINIO_REGION_NAME', None) + environ.pop('MINIO_DEFAULT_BUCKET', None) + assert build_minio_config() == { + 'endpoint_url': 'http://localhost:9000', + 'access_key': 'minioadmin', + 'secret_key': 'minioadmin', + 'region_name': 'us-east-1', + 'default_bucket': 'laborious', + } diff --git a/tests/laborious/workflows/test_minimal_retrain.py b/tests/laborious/workflows/test_minimal_retrain.py index c088702..bcbad2d 100644 --- a/tests/laborious/workflows/test_minimal_retrain.py +++ b/tests/laborious/workflows/test_minimal_retrain.py @@ -112,6 +112,7 @@ async def test_run(workflow_mock: AsyncMock, minimal_retrain: MinimalRetrain): **metadata, 'experiment_response': {'success': True, 'experiment': 'test_experiment'}, 'model_name': input_data['model_name'], + 'model_id': input_data['model_id'], 'update_report': { 'success': True, 'version': 'test_version', @@ -267,6 +268,7 @@ async def test_run_fail_retrain(workflow_mock: AsyncMock, minimal_retrain: Minim **metadata, 'experiment_response': {'success': False, 'experiment': 'test_experiment'}, 'model_name': input_data['model_name'], + 'model_id': input_data['model_id'], 'update_report': {}, }, retry_policy=ANY, diff --git a/validate.sh b/validate.sh index 8728312..aeed301 100755 --- a/validate.sh +++ b/validate.sh @@ -47,12 +47,12 @@ run_step() { FAILED_STEPS=() # Step 1: Code Formatting Check (Ruff) -if ! run_step "1. Code Formatting (Ruff)" "ruff format --check laborious/ tests/"; then +if ! run_step "1. Code Formatting (Ruff)" "ruff format laborious/ tests/ && ruff format --check laborious/ tests/"; then FAILED_STEPS+=("Code Formatting") fi # Step 2: Linting (Ruff) -if ! run_step "2. Code Linting (Ruff)" "ruff check laborious/ tests/"; then +if ! run_step "2. Code Linting (Ruff)" "ruff check --fix laborious/ tests/"; then FAILED_STEPS+=("Linting") fi From 3caa52d8b8356f7cd24a61abb2008556fe888f65 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 16 Oct 2025 09:31:23 -0300 Subject: [PATCH 44/52] SIENTIAPDE-1231 Update .gitignore and remove values.yaml for project cleanup - Added .ruff_cache/ and catboost_info/ to .gitignore to prevent tracking of temporary files and caches. - Deleted values.yaml to remove outdated configuration settings, streamlining the project structure. --- .gitignore | 3 +++ requirements-light.txt | 11 +++++++++++ 2 files changed, 14 insertions(+) create mode 100644 requirements-light.txt diff --git a/.gitignore b/.gitignore index 412222d..c347f55 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,6 @@ git_log tmp/ catboost_info/ + +.ruff_cache/ +.mypy_cache/ diff --git a/requirements-light.txt b/requirements-light.txt new file mode 100644 index 0000000..e6d4b6a --- /dev/null +++ b/requirements-light.txt @@ -0,0 +1,11 @@ +temporalio +psycopg2-binary +sqlalchemy +asyncua +redis +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.6 +prometheus-client +botocore +boto3 +s3fs +pyarrow From 358ad8b87165a01e29bcba0d40bdde623a35e5bc Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 16 Oct 2025 09:32:53 -0300 Subject: [PATCH 45/52] SIENTIAPDE-1231 --- .github/workflows/quality-gate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 9212f77..0d3ff14 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -30,12 +30,12 @@ jobs: owner: 'Aignosi' repositories: 'sientia-dataops-library,sientia-mlops-library' - - name: Prepare requirements.txt + - name: Prepare requirements-light.txt id: prepare-requirements run: | sed -e "s|git+ssh://git@github.com/|git+https://github.com/|g" \ -e "s|git@github.com:|git+https://github.com/|g" \ - requirements.txt > requirements_prepared.txt + requirements-light.txt > requirements_prepared.txt echo "PROCESSED_REQUIREMENTS_FILE=requirements_prepared.txt" >> $GITHUB_OUTPUT - name: Configure Git to use App Token From d8f400648880be7b70544adfbcde069ebc12e7c5 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 16 Oct 2025 09:49:56 -0300 Subject: [PATCH 46/52] SIENTIAPDE-1231 SIENTIAPDE-1231 Add mlflow to requirements-light.txt for enhanced model tracking capabilities --- requirements-light.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-light.txt b/requirements-light.txt index e6d4b6a..c145097 100644 --- a/requirements-light.txt +++ b/requirements-light.txt @@ -9,3 +9,4 @@ botocore boto3 s3fs pyarrow +mlflow \ No newline at end of file From f8397582d42a5d2235a43ff01c11cf2bcb405ea3 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 16 Oct 2025 10:06:42 -0300 Subject: [PATCH 47/52] SIENTIAPDE-1231 Refactor MinioRepository and MLFlowRepository for improved functionality and error handling - Updated `ensure_bucket_exists` method in MinioRepository to return None instead of a boolean, streamlining bucket existence checks. - Replaced hardcoded error messages in MLFlowRepository with a constant for better maintainability. - Adjusted column assignment in MLFlowRepository to use pd.Index for improved clarity. --- laborious/utils/repository/minio_repository.py | 5 +---- laborious/utils/repository/model_repository.py | 11 ++++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/laborious/utils/repository/minio_repository.py b/laborious/utils/repository/minio_repository.py index df20a06..b38af69 100644 --- a/laborious/utils/repository/minio_repository.py +++ b/laborious/utils/repository/minio_repository.py @@ -56,7 +56,7 @@ class MinioRepository: def close(self): self.s3_client.close() - def ensure_bucket_exists(self, metadata: dict[str, Any]) -> bool: + def ensure_bucket_exists(self, metadata: dict[str, Any]) -> None: """ Ensure the MinIO bucket exists; create it if necessary. """ @@ -64,13 +64,10 @@ class MinioRepository: try: self.logger.custom_info(f"Checking if bucket '{self.minio_bucket}' exists", metadata) self.s3_client.head_bucket(Bucket=self.minio_bucket) - return True except ClientError: self.logger.custom_info(f"Creating bucket '{self.minio_bucket}'", metadata) self.s3_client.create_bucket(Bucket=self.minio_bucket) - return True - def store_dataframe_as_parquet( self, dataframe: DataFrame, uri: str, object_name: str, metadata: dict[str, Any] ): diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 3e786c7..f9e783b 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -33,6 +33,7 @@ ARTIFACTS_PATH = './tmp/artifacts' TRANSFORMED_COMPRESSED_PATH = 'artifacts/training_transformer.pkl' PREDICTION_COMPRESSED_PATH = 'artifacts/stacking_model.pkl' +INVALID_FLAVOR_MESSAGE = "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'." def force_memory_release(logger: Logger): gc.collect() @@ -222,7 +223,7 @@ class MLFlowRepository: elif flavor == 'pytorch': model = mlflow.pytorch.load_model(model_uri) else: - raise ValueError("Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") + raise ValueError(INVALID_FLAVOR_MESSAGE) return model @@ -257,7 +258,7 @@ class MLFlowRepository: elif flavor == 'pytorch': model = mlflow.pytorch.load_model(model_uri) else: - raise ValueError("Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") + raise ValueError(INVALID_FLAVOR_MESSAGE) return model def download_model( @@ -628,7 +629,7 @@ class MLFlowRepository: # Aligns data with treated data indexes to get target variable aligned_data = data.loc[treated_data.index] aligned_series = aligned_data[target_name] - retrain_dataset = pd.merge( + retrain_dataset = pd.merge( # NOSONAR treated_data, aligned_series, left_index=True, right_index=True ) else: @@ -673,7 +674,7 @@ class MLFlowRepository: elif flavor == 'pytorch': mlflow.pytorch.log_model(model, model_type) else: - raise ValueError("Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'.") + raise ValueError(INVALID_FLAVOR_MESSAGE) def create_new_experiment( self, @@ -980,7 +981,7 @@ class MLFlowRepository: # predict_data.to_csv( # f"tmp/predicted_data_{model_name}.csv", index=True) - predict_data.columns = ['prediction'] + predict_data.columns = pd.Index(['prediction']) else: predict_data = pd.DataFrame(predict_data, columns=['prediction']) From 13644d5ebf6f9ce7a56dede8d5c20977c73d4576 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 16 Oct 2025 10:08:04 -0300 Subject: [PATCH 48/52] SIENTIAPDE-1231 Remove obsolete job and MLflow metadata files to streamline project structure and eliminate unused configurations. --- clean_job.yaml | 45 ----------------------------- mlruns/0/meta.yaml | 6 ---- mlruns/586524947870967910/meta.yaml | 6 ---- mlruns/models/test/meta.yaml | 5 ---- 4 files changed, 62 deletions(-) delete mode 100644 clean_job.yaml delete mode 100644 mlruns/0/meta.yaml delete mode 100644 mlruns/586524947870967910/meta.yaml delete mode 100644 mlruns/models/test/meta.yaml diff --git a/clean_job.yaml b/clean_job.yaml deleted file mode 100644 index 00422a1..0000000 --- a/clean_job.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: delete-old-rows - namespace: sientia -spec: - template: - spec: - containers: - - name: delete-old-rows - image: docker.io/bitnami/postgresql:16.2.0-debian-12-r10 - env: - - name: PGPASSWORD - value: "asidhsd@!#!@@!ASD!@#!ASDQ@#!FSDTRYJG#@@$#@%" - - name: PGUSER - value: temporal - - name: PGHOST - value: "paradedb-rw.paradedb.svc.cluster.local" - - name: PGDATABASE - value: "temporal_visibility" - command: - - "sh" - - "-c" - - | - # COMANDO CORRIGIDO - Excluir apenas workflows COMPLETED/FAILED antigos - # Preserva schedules (que ficam RUNNING) e workflows recentes - psql -h $PGHOST -U $PGUSER -d $PGDATABASE -c " - DELETE FROM public.executions_visibility - WHERE start_time < NOW() - INTERVAL '1 minutes' - AND status IN (2, 3, 4, 5, 7);" - - # Comando para executar VACUUM FULL após a exclusão - psql -h $PGHOST -U $PGUSER -d $PGDATABASE -c "VACUUM FULL public.executions_visibility;" - envFrom: - - secretRef: - name: postgres-credentials - restartPolicy: Never - backoffLimit: 0 - ttlSecondsAfterFinished: 3600 - -# kubectl apply -f clean_job.yaml -n sientia - -# kubectl create secret generic postgres-credentials --from-literal=postgres-password=sientia --from-literal=postgres-username=sientia -n sientia4 - -# drop database temporal; drop database temporal_visibility; create database temporal owner temporal; create database temporal_visibility owner temporal; \ No newline at end of file diff --git a/mlruns/0/meta.yaml b/mlruns/0/meta.yaml deleted file mode 100644 index 2a5def3..0000000 --- a/mlruns/0/meta.yaml +++ /dev/null @@ -1,6 +0,0 @@ -artifact_location: file:///home/grezewave/Documents/projects/sientia/sientia-dataops-laborious_temporal/mlruns/0 -creation_time: 1760447041053 -experiment_id: '0' -last_update_time: 1760447041053 -lifecycle_stage: active -name: Default diff --git a/mlruns/586524947870967910/meta.yaml b/mlruns/586524947870967910/meta.yaml deleted file mode 100644 index 7221134..0000000 --- a/mlruns/586524947870967910/meta.yaml +++ /dev/null @@ -1,6 +0,0 @@ -artifact_location: file:///home/grezewave/Documents/projects/sientia/sientia-dataops-laborious_temporal/mlruns/586524947870967910 -creation_time: 1760447067255 -experiment_id: '586524947870967910' -last_update_time: 1760447067255 -lifecycle_stage: active -name: test diff --git a/mlruns/models/test/meta.yaml b/mlruns/models/test/meta.yaml deleted file mode 100644 index 9d225cc..0000000 --- a/mlruns/models/test/meta.yaml +++ /dev/null @@ -1,5 +0,0 @@ -aliases: {} -creation_timestamp: 1760447068191 -description: null -last_updated_timestamp: 1760447068191 -name: test From d59833a5e5281097758c870a9f9e99b5fb11b7fc Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 16 Oct 2025 10:08:34 -0300 Subject: [PATCH 49/52] SIENTIAPDE-1231 Update .gitignore to include mlruns and remove Dockerfile for project simplification --- .gitignore | 1 + simulator/Dockerfile | 30 ------------------------------ 2 files changed, 1 insertion(+), 30 deletions(-) delete mode 100644 simulator/Dockerfile diff --git a/.gitignore b/.gitignore index c347f55..7cd11b3 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,4 @@ catboost_info/ .ruff_cache/ .mypy_cache/ +mlruns/ \ No newline at end of file diff --git a/simulator/Dockerfile b/simulator/Dockerfile deleted file mode 100644 index d467676..0000000 --- a/simulator/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# syntax=docker/dockerfile:1.4 - -FROM python:3.11-slim - -# Enable use of SSH agent/socket -# This line enables SSH during build -# (don't forget the syntax header above) -RUN apt-get update && apt-get install -y git openssh-client && rm -rf /var/lib/apt/lists/* - -# Use build-time SSH mount for Git clone -# The SSH key will NOT remain in the image -# IMPORTANT: this block requires BuildKit -# and the --ssh flag during docker build - -# SSH config to skip host key check (safe in CI/local dev) -RUN mkdir -p /root/.ssh && echo "StrictHostKeyChecking no" > /root/.ssh/config - -WORKDIR /app - -# Clone using SSH -ARG GIT_REPO -ARG GIT_BRANCH=main - -# Mount SSH key just for this RUN -RUN --mount=type=ssh git clone --branch ${GIT_BRANCH} ${GIT_REPO} . - -# Install requirements if exists -RUN if [ -f requirements.txt ]; then pip install --no-cache-dir -r requirements.txt; fi - -CMD ["python", "server.py"] From de47820c4ad271802a4433af46e0f7348ea67326 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 16 Oct 2025 10:15:53 -0300 Subject: [PATCH 50/52] SIENTIAPDE-1231 Refactor tests and update model_repository.py for clarity and consistency - Added a blank line in model_repository.py for improved readability. - Adjusted formatting in test_mlflow.py to streamline assertions. - Updated ensure_bucket_exists method tests in test_minio_repository.py to reflect the new return value of None instead of True. --- laborious/utils/repository/model_repository.py | 3 ++- tests/laborious/activities/test_mlflow.py | 4 +--- tests/laborious/utils/repository/test_minio_repository.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index f9e783b..72707c1 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -35,6 +35,7 @@ PREDICTION_COMPRESSED_PATH = 'artifacts/stacking_model.pkl' INVALID_FLAVOR_MESSAGE = "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'." + def force_memory_release(logger: Logger): gc.collect() @@ -629,7 +630,7 @@ class MLFlowRepository: # Aligns data with treated data indexes to get target variable aligned_data = data.loc[treated_data.index] aligned_series = aligned_data[target_name] - retrain_dataset = pd.merge( # NOSONAR + retrain_dataset = pd.merge( # NOSONAR treated_data, aligned_series, left_index=True, right_index=True ) else: diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index 3d3a550..662f0ba 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -200,9 +200,7 @@ async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflo mock_to_datetime.assert_called_once_with( mock_dataframe.return_value.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ ) - mock_to_datetime.return_value.dt.strftime.assert_called_once_with( - DATETIME_FORMAT - ) + mock_to_datetime.return_value.dt.strftime.assert_called_once_with(DATETIME_FORMAT) mock_to_datetime.assert_called_once_with( mock_dataframe.return_value.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ diff --git a/tests/laborious/utils/repository/test_minio_repository.py b/tests/laborious/utils/repository/test_minio_repository.py index 7883c1c..bebf743 100644 --- a/tests/laborious/utils/repository/test_minio_repository.py +++ b/tests/laborious/utils/repository/test_minio_repository.py @@ -67,7 +67,7 @@ def test_close(minio_repository): def test_ensure_bucket_exists_bucket_exists(minio_repository): - assert minio_repository.ensure_bucket_exists({}) is True + assert minio_repository.ensure_bucket_exists({}) is None minio_repository.s3_client.head_bucket.assert_called_once_with(Bucket='test') @@ -77,7 +77,7 @@ def test_ensure_bucket_exists_bucket_not_exists_create_success(minio_repository) error_response={'Error': {'Code': '404'}}, operation_name='head_bucket' ) - assert minio_repository.ensure_bucket_exists({}) is True + assert minio_repository.ensure_bucket_exists({}) is None minio_repository.s3_client.head_bucket.assert_called_once_with(Bucket='test') minio_repository.s3_client.create_bucket.assert_called_once_with(Bucket='test') From 644a43093ac6107eaf7f5b2b743e5112e1885f5c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 16 Oct 2025 11:01:36 -0300 Subject: [PATCH 51/52] SIENTIAPDE-1231 Enhance README and repository utilities for clarity and functionality - Updated README.md to improve descriptions and structure, adding detailed sections for features, workflows, and architecture. - Enhanced MinioRepository with comprehensive docstrings for methods and class attributes, improving usability and documentation. - Refined MLFlowRepository with clearer method descriptions and improved logging for better observability and maintainability. --- README.md | 205 ++++++++-------- .../utils/repository/minio_repository.py | 58 ++++- .../utils/repository/model_repository.py | 218 ++++++++++-------- 3 files changed, 291 insertions(+), 190 deletions(-) diff --git a/README.md b/README.md index 500153c..9c1e85f 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,108 @@ # Sientia DataOps Laborious -A high-performance, scalable machine learning prediction system built on Temporal.io for industrial data processing and ML model inference. The Laborious system provides enterprise-grade ML model management, batch prediction processing, and real-time data export capabilities with comprehensive data quality validation and monitoring. +A comprehensive, Temporal-based ML orchestration system for industrial data processing and model inference. Laborious delivers enterprise-grade batch prediction, model management, optional real-time export (OPC), and automated retraining with strong data quality validation and observability. + +## 📑 Table of Contents + +- [Features](#features) + - [Core Functionality](#core-functionality) + - [Advanced Capabilities](#advanced-capabilities) + - [Development & Quality Assurance](#development--quality-assurance) +- [Architecture](#architecture) + - [Architecture Principles](#architecture-principles) + - [Key Components](#key-components) + - [Data Flow Architecture](#data-flow-architecture) + - [Security Architecture](#security-architecture) +- [Workflows](#workflows) + - [Predictions Batch Workflow](#1-predictions-batch-workflow-predictions_batchpy) + - [Prediction Process Workflow](#2-prediction-process-workflow-prediction_processpy) + - [Format and Export Prediction Workflow](#3-format-and-export-prediction-workflow-format_and_export_predictionpy) + - [Minimal Retrain Workflow](#4-minimal-retrain-workflow-minimal_retrainpy) +- [Installation & Setup](#installation--setup) + - [Prerequisites](#prerequisites) + - [Environment Setup](#environment-setup) + - [Temporal Namespace Setup](#temporal-namespace-setup) + - [Local Development Setup](#local-development-setup) +- [How to Run](#how-to-run) + - [Running the Laborious Application](#running-the-laborious-application) + - [Running Tests and Coverage](#running-tests-and-coverage) + - [Manual Test Execution](#manual-test-execution) + - [Manual Application Execution](#manual-application-execution) +- [Code Quality & Validation](#code-quality--validation) + - [Overview](#overview) + - [Validation Tools](#validation-tools) + - [Tools Installation](#tools-installation) + - [Complete Validation](#complete-validation) + - [Automatic Fixes](#automatic-fixes) + - [Configuration](#configuration) + - [CI/CD Integration](#cicd-integration) + - [Best Practices](#best-practices) +- [Testing](#testing) + - [Test Structure](#test-structure) + - [Test Execution](#test-execution) +- [Monitoring and Metrics](#monitoring-and-metrics) + - [Application Health Metrics](#application-health-metrics) + - [Prediction Operation Metrics](#prediction-operation-metrics) + - [OPC Export Metrics](#opc-export-metrics) + - [Data Quality Metrics](#data-quality-metrics) +- [Configuration](#configuration-1) + - [Environment Variables](#environment-variables) + - [OPC Configuration](#opc-configuration) + - [Workflow Configuration](#workflow-configuration) +- [Development](#development) + - [Project Structure](#project-structure) + - [Adding New Features](#adding-new-features) +- [Troubleshooting](#troubleshooting) + - [Common Issues](#common-issues) + - [Debug Mode](#debug-mode) +- [Performance Tuning](#performance-tuning) + - [Key Parameters](#key-parameters) + - [Scaling Considerations](#scaling-considerations) +- [Contributing](#contributing) + - [Code Quality Standards](#code-quality-standards) +- [License](#license) +- [Support](#support) ## Features ### Core Functionality -- **Batch Prediction Processing**: High-throughput ML model inference using MLFlow models -- **Temporal Workflow Orchestration**: Robust workflow management with automatic retry policies and fault tolerance -- **Data Quality Gates**: Configurable filtering for data validation, MLFlow API responses, and custom validation rules -- **Multi-Model Support**: Flexible ML model management with retention policies and versioning -- **Real-time Data Export**: PostgreSQL persistence and OPC server integration for industrial systems -- **Comprehensive Monitoring**: Prometheus metrics and detailed logging for operational visibility +- **Batch Prediction Processing**: High-throughput ML inference using MLFlow models +- **Temporal Workflow Orchestration**: Robust workflow management with retries and fault tolerance +- **Data Quality Gates**: Configurable filtering for input data and MLFlow API responses +- **Multi-Model Support**: Flexible model management with retention and versioning +- **Optional Real-time Export**: PostgreSQL persistence and OPC server integration for industrial systems +- **Comprehensive Monitoring**: Prometheus metrics and structured logging for observability ### Advanced Capabilities -- **Incremental Data Processing**: Timestamp-based data loading to avoid reprocessing +- **Incremental Data Processing**: Timestamp-based loading to avoid reprocessing - **Configurable Data Retention**: Model retention policies with automatic cleanup -- **Notification System**: Integrated alerting and notification management via MongoDB -- **Scalable Architecture**: Kubernetes-ready deployment with horizontal scaling support -- **Model Retraining**: Automated model retraining workflows with production model updates +- **Notification System**: Integrated alerting via MongoDB +- **Scalable Architecture**: Kubernetes-ready with horizontal scaling +- **Model Retraining**: Automated retraining workflows with production model updates + +### Development & Quality Assurance +- **Code Quality Tools**: Ruff (lint/format), mypy (types), Bandit (security) +- **Automated Validation**: `validate.sh` and CI quality gates +- **Comprehensive Testing**: pytest with async support and high coverage +- **Type Safety**: Static type checking with mypy +- **Coverage Visualization**: Coverage Gutters integration ## Architecture -The Laborious system uses a Temporal-based workflow architecture with clear separation of concerns and robust error handling. The architecture is designed for high availability, scalability, and operational excellence in production ML environments. - +Laborious uses a Temporal-based architecture with strong separation of concerns and defensive error handling for production ML. ### Architecture Principles #### 1. **Separation of Concerns** -- **Worker Layer**: Manages Temporal workers, task queues, and application lifecycle -- **Workflow Layer**: Orchestrates business logic and process coordination -- **Activity Layer**: Implements specific operations and external system interactions -- **Data Layer**: Handles data persistence, caching, and external service connections +- **Worker Layer**: Temporal workers, task queues, lifecycle +- **Workflow Layer**: Business orchestration and coordination +- **Activity Layer**: External system interactions and isolated operations +- **Data Layer**: Persistence, caching, connectors #### 2. **Fault Tolerance & Resilience** -- **Automatic Retry Policies**: Configurable retry strategies for transient failures -- **Circuit Breaker Pattern**: Prevents cascading failures in external service calls -- **Graceful Degradation**: System continues operating with reduced functionality -- **Comprehensive Error Handling**: Detailed error reporting and notification integration +- **Automatic Retry Policies** for transient failures +- **Graceful Degradation** and circuit breaking for dependencies +- **Detailed Error Handling** with notifications #### 3. **Scalability & Performance** - **Horizontal Scaling**: Multiple worker instances for load distribution @@ -53,101 +119,57 @@ The Laborious system uses a Temporal-based workflow architecture with clear sepa ### Key Components #### **Worker (`laborious/worker/worker.py`)** -- **Purpose**: Main application orchestrator managing Temporal workers and task queues -- **Responsibilities**: - - Temporal client initialization and connection management - - Worker lifecycle management and graceful shutdown - - Task queue configuration and load balancing - - Prometheus metrics server initialization - - Notification handler setup and configuration - - OPC server connection management -- **Key Features**: - - Automatic scaling with `PollerBehaviorAutoscaling` - - Health check endpoints for Kubernetes liveness/readiness probes - - Graceful shutdown with cleanup procedures - - Multi-instance deployment support - - Two dedicated task queues: `predictions_batch-queue` and `minimal_retrain-queue` +- Temporal client setup, worker lifecycle, task queues +- Metrics server initialization, notification handler setup +- Graceful shutdown and autoscaling-friendly behavior #### **Workflows (`laborious/workflows/`)** -- **PredictionsBatch**: Main entry point for batch prediction pipelines -- **PredictionProcess**: Core prediction pipeline with MLFlow integration -- **FormatAndExportPrediction**: Data formatting and export operations -- **MinimalRetrain**: Automated model retraining and deployment -- **Key Features**: - - Temporal workflow definitions with retry policies - - Child workflow orchestration and delegation - - Comprehensive error handling and recovery - - Configurable timeout and retry strategies +- `predictions_batch.py`: Batch prediction entry point +- `sub_workflows/prediction_process.py`: Core prediction pipeline +- `sub_workflows/format_and_export_prediction.py`: Formatting and export +- `minimal_retrain.py`: Automated model retraining and production update #### **Activities (`laborious/activities/`)** -- **Activities**: Main activity orchestrator combining all functionality through multiple inheritance -- **Gates**: Data quality validation and filtering mechanisms -- **MLFlow**: Model transformation and prediction operations -- **OPC**: Real-time data export to industrial OPC servers -- **Key Features**: - - Multiple inheritance pattern for unified activity interface - - Configurable filter policies and validation rules - - MLFlow model serving integration with configurable flavors - - OPC UA client with certificate-based authentication - - Comprehensive error handling and notification integration - - Support for multiple OPC servers with independent configurations +- `gates.py`: Data quality validation and filtering +- `mlflow.py`: Transform and predict operations +- `opc.py`: OPC UA export to industrial systems (optional) +- `activities.py`: Aggregates activity interfaces #### **Data Services (`laborious/utils/`)** -- **Connectors Config**: Environment variable-based configuration management -- **Repository**: Data access layer for MLFlow and OPC operations - - `model_repository.py`: MLFlow model operations and retraining - - `opc_repository.py`: OPC server communication and data writing -- **Filters**: Data quality validation and MLFlow response filtering - - `conditional_filters.py`: Input data validation filters - - `mlflow_filters.py`: MLFlow API response validation filters -- **Key Features**: - - Environment variable-based configuration with sensible defaults - - Connection pool management and optimization - - Security credential management - - Configuration validation and error handling - - Support for multiple OPC servers and MLFlow model flavors +- `connectors_config.py`: Env-driven configuration builders +- `repository/model_repository.py`: MLFlow operations and retraining +- `repository/opc_repository.py`: OPC communication and writes +- `filters/conditional_filters.py` and `filters/mlflow_filters.py` ### Data Flow Architecture #### **1. Batch Prediction Pipeline** ``` -Input Data (PostgreSQL) → Data Quality Gates → MLFlow Transform → -MLFlow Prediction → Response Validation → Export (PostgreSQL + OPC) +Input Data (PostgreSQL) → Data Quality Gates → MLFlow Transform → +MLFlow Prediction → Response Validation → Export (PostgreSQL [+ OPC]) ``` #### **2. Model Retraining Pipeline** ``` -Training Data → Model Retraining → Quality Validation → +Training Data → Model Retraining → Quality Validation → Production Update → Notification & Monitoring ``` -#### **3. Real-time Export Pipeline** -``` -Prediction Results → Data Formatting → OPC Server Write → -Success/Failure Metrics → Notification System -``` - ### Security Architecture #### **Authentication & Authorization** -- **Certificate-based OPC Authentication**: Secure industrial communication -- **MLFlow API Authentication**: Username/password with secure transmission -- **Database Connection Security**: Encrypted connections with credential management -- **Kubernetes Secrets Integration**: Secure credential storage and access +- **MLFlow API Authentication**: Username/password +- **Database Security**: Encrypted connections and credential management +- **OPC Certificates** (if enabled): Client/server certs +- **Kubernetes Secrets**: Secure secret storage #### **Network Security** -- **TLS/SSL Encryption**: Secure communication channels -- **Network Isolation**: Kubernetes network policies and service mesh -- **Firewall Rules**: Controlled access to external services -- **VPN Integration**: Secure remote access and management +- TLS/SSL, network policies, service mesh, firewalls, VPN #### **Data Security** -- **Data Encryption**: At-rest and in-transit encryption -- **Access Control**: Role-based access control (RBAC) -- **Audit Logging**: Comprehensive access and operation logging -- **Data Retention**: Configurable data lifecycle management +- At-rest/in-transit encryption, RBAC, audit logging, lifecycle management -## 🔄 Workflows +## Workflows ### 1. Predictions Batch Workflow (`predictions_batch.py`) @@ -351,8 +373,9 @@ flowchart LR - Temporal server/cluster - PostgreSQL database - MLFlow server -- OPC server(s) +- MinIO object storage (for MLFlow artifacts) - MongoDB server (for notifications) +- OPC server(s) if using OPC export **Note**: External dependencies must be available either through: - Kubernetes cluster deployment diff --git a/laborious/utils/repository/minio_repository.py b/laborious/utils/repository/minio_repository.py index b38af69..03b0f15 100644 --- a/laborious/utils/repository/minio_repository.py +++ b/laborious/utils/repository/minio_repository.py @@ -1,3 +1,11 @@ +""" +MinIO repository utilities. + +This module provides a lightweight repository around a MinIO/S3-compatible +object storage using boto3. It supports creating buckets on demand and +storing/loading pandas DataFrames in Parquet format. +""" + from io import BytesIO from typing import Any @@ -10,6 +18,21 @@ from sientia_do.observability.logger import Logger class MinioRepository: + """ + Repository for interacting with a MinIO (S3-compatible) object storage. + + This class encapsulates a reusable `boto3` S3 client and convenience + helpers to persist and retrieve pandas DataFrames as Parquet files. + + Attributes: + storage_options (dict): Options compatible with pandas s3fs usage. + minio_bucket (str): Default bucket name used for operations. + minio_endpoint_url (str): MinIO endpoint URL. + minio_region_name (str): MinIO region name. + s3_client (Any): Reusable S3 client from `boto3`. + logger (Logger): Observability logger. + notification_handler (NotificationHandler): Notifications handler. + """ def __init__( self, minio_endpoint_url: str, @@ -20,6 +43,17 @@ class MinioRepository: logger: Logger, notification_handler: NotificationHandler, ): + """Initialize the repository and S3 client. + + Args: + minio_endpoint_url (str): MinIO endpoint URL. + minio_access_key (str): Access key (AK). + minio_secret_key (str): Secret key (SK). + minio_region_name (str): Region name for the client. + minio_default_bucket (str): Default bucket name to operate on. + logger (Logger): Logger instance for structured logs. + notification_handler (NotificationHandler): Notification handler. + """ # MinIO settings shared with pandas s3fs self.storage_options = { 'key': minio_access_key, @@ -54,11 +88,14 @@ class MinioRepository: self.notification_handler = notification_handler def close(self): + """Close the underlying S3 client.""" self.s3_client.close() def ensure_bucket_exists(self, metadata: dict[str, Any]) -> None: - """ - Ensure the MinIO bucket exists; create it if necessary. + """Ensure the default bucket exists; create it if missing. + + Args: + metadata (dict[str, Any]): Metadata used for structured logging. """ try: @@ -71,6 +108,14 @@ class MinioRepository: def store_dataframe_as_parquet( self, dataframe: DataFrame, uri: str, object_name: str, metadata: dict[str, Any] ): + """Persist a DataFrame as a Parquet object in the default bucket. + + Args: + dataframe (DataFrame): DataFrame to persist. + uri (str): Human-friendly URI used for logging context. + object_name (str): Object key (path/key within the bucket). + metadata (dict[str, Any]): Metadata used for structured logging. + """ self.ensure_bucket_exists(metadata) self.logger.custom_info(f'Storing dataframe as parquet in {uri}', metadata) @@ -83,6 +128,15 @@ class MinioRepository: self.logger.custom_info(f'Dataframe stored as parquet in {uri}', metadata) def get_parquet_as_dataframe(self, object_key: str, metadata: dict[str, Any]) -> DataFrame: + """Load a Parquet object from the default bucket into a DataFrame. + + Args: + object_key (str): Object key to retrieve from the bucket. + metadata (dict[str, Any]): Metadata used for structured logging. + + Returns: + DataFrame: Loaded DataFrame. + """ self.logger.custom_info(f'Getting parquet as dataframe from {object_key}', metadata) response = self.s3_client.get_object(Bucket=self.minio_bucket, Key=object_key) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 72707c1..3361d9a 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -1,13 +1,13 @@ """ -MLFlow Repository +MLflow repository utilities -This module contains the MLFlowRepository class, -which is responsible for handling the communication with MLFlow tracking server. +This module provides the `MLFlowRepository` class and helpers to interact with +an MLflow tracking server and model registry. It covers model discovery, +downloading/loading with multiple flavors, cached operations with retention +policies, transformation/prediction interfaces, retraining workflows, and +production model promotion. -It includes methods for model management, caching, retraining, and serving operations -using MLFlow's tracking and model registry capabilities. - -The repository provides comprehensive functionality for: +Capabilities: - Model loading and caching with retention policies - Data transformation and prediction operations - Model retraining workflows @@ -37,6 +37,15 @@ INVALID_FLAVOR_MESSAGE = "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch' def force_memory_release(logger: Logger): + """Attempt to release memory from the Python process. + + Executes a garbage collection cycle and calls `malloc_trim(0)` on glibc + where available to return free memory to the OS. This may be a no-op on + non-glibc systems. + + Args: + logger (Logger): Logger for observability. + """ gc.collect() try: @@ -48,6 +57,14 @@ def force_memory_release(logger: Logger): class MLFlowRepository: def __init__(self, host: str, username: str, password: str, logger: Logger): + """Initialize MLflow client and base state. + + Args: + host (str): MLflow tracking URI. + username (str): MLflow username. + password (str): MLflow password. + logger (Logger): Logger instance. + """ # set tracking uri mlflow.set_tracking_uri(host) @@ -64,15 +81,15 @@ class MLFlowRepository: """ def get_model_uri(self, run_id: str, prediction: bool = True): - """ - Get the model URI based on the run_id. + """Build the artifact URI for a run's model. Args: - run_id (str): The run_id of the model. - prediction (bool): Whether to get prediction model URI (default: True) + run_id (str): MLflow run identifier. + prediction (bool): If True, return `prediction_model` URI, + otherwise return `data_model` URI. Returns: - str: The model URI. + str: Artifact URI to the selected model within the run. """ run_info = mlflow.get_run(run_id) if prediction: @@ -82,15 +99,14 @@ class MLFlowRepository: return model_uri def get_model_run_id(self, model_name: str, stage: str = 'Production') -> str: - """ - Get the run_id of a model based on its name and stage. + """Resolve the run_id for a registered model at a given stage. Args: - model_name (str): The name of the model. - stage (str): The stage of the model. + model_name (str): Registered model name. + stage (str): Desired stage (e.g., 'Production'). Returns: - str: The run_id of the model. + str: Run ID for the latest version at the given stage. """ # Use search_registered_models instead of deprecated get_latest_versions registered_models = self.client.search_registered_models( @@ -120,14 +136,14 @@ class MLFlowRepository: def get_next_run_name(self, model_name: str) -> str: """ - Generate the next run name for a specific MLFlow model. + Generate the next run name for a specific MLflow model. This method calculates the next sequential run number for a model by searching existing runs and incrementing the count. It ensures unique run names for model training and retraining operations. Args: - model_name (str): The name of the MLFlow model + model_name (str): The name of the MLflow model Returns: str: The next run name in format 'model_name-run_number' @@ -140,20 +156,20 @@ class MLFlowRepository: self, experiment_name: str, create_if_not_exists: bool = False ) -> Experiment: """ - Retrieve MLFlow experiment ID by experiment name. + Retrieve MLflow experiment by name, optionally creating it. This method searches for an MLFlow experiment by name and returns its unique identifier. It provides error handling for non-existent experiments. Args: - experiment_name (str): Name of the MLFlow experiment + experiment_name (str): Name of the MLflow experiment Returns: - int: MLFlow experiment ID + Experiment: MLflow experiment object Raises: - ValueError: If the experiment name is not found + ValueError: If the experiment name is not found and creation is disabled """ experiment = mlflow.get_experiment_by_name(experiment_name) @@ -166,7 +182,14 @@ class MLFlowRepository: return experiment def get_model_params(self, run_id: str): - """Obtém os parâmetros de uma run""" + """Fetch parameters associated with a given MLflow run. + + Args: + run_id (str): Run identifier to inspect. + + Returns: + dict: Mapping of parameter names to values. + """ run_info = mlflow.get_run(run_id) return run_info.data.params @@ -176,14 +199,14 @@ class MLFlowRepository: def dowload_artifacts(self, model_name: str, artifact_path: str = 'data_model') -> str: """ - Downloads artifacts from a specific MLFlow run. + Download artifacts from the latest production run of a model. Args: - model_name (str): Name of the model - artifact_path (str): Path to the artifact within the run + model_name (str): Registered model name. + artifact_path (str): Relative path to artifacts within the run. Returns: - str: Path to the downloaded artifacts + str: Local filesystem path where artifacts are saved. """ run_id = self.get_model_run_id(model_name=model_name, stage='Production') output_dir = f'{ARTIFACTS_PATH}/{model_name}' @@ -201,7 +224,7 @@ class MLFlowRepository: def load_predict_model(self, model_name: str, flavor: str = 'sklearn') -> Any: """ - Downloads a predictive model from the MLflow Model Registry. + Load a predictive model from the MLflow Model Registry. Args: model_name (str): The name of the model to download from the registry. @@ -212,7 +235,7 @@ class MLFlowRepository: mlflow.pyfunc.PyFuncModel: The loaded predictive model. Notes: - - The model is fetched from the "production" stage of the MLflow Model Registry. + - The model is fetched from the "Production" stage of the MLflow Model Registry. - Warnings during the model loading process are suppressed. """ model_uri = f'models:/{model_name}/production' @@ -230,7 +253,7 @@ class MLFlowRepository: def load_transform_model(self, model_name: str, flavor: str) -> Any: """ - Downloads the latest production version of a specified transformation model. + Load the latest Production version of a transformation model. This method retrieves the latest production model run ID for the given model name, constructs the model URI, and loads the model using MLflow. @@ -241,11 +264,11 @@ class MLFlowRepository: artifact_path (str | None): Path to compressed artifacts if model is compressed Returns: - Any: The loaded model object, as returned by `mlflow.sklearn.load_model`. + Any: The loaded model object, depending on the flavor used. Raises: Exception: If the model run ID or URI cannot be retrieved, or if the - model cannot be loaded. + model cannot be loaded. """ latest_production_id = self.get_model_run_id(model_name=model_name, stage='Production') @@ -266,7 +289,7 @@ class MLFlowRepository: self, model_name: str, model_type: str, flavor: str, load_wrapper: bool = False ) -> tuple[Any, str | None]: """ - Download model based on type (predict or transform). + Download model based on type ("predict" or "transform"). Args: model_name (str): Name of the model to download @@ -275,7 +298,7 @@ class MLFlowRepository: load_wrapper (bool): Whether to load wrapper Returns: - tuple[Any, str]: Model object and artifact path if model is compressed + tuple[Any, str | None]: Model object and optional artifact path. """ self.logger.info( @@ -317,17 +340,19 @@ class MLFlowRepository: def detect_and_parse_datetime_index(self, data: pd.DataFrame, metadata: dict) -> pd.DataFrame: """ - Detect and parse datetime index from data. index must be a timestamp like column. - This function must detect the timestamp type (pandas Timestamp or datetime) and convert it to DATETIME_FORMAT_WITH_TZ. - If the index is a string, must be in format DATETIME_FORMAT_WITH_TZ. - If another type or format, must raise an error. + Normalize DataFrame index to the expected timestamp string format. + + The index must be timestamp-like. If the index is: + - string: it must match `DATETIME_FORMAT_WITH_TZ` + - datetime or pandas Timestamp: it will be converted to that format + Any other type raises a ValueError. Args: - data (pd.DataFrame): DataFrame with timestamp index - metadata (dict): Metadata for logging + data (pd.DataFrame): DataFrame with timestamp index. + metadata (dict): Metadata for structured logging. Returns: - pd.DataFrame: DataFrame with converted datetime index + pd.DataFrame: DataFrame with converted datetime index. """ index = data.index @@ -368,14 +393,14 @@ class MLFlowRepository: def check_cache_retention(self, cache: dict, retention: int) -> bool: """ - Check if cache is still valid based on retention time. + Check whether cached model data is still valid. Args: - cache (dict): Cached model data - retention (int): Retention time in minutes + cache (dict): Cached model data with a 'timestamp' key. + retention (int): Retention time in minutes. Returns: - bool: True if cache is still valid, False if expired + bool: True if cache is still valid, False if expired. """ current_time = datetime.now() cache_time = cache['timestamp'] @@ -385,17 +410,14 @@ class MLFlowRepository: def handle_valid_model(self, model_name: str, cache: dict) -> dict: """ - Handle valid cached model by returning appropriate model configuration. + Return the cached model configuration when retention is valid. Args: - model_name (str): Name of the model - model_type (str): Type of model ('predict' or 'transform') - compressed (bool): Whether model is compressed - retention_target (str): Retention target ('model' or 'artifact') - cache (dict): Cached model data + model_name (str): Name of the model (for logging/consistency). + cache (dict): Cached model data structure. Returns: - dict: Model configuration with model and artifact path + dict: Model configuration. """ self.logger.debug(f'Model {model_name} is still valid, using cached version') @@ -406,8 +428,8 @@ class MLFlowRepository: Clean up outdated cached model and its artifacts. Args: - model_name (str): Name of the model - model_key (str): Cache key for the model + model_name (str): Name of the model. + model_key (str): Cache key for the model. Returns: None @@ -419,16 +441,16 @@ class MLFlowRepository: def get_model(self, model_name: str, retention: int, model_type: str, flavor: str) -> Any: """ - Get model with caching support based on retention policy. + Retrieve a model with caching support based on retention policy. Args: model_name (str): Name of the model to retrieve - retention (int): Cache retention time in minutes (0 = no cache) + retention (int): Cache retention time in minutes (0 = no cache). model_type (str): Type of model ('predict' or 'transform') flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') Returns: - Any: Model object + Any: Model object. """ # Retention is 0, download a new model if retention <= 0: @@ -488,16 +510,16 @@ class MLFlowRepository: self, model_name: str, data: pd.DataFrame, operation: str, retention: int, flavor: str ) -> pd.DataFrame | ndarray: """ - Get transformed data using cached transform model. + Execute a cached operation using the requested model. Args: - model_name (str): Name of the transform model - data (pd.DataFrame): Data to transform - retention (int): Cache retention time in minutes - flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') + model_name (str): Registered model name. + data (pd.DataFrame): Input data. + retention (int): Cache retention in minutes. + flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch'). Returns: - pd.DataFrame: Transformed data + pd.DataFrame | ndarray: Operation result. """ if operation not in ['transform', 'predict']: raise ValueError("Invalid operation. Use 'transform' or 'predict'.") @@ -531,7 +553,7 @@ class MLFlowRepository: target_name: str | None = None, ) -> dict[str, dict[str, Any]]: """ - Create a new MLFlow experiment for model retraining. + Prepare models and data for a retraining run. This method sets up the complete environment for model retraining by: 1. Loading the current production prediction model @@ -541,18 +563,16 @@ class MLFlowRepository: 5. Setting up the MLFlow experiment context Args: - model_name (str): Name of the MLFlow model to retrain - data (pd.DataFrame): Training data for model retraining - transform_flavor (str): Flavor for transformation model - predict_flavor (str): Flavor for prediction model - target_name (str): Target name - metadata (dict): Metadata for logging + model_name (str): Name of the MLflow model to retrain. + data (pd.DataFrame): Training data for model retraining. + transform_flavor (str): Flavor for transformation model. + predict_flavor (str): Flavor for prediction model. + target_name (str | None): Optional target column; if None, use model target. + metadata (dict): Metadata for logging. Returns: - tuple: (prediction_model, data_model, experiment) - - prediction_model: Loaded prediction model for retraining - - data_model: Fitted transformation model - - experiment: MLFlow experiment name + dict[str, dict[str, Any]]: Mapping with prepared `prediction_model` and + `data_model`, including optional artifact paths. """ self.logger.custom_info(f'Starting model experiment creation for {model_name}', metadata) @@ -659,6 +679,14 @@ class MLFlowRepository: return retrain_data def log_model(self, model_data: dict, flavor: str, model_type: str, metadata: dict): + """Log a model into the active MLflow run. + + Args: + model_data (dict): Model holder with keys 'model' and optional 'artifact_path'. + flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch'). + model_type (str): Artifact name, e.g., 'prediction_model' or 'data_model'. + metadata (dict): Metadata for structured logging. + """ model = model_data['model'] self.logger.custom_debug(f'Logging {model_type} model to {model_type}', metadata) @@ -688,7 +716,7 @@ class MLFlowRepository: predict_flavor: str = 'sklearn', ) -> dict: """ - Execute the complete model retraining process in MLFlow. + Execute the complete model retraining process in MLflow. This method performs the actual model retraining by: 1. Starting a new MLFlow run with descriptive metadata @@ -700,17 +728,15 @@ class MLFlowRepository: Args: prediction_model: MLFlow prediction model to retrain data_model: MLFlow transformation model to retrain - experiment (str): MLFlow experiment name for the retraining - model_name (str): Name of the model being retrained - data (pd.DataFrame): Training data used for retraining - transform_flavor (str): Flavor for transformation model - predict_flavor (str): Flavor for prediction model - metadata (dict): Metadata for logging + experiment (str): MLflow experiment name for the retraining. + model_name (str): Name of the model being retrained. + data (pd.DataFrame): Training data used for retraining. + transform_flavor (str): Flavor for transformation model. + predict_flavor (str): Flavor for prediction model. + metadata (dict): Metadata for logging. Returns: - tuple: (status_message, experiment_name) - - status_message (str): Success confirmation message - - experiment_name (str): Name of the experiment + dict: Metadata about the created run and experiment. """ prediction_model = retrain_data['prediction_model'] @@ -795,16 +821,16 @@ class MLFlowRepository: self, run_id: str, model_name: str, metadata: dict ) -> dict: """ - Update production model with a specific MLFlow run. + Promote a specific run's model to Production. This method promotes a model from a specific MLFlow run to production stage. It handles model registration, versioning, and stage transitions with proper error handling. Args: - run_id (str): MLFlow run ID containing the model to promote - model_name (str): Name of the MLFlow model - metadata (dict): Metadata for logging + run_id (str): MLflow run ID containing the model to promote. + model_name (str): Name of the MLflow model. + metadata (dict): Metadata for logging. Returns: dict: Model update metadata containing: @@ -863,7 +889,7 @@ class MLFlowRepository: 5. Manages model lifecycle based on retention policy (cleanup artifacts if needed) Parameters: - model_name (str): The name of the MLFlow model to use for transformation. + model_name (str): The name of the MLflow model to use for transformation. data (pd.DataFrame): The input data to be transformed by the model. model_config (dict): Model configuration parameters metadata (dict): Metadata for logging @@ -934,7 +960,7 @@ class MLFlowRepository: 9. Handles any exceptions and returns structured error information Parameters: - model_name (str): The name of the MLFlow model to use for prediction. + model_name (str): The name of the MLflow model to use for prediction. data (pd.DataFrame): The input data to make predictions on. model_retention (int): Cache retention time in minutes (0 = no caching). model_config (dict): Model configuration parameters @@ -1031,15 +1057,13 @@ class MLFlowRepository: data (pd.DataFrame): Training data for model retraining. Must contain all features required by both transformation and prediction models, including target variable. - model_name (str): Name of the MLFlow model to retrain. Must exist - in the MLFlow Model Registry in Production stage. + model_name (str): Name of the MLflow model to retrain. Must exist + in the MLflow Model Registry in Production stage. model_config (dict): Model configuration parameters metadata (dict): Metadata for logging Returns: - tuple: Retraining operation results containing: - - status_message (str): Success confirmation message or error details - - experiment_name (str): MLFlow experiment identifier for tracking + dict: Retraining operation results and experiment details. Raises: mlflow.exceptions.MlflowException: If model not found in registry @@ -1129,7 +1153,7 @@ class MLFlowRepository: 3. Returns comprehensive update metadata Args: - experiment (str): MLFlow experiment name containing the retraining runs. + experiment (str): MLflow experiment name containing the retraining runs. Must be a valid experiment that exists in MLFlow. model_name (str): Name of the MLFlow model to update. Must exist in the MLFlow Model Registry. From a7cc9177605d6b9bb98f7a69fcf8ebd3744c6ae1 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 16 Oct 2025 11:04:37 -0300 Subject: [PATCH 52/52] SIENTIAPDE-1231 Enhance README with code quality and validation guidelines - Added a new section on code quality and validation tools, detailing the use of Ruff, mypy, Bandit, and pytest. - Included installation instructions for development dependencies and options for running validation scripts. - Provided best practices for maintaining code quality and integrating CI/CD workflows. --- README.md | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/README.md b/README.md index 9c1e85f..a941013 100644 --- a/README.md +++ b/README.md @@ -511,6 +511,68 @@ fi python -m laborious.worker.worker ``` +## Code Quality & Validation + +### Overview + +Since Python is not compiled, we validate quality, security, and correctness before execution. + +### Validation Tools + +- Ruff: Linting and formatting +- mypy: Static type checking +- Bandit: Security analysis +- pytest: Unit/integration testing with coverage + +### Tools Installation + +```bash +pip install -r requirements-dev.txt +``` + +### Complete Validation + +Option 1 (recommended): +```bash +./validate.sh +``` +The script runs, in order: +1. Format check (Ruff) +2. Linting (Ruff) +3. Type checking (mypy) +4. Security analysis (Bandit) +5. Tests with coverage (pytest) + +Option 2 (individual commands): +```bash +ruff format --check laborious/ tests/ +ruff check laborious/ tests/ +mypy laborious/ +bandit -r laborious/ -ll +pytest tests/ --cov=laborious --cov-report=term-missing +``` + +### Automatic Fixes + +```bash +ruff format laborious/ tests/ +ruff check --fix laborious/ tests/ +``` + +### Configuration + +All settings reside in `pyproject.toml` (Ruff, mypy, pytest, Bandit). + +### CI/CD Integration + +The workflow at `.github/workflows/quality-gate.yml` executes validations on each push/PR. + +### Best Practices + +- Run `./validate.sh` before committing +- Use `ruff check --watch` for continuous feedback +- Add type hints and tests for new code + ## 🧪 Testing ### Test Structure