diff --git a/.env.example b/.env.example index ece7541..36481e4 100644 --- a/.env.example +++ b/.env.example @@ -44,3 +44,5 @@ TIMEOUT_SAVE_MODEL="300" # Save model to MLFlow (5 min for artifac TIMEOUT_CLEANUP_DIRECTORY="60" # Cleanup temporary directory (1 min) TIMEOUT_DELETE_FILE="60" # Delete file from MinIO (1 min) TIMEOUT_UPDATE_DATABASE="30" # Database update operations (30 sec) + +EXTRA_PIP_REQUIREMENTS="git+https://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@github.com/Aignosi/sientia-mlops-library.git" diff --git a/model_manager/sientia/model_serving.py b/model_manager/sientia/model_serving.py index a31388c..41b50bb 100644 --- a/model_manager/sientia/model_serving.py +++ b/model_manager/sientia/model_serving.py @@ -104,14 +104,10 @@ class ModelServing: The GitHub token is hardcoded. Consider moving to environment variable or using a secure secret management solution (e.g., K8s secrets). """ - # SECURITY: Token should be in environment variable, not hardcoded - # TODO: Replace with: os.getenv('GITHUB_TOKEN') or use K8s secrets mlflow.sklearn.log_model( sk_model, artifact_path, - extra_pip_requirements=[ - 'git+https://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@github.com/Aignosi/sientia-mlops-library.git' - ], + extra_pip_requirements=[os.getenv('EXTRA_PIP_REQUIREMENTS')], **kwargs, ) diff --git a/model_manager/sientia/models.py b/model_manager/sientia/models.py index 267efca..4477297 100644 --- a/model_manager/sientia/models.py +++ b/model_manager/sientia/models.py @@ -8,6 +8,12 @@ from sklearn.base import BaseEstimator, TransformerMixin from sklearn.linear_model import LinearRegression from sklearn.preprocessing import StandardScaler +DISCONTINUITY_TREATMENT = 'Discontinuity Treatment' +LAG_SELECTION = 'Lag Selection' +STATIC_WINDOW_REMOVAL = 'Static Window Removal' +DEFINE_VARIABLES_LIMITS = 'Define Variables Limits' +NORMALIZATION = 'Normalization' + class LinearRegressionModel(BaseEstimator, TransformerMixin): """ @@ -218,11 +224,11 @@ class DataPreprocessor(BaseEstimator, TransformerMixin): # Filter steps for preprocessor class possible_steps = [ - 'Discontinuity Treatment', - 'Lag Selection', - 'Static Window Removal', - 'Define Variables Limits', - 'Normalization', + DISCONTINUITY_TREATMENT, + LAG_SELECTION, + STATIC_WINDOW_REMOVAL, + DEFINE_VARIABLES_LIMITS, + NORMALIZATION, 'Feature Creation', 'Lag Creation', ] @@ -456,39 +462,36 @@ class DataPreprocessor(BaseEstimator, TransformerMixin): for step in self.steps_order: # Discontinuity Treatment - if step == 'Discontinuity Treatment': + if step == DISCONTINUITY_TREATMENT: data_treat = self.treat_discontinuities(data_treat) # Lag for Model Training - if step == 'Lag Selection': + if step == LAG_SELECTION: data_treat = self.lag_selection(data_treat, self.lag_train) # Static Window Treatment - if step == 'Static Window Removal': + if step == STATIC_WINDOW_REMOVAL: data_treat = self.treat_static_windows(data_treat) # Adjust limits - if step == 'Define Variables Limits': + if step == DEFINE_VARIABLES_LIMITS: data_treat = self.adjust_limits(data_treat) # Normalization - if step == 'Normalization': - if self.scaler: - self.scaler = self.scaler.fit(data_treat[existing_columns]) - self.feature_names_order = list(data_treat[existing_columns].columns) - data_treat[existing_columns] = self.scaler.transform( - data_treat[existing_columns] - ) + if step == NORMALIZATION and self.scaler: + self.scaler = self.scaler.fit(data_treat[existing_columns]) + self.feature_names_order = list(data_treat[existing_columns].columns) + data_treat[existing_columns] = self.scaler.transform(data_treat[existing_columns]) - # Save scaler parameters - assert self.scaler_params is not None, 'scaler_params must be initialized' - for index, column in enumerate(list(existing_columns)): - mean = self.scaler.mean_[index] - variance = self.scaler.var_[index] - self.scaler_params[column] = { - 'mean': round(mean, 3), - 'variance': round(variance, 3), - } + # Save scaler parameters + assert self.scaler_params is not None, 'scaler_params must be initialized' + for index, column in enumerate(list(existing_columns)): + mean = self.scaler.mean_[index] + variance = self.scaler.var_[index] + self.scaler_params[column] = { + 'mean': round(mean, 3), + 'variance': round(variance, 3), + } return self @@ -515,28 +518,25 @@ class DataPreprocessor(BaseEstimator, TransformerMixin): for step in self.steps_order: # Discontinuity Treatment - if step == 'Discontinuity Treatment': + if step == DISCONTINUITY_TREATMENT: data_treat = self.treat_discontinuities(data_treat) # Lag for Model Training - if step == 'Lag Selection': + if step == LAG_SELECTION: data_treat = self.lag_selection(data_treat, self.lag_transform) # Static Window Treatment - if step == 'Static Window Removal': + if step == STATIC_WINDOW_REMOVAL: data_treat = self.treat_static_windows(data_treat) # Adjust limits - if step == 'Define Variables Limits': + if step == DEFINE_VARIABLES_LIMITS: data_treat = self.adjust_limits(data_treat) # Normalization - if step == 'Normalization': - if self.scaler: - data_treat = data_treat[self.feature_names_order] - data_treat[existing_columns] = self.scaler.transform( - data_treat[existing_columns] - ) + if step == NORMALIZATION and self.scaler: + data_treat = data_treat[self.feature_names_order] + data_treat[existing_columns] = self.scaler.transform(data_treat[existing_columns]) # Feature Creation if step == 'Feature Creation':