gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1255: Refactor: Move extra pip requirements to environment variable and use constants for preprocessor steps.

Browse Source
This commit is contained in:
Bruno Domingues
2025-10-21 09:20:23 -03:00
parent 4686b438db
commit ab7c93480b
3 changed files with 38 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.env.example
View File

@@ -44,3 +44,5 @@ TIMEOUT_SAVE_MODEL="300" # Save model to MLFlow (5 min for artifac
TIMEOUT_CLEANUP_DIRECTORY="60" # Cleanup temporary directory (1 min) TIMEOUT_CLEANUP_DIRECTORY="60" # Cleanup temporary directory (1 min)
TIMEOUT_DELETE_FILE="60" # Delete file from MinIO (1 min) TIMEOUT_DELETE_FILE="60" # Delete file from MinIO (1 min)
TIMEOUT_UPDATE_DATABASE="30" # Database update operations (30 sec) TIMEOUT_UPDATE_DATABASE="30" # Database update operations (30 sec)
EXTRA_PIP_REQUIREMENTS="git+https://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@github.com/Aignosi/sientia-mlops-library.git"

6
model_manager/sientia/model_serving.py
View File

@@ -104,14 +104,10 @@ class ModelServing:
The GitHub token is hardcoded. Consider moving to environment variable The GitHub token is hardcoded. Consider moving to environment variable
or using a secure secret management solution (e.g., K8s secrets). or using a secure secret management solution (e.g., K8s secrets).
""" """
# SECURITY: Token should be in environment variable, not hardcoded
# TODO: Replace with: os.getenv('GITHUB_TOKEN') or use K8s secrets
mlflow.sklearn.log_model( mlflow.sklearn.log_model(
sk_model, sk_model,
artifact_path, artifact_path,
extra_pip_requirements=[ extra_pip_requirements=[os.getenv('EXTRA_PIP_REQUIREMENTS')],
'git+https://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@github.com/Aignosi/sientia-mlops-library.git'
],
**kwargs, **kwargs,
) )

70
model_manager/sientia/models.py
View File

@@ -8,6 +8,12 @@ from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.linear_model import LinearRegression from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler from sklearn.preprocessing import StandardScaler
DISCONTINUITY_TREATMENT = 'Discontinuity Treatment'
LAG_SELECTION = 'Lag Selection'
STATIC_WINDOW_REMOVAL = 'Static Window Removal'
DEFINE_VARIABLES_LIMITS = 'Define Variables Limits'
NORMALIZATION = 'Normalization'
class LinearRegressionModel(BaseEstimator, TransformerMixin): class LinearRegressionModel(BaseEstimator, TransformerMixin):
""" """
@@ -218,11 +224,11 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
# Filter steps for preprocessor class # Filter steps for preprocessor class
possible_steps = [ possible_steps = [
'Discontinuity Treatment', DISCONTINUITY_TREATMENT,
'Lag Selection', LAG_SELECTION,
'Static Window Removal', STATIC_WINDOW_REMOVAL,
'Define Variables Limits', DEFINE_VARIABLES_LIMITS,
'Normalization', NORMALIZATION,
'Feature Creation', 'Feature Creation',
'Lag Creation', 'Lag Creation',
] ]
@@ -456,39 +462,36 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
for step in self.steps_order: for step in self.steps_order:
# Discontinuity Treatment # Discontinuity Treatment
if step == 'Discontinuity Treatment': if step == DISCONTINUITY_TREATMENT:
data_treat = self.treat_discontinuities(data_treat) data_treat = self.treat_discontinuities(data_treat)
# Lag for Model Training # Lag for Model Training
if step == 'Lag Selection': if step == LAG_SELECTION:
data_treat = self.lag_selection(data_treat, self.lag_train) data_treat = self.lag_selection(data_treat, self.lag_train)
# Static Window Treatment # Static Window Treatment
if step == 'Static Window Removal': if step == STATIC_WINDOW_REMOVAL:
data_treat = self.treat_static_windows(data_treat) data_treat = self.treat_static_windows(data_treat)
# Adjust limits # Adjust limits
if step == 'Define Variables Limits': if step == DEFINE_VARIABLES_LIMITS:
data_treat = self.adjust_limits(data_treat) data_treat = self.adjust_limits(data_treat)
# Normalization # Normalization
if step == 'Normalization': if step == NORMALIZATION and self.scaler:
if self.scaler: self.scaler = self.scaler.fit(data_treat[existing_columns])
self.scaler = self.scaler.fit(data_treat[existing_columns]) self.feature_names_order = list(data_treat[existing_columns].columns)
self.feature_names_order = list(data_treat[existing_columns].columns) data_treat[existing_columns] = self.scaler.transform(data_treat[existing_columns])
data_treat[existing_columns] = self.scaler.transform(
data_treat[existing_columns]
)
# Save scaler parameters # Save scaler parameters
assert self.scaler_params is not None, 'scaler_params must be initialized' assert self.scaler_params is not None, 'scaler_params must be initialized'
for index, column in enumerate(list(existing_columns)): for index, column in enumerate(list(existing_columns)):
mean = self.scaler.mean_[index] mean = self.scaler.mean_[index]
variance = self.scaler.var_[index] variance = self.scaler.var_[index]
self.scaler_params[column] = { self.scaler_params[column] = {
'mean': round(mean, 3), 'mean': round(mean, 3),
'variance': round(variance, 3), 'variance': round(variance, 3),
} }
return self return self
@@ -515,28 +518,25 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
for step in self.steps_order: for step in self.steps_order:
# Discontinuity Treatment # Discontinuity Treatment
if step == 'Discontinuity Treatment': if step == DISCONTINUITY_TREATMENT:
data_treat = self.treat_discontinuities(data_treat) data_treat = self.treat_discontinuities(data_treat)
# Lag for Model Training # Lag for Model Training
if step == 'Lag Selection': if step == LAG_SELECTION:
data_treat = self.lag_selection(data_treat, self.lag_transform) data_treat = self.lag_selection(data_treat, self.lag_transform)
# Static Window Treatment # Static Window Treatment
if step == 'Static Window Removal': if step == STATIC_WINDOW_REMOVAL:
data_treat = self.treat_static_windows(data_treat) data_treat = self.treat_static_windows(data_treat)
# Adjust limits # Adjust limits
if step == 'Define Variables Limits': if step == DEFINE_VARIABLES_LIMITS:
data_treat = self.adjust_limits(data_treat) data_treat = self.adjust_limits(data_treat)
# Normalization # Normalization
if step == 'Normalization': if step == NORMALIZATION and self.scaler:
if self.scaler: data_treat = data_treat[self.feature_names_order]
data_treat = data_treat[self.feature_names_order] data_treat[existing_columns] = self.scaler.transform(data_treat[existing_columns])
data_treat[existing_columns] = self.scaler.transform(
data_treat[existing_columns]
)
# Feature Creation # Feature Creation
if step == 'Feature Creation': if step == 'Feature Creation':