gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1255: Refactor: Move extra pip requirements to environment variable and use constants for preprocessor steps.

Browse Source
This commit is contained in:
Bruno Domingues
2025-10-21 09:20:23 -03:00
parent 4686b438db
commit ab7c93480b
3 changed files with 38 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
model_manager/sientia/model_serving.py
View File

@@ -104,14 +104,10 @@ class ModelServing:
The GitHub token is hardcoded. Consider moving to environment variable
or using a secure secret management solution (e.g., K8s secrets).
"""
# SECURITY: Token should be in environment variable, not hardcoded
# TODO: Replace with: os.getenv('GITHUB_TOKEN') or use K8s secrets
mlflow.sklearn.log_model(
sk_model,
artifact_path,
extra_pip_requirements=[
'git+https://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@github.com/Aignosi/sientia-mlops-library.git'
],
extra_pip_requirements=[os.getenv('EXTRA_PIP_REQUIREMENTS')],
**kwargs,
)

70
model_manager/sientia/models.py
View File

@@ -8,6 +8,12 @@ from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler
DISCONTINUITY_TREATMENT = 'Discontinuity Treatment'
LAG_SELECTION = 'Lag Selection'
STATIC_WINDOW_REMOVAL = 'Static Window Removal'
DEFINE_VARIABLES_LIMITS = 'Define Variables Limits'
NORMALIZATION = 'Normalization'
class LinearRegressionModel(BaseEstimator, TransformerMixin):
"""
@@ -218,11 +224,11 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
# Filter steps for preprocessor class
possible_steps = [
'Discontinuity Treatment',
'Lag Selection',
'Static Window Removal',
'Define Variables Limits',
'Normalization',
DISCONTINUITY_TREATMENT,
LAG_SELECTION,
STATIC_WINDOW_REMOVAL,
DEFINE_VARIABLES_LIMITS,
NORMALIZATION,
'Feature Creation',
'Lag Creation',
]
@@ -456,39 +462,36 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
for step in self.steps_order:
# Discontinuity Treatment
if step == 'Discontinuity Treatment':
if step == DISCONTINUITY_TREATMENT:
data_treat = self.treat_discontinuities(data_treat)
# Lag for Model Training
if step == 'Lag Selection':
if step == LAG_SELECTION:
data_treat = self.lag_selection(data_treat, self.lag_train)
# Static Window Treatment
if step == 'Static Window Removal':
if step == STATIC_WINDOW_REMOVAL:
data_treat = self.treat_static_windows(data_treat)
# Adjust limits
if step == 'Define Variables Limits':
if step == DEFINE_VARIABLES_LIMITS:
data_treat = self.adjust_limits(data_treat)
# Normalization
if step == 'Normalization':
if self.scaler:
self.scaler = self.scaler.fit(data_treat[existing_columns])
self.feature_names_order = list(data_treat[existing_columns].columns)
data_treat[existing_columns] = self.scaler.transform(
data_treat[existing_columns]
)
if step == NORMALIZATION and self.scaler:
self.scaler = self.scaler.fit(data_treat[existing_columns])
self.feature_names_order = list(data_treat[existing_columns].columns)
data_treat[existing_columns] = self.scaler.transform(data_treat[existing_columns])
# Save scaler parameters
assert self.scaler_params is not None, 'scaler_params must be initialized'
for index, column in enumerate(list(existing_columns)):
mean = self.scaler.mean_[index]
variance = self.scaler.var_[index]
self.scaler_params[column] = {
'mean': round(mean, 3),
'variance': round(variance, 3),
}
# Save scaler parameters
assert self.scaler_params is not None, 'scaler_params must be initialized'
for index, column in enumerate(list(existing_columns)):
mean = self.scaler.mean_[index]
variance = self.scaler.var_[index]
self.scaler_params[column] = {
'mean': round(mean, 3),
'variance': round(variance, 3),
}
return self
@@ -515,28 +518,25 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
for step in self.steps_order:
# Discontinuity Treatment
if step == 'Discontinuity Treatment':
if step == DISCONTINUITY_TREATMENT:
data_treat = self.treat_discontinuities(data_treat)
# Lag for Model Training
if step == 'Lag Selection':
if step == LAG_SELECTION:
data_treat = self.lag_selection(data_treat, self.lag_transform)
# Static Window Treatment
if step == 'Static Window Removal':
if step == STATIC_WINDOW_REMOVAL:
data_treat = self.treat_static_windows(data_treat)
# Adjust limits
if step == 'Define Variables Limits':
if step == DEFINE_VARIABLES_LIMITS:
data_treat = self.adjust_limits(data_treat)
# Normalization
if step == 'Normalization':
if self.scaler:
data_treat = data_treat[self.feature_names_order]
data_treat[existing_columns] = self.scaler.transform(
data_treat[existing_columns]
)
if step == NORMALIZATION and self.scaler:
data_treat = data_treat[self.feature_names_order]
data_treat[existing_columns] = self.scaler.transform(data_treat[existing_columns])
# Feature Creation
if step == 'Feature Creation':