Release 11 (from main)
This commit is contained in:
model-plugin-warehouse CI
@@ -64,6 +64,15 @@ class LinearRegressionModel(BaseEstimator, TransformerMixin):
|
||||
"""
|
||||
return self.regr
|
||||
|
||||
def _check_model_features(self, columns: pd.Index, model_features: list) -> None:
|
||||
"""Raise when columns lack any feature the regressor was fitted with."""
|
||||
missing_model_features = [c for c in model_features if c not in columns]
|
||||
if missing_model_features:
|
||||
raise ValueError(
|
||||
'Prediction data is missing required transformed feature columns: '
|
||||
+ ', '.join(missing_model_features)
|
||||
)
|
||||
|
||||
def create_poly_features(self, input_data: pd.DataFrame, fit: bool = False) -> pd.DataFrame:
|
||||
"""
|
||||
Function to create polynomial features
|
||||
@@ -110,15 +119,55 @@ class LinearRegressionModel(BaseEstimator, TransformerMixin):
|
||||
|
||||
return input_data
|
||||
|
||||
def _fittable_rows(
|
||||
self, features: pd.DataFrame, target: pd.Series
|
||||
) -> tuple[pd.DataFrame, pd.Series]:
|
||||
"""
|
||||
Keep only what LinearRegression can fit: inf becomes NaN, all-null columns are dropped
|
||||
(degree 1 only) and rows with a NaN feature or a NaN target are removed.
|
||||
|
||||
Raises:
|
||||
ValueError: If no rows are left.
|
||||
"""
|
||||
# Handle infinite values
|
||||
features = features.astype(float).replace([np.inf, -np.inf], np.nan)
|
||||
|
||||
# Drop columns with all null values. With degree > 1 they are kept (PolynomialFeatures
|
||||
# needs every variable column), so the row mask below leaves no rows and fit raises.
|
||||
if self.degree == 1:
|
||||
all_null_cols = features.columns[features.isnull().all()].tolist()
|
||||
features = features.drop(columns=all_null_cols)
|
||||
if self.verbose and all_null_cols:
|
||||
print(f'{len(all_null_cols)} columns with all null values were dropped.')
|
||||
|
||||
# Rows with NaN in any feature or in the target cannot be fitted (SIENTIAPDE-2215).
|
||||
# Masked before PolynomialFeatures, which rejects NaN.
|
||||
mask = features.notna().all(axis=1) & target.notna()
|
||||
features = features[mask]
|
||||
target = target[mask]
|
||||
if features.empty:
|
||||
raise ValueError(
|
||||
'No training rows left after removing rows with NaN features or NaN target.'
|
||||
)
|
||||
if self.verbose:
|
||||
print(f'X_train shape after removing NaN rows: {features.shape}')
|
||||
print(f'y_train shape after removing NaN rows: {target.shape}')
|
||||
return features, target
|
||||
|
||||
def fit(self, input_data: pd.DataFrame) -> 'LinearRegressionModel':
|
||||
"""
|
||||
Function to fit the model
|
||||
Function to fit the model. Rows with NaN features or a NaN target are left out of the fit.
|
||||
|
||||
Args:
|
||||
input_data (pandas.DataFrame): The data used to fit the Linear Regression model
|
||||
|
||||
Returns:
|
||||
LinearRegressionModel: The prediction model object
|
||||
|
||||
Raises:
|
||||
ValueError: If target_variable is unset, a feature column is missing, or no
|
||||
training rows are left after removing rows with NaN features or NaN target.
|
||||
KeyError: If the target column is missing from input_data.
|
||||
"""
|
||||
if self.verbose:
|
||||
# Display the header
|
||||
@@ -153,30 +202,14 @@ class LinearRegressionModel(BaseEstimator, TransformerMixin):
|
||||
print(f'X_train shape: {X_train.shape}')
|
||||
print(f'y_train shape: {y_train.shape}')
|
||||
|
||||
X_train, y_train = self._fittable_rows(X_train, y_train)
|
||||
|
||||
# Create polynomial features
|
||||
if self.degree > 1:
|
||||
X_train = self.create_poly_features(X_train, fit=True)
|
||||
|
||||
# Handle infinite values
|
||||
X_train = X_train.replace([np.inf, -np.inf], np.nan)
|
||||
|
||||
# Drop columns with all null values
|
||||
all_null_cols = X_train.columns[X_train.isnull().all()].tolist()
|
||||
X_train = X_train.drop(columns=all_null_cols)
|
||||
|
||||
if self.verbose and all_null_cols:
|
||||
print(f'{len(all_null_cols)} columns with all null values were dropped.')
|
||||
|
||||
# Get mask of rows with any NaN values
|
||||
mask = X_train.notna().all(axis=1)
|
||||
X_train = X_train[mask]
|
||||
y_train = y_train[mask]
|
||||
if self.verbose:
|
||||
print(f'X_train shape after removing NaN rows: {X_train.shape}')
|
||||
print(f'y_train shape after removing NaN rows: {y_train.shape}')
|
||||
|
||||
self.q1_target = y_train.quantile(0.25)
|
||||
self.q3_target = y_train.quantile(0.75)
|
||||
self.q1_target = float(y_train.quantile(0.25))
|
||||
self.q3_target = float(y_train.quantile(0.75))
|
||||
|
||||
# Fit the model
|
||||
self.regr.fit(X_train, y_train)
|
||||
@@ -211,7 +244,7 @@ class LinearRegressionModel(BaseEstimator, TransformerMixin):
|
||||
input_data (pandas.DataFrame): The data used to predict the target variable
|
||||
|
||||
Returns:
|
||||
numpy.ndarray: The predicted target variable
|
||||
numpy.ndarray: One prediction per input row; NaN where any model input is NaN
|
||||
"""
|
||||
if self.verbose:
|
||||
# Display the header
|
||||
@@ -228,38 +261,41 @@ class LinearRegressionModel(BaseEstimator, TransformerMixin):
|
||||
+ ', '.join(missing_features)
|
||||
)
|
||||
|
||||
X_test = input_data[self.variable_columns]
|
||||
# astype(float) first: laborious sends object columns with None, and replace() on
|
||||
# object dtype raises a pandas downcasting FutureWarning on every call.
|
||||
X_test = input_data[self.variable_columns].astype(float).replace([np.inf, -np.inf], np.nan)
|
||||
if self.verbose:
|
||||
print(f'X_test shape: {X_test.shape}')
|
||||
|
||||
# Create polynomial features
|
||||
model_features = list(self.regr.feature_names_in_)
|
||||
# Rows with any NaN model input get a NaN prediction, so the output keeps the input
|
||||
# length (SIENTIAPDE-2215). With degree > 1 the mask uses the raw variable columns,
|
||||
# because PolynomialFeatures rejects NaN.
|
||||
if self.degree > 1:
|
||||
X_test = self.create_poly_features(X_test, fit=False)
|
||||
X_test = X_test.replace([np.inf, -np.inf], np.nan)
|
||||
row_mask = X_test.notna().all(axis=1)
|
||||
else:
|
||||
self._check_model_features(X_test.columns, model_features)
|
||||
row_mask = X_test[model_features].notna().all(axis=1)
|
||||
|
||||
model_features = self.regr.feature_names_in_
|
||||
# Ensure required model features exist
|
||||
missing_model_features = [c for c in model_features if c not in X_test.columns]
|
||||
if missing_model_features:
|
||||
raise ValueError(
|
||||
'Prediction data is missing required transformed feature columns: '
|
||||
+ ', '.join(missing_model_features)
|
||||
)
|
||||
y_pred = np.full(len(input_data), np.nan)
|
||||
if not row_mask.any():
|
||||
return y_pred
|
||||
|
||||
# Get mask of rows with any NaN values
|
||||
mask = X_test.notna().all(axis=1)
|
||||
X_test = X_test[mask]
|
||||
x_valid = X_test[row_mask]
|
||||
if self.degree > 1:
|
||||
x_valid = self.create_poly_features(x_valid, fit=False)
|
||||
self._check_model_features(x_valid.columns, model_features)
|
||||
if self.verbose:
|
||||
print(f'X_test shape after removing NaN rows: {X_test.shape}')
|
||||
print(f'X_test shape after removing NaN rows: {x_valid.shape}')
|
||||
|
||||
# For simplicity and to keep behavior consistent with prior tests, drop NaN rows before prediction
|
||||
y_pred = self.regr.predict(X_test[model_features])
|
||||
valid_pred = self.regr.predict(x_valid[model_features])
|
||||
|
||||
if self.clipping:
|
||||
for i in range(len(y_pred)):
|
||||
if y_pred[i] > self.clipping['max']:
|
||||
y_pred[i] = self.q3_target
|
||||
elif y_pred[i] < self.clipping['min']:
|
||||
y_pred[i] = self.q1_target
|
||||
for i in range(len(valid_pred)):
|
||||
if valid_pred[i] > self.clipping['max']:
|
||||
valid_pred[i] = self.q3_target
|
||||
elif valid_pred[i] < self.clipping['min']:
|
||||
valid_pred[i] = self.q1_target
|
||||
|
||||
y_pred[row_mask.to_numpy()] = valid_pred
|
||||
return y_pred
|
||||
|
||||
Reference in New Issue
Block a user