gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: require date_column in training parameters and update documentation

Browse Source 
- Made `date_column` a required field in `TrainModelParams`, ensuring it must be present in the input data.
- Updated related documentation in `input-sample.md`, `README.md`, and various test scenarios to reflect the change in requirement.
- Adjusted the handling of `date_format` to default to `yyyy-MM-dd HH:mm:ss` if omitted, enhancing usability.
- Refined test scenarios to include new examples and ensure compliance with the updated parameter structure.

These changes improve the robustness of the model training workflow and clarify the expectations for input data.
This commit is contained in:
vitor-aignosi
2026-05-05 08:35:12 -03:00
parent 6bd30e3328
commit ba9eb3d7c7
38 changed files with 1027 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
model_manager/utils/repository/data_manager_repository.py
View File

@@ -24,13 +24,15 @@ import numpy as np
import pandas as pd
from sientia_do.observability.logger import Logger
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from sientia_model.wrappers.sientia_model import SientiaModel
from model_manager.runtime_paths import PROJECT_BASE_PATH, REPORTS_ROOT
from model_manager.sientia.metrics import mae, mse, r2
from model_manager.sientia.reports import Reports # type: ignore[import-untyped]
from model_manager.utils.models.train_model_params import TrainModelParams
from model_manager.utils.models.train_model_params import (
FRONTEND_DATE_FORMAT_TO_STRFTIME,
TrainModelParams,
)
from model_manager.utils.models.train_model_result import TrainModelResult
@@ -64,21 +66,26 @@ def train_test_split(
def _ensure_date_column_parsed(data: pd.DataFrame, params: TrainModelParams) -> pd.DataFrame:
"""
If date_column is set, parse the column as timezone-aware
datetime to avoid comparison errors downstream.
Parse params.date_column using the frontend date_format mapping only.
The values are expected to follow the global DATETIME_FORMAT_WITH_TZ
pattern defined in sientia_do.temporal.constants.
date_column must exist in ``data`` (callers validate before prepare). No broad
pandas inference or alternate timezone formats here—clients must send a supported
date_format or rely on the TrainModelParams default.
"""
if not params.date_column or params.date_column not in data.columns:
return data
try:
data = data.copy()
parsed = pd.to_datetime(
data[params.date_column],
format=DATETIME_FORMAT_WITH_TZ,
errors='raise',
if params.date_column not in data.columns:
raise ValueError(
f'date_column "{params.date_column}" not found in dataset columns: {list(data.columns)}'
)
data = data.copy()
col = data[params.date_column]
if params.date_format not in FRONTEND_DATE_FORMAT_TO_STRFTIME:
raise ValueError(
f'date_format "{params.date_format}" is not mapped to a strftime pattern '
'(must be one of the allowed frontend formats).'
)
strf = FRONTEND_DATE_FORMAT_TO_STRFTIME[params.date_format]
try:
parsed = pd.to_datetime(col, format=strf, errors='raise')
data[params.date_column] = parsed
except Exception as e:
raise ValueError(
@@ -113,6 +120,62 @@ class DataManagerRepository(SientiaMonitoring):
metrics_controller=None,
)
def _drop_rows_with_missing_timestamp(
self,
df: pd.DataFrame,
params: TrainModelParams,
metadata: dict[str, Any] | None,
) -> pd.DataFrame:
"""
Remove rows where the configured date_column is missing (NaN/NaT/blank string).
Empty timestamp cells cannot be placed on a DatetimeIndex and break
downstream joins and metrics.
"""
if params.date_column not in df.columns:
return df
series = df[params.date_column]
mask = series.notna()
if series.dtype == object:
stripped = series.astype(str).str.strip()
mask &= stripped.ne('')
mask &= stripped.str.lower().ne('nan')
n_drop = int((~mask).sum())
if n_drop:
self.info(
f'Dropping {n_drop} row(s) with missing or blank timestamp column '
f'"{params.date_column}"',
metadata,
)
return df.loc[mask].copy()
def _coerce_non_timestamp_columns_to_numeric(
self,
df: pd.DataFrame,
params: TrainModelParams,
metadata: dict[str, Any] | None,
) -> pd.DataFrame:
"""
Coerce all non-timestamp columns to numeric dtype.
The timestamp column defined by params.date_column is excluded from coercion.
Non-numeric values are coerced to NaN.
"""
out = df.copy()
for col in out.columns:
if col == params.date_column:
continue
original_na = int(out[col].isna().sum())
out[col] = pd.to_numeric(out[col], errors='coerce')
new_na = int(out[col].isna().sum())
introduced_na = new_na - original_na
if introduced_na > 0:
self.warning(
f'Column "{col}" had {introduced_na} non-numeric value(s) coerced to NaN',
metadata,
)
return out
def prepare_training_data(
self,
train_file_bytes: bytes,
@@ -156,9 +219,11 @@ class DataManagerRepository(SientiaMonitoring):
'Check file encoding, line separator and decimal separator.'
) from exc
train_df = self._drop_rows_with_missing_timestamp(train_df, params, metadata)
train_df = _ensure_date_column_parsed(train_df, params)
train_df = self._configure_datetime_index(train_df, params, metadata)
train_df = self._set_timezone_on_index(train_df, metadata)
train_df = self._coerce_non_timestamp_columns_to_numeric(train_df, params, metadata)
if len(train_df) <= 0:
raise ValueError('Training data view is empty after transformation')
@@ -178,9 +243,11 @@ class DataManagerRepository(SientiaMonitoring):
'Check file encoding, line separator and decimal separator.'
) from exc
val_df = self._drop_rows_with_missing_timestamp(val_df, params, metadata)
val_df = _ensure_date_column_parsed(val_df, params)
val_df = self._configure_datetime_index(val_df, params, metadata)
val_df = self._set_timezone_on_index(val_df, metadata)
val_df = self._coerce_non_timestamp_columns_to_numeric(val_df, params, metadata)
if len(val_df) <= 0:
raise ValueError('Validation data view is empty after transformation')
@@ -346,7 +413,10 @@ class DataManagerRepository(SientiaMonitoring):
tmr.r2_val = r2(y_true_val, y_pred_val)
if params.model_type == 'linear_regression':
tmr.equation = self._extract_model_equation(wrapper.model, params)
inner = getattr(wrapper, 'model', None)
regr = getattr(inner, 'regr', None) if inner is not None else None
if regr is not None and hasattr(regr, 'coef_') and hasattr(regr, 'intercept_'):
tmr.equation = self._extract_model_equation(inner, params)
return tmr
@@ -360,7 +430,8 @@ class DataManagerRepository(SientiaMonitoring):
Configure datetime index for the DataFrame.
Guards against None to avoid 'NoneType' object has no attribute 'index' downstream.
Prefers params.date_column when set; otherwise looks for common timestamp column names.
Uses only params.date_column and assumes it was already parsed exactly once by
_ensure_date_column_parsed.
Args:
data: The DataFrame to configure the datetime index for.
params: The training parameters.
@@ -375,56 +446,19 @@ class DataManagerRepository(SientiaMonitoring):
'Check file format, line separator and decimal separator.'
)
if isinstance(data.index, pd.DatetimeIndex):
self.info('DataFrame already has DatetimeIndex', metadata)
return data.sort_index()
if params.date_column not in data.columns:
raise ValueError(
f'date_column "{params.date_column}" not found in dataset columns: {list(data.columns)}'
)
common_timestamp_columns = [
'timestamp',
'Timestamp',
'TIMESTAMP',
'date',
'Date',
'DATE',
'DATA',
'datetime',
'DateTime',
]
timestamp_columns = ([params.date_column] if params.date_column else []) + [
c for c in common_timestamp_columns if c != params.date_column
]
if not pd.api.types.is_datetime64_any_dtype(data[params.date_column]):
raise ValueError(
f'date_column "{params.date_column}" must be datetime before index configuration'
)
for col in timestamp_columns:
if col in data.columns:
try:
data[col] = pd.to_datetime(data[col])
data = data.set_index(col)
data = data.sort_index()
self.info(f'Configured datetime index from column: {col}', metadata)
return data
except (ValueError, TypeError) as e:
self.warning(f'Failed to convert column {col} to datetime: {e}', metadata)
continue
# If no timestamp column found, check if first column looks like a timestamp
first_col = data.columns[0]
try:
# Try to parse first column as datetime
test_values = data[first_col].head(10).dropna()
if len(test_values) > 0:
pd.to_datetime(test_values)
data[first_col] = pd.to_datetime(data[first_col])
data = data.set_index(first_col)
data = data.sort_index()
self.info(f'Configured datetime index from first column: {first_col}', metadata)
return data
except (ValueError, TypeError):
pass
self.warning(
'No timestamp column found - some features may not work correctly',
metadata,
)
data = data.set_index(params.date_column)
data = data.sort_index()
self.info(f'Configured datetime index from column: {params.date_column}', metadata)
return data
def _set_timezone_on_index(
@@ -547,8 +581,10 @@ class DataManagerRepository(SientiaMonitoring):
)
# Generate report sections
report.add_data_quality_section(columns=data.params.variable_columns + ['target'])
report.add_data_drift_section(columns=data.params.variable_columns + ['target'])
target_col = data.params.target_variable
feature_and_target_cols = data.params.variable_columns + [target_col]
report.add_data_quality_section(columns=feature_and_target_cols)
report.add_data_drift_section(columns=feature_and_target_cols)
report.add_regression_section()
# Save HTML report