gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: require date_column in training parameters and update documentation

Browse Source 
- Made `date_column` a required field in `TrainModelParams`, ensuring it must be present in the input data.
- Updated related documentation in `input-sample.md`, `README.md`, and various test scenarios to reflect the change in requirement.
- Adjusted the handling of `date_format` to default to `yyyy-MM-dd HH:mm:ss` if omitted, enhancing usability.
- Refined test scenarios to include new examples and ensure compliance with the updated parameter structure.

These changes improve the robustness of the model training workflow and clarify the expectations for input data.
This commit is contained in:
vitor-aignosi
2026-05-05 08:35:12 -03:00
parent 6bd30e3328
commit ba9eb3d7c7
38 changed files with 1027 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
model_manager/utils/models/train_model_params.py
View File

@@ -14,6 +14,9 @@ FRONTEND_DATE_FORMAT_TO_STRFTIME = {
}
ALLOWED_FRONTEND_DATE_FORMATS = frozenset(FRONTEND_DATE_FORMAT_TO_STRFTIME.keys())
# When the client omits date_format (or sends null/blank), parsing uses this frontend format.
DEFAULT_TRAIN_DATE_FORMAT = 'yyyy-MM-dd HH:mm:ss'
def validate_frontend_date_format(fmt: str | None) -> None:
"""Raise ValueError if fmt is set and not one of the allowed frontend date formats."""
@@ -49,8 +52,9 @@ class TrainModelParams:
file_name (str): Name of the file in the MinIO bucket.
line_separator (str): Line separator used in the CSV file.
decimal_separator (str): Decimal separator used in the CSV file.
date_column (str | None): Name of the date/time column. If set with date_format, the column is parsed as datetime.
date_format (str | None): Format of the date column (e.g. dd/MM/yyyy HH:mm:ss). Used when date_column is set.
date_column (str): Name of the date/time column in the dataset (required).
date_format (str): Format of the date column (allowed frontend strings). If omitted or blank
in the input dict, defaults to DEFAULT_TRAIN_DATE_FORMAT.
train_size (int): Percentage of data to use for training (0-100).
shuffle (bool): Whether to shuffle the data during train/test split.
experiment_run_id (int): Unique identifier for the experiment run.
@@ -70,8 +74,8 @@ class TrainModelParams:
file_name: str
line_separator: str
decimal_separator: str
date_column: str | None
date_format: str | None
date_column: str
date_format: str
train_size: int
shuffle: bool
random_state: int
@@ -102,7 +106,8 @@ class TrainModelParams:
Args:
data: Dictionary containing training parameters with keys matching the
attribute names (e.g. variable_columns, data_model_kwargs, model_kwargs, opt_params).
attribute names (e.g. variable_columns, date_column, data_model_kwargs, model_kwargs, opt_params).
Unknown keys are ignored by from_dict; missing required snake_case keys raise.
model_metadata may be omitted or None until load_model_metadata fills it.
experiment_run_id may be an int or numeric string.
@@ -131,8 +136,8 @@ class TrainModelParams:
decimal_separator=cls._check_none(
data.get('decimal_separator'), str, 'decimal_separator'
),
date_column=data.get('date_column'),
date_format=data.get('date_format'),
date_column=cls._check_none(data.get('date_column'), str, 'date_column'),
date_format=cls._resolve_date_format(data.get('date_format')),
train_size=cls._check_none(data.get('train_size'), int, 'train_size'),
shuffle=cls._check_none(data.get('shuffle'), bool, 'shuffle'),
random_state=cls._check_none(data.get('random_state', 42), int, 'random_state'),
@@ -150,6 +155,29 @@ class TrainModelParams:
model_metadata=cls._parse_optional_model_metadata(data.get('model_metadata')),
)
@staticmethod
def _resolve_date_format(raw: Any) -> str:
"""
Resolve date_format from workflow input.
Omitted, null, or blank values use DEFAULT_TRAIN_DATE_FORMAT. Non-string types raise.
Args:
raw: Raw date_format from the payload, or None if absent.
Return:
str: Canonical frontend date format string.
"""
if raw is None:
return DEFAULT_TRAIN_DATE_FORMAT
if isinstance(raw, str) and not raw.strip():
return DEFAULT_TRAIN_DATE_FORMAT
if not isinstance(raw, str):
raise TypeError(
f'date_format must be a string or omitted, but got {type(raw).__name__}.'
)
return raw.strip()
def to_dict(self) -> dict[str, Any]:
"""
Convert TrainModelParams to a dictionary.
@@ -327,7 +355,9 @@ class TrainModelParams:
if not self.model_name.strip():
raise ValueError('model_name cannot be empty or whitespace')
if not self.date_column.strip():
raise ValueError('date_column cannot be empty or whitespace')
def _validate_date_format(self) -> None:
"""Validate date_format is one of the allowed frontend formats when set."""
if self.date_format:
validate_frontend_date_format(self.date_format)
"""Validate date_format is one of the allowed frontend formats."""
validate_frontend_date_format(self.date_format)

168
model_manager/utils/repository/data_manager_repository.py
View File

@@ -24,13 +24,15 @@ import numpy as np
import pandas as pd
from sientia_do.observability.logger import Logger
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from sientia_model.wrappers.sientia_model import SientiaModel
from model_manager.runtime_paths import PROJECT_BASE_PATH, REPORTS_ROOT
from model_manager.sientia.metrics import mae, mse, r2
from model_manager.sientia.reports import Reports # type: ignore[import-untyped]
from model_manager.utils.models.train_model_params import TrainModelParams
from model_manager.utils.models.train_model_params import (
FRONTEND_DATE_FORMAT_TO_STRFTIME,
TrainModelParams,
)
from model_manager.utils.models.train_model_result import TrainModelResult
@@ -64,21 +66,26 @@ def train_test_split(
def _ensure_date_column_parsed(data: pd.DataFrame, params: TrainModelParams) -> pd.DataFrame:
"""
If date_column is set, parse the column as timezone-aware
datetime to avoid comparison errors downstream.
Parse params.date_column using the frontend date_format mapping only.
The values are expected to follow the global DATETIME_FORMAT_WITH_TZ
pattern defined in sientia_do.temporal.constants.
date_column must exist in ``data`` (callers validate before prepare). No broad
pandas inference or alternate timezone formats here—clients must send a supported
date_format or rely on the TrainModelParams default.
"""
if not params.date_column or params.date_column not in data.columns:
return data
try:
data = data.copy()
parsed = pd.to_datetime(
data[params.date_column],
format=DATETIME_FORMAT_WITH_TZ,
errors='raise',
if params.date_column not in data.columns:
raise ValueError(
f'date_column "{params.date_column}" not found in dataset columns: {list(data.columns)}'
)
data = data.copy()
col = data[params.date_column]
if params.date_format not in FRONTEND_DATE_FORMAT_TO_STRFTIME:
raise ValueError(
f'date_format "{params.date_format}" is not mapped to a strftime pattern '
'(must be one of the allowed frontend formats).'
)
strf = FRONTEND_DATE_FORMAT_TO_STRFTIME[params.date_format]
try:
parsed = pd.to_datetime(col, format=strf, errors='raise')
data[params.date_column] = parsed
except Exception as e:
raise ValueError(
@@ -113,6 +120,62 @@ class DataManagerRepository(SientiaMonitoring):
metrics_controller=None,
)
def _drop_rows_with_missing_timestamp(
self,
df: pd.DataFrame,
params: TrainModelParams,
metadata: dict[str, Any] | None,
) -> pd.DataFrame:
"""
Remove rows where the configured date_column is missing (NaN/NaT/blank string).
Empty timestamp cells cannot be placed on a DatetimeIndex and break
downstream joins and metrics.
"""
if params.date_column not in df.columns:
return df
series = df[params.date_column]
mask = series.notna()
if series.dtype == object:
stripped = series.astype(str).str.strip()
mask &= stripped.ne('')
mask &= stripped.str.lower().ne('nan')
n_drop = int((~mask).sum())
if n_drop:
self.info(
f'Dropping {n_drop} row(s) with missing or blank timestamp column '
f'"{params.date_column}"',
metadata,
)
return df.loc[mask].copy()
def _coerce_non_timestamp_columns_to_numeric(
self,
df: pd.DataFrame,
params: TrainModelParams,
metadata: dict[str, Any] | None,
) -> pd.DataFrame:
"""
Coerce all non-timestamp columns to numeric dtype.
The timestamp column defined by params.date_column is excluded from coercion.
Non-numeric values are coerced to NaN.
"""
out = df.copy()
for col in out.columns:
if col == params.date_column:
continue
original_na = int(out[col].isna().sum())
out[col] = pd.to_numeric(out[col], errors='coerce')
new_na = int(out[col].isna().sum())
introduced_na = new_na - original_na
if introduced_na > 0:
self.warning(
f'Column "{col}" had {introduced_na} non-numeric value(s) coerced to NaN',
metadata,
)
return out
def prepare_training_data(
self,
train_file_bytes: bytes,
@@ -156,9 +219,11 @@ class DataManagerRepository(SientiaMonitoring):
'Check file encoding, line separator and decimal separator.'
) from exc
train_df = self._drop_rows_with_missing_timestamp(train_df, params, metadata)
train_df = _ensure_date_column_parsed(train_df, params)
train_df = self._configure_datetime_index(train_df, params, metadata)
train_df = self._set_timezone_on_index(train_df, metadata)
train_df = self._coerce_non_timestamp_columns_to_numeric(train_df, params, metadata)
if len(train_df) <= 0:
raise ValueError('Training data view is empty after transformation')
@@ -178,9 +243,11 @@ class DataManagerRepository(SientiaMonitoring):
'Check file encoding, line separator and decimal separator.'
) from exc
val_df = self._drop_rows_with_missing_timestamp(val_df, params, metadata)
val_df = _ensure_date_column_parsed(val_df, params)
val_df = self._configure_datetime_index(val_df, params, metadata)
val_df = self._set_timezone_on_index(val_df, metadata)
val_df = self._coerce_non_timestamp_columns_to_numeric(val_df, params, metadata)
if len(val_df) <= 0:
raise ValueError('Validation data view is empty after transformation')
@@ -346,7 +413,10 @@ class DataManagerRepository(SientiaMonitoring):
tmr.r2_val = r2(y_true_val, y_pred_val)
if params.model_type == 'linear_regression':
tmr.equation = self._extract_model_equation(wrapper.model, params)
inner = getattr(wrapper, 'model', None)
regr = getattr(inner, 'regr', None) if inner is not None else None
if regr is not None and hasattr(regr, 'coef_') and hasattr(regr, 'intercept_'):
tmr.equation = self._extract_model_equation(inner, params)
return tmr
@@ -360,7 +430,8 @@ class DataManagerRepository(SientiaMonitoring):
Configure datetime index for the DataFrame.
Guards against None to avoid 'NoneType' object has no attribute 'index' downstream.
Prefers params.date_column when set; otherwise looks for common timestamp column names.
Uses only params.date_column and assumes it was already parsed exactly once by
_ensure_date_column_parsed.
Args:
data: The DataFrame to configure the datetime index for.
params: The training parameters.
@@ -375,56 +446,19 @@ class DataManagerRepository(SientiaMonitoring):
'Check file format, line separator and decimal separator.'
)
if isinstance(data.index, pd.DatetimeIndex):
self.info('DataFrame already has DatetimeIndex', metadata)
return data.sort_index()
if params.date_column not in data.columns:
raise ValueError(
f'date_column "{params.date_column}" not found in dataset columns: {list(data.columns)}'
)
common_timestamp_columns = [
'timestamp',
'Timestamp',
'TIMESTAMP',
'date',
'Date',
'DATE',
'DATA',
'datetime',
'DateTime',
]
timestamp_columns = ([params.date_column] if params.date_column else []) + [
c for c in common_timestamp_columns if c != params.date_column
]
if not pd.api.types.is_datetime64_any_dtype(data[params.date_column]):
raise ValueError(
f'date_column "{params.date_column}" must be datetime before index configuration'
)
for col in timestamp_columns:
if col in data.columns:
try:
data[col] = pd.to_datetime(data[col])
data = data.set_index(col)
data = data.sort_index()
self.info(f'Configured datetime index from column: {col}', metadata)
return data
except (ValueError, TypeError) as e:
self.warning(f'Failed to convert column {col} to datetime: {e}', metadata)
continue
# If no timestamp column found, check if first column looks like a timestamp
first_col = data.columns[0]
try:
# Try to parse first column as datetime
test_values = data[first_col].head(10).dropna()
if len(test_values) > 0:
pd.to_datetime(test_values)
data[first_col] = pd.to_datetime(data[first_col])
data = data.set_index(first_col)
data = data.sort_index()
self.info(f'Configured datetime index from first column: {first_col}', metadata)
return data
except (ValueError, TypeError):
pass
self.warning(
'No timestamp column found - some features may not work correctly',
metadata,
)
data = data.set_index(params.date_column)
data = data.sort_index()
self.info(f'Configured datetime index from column: {params.date_column}', metadata)
return data
def _set_timezone_on_index(
@@ -547,8 +581,10 @@ class DataManagerRepository(SientiaMonitoring):
)
# Generate report sections
report.add_data_quality_section(columns=data.params.variable_columns + ['target'])
report.add_data_drift_section(columns=data.params.variable_columns + ['target'])
target_col = data.params.target_variable
feature_and_target_cols = data.params.variable_columns + [target_col]
report.add_data_quality_section(columns=feature_and_target_cols)
report.add_data_drift_section(columns=feature_and_target_cols)
report.add_regression_section()
# Save HTML report