gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-2201: fix: keep the timestamp column in train/test data CSVs

Browse Source 
generate_report wrote train_data.csv and test_data.csv with index=False, which
dropped the UTC time index. laborious drift reads test_data.csv from MLflow as
its reference and needs a timestamp column; without it, every model trained here
drifted against the first 30% of its own target window. Both CSVs now write the
index as timestamp. If a column already has that name, the CSV is written as
before and a warning is logged, so training never fails over drift.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
PedroHMCosme
2026-10-15 10:09:06 -03:00
parent dd981b8027
commit 37f16f34ff
2 changed files with 80 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
model_manager/utils/repository/data_manager_repository.py
View File

@@ -527,6 +527,32 @@ class DataManagerRepository(SientiaMonitoring):
self.error(error_msg, metadata)
raise OSError(error_msg) from e
def _save_data_csv(
self, frame: pd.DataFrame, file_path: str, metadata: dict[str, Any] | None = None
) -> None:
"""
Write a train/test frame to CSV with its time index as a ``timestamp`` column.
laborious drift reads ``test_data.csv`` from MLflow as its reference and needs
``timestamp`` (SIENTIAPDE-2201). If a column already has that name, the frame is
written without the index and that model's drift keeps falling back to the target
window: training never fails over drift.
Args:
frame: Frame indexed by the UTC training timestamps.
file_path: Destination CSV path.
metadata: Workflow metadata for logging.
"""
if 'timestamp' in frame.columns:
self.warning(
f"Column 'timestamp' already exists; {path.basename(file_path)} saved without "
'the time column (drift reference falls back to the target window)',
metadata,
)
frame.to_csv(file_path, index=False)
return
frame.to_csv(file_path, index_label='timestamp')
def generate_report(
self, data: TrainModelResult, metadata: dict[str, Any] | None = None
) -> TrainModelResult:
@@ -598,11 +624,11 @@ class DataManagerRepository(SientiaMonitoring):
# Save training / validation CSVs using the same frames as the report (includes
# literal `target` alias for Evidently, plus predictions and float-cast features).
data.train_data_path = path.join(data.run_dir, 'train_data.csv')
reference_data_float.to_csv(data.train_data_path, index=False)
self._save_data_csv(reference_data_float, data.train_data_path, metadata)
# Save test data CSV
data.test_data_path = path.join(data.run_dir, 'test_data.csv')
current_data_float.to_csv(data.test_data_path, index=False)
self._save_data_csv(current_data_float, data.test_data_path, metadata)
# Save equation as JSON
if data.equation is not None and data.params.model_type == 'linear_regression':