gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1231

Browse Source 
Comment out CSV export lines in MLFlowRepository to prevent temporary file creation during model operations. This change enhances data handling by avoiding unnecessary file writes while maintaining logging functionality.
This commit is contained in:
vitor-aignosi
2025-10-13 11:35:18 -03:00
parent 49b6e504ae
commit ac9b2e0ec6
1 changed files with 17 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
laborious/utils/repository/model_repository.py
View File

@@ -680,8 +680,8 @@ class MLFlowRepository():
self.logger.custom_debug( self.logger.custom_debug(
f"Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, fit_config: {fit_config}, target_name: {target_name}", metadata) f"Model configuration - transform_flavor: {transform_flavor}, predict_flavor: {predict_flavor}, fit_config: {fit_config}, target_name: {target_name}", metadata)
data.to_csv( # data.to_csv(
f"tmp/retrain_data_{model_name}.csv", index=True) # f"tmp/retrain_data_{model_name}.csv", index=True)
self.logger.custom_info( self.logger.custom_info(
f"Retrieved latest production run ID: {latest_production_id}", metadata) f"Retrieved latest production run ID: {latest_production_id}", metadata)
@@ -721,8 +721,8 @@ class MLFlowRepository():
self.logger.custom_debug( self.logger.custom_debug(
f"Treated data index: {treated_data.index}", metadata) f"Treated data index: {treated_data.index}", metadata)
treated_data.to_csv( # treated_data.to_csv(
f"tmp/retrain_treated_data_{model_name}.csv", index=True) # f"tmp/retrain_treated_data_{model_name}.csv", index=True)
self.logger.custom_debug( self.logger.custom_debug(
f"Transformed data shape: {treated_data.shape}", metadata) f"Transformed data shape: {treated_data.shape}", metadata)
@@ -749,8 +749,8 @@ class MLFlowRepository():
f"Target variable {target_name} found in treated data, using it", metadata) f"Target variable {target_name} found in treated data, using it", metadata)
retrain_dataset = treated_data retrain_dataset = treated_data
retrain_dataset.to_csv( # retrain_dataset.to_csv(
f"tmp/retrain_retrain_dataset_{model_name}.csv", index=True) # f"tmp/retrain_retrain_dataset_{model_name}.csv", index=True)
prediction_model.fit(retrain_dataset) prediction_model.fit(retrain_dataset)
@@ -896,7 +896,7 @@ class MLFlowRepository():
data_path = f"{model_temp_path}/retrain_data.csv" data_path = f"{model_temp_path}/retrain_data.csv"
data.to_csv(data_path, index=True) # data.to_csv(data_path, index=True)
self.logger.custom_info( self.logger.custom_info(
f"Starting model upload for {experiment_name} with run name {current_run_name}", metadata) f"Starting model upload for {experiment_name} with run name {current_run_name}", metadata)
@@ -1051,8 +1051,8 @@ class MLFlowRepository():
self.logger.custom_debug( self.logger.custom_debug(
f"Data received for model transformation: {data.head(5).to_csv()}", metadata) f"Data received for model transformation: {data.head(5).to_csv()}", metadata)
data.to_csv( # data.to_csv(
f"tmp/data_{model_name}.csv", index=True) # f"tmp/data_{model_name}.csv", index=True)
model_retention = model_config.get('retention_minutes', 0) model_retention = model_config.get('retention_minutes', 0)
flavor = model_config.get('transform_flavor', 'sklearn') flavor = model_config.get('transform_flavor', 'sklearn')
@@ -1065,8 +1065,8 @@ class MLFlowRepository():
self.logger.custom_debug( self.logger.custom_debug(
f"Data received from model transformation: {transformed_data.head(5).to_csv()}", metadata) f"Data received from model transformation: {transformed_data.head(5).to_csv()}", metadata)
transformed_data.to_csv( # transformed_data.to_csv(
f"tmp/transformed_data_{model_name}.csv", index=True) # f"tmp/transformed_data_{model_name}.csv", index=True)
transformed_data = self.detect_and_parse_datetime_index( transformed_data = self.detect_and_parse_datetime_index(
transformed_data, metadata) transformed_data, metadata)
@@ -1138,8 +1138,8 @@ class MLFlowRepository():
self.logger.custom_debug( self.logger.custom_debug(
f"Data received for model prediction: {data.head(5).to_csv()}", metadata) f"Data received for model prediction: {data.head(5).to_csv()}", metadata)
data.to_csv( # data.to_csv(
f"tmp/treated_data_{model_name}.csv", index=True) # f"tmp/treated_data_{model_name}.csv", index=True)
predict_data = self.get_cached_predict( predict_data = self.get_cached_predict(
model_name, data, model_retention, flavor) model_name, data, model_retention, flavor)
@@ -1150,15 +1150,15 @@ class MLFlowRepository():
self.logger.custom_debug( self.logger.custom_debug(
f"Data received from model prediction: {data.head(5).to_csv()}", metadata) f"Data received from model prediction: {data.head(5).to_csv()}", metadata)
predict_data.to_csv( # predict_data.to_csv(
f"tmp/predicted_data_{model_name}.csv", index=True) # f"tmp/predicted_data_{model_name}.csv", index=True)
data.columns = ['prediction'] data.columns = ['prediction']
else: else:
predict_data = pd.DataFrame( predict_data = pd.DataFrame(
predict_data, columns=['prediction']) predict_data, columns=['prediction'])
predict_data.to_csv( # predict_data.to_csv(
f"tmp/predicted_data_{model_name}.csv", index=True) # f"tmp/predicted_data_{model_name}.csv", index=True)
predict_data.index = input_index predict_data.index = input_index
predict_data['response_time'] = ( predict_data['response_time'] = (