SIENTIAPDE-1252: Implement save_model activity to save trained models to MLflow with comprehensive error handling and add corresponding unit tests.

This commit is contained in:
Bruno Domingues
2025-10-13 10:18:45 -03:00
parent 3fd5ab79fe
commit b3c749872c
2 changed files with 352 additions and 0 deletions

View File

@@ -344,3 +344,107 @@ class MLFlow(BaseActivity):
)
self.error(trace, metadata=metadata)
raise e
@activity.defn(name='save_model')
async def save_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Save a trained ML model and its artifacts to MLflow with comprehensive error handling.
This activity orchestrates the complete model saving pipeline:
1. Generates the next run name for the experiment
2. Creates and organizes artifacts (reports, data files)
3. Logs model, parameters, metrics, and artifacts to MLflow
4. Returns success/failure status with results or error message
The activity does NOT raise exceptions on failure - it catches all errors,
sends notifications, and returns a failure status. This allows the workflow
to handle the error gracefully and update the database accordingly.
Args:
input_data: Configuration for model saving operation
Required keys:
- metadata (dict): Workflow execution metadata
- train_result (TrainModelResult): Training result with model and metrics
Returns:
dict: Save result with the following structure:
{
'success': bool, # True if saving succeeded, False otherwise
'result': TrainModelResult | None, # Updated result if success=True
'error_message': str | None # Error message if success=False
}
Example:
# Successful save
result = await save_model({
'metadata': {'workflow_id': 'save-123', 'experiment_run_id': 456},
'train_result': TrainModelResult(...)
})
# Returns: {'success': True, 'result': TrainModelResult(...), 'error_message': None}
# Failed save
# Returns: {'success': False, 'result': None, 'error_message': 'Error details...'}
"""
metadata = input_data.get('metadata', {})
train_result = input_data['train_result']
try:
experiment_name = train_result.params.experiment_name
self.info(
f'Starting model save for experiment: {experiment_name}',
metadata,
)
# Step 1: Generate next run name
self.info('Generating run name', metadata)
train_result.run_name = self.model_monitoring_repository.get_next_run_name(
experiment_name
)
self.info(f'Generated run name: {train_result.run_name}', metadata)
# Step 2: Generate artifacts (reports, CSV files)
self.info('Generating artifacts', metadata)
train_result = self.model_monitoring_repository.generate_artifacts(train_result)
self.info('Artifacts generated successfully', metadata)
# Step 3: Save run to MLflow
self.info('Saving run to MLflow', metadata)
self.model_monitoring_repository.save_run(train_result)
self.info(
f'Model saved successfully - Run: {train_result.run_name}, '
f'Experiment: {experiment_name}',
metadata,
)
return {
'success': True,
'result': train_result,
'error_message': None,
}
except Exception as e: # noqa: BLE001
error_msg = f'Error saving model - Experiment: {train_result.params.experiment_name if train_result and train_result.params else "unknown"}, Error: {str(e)}'
trace = traceback.format_exc()
# Send notification (MongoDB)
self.send_notification(
metadata=metadata,
notification_id='SAVE_MODEL_ERROR',
message=error_msg,
block='save_model',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
# Log error with metadata
self.error(trace, metadata=metadata)
# Return failure result (do NOT raise exception)
# This allows workflow to update database with error status
return {
'success': False,
'result': None,
'error_message': str(e),
}