feat: enhance training workflow with model metadata loading and refactor data handling

- Introduced a new activity to load model metadata from the model store.
- Refactored training logic to utilize new model metadata and improved parameter handling.
- Updated the `TrainModelParams` class to include additional fields for model configuration.
- Replaced deprecated utility functions with a custom train-test split implementation.
- Removed unused utility functions and cleaned up the data manager repository.
- Adjusted experiment tracking to include model-specific metadata in notifications.

model_manager/workflows/train_model.py

@@ -53,7 +53,6 @@ with workflow.unsafe.imports_passed_through():
         maximum_attempts=5,
     )
 
-POD_ID = os.getenv('POD_ID')
 
 
 @workflow.defn(name='train_model')
@@ -97,12 +96,15 @@ class TrainModel:
             ValueError: If experiment_run_id is missing or invalid
         """
         experiment_run_id = self._validate_experiment_run_id(input_data)
-
+        model_name = input_data.get('model_name')
+        model_id = input_data.get('model_id')
+        
         metadata = {
             'metadata': {
-                'pod_id': POD_ID,
                 'experiment_run_id': experiment_run_id,
                 'workflow_name': 'train_model',
+                'model_name': model_name,
+                'model_id': model_id,
             }
         }
 
@@ -177,18 +179,28 @@ class TrainModel:
             Exception: If validation fails (after updating DB status)
         """
         try:
+            input_data = await workflow.execute_activity_method(
+                Activities.load_model_metadata,
+                {
+                    **input_data,
+                    **metadata,
+                },
+                retry_policy=no_retry_policy,
+                start_to_close_timeout=timedelta(seconds=TIMEOUT_VALIDATE_PARAMS),
+            )
+
             train_params = await workflow.execute_activity_method(
                 Activities.validate_train_params,
                 {
-                    **metadata,
                     **input_data,
+                    **metadata,
                 },
                 retry_policy=no_retry_policy,
                 start_to_close_timeout=timedelta(seconds=TIMEOUT_VALIDATE_PARAMS),
             )
 
             await self._update_experiment_run(
-                metadata=metadata,
+                metadata=metadata,  
                 experiment_run_id=experiment_run_id,
                 update_type=UpdateType.STATUS,
                 status=ExperimentStatus.ORCHESTRATOR_WAITING_PROC,