SIENTIAPDE-1253: Expose workflow activity timeouts as environment variables and document them.

.env.example

@@ -33,4 +33,14 @@ MINIO_USE_SSL="false"
 MINIO_MAX_RETRY_ATTEMPTS="3"
 MINIO_RETRY_MODE="adaptive"
 MINIO_CONNECT_TIMEOUT="10"
-MINIO_READ_TIMEOUT="60"
+MINIO_READ_TIMEOUT="60"
+
+# Workflow Activity Timeouts (in seconds)
+# These timeouts are designed to handle large files (up to 200MB)
+TIMEOUT_VALIDATE_PARAMS="30"           # Parameter validation (fast operation)
+TIMEOUT_DOWNLOAD_FILE="600"            # File download from MinIO (10 min for 200MB @ 1MB/s with 3x buffer)
+TIMEOUT_TRAIN_MODEL="1800"             # Model training (30 min for large datasets)
+TIMEOUT_SAVE_MODEL="300"               # Save model to MLFlow (5 min for artifacts upload)
+TIMEOUT_CLEANUP_DIRECTORY="60"         # Cleanup temporary directory (1 min)
+TIMEOUT_DELETE_FILE="60"               # Delete file from MinIO (1 min)
+TIMEOUT_UPDATE_DATABASE="30"           # Database update operations (30 sec)

README.md

@@ -892,6 +892,22 @@ The Model Manager system exposes comprehensive Prometheus metrics for operationa
 | `HTTP_SDK_METRICS_PORT` | Temporal SDK metrics port | `9091` | No |
 | `POD_ID` | Kubernetes pod identifier | `None` | No |
 
+#### Workflow Activity Timeouts
+
+These timeouts control how long each activity in the training workflow can run before timing out. All values are in seconds and are designed to handle large files (up to 200MB).
+
+| Variable | Description | Default | Calculation Basis |
+|----------|-------------|---------|-------------------|
+| `TIMEOUT_VALIDATE_PARAMS` | Parameter validation timeout | `30` | Fast operation, no I/O |
+| `TIMEOUT_DOWNLOAD_FILE` | File download from MinIO timeout | `600` | 200MB @ 1MB/s with 3x buffer (10 min) |
+| `TIMEOUT_TRAIN_MODEL` | Model training timeout | `1800` | Large dataset processing (30 min) |
+| `TIMEOUT_SAVE_MODEL` | Save model to MLFlow timeout | `300` | Artifact upload and logging (5 min) |
+| `TIMEOUT_CLEANUP_DIRECTORY` | Cleanup temporary directory timeout | `60` | Local filesystem operation (1 min) |
+| `TIMEOUT_DELETE_FILE` | Delete file from MinIO timeout | `60` | MinIO delete operation (1 min) |
+| `TIMEOUT_UPDATE_DATABASE` | Database update timeout | `30` | PostgreSQL update query (30 sec) |
+
+**Note**: These timeouts can be adjusted based on your infrastructure performance and file sizes. If you're processing files larger than 200MB or have slower network/compute resources, increase these values accordingly.
+
 ### Workflow Configuration
 
 MongoDB pipeline configuration:

model_manager/workflows/train_model.py

@@ -12,6 +12,7 @@ This workflow orchestrates the complete ML model training process, including:
 from temporalio import workflow
 
 with workflow.unsafe.imports_passed_through():
+    import os
     from datetime import timedelta
     from typing import Any
 
@@ -23,6 +24,16 @@ with workflow.unsafe.imports_passed_through():
     from model_manager.utils.models.train_model_params import TrainModelParams
     from model_manager.utils.models.train_model_result import TrainModelResult
 
+    # Activity Timeouts (in seconds) - Configurable via environment variables
+    # Defaults are designed to handle large files (up to 200MB)
+    TIMEOUT_VALIDATE_PARAMS = int(os.getenv('TIMEOUT_VALIDATE_PARAMS', '30'))
+    TIMEOUT_DOWNLOAD_FILE = int(os.getenv('TIMEOUT_DOWNLOAD_FILE', '600'))
+    TIMEOUT_TRAIN_MODEL = int(os.getenv('TIMEOUT_TRAIN_MODEL', '1800'))
+    TIMEOUT_SAVE_MODEL = int(os.getenv('TIMEOUT_SAVE_MODEL', '300'))
+    TIMEOUT_CLEANUP_DIRECTORY = int(os.getenv('TIMEOUT_CLEANUP_DIRECTORY', '60'))
+    TIMEOUT_DELETE_FILE = int(os.getenv('TIMEOUT_DELETE_FILE', '60'))
+    TIMEOUT_UPDATE_DATABASE = int(os.getenv('TIMEOUT_UPDATE_DATABASE', '30'))
+
 
 @workflow.defn(name='train_model')
 class TrainModel:
@@ -173,7 +184,7 @@ class TrainModel:
                 Activities.validate_train_params,
                 validation_input,
                 retry_policy=retry_policy,
-                start_to_close_timeout=timedelta(seconds=30),
+                start_to_close_timeout=timedelta(seconds=TIMEOUT_VALIDATE_PARAMS),
             )
 
             # Validation succeeded: Update status
@@ -243,7 +254,7 @@ class TrainModel:
                 Activities.fetch_file_from_minio,
                 download_input,
                 retry_policy=retry_policy,
-                start_to_close_timeout=timedelta(seconds=60),
+                start_to_close_timeout=timedelta(seconds=TIMEOUT_DOWNLOAD_FILE),
             )
 
             # Step 2: Train model with downloaded file
@@ -257,7 +268,7 @@ class TrainModel:
                 Activities.train_model,
                 train_input,
                 retry_policy=retry_policy,
-                start_to_close_timeout=timedelta(seconds=300),
+                start_to_close_timeout=timedelta(seconds=TIMEOUT_TRAIN_MODEL),
             )
 
             # Training succeeded: Update status
@@ -330,7 +341,7 @@ class TrainModel:
                 Activities.save_model,
                 save_input,
                 retry_policy=retry_policy,
-                start_to_close_timeout=timedelta(seconds=120),
+                start_to_close_timeout=timedelta(seconds=TIMEOUT_SAVE_MODEL),
             )
 
             # Model saved successfully: Update status to MLFLOW_SENT with run_name
@@ -396,7 +407,7 @@ class TrainModel:
                     Activities.cleanup_run_directory,
                     cleanup_input,
                     retry_policy=retry_policy,
-                    start_to_close_timeout=timedelta(seconds=30),
+                    start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_DIRECTORY),
                 )
 
                 workflow.logger.info(
@@ -414,7 +425,7 @@ class TrainModel:
                 Activities.delete_file_from_minio,
                 delete_input,
                 retry_policy=retry_policy,
-                start_to_close_timeout=timedelta(seconds=30),
+                start_to_close_timeout=timedelta(seconds=TIMEOUT_DELETE_FILE),
             )
 
             # Cleanup succeeded: Update status to FILE_DELETED
@@ -486,5 +497,5 @@ class TrainModel:
             Activities.update_experiment_run,
             update_input,
             retry_policy=retry_policy,
-            start_to_close_timeout=timedelta(seconds=30),
+            start_to_close_timeout=timedelta(seconds=TIMEOUT_UPDATE_DATABASE),
         )