Merge pull request #34 from Aignosi/fix/SIENTIAPDE-1712

SIENTIAPDE-1712: Enhance Error Handling and Configuration Alignment

notification_generator.ipynb

@@ -151,7 +151,7 @@
     }
    ],
    "source": [
-    "error_models = general_events.get('ERROR', {}).get('models', [])\n",
+    "error_models = general_events.get('ERROR', {}).get('', [])\n",
     "warning_models = general_events.get('WARNING', {}).get('models', [])\n",
     "info_models = general_events.get('INFO', {}).get('models', [])\n",
     "mail_type = 'Alerts'\n",

orchestrator/utils/orchestrator_functions.py

@@ -38,6 +38,7 @@ def common_config(config: dict[str, Any]):
         'model_config': model.get('model_config', {}),
         'execution_timeout_seconds': config.get('execution_timeout_seconds', 300),
         'task_timeout_seconds': config.get('task_timeout_seconds', 300),
+        'on_conflict': config.get('on_conflict', 'error'),
     }
 
 

tests/orchestrator/utils/test_orchestrator_functions.py

@@ -33,6 +33,7 @@ def test_common_config():
         'model_id': 'test_model_id',
         'model_name': 'test_model_name',
         'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
         'execution_timeout_seconds': 300,
         'task_timeout_seconds': 300,
     }
@@ -58,6 +59,7 @@ def test_drift():
         'model_id': 'test_model_id',
         'model_name': 'test_model_name',
         'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
         'schema': 'sientia_data',
         'source_table_name': 'laborious_data',
         'target_table_name': 'drift_metrics',
@@ -88,6 +90,7 @@ def test_simple_metrics():
         'model_id': 'test_model_id',
         'model_name': 'test_model_name',
         'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
         'schema': 'sientia_data',
         'predictions_table_name': 'predictions',
         'data_table_name': 'laborious_data',
@@ -119,6 +122,7 @@ def test_minimal_retrain():
         'model_id': 'test_model_id',
         'model_name': 'test_model_name',
         'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
         'query': 'select * from sientia_data.laborious_data order by "timestamp" desc limit 30;',
         'schema': 'sientia_data',
         'table_name': 'log_retrain',
@@ -153,6 +157,7 @@ def test_scouter():
         'model_id': 'test_model_id',
         'model_name': 'test_model_name',
         'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
         'topic': 'raw_test_schedule',
         'trigger_laborious': False,
         'filters': {'test_filter_name': {'policy': 'test_policy'}},
@@ -254,6 +259,7 @@ def test_predictions_batch(mock_process_path_priority, mock_overlap_filter_confi
         'model_id': 'test_model_id',
         'model_name': 'test_model_name',
         'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
         'query': 'test_query',
         'schema': 'sientia_data',
         'table_name': 'predictions',
@@ -556,6 +562,7 @@ def test_base_scouter():
         'model_id': 'test_model_id',
         'model_name': 'test_model_name',
         'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
         'trigger_laborious': False,
         'filters': {
             'test_filter_name': {'policy': 'test_policy'},
@@ -605,6 +612,7 @@ def test_pi_web_api_scouter():
         'model_id': 'test_model_id',
         'model_name': 'test_model_name',
         'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
         'trigger_laborious': False,
         'filters': {'test_filter_name': {'policy': 'test_policy'}},
         'schema': 'sientia_data',
@@ -660,6 +668,7 @@ def test_pi_web_api_scouter_with_timeout_greater_than_frequency():
         'model_id': 'test_model_id',
         'model_name': 'test_model_name',
         'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
         'trigger_laborious': False,
         'filters': {},
         'schema': 'sientia_data',
@@ -710,6 +719,7 @@ def test_pi_web_api_scouter_with_no_timeout():
         'model_id': 'test_model_id',
         'model_name': 'test_model_name',
         'model_config': {'test_config': 'test_config'},
+        'on_conflict': 'error',
         'trigger_laborious': False,
         'filters': {},
         'schema': 'sientia_data',

values.yaml

@@ -164,7 +164,7 @@ env:
   - name: GITHUB_REPO_URL
     value: "git@github.com:Aignosi/sientia-dataops-orchestrator_temporal.git"
   - name: GITHUB_BRANCH
-    value: "feature/SIENTIAPDE-1478"
+    value: "fix/SIENTIAPDE-1712"
   - name: PYTHON_APP
     value: "orchestrator.worker.worker"