SIENTIAPDE-994

Update requirements.txt with new dependencies and refactor activity methods for improved functionality and error handling

laborious/workflows/predictions_batch.py

@@ -0,0 +1,85 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from laborious.activities.postgres import Postgres
+    from laborious.activities.mlflow import MLFlow
+    from laborious.activities.gates import Gates
+    from laborious.activities.opc import OPC
+    from typing import Any
+
+
+@workflow.defn(name="predictions_batch")
+class PredictionsBatch():
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]):
+
+        await workflow.execute_activity_method(
+            Postgres.prepare_activity,
+            {
+                'schedule_name': input_data['schedule_name'],
+                'model_name': input_data['model_name'],
+                'model_id': input_data['model_id']
+            }
+        )
+
+        data = await workflow.execute_activity_method(
+            Postgres.load_custom_query,
+            input_data['query']
+        )
+
+        path_flag, confidence = await workflow.execute_activity_method(
+            Gates.input_gate,
+            {
+                'filters': input_data['filters'],
+                'data': data
+            }
+        )
+
+        if path_flag == 'stop':
+            return
+
+        if path_flag == 'continue':
+            # repeat last prediction
+            await workflow.execute_activity_method(
+                Postgres.repeat_last_prediction,
+                {
+                    'schema': input_data['schema'],
+                    'table_name': input_data['table_name'],
+                    'model': input_data['model']
+                }
+            )
+            return
+
+        response_data, last_timestamp = await workflow.execute_activity_method(
+            MLFlow.transform_data,
+            {
+                'data': data,
+                'model_name': input_data['model_name'],
+                'model_retention': input_data['model_retention']
+            }
+        )
+
+        path_flag, confidence = await workflow.execute_activity_method(
+            Gates.mlflow_gate,
+            {
+                'filters': input_data['filters'],
+                'data': response_data,
+                'type': 'transform'
+            }
+        )
+
+        if path_flag == 'stop':
+            return
+
+        if path_flag is None:
+            # procced with prediction
+            response_data = await workflow.execute_activity_method(
+                MLFlow.request_predict,
+                {
+                    'data': response_data,
+                    'model_name': input_data['model_name'],
+                    'model_retention': input_data['model_retention']
+                }
+            )
+
+            path_flag

laborious/workflows/sub_workflows/format_and_export_prediction.py

@@ -0,0 +1,60 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from laborious.activities.activities import Activities
+    from typing import Any
+
+
+@workflow.defn(name="format_and_export_prediction")
+class FormatAndExportPrediction():
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]):
+        path_flag = input_data['path_flag']
+        data = input_data['data']
+        confidence = input_data['confidence']
+
+        if path_flag is None:
+            # proceed with formatting and exporting
+            prediction = await workflow.execute_activity_method(
+                Activities.format_prediction,
+                {
+                    'data': data,
+                    'timestamp': input_data['timestamp'],
+                    'model_id': input_data['model_id'],
+                    'prediction_confidence': confidence,
+                }
+            )
+
+        else:
+            # create default prediction
+            prediction = await workflow.execute_activity_method(
+                Activities.format_default_prediction,
+                {
+                    'timestamp': input_data['timestamp'],
+                    'model_id': input_data['model_id'],
+                    'prediction_confidence': confidence,
+                    'comment': input_data['comment']
+                }
+            )
+
+        # write to postgres
+        postgres_holder = workflow.execute_activity_method(
+            Activities.export_data_to_postgres,
+            {
+                'schema': input_data['schema'],
+                'table_name': input_data['table_name'],
+                'data': prediction
+            }
+        )
+
+        opc_holder = workflow.execute_activity_method(
+            Activities.write_opc_data,
+            {
+                'opc_servers': input_data['opc_servers'],
+                'opc_output_config': input_data['opc_output_config'],
+                'data': prediction
+            }
+        )
+
+        await postgres_holder
+        await opc_holder

laborious/workflows/sub_workflows/prediction_process.py

@@ -0,0 +1,59 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from laborious.activities.postgres import Postgres
+    from laborious.activities.mlflow import MLFlow
+    from laborious.activities.gates import Gates
+    from laborious.activities.opc import OPC
+    from typing import Any
+
+
+@workflow.defn(name="prediction_process")
+class PredictionProcess():
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]):
+        data = input_data['data']
+
+        path_flag, _confidence = await workflow.execute_activity_method(
+            Gates.input_gate,
+            {
+                'filters': input_data['filters'],
+                'data': data
+            }
+        )
+
+        if path_flag == 'stop':
+            return
+
+        if path_flag == 'continue':
+            # repeat last prediction
+            await workflow.execute_activity_method(
+                Postgres.repeat_last_prediction,
+                {
+                    'schema': input_data['schema'],
+                    'table_name': input_data['table_name'],
+                    'model': input_data['model']
+                }
+            )
+            return
+
+        response_data, last_timestamp = await workflow.execute_activity_method(
+            MLFlow.transform_data,
+            {
+                'data': data,
+                'model_name': input_data['model_name'],
+                'model_retention': input_data['model_retention']
+            }
+        )
+
+        path_flag, confidence = await workflow.execute_activity_method(
+            Gates.mlflow_gate,
+            {
+                'filters': input_data['filters'],
+                'data': response_data,
+                'type': 'transform'
+            }
+        )
+
+        if path_flag == 'stop':
+            return