SIENTIAPDE-994

Refactor and enhance the laborious workflow and utilities

- Removed outdated test file `test_predictions_batch.py` from workflows.
- Added `input_sample.json` for standardized input configuration.
- Introduced `connectors_config.py` to manage database and service configurations.
- Implemented a logging utility in `logger.py` for consistent logging across the application.
- Created `policies.py` to define retry policies for workflows.
- Developed comprehensive tests for `MLFlowRepository` in `test_model_repository.py`.
- Added extensive tests for `OpcRepository` in `test_opc_repository.py`.
- Updated `test_predictions_batch.py` to reflect new workflow structure and testing methodology.

laborious/worker/worker.py

@@ -2,30 +2,32 @@ from temporalio import workflow, client
 from temporalio.worker import Worker
 
 with workflow.unsafe.imports_passed_through():
-    from laborious.workflows.predictions_batch import PredictionsBatch
-    from laborious.activities.activities import Activities
     import os
-    import logging
+    import asyncio
+    from laborious.workflows.predictions_batch import PredictionsBatch
+    from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
+    from laborious.workflows.sub_workflows.format_and_export_prediction import \
+        FormatAndExportPrediction
+    from laborious.activities.activities import Activities
+    from laborious.utils.logger import get_logger
+    from laborious.utils.connectors_config import (
+        build_postgres_config,
+        build_mlflow_config,
+        build_opc_config
+    )
     from sientia_do.notifications.handlers import NotificationHandler
 
 
 async def main():
     host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
-    logger = logging.getLogger(__name__)
-    stream_handler = logging.StreamHandler()
-    stream_handler.setLevel(
-        os.getenv('LOG_LEVEL', 'INFO').upper()
-    )
-    stream_handler.setFormatter(
-        logging.Formatter(
-            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-        )
-    )
+    logger = get_logger(__name__)
 
-    logger.addHandler(stream_handler)
+    logger.info('Starting Worker...')
+
+    logger.info('Starting Notification Handler...')
 
     notification_handler = NotificationHandler(
-        servers=os.getenv('NOTIFICATION_SERVERS', 'http://localhost:29092'),
+        servers=os.getenv('KAFKA_SERVERS', 'http://localhost:9092'),
         logger=logger,
         project_name=os.getenv('PROJECT_NAME', 'laborious'),
         pipeline_name='-',
@@ -34,46 +36,31 @@ async def main():
         model='-'
     )
 
-    postgres_config = {
-        'host': os.getenv('POSTGRES_HOST', 'localhost'),
-        'port': int(os.getenv('POSTGRES_PORT', '5432')),
-        'user': os.getenv('POSTGRES_USER', 'sientia'),
-        'password': os.getenv('POSTGRES_PASSWORD', 'sientia'),
-        'dbname': os.getenv('POSTGRES_DBNAME', 'sientia'),
-        'min_connections': int(os.getenv('POSTGRES_MIN_CONNECTIONS', '5')),
-        'max_connections': int(os.getenv('POSTGRES_MAX_CONNECTIONS', '20'))
-    }
-
-    mlflow_config = {
-        'host': os.getenv('MLFLOW_HOST', 'localhost'),
-        'port': int(os.getenv('MLFLOW_PORT', '5000')),
-        'username': os.getenv('MLFLOW_USERNAME', 'aignosi'),
-        'password': os.getenv('MLFLOW_PASSWORD', 'aignosi')
-    }
-
-    opc_config = {
-        'name': os.getenv('OPC_NAME', 'opc'),
-        'url': os.getenv('OPC_URL', 'opc.tcp://localhost:4840'),
-        'server_uri': os.getenv('OPC_SERVER_URI', 'opc.tcp://localhost:4840'),
-        'cert_path': os.getenv('OPC_CERT_PATH', None),
-        'private_key_path': os.getenv('OPC_PRIVATE_KEY_PATH', None),
-        'server_cert_path': os.getenv('OPC_SERVER_CERT_PATH', None)
-    }
+    logger.info('Starting Activities...')
 
     activities = Activities(
-        postgres_config=postgres_config,
-        mlflow_config=mlflow_config,
-        opc_config=opc_config,
+        postgres_config=build_postgres_config(),
+        mlflow_config=build_mlflow_config(),
+        opc_config=build_opc_config(),
         logger=logger,
         notification_handler=notification_handler
     )
 
-    temporal_client = await client.Client.connect(target_host=host)
+    logger.info('Starting Temporal Client...')
+
+    temporal_client = await client.Client.connect(
+        target_host=host,
+        namespace=os.getenv('TEMPORAL_NAMESPACE', 'default')
+    )
+
+    logger.info('Starting Workers...')
+
     workers = [
         Worker(
             temporal_client,
-            task_queue='predictions',
-            workflows=[PredictionsBatch],
+            task_queue='predictions-queue',
+            workflows=[PredictionsBatch, PredictionProcess,
+                       FormatAndExportPrediction],
             activities=[
                 # Base
                 activities.prepare_activity,
@@ -97,9 +84,13 @@ async def main():
         )
     ]
 
+    handlers = []
     for w in workers:
-        await w.run()
+        handlers.append(w.run())
+
+    logger.info('Workers started successfully')
+
+    await asyncio.gather(*handlers)
 
 if __name__ == '__main__':
-    import asyncio
     asyncio.run(main())