SIENTIAPDE-994Add shutdown methods for Activities and OPC classes; enhance worker error handling

laborious/activities/activities.py

@@ -47,3 +47,7 @@ class Activities(Postgres, MLFlow, Gates, OPC):
     @activity.defn(name="prepare_activity")
     async def prepare_activity(self, input_data: dict[str, Any]):
         await super().prepare_activity(input_data)
+
+    def shutdown(self):
+        Postgres.close(self)
+        OPC.shutdown(self)

laborious/activities/opc.py

@@ -99,3 +99,7 @@ class OPC(BaseActivity):
                         data_type=tag_config['data_type'],
                         tag_type='confidence'
                     )
+
+    def shutdown(self):
+        for opc in self.opc_repository.values():
+            opc.disconnect()

laborious/worker/worker.py

@@ -1,5 +1,6 @@
 from temporalio import workflow, client
 from temporalio.worker import Worker
+import sys
 
 with workflow.unsafe.imports_passed_through():
     import os
@@ -90,7 +91,19 @@ async def main():
 
     logger.info('Workers started successfully')
 
-    await asyncio.gather(*handlers)
+    try:
+        # This will run the workers and wait for them to complete.
+        # If an exception occurs in any of the worker handlers, it will be propagated here.
+        await asyncio.gather(*handlers)
+    except BaseException as e:
+        logger.error("An unhandled exception occurred: %s", e, exc_info=True)
+    finally:
+        if notification_handler:
+            notification_handler.shutdown()
+        if activities:
+            activities.shutdown()
+        # Exit with a non-zero status code to indicate failure to Kubernetes
+        sys.exit(1)
 
 if __name__ == '__main__':
     asyncio.run(main())

values.yaml

@@ -0,0 +1,183 @@
+# Default values for sientia-module.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
+replicaCount: 1
+
+# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
+image:
+  repository: aignosi.azurecr.io/sientia-module
+  # This sets the pull policy for images.
+  pullPolicy: Always
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "0.0.2"
+
+# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+imagePullSecrets:
+- name: docker-hub-secret
+# This is to override the chart name.
+nameOverride: "sientia-laborious-worker"
+fullnameOverride: "sientia-laborious-worker"
+namespace: sientia
+
+# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Automatically mount a ServiceAccount's API credentials?
+  automount: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: "sientia-laborious-worker"
+
+# This is for setting Kubernetes Annotations to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+podAnnotations: {}
+# This is for setting Kubernetes Labels to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+podLabels: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+livenessProbe:
+  exec:
+    command:
+      - sh
+      - -c
+      - pgrep -f "laborious.worker.worker"
+  initialDelaySeconds: 20
+  periodSeconds: 30
+
+readinessProbe:
+  exec:
+    command:
+      - sh
+      - -c
+      - pgrep -f "laborious.worker.worker"
+  initialDelaySeconds: 10
+  periodSeconds: 15
+
+
+# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+# Additional volumes on the output Deployment definition.
+volumes: []
+# - name: foo
+#   secret:
+#     secretName: mysecret
+#     optional: false
+
+# Additional volumeMounts on the output Deployment definition.
+volumeMounts: []
+# - name: foo
+#   mountPath: "/etc/foo"
+#   readOnly: true
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+service:
+  enabled: false
+  type: ClusterIP
+  port: 4840
+  targetPort: 4840
+  
+
+env:
+  # Entrypoint variables 
+  - name: GITHUB_REPO_URL
+    value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git"
+  - name: GITHUB_BRANCH
+    value: "SIENTIAPDE-994-implementar-os-workflows-mapeados-utilizando-as-workers-e-activities-apropriadas"
+  - name: PYTHON_APP
+    value: "laborious.worker.worker"
+
+  # Application variables
+  - name: POSTGRES_HOST
+    value: "paradedb-rw.paradedb.svc.cluster.local"
+  - name: POSTGRES_PORT
+    value: "5432"
+  - name: POSTGRES_USER
+    value: "sientia"
+  - name: POSTGRES_PASSWORD
+    value: "sientia"
+  - name: POSTGRES_DBNAME
+    value: "sientia"
+  - name: POSTGRES_MIN_CONNECTIONS
+    value: "10"
+  - name: POSTGRES_MAX_CONNECTIONS
+    value: "20"
+
+  - name: MLFLOW_HOST
+    value: "sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local"
+  - name: MLFLOW_PORT
+    value: "80"
+  - name: MLFLOW_USERNAME
+    value: "aignosi"
+  - name: MLFLOW_PASSWORD
+    value: "aignosi"
+
+  - name: OPC_NAME
+    value: "server-1"
+  - name: OPC_URL
+    value: "opc.tcp://sientia-opc-simulator.sientia.svc.cluster.local:4840"
+
+  - name: LOG_LEVEL
+    value: "DEBUG"
+  - name: PROJECT_NAME
+    value: "sientia-laborious"
+
+  - name: TEMPORAL_HOST
+    value: "temporal-frontend.temporal.svc.cluster.local:7233"
+  - name: TEMPORAL_NAMESPACE
+    value: "default"
+
+ssh:
+  enabled: true
+  secretName: git-ssh-key-sientia-laborious-worker
+  sshPath: /mnt/.ssh
+  knownHostsPath: /mnt/known_hosts
+
+# kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp
+
+# helm upgrade --install sientia-laborious-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.1.0-uat
+
+# kubectl create secret generic git-ssh-key-sientia-laborious-worker \
+#   --namespace sientia \
+#   --from-file=ssh-privatekey=git_key \
+#   --type=kubernetes.io/ssh-auth